#!/usr/bin/perl

use strict;
use warnings;
use POSIX qw(setsid);

# import modules
use Getopt::Long;
use threads;
use threads::shared;
use Time::HiRes qw(usleep);
use Time::HiRes qw(time);
use Scalar::Util qw(looks_like_number);
use Cwd;
use Switch;
use Fcntl ':flock';
use Socket;
use POSIX ":sys_wait_h";

my $upd_version = "11.0.22"; # derived from "11.551"
 
my $file        = $0;
my $pid         = $$;
my $status_protocol_version = 1;
my $webservice_dir      = '/opt/webservice/';
my $softgate_dir        = '/opt/soco/';
my $isoftgate_dir       = '/sg2/opt/soco/';
my $softgate_data_dir   = '/var/opt/soco/';
my $isoftgate_data_dir  = '/var/sg2/opt/soco/';
my $softgate_config_dir = $softgate_data_dir.'config/';
my $isoftgate_config_dir= $isoftgate_data_dir.'config/';
my $sg_nuc_status_dir   = $softgate_data_dir.'nuc/';
my $opensips_sql_dump_file = $softgate_config_dir.'opensips.sql';
my $opensips_sql_dump_fallback_file = $softgate_config_dir.'opensips_fallback.sql';
my $yaps_dir            = '/opt/yaps/';
my $ha_dir              = '/opt/ha/';
my $cla_license_dir     = '/opt/cla/license/';
my $cla_import_dir      = '/opt/cla/import/';
my $cla_temp_dir        = '/var/opt/soco2/cla/';
my $snmp_upgrade_dir    = '/var/opt/soco2/snmp/';
my $generic_upgrade_dir = '/var/opt/soco2/generic_update/';
my $csta_os_hf_pkg_dir  = '/var/opt/soco2/os_pkgs_for_csta/';
my $assi_os_hf_pkg_dir  = '/var/opt/soco2/os_pkgs_for_asmgr/';
my $ha_bin_dir          = $ha_dir.'bin/';
my $ha_data_dir         = '/var/opt/ha/';
my $openais_conf_file   = '/etc/ais/openais.conf';
my $dscxl_dir           = '/opt/dscxl/';
my $soco2_data_dir      = '/var/opt/soco2/';
my $mek_blobber_dir     = $soco2_data_dir.'mek/';
my $syslog_dir          = "/var/opt/soco2/Syslog/";
my $sysloghost_file     = $syslog_dir . "AStraceForwardHost.conf";
my $syslogconfd         = "/etc/syslog-ng/conf.d/";
my $syslogforward_file  = $syslogconfd . "AStraceForward.conf";
my $syslogepoch_file    = $syslog_dir . "AStraceForward.epoch";
my $soco2_tmp_dir       = $soco2_data_dir.'tmp/';
my $soco2_sg_dir        = $soco2_data_dir.'sg_repo/';           # OLD/NEW world takeover dir for SG LW
my $soco2_sg_force_dir  = $soco2_data_dir.'sg_repo_forced/';    # softgate.rpm in this dir will be forcibly updated by SG startup (and then removed))
my $soco2_isg_force_dir = '/var/sg2/opt/soco2/sg_repo_forced/'; # the same as above but for iSG
my $cancel_upgrade_file = $soco2_data_dir.'/cancel_nuc_update';     # this trigger file will be created if SWA calls teh NUC script for setting failure status
my $sg_os_rlc_dir       = '/instmedia/sg_os_rlc';
my $nuc_bin_dir         = '/instmedia/opt/yaps/script/update_new/';
my $nuc_copy_other_script   = $nuc_bin_dir.'copy_other.sh';
my $nuc_exec_other_script   = $nuc_bin_dir.'exec_other.sh';
my $nuc_reboot_script       = $nuc_bin_dir.'update-new-reboot-to-activate.sh';
my $stmix_os_confirm_script = $nuc_bin_dir.'stmix_os_update.sh';
my $nuc_getworld_script     = $nuc_bin_dir.'get_world.sh';
my $nuc_rollback_script     = $nuc_bin_dir.'world_rollback.sh';
my $nuc_sg_os_state_script= '/opt/soco-common/set_os_update_state.sh';
my $webservice_data_dir   = '/var/opt/webservice/';
my $sg_os_version_file    = $webservice_data_dir.'sg_os_version';
my $webservice_log_dir    = '/var/log/webservice/';
my $softgate_log_dir      = '/var/opt/soco/log/';
my $assistant_data_area   = '/var/opt/Assistant/data/';
my $assistant_nuc_downtime_file = $assistant_data_area.'nuc_downtime';
my $assistant_nuc_downtime_file_step = 0;
my $assistant_sws_area    = $assistant_data_area.'SWS/';
my $assistant_gw_area     = $assistant_data_area.'GW/';
my $assistant_backup_area = $assistant_data_area.'BACKUP/';
my $assistant_controlled_sg_reboot_file = $assistant_backup_area.'hbr-auto-os-activation';
my $assistant_physical_partition = '/var/pgsql/Assistant/';
my $assistant_physical_data_area = $assistant_physical_partition.'data/';
my $assistant_physical_sws_area  = $assistant_physical_data_area.'SWS/';
my $skip_quorum_flagfile= $assistant_sws_area.'flags/skip_quorum_rpm_update';
my $plt_dir             = 'os4k_platform/';
my $plt_hf_dir          = 'os4k_platform_hotfix/';
my $rpm_dir             = $soco2_data_dir.$plt_dir;
my $rpm_pre_dir         = $soco2_data_dir.'os4k_platform_pre/';
my $rpm_post_dir        = $soco2_data_dir.'os4k_platform_post/';
my $osdeps_rpm_dir      = $rpm_dir.'os_deps';
my $osupdate_rpm_dir    = $rpm_dir.'os_update';
my $assistant_rpm_dir   = $assistant_sws_area.$plt_dir;
my $hotfix_dir          = $soco2_data_dir.$plt_hf_dir;
my $assistant_hotfix_dir= $assistant_sws_area.$plt_hf_dir;
my $snmp_dir            = $soco2_data_dir."SNMP/";
my $license_dir         = $soco2_data_dir."license/";
my $license_storage_dir = $soco2_data_dir."license_storage/";
my $assistant_license_dir= $assistant_sws_area."license/";
my $assistant_snmp_dir  =  $assistant_sws_area."SNMP/";
my $cur_hf_dir_old      = $webservice_data_dir.'current_hotfix/';
my $cur_hf_dir          = $soco2_data_dir.'current_hotfix/';
my $cur_hf_rpm_dir      = $soco2_data_dir.'current_hotfix_rpms/';
my $backup_dir          = $soco2_data_dir.'installrepo/';
my $osdeps_backup_dir   = $soco2_data_dir.'os_deps/';
my $osupdate_backup_dir = $soco2_data_dir.'os_update/';
my $tmp_rpm_dir         = $soco2_data_dir.'work/';
my $ws_rpm_file         = $rpm_dir.'webservice*.rpm';
my $yaps_rpm_file       = $rpm_dir.'ASyaps*.rpm';
my $cld_rpm_file        = $rpm_dir.'cld-os4k*.rpm';
my $dtb_rpm_file        = $rpm_dir.'dtb-*.rpm';
my $hfinfo_file         = $rpm_dir.'hf.info';
my $upginfo_file        = $rpm_dir.'upgrade.info';
my $torrent_file        = "sg_os.torrent";
my $torrent_path_share  = $assistant_gw_area . $torrent_file;
my $torrent_path        = "/instmedia/" . $torrent_file;
my $private_os_ssh_key  = "sshkey_sg_os_update";
my $log_file            = $webservice_log_dir.'update-daemon.log';
my $log_alt_file        = $webservice_log_dir.'update-daemon_transfer.log';
my $upgrade_log_file    = $webservice_log_dir.'plt-upgrade.log';
my $status_file         = $webservice_log_dir.'update-status.log';
my $status_file_tmp     = $webservice_log_dir.'update-status.tmp.log';
my $status_local_file   = $webservice_log_dir.'update-status_local.log';
my $status_remote_file  = $webservice_log_dir.'update-status_remote.log';
my $status_quorum_file  = $webservice_log_dir.'update-status_quorum.log';
my $sysinfo_data_dir    = '/var/opt/sysinfo/';
my $sysinfo_log             = $sysinfo_data_dir.'input/upgrade_info.xml';
my $sysinfo_standby_log     = $sysinfo_data_dir.'input/upgrade_info_standby.xml';
my $sysinfo_quorum_log      = $sysinfo_data_dir.'input/upgrade_info_quorum.xml';
my $sysinfo_pkg_log         = $sysinfo_data_dir.'input/package_installation.xml';
my $sysinfo_pkg_standby_log = $sysinfo_data_dir.'input/package_installation_standby.xml';
my $sysinfo_pkg_quorum_log  = $sysinfo_data_dir.'input/package_installation_quorum.xml';
my $dscxlbios_install_dir   = "/opt/biosdscxl/";
my $dscxlbios_activation_dir= "/var/opt/biosdscxl/";
my $yaps_self           = '/opt/yaps/script/swupdated';
my $self                = $webservice_dir.'scripts/swupdated';
my $old_self            = $webservice_dir.'scripts/swupdated.old';
my $remote_self_upg     = $soco2_data_dir.'swupdated';
my $remote_self         = $webservice_dir.'scripts/swupdated';
my $prio_script_ws      = $webservice_dir.'scripts/re_prio.sh';
my $prio_script_sg      = $softgate_dir.'native/os/linux/re_prio.sh';
my $sshkeyexch_script	= '/opt/webservice/install/sshkeyexch'; # deprecated
my $lock_dir            = "/dev/webservice/";
my $lock_file           = $lock_dir.'swupdated.lock';
my $remote_lock_file    = $lock_dir.'swupdated_remote.lock';
my $def_act_lock_file   = $lock_dir.'swupdated_deferred_activation.lock';
my $version_lock_file   = $lock_dir.'swupdated_versioninfo.lock';
my $getif_lock_file     = $lock_dir.'swupdated_getif.lock';
my $gettz_lock_file     = $lock_dir.'swupdated_gettz.lock';
my $setui_lock_file     = $lock_dir.'swupdated_setui.lock';
my $settz_lock_file     = $lock_dir.'swupdated_settz.lock';
my $activatelicense_lock_file = $lock_dir.'swupdated_activatelicense.lock';
my $configuresnmp_lock_file = $lock_dir.'swupdated_configuresnmp.lock';
my $configuremek_lock_file  = $lock_dir.'swupdated_configuremek.lock';
my $configuresyslog_lock_file  = $lock_dir.'swupdated_configuresyslog.lock';
my $sshd_lock_file      = $lock_dir.'swupdated_sshd.lock';
my $sshq_lock_file      = $sshd_lock_file;       # no separate lock file for sshq since we have general ssh private key file 
my $shutdown_lock_file  = $lock_dir.'swupdated_shutdown.lock';
my $os_transfer_lock_file=$lock_dir.'swupdated_os_transfer.lock';
my $sg_os_prep_lock_file=$lock_dir.'swupdated_sg_os_prep.lock';
my $job_file            = $webservice_data_dir.'jobs.xml';
my $swupdated_call_file = $webservice_data_dir.'plt_upgrade_running';
my $sg_necessary_file   = $softgate_data_dir.'sg_is_necessary';
my $isg_necessary_file  = $isoftgate_data_dir.'sg_is_necessary';
my $swupdated_call_status_file = $webservice_data_dir.'plt_upgrade_running.status';
my $pgsql_before_upgrade= $webservice_dir.'install/all-db-backup-and-clean.sh';
my $beer_backup_script  = $webservice_dir.'install/beer_db_backup_for_update.sh';
my $pgsql_data_zip_file = $webservice_data_dir.'ngsdb-backup.zip';
my $pgsql_data_zip_resc = $webservice_data_dir.'ngsdb-backup-rescue.zip';
my $pgsql_data_zip_save = $webservice_log_dir.'ngsdb-backup.zip';
my $upd_ok_sh           = 'upgrade_successful.sh';
my $before_upd_sh       = 'exec_before_upgrade.sh';
my $after_upd_sh        = 'exec_after_upgrade.sh';
my $after_rpm_install_sh= 'exec_after_rpm_install.sh';
my $after_stop_sh       = 'exec_after_openais_stop.sh';
my $manager_cluster_sh  = 'cluster_manage_for_reduced_downtime.sh';
my $make_this_cc_active_sh = $dscxl_dir.'bin/make_this_cc_active.sh';
my $rd_exec_after_sh       = $dscxl_dir.'bin/reduce_downtime_exec_after_plt_upgrade.sh';
my $ha_reboot_host_cmd  = $ha_dir.'bin/reboot';
my $ws_reboot_host_cmd  = $webservice_dir.'scripts/rebootHost.sh';
my $ws_shutdown_host_cmd  = $webservice_dir.'scripts/shutdownHost.sh';
my $before_stop_diagnose_script  = '/tmp/before_stop';
my $after_stop_diagnose_script   = '/tmp/after_stop';
my $before_start_diagnose_script = '/tmp/before_start';
my $after_start_diagnose_script  = '/tmp/after_start';
my $backup_rpm_script            = $webservice_dir.'scripts/backup_rpm_db_area.sh';
my $configure_snmp_script        = $webservice_dir.'scripts/configureSNMP.sh';
my $configure_mek_script         = $webservice_dir.'scripts/configureMEK.sh';
my $configure_syslog_script      = $webservice_dir.'scripts/configureSyslog.sh';
my $activate_license_script      = $webservice_dir.'scripts/activateLicense.sh';
my $get_interface_list_script    = $webservice_dir.'scripts/getInterfaceList.sh';
my $get_version_info_script      = $webservice_dir.'scripts/getVersionInfo.sh';
my $set_ui_mode_script           = $webservice_dir.'scripts/set_ui_mode.sh';
my $get_timezone_script          = $webservice_dir.'scripts/get_timezone.sh';
my $set_timezone_script          = $webservice_dir.'scripts/set_timezone.sh';
my $ha_insserv_script            = $webservice_dir.'scripts/insserv_HA.sh';
my $zypper_protect_script        = $webservice_dir.'scripts/protect_zypper_logs.sh';
my $is_hpa_script                = $webservice_dir.'scripts/is_hpa';
my $is_dscxl_script              = $webservice_dir.'scripts/is_dscxl';
my $is_ecoap_script              = $webservice_dir.'scripts/is_ecoap';
my $is_manager_script            = $webservice_dir.'scripts/is_manager';
my $get_deployment_string_script = $webservice_dir.'scripts/get_deployment_as_string.sh';
my $store_package_script         = $webservice_dir.'scripts/store_package_unique.sh';
my $boarddatXmlFile              = "/var/opt/soco/pbc_21/boarddat.xml";
my $major_release_update_file    = $webservice_data_dir.'is_major_release_update';
my $pkey_dir                     = "/root/.ssh/";
my $pkey_file                    = $pkey_dir."id_rsa";
my $pkey_file_save               = $pkey_file."_haupdate";
my $pubkey_file                  = $pkey_file."id_rsa.pub";
my $ssh_option                   = "-o 'BatchMode yes' -o 'CheckHostIP no' -o 'PasswordAuthentication no' -o 'ServerAliveInterval 60' -o 'ServerAliveCountMax 3'";
my $ssh_executable_ws            = "/opt/webservice/scripts/ssh_swupdated";
my $ssh_executable_linux         = "/usr/bin/ssh";
my $ssh_cmd                      = "$ssh_executable_linux $ssh_option";
my $rsync_cmd                    = "/usr/bin/rsync -a --delete --force -v -e \"$ssh_cmd\"";
my $node_reach_test_method       = "none";    # "none", "ping", "crm"
my $ssh_cancel_wait_file         = "";  # if the file this variable points to exists then the ssh wait will be terminated. 
my ($call_stack_depth, $log_msg_indent) = (0, "");
my $global_rc   = 0;
my $hotfix_mode = 0; # 1 = activate, 2 = deactivate
my $exec_remote = 0;
my $hotfix_number_to_fallback_to_by_deactivation = -1;
my $rmx_reboot  = 1; # 0 = no, 1 = yes
my $host_reboot = 0; # 0 = no, 1 = yes
my $upgrade_with_completely_stopped_ha = 1; # 0 = no, 1 = yes
my $upgrade_with_full_drbd_sync        = 1; # 0 = no, 1 = yes
my $upgrade_with_reduce_downtime       = 0; # 0 = no, 1 = yes
my $upgrade_via_image                  = 0; # 0 = no, 1 = yes
my $continue_upgrade_with_completely_stopped_ha = $upgrade_with_completely_stopped_ha;
my $continue_upgrade_with_full_drbd_sync        = $upgrade_with_full_drbd_sync;
my $continue_upgrade_with_reduce_downtime       = $upgrade_with_reduce_downtime;
my $continue_upgrade_via_image                  = $upgrade_via_image;
my $full_drbd_sync_flag_really_read    = 0;
my $autodeact   = 0;
my $biosupdate = 0;
my $biosupdate_remote = 0;
my $intended_bios_version  = "12/12/2011"; # VXCC0004.UPD
my $intended_bios_file     = "VXCC0004.UPD"; 
my $bad_bios_version       = "08/06/2010";
my $delivered_bios_version = "";
my ($openj9_java8_version_before_upgrade, $openj9_java8_version_after_upgrade, $java8_version_before_upgrade, $java8_version_after_upgrade, $java7_version_before_upgrade, $java7_version_after_upgrade) = ("", "", "", "", "", "");
my ($sg_os_update, $deferred_activation, $keep_installed_sg) = (0, 0, 0);
my ($continue_upgrade, $continue_upgrade_iteration, $max_continue_upgrade_iteration) = (0, 0, 5);
my $duplex_with_one_board_allowed = 0;
my $local_ha_has_been_stopped = 0;
my $max_wait_for_up = 120; # minutes
my $max_wait_for_ssh_lock_default = 30;  # seconds
my $max_wait_for_ssh_lock_upgrade = 600; # seconds = 10 minutes
my $wait_for_ssh_connectivity = 80; # minutes
my $max_wait_for_ssh_client_execution = 6 * 60; # seconds
my $ignore_remote_return_code = 0;
my $sftp_transfer_user = "os_transfer"; # "os_transfer"; # linuser can not be sued, it may be locked
my $backup_set  = '';
my $extract     = 0;
my $nodaemon    = 0;
my $forcedaemon = 0;
my $forceoption = 0;
my $avoidminicib= 0;
my $nowait      = 0;
my $testmode    = 0;
my $isquorum    = 0; 
my $skip_quorum_rpm_upgrade = 0;
my $no_self_xtract = 0;
my $is_old      = 0;
my $is_dscxl    = 0;
my $is_eco      = 0;
my $is_stmix    = 0;
my $is_cots     = 0;
my $is_entgw    = 0;
my $is_virtual  = 0;
my $is_manager  = 0;
my $is_FI = 0;
my $is_HD_layout_A      = 0;
my $is_HD_layout_B      = 0;
my $is_major_release_update = 0;
my $platform_has_beer = 0;
my $params        = "";
my $params_remote = "";
my $write_sysinfo_status_enabled = 0;
my $upgrade_dir_has_been_removed_due_to_duplex_hf_deactivation = 0;
my $remote_updates_were_already_successfully_done_by_previous_update = 0;
my ($installed_version_at_beginning_of_upgrade, $installed_effective_version_at_beginning_of_upgrade, $running_version, $upgrade_version) = ("", "", "", "");
my ($sysinfo_sec,$sysinfo_min,$sysinfo_hour,$sysinfo_mday,$sysinfo_mon,$sysinfo_year, $sysinfo_wday,$sysinfo_yday,$sysinfo_iddst) = (0,0,0,0,0,0,0,0,0);
my ($sysinfo2_sec,$sysinfo2_min,$sysinfo2_hour,$sysinfo2_mday,$sysinfo2_mon,$sysinfo2_year, $sysinfo2_wday,$sysinfo2_yday,$sysinfo2_iddst) = (0,0,0,0,0,0,0,0,0);
my $internalUpgradeStepValue = "";
my $UpgradeRunning = "no";
my ($global_activation_rc, $quorum_activation_rc, $standby_activation_rc, $global_autodeactivation_rc, $activation_has_been_avoided ) = (0, 0, 0, 0, 0);
my $activation_status_text = "";
my $ts_rlc_upd_start = 0;
my $ts_rlc_upd_stop = 0;
my $ts_delta = 30;

# control variable to kill the update sysinfo status file transfer thread after the upgrade
my $die_background_thread : shared = 0;

  
my $ERROR_CanNotCopyAssistantFiles            = 1;
my $ERROR_AssistantDirectoryDoesNotExist      = 2;
my $ERROR_HotfixNotFound                      = 3;
my $ERROR_CanNotReadHotfixDir                 = 4;
my $ERROR_HotfixDirDoesNotContainValidHotfix  = 5;
my $ERROR_HotfixDirContainsTooMuchHotfixFiles = 6;
my $ERROR_CouldNotGetPackagesInfo             = 7;
my $ERROR_ExecutePreHaUpdate                  = 8;
my $ERROR_RemoteUpdateError                   = 9;
my $ERROR_ExecutePreinstallFailed             = 10;
my $ERROR_ExecutePostinstallFailed            = 11;
my $ERROR_UpdatePackagesFailed                = 12;
my $ERROR_WaitForUpgradeSuccessFailed         = 13;
my $ERROR_LicenseDirectoryDoesNotExist        = 14;
my $ERROR_LicenseDirectoryIsEmpty             = 15;
my $ERROR_HaipsetupDoesNotExist               = 16;
my $ERROR_StartingOpenaisFailed               = 17;
my $ERROR_StoppingOpenaisFailed               = 18;
my $ERROR_SysbackupDirectoryDoesNotExist      = 19;
my $ERROR_ExceededMaxNumberOfPltUpgradeRetries= 20;
my $ERROR_ExecuteFixMenuListFailed            = 21;
my $ERROR_BIOS_CheckFailed                    = 22;
my $ERROR_CanNotCopySysbackupFiles            = 23;
my $ERROR_AnotherShutdownAlreadyRunning       = 24;
my $ERROR_CanNotGetLock                       = 25;
my $ERROR_CanNotDaemonize                     = 26;
my $ERROR_ExecutePrePostinstallFailed         = 27;
my $ERROR_CanNotActicateLicenseDuringUpgrade  = 28;
my $ERROR_CanNotReachRemoteNode               = 29;
my $ERROR_RootPwdExpired                      = 30;
my $ERROR_UpgradeAlreadyRunning               = 31;
my $ERROR_CanNotCopyFromRemoteNode            = 32;
my $ERROR_NUCPreparationFailed                = 33;
my $ERROR_NUCDataUpgradeFailed                = 34;
my $ERROR_NoUpgradePackagesFound              = 35;
my $ERROR_NoMultiNodeSystem                   = 36;
my $ERROR_CouldNotReadLWUMInputFile           = 37;
my $ERROR_generic_SG_OS_Update                = 38;
my $ERROR_NUCRemoteNodeActivationRebootFailed = 39;
my $ERROR_SSH_KeygenFailed                    = 40;
my $ERROR_CreateUserFailed                    = 41;
my $ERROR_NUCActivationRebootFailed           = 42;
my $ERROR_SG_OS_RLC_DirDoesNotExists          = 43;
my $ERROR_SG_OS_TransferNotCancelled          = 44;
my $ERROR_SNMPDirectoryDoesNotExist           = 45;
my $ERROR_SNMPDirectoryIsEmpty                = 46;
my $ERROR_CanNotDetermineADPHost              = 47;
my $ERROR_RemoteNodeExecutionTimeout          = 48;
my $ERROR_SNMPConfigRejected                  = 49;
my $ERROR_GenericRemoteExecutionFailure       = 50;
my $ERROR_LocalLicenseActivationError         = 51;
my $ERROR_RemoteLicenseActivationError        = 52;

my %ERROR_text;
$ERROR_text{$ERROR_CanNotCopyAssistantFiles}        = "Error: can not copy Assistant files from /var/opt/Assistant/data/SWS/os4k_platform/ because dir is empty";
$ERROR_text{$ERROR_HaipsetupDoesNotExist}           = "haipsetup.xml file does not exist";
$ERROR_text{$ERROR_BIOS_CheckFailed}                = "BIOS check failed";
$ERROR_text{$ERROR_CanNotGetLock}                   = "did not get lock";
$ERROR_text{$ERROR_CanNotReachRemoteNode}           = "remote node not reachable";
$ERROR_text{$ERROR_RootPwdExpired}                  = "expired root password on platform node";
$ERROR_text{$ERROR_CanNotCopyFromRemoteNode}        = "can not copy file from remote node";
$ERROR_text{$ERROR_LocalLicenseActivationError}     = "License could not be activated/imported on primary node";
$ERROR_text{$ERROR_RemoteLicenseActivationError}    = "License could not be activated/imported on secondary/remote node";

my $NUC_SG_OS_ERROR_GenericActivationError   = 61;
my $NUC_SG_OS_ERROR_BootToNewWorldFailed     = 62;

my $rmx_hista_no_alarm    = 0;
my $rmx_hista_minor_alarm = 1;
my $rmx_hista_major_alarm = 2;


my %STEPS_text;

my $STEP_platform_upgrade_started             = 1;
my $STEP_preparation                          = 2;
my $STEP_start_openais_q_t3                   = 3;
my $STEP_start_openais_s_t3                   = 4;
my $STEP_platform_upgrade_success             = 5;
my $STEP_platform_upgrade_failed              = 6;
my $STEP_stop_openais_t3                      = 7;
my $STEP_remote_installation                  = 8;
my $STEP_startup_verification                 = 9;
my $STEP_packet_installation                  = 10;
my $STEP_post_processing                      = 11;
my $STEP_wait_for_status                      = 12;
my $STEP_reboot_host                          = 13;
my $STEP_wait_for_openais                     = 14;
my $STEP_stop_openais                         = 15;
my $STEP_remote_update                        = 16;
my $STEP_remote_update_complete               = 17;
my $STEP_NUC_preparation                      = 20;
my $STEP_wait_for_remote_NUC_host             = 21;
my $STEP_NUC_rollback                         = 22;

$STEPS_text{$STEP_platform_upgrade_started}   = "platform upgrade started";
$STEPS_text{$STEP_preparation}                = "upgrade preparation";
$STEPS_text{$STEP_start_openais_q_t3}         = "start of openais on quorum due to special HA upgrade";
$STEPS_text{$STEP_start_openais_s_t3}         = "start of openais on standby node due to special HA upgrade";
$STEPS_text{$STEP_platform_upgrade_success}   = "platform upgrade finished successfully";
$STEPS_text{$STEP_platform_upgrade_failed}    = "platform upgrade failed";
$STEPS_text{$STEP_stop_openais_t3}            = "Stopping HA on all nodes";
$STEPS_text{$STEP_remote_installation}        = "Remote Installation";
$STEPS_text{$STEP_startup_verification}       = "Startup Verification";
$STEPS_text{$STEP_packet_installation}        = "Packet Installation";
$STEPS_text{$STEP_post_processing}            = "Post processing";
$STEPS_text{$STEP_wait_for_status}            = "waiting for status Active of the CC";
$STEPS_text{$STEP_reboot_host}                = "Reboot the Linux Host";
$STEPS_text{$STEP_wait_for_openais}           = "waiting for remote host";
$STEPS_text{$STEP_stop_openais}               = "Stopping HA";
$STEPS_text{$STEP_remote_update}              = "Remote Update";
$STEPS_text{$STEP_remote_update_complete}     = "Remote Update finished";
$STEPS_text{$STEP_NUC_preparation}            = "Preparation of an RLC update via image";
$STEPS_text{$STEP_wait_for_remote_NUC_host}   = "waiting for remote host re-appearance after reboot";
$STEPS_text{$STEP_NUC_rollback}               = "rollback of image activation";


my %STATUS_text;

my $STATUS_nyd                                = 1;
my $STATUS_success                            = 2;
my $STATUS_failed                             = 3;
my $STATUS_running                            = 4;
my $STATUS_dropped                            = 5;

$STATUS_text{$STATUS_nyd}                     = "to do";
$STATUS_text{$STATUS_success}                 = "successfully done";
$STATUS_text{$STATUS_failed}                  = "failed";
$STATUS_text{$STATUS_running}                 = "running";
$STATUS_text{$STATUS_dropped}                 = "dropped";

my $upgrade_sysinfo_status_a = $STATUS_text{$STATUS_nyd}; # node_A
my $upgrade_sysinfo_status_b = $STATUS_text{$STATUS_nyd}; # node_B
my $upgrade_sysinfo_status_q = $STATUS_text{$STATUS_nyd}; # node_Q


my %OPENAIS_STATE_text;

my $OPENAIS_STATE_running                           = 1;
my $OPENAIS_STATE_stopping                          = 2;
my $OPENAIS_STATE_stopped                           = 3;
my $OPENAIS_STATE_stopped_with_error                = 4;
my $OPENAIS_STATE_starting                          = 5;
my $OPENAIS_STATE_unknown                           = 6;


$OPENAIS_STATE_text{$OPENAIS_STATE_running}             = "running";
$OPENAIS_STATE_text{$OPENAIS_STATE_stopping}            = "stopping";
$OPENAIS_STATE_text{$OPENAIS_STATE_stopped}             = "stopped";
$OPENAIS_STATE_text{$OPENAIS_STATE_stopped_with_error}  = "error when stopping";
$OPENAIS_STATE_text{$OPENAIS_STATE_starting}            = "starting";
$OPENAIS_STATE_text{$OPENAIS_STATE_unknown}             = "unknown";

my $openais_sysinfo_state_a = $OPENAIS_STATE_text{$OPENAIS_STATE_running}; # node_A
my $openais_sysinfo_state_b = $OPENAIS_STATE_text{$OPENAIS_STATE_running}; # node_B
my $openais_sysinfo_state_q = $OPENAIS_STATE_text{$OPENAIS_STATE_running}; # node_Q



my $DATA_CURRENTNESS_STATUS_irrelevant          = -1;
my $DATA_CURRENTNESS_STATUS_up_to_date          = 0;
my $DATA_CURRENTNESS_STATUS_cached_sshlock      = 1;
my $DATA_CURRENTNESS_STATUS_cached_connectivity = 2;
my $DATA_CURRENTNESS_STATUS_cached_misc         = 3;

my %DATA_CURRENTNESS_STATUS_text_en;
$DATA_CURRENTNESS_STATUS_text_en{$DATA_CURRENTNESS_STATUS_irrelevant}             = "no read operation -> caching state irrelevant";
$DATA_CURRENTNESS_STATUS_text_en{$DATA_CURRENTNESS_STATUS_up_to_date}             = "data up-to-date";
$DATA_CURRENTNESS_STATUS_text_en{$DATA_CURRENTNESS_STATUS_cached_sshlock}         = "data from remote cache due to ssh lock";
$DATA_CURRENTNESS_STATUS_text_en{$DATA_CURRENTNESS_STATUS_cached_connectivity}    = "data from remote cache due to connectivity failure";
$DATA_CURRENTNESS_STATUS_text_en{$DATA_CURRENTNESS_STATUS_cached_misc}            = "data from remote cache due to misc. problems";

my %DATA_CURRENTNESS_STATUS_text_de;
$DATA_CURRENTNESS_STATUS_text_de{$DATA_CURRENTNESS_STATUS_irrelevant}             = "keine Leseoperation -> Caching-Status irrelevant";
$DATA_CURRENTNESS_STATUS_text_de{$DATA_CURRENTNESS_STATUS_up_to_date}             = "Daten sind up-to-date";
$DATA_CURRENTNESS_STATUS_text_de{$DATA_CURRENTNESS_STATUS_cached_sshlock}         = "Daten stammen aus dem Remote-Cache wegen des ssh-Locks";
$DATA_CURRENTNESS_STATUS_text_de{$DATA_CURRENTNESS_STATUS_cached_connectivity}    = "Daten stammen aus dem Remote-Cache wegen Verbindungsproblemen";
$DATA_CURRENTNESS_STATUS_text_de{$DATA_CURRENTNESS_STATUS_cached_misc}            = "Daten stammen aus dem Remote-Cache wegen sonstigen Problemen";

my $Wiederaufsetzpunkt1 = 1;
my $Wiederaufsetzpunkt2 = 2;
my $Wiederaufsetzpunkt3 = 3;
my $Wiederaufsetzpunkt4 = 4;
my $Wiederaufsetzpunkt5 = 5;

my $fss_work_dir = "/dev/shm/fast_swupdated";
my $fss_server_socket = $fss_work_dir."/server-sock";
my $fss_server_pid = $fss_work_dir."/server.pid";
my $fss_server_cmd = $fss_work_dir."/server.cmd";

#will be 0 by default ; 1 = instance is a fss server ; 2 = instance is a fss child
my $fss_instance_type=0;

my $fss_child_continue=0;
my $fss_next_child_pid=0;

#some count which will be increased with each new command invocation to ensure uniqueness of filenames
my $fss_cmd_count=0;

sub REAPER {
  my $stiff;
  while (($stiff = waitpid(-1, &WNOHANG)) > 0) {
    #dummy instruction - do nothing
    $stiff = $stiff;
  }
  $SIG{CHLD} = \&REAPER; # install *after* calling waitpid
}

if(defined $ARGV[0]) {
  if ($ARGV[0] eq "--fast-swupdated-server") {
    #this instance is a fss server
    $fss_instance_type=1;
    log_msg("Launching FAST swupdated Server using $0 ...");
    system("mkdir -p $fss_work_dir");
    unlink($fss_server_pid);
    $fss_cmd_count=0;
    #create socket and start listening
    #start server socket
    my $fss_uaddr = sockaddr_un($fss_server_socket);
    socket( FSS_SRVSOCK, AF_UNIX, SOCK_STREAM, 0 ) || die "socket: $!";
    unlink($fss_server_socket);
    bind( FSS_SRVSOCK, $fss_uaddr ) || die "bind: $!";
    listen( FSS_SRVSOCK, SOMAXCONN ) || die "listen: $!";
    #finally announce server is running for easy checking by clients
    $pid = $$;
    open(FSS_PID,">".$fss_server_pid.".tmp") || die "Can't write to $fss_server_pid : $!";
    print FSS_PID $pid;
    close(FSS_PID);
    open(FSS_CMD,">".$fss_server_cmd) || die "Can't write to $fss_server_cmd : $!";
    print FSS_CMD $0;
    close(FSS_CMD);
    system("mv ".$fss_server_pid.".tmp ".$fss_server_pid);
    log_msg("FAST swupdated Server (pid $pid) is now listening ...");
    $fss_child_continue=0;
    $fss_next_child_pid=0;
    while ( $fss_child_continue == 0 ) {
      my $fss_con;
      my $fss_addr = accept( $fss_con, FSS_SRVSOCK );
      unless ($fss_addr) {
        log_msg("accept: $!");
        next;
      }
      my $fss_requestLine = <$fss_con>;
      chomp($fss_requestLine);
      log_msg("FAST swupdated Server got a request : $fss_requestLine");
      if ( $fss_requestLine =~ /^QUIT/ ) {
        print $fss_con "OK\n";
        close($fss_con);
        close(FSS_SRVSOCK);
        unlink($fss_server_socket);
        unlink($fss_server_pid);
        unlink($fss_server_cmd);
        log_msg("FAST swupdated Server (pid $pid) is now stopped.");
        exit(0);
      }
      if ( $fss_requestLine =~ /^PING/ ) {
        print $fss_con "OK\n";
      }
      if ( $fss_requestLine =~ /^CMD:([0-9]*)/ ) {
        $fss_next_child_pid = $1;
        #increase command count
        $fss_cmd_count = $fss_cmd_count + 1;
        #reply to the caller that child process is about to be launched
        print $fss_con "OK:$fss_cmd_count\n";
        close($fss_con);
        #now fork the child process
        defined($pid = fork)     or die "Can't fork: $!";
        if($pid) {
          #still inside parent => continue and listen to another connection
          $fss_child_continue=0;
          #install signal to reap children in order to avoid zombies
          $SIG{CHLD} = \&REAPER;
        } else {
          #here we are inside child process => replace ARGV, redirect stdout and stderr and then continue
          close(FSS_SRVSOCK);
          $pid = $$;
          chdir '/'                or die "Can't chdir to /: $!";
          # umask 0;
          open STDIN, '/dev/null'  or die "Can't read /dev/null: $!";
          open STDOUT, ">$fss_work_dir/$fss_next_child_pid-$fss_cmd_count.out" or die "Can't write to $fss_work_dir/$fss_next_child_pid-$fss_cmd_count.out : $!";
          open STDERR, ">$fss_work_dir/$fss_next_child_pid-$fss_cmd_count.err" or die "Can't write to $fss_work_dir/$fss_next_child_pid-$fss_cmd_count.err : $!";
          #replace ARGV
          unless ( do "$fss_work_dir/$fss_next_child_pid.args" ) {
            log_msg("error : could not parse $fss_work_dir/$fss_next_child_pid.args");
            exit(2);
          }
          #this instance is a fss client -> this will ensure return code will be handled in END block
          $fss_instance_type=2;
          #make sure to exit server loop so that child execution continues
          $fss_child_continue=1;
        }
      } else {
        #unknown or already processed request - just close the connection to make sure we do not leak file handles
        close($fss_con);
      }
    }
  }
}

#exit trap - in case of fss children make sure to write the exit code to the designated file
END {
  my $fss_retcode=$?;
  if ( $fss_instance_type == 2 ) {
    log_msg("FAST swupdated Client (pid $pid) ended with rc $fss_retcode");
    open(FSS_RC,">$fss_work_dir/$fss_next_child_pid-$fss_cmd_count.rc.tmp") || die "Can't write to $fss_work_dir/$fss_next_child_pid-$fss_cmd_count.rc.tmp : $!";
    print FSS_RC $fss_retcode;
    close(FSS_RC);
    #close STDOUT and STDERR to make sure files are flushed
    close(STDOUT);
    close(STDERR);
    system("mv $fss_work_dir/$fss_next_child_pid-$fss_cmd_count.rc.tmp $fss_work_dir/$fss_next_child_pid-$fss_cmd_count.rc");
  }
  exit($fss_retcode);
}

my $hostname = exec_cmd_out("/bin/hostname", "silent");
chomp($hostname);
# for NNUC the cla temp dir has to be hostname specific to that the NEW world can pick the correct CLA files to import
$cla_temp_dir=$cla_temp_dir.$hostname.'/';

my $uname = exec_cmd_out("/bin/uname -a", "silent");
chomp($uname);

my $ppid        = exec_cmd_out("ps -o ppid= -p $pid");
chomp($ppid);

# $currentHostname is set to the hostname on which currently actions are done (e.g. upgrade or reboot or stop openais)
my $currentHostName = $hostname;
my $currentHostType = "";
my $defaultHostType = "";

# remember the original paths and args
my $call_string =  "$0 @ARGV";
my $call_script =  "$0";

#strip .swupdated_slow suffix if present
$call_string =~ s/.swupdated_slow//;
$call_script =~ s/.swupdated_slow//;
$file =~ s/.swupdated_slow//;

my $sles_major_version = get_SLES_Major_Version();
# my $sles_minor_version = get_SLES_Minor_Version();  # we do not need SLES Minor version yet, so skip parsing for it.
  
# parse parameters

my ($multiNodeType, $other_node, $node_q, $isMultiNode, $has_a_quorum) = (100, "", "", 0, 0);

initialize_sysinfo_values();

if(defined $ARGV[0]) {
  if ($ARGV[0] eq "-h" || $ARGV[0] eq "--help" || $ARGV[0] eq "-?" ) {
    usage();
  }
}

$is_stmix = stmix_discover();
# on STMIX I need a different location for the logfiles since there is no webservice package
if ($is_stmix == 1) {
  $log_file            = $softgate_log_dir.'update_os-daemon.log';
  $status_local_file   = $softgate_log_dir.'update-status_local.log';
  $status_file         = $softgate_log_dir.'update-status.log';
  $status_file_tmp     = $softgate_log_dir.'update-status.tmp.log';
  $store_package_script= $softgate_dir.'native/os/linux/store_package_unique.sh';
}

check_log_file($log_file);
check_log_file($log_alt_file);
      
print_log_header();

# read options
my $arg_result = GetOptions ("no_self_xtract"          => \$no_self_xtract,
                             "extract"                 => \$extract,
                             "nodaemon"                => \$nodaemon,
                             "forcedaemon"             => \$forcedaemon,
                             "avoidminicib"            => \$avoidminicib,
                             "nowait"                  => \$nowait,
                             "testmode"                => \$testmode,
                             "force"                   => \$forceoption,
                             "isquorum"                => \$isquorum,
                             "is_FI"                   => \$is_FI,
                             "skip_quorum_rpm_upgrade" => \$skip_quorum_rpm_upgrade,
                             "autodeact"               => \$autodeact,
                             "biosupdate"              => \$biosupdate, 
                             "continue_upgrade"        => \$continue_upgrade, 
                             "sg_os_update"            => \$sg_os_update,
                             "deferred_activation"     => \$deferred_activation,
                             "keep_installed_sg"       => \$keep_installed_sg,
                             "eff_vers=s"              => \$installed_effective_version_at_beginning_of_upgrade,
                             "irrc"                    => \$ignore_remote_return_code,
                             "nodecheck=s"             => \$node_reach_test_method,
                             "sshlock_timeout=i"       => \$max_wait_for_ssh_lock_default,
                             "sshexec_timeout=i"       => \$max_wait_for_ssh_client_execution,
                             );

# print STDERR "no_self_xtract=$no_self_xtract\n";
# print STDERR "extract=$extract\n";
# print STDERR "nodaemon=$nodaemon\n";
# print STDERR "autodeact=$autodeact\n";

if ( $extract   == 1 ) {
  $params_remote .= " --extract";
}
if ( $autodeact == 1 ) {
  $params        .= " --autodeact";
  $params_remote .= " --autodeact";
}
if ( $biosupdate == 1 ) {
  $params        .= " --biosupdate";
  $params_remote .= " --biosupdate";
}
if ( $keep_installed_sg == 1 ) {
  $params        .= " --keep_installed_sg";
  $params_remote .= " --keep_installed_sg";
}
if ( $is_FI == 1 ) {
  $params        .= " --is_FI";
}

if(defined $ARGV[0] && ($ARGV[0] eq "ssh")) {
  $exec_remote = 1;
  my $node = "";
  my ($node_is_reachable, $password_is_expired) = (0, 0);
  
  my $remote_cmd = "";
  
  my $i = 1; 
  if(defined $ARGV[$i]) {
    $node = $ARGV[$i];
    $i++;
  }
  
  if(defined $ARGV[$i] ) {
    $remote_cmd = $ARGV[$i];
    $i++;
  }  
  
  if (length($node) > 0 && length($remote_cmd) > 0) {
    my $output = "";
    my $rc = 0;

    $global_rc = wait_for_ssh_lock($sshd_lock_file, $node);
    if ($global_rc == 0) {
      create_lock_file($sshd_lock_file);

      $global_rc = execute_remote_cmd($node, \$node_is_reachable, \$password_is_expired, $remote_cmd, \$output, 1, 60, $ignore_remote_return_code);
      print("$output");
      remove_lock_file($sshd_lock_file);
    }
    log_msg_info("return value of execute_remote_cmd:$global_rc");
  } else {
    log_msg_info("can not start command on remote host: either hostname or command are empty");  
    $global_rc = 1;
  }
  exit($global_rc);
  
} elsif (defined $ARGV[0] && ($ARGV[0] eq "prepare_ssh")) {
  $global_rc = 0; 
  my $max_try = 1;
  my $period  = 10; # secs
  my $node = "";
  my $node_is_reachable = 0;
  my $password_is_expired = 0;

  my $i = 1;
  if(defined $ARGV[$i]) {
    $node = $ARGV[$i];
    $i++;
  } else {
    print("no node is specified! \n");
    exit(1);
  }  
  if(defined $ARGV[$i] ) {
    $max_try = $ARGV[$i];
  }
  $global_rc = prepare_remote_ssh_access($node, $max_try, $period, \$password_is_expired);
  print("prepare_remote_ssh_access status=$global_rc password_is_expired=$password_is_expired \n");
  if ($password_is_expired == 1) {
    print("Password on remote node $node is expired or was never initialized. Please login directly and set the password. \n");
  }
  exit ($global_rc);
  
} elsif (defined $ARGV[0] && ($ARGV[0] eq "cleanup_ssh")) {
  $global_rc = 0;
  my $node = "";
  my $node_is_reachable = 1;

  my $i = 1;
  if(defined $ARGV[$i]) {
    $node = $ARGV[$i];
    $i++;
  } else {
    print("no node is specified! \n");
    exit(1);
  }  

  $global_rc = cleanup_remote_ssh_access($node, $node_is_reachable);
  print("cleanup_remote_ssh_access status=$global_rc \n");
  exit ($global_rc);

} elsif (defined $ARGV[0] && ($ARGV[0] eq "get_adp_host")) {
  my $output = get_node_where_adp_is_running();
  if ($output eq "") {
    $global_rc = $ERROR_CanNotDetermineADPHost;
    $output = "Error: No ADP found"
  }
  print("$output\n");
  exit ($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "check_node_connectivity")) {
  my ($node, $output, $node_is_reachable) = ("", "", 1);
  
  my $i = 1;
  if(defined $ARGV[$i]) {
    $node = $ARGV[$i];
    $i++;
  } else {
    print("no node is specified! \n");
    exit(1);
  }
  $node_reach_test_method = "ping";
  $node_is_reachable = is_node_reachable($node);
  if ($node_is_reachable == 1) {
    $output = "Node $node is reachable via $node_reach_test_method";
    $global_rc = 0;
  } else {
    $output = "Node $node is NOT reachable via $node_reach_test_method";
    $global_rc = 1;
  }
  print("$output\n");

  $node_reach_test_method = "crm";
  $node_is_reachable = is_node_reachable($node);
  if ($node_is_reachable == 1) {
    $output = "Node $node is reachable via $node_reach_test_method";
    $global_rc = 0;
  } else {
    $output = "Node $node is NOT reachable via $node_reach_test_method";
    $global_rc = 1;
  }
  print("$output\n");
  
  exit ($global_rc);  
} elsif (defined $ARGV[0] && ($ARGV[0] eq "create_upgrade_lock")) {
  $global_rc = 0;
  my $wait_time = 300;
  if(defined $ARGV[1] ) {
    $wait_time = $ARGV[1];
  }

  if(is_daemon_running()) {
    log_msg_info("daemon already running");
    exit(1);
  }
  $global_rc = create_upgrade_lock_file();
  log_msg_info("Creating upgrade lock status=$global_rc; waiting $wait_time", 1);
  sleep($wait_time);
  remove_upgrade_lock_file();
  exit ($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "sshadp")) {
  my $node = get_node_where_adp_is_running();
  my ($node_is_reachable, $password_is_expired) = (0, 0);
  my $remote_cmd = "";

  my $i = 1;
  if(defined $ARGV[$i] ) {
    $remote_cmd = $ARGV[$i];
    $i++;
  }

  if (length($node) > 0 && length($remote_cmd) > 0) {
    my $output = "";
    my $rc = 0;

    if ($node eq $hostname) {
      log_msg_info("since ADP host is the same than current host -> execute local");
      # the command is local
      $output = exec_cmd_out($remote_cmd);
      print("$output");
      
    } else {
      # the command has to be executed on a remnote host    
      $rc = wait_for_ssh_lock($sshd_lock_file, $node);
      if ($rc == 0) {
        create_lock_file($sshd_lock_file);

        $global_rc = execute_remote_cmd($node, \$node_is_reachable, \$password_is_expired, $remote_cmd, \$output, 1, 60, $ignore_remote_return_code);
        print("$output");
        remove_lock_file($sshd_lock_file);
      }
      log_msg_info("return value of execute_remote_cmd:$global_rc");
    }
  } else {
    log_msg_info("can not start command on remote host: either no ADP running or command is empty");
    if (length($node) == 0) {
      log_msg_info("no ADP running");
      $global_rc = $ERROR_CanNotDetermineADPHost;
    } else {
      log_msg_info("ADP is running, but another error");
      $global_rc = 1;
    }
  }
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "sshx")) {
  $exec_remote = 1;
  my $remote_cmd = "";
  my $output = "";
  
  my $i = 1; 
  
  if(defined $ARGV[$i] ) {
    $remote_cmd = $ARGV[$i];
    $i++;
  }  
  my $msg = "";
  if (length($remote_cmd) > 0) {
    my ($other_node_is_reachable, $password_is_expired) = (0, 0);

    $global_rc = execute_remote_duplex_cmd($remote_cmd, \$output, 1, \$other_node_is_reachable, \$password_is_expired, $ignore_remote_return_code);
    $msg = "return value of execute_remote_duplex_cmd:$global_rc";
  } else {
    $msg = "can not start command on remote host: command is empty"; 
    print STDERR "$msg\n";    
    $global_rc = 1;
  }
  print("$output");  
  log_msg_info("$msg"); 
  exit($global_rc);
} 
elsif (defined $ARGV[0] && ($ARGV[0] eq "putx")) {
  $exec_remote = 1;
  my $local_path  = "";
  my $remote_path = "";  
  
  my $i = 1; 
  
  if(defined $ARGV[$i] ) {
    $local_path = $ARGV[$i];
    $i++;
  }  
  
  if(defined $ARGV[$i] ) {
    $remote_path = $ARGV[$i];
    $i++;
  }

  my $msg = "";    
  if (length($local_path) > 0 && length($remote_path) > 0) {
    $global_rc = execute_remote_duplex_copy($local_path, $remote_path, "put");
    $msg = "return value of execute_remote_duplex_copy:$global_rc";
  } else {
    $msg = "can not copy to remote host: either local or remote path is empty/missing"; 
    $global_rc = 1;
  }
  log_msg_info("$msg");   
  print STDERR "$msg\n";  
  exit($global_rc);
} 
elsif (defined $ARGV[0] && ($ARGV[0] eq "getx")) {
  $exec_remote = 1;
  my $local_path  = "";
  my $remote_path = "";  
  
  my $i = 1; 
  
  if(defined $ARGV[$i] ) {
    $local_path = $ARGV[$i];
    $i++;
  }  
  
  if(defined $ARGV[$i] ) {
    $remote_path = $ARGV[$i];
    $i++;
  }

  my $msg = "";    
  if (length($local_path) > 0 && length($remote_path) > 0) {
    $global_rc = execute_remote_duplex_copy($local_path, $remote_path, "get");
    $msg = "return value of execute_remote_duplex_copy:$global_rc";
  } else {
    $msg = "can not copy from remote host: either local or remote path is empty/missing"; 
    $global_rc = 1;
  }
  log_msg_info("$msg");   
  print STDERR "$msg\n";  
  exit($global_rc);
}
elsif (defined $ARGV[0] && ($ARGV[0] eq "putq")) {
  $exec_remote = 1;
  my $local_path  = "";
  my $remote_path = "";  
  
  my $i = 1; 
  
  if(defined $ARGV[$i] ) {
    $local_path = $ARGV[$i];
    $i++;
  }  
  
  if(defined $ARGV[$i] ) {
    $remote_path = $ARGV[$i];
    $i++;
  }

  my $msg = "";    
  if (length($local_path) > 0 && length($remote_path) > 0) {
    $global_rc = execute_remote_quorum_copy($local_path, $remote_path, "put");
    $msg = "return value of execute_remote_quorum_copy:$global_rc";
  } else {
    $msg = "can not copy to remote host: either local or remote path is empty/missing"; 
    $global_rc = 1;
  }
  log_msg_info("$msg");   
  print STDERR "$msg\n";  
  exit($global_rc);
}
elsif (defined $ARGV[0] && ($ARGV[0] eq "getq")) {
  $exec_remote = 1;
  my $local_path  = "";
  my $remote_path = "";  
  
  my $i = 1; 
  
  if(defined $ARGV[$i] ) {
    $local_path = $ARGV[$i];
    $i++;
  }  
  
  if(defined $ARGV[$i] ) {
    $remote_path = $ARGV[$i];
    $i++;
  }

  my $msg = "";    
  if (length($local_path) > 0 && length($remote_path) > 0) {
    $global_rc = execute_remote_quorum_copy($local_path, $remote_path, "get");
    $msg = "return value of execute_remote_quorum_copy:$global_rc";
  } else {
    $msg = "can not copy from remote host: either local or remote path is empty/missing"; 
    $global_rc = 1;
  }
  log_msg_info("$msg");   
  print STDERR "$msg\n";  
  exit($global_rc);  
}
elsif (defined $ARGV[0] && ($ARGV[0] eq "put")) {
  $exec_remote = 1;
  my ($node, $node_is_reachable, $local_path, $remote_path) = ("", 1, "", "");
  
  my $i = 1; 
  
  if(defined $ARGV[$i]) {
    $node = $ARGV[$i];
    $i++;
  }
    
  if(defined $ARGV[$i] ) {
    $local_path = $ARGV[$i];
    $i++;
  }  
  
  if(defined $ARGV[$i] ) {
    $remote_path = $ARGV[$i];
    $i++;
  }

  my $msg = "";    
  if (length($node) > 0 && length($local_path) > 0 && length($remote_path) > 0) {
    $global_rc = wait_for_ssh_lock($sshd_lock_file, $node);
    if ($global_rc == 0) {
      create_lock_file($sshd_lock_file);

      $global_rc = execute_remote_copy($node, \$node_is_reachable, $local_path, $remote_path, "put");
      remove_lock_file($sshd_lock_file);
      $msg = "return value of execute_remote_copy:$global_rc";
    }
  } else {
    $msg = "can not copy to remote host: either host or local or remote path is empty/missing"; 
    $global_rc = 1;
  }
  log_msg_info("$msg");   
  print STDERR "$msg\n";  
  exit($global_rc);
}
elsif (defined $ARGV[0] && ($ARGV[0] eq "get")) {
  $exec_remote = 1;
  my ($node, $node_is_reachable, $local_path, $remote_path) = ("", 1, "", "");
  
  my $i = 1; 
  
  if(defined $ARGV[$i]) {
    $node = $ARGV[$i];
    $i++;
  }
    
  if(defined $ARGV[$i] ) {
    $local_path = $ARGV[$i];
    $i++;
  }  
  
  if(defined $ARGV[$i] ) {
    $remote_path = $ARGV[$i];
    $i++;
  }

  my $msg = "";    
  if (length($node) > 0 && length($local_path) > 0 && length($remote_path) > 0) {
    $global_rc = wait_for_ssh_lock($sshd_lock_file, $node);
    if ($global_rc == 0) {
      create_lock_file($sshd_lock_file);

      $global_rc = execute_remote_copy($node, \$node_is_reachable, $local_path, $remote_path, "get");
      remove_lock_file($sshd_lock_file);
    }
    $msg = "return value of execute_remote_copy:$global_rc";
  } else {
    $msg = "can not copy from remote host: either host or local or remote path is empty/missing"; 
    $global_rc = 1;
  }
  log_msg_info("$msg");   
  print STDERR "$msg\n";  
  exit($global_rc);      
} 
elsif (defined $ARGV[0] && ($ARGV[0] eq "sshq")) {
  $exec_remote = 1;
  my $remote_cmd = "";
  my $output = ""; 
  my ($other_node_is_reachable, $password_is_expired) = (0, 0);
   
  my $i = 1; 
  
  if(defined $ARGV[$i] ) {
    $remote_cmd = $ARGV[$i];
    $i++;
  }  
  
  if (length($remote_cmd) > 0) {
    $global_rc = execute_remote_quorum_cmd($remote_cmd, \$output, 1, \$other_node_is_reachable, \$password_is_expired, $ignore_remote_return_code);
    print("$output");
    log_msg_info("return value of execute_remote_quorum_cmd:$global_rc");      
  } else {
    log_msg_info("can not start command on remote host: command is empty");  
    $global_rc = 1;
  }
  exit($global_rc);
}
elsif (defined $ARGV[0] && ($ARGV[0] eq "get_rpm_consistency_status")) {
  my ($node_to_process, $version_to_check) = ("", "");
  my $i = 1;
  if(defined $ARGV[$i] ) {
    $node_to_process = $ARGV[$i];
    $i++;
  }
  if(defined $ARGV[$i] ) {
    $version_to_check = $ARGV[$i];
    $i++;
  }
  
  if (length($node_to_process) == 0) {
    log_msg_info("can not start get_rpm_consistency_status command: no host argument given", 1);
    $global_rc = 1;
  } else {
    if (length($version_to_check) == 0) {
      log_msg_info("can not start get_rpm_consistency_status command: no version argument given", 1);
      $global_rc = 1;
    } else {
      $global_rc = get_rpm_consistency_status($node_to_process, $version_to_check);
      log_msg_info("return value of get_rpm_consistency_status:$global_rc");
    }
  }
  exit($global_rc);
}
elsif (defined $ARGV[0] && ($ARGV[0] eq "reboot")) {
  my @node_to_process = ();
  my ($work_on_active, $work_on_standby, $work_on_quorum, $work_on_all) = (0, 0, 0, 0);
  my $subcmd = 1; # reboot
  my $do_daemonize = 1;
  my $verbose = 0;

 for(my $i = 1; $i < 5; $i++) {
    if(defined $ARGV[$i] ) {
      @node_to_process = (@node_to_process, $ARGV[$i]);
      log_msg_info("checking argument $ARGV[$i]");

      if ($ARGV[$i] eq "a" )  {
        $work_on_active = 1;
      }
      if ($ARGV[$i] eq "s" )  {
        $work_on_standby = 1;
      }
      if ($ARGV[$i] eq "q" )  {
        $work_on_quorum = 1;
      }
      if ($ARGV[$i] eq "all" )  {
        $work_on_all = 1;
      }
    }
  }

  if (scalar(@node_to_process) > 0) {
    $global_rc = execute_reboot($subcmd, $do_daemonize, $verbose, $work_on_active, $work_on_standby, $work_on_quorum, $work_on_all);
    log_msg_info("return value of execute_reboot:$global_rc");
  } else {
    log_msg_info("can not start reboot command: no host argument given", 1);
    $global_rc = 1;
  }
  exit($global_rc);
}
elsif (defined $ARGV[0] && ($ARGV[0] eq "shutdown")) {
  my @node_to_process = ();
  my ($work_on_active, $work_on_standby, $work_on_quorum, $work_on_all) = (0, 0, 0, 0);
  my $subcmd = 0;  # shutdown
  my $do_daemonize = 1;
  my $verbose = 0; 

  for(my $i = 1; $i < 5; $i++) {
    if(defined $ARGV[$i] ) {
      @node_to_process = (@node_to_process, $ARGV[$i]);
      if ($ARGV[$i] eq "a" )  {
        $work_on_active = 1;
      }
      if ($ARGV[$i] eq "s" )  {
        $work_on_standby = 1;
      }
      if ($ARGV[$i] eq "q" )  {
        $work_on_quorum = 1;
      }
      if ($ARGV[$i] eq "all" )  {
        $work_on_all = 1;
      }
    }
  }

  if (scalar(@node_to_process) > 0) {
    $global_rc = execute_reboot($subcmd, $do_daemonize, $verbose, $work_on_active, $work_on_standby, $work_on_quorum, $work_on_all);
    log_msg_info("return value of execute_reboot:$global_rc");
  } else {
    log_msg_info("can not start shutdown command: no host argument given", 1);
    $global_rc = 1;
  }
  exit($global_rc);
}
elsif (defined $ARGV[0] && ($ARGV[0] eq "stop_openais")) {
  my @node_to_process = ();
  my ($work_on_active, $work_on_standby, $work_on_quorum, $work_on_all) = (0, 0, 0, 0);
  my $subcmd = 2; # stop openais
  my $do_daemonize = 0;
  my $verbose = 1;

  for(my $i = 1; $i < 5; $i++) {
    if(defined $ARGV[$i] ) {
      @node_to_process = (@node_to_process, $ARGV[$i]);
      if ($ARGV[$i] eq "a" )  {
        $work_on_active = 1;
      }
      if ($ARGV[$i] eq "s" )  {
        $work_on_standby = 1;
      }
      if ($ARGV[$i] eq "q" )  {
        $work_on_quorum = 1;
      }
      if ($ARGV[$i] eq "all" )  {
        $work_on_all = 1;
      }
    }
  }

  if (scalar(@node_to_process) > 0) {
    $global_rc = execute_reboot($subcmd, $do_daemonize, $verbose, $work_on_active, $work_on_standby, $work_on_quorum, $work_on_all);
    log_msg_info("return value of execute_reboot:$global_rc");
  } else {
    log_msg_info("can not stop openais: no host argument given", 1);
    $global_rc = 1;
  }
  exit($global_rc);
}
elsif (defined $ARGV[0] && ($ARGV[0] eq "shutdownstatus")) {
  # use the LSB service return codes ->  service not running (unused)
  $global_rc = 3;

  if(is_running($shutdown_lock_file) != 0) {
    log_msg_info("another reboot/shutdown daemon action is currently running");
    # service up and running
    $global_rc = 0;
  }
  log_msg_info("shutdownstatus rc=$global_rc");
  exit($global_rc);
}
elsif (defined $ARGV[0] && ($ARGV[0] eq "redistribute_licenses")) {
  my $locale = "en";

  $global_rc = redistribute_licenses($locale);
  log_msg_info("return value of redistribute_licenses: $global_rc");
  exit($global_rc);
}
elsif (defined $ARGV[0] && ($ARGV[0] eq "activate_license")) {
  my $i = 1; 
  my $locale = "";
  if(defined $ARGV[$i] ) {
    $locale = $ARGV[$i];
    $i++;
  }

  $global_rc = activate_license($locale);
  log_msg_info("return value of activate_license: $global_rc");      
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "configure_snmp")) {
  my $devicetype = "";
  if(defined $ARGV[1] ) {
    $devicetype = $ARGV[1];
  }
  
  $global_rc = configure_snmp($devicetype);
  log_msg_info("return value of configure_snmp: $global_rc");
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "configure_mek")) {
  my $mek = "";
  if(defined $ARGV[1] ) {
    $mek = $ARGV[1];
    if (length($mek) != 16) {
      log_msg_info ("mek is not 16 chars", 1);
      usage();
    } else {
      # mek is 16 chars
      $global_rc = configure_mek($mek);
    }
  } else {
    log_msg_info ("mek is not given", 1);
    usage();
  }

  log_msg_info("return value of configure_mek: $global_rc");
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "configure_syslog")) {
  my $syslogsrc = "";
  my $epoch=0;
  if(defined $ARGV[1] && defined $ARGV[2]) {
    $syslogsrc = $ARGV[1];
	$epoch = $ARGV[2];
  } else {
    log_msg_info ("Syslog config is not given", 1);
    usage();
  }
  $global_rc = configure_syslog($syslogsrc,$epoch);
  log_msg_info("return value of configure_syslog: $global_rc");
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "get_interface_list")) {
  $global_rc = get_interface_list();
  log_msg_info("return value of get_interface_list:$global_rc");      
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "set_ui_mode")) {
  my $mode = 0;
  if (defined $ARGV[1]) {
    $mode = $ARGV[1];
  } else {
    log_msg_info ("mode is not given", 1);
    usage();
  }
  if ( "$mode" eq "0" || "$mode" eq "1" || "$mode" eq "2" ) {
    my ($tls_version) = (2);
    if (defined $ARGV[2]) {
      $tls_version = $ARGV[2];
    }
    $global_rc = set_ui_mode($mode, $tls_version);
    log_msg_info("return value of set_ui_mode:$global_rc");
  } else {
    $global_rc = 1;
    log_msg_info ("allowed values for mode are: 0 or 1 or 2", 1);
  }
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "get_timezone")) {
  $global_rc = get_timezone();
  log_msg_info("return value of get_timezone:$global_rc");
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "replicate_ssh_banner")) {
  $global_rc = replicate_ssh_banner();
  log_msg_info("return value of replicate_ssh_banner:$global_rc");
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "set_timezone")) {
  my $tzone = "";
  if (defined $ARGV[1]) {
    $tzone = "$ARGV[1]";
  } else {
    log_msg_info ("tzone is not given", 1);
    usage();
  }
  if ( "$tzone" ne "" ) {
    $global_rc = set_timezone($tzone);
    log_msg_info("return value of set_timezone:$global_rc");
  } else {
    $global_rc = 1;
    log_msg_info ("tzone must not be empty", 1);
  }
  exit($global_rc);  
} elsif (defined $ARGV[0] && ($ARGV[0] eq "get_version_info")) {
  $global_rc = get_version_info();
  log_msg_info("return value of get_version_info:$global_rc");      
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "get_upd_status")) {
  $global_rc = get_upd_status();
  log_msg_info("return value of get_upd_status:$global_rc");
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "configure_cib")) {
  $global_rc = configure_cib($ARGV[1], $ARGV[2]);
  log_msg_info("return value of configure_cib:$global_rc");      
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "isolate_node")) {
  $global_rc = isolate_node($ARGV[1]);
  log_msg_info("return value of isolate_node:$global_rc");
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "reimport_licences")) {
  $global_rc = reimport_the_licences_to_cla();
  log_msg_info("return value of reimport_the_licences_to_cla:$global_rc");
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "configure_cib_local")) {
  $global_rc = configure_cib_local();
  log_msg_info("return value of configure_cib_local:$global_rc");      
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "gen_os_transfer_file_for_sg")) {
  my ($path_to_sg_list_file, $path_to_stmix_list_file) = ("", "");
  if (defined $ARGV[1]) {
    $path_to_sg_list_file = "$ARGV[1]";
  } else {
    log_msg_info ("error: path to Softgate list file is not given", 1);
    usage();
  }
  if (defined $ARGV[2]) {
    $path_to_stmix_list_file = "$ARGV[2]";
  } else {    
    log_msg_info ("path to STMIX list file is not given, assuming empty list", 0);
  }
  $global_rc = generate_os_transfer_file_for_sg($path_to_sg_list_file, $path_to_stmix_list_file);
  log_msg_info("return value of generate_os_transfer_file_for_sg:$global_rc");
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "cleanup_after_os_sg_update")) {
  $global_rc = cleanup_after_os_sg_update();
  log_msg_info("return value of cleanup_after_os_sg_update:$global_rc");
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "transfer_os_from_host")) {
  my $host = "";
  my $workdir = "/tmp/sg_os";
  my $local_path = "";
  my $remote_path = "";

  if (! -d $webservice_log_dir) { mkdir($webservice_log_dir); }
  
  if (defined $ARGV[1]) {
    $host = $ARGV[1];
  } else {
    log_msg_info ("host argument is not given", 1);
    usage();
  }
  if (defined $ARGV[2]) {
    $local_path = $ARGV[2];
  } else {
    log_msg_info ("local_path argument is not given", 1);
    usage();
  }
  if (defined $ARGV[3]) {
    $remote_path = $ARGV[3];
  } else {
    log_msg_info ("remote_path argument is not given", 1);
    usage();
  }
  if (defined $ARGV[4]) {
    $workdir = $ARGV[4];
  } else {
    log_msg_info ("workdir argument is not given, using default $workdir");
  }
  
  $global_rc = transfer_os_from_host($host, $workdir, $local_path, $remote_path);
  log_msg_info("return value of transfer_os_from_host:$global_rc");
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "cancel_os_transfer")) {
  $global_rc = cancel_os_transfer();
  log_msg_info("return value of cancel_os_transfer:$global_rc");
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "prepare_sftp_host")) {
  my ($sentinel, $sftp_transfer_user) = ("", "");
  my ($user_id, $group_id) = (0, 0);

  if (! -d $webservice_log_dir) { mkdir($webservice_log_dir); }

  if (defined $ARGV[1]) {
    $sentinel = $ARGV[1];
  } else {
    log_msg_info ("sentinel argument is not given", 1);
    usage();
  }
  if (defined $ARGV[2]) {
    $sftp_transfer_user = $ARGV[2];
  } else {
    log_msg_info ("user argument is not given", 1);
    usage();
  }
  if (defined $ARGV[3]) {
    $user_id = $ARGV[3];
    if ($ARGV[3] eq "nouid") { $user_id = -1; } else { $user_id = 0 + $ARGV[3]; }
  } else {
    log_msg_info ("uid argument is not given", 1);
    usage();
  }
  if (defined $ARGV[4]) {
    $group_id = $ARGV[4];
  } else {
    log_msg_info ("gid argument is not given", 1);
    usage();
  }

  $global_rc = prepare_sftp_host($sentinel, $sftp_transfer_user, \$user_id, $group_id);
  log_msg_info("created uid: $user_id", 1);
  
  log_msg_info("return value of prepare_sftp_host:$global_rc");
  exit($global_rc);
  
} elsif (defined $ARGV[0] && ($ARGV[0] eq "test")) {
  log_msg_info("starting test frame");

  # test case 1: BIOS activation
  # check_if_local_bios_activation_has_to_be_enforced();
  # log_msg_info("test data: biosupdate=$biosupdate");
  
  # test case 2: Assistant share migration on Simplex
  #my ($use_tiptoe, $reboot_if_tiptoe_fails, $use_reduce_downtime, $do_insserv_rm) = (1, 1, 0, 0);
  #stop_openais_service("local", 1, $use_tiptoe, $reboot_if_tiptoe_fails, $use_reduce_downtime, $do_insserv_rm);
  #my $isMultiNode = 0;
  #migrate_assistant_share_from_var_to_pgsql_partition($isMultiNode);

  # test case 3: BIOS activation
  #remove_remote_status_file();

  # test case 4: asyaps post install
  #$global_rc=execute_yaps_prepostinstall("-u");
  #log_msg_info("global_rc=$global_rc");
  #$global_rc=execute_yaps_postinstall("-u");

  # test case 5
  #$hotfix_mode = 1;
  #$rmx_reboot  = 0;
  #$isMultiNode = 1;
  # my $rc_status = is_resource_running("rsc4k_dtb");
  # if ($rc_status == 1) { stop_resource("rsc4k_dtb"); }
  # $global_rc = start_all_stopped_resources_if_necessary(1, 0, $rc_status, "primary");

  # test case 6
  # create_lock_file($setui_lock_file);
  # sleep(120);

  # test case 8
  # print_package_info("before");

  # test case 9
  # my ($upgrade_version, $multiNodeType, $isMultiNode, $has_a_quorum) = ("V6_R2.14.3", 60, 1, 0);
  # $global_rc = get_rpm_consistency_status_for_all_nodes($upgrade_version, $multiNodeType, $isMultiNode, $has_a_quorum);

  #my $takeover_mode = 0; # RLC
  #$global_rc = takeover_files($takeover_mode);
  #$global_rc = wait_for_NUC_PREP();
  #$global_rc = determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 1, "/tmp/haipsetup.xml");
  #$global_rc = store_hotfix_into_instmedia();
  #$global_rc = trigger_watchdog();

  #my $password_is_expired = 0;
  #check_local_password_validity(\$password_is_expired);

  #$global_rc = do_NUC_data_upgrade(0, 0);
  #$global_rc = do_NUC_data_sg_upgrade();

  # $global_rc = wait_for_remote_node_world_appearance($ARGV[1], "OLD");

  # my $hsr_status = kill_via_pidfilewait_till_hsr_goes_down();
  # kill_via_pidfile($def_act_lock_file);
  # $global_rc = prepare_local_ssh_credentials( "os_update", $sftp_transfer_user, 1005, 1000);

  # my $uid = -1;
  #$global_rc = create_linux_user("os_transfer", \$uid, 1000);

  # get_current_real_version_without_hotfixnumber();
  # $global_rc = stop_openais_service("local", 5, 1, 1, 0, 1);
  
  #set_internal_upgrade_step($STEP_platform_upgrade_started, $defaultHostType, $hostname);
  #sleep(3);
  #set_internal_upgrade_step($STEP_preparation, $defaultHostType, $hostname);
  #sleep(5);
  #set_internal_upgrade_step($STEP_stop_openais_t3, "Active", $hostname);
  #sleep(20);
  #set_internal_upgrade_step($STEP_remote_installation, "Active", $hostname);
  #sleep(30);
  #set_internal_upgrade_step($STEP_platform_upgrade_success, $defaultHostType, $hostname);
  #sleep(3);

  # check_for_controlled_NUC_activation_for_SurvSoftGates();

  # my $rc = write_combined_upgrade_status("/var/log/webservice/update-status.tmp.log", 60, 1, "Katrina-A", "");
  # $global_rc = determine_prerequisites_for_keep_sg($ARGV[1]);
  # $global_rc = copy_filelist_content($generic_upgrade_dir.'files_to_copy_A*.conf');

  # $global_rc = concatFileAndMove ("/tmp/a", "/tmp/b", "processed");

  # $global_rc = start_resource("rsc4k_dtb");
  # $global_rc = is_DTB_configured();
  #$upgrade_version = "V7_R2.3.4";
  #$hotfix_mode = 1;
  # $global_rc = copy_upgrade_log($upgrade_version);
  #$global_rc = wait_for_successful_startup(1, 0, 0, 50, 0, "secondary", $max_wait_for_up, 0);
  # $global_rc = is_yaps_version_identical($yaps_rpm_file);

  #$global_rc = check_for_major_release_update($ARGV[1], $ARGV[2]);
  #$global_rc = entgw_discover();

  # $global_rc = wait_for_cluster_node_reappearance($ARGV[1], 10, 30);
  #$global_rc = dump_file_to_log("/opt/cla/log/error.log");
  
  #get_ali();
  #get_ali_p();

  #$global_rc = reimport_the_licences_to_cla();
  # $global_rc = postprocess_sg_configdata_tls_version("/var/opt/soco/config");
  # $global_rc = postprocess_ha_dhcp_upper_range();
  # $global_rc = postprocess_portal_configdata_tls_version();
  # $global_rc = is_integrated_isg_softgate();
  # $global_rc = check_for_major_release_update_in_new_world();
  # $global_rc = is_SBC_configured();
  # $global_rc = log_iproute2_table("test");
  # $global_rc = is_service_running($ARGV[1]);
  # $global_rc = stop_openais_service("local", 2, 1, 1, 1, 1);
  # $global_rc = verify_md5sum($ARGV[1], $ARGV[2]);
  # $global_rc = stop_openvpn_service();
  # $global_rc = restart_systemd_daemon();
  # $global_rc = update_packages_with_rpm("/tmp/plthf/", 1, 1);
  # $global_rc = stop_cla_service();
  # $global_rc = NUC_copy_other($license_storage_dir, $soco2_data_dir);
  $global_rc = erase_obsolete_packages();
  #  $global_rc = patch_stmix_checkNecessaryFreeDiskSpaceForOSTransfer_file_via_inventory();
  # get_swupdated_path_for_OS_update();
  log_msg_info("leaving test frame rc=$global_rc");
  exit($global_rc);
} elsif (defined $ARGV[0] && ($ARGV[0] eq "hfactivate")) {
  $hotfix_mode = 1;
  $params        .= " $ARGV[0]";
  $params_remote .= " $ARGV[0]";
  log_msg_info("Hotfix activation mode");  
} elsif (defined $ARGV[0] && ($ARGV[0] eq "hfdeactivate")) {
  $hotfix_mode = 2;
  $params        .= " $ARGV[0]";
  $params_remote .= " $ARGV[0]";
  log_msg_info("Hotfix deactivation mode");   
} elsif (defined $ARGV[0] && defined $ARGV[1] && ($ARGV[0] eq "extract") && ($ARGV[1] eq "hfactivate")) {
  $hotfix_mode = 1;
  $extract     = 1;
  $params        .= " $ARGV[1]";
  $params_remote .= " $ARGV[1]";
  if (defined $ARGV[2] && ($ARGV[2] eq "autodeact") ) {
    $autodeact = 1;
    log_msg_info("autodeact mode (deprecated but still supported syntax using autodeact instead of --autodeact)");
  }
  log_msg_info("Hotfix activation mode (deprecated but still supported syntax using extract instead of --extract)");  
} elsif (defined $ARGV[0] && defined $ARGV[1] && ($ARGV[0] eq "extract") && ($ARGV[1] eq "hfdeactivate")) {
  $hotfix_mode = 2;
  $extract     = 1;
  $params        .= " $ARGV[1]";
  $params_remote .= " $ARGV[1]";
  log_msg_info("Hotfix deactivation mode (deprecated but still supported syntax using extract instead of --extract)"); 
} elsif (defined $ARGV[0] && !defined $ARGV[1] && ($ARGV[0] eq "extract") ) {
  $extract     = 1;
  log_msg_info("platform upgrade (deprecated but still supported syntax using extract instead of --extract)");   
}
elsif ( defined $ARGV[0] ) {
  print STDERR "unknown argument or sub command:  $ARGV[0]\n";
  usage();
} else {
  log_msg_info("No argument or sub command given -> assuming platform upgrade"); 
}

if ($nodaemon == 0) {
  if ($continue_upgrade == 1) {
    log_msg_info("Skipping check_preconditions() in continue mode, where openais and assistant may not run");
  } else {
  # assistant dirs must not be checked in continue mode where the openais probably is not running
    if ($sg_os_update == 0) {
      check_preconditions ();
    }
  }
}

if ($extract) {
  # write to syslog only once (in the extract phase))
  log_msg_info("*** Starting platform software upgrade (continue mode = $continue_upgrade)", 0, 1); 
} else {
  log_msg_info("*** Starting platform software upgrade (continue mode = $continue_upgrade)", 0, 0); 
}


# make a copy to original daemon location
if (! -e $self and -e $old_self) {
  log_msg("There is currently no swupdated at the original location $self (webservice java part obviously moved to old) -> make a back copy");
  exec_cmd("/bin/cp -vfL $old_self $self");
  log_msg("copied file $old_self to $self to allow putx,getx,putq,getq also during upgrade");
}
      

create_call_file();

if ($is_stmix == 0) {
  migrate_current_hotfix_dir();
}

if($hotfix_mode > 0) {
  log_msg_info("start in hotfix mode, setting defaults for the upgrade flags (setting all to 0)");
  $rmx_reboot = 0;
  $host_reboot = 0; 
  $upgrade_with_completely_stopped_ha = 0; 
  $upgrade_with_full_drbd_sync        = 0;
  $full_drbd_sync_flag_really_read    = 0;
  $upgrade_with_reduce_downtime       = 0;
  $upgrade_via_image                  = 0;
  
  
  if($hotfix_mode == 1) {
    if ( $autodeact == 1) {
      log_msg_info("hotfix will be automatically deactivated if a package installation failure occurs");
    } else {
      log_msg_info("hotfix will not be automatically deactivated if a package installation failure occurs");    
    }
  }
}

if(!$nodaemon) {
  if ($is_FI == 0) {
    log_msg_info("start in daemon mode");
    if(is_daemon_running()) {
      log_msg_info("daemon already running, exiting");
      write_status(3, "update daemon already running");
      exit(0);
    }

    chdir '/'                or die "Can't chdir to /: $!";
    # umask 0;
    open STDIN, '/dev/null'  or die "Can't read /dev/null: $!";
    open STDOUT, '/dev/null' or die "Can't write to /dev/null: $!";
    open STDERR, '/dev/null' or die "Can't write to /dev/null: $!";
    defined($pid = fork)     or die "Can't fork: $!";
    if($pid) {
      $pid = $$;
      log_msg("exiting parent process (daemonizing)");
      exit(0);
    } else {
      sleep(1);
      $pid = $$;
      log_msg("proceeding child process (daemonized)");
    }
    setsid                   or die "Can't start a new session: $!";

    $SIG{'INT' } = 'interrupt';
    $SIG{'HUP' } = 'interrupt';
    $SIG{'ABRT'} = 'interrupt';
    $SIG{'QUIT'} = 'interrupt';
    $SIG{'TRAP'} = 'interrupt';
    $SIG{'STOP'} = 'interrupt';
    $SIG{'TERM'} = 'interrupt';
  } else {
    log_msg_info("start in first install mode, therefore skipping daemonization");
  }
  create_upgrade_lock_file();
}

# enable sysinfo status file writing
if ($is_stmix == 0) {
  initialize_sysinfo_for_upgrade();
}

remove_status_files();

$is_dscxl   = dscxl_discover();
$is_eco     = eco_discover();
$is_entgw   = entgw_discover();
$is_stmix   = stmix_discover();
$is_cots    = cots_discover();
$is_virtual = virtual_discover();
$is_manager = manager_discover();
discover_HD_layout();

if (!$nodaemon) {
  if ($is_stmix == 0) {
    log_msg_info("*** removing openais service from sys5 startup to keep it down if an unexpected reboot happens");
    execute_ha_rm_insserv();
  }
  $defaultHostType = "Active";  
} else {
  if ($isquorum == 1) {
    $defaultHostType = "Quorum"; 
  } else {
    $defaultHostType = "Standby";
  }
}
$currentHostType=$defaultHostType;
$UpgradeRunning = "yes";

set_internal_upgrade_step($STEP_platform_upgrade_started, 1, $defaultHostType, $hostname);

if ($is_dscxl == 1 && -d $dscxlbios_activation_dir) {
  log_msg_info("because of DSCXL hardware determine a possibly not activated BIOS file");
  my $output = exec_cmd_out("ls -l $dscxlbios_activation_dir");
  log_msg_info($output);
}

log_msg("trying to reduce CPU and IO priority with ionice and renice");
if ($is_stmix == 0) {
  exec_cmd($prio_script_ws . " $pid");
} else {
  # on STMIX HW use the re_prio script from SG
  exec_cmd($prio_script_sg . " $pid");
}

if (-e $def_act_lock_file and $sg_os_update == 1) {
  log_msg("old SG OS activation lockfile ($def_act_lock_file) exists -> I have to kill the old instance to prevent double NUC reboot");
  kill_via_pidfile($def_act_lock_file);
  remove_lock_file($def_act_lock_file);
}

removeDir($tmp_rpm_dir);
if ($is_stmix == 0) {
  stop_nscp_service();
  cleanup_sysrestore_files();
  copy_important_ha_config_files();
}

my ($force_reboot_of_host, $force_reboot_of_RMX, $force_completely_stopped_ha, $force_full_drbd_sync, $force_reduce_downtime, $force_upgrade_via_image) = (0, 0, 0, 0, 0, 0);

log_msg_info("determine current version at begin of upgrade");
$installed_version_at_beginning_of_upgrade = get_current_real_version();
log_msg_info("real version: $installed_version_at_beginning_of_upgrade");
log_msg_info("effective version: $installed_effective_version_at_beginning_of_upgrade");
$running_version = $installed_version_at_beginning_of_upgrade;


$global_activation_rc = determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 1);
log_msg_info("multi node values:  rc=$global_activation_rc multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q");

if (!$nodaemon) {
  # bios check as pre-requisite for upgrade has only to be called on active node, it will perform a remote check
  my $bios_check_status = execute_bios_check($isMultiNode, 1, 120);
  if ($bios_check_status != 0) {
      log_msg_info("BIOS check for DSCXL2 failed, exiting");
      write_short_status($ERROR_BIOS_CheckFailed, "BIOS check for DSCXL2 failed, exiting");
      cleanup_after_upgrade();
      $UpgradeRunning = "no";
      finalize_sysinfo_values();
      exit(1);
  }
}

# remove the possible remnants of a previous upgrade cancel
removeFile($cancel_upgrade_file);

# in case of a crash ws would start on the standby and report an old status file if we do not delete it remotely
if (!$nodaemon) {
  remove_remote_status_file();
}

# check if the standby DSCXL machine would enforce a BIOS update, we need this info to know if the standby will reboot
if ($isMultiNode == 1) {
  if (!$nodaemon) {  
    $biosupdate_remote = check_if_remote_bios_activation_would_be_enforced();
  }
}

# check if the bios update has to be enforced (redmine story #10)
if ($biosupdate_remote == 1 || check_if_local_bios_activation_has_to_be_enforced()) {
  log_msg_info("*** enforcing activation of the BIOS for DSCXL2 hw (activation option may have been lost by webservice)");
  $biosupdate = 1
  }

retrieve_java_versions("before");

if ($is_stmix == 0) {
  # initialize the current openais status
  if (is_openais_service_running() == 1) {
    $local_ha_has_been_stopped = 0;
  } else {
    $local_ha_has_been_stopped = 1;
  }
}

log_msg_info("disable the remote execution timeout from $max_wait_for_ssh_client_execution to 0 for upgrade context");
$max_wait_for_ssh_client_execution = 0;

set_internal_upgrade_step($STEP_preparation, 1, $defaultHostType, $hostname);

if ( $continue_upgrade == 1) {
  log_msg_info("continuing an interrupted platform upgrade");
  my $step = get_callstatus_step();
  if ($step != 0) {
    log_msg_info("continuing at interrupt point: $step");
    if ($continue_upgrade_iteration > $max_continue_upgrade_iteration) {
      $global_activation_rc   = $ERROR_ExceededMaxNumberOfPltUpgradeRetries; 
      set_activation_status_text("number of plt upgrade retries $continue_upgrade_iteration exceeds max number of allowed retries $max_continue_upgrade_iteration");
      log_msg_info("$activation_status_text -> cancel upgrade");
      goto FAILURE_SPOT1;
    }

    if ($step == $Wiederaufsetzpunkt1) {
      log_msg_info("starting local openais service on active node to be able to continue at interrupt point: $step");
      start_openais_service("local", 0);
      # restore the upgrade flags read from the call status file 
      $upgrade_with_completely_stopped_ha = $continue_upgrade_with_completely_stopped_ha;
      $upgrade_with_full_drbd_sync        = $continue_upgrade_with_full_drbd_sync;
      $upgrade_with_reduce_downtime       = $continue_upgrade_with_reduce_downtime;
      $upgrade_via_image                  = $continue_upgrade_via_image;

      goto CONTINUE_SPOT1; 
    }
    elsif ($step == $Wiederaufsetzpunkt2) {
      log_msg_info("interrupt point2 was stopping openais on all involved machines. in this case we just can re-start upgrade from the beginning");
    } 
    elsif ($step == $Wiederaufsetzpunkt3) {
      log_msg_info("interrupt point3 was stopping openais on the active node. in this case we just can re-start upgrade from the beginning and avoid any remote updates which are already succsessfully done");
      $remote_updates_were_already_successfully_done_by_previous_update = 1;
    }
    elsif ($step == $Wiederaufsetzpunkt4) {
      log_msg_info("interrupt point4 was rebooting the active node after type 3 update -> CiB has still to be generated");
      log_msg_info("this Wiederaufsetzpunkt should only be used with command configure_cib_local, should never occur here !!!");
    }
    elsif ($step == $Wiederaufsetzpunkt5) {
      log_msg_info("interrupt point5 was rebooting the active node after type 3 NUC update -> fallback of all worlds has to be initiated");
      log_msg_info("this Wiederaufsetzpunkt should only be used with command configure_cib_local, should never occur here !!!");
    }
    else {
      log_msg_info("Error: unknown interrupt point: $step");
      log_msg_info("Continuing with complete upgrade");
    }
  }
}

if(!$nodaemon && $extract) {
      # takeover from Assistant must only be done on the active side, the files will be replicated with the /var/opt/soco2
      # and it must only be done in the first upgrade invocation with extract. The second invocation with the extracted daemon must not takeover again.
      # and it must not be done in continue mode, where the openais (and Assi) may not run
      my $takeover_mode = 0; # plt upgrade
      if($hotfix_mode) {
        $takeover_mode = 1; # plt hotfix upgrade
      }
      if ($continue_upgrade == 1) {
        log_msg_info("Skipping takeover_files() because in continue mode, where openais and assistant may not run");
      } else {
        $global_activation_rc = takeover_files($takeover_mode);
        if ($global_activation_rc != 0) {
          set_activation_status_text($ERROR_text{$ERROR_CanNotCopyAssistantFiles});
        }
      }
  }

# check for attended NUC activation for Surv. Softgates
check_for_controlled_NUC_activation_for_SurvSoftGates();

if($global_activation_rc == 0) {
  $global_activation_rc = do_upgrade($force_reboot_of_host, $force_reboot_of_RMX, $force_completely_stopped_ha, $force_full_drbd_sync, $force_reduce_downtime, $force_upgrade_via_image, 0, $multiNodeType, $isMultiNode, $other_node, $node_q);
  if ($global_activation_rc != 0) {
    if ($hotfix_mode == 1 && $autodeact == 1) {
      if (($global_activation_rc == $ERROR_RemoteUpdateError) || ($host_reboot == 1) ) {
        log_msg_info("HF activation failed but skipping auto deact because reason for failure was remote node update failure rc=$global_activation_rc or host reboot");
      } else {        
        $global_autodeactivation_rc = prepare_fallback_hotfix($installed_version_at_beginning_of_upgrade);
        if ($global_autodeactivation_rc == 0) {
          # if the first activation had detected that a Host reboot is needed, I have to make the reboot also after the fallback auto deactivation
          $force_reboot_of_host        = $host_reboot;
          $force_reboot_of_RMX         = $rmx_reboot;
          $force_completely_stopped_ha = $upgrade_with_completely_stopped_ha;
          $force_full_drbd_sync        = $upgrade_with_full_drbd_sync;
          $force_reduce_downtime       = $upgrade_with_reduce_downtime;
          $force_upgrade_via_image     = $upgrade_via_image;
          $full_drbd_sync_flag_really_read = 0;
          
          # go into hf deact mode
          $hotfix_mode = 2;
          # enable the xtract flag again
          $extract = 1;
         
          $global_autodeactivation_rc = do_upgrade($force_reboot_of_host, $force_reboot_of_RMX, $force_completely_stopped_ha, $force_full_drbd_sync, $force_reduce_downtime, $force_upgrade_via_image, 1, $multiNodeType, $isMultiNode, $other_node, $node_q);
          
          if ($global_autodeactivation_rc == 0) {
            set_activation_status_text("Activation failed but Autodeact fallback succeeded. You may retry HF activation.");
          } else {
            set_activation_status_text("Activation failed and Autodeact fallback also failed. You may retry HF activation.");
          }
        } else {
          set_activation_status_text("Activation failed and Autodeact failed because HF file missing");
        }
      }
    } else {
        set_activation_status_text("Activation failed");
    }
  } else {
    # activation success
    set_activation_status_text("");
  }
}

# handle the openais on standby and quorum side from the active side
if (!$nodaemon && $upgrade_with_completely_stopped_ha && $activation_has_been_avoided == 0 && $is_stmix == 0) {
  my $enforce_remote_insserv = 0;
  my $is_failure = 0;
  if ($upgrade_via_image == 1) {
    log_msg_info("NUC: insserv flag for B / Q has to be set");
    $enforce_remote_insserv = 1;
  }
  

  log_msg_info("node_A on non_NUC activations with type 3 (complete down)");
   
  if ($quorum_activation_rc != 0 || $standby_activation_rc != 0 ) {
    log_msg_info("*** quorum or standby error -> set failure mode for configure_cib");
    $is_failure = 1;
  }

  if ( $is_failure == 0 ) {
    if (has_quorum($node_q, $multiNodeType) == 1) {
      log_msg_info("*** start of openais on quorum due to special HA upgrade (type3 update)");

      set_internal_upgrade_step($STEP_start_openais_q_t3, 1, "Quorum", $node_q);

      $global_rc = start_openais_service("quorum", $enforce_remote_insserv, $is_failure);
      if ($global_rc != 0) {
        $global_activation_rc = $ERROR_StartingOpenaisFailed;
        set_activation_status_text("Starting Openais for the quorum node failed");
      }
    }
    if ($isMultiNode == 1) {
      if ($upgrade_with_reduce_downtime == 1 && $isMultiNode == 1) {
        log_msg_info("*** skipped start of openais on the other (standby) side for special HA upgrade (type3 update) due to reduce downtime feature");
      } else {

        log_msg_info("*** start of openais on the other (standby) side due to special HA upgrade (type3 update)");

        set_internal_upgrade_step($STEP_start_openais_s_t3, 1, "Standby", $other_node);

        $global_rc = start_openais_service("remote", $enforce_remote_insserv, $is_failure);
        if ($global_rc != 0) {
          $global_activation_rc = $ERROR_StartingOpenaisFailed;
          set_activation_status_text("Starting Openais for the standby node failed");
        }
      }
    }
  } else {
    if ($upgrade_via_image == 1) {
      log_msg_info("error case (at least one remote node error), wiederaufsetzpunkt 5 will get the world back into operation in case of NUC");
    } else {
      log_msg_info("error case (at least one remote node error) in non-NUC scenario -> dont know what to do ????");
    }
  }
}

# check if HA has been stopped and due to some reasons not yet started again
if($local_ha_has_been_stopped == 1 && $is_stmix == 0) {
  if ($nodaemon && $upgrade_with_completely_stopped_ha) {
    log_msg_info("*** skipping start of openais due to special HA upgrade on the standby/Quorum side (will be done later from the active side)");
  } else {
    if ($host_reboot == 1) {
      log_msg_info("*** skipping start of openais due to Host reboot request or NUC reboot");
    } else {
      log_msg_info("*** start of openais on local machine (type 1/2/3)");
      $global_rc = start_openais_service("local", 0);
      $local_ha_has_been_stopped = 0;
    }
  }
}

if ($is_stmix == 0) {
  my $drbd_output = exec_cmd_out("/usr/sbin/drbd-overview");
  log_msg_info("drbd status after installation:");
  log_msg_info("$drbd_output");
}

create_call_file();
create_callstatus_file($Wiederaufsetzpunkt1, $continue_upgrade_iteration, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $upgrade_with_reduce_downtime, $upgrade_via_image);
CONTINUE_SPOT1:
# if a HA upgrade I can check for success only here , but NOT if a host reboot requested and only on the active side
  if ($global_activation_rc == 0 && $activation_has_been_avoided == 0) {
    if (($continue_upgrade == 1 || $upgrade_with_completely_stopped_ha == 1 )) {
      if (!$nodaemon && $is_FI == 0) {
        if ($host_reboot != 0) {
          log_msg_info(" skipping call of the wait for success script upgrade_successful.sh for active node due to planned host reboot");
        } else {
          $global_activation_rc = wait_for_successful_startup(1, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $multiNodeType, $isMultiNode, "primary", $max_wait_for_up, 0);
          if ($global_activation_rc != 0) {
            set_activation_status_text("rpm upgrade successful, but startup verification (reach appropriate state of the CCs) failed");
          } else {
            append_timestamp_file_for_assi(3, 1); # third entry after start with forced write
          }
        }
      }
    }

    if (!$nodaemon && $host_reboot == 1 && $upgrade_with_reduce_downtime == 1) {
        log_msg_info("Error: I would need to reboot myself (active node) and reboot the remote node due to reduced downtime feature, dont know what to do now !!!");
    }

    if (!$nodaemon && $host_reboot == 0) {
      if ($upgrade_with_reduce_downtime == 1 && $isMultiNode == 1) {
        log_msg_info("forcing the CC active to the node_A again after reduced downtime feature");
        make_this_cc_active();
        sleep(5);

        log_msg_info("rebooting the node_B after reduced downtime feature");
        reboot_host("remote", $upgrade_with_reduce_downtime, $upgrade_via_image);

        log_msg_info("waiting for the node_b to come up again after reboot");
        sleep_for_openais(5, "Standby", $other_node, 1);

        log_msg_info("***generating the CIB on the standby node due to the reduced downtime feature");

        set_internal_upgrade_step($STEP_start_openais_s_t3, 1, "Standby", $other_node);

        $global_rc = start_openais_service("remote", 0);
        if ($global_rc != 0) {
          $global_activation_rc = $ERROR_StartingOpenaisFailed;
          set_activation_status_text("Starting Openais for the standby node failed");
        }
      }
    }
  }

FAILURE_SPOT1:
$UpgradeRunning = "no";
# always use the original activation status as overall status, not the deactivation status
my $host_reboot_will_really_be_executed = 0;
if ($host_reboot == 1) {
  log_msg_info("Host reboot requested, checking if it will really be executed.");
  if ($global_activation_rc != 0 && $upgrade_via_image == 1) {
    log_msg_info("It is a NUC update with failure -> no reboot");
  } else {
    log_msg_info("It is either a NUC update with success or a non NUC -> reboot");
    $host_reboot_will_really_be_executed = 1;
  }
}
    
if ($nodaemon == 1) {
  if ($isquorum == 1) {
    write_upgrade_quorum_status($global_activation_rc, $activation_status_text);
  } else {
    write_upgrade_remote_status($global_activation_rc, $activation_status_text);
  }
} else {
  # active node
  write_upgrade_local_status($global_activation_rc, $activation_status_text);
    
  if ($host_reboot_will_really_be_executed == 1 &&  $upgrade_with_completely_stopped_ha == 1) {
    log_msg_info("do not generate overall upgrade status because a host reboot will occur now and CIB has still to be generated:");
  } else {
    $global_rc = write_combined_upgrade_status($status_file_tmp, $multiNodeType, $isMultiNode, $other_node, $node_q);
    my $transfer_rc = transfer_upgrade_status_to_remote_node();
    if ($transfer_rc != 0) {
      log_msg_info("propagation of the overall upgrade status $global_rc to the former standby node failed !!!");
      log_msg_info("This may lead to a missing upgrade response from platform to Assistant SWA!!!"); 
    }
    $global_rc = renameFile($status_file_tmp, $status_file);
  }
}
# write_status(0, "");


cleanup_after_upgrade();

# I have to generate the sysinfo status before a potential reboot, because afterwards I dont have the approriate info available.
$running_version = get_current_real_version();
if ($is_stmix == 0) {
  finalize_sysinfo_values();
}
if ($global_activation_rc == 0) {
  if ($upgrade_via_image == 1) {
    if ($upgrade_version eq "") {
      log_msg_info("I have a successful NUC activation which will be completed by reboot but upgrade_version is empty -> obviously RAR -> get the upgrade_version from new world");
      $running_version = NUC_execute_other_pipe("/bin/getydata -v");
    } else {
      log_msg_info("I have a successful NUC activation which will be completed by reboot -> use the new image version $upgrade_version as running_version for the last sysinfo upgrade_info.xml generation");
      $running_version =  $upgrade_version;
    }
  } else {
    log_msg_info("I have a successful non NUC activation -> use the new real version $running_version as upgrade_version (to Version) for the last sysinfo upgrade_info.xml generation to avoid RAR pseudo hotfix number to appear in sysinfo after upgrade");
    $upgrade_version = $running_version;
  }
  set_internal_upgrade_step($STEP_platform_upgrade_success, 1, $defaultHostType, $hostname);
} else {
  set_internal_upgrade_step($STEP_platform_upgrade_failed, 1, $defaultHostType, $hostname);
}

if ($upgrade_via_image == 1 && $is_stmix == 0) {
  # NUC
  log_msg_info("I have a NUC activation with generated sysinfo upgrade.xml -> copy into new world.");
  NUC_copy_other($sysinfo_log);

  if ($nodaemon == 0) {
    log_msg_info("In case a rollback to old world is executed, I have to prepare a failure sysinfo upgrade .xml, but only on the active node");
    if ($global_activation_rc != 0) {
      log_msg_info("NUC activation failed, no reboot will happen, therefore NOT stopping sysinfod");
    } else {
      log_msg_info("I have to stop the sysinfo in old world on active node to prevent the sysinfo from displaying this pre-emptive failure");
      exec_cmd("service sysinfod stop");
    }
    set_internal_upgrade_step($STEP_platform_upgrade_failed, 1, $defaultHostType, $hostname);
  }
  dump_sysinfo_status_to_log();
} else {
  # no NUC
  if ($nodaemon == 0) {
    if ($is_stmix == 0) {    
      transfer_sysinfo_status_to_remote_nodes(0);
    }
  }
}

start_other_service_after_upgrade();
if ($is_stmix == 0) {
  ensure_self_existence();
}

retrieve_java_versions("after");
restart_softgate_if_necessary();

# handle a reboot (normal or NUC)
if ($host_reboot == 1 && $activation_has_been_avoided == 0) {
    if ($global_activation_rc != 0 && $upgrade_via_image == 0) {
      log_msg_info("*** doing reboot of the Host even if upgrade was not successful global_rc=$global_rc ($uname)");
    }
    if ($upgrade_with_completely_stopped_ha == 1) {
      log_msg_info("*** software update not yet finished, will be continued after host reboot", 0, 1);
    } else {
      log_msg_info("*** software update finished after host reboot rc=$global_activation_rc ($uname)", 0, 1);
    }
    if ($upgrade_via_image == 1) {
      if ($nodaemon == 0) {
        log_msg_info("In case of NUC I must not insserv the HA in old world, primary has to execute Wiederaufsetzpunkt5 if it is really started again");
        create_call_file_for_reboot();
        create_callstatus_file($Wiederaufsetzpunkt5, $continue_upgrade_iteration, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $upgrade_with_reduce_downtime, $upgrade_via_image);
      } else {
        log_msg_info("non primary node in case of NUC must not execute Wiederaufsetzpunkt5 -> it has to be taken into operation from primary node ");
      }

      if ($global_activation_rc == 0) {
        log_msg_info("NUC update finished with success -> initiate activation reboot");
        execute_NUC_reboot($isMultiNode, $isquorum);
      } else {
        log_msg_info("NUC update finished with error -> skipping activation reboot");
        log_msg_info("I have to take the remote nodes into operation again like Wiederaufsetzpunkt 5 in case of a failed reboot to new world on A");
        configure_cib_local();
      }
    } else {
      copy_upgrade_log($upgrade_version);
      reboot_host("local", $upgrade_with_reduce_downtime, $upgrade_via_image);
    }
} else {

  if ($is_stmix == 0) {
    start_sysinfo();
  }
  restart_snmp();
 
  $uname = exec_cmd_out("/bin/uname -a", "silent");
  chomp($uname);  
  log_msg_info("*** software update finished rc=$global_activation_rc ($uname)", 0, 1);
  copy_upgrade_log($upgrade_version);
}

exit (0);


sub usage {
  print STDERR "Usage: $0 [<options>] <command> [<command_args>] \n";
  print STDERR "ssh commands:           prepare_ssh <host> | cleanup_ssh <host> | ssh <host> <cmd> | sshx <cmd>  | sshq <cmd> | sshadp <cmd>  \n";
  print STDERR "sftp get commands:      get <host> <localpath> <remotepath> | getx <localpath> <remotepath> | getq <localpath> <remotepath>  \n";
  print STDERR "sftp put commands:      put <host> <localpath> <remotepath> | putx <localpath> <remotepath> | putq <localpath> <remotepath> \n";
  print STDERR "Assistant helper cmds:  configure_mek <mek> | replicate_ssh_banner | configure_snmp [sg] | configure_syslog <config> <epoch> | activate_license [<locale>] |redistribute_licenses | get_interface_list | set_ui_mode <mode=0,1,2> [<TLS version=1,2,3>] | get_timezone | set_timezone <tzone> | get_version_info\n";
  print STDERR "upgrade commands:       hfactivate | hfdeactivate | get_upd_status | configure_cib <ha_type> <is_failure>| configure_cib_local | isolate_node <type> | reimport_licences \n";
  print STDERR "reboot commands:        reboot a|s|q|all | shutdown a|s|q|all | stop_openais  a|s|q|all | shutdownstatus\n";
  print STDERR "misc commands:          get_adp_host  | get_rpm_consistency_status local|remote|quorum|all <version> | check_node_connectivity <host> | prepare_sftp_host <sentinel> <user> <uid> <gid>\n";
  print STDERR "NUC commands:           gen_os_transfer_file_for_sg <path_to_sg_list_file> [<path_to_stmix_list_file>] | cleanup_after_os_sg_update | transfer_os_from_host <host> <localpath> <remotepath> [<workdir>] | cancel_os_transfer\n";
  print STDERR "test commands:          test | create_upgrade_lock [<waiting_time [300]]>] \n";
  print STDERR "\n";
  print STDERR "upgrade options:  --no_self_xtract | --extract | --sg_os_update | --deferred_activation | --nodaemon | --avoidminicib | --skip_quorum_rpm_upgrade | --autodeact | --biosupdate | --eff_vers=<version> | --is_FI \n";
  print STDERR "NUC SG options:   --sg_os_update | --deferred_activation | --keep_installed_sg \n";
  print STDERR "ssh options:      --irrc | --sshexec_timeout=<TimeInSec> | --sshlock_timeout=<TimeInSec> | --nodecheck=none|crm|ping (default is none)  \n";
  print STDERR "general options:  --testmode | --force | --forcedaemon | --isquorum | --replacement \n";
  print STDERR "\n";
  exit(1);
}

sub check_for_controlled_NUC_activation_for_SurvSoftGates {
  function_enter("check_for_controlled_NUC_activation_for_SurvSoftGates", "");

  if ( -e $assistant_controlled_sg_reboot_file ) {
    my $assistant_controlled_sg_reboot = read_file_content($assistant_controlled_sg_reboot_file);
    chomp($assistant_controlled_sg_reboot);
    if ($assistant_controlled_sg_reboot eq "0") {
      if ($sg_os_update == 1) {
        log_msg_info("activation via LWUM and the special attended Surv. SG OS activation trigger file $assistant_controlled_sg_reboot_file detected and content is 0 -> make attended full NUC Activation");

        log_msg_info("switching to full NUC activation mode (not only softgate)");
        $sg_os_update = 0;

        log_msg_info("removing special attended Surv. SG OS activation trigger file");
        removeFile($assistant_controlled_sg_reboot_file);
      } else {
        log_msg_info("activation via SWA and the special attended Surv. SG OS activation trigger file detected and content is 0, check conditions");
        if ( !$hotfix_mode ) {
          log_msg_info("Its not a hotfix update, check deployment");
          if (is_survivable_deployment($multiNodeType)) {
            log_msg_info("Its a surv. SG/Enterprise GW -> quit upgrade and let SG trigger the activation");

            my $trigger_file = $soco2_data_dir.'controlled_NUC_activation_via_SG_possible';
            log_msg_info("Creating trigger file $trigger_file for SG to allow user in WBM the controlled activation");
            write_file_exclusive($trigger_file, "");
            
            ensure_self_existence();

            remove_upgrade_lock_file();
            remove_remote_lock_file();
  
            log_msg_info("skipping upgrade without generating status for SWA");
            $uname = exec_cmd_out("/bin/uname -a", "silent");
            chomp($uname);
            log_msg_info("*** software update skipped due to attended NUC activation for Surv. SoftGate/Enterprise GW ($uname)", 0, 1);
            exit(0);
          }
        }
      }
    } else {
      log_msg_info("special attended Surv. SoftGate/Enterprise GW OS activation trigger file $assistant_controlled_sg_reboot_file detected -> but content is not 0: $assistant_controlled_sg_reboot");
    }
  }  else {
    log_msg_info("special attended Surv. SoftGate/Enterprise GW OS activation trigger file $assistant_controlled_sg_reboot_file does not exist");
  }

  function_leave("check_for_controlled_NUC_activation_for_SurvSoftGates" , "");
}

sub ensure_self_existence {
  function_enter("ensure_self_existence", "");  
  my $self                = $webservice_dir.'scripts/swupdated';
  my $old_self            = $webservice_dir.'scripts/swupdated.old';
  my $self_save = $self.".save";
  my $self_old_save = $self.".save.old";
  my $rc = 0;

  if (-e $self) {
    log_msg_info("swupdated exists on official location: $self ");
  } else {
    log_msg_info("swupdated does not exist on official location: $self ");
    log_msg_info("maybe hf activation avoidance or lack of webservice package did not restore the swupdated -> I have to do it now ");
    if (-e $old_self) {
       my $cmd = "/bin/cp -vf $old_self $self";
       $rc = exec_cmd($cmd);
    } else {
      if (-e $self_save) {
       my $cmd = "/bin/cp -vf $self_save $self";
       $rc = exec_cmd($cmd);
      } else {
        if (-e $self_old_save) {
          my $cmd = "/bin/cp -vf $self_old_save $self";
          $rc = exec_cmd($cmd);
        } else {
          log_msg_info("Error: swupdated does not exist on any known location !!! Assistant will have problems !!!! ");
          log_msg_info("Error: This situation can not happen and must not happen !!!! ");
          $rc = -1;
        }
      }
    }
  }
  exec_cmd("/bin/chmod +x $self");
  function_leave("ensure_self_existence" , "rc=$rc");
  return ($rc);
}

sub isolate_node {
  my $type = shift;
  function_enter("isolate_node", "type=$type");
  my $rc = 0;

  $rc = make_this_cc_active();

#  if ($rc == 0) {
#    log_msg_info("Trying to put the standby node into $type state");
#    $rc = manage_cluster_for_reduced_downtime($type);
#  }
  
  function_leave("isolate_node" , "rc=$rc");
  return ($rc);
}

sub configure_cib {
  my $ha_type = shift;
  my $is_failure = shift;
  if (!defined($is_failure)) { $is_failure = 0; }
  function_enter("configure_cib", "ha_type=$ha_type is_failure=$is_failure");
  my $rc = 0;

  $is_manager = manager_discover();

  $rc = execute_ha_do_insserv();

  # does not make sense to call this function if not a type 3 upgrade
  $upgrade_with_completely_stopped_ha = 1;
  $upgrade_with_full_drbd_sync = $upgrade_with_completely_stopped_ha;

  determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 1);

  log_msg_info("starting ASyaps pre-postinstall (before initHA)");
  $rc = execute_yaps_prepostinstall("-u");
  if($rc != 0) {
    log_msg_info("ASyaps prepostinstall failed");
  }

  log_msg_info("for diagnosis print the ALI before initHA");
  get_ali();
  get_ali_p();

  # new world
  # import the licensing info
  if ($is_failure == 0) {
    # currently no dependecies of the snmp update known
    $rc = take_snmp_into_operation_in_new_world();

    $rc = take_cla_into_operation_in_new_world();
  }
  else {
    log_msg_info("configure_cib in failure mode- > skip take_cla_into_operation_in_new_world");
  }

  log_msg_info("plt-update: SG on Quorum has to be handled after CLA since it requires the licensing data");
  take_sg_into_operation_in_new_world(0, $softgate_config_dir,   $sg_necessary_file);
  take_sg_into_operation_in_new_world(1, $isoftgate_config_dir, $isg_necessary_file);
  
  log_msg_info("initHA in any case (success in new world or failure in old or rollback)");
  $rc = execute_after_rpm_install_ha($ha_type, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $multiNodeType, $isMultiNode);

  start_openais_service("local", 0);

  if ($is_failure == 0) {
    my $ha_rc = execute_post_ha_update($upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $multiNodeType, $isMultiNode);
    if ($ha_rc != 0) {
      log_msg_info("ignoring return code of HA script");
    }
  } else {
    log_msg_info("configure_cib in failure mode- > skip execute_post_ha_update");
  }

  log_msg_info("starting ASyaps postinstall (after initHA)");
  $rc = execute_yaps_postinstall("-u");

  if ($ha_type eq "quorum") {
    log_msg_info("removing transmission packets, which are not needed on a quorum");
    exec_cmd("/opt/webservice/scripts/clean_rpms_nodeq.sh");
  }

  log_msg_info("for diagnosis print the ALI after initHA");
  get_ali();
  get_ali_p();
  
  function_leave("configure_cib", "rc=$rc");
  return ($rc);
}

sub take_snmp_into_operation_in_new_world {
  function_enter("take_snmp_into_operation_in_new_world", "");
  my $rc = 0;  

  if ( -d $snmp_upgrade_dir ) {
    log_msg_info("snmp upgrade dir $snmp_upgrade_dir exists, continue with snmp update");

    # stop the daemons
    log_msg_info("stopping the snmp related daemons in order to restore the config in new world");
    
    $rc = exec_cmd("service syslog stop");
    $rc = exec_cmd("service snmp-syslogagt stop");
    $rc = exec_cmd("service snmpd stop");

    my $configfile = $snmp_upgrade_dir.'config_file.txt';
    if ( -f $configfile) {
      $rc = exec_cmd ("/bin/mv -vf $configfile /etc/subagents/snmp-syslogagt/");
    }

    $configfile =  $snmp_upgrade_dir.'syslogagt-filter.conf';
    if ( -f $configfile) {
      $rc = exec_cmd ("/bin/mv -vf $configfile /etc/subagents/snmp-syslogagt/");
    }

    $configfile = $snmp_upgrade_dir.'snmp_snmpd.conf';
    if ( -f $configfile) {
      $rc = exec_cmd ("/bin/mv -vf $configfile /etc/snmp/snmpd.conf");
    }

    $configfile = $snmp_upgrade_dir.'net-snmp_snmpd.conf';
    if ( -f $configfile) {
      $rc = exec_cmd ("/bin/mv -vf $configfile /var/lib/net-snmp/snmpd.conf");
    }

    # re-initialize the /etc/snmp/snmpd.conf in case e.g. upgrade from a 7.1 system
    $rc = exec_cmd ("/opt/subagents/snmp-syslog-agent/configSnmpSystem.sh");
    if ($rc != 0) {
      log_msg_info("error: invocation of SNMP script /opt/subagents/snmp-syslog-agent/configSnmpSystem.sh failed, rc=$rc, continuing");
    }
    
    # start the daemons
    log_msg_info("starting the snmp related daemons");
   
    $rc = exec_cmd("service snmpd start");
    $rc = exec_cmd("sleep 2");
    $rc = exec_cmd("service snmp-syslogagt start");
    $rc = exec_cmd("sleep 2");
    $rc = exec_cmd("service syslog start");

    # insserv them (on STMIX the phase2 does not insserv the snmpd)
    $rc = exec_cmd("systemctl enable snmpd");  # insserv does not work anymore due to missing /etc/init.d/snmpd script
    $rc = exec_cmd("insserv /etc/init.d/snmp-syslogagt");

  } else {
    log_msg_info("snmp upgrade dir $snmp_upgrade_dir does not exists, skipping snmp update. Has to be done by Assistant later on.");
  }
 
  function_leave("take_snmp_into_operation_in_new_world", "rc=$rc");
  return ($rc);
}

sub log_iproute2_table {
  my $location = shift;

  function_enter("log_iproute2_table", "location=$location");
  my $rc = 0;
  my $cmd = "cat /etc/iproute2/rt_tables";
  $rc = exec_cmd ($cmd);
  function_leave("log_iproute2_table", "rc=$rc");
  return ($rc);
}

sub get_ali {
  function_enter("get_ali", "");
  my $ali = "";
  if ($is_stmix == 0) {
    my $cmd = "/opt/webservice/scripts/get_ali.sh";
    $ali = exec_cmd_out ($cmd);
    $ali = my_trim($ali);
  } else {
    log_msg_info("on STMIX skipping get_ali.sh");
  }
  function_leave("get_ali", "ali=$ali");
  return ($ali);
}

sub get_ali_p {
  function_enter("get_ali_p", "");
  my $rc = 0;
  if ($is_stmix == 0) {
    my $cmd = "LD_LIBRARY_PATH=/opt/cla/bin /opt/cla/bin/get_ali -p";
    $rc = exec_cmd ($cmd);
  } else {
    log_msg_info("on STMIX skipping get_ali -p");
  }
  function_leave("get_ali_p", "rc=$rc");
  return ($rc);
}

sub install_cla {
  function_enter("install_cla", "");
  my $rc = 0;

  log_msg_info("note: cla package is taken from SoftGate installation");
  $rc = exec_cmd("rpm -Uvh --force /opt/soco/rpm/cla/cla*rpm");
  if ($rc != 0) {
    log_msg_info("installing/updateing cla failed with rc=$rc, but continue anyway");
  }
  sleep(2);
  $rc = exec_cmd("cp -v /opt/soco/config/templates/cla_45.0.0_full_tracing.xml /opt/cla/bin/cla.xml");

  # now do the insserv as the cla.rpm itself sometimes does not do it on SLES15
  # first check the current insservstatus of the cla rpm scriptlet
        exec_cmd("find /etc/init.d/ | grep cla");
  $rc = exec_cmd("insserv cla");
        exec_cmd("find /etc/init.d/ | grep cla");

  function_leave("install_cla", "rc=$rc");
  return ($rc);
}

sub take_cla_into_operation_in_new_world {
  function_enter("take_cla_into_operation_in_new_world (aligned with install-appliance-phase2.sh)", "");
  my $rc = 0;
  my $skip_license_file_takeover = 0;

  log_msg_info("check for major release update");
  $is_major_release_update = check_for_major_release_update_in_new_world();
  if ($is_major_release_update == 0) {
    log_msg_info("no major release update, I can take over the licenses");
  } else {
    log_msg_info("it is a major release update, check central host");
      if (isCentralHost($multiNodeType) == 1) {
        log_msg_info("For central host and major release update we have to skip the license file takeover to avoid cummulation of Multiple Product Licenses ( MPL) of the Flex Licenses (OSFOURK-16597)");
        $skip_license_file_takeover = 1;
      } else {
        log_msg_info("it is as major release update, but not on central host, I can take over the licenses");
      }
  }

 
  log_msg_info("update license data step 1: stopping cla");
  $rc = stop_cla_service();

  log_msg_info("first erase a cla package like install-appliance-phase2.sh");
  $rc = exec_cmd("rpm -e cla");
  if ($rc != 0) {
    log_msg_info("erasing cla failed with rc=$rc, but continue anyway");
  }

  log_msg_info("now ensure that a cla package is installed");
  $rc = install_cla();
  
  log_msg_info("check the transaction.logs before starting cla");
  $rc = exec_cmd("ls -l /opt/cla/mirror/transaction.log /opt/cla/data/transaction.log");
 
  log_msg_info("for diagnosis print the ALI before license import");
  get_ali();

  # OSFOURK-23167: do not restore the hidden system dir of CLA due to CLA 45 crashing, maybe re-enabled later
  #  log_msg_info("stopping cla again in case that it started itself due to rpm install");
  #  $rc = stop_cla_service();

  #  my $cla_temp_system_dir  = "$cla_temp_dir"."system/";
  #  my $cla_temp_license_dir = "$cla_temp_dir"."license/";
  #  if (-e $cla_temp_license_dir) {
  #    if (-e "/var/system/") {
  #      log_msg_info("update license date step 2a: removing system dir");
  #      $rc = exec_cmd ("/bin/rm -rf /var/system/");
  #    }

  #    log_msg_info("update license date step 2b: restoring system dir");
  #    $rc = exec_cmd ("/bin/mv -vf $cla_temp_system_dir /var/");
  #  }

  log_msg_info("before starting cla check the current status (may have started itself already)");
  $rc = exec_cmd("service cla status");

  log_msg_info("update license date step 3: starting cla");
  $rc = start_cla_service();

  log_msg_info("check the transaction.logs after starting cla");
  $rc = exec_cmd("ls -l /opt/cla/mirror/transaction.log /opt/cla/data/transaction.log");

  log_msg_info("check the current status after start cla");
  $rc = exec_cmd("service cla status");
  if ($rc == 0) {
    log_msg_info("cla service seems to run");
  } else {
    log_msg_info("cla service seems not to run, try deleting the transaction files before retry start cla");
    $rc = exec_cmd("rm -f /opt/cla/mirror/transaction.log /opt/cla/data/transaction.log");
    $rc = start_cla_service();
  }

  if ($skip_license_file_takeover == 1) {
    log_msg_info("skip license date step 4a (importing the license files)");
  } else {
    log_msg_info("update license date step 4a: importing the license files (before initHA so that the first license fetch by Assi and SG in normal case succeeds)");
    reimport_the_licences_to_cla();
  }

  dump_file_to_log("/opt/cla/log/audit.log");
  dump_file_to_log("/opt/cla/log/error.log");
  function_leave("take_cla_into_operation_in_new_world", "rc=$rc");
  return ($rc);
}

sub reimport_the_licences_to_cla {
  function_enter("reimport_the_licences_to_cla", "");
  my $rc = 0;

  log_msg_info("update license date: importing the license files");
  my $cla_temp_license_dir = "$cla_temp_dir"."license/";

  # main license import based upon rlf_*.xml files from /opt/cla/license/ in old world
  if (-e $cla_import_dir) {
    $rc = exec_cmd ("/bin/cp -fv $cla_temp_license_dir/* $cla_import_dir");
  } else {
    log_msg_info("error: cla import dir $cla_import_dir does not exist -> skip importing license files, but continue with upgrade");
  }
  # in case of Manager try also to re-import the original license files (OSFOURK-24598)
  if ($is_manager == 1) {
    if (-e $cla_import_dir) {
      sleep(1);
      log_msg_info("re-importing the original license files from $license_storage_dir");
      log_msg_info("this may cause errors in audit.log complaining about already imported licenses");
      $rc = exec_cmd ("/bin/cp -fv $license_storage_dir/*.lic $cla_import_dir");
    } else {
      log_msg_info("error: cla import dir $cla_import_dir does not exist -> skip importing license files, but continue with upgrade");
    }
  } else {
    log_msg_info("not a Manager deployment. Identical license ID for Manager and OS4K network license can not happen. re-import of original license files not necessary");
  }
  
  # wait a little bit to give CLA time to import the licenses
  sleep(5);

  dump_file_to_log("/opt/cla/log/audit.log");
  dump_file_to_log("/opt/cla/log/error.log");

  exec_cmd ("ls -l $cla_license_dir");

  function_leave("reimport_the_licences_to_cla", "rc=$rc");
  return ($rc);
}

sub reimport_the_licences_to_cla_on_b {
  function_enter("reimport_the_licences_to_cla_on_b");
  my ($output, $node_is_reachable, $password_is_expired, $ignore_remote_return_code) = ("", 1, 0, 1);

  my $remote_cmd = "/opt/webservice/scripts/swupdated reimport_licences";
  my $rc = 0;

  $rc = execute_remote_duplex_cmd($remote_cmd, \$output, 1, \$node_is_reachable, \$password_is_expired, $ignore_remote_return_code);

  function_leave("reimport_the_licences_to_cla_on_b", "rc=$rc");
  return($rc);
}

sub reimport_the_licences_to_cla_on_q {
  function_enter("reimport_the_licences_to_cla_on_q");
  my ($output, $node_is_reachable, $password_is_expired, $ignore_remote_return_code) = ("", 1, 0, 1);

  my $remote_cmd = "/opt/webservice/scripts/swupdated reimport_licences";
  my $rc = 0;

  $rc = execute_remote_quorum_cmd($remote_cmd, \$output, 1, \$node_is_reachable, \$password_is_expired, $ignore_remote_return_code);

  function_leave("reimport_the_licences_to_cla_on_q", "rc=$rc");
  return($rc);
}

sub postprocess_ha_dhcp_upper_range {
  function_enter("postprocess_ha_dhcp_upper_range");
  my $rc = 0;
#  #OSFOURK-14897 : disable changes done in OSFOURK-13949 by not doing anything here until we clarify a proper solution
#  log_msg_info("postprocess_ha_dhcp_upper_range will not do anything => OSFOURK-13949 is possibly still not fixed");
#  function_leave("postprocess_ha_dhcp_upper_range", "rc=$rc");
#  return ($rc);
  
  my $property_name   = "internal-lan.end-ip-address";
  my $cur_upper_range = retrieve_from_xml("$property_name");
  my $intended_upper_range = $cur_upper_range;
  #[OSFOURK-13949] Increase DHCP range of dnsmasq
  $intended_upper_range =~ s/(.*)\.(\d+)/$1.139/;
  
  if ($cur_upper_range eq $intended_upper_range) {
    log_msg_info("current upper dhcp range is $cur_upper_range is already identical with new default, nothing to do");
  } else {
    log_msg_info("current upper dhcp range is $cur_upper_range, need to force change to new default $intended_upper_range");
    log_msg_info("note: this should not cause an Assistant/CSTA restart if haconfd.pl is also prepared for this change");
    $rc = ngs_data_set_ha_ip_properties("internal-lan.end-ip-address", $intended_upper_range);
  }
  
  function_leave("postprocess_ha_dhcp_upper_range", "rc=$rc");
  return ($rc);
}

sub ngs_data_set_ha_ip_properties {
  my $property_name = shift;
  my $property_value = shift;

  function_enter("ngs_data_set_ha_ip_properties", "property_name=$property_name, property_value=$property_value");
  my $rc = exec_cmd("/opt/webservice/install/ngs-data-set-ha-ip-properties.sh $property_name=$property_value");

  function_leave("ngs_data_set_ha_ip_properties", "rc=$rc");
  return ($rc);
}

sub postprocess_portal_configdata_tls_version {
  function_enter("postprocess_portal_configdata_tls_version");
  my $rc = 0;
  my $tls_configfile= "/var/opt/webservice/tls_version.conf";

  if(open(TLSCFILE, ">>$tls_configfile")) {
    print TLSCFILE ("3\n");
    close TLSCFILE;
    protectFile($tls_configfile);
  }
  
  function_leave("postprocess_portal_configdata_tls_version", "rc=$rc");
  return ($rc);
}

sub postprocess_sg_configdata_tls_version {
  my $softgate_config_dir = shift;
  function_enter("postprocess_sg_configdata_tls_version", "softgate_config_dir=$softgate_config_dir");
  my $rc = 0;
  my $cmd_file="/tmp/postprocess_sg_tls.sh";

  # create temporary command file
  if(open(SGPPCMDFILE, ">$cmd_file")) {
    # goal is to change SG config data (all slots) about TLS protocol versions for HFA and HTTPS (not SIP) during RLC upgrade to V10
    # I can afford to include also the SIP spesd.txt files, since parameter is different here with underscore: PSPE_SDTLSProtocol...
    print SGPPCMDFILE "find $softgate_config_dir -name spesd.txt -exec sed -i 's/SpeSDTlsProtocols..[1-2]/SpeSDTlsProtocols\">3/' {} \\; \n";
    print SGPPCMDFILE "find $softgate_config_dir -name fips.xml  -exec sed -i 's/PFips_WbmTLSProtocolHTTPS..[0-2]/PFips_WbmTLSProtocolHTTPS\">3/' {} \\; \n";
    close SGPPCMDFILE;
  }
  exec_cmd("chmod +x $cmd_file");
  $rc = exec_cmd("$cmd_file");

  removeFile($cmd_file);
  
  function_leave("postprocess_sg_configdata_tls_version", "rc=$rc");
  return ($rc);
}

sub take_sg_into_operation_in_new_world {
  my $is_sg2d = shift;
  my $softgate_config_dir = shift;
  my $sg_necessary_file = shift;
  function_enter("take_sg_into_operation_in_new_world", "is_sg2d=$is_sg2d softgate_config_dir=$softgate_config_dir sg_necessary_file=$sg_necessary_file");
  my $rc = 0;

  # insserv the SG if necessary
  my $sg_is_necessary = 0;
  
  if ( (-e $sg_necessary_file and $is_sg2d == 0) or (-e $isg_necessary_file and $is_sg2d == 1) ) {
    log_msg_info("Softgate was running in old world (native or container) -> SG necessary", 1, 0);
    $sg_is_necessary = 1;
    removeFile($sg_necessary_file);
  } else {
    if ($is_stmix == 0) {
      log_msg_info("Softgate was NOT running in old world (file $sg_necessary_file missing)", 1, 0);
    } else {
      log_msg_info("Softgate was NOT running in old world (file $sg_necessary_file missing), but this does not make sense due to STMIX HW", 1, 0);
      $sg_is_necessary = 1;
    }
  }
  
  if ($sg_is_necessary == 1) {
    my $old_sg_rpm = $soco2_sg_dir . "softgate.rpm";

    log_msg_info("checking currently installed SoftGate version");
    my $cur_sg_version = exec_cmd_out("rpm -q softgate");

    if (-e $old_sg_rpm) {
      log_msg_info("previously installed softgate package from Old world found: $old_sg_rpm ");
      log_msg_info("check minimum version of this LW, so not to trust the check of swupdated in Old world");
      my $keep_sg_option_allowed = determine_prerequisites_for_keep_sg($old_sg_rpm);
      if ($keep_sg_option_allowed == 0) {
        log_msg_info(" old sg_rpm from Old world $old_sg_rpm does NOT meet prerequisites -> do not activate it");
        my $old_sg_rpm_fallback = "/olddisk/opt/soco/softgate.rpm";
        if (-f $old_sg_rpm_fallback) {
          log_msg_info("fallback rpm $old_sg_rpm_fallback exists. This maybe a transferred but not activated SG LW in OLD world");
          log_msg_info("check minimum version of this LW. It is is fulfilled we can activate this one.");
          $keep_sg_option_allowed = determine_prerequisites_for_keep_sg($old_sg_rpm_fallback);
          if ($keep_sg_option_allowed == 1) {
            log_msg_info("minimum version of the fallback LW is fullfilled. ");
            $old_sg_rpm=$old_sg_rpm_fallback;
          } else {
            log_msg_info("minimum version of the fallback LW is not fulfilled. Cannot activate it.");
          }
        }
      } 
      if ($keep_sg_option_allowed == 1) {
        log_msg_info(" old sg_rpm from Old world meets prerequisites -> activate it");
        log_msg_info("-> try to install it ");
        if ($is_sg2d == 0) {
          log_msg_info("native SG");

          $rc = exec_cmd("rpm -Uhv --force $old_sg_rpm");

          if ($rc == 0) {
            log_msg_info("SG upgrade successful");
          } else {
            log_msg_info("SG upgrade failed");
          }
        }
        else {
          log_msg_info("container SG, copy the desired LW into the path where the iSG will automatically update before start ");
          log_msg_info("LW rpm file will automatically be deleted by iSG after update");
          $rc = exec_cmd("rm -rfv   $soco2_isg_force_dir/softgate.rpm");
          $rc = exec_cmd("mkdir -pv $soco2_isg_force_dir");
          $rc = exec_cmd("cp -v $old_sg_rpm $soco2_isg_force_dir/softgate.rpm");

          # protocol the currently installed iSG version
          dump_file_to_log("/sg2/opt/soco/version.properties");

          if ($is_entgw == 1) {
            log_msg_info("on EntGW I can restart the sg2d to force the LW update inside the container (sg2d not under HA control)");
            $rc = exec_cmd("service sg2d restart");
          } else {
            log_msg_info("on deployments where sg2d is under HA control, I can not restart sg2d so easily, what to Do ????");
          }

          log_msg_info("LW rpm file will automatically be deleted by iSG after update");
        }
      }
    } else {
      log_msg_info("previously installed softgate package not found: $old_sg_rpm ");
    }

    log_msg_info("check the content of SG config dir e.g. mekdata");
    exec_cmd("ls -l $softgate_config_dir/pbc_21/");

    # check for config data migration
    postprocess_sg_configdata_tls_version($softgate_config_dir);

    if ($is_sg2d == 0) {
      log_msg_info("Starting softgate services (insserv)", 1, 0);
      # necessary to do this AFTER the haipsetup.xml is generated
      $global_rc = exec_cmd("/opt/soco/native/os/linux/insserv_SG.sh -s");
      if ($global_rc != 0) {
        log_msg_info("Starting softgate (insserv) services failed!", 1, 0);
      }
    }

    # handle the opensips loadbalancer pgsql config data after insserv_SG.sh to be sure opensips is really installed
    if (-e $opensips_sql_dump_file) {
      if ($is_sg2d == 0) {
        log_msg_info("opensips loadbalancer pgsql config file found: $opensips_sql_dump_file ");
        log_msg_info("But we do not import it anymore starting with V10 R1.");
        log_msg_info("Import and remove will be done by SoftGate itself");
        # rc = exec_cmd("/opt/opensips/dbscripts/data-restore2.sh $opensips_sql_dump_file");
        # if ($rc == 0) {
        #  log_msg_info("import of opensips loadbalancer pgsql file $opensips_sql_dump_file successful. Deleting it.");
        #  removeFile($opensips_sql_dump_file);
        # } else {
        #   log_msg_info("import of opensips loadbalancer pgsql file $opensips_sql_dump_file failed. Not deleting it for diagnosis purposes.");
        # }
      } else {
        log_msg_info("opensips loadbalancer for iSG not yet supported");
      }
    } else {
      log_msg_info("opensips loadbalancer pgsql config file $opensips_sql_dump_file not found");
    }
  } else {
    log_msg_info("Starting softgate (insserv) services skipped since no SoftGate active!", 1, 0);
  }
  
  function_leave("take_sg_into_operation_in_new_world", "sg_is_necessary=$sg_is_necessary");
  return ($sg_is_necessary);
}


sub configure_cib_local {
  function_enter("configure_cib_local", "");
  
  my $rc = 0;
  my $ha_type = "";
  my $step = get_callstatus_step();
  my ($sg_is_necessary, $isg_is_necessary) = (0, 0);
  my $is_fallback_in_old_world = 0;
  my $is_retry_wait_for_success_in_new_world = 0;
  my $too_many_retry_wait_for_success_in_new_world = 0;
  my $world = "unknown";
  my $activation_status_text = "";
  my $log_on_stdout = 1;


  # restore the upgrade flags read from the call status file
  $upgrade_with_completely_stopped_ha = $continue_upgrade_with_completely_stopped_ha;
  $upgrade_with_full_drbd_sync        = $continue_upgrade_with_full_drbd_sync;
  $upgrade_with_reduce_downtime       = $continue_upgrade_with_reduce_downtime;
  $upgrade_via_image                  = $continue_upgrade_via_image;

  log_msg_info("plt-update: sleep 20 seconds to be sure that network layer is already up and usable", $log_on_stdout);
  sleep(20);

  determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 1);
  $is_entgw   = entgw_discover();
  $is_manager = manager_discover();

  if ( -e $nuc_getworld_script ) {
    $world = exec_cmd_out($nuc_getworld_script);
  }
  log_msg_info("plt-update: world is : $world");

  # now do the local stuff

  if ($step == $Wiederaufsetzpunkt4) {
    log_msg_info("plt-update: internal step $step to continue setup in new world after NUC activation reboot", $log_on_stdout);
    $global_rc = 0;
    $is_fallback_in_old_world = 0; # we are in new world
    if ($world ne "NEW") {
      log_msg_info("plt-update: assuming world is new, but world detection says it is not new, What to do now ???", $log_on_stdout);
    }
  } elsif ($step == $Wiederaufsetzpunkt5) {
    #  !!!!!!!!!!!!!!!!! Fallback !!!!!!!!!!!!!!!!!
    $global_rc = $ERROR_NUCActivationRebootFailed;
    $is_fallback_in_old_world = 1;
    if ($world ne "OLD") {
      log_msg_info("plt-update: assuming world is old, but world detection says it is not old, What to do now ???", $log_on_stdout);
    }
  } elsif ($step == $Wiederaufsetzpunkt1) {
    if ($continue_upgrade_iteration > $max_continue_upgrade_iteration) {
      $is_retry_wait_for_success_in_new_world = 0;
      $too_many_retry_wait_for_success_in_new_world = 1;
      log_msg_info("plt-update: too many retries (reboots) in the startup verification phase occurred. Cancel upgrade and initiate rollback.", $log_on_stdout);
    } else {
      $is_retry_wait_for_success_in_new_world = 1;
      log_msg_info("plt-update: unexpected reboot occurred during startup of HA resources in new world -> continue with wait for resource status success", $log_on_stdout);
    }    
  } else {
    log_msg_info("plt-update: warning: internal step $step does not match the command configure_cib_local, obviously called at the end of a failed NUC in old world", $log_on_stdout);
    $is_fallback_in_old_world = 1;
  }

  trigger_watchdog();

  # handle failure
  if ($is_fallback_in_old_world == 1 || $too_many_retry_wait_for_success_in_new_world == 1) {
    log_msg_info("plt-update: internal step $step to rollback all in case of a failed NUC activation reboot", $log_on_stdout);
    if ($isMultiNode == 1) {
      if (has_quorum($node_q, $multiNodeType)) {
        log_msg_info("plt-update: Sep. Duplex detected -> I have to rollback the Quorum and wait", $log_on_stdout);
        $ha_type = "quorum";
        NUC_remote_rollback($node_q, $ha_type);
        trigger_watchdog();
      }
      log_msg_info("plt-update: Duplex detected -> I have to rollback the Standby and wait", $log_on_stdout);
      $ha_type = "secondary";
      NUC_remote_rollback($other_node, $ha_type);
      trigger_watchdog();

      log_msg_info("plt-update: remote nodes are back in old world -> generate the empty CIB", $log_on_stdout);
      my $enforce_remote_insserv = 1;
      my $is_failure = 1;
      if (has_quorum($node_q, $multiNodeType)) {
        start_openais_service("quorum", $enforce_remote_insserv, $is_failure);
        trigger_watchdog();
      }
      start_openais_service("remote", $enforce_remote_insserv, $is_failure);
      trigger_watchdog();
    }

    log_msg_info("plt-update: generate again the local upgrade status (primary) to failure", $log_on_stdout);
    if ($too_many_retry_wait_for_success_in_new_world == 1) {
      $global_rc = $ERROR_ExceededMaxNumberOfPltUpgradeRetries;
      set_activation_status_text("number of plt upgrade retries $continue_upgrade_iteration exceeds max number of allowed retries $max_continue_upgrade_iteration");
      log_msg_info("$activation_status_text -> cancel upgrade");
    } else {
      $global_rc = $ERROR_NUCActivationRebootFailed;
      set_activation_status_text("reboot to new world on primary failed");
    }
    write_upgrade_local_status($global_rc, $activation_status_text);
    set_NUC_SG_OS_state(10, $NUC_SG_OS_ERROR_BootToNewWorldFailed);
  }


  if ($is_retry_wait_for_success_in_new_world == 0) {

    log_msg_info("for diagnosis print the ALI before initHA");
    get_ali();
    get_ali_p();

    if ($is_fallback_in_old_world == 0) {
      # currently no dependecies of the snmp update known
      take_snmp_into_operation_in_new_world();

      if ($is_stmix == 0) {
        # import the licensing info
        log_msg_info("starting CLA and SG before initHA so that webservice startup is not disturbed by a potential Java upgrade of SG");
        $rc = take_cla_into_operation_in_new_world();
      }

      log_iproute2_table("before take_sg_into_operation_in_new_world");
      log_msg_info("plt-update: SG has to be handled after CLA since it requires the licensing data", $log_on_stdout);
       $sg_is_necessary = take_sg_into_operation_in_new_world(0, $softgate_config_dir,   $sg_necessary_file);
      $isg_is_necessary = take_sg_into_operation_in_new_world(1, $isoftgate_config_dir, $isg_necessary_file);

      log_msg_info("prevent TLS 1.0/1.1 on Standalone deployments before init_apache is called");
      postprocess_portal_configdata_tls_version();
      log_iproute2_table("after take_sg_into_operation_in_new_world");
    }

    log_msg_info("plt-update: starting ASyaps pre-postinstall (before initHA)", $log_on_stdout);
    $rc = execute_yaps_prepostinstall("-u");
    if($rc != 0) { log_msg_info( "plt-update: ASyaps prepostinstall failed", $log_on_stdout); }
    trigger_watchdog();

    log_msg_info("plt-update: creating the NUC status file for webservice before initHA, because it is checked and removed by the first webservice startup", $log_on_stdout);

    my $make_init_ha = 1;
    if ($is_stmix == 0) {
      if ($is_fallback_in_old_world == 1) {
        notify_NUC_status_for_WS(-1);
        log_msg_info("plt-update: NUC Fallback -> check if initHA is necessary e.g. due to DRBD", $log_on_stdout);
        if ($isMultiNode == 1) {
          log_msg_info("plt-update: NUC Fallback on Duplex->  initHA is necessary due to DRBD", $log_on_stdout);
          $make_init_ha = 1;
        } else {
          log_msg_info("plt-update: NUC Fallback on non MultiNode ->  initHA is not necessary (no DRBD)", $log_on_stdout);
          $make_init_ha = 0;
        }
      } else {
        notify_NUC_status_for_WS(0);
      }

      if ($make_init_ha == 1) {
        $ha_type = "primary";
        $rc = execute_after_rpm_install_ha($ha_type, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $multiNodeType, $isMultiNode);
      }
    }
  }

  if ($is_stmix == 0) {
    start_openais_service("local", 0);
  }
  
  trigger_watchdog();

  if ($is_retry_wait_for_success_in_new_world == 0) {

    if ($is_stmix == 0) {    
      my $ha_rc = execute_post_ha_update($upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $multiNodeType, $isMultiNode);
      if ($ha_rc != 0) {
        log_msg_info("plt-update: ignoring return code of HA script", $log_on_stdout);
      }
    }

    trigger_watchdog();

    log_msg_info("plt-update: starting ASyaps postinstall (after initHA)", $log_on_stdout);
    $rc = execute_yaps_postinstall("-u");

    trigger_watchdog();

    if ($upgrade_via_image != 0) {
      if ( ! -e $torrent_path_share ) {
        log_msg_info("plt-update: torrent file $torrent_path_share does not exist -> but I can not generate it since Assistant share not available", $log_on_stdout);
      }
    }
  }

  log_iproute2_table("after execute_yaps_postinstall");

  if ($is_fallback_in_old_world == 0) {
    # new world
    if ($global_rc == 0) {
      if ($is_stmix == 1) {
        # success on STMIX
        log_msg_info("ToDo: Confirmation of the STMIX OS Update should not be done by swupdated, but later on by SG itslef when sending E3 / ready", $log_on_stdout);
        confirm_new_stmix_os_activation();
      } else {
        # success on non STMIX
        if ($multiNodeType != 21 and $multiNodeType != 100) {
          log_msg_info("plt-update: wait for CC and ADP resources if soco2, but create an activation failure status before in case a watchdog occur", $log_on_stdout);
          write_upgrade_local_status($ERROR_NUCActivationRebootFailed, "watchdog occurred during resource verification phase");

          log_msg_info("plt-update: creating a retry step for resource verification in case a watchdog occurs", $log_on_stdout);
          create_callstatus_file($Wiederaufsetzpunkt1, $continue_upgrade_iteration, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $upgrade_with_reduce_downtime, $upgrade_via_image);

          my $specific_wait_for_up = $max_wait_for_up;
          $global_rc = wait_for_successful_startup(1, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $multiNodeType, $isMultiNode, "primary", $specific_wait_for_up, $log_on_stdout);
          if ($global_rc != 0)  {
            $activation_status_text = "resources did not come up";
          } else {
            $assistant_nuc_downtime_file_step = 2;
            append_timestamp_file_for_assi(3, 1); # third entry after start with forced write
          }
          write_upgrade_local_status($global_rc, $activation_status_text);
        } else {
          log_msg_info("plt-update: SoftGate Standalone scenario ($multiNodeType) does not need any local CC resources", $log_on_stdout);
        }

        # re-import again the licenses if not a major release update on central host
        log_msg_info("check for potential license reimport");
        $is_major_release_update = check_for_major_release_update_in_new_world();
        my $do_license_reimport = 1;
        if ($is_stmix == 1) {
          $do_license_reimport = 0;
        }
        if ($is_major_release_update == 1) {
          if (isCentralHost($multiNodeType) == 1) {
          log_msg_info("For central host and major release update we have to skip the license file takeover to avoid cummulation of Multiple Product Licenses ( MPL) of the Flex Licenses (OSFOURK-16597)");
            $do_license_reimport = 0;
          } else {
            log_msg_info("Major release update, but not a central host -> we need license file re-import");
            $do_license_reimport = 1;
          }
        } else {
          log_msg_info("Not a major release update -> we need license file re-import");
          $do_license_reimport = 1;
        }
        if ($do_license_reimport == 1) {        
	        log_msg_info("update license date step 4b: re-importing the license files (after initHA) in case that the first license import failed due to missing ALI (on node A)");
	        reimport_the_licences_to_cla();
	        if ($isMultiNode == 1) {
	          log_msg_info("re-import the SG licenses on remote Duplex node", $log_on_stdout);
	          reimport_the_licences_to_cla_on_b();
	          if (has_quorum($node_q, $multiNodeType)) {
	            log_msg_info("re-import the SG licenses on Quorum", $log_on_stdout);
	            reimport_the_licences_to_cla_on_q();
	          }
	        }
	      }
        
        # check for DHCP upper range (OSFOURK-13949)
        postprocess_ha_dhcp_upper_range();
      }

      # after a N-NUC we have to remove the superfluous ngsdb-backup.zip file from the remote node since webservice could not do that due to N-NUC
      if ($isMultiNode == 1) {
        remove_remote_pgsql_backup_file();
      }

    }
    set_NUC_SG_OS_state(10);
  }

  if ($is_stmix == 0) {
    log_msg_info("plt-update: inserv of HA after resource verification is done", $log_on_stdout);
    $rc = execute_ha_do_insserv();
    log_msg_info("plt-update: removing call files now that I am sure HA would start after watchdog", $log_on_stdout);
  } else {
    log_msg_info("plt-update: remove inserv of plt_upgrade", $log_on_stdout);
    exec_cmd("/sbin/insserv -r /etc/init.d/plt_upgrade");
  }

  remove_call_file();
  remove_call_status_file();
  
  $rc = write_combined_upgrade_status($status_file_tmp, $multiNodeType, $isMultiNode, $other_node, $node_q);
  if ($is_stmix == 0) {  
    my $transfer_rc = transfer_upgrade_status_to_remote_node();
    if ($transfer_rc != 0) {
      log_msg_info("plt-update: propagation of the overall upgrade status $global_rc to the former standby node failed !!!", $log_on_stdout);

      log_msg_info("plt-update: give the network another 20 seconds to come up after this failure", $log_on_stdout);
      sleep(20);

      $transfer_rc = transfer_upgrade_status_to_remote_node();
      if ($transfer_rc != 0) {
        log_msg_info("plt-update: propagation of the overall upgrade status $global_rc to the former standby node failed a second time !!!", $log_on_stdout);
        log_msg_info("plt-update: This may lead to a missing upgrade response from platform to Assistant SWA!!!", $log_on_stdout);
      }
    }
  }
  $rc = renameFile($status_file_tmp, $status_file);

  if ($is_stmix == 0) {
    transfer_sysinfo_status_to_remote_nodes($log_on_stdout);
  }
  trigger_watchdog();

  if ($sg_is_necessary != 0) {
    notify_NUC_status_for_SG($global_rc);
  }
  dump_upgrade_log_to_log();

  ensure_self_existence();

  log_msg_info("for diagnosis print the ALI after initHA");
  get_ali();
  get_ali_p();
  
  if ($is_fallback_in_old_world == 0 and $global_rc != 0) {
      # we are in new world, but have to fall back
      log_msg_info("plt-update: Activation failure -> notify SWA in old world of activation failure status and rollback into old world", $log_on_stdout);
      # Status for SWA
      NUC_copy_other($status_file);

      # Status for SG and WS
      log_msg_info("plt-update: Activation status for WS and SG is already set to failure before reboot -> no copy necessary now", $log_on_stdout);

      NUC_rollback("all");
  }

  log_msg_info("**** plt-update: NUC activation on platform finished *****", $log_on_stdout);

  copy_upgrade_log("RLC");
      
  function_leave("configure_cib_local", "rc=$rc");
  return ($rc);
}

sub reset_NUC_status_for_WS {
  function_enter("reset_NUC_status_for_WS", "");
  if (-e $webservice_data_dir) {
    removeFile($webservice_data_dir . "nuc_update.success");
    removeFile($webservice_data_dir . "nuc_update.failure");
  }
  function_leave("reset_NUC_status_for_WS", "");
}

sub reset_NUC_status_for_SG {
  function_enter("reset_NUC_status_for_SG", "");
  if (-e $sg_nuc_status_dir) {
    removeFile($sg_nuc_status_dir . "nuc_update.success");
    removeFile($sg_nuc_status_dir . "nuc_update.failure");
  }
  function_leave("reset_NUC_status_for_SG", "");  
}

sub notify_NUC_status_for_WS {
  my $activation_rc = shift;
  function_enter("notify_NUC_status_for_WS", "activation_rc=$activation_rc");
  log_msg_info("webservice has to be informed about the upgrade status");
  my $rc = 0;
  $rc = makeDir ($webservice_data_dir);
  if ($rc == 0) {
    if ($activation_rc == 0) {
      # success
      create_generic_file("", $webservice_data_dir . "nuc_update.success");
      removeFile         ($webservice_data_dir . "nuc_update.failure");

    } else {
      # failure
      create_generic_file("", $webservice_data_dir . "nuc_update.failure");
      removeFile         ($webservice_data_dir . "nuc_update.success");
    }
  } else {
    log_msg_info("Error: webservice data dir $webservice_data_dir could not be created");
  }

  function_leave("notify_NUC_status_for_WS", "rc=$rc");
  return ($rc);
}

sub notify_NUC_status_for_SG {
  my $activation_rc = shift;
  function_enter("notify_NUC_status_for_SG", "activation_rc=$activation_rc");
  log_msg_info("Softgate has to be informed about the upgrade status");
  my $rc = 0;
  $rc = makeDir ($sg_nuc_status_dir);
  if ($rc == 0) {
    if ($activation_rc == 0) {
      # success
      create_generic_file("", $sg_nuc_status_dir . "nuc_update.success");
      removeFile         ($sg_nuc_status_dir . "nuc_update.failure");

    } else {
      # failure
      create_generic_file("", $sg_nuc_status_dir . "nuc_update.failure");
      removeFile         ($sg_nuc_status_dir . "nuc_update.success");
    }
  } else {
    log_msg_info("Error: Softgate nuc status dir $sg_nuc_status_dir could not be created");
  }
  
  function_leave("notify_NUC_status_for_SG", "rc=$rc");
  return ($rc);
}

sub prepare_os_packages_for_container {
  function_enter("prepare_os_packages_for_container");
  my $rc = 0;
  # first remove the os package dir from previous plt HF, the new one is cummulative
  # then create it new as empty
  $rc = exec_cmd("rm -rf $csta_os_hf_pkg_dir");
  $rc = makeDir ($csta_os_hf_pkg_dir);

  $rc = exec_cmd("rm -rf $assi_os_hf_pkg_dir");
  $rc = makeDir ($assi_os_hf_pkg_dir);

  function_leave("prepare_os_packages_for_container", "rc=$rc");
  return ($rc);
}

sub create_os_package_update_scripts {
  function_enter("create_os_package_update_scripts");
  # do not use --force option for CSTA as it would stop the cld service during cld.rpm update
  # for Assistant we can use --force
  my $assi_script_content =
  "cd /.AS/SWS/os4k_platform_hotfix/os_pkgs_for_asmgr/;
  rpm -Uhv --force *.rpm;
  bash ../fixup-rewritemap-vhost-unity.sh /etc/apache2/vhosts.d/vhost-unity.conf;
  bash ../fixup-rewritemap-vhost-unity.sh /opt/apache/conf/vhost-unity.conf;
  sed -i -e \"s/PrivateTmp=true/PrivateTmp=false/\" /usr/lib/systemd/system/apache2.service;
  service apache2 restart;
  cd /tmp;
  rm -rf /.AS/SWS/os4k_platform_hotfix/os_pkgs_for_asmgr/;";
  
  if ($is_FI == 1) {
      log_msg_info("FI context, adapting assistant os_package_update_script ...");
      $assi_script_content =
  "cd /tmp/os_pkgs_for_asmgr/;
  rpm -Uhv --force *.rpm;
  bash ../fixup-rewritemap-vhost-unity.sh /etc/apache2/vhosts.d/vhost-unity.conf;
  bash ../fixup-rewritemap-vhost-unity.sh /opt/apache/conf/vhost-unity.conf;
  sed -i -e \"s/PrivateTmp=true/PrivateTmp=false/\" /usr/lib/systemd/system/apache2.service;
  cd /tmp;
  rm -rf /tmp/os_pkgs_for_asmgr/;";
  }
  
  my $assi_script = $assi_os_hf_pkg_dir."update_os_packages.sh";

  create_generic_file($assi_script_content, $assi_script);
  exec_cmd("chmod +x $assi_script");

  my $csta_script_content =
  "cd /.CAP_Inside/reinstall_data/hf/os_pkgs_for_csta/;
  rpm -ihv cld-os4k*rpm;
  rm -fv cld-os4k*rpm;
  rpm -Uhv --force *.rpm;
  sed -i -e \"s/PrivateTmp=true/PrivateTmp=false/\" /usr/lib/systemd/system/apache2.service;
  service apache2 restart;
  cd /tmp;
  rm -rf /.CAP_Inside/reinstall_data/hf/os_pkgs_for_csta/;";
  my $csta_script = $csta_os_hf_pkg_dir."update_os_packages.sh";

  if ($is_FI == 1) {
      log_msg_info("FI context, adapting CSTA os_package_update_script ...");
      $csta_script_content =
      "cd /tmp/os_pkgs_for_csta/;
      rpm -ihv cld-os4k*rpm;
      rm -fv cld-os4k*rpm;
      rpm -Uhv --force *.rpm;
      sed -i -e \"s/PrivateTmp=true/PrivateTmp=false/\" /usr/lib/systemd/system/apache2.service;
      cd /tmp;
      rm -rf /tmp/os_pkgs_for_csta/;";
  }
  
  create_generic_file($csta_script_content, $csta_script);
  exec_cmd("chmod +x $csta_script");

  log_msg_info("apache2 restart requested after package update in Assistant and CSTA");

  function_leave("create_os_package_update_scripts");
}

sub node_has_an_assistant {
  function_enter("node_has_an_assistant");
  my $have_an_assistant = 1;

  # check deployment
  if ($is_stmix == 1)                  { $have_an_assistant = 0 ; }
  if (is_standalone_deployment($multiNodeType) == 1) { $have_an_assistant = 0 ; }
  if (isQuorumNode() == 1)             { $have_an_assistant = 0 ; }
  if ( ! -d "/var/opt/Assistant/data/SWS/" ) {
      log_msg_info("no /var/opt/Assistant/data/SWS/");
      $have_an_assistant = 0 ;
  }
  if ($is_FI == 1) {
      log_msg_info("FI context, no Assistant existing yet");
      $have_an_assistant = 0 ;
  }
  if ($have_an_assistant == 0) {
    log_msg_info("there is no Assistant or CSTA container");
  }
  function_leave("node_has_an_assistant", "have_an_assistant=$have_an_assistant");
  return($have_an_assistant);
}

sub propagate_os_packages_to_containers {
  function_enter("propagate_os_packages_to_containers");
  my ($rc, $dir_on_share, $no_of_files_assi, $no_of_files_csta, $have_an_assistant) = (0, "", 0, 0, 1);

  # the scripts have to be created also in FI context for later usage
  $rc = create_os_package_update_scripts();

  # restart the Apache on V11 R0.22 due to the Apache package security update via Plt HF V11 R0.22.3
  # the same Apache restart will be done in the container update scripts.
  #OSFOURK-28965 - ensure apache2 does not use a Private Temp folder
  $rc = exec_cmd("sed -i -e \"s/PrivateTmp=true/PrivateTmp=false/\" /usr/lib/systemd/system/apache2.service");
  log_msg_info("restarting apache server due to security update");
  $rc = exec_cmd("service apache2 restart");
  
  $have_an_assistant = node_has_an_assistant();
  if ($have_an_assistant == 0) {
    if ($is_FI == 1) {
      log_msg_info("FI case => doing package installation using chroot");
    } else {
      log_msg_info("not FI case and there is no Assistant or CSTA container to update, skipping package installation");
      function_leave("propagate_os_packages_to_containers", "rc=$rc");
      return (0);
    }
  }

  $no_of_files_assi = get_number_of_files_in_dir ($assi_os_hf_pkg_dir);
  $no_of_files_csta = get_number_of_files_in_dir ($csta_os_hf_pkg_dir);

  # first Assistant
  if ($no_of_files_assi == 0) {
    log_msg_info("no OS packages for Assistant/Manager in $assi_os_hf_pkg_dir, skipping update in Assi container");
  } else {
    log_msg_info("$no_of_files_assi OS packages for Assistant/Manager in $assi_os_hf_pkg_dir found, updating in container");
    if ($is_FI == 0) {
        #$dir_on_share = "/var/opt/Assistant/data/SWS/os4k_platform_hotfix/os_pkgs_for_asmgr/";
        $rc = exec_cmd("cp -rv $assi_os_hf_pkg_dir /opt/yaps/script/fixup-rewritemap-vhost-unity.sh /var/opt/Assistant/data/SWS/os4k_platform_hotfix/");
        if ($rc != 0) {
          log_msg_info("error when copying OS packages from soco2 to /var/opt/Assistant/data/SWS/os4k_platform_hotfix/");
        } else {
          $rc = exec_cmd("sshr /.AS/SWS/os4k_platform_hotfix/os_pkgs_for_asmgr/update_os_packages.sh");
        }
    } else {
        my $assi_share_rootfs = `ls -1vd /opt/Assistant/share/rootfs_* | tail -n 1`;
        chomp($assi_share_rootfs);
        #$dir_on_share = "$assi_share_rootfs/tmp/os_pkgs_for_asmgr/";
        $rc = exec_cmd("cp -rv $assi_os_hf_pkg_dir /opt/yaps/script/fixup-rewritemap-vhost-unity.sh $assi_share_rootfs/tmp/");
        if ($rc != 0) {
          log_msg_info("error when copying OS packages from soco2 to $assi_share_rootfs/tmp/");
        } else {
          $rc = exec_cmd("chroot $assi_share_rootfs bash /tmp/os_pkgs_for_asmgr/update_os_packages.sh");
        }
    }
  }

  # second CSTA
  if ($no_of_files_csta == 0) {
    log_msg_info("no OS packages for CSTA in $csta_os_hf_pkg_dir, skipping update in CSTA container");
  } else {
    log_msg_info("$no_of_files_csta OS packages for CSTA in $csta_os_hf_pkg_dir found, updating in container");
    if ($is_FI == 0) {
        #$dir_on_share = "/var/opt/CAP_Inside/data/reinstall_data/hf/os_pkgs_for_csta/";
        $rc = exec_cmd("cp -rv $csta_os_hf_pkg_dir /var/opt/CAP_Inside/data/reinstall_data/hf/");
        if ($rc != 0) {
          log_msg_info("error when copying OS packages from soco2 to /var/opt/CAP_Inside/data/reinstall_data/hf/");
        } else {
          $rc = exec_cmd("sshr -c cap /.CAP_Inside/reinstall_data/hf/os_pkgs_for_csta/update_os_packages.sh");
        }
    } else {
        my $csta_share_rootfs = `ls -1vd /opt/CAP_Inside/share/rootfs_* | tail -n 1`;
        chomp($csta_share_rootfs);
        $rc = exec_cmd("cp -rv $csta_os_hf_pkg_dir $csta_share_rootfs/tmp/");
        if ($rc != 0) {
          log_msg_info("error when copying OS packages from soco2 to $csta_share_rootfs/tmp/");
        } else {
          $rc = exec_cmd("chroot $csta_share_rootfs bash /tmp/os_pkgs_for_csta/update_os_packages.sh");
        }
    }
  }


  # iSG containers ??

  function_leave("propagate_os_packages_to_containers", "rc=$rc");
  return ($rc);
}

sub erase_obsolete_packages {
  function_enter("erase_obsolete_packages");
  my ($rc, $overall_rc, $have_an_assistant) = (0, 0, 1);

  $have_an_assistant = node_has_an_assistant();
 
  # OSFOURK-26994 (V11 R0.22.1)
  $rc = exec_cmd("rpm -e docker docker-buildx docker-rootless-extras");
  if ($rc != 0) {
    log_msg_info("erasing docker failed with rc=$rc, but continue anyway");
    $overall_rc = 1;
  }

  if ($have_an_assistant == 1) {
    log_msg_info("erasing docker also in Assistant container. CSTA still needs it for CLD");
    $rc = exec_cmd("sshr        rpm -e docker docker-buildx docker-rootless-extras");
  }

  # OSFOURK-27001 (V11 R0.22.1)
  $rc = exec_cmd("rpm -e gstreamer-plugins-bad");
  $rc = exec_cmd("rpm -e libaom3");
  if ($rc != 0) {
    log_msg_info("erasing libaom3 failed with rc=$rc, but continue anyway");
    $overall_rc = 1;
  }

  if ($have_an_assistant == 1) {
    log_msg_info("erasing libaom3 also in Assistant and CSTA container");
    $rc = exec_cmd("sshr -c cap rpm -e gstreamer-plugins-bad");
    $rc = exec_cmd("sshr -c cap rpm -e libaom3");
    $rc = exec_cmd("sshr        rpm -e gstreamer-plugins-bad");
    $rc = exec_cmd("sshr        rpm -e libaom3");
  }

  # OSFOURK-27044 (V11 R0.22.1)
  # on Plt we do not have anything to do here
  if ($have_an_assistant == 1) {
    log_msg_info("erasing obsolete Tomcat examples directory in Assistant and CSTA container");
    $rc = exec_cmd("sshr -c cap rm -rfv /usr/share/tomcat/tomcat-webapps/examples ");
    $rc = exec_cmd("sshr        rm -rfv /usr/share/tomcat/tomcat-webapps/examples ");
    
    # OSFOURK-27365 additional cleanup in Assistant /etc/tomcat/server.xml necessary
    # remove the resource definition in Tomcats server.xml
    $rc = exec_cmd("sshr sed -i -e /examples/d /etc/tomcat/server.xml");
    # in case Assistant deploys again /opt/apache/tomcat/conf/server.xml to /etc/tomcat/server.xml create the empty examples
    $rc = exec_cmd("sshr mkdir /usr/share/tomcat/tomcat-webapps/examples");
  }

  function_leave("erase_obsolete_packages", "overall_rc=$overall_rc");
  return ($overall_rc);
}

sub do_rlc_actions_before_openais_is_stopped {
  my $multiNodeType = shift;
  my $isMultiNode = shift;

  my $is_sg2d = 0;
  my $do_lw_swupdated_dump = 0;
  my $sip_data_dump_already_done = 0; # local or global variable?

  function_enter("do_rlc_actions_before_openais_is_stopped", "multiNodeType=$multiNodeType isMultiNode=$isMultiNode");

  if (isCentralHost($multiNodeType) == 1) {
    log_msg_info("we are running on the central host, action may be required");

    my  $sg_initialcfg =  $softgate_config_dir . "initialcfg.xml";
    my $isg_initialcfg = $isoftgate_config_dir . "initialcfg.xml";

     my $is_sg_running = is_service_running("socod");
     my $is_sg2d_running = 0;
     if ($is_stmix == 0) {
       $is_sg2d_running = is_service_running("sg2d");
     }
     my $sg_is_configured = 0;
     my $number_of_sg_on_central_hosts = get_number_of_sg_on_central_hosts(0);
     if ($number_of_sg_on_central_hosts != 0) {
       log_msg_info("SoftGate flag in haipsetup.xml is set");
       if ($isMultiNode == 0) {
          $sg_is_configured = 1;
        }
      }

      my $sg2d_is_configured = 0;
      $sg2d_is_configured = is_integrated_isg_softgate();
      my $isg_is_configured = 0;
      if ( -e $isg_initialcfg ) {
        log_msg_info("found $isg_initialcfg, assuming iSG is active ");
        $isg_is_configured = 1; # using the existence of the initialcfg.xml inside the iSG container as criterium, not the system.iSG.node1/system.iSG.node2
      }

      if ($is_sg_running == 1 or $sg_is_configured == 1) {
          log_msg_info("socod running or configured-> SoftGate on Simplex active and needed ");
          $do_lw_swupdated_dump = 1;
      }

      if ($isMultiNode == 1)
      {
        # what to do on Duplex??
        if ($is_sg2d_running == 1 or $sg2d_is_configured == 1 or $isg_is_configured == 1) {
          $is_sg2d = 1;
          $do_lw_swupdated_dump = 1;
        }
      }
      if ($do_lw_swupdated_dump == 1) {
        log_msg_info("create the internal xml files via http commands");
        if (-x "/opt/soco/native/daemon/swupdated" and $sip_data_dump_already_done == 0) {
        log_msg_info("try to connect all vHG(SIP) boards via HTTPS and make fresh int-cfg.xml and int-scfg.xml");
        if ($is_sg2d == 0) {
          log_msg_info("calling SoftGate swupdated natively");
          exec_cmd("/opt/soco/native/daemon/swupdated rpm");

          # diagnosis
          exec_cmd("find $softgate_config_dir -name int-cfg.xml -o -name int-scfg.xml");
         } else {
          log_msg_info("calling SoftGate swupdated via chroot for Container instance");
          exec_cmd("/etc/init.d/sg2d chroot '/opt/soco/native/daemon/swupdated rpm'");

          # diagnosis
          exec_cmd("find $isoftgate_config_dir -name int-cfg.xml -o -name int-scfg.xml");
       }
        # ensure by flag that the dump is only executed once
        $sip_data_dump_already_done = 1;

      }
    }
  } else {
    log_msg_info("we are not running on the central host, no actions required");
  }

  function_leave("do_rlc_actions_before_openais_is_stopped", "");
  return ();
}

sub do_upgrade {
  my $force_host_reboot = shift;
  my $force_rmx_reboot = shift;  
  my $force_upgrade_with_completely_stopped_ha = shift;
  my $force_upgrade_with_full_drbd_sync = shift;
  my $force_upgrade_with_reduce_downtime = shift;
  my $force_upgrade_via_image = shift;
  my $is_autodeact_downgrade = shift;
  my $multiNodeType = shift;
  my $isMultiNode = shift;
  my $other_node = shift;
  my $node_q = shift;
  my $ha_type = "";
   
  $global_rc   = 0;
  
  my $other_node_is_reachable = 0;
  my ($node_q_is_reachable, $dtb_resource_has_been_stopped) = (0, 0);
  
  function_enter("do_upgrade", "with force_host_reboot=$force_host_reboot force_rmx_reboot=$force_rmx_reboot force_upgrade_with_completely_stopped_ha=$force_upgrade_with_completely_stopped_ha force_upgrade_with_full_drbd_sync=$force_upgrade_with_full_drbd_sync force_upgrade_with_reduce_downtime=$force_upgrade_with_reduce_downtime force_upgrade_via_image=$force_upgrade_via_image is_autodeact_downgrade=$is_autodeact_downgrade multiNodeType=$multiNodeType isMultiNode=$isMultiNode other_node=$other_node node_q=$node_q");
  while(1) {
    sleep(3);
    
    if($global_rc == 0 && $hotfix_mode && $extract) {
      log_msg_info("*** starting to extract hotfix");
      $global_rc = extract_hotfix();
      if($global_rc != 0) {
        write_status($global_rc, "could not extract hotfix files");
        log_msg_info("hotfix mode finished");
        $is_old = 1;
      } 
      
      if ($global_rc == 0 && $hotfix_mode == 2 && $hotfix_number_to_fallback_to_by_deactivation == 0 ) {
        log_msg_info("hotfix deactivation to base FR detected -> cleaning the storage of the installed hotfix rpms"); 
        removeDir($cur_hf_rpm_dir);		
      }
      
    }
    if($global_rc == 0 && $extract) {
      log_msg("renaming all rpm files in repository");
      rename_rpm_files($rpm_dir);
    }
    
    if($global_rc == 0 && $extract) {
      if ($no_self_xtract == 1 || $hotfix_mode == 2 || $nodaemon == 1 || $hotfix_mode == 0) {
        log_msg_info("not trying to extract new software updater daemon due to option no_self_xtract or hotfix deactivation or remote nodaemon mode or RLC update. Using old one."); 
        $is_old = 1;	
      } else {
        log_msg_info("starting to extract new software updater daemon");
        if(-e glob($yaps_rpm_file)) {
          $global_rc = exec_cmd("cd / && /usr/bin/rpm2cpio $yaps_rpm_file | cpio -ivmdu \"*swupdated\"");
          if ($global_rc == 0) {
            log_msg_info("extract new software updater daemon from ASyaps package succeeded, but do not copy to webservice");
            # exec_cmd("/bin/cp -vf /opt/yaps/script/swupdated $self");
          }
        } else {
          if(-e glob($ws_rpm_file)) {
            $global_rc = exec_cmd("cd / && /usr/bin/rpm2cpio $ws_rpm_file | cpio -ivmdu \"*swupdated\"");
          } else {
            $global_rc = 1; # webservice rpm and ASyaps does not exist, force using old one
          }
        }
        if($global_rc == 0) {
          # remove lock file
          remove_upgrade_lock_file();
          # start newly extracted daemon (from yaps))
          $global_rc = exec_cmd($yaps_self.$params);
          if($global_rc == 0) {
            log_msg_info("exiting old software updater daemon");
            exit(0);
          } else {
            log_msg_info("extracting of new software updater succeeded but executing it failed -> continuing using old one, but require lock again");
            if(is_daemon_running()) {
              log_msg_info("a new daemon was already started since I removed my lock, obviously Assistant sent retry -> I must not continue but let the new daemon work");
              exit(1);
            } else {
              log_msg_info("a new daemon is not yet running, -> I can continue if I request the lock again");
              $global_rc = create_upgrade_lock_file();
            }
          }
        }
        if($global_rc != 0) {
          log_msg_info("could not extract or start new software updater, using old one");
          $global_rc = 0;
          $is_old = 1;
        }
      }
    }

    # plausi check for existence of at least one rpm package to be installed
    my $no_of_files = get_number_of_files_in_dir ($rpm_dir);
    if ($no_of_files == 0 and $sg_os_update == 0) {
      log_msg_info("Error: Did not find any rpm package in upgrade dir $plt_dir -> cancel upgrade before stopping any resources");
      $global_rc = $ERROR_NoUpgradePackagesFound;
    }

    if($global_rc == 0) {
      log_msg_info("*** starting software update");
      
      if($global_rc == 0) {

        log_msg_info("upgrade flags immediately before get_upgrade_flags_from_file: rmx_reboot=$rmx_reboot host_reboot=$host_reboot upgrade_with_completely_stopped_ha=$upgrade_with_completely_stopped_ha upgrade_with_full_drbd_sync=$upgrade_with_full_drbd_sync full_drbd_sync_flag_really_read=$full_drbd_sync_flag_really_read upgrade_with_reduce_downtime=$upgrade_with_reduce_downtime upgrade_via_image=$upgrade_via_image ");

        my $activation_avoidance_possible = 0;
        if ($hotfix_mode) {
          my $hf_info_file_to_analyze = "";
          $hf_info_file_to_analyze = get_hotfix_info_file();
          $activation_avoidance_possible = get_upgrade_flags_from_file($hf_info_file_to_analyze);
        } else {
          my $info_file_to_analyze = get_upgrade_info_file();
          get_upgrade_flags_from_file($info_file_to_analyze);
        }

        # only check on active node
        if ($activation_avoidance_possible == 1 && $nodaemon == 0) {
          log_msg_info("activation avoidance maybe possible because destination and current versions on master node are equal");
          my $yaps_versions_are_identical_on_main_node = is_yaps_version_identical($yaps_rpm_file);

          if ($yaps_versions_are_identical_on_main_node == 1 ) {
            log_msg_info("activation avoidance maybe still possible because ASyaps versions are identical in HF and installed");
            
            my $has_a_quorum = has_quorum($node_q, $multiNodeType);
            my $consistency_status = get_rpm_consistency_status_for_all_nodes ($upgrade_version, $multiNodeType, $isMultiNode, $has_a_quorum);
            if ($consistency_status != 0) {
              log_msg_info("activation avoidance is not possible because rpm versions on current node and remote nodes are not consistent or activation version != current version");
            } else {
              log_msg_info("activation avoidance is possible because rpm versions on current node and remote nodes are consistent and activation version $upgrade_version == current version");
              $activation_has_been_avoided = 1;
              log_msg_info("<<< do_upgrade with activation_has_been_avoided=$activation_has_been_avoided global_rc=$global_rc");
              return($global_rc);
            }
          } else {
          log_msg_info("activation avoidance not possible since ASyaps versions differ, obviously new release candidate of the same hotfix");
          }
        }

        if ($rmx_reboot == 0) {
          log_msg_info("flag REST_RMX is not set, -> cheap type 1 hotfix/rlc");
          if ($hotfix_mode) {
            # check for DTB package in Hotfix
            my $local_dtb_rpm_file = glob($dtb_rpm_file);
            if( defined($local_dtb_rpm_file) and -e $local_dtb_rpm_file ) {

              # check Quorum
              if ($isquorum == 1) {
                log_msg_info("DTB rpm package detected in HF set but node is a Q -> remove package as rpm installation will fail due to missing wine dependency");
                exec_cmd ("rm -vf $dtb_rpm_file");
              } else {
                log_msg_info("DTB rpm package detected in HF set but node is not a Q -> keeping package for update");
              }

              log_msg_info("type 1 hotfix and DTB rpm package detected in the hotfix set -> check status of DTB resource");
              if (is_resource_running("rsc4k_dtb") == 1) {
                if ($nodaemon == 0) {
                  log_msg_info("DTB resource is currently running on active node -> stop it before rpm update of type 1 hotfix");
                  stop_resource("rsc4k_dtb");
                  $dtb_resource_has_been_stopped = 1;
                  log_msg_info("wait 5 seconds after stopping DTB resource, processes may still be running");
                  sleep(5);
                } else {
                  log_msg_info("DTB resource is currently running on non active node (node_B or Q) -> not stopping it");
                }
              }
            } else {
                log_msg_info("no DTB rpm package detected in the hotfix set");
            }
          }
        }
        
        log_msg_info("upgrade flags immediately after get_upgrade_flags_from_file: rmx_reboot=$rmx_reboot host_reboot=$host_reboot upgrade_with_completely_stopped_ha=$upgrade_with_completely_stopped_ha upgrade_with_full_drbd_sync=$upgrade_with_full_drbd_sync full_drbd_sync_flag_really_read=$full_drbd_sync_flag_really_read upgrade_with_reduce_downtime=$upgrade_with_reduce_downtime upgrade_via_image=$upgrade_via_image ");
        $rmx_reboot  =  $rmx_reboot  || $force_rmx_reboot;   # 0 = no, 1 = yes
        $host_reboot =  $host_reboot || $force_host_reboot ; # 0 = no, 1 = yes
        $upgrade_with_completely_stopped_ha = $upgrade_with_completely_stopped_ha || $force_upgrade_with_completely_stopped_ha; # 0 = no, 1 = yes
        $upgrade_with_full_drbd_sync        = $upgrade_with_full_drbd_sync        || $force_upgrade_with_full_drbd_sync;        # 0 = no, 1 = yes
        $upgrade_with_reduce_downtime       = $upgrade_with_reduce_downtime       || $force_upgrade_with_reduce_downtime;       # 0 = no, 1 = yes
        $upgrade_via_image                  = $upgrade_via_image                  || $force_upgrade_via_image;                  # 0 = no, 1 = yes
        log_msg_info("upgrade flags after calculating the auto deact params: rmx_reboot=$rmx_reboot host_reboot=$host_reboot upgrade_with_completely_stopped_ha=$upgrade_with_completely_stopped_ha upgrade_with_full_drbd_sync=$upgrade_with_full_drbd_sync full_drbd_sync_flag_really_read=$full_drbd_sync_flag_really_read upgrade_with_reduce_downtime=$upgrade_with_reduce_downtime upgrade_via_image=$upgrade_via_image"); 

        if ($upgrade_via_image != 0) {
            log_msg_info("NUC update detected");
            reset_NUC_status_for_SG();
            reset_NUC_status_for_WS();

            log_msg_info("NUC update detected -> wait for PREP to finish");
            $global_rc = wait_for_NUC_PREP($multiNodeType);
            if ($global_rc != 0) {
              log_msg_info("<<< do_upgrade with activation_has_been_avoided=$activation_has_been_avoided global_rc=$global_rc");
              return($global_rc);
            }
        }
      }
      
      # print packages info
      if($global_rc == 0) {
        $global_rc = exec_cmd("/bin/rpm -qi -p ".$rpm_dir."*.rpm");
        if($global_rc != 0) {
          $global_rc = $ERROR_CouldNotGetPackagesInfo;
          write_status($global_rc, "could not get packages info");
        }
      }

      if ($is_stmix == 0) {
        if($global_rc == 0) {
          $global_rc = execute_pre_ws_update();
        }
      }
      
      if($global_rc == 0) {
        log_msg_info("starting ASyaps pre install");
        $global_rc = execute_yaps_preinstall();
        if($global_rc != 0) { write_status($global_rc, "ASyaps preinstall failed"); }
      }      

      if ($is_stmix == 0) {
        if ($global_rc == 0) {
          $global_rc = execute_pre_ha_update($upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $multiNodeType, $isMultiNode);
        }

        my $drbd_output = exec_cmd_out("/usr/sbin/drbd-overview");
        log_msg_info("drbd status before openais may be shut down:");
        log_msg_info("$drbd_output");
      }

      create_call_file();
      create_callstatus_file($Wiederaufsetzpunkt2, $continue_upgrade_iteration, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $upgrade_with_reduce_downtime, $upgrade_via_image);

      if ( -e "/var/log/nuc/swupdated.use.nnuc" ) {
        log_msg_info("skipped openais stopped on all nodes due to special N-NUC case");
      } else {
      if($global_rc == 0 && !$nodaemon && $is_stmix == 0 && $upgrade_with_completely_stopped_ha == 1 && $remote_updates_were_already_successfully_done_by_previous_update == 0 ) {
        
        set_internal_upgrade_step($STEP_stop_openais_t3, 1, "Active", $hostname);

        log_msg_info("before stopping openais service do all the steps which need a running RMX, e.g. the data backup of integrated SGs on central host");
        do_rlc_actions_before_openais_is_stopped($multiNodeType, $isMultiNode);
        
        log_msg_info("stopping local AND remote openais service because HA or image package has to be upgraded (initiated always from the active side)");

        log_msg_info("making the next stop operations interruptable by existence of cancel trigger file $cancel_upgrade_file");
        enable_ssh_wait_cancel($cancel_upgrade_file);
        
        my $angstwart_dauer = 0; # in Minutes
        if (($global_rc == 0) && (has_quorum($node_q, $multiNodeType) == 1)) {
          $global_rc = stop_openais_service("quorum", $angstwart_dauer, 1, 1, 1, 1);
          
          # after stopping HA on Q we have to keep the watchdog away at least for EcoServer with only 10 minutes watchdog timeout
          start_watchdog_daemon_remote($node_q);          
        } 
        if (($global_rc == 0) && ($isMultiNode == 1)) {
          $global_rc = stop_openais_service("remote", $angstwart_dauer, 1, 1, 1, 1);
          
          # after stopping HA on Standby we have to keep the watchdog away at least for EcoServer with only 10 minutes watchdog timeout
          start_watchdog_daemon_remote($other_node);             
        }

        if($global_rc == 0) {
          # it is enough to sleep the 5 minutes only once, since the remote nodes now have also time to crash and recover
          my $is_sg_standalone = is_softgate_standalone($multiNodeType);
          if ($is_sg_standalone == 1) {
            log_msg_info("Also for SG Standalone I have to execute Angstwarten after stop. The reboot to activate would be othererwise in danger of watchdog when generating the initrd.");
            $angstwart_dauer = 5; # Minutes
          } else {
            $angstwart_dauer = 5; # Minutes
          }
          $global_rc = stop_openais_service("local", $angstwart_dauer, 1, 1, 1, 1);
          
          # after stopping HA on primary we have to keep the watchdog away at least for EcoServer with only 10 minutes watchdog timeout
          start_watchdog_daemon();          
        }

        log_msg_info("disable again the cancel ssh wait mechanism");
        disable_ssh_wait_cancel();
      }
      } # ( -e "/var/log/nuc/swupdated.use.nnuc" )
      
      trigger_watchdog();
      remove_call_status_file();

      if ($hotfix_mode == 0) {
        log_msg_info("Creating a new (failure) Wiederaufsetzpunkt5 to cover watchdog/power-off during remote NUC installation");
        create_call_file_for_reboot();
        create_callstatus_file($Wiederaufsetzpunkt5, $continue_upgrade_iteration, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $upgrade_with_reduce_downtime, $upgrade_via_image);
      } else {
        log_msg_info("Creating a new (failure) Wiederaufsetzpunkt2 to cover watchdog/power-off during Hotfix installation -> retry complete activation");
        create_call_file();
        create_callstatus_file($Wiederaufsetzpunkt2, $continue_upgrade_iteration, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $upgrade_with_reduce_downtime, $upgrade_via_image);
      }
      
      $ha_type = get_ha_type();
      if ($is_stmix == 0) {
        if($global_rc == 0) {
          execute_yaps_preinstall2($ha_type, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $multiNodeType, $isMultiNode);
        }
      }
      
      if($global_rc == 0) {
        if(!$nodaemon && $is_stmix == 0 && $is_FI == 0 && $is_autodeact_downgrade != 1  && $remote_updates_were_already_successfully_done_by_previous_update == 0) {
          set_internal_upgrade_step($STEP_remote_installation, 1, "Active", $hostname);
          $global_rc = execute_remote_update(\$other_node_is_reachable, \$node_q_is_reachable, $multiNodeType, $isMultiNode, $other_node, $node_q);
          if($global_rc != 0) {
            if ($other_node_is_reachable == 0 && $duplex_with_one_board_allowed == 1) {
              log_msg_info("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");            
              log_msg_info("!!!!! could not reach remote HA node via corosync IP");
              log_msg_info("!!!!! maybe this is a configured Duplex scenario with only one board running");              
              log_msg_info("!!!!! ignoring this error situation and trying to upgrade the local node");              
              log_msg_info("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");            
              $global_rc = 0;
            } else {
              write_status($global_rc, "failed to update remote HA node, too dangerous to continue with upgrade of local node"); 
            }
          }
        }
        if($global_rc == 0) {
          set_internal_upgrade_status($STATUS_running, $defaultHostType, $hostname);
          
          # stopping daemon
          if(!$nodaemon && $rmx_reboot) {
            # on request from assistant to get status before ha shutdown
            log_msg_info("sleeping 30secs...");
            sleep(30);
            if ($isMultiNode == 1 && $upgrade_with_completely_stopped_ha == 0) {
              log_msg_info("for type2 HF we sleep additionally 60 seconds to be sure the StandBy Node is really up again ...");
              sleep(60);
            }
          }
          if($rmx_reboot) {
            trigger_watchdog();

            create_call_file();
            create_callstatus_file($Wiederaufsetzpunkt3, $continue_upgrade_iteration, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $upgrade_with_reduce_downtime, $upgrade_via_image);
            if($rmx_reboot && $is_stmix == 0) {
              # I will stop the openais anyway even if the remote nodes (B and Q) are already down, just to be sure and to set the flag to start it again after type2 upgrades
              if ( -e "/var/log/nuc/swupdated.use.nnuc" ) {
                log_msg_info("skipped openais stop on local node due to special N-NUC case");
              } else {
                $global_rc = stop_openais_service("local", 5, 1, 1, 0, 1);
              }
            }
          } else {
            log_msg_info("not stopping openais service (not requested by e.g. Hotfix info file)");
          }
        }
      } else {
        log_msg_info("!!! Skipped remote update due to previous error"); 
      }
    }
    
    remove_call_status_file();
    
    trigger_watchdog();

    if($is_stmix == 0) {
      if ($global_rc == 0) {
        # only activate this if the HA resource agent creates the bind mount /var/opt/Assistant/data also for simplex deployments
        migrate_assistant_share_from_var_to_pgsql_partition($isMultiNode);
        trigger_watchdog();
      }
    }
    

    if ($global_rc == 0 && $upgrade_dir_has_been_removed_due_to_duplex_hf_deactivation == 1) {
      log_msg_info("*** starting again to extract hotfix for deactivation");
      $global_rc = extract_hotfix();
      if($global_rc != 0) {
        write_status($global_rc, "could not extract hotfix files");
        log_msg_info("hotfix mode finished");
      }

      if($global_rc == 0) {
        log_msg("renaming all rpm files in repository");
        rename_rpm_files($rpm_dir);
      }
    }

    # package info list not really useful for NUC upgrade, since we get a new image
    if($global_rc == 0 and $upgrade_via_image == 0) {
      print_package_info("before");
      dump_cla_diagnosis_data();
    }
  
    if($global_rc == 0 and $upgrade_via_image == 0) {
      log_msg_info("*** starting update of the rpm packages");   
      set_internal_upgrade_step($STEP_packet_installation, 1, $defaultHostType, $hostname);
      if ( $skip_quorum_rpm_upgrade == 1 ) {
        log_msg_info("*** trigger file $skip_quorum_flagfile was detected on node_A -> skipping the rpm update on the quorum");
      } else {
        
        my $separate_rc = separate_packages_into_own_directory("ASyaps-", $rpm_dir, $rpm_post_dir);

        my $do_java_update = 0;
        my $do_trigger_watchdog = 1;
        if ($is_stmix == 1) {
          log_msg_info("on STMIX do not trigger the watchdog");
          $do_trigger_watchdog = 0;
        }

        prepare_os_packages_for_container();

        # $global_rc = update_packages_with_zypper($rpm_dir, 1, 1, $do_java_update, 1, 0, $do_trigger_watchdog);
        $global_rc = update_packages_with_rpm($rpm_dir, 1, $do_trigger_watchdog);
        if ($global_rc == 0 && $separate_rc == 0) {
          # $global_rc = update_packages_with_zypper($rpm_post_dir, 0, 0, $do_java_update, 1, 0, $do_trigger_watchdog);
          $global_rc = update_packages_with_rpm($rpm_post_dir, 0, $do_trigger_watchdog);
        }
        if ($is_stmix == 0) {
          exec_after_rpm_installation();
        }

        propagate_os_packages_to_containers();

        print_package_info("after");
        dump_cla_diagnosis_data();
        
        restart_systemd_daemon();
      }
      set_internal_upgrade_step ($STEP_post_processing, 1, $defaultHostType, $hostname);
      
      if($global_rc == 0) {
        if ($hotfix_mode == 0) {
          # FR update
          $global_rc = store_upgrade_rpms_into_installrepo();

          # for RLC upgrades which are coming from V6 R1, we have to harden the platform
          # but not for NUC, since here the new world is hardened by the activation script
          if ($upgrade_via_image == 0) {
            activate_platform_hardening();
          } else {
              log_msg_info("platform hardening is skipped in case of NUC");
          }
        } else {
          # HF

          # erase packages which have become obsolete via Hotfix (OSFOURK-25620)
          erase_obsolete_packages();
          
          if ($hotfix_mode == 1) {
            # hotfix activation
            $global_rc = store_hotfix_into_instmedia();
            $global_rc = store_hotfix_rpms_into_hotfixrepo();

            if ($isquorum == 1) {
              log_msg_info("handle Quorum specific package cleanup");
              log_msg_info("dtb package may have been removed from packages installed on Q, need to relax checkydata");
              relax_checkydata("dtb");

              log_msg_info("removing transmission packets, which are not needed on a quorum, but at least relax checkydata");
              exec_cmd("/opt/webservice/scripts/clean_rpms_nodeq.sh");
            }
          } else {
            # hotfix deactivation
            if ($hotfix_number_to_fallback_to_by_deactivation == 0) {
              # hotfix deactivation to base  
              log_msg_info("*** hotfix deactivation to base FR detected -> removing the storage of the installed hotfix rpms ($cur_hf_rpm_dir)"); 
              removeDir($cur_hf_rpm_dir);
            } else {
              # hotfix deactivation to other HF  
              $global_rc = store_hotfix_rpms_into_hotfixrepo();
            }
          }        
        }
        log_msg_info("removing the temporary rpm storage dir ($tmp_rpm_dir)");   
        removeDir($tmp_rpm_dir);      
      }
      if ($global_rc == 0) {
        $global_rc = store_osdep_rpms_into_osdeps_repo();
      }
      
      if($global_rc != 0) { write_status($global_rc, "update rpm files failed"); }
    }
   
    if ($is_FI == 0 and $is_stmix == 0) {
      # start webservice, tunnelbroker and DTB resource again even if upgrade failed
      start_all_stopped_resources_if_necessary($hotfix_mode, $rmx_reboot, $dtb_resource_has_been_stopped);
    }
    
    if($is_old != 0 && $no_self_xtract != 1 && !$nodaemon) {
      # rename update daemon script if no new one supplied
      rename($old_self, $self);
      log_msg("renamed update daemon file $old_self to $self");
    }
    # clean intermediate files
    if($global_rc == 0) {
      clean_intermediate_files()
    }

    # bios update has to be made before openais is started again and before the host_reboot flag is evaluated
    if ($global_rc == 0 && $biosupdate != 0) {
      if ($upgrade_via_image == 0) {
        activate_bios();
      } else {
        log_msg_info("BIOS update requested by Assistant, but on NUC update this is handled by image installation");
      }
    }
   
    trigger_watchdog();

    if ($global_rc == 0 && $upgrade_via_image == 1) {
      $global_rc = do_NUC_data_upgrade($multiNodeType, $isMultiNode, $isquorum);
    }
   
    my $generate_cib = 1;

    if ($nodaemon == 1 && $host_reboot == 1 && $upgrade_with_completely_stopped_ha == 1) {
      log_msg_info("*** do not Generating CIB on a standby/quorum node because of the Host reboot, active node will do it later instead of the openais start");
      $generate_cib = 0;
    }

    if ($nodaemon == 1 && $isquorum == 0 && $upgrade_with_reduce_downtime == 1 && $upgrade_with_completely_stopped_ha == 1) {
      log_msg_info("*** do not Generating CIB on a standby node because of the reduce downtime, active node will do it later instead of the openais start");
      $generate_cib = 0;
    }
    
    if($nodaemon == 0 && $host_reboot == 1 && $upgrade_with_completely_stopped_ha == 1) {
      log_msg_info("*** do not Generating CIB on a active node because of the Host reboot, the plt service will do it after the reboot");
      $generate_cib = 0;
    }    
    
    if($nodaemon == 0 && $upgrade_with_completely_stopped_ha == 1 && $local_ha_has_been_stopped == 0) {
      log_msg_info("*** do not Generating CIB on a active node with type 3 upgrade in case that the local openais has not yet been stopped");
      $generate_cib = 0;
    }
    if ($is_FI == 1) {
      log_msg_info("*** do not Generating CIB in FI context");
      $generate_cib = 0;
    }
    if($global_rc == 0 && $hotfix_mode == 1) {
      log_msg_info("restarting inventory daemon after hotfix activation so that inventory will be aware of the new plt HF version");
      exec_cmd("service inventoryd restart");
    }
    if($global_rc == 0 && $upgrade_via_image == 0) {
      log_msg_info("starting ASyaps pre-postinstall (independant of the generating CIB)");
      $global_rc = execute_yaps_prepostinstall("-u");
      if($global_rc != 0) { write_status($global_rc, "ASyaps prepostinstall failed"); }
    }

    if ($generate_cib == 1 && $upgrade_via_image == 0) {
      if($global_rc != 0) {
        log_msg_info("*** generating CIB anyway even if the upgrade failed (H97564)");
      }
      # ignoring return code !
      $ha_type = get_ha_type();
      execute_after_rpm_install_ha($ha_type, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $multiNodeType, $isMultiNode);
    }

   trigger_watchdog();

   my $start_rc = 0;
   if($local_ha_has_been_stopped == 1 && $is_stmix == 0) {
      if ($global_rc != 0 && $hotfix_mode != 0 && $autodeact == 1 && $is_autodeact_downgrade == 0) {
        log_msg_info("*** HF upgrade failed and autodeact is configured ");
        log_msg_info("*** That means an automatic Hotfix deactivation downgrade will follow ");
        log_msg_info("*** skipping start of openais due to optimization");
      } else {
        if ($host_reboot == 1) {
          log_msg_info("*** upgrade type with Linux host reboot is requested");
          log_msg_info("*** skipping start of openais due to optimization (linux reboot will start it anyway)");
        } else {
          if ($upgrade_with_completely_stopped_ha) {
            log_msg_info("*** skipping start of openais due to special HA upgrade (will be done later)");
          } else {
            $start_rc = start_openais_service("local", 0);
            $local_ha_has_been_stopped = 0;
          }
        }
      }
    }
    if ($global_rc == 0 && $start_rc != 0) {
      log_msg_info("*** upgrade was successful, but starting openais service failed");
      $global_rc = $ERROR_StartingOpenaisFailed;
    }

    if($global_rc == 0) {
      if ($generate_cib == 1) {
        my $ha_rc = execute_post_ha_update($upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $multiNodeType, $isMultiNode);
        if ($ha_rc != 0) {
          log_msg_info("ignoring return code of HA script");
        }
      } else {
        log_msg_info("skipping execute_post_ha_update/exec_after_upgrade.sh (no initHA necessary)");
      }

      log_msg_info("starting ASyaps postinstall (also in FI context without initHA)");
      $global_rc = execute_yaps_postinstall("-u");
      if($global_rc != 0) { write_status($global_rc, "ASyaps postinstall failed"); }
    }

    if ($is_stmix == 0) {
      start_sysinfo();
    }
    
    trigger_watchdog();

    # waiting for successful startup of active node - perform automatic restore if needed and set
    if($global_rc == 0 && !$nodaemon && $is_FI == 0 && $upgrade_via_image == 0) {
      if ($upgrade_with_completely_stopped_ha == 1) {
        log_msg_info("skipping call of the wait for success script $upd_ok_sh for active node due to upgrade with complete HA down");
      } else {
        if ($host_reboot == 1) {
          log_msg_info("skipping call of the wait for success script $upd_ok_sh for active node due to host reboot");
        } else {
          $global_rc = wait_for_successful_startup(1, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $multiNodeType, $isMultiNode, "primary", $max_wait_for_up, 0);
          if ($global_rc == 0) {
            append_timestamp_file_for_assi(3, 1); # third entry after start with forced write
          }
        }
      }
    }

    if ($is_stmix == 0) {
      # clean update repos after upd_ok_sh 
      clean_update_repos();
    
      sleep(1);
    
      my $drbd_output = exec_cmd_out("/usr/sbin/drbd-overview");
      log_msg_info("drbd status after installation (openais may be still down):");
      log_msg_info("$drbd_output");
    }

    function_leave("do_upgrade", "with activation_has_been_avoided=$activation_has_been_avoided global_rc=$global_rc");
    return($global_rc);
  }
}

sub dump_cla_diagnosis_data {
  function_enter("dump_cla_diagnosis_data", "");
  my $output = "";

  $output = exec_cmd_out("ls -l /opt/cla/license/");
  log_msg_info("$output");

  $output = exec_cmd_out("ls -ld /opt/cla/license/");
  log_msg_info("$output");

  $output = exec_cmd_out("rpm -q -i cla");
  log_msg_info("$output");

  $output = exec_cmd_out("ls -l /opt/cla/log/");
  log_msg_info("$output");

  function_leave("dump_cla_diagnosis_data", "");
  return;
}

sub get_local_bios_version {
  function_enter("get_local_bios_version", "");
  my $current_version = exec_cmd_out("/usr/sbin/dmidecode -s bios-release-date");
  $current_version=my_trim($current_version);
  function_leave("get_local_bios_version", "current_version=$current_version");
  return($current_version);
}

sub get_local_firmware_version {
  function_enter("get_local_firmware_version", "");
  my $current_version = exec_cmd_out("/bin/rpm -q --qf '%{VERSION}-%{RELEASE}' os4k-platform-support");
  $current_version=my_trim($current_version);
  function_leave("get_local_firmware_version", "current_version=$current_version");
  return($current_version);
}

sub get_remote_bios_version {
  my $bios_wait_for_ssh_lock = shift;
  function_enter("get_remote_bios_version", "bios_wait_for_ssh_lock=$bios_wait_for_ssh_lock");
  
  my $output = "";
  my ($other_node_is_reachable, $password_is_expired) = (0, 0);
  
  my $cmd_str = "/usr/sbin/dmidecode -s bios-release-date";
  my $ignore_return_code = 1;
  my $rc = execute_remote_duplex_cmd($cmd_str, \$output, 1, \$other_node_is_reachable, \$password_is_expired, $ignore_return_code, $bios_wait_for_ssh_lock);

  # $ignore_return_code = 1 is chosen to omit thr rc=x output
  $output=my_trim($output);
  function_leave("get_remote_bios_version", "output=$output");
  return($output);
}

sub check_if_remote_bios_activation_would_be_enforced {
  function_enter("check_if_remote_bios_activation_would_be_enforced", "");
  my $rc = 0;
  
  if($hotfix_mode == 1) {
    log_msg_info("in hotfix mode we will not enforce a DSCXL BIOS activation on the standby machine");
  } else {
    if ($is_dscxl == 1) {
      if ($biosupdate == 1 ) {
        log_msg_info("the local BIOS update flag is already set -> set also the remote one");
        $biosupdate_remote = $biosupdate;
      } else {
        my $current_bios_version = get_remote_bios_version(120);
        if ($current_bios_version eq $intended_bios_version) {
          log_msg_info("the currently installed DSCXL2 BIOS on the standby is already the bios $intended_bios_version / $intended_bios_file -> no need to enforce activation");
        } else {
          log_msg_info("*** the currently installed DSCXL2 BIOS $current_bios_version on the standby differs from the intended $intended_bios_version / $intended_bios_file");
          $rc = 1;
        }
      }
    } else {
        log_msg_info("no DSCXL HW -> no bios activation on the standby");
    }
  }
  function_leave("check_if_remote_bios_activation_would_be_enforced", "");
  return($rc);
}

sub check_if_local_bios_activation_has_to_be_enforced {
  function_enter("check_if_local_bios_activation_has_to_be_enforced", "");
  my $rc = 0;
  
  if($hotfix_mode == 1) {
    log_msg_info("in hotfix mode we will not enforce a DSCXL BIOS activation");
  } else {
    if ($is_dscxl == 1) {
      if ($biosupdate == 1 ) {
        log_msg_info("the BIOS update flag is already set -> no need to enforce it");
      } else {
        my $current_bios_version = get_local_bios_version();
        if ($current_bios_version eq $intended_bios_version) {
          log_msg_info("the currently installed DSCXL2 BIOS is already the bios $intended_bios_version / $intended_bios_file -> no need to enforce activation");
        } else {
          log_msg_info("*** the currently installed DSCXL2 BIOS $current_bios_version differs from the intended $intended_bios_version / $intended_bios_file");
          $rc = 1;
        }
      }
    } else {
        log_msg_info("no DSCXL HW -> no enforced bios activation");
    }
  }
  function_leave("check_if_local_bios_activation_has_to_be_enforced", "rc=$rc");
  return($rc);
}

sub remove_remote_status_file {
  function_enter("remove_remote_status_file", "");

  my $output = "";
  my ($other_node_is_reachable, $password_is_expired) = (0, 0);

  my $cmd_str = "/bin/rm -vf $status_file";
  my $ignore_return_code = 1;
  
  my $rc = 0;

  if ($isMultiNode == 1) {
    $rc = execute_remote_duplex_cmd($cmd_str, \$output, 1, \$other_node_is_reachable, \$password_is_expired, $ignore_return_code);
  }
  function_leave("remove_remote_status_file", "rc=$rc");
  return($rc);
}


sub execute_bios_check {
  my $isMultiNode = shift;
  my $remote_host_is_reachable = shift;
  my $ssh_timeout = shift;
  
  function_enter("execute_bios_check", "isMultiNode=$isMultiNode remote_host_is_reachable=$remote_host_is_reachable  ssh_timeout=$ssh_timeout");
   my $rc = 0;

   if ($is_dscxl == 0) {
     log_msg_info("hw is not a DSCXL2 -> bios check has to be skipped ok");
     $rc = 0;
   } else {
     log_msg_info("hw is a DSCXL2 -> bios check for bad version $bad_bios_version has to be done on node_A and node_B");
     my $current_version_a = get_local_bios_version();
     my $len = length($current_version_a);
     log_msg_info("local bios version: $current_version_a length=$len");

     if ($current_version_a eq $bad_bios_version) {
       log_msg_info("bad bios version detected: $current_version_a");
       $rc = 1;
     }
     if ($isMultiNode == 1) {
      if ($remote_host_is_reachable == 1) {
        my $current_version_b = get_remote_bios_version($ssh_timeout);
        $len = length($current_version_b);
        log_msg_info("remote bios version: $current_version_b length=$len");
        if ($current_version_b eq $bad_bios_version) {
          log_msg_info("bad bios version detected: $current_version_b");
          $rc = 1;
        }
      } else {
        log_msg_info("data_currentness indicates that remote node is not reachable -> skip checking remote bios since upgrade is blocked anyway");
      }
     }
   }
     
  function_leave("execute_bios_check", "rc=$rc");
  return ($rc);
}

sub get_bios_error_string_en {
  return("The BIOS (version 08/06/2010) of at least one of the DSCXL2 boards contains an error which could prevent startup in case of a board reboot.
No platform hotfix activation will be possible. Please correct the erroneous BIOS according to service info INF-12-000040.");
}

sub get_bios_error_string_de {
  return("Das BIOS (Version 08/06/2010) mindestens eines DSCXL2-Boards weist einen Fehler auf, der im Falle eines Board-Reboots den Startup verhindern kann.
Die Aktivierung eines Platform-Hotfix ist deswegen nicht erlaubt. Bitte korrigieren Sie das fehlerhafte BIOS laut Service-Info INF-12-000040.");
}

sub activate_bios {
  function_enter("activate_bios", "");
   my $rc = 0;
   if ($is_dscxl == 1) {
     log_msg_info("*** skipping activation of the BIOS reboot on DSCXL2 HW. This will be handled automatically at next startup.");
   }
   if ($is_eco == 0) {
     log_msg_info("*** skipping activation of the BIOS due to hardware != EcoServer ");
     $rc = 1;
   } else {
     log_msg_info("*** activation of the BIOS  will be initiated due to hardware == EcoServer ");
     my $current_version = get_local_bios_version();
     log_msg_info("current BIOS version before BIOS upgrade: $current_version ");
     $current_version = get_local_firmware_version();
     log_msg_info("current Firmware version before BIOS upgrade: $current_version ");
     
     log_msg_info("On EcoServer the BIOS can not be upgraded exclusivly anymore. It is part of the firmware in package os4k-platform-support and will be activated automatically at next startup");

     if ($host_reboot == 0) {
       log_msg_info("*** Due to BIOS update option a linux reboot will be initiated even if not requested by upgrade info type.");
       $host_reboot = 1;
     } else {
       log_msg_info("*** activation of the BIOS file succeeded. A linux reboot was already requested upgrade info type.");
     }
   }
   function_leave("activate_bios", "rc=$rc");
   return ($rc);
}

sub wait_for_successful_startup {
   my $report_to_SWA = shift;
   my $upgrade_with_completely_stopped_ha = shift;
   my $upgrade_with_full_drbd_sync = shift;
   my $multiNodeType = shift;
   my $isMultiNode = shift;
   my $side = shift;
   my $specific_wait_for_up = shift;
   my $report_on_stdout = shift;
   my $rc = 0;

   if (!defined($specific_wait_for_up)) {
     $specific_wait_for_up = $max_wait_for_up;
   }
     
   set_internal_upgrade_step($STEP_wait_for_status, $report_to_SWA, $defaultHostType, $hostname);

   my $script = $ha_bin_dir . "$upd_ok_sh";
   if ( ! -e "$script" ) {
     log_msg_info("Did not find script $script, checking old location ");
     $script =  $dscxl_dir . "bin/". "$upd_ok_sh";
   }
     
   function_enter("wait_for_successful_startup", "report_to_SWA=$report_to_SWA side=$side upgrade_with_completely_stopped_ha=$upgrade_with_completely_stopped_ha upgrade_with_full_drbd_sync=$upgrade_with_full_drbd_sync multiNodeType=$multiNodeType isMultiNode=$isMultiNode specific_wait_for_up=$specific_wait_for_up report_on_stdout=$report_on_stdout");

   if ($full_drbd_sync_flag_really_read == 0) {
     $upgrade_with_full_drbd_sync = $upgrade_with_completely_stopped_ha;
     log_msg_info(">>> forcing upgrade_with_full_drbd_sync to value of completely_stopped_ha ");
   }
  
   if( -e $script) {
      $rc = 5;
      my ($startup_rc, $startup_rc2) = (0 , 0);
      for(my $i = 0; $i < $specific_wait_for_up; $i++) {
        my $wait_cmd = "";
        $wait_cmd = "$script $side $upgrade_with_completely_stopped_ha $multiNodeType $isMultiNode $upgrade_with_full_drbd_sync";

        if ($specific_wait_for_up != 1) {
          set_internal_upgrade_step($STEP_startup_verification, $report_to_SWA, $defaultHostType, $hostname,  "Startup Verification ($i of max. $specific_wait_for_up minutes)");
        }

        log_msg_info("calling of the wait for success script $upd_ok_sh", $report_on_stdout);
        $startup_rc = exec_cmd($wait_cmd);

        if($startup_rc == 0) {
          if ($specific_wait_for_up != 1) {
            log_msg_info("startup successful, all necessary resources running, checking for confirmation", $report_on_stdout);
            sleep(5);
            $startup_rc2 = exec_cmd($wait_cmd);
            if($startup_rc2 == 0) {
              log_msg_info("startup successful confirmed", $report_on_stdout);
              $rc = 0;
              last;
            } else {
              log_msg_info("first startup successful not confirmed ... waiting ", $report_on_stdout);
            }
          } else {
            log_msg_info("startup successful, all necessary resources running", $report_on_stdout);
            $rc = 0;
            last;
          }
        } else {
          if ($specific_wait_for_up == 1) {
            # for the special once check, we do no want to wait at all
            last;
          } else {
            log_msg_info("startup in progress... waiting", $report_on_stdout);
          }
        }
        sleep(60);
      }
   } else {
     log_msg_info("Did not find script $script, skipping startup test ");   
   }
    
   if ($rc != 0) {
     $rc = $ERROR_WaitForUpgradeSuccessFailed;
   }
   function_leave("wait_for_successful_startup", "rc=$rc");
   return ($rc);
}


sub prepare_fallback_hotfix {
  my $fallback_version = shift;
  
  function_enter("prepare_fallback_hotfix", "fallback_version=$fallback_version");
  my $rc = 0;
  # version schema:          V6_R1.2.0
  # hotfix name schema:  PLT-V6_R1.2.1.tar
  # to do
  my $needed_hf_basename = "PLT-$fallback_version.tar";
  my $needed_hf_file     = $cur_hf_dir.$needed_hf_basename;
  
  my $fallback_hf_version = $fallback_version;
  $fallback_hf_version =~  /\.([0-9]*)/;
  $fallback_hf_version = $1;
  if (! defined($fallback_hf_version) ) {
    log_msg_info("error: could not extract the fallback hf version out of version string $fallback_version. Assuming 0.");
    $fallback_hf_version = 0;
  }
  log_msg_info("detected fallback HF version : $fallback_hf_version ");

  # clean the platform upgrade dirs
  removeDir($rpm_dir);
  makeDir($rpm_dir);  

  removeDir($hotfix_dir);
  makeDir($hotfix_dir);  

  
  if (-e $needed_hf_file ) {
    log_msg_info("found necessary fallback hotfix file: $needed_hf_file ");
    log_msg_info("deploying fallback hotfix file to dir $hotfix_dir ");
    exec_cmd("/bin/cp -vf $needed_hf_file $hotfix_dir");
  } else {
    log_msg_info("did not have necessary hotfix file: $needed_hf_file ");
    if ( $fallback_hf_version == 0) {
      $needed_hf_file =~ s/\.[0-9]*\.tar/.0.tar/;
      log_msg_info("creating base KV fallback hotfix file $needed_hf_file");
      exec_cmd("/usr/bin/touch $needed_hf_file");
    } else {
      log_msg_info("Autodeactivation not possible");
      $rc = 1;
    }
  }
  function_leave("prepare_fallback_hotfix", "rc=$rc");
  return($rc);    
}

sub create_upgrade_lock_file {
  function_enter("create_upgrade_lock_file", "");
  my $rc = 0;
  $rc = create_lock_file($lock_file);
  #remember timestamp when RLC activation invocation was executed
  $ts_rlc_upd_start = `date +%s`;
  log_msg("Timestamp when upgrade was started: $ts_rlc_upd_start");
  function_leave("create_upgrade_lock_file", "rc=$rc");
  return($rc);
}

sub remove_upgrade_lock_file {
  function_enter("remove_upgrade_lock_file", "");
  if (-e $lock_file) {
    remove_lock_file($lock_file);
    removeFile($lock_file);
    log_msg("deleted lock file for active node: $lock_file");
  }
  function_leave("remove_upgrade_lock_file", "");
}

sub create_remote_lock_file {
  create_lock_file($remote_lock_file);
}

sub remove_remote_lock_file {
    if (-e $remote_lock_file) {
      remove_lock_file($remote_lock_file);
      log_msg("deleted lock file for remote (passive) node update: $remote_lock_file");
    }
}

sub is_daemon_running {
  my $rc = 0;
  if(-e $lock_file) {
    log_msg("lockfile $lock_file exists -> check proc");
    my $pid_to_check = exec_cmd_out("cat $lock_file");
    log_msg("pid is: $pid_to_check");

    if( 
       exec_cmd("/sbin/checkproc -v -p $lock_file $self") == 0
       ||
       exec_cmd("/sbin/checkproc -v -p $lock_file $old_self") == 0
       ||
       exec_cmd("/sbin/checkproc -v -p $lock_file $yaps_self") == 0
       ||
       exec_cmd("/sbin/checkproc -v -p $lock_file $self.swupdated_slow") == 0
       ||
       exec_cmd("/sbin/checkproc -v -p $lock_file $old_self.swupdated_slow") == 0
      ) {
      $rc = 1;
    } else {
      log_msg("lockfile $lock_file exists, but no current process attached -> check for stale lockfile");
      if (-M $lock_file > 1) {
        log_msg("lockfile $lock_file is older than one day (age of modified) -> remove it");
        remove_lock_file($lock_file);
        $rc = 0;
      } else {
        log_msg("lockfile $lock_file is not older than one day (age of modified) -> it may still valid -> assume older instance is running");
        $rc = 1;
      }
    }
  }

  log_msg("daemon is running $rc (0=no)");
  return($rc);
}

sub is_running {
  my $lock_file =  $_[0];
  function_enter("is_running", "lockfile=$lock_file");

  my $rc = 0;
  if(-e $lock_file
     &&
     ( 
       exec_cmd("/sbin/checkproc -v -p $lock_file $self") == 0
       ||
       exec_cmd("/sbin/checkproc -v -p $lock_file $old_self") == 0
       ||
       exec_cmd("/sbin/checkproc -v -p $lock_file $yaps_self") == 0
       ||
       exec_cmd("/sbin/checkproc -v -p $lock_file $self.swupdated_slow") == 0
       ||
       exec_cmd("/sbin/checkproc -v -p $lock_file $old_self.swupdated_slow") == 0
     )
    ) {
    $rc = 1;
  }
  function_leave("is_running", "action is running $rc (0=no)");
  return($rc);
}

sub wait_for_service_running {
  my $service     = shift;
  my $max_try     = shift;
  my $period      = shift;
  my $rc = 1;

  for(my $i = 0; $i < $max_try; $i++) {
    if(is_service_running($service) == 1) {
        $rc = 0;
        last;
      }
    sleep($period);
  }
  return($rc);
}

sub wait_for_ssh_lock {
  my $lock_file   = shift;
  my $remote_node = shift;
  my $max_try     = shift;
  my $rc = $ERROR_CanNotGetLock;
  if (!defined($max_try)) {
    $max_try = $max_wait_for_ssh_lock_default;
  }
  function_enter("wait_for_ssh_lock", "lock_file=$lock_file remote_node=$remote_node max_try=$max_try");

  for(my $i = 0; $i < $max_try; $i++) {
        if(create_flock($lock_file) == 0) {
          $rc = 0;
          last;
        } else {
          my $msg = "ssh access to remote node $remote_node is currently blocked. waiting $i of $max_try ...\n";
          log_msg("$msg");
          print STDERR "$msg";
        }
        sleep(1);
  }
  if ($rc == $ERROR_CanNotGetLock) {
    my $msg = "too many retries ($max_try) for ssh access to remote node $remote_node. aborting \n";
    print STDERR "$msg";
  }
  function_leave("wait_for_ssh_lock", "rc=$rc");
  return ($rc);
}

# return code: 0=got the lock, $ERROR_CanNotGetLock
sub wait_for_lock {
  my $lock_file   = shift;
  my $wait_for_lock_time = shift;
  my $action = shift;
  my $got_the_lock_immediately_ref = shift;
  my $rc = $ERROR_CanNotGetLock;
  my $wait_between_successful_lock_fetches = rand(100);  # at max 100 micro seconds

  function_enter("wait_for_lock", "lock_file=$lock_file time=$wait_for_lock_time");

  $$got_the_lock_immediately_ref = 1;
  for(my $i = 0; $i < $wait_for_lock_time; $i++) {
        if(is_running($lock_file) == 0) {

          # do not rely on the fact that lock is not hold
          # sleep randomly and try again
          log_msg("sleeping $wait_between_successful_lock_fetches micro seconds before confirming that nobody has the log");
          usleep($wait_between_successful_lock_fetches);
          if(is_running($lock_file) == 0) {
            $rc = 0;
            last;
          }
        } else {
          my $msg = "$action is currently running. waiting  $i of $wait_for_lock_time ...\n";
          $$got_the_lock_immediately_ref = 0;
          log_msg("$msg");
          print STDERR "$msg";
        }
        sleep(1);
  }
  if ($rc == $ERROR_CanNotGetLock) {
    my $msg = "too many retries ($wait_for_lock_time) for $action. aborting \n";
    log_msg("$msg");
    print STDERR "$msg";
    $$got_the_lock_immediately_ref = 0;
  }

  function_leave("wait_for_lock", "lock_file=$lock_file time=$wait_for_lock_time got_the_lock_immediately=$$got_the_lock_immediately_ref");
  return ($rc);
}

sub create_flock {
  my $lock_file  = shift;

  function_enter("create_flock", "lock_file=$lock_file");
  my $rc = 0;

  # since any lock file resides in the lock_dir, I have to guarantee that it exists
  if ( ! -e $lock_dir) {
    log_msg_info("creating lock dir $lock_dir since it does not exist yet");
    makeDir($lock_dir);
  }

  $rc = open(MYFLOCKFILE, ">$lock_file");
  if ($rc) {
    $rc = flock(MYFLOCKFILE,  LOCK_EX | LOCK_NB); # LOCK_EX exclusive , LOCK_NB { 4 } ## non-blocking
    if ($rc) {
      log_msg_info("flock on $lock_file succeeded (got the lock, rc=$rc)");
      $rc=0;
    } else {
      log_msg_info("flock on $lock_file failed (did not get the lock, rc=$rc)");
      $rc = $ERROR_CanNotGetLock;
    }
  } else {
    log_msg_info("creating lock file $lock_file failed, rc=$rc");
    $rc = $ERROR_CanNotGetLock;
  }

  function_leave("create_flock", "lock_file=$lock_file rc=$rc");
  return ($rc);
}

sub remove_flock {
  my $lock_file  = shift;  
  function_enter("remove_flock", "lock_file=$lock_file");
  my $rc = 0;

  $rc = remove_lock_file($lock_file);
  function_leave("remove_flock", "lock_file=$lock_file rc=$rc");
  return ($rc);
}

sub has_the_flock {
  function_enter("has_the_flock", "lock_file=$lock_file");
  my $rc = 0;

  log_msg_info("Error: has_the_flock not yet implemented !!!");
  $rc = 1;

  function_leave("has_the_flock", "lock_file=$lock_file rc=$rc");
  return ($rc);
}

sub create_lock_file {
  my $lock_file  = shift;
  function_enter("create_lock_file", "lock_file=$lock_file");
  
  my $rc = 0;

  # since any lock file resides in the lock_dir, I have to guarantee that it exists
  if ( ! -e $lock_dir) {
    log_msg("creating lock dir $lock_dir since it does not exist yet");
    makeDir($lock_dir);
  }
  
  if(open(PID, ">$lock_file")) {
    $rc = 0;
    protectFile($lock_file);
    print PID $pid;
    log_msg("saved pid ($pid) to file ($lock_file)");
    close(PID);
  } else {
    $rc = 1;
  }
  function_leave("create_lock_file", "lock_file=$lock_file rc=$rc");
  return ($rc);
}

sub remove_lock_file {
  my $lock_file  = shift;
  function_enter("remove_lock_file", "lock_file=$lock_file");
  
  if (-e $lock_file) {
    removeFile($lock_file);
    log_msg("deleted lock file: $lock_file");
  }
  function_leave("remove_lock_file", "lock_file=$lock_file");
}

sub kill_via_pidfile {
  my $pid_file  = shift;
  function_enter("kill_via_pidfile", "pid_file=$pid_file");

  if (-e $pid_file) {
    log_msg("pid file $pid_file exists -> read pid for killing");
    my $read_pid = exec_cmd_out("cat $pid_file");
    chomp($read_pid);
    if ($read_pid eq "") {
      log_msg("can not determine pid for killing");
    } else {
      if ( looks_like_number($read_pid) ) {
        log_msg("pid for killing: $read_pid");
        exec_cmd_out("kill $read_pid");
      } else {
        log_msg("seems not to be a numeric value: $read_pid");
      }
    }
  } else {
    log_msg("pid file $pid_file does not exist -> cancel killing");
  }
  function_leave("kill_via_pidfile", "pid_file=$pid_file");
}

sub takeover_files {
  my $mode = shift;
  my $from_dir = shift;
  my $rc = 0;
  my $copy_rc = 0;
  my $to_dir = "";

  if (!defined($from_dir)) {
    $from_dir = "";
  }
  function_enter("takeover_files", "mode=$mode from_dir=$from_dir");
  # cleanup phase
  if ($mode == 0 || $mode == 1 || $mode == 3) {
    removeDir($rpm_dir);
    removeDir($hotfix_dir);
  }
  
  if ($mode == 2) {
    removeDir($license_dir);
  }

  # copy phase  
  if ($mode == 0) {
    # RLC update
    $from_dir = $assistant_rpm_dir;
    $to_dir   = $rpm_dir;

    # since the RLC files are copied on a per-file basis since NUC, I have to ensure that the to dir exists
    makeDir($rpm_dir);
    
    if (-e $from_dir) {
      my $no_of_files = get_number_of_files_in_dir ($from_dir);
      if ($no_of_files == 0) {
        log_msg_info("Error: can not copy Assistant files from $from_dir because dir is empty");
        $rc = $ERROR_CanNotCopyAssistantFiles;
      } else {
        log_msg_info("copying Assistant files from $from_dir to $to_dir");
        # $copy_rc = exec_cmd("/bin/cp -rvfL $from_dir $to_dir");

        my $filename = "";
        opendir ( DIR, $from_dir ) || log_msg_info ("Error in opening dir $from_dir");
        while( ($filename = readdir(DIR))){
          if($filename ne "." && $filename ne "..") {
            if(-d "$from_dir/$filename") {
              log_msg_info("$filename is a directory -> skip copy");
            } else {
                if($filename =~ /hp4k_instkit-.*rpm/) {
                log_msg_info("found NUC image file: $filename -> skip copy. This is the job of NUC PREP");
              } else {             
                log_msg_info("copying file $filename from $from_dir to $to_dir");
                my $from_file = "$from_dir"."$filename";
                $copy_rc = exec_cmd("/bin/cp -vfL $from_file $to_dir");
              }
            }
          }
        }
        closedir(DIR);
        
        if ( $copy_rc != 0) {
          log_msg_info("Error: can not copy Assistant files from $from_dir to $to_dir");
          $rc = $ERROR_CanNotCopyAssistantFiles;
        }
      }
    } else {
      log_msg_info("Error: can not copy Assistant files from $from_dir because dir does not exist");
      $rc = $ERROR_CanNotCopyAssistantFiles;
    }
  }

  if ($mode == 1) {
    # PLT HF update
    $from_dir = $assistant_hotfix_dir;
    $to_dir   = $hotfix_dir;   
    if (-e $from_dir) {
      log_msg_info("copying Assistant files from $from_dir to $to_dir");
      $copy_rc = exec_cmd("/bin/cp -rvfL $from_dir $to_dir");
      if ( $copy_rc != 0) {
        log_msg_info("Error: can not copy Assistant files from $from_dir to $to_dir");
        $rc = $ERROR_CanNotCopyAssistantFiles;
      }
    }
    # ensure that the upgrade dir always exist for extract hotfix
    makeDir($rpm_dir);
  }

  if ($mode == 3) {
    # system restore
    $to_dir   = $rpm_dir;
    if (-e $from_dir) {
      makeDir($rpm_dir);
      makeDir($soco2_tmp_dir);

      log_msg_info("copying system backup files non-recursive from $from_dir to $to_dir");
      $copy_rc = exec_cmd("/bin/cp -vfL $from_dir/*.rpm $to_dir");
      if ( $copy_rc != 0) {
        log_msg_info("Error: can not copy system backup files from $from_dir to $to_dir");
        $rc = $ERROR_CanNotCopySysbackupFiles;
      }

      if ($rc == 0) {
        log_msg_info("copying special system backup files recursive from $from_dir/os_deps to $to_dir");
        $copy_rc = exec_cmd("/bin/cp -rvfL $from_dir/os_deps $to_dir");
        if ( $copy_rc != 0) {
          log_msg_info("Error: can not copy system backup os_deps dir from $from_dir/os_deps to $to_dir");
          $rc = $ERROR_CanNotCopySysbackupFiles;
        }
      }
    }
  }
  
  if ($mode == 2) {
    $from_dir = $assistant_license_dir;
    $to_dir   = $license_dir;
    if (-e $from_dir) {
      log_msg_info("copying Assistant files from $from_dir to $to_dir");
      $copy_rc = exec_cmd("/bin/cp -rvfL $from_dir $to_dir");
      if ( $copy_rc != 0) {
        log_msg_info("Error: can not copy Assistant files from $from_dir to $to_dir");
        $rc = $ERROR_CanNotCopyAssistantFiles;
      }
    } else {
        $rc = $ERROR_AssistantDirectoryDoesNotExist;
    }
    
    if ($copy_rc == 0) {
      log_msg_info("copying Assistant files from $from_dir to $license_storage_dir for long term storage");
      makeDir($license_storage_dir);
      exec_cmd("/bin/cp -vf $from_dir* $license_storage_dir");
      }
  }

  if ($mode == 4) {
    $from_dir = $assistant_snmp_dir;
    $to_dir   = $snmp_dir;
    if (-e $from_dir) {
      log_msg_info("copying Assistant files from $from_dir to $to_dir");
      if ( ! -e $to_dir) {
        makeDir($to_dir);
      }
      my $from_pattern = $from_dir . "snmp.xml";
      $copy_rc = exec_cmd("/bin/cp -rvfL $from_pattern $to_dir");
      if ( $copy_rc != 0) {
        log_msg_info("Error: can not copy Assistant files from $from_pattern to $to_dir");
        $rc = $ERROR_CanNotCopyAssistantFiles;
      }
    } else {
        $rc = $ERROR_AssistantDirectoryDoesNotExist;
    }
  }  

  function_leave("takeover_files", "rc=$rc");
  return ($rc);
}

sub store_osdep_rpms_into_osdeps_repo {
  function_enter("store_osdep_rpms_into_osdeps_repo", "storing of the osdeps rpm packages for system backup");
  my $rc = 0;
  if ( -e $osdeps_rpm_dir ) {
    if (! -e $osdeps_backup_dir ) {
      makeDir($osdeps_backup_dir);
    }    
    my $cmd  = 'export PATH=/usr/bin:/bin; ';
    $cmd .= 'BACKUP="'.$osdeps_backup_dir.'"; ';
    $cmd .= 'WEBSERVICE_DIR="'.$webservice_dir.'"; ';   
    $cmd .= 'REPO="'.$osdeps_rpm_dir.'"; ';
    $cmd .= 'cd $REPO; ';
    $cmd .= 'RPM_FILES=`ls $REPO | grep .rpm`; ';
    $cmd .= 'echo "--> rpm files: " >> '.$log_file.'; ';  
    $cmd .= 'echo "${RPM_FILES}" >> '.$log_file.'; ';
    $cmd .= 'echo "before copy is done, those rpms from the storage (${BACKUP}) have to be removed which were delivered with the upgrade to avoid multiple rpms with different versions" >> '.$log_file.'; ';
    $cmd .= 'for RPM in ${RPM_FILES}; do ';
    $cmd .= $store_package_script.' ${RPM} ${BACKUP}  >> '.$log_file.' 2>&1 ;';
    $cmd .= 'done; ';
    
    $rc = exec_cmd($cmd);
  } else {
    log_msg_info("directory $osdeps_rpm_dir does not exist -> no add. OS dependency packages were delivered with the upgrade. Skipping storage.");
  }
  function_leave("store_osdep_rpms_into_osdeps_repo", "rc=$rc");
  return ($rc);
}

sub store_hotfix_into_instmedia {
  function_enter("store_hotfix_into_instmedia", "storing of the hotfix file for potential hotfix activation after node replacement");
  # make sure the instmedia dest dir exists
  my $hf_instmedia_dir = "/instmedia/Hotfixes/";
  if (! -e $hf_instmedia_dir ) {
      makeDir($hf_instmedia_dir);
  }

  my $cmd  = 'export PATH=/usr/bin:/bin; ';
  $cmd .= 'DEST="'.$hf_instmedia_dir.'"; ';
  $cmd .= 'SOURCE="'.$hotfix_dir.'"; ';
  $cmd .= 'rm -vf ${DEST}PLT-*.tar  >> '.$log_file.'; ';
  $cmd .= 'cp -vf ${SOURCE}/PLT-*.tar ${DEST} >> '.$log_file.'; exit $?';
  my $rc = exec_cmd($cmd);

  function_leave("store_hotfix_into_instmedia", "rc=$rc");
  return ($rc);
}

sub store_hotfix_rpms_into_hotfixrepo {
  function_enter("store_hotfix_rpms_into_hotfixrepo", "storing of the hotfix rpm packages for potential hotfix deactivation");
   
  my $cmd  = 'export PATH=/usr/bin:/bin; ';
  $cmd .= 'BACKUP="'.$cur_hf_rpm_dir.'"; ';
  $cmd .= 'WEBSERVICE_DIR="'.$webservice_dir.'"; ';   
  $cmd .= 'REPO="'.$tmp_rpm_dir.'"; ';
  $cmd .= 'cd $REPO; ';
  $cmd .= 'RPM_FILES=`ls $REPO | grep .rpm`; ';
  $cmd .= 'echo "--> rpm files: " >> '.$log_file.'; ';  
  $cmd .= 'echo "${RPM_FILES}" >> '.$log_file.'; ';
  $cmd .= 'echo "before copy is done, those rpms from the storage (${BACKUP}) have to be removed which were delivered with the hotfix to avoid multiple rpms with different versions" >> '.$log_file.'; ';
  $cmd .= 'for RPM in ${RPM_FILES}; do ';
  $cmd .= $store_package_script.' ${RPM} ${BACKUP}  >> '.$log_file.' 2>&1 ;';
  $cmd .= 'done; ';
  
  my $rc = exec_cmd($cmd);

  function_leave("store_hotfix_rpms_into_hotfixrepo", "rc=$rc");
  return ($rc);
}

sub store_upgrade_rpms_into_installrepo {
  function_enter("store_upgrade_rpms_into_installrepo", "storing of the update rpm packages for potential hotfix deactivation");
   
  my $cmd  = 'export PATH=/usr/bin:/bin; ';
  $cmd .= 'BACKUP="'.$backup_dir.'"; ';
  $cmd .= 'WEBSERVICE_DIR="'.$webservice_dir.'"; ';   
  $cmd .= 'REPO="'.$tmp_rpm_dir.'"; ';
  $cmd .= 'cd $REPO; ';  
  $cmd .= 'RPM_FILES=`ls $REPO | grep .rpm`; ';
  $cmd .= 'echo "--> rpm files: " >> '.$log_file.'; ';
  $cmd .= 'echo "${RPM_FILES}" >> '.$log_file.'; ';  
  $cmd .= 'echo "before copy is done, those rpms from the storage (${BACKUP}) have to be removed which were delivered with the upgrade to avoid multiple rpms with different versions" >> '.$log_file.'; ';
  $cmd .= 'for RPM in ${RPM_FILES}; do ';
  $cmd .= $store_package_script.' ${RPM} ${BACKUP}  >> '.$log_file.' 2>&1 ;';
  $cmd .= 'done; ';
  
  my $rc = exec_cmd($cmd);
  
  function_leave("store_upgrade_rpms_into_installrepo", "rc=$rc");
  return ($rc);
}


sub check_preconditions {
  function_enter("checking preconditions", "");
  if($hotfix_mode && ! -d $assistant_hotfix_dir) {
    log_msg_info("starting preconditions not fullfilled!");
    log_msg_info("Directory $assistant_hotfix_dir is missing. exiting!");
    write_status(2, "could not start update daemon. Directory $assistant_hotfix_dir is missing");
    exit(1);
   }
  
  if($hotfix_mode == 0 &&! -d $assistant_rpm_dir) {  
    log_msg_info("starting preconditions not fullfilled!");
    log_msg_info("Directory $assistant_rpm_dir is missing. exiting!");
    write_status(2, "could not start update daemon. Directory $assistant_rpm_dir is missing");
    exit(1);
  }
  function_leave("checking preconditions", "");
}


sub start_sysinfo {
  function_enter("start_sysinfo", "");

  log_msg_info("starting sysinfo to be it is running since it does not support restarting from unused state");
  exec_cmd("service sysinfod start");
  
  log_msg_info("restarting sysinfo to be sure potential memory leaks are cleared");
  exec_cmd("service sysinfod restart");

  log_msg_info("insserving sysinfo daemon to undo a potential memory tuning");
  exec_cmd("insserv sysinfod");
  
  function_leave("start_sysinfo" , "");
}

sub restart_snmp {
  function_enter("restart_snmp", "");
  log_msg_info("restarting snmp subagent service so that the new version string can be retrieved via get_SNMP_sysDescription.sh");
  exec_cmd("service snmp-syslogagt restart");  
  function_leave("restart_snmp" , "");
}

sub start_other_service_after_upgrade {
  function_enter("start_other_service_after_upgrade", "");
  
  # be sure the cron daemon is running (for CLA See H99300)
  my $rc = exec_cmd("service cron start");
  if ($rc != 0) {
    log_msg_info("Error when trying to start cron service");
  }
  log_msg_info("checking the current cron jobs");
  exec_cmd("/usr/bin/crontab -l");
  function_leave("start_other_service_after_upgrade" , "");  
}

sub restart_softgate_if_necessary {
  function_enter("restart_softgate_if_necessary", "");
  my $rc = 0;
  if ($host_reboot == 1 || $activation_has_been_avoided == 1) {
    log_msg_info("restart SoftGate not necessary because Linux reboot will do it anyway");
  } else {
    if ($hotfix_mode == 0) {
      log_msg_info("restart SoftGate not necessary because RLC NUC updates always reboot the platform");
    } else {
      my $is_running = is_service_running("socod");
      if ($is_running == 1) {
        log_msg_info("SoftGate is running, check java versions");
        if ( ($openj9_java8_version_before_upgrade ne $openj9_java8_version_after_upgrade) or ($java8_version_before_upgrade ne $java8_version_after_upgrade) or ($java7_version_before_upgrade ne $java7_version_after_upgrade)) {
          log_msg_info("restarting SoftGate because java update $java7_version_after_upgrade/$java8_version_after_upgrade/$openj9_java8_version_after_upgrade during platform update makes it necessary", 1, 1);
          my $deferred_restart=0;
          if ($is_stmix == 1)                  { $deferred_restart = 1 ; }
          
          $rc = restart_softgate($deferred_restart);
        }
      } else {
        log_msg_info("no SoftGate running, restart not necessary");
      }
    }
  }
  function_leave("restart_softgate_if_necessary", "rc=$rc");
  return ($rc);
}

sub restart_softgate {
  my $deferred_restart = shift;
  function_enter("restart_softgate", "deferred_restart=$deferred_restart");
  my $rc = 0;
  $rc = exec_cmd("service socosd restart");
  if ($rc != 0) {
    log_msg_info("Error when trying to restart socosd service");
  }

  if ($deferred_restart == 1) {
    # deferred  restart for STMIX and STMIY to give swupdated time to finish HF activation before IP address of STMIX gone
    my $tmp_file = "/tmp/swupdated_restart_sg.$pid";
    my $cmd_content = "sleep 20; service socod restart;";
    create_generic_file($cmd_content, $tmp_file);
    exec_cmd("chmod +x $tmp_file");
    log_msg_info("Calling nohuped script to restart socod deferred");
    $rc = exec_cmd("nohup $tmp_file&");
    log_msg_info("nohuped script to restart socod deferred returned $rc");
    sleep(1); # give the nohup script time to be read by sub shell before removing it
    removeFile($tmp_file);
  } else {
    # immediate restart
    $rc = exec_cmd("service socod restart");
    if ($rc != 0) {
      log_msg_info("Error when trying to restart socod service");
    }
  }
  
  function_leave("restart_softgate", "rc=$rc");
  return ($rc);
}

sub start_softgate {
  function_enter("start_softgate", "");  
  my $rc = 0;
  $rc = exec_cmd("service socosd start");
  if ($rc != 0) {
    log_msg_info("Error when trying to start socosd service");
  }  

  $rc = exec_cmd("service socod start");
  if ($rc != 0) {
    log_msg_info("Error when trying to start socod service");
  }
  function_leave("start_softgate", "rc=$rc");
  return ($rc);
}

sub retrieve_java_versions {
  my $type = shift;
  function_enter("retrieve_java_versions", "type=$type");
  if ($type eq "before") {
    $java7_version_before_upgrade = get_rpm_version("java-1_7_1-ibm");
    $java8_version_before_upgrade = get_rpm_version("java-1_8_0-ibm");
    $openj9_java8_version_before_upgrade = get_rpm_version("java-1_8_0-openj9");
  } else {
    $java7_version_after_upgrade = get_rpm_version("java-1_7_1-ibm");
    $java8_version_after_upgrade = get_rpm_version("java-1_8_0-ibm");
    $openj9_java8_version_after_upgrade = get_rpm_version("java-1_8_0-openj9");
  }
  function_leave("retrieve_java_versions", "");
}

sub check_corosync_hostnames {
  my $ref = shift;
  my $other_node_is_reachable_ref = shift;
  function_enter("check_corosync_hostnames", "other_node_is_reachable=$$other_node_is_reachable_ref");

  my $hostname = exec_cmd_out("/bin/hostname", "silent");
  chomp($hostname);
  log_msg_info("local /bin/hostname:\"$hostname\"");

  my $password_is_expired_ref = 0;
  my $rc = 0;
  my $remote_hostname = "";

  my ($max_try, $wait_for_ssh_lock, $ignore_remote_return_code) = (1, 30, 1);

  $rc = execute_remote_duplex_cmd("/bin/hostname", \$remote_hostname, $max_try, $other_node_is_reachable_ref, \$password_is_expired_ref, $ignore_remote_return_code, $wait_for_ssh_lock);
  $remote_hostname =~ s/remote rc: [0-9]*\n*//;
  chomp($remote_hostname);
  log_msg_info("remote /bin/hostname:\"$remote_hostname\"");

  my $node1name=retrieve_from_xml("system.name-node1");
  log_msg_info("node1 hostname:\"$node1name\"");
  my $node2name=retrieve_from_xml("system.name-node2");
  log_msg_info("node2 hostname:\"$node2name\"");

  unless ( ($node1name eq $hostname && $node2name eq $remote_hostname) ||
       ($node1name eq $remote_hostname && $node2name eq $hostname) ) {
    $$ref = 1;
  }

#  grep corosync-name.node1 $fixml | grep ">`hostname`<" && whoisme="node1"
#   1510    grep corosync-name.node2 $fixml | grep ">`hostname`<" && whoisme="node2"

#  log_msg_info("script $script not available -> skipping password expiration check, assuming password is valid");

  function_leave("check_corosync_hostnames", "status=$$ref, other_node_is_reachable=$$other_node_is_reachable_ref");
  return(0);
}

sub check_vmguest_remote {
  my $node=shift;
  my $remote_cmd = shift;
  function_enter("check_vmguest_remote", "node=$node remote_cmd=$remote_cmd");

  my ($ignore_remote_return_code, $suppress_diagnosis_output, $skip_sshkeygen, $skip_sshkeycleanup, $other_node_is_reachable, $password_is_expired, $ssh_timeout) = (0,0,0,0,0,0,30);

  my $rc = 0;
  my $output = "";

  # $rc = execute_remote_cmd($node, \$other_node_is_reachable, \$password_is_expired, $remote_cmd, \$output, 1, $ssh_timeout, $ignore_remote_return_code, $suppress_diagnosis_output, $skip_sshkeygen, $skip_sshkeycleanup);
  # use here the execute_remote_quorum_cmd to fetch the ssh lock OSFOURK-11064
  $rc = execute_remote_quorum_cmd($remote_cmd, \$output, 1, \$other_node_is_reachable, \$password_is_expired, $ignore_remote_return_code);

  function_leave("check_vmguest_remote", "rc=$rc");
  return($rc);
}

sub check_local_password_validity {
  my $password_is_expired_ref = shift;
  function_enter("check_local_password_validity", "");

  # do not use chage. It hangs sometimes -> parse directly the /etc/shadow
  my $account = "root";
  my $script =  "/opt/webservice/scripts/is_account_expired.sh";

  if (-e $script) {    
    my $cmd = "/opt/webservice/scripts/is_account_expired.sh $account";
    my $rc = exec_cmd($cmd);

    if ($rc == 1 or $testmode != 0) {
      $$password_is_expired_ref = 1;
    }
  } else {
    log_msg_info("script $script not available -> skipping password expiration check, assuming password is valid");
  }
  function_leave("check_local_password_validity", "password_is_expired=$$password_is_expired_ref");
  return(0);
}

sub check_password_expired {
  my $node_to_process = shift;
  my $other_node_is_reachable_ref = shift;
  my $password_is_expired_ref = shift;
  my $rc = 0;
  my $output = "";  
  function_enter("check_password_expired", "node_to_process=$node_to_process");

  if ($node_to_process eq "q" || $node_to_process eq "all") {
    if (has_quorum($node_q, $multiNodeType) == 1) {
      $rc = execute_remote_quorum_cmd("/bin/hostname", \$output, $wait_for_ssh_connectivity, $other_node_is_reachable_ref, $password_is_expired_ref );
    }
  }

  # if quorum has already failed I can skip the standby node
  if ($$other_node_is_reachable_ref == 1 and $$password_is_expired_ref == 0) {
    if ($node_to_process eq "s" || $node_to_process eq "all") {
      if ($isMultiNode == 1) {
        $rc = execute_remote_duplex_cmd("/bin/hostname", \$output, $wait_for_ssh_connectivity, $other_node_is_reachable_ref, $password_is_expired_ref );
      }
    }
  }

  # if quorum or standby have already failed I can skip the active node
  if ($$other_node_is_reachable_ref == 1 and $$password_is_expired_ref == 0) {
    check_local_password_validity($password_is_expired_ref);
  }
    
  function_leave("check_password_expired", "node_to_process=$node_to_process other_node_is_reachable=$$other_node_is_reachable_ref password_is_expired=$$password_is_expired_ref");
  return(0);
}

sub generate_hista_entry {
  my $hista_arg         = shift;
  function_enter("generate_hista_entry", "hista_arg=$hista_arg ");

  # /opt/inventory/inventory-client.sh notification HISTA_MESSAGE "--alarm 0 --event 0x2b --subevent 0x37 messagetext"

  my $cmd="/opt/inventory/inventory-client.sh notification HISTA_MESSAGE " . "$hista_arg";
  my $rc = exec_cmd($cmd);

  function_leave("generate_hista_entry", "rc=$rc");
  return($rc);
}

sub execute_reboot {
  my $sub_cmd         = shift;
  my $do_daemonize    = shift;
  my $verbose         = shift;
  my $work_on_active  = shift;
  my $work_on_standby = shift;
  my $work_on_quorum  = shift;
  my $work_on_all     = shift;

  my $rc = 0;
  my $node_to_process = "";
  my ($other_node_is_reachable, $password_is_expired, $rmx_alarm, $hista_text, $LTU_info, $hista_for_A_already_sent) = (1, 0, $rmx_hista_no_alarm, "", "", 0);

  function_enter("execute_reboot", "sub_cmd=$sub_cmd do_daemonize=$do_daemonize verbose=$do_daemonize work_on_active=$work_on_active work_on_standby=$work_on_standby work_on_quorum=$work_on_quorum work_on_all=$work_on_all");

  $node_reach_test_method = "ping";
  
  determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 0);
    
  # set the timeout for ssh for remote check and stop_openais_service and remote shutdown_host
  log_msg_info("reducing timeout for ssh (connectivity) from $wait_for_ssh_connectivity to 10 minutes");
  $wait_for_ssh_connectivity = 10; # minutes
  log_msg_info("increasing timeout for ssh (lock) from $max_wait_for_ssh_lock_default to 180 seconds");
  $max_wait_for_ssh_lock_default = 180; # seconds
  
  # check the shutdown lock before daemonizing. Otherwise I can give no return code
  if(is_running($shutdown_lock_file) != 0) {
    log_msg_info("another reboot/shutdown daemon action is currently running -> quit", $verbose);
    return ($ERROR_AnotherShutdownAlreadyRunning);
  }

  # check the upgrade lock before daemonizing. Otherwise I can give no return code
  if(is_daemon_running()) {
    log_msg_info("an upgrade is currently running -> quit", $verbose);
    return ($ERROR_UpgradeAlreadyRunning);
  }

  # create the first lock before daemonizing to protect the ssh wait
  create_lock_file($shutdown_lock_file);
  
  if ($work_on_quorum == 1) {
    if (has_quorum($node_q, $multiNodeType) == 0) {
        log_msg_info("system has no quorum", $verbose);
        remove_lock_file($shutdown_lock_file);
        return ($ERROR_NoMultiNodeSystem);
    }
  }
  if ($work_on_quorum == 1 or $work_on_all == 1) {
    $node_to_process = "q";
    check_password_expired($node_to_process, \$other_node_is_reachable, \$password_is_expired);
    if($other_node_is_reachable == 0) {
      log_msg_info("remote node Q not reachable", $verbose);
      if ($work_on_all) {
        log_msg_info("remote node Q not reachable but multi action-> do not quit, continue with other nodes", $verbose);
      } else {
        log_msg_info("remote node Q not reachable but single action-> quit with exit code", $verbose);
        remove_lock_file($shutdown_lock_file);
        return ($ERROR_CanNotReachRemoteNode);
      }
    }
    if($password_is_expired == 1) {
      log_msg_info("root password expired on Q -> do not quit, but continue with other nodes", $verbose);
      if ($work_on_all) {
        log_msg_info("remote node Q not reachable but multi action-> do not quit, continue with other nodes", $verbose);
      } else {
        log_msg_info("remote node Q not reachable but single action-> quit with exit code", $verbose);
        remove_lock_file($shutdown_lock_file);
        return ($ERROR_RootPwdExpired);
      }
    }
  }

  if ($work_on_standby == 1) {
    if ($isMultiNode == 0) {
        log_msg_info("system is single node, requested standby host not defined", $verbose);
        remove_lock_file($shutdown_lock_file);
        return ($ERROR_NoMultiNodeSystem);
    }
  }
  if ($work_on_standby == 1 or $work_on_all == 1) {
    $node_to_process = "s";
    check_password_expired($node_to_process, \$other_node_is_reachable, \$password_is_expired);
    if($other_node_is_reachable == 0) {
      log_msg_info("remote node B not reachable", $verbose);
      if ($work_on_all) {
        log_msg_info("remote node B not reachable but multi action-> do not quit, continue with active node", $verbose);
      } else {
        log_msg_info("remote node B not reachable but single action-> quit with exit code", $verbose);
        remove_lock_file($shutdown_lock_file);
        return ($ERROR_CanNotReachRemoteNode);
      }
    }
    if($password_is_expired == 1) {
      log_msg_info("root password expired on B ", $verbose);
      if ($work_on_all) {
        log_msg_info("remote node B not reachable but multi action-> do not quit, continue with active node", $verbose);
      } else {
        log_msg_info("remote node B not reachable but single action-> quit with exit code", $verbose);
        remove_lock_file($shutdown_lock_file);
        return ($ERROR_CanNotReachRemoteNode);
      }
    }
  }

  if ($do_daemonize ==1)  {
    $rc = daemonize_me();
    if ($rc != 0) {
      log_msg_info("error when daemonizing, continuing and trying the best");
    }
  }

  # create the lock after daemonizing to write the correct pid into
  create_lock_file($shutdown_lock_file);

  # prevent any sysinfo status file writing
  $write_sysinfo_status_enabled = 0;
  my ($upgrade_with_reduce_downtime, $upgrade_via_image) = (0, 0);
  my $type = "";

  # parameter for openais stop, use tiptoe but no tiptoe reboot reboot
  my $use_tiptoe_if_available = 1;
  my $reboot_if_tiptoe_failes = 0;

  # phase 1 in case of reboot and shutdown and stop

  log_msg_info("increasing the ssh client execution supervision timeout from $max_wait_for_ssh_client_execution to 900 sec (15 min)", $verbose);
  $max_wait_for_ssh_client_execution = 900;

  my $number_of_sg_on_central_hosts = get_number_of_sg_on_central_hosts(0);
  if ($number_of_sg_on_central_hosts == 0) {
      log_msg_info("no SG configured, therefore no LTU info needed");
  } else {
      log_msg_info("SG configured, therefore LTU info needed");
      if ($isMultiNode == 1) {
        # multi node, SG can only be on Q
        $LTU_info = " with SoftGate";
      } else {
        # single node, fetch the LTU locally
        my $LTU_number = get_LTU();
        if ($LTU_number eq "") {
        log_msg_info("Warn: SG configured, but no LTU info read, maybe SG never started up");
          $LTU_info = ", LTU unknown";
        } else {
          $LTU_info = ", LTU $LTU_number";
        }
     }
  }
    
  # Quorum
  if ($work_on_quorum == 1 or $work_on_all == 1) {
    if (has_quorum($node_q, $multiNodeType) == 1) {
      $type = "quorum";
      if ($sub_cmd == 0) {
        ($rmx_alarm, $hista_text) = ($rmx_hista_no_alarm, "Admin Shutdown of the Quorum host node $node_q"."$LTU_info");
      } else {
        ($rmx_alarm, $hista_text) = ($rmx_hista_no_alarm, "Admin Reboot of the Quorum host node $node_q"."$LTU_info");
      }
      generate_hista_entry("--alarm $rmx_alarm --event 0x2b --subevent 0x37 $hista_text");
  
      # stop openais to reduce watchdog probability
      log_msg_info("stopping openais service on quorum node $node_q", $verbose);
      $rc = stop_openais_service($type, 0, $use_tiptoe_if_available, $reboot_if_tiptoe_failes, 0, 0);

      if ($sub_cmd == 0) {
        # executing the real shutdown command
        log_msg_info("initiating a remote admin shutdown on quorum node $node_q", $verbose);
        shutdown_host($type, "initiating a remote admin shutdown on quorum");
      }
    }
  }

  # Standby
  if ($work_on_standby == 1 or $work_on_all == 1) {
    if ($isMultiNode == 1) {
      $type = "remote";
      if ($sub_cmd == 0) {
        ($rmx_alarm, $hista_text) = ($rmx_hista_no_alarm, "Admin Shutdown of the Standby host node $other_node");
      } else {
        ($rmx_alarm, $hista_text) = ($rmx_hista_no_alarm, "Admin Reboot of the Standby host node $other_node");
      }
      generate_hista_entry("--alarm $rmx_alarm --event 0x2b --subevent 0x37 $hista_text");
      
      # strange thing: on sep. Duplex the hista notification for A is lost, so I have to do it now
      if (has_quorum($node_q, $multiNodeType) == 1) {
        # its a sep. Duplex
        log_msg_info("separate Duplex -> check for Hista workaround", $verbose);
        if ($work_on_active == 1 or $work_on_all == 1) {
          if ($sub_cmd == 0) {
            ($rmx_alarm, $hista_text) = ($rmx_hista_no_alarm, "Admin Shutdown of the Active host node $hostname");
          } else {
            ($rmx_alarm, $hista_text) = ($rmx_hista_no_alarm, "Admin Reboot of the Active host node $hostname");
          }
          generate_hista_entry("--alarm $rmx_alarm --event 0x2b --subevent 0x37 $hista_text");
          $hista_for_A_already_sent = 1;
        } else {
        log_msg_info("Hista workaround for separate Duplex not necessary since node_A not affected", $verbose);
        }
      }
  
      log_msg_info("stopping openais service on standby node $other_node", $verbose);
      $rc = stop_openais_service($type, 0, $use_tiptoe_if_available, $reboot_if_tiptoe_failes, 0, 0);

      if ($sub_cmd == 0) {
        # executing the real shutdown command
        log_msg_info("initiating a remote admin shutdown on standby node $other_node", $verbose);
        shutdown_host($type, "initiating a remote admin shutdown on standby");
      }
    }
  }

  # Active
  if ($work_on_active == 1 or $work_on_all == 1) {
    $type = "local";
    my $host_specifier = "Active host node";
    switch ($multiNodeType) {
      case 20  {
        log_msg_info("Surv. SG detected, updating noun", $verbose);
        $host_specifier = "Surv. SoftGate host";
        }
      case 21  {
        log_msg_info("SoftGate StandAlone detected, updating noun", $verbose);
        $host_specifier = "SoftGate host";
        }        
      case 71  {
        log_msg_info("Survivable Enterprise GW detected, updating noun", $verbose);
        $host_specifier = "Survivable Enterprise GW host";
        }
      else  {
        log_msg_info("not a Survivable");
      }
    }

    if ($hista_for_A_already_sent == 0) {
      if ($sub_cmd == 0) {
        ($rmx_alarm, $hista_text) = ($rmx_hista_no_alarm, "Admin Shutdown of the $host_specifier $hostname"."$LTU_info");
      } else {
        ($rmx_alarm, $hista_text) = ($rmx_hista_no_alarm, "Admin Reboot of the $host_specifier $hostname"."$LTU_info");
      }
      generate_hista_entry("--alarm $rmx_alarm --event 0x2b --subevent 0x37 $hista_text");
    } else {
      log_msg_info("hista entry for node_A already sent earlier.", $verbose);
    }
      
    log_msg_info("stopping openais service on active node $hostname", $verbose);
    $rc = stop_openais_service($type, 0, $use_tiptoe_if_available, $reboot_if_tiptoe_failes, 0, 0);

    if ($sub_cmd == 0) {
      # executing the real shutdown command
      log_msg_info("initiating a local admin shutdown on active node $hostname", $verbose);
      shutdown_host($type, "initiating a local admin shutdown on active node");
    }
  }

  # phase 2 only in case of reboot
  if ($sub_cmd == 1) {
    log_msg_info("starting phase 2 due to reboot");

    # Quorum
    if ($work_on_quorum == 1 or $work_on_all == 1) {
      if (has_quorum($node_q, $multiNodeType) == 1) {
        log_msg_info("initiating a remote admin reboot on quorum node $node_q", $verbose);
        reboot_host("quorum", $upgrade_with_reduce_downtime, $upgrade_via_image, "initiating a remote admin reboot on quorum");
      }
    }

    # Standby
    if ($work_on_standby == 1 or $work_on_all == 1) {
      if ($isMultiNode == 1) {
        log_msg_info("initiating a remote admin reboot on standby node $other_node", $verbose);
        reboot_host("remote", $upgrade_with_reduce_downtime, $upgrade_via_image, "initiating a remote admin reboot on standby");
      }
    }

    # Active
    if ($work_on_active == 1 or $work_on_all == 1) {
      log_msg_info("initiating a remote admin reboot on active node $hostname", $verbose);
      reboot_host("local", $upgrade_with_reduce_downtime, $upgrade_via_image, "initiating an admin reboot on active node");
    }
  }

  remove_lock_file($shutdown_lock_file);

  # does it make sense to return anything other than success?
  $rc = 0;

  function_leave("execute_reboot", "rc=$rc");
  return ($rc);
}


sub shutdown_host {
  my $type =  shift;
  my $reboot_for_reduce_downtime = shift;
  my $msg  =  shift;
  my $rc = 0;
  my $output = "";
  my ($other_node_is_reachable, $password_is_expired) = (0, 0);

  if (!defined($msg)) {
    $msg = "Shutdown the Linux Host";
  }
  function_enter("shutdown_host", "type=$type msg=$msg");
  my $cmd = "$ws_shutdown_host_cmd 20";

  if ($type eq "remote") {
    set_internal_upgrade_step($STEP_reboot_host, 0, "Standby", $other_node, $msg);
    $rc = execute_remote_duplex_cmd("/opt/webservice/scripts/initiateReboot.sh $ws_shutdown_host_cmd 20 secondary ", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
  } else {
    if ($type eq "quorum") {
      set_internal_upgrade_step($STEP_reboot_host, 0, "Quorum", $node_q, $msg);
      $rc = execute_remote_quorum_cmd("/opt/webservice/scripts/initiateReboot.sh $ws_shutdown_host_cmd 20 quorum ", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired );
    } else {
      set_internal_upgrade_step($STEP_reboot_host, 0, "Active", $hostname, $msg);

      # be sure the at daemon is running
      $rc = exec_cmd("service atd start");
      $cmd = $cmd . " primary";

      if ($rc == 0) {
        exec_cmd("/opt/webservice/scripts/initiateReboot.sh $ws_shutdown_host_cmd 20 primary ");
      } else {
        log_msg_info("error: service atd can not be started (not installed or wrong configuration) ");
        log_msg_info("Trying to reboot directly with the risk that the current ssh invocation may hang ");
        $rc = exec_cmd("$cmd");
      }
    }
  }

  function_leave("shutdown_host", "type=$type rc=$rc");
  return ($rc);
}

sub reboot_host {
  my $type =  shift;
  my $reboot_for_reduce_downtime = shift;
  my $upgrade_via_image = shift;
  my $msg  =  shift;
  my $rc = 0;
  my $output = "";
  my ($other_node_is_reachable, $password_is_expired) = (0, 0);
  
  if (!defined($msg)) {
    $msg = "Reboot the Linux Host";
  }
  function_enter("reboot_host", "type=$type reboot_for_reduce_downtime=$reboot_for_reduce_downtime upgrade_via_image=$upgrade_via_image msg=$msg");
  my $cmd = "$ws_reboot_host_cmd $ha_reboot_host_cmd 20";
  
  if ($type eq "remote") {
    set_internal_upgrade_step($STEP_reboot_host, 0, "Standby", $other_node, $msg);
    $rc = execute_remote_duplex_cmd("/opt/webservice/scripts/initiateReboot.sh $ws_reboot_host_cmd $ha_reboot_host_cmd 20 secondary $reboot_for_reduce_downtime", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
  } else {
    if ($type eq "quorum") {
      set_internal_upgrade_step($STEP_reboot_host, 0, "Quorum", $node_q, $msg);
      $rc = execute_remote_quorum_cmd("/opt/webservice/scripts/initiateReboot.sh $ws_reboot_host_cmd $ha_reboot_host_cmd 20 quorum $reboot_for_reduce_downtime", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired );
    } else {
      # LOCAL
      set_internal_upgrade_step($STEP_reboot_host, 0, "Active", $hostname, $msg);

      if ($upgrade_via_image == 0) {
          log_msg_info("Warning: reboot requested and its not a NUC -> initiating a normal reboot with 30 seconds delay");
      }
      
      # be sure the at daemon is running
      $rc = exec_cmd("service atd start");
      $cmd = $cmd . " primary $reboot_for_reduce_downtime";

      if ($rc == 0) {
        exec_cmd("/opt/webservice/scripts/initiateReboot.sh $ws_reboot_host_cmd $ha_reboot_host_cmd 30 primary $reboot_for_reduce_downtime");
      } else {
        log_msg_info("error: service atd can not be started (not installed or wrong configuration) ");
        log_msg_info("Trying to reboot directly with the risk that the current ssh invocation may hang ");
        $rc = exec_cmd("$cmd");
      }
    }
  }
  
  function_leave("reboot_host", "type=$type rc=$rc ");
  return ($rc);
}

sub start_cla_service {
  function_enter("start_cla_service", "");

  my $cmd = "service cla start";
  my $rc = exec_cmd($cmd);
  if ($rc != 0) {
    log_msg_info("starting cla failed with rc=$rc, but continue anyway");
  }
  log_msg_info("sleeping 2 seconds to give cla time to figure out its real condition and then start again to be really sure it is running");
  sleep(2);
  exec_cmd($cmd);
  
  function_leave("start_cla_service", "rc=$rc");
  return ($rc);
}    

sub stop_cla_service {
  function_enter("stop_cla_service", "");

  log_msg_info("before stopping cla, check if it is running");
  exec_cmd("service cla status");
  exec_cmd("ps -elf | grep ./cla");

  my $cmd = "service cla stop";
  my $rc = exec_cmd($cmd);
  if ($rc != 0) {
    log_msg_info("stopping cla failed with rc=$rc, but continue anyway");
  }
  log_msg_info("sleeping 2 seconds to give cla time to to figure out its real condition and then stop again to be really sure it is down");
  sleep(2);
  exec_cmd($cmd);
  sleep(2);

  # it happened once (OSFOURK-24475) that even stopping twice did not kill the cla process, which prevented starting new one
  exec_cmd("ps -elf | grep ./cla");
  my $kill_rc = exec_cmd("/usr/bin/killall -9 cla");
  exec_cmd("ps -elf | grep ./cla");

  function_leave("stop_cla_service", "rc=$rc");
  return $rc;
}

sub stop_nscp_service {
  function_enter("stop_nscp_service", "(maybe dangerous if the CSTA rpm is installed -> H99338)");

  my $cmd = "service nscd stop";
  my $rc = exec_cmd($cmd);

  function_leave("stop_nscp_service", "rc=$rc");
  return;
}

sub get_openvpn_servname {
  function_enter("get_openvpn_servname");
  my $servName="";
  my $vpn_pathname = exec_cmd_out("ls /etc/openvpn/*.conf 2>/dev/null| head -1");
  $vpn_pathname = my_trim($vpn_pathname);
  log_msg_info("vpn_pathname = $vpn_pathname");

  if ($vpn_pathname eq "") {
    log_msg_info("no openvpn service file found");
  } else {
    my $vpn_basename = exec_cmd_out("basename -s .conf $vpn_pathname");
    $vpn_basename = my_trim($vpn_basename);
    log_msg_info("vpn_basename = $vpn_basename");
    $servName='openvpn@'.$vpn_basename.".service";
  }
  function_leave("get_openvpn_servname", "servName=$servName");
  return($servName);
}

sub stop_openvpn_service {
  function_enter("stop_openvpn_service", "");

  my ($cmd, $rc, $is_openvpn_running) = ("", 0, 0);
  my $servName = get_openvpn_servname();

  if ($sles_major_version == 11)  {
    $is_openvpn_running = is_service_running("openvpn");
  } else {
    if ($servName eq "") {
      log_msg_info("openvpn does not have a service config file, obviously not used");
    } else {
      $cmd = "systemctl status --no-pager $servName";
      $rc = exec_cmd($cmd);
      if ($rc == 0) {
        $is_openvpn_running = 1;
      }
    }
  }

  if ($is_openvpn_running == 1)
  {
    log_msg_info("openvpn is running, we have to stop it to get rid of any dynamic ip addess on a tun device");
  } else {
    log_msg_info("openvpn is not running, but stopping it anyway to be sure");
  }

  # first try the traditional service stop from SLES11
  $cmd = "service openvpn stop";
  $rc = exec_cmd($cmd);

  if ($sles_major_version == 11)  {
    log_msg_info("on SLES11 we do not need systemctl");
  } else {
    #in order to use systemctl, the openvpn service must be addressed like this:
    #systemctl start openvpn@SoftGate19.service
    #i.e. we have to retrieve the config file basename

    if ($servName eq "") {
      log_msg_info("openvpn does not have a service config file, no need to stop via systemctl");
    } else {
      $cmd = "systemctl stop $servName";
      $rc = exec_cmd($cmd);
    }
  }

  if ($is_openvpn_running == 1) {
    log_msg_info("openvpn service was running, check for any left over ip addresses");
    my $ip_addresses_list = exec_cmd_out("ip addr show");
    log_msg_info("ip addresses: $ip_addresses_list");
  }

  function_leave("stop_openvpn_service", "rc=$rc");
  return($rc);
}

sub stop_softgate_service {
  function_enter("stop_softgate_service", "");

  my $cmd = "service socosd stop";
  my $rc = exec_cmd($cmd);
  
  $cmd = "service socod stop";
  $rc = exec_cmd($cmd);

  function_leave("stop_softgate_service", "rc=$rc");
  return;
}

sub is_resource_running
{
  my $resource = shift;

  function_enter("is_resource_running", "resource=$resource");
  my $rc = 1;
  
  my $cmd = "/usr/sbin/crm resource list $resource 2>&1 | grep 'is NOT running'";
  my $status = exec_cmd($cmd);
  if ($status == 0) {
    $rc = 0;
  }
  
  function_leave("is_resource_running", "rc=$rc (1 is running, 0 is not running)");
  return $rc;
}

sub start_resource
{
  my $resource = shift;

  function_enter("start_resource", "resource=$resource");

  my $cmd = "/usr/sbin/crm resource start ";
  my $rc = exec_cmd($cmd . "$resource");

  function_leave("start_resource", "rc=$rc");
  return $rc;
}

sub stop_resource
{
  my $resource = shift;

  function_enter("stop_resource", "resource=$resource");

  my $cmd = "/usr/sbin/crm resource stop ";
  my $rc = exec_cmd($cmd . "$resource");

  function_leave("stop_resource", "rc=$rc");
  return $rc;
}

sub sleep_for_openais {
  my $max_tries = shift;
  my $hosttype  = shift;
  my $host      = shift;
  my $inform_sysinfo = shift;
  
  my $count = 0;
   
  if ($nowait != 0) {
    log_msg_info("skip the waiting of $max_tries minutes after openais stop or before openais start due to the --nowait flag");
    return;
  }
  while($count < $max_tries) {
    log_msg_info("sleep_for_openais max_tries=$max_tries count=$count");
    sleep(60);
    $count++;
    if ($inform_sysinfo == 1) {
      my $report_to_SWA = 0;
      if ($UpgradeRunning eq "yes") { $report_to_SWA = 1; }
      set_internal_upgrade_step($STEP_wait_for_openais, $report_to_SWA, $hosttype, $host, "waiting $count of $max_tries minutes");
    }
  }
}

sub is_service_running {
  my $service = shift;
  function_enter("is_service_running", "service=$service");

  my ($is_running, $rc, $link_rc) = (0, 0, 0);

  # on SLES 15 with systemd sometimes the service command does not tell the truth
  # status and stop are not reliable. e.g. a orphaned still running child process keeps system from calling the rc script, but just saying "still running"
  # trick: create a symlink to the service script and use the symlink, here systemd does not intervene and the service script is really called.

  if (-e "/etc/init.d/$service" ) {
    my $link_cmd ="ln -sf /etc/init.d/$service /bin/legacy-$service";
    $link_rc = exec_cmd($link_cmd);
  } else {
    log_msg_info("legacy link /bin/legacy-$service can not be created since service script /etc/init.d/$service does not exist");
  }
  
  if ($link_rc == 0 and -e "/bin/legacy-$service") {
    log_msg_info("using legacy link /bin/legacy-$service for status retrieval");
    $rc = exec_cmd("/bin/legacy-$service status");
  } else {
    log_msg_info("could not create the legacy link. I have to use the old method via service command");
    my $cmd = "service $service status";
    $rc = exec_cmd($cmd);
  }

  if ($rc == 0) {
      $is_running = 1;
  } else {
      $is_running = 0;
  }
  function_leave("is_service_running", "service=$service is_running=$is_running (rc=$rc)");
  return ($is_running); 
}

sub is_openais_service_running {
  function_enter("is_openais_service_running", "");

  my $is_running = is_service_running("openais");
  if ($is_running == 1) {
    set_openais_state($OPENAIS_STATE_running, "Active", $hostname);
  } else {
    set_openais_state($OPENAIS_STATE_stopped, "Active", $hostname);
  }

  function_leave("is_openais_service_running", "is_running=$is_running");
  return ($is_running); 
}

sub does_openais_service_support_tiptoe_stop {
  function_enter("does_openais_service_support_tiptoe_stop", "");
  
  my $cmd = " service openais help | grep tiptoe-stop";
  my $rc = exec_cmd($cmd);
  
  my $is_supporting = 0;
  if ($rc == 0) {
    $is_supporting = 1;
  }
  function_leave("does_openais_service_support_tiptoe_stop", "is_supporting=$is_supporting (rc=$rc)");
  return ($is_supporting); 
}

sub create_stoptime_file_for_assi {
  function_enter("create_stoptime_file_for_assi", "");

  if ($write_sysinfo_status_enabled == 0) {
    log_msg_info("skip notifying NUC downtime time in file: $assistant_nuc_downtime_file because it is no upgrade context");
  } else {
    if ($assistant_nuc_downtime_file_step != 0) {
      log_msg_info("since SWA NUC downtime time file: $assistant_nuc_downtime_file has already been created -> skip creating it again ");
    } else {
      $assistant_nuc_downtime_file_step = 1;
      my $stop_time = exec_cmd_out("/bin/date +%s");
      log_msg_info("local openais is still running now -> write down the stop time ($stop_time) for Assistant SWA in file $assistant_nuc_downtime_file");
      create_generic_file($stop_time, $assistant_nuc_downtime_file);
    }
  }
  function_leave("create_stoptime_file_for_assi", "");
}

sub append_timestamp_file_for_assi {
  my $step = shift;
  my $force_write = shift;
  function_enter("append_timestamp_file_for_assi", "step=$step $force_write=force_write");

  my $do_write = $force_write;
  if ($write_sysinfo_status_enabled == 1) {
    $do_write = 1;
  }
   
  if ($do_write == 0) {
    log_msg_info("skip notifying NUC timestamp time in file: $assistant_nuc_downtime_file because it is no upgrade context");
  } else {    
    if ($assistant_nuc_downtime_file_step == 0) {
      log_msg_info("error: can not append the start/success time for Assistant SWA in file $assistant_nuc_downtime_file because stop time not yet written down");
    } else {
      if ($assistant_nuc_downtime_file_step +1 == $step) {
        $assistant_nuc_downtime_file_step = $step;
        my $cur_time = exec_cmd_out("/bin/date +%s");
        log_msg_info("adding timestamp $cur_time to $assistant_nuc_downtime_file");
        append_generic_file($cur_time, $assistant_nuc_downtime_file);
      } else {
      log_msg_info("error: can not append the start/success time for Assistant SWA in file $assistant_nuc_downtime_file because previous entry is still missing (assistant_nuc_downtime_file_step=$assistant_nuc_downtime_file_step)");
      }        
    }
  }
  function_leave("append_timestamp_file_for_assi", "step=$step");
}

sub stop_openais_service {
  my $type = shift;
  my $sleep_time = shift;
  my $use_tiptoe_if_available = shift;
  my $reboot_if_tiptoe_failes = shift;
  my $use_reduce_downtime_if_available = shift;
  my $do_insserv_rm = shift;
  my $rc = 0;
  my $output = "";
  my ($other_node_is_reachable, $password_is_expired) = (0, 0);
  my $report_to_SWA=0;
  if ($UpgradeRunning eq "yes") { $report_to_SWA = 1; }

  function_enter("stop_openais_service", "type=$type currentHostType=$currentHostType sleep_time=$sleep_time use_tiptoe_if_available=$use_tiptoe_if_available reboot_if_tiptoe_failes=$reboot_if_tiptoe_failes use_reduce_downtime_if_available=$use_reduce_downtime_if_available do_insserv_rm=$do_insserv_rm report_to_SWA=$report_to_SWA");

  # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
  # log_msg_info("*** stop_openais_service dummy !!!!!!!!!!!!!!!!!!!!!!");
  # return (0);

  my $is_supporting_tiptoe = does_openais_service_support_tiptoe_stop();
  my $stop_cmd = "stop";
  my $use_tiptoe = 0;
  my $cluster_is_unmanaged = 0;

  if ($is_supporting_tiptoe == 1 && $use_tiptoe_if_available == 1) {
    $use_tiptoe = 1;
  }

  if ($use_tiptoe == 1) {
    log_msg("using the special tiptoe stop command for openais");
    $stop_cmd = "tiptoe-stop";
  }

  if ($type eq "remote") {
    ########### REMOTE ###########################

    set_internal_upgrade_step($STEP_stop_openais, $report_to_SWA, "Standby", $other_node, "Stopping HA on Standby node");
    set_openais_state        ($OPENAIS_STATE_stopping, "Standby", $other_node);

    if ($do_insserv_rm == 1) {
      log_msg_info("removing openais service from Linux startup in case a crash happens during stop on the standby node");
      $rc = execute_remote_duplex_cmd("/sbin/insserv -r /etc/init.d/openais", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
    }

    if ($use_reduce_downtime_if_available == 1) {
      if ($upgrade_with_completely_stopped_ha == 0 && $upgrade_with_reduce_downtime == 1) {
        log_msg_info("*** disabling reduce downtime feature because not a type3 upgrade");
        $upgrade_with_reduce_downtime = 0;
      }
      if ($upgrade_with_reduce_downtime == 1 && $multiNodeType != 60) {
        log_msg_info("*** disabling reduce downtime feature because not supported deployment: $multiNodeType. Only DSCXL Duplex supports this feature up to now.");
        $upgrade_with_reduce_downtime = 0;
      }

      if ($upgrade_with_reduce_downtime == 1) {
        my $is_reduced_downtime_possible_rc = is_reduced_downtime_possible();
        if ($is_reduced_downtime_possible_rc == 0) {
          log_msg_info("*** disabling reduce downtime feature because scripts are not available");
          $upgrade_with_reduce_downtime = 0;
        }
      }

      if ($upgrade_with_reduce_downtime == 1) {
        # arg1=1 -> type duplex
        $rc = execute_remote_duplex_cmd("$make_this_cc_active_sh 1", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
        if ($rc != 0) {
          log_msg_info("*** disabling reduce downtime feature because remote node does not support it");
          $upgrade_with_reduce_downtime = 0;
        }
      }
      if ($upgrade_with_reduce_downtime == 1) {
         log_msg_info("Trying to put the cluster into unmanaged state");

         log_msg_info("First unmanage the standby");
         my $manage_script = $ha_bin_dir . "$manager_cluster_sh";

         # ignore return code of this script. A reboot on the standby may occur
         $rc = execute_remote_duplex_cmd("$manage_script unmanage secondary", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);


         log_msg_info("Second unmanage the active");
         $rc = manage_cluster_for_reduced_downtime("unmanage", "primary");
         $cluster_is_unmanaged = 1;
         if ($rc != 0) {
           log_msg_info("*** the attempt to bring the cluster into unmanaged state failed -> ???");
           # $cluster_is_unmanaged = 0;  ?????
         }
      }
    }

    if ($multiNodeType == 10 or $multiNodeType == 11 or $multiNodeType == 62) {
      log_msg_info("deployment is a sep. Duplex -> create stop file when stopping secondary since primary will not be able to create it because the Assistant share is not mounted anymore ");
      create_stoptime_file_for_assi();
    }
          
    # on the remote node prepare the legacy-openais symlink and use it instead of service
    # $rc = execute_remote_duplex_cmd("service openais $stop_cmd", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
    $rc = execute_remote_duplex_cmd("ln -sf /etc/init.d/openais /bin/legacy-openais", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
    $rc = execute_remote_duplex_cmd("/bin/legacy-openais $stop_cmd", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
    
    if ($rc != 0 && $use_tiptoe == 1) {
      set_openais_state        ($OPENAIS_STATE_stopped_with_error, "Standby", $other_node);

      log_msg_info("*** disabling reduce downtime feature because standby node had tiptoe stop failure");
      $upgrade_with_reduce_downtime = 0;
      if ($reboot_if_tiptoe_failes == 1) {
        log_msg_info("*** service openais tiptoe-stop returned failure (rc=$rc) -> initiating a remote reboot  on standby node to recover the system");
        reboot_host($type, $upgrade_with_reduce_downtime, 0, "initiating a remote reboot to recover the system after stop failure");
        $rc = 0;
      }
    } else {
      set_internal_upgrade_step($STEP_wait_for_openais, $report_to_SWA, "Standby", $other_node, "sleeping $sleep_time min. on the active node in case a crash happens during stop of HA at the standby node");
      set_openais_state        ($OPENAIS_STATE_stopped, "Standby", $other_node);

      my $script =  $ha_bin_dir . "$after_stop_sh";
      if (-e $script) {
        $rc = execute_remote_duplex_cmd("$script secondary", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
        if ($rc != 0) {
          log_msg_info("*** disabling reduce downtime feature because remote node reported trouble");
          $upgrade_with_reduce_downtime = 0;
          log_msg_info("*** script $script returned failure (rc=$rc) -> initiating a remote reboot on standby node to recover the system");
          reboot_host($type, $upgrade_with_reduce_downtime, 0, "initiating a remote reboot to recover the system after stop failure");
          $rc = 0;
        }
      } else {
          log_msg_info("*** can not find script $script, skipping remote script execution on standby node");
      }
      if ($cluster_is_unmanaged == 1) {
        $rc = manage_cluster_for_reduced_downtime("manage", "primary");
        if ($rc != 0) {
          log_msg_info("*** Error: Could not bring cluster in managed state");
        }
      }
    }
    sleep_for_openais($sleep_time, "Standby", $other_node, 1);
  } else {
    if ($type eq "quorum") {
      ########### QUORUM ###########################
      set_internal_upgrade_step($STEP_stop_openais, $report_to_SWA, "Quorum", $node_q, "Stopping HA on Quorum node" );
      set_openais_state        ($OPENAIS_STATE_stopping, "Quorum", $node_q);

      if ($do_insserv_rm == 1) {
        log_msg_info("removing openais service from Linux startup in case a crash happens during stop on the quorum node");
        $rc = execute_remote_quorum_cmd("/sbin/insserv -r /etc/init.d/openais", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
      }

      # on the Q node prepare the legacy-openais symlink and use it instead of service
      # $rc = execute_remote_quorum_cmd("service openais $stop_cmd", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
      $rc = execute_remote_quorum_cmd("ln -sf /etc/init.d/openais /bin/legacy-openais", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
      $rc = execute_remote_quorum_cmd("/bin/legacy-openais $stop_cmd", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);

      if ($rc != 0 && $use_tiptoe == 1) {
        set_openais_state        ($OPENAIS_STATE_stopped_with_error, "Quorum", $node_q);
        if ($reboot_if_tiptoe_failes == 1) {
          log_msg_info("*** service openais tiptoe-stop returned failure (rc=$rc) -> initiating a remote reboot on quorum node to recover the system");
          reboot_host($type, $upgrade_with_reduce_downtime, 0, "initiating a remote reboot to recover the system after stop failure");
          $rc = 0;
        }
      } else {
        set_internal_upgrade_step($STEP_wait_for_openais, $report_to_SWA, "Quorum", $node_q, "sleeping $sleep_time min. on the active node in case a crash happens during stop of HA at the quorum node");
        set_openais_state        ($OPENAIS_STATE_stopped, "Quorum", $node_q);

        my $script =  $ha_bin_dir . "$after_stop_sh";
        if (-e $script) {
          $rc = execute_remote_quorum_cmd("$script quorum", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
          if ($rc != 0) {
            log_msg_info("*** script $script returned failure (rc=$rc) -> initiating a remote reboot on quorum node to recover the system");
            reboot_host($type, $upgrade_with_reduce_downtime, 0, "initiating a remote reboot to recover the system after stop failure");
            $rc = 0;
          }
        } else {
            log_msg_info("*** can not find script $script, skipping remote script execution on quorum");
        }
      }
      sleep_for_openais($sleep_time, "Quorum", $node_q, 1);
    } else {
        ########### LOCAL ###########################
        set_internal_upgrade_step($STEP_stop_openais, $report_to_SWA, $defaultHostType, $hostname);
        set_openais_state        ($OPENAIS_STATE_stopping, $defaultHostType, $hostname);

        if (is_openais_service_running() == 1) {
          $local_ha_has_been_stopped = 0;
        } else {
          $local_ha_has_been_stopped = 1;
        }

        if ($local_ha_has_been_stopped == 1) {
          log_msg_info("local openais was already stopped but stopping it again anyway");
        } else {
          if ($multiNodeType != 10 and $multiNodeType != 11 and $multiNodeType != 62) {
            log_msg_info("deployment is not a sep. Duplex -> create stop file when stopping primary");
            create_stoptime_file_for_assi();
          }
        }
        
        #make sure NOT to stop openais earlier than 20 seconds since daemon was started otherwise we risk that SWA in Assistant is terminated before it can finish all its activities
        $ts_rlc_upd_stop = `date +%s`;
        if ( $ts_rlc_upd_start == 0 ) {
            log_msg_info("found no timestamp for the daemon start -> will use the default sleep time $ts_delta");
        } else {
            #intention is to sleep only if the time since the daemon start is less than 20 seconds
            $ts_rlc_upd_start = $ts_rlc_upd_start + 20;
            if ( $ts_rlc_upd_stop >= $ts_rlc_upd_start ) {
                log_msg_info("no need to sleep because more than 20 seconds passed since the daemon start");
                $ts_delta = 0;
            } else {
                $ts_delta = $ts_rlc_upd_start - $ts_rlc_upd_stop;
                log_msg_info("need to sleep $ts_delta seconds so that at least 20 seconds have passed since the daemon started");
            }
        }
        if ( $ts_delta > 0 ) {
            log_msg_info("sleeping $ts_delta seconds ...");
            sleep($ts_delta);
        }

        set_openais_state        ($OPENAIS_STATE_stopping, $defaultHostType, $hostname);
        $rc = exec_cmd("service openais $stop_cmd");
        if ($rc != 0 && $local_ha_has_been_stopped == 1) {
          log_msg_info("local openais was already stopped therefore forcing the bad return code to success");
          $rc = 0;
        }
        # make a second status check with legacy status script and amek a secodn stop if still running_version
        log_msg_info("check local openais status via legacy script after the stop via service. ");
        if (is_openais_service_running() == 1) {
          log_msg_info("local openais is still running after stop. That's bad. need to stop again, but this time using the legacy script");
           exec_cmd("/bin/legacy-openais $stop_cmd");
        }

        if ($rc != 0 && $use_tiptoe == 1) {
          set_openais_state        ($OPENAIS_STATE_stopped_with_error, $defaultHostType, $hostname);

          if ($reboot_if_tiptoe_failes == 1) {
            log_msg_info("*** service openais tiptoe-stop returned failure (rc=$rc) -> initiating a local reboot to recover the system");
            reboot_host($type, $upgrade_with_reduce_downtime, 0, "initiating a reboot to recover the system after stop failure");
            # I must not survive the reboot, so I will wait here till the reboot kills me. The plt_upgrade service will retry the upgrade.
            sleep_for_openais($sleep_time, $defaultHostType, $hostname, 1);
          }
        } else {
          set_openais_state        ($OPENAIS_STATE_stopped, $defaultHostType, $hostname);

          my $script =  $ha_bin_dir . "$after_stop_sh";
          if ( -e $script ) {
            $rc = exec_cmd("$script primary");
            if ($rc != 0) {
              log_msg_info("*** script $script returned failure (rc=$rc) -> initiating a local reboot to recover the system");
              reboot_host($type,$upgrade_with_reduce_downtime, 0, "initiating a reboot to recover the system after stop failure");
            }
          } else {
              log_msg_info("*** can not find script $script, skipping script execution");
          }
        }

        if(!$nodaemon) {
          if ($local_ha_has_been_stopped == 1) {
            log_msg_info("since openais was already stopped before the current stop, I skip the Angst waiting");
          } else {
            set_internal_upgrade_step($STEP_wait_for_openais, $report_to_SWA, $defaultHostType, $hostname, "sleeping $sleep_time min. on the active node in case a crash happens during stop of HA");
            sleep_for_openais($sleep_time, $defaultHostType, $hostname, 1);
          }
        }

        # do not start openais if stop failed
        if($rc != 0) {
          write_status($rc, "could not stop openais, therefore do not set the down flag");
          # do not restart openais
        } else {
          $local_ha_has_been_stopped = 1;
        }
    }
  }
  sleep(5);
  function_leave("stop_openais_service", "type=$type rc=$rc local_ha_has_been_stopped=$local_ha_has_been_stopped");
  return ($rc);
}

sub start_openais_service {
  my $type =  shift;
  my $enforce_remote_insserv = shift;
  my $is_failure = shift;
  if (!defined($is_failure)) { $is_failure = 0; }
  
  my $startup_script_rc = 0; 
  my $start_rc = 0;
  my $rc = 0;
  my $output = "";
  my ($other_node_is_reachable, $password_is_expired) = (0, 0);
  my $sleep_time = 5; # 5 minutes Angstwarten
  
  function_enter("start_openais_service", "type=$type currentHostType=$currentHostType enforce_remote_insserv=$enforce_remote_insserv is_failure=$is_failure");
  if ( -e "/var/log/nuc/swupdated.use.nnuc" ) {
    log_msg_info("start_openais_service skipped due to special N-NUC case");
    function_leave("start_openais_service", "type=$type rc=$rc");
    return($rc);
  }

  if ($is_FI == 1) {
    log_msg_info("start_openais_service skipped due to firstinstall scenario");
    function_leave("start_openais_service", "type=$type rc=$rc");
    return($rc);
  }

  
  if ($type eq "remote") {

    set_openais_state        ($OPENAIS_STATE_starting, "Standby", $other_node);

    if ($nodaemon == 0 && $upgrade_with_completely_stopped_ha == 1 && ($host_reboot == 1 || $upgrade_with_reduce_downtime == 1)) {
      log_msg_info("enforce_remote_insserv activated");
      $enforce_remote_insserv = 1;
    }
    if ($enforce_remote_insserv == 1) {
      if ($upgrade_via_image == 0) {
        sleep_for_openais($sleep_time, "Standby", $other_node, 1);
      } else {
        log_msg_info("its a NUC, therefore waiting for re-appearance of B is not necessary (worldness is already verified)");
      }
      $rc = execute_remote_duplex_cmd("$remote_self configure_cib secondary $is_failure", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
    } else {
      $rc = execute_remote_duplex_cmd("service openais start", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
    }

    set_openais_state        ($OPENAIS_STATE_running, "Standby", $other_node);

    if ($upgrade_with_reduce_downtime == 1) {
      sleep(10);
      manage_cluster_for_reduced_downtime("manage", "primary");
    }
  } else {    
    
    if ($type eq "quorum") {
      set_openais_state        ($OPENAIS_STATE_starting, "Quorum", $node_q);
      
      if(($nodaemon == 0 && $upgrade_with_completely_stopped_ha == 1 && $host_reboot == 1) || ($enforce_remote_insserv == 1)) {
        if ($upgrade_via_image == 0) {
          sleep_for_openais($sleep_time, "Quorum", $node_q, 1);
        } else {
          log_msg_info("its a NUC, therefore waiting for re-appearance of Q is not necessary (worldness is already verified)");
        }
        $rc = execute_remote_quorum_cmd("$remote_self configure_cib quorum $is_failure", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
      } else {      
        $rc = execute_remote_quorum_cmd("service openais start", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
      }
      
      set_openais_state        ($OPENAIS_STATE_running, "Quorum", $node_q);
    } else {
  
      if ( -x $before_start_diagnose_script )  {
        $startup_script_rc = exec_cmd("$before_start_diagnose_script");
      }
    
      if ($startup_script_rc != 0) {
        $rc = $startup_script_rc;
        write_status($global_rc, "Not starting openais service due to return status of before_start diagnosis script");
      } else {
        # starting daemon - ignore rc
        set_openais_state        ($OPENAIS_STATE_starting, $defaultHostType, $hostname);

        log_msg_info("starting openais service");
        $start_rc = exec_cmd("service openais start");
        if ($start_rc != 0) {
          write_status($rc, "error when starting openais service rc=$start_rc. Continuing anyway ...");
        }
        set_openais_state        ($OPENAIS_STATE_running, $defaultHostType, $hostname);
        append_timestamp_file_for_assi(2, 0); # second entry after stop
      }
      
      if ($rc != 0 && -x $after_start_diagnose_script) {     
        $rc = exec_cmd("$after_start_diagnose_script");
        if ($rc != 0) {
          write_status($rc, "error indicated by after_start diagnosis script. You may reboot your system.");
        }
      }
    }
  }  
  function_leave("start_openais_service", "type=$type rc=$rc");
  
  return ($rc);
}


sub execute_yaps_preinstall {
  my $rc = 0;
  
  function_enter("execute_yaps_preinstall", "");
  
  my $cmd = 'export PATH=/usr/bin:/bin; ';
  $cmd .= 'PREINSTALL=/opt/yaps/script/preinstall; ';
  $cmd .= 'cd /tmp && /usr/bin/rpm2cpio '.$rpm_dir.'ASyaps*.rpm | ';
  $cmd .= 'cpio -ivdu --no-absolute-filename .${PREINSTALL} >> '.$log_file.' 2>&1; ';
  $cmd .= 'if [ "$?" == "0" ] && [ -e "/tmp${PREINSTALL}" ]; then ';
  $cmd .= '/tmp${PREINSTALL} -p '.$rpm_dir.' -u >> '.$log_file.' 2>&1 || exit 1; ';
  $cmd .= 'fi;';
  log_msg_info("executing preinstall");
  exec_cmd("/bin/df -h");
  exec_cmd("/bin/df -i");
  exec_cmd("/bin/cat /etc/mtab");
  exec_cmd("/bin/checkydata");

  $rc = exec_cmd($cmd);
  
  if ($rc != 0) {
    $rc = $ERROR_ExecutePreinstallFailed;
  }
  function_leave("execute_yaps_preinstall", "rc=$rc");
  return($rc);  
}

sub execute_yaps_preinstall2 {
  my $type = shift;
  my $upgrade_with_completely_stopped_ha = shift;
  my $upgrade_with_full_drbd_sync        = shift;
  my $multiNodeType = shift;
  my $isMultiNode   = shift;
  
  my $rc = 0;
  function_enter("execute_yaps_preinstall2", "type=$type upgrade_with_completely_stopped_ha=$upgrade_with_completely_stopped_ha upgrade_with_full_drbd_sync=$upgrade_with_full_drbd_sync multiNodeType=$multiNodeType isMultiNode=$isMultiNode ");

  my $cmd = 'export PATH=/usr/bin:/bin; ';
  $cmd .= 'PREINSTALL2=/opt/yaps/script/preinstall2; ';
  $cmd .= 'cd /tmp && /usr/bin/rpm2cpio '.$rpm_dir.'ASyaps*.rpm | ';
  $cmd .= 'cpio -ivdu --no-absolute-filename .${PREINSTALL2} >> '.$log_file.' 2>&1; ';
  $cmd .= 'if [ "$?" == "0" ] && [ -e "/tmp${PREINSTALL2}" ]; then ';
  $cmd .= '/tmp${PREINSTALL2} '.$type.'  '.$upgrade_with_completely_stopped_ha.'  '.$upgrade_with_full_drbd_sync.'  '.$multiNodeType.' '.$isMultiNode.' >> '.$log_file.' 2>&1 || exit 1; ';
  $cmd .= 'fi;';
  log_msg_info("executing preinstall");
  exec_cmd("/bin/df -h");
  exec_cmd("/bin/df -i");
  exec_cmd("/bin/cat /etc/mtab");
  exec_cmd("/bin/checkydata");

  $rc = exec_cmd($cmd);

  if ($rc != 0) {
    $rc = $ERROR_ExecutePreinstallFailed;
  }
  function_leave("execute_yaps_preinstall2", "rc=$rc");
  return($rc);
}

sub get_ha_type {
  function_enter("get_ha_type", "");
  my $type = "primary";
  if(!$nodaemon) {
    # node_A
    $type = "primary";
  } else {
    if ($isquorum == 1) {
      $type = "quorum";
    } else {
      $type = "secondary";
    }
  }
  function_leave("get_ha_type", "type=$type");
  return($type);
}

sub get_LTU {
  function_enter("get_LTU", "");
  my ($LTU, $script) = ("", "/opt/soco/native/os/linux/get_LTU.sh");

  if (-e $script) {
    $LTU = exec_cmd_out($script);
    chomp($LTU);
  } else {
    log_msg_info("script $script does not exist, can not retrieve SGs LTU number");
  }
  function_leave("get_LTU", "LTU=$LTU");
  return($LTU);
}

sub fix_menu_list_after_os_update {
  my $rc = 0;
  function_enter("fix_menu_list_after_os_update", "");

  my $type="";
  if(!$nodaemon) {
      $type = "primary";
      log_msg_info(" running fix_menu.lst_after_osupdate.sh in primary mode ");
  } else {
      if ($isquorum == 1) {
        $type = "quorum";
        log_msg_info(" running fix_menu.lst_after_osupdate.sh in quorum mode ");
      } else {
        $type = "secondary";
        log_msg_info(" running fix_menu.lst_after_osupdate.sh in secondary mode ");
      }
  }
    
  $rc = exec_cmd("/opt/os4k-linux-basicinit/fix_menu.lst_after_osupdate.sh $type");

  if ($rc != 0) {
    $rc = $ERROR_ExecuteFixMenuListFailed;
  }
  function_leave("fix_menu_list_after_os_update", "rc=$rc");
  return($rc);
}

sub execute_yaps_prepostinstall {
  my $rc = 0;
  my $type = shift;
  
  function_enter("execute_yaps_prepostinstall", "type=$type");
  my $script = $yaps_dir.'script/pre-postinstall';
  if (-e $script) {
    $rc = exec_cmd("$script -p $rpm_dir $type");
    if ($rc != 0) {
      $rc = $ERROR_ExecutePrePostinstallFailed;
    }
  } else {
    log_msg_info("script $script does not exist, skipping");
  }
  function_leave("execute_yaps_prepostinstall", "rc=$rc");
  return($rc);
}

sub execute_yaps_postinstall {
  my $rc = 0;
  my $type = shift;
  
  function_enter("execute_yaps_postinstall", "type=$type");
  exec_cmd("/bin/checkydata");

  my $script = $yaps_dir.'script/postinstall';
  if (-e $script) {  
    $rc = exec_cmd("$script -p $rpm_dir $type");

    if ($rc != 0) {
      $rc = $ERROR_ExecutePostinstallFailed;
    }
  } else {
    log_msg_info("script $script does not exist, skipping");
  }
  function_leave("execute_yaps_postinstall", "rc=$rc");
  return($rc);
}

sub execute_pre_ws_update {
  function_enter("execute_pre_ws_update", "");
  my $cmd = "$pgsql_before_upgrade";
  my $rc = 0;  

  if($nodaemon) {
    log_msg_info("skip dump of pgsql db because we are running on the passive or quorum node (no webservice resource is running)");
  } else {
    # pgsql backup/restore must be done only on the active node

    # check deployment
    if ($multiNodeType == 100) {
      log_msg_info("node is still unconfigured (deployment == 100) -> there is no DB content to upgrade -> skip db dump");
    } else {
      # check if there is already a BEER backup available
      if ( -e $beer_backup_script) {
        log_msg_info("platform already supports scripted BEER backup -> use it");
        $cmd = "$beer_backup_script";
        $platform_has_beer = 1;
      } else {
        log_msg_info("platform does not support scripted BEER backup -> use as fallback the ngs backup");
        $cmd = "$pgsql_before_upgrade";
        $platform_has_beer = 0;
      }
      my $output = exec_cmd_out($cmd);
      # $rc = exec_cmd($cmd);
      log_msg_info("*** output of $pgsql_before_upgrade=$output");
    }

    # check opensips pgsql for NUC update, but not on STMIX
    if ($is_stmix == 0) {
      if ($upgrade_via_image == 1) {
        if (get_number_of_sg_on_central_hosts(0) != 0) {
          log_msg_info("It is a NUC scenario on a host where a Softgate is configured -> handle opensips pgsql when openais is still running");
          exec_cmd ("/opt/opensips/dbscripts/data-backup2.sh $opensips_sql_dump_file");
          if (-e $opensips_sql_dump_file) {
             log_msg_info("creating a fallback copy $opensips_sql_dump_fallback_file of the $opensips_sql_dump_file in case the SG removes the original");
             $rc = exec_cmd("cp -v $opensips_sql_dump_file $opensips_sql_dump_fallback_file");
          }
        }
      }
    }
  }

  # for diagnosis
  exec_cmd("ls -l $pgsql_data_zip_file");
  exec_cmd("ls -l $pgsql_data_zip_save");
  exec_cmd("ls -l $pgsql_data_zip_resc");
  exec_cmd("ls -l $opensips_sql_dump_file");
  exec_cmd("ls -l $opensips_sql_dump_fallback_file");
        
  function_leave("execute_pre_ws_update", "rc=$rc");
  return ($rc);
}

sub execute_ha_do_insserv {
  function_enter("execute_ha_do_insserv", "");
  log_msg_info("*** insserv openais service to be rebootable again ");
  
  my $call = "$ha_insserv_script";
  my $rc = 0;  

  if (-e $ha_insserv_script ) {
    $rc = exec_cmd($call);
  } else {
    log_msg_info("Script $ha_insserv_script does not exist -> executing the steps manually ");
    
    $call = "/sbin/insserv /etc/init.d/openais ";
    $rc = exec_cmd($call);

    $call = "/sbin/insserv -r /etc/init.d/plt_upgrade ";
    $rc = exec_cmd($call);
  }
  function_leave("execute_ha_do_insserv", "");
  return ($rc);
}

sub execute_ha_rm_insserv {
  function_enter("execute_ha_rm_insserv", "");
  my $call = "$ha_insserv_script -r";
  my $rc = 0;
  if (-e $ha_insserv_script ) {
    $rc = exec_cmd($call);
  } else {
    log_msg_info("Script $ha_insserv_script does not exist -> executing the steps manually ");

    $call = "/sbin/insserv -r /etc/init.d/openais ";
    $rc = exec_cmd($call);

    $call = "/sbin/insserv /etc/init.d/plt_upgrade ";
    $rc = exec_cmd($call);
  }
  function_leave("execute_ha_rm_insserv", "");
  return ($rc);
}


sub execute_pre_ha_update {
  my $upgrade_with_completely_stopped_ha = shift;  
  my $upgrade_with_full_drbd_sync = shift;
  my $multiNodeType = shift;
  my $isMultiNode   = shift;
  
  my $cmd = "";
  my $rc = 0;
  function_enter("execute_pre_ha_update", "upgrade_with_completely_stopped_ha=$upgrade_with_completely_stopped_ha upgrade_with_full_drbd_sync=$upgrade_with_full_drbd_sync multiNodeType=$multiNodeType isMultiNode=$isMultiNode");

  if ($full_drbd_sync_flag_really_read == 0) {
    $upgrade_with_full_drbd_sync = $upgrade_with_completely_stopped_ha;
    log_msg_info("execute_pre_ha_update: forcing upgrade_with_full_drbd_sync to value of completely_stopped_ha ");
  }
  
  my $script = $ha_bin_dir . "$before_upd_sh";
  if ( ! -e "$script" ) {
    log_msg_info("Did not find script $script, checking old location ");
    $script =  $ha_dir . "$before_upd_sh";
  }
  
  if ( -e "$script" ) {
    if(!$nodaemon) {
      log_msg_info(" running pre_ha_update in primary mode ");    
      $cmd = "$script primary $upgrade_with_completely_stopped_ha $multiNodeType $isMultiNode $upgrade_with_full_drbd_sync"
    } else {
      if ($isquorum == 1) {
        log_msg_info(" running execute_pre_ha_update in quorum mode ");     
        $cmd = "$script quorum $upgrade_with_completely_stopped_ha $multiNodeType $isMultiNode $upgrade_with_full_drbd_sync" 
      } else {       
        log_msg_info(" running pre_ha_update in secondary mode ");     
        $cmd = "$script secondary $upgrade_with_completely_stopped_ha $multiNodeType $isMultiNode $upgrade_with_full_drbd_sync"
      }
    }
    $rc = exec_cmd($cmd)
  } else {
    log_msg_info("skipping pre_ha_update because script $before_upd_sh not available (not on new and not on old location)"); 
    $rc = 0;
  }
  if ($rc != 0) {
    $rc = $ERROR_ExecutePreHaUpdate;
  }
  
  function_leave("execute_pre_ha_update", "rc=$rc");
  return($rc);
}

sub execute_after_rpm_install_ha {
  my $type = shift;
  my $upgrade_with_completely_stopped_ha = shift;
  my $upgrade_with_full_drbd_sync        = shift;
  my $multiNodeType = shift;  
  my $isMultiNode   = shift;
  
  my $cmd = "";
  my $rc = 0;  

  function_enter("execute_after_rpm_install_ha", "type=$type upgrade_with_completely_stopped_ha=$upgrade_with_completely_stopped_ha upgrade_with_full_drbd_sync=$upgrade_with_full_drbd_sync multiNodeType=$multiNodeType isMultiNode=$isMultiNode");

  if ($is_stmix == 1) {
      log_msg_info("on STMIX skip $after_rpm_install_sh execution");

  } else {
    # restore the saved openais.conf after upgrade before initHA may re-generate it
    restore_important_ha_config_files();

    if ($full_drbd_sync_flag_really_read == 0) {
      $upgrade_with_full_drbd_sync = $upgrade_with_completely_stopped_ha;
      log_msg_info(">>> forcing upgrade_with_full_drbd_sync to value of completely_stopped_ha ");
    }

    my $script = $ha_bin_dir . "$after_rpm_install_sh";
    if ( ! -e "$script" ) {
      log_msg_info("Did not find script $script, checking old location ");
      $script =  $ha_dir . "$after_rpm_install_sh";
    }

    if ( -e "$script" ) {
      $cmd = "$script $type $upgrade_with_completely_stopped_ha $multiNodeType $isMultiNode $upgrade_with_full_drbd_sync";
      $rc = exec_cmd($cmd);
   } else {
      log_msg_info("skipping execute_after_rpm_install_ha because script $after_rpm_install_sh not available");
      $rc = 0;
    }
  }

  function_leave("execute_after_rpm_install_ha", "type=$type rc=$rc");
  return($rc);
}

sub is_reduced_downtime_possible {
  function_enter("is_reduced_downtime_possible");
  my $is_possible = 1;

  my $script = "";

  $script = "$make_this_cc_active_sh";
  if ( ! -e "$script" ) {
    log_msg_info("*** Did not find script $script, disabling reduced downtime feature");
    $is_possible = 0;
  }

  $script = $ha_bin_dir . "$manager_cluster_sh";
  if ( ! -e "$script" ) {
    log_msg_info("*** Did not find script $script, disabling reduced downtime feature");
    $is_possible = 0;
  }

  $script = $ha_bin_dir . "$after_stop_sh";
  if ( ! -e "$script" ) {
    log_msg_info("*** Did not find script $script, disabling reduced downtime feature");
    $is_possible = 0;
  }

  if ($host_reboot == 1) {
    log_msg_info("*** Upgrade enforces a Linux Host reboot (e.g. due to Kernel update) -> Feature reduced_downtime is not possible in this case");
    $is_possible = 0;
  }
  
  if ($biosupdate != 0) {
    log_msg_info("*** Bios upgrade enforces a Linux Host reboot -> Feature reduced_downtime is not possible in this case");
    $is_possible = 0;
  }  
  
  function_leave("is_reduced_downtime_possible", "is_possible=$is_possible");
  return($is_possible);
}

sub start_all_stopped_resources_if_necessary {
  my $hotfix_mode = shift;
  my $rmx_reboot = shift;
  my $dtb_resource_has_been_stopped = shift;
  my $type = shift;
  my $cmd = "";
  my $rc = 0;
  
  if (! defined($type)) {
    if(!$nodaemon) {
      $type = "primary";
    } else {
      if ($isquorum == 1) {
        $type = "quorum";
      } else {
        $type = "secondary";
      }
    }
  }
  function_enter("start_all_stopped_resources_if_necessary", "hotfix_mode=$hotfix_mode, rmx_reboot=$rmx_reboot, dtb_resource_has_been_stopped=$dtb_resource_has_been_stopped, type=$type");

  my $max_tries = 36; 
  my $period    = 10; # seconds
  if ($type eq  "primary") {
    if ($hotfix_mode != 0 && $rmx_reboot == 0) {
      log_msg_info("it is a type 1 platform hotfix -> restart potentially stopped resources, but only on the active side");
      $cmd = " /usr/sbin/crm resource start ";

      my $dtb_is_configured = is_DTB_configured();
      if ($dtb_resource_has_been_stopped == 1 and $dtb_is_configured == 0) {
         log_msg_info("WARN: DTB resource was running before HF update, but it is not configured -> keeping it down", 0, 1);
      }
      if ($dtb_resource_has_been_stopped == 0 and $dtb_is_configured == 1) {
         log_msg_info("DTB resource was not stopped before HF update, but it is configured -> starting it to be sure it is running");
      }

      if ($dtb_is_configured == 1) {
        log_msg_info("starting the configured DTB resource on active node for type 1 plt HF independantly of resource status before the update");
        $rc = exec_cmd($cmd . "rsc4k_dtb");
      }
      
      $rc = exec_cmd($cmd . "rsc4k_webservice");
      if ($isMultiNode != 0) {
        log_msg_info("redundancy scenario -> I have to wait till the webservice is really running before adding the tunnelbroker resource again");
        $rc = wait_for_service_running("webservice", $max_tries, $period);
        if ($rc != 0) {
          log_msg_info("webservice not running as a service on current node during 6 minutes, skip waiting. It may have been started on the other side");
        }
      }
      $rc = exec_cmd($cmd . "rsc4k_tunnelbroker");
      log_msg_info("not necessary to wait for the status of the tunnelbroker service");
    }
  }
  function_leave("start_all_stopped_resources_if_necessary", "type=$type rc=$rc");
  return($rc);
}

sub restart_systemd_daemon {
  function_enter("restart_systemd_daemon");
  log_msg_info("restarting the systemd in case a new rpm package with init.d script has been installed");
  # we do not need a check for SLES11
  my $rc = exec_cmd("systemctl daemon-reload");
  function_leave("restart_systemd_daemon", "rc=$rc");
  return($rc);
}

sub exec_after_rpm_installation {
  my $type = "";
  my $cmd = "";
  my $rc = 0;

  function_enter("exec_after_rpm_installation");

  if(!$nodaemon) {
    $type = "primary";
  } else {
    if ($isquorum == 1) {
       $type = "quorum";
    } else {
       $type = "secondary";
    }
  }
  
  my $script = "$rd_exec_after_sh";
  if ( ! -e "$script" ) {
    log_msg_info("*** Did not find script $script, skipping execution");
    $rc = 1;
  } else {
    if ($upgrade_with_reduce_downtime == 0) {
      log_msg_info("*** no reduce downtime upgrade -> skipping call of $script");
    } else {
      $cmd = "$script $type";
      log_msg_info("reduce downtime upgrade -> call $cmd");
      $rc = exec_cmd($cmd);
    }
  }
  
  protect_zypper_logs();
  
  function_leave("exec_after_rpm_installation", "type=$type rc=$rc");
  return($rc);
}


sub make_this_cc_active {
  my $cmd = "";
  my $rc = 0;

  function_enter("make_this_cc_active");

  my $script = "$make_this_cc_active_sh";
  if ( ! -e "$script" ) {
    log_msg_info("*** Did not find script $script, disabling reduced downtime feature on active node");
    $rc = 1;
  } else {
    my $type = 1; # 1 means duplex deployment
    $cmd = "$script $type ";
    $rc = exec_cmd($cmd);
  }

  function_leave("make_this_cc_active", "rc=$rc");
  return($rc);
}

sub manage_cluster_for_reduced_downtime {
  my $managetype = shift;

  # Die Bedeutung ist nodeType, d.h. auf die primaere Seite es soll "primary" sein und auf secondary es soll "secondary" angegeben.
  my $nodetype   = shift;

  my $cmd = "";
  my $rc = 0;

  function_enter("manage_cluster_for_reduced_downtime", "managetype=$managetype nodetype=$nodetype upgrade_with_reduce_downtime=$upgrade_with_reduce_downtime");

  my $script = $ha_bin_dir . "$manager_cluster_sh";
  if ( ! -e "$script" ) {
    log_msg_info("*** Did not find script $script, disabling reduced downtime feature");
    $upgrade_with_reduce_downtime = 0;
    $rc = 1;
  } else {
    $cmd = "$script $managetype $nodetype";
    $rc = exec_cmd($cmd);
  }

  function_leave("manage_cluster_for_reduced_downtime", "managetype=$managetype nodetype=$nodetype upgrade_with_reduce_downtime=$upgrade_with_reduce_downtime rc=$rc");
  return($rc);
}

sub execute_post_ha_update {
  my $upgrade_with_completely_stopped_ha = shift;
  my $upgrade_with_full_drbd_sync        = shift;
  my $multiNodeType = shift;  
  my $isMultiNode   = shift;
  
  my $cmd = "";
  my $rc = 0;  

  function_enter("execute_post_ha_update", "upgrade_with_completely_stopped_ha=$upgrade_with_completely_stopped_ha upgrade_with_full_drbd_sync=$upgrade_with_full_drbd_sync multiNodeType=$multiNodeType isMultiNode=$isMultiNode");

  if ($full_drbd_sync_flag_really_read == 0) {
    $upgrade_with_full_drbd_sync = $upgrade_with_completely_stopped_ha;
    log_msg_info(">>> forcing upgrade_with_full_drbd_sync to value of completely_stopped_ha ");
  }
  
  my $script = $ha_bin_dir . "$after_upd_sh";
  if ( ! -e "$script" ) {
    log_msg_info("Did not find script $script, checking old location ");
    $script =  $ha_dir . "$after_upd_sh";
  }

  if ($is_FI == 0) {
    if ( -e "$script" ) {
      if(!$nodaemon) {
        log_msg_info(" running post_ha_update in primary mode ");
        $cmd = "$script primary $upgrade_with_completely_stopped_ha $multiNodeType $isMultiNode $upgrade_with_full_drbd_sync"
      } else {
        if ($isquorum == 1) {
          log_msg_info(" running post_ha_update in quorum mode ");
          $cmd = "$script quorum $upgrade_with_completely_stopped_ha $multiNodeType $isMultiNode $upgrade_with_full_drbd_sync"
        } else {
          log_msg_info(" running post_ha_update in secondary mode ");
          $cmd = "$script secondary $upgrade_with_completely_stopped_ha $multiNodeType $isMultiNode $upgrade_with_full_drbd_sync"
        }
      }

      $rc = exec_cmd($cmd)
    } else {
      log_msg_info("skipping post_ha_update because script $after_upd_sh not available");
      $rc = 0;
    }
  } else {
      log_msg_info("skipping post_ha_update via script $after_upd_sh in first install mode");
  }
  
  function_leave("execute_post_ha_update", "rc=$rc");
  return($rc);
}

sub is_node_reachable {
  my $node_to_check = shift;

  function_enter("is_node_reachable", "node_to_check=$node_to_check test_method=$node_reach_test_method");
  my ($cmd, $script) = ("", "");
  my $rc = 0;

  if ($node_reach_test_method eq "none") {
    log_msg_info("do not check the node reachability due to none method -> assuming it is reachable");
    $rc = 1;
  } else {
    $script = "$webservice_dir"."scripts/is_node_reachable.sh";
    if ($node_reach_test_method eq "ping") {
      $cmd = "$script -p $node_to_check";
    } else {
      $cmd = "$script -c $node_to_check";
    }
    if (-e $script) {
      $rc = exec_cmd($cmd);
    } else {
      log_msg_info("script $script does not exist for checking node connectivity -> assuming it is reachable");
      $rc = 1;
    }
  }

  function_leave("is_node_reachable", "node_to_check=$node_to_check is_reachable=$rc");
  return($rc);
}

sub prepare_sftp_host {
  my $key_sentinel = shift;
  my $user = shift;
  my $user_id_ref = shift;
  my $group_id = shift;
  
  function_enter("prepare_sftp_host", "key_sentinel=$key_sentinel user=$user uid=$$user_id_ref, gid=$group_id");
  my $rc = 0;

  $rc = create_linux_user($sftp_transfer_user, $user_id_ref, $group_id);
  if ($rc == 0) {
    $rc = prepare_local_ssh_credentials($key_sentinel, $sftp_transfer_user, $$user_id_ref, $group_id);
  }

  # diagnosis
  exec_cmd("systemctl --no-pager status sshd");
  
  function_leave("prepare_sftp_host", "rc=$rc");
  return($rc);
}  

sub generate_torrent_file {
  my $cur_platform_version = shift;
  my $exec_in_background = shift;
  function_enter("generate_torrent_file", "cur_platform_version=$cur_platform_version exec_in_background=$exec_in_background");
  my $rc = 0;
  my $uwe_script="/opt/soco-common/create_sg_os_metadata.sh";
  my $cmd = "$uwe_script -v $cur_platform_version";
  if ( $exec_in_background == 1) {
    $cmd .= "&";
  }
  $rc = exec_cmd($cmd);
  if ($rc != 0) {
    log_msg_info("script $uwe_script returned error, we do not cancel complete tar file generation, but just continue");
    $rc = 0; # $ERROR_generic_SG_OS_Update;
  }
  
  function_leave("generate_torrent_file", "rc=$rc");
  return($rc);  
}

# it would be necessary to patch the SG file only on STMIX with OS V10 R0.28 with initial SG version
# but it is too complicated to check that, so we copy to all STMIX as on central host we anyhow have the newer version of the script
sub patch_stmix_checkNecessaryFreeDiskSpaceForOSTransfer_file_via_inventory {
  function_enter("patch_stmix_checkNecessaryFreeDiskSpaceForOSTransfer_file_via_inventory");
  my ($rc, $copy_rc, $inventory_id, $ip_address) = (0, 0, "", "");
  my $retrieve_cmd = "/opt/inventory/inventory appliance_list --include-stmix";
  my $copy_cmd = "/opt/inventory/inventory put";
  my $sg_file_to_patch = "/opt/soco/native/os/linux/checkNecessaryFreeDiskSpaceForOSTransfer.sh";


  if (! -f $sg_file_to_patch) {
    log_msg_info("file $sg_file_to_patch not found, cannot patch on STMIX");
    $rc=1;
    function_leave("patch_stmix_checkNecessaryFreeDiskSpaceForOSTransfer_file_via_inventory", "rc=$rc");
    return($rc);
  }

  # retrieve the list of all appliances from inventory (it should run at this time)
  # we expect output like this:
  # 101700 172.29.178.80
  # 23900 172.29.178.88
  # 25001 172.29.178.70
  # 25003 1.2.3.4
  my $retrieve_out = exec_cmd_out($retrieve_cmd);
  # log_msg_info("retrieve_out=$retrieve_out");

  # go over all lines and get the appliance_id
  my @inventory_lines = split(/\n/, $retrieve_out);
  foreach(@inventory_lines) {
    log_msg_info("line=$_");

      if(length($_) >= 0) {
        log_msg_info("line is not empty");

        my @parts = split(/ /, $_);
        $inventory_id = $parts[0];
        $ip_address = $parts[1];
        log_msg_info("inventory_id=$inventory_id, ip_address=$ip_address");
        if (!defined($inventory_id)) { $inventory_id=""; }
        if (!defined($ip_address))   { $ip_address=""; }
        if ($inventory_id ne "") {
          if (looks_like_number($inventory_id)) {
            if ($inventory_id % 100 == 0) {
              log_msg_info("inventory_id=$inventory_id is modulo 100, it has to be a SG or EntGW");
            } else {
              log_msg_info("inventory_id=$inventory_id is not modulo 100, it has to be a STMIX or STMIY");
              $copy_rc = exec_cmd("$copy_cmd $inventory_id $sg_file_to_patch $sg_file_to_patch");
              if ($copy_rc != 0) {
                log_msg_info("copy of file $sg_file_to_patch to appliance $inventory_id failed, continue");
                $rc = 1;
              }
            }
          } else {
            log_msg_info("inventory_id $inventory_id is not a number");
          }
        } else {
          log_msg_info("inventory_id empty");
        }
      } else {
        log_msg_info("line is empty");
      }
  }

  function_leave("patch_stmix_checkNecessaryFreeDiskSpaceForOSTransfer_file_via_inventory", "rc=$rc");
  return($rc);  
}

sub concatFileAndMove {
  my $path_to_file_to_keep = shift;
  my $path_to_file_to_concat = shift;
  my $extension = shift;
  function_enter("concatFileAndMove", "path_to_file_to_keep=$path_to_file_to_keep path_to_file_to_concat=$path_to_file_to_concat extension=$extension");
  my $rc = 0;

  my $cmd = "/bin/cat $path_to_file_to_concat >> $path_to_file_to_keep";
  # cant use exec_cmd here due to the >>
  my $out = exec_cmd_pipe($cmd);

  $cmd = "/bin/mv -vf $path_to_file_to_concat $path_to_file_to_concat.$extension";
  $rc = exec_cmd($cmd);
  if ($rc != 0) {
    log_msg_info("moving $path_to_file_to_concat to $path_to_file_to_concat.$extension failed");
    $rc = $ERROR_generic_SG_OS_Update;
  }
  
  function_leave("concatFileAndMove", "rc=$rc");
  return($rc);  
}

sub get_swupdated_path_for_OS_update {
  function_enter("get_swupdated_path_for_OS_update");
  my $rc = 0;
  my $swupdated_path_1 = "/opt/yaps/script/swupdated";
  my $swupdated_path_2 = "/instmedia/opt/yaps/script/swupdated";
  my $swupdated_path = "";
  # on RLC activations the /instmedia seems to be the most recent (containing the Plt HF transferred and activated together with RLC)
  if (-f $swupdated_path_2) {
    $swupdated_path=$swupdated_path_2;
  } else {
    log_msg_info("$swupdated_path_2 does not exist, using the currently installed yaps swupdated");
    $swupdated_path=$swupdated_path_1;
  }

  my $check_swupdated_version = exec_cmd_out("grep 'upd_version =' $swupdated_path");
  log_msg_info("swupdated version to put into transfer.tar: $check_swupdated_version");

  function_leave("get_swupdated_path_for_OS_update", "swupdated_path=$swupdated_path");
  return($swupdated_path);
}

sub generate_os_transfer_file_for_sg {
  my $path_to_sg_list_file_from_lwum = shift;
  my $path_to_stmix_list_file_from_lwum = shift;
  function_enter("generate_os_transfer_file_for_sg", "path_to_sg_list_file=$path_to_sg_list_file_from_lwum path_to_stmix_list_file=$path_to_stmix_list_file_from_lwum");
  my ($rc, $remote_rc) = (0,0);
  my $working_dir = $assistant_gw_area;
  my $type_of_operation =  "gen_os_transfer_file";
  my $combined_log = $webservice_log_dir."$type_of_operation".".log";
  my ($global_text, $remote_text, $remote_node_is_reachable, $remote_node_password_is_expired, $local_text, $quorum_text, $quorum_node_is_reachable, $quorum_node_password_is_expired) = ("", "", 1, 0, "", "", 1, 0);
  my ($other_node_is_reachable, $password_is_expired, $remote_cmd) = (0, 0, "");
  my $work_on_remote = 0;   # we do not want to operate on Duplex standby node anymore (OSFOURK-25063)
  my ($path_to_stmix_list_file, $path_to_sg_list_file) = ("", "");

  # check the lock: $sg_os_prep_lock_file
  my ($lock_timeout, $got_the_lock_immediately) = (2, 0);  # 2 seconds
  
  $rc = wait_for_lock ($sg_os_prep_lock_file, $lock_timeout, $type_of_operation, \$got_the_lock_immediately);
  if ($rc == 0) {
    create_lock_file($sg_os_prep_lock_file);
    log_msg_info("got the lock for $sg_os_prep_lock_file");

    if ( ! -e $working_dir ) {
      $global_text = "error: working dir $working_dir with torrent file does not exist";
      log_msg_info($global_text);
      $rc = $ERROR_generic_SG_OS_Update;
    } else {
      log_msg_info("changing working dir to: $working_dir");
      chdir $working_dir or $rc = $ERROR_generic_SG_OS_Update;
    }

    determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 0);

    # check the existence of the SG list from LWUM
    $path_to_sg_list_file = $assistant_data_area . $path_to_sg_list_file_from_lwum;
    if ( -f $path_to_sg_list_file) {
      log_msg_info("using $path_to_sg_list_file (precedence 1)");
    } else {
      $path_to_sg_list_file = $assistant_gw_area . $path_to_sg_list_file_from_lwum;
      if ( -f $path_to_sg_list_file) {
        log_msg_info("using $path_to_sg_list_file (precedence 2)");
      } else {
        $path_to_sg_list_file = $path_to_sg_list_file_from_lwum;
        if ( -f $path_to_sg_list_file) {
          log_msg_info("using $path_to_sg_list_file (obviously absolute path, precedence 3)");
        } else {
          log_msg_info("neither relative path nor absolute path working -> error");
          $rc = $ERROR_CouldNotReadLWUMInputFile;
          $global_text = "input file with list of SG IPs not found";
        }
      }
    }

    if ($rc == 0) {
      # check the existence of the STMIX list from LWUM
      $path_to_stmix_list_file = $assistant_data_area . $path_to_stmix_list_file_from_lwum;
      if ( -f $path_to_stmix_list_file) {
        log_msg_info("using $path_to_stmix_list_file (precedence 1)");
      } else {
        $path_to_stmix_list_file = $assistant_gw_area . $path_to_stmix_list_file_from_lwum;
        if ( -f $path_to_stmix_list_file) {
          log_msg_info("using $path_to_stmix_list_file (precedence 2)");
        } else {
          $path_to_stmix_list_file = $path_to_stmix_list_file_from_lwum;
          if ( -f $path_to_stmix_list_file) {
            log_msg_info("using $path_to_stmix_list_file (obviously absolute path, precedence 3)");
          } else {
            log_msg_info("neither relative path nor absolute path working -> assuming list is empty, no STMIX boards");
          }
        }
      }
      if ( -f $path_to_stmix_list_file) {
        log_msg_info("concat the STMIX lists to the list of SGs, its the same instkit");
        concatFileAndMove($path_to_sg_list_file, $path_to_stmix_list_file, "processed");
      }
    }

    # check the existence of the sg_os_rlc dir in instmedia
    if ( ! -e $sg_os_rlc_dir) {
      $global_text = "Error: dir $sg_os_rlc_dir does not exist, transfer not possible";
      log_msg_info($global_text);
      $rc = $ERROR_SG_OS_RLC_DirDoesNotExists;
    }

    if ($rc == 0) {
      log_msg_info("enhance the ip list of all softgates with port info");
      my $uwe_script ="/opt/soco-common/prepare_ip_list_for_os_transfer.sh";
      if (! -e $uwe_script) {
        $global_text = "Error: script $uwe_script does not exists, maybe soco-common version too old";
        log_msg_info($global_text);
        $rc = $ERROR_generic_SG_OS_Update;
      } else {
        $rc = exec_cmd("/opt/soco-common/prepare_ip_list_for_os_transfer.sh $path_to_sg_list_file");
        if ($rc != 0) {
          $global_text = "script $uwe_script returned error, skipping generation";
          log_msg_info($global_text);
          $rc = $ERROR_generic_SG_OS_Update;
        } else {
          log_msg_info("script $uwe_script returned success, input file is also output file");
        }
      }
    }

    # check torrent
    if ($rc == 0) {

      my $torrent_diagnose = "";
      if (-f "/var/pgsql/Assistant/data/GW/sg_os.torrent") {
        $torrent_diagnose = exec_cmd_out ("ls -al /var/pgsql/Assistant/data/GW/sg_os.torrent");
        log_msg_info($torrent_diagnose);
      } 
      if (-f "/var/opt/Assistant/data/GW/sg_os.torrent") {
        $torrent_diagnose = exec_cmd_out ("ls -al /var/opt/Assistant/data/GW/sg_os.torrent");
        log_msg_info($torrent_diagnose);
      }
      if (-f "/instmedia/sg_os.torrent") {
        $torrent_diagnose = exec_cmd_out ("ls -al /instmedia/sg_os.torrent");
        log_msg_info($torrent_diagnose);
      }

      if ( ! -e $torrent_path ) {
        $global_text = "Warning: torrent file $torrent_path missing, trying to create it";
        log_msg_info($global_text);
      } else {
        log_msg_info("torrent file $torrent_path found, but anyway creating new one as it may contain node_B addresses when created in OLD world on V7 systems");
      }

      my $plt_version=get_current_real_version_without_hotfixnumber();
      my $do_in_background = 0;
      log_msg_info("creating torrent file in foreground as we need it for tar file creation");
      $rc = generate_torrent_file($plt_version, $do_in_background);

      # check torrent file again
      if ( $rc == 0 ) {
        if ( ! -e $torrent_path ) {
          $global_text = "Warning: torrent file $torrent_path missing, taking it out of the tar file generation";
          log_msg_info($global_text);
          $torrent_path = "";
          $torrent_file = "";
        } else {
          log_msg_info("torrent file found, using it for tar generation in the share");
          exec_cmd ("cp -vfa $torrent_path $torrent_path_share");
        }
      }
    }

    # copy the current upgrade daemon
    if ($rc == 0) {
      my $daemon_file_to_include = $working_dir . "swupdated";
      #copy the best and latest (and slowest :) ) version from YAPS for OS Update
      my $daemon_file_to_copy_from = get_swupdated_path_for_OS_update();
      exec_cmd("/bin/cp -vfa $daemon_file_to_copy_from $daemon_file_to_include");
    }

    # copy the MD5SUM file from instmedia
    if ($rc == 0) {
      exec_cmd("/bin/cp -vfa /instmedia/sg_os_rlc/MD5SUMS .");
    } else {
      log_msg_info("Warning: /instmedia/sg_os_rlc/MD5SUMS does not exist -> drop it from tar list");
    }

    if ($rc == 0) {
        # check if new credentials have to generated or the old ones can be used
        my $private_key = $private_os_ssh_key;
        my $public_key  = $private_key.".pub";
        if (-e $private_key and -e $public_key) {
          log_msg_info("private and public ssh credential files $private_key, $public_key already exists in working dir ");
          exec_cmd("ls -l $private_key");
          exec_cmd("ls -l $public_key");
          log_msg_info("But we cannot dare to keep them and use them as they may be from old Assistant share (OSFOURK-24029)");
          removeFile("$private_key");
          removeFile("$public_key");
        }
        log_msg_info("private and public ssh credential files $private_key, $public_key do not exist in working dir -> generate them");

        my $key_sentinel =  "os_update";
        my $user_id  = -1;             # 1003; # linuser
        my $group_id = 1000;           # hp4k_users
        $rc = prepare_sftp_host($key_sentinel, $sftp_transfer_user, \$user_id, $group_id);

        # per default no action for the node_B anymore as it would fail the complete operation for Duplex with only one node
        if ($isMultiNode == 1) {
          if ($work_on_remote == 1) {
            $remote_cmd = "$yaps_self prepare_sftp_host $key_sentinel $sftp_transfer_user nouid $group_id";
            my $output = "";

            log_msg_info("increasing timeout for ssh (lock) from $max_wait_for_ssh_lock_default to 180 seconds");
            $max_wait_for_ssh_lock_default = 180; # seconds

            $remote_rc = execute_remote_duplex_cmd($remote_cmd, \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
            if ($remote_rc == 0) {
              log_msg_info("Copy the exact .ssh/authorized_keys file from node_A to node_B to allow access with the same private key as on node_A");
              my $auth_keyfile = "/home/$sftp_transfer_user/.ssh/authorized_keys";
              $rc = execute_remote_duplex_copy($auth_keyfile, $auth_keyfile, "put");
            } else {
              log_msg_info("can not copy the exact .ssh/authorized_keys file from node_A to node_B but continuing the operation");
            }
          } else {
            log_msg_info("skip prepare_sftp_host on Duplex remote node (not needed)");
          }
        }
      }

    # prepare the SFTP ip adresses
    my $sftp_addresses_file = "sftp_addresses";
    if ($rc == 0) {
      my ($mgmg_ip1, $mgmg_ip2, $ipda_ip1, $ipda_ip2, $portal_ip) = ("", "", "", "", "");
      $mgmg_ip1 = get_local_mgmt_IP();
      $ipda_ip1 = get_local_IPDA_IP();
      if ($isMultiNode == 1 and $work_on_remote == 1 and $remote_rc == 0) {
        $ipda_ip2 =  retrieve_from_xml("ipda-lan.server-ip-address-node2");
      }
      $portal_ip =  retrieve_from_xml("customer-lan.ip-address");

      if ($mgmg_ip1 eq "dhcp") {
        log_msg_info("Management ip customer-lan.mgmt-node1-ip-address is set to $mgmg_ip1 -> use customer-lan.ip-address instead as fallback");
        $mgmg_ip1=$portal_ip;
      }

      # we do not want to deliver node_B / standby IPs to the SG as standby may not have the correct torrent file or SFTP credentials.
      # if ($mgmg_ip2 eq "dhcp") {
      #  log_msg_info("Management ip customer-lan.mgmt-node2-ip-address is set to $mgmg_ip2 -> do not use it");
      #  $mgmg_ip2="";
      # }

      if(open(SFTP_ADDR, ">$sftp_addresses_file")) {
        log_msg_info("Storing local mgmt ip address $mgmg_ip1 to $sftp_addresses_file");
        print SFTP_ADDR "$mgmg_ip1\n";
        # if (defined($mgmg_ip2) and length($mgmg_ip2) > 0) {
        #  log_msg_info("Since duplex storing ip address $mgmg_ip2 to $sftp_addresses_file");
        #  print SFTP_ADDR "$mgmg_ip2\n";
        # }
        # add now the IPDA addresses with - as delimiter
        log_msg_info("Storing local IPDA ip address $ipda_ip1 to $sftp_addresses_file");
        print SFTP_ADDR "-\n";
        print SFTP_ADDR "$ipda_ip1\n";
        # if (defined($ipda_ip2) and length($ipda_ip2) > 0) {
        #  log_msg_info("Since duplex storing ip address $ipda_ip2 to $sftp_addresses_file");
        #  print SFTP_ADDR "$ipda_ip2\n";
        # }
        close(SFTP_ADDR);
      }
    }

    # generate the tar.gz file and place it in working dir_to_check
    if ($rc == 0) {
      # missing !
      my $tar_file = "sg_os.tar.gz";

      my $cmd = 'export PATH=/usr/bin:/bin; ';
      $cmd .= 'tar -czf '.$tar_file.' '.$torrent_file.' `basename '.$path_to_sg_list_file.'` '.$sftp_addresses_file.' swupdated sshkey_sg_os_update sshkey_sg_os_update.pub ';
      if (-e "MD5SUMS") {
        $cmd .= ' MD5SUMS ';
      }
      if (-e 'ost-common') {
        log_msg_info("directory ost-common exists, adding to tar file");
        $cmd .= ' ost-common ';
      } else {
        log_msg_info("directory ost-common missing, skipping from tar file");
      }
      $cmd .= ' >> '.$log_file.' 2>&1 || exit 1; ';
      $rc = exec_cmd($cmd);

      if ($rc != 0) {
        $global_text = "generating tarfile $tar_file failed";
        log_msg_info($global_text);
        $rc = $ERROR_generic_SG_OS_Update;
      } else {
        log_msg_info("generating tarfile $tar_file succeeded");
        $local_text = "<path_to_transfer_file>GW/$tar_file</path_to_transfer_file>";
      }
      log_msg_info("protecting tarfile $tar_file to be readable by Assistant (world)");
      chmod(0664, $tar_file);
    }

    # transmission-daemon has to be insserved
    if ($rc == 0) {
      log_msg_info("before starting SG OS update the transmission-daemon has to be insserved in case that this central platform node reboots (will be undone later)");
      exec_cmd("/sbin/insserv transmission-daemon 2>/dev/null");
      if ($isMultiNode == 1) {
        if ($work_on_remote == 1) {
          my $output = "";
          $remote_cmd = "/sbin/insserv transmission-daemon 2>/dev/null";
          my $irc = execute_remote_duplex_cmd($remote_cmd, \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
          if ($irc != 0) {
            $global_text = "insserv transmission-daemon failed";
          }
        } else {
          log_msg_info("skip insserv transmission-daemon on Standby node");
        }
      }
    }
    # OSFOURK-25570: try to update the checkNecessaryFreeDiskSpaceForOSTransfer.sh on all STMIX boards (in case they are V10 R0 GA)
    patch_stmix_checkNecessaryFreeDiskSpaceForOSTransfer_file_via_inventory();

    remove_lock_file($sg_os_prep_lock_file);

  } else {
    $rc = $ERROR_CanNotGetLock;
    log_msg_info("generate_os_transfer_file_for_sg failed to get the lock after $lock_timeout seconds");
  }

  # generate XML response for Assistant
  my $xml_tag = $type_of_operation;
  my $do_bios_check = 0;
  my ($global_status, $local_status, $remote_status, $quorum_status) = ($rc, $rc, 0, 0);
  my $data_currentness = $DATA_CURRENTNESS_STATUS_irrelevant;

  write_extended_status($multiNodeType, $isMultiNode, $has_a_quorum, $xml_tag, $do_bios_check, $combined_log, $global_status, $global_text, $hostname, $local_status, $local_text, "node_a", $other_node, $remote_status, $remote_node_is_reachable, $remote_text, "node_b", $node_q, $quorum_status, $quorum_node_is_reachable, $quorum_text, "node_q", $data_currentness);

  my $content_xml = read_file_content($combined_log);
  print ("$content_xml\n");
    
  function_leave("generate_os_transfer_file_for_sg", "rc=$rc");
  return($rc);  
}

sub cleanup_after_os_sg_update {
  function_enter("cleanup_after_os_sg_update");
  my $rc = 0;

  determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 0);


  my $output = "";
  my ($other_node_is_reachable, $password_is_expired) = (0, 0);
 
  # remove the SSH credentials (private key id_rsa) and maybe restart ssh daemon
  cleanup_local_ssh_credentials($sftp_transfer_user);

  delete_linux_user($sftp_transfer_user);
  # the same for the node_B
  if ($isMultiNode == 1) {
    my $remote_cmd = "/usr/sbin/userdel -r --force $sftp_transfer_user";
    execute_remote_duplex_cmd($remote_cmd, \$output, 1, \$other_node_is_reachable, \$password_is_expired);
  }

  # remove the SSH credentials from tar creation directory, so that a new gen_tar command will create new ones
  my $private_key = $assistant_gw_area.$private_os_ssh_key;
  my $public_key  = $private_key.".pub";
  if (-e $private_key) {
    log_msg_info("deleting private ssh key file $private_key");
    removeFile($private_key);    
  }
  if (-e $public_key) {  
    log_msg_info("deleting public ssh key file $public_key");
    removeFile($public_key);
  }

  # undo the insserv_HA
  log_msg_info("since SG OS update has finished for all SGs, the insserv of the transmission daemon can be undone and stopped on both nodes");
  exec_cmd("/sbin/insserv -r transmission-daemon");
  exec_cmd("service transmission-daemon stop");
  if ($isMultiNode == 1) {
    execute_remote_duplex_cmd("/sbin/insserv -r transmission-daemon", \$output, 1, \$other_node_is_reachable, \$password_is_expired);
    execute_remote_duplex_cmd("service transmission-daemon stop", \$output, 1, \$other_node_is_reachable, \$password_is_expired);
  }

  function_leave("cleanup_after_os_sg_update", "rc=$rc");
  return($rc);   
}

sub cancel_os_transfer {
  function_enter("cancel_os_transfer", "");
  my $rc = 0;
  # check the os lock 
  if(-e $os_transfer_lock_file) {
    log_msg_info("os transfer action is probably currently running (pid file exists) -> proceed with cancel");
    log_msg_info("killing all rsync processes");
    $rc = exec_cmd("/usr/bin/killall -v rsync");
  } else {
    log_msg_info("no os transfer action is currently running -> cancel the cancel");
    $rc = $ERROR_SG_OS_TransferNotCancelled;
  } 
  
  function_leave("cancel_os_transfer", "rc=$rc");
  return($rc);
}

sub transfer_os_from_host {
  my $host   = shift;
  my $working_dir = shift;
  my $local_path = shift;
  my $remote_path = shift;
  function_enter("transfer_os_from_host", "host=$host workdir=$working_dir local_path=$local_path remote_path=$remote_path");
  my $rc = 0;

  # check the os lock before 
  if(is_running($os_transfer_lock_file) != 0) {
    log_msg_info("another os transfer action is currently running -> quit");
    return ($ERROR_generic_SG_OS_Update);
  }
  
  create_lock_file($os_transfer_lock_file);

  $rc = daemonize_me();
  if ($rc != 0) {
    log_msg_info("error when daemonizing, continuing and trying the best");
    $rc = 0;
  }

  log_msg_info("setting working dir to $working_dir");
  chdir $working_dir or $rc = $ERROR_generic_SG_OS_Update;

  my $skip_ssh_credential_generation=1;
  $node_reach_test_method = "none";
  my $provided_private_key_file = $working_dir."/sshkey_sg_os_update";
  my $node_is_reachable = 1;

  if ($rc == 0) {
    # the real transfer with provided private key file
    $rc = execute_remote_copy($host, \$node_is_reachable, $local_path, $remote_path, "get", $sftp_transfer_user, $provided_private_key_file);
  }

  if ($rc != 0) {
    log_msg_info("first SSH/rsync transfer from central host failed. Retry ...");
    exec_cmd ("ping -c 3 $host");
    exec_cmd ("/usr/sbin/traceroute $host");

    sleep (30); # 0.5 minutes
    # the second try transfer with provided private key file
    $rc = execute_remote_copy($host, \$node_is_reachable, $local_path, $remote_path, "get", $sftp_transfer_user, $provided_private_key_file);
  }

  if ($rc != 0) {
    log_msg_info("second SSH/rsync transfer from central host failed. Retry ...");
    exec_cmd ("ping -c 3 $host");
    exec_cmd ("/usr/sbin/traceroute $host");

    sleep (60); # 1 minute
    # the third try transfer with provided private key file
    $rc = execute_remote_copy($host, \$node_is_reachable, $local_path, $remote_path, "get", $sftp_transfer_user, $provided_private_key_file);
  }

  if ($rc != 0) {
    log_msg_info("third SSH/rsync transfer from central host failed. Retry ...");
    exec_cmd ("ping -c 3 $host");
    exec_cmd ("/usr/sbin/traceroute $host");

    sleep (300); # 5 minutes
    # the fourth try transfer with provided private key file
    $rc = execute_remote_copy($host, \$node_is_reachable, $local_path, $remote_path, "get", $sftp_transfer_user, $provided_private_key_file);
  }

  # check the MD5sum in case of success
  if ($rc == 0) {
    my $md5sum_file="$local_path". "MD5SUMS"; # /instmedia/sg_os_rlc/MD5SUMS
    my $md5_rc = verify_md5sum($local_path, $md5sum_file);
    if ($md5_rc != 0) {
      log_msg_info("md5sum check failed ($md5_rc), that means complete transfer failed");
      $rc = $md5_rc;
    } else {
      log_msg_info("md5sum check succeeded ($md5_rc), that means complete transfer suceeded");
    }
  }

  # notify Uwe that the transfer is done or failed
  exec_cmd("/opt/soco-common/exec_after_os_transfer.sh -F $rc");

  remove_lock_file($os_transfer_lock_file);
  function_leave("transfer_os_from_host", "rc=$rc");
  return($rc);
}

sub verify_md5sum {
  my $working_dir = shift;
  my $md5sum_file = shift;
  function_enter("verify_md5sum", "working_dir=$working_dir md5sum_file=$md5sum_file");
  my $rc = 0;
  if (-e $md5sum_file ) {
    log_msg_info("md5sum file $md5sum_file exist, use it for verification");
    log_msg_info("md5sum file $md5sum_file contains absolute pathnames, therefore we can skip cd to $working_dir");
    $rc = exec_cmd("md5sum -c $md5sum_file");
  } else {
    log_msg_info("md5sum file $md5sum_file does not exist, no verification possible. Assuming transfer success");
    $rc = 0;
  }

  function_leave("verify_md5sum", "rc=$rc");
  return($rc);
}

sub set_NUC_SG_OS_state {
  my $state = shift;
  my $error_cond = shift;
  function_enter("set_NUC_SG_OS_state", "state=$state");
  my $rc = 0;
  my $cmd = "$nuc_sg_os_state_script";
  if (defined($error_cond)) {
    $cmd .= " -e $error_cond";
  }
  $cmd .= " -s $state";

  $rc = exec_cmd($cmd);

  function_leave("set_NUC_SG_OS_state");
  return($rc);  
}

sub generate_sg_version_file_for_dashboard {
  my $sg_version_local = shift;
  my $sg_version_remote = shift;
  my $sg_version_quorum = shift;
  function_enter("generate_sg_version_file_for_dashboard", "$sg_version_local / $sg_version_remote / $sg_version_quorum");
  my $rc = 0;
  my $sg_version_file_for_dashboard = "$webservice_log_dir"."sg_version_file_for_dashboard.info";
  
  # format the display string for Assistant dashboard
  my $sg_version_content = "$sg_version_local";
  if ($sg_version_remote ne "") {
    $sg_version_content = $sg_version_content . " / " . $sg_version_remote;
  }
  if ($sg_version_quorum ne "") {
    $sg_version_content = $sg_version_content . " / " . $sg_version_quorum;
  }
  
  create_generic_file($sg_version_content, $sg_version_file_for_dashboard);

  function_leave("generate_sg_version_file_for_dashboard", "sg_version_content=$sg_version_content");
  return($rc);
}

sub is_cachefile_still_valid {
  my $cachefile = shift;
  my $deltatime_in_secs = shift;
  my $rc = 0;
  function_enter("is_cachefile_still_valid", "cachefile=$cachefile deltatime_in_secs=$deltatime_in_secs");
  my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($cachefile);
  if (!defined($mtime) || $mtime eq "") {
    log_msg_info("could not retrieve mtime from stat()");
    $rc = 0;
  } else {
    my $cur_time = time();
    log_msg_info("$cachefile mtime=$mtime cur_time=$cur_time");
    my $oldness = $cur_time - $mtime;
    if ($oldness > $deltatime_in_secs) {
      log_msg_info("oldness=$oldness (sec) is bigger than threshold $deltatime_in_secs (sec)");
      $rc = 0;
    } else {
      log_msg_info("oldness=$oldness (sec) is smaller or equals than threshold $deltatime_in_secs (sec) ");
      $rc = 1;
    }
  }

  function_leave("is_cachefile_still_valid", "rc=$rc (0 is not valid anymore, 1 is still valid)");
  return($rc);
}

sub data_retrieval_template {
  my $type_of_operation = shift;
  my $xml_tag = shift;
  my $lockfile_to_use = shift;
  my $do_bios_check  = shift;
  my $operate_on_quorum = shift;
  my $local_cmd_ref  = shift;
  my $remote_cmd_ref = shift;
  
  function_enter("data_retrieval_template", "type=$type_of_operation xml_tag=$xml_tag do_bios_check=$do_bios_check operate_on_quorum=$operate_on_quorum $local_cmd_ref $remote_cmd_ref");

  my ($rc) = (0);
  my ($local_status, $remote_status, $quorum_status, $global_status) = (0, 0, 0, 0);
  my ($sg_version_local, $sg_version_remote, $sg_version_quorum) = ("", "", "");
  my $remote_log   = $webservice_log_dir."$type_of_operation"."_remote.log";
  my $local_log    = $webservice_log_dir."$type_of_operation"."_local.log";
  my $quorum_log   = $webservice_log_dir."$type_of_operation"."_quorum.log";
  my $combined_log = $webservice_log_dir."$type_of_operation"."_all.log";
  my ($global_text, $remote_text, $remote_node_is_reachable, $remote_node_password_is_expired, $local_text, $quorum_text, $quorum_node_is_reachable, $quorum_node_password_is_expired) = ("", "", 1, 0, "", "", 1, 0);
  my ($lock_timeout, $ssh_timeout, $ssh_timeout_retry, $got_the_lock_immediately, $skip_retrieval) = (70, 30, 10, 0, 0);
  my $data_currentness = $DATA_CURRENTNESS_STATUS_up_to_date;
  my $content_xml = "";

  $is_dscxl   = dscxl_discover();
  $is_cots    = cots_discover();
  $is_virtual = virtual_discover();
  $is_manager = manager_discover();

  determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 1);
  log_msg_info("$type_of_operation rc=$rc multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q");
      
  $rc = wait_for_lock ($lockfile_to_use, $lock_timeout, $type_of_operation, \$got_the_lock_immediately);
  if ($rc == 0) {
    create_lock_file($lockfile_to_use);

    if (-e "$combined_log") {
      log_msg_info("cached file exists: $combined_log");
      if ($got_the_lock_immediately == 0 ) {
        $skip_retrieval = 1;
        log_msg_info("I had to wait for the lock, therefore I will immediately take the existing log, which has to be still consistent");
      } else {
         log_msg_info("I didn't wait for the lock, check oldness of the cachefile");
         my $oldness_allowed_threshold = 10; # in seconds
         if (is_cachefile_still_valid($combined_log, $oldness_allowed_threshold) == 1) {
           log_msg_info("cachefile still new enough, use it");
           $skip_retrieval = 1;
         } else {
           log_msg_info("cachefile too old, generate new one");
         }
      }
    }

    if ($skip_retrieval == 0) {

      if ($isMultiNode == 0) {
        log_msg_info("not running in redundancy scenario - no $type_of_operation on remote nodes performed");
      }

      if ( $isMultiNode != 0) {

        # set the method for the reachability test before curl commands to save time (prevent timeouts in Assistant rcon)
        $node_reach_test_method = "crm"; # "ping"
        log_msg_info("setting reachability test method: $node_reach_test_method for current process");

        $rc = wait_for_ssh_lock($sshd_lock_file, $other_node, $ssh_timeout);
        if ($rc == 0) {
          create_lock_file($sshd_lock_file);

          removeFile($remote_log);
          $remote_status = $remote_cmd_ref->($other_node, $remote_log, \$remote_node_is_reachable, \$remote_node_password_is_expired, $ssh_timeout, 0);
          $remote_text = read_file_content($remote_log);
          
          # plausy check
          if ($remote_text eq "" or $remote_status != 0 or $remote_node_is_reachable == 0) {
            log_msg_info("obviously file $remote_log does not exist or is empty or status ($remote_status) failed or node not reachable ($remote_node_is_reachable) -> retry");
            sleep(1);
                       
            $remote_status = $remote_cmd_ref->($other_node, $remote_log, \$remote_node_is_reachable, \$remote_node_password_is_expired, $ssh_timeout_retry, 0);
            $remote_text = read_file_content($remote_log);
          }

          if ($remote_node_is_reachable == 0) {
            log_msg_info("forcing remote_status to $ERROR_CanNotReachRemoteNode due to not reachable");
            $remote_status = $ERROR_CanNotReachRemoteNode;
          }
          if ($remote_node_password_is_expired == 1) {
            log_msg_info("forcing remote_status to $ERROR_RootPwdExpired due to expired password");
            $remote_status = $ERROR_RootPwdExpired;
          }
          
          remove_lock_file($sshd_lock_file);
        } else {
          log_msg_info("did not get the remote lock $sshd_lock_file");
        }

        if ($rc != 0) {
          log_msg_info("didnt get the remote lock ->  checking for cached data");
          if (-e $remote_log ) {
            log_msg_info("cached data file $remote_log found -> using it");
            $remote_status    = 0; # this is seen as no error
            if ($rc != 0 ) { $data_currentness = $DATA_CURRENTNESS_STATUS_cached_sshlock; } elsif ($remote_node_is_reachable == 0) { $data_currentness = $DATA_CURRENTNESS_STATUS_cached_connectivity; } else { $data_currentness = $DATA_CURRENTNESS_STATUS_cached_misc; }
            $remote_text = read_file_content($remote_log);
          } else {
            log_msg_info("cached data file $remote_log not found -> still an error");
            $data_currentness = $DATA_CURRENTNESS_STATUS_irrelevant;
            if ($rc != 0) {
              $remote_status = $ERROR_CanNotGetLock;
            }
          }
        }
        if ($rc == 0) {
          if ( $type_of_operation eq "get_version") {
            $sg_version_remote = parse_xml_tag("version_of_sg_lw", $remote_text);
          }
        }
      }

      if ($operate_on_quorum == 1 and has_quorum($node_q, $multiNodeType) == 1) {
        $has_a_quorum = 1;

        $rc = wait_for_ssh_lock($sshq_lock_file, $node_q, $ssh_timeout);
        if ($rc == 0) {
          create_lock_file($sshq_lock_file);

          removeFile($quorum_log);
          $quorum_status = $remote_cmd_ref->($node_q, $quorum_log, \$quorum_node_is_reachable, \$quorum_node_password_is_expired, $ssh_timeout, 1);
          $quorum_text = read_file_content($quorum_log);

          # plausy check
          if ($quorum_text eq "" or $quorum_status != 0 or $quorum_node_is_reachable == 0) {
            log_msg_info("obviously file $quorum_log does not exist or is empty or status ($quorum_status) failed or node not reachable ($quorum_node_is_reachable) -> retry");
            sleep(1);

            $quorum_status = $remote_cmd_ref->($node_q, $quorum_log, \$quorum_node_is_reachable, \$quorum_node_password_is_expired, $ssh_timeout_retry, 1);
            $quorum_text = read_file_content($quorum_log);
          }

          if ($quorum_node_is_reachable == 0) {
            log_msg_info("forcing quorum_status to $ERROR_CanNotReachRemoteNode due to not reachable");
            $quorum_status = $ERROR_CanNotReachRemoteNode;
          }
          if ($quorum_node_password_is_expired == 1) {
            log_msg_info("forcing quorum_status to $ERROR_RootPwdExpired due to expired password");
            $quorum_status = $ERROR_RootPwdExpired;
          }

        #  if ($testmode != 0) {
        #    log_msg_info("!!! testmode -> forcing quorum_status to $ERROR_RootPwdExpired to simulate expired password");
        #    $quorum_status = $ERROR_RootPwdExpired;
        #    $quorum_node_is_reachable = 0;
        #  }
          
          remove_lock_file($sshq_lock_file);
        } else {
          log_msg_info("did not get the remote lock $sshq_lock_file");
        }

        if ($rc != 0 ) {
          log_msg_info("didnt get the remote quorum lock ->  checking for cached data");

          if (-e $quorum_log ) {
            log_msg_info("cached data file $quorum_log found -> using it");
            $quorum_status    = 0; # this is seen as no error
            if ($rc != 0 ) { $data_currentness = $DATA_CURRENTNESS_STATUS_cached_sshlock; } elsif ($quorum_node_is_reachable == 0) { $data_currentness = $DATA_CURRENTNESS_STATUS_cached_connectivity; } else { $data_currentness = $DATA_CURRENTNESS_STATUS_cached_misc; }
            $quorum_text = read_file_content($quorum_log);
          } else {
            log_msg_info("cached data file $quorum_log not found -> still an error");
            $data_currentness = $DATA_CURRENTNESS_STATUS_irrelevant;
            if ($rc != 0) {
              $quorum_status = $ERROR_CanNotGetLock;
            }
          }
        }
        if ($rc == 0 ) {
          if ( $type_of_operation eq "get_version") {
            $sg_version_quorum = parse_xml_tag("version_of_sg_lw", $quorum_text);
          }
        }
      }

      # do the local retrieval even if remote retrieval fails
      $local_status = $local_cmd_ref->($local_log);
      $local_text  = read_file_content($local_log);
      # plausy check
      if ($local_text eq "" or $local_status != 0) {
        log_msg_info("obviously file $local_log does not exist or is empty or status failed -> retry");
        sleep(1);
        $local_status = $local_cmd_ref->($local_log);
        $local_text  = read_file_content($local_log);
      }

      if ( $type_of_operation eq "get_version") {
        $sg_version_local = parse_xml_tag("version_of_sg_lw", $local_text);
        generate_sg_version_file_for_dashboard($sg_version_local, $sg_version_remote, $sg_version_quorum);
      }
      
      if ($quorum_status != 0) { $global_status = $quorum_status}
      if ($remote_status != 0) { $global_status = $remote_status}
      if ($local_status != 0)  { $global_status = $local_status}
      if (($global_status != 0) and ($remote_node_is_reachable == 0 or $quorum_node_is_reachable == 0)) {
        if ($global_status == $ERROR_RootPwdExpired) {
          $global_text = $ERROR_text{$ERROR_RootPwdExpired};
        } else {
          $global_text = $ERROR_text{$ERROR_CanNotReachRemoteNode};
        }
      }

      write_extended_status($multiNodeType, $isMultiNode, $has_a_quorum, $xml_tag, $do_bios_check, $combined_log, $global_status, $global_text, $hostname, $local_status, $local_text, "node_a", $other_node, $remote_status, $remote_node_is_reachable, $remote_text, "node_b", $node_q, $quorum_status, $quorum_node_is_reachable, $quorum_text, "node_q", $data_currentness);
    }

    $content_xml = read_file_content($combined_log);
    print ("$content_xml\n");
    if ($skip_retrieval == 1) {
      # in case of no retrieval I need additional logging to see the content
      log_msg_info("output=$content_xml");
    }

    remove_lock_file($lockfile_to_use);
  } else {
    log_msg_info("$type_of_operation failed to get the lock after $lock_timeout seconds");

    $global_status = $ERROR_CanNotGetLock;
    $do_bios_check = 0;
    $data_currentness = $DATA_CURRENTNESS_STATUS_irrelevant;
    $combined_log .= ".$pid";

    write_extended_status($multiNodeType, $isMultiNode, $has_a_quorum, $xml_tag, $do_bios_check, $combined_log, $global_status, $global_text, $hostname, $local_status, $local_text, "node_a", $other_node, $remote_status, $remote_node_is_reachable, $remote_text, "node_b", $node_q, $quorum_status, $quorum_node_is_reachable, $quorum_text, "node_q", $data_currentness);
    
    $content_xml = read_file_content($combined_log);
    print ("$content_xml\n");
  }

  function_leave("data_retrieval_template");
  return($global_status);
}

sub get_upd_status {
  function_enter("get_upd_status");

  # no lock or data_retrieval_template necessary since only read operations
  my $rc = 0;
  my $is_default = 1;
  my $content_xml =
"<date>0000-00-00 00:00:00</date>
<errortext>no update performed</errortext>
<rc>1</rc>";

  if (-e $status_file) {
    log_msg_info("found upgrade status file: $status_file -> reading it");
    $content_xml = read_file_content($status_file);
    $is_default = 0;    
  } else {
    log_msg_info("did not find upgrade status file: $status_file -> answering with default response: <rc>1</rc>");
    $is_default = 1;
  }

  my $response = "<wbm>\n<SWCONF>\n$content_xml\n</SWCONF>\n</wbm>\n";
  print ($response);

  if ($is_default == 0) {
    # print diagnosis
    log_msg_info("response=$response");
  }

  function_leave("get_upd_status", "rc=$rc");
  return($rc);
}

sub get_version_info {
  function_enter("get_version_info");

  my ($rc, $do_bios_check, $operate_on_quorum) = (0, 1, 1);

  $rc = data_retrieval_template( "get_version", "version_info", $version_lock_file, $do_bios_check, $operate_on_quorum, \&get_version_info_local, \&get_version_info_remote);

  # always return 0 for Assistant
  $rc = 0;
  function_leave("get_version_info", "rc=$rc");
  return($rc);
}

sub get_version_info_local {
  my $local_log = shift;
  my $rc = 0;
  
  function_enter("get_version_info_local", "local_log=$local_log");

  my $firstinst_netw_local_log_name = $webservice_log_dir."firstinst_netw_check_local.log";
  my $nuc_prep_local_log_name       = $webservice_log_dir."nuc_prep_status_checkoutput_local.log";

  removeFile($local_log);
  removeFile($firstinst_netw_local_log_name);
  removeFile($nuc_prep_local_log_name);

  my $cmd = "$get_version_info_script $local_log $firstinst_netw_local_log_name $nuc_prep_local_log_name";
  $rc = exec_cmd($cmd);
  
  function_leave("get_version_info_local", "rc=$rc");
  return($rc);
}

sub get_version_info_remote {
  my $node = shift;
  my $remote_log = shift;
  my $node_is_reachable_ref = shift;
  my $password_is_expired_ref = shift;
  my $ssh_timeout = shift;
  my $is_quorum = shift;
  
  
  my $cmd_rc = 0;
  my $copy_rc = 0; 
  my $prepare_rc = 0;
  my $rc = 0;
  my $cmd = "";
  my $output = "";
  my ($ignore_remote_return_code, $suppress_diagnosis_output, $skip_sshkeygen, $skip_sshkeycleanup, $provided_private_key_file, $user) = (0,0,0,0,"","root");

  function_enter("get_version_info_remote", "node=$node remote_log=$remote_log timeout=$ssh_timeout is_quorum=$is_quorum");

  my $firstinst_netw_remote_log_name = "";
  my $nuc_prep_remote_log_name = "";

  if ($is_quorum == 1) {
    $firstinst_netw_remote_log_name = $webservice_log_dir."firstinst_netw_check_quorum.log";
    $nuc_prep_remote_log_name       = $webservice_log_dir."nuc_prep_status_checkoutput_quorum.log";
  } else {
    $firstinst_netw_remote_log_name = $webservice_log_dir."firstinst_netw_check_remote.log";
    $nuc_prep_remote_log_name       = $webservice_log_dir."nuc_prep_status_checkoutput_remote.log";
  }
  
  my $remote_cmd = "$get_version_info_script $remote_log $firstinst_netw_remote_log_name $nuc_prep_remote_log_name";
  $skip_sshkeycleanup = 1;
  $rc = execute_remote_cmd($node, $node_is_reachable_ref, $password_is_expired_ref, $remote_cmd, \$output, 1, $ssh_timeout, $ignore_remote_return_code, $suppress_diagnosis_output, $skip_sshkeygen, $skip_sshkeycleanup);

  if ($rc == 0 && $$node_is_reachable_ref != 0) {
    removeFile($remote_log);
    $skip_sshkeygen = 1;
    $skip_sshkeycleanup = 1;
    $copy_rc = execute_remote_copy($node, $node_is_reachable_ref, $remote_log, $remote_log, "get", $user, $provided_private_key_file, $skip_sshkeygen, $skip_sshkeycleanup);

    if ($rc == 0 and $copy_rc != 0) {
      # ssh sucess, but rsync error -> try again with keygen
      log_msg_info("error on copy, retry with keygen");
      $skip_sshkeygen = 0;
      $copy_rc = execute_remote_copy($node, $node_is_reachable_ref, $remote_log, $remote_log, "get", $user, $provided_private_key_file, $skip_sshkeygen, $skip_sshkeycleanup);

      if ($rc == 0 and $copy_rc != 0) {
        log_msg_info("enforcing the erroneous copy_rc=$copy_rc to rc");
        $rc = $copy_rc;
      }
    }

    # copy additionally also the nuc_prep check status files, but do not evaluate the rc
    removeFile($nuc_prep_remote_log_name);
    $skip_sshkeygen = 1;
    $skip_sshkeycleanup = 1;
    execute_remote_copy($node, $node_is_reachable_ref, $nuc_prep_remote_log_name, $nuc_prep_remote_log_name, "get", $user, $provided_private_key_file, $skip_sshkeygen, $skip_sshkeycleanup);

    # copy additionally also the firstinst_netw check status files, but do not evaluate the rc
    removeFile($firstinst_netw_remote_log_name);
    $skip_sshkeygen = 1;
    $skip_sshkeycleanup = 0;
    execute_remote_copy($node, $node_is_reachable_ref, $firstinst_netw_remote_log_name, $firstinst_netw_remote_log_name, "get", $user, $provided_private_key_file, $skip_sshkeygen, $skip_sshkeycleanup);

  }
  function_leave("get_version_info_remote", "on $node finished rc=$rc node_is_reachable=$$node_is_reachable_ref password_is_expired=$$password_is_expired_ref copy_rc=$copy_rc");
  return($rc);
}  

sub get_drbd_status {
  my $is_duplex = shift;
  my $host_b_reachable = shift;
  function_enter("get_drbd_status", "is_duplex=$is_duplex host_b_reachable=$host_b_reachable ");
  
  # default is in sync (value = 0) and connected (value = 0))
  my ($drbd_sync_status, $drbd_connect_status) = (0, 0);

#  if (-e "/var/run/soco2/drbd_out_of_sync" ) {
#    log_msg_info("get_drbd_status file /var/run/soco2/drbd_out_of_sync exists -> forcing drbd_status to 1");
#    $drbd_status = 1;
#  }
#  if (-e "/var/run/soco2/drbd_synced" ) {
#    log_msg_info("get_drbd_status file /var/run/soco2/drbd_out_of_sync exists -> forcing drbd_status to 0");
#    $drbd_status = 0;
#  }

  my $drbd_status_file = "/proc/drbd"; #  "/tmp/drbd"

  log_msg_info("checking $drbd_status_file");
  if(open(DRBDSTAT, "<$drbd_status_file")) {
    my @drbd_info = <DRBDSTAT>;
    my $line = "";
    my $pattern = "SyncSource";

    foreach(@drbd_info) {
      $line = my_trim($_);
      log_msg("drbd: $line");

      $pattern = "SyncSource";
      if ( $_ =~ /$pattern/) {
        log_msg_info("found trigger word $pattern");
        $drbd_sync_status = 1;
      }

      $pattern = "SyncTarget";
      if ( $_ =~ /$pattern/) {
        log_msg_info("found trigger word $pattern");
        $drbd_sync_status = 1;
      }

      $pattern = "WFConnection";
      if ( $_ =~ /$pattern/) {
        log_msg_info("found trigger word $pattern as requested by I45717");
        $drbd_connect_status = 1;
      }       
    }
    close(DRBDSTAT);
  }  else {
      log_msg_info("drbd_status can not be retrieved because file $drbd_status_file does not exist -> assuming consistent");
      ($drbd_sync_status, $drbd_connect_status) = (0, 0);
  }

  
  if ($is_duplex == 0 and $drbd_sync_status != 0) {
    log_msg_info("get_drbd_status is $drbd_sync_status but deployment is simplex or mono -> drbd status seems to be inconsistent (maybe due to memory tuning) -> forcing it to 0");
      ($drbd_sync_status, $drbd_connect_status) = (0, 0);
  }

  if ($is_duplex == 1 and ($drbd_sync_status != 0 or $drbd_connect_status != 0)) {
    log_msg_info("drbd_status is $drbd_sync_status,$drbd_connect_status for duplex ->  check if other node is reachable ");
    if ($host_b_reachable == 0)  {
      log_msg_info("get_drbd_status: other node is NOT reachable, probably Duplex with one node-> setting drbd state to 0, the health check should report the error");
      ($drbd_sync_status, $drbd_connect_status) = (0, 0);
    } else {
      log_msg_info("get_drbd_status: other node IS reachable -> use the drbd status from $drbd_status_file");
    }
  }

  function_leave("get_drbd_status", "drbd_sync_status=$drbd_sync_status drbd_connect_status=$drbd_connect_status");
  return ($drbd_sync_status, $drbd_connect_status);
}  

sub get_timezone {
  function_enter("get_timezone");

  my ($rc, $do_bios_check, $operate_on_quorum) = (0, 0, 1);

  $rc = data_retrieval_template( "get_timezone", "get_timezone", $gettz_lock_file, $do_bios_check, $operate_on_quorum, \&get_timezone_local, \&get_timezone_remote);

  # always return 0 for Assistant
  $rc = 0;

  function_leave("get_timezone", "rc=$rc");
  return($rc);
}

sub get_timezone_local {
  my $local_log = shift;
  my $rc = 0;

  function_enter("get_timezone_local", "local_log=$local_log");

  removeFile($local_log);
  my $cmd = "$get_timezone_script $local_log";
  $rc = exec_cmd($cmd);

  function_leave("get_timezone_local", "rc=$rc");
  return($rc);
}

sub get_timezone_remote {
  my $node = shift;
  my $remote_log = shift;
  my $node_is_reachable_ref = shift;
  my $password_is_expired_ref = shift;
  my $ssh_timeout = shift;
  my $is_quorum = shift;

  my $cmd_rc = 0;
  my $copy_rc = 0;
  my $prepare_rc = 0;
  my $rc = 0;
  my $cmd = "";
  my $output = "";

  my ($ignore_remote_return_code, $suppress_diagnosis_output, $skip_sshkeygen, $skip_sshkeycleanup, $provided_private_key_file, $user) = (0,0,0,0,"","root");


  function_enter("get_timezone_remote", "on node $node timeout=$ssh_timeout remote_log=$remote_log");
    
  my $remote_cmd = "$get_timezone_script $remote_log";
  $skip_sshkeycleanup = 1;
  $rc = execute_remote_cmd($node, $node_is_reachable_ref, $password_is_expired_ref, $remote_cmd, \$output, 1, $ssh_timeout, $ignore_remote_return_code, $suppress_diagnosis_output, $skip_sshkeygen, $skip_sshkeycleanup);

  if ($rc == 0 && $$node_is_reachable_ref != 0) {
    removeFile($remote_log);
        $skip_sshkeygen = 1;
    $skip_sshkeycleanup = 0;
    $copy_rc = execute_remote_copy($node, $node_is_reachable_ref, $remote_log, $remote_log, "get", $user, $provided_private_key_file, $skip_sshkeygen, $skip_sshkeycleanup);

    if ($rc == 0 and $copy_rc != 0) {
      log_msg_info("enforcing the erroneous copy_rc=$copy_rc to rc");
      $rc = $copy_rc;
    }
  }

  function_leave("get_timezone_remote", "on $node finished rc=$rc copy_rc=$copy_rc node_is_reachable=$$node_is_reachable_ref password_is_expired=$$password_is_expired_ref");
  return($rc);   
}

sub set_timezone {
  my $tzone = shift;
  function_enter("set_timezone", "tzone=$tzone");

  my $rc = 0;
  my $remote_rc = 0;
  my ($local_status, $remote_status, $quorum_status) = (0, 0, 0);
  my $wait_rc = 0;
  my $quorum_log   = $webservice_log_dir.'set_timezone_quorum.log';
  my $remote_log   = $webservice_log_dir.'set_timezone_remote.log';
  my $local_log    = $webservice_log_dir.'set_timezone_local.log';
  my $combined_log = $webservice_log_dir.'set_timezone_all.log';
  my ($text, $remote_text, $local_text, $quorum_text) = ("", "", "", "");
  my ($lock_timeout, $got_the_lock_immediately) = (30, 0);
  my $data_currentness = $DATA_CURRENTNESS_STATUS_irrelevant;
  my ($content_xml, $toplevel_tag) = ("", "set_timezone");
  

  $rc = determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 1);
  log_msg_info("set_timezone rc=$rc multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q");
    
  $rc = wait_for_lock ($settz_lock_file, $lock_timeout, "set_timezone", \$got_the_lock_immediately);
  if ($rc == 0) {
    create_lock_file($settz_lock_file);

    if ($isMultiNode == 0) {
      log_msg_info("not running in redundancy scenario - no set_timezone on remote nodes performed");
      $remote_rc = 0;
    }
    
    if ( $isMultiNode != 0) {
      $rc = wait_for_ssh_lock($sshd_lock_file, $other_node);      
      if ($rc == 0) {
        create_lock_file($sshd_lock_file);

        $remote_status = set_timezone_remote($other_node, $remote_log, $tzone);
        $remote_text = read_file_content($remote_log);

        # plausy check
        if ($remote_text eq "" or $remote_status != 0) {
          log_msg_info("obviously file $remote_log does not exist or is empty or status failed -> retry");
          sleep(1);
          $remote_status = set_timezone_remote($other_node, $remote_log, $tzone);
          $remote_text = read_file_content($remote_log);
        }
        
        remove_lock_file($sshd_lock_file);
      } else {
        log_msg_info("did not get the remote lock $sshd_lock_file");
        $remote_status = $rc;
      }
    }

    if (has_quorum($node_q, $multiNodeType) == 1) {
      $has_a_quorum = 1;
      $rc = wait_for_ssh_lock($sshq_lock_file, $node_q);
      if ($rc == 0) {
        create_lock_file($sshq_lock_file);
        
        $quorum_status = set_timezone_remote($node_q, $quorum_log, $tzone);
        $quorum_text = read_file_content($quorum_log);

        # plausy check
        if ($quorum_text eq "" or $quorum_status != 0) {
          log_msg_info("obviously file $quorum_log does not exist or is empty or status failed -> retry");
          sleep(1);

          $quorum_status = set_timezone_remote($node_q, $quorum_log, $tzone);
          $quorum_text = read_file_content($quorum_log);
        }
        
        remove_lock_file($sshq_lock_file);
      } else {
        log_msg_info("did not get the remote lock $sshq_lock_file");
        $quorum_status = $rc;
      }
    }

    if ($rc == 0) {
      # do the local retrieval even if remote retrieval fails
      removeFile($local_log);
      my $cmd = "$set_timezone_script $local_log $tzone";
      $local_status = exec_cmd($cmd);
      $local_text  = read_file_content($local_log);

      # plausy check
      if ($local_text eq "" or $local_status != 0) {
        log_msg_info("obviously file $local_log does not exist or is empty or status failed -> retry");
        sleep(1);

        $local_status = exec_cmd($cmd);
        $local_text  = read_file_content($local_log);
        }
    }

    write_extended_status($multiNodeType, $isMultiNode, $has_a_quorum, $toplevel_tag, 0, $combined_log, $rc, $text, $hostname, $local_status, $local_text, "node_a", $other_node, $remote_status, 1, $remote_text, "node_b", $node_q, $quorum_status, 1, $quorum_text, "node_q", $data_currentness);

    $content_xml = read_file_content($combined_log);
    print ("$content_xml\n");

    remove_lock_file($settz_lock_file);
  } else {
    log_msg_info("set_timezone failed to get the lock after $lock_timeout seconds");
    $rc = $ERROR_CanNotGetLock;
    $combined_log .= ".$pid";

    write_extended_status($multiNodeType, $isMultiNode, $has_a_quorum, $toplevel_tag, 0, $combined_log, $rc, $text, $hostname, $local_status, $local_text, "node_a", $other_node, $remote_status, 1, $remote_text, "node_b", $node_q, $quorum_status, 1, $quorum_text, "node_q", $data_currentness);

    $content_xml = read_file_content($combined_log);
    print ("$content_xml\n");     
  }

  function_leave("set_timezone", "tzone=$tzone rc=$rc");
  
  return($rc);
}

sub set_timezone_remote {
  my $node = shift;
  my $remote_log = shift;
  my $tzone = shift;

  my $cmd_rc = 0;
  my $copy_rc = 0;
  my $prepare_rc = 0;
  my $rc = 0;
  my $cmd = "";
  my $output = "";
  my ($node_is_reachable, $password_is_expired) = (0, 0);

  function_enter("set_timezone_remote", "on node $node remote_log=$remote_log tzone=$tzone");

  my $remote_cmd = "$set_timezone_script $remote_log $tzone";
  # $rc = execute_remote_duplex_cmd($remote_cmd, \$output, 1, \$other_node_is_reachable);
  $rc = execute_remote_cmd($node, \$node_is_reachable, \$password_is_expired, $remote_cmd, \$output, 1, 60);

  removeFile($remote_log);
  $copy_rc = execute_remote_copy($node, \$node_is_reachable, $remote_log, $remote_log, "get");

  if ($rc == 0 and $copy_rc != 0) {
    log_msg_info("enforcing the erroneous copy_rc=$copy_rc to rc");
    $rc = $copy_rc;
  }
  
  function_leave("set_timezone_remote", "on $node finished rc=$rc copy_rc=$copy_rc");
  return($rc);
  }
  
sub set_ui_mode {
  my $mode = shift;
  my $tls_version = shift;
  function_enter("set_ui_mode", "mode=$mode, tls_version=$tls_version");

  my $rc = 0; 
  my $remote_rc = 0;
  my ($local_status, $remote_status, $quorum_status) = (0, 0, 0);
  my $wait_rc = 0;
  my $quorum_log   = $webservice_log_dir.'set_ui_mode_quorum.log';
  my $remote_log   = $webservice_log_dir.'set_ui_mode_remote.log';
  my $local_log    = $webservice_log_dir.'set_ui_mode_local.log';  
  my $combined_log = $webservice_log_dir.'set_ui_mode_all.log';
  my ($text, $remote_text, $local_text, $quorum_text) = ("", "", "", "");
  my ($lock_timeout, $got_the_lock_immediately) = (30, 0);
  my $data_currentness = $DATA_CURRENTNESS_STATUS_irrelevant; 
  my ($content_xml, $toplevel_tag) = ("", "set_ui_mode");

  $rc = determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 1);
  log_msg_info("set_ui_mode rc=$rc multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q");

  $rc = wait_for_lock ($setui_lock_file, $lock_timeout, "set_ui_mode", \$got_the_lock_immediately);
  if ($rc == 0) {

    create_lock_file($setui_lock_file);  

    if ($isMultiNode == 0) {
      log_msg_info("not running in redundancy scenario - no set_ui_mode on remote nodes performed"); 
      $remote_rc = 0;
    }

    if ( $isMultiNode != 0) {

      $rc = wait_for_ssh_lock($sshd_lock_file, $other_node);
      if ($rc == 0) {
        create_lock_file($sshd_lock_file);
        
        $remote_status = set_ui_mode_remote($other_node, $remote_log, $mode, $tls_version);
        $remote_text = read_file_content($remote_log);
        
        # plausy check
        if ($remote_text eq "" or $remote_status != 0) {
          log_msg_info("obviously file $remote_log does not exist or is empty or status failed -> retry");
          sleep(1);

          $remote_status = set_ui_mode_remote($other_node, $remote_log, $mode, $tls_version);
          $remote_text = read_file_content($remote_log);
        }        

        remove_lock_file($sshd_lock_file);
      } else {
        log_msg_info("did not get the remote lock $sshd_lock_file");
        $remote_status = $rc;
      }
    }
    
    if (has_quorum($node_q, $multiNodeType) == 1) {
      $has_a_quorum = 1;
      $rc = wait_for_ssh_lock($sshq_lock_file, $node_q);
      if ($rc == 0) {
        create_lock_file($sshq_lock_file);
        
        $quorum_status = set_ui_mode_remote($node_q, $quorum_log, $mode, $tls_version);
        $quorum_text = read_file_content($quorum_log);

        # plausy check
        if ($quorum_text eq "" or $quorum_status != 0) {
          log_msg_info("obviously file $quorum_log does not exist or is empty or status failed -> retry");
          sleep(1);

          $quorum_status = set_ui_mode_remote($node_q, $quorum_log, $mode, $tls_version);
          $quorum_text = read_file_content($quorum_log);
        }
        
        remove_lock_file($sshq_lock_file);
      } else {
        log_msg_info("did not get the remote lock $sshq_lock_file");
        $quorum_status = $rc;
      }
    }

    if ($rc == 0) {  
      # do the local retrieval even if remote retrieval fails
      removeFile($local_log);
      my $cmd = "$set_ui_mode_script $local_log $mode $tls_version";
      $local_status = exec_cmd($cmd);
      $local_text  = read_file_content($local_log);

      # plausy check
      if ($local_text eq "" or $local_status != 0) {
        log_msg_info("obviously file $local_log does not exist or is empty or status failed -> retry");
        sleep(1);

        $local_status = exec_cmd($cmd);
        $local_text  = read_file_content($local_log);
      }
    }

    write_extended_status($multiNodeType, $isMultiNode, $has_a_quorum, $toplevel_tag, 0, $combined_log, $rc, $text, $hostname, $local_status, $local_text, "node_a", $other_node, $remote_status, 1, $remote_text, "node_b", $node_q, $quorum_status, 1, $quorum_text, "node_q", $data_currentness);
      
    $content_xml = read_file_content($combined_log);
    print ("$content_xml\n");

    remove_lock_file($setui_lock_file);
  } else {
    log_msg_info("set_ui_mode failed to get the lock after $lock_timeout seconds");
    $rc = $ERROR_CanNotGetLock;
    $combined_log .= ".$pid";

    write_extended_status($multiNodeType, $isMultiNode, $has_a_quorum, $toplevel_tag, 0, $combined_log, $rc, $text, $hostname, $local_status, $local_text, "node_a", $other_node, $remote_status, 1, $remote_text, "node_b", $node_q, $quorum_status, 1, $quorum_text, "node_q", $data_currentness);

    $content_xml = read_file_content($combined_log);
    print ("$content_xml\n");     
  }
  function_leave("set_ui_mode", "mode=$mode rc=$rc");
  
  return($rc);
}

sub set_ui_mode_remote {
  my $node = shift;
  my $remote_log = shift;
  my $mode = shift;
  my $tls_version = shift;
  
  my $cmd_rc = 0;
  my $copy_rc = 0; 
  my $prepare_rc = 0;
  my $rc = 0;
  my $cmd = "";
  my $output = "";
  my ($node_is_reachable, $password_is_expired) = (0, 0);

  function_enter("set_ui_mode_remote", "on node $node remote_log=$remote_log mode=$mode, tls_version=$tls_version");

  my $remote_cmd = "$set_ui_mode_script $remote_log $mode $tls_version";
  $rc = execute_remote_cmd($node, \$node_is_reachable, \$password_is_expired, $remote_cmd, \$output, 1, 60);

  removeFile($remote_log);
  $copy_rc = execute_remote_copy($node, \$node_is_reachable, $remote_log, $remote_log, "get");

  if ($rc == 0 and $copy_rc != 0) {
    log_msg_info("enforcing the erroneous copy_rc=$copy_rc to rc");
    $rc = $copy_rc;
  }
  
  function_leave("set_ui_mode_remote", "on $node finished rc=$rc copy_rc=$copy_rc");
  return($rc);  
  }

sub get_interface_list {
  function_enter("get_interface_list");

  my ($rc, $do_bios_check, $operate_on_quorum) = (0, 0, 0);

  $rc = data_retrieval_template( "get_interface_list", "interface_list", $getif_lock_file, $do_bios_check, $operate_on_quorum, \&get_interface_list_local, \&get_interface_list_remote);

  # always return 0 for Assistant
  $rc = 0;

  function_leave("get_interface_list", "rc=$rc");
  return($rc);
}

sub get_interface_list_local {
  my $local_log = shift;
  my $rc = 0;

  function_enter("get_interface_list_local", "local_log=$local_log");

  removeFile($local_log);
  my $cmd = "$get_interface_list_script $local_log node1";
  $rc = exec_cmd($cmd);

  function_leave("get_interface_list_local", "rc=$rc");
  return($rc);
}

sub get_interface_list_remote {
  my $node = shift;
  my $remote_log = shift;
  my $node_is_reachable_ref = shift;
  my $password_is_expired_ref = shift;
  my $ssh_timeout = shift;
  my $is_quorum = shift;

  my $cmd_rc = 0;
  my $copy_rc = 0; 
  my $prepare_rc = 0;
  my $rc = 0;
  my $cmd = "";
  my $output = "";
  

  function_enter("get_interface_list_remote", "on node=$node remote_log=$remote_log ssh_timeout=$ssh_timeout");

  my $remote_cmd = "$get_interface_list_script $remote_log node2";
  $rc = execute_remote_cmd($node, $node_is_reachable_ref, $password_is_expired_ref, $remote_cmd, \$output, 1, $ssh_timeout);

  if ($rc == 0 and  $$node_is_reachable_ref != 0) {
    removeFile($remote_log);
    $copy_rc = execute_remote_copy($node, $node_is_reachable_ref, $remote_log, $remote_log, "get");
  }

  if ($rc == 0 and $copy_rc != 0) {
    log_msg_info("enforcing the erroneous copy_rc=$copy_rc to rc");
    $rc = $copy_rc;
  }
  
  function_leave("get_interface_list_remote", "on $node finished node_is_reachable=$$node_is_reachable_ref password_is_expired=$$password_is_expired_ref rc=$rc copy_rc=$copy_rc");
  return($rc);  
  }

sub replicate_ssh_banner {
  my $rc = 0;
  my $remote_rc = 0;
  my $banner_file = "/var/lib/sshd/Banner";

  $is_dscxl = dscxl_discover();
  $rc = determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 1);
  function_enter("replicate_ssh_banner", "multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q");

  if ($isMultiNode == 0) {
    log_msg_info("not in Duplex mode, do nothing, report success");
  } else {
    log_msg_info("in Duplex mode, distribute the banner to remote nodes");
    if (-e $banner_file) {
      $remote_rc = execute_remote_duplex_copy($banner_file, $banner_file, "put");
      #$remote_rc = execute_remote_copy($other_node, \$node_is_reachable, $banner_file, $banner_file, "put");
      if ($remote_rc != 0) {
        log_msg_info("copy to standby node failed but in any case try to reach the Q, but report error");
        $rc = 1;
      }
      if (has_quorum($node_q, $multiNodeType) == 1) {
        $remote_rc = execute_remote_quorum_copy($banner_file, $banner_file, "put");
        #$remote_rc = execute_remote_copy($node_q, \$node_is_reachable, $banner_file, $banner_file, "put");
      }
      if ($remote_rc != 0) {
        log_msg_info("copy to Quorum node failed, report error");
        $rc = 1;
      }

    } else {
      log_msg_info("banner_file $banner_file does not exist, can not distribute to remote nodes, report error");
      $rc = 1;
    }
  }

  function_leave("replicate_ssh_banner");
  return($rc);
}

# return code for configure_mek:
# 0   OK, binary MEK could be successfully generated (with history) and distributed
# 1   mek is not given or given mek is not 16 chars
# 2   SG mektool does not exist
# 3   SG mektool returned error
# 4   platform mek_script returned error
# 5   only on Duplex: can not replicate mek binary to Standby or Quorum node due to missing remote lock_file
# 6   only on Duplex: can not replicate mek binary to Standby or Quorum node 
# 25  error, cannot get lock for MEK generation. Another MEK operation already in progress
# 50  only on Duplex: generic remote execution error

sub configure_mek {
  my $mek = shift;
  my $rc = 0;
  my $remote_rc = 0;
  my ($local_status, $remote_status, $quorum_status) = (0, 0, 0);
  my $wait_rc = 0;
  my ($text, $remote_text, $local_text, $quorum_text) = ("", "", "", "");
  my ($remote_node_is_reachable, $remote_node_password_is_expired, $quorum_node_is_reachable, $quorum_node_password_is_expired) = (1, 0, 1, 0);
  my ($lock_timeout, $got_the_lock_immediately) = (120, 0);
  my ($ignore_remote_return_code, $suppress_diagnosis_output, $skip_sshkeygen, $skip_sshkeycleanup, $provided_private_key_file, $user) = (0,0,0,0,"","root");
  my $data_currentness = $DATA_CURRENTNESS_STATUS_irrelevant;
  my $toplevel_tag = "configure_mek";
  my $content_xml = "";
  my $mek_tool     = "/opt/soco/native/os/linux/mektool";
  my $mek_dir      = "/var/opt/soco/config/pbc_21/";
  my $mekdata_file = "$mek_dir"."mekdata";
  my $mekdata_file_copy = "$mek_dir"."mekdata.bak1";
  my $output = "";

  $is_dscxl = dscxl_discover();
  $rc = determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 1);
  function_enter("configure_mek", "mek=$mek multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q");

  $rc = wait_for_lock ($configuremek_lock_file, $lock_timeout, "configure_mek", \$got_the_lock_immediately);
  if ($rc == 0) {

    create_lock_file($configuremek_lock_file);

    # first generate the binary mek file with the tool
    if ( -f "$mek_tool" ) {
      log_msg_info("SG mektool exists, use it");
      if (-f $mekdata_file) {
        exec_cmd("ls -l $mekdata_file");
        log_msg_info("SG data file exists, make a copy");
        exec_cmd("cp -v $mekdata_file $mekdata_file_copy");
      }
      if (-d $mek_dir) {
        log_msg_info("mekdir $mek_dir already exists");
      } else {
        log_msg_info("mekdir $mek_dir does not exists, create it");
        exec_cmd("/bin/mkdir -pv $mek_dir");
      }
      
      my $cmd = "$mek_tool --mek=$mek";
      $rc = exec_cmd($cmd);
      if ($rc != 0) {
        log_msg_info("$mek_tool returned error");
        $rc = 3;
      }

      exec_cmd("ls -l $mekdata_file");

    } else {
      log_msg_info("SG mektool $mek_tool does not exist, error");
      $rc = 2;
    }

    # deploy to container iSG@Duplex
    if ($rc == 0) {
      if (-d "/var/sg2/opt/soco/config/pbc_21/" ) {
        log_msg_info("iSG container exists, distribute MEK to it");
        exec_cmd("cp -v $mekdata_file /var/sg2/opt/soco/config/pbc_21/");
      } else {
        log_msg_info("iSG container does not exist, do not distribute MEK");
      }
    }

    # deploy to inventory blobber
    if ($rc == 0) {
      log_msg_info("forward the newly created binary mekfile to inventory");
      my $script_cmd = "$configure_mek_script $mekdata_file $mek_blobber_dir";
      if (-x $configure_mek_script) {
        my $script_rc = exec_cmd($script_cmd);
        if ($script_rc != 0) {
          log_msg_info("$configure_mek_script returned error $script_rc");
          $rc = 4;
        }
      } else {
        log_msg_info("$configure_mek_script does not exist");
        $rc = 4;
      }
    }

    if ($rc == 0) {
      if ($isMultiNode == 0) {
        log_msg_info("not running in redundancy scenario - no configure mek on remote nodes performed");
        $remote_rc = 0;
      } else {

        log_msg_info("increasing timeout for ssh (lock) from $max_wait_for_ssh_lock_default to 180 seconds");
        $max_wait_for_ssh_lock_default = 180; # seconds
        my ($copy_rc_B, $copy_rc_Q, $tmp_rc) = (0, 0, 0);
        $rc = wait_for_ssh_lock($sshd_lock_file, $other_node);
        if ($rc == 0) {
          create_lock_file($sshd_lock_file);
          
          my $remote_cmd = "/bin/mkdir -pv $mek_dir";
          $skip_sshkeygen = 1;
          $skip_sshkeycleanup = 0;
          $tmp_rc = execute_remote_cmd($other_node, \$remote_node_is_reachable, \$remote_node_password_is_expired, $remote_cmd, \$output, 1, 60, $ignore_remote_return_code);
          if (!defined($tmp_rc)) { $rc = $ERROR_GenericRemoteExecutionFailure; }

          if ($tmp_rc == 0) {
            log_msg_info("copying binary mek to remote node");
            $skip_sshkeygen = 0;
            $skip_sshkeycleanup = 0;
            $copy_rc_B = execute_remote_copy($other_node, \$remote_node_is_reachable, $mekdata_file, $mekdata_file, "put", "root", $provided_private_key_file, $skip_sshkeygen, $skip_sshkeycleanup);

            log_msg_info("copying blobber dir to remote node");
            $skip_sshkeygen = 0;
            $skip_sshkeycleanup = 1;
            $copy_rc_B = execute_remote_copy($other_node, \$remote_node_is_reachable, $mek_blobber_dir, $mek_blobber_dir, "put", "root", $provided_private_key_file, $skip_sshkeygen, $skip_sshkeycleanup);
          } 

          if (has_quorum($node_q, $multiNodeType) == 1) {
            $has_a_quorum = 1;
            # wait for sshq lock not necessary, its only one lock
            $skip_sshkeygen = 1;
            $skip_sshkeycleanup = 0;
            $tmp_rc = execute_remote_cmd($node_q, \$remote_node_is_reachable, \$remote_node_password_is_expired, $remote_cmd, \$output, 1, 60, $ignore_remote_return_code);
            if (!defined($tmp_rc)) { $rc = $ERROR_GenericRemoteExecutionFailure; }

            if ($tmp_rc == 0) {
              log_msg_info("copying binary mek to Q node");
              $skip_sshkeygen = 0;
              $skip_sshkeycleanup = 1;
              $copy_rc_Q = execute_remote_copy($node_q, \$remote_node_is_reachable, $mekdata_file, $mekdata_file, "put", "root", $provided_private_key_file, $skip_sshkeygen, $skip_sshkeycleanup);
            }
            # blobber dir not necessary to replicate to Q
          }
          if ($copy_rc_B != 0 or $copy_rc_Q != 0) {
            log_msg_info("error when distributing binary mek to remote node");
            $rc = 6;
          }

          remove_lock_file($sshd_lock_file);
        } else {
          log_msg_info("did not get the remote lock $sshd_lock_file");
          $remote_status = $rc;
          $remote_text = "did not get the remote lock";
          $rc = 5;
        }
      }
    } else {
        log_msg_info("generate binary mek failed, this is a general failure");
    }

    remove_lock_file($configuremek_lock_file);

  } else {
    $rc = $ERROR_CanNotGetLock;
    log_msg_info("configure_mek failed to get the lock after $lock_timeout seconds");
  }

  function_leave("configure_mek");
  return($rc);
}

# return code for configure_syslog:
# 0   OK, syslog config could be successfully generated and distributed
# 1   syslog config is not given
# 4   platform script returned error
# 5   only on Duplex: can not replicate to Standby or Quorum node due to missing remote lock_file
# 6   only on Duplex: can not replicate to Standby or Quorum node 
# 25  error, cannot get lock for generation. Another operation already in progress
# 50  only on Duplex: generic remote execution error

sub configure_syslog {
  my ($syslogsrc,$epoch) = @_;
  my $rc = 0;
  my $remote_rc = 0;
  my ($local_status, $remote_status, $quorum_status) = (0, 0, 0);
  my $wait_rc = 0;
  my ($text, $remote_text, $local_text, $quorum_text) = ("", "", "", "");
  my ($remote_node_is_reachable, $remote_node_password_is_expired, $quorum_node_is_reachable, $quorum_node_password_is_expired) = (1, 0, 1, 0);
  my ($lock_timeout, $got_the_lock_immediately) = (120, 0);
  my ($ignore_remote_return_code, $suppress_diagnosis_output, $skip_sshkeygen, $skip_sshkeycleanup, $provided_private_key_file, $user) = (0,0,0,0,"","root");
  my $data_currentness = $DATA_CURRENTNESS_STATUS_irrelevant;
  my $output = "";

  $is_dscxl = dscxl_discover();
  $rc = determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 1);
  function_enter("configure_syslog", "config=$syslogsrc multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q");

  $rc = wait_for_lock ($configuresyslog_lock_file, $lock_timeout, "configure_syslog", \$got_the_lock_immediately);
  if ($rc == 0) {

    create_lock_file($configuresyslog_lock_file);

	log_msg_info("Process the config file: $syslogsrc, epoch: $epoch");
	if (-e $syslogsrc) {
		exec_cmd("/bin/mkdir -pv $syslog_dir");
		#prepare for /opt/inventory/get_syslog_forward_config.sh
		exec_cmd("/usr/bin/cp -v $syslogsrc $sysloghost_file");
		#process for Host nodes 
		exec_cmd_out(qq(/usr/bin/sed -n "/#HOST_BEGIN/,/#HOST_END/p" $syslogsrc > $syslogforward_file));
	} else {
		#empty/non existent --> deactivate forwarding
		exec_cmd("/usr/bin/rm -f $sysloghost_file $syslogforward_file");
	}
	exec_cmd("/usr/bin/rm -f $syslogsrc");
	exec_cmd_out("/bin/echo $epoch >$syslogepoch_file");
    exec_cmd("/usr/bin/systemctl restart --now syslog-ng.service");

#    # deploy to container iSG@Duplex
#    if ($rc == 0) {
#      if (-d "/var/sg2/opt/soco/config/pbc_21/" ) {
#        log_msg_info("iSG container exists, distribute to it");
#        exec_cmd("cp -v $syslogforward_file /var/sg2/opt/soco/config/pbc_21/");
#      } else {
#        log_msg_info("iSG container does not exist, do not distribute");
#      }
#    }


    if ($rc == 0) {
      if ($isMultiNode == 0) {
        log_msg_info("not running in redundancy scenario - no configure syslog on remote nodes performed");
        $remote_rc = 0;
      } else {

        log_msg_info("increasing timeout for ssh (lock) from $max_wait_for_ssh_lock_default to 180 seconds");
        $max_wait_for_ssh_lock_default = 180; # seconds
        my ($copy_rc_B, $copy_rc_Q, $tmp_rc) = (0, 0, 0);
        $rc = wait_for_ssh_lock($sshd_lock_file, $other_node);
        if ($rc == 0) {
          create_lock_file($sshd_lock_file);
          
          my $remote_cmd = "/bin/mkdir -pv $syslog_dir";
          $skip_sshkeygen = 1;
          $skip_sshkeycleanup = 0;
          $tmp_rc = execute_remote_cmd($other_node, \$remote_node_is_reachable, \$remote_node_password_is_expired, $remote_cmd, \$output, 1, 60, $ignore_remote_return_code);
          if (!defined($tmp_rc)) { $rc = $ERROR_GenericRemoteExecutionFailure; }

          if ($tmp_rc == 0) {
            log_msg_info("copying syslog config to remote node");
            $skip_sshkeygen = 0;
            $skip_sshkeycleanup = 1;
			if (-e $sysloghost_file) {
				$copy_rc_B += execute_remote_copy($other_node, \$remote_node_is_reachable, $sysloghost_file, $sysloghost_file, "put", "root", $provided_private_key_file, $skip_sshkeygen, $skip_sshkeycleanup);
				$copy_rc_B += execute_remote_copy($other_node, \$remote_node_is_reachable, $syslogforward_file, $syslogforward_file, "put", "root", $provided_private_key_file, $skip_sshkeygen, $skip_sshkeycleanup);
			} else {
				$copy_rc_B += execute_remote_cmd($other_node, \$remote_node_is_reachable, \$remote_node_password_is_expired,
					"/usr/bin/rm -f $sysloghost_file $syslogforward_file", 
					\$output, 1, 60, $ignore_remote_return_code);
			}
            $copy_rc_B += execute_remote_copy($other_node, \$remote_node_is_reachable, $syslogepoch_file, $syslogepoch_file, "put", "root", $provided_private_key_file, $skip_sshkeygen, $skip_sshkeycleanup);
			$copy_rc_B += execute_remote_cmd($other_node, \$remote_node_is_reachable, \$remote_node_password_is_expired,
				"/usr/bin/systemctl restart --now syslog-ng.service",
				\$output, 1, 60, $ignore_remote_return_code);
          }

          if (has_quorum($node_q, $multiNodeType) == 1) {
            $has_a_quorum = 1;
            # wait for sshq lock not necessary, its only one lock
            $skip_sshkeygen = 1;
            $skip_sshkeycleanup = 0;
            $tmp_rc = execute_remote_cmd($node_q, \$remote_node_is_reachable, \$remote_node_password_is_expired, $remote_cmd, \$output, 1, 60, $ignore_remote_return_code);
            if (!defined($tmp_rc)) { $rc = $ERROR_GenericRemoteExecutionFailure; }

            if ($tmp_rc == 0) {
              log_msg_info("copying syslog config to Q node");
              $skip_sshkeygen = 0;
              $skip_sshkeycleanup = 1;
			  if (-e $sysloghost_file) {
			    $copy_rc_Q += execute_remote_copy($node_q, \$remote_node_is_reachable, $syslogforward_file, $syslogforward_file, "put", "root", $provided_private_key_file, $skip_sshkeygen, $skip_sshkeycleanup);
			  } else {
				$copy_rc_Q += execute_remote_cmd($node_q, \$remote_node_is_reachable, \$remote_node_password_is_expired, 
					"/usr/bin/rm -f $syslogforward_file", 
					\$output, 1, 60, $ignore_remote_return_code);
			  }
              $copy_rc_Q += execute_remote_copy($node_q, \$remote_node_is_reachable, $syslogepoch_file, $syslogepoch_file, "put", "root", $provided_private_key_file, $skip_sshkeygen, $skip_sshkeycleanup);
			  $copy_rc_Q += execute_remote_cmd($node_q, \$remote_node_is_reachable, \$remote_node_password_is_expired, 
				"/usr/bin/systemctl restart --now syslog-ng.service",
				\$output, 1, 60, $ignore_remote_return_code);
            }
          }
          if ($copy_rc_B != 0 or $copy_rc_Q != 0) {
            log_msg_info("error when distributing syslog config to remote node");
            $rc = 6;
          }

          remove_lock_file($sshd_lock_file);
        } else {
          log_msg_info("did not get the remote lock $sshd_lock_file");
          $remote_status = $rc;
          $remote_text = "did not get the remote lock";
          $rc = 5;
        }
      }
    } else {
        log_msg_info("generate syslog config failed, this is a general failure");
    }

    remove_lock_file($configuresyslog_lock_file);

  } else {
    $rc = $ERROR_CanNotGetLock;
    log_msg_info("configure_syslog failed to get the lock after $lock_timeout seconds");
  }

  #TODO how to get output trough to the Assistant?
  print STDERR $output;
  function_leave("configure_syslog");
  return($rc);
} #configure_syslog

sub configure_snmp {
  my $devicetype = shift;  
  my $rc = 0;
  my $remote_rc = 0;
  my ($local_status, $remote_status, $quorum_status) = (0, 0, 0); 
  my $wait_rc = 0;
  my $remote_log   = $webservice_log_dir.'configure_snmp_remote.log';
  my $quorum_log   = $webservice_log_dir.'configure_snmp_quorum.log';
  my $local_log    = $webservice_log_dir.'configure_snmp_local.log';
  my $combined_log = $webservice_log_dir.'configure_snmp_all.log';
  my ($text, $remote_text, $local_text, $quorum_text) = ("", "", "", "");
  
  my ($lock_timeout, $got_the_lock_immediately) = (120, 0);
  my $data_currentness = $DATA_CURRENTNESS_STATUS_irrelevant;
  my $toplevel_tag = "configure_snmp";
  my $content_xml = "";
  my $propagate_to_ngs_option = "";

  $is_dscxl = dscxl_discover();
  $rc = determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 1);
  function_enter("configure_snmp", "devicetype=$devicetype multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q");

  $rc = wait_for_lock ($configuresnmp_lock_file, $lock_timeout, "configure_snmp", \$got_the_lock_immediately);
  if ($rc == 0) {

    create_lock_file($configuresnmp_lock_file);

    my $takeover_mode = 4; # snmp config file
    $rc = takeover_files($takeover_mode);
    if ($rc != 0) {
      $text = "Takeover snmp config file from assistent area $assistant_snmp_dir failed"
    }

    # check for Surv SG
    # on Surv SG the SG has precedence to enable central config, so I will reject requests from Assistant
    if ($rc == 0) {
      if (is_survivable_deployment($multiNodeType)) {
        if ($devicetype eq "sg") {
          log_msg_info("deployment is Surv. SG/Enterprise GW and request is coming from SG -> allow due to central config");
        } else {
          $text = "deployment is Surv. SG/Enterprise GW and request is coming from Assistant -> reject due to central config";
          log_msg_info($text);
          $rc = $ERROR_SNMPConfigRejected;
        }
      }
    }
    
    if ($rc == 0) {
      # on central nodes with integr. SG, the Assistant has precedence over SG
      if (isCentralHost($multiNodeType)) {
        if ($isMultiNode == 0) {
          log_msg_info("deployment is Central Simplex -> put snmp config to NGS");
          $propagate_to_ngs_option = "-n";
        } else {
          log_msg_info("deployment is Central Duplex -> put snmp config to NGS since this call is always on the active node");
          $propagate_to_ngs_option = "-n";          
        }
        if ($devicetype eq "sg") {        
          $text =  "deployment is Integrated SG and request is coming from SoftGate -> reject";
          log_msg_info($text);          
          $rc = $ERROR_SNMPConfigRejected;
        } else {
          log_msg_info("deployment is Integrated SG and request is coming from Assistant -> allow");
        }
      }
    }

    if ($rc == 0) {
      # do the local configuration before remote configuration for check
      removeFile($local_log);
      my $cmd = "$configure_snmp_script $propagate_to_ngs_option $local_log $snmp_dir $multiNodeType";
      $local_status = exec_cmd($cmd);
      $local_text  = read_file_content($local_log);

      $rc = $local_status;
    }


    if ($rc == 0) {
      if ($isMultiNode == 0) {
        log_msg_info("not running in redundancy scenario - no configure snmp on remote nodes performed");
        $remote_rc = 0;
      } else {

        $rc = wait_for_ssh_lock($sshd_lock_file, $other_node);
        if ($rc == 0) {
          create_lock_file($sshd_lock_file);
        
          $remote_status = configure_snmp_remote($other_node, $multiNodeType, $remote_log);
          $remote_text = read_file_content($remote_log);

          if (has_quorum($node_q, $multiNodeType) == 1) {
            $has_a_quorum = 1;
            # wait for sshq lock not necessary, its only one lock

            $quorum_status = configure_snmp_remote($node_q, $multiNodeType, $quorum_log);
            $quorum_text = read_file_content($quorum_log);
          }

          remove_lock_file($sshd_lock_file);
        } else {
          log_msg_info("did not get the remote lock $sshd_lock_file");
          $remote_status = $rc;
          $remote_text = "did not get the remote lock";
        }
      }
    } else {
        log_msg_info("local snmp config failed, FDB5 requires to skip the remote calls then");
    }

    write_extended_status($multiNodeType, $isMultiNode, $has_a_quorum, $toplevel_tag, 0, $combined_log, $rc, $text, $hostname, $local_status, $local_text, "node_a", $other_node, $remote_status, 1, $remote_text, "node_b", $node_q, $quorum_status, 1, $quorum_text,  "node_q", $data_currentness);

    $content_xml = read_file_content($combined_log);
    print ("$content_xml\n");

    remove_lock_file($configuresnmp_lock_file);

  } else {
    $rc = $ERROR_CanNotGetLock;
    log_msg_info("configure_snmp failed to get the lock after $lock_timeout seconds");
    $combined_log .= ".$pid";

    write_extended_status($multiNodeType, $isMultiNode, $has_a_quorum, $toplevel_tag, 0, $combined_log, $rc, $text, $hostname, $local_status, $local_text, "node_a", $other_node, $remote_status, 1, $remote_text, "node_b", $node_q, $quorum_status, 1, $quorum_text, "node_q", $data_currentness);

    $content_xml = read_file_content($combined_log);
    print ("$content_xml\n");
  }

  function_leave("configure_snmp");
  return($rc);
}

sub configure_snmp_remote {
  my $node = shift;
  my $multiNodeType = shift;
  my $remote_log = shift;
  my $cmd_rc = 0;
  my $copy_rc = 0;
  my $prepare_rc = 0;
  my $rc = 0;
  my $cmd = "";
  my $output = "";
  my ($node_is_reachable, $password_is_expired) = (0, 0);


  function_enter("configure_snmp_remote", "on node $node multiNodeType=$multiNodeType remote_log=$remote_log");

  removeFile($remote_log);

  $rc = execute_remote_copy($node, \$node_is_reachable, $snmp_dir, $snmp_dir, "put");

  if ($rc == 0) {
    my $remote_cmd = "$configure_snmp_script $remote_log $snmp_dir $multiNodeType";
    $rc = execute_remote_cmd($node, \$node_is_reachable, \$password_is_expired, $remote_cmd, \$output, 1, 60);
  }

  removeFile($remote_log);
  $copy_rc = execute_remote_copy($node, \$node_is_reachable, $remote_log, $remote_log, "get");

  if ($rc == 0 and $copy_rc != 0) {
    log_msg_info("enforcing the erroneous copy_rc=$copy_rc to rc");
    $rc = $copy_rc;
  }

  function_leave("configure_snmp_remote", "on $node finished rc=$rc copy_rc=$copy_rc");
  return($rc);
}

sub redistribute_licenses {
  my $locale = shift;
  my $rc = 0;
  my $remote_rc = 0;
  my $remote_status = 0;
  my $remote_errortext = "";
  my ($lock_timeout, $got_the_lock_immediately) = (120, 0);
  my $remote_log   = $webservice_log_dir.'redistribute_licenses_remote.log';
  
  $is_dscxl = dscxl_discover();
  $rc = determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 1);
  function_enter("redistribute_licenses (for node replacement)", "locale=$locale rc=$rc multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q");

  $rc = wait_for_lock ($activatelicense_lock_file, $lock_timeout, "activate_license", \$got_the_lock_immediately);
  if ($rc == 0) {

    if ($rc == 0) {
      if ($isMultiNode == 0) {
        log_msg_info("not running in Duplex scenario - no license activation on remote nodes performed");
        $rc = $ERROR_RemoteLicenseActivationError;
      } else {
          if(!-d $license_storage_dir) {
            log_msg_info("license storage directory $license_storage_dir not found. Can not redistribute license files to replaced node.");
            $rc = $ERROR_RemoteLicenseActivationError;
          } else {
            $remote_status = activate_license_remote($other_node, $multiNodeType, $remote_log, $locale, 1);
            $remote_errortext = read_file_content($remote_log);
            $remote_errortext = "<![CDATA[$remote_errortext]]>";
            if ($remote_status != 0 ) {
              $remote_status = $ERROR_RemoteLicenseActivationError;
            } else {
              log_msg_info("report success from remote side:");
              log_msg_info("remote_errortext=$remote_errortext");
              $rc = 0
            }
          }
      }
    }

  } else {
    $rc = $ERROR_CanNotGetLock;
    log_msg_info("redistribute_licenses failed to get the lock after $lock_timeout seconds");
  }

  function_leave("redistribute_licenses", "rc=$rc");
  return($rc);
}


sub activate_license {
  my $locale = shift;
  my $rc = 0; 
  my $remote_rc = 0;
  my $remote_status = 0;
  my $local_status = 0;
  my $wait_rc = 0;
  my $remote_log   = $webservice_log_dir.'activate_license_remote.log';
  my $local_log    = $webservice_log_dir.'activate_license_local.log';  
  my $combined_log = $webservice_log_dir.'activate_license_all.log';
  my ($errortext, $remote_errortext, $local_errortext) = ("", "", "");
  my ($lock_timeout, $got_the_lock_immediately) = (120, 0);
  my $data_currentness = $DATA_CURRENTNESS_STATUS_irrelevant; 
  my $toplevel_tag = "license_activation";
  my $content_xml = "";

  $is_dscxl = dscxl_discover();
  $rc = determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 1);
  function_enter("activate_license", "rc=$rc multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q");

  $rc = wait_for_lock ($activatelicense_lock_file, $lock_timeout, "activate_license", \$got_the_lock_immediately);
  if ($rc == 0) {

    create_lock_file($activatelicense_lock_file);
  
    log_msg_info("activate_license for locale: $locale");

    my $takeover_mode = 2; # license files
    $rc = takeover_files($takeover_mode);
    if ($rc != 0) {
      $errortext = "Takeover license files from assistent area $assistant_license_dir failed"
    }

    if ($rc == 0) {
      if ($isMultiNode == 0) {
        log_msg_info("not running in redundancy scenario - no license activation on remote nodes performed"); 
        $remote_rc = 0;
      }

      if ( $isMultiNode != 0) {
          if(is_daemon_running()) {
            log_msg_info("daemon already running -> upgrade in progress, no not try to active license on standby");
            $remote_status = $ERROR_CanNotActicateLicenseDuringUpgrade;
            $remote_errortext = "Cannot activate license on standby node during running update.";
          } else {
            $remote_status = activate_license_remote($other_node, $multiNodeType, $remote_log, $locale,0);
            $remote_errortext = read_file_content($remote_log);
            $remote_errortext = "<![CDATA[$remote_errortext]]>";
            if ($remote_status != 0 ) {
              $remote_status = $ERROR_RemoteLicenseActivationError;
            }
          }
      }
    }

    if ($rc == 0) {
      # do the local activation even if remote activation fails
      removeFile($local_log);
      my $cmd = "$activate_license_script $local_log $license_dir $multiNodeType $locale";
      $local_status = exec_cmd($cmd);

      $local_errortext  = read_file_content($local_log);
      $local_errortext = "<![CDATA[$local_errortext]]>";
      if ($local_status != 0 ) {
        $local_status = $ERROR_LocalLicenseActivationError;
      }
    }
  
    write_extended_status($multiNodeType, $isMultiNode, 0, $toplevel_tag, 0, $combined_log, $rc, $errortext, $hostname, $local_status, $local_errortext, "node_a_errortext", $other_node, $remote_status, 1, $remote_errortext, "node_b_errortext", "", 0, 1, "", "", $data_currentness);
   
    $content_xml = read_file_content($combined_log);
    print ("$content_xml\n");
  
    remove_lock_file($activatelicense_lock_file);

  } else {
    $rc = $ERROR_CanNotGetLock;
    log_msg_info("activate_license failed to get the lock after $lock_timeout seconds");
    $combined_log .= ".$pid";

    write_extended_status($multiNodeType, $isMultiNode, 0, $toplevel_tag, 0, $combined_log, $rc, $errortext, $hostname, $local_status, $local_errortext, "node_a_errortext", $other_node, $remote_status, 1, $remote_errortext, "node_b_errortext", "", 0, 1, "", "", $data_currentness);

    $content_xml = read_file_content($combined_log);
    print ("$content_xml\n");
  }

  function_leave("activate_license", "rc=$rc");
  return($rc);
}

sub activate_license_remote {
  my $node = shift;
  my $multiNodeType = shift;
  my $remote_log = shift;
  my $locale = shift;
  my $mode = shift;
  my $cmd_rc = 0;
  my $copy_rc = 0; 
  my $prepare_rc = 0;
  my $rc = 0;
  my $cmd = "";
  my $output = "";
  my ($other_node_is_reachable, $password_is_expired) = (0, 0);
  

  function_enter("activate_license_remote", "on node $node multiNodeType=$multiNodeType remote_log=$remote_log locale=$locale mode=$mode");

  if ($mode == 1) {
    log_msg_info("operating in restribution mode");
    $rc = execute_remote_duplex_copy($license_storage_dir, $license_dir, "put");
  } else {
    log_msg_info("operating in activating (normal) mode");
    $rc = execute_remote_duplex_copy($license_dir, $license_dir, "put");
    $rc = execute_remote_duplex_copy($license_storage_dir, $license_storage_dir, "put");
  }

  if ($rc == 0) {
    my $remote_cmd = "$activate_license_script $remote_log $license_dir $multiNodeType $locale";
    $rc = execute_remote_duplex_cmd($remote_cmd, \$output, 1, \$other_node_is_reachable, \$password_is_expired);
  }

  removeFile($remote_log);   
  $copy_rc = execute_remote_duplex_copy($remote_log, $remote_log, "get");

  if ($rc == 0 and $copy_rc != 0) {
    log_msg_info("enforcing the erroneous copy_rc=$copy_rc to rc");
    $rc = $copy_rc;
  }
  
  function_leave("activate_license_remote", "on $node finished rc=$rc copy_rc=$copy_rc");
  return($rc);
}

sub execute_remote_update {
  my $other_node_is_reachable_ref = shift;
  my $node_q_is_reachable_ref = shift;  
  my $multiNodeType = shift;
  my $isMultiNode = shift;
  my $other_node = shift;
  my $node_q = shift;
  my $rc = 0;
  
  function_enter("execute_remote_update", "rc=$rc multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q");
  
  if ( -e "/var/log/nuc/swupdated.use.nnuc" ) {
    log_msg_info("execute_remote_update : special N-NUC case detected.");
    # indicate to N-NUC P daemon that we were started - now the ip_cust can be stopped
    exec_cmd("touch /var/log/nuc/swupdated.nnuc.swupdated.initiated");
    while( 1 == 1 ) {
      $rc = $rc + 1;
      if ( $rc == 120 ) {
        log_msg_info("execute_remote_update : ERROR : waiting for continuation indication from N-NUC daemon timed out. Will immediately exit swupdated");
        exit(1);
      }
      log_msg_info("execute_remote_update : waiting for continuation indication from N-NUC daemon ( $rc out of max 120 times ) ...");
      sleep(60);
      if ( -e "/var/log/nuc/swupdated.nnuc.secondary.is.ready" ) {
        last;
      }
    }
    log_msg_info("execute_remote_update : got indication to continue. Skipping rest of usual NUC actions done in execute_remote_update. S and Q should now already be updated and ready for takeover.");
    function_leave("execute_remote_update", "rc=$rc other_node_is_reachable=$$other_node_is_reachable_ref node_q_is_reachable=$$node_q_is_reachable_ref");
    return(0);
  }

  if ($isMultiNode == 0) {
     log_msg_info("not running in redundancy scenario - no update of redundant nodes performed"); 
  }

  # check if running node Q scenario 
  if($rc == 0 && $isMultiNode == 1 && has_quorum($node_q, $multiNodeType)) {
    set_internal_upgrade_step($STEP_remote_update, 1, "Quorum", $node_q);

    if ( $rmx_reboot != 0 && $upgrade_with_completely_stopped_ha == 0) {
      log_msg_info("stopping openais on quorum BEFORE starting upgrade there in case a reboot happens during stop ");	  
      # ignoring return code	  
      stop_openais_service("quorum", 5, 1, 1, 0, 1);
    }
    
    # lock must be taken after stop_openais_service to prevent deadlock
    $rc = wait_for_ssh_lock($sshq_lock_file, $node_q, $max_wait_for_ssh_lock_upgrade);
    create_lock_file($sshq_lock_file);
    
    if ($rc == 0 ) {
      set_internal_upgrade_status($STATUS_running,   "Quorum",  $node_q);

      # trigger watchdog and hope that the quorum update is less than one hour
      trigger_watchdog();
      $rc = ha_update_node($node_q, $node_q_is_reachable_ref, 1, $status_quorum_file);
      if ($rc == 0) {
        set_internal_upgrade_status($STATUS_success, "Quorum",  $node_q);
        if ($upgrade_with_completely_stopped_ha == 0) {
          set_openais_state        ($OPENAIS_STATE_running, "Quorum", $node_q);
        }
      } else {
        if ($upgrade_via_image == 0) {
          log_msg_info("remote update on Quorum node failed and it is no NUC -> error");
          set_internal_upgrade_status($STATUS_failed,  "Quorum",  $node_q);
          set_internal_upgrade_status($STATUS_dropped, "Standby", $other_node);
          set_internal_upgrade_status($STATUS_dropped, "Active",  $hostname);
          set_openais_state          ($OPENAIS_STATE_unknown, "Quorum", $node_q);
        } else {
          log_msg_info("remote update on Quorum node failed but it is a NUC -> not necessarily an error -> continue to check worldness");
        }
      }

      if ($upgrade_via_image != 0) {
        # NUC      
        $rc = wait_for_remote_node_world_appearance($node_q, "NEW");
        if ($rc == 0) {
          log_msg_info("NUC activation reboot on Quorum node $node_q was successful -> continue with B");
        } else {
          log_msg_info("NUC activation reboot on Quorum node $node_q failed (still old) -> do NOT continue with B");
          $wait_for_ssh_connectivity = 15;
          log_msg_info("due to remote node failure reducing wait time to $wait_for_ssh_connectivity min. (test method=$node_reach_test_method)");
          $quorum_activation_rc = $ERROR_RemoteUpdateError;
        }
      } else {
        # non-NUC -> HF
        if ($host_reboot == 1) {
          log_msg_info("sleeping two minutes to give the Q node time to go down for Linux reboot");
          trigger_watchdog();
          sleep(60); # 1 minute
          trigger_watchdog();
          sleep(60); # 1 minute
          my $rc_status = wait_for_cluster_node_reappearance($node_q, 60, 30); # 30 minutes
          if ($rc_status == 0) {
            log_msg_info("node Q re-appeared in time");
          } else {
            log_msg_info("node Q did not re-appear in time (after 2 + 30 minutes), error situation");
            $quorum_activation_rc = $ERROR_RemoteUpdateError;
            $rc = $quorum_activation_rc;
          }
        }
      }
    }
    remove_lock_file($sshq_lock_file);
  }
     
  if ($rc == 0 && $isMultiNode == 1) {
    set_internal_upgrade_step($STEP_remote_update, 1, "Standby", $other_node);

    create_remote_lock_file();

    if ( $rmx_reboot != 0 && $upgrade_with_completely_stopped_ha == 0) {
      log_msg_info("stopping openais on standby node BEFORE starting upgrade there in case a reboot happens during stop ");
      # ignoring return code
      stop_openais_service("remote", 5, 1, 1, 0, 1);
    }

    # lock must be taken after stop_openais_service to prevent deadlock
    $rc = wait_for_ssh_lock($sshd_lock_file, $other_node, $max_wait_for_ssh_lock_upgrade);
    create_lock_file($sshd_lock_file);
    
    if ($rc == 0 ) {
      set_internal_upgrade_status($STATUS_running,   "Standby", $other_node);

      # trigger watchdog and hope that the standby update is less than one hour
      trigger_watchdog();      
      $rc = ha_update_node($other_node, $other_node_is_reachable_ref, 0, $status_remote_file);
      if ($rc == 0) {
        set_internal_upgrade_status($STATUS_success, "Standby", $other_node);
        if ($upgrade_with_completely_stopped_ha == 0) {        
          set_openais_state        ($OPENAIS_STATE_running, "Standby", $other_node);
        }
      } else {
        set_internal_upgrade_status($STATUS_failed,  "Standby", $other_node);
        set_internal_upgrade_status($STATUS_dropped, "Active",  $hostname);
        set_openais_state          ($OPENAIS_STATE_unknown, "Standby", $other_node);
      }
    }

    if ($upgrade_via_image != 0) {
      # NUC
      $rc = wait_for_remote_node_world_appearance($other_node, "NEW");
      if ($rc == 0) {
        log_msg_info("NUC activation reboot on node-B $other_node was successful -> continue with A");
      } else {
        log_msg_info("NUC activation reboot on node-B node $other_node failed (still old) -> do NOT continue with A and rollback Q if necessary");
        $wait_for_ssh_connectivity = 15;
        log_msg_info("due to remote node failure reducing wait time to $wait_for_ssh_connectivity min. (test method=$node_reach_test_method)");
        $standby_activation_rc = $ERROR_RemoteUpdateError;
        if ( has_quorum($node_q, $multiNodeType) ) {
          my $ha_type_to_rollback = "quorum";  # get_ha_type() geht hier nicht
          NUC_remote_rollback($node_q, $ha_type_to_rollback);
        }
      }
    } else {
      # non-NUC -> HF
      if ($host_reboot == 1) {
        log_msg_info("sleeping two minutes to give the B node time to go down after Linux reboot");
        sleep(120); # 2 minutes
        my $rc_status = wait_for_cluster_node_reappearance($other_node, 60, 30); # 30 minutes
        if ($rc_status == 0) {
          log_msg_info("node B re-appeared in time");
        } else {
          log_msg_info("node B did not re-appear in time (after 2 + 30 minutes), error situation");
          $standby_activation_rc = $ERROR_RemoteUpdateError;
          $rc = $standby_activation_rc;        }
      }
    }
      
    remove_remote_lock_file();
    remove_lock_file($sshd_lock_file);
  }

  # trigger watchdog immediately after the standby update (it could be nearly an hour)
  trigger_watchdog();

  set_internal_upgrade_step($STEP_remote_update_complete, 1, "Active", $hostname);

  if ($rc == 0 && $isMultiNode == 1 && $is_FI == 0) {
    # wait for the Active/Ready state on the active node
    if ($upgrade_with_completely_stopped_ha == 1) {
      log_msg_info("skipping call of the wait for success script $upd_ok_sh for passive node due to upgrade with complete HA down");
    } else {
      $rc = wait_for_successful_startup(1, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $multiNodeType, $isMultiNode, "secondary", $max_wait_for_up, 0);
    }
  }
  
 
  function_leave("execute_remote_update", "rc=$rc other_node_is_reachable=$$other_node_is_reachable_ref node_q_is_reachable=$$node_q_is_reachable_ref");
  return($rc);
}


sub has_quorum {
  my $node_q = shift;
  my $multiNodeType = shift;
  my $rc = 0;

  if(length($node_q) > 0 && ($multiNodeType == 10 || $multiNodeType == 11 || $multiNodeType == 62)) {
    $rc = 1;
  }
  return($rc);
}

sub retrieve_from_xml {
  my $entry = shift;
  my $given_xml_file  = shift;
  my $value = "";

  my $xml_file = $ha_data_dir."conf/haipsetup.xml";
  if (defined($given_xml_file)) {
    if (-e $given_xml_file) {
      $xml_file = $given_xml_file;
    }
  }
  function_enter("retrieve_from_xml", "entry=$entry xml_file=$xml_file");
  
  if(-e $xml_file && open(HAIPSETUP, "<$xml_file")) {
    my @setup_xml = <HAIPSETUP>;
    my @parts;
    foreach(@setup_xml) {
      if(index($_, "$entry") >= 0) {
        @parts = split(/<|>/, $_);
        $value = $parts[2];
        log_msg_info("$entry=$value");
      }
    }
    close(HAIPSETUP);
  } else {
    log_msg("$xml_file does not exist or could not be opened");
  }
  chomp($value);
  function_leave("retrieve_from_xml", "$entry=$value");
  return ($value);
}


sub parse_xml_tag {
  my $tag = shift;
  my $xml_content  = shift;
  function_enter("parse_xml_tag", "tag=$tag (using SimpleXML parser)");
  my $value="";

  use XML::Simple;

  my $tree = 0;
  my $xml_attribute = "";
  my $simple = XML::Simple->new( );             # initialize the object

  # eval is necessary, otherwise we would die on error
  # put a toplevel tag <content> around it to get well formed XML
  # second arg of XMLin() can be a filename or a complete XML content as string
  $tree = eval { $simple->XMLin("<content>$xml_content</content>", ForceArray => 0 ) };
  if($@) {
    log_msg_info("error: content could not successfully be parsed: $@");
  } else {
    log_msg_info("content could successfully be parsed");

    log_msg_info("tree=$tree");
    
    $value = eval { $tree->{$tag} };
    if (!defined($value)) { $value=""; }    
  }
  
  function_leave("parse_xml_tag", "value=$value");
  return ($value);
}

sub determine_ha_nodes {

  my $enhanced_diagnosis = $_[4];
  my $given_ha_xml_file  = $_[5];
  my $rc = 0;

  my $haipsetup = $ha_data_dir."conf/haipsetup.xml";
  
  if (defined($given_ha_xml_file)) {
    if (-e $given_ha_xml_file) {
      $haipsetup = $given_ha_xml_file;
    }
  }
  my $multiNodeType = 0; # single node
  my $isMultiNode = 0;
  my $node_a = "";
  my $node_b = "";
  my $node_q = "";
  my $other_node = "";
  function_enter("determine_ha_nodes", "(using SimpleXML parser)");

  if ($is_stmix == 1) {
    log_msg_info("no ha parameters in STMIX mode, keeping the values");
    function_leave("determine_ha_nodes", "finished rc=$rc multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q");
    return($rc);
  }
  
  if ( defined($enhanced_diagnosis) && $enhanced_diagnosis == 1) {
    my $haipsetup_diagnose = "";
    log_msg_info("current content of $haipsetup:");
    $haipsetup_diagnose = exec_cmd_out("cat $haipsetup");
    log_msg_info("$haipsetup_diagnose\n");
  }
  
  use XML::Simple;

  my $ref = 0;
  my $xml_attribute = "";

  # eval is necessary, otherwise we would die on error
  $ref = eval { XMLin($haipsetup, ForceArray => 0 ) };
  if($@) {
    log_msg_info("error: $haipsetup could not successfully be parsed: $@");
  } else {
    log_msg_info("$haipsetup could successfully be parsed");

    $xml_attribute = "system.multiNodeSystem";
    $multiNodeType=$ref->{entry}{$xml_attribute}{content};
    if (!defined($multiNodeType)) {
      log_msg_info("$xml_attribute not found in $haipsetup");
      $multiNodeType = 0;
    }

    $xml_attribute = "system.name-node1";
    $node_a=$ref->{entry}{$xml_attribute}{content};
    if (!defined($node_a)) {
      log_msg_info("$xml_attribute not found in $haipsetup");
      $node_a = "";
    }

    $xml_attribute = "system.name-node2";
    $node_b=$ref->{entry}{$xml_attribute}{content};
    if (!defined($node_b)) {
      log_msg_info("$xml_attribute not found in $haipsetup");
      $node_b = "";
    }

    $xml_attribute = "system.nameQuorumNode";
    $node_q=$ref->{entry}{$xml_attribute}{content};
    if (!defined($node_q)) {
      log_msg_info("$xml_attribute not found in $haipsetup");
      $node_q = "";
    }
  }

  # check if running redundancy scenario (10-19, 60-69, 99)
  log_msg_info("detected multi node type $multiNodeType");
  if($rc == 0 &&
     length($node_a) > 0 &&
     length($node_b) > 0 &&
     (($multiNodeType >= 10 && $multiNodeType < 20) || ($multiNodeType >= 60 && $multiNodeType < 70) || ($multiNodeType == 99)))
  {
     $isMultiNode = 1;
     $other_node = $node_a;

     if($node_a eq $hostname) { $other_node = $node_b; }
     log_msg_info("other node is $other_node");

  } else {
    log_msg_info("not running in redundancy scenario");
    $isMultiNode = 0;
  }
  
  # give back the values
  $_[0] = $multiNodeType;
  $_[1] = $isMultiNode;
  $_[2] = $other_node;
  $_[3] = $node_q;

  function_leave("determine_ha_nodes", "finished rc=$rc multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q");
  return($rc);
}

sub execute_remote_duplex_copy {
  my $local_path  = shift;
  my $remote_path = shift; 
  my $type = shift;       
  my $rc = 0;
  function_enter("execute_remote_duplex_copy", "type=$type local_path=$local_path remote_path=$remote_path");

  my $node_is_reachable = 1;
  
  $rc = determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 0);
  log_msg_info("execute_remote_duplex_copy rc=$rc multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q");

  if ($isMultiNode == 0) {
    print STDERR "Not running in redundancy/multi-node scenario.\nSkipping execution of remote copy.\n";
  }
  
  if ($rc == 0 && $isMultiNode == 1) {

    # make first a connectivity check to avoid time consuming wait for lock in case remote node is not reachable at all
    $node_is_reachable = is_node_reachable($other_node);
    if ($node_is_reachable == 0) {
      log_msg_info("skip wait_for_ssh_lock since node $other_node is NOT reachable");
      $rc  = $ERROR_CanNotReachRemoteNode;
    } else {
      log_msg_info("continuing since node $other_node is reachable");    
      $rc = wait_for_ssh_lock($sshd_lock_file, $other_node);
    }
  }

  if ($rc == 0 && $isMultiNode == 1) {
    create_lock_file($sshd_lock_file);
    $rc = execute_remote_copy($other_node, \$node_is_reachable, $local_path, $remote_path, $type);
    remove_lock_file($sshd_lock_file);
  }
  function_leave("execute_remote_duplex_copy", "rc=$rc");
  return($rc);
}

sub execute_remote_quorum_copy {
  my $local_path  = shift;
  my $remote_path = shift;    
  my $type = shift;    
  my $rc = 0;
  
  function_enter("execute_remote_quorum_copy", "type=$type local_path=$local_path remote_path=$remote_path");

  my $node_is_reachable = 1;
  
  $rc = determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 0);
  log_msg_info("execute_remote_quorum_copy rc=$rc multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q");

  if ($isMultiNode == 0) {
    print STDERR "Not running in redundancy/multi-node scenario.\nSkipping execution of remote copy.\n";
  }

  if ($rc == 0 && $isMultiNode == 1) {

    # make first a connectivity check to avoid time consuming wait for lock in case remote node is not reachable at all
    $node_is_reachable = is_node_reachable($node_q);
    if ($node_is_reachable == 0) {
      log_msg_info("skip wait_for_ssh_lock since node $node_q is NOT reachable");
      $rc  = $ERROR_CanNotReachRemoteNode;
    } else {
      log_msg_info("continuing since node $other_node is reachable");
      $rc = wait_for_ssh_lock($sshq_lock_file, $node_q);
    }
  }    

  if ($rc == 0 && $isMultiNode == 1) {
    create_lock_file($sshq_lock_file);
    $rc = execute_remote_copy($node_q, \$node_is_reachable, $local_path, $remote_path, $type);
    remove_lock_file($sshq_lock_file);
  }
  function_leave("execute_remote_quorum_copy", "rc=$rc");
  return($rc);
}

sub execute_remote_duplex_cmd {
  my $cmd = shift; 
  my $output_ref = shift; 
  my $max_try = shift;
  my $other_node_is_reachable_ref = shift;
  my $password_is_expired_ref = shift;
  my $ignore_remote_return_code = shift;
  my $duplex_wait_for_ssh_lock = shift;

  if (!defined($ignore_remote_return_code)) {
    $ignore_remote_return_code = 0;
  }
  if (!defined($duplex_wait_for_ssh_lock)) {
    $duplex_wait_for_ssh_lock = $max_wait_for_ssh_lock_default;
  }
  my $output = "";
  my $rc = 0;
  function_enter("execute_remote_duplex_cmd", "cmd=$cmd max_try=$max_try other_node_is_reachable=$$other_node_is_reachable_ref ignore_remote_return_code=$ignore_remote_return_code duplex_wait_for_ssh_lock=$duplex_wait_for_ssh_lock");

  $rc = determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 0);
  log_msg_info("execute_remote_duplex_cmd rc=$rc multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q");

  if ($isMultiNode == 0) {
    print STDERR "Not running in redundancy/multi-node scenario.\nSkipping execution of remote command $cmd\n";
  }

  if ($rc == 0 && $isMultiNode == 1) {

    # make first a connectivity check to avoid time consuming wait for lock in case remote node is not reachable at all
    my $node_is_reachable = is_node_reachable($other_node);
    if ($node_is_reachable == 0) {
      log_msg_info("skip wait_for_ssh_lock since node $other_node is NOT reachable");
      $rc  = $ERROR_CanNotReachRemoteNode;
      $$other_node_is_reachable_ref = 0;
    } else {
      log_msg_info("continuing since node $other_node is reachable");
      $rc = wait_for_ssh_lock($sshd_lock_file, $other_node, $duplex_wait_for_ssh_lock);
    }
  }

  if ($rc == 0 && $isMultiNode == 1) {
    create_lock_file($sshd_lock_file);
    $rc = execute_remote_cmd($other_node, $other_node_is_reachable_ref, $password_is_expired_ref, $cmd, \$output, $max_try, 60, $ignore_remote_return_code);
    if (!defined($rc)) { $rc = $ERROR_GenericRemoteExecutionFailure; }

    log_msg_info("execute_remote_duplex_cmd got output=$output");    
    remove_lock_file($sshd_lock_file);
  }  
  
  function_leave("execute_remote_duplex_cmd", "rc=$rc node_is_reachable=$$other_node_is_reachable_ref password_is_expired=$$password_is_expired_ref output=$output");
  $$output_ref = $output;
  return($rc);
}

sub execute_remote_quorum_cmd {
  my $cmd = shift; 
  my $output_ref = shift;
  my $max_try = shift;
  my $other_node_is_reachable_ref = shift;
  my $password_is_expired_ref = shift;
  my $ignore_remote_return_code = shift;
  my $skip_ssh_lock = shift;

  if (!defined($ignore_remote_return_code)) {
    $ignore_remote_return_code = 0;
  }
  if (!defined($skip_ssh_lock)) {
    $skip_ssh_lock = 0;
  }

  my $output = "";
  my $rc = 0;
  function_enter("execute_remote_quorum_cmd", "cmd=$cmd max_try=$max_try skip_ssh_lock=$skip_ssh_lock");
    
  $rc = determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 0);
  log_msg_info("inside execute_remote_quorum_cmd rc=$rc multiNodeType=$multiNodeType, isMultiNode=$isMultiNode, other_node=$other_node, node_q=$node_q cmd=$cmd");

  if ($isMultiNode == 0) {
    print  STDERR "Not running in redundancy/multi-node scenario.\nSkipping execution of remote command $cmd\n";
    $rc=1;
  }

  if (has_quorum($node_q, $multiNodeType) == 0) {
    print  STDERR "Not running in quorum scenario.\nSkipping execution of remote command $cmd\n";
    $rc=1;
  }
    
  if ($rc == 0 && $isMultiNode == 1) {

    # make first a connectivity check to avoid time consuming wait for lock in case remote node is not reachable at all
    my $node_is_reachable = is_node_reachable($node_q);
    if ($node_is_reachable == 0) {
      log_msg_info("skip wait_for_ssh_lock since node $node_q is NOT reachable");
      $rc  = $ERROR_CanNotReachRemoteNode;
    } else {
      log_msg_info("continuing since node $other_node is reachable");

      if ($skip_ssh_lock == 0) {
        $rc = wait_for_ssh_lock($sshq_lock_file, $node_q);
      }
    }
  }  
     
  if ($rc == 0 && $isMultiNode == 1) {  
    if ($skip_ssh_lock == 0) {
      create_lock_file($sshq_lock_file);
    }
    $rc = execute_remote_cmd($node_q, $other_node_is_reachable_ref, $password_is_expired_ref, $cmd, \$output, $max_try, 60, $ignore_remote_return_code);
    if ($skip_ssh_lock == 0) {
      remove_lock_file($sshq_lock_file);
    }
  }  
  function_leave("execute_remote_quorum_cmd", "rc=$rc node_is_reachable=$$other_node_is_reachable_ref password_is_expired=$$password_is_expired_ref output=$output");
  $$output_ref = $output;
  return($rc);
}

sub execute_remote_copy {
  my $node = shift;
  my $node_is_reachable_ref = shift;
  my $from_path  = shift;
  my $to_path = shift;   
  my $type = shift;
  my $user = shift;
  my $provided_private_key_file = shift;
  my $skip_sshkeygen     = shift;
  my $skip_sshkeycleanup = shift;
  my $node_is_reachable = 0;

  if (!defined($skip_sshkeygen)) {
    $skip_sshkeygen = 0;
  }
  if (!defined($skip_sshkeycleanup)) {
    $skip_sshkeycleanup = 0;
  }
 
  my $rc = 0;
  my $cmd = "";
  my $password_is_expired = 0;
  if (!defined($provided_private_key_file)) { $provided_private_key_file = ""; }
  if (!defined($user)) { $user = "root"; }
  
  function_enter("execute_remote_copy", "type=$type node:$node from $from_path to $to_path, user=$user provided_private_key_file=$provided_private_key_file skip_sshkeygen=$skip_sshkeygen skip_sshkeycleanup=$skip_sshkeycleanup node_is_reachable=$$node_is_reachable_ref");

  if ($skip_sshkeygen == 0) {
    $rc = prepare_remote_ssh_access($node, 1, 60, \$password_is_expired, $user, $provided_private_key_file);
    if ( $rc != 0) {
      $$node_is_reachable_ref = 0;
    } else {
      $$node_is_reachable_ref = 1;
    }
  } else {
    log_msg_info("execute_remote_copy: skipping ssh key generation");
  }
  
  if($rc == 0) {
    if ($type eq "put") {
      $cmd = $rsync_cmd." -r -v $from_path $user\@$node:$to_path";
    } else {
      $cmd = $rsync_cmd." -r -v $user\@$node:$from_path $to_path";
    }
    # $cmd .= " ; echo \$?;";
    $rc = exec_cmd($cmd);
  }

  if ($skip_sshkeycleanup == 0) {
    cleanup_remote_ssh_access($node, $$node_is_reachable_ref, $user, $provided_private_key_file);
  }  else {
    log_msg_info("execute_remote_copy: skipping ssh key cleanup");
  }
  
  if($rc != 0) {
    $rc = $ERROR_CanNotCopyFromRemoteNode;
  }
  
  function_leave("execute_remote_copy", "type=$type node:$node from $from_path to $to_path password_is_expired=$password_is_expired rc=$rc ");
  return ($rc);    
  }

sub execute_remote_cmd_in_background {
  my $cmd = shift;
  my $node = shift;
  my $ignore_remote_return_code = shift;
  
  function_enter("execute_remote_cmd_in_background", "node=$node cmd=$cmd");

  my $output = exec_cmd_out($cmd);
  log_msg_info("asynchron remote cmd returned");

  if (!defined( $output )) {
    log_msg_info("since output of remote cmd is not defined assuming empty string and rc=1");
    $output = "";
  }

  function_leave("execute_remote_cmd_in_background", "node=$node finished");
  return ($output);
}

sub execute_remote_cmd {
  my $node = shift;
  my $node_is_reachable_ref = shift;
  my $password_is_expired_ref = shift;
  my $cmd = shift;   
  my $output_ref = shift;
  my $max_try = shift;
  my $period = shift;
  my $ignore_remote_return_code = shift;
  my $suppress_diagnosis_output = shift;
  my $skip_sshkeygen     = shift;
  my $skip_sshkeycleanup = shift;
  my $output = "";  
  my $rc = 0;
  my $count = 0;

  if (!defined($ignore_remote_return_code)) {
    $ignore_remote_return_code = 0;
  }
  if (!defined($suppress_diagnosis_output)) {
    $suppress_diagnosis_output = 0;
  }
  if (!defined($skip_sshkeygen)) {
    $skip_sshkeygen = 0;
  }
  if (!defined($skip_sshkeycleanup)) {
    $skip_sshkeycleanup = 0;
  }

  function_enter("execute_remote_cmd", "remote on node:$node cmd:$cmd max_try=$max_try period=$period ignore_remote_return_code=$ignore_remote_return_code suppress_diagnosis_output=$suppress_diagnosis_output max_wait_for_ssh_client_execution=$max_wait_for_ssh_client_execution skip_sshkeygen=$skip_sshkeygen skip_sshkeycleanup=$skip_sshkeycleanup ---");

  if ($skip_sshkeygen == 0 ) {
    $rc = prepare_remote_ssh_access($node, $max_try, $period, $password_is_expired_ref, "root", "", $suppress_diagnosis_output);
    if ( $rc != 0) {
      $$node_is_reachable_ref = 0;
    } else {
      $$node_is_reachable_ref = 1;
    }
  }
  
  if($rc == 0) {
    if ($ignore_remote_return_code == 1) {
      $cmd .= " ; ";
    } else {
      $cmd .= " ; echo 'remote rc: '\$?;";
    }

    my $local_cmd = "$ssh_cmd -q root\@$node \'$cmd\'";
    
    if ($max_wait_for_ssh_client_execution > 0 ) {

      # redefine the ssh command to be sure to use the copied ssh client from the ws
      if (! -x $ssh_executable_ws) {
        log_msg_info("copied ssh client from ws installation missing: $ssh_executable_ws, making a copy now");
        exec_cmd("/bin/cp -vfa $ssh_executable_linux $ssh_executable_ws");
        exec_cmd("chmod +x $ssh_executable_ws");
      }
      $local_cmd = "$ssh_executable_ws $ssh_option -q root\@$node \'$cmd\'";
      # do in background
      my $thr = threads->new(\&execute_remote_cmd_in_background, $local_cmd, $node, $ignore_remote_return_code); # Spawn the thread
      my $tid = $thr->tid;
      #$thr->detach; # Now we officially don't care any more
      log_msg_info("*** spawning background thread tid=$tid");

      while($count < $max_wait_for_ssh_client_execution ) {
        # log_msg_info("execute_remote_cmd_in_background $node count=$count", 0, 0);

        if ($thr->is_joinable()) {
          $output = $thr->join();
          last;
        } else {
          $count++;
          sleep(1);
          log_msg_info("waiting for background execution $tid to finish, count=$count of $max_wait_for_ssh_client_execution");
        }
      }
      if ($count >= $max_wait_for_ssh_client_execution) {
        log_msg_info("waiting for background execution timout $max_wait_for_ssh_client_execution of thread $tid reached");
        log_msg_info("discarding the background thread $tid");
        $thr->detach();
        # $thr->kill('SIGTERM');
        log_msg_info("killing all ssh clients, which were invoked by asynchroneous swupdated instances");
        exec_cmd("/usr/bin/killall -v $ssh_executable_ws");
        $rc = $ERROR_RemoteNodeExecutionTimeout;
      }
      
    } else {
      # do in foreground
      
      $output = exec_cmd_out($local_cmd);
      log_msg_info("synchron remote cmd returned");
    }
  }

  if (!defined( $output )) {
    log_msg_info("since output of remote cmd is not defined assuming empty string and rc=1");
    $output = "";
    $rc = 1;
  }
      
  if($rc == 0 && $ignore_remote_return_code == 0) {
    $output =~ /remote rc: ([0-9]*)/;
    if (!defined($1)) {
      log_msg_info("undefined rc");
    } else {
      $rc = $1;
    }
  }

  if (!defined($rc) || $rc eq "") {
    log_msg_info("undefined or empty rc, setting error code");
    $rc = $ERROR_GenericRemoteExecutionFailure;
  }

  if ($skip_sshkeycleanup == 0 ) {  
    cleanup_remote_ssh_access($node, $$node_is_reachable_ref);
  }
  function_leave("execute_remote_cmd", "rc=$rc, remote on node:$node cmd:$cmd node_is_reachable=$$node_is_reachable_ref password_is_expired=$$password_is_expired_ref output=$output");
  $$output_ref = $output;
  return ($rc);    
}


sub enable_ssh_wait_cancel {
  my $wait_file = shift;
  function_enter("enable_ssh_wait_cancel", "wait_file=$wait_file");

  $ssh_cancel_wait_file = $wait_file;
  
  function_leave("enable_ssh_wait_cancel", "");
  }

sub disable_ssh_wait_cancel {
  function_enter("disable_ssh_wait_cancel", "current value = $ssh_cancel_wait_file");

  $ssh_cancel_wait_file = "";

  function_leave("disable_ssh_wait_cancel", "");
  }  

sub prepare_remote_ssh_access {
  my $node = shift;
  my $max_try = shift;
  my $period = shift; # seconds
  my $password_is_expired_ref = shift;
  my $user = shift;
  if (!defined($user)) { $user = "root"; }
  my $provided_private_key_file = shift;
  if (!defined($provided_private_key_file)) { $provided_private_key_file = ""; }
  my $suppress_diagnosis_output = shift;
  if (!defined($suppress_diagnosis_output)) { $suppress_diagnosis_output = 0; }
  
  
  my $rc = 0;
  my $connectivity_failed_at_least_once = 0;
  function_enter("prepare_remote_ssh_access", "for node:$node max_try=$max_try period=$period (sec) user=$user provided_private_key_file=$provided_private_key_file suppress_diagnosis_output=$suppress_diagnosis_output");

  for(my $i = 0; $i < $max_try; $i++) {
    $rc = prepare_remote_ssh_access_internal($node, $password_is_expired_ref, $user, $provided_private_key_file, $suppress_diagnosis_output);

    if($rc == 0) {
      log_msg_info("connectivity successful i=$i");
      if ($connectivity_failed_at_least_once == 1) {
        log_msg_info("connectivity successful after failures before, obviously the remote system is starting up currently -> waiting additionally 1 minute to give it time to start services properly (not only the ip stack and SSH daemon) ");
        sleep(60);
      }
      last;
    } else {
      $connectivity_failed_at_least_once = 1;
      log_msg_info("connectivity failed i=$i of $max_try ... waiting");
    }
    if ($max_try != 1) {
      # only wait for non trivial (!= 1) requests
      sleep($period);
      trigger_watchdog();
      if ( $connectivity_failed_at_least_once == 1) {
        if ($ssh_cancel_wait_file ne "" and -e $ssh_cancel_wait_file) {
          log_msg_info("The ssh cancel wait file $ssh_cancel_wait_file exists -> I will terminate the wait for the remote node and remove this trigger file");
          removeFile($ssh_cancel_wait_file);
          last;
        }
      }
    }
  }  

  if ($rc != 0 and $suppress_diagnosis_output != 1) {
    print_additional_ssh_diagnostic();
  }
  function_leave("prepare_remote_ssh_access", "rc=$rc password_is_expired=$$password_is_expired_ref");
  return ($rc);    
}

sub print_additional_ssh_diagnostic {
    log_msg_info("remote ssh access connectivity failed -> print additional diagnostic file");
    exec_cmd ("/bin/cat /etc/hosts");
    exec_cmd ("/sbin/ifconfig");
    exec_cmd ("/bin/ls -al /root/.ssh/");
}

sub is_group_existing {
  my $group = shift;
  function_enter("is_group_existing", "group=$group");
  my $is_existing = 0;

  my $rc = exec_cmd ("/usr/bin/getent group $group");
  if ($rc == 0) {
    $is_existing = 1;
  }
  function_leave("is_group_existing", "is_existing=$is_existing");
  return ($is_existing);
}

sub is_user_existing {
  my $user = shift;
  function_enter("is_user_existing", "user=$user");
  my $is_existing = 0;

  my $rc = exec_cmd ("/usr/bin/getent passwd $user");
  if ($rc == 0) {
    $is_existing = 1;
  }
  function_leave("is_user_existing", "is_existing=$is_existing");
  return ($is_existing);    
}

sub delete_linux_user {
  my $user = shift;
  function_enter("delete_linux_user", "user=$user");
  my $rc = 0;

  $rc = exec_cmd("/usr/sbin/userdel -r --force $user ");
  
  function_leave("delete_linux_user", "rc=$rc");
  return ($rc);
}

sub create_linux_user {
  my $user = shift;
  my $user_id_ref = shift;
  my $group_id = shift;
  function_enter("create_linux_user", "user=$user requested_uid=$$user_id_ref gid=$group_id in current working dir");
  my $rc = 0;

  if (is_user_existing($user) == 1) {
    log_msg_info("user $user already exists -> do not need to create him");
    log_msg_info("delete the potentially misprotected .ssh directory");
    $rc = exec_cmd("rm -rf /home/$user/.ssh");
  } else {
    log_msg_info("user $user does not exist -> create him");
    my $passwd_crypted = crypt("geheim", "salz");

    # check the group
    my $group_option = "";
    if (is_group_existing($group_id) == 1) {    
      log_msg_info("group $group_id exists -> I can use it");
      $group_option = "--gid=$group_id";
    } else {
      log_msg_info("group $group_id does not exist, I have to to create it");
       $rc = exec_cmd("/usr/sbin/groupadd --gid $group_id hp4k_users");
       if ($rc == 0) {
          log_msg_info("groupadd for hp4k_users succeeded");
          $group_option = "--gid=$group_id";
       }
    }

    my $log_shell_option = "-s /bin/false";
    $log_shell_option = ""; # unfortunately such a restricted user can not be used by rsync via ssh
    
    if ($$user_id_ref == -1) {
      # uid not specified
      $rc = exec_cmd("/usr/sbin/useradd -c 'user for transfer of OS images via SFTP' -p '$passwd_crypted' --create-home $log_shell_option --inactive=-1 $group_option $user ");
      if ($rc != 0) {
        log_msg_info("useradd $user failed");
        $rc = $ERROR_CreateUserFailed;
      } else {
        log_msg_info("useradd $user succeeded");
      }
    } else {
      $rc = exec_cmd("/usr/sbin/useradd -c 'user for transfer of OS images via SFTP' -p '$passwd_crypted' --create-home $log_shell_option --inactive=-1 --uid=$$user_id_ref $group_option $user ");
    }
  }

  # get the uid
  if ($rc == 0) {
    my $output = exec_cmd_out ("/usr/bin/getent passwd $user");
    my @parts = split(':', $output);
    if ( !defined($parts[2]) or (defined($parts[2]) && $parts[2] eq "")) {
      log_msg_info("can not determine uid for $user ");
    } else {
      $$user_id_ref = 0 + $parts[2];
      log_msg_info("uid for $user: $$user_id_ref");
    }
  }

  function_leave("create_linux_user", "rc=$rc uid=$$user_id_ref");
  return ($rc);   
}


sub cleanup_local_ssh_credentials {
  my $user = shift;
  function_enter("cleanup_local_ssh_credentials", "user=$user");

  # doesn't remove authorized_keys for root user, it collide with new sshkey mechanism
  if ($user eq "root") {
    return(0);
  }
  my $auth_keyfile = "/home/$user/.ssh/authorized_keys";
  my $rc = removeFile($auth_keyfile);
  function_leave("cleanup_local_ssh_credentials", "rc=$rc");
  return ($rc);    
}


sub prepare_local_ssh_credentials {
  my $sentinel = shift;
  my $user = shift;
  my $user_id = shift;
  my $group_id = shift;
  function_enter("prepare_local_ssh_credentials", "for sentinel=$sentinel user=$user uid=$user_id gid=$group_id in current working dir");
  my $rc = 0;
  my $cmd_keygen = "/opt/webservice/ssh-keygen/bin/ssh-keygen";

  # try if pubkey authentication isn't working autmatically

  determine_ha_nodes($multiNodeType, $isMultiNode, $other_node, $node_q, 0);

  $rc = exec_cmd("$cmd_keygen $sentinel $user $user_id $group_id");

  my $private_key = "/tmp/sshkey_sg_os_update";
  my $public_key = $private_key.".pub";

  if($rc == 0 && -e $private_key && -e $public_key) {
    log_msg_info("keygen succeeded, keyfiles $private_key and $public_key exist -> moving files to working dir");
          exec_cmd ("ls -l $private_key");
    $rc = exec_cmd ("/bin/mv -vf $private_key .");
          exec_cmd ("ls -l $public_key");
    $rc = exec_cmd ("/bin/mv -vf $public_key .");
  } else {
    log_msg_info("keygen failed or private or public key file does not exist, maybe swupdated is newer than webservice rpm version");
    $rc = $ERROR_SSH_KeygenFailed;
  }

  function_leave("prepare_local_ssh_credentials", "rc=$rc");
  return ($rc);  	
}

sub generate_keys_via_curl {
  my $node = shift;
  my $suppress_diagnosis_output = shift;
  if (!defined($suppress_diagnosis_output)) { $suppress_diagnosis_output = 0; }
  function_enter("generate_keys_via_curl", "node=$node");

  my $pkey = "";
  my ($rc, $is_fallback) = (0, 0);
  
  my $get_key_cmd      = "/usr/bin/curl -m 15 --silent --insecure https://$node/ssh-keygen/cgi-bin/ssh-keygen.cgi";
  my $get_key_alt_cmd  = "/usr/bin/curl -m 15 --silent --insecure http://$node/ssh-keygen/cgi-bin/ssh-keygen.cgi";
  my $get_key_diag_cmd = "/usr/bin/curl -m 15          --insecure https://$node/ssh-keygen/cgi-bin/ssh-keygen.cgi 2>&1";

  my $begin_pkey = "-----BEGIN OPENSSH PRIVATE KEY-----";
  my $end_pkey   = "-----END OPENSSH PRIVATE KEY-----";
  if ($sles_major_version == 11)  {
    # on SLES11 the keygen generates RSA keys
    $begin_pkey = "-----BEGIN RSA PRIVATE KEY-----";
    $end_pkey   = "-----END RSA PRIVATE KEY-----";
  }

  # generate keys for ssh usage
  my $html = exec_cmd_out($get_key_cmd);

  if (length($html) == 0) {
    log_msg_info("html length is zero: failed to access keygen vi https -> fallback to http for node:$node");
    $is_fallback = 1;
    $html = exec_cmd_out($get_key_alt_cmd);
  }

  if(length($html) > 0) {
    my $found = 0;
    my @html_lines = split(/\n/, $html);
    foreach(@html_lines) {
      if($found == 0 && index($_, $begin_pkey) >= 0) {
        $found = 1;
      }
      if($found == 1 && index($_, $end_pkey) >= 0) {
          $pkey .= $_."\n";
          $found = 0;
      }
      if($found == 1) {
        $pkey .= $_."\n";
      }
    }
  }

  if(length($pkey) == 0) {
    if ($is_fallback) {
      log_msg_info("pkey length is zero: failed to parse keys from response, but already fallback/http -> give up");
    } else {
      log_msg_info("pkey length is zero: failed to parse keys from response -> fallback to http for node:$node");
      $html = exec_cmd_out($get_key_alt_cmd);
      $is_fallback = 1;

      # parse again the (new http) response
      if(length($html) > 0) {
        my $found = 0;
        my @html_lines = split(/\n/, $html);
        foreach(@html_lines) {
          if($found == 0 && index($_, $begin_pkey) >= 0) {
            $found = 1;
          }
          if($found == 1 && index($_, $end_pkey) >= 0) {
              $pkey .= $_."\n";
              $found = 0;
          }
          if($found == 1) {
            $pkey .= $_."\n";
          }
        }
      }
    }
  }

  if(length($pkey) > 0) {
    if (! -e $pkey_dir ) {
      log_msg_info("ssh config dir $pkey_dir does not exist -> create it");
      makeDir ($pkey_dir);
      log_msg_info("ssh config dir $pkey_dir has to be protected well with 700");
      chmod(0700, $pkey_dir);
    }
    if(open(PKEY, ">$pkey_file")) {
      print PKEY $pkey;
      close(PKEY);
    } else {
      log_msg_info("error: could not write private key to $pkey_file");
    }
  } else {
    $rc = 1;
    if ($suppress_diagnosis_output == 0) {
      log_msg_info("prepare_remote_ssh_access for node:$node failed due to lenght=0 of generated key. rc=$rc");
      log_msg_info("generating diagnosis curl call without silent option");
      $html = exec_cmd_out($get_key_diag_cmd);
      log_msg_info("answer from curl: $html");
    }
  }
  
  function_leave("generate_keys_via_curl", "rc=$rc");
  return ($rc);    
}

sub prepare_remote_ssh_access_internal {
  my $node = shift;
  my $password_is_expired_ref = shift;
  my $user = shift;
  my $provided_private_key_file = shift;
  my $suppress_diagnosis_output = shift;
  my $rc = 0;

  if (!defined($user)) { $user = "root"; }
  if (!defined($provided_private_key_file)) { $provided_private_key_file = ""; }
  if (!defined($suppress_diagnosis_output)) { $suppress_diagnosis_output = 0; }
  
  function_enter("prepare_remote_ssh_access_internal", "for node=$node user=$user provided_private_key_file=$provided_private_key_file");

  # make first a connectivity check to avoid time consuming curls in case of node down
  my $node_is_reachable = is_node_reachable($node);
  if ($node_is_reachable == 0) {
    log_msg_info("skip curls commands since node $node is NOT reachable");
    return ($ERROR_CanNotReachRemoteNode);
  } else {
    log_msg_info("continuing since node $node is or is assumed reachable");
  }

  # save current private key
  if (-e $pkey_file) {
    removeFile($pkey_file_save);
    renameFile($pkey_file, $pkey_file_save);
  }

  if ($provided_private_key_file eq "") {
    log_msg_info("no private key file given -> generate keys via curl");

    # no private key file provided -> generate it via curl
    $rc = generate_keys_via_curl($node, $suppress_diagnosis_output);
    if ($rc != 0) {
      log_msg_info("got no private key from remote node $node");
      # restore the saved private key file
      if (-e $pkey_file_save) {
        removeFile($pkey_file);
        renameFile($pkey_file_save, $pkey_file);
      }
      return ($rc);
    }
  } else {
    log_msg_info("private key file given -> use it");
    if (! -e $provided_private_key_file) {
      log_msg_info("private key file $provided_private_key_file does not exist -> error");
      $rc = $ERROR_CanNotCopyFromRemoteNode;
    } else {
      $rc = exec_cmd("/bin/cp -vfa $provided_private_key_file $pkey_file");
    }
  }

  if ($rc == 0) {
    log_msg_info("protect private key file $pkey_file with 600");
    chmod(0600, $pkey_file);
  }
  exec_cmd("ls -al $pkey_file*"); # diagnosis

  # delete old entries in known_hosts
  if($rc == 0) {
    remove_ssh_key_entry_from_local_known_hosts_file($node);
  }
  # check ssh connection (will add host to known_hosts)
  if($rc == 0) {
    $$password_is_expired_ref = 0;
    my $hoststore_cmd = "$ssh_cmd -o 'StrictHostKeyChecking no' $user\@$node /bin/hostname 2>&1";
    $rc = exec_cmd($hoststore_cmd);
    if ($rc != 0) {
      # try to find out if password is expired situation

      log_msg_info("ssh client for establishing session to node $node failed, retrying for diagnosis purposes, but check reachability again");

      $node_is_reachable = is_node_reachable($node);
      if ($node_is_reachable == 0) {
        log_msg_info("skip second diagnosis ssh command since node $node is NOT reachable");
        return ($ERROR_CanNotReachRemoteNode);
      } else {
        log_msg_info("continuing since node $node is or is assumed reachable");
  
        my $output = exec_cmd_out($hoststore_cmd);
        my @output_lines = split(/\n/, $output);
        foreach(@output_lines) {
          if(index($_, 'WARNING: Your password has expired') >= 0) {
            log_msg_info("detected an expired or uninitialized root password on remote node $node");
            $$password_is_expired_ref = 1;
          }
        }
        if ($$password_is_expired_ref != 1) {
          log_msg_info("remote ssh command on node $node failed, but it is not an expired password situation");
          log_msg_info("output=$output");

          log_msg_info("checking for possible outdated public key file");
          if (-f $pubkey_file) {
            log_msg_info("found possible outdated public key file, remove it");
            removeFile($pubkey_file);
            log_msg_info("now retry the test ssh command");
            $rc = exec_cmd($hoststore_cmd);
            if ($rc == 0) {
              log_msg_info("after workaround ssh is working, OK");
            } else {
              log_msg_info("after workaround ssh still not working");
            }
          }
        }
      }      
    }
  }
  function_leave("prepare_remote_ssh_access_internal", "node=$node rc=$rc");
  return ($rc);  
}

sub cleanup_remote_ssh_access {
  my $node = shift;
  my $node_is_reachable = shift;
  my $user = shift;
  my $provided_private_key_file = shift;  
  my $rc = 0;
  if (!defined($provided_private_key_file)) { $provided_private_key_file = ""; }

  function_enter("cleanup_remote_ssh_access", "for node:$node node_is_reachable=$node_is_reachable provided_private_key_file=$provided_private_key_file");
  
  # remove public authkey from remote machine if remote machine is reachable and private kay was generated via curl
  if ($node_is_reachable == 1 and $provided_private_key_file eq "") {
    exec_cmd("$ssh_cmd -q root\@$node /usr/bin/sed -i /haupdatekey/d /root/.ssh/authorized_keys");
  } else {
    log_msg_info("skipping cleanup of authkey on remote node $node since this node is not reachable or cleanup not necessary -> only local cleanup ");
  }

  remove_ssh_key_entry_from_local_known_hosts_file($node);

  # cleanup used private key
  if(-e $pkey_file_save) {
    removeFile($pkey_file);
    removeFile($pubkey_file);    
    renameFile($pkey_file_save, $pkey_file);
  }
  function_leave("cleanup_remote_ssh_access", "rc=$rc");
  return ($rc);  
}

sub remove_ssh_key_entry_from_local_known_hosts_file {
  my $node = shift;
  my $known_hosts_file = "/root/.ssh/known_hosts";
  my $rc = 1;
  my $be_silent = 1;
  if (-e $known_hosts_file) {
    # only in case that the known_hosts file really exists, I can call "ssh-keygen -R" with full logging. Otherwise error would occur.
    $be_silent = 0;
  } else {
    $be_silent = 1;
    log_msg_info("using silent mode for key removal in known_hosts file to avoid error messagess");
  }

  if ($be_silent == 1) {
    # delete old entries in known_hosts
    exec_cmd("/usr/bin/ssh-keygen -R $node", "silent");
    # delete also a lowercase version of the hostname in case that the hostname is in upper case.
    my $lowercase_node = lc($node);
    exec_cmd("/usr/bin/ssh-keygen -R $lowercase_node", "silent");
  } else {
    # delete old entries in known_hosts
    exec_cmd("/usr/bin/ssh-keygen -R $node");
    # delete also a lowercase version of the hostname in case that the hostname is in upper case.
    my $lowercase_node = lc($node);
    exec_cmd("/usr/bin/ssh-keygen -R $lowercase_node");
  }

  return ($rc);
}


sub transfer_remote_sysinfo_upgrade_file_to_master_node {
  my $node = shift;
  my $node_is_a_quorum = shift;
  
  function_enter("transfer_remote_sysinfo_upgrade_file_to_master_node", "$node node_is_a_quorum=$node_is_a_quorum");

  my $remote_log     = $sysinfo_standby_log;
  my $remote_pkg_log = $sysinfo_pkg_standby_log;
  if ($node_is_a_quorum == 1) {
    $remote_log     = $sysinfo_quorum_log;
    $remote_pkg_log = $sysinfo_pkg_quorum_log;
  }
  my $max_tries = 1000;
  my $count = 0;

  my $use_altfile_for_logging = 1;
  
  while($count < $max_tries && $die_background_thread == 0) {
    log_msg_info("transfer_remote_sysinfo_upgrade_file_to_master_node $node count=$count", 0, 0, $use_altfile_for_logging);
    my $statustransfer_rc = exec_cmd($rsync_cmd." -r -v  root\@$node:$sysinfo_log $remote_log", "alt");
    if ($statustransfer_rc != 0) {
      log_msg_info("error when transfering the remote sysinfo status file $sysinfo_log from node $node to active node: rc=$statustransfer_rc", 0, 0, $use_altfile_for_logging);
    }

    $statustransfer_rc = exec_cmd($rsync_cmd." -r -v  root\@$node:$sysinfo_pkg_log $remote_pkg_log",  "alt"); 
    if ($statustransfer_rc != 0) {
      log_msg_info("could not transfer the remote sysinfo package status file $sysinfo_pkg_log from node $node to active node: rc=$statustransfer_rc", 0, 0, $use_altfile_for_logging);
      # obviously the sysinfo package status file does not exists anymore on the remote node -> remove it also on the local node
      removeFile($remote_pkg_log);
    } else {
      my $content = read_file_content($remote_pkg_log);
      log_msg_info("$content", 0, 0, $use_altfile_for_logging);
    }
    
    if ($die_background_thread == 1) {  last;  }
    sleep(3);
    if ($die_background_thread == 1) {  last;  }

    trigger_watchdog();

    $count++;      
  }
  function_leave("transfer_remote_sysinfo_upgrade_file_to_master_node", "$node finished after $count cycles");
}

sub activate_platform_hardening {
  my $rc = 0;
  function_enter("activate_platform_hardening", "for RLC without NUC ");
  $rc = exec_cmd("/opt/linux-hardening/Start_modify.sh -x -g");
  if ($rc != 0) {
    log_msg_info("platform_hardening returned error, but ignoring and continuing anyway");
  }
  function_leave("activate_platform_hardening");
  return($rc);  
}

sub ha_update_node {
  my $node = shift;
  my $node_is_reachable_ref = shift;  
  my $node_is_a_quorum = shift;
  my $logfile_to_use = shift;
  my $password_is_expired = 0;
  my ($rc, $cleanup_rc) = (0, 0);
  my $wait_for_remote_node_timeout = 20 ; # min

  function_enter("ha_update_node", "start update node $node");
  
  $rc = prepare_remote_ssh_access($node, $wait_for_remote_node_timeout, 60, \$password_is_expired);
  if ( $rc != 0) {
    $$node_is_reachable_ref = 0;
  } else {
    $$node_is_reachable_ref = 1;  
  }
  
  if($rc == 0) {
    $die_background_thread = 0;
  
    my $thr = threads->new(\&transfer_remote_sysinfo_upgrade_file_to_master_node, $node, $node_is_a_quorum); # Spawn the thread
    my $tid = $thr->tid;
    log_msg_info("*** spawning background transfer thread tid=$tid");
  }
    
  # create directories if not exists on remote host
  if($rc == 0) {
    $rc = exec_cmd("$ssh_cmd -q root\@$node mkdir -p $rpm_dir $hotfix_dir $soco2_data_dir");
  }
  # transfer installation packages
  if($rc == 0) {
    
    if($hotfix_mode == 2) {
      my $rm_cmd = "/bin/rm -rfv $rpm_dir"."*";
      log_msg("remove the freshly extracted upgrade dir to avoid syncing it to remote node");
      $rc = exec_cmd($rm_cmd);
      $upgrade_dir_has_been_removed_due_to_duplex_hf_deactivation = 1;
    }
    
    # log_msg_info("do not transfer assistant SWS area to remote node anymore (DRBD will sync it)");
    log_msg_info("not transferring assistant SWS area to remote node anymore. It is done by DRBD now.");
    # $rc = exec_cmd($rsync_cmd." -r -v $assistant_sws_area root\@$node:$assistant_sws_area");    

    my $cur_self = $yaps_self;
    if ($is_old && -e $old_self) {
      $cur_self = $old_self;
    }
      
    log_msg_info("copying current upgrade daemon from /opt to /var/opt/soco2 (for later rsync)");
    
    log_msg_info("will analyze if current script is the fast-swupdated-client (small script) or the regular script (the big one) ...");
    my $nrofscriptlines=exec_cmd_out("wc -l $cur_self");
    $nrofscriptlines =~ /^(\d+).*/;
    $nrofscriptlines=$1;
    log_msg_info("$cur_self script has ($nrofscriptlines) number of lines");
    if ( $nrofscriptlines < 1000 ) {
      my $scriptocopy=$self.".swupdated_slow";
      log_msg_info("$cur_self is obviously the small script => will use the $scriptocopy for the remote update");
      $rc = exec_cmd("cp -vf $scriptocopy $remote_self_upg");
    } else {
      log_msg_info("$cur_self is obviously the big script => it will be used as-is for the remote node update");
      $rc = exec_cmd("cp -vf $cur_self $remote_self_upg");
    }
    chmod(0750,$remote_self_upg);
    
    if ($rc != 0) {
      log_msg_info("*** error when transferring the current upgrade daemon from active node to remote node, but ignoring it and using the existing upgrade daemon (e.g. multi quorum with readonly FS)");
    }
    
    log_msg_info("transferring pg sql restore zip file from active node to passive node (for restore)");
    $rc = exec_cmd($rsync_cmd." -r -v $pgsql_data_zip_file root\@$node:$pgsql_data_zip_file");   
    if ($rc != 0) {
      log_msg_info("*** error when transferring the pg sql restore zip file from active node to passive node, but ignoring it ");
    }

    log_msg_info("transferring pg sql restore zip save file from active node to passive node (for diagnosis only)");
    $rc = exec_cmd($rsync_cmd." -r -v $pgsql_data_zip_save root\@$node:$pgsql_data_zip_save");
    if ($rc != 0) {
      log_msg_info("*** error when transferring the pg sql restore save zip file from active node to passive node, but ignoring it ");
    }

    trigger_watchdog();
    
    log_msg_info("transferring complete soco2 dir (without tmp) from active node to passive node");
    $rc = exec_cmd($rsync_cmd." -r -v --exclude 'tmp' $soco2_data_dir root\@$node:$soco2_data_dir");    
    if ($rc != 0) {
      log_msg_info("*** error when transferring the complete soco2 dir from active node to passive node ");
    }

    trigger_watchdog();

#    if(-e $rpm_dir) {
#     $rc = exec_cmd($rsync_cmd."$rpm_dir root\@$node:$rpm_dir");
#  } else {
#   log_msg("$rpm_dir does not exist - skipping rsync");
#}
#    if($rc == 0) {
#      if(-e $hotfix_dir) {
#        $rc = exec_cmd($rsync_cmd."$hotfix_dir root\@$node:$hotfix_dir");
#      } else {
#        log_msg("$hotfix_dir does not exist - skipping rsync");
#      }
#   }
  }
  if($rc == 0) {
    # always call the .old daemon (to not overwriting it via a new webservice package)
    my $install_cmd = "$ssh_cmd -q root\@$node $remote_self_upg --nodaemon --no_self_xtract";
    if ($node_is_a_quorum == 1) {
      # Quorum
      $install_cmd .= " --isquorum";
      if (-e $skip_quorum_flagfile) {
        log_msg_info("flag file $skip_quorum_flagfile exists -> therefore instruct the upgrade on the quorum to skip the rpm update"); 
        $install_cmd .= " --skip_quorum_rpm_upgrade";
        my $eff_vers_string = get_current_effective_version();
        $install_cmd .= " --eff_vers $eff_vers_string";
      }
    } else {
      # Standby Node
      if ($biosupdate != 0) {
        $install_cmd .= " --biosupdate";
      }
    }
    $install_cmd .= $params_remote;
    $rc = exec_cmd($install_cmd);
  } else {
    log_msg_info("*** skipping remote installation due to previous error rc=$rc");
  }

  sleep(4);  # wait till the  sysinfo background thread has fetched the final upgrade_info.xml from remote node
  log_msg_info("terminate the sysinfo background thread");
  $die_background_thread = 1;
  sleep(4);

  if ($upgrade_via_image == 1) {
    log_msg_info("for NUC updates I skip the remote status fetching (risk of hanging ssh clients) and assume success (keep rc as 0)");
  } else {

    $cleanup_rc = cleanup_remote_ssh_access($node, $$node_is_reachable_ref);
      
    # build new ssh keys if anyone has removed them during the upgrade ignoring the lock
    $rc = prepare_remote_ssh_access($node, $wait_for_remote_node_timeout, 60, \$password_is_expired);
    if ( $rc != 0) {
      $$node_is_reachable_ref = 0;
    } else {
      $$node_is_reachable_ref = 1;
    }

    # print remote installation status file
    if($rc == 0) {
      my $remote_status = exec_cmd_out("$ssh_cmd -q root\@$node /bin/cat $logfile_to_use");
      my @status_lines = split(/\n/, $remote_status);
      $rc = 123;
      foreach(@status_lines) {
        if(index($_, 'rc') >= 0) {
          my @parts = split(/<|>/, $_);
          $rc = $parts[2];
          log_msg_info("remote status rc: $rc");
        }
      }
    } else {
      log_msg_info("*** skipping output of remote installation status file due to previous error rc=$rc");
    }

    if($rc == 0) {
      log_msg_info("transferring status file from remote node to active node");
      my $statustransfer_rc = exec_cmd($rsync_cmd." -r -v  root\@$node:$logfile_to_use $logfile_to_use");
      if ($statustransfer_rc != 0) {
        log_msg_info("error when transfering the remote status file $logfile_to_use to active node: $statustransfer_rc");
      }
    } else {
      log_msg_info("*** skipping transfer of remote installation status file due to previous error rc=$rc");
    }

    $cleanup_rc = cleanup_remote_ssh_access($node, $$node_is_reachable_ref);
  }
  
  if ($rc != 0) {
    $rc = $ERROR_RemoteUpdateError;
  }
  function_leave("ha_update_node", "update node $node finished rc=$rc");    
  return($rc);
}

sub clean_intermediate_files {
  function_enter("clean_intermediate_files", "cleaning intermediate files");

  removeFile($job_file);

  function_leave("clean_intermediate_files", "cleaning intermediate files finished");
}

sub migrate_assistant_share_from_var_to_pgsql_partition {
  my $isMultiNode = shift;
  my $rc = 0;
  my $migration_necessary = 0;
  function_enter("migrate_assistant_share_from_var_to_pgsql_partition", "isMultiNode=$isMultiNode");
  if ($local_ha_has_been_stopped == 0) {
    log_msg_info("info: assistant share migration can not be done with runnning openais -> skipping");
    function_leave("migrate_assistant_share_from_var_to_pgsql_partition");    
    return;
  }
  if($hotfix_mode == 1) {
    log_msg_info("info: assistant share migration will not be done by a hotfix activation");
    function_leave("migrate_assistant_share_from_var_to_pgsql_partition");    
    return;
  }
  if ($isMultiNode == 1) {
    log_msg_info("info: assistant share migration will not be done on Duplex deployments");
    function_leave("migrate_assistant_share_from_var_to_pgsql_partition");    
    return;
  }

  if ($upgrade_via_image != 0) {
    log_msg_info("info: assistant share migration will not be done during NUC update -> it would make a rollback impossible");
    function_leave("migrate_assistant_share_from_var_to_pgsql_partition");    
    return;
  }

  # check if necessary
  # if /var/opt/Assistant/data/ has content (e.g. SWS) after openais is stopped -> migrate it
  if (-e "$assistant_sws_area") {
    log_msg_info("*** assistant share migration has to be done since $assistant_sws_area exists after openais has been stopped");
    $migration_necessary = 1;
  }
  if (-e "$assistant_data_area"."BACKUP") {
    log_msg_info("*** assistant share migration has to be done since BACKUP exists after openais has been stopped");
    $migration_necessary = 1;
  }
  if (-e "$assistant_data_area"."install") {
    log_msg_info("*** assistant share migration has to be done since install exists after openais has been stopped");
    $migration_necessary = 1;
  }
  
  if ($migration_necessary == 1) {
    if ( -e $assistant_physical_sws_area) {
      log_msg_info("*** assistant share migration would have to be done but the target dir $assistant_physical_sws_area already exists -> target dir has precedence -> ignore old location");
    } else {
      # migrate it
      makeDir($assistant_physical_partition);
      $rc = exec_cmd("/bin/mv -vf $assistant_data_area $assistant_physical_partition");

      log_msg_info("create an empty mountover dir for the bind mount");
      makeDir($assistant_data_area);
    }
  } else {
    log_msg_info("assistant share migration not necessary since $assistant_data_area seems empty after openais stop");
  }
  function_leave("migrate_assistant_share_from_var_to_pgsql_partition");
}

sub migrate_current_hotfix_dir {
  if(! -e $cur_hf_dir) {
    if( -e $cur_hf_dir_old) { 
      log_msg_info("found an old current hotfix dir -> moving it to new location");    
      exec_cmd("/bin/mv -vf $cur_hf_dir_old $cur_hf_dir");      
    }
  }

  if( -e $cur_hf_dir_old) {
      log_msg_info("found an old current hotfix dir -> removing it");
      removeDir($cur_hf_dir_old);
  }
}

sub clean_installrepo_based_on_buildtime {
      my $cmd  = 'export PATH=/usr/bin:/bin; ';
      $cmd .= 'RPM_FILES=`ls '.$backup_dir.'*.rpm`; ';
      $cmd .= 'for RPM in ${RPM_FILES}; do ';
      $cmd .= 'NAME=`rpm -q --qf %{NAME} -p ${RPM}`; ';
      $cmd .= 'BT=`rpm -q --qf %{BUILDTIME} ${NAME}`; ';
      $cmd .= 'BT_FILE=`rpm -q --qf %{BUILDTIME} -p ${RPM}`; ';
      $cmd .= 'if [ ${BT} != ${BT_FILE} ]; then ';
      $cmd .= 'rm -vf ${RPM} >> '.$log_file.' 2>&1; ';
      $cmd .= 'fi; ';
      $cmd .= 'done; ';
      exec_cmd($cmd);  
}

sub clean_update_repos {
  function_enter("clean_update_repos", "cleaning update repositories");
  my $rc = 0;    
  # copy hotfixes to current hotfix directory
  if(! -e $cur_hf_dir) {
    makeDir($cur_hf_dir);
  }
  removeDir( $cur_hf_dir."work" );
  if($global_rc == 0 && $hotfix_mode != 0) {
    exec_cmd("/bin/cp -vf $hotfix_dir* $cur_hf_dir");
    exec_cmd("/bin/rm -vf $hotfix_dir*"); 
  } elsif($hotfix_mode == 0) {
    if($global_rc == 0) {
      # this plausibility rm is not needed as far as I can see it (gg 01.08.2011))
      # clean_installrepo_based_on_buildtime();
    }
    exec_cmd("/bin/rm -vf $cur_hf_dir*");
  }
  if (-e "$rpm_dir"."os_deps") {
    removeDir( "$rpm_dir"."os_deps" );
  }
  if (-e "$rpm_dir"."os_update") {
    removeDir( "$rpm_dir"."os_update" );
  }
  exec_cmd("/bin/rm -vf $rpm_dir*");
  exec_cmd("/bin/rm -vf $rpm_post_dir*");
  
  log_msg("removing image files from $rpm_dir ");
  
  
  my $cmd_rm = "/bin/rm -vf $rpm_dir*-image-*.rpm";  
  $rc = removeFile("$rpm_dir*-image-*.rpm");
  if ($rc != 0) {
    log_msg_info("error when removing image files from $rpm_dir rc=$rc");
  }  
      
  function_leave("clean_update_repos", "cleaning update repositories finished");
}

sub get_current_real_version_as_array {
  function_enter("get_current_real_version_as_array", "");
  my @soco_version = ("", "", "", "");
  if (! -e "/bin/getydata") {
    log_msg_info("get_current_real_version_as_array : no getydata found");
  } else {
    my $plt_version = exec_cmd_pipe("/bin/getydata -v");
    $plt_version = my_trim($plt_version);
    $plt_version =~ /V([0-9]+)_R([0-9]+)\.([0-9]+)\.([0-9]+)/;
    my ($maj, $min, $fr, $hf) = ($1, $2, $3, $4);
    if (! defined($maj) ) { $maj = ""; }
    if (! defined($min) ) { $min = ""; }
    if (! defined($fr)  ) { $fr  = ""; }
    if (! defined($hf)  ) { $hf  = ""; }
    @soco_version = ($maj, $min, $fr, $hf);
  }
  function_leave("get_current_real_version_as_array", "@soco_version");
  return (@soco_version);
}

sub get_current_real_version_without_hotfixnumber {
  function_enter("get_current_real_version_without_hotfixnumber", "");
  
  my $cur_version = get_current_real_version();
  # schnibbel away the trailing hotfix number
  $cur_version =~ s/\.[0-9]*$//;
  function_leave("get_current_real_version_without_hotfixnumber", "cur_version=$cur_version");
  return ($cur_version);
}

sub get_current_real_version {
    my $cur_version = exec_cmd_pipe("/bin/getydata -v");
    $cur_version = my_trim($cur_version);
    return ($cur_version);
}

sub get_current_effective_version {
  
    my $cur_version = "";
    if ($installed_effective_version_at_beginning_of_upgrade eq "") {
      $cur_version = get_current_real_version();
    } else {
      $cur_version = $installed_effective_version_at_beginning_of_upgrade;
    }
    $cur_version = my_trim($cur_version);
    return ($cur_version);
}

sub get_current_version_in_brackets {
    my $cur_version = get_current_effective_version(); 
    $cur_version =~ s/_/ /;
    $cur_version = "[".my_trim($cur_version)."]";
    return ($cur_version);
}

sub get_upgrade_info_file {
  my $info_file_to_analyze = $upginfo_file;
  function_enter("get_upgrade_info_file");
  function_leave("get_upgrade_info_file", "info_file_to_analyze=$info_file_to_analyze");
  return($info_file_to_analyze);
}

sub get_hotfix_info_file {

  my $hfinfo_file_to_analyze = "";
  
  if($hotfix_mode == 1) {
    # activate HF
    $hfinfo_file_to_analyze = $hfinfo_file;
  }
  function_enter("get_hotfix_info_file", "hfinfo_file_to_analyze=$hfinfo_file_to_analyze");
  
  if($hotfix_mode == 2) {
    # deactivate HF
    my $cur_version = get_current_effective_version();
    
    # trying to get the correct stored HF tarfile
    my $cur_hftar_file_to_analyze = glob($cur_hf_dir.'*PLT-'.my_trim($cur_version).'.tar');
    if ( !defined($cur_hftar_file_to_analyze)) {
      $cur_hftar_file_to_analyze = "";
    }
    if (-e "$cur_hftar_file_to_analyze") {
      log_msg("get_hotfix_info_file: necessary hotfix tarfile corresponding to current version $cur_hftar_file_to_analyze exists");
          my $rc = 0;
          my $cmd = 'export PATH=/usr/bin:/bin; ';
          $cmd .= 'WORK="'.$cur_hf_dir.'work/"; ';    
          $cmd .= 'mkdir -p ${WORK} ; ';
          $cmd .= 'echo "extracting current hotfix tar to temporary directory" >> '.$log_file.'; ';
          $cmd .= 'nice -n 10 tar --directory="${WORK}" -x hf.info -vf '.$cur_hftar_file_to_analyze.' >> '.$log_file.' 2>&1 || exit 1; ';
          $rc = exec_cmd($cmd);
          if ( $rc == 0 ){
             $hfinfo_file_to_analyze = $cur_hf_dir.'work/hf.info';
          } else {
            log_msg("get_hotfix_info_file: error: extracting hf info file out of current hotfix tarfile $cur_hftar_file_to_analyze failed");
          }

    } else {
      log_msg("get_hotfix_info_file: can not access the necessary hotfix tarfile $cur_hftar_file_to_analyze to extract current hf.info");
    }
  }
  function_leave("get_hotfix_info_file", "hfinfo_file_to_analyze=$hfinfo_file_to_analyze");
  return ($hfinfo_file_to_analyze);
}

sub relax_checkydata {
  my $yaps_package_to_relax = shift;
  function_enter("relax_checkydata", "yaps_package_to_relax=$yaps_package_to_relax");
  my $yaps_data_file = "/opt/yaps/data";
  my $yaps_data_orig_file = $yaps_data_file.".orig";

  if (! -e $yaps_data_orig_file) {
    log_msg("keeping the original version of the yaps data version file as a copy to $yaps_data_orig_file");
    exec_cmd("cp -v $yaps_data_file $yaps_data_orig_file");
  }

  my $translate_rc =  exec_cmd("sed --in-place=.old -e '/^".$yaps_package_to_relax."/d' /opt/yaps/data");

  my $check_rc =  exec_cmd("/bin/checkydata");
  if ($check_rc == 0) {
    log_msg("checkydata consistent");
  } else {
    log_msg("checkydata inconsistent");
  }
  function_leave("relax_checkydata");
  return;
}

sub is_yaps_version_identical {
  my $yaps_package_file_to_check = shift;
  my ($is_identical, $new_yaps_version, $installed_yaps_version) = (0, "", "");
  function_enter("is_yaps_version_identical", "yaps_package_file_to_check=$yaps_package_file_to_check");

  my $yaps_package_file_to_check_after_glob = glob($yaps_package_file_to_check);

  if ( defined ($yaps_package_file_to_check_after_glob) and -e $yaps_package_file_to_check_after_glob) {
    log_msg("yaps package file to check exists: $yaps_package_file_to_check_after_glob");

    $new_yaps_version = exec_cmd_out("/bin/rpm -q -p $yaps_package_file_to_check_after_glob");
    $new_yaps_version = my_trim($new_yaps_version);

    $installed_yaps_version = get_rpm_version("ASyaps");

    if ($new_yaps_version eq $installed_yaps_version) {
      log_msg("installed and intended ASyaps versions are identical: $installed_yaps_version. Activation avoidance possible.");
      $is_identical = 1;
    } else {
      log_msg("installed $installed_yaps_version and intended $new_yaps_version ASyaps versions differ. No activation avoidance possible.");
      $is_identical = 0;
    }

  } else {
    log_msg("yaps package file to check does not exist: $yaps_package_file_to_check");
  }

  function_leave("is_yaps_version_identical", "is_identical=$is_identical");
  return ($is_identical);
}

sub get_rpm_consistency_status_for_all_nodes {
  my $upgrade_version = shift;
  my $multiNodeType = shift;
  my $isMultiNode = shift;
  my $has_a_quorum = shift;
  function_enter("get_rpm_consistency_status_for_all_nodes", "upgrade_version=$upgrade_version multiNodeType=$multiNodeType isMultiNode=$isMultiNode has_a_quorum=$has_a_quorum");
  my ($status, $node_status) = (0, 0); # consistent

  # check local
  $node_status = get_rpm_consistency_status("local", $upgrade_version);
  if ($node_status != 0) {
    $status = 1;
  }

  if ($status == 0) {
    if ($isMultiNode != 0) {
      # standby
      $node_status = get_rpm_consistency_status("remote", $upgrade_version);
      if ($node_status != 0) {
        $status = 1;
      }

      if ($status == 0) {
        # quorum
        if ($has_a_quorum) {
          $node_status = get_rpm_consistency_status("quorum", $upgrade_version);
          if ($node_status != 0) {
            $status = 1;
          }          
        }
      }
    }    
  }
  
  function_leave("get_rpm_consistency_status_for_all_nodes", "status=$status");
  return ($status);
}

sub get_rpm_consistency_status {
  my $type = shift;
  my $version_to_check = shift;
  function_enter("get_rpm_consistency_status", "type=$type version_to_check=$version_to_check");
  my ($status, $rc, $output, $cur_version) = (1, 0, "", "");
  my ($other_node_is_reachable, $password_is_expired) = (0, 0);
  my $max_try = 1; # 1 x 60s
  
  if ($type eq "local") {
    my $cmd = "/bin/checkydata";
    my $rc = exec_cmd($cmd);
    if ($rc == 0) {
      $cur_version = get_current_real_version();
      if ($cur_version eq $version_to_check) {
        log_msg("local and to_check_version are equal");
        $status = 0;
      } else {
        $status = 1;
      }
    }
  }

  if ($type eq "quorum") {
    $rc = execute_remote_quorum_cmd("$remote_self get_rpm_consistency_status local $version_to_check", \$output, $max_try, \$other_node_is_reachable, \$password_is_expired);
    if ($rc == 0) {
      $status = 0;
    }
  }

  if ($type eq "remote") {
    $rc = execute_remote_duplex_cmd("$remote_self get_rpm_consistency_status local $version_to_check", \$output, $max_try, \$other_node_is_reachable, \$password_is_expired);
    if ($rc == 0) {
      $status = 0;
    }
  }

  function_leave("get_rpm_consistency_status", "type=$type status=$status");
  return ($status);
}


sub NUC_copy_other_if_exist {
  my $from_path = shift;
  my $dest_path = shift;
  if (!defined($dest_path)) {
    $dest_path = $from_path;
  }
  if (! -e $from_path) {
    log_msg_info("NUC_copy_other_if_exist from_path=$from_path does not exist, skip copy");
    return 0;
  }
  my $rc = NUC_copy_other($from_path, $dest_path);
  return($rc);
}

sub NUC_copy_other {
  my $from_path = shift;
  my $dest_path = shift;
  if (!defined($dest_path)) {
    $dest_path = $from_path;
  }
  function_enter("NUC_copy_other", "from_path=$from_path dest_path=$dest_path");
  my ($cmd) = ("");
  if (-d $from_path) {
    log_msg_info("from_path=$from_path is a directory -> using -r option for recurse copy");
    $cmd = "$nuc_copy_other_script -rpv $from_path $dest_path";
  } else {
    log_msg_info("from_path=$from_path is not a directory -> skip -r option");
    $cmd = "$nuc_copy_other_script -pv $from_path $dest_path";
}
  my $rc = exec_cmd($cmd);
  function_leave("NUC_copy_other", "rc=$rc");
  return($rc);
}

sub NUC_execute_other {
  my $other_cmd = shift;
  function_enter("NUC_execute_other", "other_cmd=$other_cmd");
  my ($cmd) = ("");
  $cmd = "$nuc_exec_other_script $other_cmd";
  my $rc = exec_cmd($cmd);
  function_leave("NUC_execute_other", "rc=$rc");
  return($rc);
}

sub NUC_execute_other_pipe {
  my $other_cmd = shift;
  function_enter("NUC_execute_other_pipe", "other_cmd=$other_cmd");
  my ($cmd) = ("");
  $cmd = "$nuc_exec_other_script $other_cmd";
  my $output = exec_cmd_pipe($cmd);
  function_leave("NUC_execute_other_pipe", "output=$output");
  return($output);
}

sub copy_filelist_content {
  my $pattern = shift;
  function_enter("copy_filelist_content", "pattern=$pattern");
  
  my @file_list = exec_cmd_out("cat $pattern 2>/dev/null");
  log_msg_info("file_list=@file_list");
  foreach (@file_list) {
    $file=$_;
    log_msg_info("file=$file");
  }
  
  function_leave("copy_filelist_content");  
}

sub do_NUC_generic_data_upgrade {
  my $multiNodeType = shift;
  my $isMultiNode = shift;
  my $isquorum = shift;  
  function_enter("do_NUC_generic_data_upgrade", "multiNodeType=$multiNodeType isMultiNode=$isMultiNode isquorum=$isquorum");
  my ($rc, $file_to_create) = (0, "");

  $rc = NUC_execute_other("/bin/rm    -rf $generic_upgrade_dir");

  if ( -d $generic_upgrade_dir ) {
    log_msg_info("generic update data dir $generic_upgrade_dir exists already in old world -> copy the complete content to new world");    
    $rc = NUC_copy_other($generic_upgrade_dir, $soco2_data_dir);

    log_msg_info("copy the node specific filelists");
    if(!$nodaemon) {
      log_msg_info("copy the node_A specific filelist");
      copy_filelist_content($generic_upgrade_dir.'files_to_copy_A*.conf');
    } else {
      if ($isquorum == 1) {
        log_msg_info("copy the node_Q specific filelist");
        copy_filelist_content($generic_upgrade_dir.'files_to_copy_Q*.conf');
      } else {
        log_msg_info("copy the node_B specific filelist");
        copy_filelist_content($generic_upgrade_dir.'files_to_copy_B*.conf');
      }      
    } 
  } else {
    # create default empty file lists
    $rc = NUC_execute_other("/bin/mkdir -pv $generic_upgrade_dir");
    if ($isquorum == 1) {
      $file_to_create = $generic_upgrade_dir.'files_to_copy_Q.conf';
      $rc = NUC_execute_other("/usr/bin/touch $file_to_create");
    } else {        
      $file_to_create = $generic_upgrade_dir.'files_to_copy_A.conf';
      $rc = NUC_execute_other("/usr/bin/touch $file_to_create");
      if ($isMultiNode == 1) {
        $file_to_create = $generic_upgrade_dir.'files_to_copy_B.conf';
        $rc = NUC_execute_other("/usr/bin/touch $file_to_create");
      }
    }
  }

  function_leave("do_NUC_generic_data_upgrade", "rc=$rc");
  return($rc);
}

sub do_NUC_data_syslog_forward {
  function_enter("do_NUC_data_syslog_forward");
  my ($rc) = (0);

  $rc = NUC_execute_other("/bin/mkdir -pv $syslog_dir");
  $rc = NUC_copy_other_if_exist($sysloghost_file,$syslog_dir);
  $rc = NUC_copy_other_if_exist($syslogepoch_file,$syslog_dir);
  $rc = NUC_copy_other_if_exist($syslogforward_file,$syslogconfd);

  function_leave("do_NUC_data_syslog_forward", "rc=$rc");
  return($rc);
}

sub do_NUC_data_mek_blobber {
  function_enter("do_NUC_data_mek_blobber");
  my ($rc) = (0);

  $rc = NUC_execute_other("/bin/mkdir -pv $mek_blobber_dir");
  $rc = NUC_copy_other_if_exist("$mek_blobber_dir.'mekdata' ",       $mek_blobber_dir );
  $rc = NUC_copy_other_if_exist("$mek_blobber_dir.'mekdata.epoch' ", $mek_blobber_dir );

  function_leave("do_NUC_data_mek_blobber", "rc=$rc");
  return($rc);
}

sub do_NUC_data_snmp_upgrade {
  function_enter("do_NUC_data_snmp_upgrade");
  my ($rc) = (0);

  $rc = NUC_execute_other("/bin/mkdir -pv $snmp_upgrade_dir");
  $rc = NUC_copy_other_if_exist("/etc/subagents/snmp-syslogagt/config_file.txt",       $snmp_upgrade_dir );
  $rc = NUC_copy_other_if_exist("/etc/subagents/snmp-syslogagt/syslogagt-filter.conf", $snmp_upgrade_dir );
  $rc = NUC_copy_other_if_exist("/etc/snmp/snmpd.conf",                                $snmp_upgrade_dir.'snmp_snmpd.conf' );
  $rc = NUC_copy_other_if_exist("/var/lib/net-snmp/snmpd.conf",                        $snmp_upgrade_dir.'net-snmp_snmpd.conf' );

  log_msg_info("copy the SNMP storage dir on platform into NEW word");
  $rc = NUC_execute_other("/bin/rm -rf $snmp_dir");
  $rc = NUC_copy_other_if_exist($snmp_dir);
  
  function_leave("do_NUC_data_snmp_upgrade", "rc=$rc");
  return($rc);
}
  
sub do_NUC_data_DTB_upgrade {
  function_enter("do_NUC_data_DTB_upgrade");
  my ($rc, $dtb_data_dir, $dtb_data_file) = (0, "", "");


  # ini file (Master)
  $dtb_data_dir  = "/var/pgsql/dtb/DTBMaster/config/";
  $dtb_data_file = $dtb_data_dir . "dtbmaster.ini";
  if (-e $dtb_data_dir) {
    log_msg_info("DTB directory $dtb_data_dir exists -> creating it in new world");
    $rc = NUC_execute_other("/bin/mkdir -pv $dtb_data_dir");
  }
  if (-e $dtb_data_file) {
    log_msg_info("processing: DTB ini file $dtb_data_file");
    $rc = NUC_copy_other($dtb_data_file, $dtb_data_file );
  }

  # ini file (Server)
  $dtb_data_dir  = "/var/pgsql/dtb/DTBServer/config/";
  $dtb_data_file = $dtb_data_dir . "DTBServer.ini";
  if (-e $dtb_data_dir) {
    log_msg_info("DTB directory $dtb_data_dir exists -> creating it in new world");
    $rc = NUC_execute_other("/bin/mkdir -pv $dtb_data_dir");
  }
  if (-e $dtb_data_file) {
    log_msg_info("processing: DTB ini file $dtb_data_file");
    $rc = NUC_copy_other($dtb_data_file, $dtb_data_file );
  }
  
  # Journal files (Master)
  my @master_filelist = ("rjc.*", "rjg.*", "rjk.*", "*.nam", "*.nax", "*.rnr", "*.rnx", "*.t9i", "*.req", "*.cfg", "*.pfx");
  $dtb_data_dir  = "/var/pgsql/dtb/DTBMaster/winedir/";
  if (-e $dtb_data_dir) {
    log_msg_info("DTB directory $dtb_data_dir exists -> creating it in new world");
    $rc = NUC_execute_other("/bin/mkdir -pv $dtb_data_dir");

    foreach (@master_filelist) {
      $dtb_data_file = $dtb_data_dir . $_;
      log_msg_info("processing: fournal files $dtb_data_file");
      $rc = NUC_copy_other($dtb_data_file, $dtb_data_dir );
    }  
  }
  # Journal files (Server)
  my @server_filelist = ("rjc.*", "rjg.*", "rjk.*", "*.nam", "*.nax", "*.rnr", "*.rnx", "*.t9i", "*.dtb");
  $dtb_data_dir  = "/var/pgsql/dtb/DTBServer/winedir/";
  if (-e $dtb_data_dir) {
    log_msg_info("DTB directory $dtb_data_dir exists -> creating it in new world");
    $rc = NUC_execute_other("/bin/mkdir -pv $dtb_data_dir");

    foreach (@server_filelist) {
      $dtb_data_file = $dtb_data_dir . $_;
      log_msg_info("processing: fournal files $dtb_data_file");
      $rc = NUC_copy_other($dtb_data_file, $dtb_data_dir );
    }
  }  

  function_leave("do_NUC_data_DTB_upgrade", "rc=$rc");
  return($rc);  
}

sub do_NUC_data_sg_upgrade {
  my $is_sg2d = shift;
  my $softgate_config_dir = shift;
  function_enter("do_NUC_data_sg_upgrade", "is_sg2d=$is_sg2d softgate_config_dir=$softgate_config_dir");
  my ($rc, $PBC_MAX, $sip_data_dump_already_done) = (0, 21, 0);

  # fix potentially permission misprotection of config dir_to_check OSFOURK-11922
  if (-e "$softgate_config_dir") {
    exec_cmd("chmod -c o-w $softgate_config_dir/pbc_21/");
    exec_cmd("chmod -c o-w $softgate_config_dir/tones/extann/");
    exec_cmd("chmod -c o-w $softgate_config_dir/tones/");
  }

  # create the stucture in the new world
  $rc = NUC_execute_other("/bin/mkdir -pv $softgate_config_dir");

  # copy the central files in config
  $rc = NUC_copy_other($softgate_config_dir . "initialcfg.xml");
  if ($rc != 0) {
    log_msg_info("copying SoftGate initialcfg.xml failed -> ???");
  }

  $rc = NUC_copy_other_if_exist($softgate_config_dir . "telnumlookup.properties");
  if ($rc != 0) {
    log_msg_info("copying telnumlookup.properties failed");
  }

  # handle the peri boards config
  # on STMIX we have only one board/pbc with multi functionality (SIP or HFA)
  # on SG we have multiple boards/pbc (max 21) but with only one functionality (SIP or HFA)
  # also on STMIX the max pbc number is < 21

  for(my $i = 1; $i < $PBC_MAX; $i++) {
    my $vhg_dir=$softgate_config_dir . "pbc_$i/";
    log_msg_info("checking $vhg_dir");

    if ( -d $vhg_dir ) {
      log_msg_info("peri board directory $vhg_dir exists (either SIP or HFA)");
      $rc = NUC_execute_other("/bin/mkdir -pv $vhg_dir");

      my $vhg_data_dir = $vhg_dir . "data/";
      if ( -d $vhg_data_dir ) {
        # gateway data directory exists -> its a SIP vHG3500 (either SG or STMIX)
        log_msg_info("processing: vSIP dir $vhg_data_dir");
        $rc = NUC_execute_other("/bin/mkdir -pv $vhg_data_dir");

        log_msg_info("create the internal xml files via http commands");
        if (-x "/opt/soco/native/daemon/swupdated" and $sip_data_dump_already_done == 0) {
          log_msg_info("try to connect all vHG(SIP) boards via HTTPS and make fresh config.xml and secconf.xml");
          if ($is_sg2d == 0) {
            log_msg_info("calling SoftGate swupdated natively");
            exec_cmd("/opt/soco/native/daemon/swupdated rpm");
          } else {
            log_msg_info("calling SoftGate swupdated via chroot for Container instance");
            exec_cmd("/etc/init.d/sg2d chroot '/opt/soco/native/daemon/swupdated rpm'");
          }
          # ensure by flag that the dump is only executed once
          $sip_data_dump_already_done = 1;
        }

        # copy the insecure
        my $vhg_cfg_file     = $vhg_data_dir . "config.xml";
        my $vhg_int_cfg_file = $vhg_data_dir . "int-cfg.xml";

        if ( -f $vhg_int_cfg_file ) {
          $rc = NUC_copy_other($vhg_int_cfg_file);
          log_msg_info("pbc $i: I can not remove the int-cfg.xml in old world to prevent data loss in case of fallback to old world (ldh_db_copied trigger file removed by SG swupdated) ");
          # removeFile ($vhg_int_cfg_file);
        } else {
          if ( -f $vhg_cfg_file ) {
            log_msg_info("pbc $i: internal config file does not exist, but at least the normal exists -> use it");
            $rc = NUC_copy_other($vhg_cfg_file, $vhg_int_cfg_file );
            # removeFile ($vhg_cfg_file);
          } else {
            log_msg_info("pbc $i: neither internal config file , nor normal config file exist. Either pure HFA slot or export error in vHG");
          }
        }

        # copy the secure
        my $vhg_sec_cfg_file     = $vhg_data_dir . "secconf.xml";
        my $vhg_int_sec_cfg_file = $vhg_data_dir . "int-scfg.xml";

        if ( -f $vhg_int_sec_cfg_file ) {
          $rc = NUC_copy_other($vhg_int_sec_cfg_file);
          log_msg_info("pbc $i: I can not remove the int-scfg.xml in old world to prevent data loss in case of fallback to old world (ldh_db_copied trigger file removed by SG swupdated) ");
          # removeFile ($vhg_int_sec_cfg_file);
        } else {
          if ( -f $vhg_sec_cfg_file ) {
            log_msg_info("pbc $i: internal sec config file does not exist, but at least the normal exists -> use it");

            $rc = NUC_copy_other($vhg_sec_cfg_file, $vhg_int_sec_cfg_file );
            # removeFile ($vhg_sec_cfg_file);
          } else {
            log_msg_info("pbc $i: neither internal sec config file, nor normal sec config file exist. Either pure HFA slot or export error in vHG");
          }
        }
      }
      log_msg_info("processing: vHFA dir $vhg_dir");

      log_msg_info("processing: vHFA config files (e.g. dlscparams.xml, qdccfgparams.xml, ...)");
      $rc = NUC_copy_other("$vhg_dir"."*.xml", "$vhg_dir");
      log_msg_info("processing: vHFA config files (e.g. gwglobal.txt, spesd.txt, ...)");
      $rc = NUC_copy_other("$vhg_dir"."*.txt", "$vhg_dir");
      log_msg_info("processing: vHFA SPE certificates files");
      $rc = NUC_copy_other("$vhg_dir"."*.cer", "$vhg_dir");
      $rc = NUC_copy_other("$vhg_dir"."*.p12", "$vhg_dir");
      log_msg_info("processing: vHFA DLS key store");
      $rc = NUC_copy_other("$vhg_dir"."*.jks", "$vhg_dir");
    } else {
        log_msg_info("skipping non-existent pbc: $vhg_dir");
    }
  }
  
  # handle the global config like fips.xml (SG and STMIX)
  $rc = NUC_copy_other("$softgate_config_dir"."*.xml", "$softgate_config_dir");
  

  # handle the NCUI config
  my $ncui_dir=$softgate_config_dir . "pbc_21/";
  if ( -d $ncui_dir ) {
    log_msg_info("processing: vNCUI dir $ncui_dir, but remove before to be sure it is empty");
    $rc = NUC_execute_other("/bin/rm -rf $ncui_dir");
    # copy the whole pbc_21 recursively including the mekdata, telnumlookup.properties, ...
    $rc = NUC_copy_other($ncui_dir, $softgate_config_dir);
    log_msg_info("verify that vNCUI dir $ncui_dir was really copied (e.g. mekdata):");
    NUC_execute_other("ls -l $ncui_dir");
  }

  # handle the LB cert data in pbc_22
  my $lb_dir=$softgate_config_dir . "pbc_22/";
  if ( -d $lb_dir ) {
    log_msg_info("processing: LB dir $lb_dir, but remove before to be sure it is empty");
    $rc = NUC_execute_other("/bin/rm -rf $lb_dir");
    # copy the whole pbc_22 recursively including the p12, txt and jks 
    $rc = NUC_copy_other($lb_dir, $softgate_config_dir);
  } else {
    log_msg_info("skipping: LB dir $lb_dir because it does not exist");
  }
  
  # handle the trace config
  my $trace_dir=$softgate_config_dir . "logging/";
  if ( -d $trace_dir ) {
    log_msg_info("info: There is a SG trace dir $trace_dir, but since trace info must not survive the upgrade -> do not copy to new world ");
  }
  
 # handle all boards for apache certificates
  for(my $i = 1; $i <= $PBC_MAX; $i++) {
    my $vhg_dir=$softgate_data_dir . "pbc_$i/";
    log_msg_info("checking $vhg_dir");
    if ( -d $vhg_dir ) {
      log_msg_info("board directory $vhg_dir exists (either SIP or HFA or NCUI)");
      my $vhg_cert_dir = $vhg_dir . "certificates/";
      if ( -d $vhg_cert_dir ) {
        $rc = NUC_execute_other("/bin/mkdir -pv $vhg_cert_dir");
        $rc = NUC_copy_other("$vhg_cert_dir"."*.cert", "$vhg_cert_dir");
        $rc = NUC_copy_other("$vhg_cert_dir"."*.key", "$vhg_cert_dir");
      } else {
        log_msg_info("board cert directory $vhg_cert_dir does not exist -> skipping");
      }
    } else {
        log_msg_info("skipping non-existent pbc: $vhg_dir");
    }
  }

 # handle all boards for auto HBR restore
  for(my $i = 1; $i <= $PBC_MAX; $i++) {
    my $vhg_dir=$softgate_data_dir . "pbc_$i/";
    log_msg_info("checking $vhg_dir");
    if ( -d $vhg_dir ) {
      log_msg_info("board directory $vhg_dir exists (either SIP or HFA or NCUI)");
      my $counterfile = $vhg_dir . "counters.xml";
      if ( -f $counterfile ) {
        log_msg_info("copying HBR counter file to prevent unnecessary restore rand reboot in new world");
        $rc = NUC_execute_other("/bin/mkdir -pv $vhg_dir");
        $rc = NUC_copy_other("$counterfile", "$vhg_dir");
      } else {
        log_msg_info("board HBR counter file does not exist -> skipping");
      }
    } else {
        log_msg_info("skipping non-existent pbc: $vhg_dir");
    }
  }  

  # handle the tones and announcements
  my $tones_dir="/opt/soco/tones/extann/"; 
  if ( -d $tones_dir and $is_sg2d == 0) {
    log_msg_info("processing: SoftGate (63/A1/A2/A3) media server announcements dir $tones_dir");
     # copy the whole tones recursively 
    $rc = NUC_execute_other("/bin/rm -rf $tones_dir");
    $rc = NUC_copy_other($tones_dir);
  } else {
    log_msg_info("SoftGate media server announcements dir $tones_dir does not exist -> skip copy");
  }

  $tones_dir=$softgate_config_dir."/tones/extann/";
  if ( -d $tones_dir ) {
    log_msg_info("processing: SoftGate (A4 and higher) media server announcements dir $tones_dir");
     # copy the whole tones recursively 
    $rc = NUC_execute_other("/bin/rm -rf $tones_dir");
    $rc = NUC_copy_other($tones_dir);
  } else {
    log_msg_info("SoftGate media server announcements dir $tones_dir does not exist -> skip copy");
  }
  
  # handle the opensips pgsql data
  log_msg_info("processing: opensips pgsql data");
  if ($is_sg2d == 0) {
    if (-e $opensips_sql_dump_file ) {
      $rc = NUC_copy_other($opensips_sql_dump_file);
    } else {
      log_msg_info("could not copy the opensips sql file $opensips_sql_dump_file to new world, since it is missing");
      if (-e $opensips_sql_dump_fallback_file) {
        log_msg_info("found the fallback sql file, using this instead of $opensips_sql_dump_file");
        $rc = NUC_copy_other($opensips_sql_dump_fallback_file, $opensips_sql_dump_file);
      } else {
        log_msg_info("could not copy the opensips sql fallback file $opensips_sql_dump_fallback_file as well. Probably LB data loss!!!!!!");
      }
    }
  } else {
      log_msg_info("for iSG currently there is no opensips LB installation allowed. So we can skip");
  }

  # handle the keep_sg version flag 
  if ($is_sg2d == 0) {
    log_msg_info("is_sg2d flag not set, it is a native SG/EntGW/STMIX");
    if ($keep_installed_sg != 0 and $is_sg2d == 0) {
      log_msg_info("keep_installed_sg flag is set -> I will try to save the current SoftGate version into new world");
      my $old_sg_rpm = $soco2_sg_dir . "softgate.rpm";
      if (-e $old_sg_rpm ) {
        log_msg_info("Found old already activated softgate rpm package: $old_sg_rpm, but not using it keep_installed_sg");
      }
      
      $old_sg_rpm = "/opt/soco/softgate.rpm";
      if (-e $old_sg_rpm ) {
        log_msg_info("Found old not already activated, but only transferred softgate rpm package: $old_sg_rpm, using it for keep_installed_sg");
      }
      
      if (-e $old_sg_rpm ) {
        log_msg_info("Found old softgate rpm package: $old_sg_rpm -> check minimum version");
        my $keep_sg_option_allowed = determine_prerequisites_for_keep_sg($old_sg_rpm);
        if ($keep_sg_option_allowed == 1) {
          log_msg_info(" old sg_rpm meets prerequisites -> copy to new world");
          NUC_execute_other("/bin/mkdir -pv $soco2_sg_dir");
          $rc = NUC_copy_other($old_sg_rpm, $soco2_sg_dir);
        } else {
          log_msg_info(" old sg_rpm does not meet prerequisites e.g. minimum SG version due to Apache update -> do NOT copy to new world");
        }
      } else {
        log_msg_info("Warning: old softgate rpm package: $old_sg_rpm not found, but keep_installed_sg flag is set. have to ignore it");
      }
    } else {
      log_msg_info("keep_installed_sg flag is not set -> SoftGate version will be that of the new world image");
      log_msg_info("remove the SG LW takeover directory in NEW world in case it has been populated already by an earlier activation attempt with --keep_installed_sg option set");
      NUC_execute_other("/bin/rm -rvf $soco2_sg_dir");
    }
  } else {
    log_msg_info("is_sg2d flag is set, skipping old SG version handling");
  }
  
  function_leave("do_NUC_data_sg_upgrade", "rc=$rc");
  return($rc);
}

sub determine_prerequisites_for_keep_sg {
  my $old_sg_rpm = shift;

  if (! defined($old_sg_rpm)) {
    $old_sg_rpm = $soco2_sg_dir . "softgate.rpm";
  }
  function_enter("determine_prerequisites_for_keep_sg", "old_sg_rpm=$old_sg_rpm");
  my $rc = 0; # not fullfilled;

  if (! -e $old_sg_rpm ) {
    log_msg_info("old_sg_rpm file does not exist -> return false");
    $rc = 0;
  } else {
    # my $sg_short_version = exec_cmd_out("rpm -q --queryformat '%{VERSION}' -p $old_sg_rpm");
    # my $sg_short_release = exec_cmd_out("rpm -q --queryformat '%{RELEASE}' -p $old_sg_rpm");
    my $sg_version = exec_cmd_out("rpm -q -p $old_sg_rpm");
    my ($maj, $min, $fixrel, $overall, $delta) = get_softgate_versions($sg_version);
    if ($maj ne "" and $maj >= 7) {
      if ($maj >= 10) {
        if ($min eq "") {
          # its A0/B0 
          $rc = 1;
          # A0 has to be at least L0-T4T.A0.013-006 (OSFOURK-18480), but no check since no customer scenario
        } else {
          if ($min == 0) {
            # its A9 or B1
            log_msg_info("keeping SG A9/B1 loadware on V11 platform or newer not possible.");
            $rc = 0;
          } else {
            # its B2 or later
            log_msg_info("keeping SG B2 loadware on V11 platform or newer not possible.");
            $rc = 0;          }
        }
      } else {
        # its V7 or V8 SG (block, see: OSFOURK-6887)
        log_msg_info("keeping SG V7/V8 loadware on V11 platform or newer not possible.");
        $rc = 0;
      }
    }
  }

  if ($rc == 1) {
    log_msg_info("SG prerequisite fulfilled");
  } else {
    log_msg_info("SG prerequisite not fulfilled");
  }
  function_leave("determine_prerequisites_for_keep_sg", "rc=$rc");
  return($rc);
}

sub get_softgate_versions {
  my $softgate_version_string = shift;
  function_enter("get_softgate_versions", "softgate_version_string=$softgate_version_string");
  
  my @soco_version = ("", "", "", "");
  my ($version, $release, $maj, $min, $fixrel, $overall, $delta) = ("", "", "", "", "", "", "");

  # example: softgate-7.1.1-4.1
  $softgate_version_string = my_trim($softgate_version_string);
  $softgate_version_string =~ /softgate-(.*)-(.*)/;
 
  $version = $1;
  $release = $2;
  log_msg_info("version=$version release=$release");

  if (! defined($version) )  { $version = ""; }
  if (! defined($release)  ) { $release  = ""; }

  my @version_parts = split(/\./, $version);
  my @release_parts = split(/\./, $release);

  log_msg_info("version_parts=@version_parts release_parts=@release_parts");

  ($maj, $min, $fixrel) = @version_parts;
  if (! defined($maj) )      { $maj = ""; }
  if (! defined($min) )      { $min = ""; }
  if (! defined($fixrel) )   { $fixrel = ""; }

  ($overall, $delta) = @release_parts;
  if (! defined($overall) )   { $overall = ""; }
  if (! defined($delta) )     { $delta = ""; }
  
  @soco_version = ($maj, $min, $fixrel, $overall, $delta);

  function_leave("get_softgate_versions", "soco_version=@soco_version");
  return (@soco_version);
}

sub nnuc_force_local_copyexec {
  function_enter("nnuc_force_local_copyexec");
  $ENV{'_PLT_NNUC_FORCE_LOCAL_COPY_'}='ON';
  function_leave("nnuc_force_local_copyexec");
}

sub nnuc_unforce_local_copyexec {
  function_enter("nnuc_unforce_local_copyexec");
  $ENV{'_PLT_NNUC_FORCE_LOCAL_COPY_'}='DEFAULT';
  function_leave("nnuc_unforce_local_copyexec");
}

sub do_NUC_data_upgrade {
  my $multiNodeType = shift;
  my $isMultiNode = shift;
  my $isquorum = shift;
  function_enter("do_NUC_data_upgrade", "multiNodeType=$multiNodeType isMultiNode=$isMultiNode isquorum=$isquorum");
  my $rc = 0; # $ERROR_NUCDataUpgradeFailed;
  my ($cmd, $script, $copy_the_swupdated_to_new_world) = ("", "", 0);

  if ($copy_the_swupdated_to_new_world == 1) {
    # we just have to ensure that on new world the newest daemon script is used, not the potential older in the image
    log_msg_info("copying current (newest) daemon script to new world (from Plt HF on top of teh RLC)");
    $rc = NUC_copy_other($yaps_self);
    if ($rc != 0) {
      log_msg_info("copying current (newest) daemon script to new world failed -> ignore it and continue");
      $rc = 0;
    }
  } else {
    log_msg_info("copying current (newest) daemon script to new world is skipped. Too dangerous to copy a non SLES15 compliant swupdated to V10 R1 or later");
  }

  # save generic private files (temp. disabled till A4 branch if off)
  # do_NUC_generic_data_upgrade($multiNodeType, $isMultiNode, $isquorum);

  nnuc_force_local_copyexec();
  if ( -f $major_release_update_file) {
      log_msg_info("it is major release update, copy trigger file to NEW world");
      $rc = NUC_copy_other($major_release_update_file);
  }
  nnuc_unforce_local_copyexec();
  
  if ($is_stmix == 0) {
    if ($platform_has_beer == 1) {
      log_msg_info("from-platform already provides BEER backup, which automatically contains also the DTB data/journal as well as the SBC data");
    } else  {
      log_msg_info("from-platform does not provide BEER backup, I have to do it in the old way (copy data)");
      # DTB data_currentness
      do_NUC_data_DTB_upgrade();
    }
  }

  nnuc_force_local_copyexec();
  # SNMP data
  do_NUC_data_snmp_upgrade();

  do_NUC_data_mek_blobber();

  #Syslog Forward
  do_NUC_data_syslog_forward();
  
  nnuc_unforce_local_copyexec();
  
  # rcon/sshr data, they are intentionally not part of BEER to be independant of Assistant, portal amd resources.
  # do not set rc if copy of those files fails
  # we do not cover other portal user, but only engr
  log_msg_info("copy the fsshr files to new world");
  if ( -e "/var/pgsql/engr-cached.pwd" ) {
    NUC_copy_other("/var/pgsql/engr-cached.pwd");
    } else {
    log_msg_info("/var/pgsql/engr-cached.pwd missing in OLD world");
  }
  if ( -e "/opt/yapp/engr-cached.pwd" ) {
    NUC_copy_other("/opt/yapp/engr-cached.pwd");
    } else {
    log_msg_info("/opt/yapp/engr-cached.pwd missing in OLD world");
  }

  # webservice data
  if ($is_stmix == 0) {  
    if(!$nodaemon) {
      log_msg_info("preparing upgrade data for webservice on primary node (openais is probably already down)");
      if ($rc == 0) {
        if ( -e $pgsql_data_zip_file ) {
          log_msg_info("copying pgsql database dump to new world for import on first webservice start");
          $rc = NUC_copy_other($pgsql_data_zip_file);
          log_msg_info("removing pgsql database dump in old world to prevent import on first webservice start in case of rollback to old world");
          removeFile($pgsql_data_zip_file);

        } else {
          log_msg_info("no pgsql dump file $pgsql_data_zip_file found, checking rescue file");
          if ( -e $pgsql_data_zip_resc ) {
            $rc = NUC_copy_other($pgsql_data_zip_resc, $pgsql_data_zip_file);
          } else {
            log_msg_info("no pgsql dump file $pgsql_data_zip_file found, and no rescue file $pgsql_data_zip_resc -> error");
            $rc = $ERROR_NUCDataUpgradeFailed;
          }
        }
      }
    }
    my $ws_cert_dir = $webservice_data_dir . "certificates/";
    if ( -d $ws_cert_dir ) {
      nnuc_force_local_copyexec();
      $rc = NUC_execute_other("/bin/mkdir -pv $ws_cert_dir");
      $rc = NUC_copy_other("$ws_cert_dir"."*.cert", "$ws_cert_dir");
      $rc = NUC_copy_other("$ws_cert_dir"."*.key", "$ws_cert_dir");
      nnuc_unforce_local_copyexec();
    } else {
      log_msg_info("webservice cert directory $ws_cert_dir does not exist -> skipping");
    }

    if (-e "/var/opt/soco2/tmp/hp4k-base.installed") {
      log_msg_info("found an existing /var/opt/soco2/tmp/hp4k-base.installed, but do not copy it to new world");
    }
    $rc = NUC_copy_other_if_exist("/var/log/webservice/local_hw_type.log");
  }


  if ($is_stmix == 0) {
    # HA data also for secondary and quorum
    # copy also the haipsetup.xml
    log_msg_info("copy also the haipsetup.xml to new world, even if webservice would generate it newly from pgsql at first startup");
    $rc = NUC_copy_other($ha_data_dir."conf/haipsetup.xml");

    # copy also the openais.conf
    log_msg_info("copy also the $openais_conf_file to new world, it will be replaced by initHA later");
    $rc = NUC_copy_other($openais_conf_file);
  }

  # SoftGate data
  if ($rc == 0) {
    $script = $softgate_dir . "upgrade_nuc_data.sh";
    if ( -e $script) {
      log_msg_info("found NUC data script, but ignoring it");
    }
      
    my  $sg_initialcfg =  $softgate_config_dir . "initialcfg.xml";
    my $isg_initialcfg = $isoftgate_config_dir . "initialcfg.xml";
    if (-e $sg_initialcfg or -e $isg_initialcfg or $is_stmix == 1 or $is_entgw == 1 or $isMultiNode == 1) {
        log_msg_info("SG or iSG initialcfg.xml found or STMIX HW or EntGW or Duplex -> SoftGate could be active and needed ");
        my $is_sg_running = is_service_running("socod");
        my $is_sg2d_running = 0;
        if ($is_stmix == 0) {
          $is_sg2d_running = is_service_running("sg2d");
        }
        my $sg_is_configured = 0;
        my $number_of_sg_on_central_hosts = get_number_of_sg_on_central_hosts(0);
        if ($number_of_sg_on_central_hosts != 0) {
          log_msg_info("Softgate flag in haipsetup.xml is set");
          if ($isMultiNode == 0) {
            $sg_is_configured = 1;
          }
        }
        my $sg2d_is_configured = 0;
        $sg2d_is_configured = is_integrated_isg_softgate();
        my $isg_is_configured = 0;
        if ( -e $isg_initialcfg ) {
          log_msg_info("found $isg_initialcfg, assuming iSG is active ");
          $isg_is_configured = 1; # using the existence of the initialcfg.xml inside the iSG container as criterium, not the system.iSG.node1/system.iSG.node2
        }

        nnuc_force_local_copyexec();
        if ($is_sg_running == 1 or $sg_is_configured == 1 or $is_stmix == 1 or $is_entgw == 1) {
          log_msg_info("socod running or configured-> SoftGate active and needed ");
          $rc = do_NUC_data_sg_upgrade(0, $softgate_config_dir);
          create_generic_file("", $sg_necessary_file);
          NUC_copy_other_if_exist($sg_necessary_file);
        } else {
          log_msg_info("socod not running -> SoftGate obviously configured but inactive and not needed in new world");
        }

        if ($is_sg2d_running == 1 or $sg2d_is_configured == 1 or $isg_is_configured == 1) {
          # handle iSG on EntGW or Surv EntGW
          log_msg_info("sg2d running or sg2d insserved or iSG initialcfg.xml found -> iSoftGate in container either active or needed ");
          $rc = do_NUC_data_sg_upgrade(1, $isoftgate_config_dir);
          create_generic_file("", $isg_necessary_file);
          NUC_copy_other_if_exist($isg_necessary_file);
        } else {
          log_msg_info("sg2d not running -> no action needed");
        }
        nnuc_unforce_local_copyexec();
        
    } else {
        log_msg_info("SoftGate initialcfg.xml missing -> SoftGate obviously not configured -> skipping ");
    }
    
    if ($rc != 0) {
      log_msg_info("Error: Softgate data upgrade failed, continuing ... ");    
    }
  }

  # Softgate OS update
  log_msg_info("checking if opentracker has to be insserved in new world for central hosts");
  my $is_opentracker_running = is_service_running("opentracker-ipv4");
  if ($is_opentracker_running == 1) {
      log_msg_info("opentracker is running -> obviously an central host -> insserv it in the new world");
      NUC_execute_other("insserv opentracker-ipv4");
  }

  # we have to copy $sg_os_version_file to NEW world as the preparation provides the install area in OLD and copies to NEW world (the change for OSFOURK-25263 was a mistake)
  NUC_copy_other_if_exist($sg_os_version_file);
  
  NUC_copy_other_if_exist("/var/opt/soco-common/local_transmission.config");
  NUC_copy_other_if_exist("/var/lib/opentracker/opentracker.whitelist");
  NUC_copy_other_if_exist("/etc/opentracker-ipv4.conf");
  NUC_copy_other_if_exist("/etc/opentracker-ipv6.conf");
  NUC_copy_other_if_exist("/etc/sysconfig/transmission-daemon");

  
  log_msg_info("special handling of transmission dir, remove it in other world before copy (to prevent Verschachtelung)");

  my $transmission_config_dir = "/var/cache/transmission/.config/transmission/";
  if ( -e $transmission_config_dir) {
    log_msg_info("special handling of transmission config dir $transmission_config_dir, remove it in other world before copy (to prevent Verschachtelung)");
    NUC_execute_other("/bin/rm -rvf $transmission_config_dir");
    NUC_copy_other_if_exist($transmission_config_dir); # dir
  } else {
    log_msg_info("transmission config dir $transmission_config_dir does not exist in old world, skip removal and copy to new world to keep at least the defaults of new world.");
  }

  # CLA / licensing
  my $do_license_update = 1;
  if ($is_stmix == 1) {
    $do_license_update = 0;
  }
  if ($is_major_release_update == 1) {
    if (isCentralHost($multiNodeType) == 1) {
      log_msg_info("For central host and major release update we have to skip the license file takeover to avoid cummulation of Multiple Product Licenses ( MPL) of the Flex Licenses (OSFOURK-16597)");
      $do_license_update = 0;
    } else {
      log_msg_info("Major release update, but not a central host -> we need license file takeover");
      $do_license_update = 1;
    }
  } else {
    log_msg_info("Not a major release update -> we need license file takeover");
    $do_license_update = 1;
  }

  if ($do_license_update == 1) {
    log_msg_info("preparing upgrade data for CLA/Licensing (also on quorum for e.g. SG on Q)");
    if (-e $cla_license_dir) {

      log_msg_info("for diagnosis print the ALI");
      get_ali();
      get_ali_p();

      log_msg_info("do not copy the DNS config file from OLD to NEW, as for Duplex (N-NUC) it would be copied to other node");
      #$rc = NUC_copy_other_if_exist("/etc/resolv.conf");
      dump_file_to_log("/etc/resolv.conf");


      log_msg_info("update license data step 0: copying already imported original license files to new world");
      $rc = NUC_copy_other($license_storage_dir, $soco2_data_dir);
      if ($rc != 0) {
        log_msg_info("error: copying already imported original license files failed with rc=$rc, but I can not let complete upgrade fail -> continue");
      }
      
      log_msg_info("update license data step 1: stopping cla");
      $rc = stop_cla_service();

      log_msg_info("update license data step 2: creating temp. dir in new world to store the licensing data, but make sure it does not yet exist there (has to be empty))");

      $rc = NUC_execute_other("/bin/rm -rvf $cla_temp_dir");
      $rc = NUC_execute_other("/bin/mkdir -p $cla_temp_dir");
      if ($rc != 0) {
        log_msg_info("error:  creating temp. license dir failed with rc=$rc, but I can not let complete upgrade fail -> continue");
      }
      
      log_msg_info("update license data step 3: copy CLA system dir to temp. dir in other world");
      $rc = NUC_copy_other("/var/system/", $cla_temp_dir);
      if ($rc != 0) {
        log_msg_info("error: copy CLA system dir failed with rc=$rc, but I can not let complete upgrade fail -> continue");
      }

      log_msg_info("update license data step 4a: create the empty license dir in temp folder");
      $rc = NUC_execute_other("/bin/mkdir -p $cla_temp_dir/license");

      log_msg_info("update license data step 4b: copy the real license files in license dir $cla_license_dir to temp. dir in other world, avoid gpcf and alf files");
      $rc = NUC_copy_other($cla_license_dir."rlf_*.xml", $cla_temp_dir."license/");
      if ($rc != 0) {
        log_msg_info("error: copy CLA licenses failed with rc=$rc, but I can not let complete upgrade fail -> continue");
      }

      log_msg_info("taking a look in the temp. license dir");
      NUC_execute_other("ls -lR $cla_temp_dir");
      
      log_msg_info("update license data step 5: start CLA again in old world in case that the upgrade fails");
      $rc = start_cla_service();

    } else {
      log_msg_info("CLA license dir $cla_license_dir does not exist -> no CLA data upgrade");
    }
  }

  if ($is_stmix == 0) {  
    # SWA files
    if (-e $assistant_nuc_downtime_file) {
      $rc = NUC_copy_other($assistant_nuc_downtime_file);
    } else {
      log_msg_info("SWA downtime file $assistant_nuc_downtime_file does not exist, skip copy to new world");
    }
    log_msg_info("copying the $upgrade_log_file for Assistant SWA into new world, so that it has history and can can be completed in new world.");
    $rc = NUC_copy_other($upgrade_log_file);
  }

  # SBC
  my $sbc_is_configured = is_SBC_configured();
  if ($is_stmix == 0 and $sbc_is_configured == 1) {
    my $sbc_vmimport_dir = "/var/vm_import/hp4k-SBC";
    my $sbc_current_link = $sbc_vmimport_dir."/SBC-current";
    my $sbc_incoming_rpm_dir = $sbc_vmimport_dir."/incoming-rpm";
    my $sbc_factory_img = "SBC.img.bz2";
    my $rcx = 0;
    log_msg_info("SBC: skip manipulating SBC image because NUC Preparation should already have executed all LXC import actions");

    #  Force network reconfiguration
    NUC_execute_other("rm -f /opt/SBC/etc/networking.conf.old");
    log_msg_info("SBC: custom image detection end");
  }

  function_leave("do_NUC_data_upgrade", "rc=$rc");
  return ($rc);  
}

sub NUC_remote_rollback {
  my $node     = shift;
  my $nodetype = shift;
  function_enter("NUC_remote_rollback", "node=$node nodetype=$nodetype");

  my $rc  = 0;
  my $remote_cmd = "$nuc_rollback_script $nodetype";
  my $output = "";
  my ($node_is_reachable, $password_is_expired) = (0, 0);

  if ($upgrade_via_image == 0 ) {
    log_msg_info("NUC rollback on $nodetype skipped because no NUC scenario");
  } else {
    set_internal_upgrade_step($STEP_NUC_rollback, 1, $nodetype, $node );

    log_msg_info("initiating a NUC rollback on $nodetype");
    $rc = execute_remote_cmd($node, \$node_is_reachable, \$password_is_expired, $remote_cmd, \$output, 1, 60, 1);

    log_msg_info("waiting till the old world is back");
    $rc = wait_for_remote_node_world_appearance($node, "OLD");
  }
          
  function_leave("NUC_remote_rollback", "rc=$rc");
  return ($rc);      
}

sub NUC_rollback {
  my $nodetype = shift;
  function_enter("NUC_rollback", "nodetype=$nodetype");
  my $rc  = 0;
  my $rollback_cmd = "$nuc_rollback_script $nodetype";
  my $output = "";
    
  if ($upgrade_via_image == 0 ) {
    log_msg_info("NUC rollback on $nodetype skipped because no NUC scenario");
  } else {
    set_internal_upgrade_step($STEP_NUC_rollback, 1, $nodetype );

    log_msg_info("initiating a NUC rollback type=$nodetype");
    $rc = exec_cmd( $rollback_cmd );
  }

  function_leave("NUC_rollback", "rc=$rc");
  return ($rc);
}

sub wait_for_NUC_PREP {
  my $multiNodeType = shift;
  function_enter("wait_for_NUC_PREP", "multiNodeType=$multiNodeType");
  my $rc = $ERROR_NUCPreparationFailed; # assume error for timeout
  my $get_prep_script   = "$nuc_bin_dir"."get_prep_status.sh";
  my $start_prep_script = "/instmedia/sg_os_rlc/plt-update-new.sh";
  my $start_prep_cmd = "$start_prep_script sg_standalone";
  my $nuc_prep_timeout = 120; # minutes
  my $attempt_to_start_prep = 0; # flag
  
  if (-e $get_prep_script) {
    for(my $i = 0; $i < $nuc_prep_timeout; $i++) {

      set_internal_upgrade_step($STEP_NUC_preparation, 1, $defaultHostType, $hostname,  "Image Preparation ($i of max. $nuc_prep_timeout minutes)");

      log_msg_info("calling of the NUC get prep status script");
      $rc = exec_cmd($get_prep_script);      

      if($rc == 0) {
        log_msg_info("NUC preparation successful");
        last;
      } else {
        if($rc == 1) {
          log_msg_info("NUC preparation in progress... waiting ($i of max. $nuc_prep_timeout minutes)");
        } else {
          log_msg_info("NUC preparation failure rc=$rc (maybe never started)");
          if (($attempt_to_start_prep == 0) and (is_softgate_standalone($multiNodeType) == 1) and ($sg_os_update == 1)) {
            log_msg_info("NUC preparation failure -> try once starting it for SG OS activation for SG StandAlone");
            $attempt_to_start_prep = 1;
            if (-e $start_prep_script) {
              exec_cmd($start_prep_cmd);
            } else {
              log_msg_info("NUC preparation can not be started because script $start_prep_script does not exist");
              $rc = $ERROR_NUCPreparationFailed;
            }
          } else {
            log_msg_info("NUC preparation failure -> try starting it already done or wrong deployment -> give up");
            $rc = $ERROR_NUCPreparationFailed;
            last;
          }
        }
      }
      sleep(60); # 1 minute
    }
  } else {
    log_msg_info("Error: NUC get prep status script $get_prep_script does not exist");
  }
  function_leave("wait_for_NUC_PREP", "rc=$rc");
  return ($rc);
}

sub wait_for_remote_node_world_appearance {
  my $node = shift;
  my $expected_world = shift;
  function_enter("wait_for_remote_node_world_appearance", "node=$node expected_world=$expected_world");
  
  my ($rc, $check_rc)  = ($ERROR_NUCRemoteNodeActivationRebootFailed, 0);
  my $nuc_wait_timeout = 30; # minutes
  my $remote_cmd = $nuc_getworld_script;
  my ($output, $reboot_type) = ("", "activation");
  my ($node_is_reachable, $password_is_expired) = (0, 0);  

  if ($expected_world eq "OLD") { $reboot_type = "rollback" ; }
  log_msg_info("sleeping one minute to give the remote node the time to really go down (reducing risk of hanging ssh client)");
  sleep(60); # 1 minute

  for(my $i = 0; $i < $nuc_wait_timeout; $i++) {

      set_internal_upgrade_step($STEP_wait_for_remote_NUC_host, 1, $defaultHostType, $hostname,  "Wait for remote host $node to re-appear after activation reboot ($i of max. $nuc_wait_timeout minutes)");

      log_msg_info("checking worldness of remote host $node");
      $output = "";
      my $suppress_diagnosis=1;
      if ($i == ($nuc_wait_timeout -1)) {
        log_msg_info("its the last run for node $node, increasing verbosity");
        $suppress_diagnosis=0;
      }
      $check_rc = execute_remote_cmd($node, \$node_is_reachable, \$password_is_expired, $remote_cmd, \$output, 1, 60, 1, $suppress_diagnosis);
      chomp($output);
      log_msg_info("world is: $output, rc=$check_rc");

      if($output eq $expected_world) {
        log_msg_info("remote node is expected world ($expected_world), skip waiting");
        $rc = 0;
        last;
      } else {
        log_msg_info("NUC $reboot_type reboot in progress on remote node $node ... waiting ($i of max. $nuc_wait_timeout minutes)");
      }
      trigger_watchdog();
      sleep(60); # 1 minute
    }
    
  function_leave("wait_for_remote_node_world_appearance", "rc=$rc");
  return ($rc);
}

sub wait_for_cluster_node_reappearance {
  my $node = shift;
  my $max_wait = shift;
  my $wait_delta = shift;

  function_enter("wait_for_cluster_node_reappearance", "node=$node max_wait=$max_wait wait_delta=$wait_delta (sec)");

  my ($rc, $check_rc)  = ($ERROR_NUCRemoteNodeActivationRebootFailed, 0);

  my $old_node_reach_test_method = $node_reach_test_method;
  $node_reach_test_method = "crm";

  for(my $i = 0; $i < $max_wait; $i++) {

      log_msg_info("checking upness of remote cluster host $node");

      $check_rc = is_node_reachable($node);

      log_msg_info("rc=$check_rc");

      if ($check_rc == 1) {
        log_msg_info("HA on remote node is up");
        $rc = 0;
        last;
      } else {
        log_msg_info("HA on remote node $node not yet up ... waiting ($i of max. $max_wait)");
      }
      trigger_watchdog();
      sleep($wait_delta); # in seconds
    }

  $node_reach_test_method = $old_node_reach_test_method;
  function_leave("wait_for_cluster_node_reappearance", "rc=$rc");
  return ($rc);
}

sub confirm_new_stmix_os_activation {
  my ($rc) = (0);

  function_enter("confirm_new_stmix_os_activation", "");
  

  $rc = exec_cmd("$stmix_os_confirm_script confirm_new_os_activation");
  if ($rc != 0) {
    log_msg_info("Error: STMIX OS confirm script recturned error $rc -> ????");
  }
    
  function_leave("confirm_new_stmix_os_activation", "rc=$rc");
  return ($rc);
}
 
sub execute_NUC_reboot {
  
  my ($rc) = (0);
  my $reboot_type = get_ha_type();
  if ($is_stmix == 1) {
    $reboot_type = "stmix";
  }
  
  function_enter("execute_NUC_reboot", "reboot_type=$reboot_type");

  if (-e $softgate_data_dir) {

    log_msg_info("stopping Softgate before creating activation failure state. Gateway Manager must not see this failure before the reboot has been tried");
    stop_softgate_service();
    
    log_msg_info("creating a NUC update failure status file in case that the reboot to new world fails");
    notify_NUC_status_for_SG(-1);
  }
  notify_NUC_status_for_WS(-1);

  # stop the openvpn as it may still have ip addresses on tun device
  stop_openvpn_service();

  if ($nodaemon == 1) {
    log_msg_info("NUC activation reboot on non primary node");
    log_msg_info("insserv on new world before reboot not necessary since the primary node will do this from remote");
    
  } else {
    
    log_msg_info("NUC activation reboot on primary node");

    if ($is_stmix == 1) {
      $reboot_type = "stmix";
      $upgrade_via_image = 1;
    } else {
      $reboot_type = "primary";
      # insserve on the other side
      log_msg_info("insserv removal of HA in new world -> plt_upgrade has to do this via configure_cib_local");
      $rc = NUC_execute_other("/sbin/insserv -r /etc/init.d/openais");
      if ($rc != 0) {
        log_msg_info("Error: insserv removal of HA on new world failed  -> continue ???");
      }
    }

    log_msg_info("insserv removal of Softgate in new world -> plt_upgrade has to do this via configure_cib_local");
    $rc = NUC_execute_other("/sbin/insserv -r /etc/init.d/socod");
    if ($rc != 0) {
      log_msg_info("Error: insserv removal of SoftGate on new world failed  -> continue ???");
    }
    
    $rc = NUC_execute_other("/sbin/insserv /etc/init.d/plt_upgrade");
    if ($rc != 0) {
      log_msg_info("Error: insserv plt_upgrade on new world failed  -> copy it from old world to new world");
      $rc = NUC_execute_other("cp -av /opt/soco/native/daemon/plt_upgrade /etc/init.d/plt_upgrade");
      if ($rc != 0) {
        log_msg_info("copy plt_upgrade to /etc/init.d failed, what now??");
      }
      log_msg_info("insserv plt_upgrade on new world a second time in hope that now its possible");
      $rc = NUC_execute_other("/sbin/insserv /etc/init.d/plt_upgrade");       
    }

    log_msg_info("create the daemon call file with configure_cib_local");
    create_call_file_for_reboot();
    log_msg_info("copy the daemon call file to new world to finish the NUC update there");
    $rc = NUC_execute_other("/bin/mkdir -pv $webservice_data_dir");
    $rc = NUC_copy_other($swupdated_call_file);

    log_msg_info("create the daemon call status file with configure_cib_local state to finish the NUC update in new world");
    create_callstatus_file($Wiederaufsetzpunkt4, $continue_upgrade_iteration, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $upgrade_with_reduce_downtime, $upgrade_via_image);
    log_msg_info("copy the daemon call status file to new world to finish the NUC update in new world");
    $rc = NUC_copy_other($swupdated_call_status_file);

    
    log_msg_info("generate the daemon state file again in old world, in case that activation reboot fails");
    create_callstatus_file($Wiederaufsetzpunkt5, $continue_upgrade_iteration, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $upgrade_with_reduce_downtime, $upgrade_via_image);

        
    log_msg_info("copy the node specific upgrade status files to new world, so that the configure_cib_local in the new world can generate the final upgrade status");
    $rc = NUC_copy_other($status_local_file);
    if (-r $status_remote_file) {
      $rc = NUC_copy_other($status_remote_file);
    }
    if (-r $status_quorum_file) {
      $rc = NUC_copy_other($status_quorum_file);
    }
  }

  my $do_the_Reboot = 1;
  if ($sg_os_update == 1 and $deferred_activation == 1) {
    log_msg_info("A deferred NUC OS activation for SG Standalone requested -> wait till the HSR connection goes down (max. 24 hours)", 0, 1);
    create_lock_file($def_act_lock_file);
    my $hsr_status = wait_till_hsr_goes_down();
    switch ($hsr_status) {
      case 0   {
        log_msg_info("HSR connection went down -> continue with NUC reboot");
        $do_the_Reboot = 1;
      }
      case 1   {
        log_msg_info("HSR connection still up after timeout -> cancel NUC reboot");
        $do_the_Reboot = 0;
      }
      else {
        log_msg_info("unkown HSR status -> cancel NUC reboot");
        $do_the_Reboot = 0;
      }
    }
    remove_lock_file($def_act_lock_file);
  }
      
  # the real NUC reboot initiated on the old world
  log_msg_info("It is guaranteed that a reboot occurs when calling the NUC reboot script. Either booting to new (success) or booting to old (failure) world", 0, 1);

  log_msg_info("copy my own daemon log file into new world as last action");
  if ($is_stmix == 1) {
    NUC_copy_other($log_file,                "/var/opt/soco/log/update_os-daemon_oldworld.log");
    NUC_copy_other("/var/log/os_update.log", "/var/opt/soco/log/os_update_oldworld.log");
  } else {
    # on Duplex with N-NUC copy_other from OLD world node_A will copy to NEW world of node_B, so add the hostname to be unique
    NUC_copy_other($log_file, "/var/log/webservice/update-daemon_oldworld_"."$hostname".".log");
  }
  
  if ($do_the_Reboot == 1) {
    if ( -e "/var/log/nuc/swupdated.use.nnuc" ) {
      log_msg_info("special N-NUC case - not calling reboot for primary but make special indication to N-NUC daemon to continue from here");
      $rc = exec_cmd("touch /var/log/nuc/swupdated.nnuc.finished");
    } else {
      $rc = exec_cmd("$nuc_reboot_script $reboot_type");
    }
    if ($rc != 0) {
      log_msg_info("Error: NUC reboot script reported error -> ????");
    }
  } else {
      log_msg_info("Error: NUC reboot cancelled");
  }
  
  function_leave("execute_NUC_reboot", "rc=$rc");
  return ($rc);
}


sub get_upgrade_flags_from_file {

  my $hotfix_info_file_to_analyze = $_[0];
  function_enter("get_upgrade_flags_from_file", "file_to_analyze=$hotfix_info_file_to_analyze");
  my ($cur_version, $destination_upgrade_version, $cur_ver_equals_dest_ver) = ("", "", 0);
  
  # if webservice can be installed with running HA, remove HA check from rpm
  # check RMX flag only
  if(-e $hotfix_info_file_to_analyze and $sg_os_update == 0) {
    
    my $content = read_file_content($hotfix_info_file_to_analyze);
    log_msg("\n$content\n");    
    
    $cur_version = get_current_version_in_brackets();
    log_msg("currently installed version: $cur_version");
    
    # replace the hotfix number
    my $version_with_hotfix_number_to_fallback_to_by_deactivation = $cur_version;
    $version_with_hotfix_number_to_fallback_to_by_deactivation =~ s/\.[0-9]*\]/.$hotfix_number_to_fallback_to_by_deactivation]/;
    log_msg("intended version for hotfix deactivation: $version_with_hotfix_number_to_fallback_to_by_deactivation");
        
    my $cur_version_substr = substr($cur_version, 0, 6);
    log_msg("currently installed version substring: $cur_version_substr");    
    if(open(HFINFO, "<$hotfix_info_file_to_analyze")) {
      my @hf_info = <HFINFO>;
      my $line = "";
      my $first_line = 0;
      my $state = 0;
      if($hotfix_mode == 0) {
        log_msg("For FR or MR upgrades we get customized upgrade.info for every upgrade path -> no cummulativenesss necessary. So we start analyzing from the first line.");
        $state = 1;
      }
      if($hotfix_mode == 0) {
        # RLC
        $rmx_reboot = 1;
      } else {
        # HF
        $rmx_reboot = 0;
      }        
      $upgrade_with_completely_stopped_ha=0;
      $upgrade_with_full_drbd_sync       =0;
      $upgrade_with_reduce_downtime      =0;
      $upgrade_via_image                 =0;
      my $found_upgrade_with_reduce_downtime_set_to_0 = 0;
      my $found_upgrade_with_reduce_downtime_set_to_1 = 0;
      $host_reboot = 0;      
      foreach(@hf_info) {
        $line = my_trim($_);
        log_msg("hf.info: $line");
        
        if($first_line == 0) {
          $destination_upgrade_version = $line;
          $upgrade_version = $line;
          $upgrade_version =~ s/\[/ /;
          $upgrade_version =~ s/\]/ /;
          $upgrade_version = my_trim($upgrade_version);
          $upgrade_version =~ s/ /_/;
          log_msg_info("found upgrade_version=$upgrade_version for sysinfo");
          log_msg_info("found upgrade_version=$destination_upgrade_version for activation avoidance");
          $first_line = 1;
        }        

        if($state == 1) {
          if (index($line, "REST_RMX=yes;") >= 0) {
            $rmx_reboot = 1;
            log_msg_info("Restart of HA/openais (and therefore RMX), because $line");
          }
          if(index($line, "REST_COMPLETE_HA=yes;") >= 0) {
            $upgrade_with_completely_stopped_ha = 1;
            log_msg_info("Upgrade with complete shutdown of HA/openais on duplex, because $line");
            }
          if(index($line, "DRBD_SYNC=yes;") >= 0) {
            $upgrade_with_full_drbd_sync = 1;
            $full_drbd_sync_flag_really_read = 1;
            log_msg_info("Upgrade with full DRBD sync on duplex, because $line");			
            }
          if(index($line, "UPGRADE_VIA_IMAGE=yes;") >= 0) {
            $upgrade_via_image = 1;
            log_msg_info("Upgrade via image detected, because $line");
            }            
          if(index($line, "REDUCE_DOWNTIME=yes;") >= 0) {
            $found_upgrade_with_reduce_downtime_set_to_1 = 1;
            log_msg_info("Upgrade with reduce downtime, because $line");
            }
          if(index($line, "REDUCE_DOWNTIME=no;") >= 0) {
            $found_upgrade_with_reduce_downtime_set_to_0 = 1;
            log_msg_info("Upgrade with reduce downtime, because $line");
            }
          if(index($line, "REST_HOST=yes;") >= 0) {
            $host_reboot = 1;
            log_msg_info("restart of the OS after upgrade on each node, because $line");
            }
          if ( $line =~ /BIOS_VER="(.*)";/ ) {
            $delivered_bios_version = $1;
            log_msg_info("EcoServer bios version delivered with upgrade package: delivered_bios_version=$delivered_bios_version");
            }
        }        
        if(index($line, $cur_version_substr) == 0) {
          if($hotfix_mode == 1) {
            if (cmp_hf_version($line, $cur_version) == 1) {
              $state = 1;
              log_msg_info("found hf.info entry for version greater current ($line > $cur_version)   -> using this HF entry");
            } else {
              $state = 0;
              log_msg_info("found hf.info entry for version lesser or equal current ($line > $cur_version)  -> skipping this HF entry");
            }
          } elsif($hotfix_mode == 2) {
            my $compare_result = cmp_hf_version($line, $cur_version);
            if ( $compare_result == 0) {
              $state = 1;
              log_msg_info("found hf.info entry for current version ($line = $cur_version)   -> using this HF entry as start criteria");
            } 
            
            if ( $state == 1) {
              my $compare_result = cmp_hf_version($line, $version_with_hotfix_number_to_fallback_to_by_deactivation );
              if ( $compare_result == 0) {
                $state = 0;
                log_msg_info("found hf.info entry for intended target version ($line = $version_with_hotfix_number_to_fallback_to_by_deactivation)   -> using this HF entry as stop criteria");
              }
            }
          }elsif($hotfix_mode == 0) {
              $state = 1;
              log_msg_info("found upgrade.info entry for current version ($line = $cur_version)   -> using this entry as start criteria");            
          }          
        }
      }
      close(HFINFO);

      if ($found_upgrade_with_reduce_downtime_set_to_1 == 1)
      {
        log_msg_info("found at least one entry requiring the reduce downtime");
        $upgrade_with_reduce_downtime = 1;
      }
      if ($found_upgrade_with_reduce_downtime_set_to_0 == 1)
      {
        log_msg_info("found at least one entry forbidding the reduce downtime -> this wins");
        $upgrade_with_reduce_downtime = 0;
      }
      
    } else {
      log_msg_info("Could not open file $hotfix_info_file_to_analyze!");
    }
  } else {
    if ($sg_os_update == 0) {
      log_msg_info("upgrade info file $hotfix_info_file_to_analyze not available - not a hotfix update");
    } else {
      log_msg_info("upgrade info file not available - it is NUC update for SG standalone");
      log_msg_info("setting upgrade flags hardcoded");
      $upgrade_via_image = 1;
      $rmx_reboot = 1;
      $upgrade_with_completely_stopped_ha = 1;
      $upgrade_with_full_drbd_sync = 1;
      $host_reboot = 1;
    }
  }

  if ( $is_FI == 1 ) {
    log_msg_info("It is first installation mode, ignore the hf.info values but use hardcoded ones");
    $rmx_reboot = 0;
    $upgrade_with_completely_stopped_ha = 0;
    $upgrade_with_full_drbd_sync = 0;
    $host_reboot = 0;
  }
  
  if ($cur_version eq $destination_upgrade_version) {
    $cur_ver_equals_dest_ver = 1;
    log_msg_info("Current installed version $cur_version is the same as the intended version of the activation -> activation avoidance possible");
  }
  $is_major_release_update = check_for_major_release_update($cur_version, $destination_upgrade_version);

  function_leave("get_upgrade_flags_from_file", "cur_ver_equals_dest_ver=$cur_ver_equals_dest_ver is_major_release_update=$is_major_release_update");
  return($cur_ver_equals_dest_ver);
}

sub get_SLES_Major_Version {
  my $maj_version = 11; # default SLES 11
  if (-f "/etc/os-release") {
    $maj_version = exec_cmd_out(" /usr/bin/grep -m 1 VERSION_ID /etc/os-release | /usr/bin/awk  -F= '/VERSION_ID/ {print \$2;}' | sed -e 's/\"//g'  | /usr/bin/awk -F. ' {print \$1;}'");
  } else {
    $maj_version = exec_cmd_out("/usr/bin/awk  -F= '/VERSION/ {print \$2;}' < /etc/SuSE-release");
  }
  chomp($maj_version);
  $maj_version = my_trim($maj_version);
  log_msg("SLES Major Version $maj_version detected");
  return $maj_version;
}

sub get_SLES_Minor_Version {
  my $min_version = 4; # default SLES 11 SP4
  if (-f "/etc/os-release") {
    $min_version = exec_cmd_out(" /usr/bin/grep -m 1 VERSION_ID /etc/os-release | /usr/bin/awk  -F= '/VERSION_ID/ {print \$2;}' | sed -e 's/\"//g'  | /usr/bin/awk -F. '{print \$2;}'");
  } else {
    $min_version = exec_cmd_out("/usr/bin/awk  -F= '/PATCHLEVEL/ {print \$2;}' < /etc/SuSE-release");
  }
  chomp($min_version);
  $min_version = my_trim($min_version);
  log_msg("SLES Minor Version $min_version detected");
  return $min_version;
}

sub check_for_major_release_update_in_new_world {
  function_enter("check_for_major_release_update_in_new_world", "");
  my ($rc) = (0);

  if (-f $major_release_update_file) {
    log_msg_info("It is a major release update because trigger file $major_release_update_file copied from OLD world exists");
    $rc = 1;
  } else {
    log_msg_info("It is not a major release update because trigger file $major_release_update_file does not exist");
    $rc = 0;
  }

  function_leave("check_for_major_release_update_in_new_world", "rc=$rc (0 = no major upgrade, 1 = its a major upgrade)");
  return($rc);
}

# expecting version string like: [V8 R0.14.3]
sub check_for_major_release_update {
  my $from_version = shift;
  my $to_version = shift;
  function_enter("check_for_major_release_update", "from_version=$from_version to_version=$to_version");
  my ($rc, $from_major_version, $to_major_version) = (0, 0, 0);


  $from_version =~ /\[V([0-9]*).*\]/;
  if (defined($1)) {
    $from_major_version = $1;
    log_msg_info("found from_major_version $from_major_version");
  }

  $to_version =~ /\[V([0-9]*).*\]/;
  if (defined($1)) {
    $to_major_version = $1;
    log_msg_info("found to_major_version $to_major_version");
  }

  if ($from_major_version == $to_major_version) {
      log_msg_info("not a major release upgrade ");
      $rc = 0;
  } else {
      log_msg_info("its a major release upgrade from $from_major_version to $to_major_version");
      $rc = 1;
  }
  if ($rc == 1) {
      log_msg_info("its a major release upgrade, so create flag file");
      create_generic_file($major_release_update_file, "");
  } else {
      log_msg_info("its not a major release upgrade, so remove flag file");
      removeFile($major_release_update_file);
  }
  function_leave("check_for_major_release_update", "rc=$rc (0 = no major upgrade, 1 = its a major upgrade)");
  return($rc);
}

sub cmp_hf_version {
  # 0 = equal, 1 = greater, 2 = lower, 3 = error
  my $base = shift;
  my $ver = shift;
  function_enter("cmp_hf_version");
  my $rc = ($base eq $ver) ? 0 : 3;
  if($rc == 3) {
    my @base_parts = split(/\./, substr($base, 0, length($base) - 1));
    my @ver_parts = split(/\./, substr($ver, 0, length($ver) - 1));
    if($#base_parts == 2 && $#ver_parts == 2 && $base_parts[0] eq $ver_parts[0] && $base_parts[1] eq $ver_parts[1]) {
      if($base_parts[2] > $ver_parts[2]) {
        $rc = 1;
      } elsif($base_parts[2] < $ver_parts[2]) {
        $rc = 2;
      }
    }
  }
  function_leave("cmp_hf_version", "$base $rc $ver (0 = equal, 1 = greater, 2 = lower, 3 = error)");
  return($rc);
}

sub separate_packages_into_own_directory {
  my $pattern = shift;
  my $dir_to_get_from = shift;
  my $dir_to_store_to = shift;
  my $rc = 0;
  function_enter("separate_packages_into_own_directory", "pattern=$pattern dir_to_get_from=$dir_to_get_from dir_to_store_to=$dir_to_store_to");

  if (! -e "$dir_to_get_from") {
    log_msg_info("separate_packages_into_own_directory dir_to_get_from=$dir_to_get_from does not exists -> skipping");
    return (-1);
  }
  if (! -e "$dir_to_store_to") {
    makeDir($dir_to_store_to);
  }

  if($rc == 0 && opendir(DIR, $dir_to_get_from)) {
    my @files = readdir(DIR);
    my $cmd_mv = "";
    foreach(@files) {
      last if($rc);
      if($_ ne "." && $_ ne ".." && ! -d "$dir_to_get_from/$_") {
        log_msg_info("checking $_ for pattern $pattern");
        if ( $_ =~ /$pattern/) {
          log_msg_info("found pattern $pattern");
          # use mv to store the rpm files to the final storade dir to save IO and time if on the same partition
          $cmd_mv = "$store_package_script $dir_to_get_from/$_ $dir_to_store_to";
          $rc = exec_cmd($cmd_mv);
          if($rc != 0) {
            log_msg_info("could not move file $dir_to_get_from/$_ to $dir_to_store_to/$_");
          }
        }
      }
    }
    closedir(DIR);
  } else {
    $rc = 1;
    log_msg_info("could not open/process directory $dir_to_get_from");
  }   
  function_leave("separate_packages_into_own_directory", "rc=$rc");
  return ($rc);
}

sub print_package_info {
  my $type = shift;
  function_enter("print_package_info", "$type");

  log_msg("listing installed repositories:");
  my $all_repos = exec_cmd_out("/usr/bin/zypper lr");
  log_msg("listing installed repositories: $all_repos");
  
  my $all_rpms = exec_cmd_out("/bin/rpm -qa | sort");
  log_msg("listing all installed rpm packages: \n$all_rpms");
  
  my $rpm_packacke_info_file = $webservice_log_dir . "rpm_packages_" . "$type" . ".txt";
  if(open(PPIFILE, ">$rpm_packacke_info_file")) {
    protectFile($rpm_packacke_info_file);
    print PPIFILE "$all_rpms\n";
    close PPIFILE;
  }
  function_leave("print_package_info");
}

sub update_packages_with_rpm {
  my $rpm_repo_dir = shift;
  my $do_rpm_backup = shift;
  my $do_trigger_watchdog = shift;

  my $rc = 0;
  my $repo = 'soco2';

  function_enter("update_packages_with_rpm", "rpm_repo_dir=$rpm_repo_dir do_rpm_backup=$do_rpm_backup do_trigger_watchdog=$do_trigger_watchdog ");

  removeFile($sysinfo_pkg_log);

  my $cmd  = 'export PATH=/usr/bin:/bin; ';
  $cmd .= 'WEBSERVICE_DIR="'.$webservice_dir.'"; ';
  $cmd .= 'WORK="'.$tmp_rpm_dir.'"; ';
  if  ($do_rpm_backup == 1 and $is_stmix == 0) {
    $cmd .= $backup_rpm_script.' >> '.$log_file.' 2>&1 || exit 1; ';
  }
  $cmd .= 'echo "--> started updating packages in '.$rpm_repo_dir.'" >> '.$log_file.'; ';
  $cmd .= 'Z_OPTS="-Uhv --force"; ';
  $cmd .= 'PACKAGES=`ls '.$rpm_repo_dir.'*.rpm 2>/dev/null`; ';
  $cmd .= 'echo "--> rpm files: " >> '.$log_file.'; ';
  $cmd .= 'echo "${PACKAGES}" >> '.$log_file.'; ';
  $cmd .= 'for PKG in ${PACKAGES}; do ';
  $cmd .= ' PKG_NAME=`rpm -q --qf %{NAME} -p ${PKG}`; ';
  $cmd .= ' echo >> '.$log_file.'; ';
  $cmd .= ' echo `date` "--> package name: ${PKG_NAME}" >> '.$log_file.'; ';
  $cmd .= ' PKG_NAME_VERSION=`rpm -q --qf "%{NAME}-%|EPOCH?{%{EPOCH}:}:{}|%{VERSION}-%{RELEASE}" -p ${PKG}`; ';
  $cmd .= ' SKIP_PKG=0; ';
  $cmd .= ' echo "--> package name and version: ${PKG_NAME_VERSION}" >> '.$log_file.'; ';
  $cmd .= ' echo "<packageinfo><packagename>${PKG_NAME}</packagename></packageinfo>"                   >  '.$sysinfo_pkg_log.'; ';
  $cmd .= ' ls -al '.$rpm_repo_dir.' >> '.$log_file.' 2>&1 ; ';
  
  $cmd .= ' if [ $PKG_NAME = soco-common ]; then ';
  $cmd .= '  echo "for soco-common use only upgrade, no downgrade option" >> '.$log_file.'; ';
  $cmd .= '  Z_OPTS_TO_USE="-Uhv"; ';
  $cmd .= ' else ';
  $cmd .= '  Z_OPTS_TO_USE="$Z_OPTS"; ';
  $cmd .= ' fi; ';

  $cmd .= '  if [ $PKG_NAME = cld-os4k ]; then ';
  $cmd .= '  echo "for Cloud Link daemon cld-os4k package prevent installation on Plt, mark for CSTA installation only" >> '.$log_file.'; ';
  $cmd .= '    mv -v ${PKG} '.$csta_os_hf_pkg_dir.' 2>&1 ; ';
  $cmd .= '    SKIP_PKG=1; ';
  $cmd .= '  fi; ';

  $cmd .= '  if [ $PKG_NAME = docker ]; then ';
  $cmd .= '  echo "for Cloud Link daemon $PKG_NAME package prevent installation on Plt, mark for CSTA installation only" >> '.$log_file.'; ';
  $cmd .= '    mv -v ${PKG} '.$csta_os_hf_pkg_dir.' 2>&1 ; ';
  $cmd .= '    SKIP_PKG=1; ';
  $cmd .= '  fi; ';

  $cmd .= '  if [ $PKG_NAME = docker-bash-completion ]; then ';
  $cmd .= '  echo "for Cloud Link daemon $PKG_NAME package prevent installation on Plt, mark for CSTA installation only" >> '.$log_file.'; ';
  $cmd .= '    mv -v ${PKG} '.$csta_os_hf_pkg_dir.' 2>&1 ; ';
  $cmd .= '    SKIP_PKG=1; ';
  $cmd .= '  fi; ';

  $cmd .= '  if [ $PKG_NAME = docker-rootless-extras ]; then ';
  $cmd .= '  echo "for Cloud Link daemon $PKG_NAME package prevent installation on Plt, mark for CSTA installation only" >> '.$log_file.'; ';
  $cmd .= '    mv -v ${PKG} '.$csta_os_hf_pkg_dir.' 2>&1 ; ';
  $cmd .= '    SKIP_PKG=1; ';
  $cmd .= '  fi; ';

  $cmd .= '  if [ $SKIP_PKG = 0 ]; then ';
  $cmd .= '    rpm ${Z_OPTS_TO_USE} ${PKG} >> '.$log_file.' 2>&1 ; rpmretc=$? ; ';
  $cmd .= '  fi; ';
  
  $cmd .= '  if [ $PKG_NAME = python3-scipy ]; then ';
  $cmd .= '    cp -v ${PKG} '.$csta_os_hf_pkg_dir.' 2>&1 ; ';
  $cmd .= '    cp -v ${PKG} '.$assi_os_hf_pkg_dir.' 2>&1 ; ';
  $cmd .= '  fi; ';

  $cmd .= '  if [ $PKG_NAME = google-opensans-fonts ]; then ';
  $cmd .= '    cp -v ${PKG} '.$assi_os_hf_pkg_dir.' 2>&1 ; ';
  $cmd .= '  fi; ';

  $cmd .= '  if [ $PKG_NAME = apache2 ]; then ';
  $cmd .= '    cp -v ${PKG} '.$csta_os_hf_pkg_dir.' 2>&1 ; ';
  $cmd .= '    cp -v ${PKG} '.$assi_os_hf_pkg_dir.' 2>&1 ; ';
  $cmd .= '  fi; ';

  $cmd .= '  if [ $PKG_NAME = apache2-prefork ]; then ';
  $cmd .= '    cp -v ${PKG} '.$csta_os_hf_pkg_dir.' 2>&1 ; ';
  $cmd .= '    cp -v ${PKG} '.$assi_os_hf_pkg_dir.' 2>&1 ; ';
  $cmd .= '  fi; ';

  $cmd .= '  if [ $PKG_NAME = apache2-utils ]; then ';
  $cmd .= '    cp -v ${PKG} '.$csta_os_hf_pkg_dir.' 2>&1 ; ';
  $cmd .= '    cp -v ${PKG} '.$assi_os_hf_pkg_dir.' 2>&1 ; ';
  $cmd .= '  fi; ';
  
  $cmd .= ' ls -al '.$rpm_repo_dir.' >> '.$log_file.' 2>&1 ; ';
  $cmd .= ' if [ $rpmretc -eq 0 ]; then ';
  $cmd .= '  ' . $store_package_script . '   ${PKG} ${WORK} -v >> '.$log_file.' 2>&1 ; ';
  $cmd .= ' else ';
  $cmd .= '  if [ $PKG_NAME = soco-common ]; then ';
  $cmd .= '   echo "--> return code of rpm for ${PKG_NAME} is $? (!= 0) but it is soco-common, so lets assume it was a downgrade request"  >> '.$log_file.'; ';
  $cmd .= '   rm -v ${PKG} >> '.$log_file.' 2>&1 ; ';
  $cmd .= '  else ';
  $cmd .= '   echo "--> return code of rpm for ${PKG_NAME} is $? (!= 0). Continuing in hope that wildcard update will succeed later on"  >> '.$log_file.'; ';
  $cmd .= '  fi; ';
  $cmd .= ' fi; ';
  if ( $do_trigger_watchdog == 1 ) {
    $cmd .= ' if [ -f /usr/sbin/wd_keepalive ]; then /usr/sbin/wd_keepalive -1 2>/dev/null; fi; ';
  }
  $cmd .= 'done; ';
  $cmd .= 'echo "--> starting wildcard update of the remaining packages (best case is: error: File not found by glob: ...)" >> '.$log_file.'; ';
  $cmd .= 'echo `date` "--> cmd: rpm ${Z_OPTS} " >> '.$log_file.'; ';
  $cmd .= 'if ls -al '.$rpm_repo_dir.'*.rpm >> '.$log_file.' 2>&1 ; then rpm ${Z_OPTS} '.$rpm_repo_dir.'*.rpm >> '.$log_file.' 2>&1 || exit 1 ; fi ; ';
  $cmd .= 'echo "--> executed rpm wildcard update with command: rpm ${Z_OPTS} *.rpm" >> '.$log_file.'; ';
  $cmd .= 'echo "--> finished updating packages" >> '.$log_file.'; ';
  log_msg_info("*** updating packages using repository $rpm_repo_dir");
  $rc = exec_cmd($cmd);

  if ($rc != 0) {
    $rc = $ERROR_UpdatePackagesFailed;
  }
  removeFile($sysinfo_pkg_log);

  function_leave("update_packages_with_rpm", "rc=$rc");
  return($rc);
}

sub update_packages_with_zypper {
  my $rpm_repo_dir = shift;
  my $do_os_update = shift;
  my $do_rpm_backup = shift;
  my $do_java_update = shift;
  my $report_to_sysinfo = shift;
  my $report_to_stdout = shift;
  my $do_trigger_watchdog = shift;

  my $os_update_repo_dir = $osupdate_rpm_dir;
  my $rc = 0;
  my $repo = 'soco2';
  my $os_update_repo = 'soco2_os_update';

  function_enter("update_packages_with_zypper", "rpm_repo_dir=$rpm_repo_dir do_os_update=$do_os_update do_rpm_backup=$do_rpm_backup do_java_update=$do_java_update do_trigger_watchdog=$do_trigger_watchdog ");
  
  if ( -e "$os_update_repo_dir" ) {
    log_msg("update_packages_with_zypper: OS update dir $os_update_repo_dir exists -> making special dist-upgrade before doing the normal upgrade");
  } else {
    log_msg("update_packages_with_zypper: OS update dir $os_update_repo_dir does not exists -> no special dist-upgrade before normal upgrade");
    $do_os_update = 0;
  }

  removeFile($sysinfo_pkg_log);
 
  # RPM_INSTALL_PREFIX is needed for zypper install of cla package cla-v1-r19.0.0.x86_64.rpm
  # zypper global option --no-gpg-checks is for overcoming the SLES15 zypper default to require signed rpms. it exists already on SLES11 SP4.

  my $cmd  = 'export PATH=/usr/bin:/bin; ';
  $cmd .= 'export RPM_INSTALL_PREFIX=/opt; ';
  $cmd .= 'WEBSERVICE_DIR="'.$webservice_dir.'"; '; 
  $cmd .= 'WORK="'.$tmp_rpm_dir.'"; ';
  if  ($do_rpm_backup == 1 and $is_stmix == 0 ) {
    $cmd .= $backup_rpm_script.' >> '.$log_file.' 2>&1 || exit 1; ';
  }
  $cmd .= 'echo "--> started updating packages in '.$rpm_repo_dir.'" >> '.$log_file.'; ';
  $cmd .= 'Z_OPTS="--non-interactive --verbose --xmlout --no-gpg-checks"; ';
  $cmd .= 'zypper ${Z_OPTS} removerepo '.$repo.' >/dev/null 2>&1 || exit 1; ';
  $cmd .= 'zypper ${Z_OPTS} removerepo '.$os_update_repo.' >/dev/null  2>&1 || exit 1; ';
  $cmd .= 'PACKAGES=`ls '.$rpm_repo_dir.'*.rpm 2>/dev/null`; ';
  $cmd .= 'echo "--> rpm files: " >> '.$log_file.'; ';
  $cmd .= 'echo "${PACKAGES}" >> '.$log_file.'; ';
  if ( $do_os_update == 1 ) {
    $cmd .= 'zypper ${Z_OPTS} addrepo --type plaindir --refresh '.$os_update_repo_dir.' '.$os_update_repo.' >> '.$log_file.' 2>&1 || exit 1; ';
    $cmd .= 'echo "--> added special os_update repo '.$os_update_repo.' for OS updates" >> '.$log_file.'; ';
  }
  $cmd .= 'zypper ${Z_OPTS} addrepo --type plaindir --refresh '.$rpm_repo_dir.' '.$repo.' >> '.$log_file.' 2>&1 || exit 1; ';
  $cmd .= 'echo "--> added repo '.$repo.' for normal install/upgrade" >> '.$log_file.'; ';
  if ( $do_os_update == 1 ) {
    $cmd .= 'echo "--> starting special dist-upgrade for OS update" >> '.$log_file.'; ';
    $cmd .= 'echo "--> cmd: zypper ${Z_OPTS} dist-upgrade --no-recommends --repo '.$os_update_repo.'  --repo '.$repo.'" >> '.$log_file.'; ';
    $cmd .= 'zypper ${Z_OPTS} dist-upgrade --no-recommends --repo '.$os_update_repo.' --repo '.$repo.' >> '.$log_file.' 2>&1 || exit 1; ';
    $cmd .= 'echo "--> executed special dist-upgrade" >> '.$log_file.'; ';
  }
  $cmd .= 'for PKG in ${PACKAGES}; do ';
  $cmd .= ' PKG_NAME=`rpm -q --qf %{NAME} -p ${PKG}`; ';
  $cmd .= ' echo >> '.$log_file.'; ';
  $cmd .= ' echo `date` "--> package name: ${PKG_NAME}" >> '.$log_file.'; ';
  if ( $report_to_sysinfo == 1 ) {
    $cmd .= ' echo "<packageinfo><packagename>${PKG_NAME}</packagename></packageinfo>"                   >  '.$sysinfo_pkg_log.'; ';
  }
  if ( $report_to_stdout == 1 ) {
    $cmd .= ' echo "installing package: ${PKG_NAME}" ; ';
  }
  $cmd .= ' PKG_NAME_VERSION=`rpm -q --qf "%{NAME}-%|EPOCH?{%{EPOCH}:}:{}|%{VERSION}-%{RELEASE}" -p ${PKG}`; ';
  $cmd .= ' echo "--> package name and version: ${PKG_NAME_VERSION}" >> '.$log_file.'; ';
  $cmd .= ' zypper ${Z_OPTS} install --no-recommends --force-resolution --repo '.$repo.' ${PKG_NAME_VERSION} >> '.$log_file.' 2>&1 ; ';
  $cmd .= ' if [ $? -eq 0 ]; then ';
  $cmd .= '  ' . $store_package_script . '   ${PKG} ${WORK} -v >> '.$log_file.' 2>&1 ; ';
  $cmd .= ' else ';
  $cmd .= '  echo "--> return code of zypper install ${PKG_NAME} is $? (!= 0). Continuing in hope that dist-upgrade will succeed later on"  >> '.$log_file.'; ';
  $cmd .= ' fi; ';
  if ( $do_trigger_watchdog == 1 ) {
    $cmd .= ' if [ -f /usr/sbin/wd_keepalive ]; then /usr/sbin/wd_keepalive -1 2>/dev/null; fi; ';
  }
  $cmd .= 'done; ';
  $cmd .= 'zypper ${Z_OPTS} removerepo '.$repo.' >> '.$log_file.' 2>&1 || exit 1; ';
  $cmd .= 'zypper ${Z_OPTS} removerepo '.$os_update_repo.' >> '.$log_file.' 2>&1 || exit 1; ';
  $cmd .= 'zypper ${Z_OPTS} addrepo --type plaindir --refresh '.$rpm_repo_dir.' '.$repo.' >> '.$log_file.' 2>&1 || exit 1; ';
  $cmd .= 'echo "--> added repo '.$repo.' for dist-upgrade" >> '.$log_file.'; ';
  $cmd .= 'echo "--> starting dist-upgrade" >> '.$log_file.'; ';
  $cmd .= 'echo `date` "--> cmd: zypper ${Z_OPTS} dist-upgrade --no-recommends --repo '.$repo.'" >> '.$log_file.'; ';
  $cmd .= 'zypper -v -v ${Z_OPTS} dist-upgrade --no-recommends --repo '.$repo.' >> '.$log_file.' 2>&1 || exit 1; ';
  $cmd .= 'echo "--> executed dist-upgrade with command: zypper -v ${Z_OPTS} dist-upgrade --no-recommends --repo '.$repo.'" >> '.$log_file.'; ';
  $cmd .= 'zypper ${Z_OPTS} removerepo '.$repo.' >> '.$log_file.' 2>&1 || exit 1; ';
  $cmd .= 'zypper ${Z_OPTS} removerepo '.$os_update_repo.' >> '.$log_file.' 2>&1 || exit 1; ';
  $cmd .= 'echo "--> store temporary remaining upgrade packages after dist-upgrade" >> '.$log_file.'; ';
  $cmd .= 'PACKAGES=`ls '.$rpm_repo_dir.'*.rpm 2>/dev/null`; ';
  $cmd .= 'for PKG in ${PACKAGES}; do ';
  $cmd .= ' echo "--> package name for storage after successful dist-upgrade: ${PKG_NAME}" >> '.$log_file.'; ';
  $cmd .= '  ' . $store_package_script . ' ${PKG} ${WORK} -v  >> '.$log_file.' 2>&1  || exit 1;';
  $cmd .= 'done; ';
  if ( $do_java_update == 1 ) {
    $cmd .= 'echo "--> updating java jre" >> '.$log_file.'; ';
    $cmd .= '  ${WEBSERVICE_DIR}/scripts/update_java.sh '.$rpm_repo_dir.'  >> '.$log_file.' 2>&1  || exit 1;';
  }
  $cmd .= 'echo "--> finished updating packages" >> '.$log_file.'; ';
  log_msg_info("*** updating packages using repository $rpm_repo_dir");
  $rc = exec_cmd($cmd);
  
  if ($rc != 0) {
    $rc = $ERROR_UpdatePackagesFailed;
  }

  if ( $do_os_update == 1 ) {
    my $fix_menulist_rc = fix_menu_list_after_os_update();
  }  
    
  removeFile($sysinfo_pkg_log);

  function_leave("update_packages_with_zypper", "rc=$rc");
  return($rc);
}


sub extract_hotfix {
  my $rc = 0;
  my $hf_size = -1;
  my $count = 0;
  function_enter("extract_hotfix");
  log_msg_info("removing existing files in update directory");
  exec_cmd("/bin/rm -rvf $rpm_dir*");
  log_msg_info("extracting hotfix files to update directory");
  if(opendir(DIR, $hotfix_dir)) {
    my @files = readdir(DIR);
    foreach(@files) {
      if($_ ne "." and $_ ne ".." and $_ =~ /.*PLT(.*).tar/) {
        $count += 1;
        $hf_size = -s "$hotfix_dir$_";
        log_msg_info("extract_hotfix hotfix_mode=$hotfix_mode hf_size=$hf_size file_to_check=$hotfix_dir$_");       
        if($hotfix_mode == 2) {
          my $cur_version = exec_cmd_pipe('/bin/getydata -v');
          my $hf_file = "$hotfix_dir$_";
          my $hf_file_basename ="$_";
          $hf_file_basename =~ /.*PLT(.*)\.([0-9]*)\.tar/;
          my $extracted_hf_number = $2;
          log_msg_info("hotfix number extracted out of filename of hotfix file: $extracted_hf_number"); 
          $hotfix_number_to_fallback_to_by_deactivation = $extracted_hf_number;
          
          my $cmd = 'export PATH=/usr/bin:/bin; ';
          $cmd .= 'BACKUP="'.$backup_dir.'"; ';
          $cmd .= 'REPO="'.$rpm_dir.'"; ';   
          $cmd .= 'WORK="'.$cur_hf_dir.'work/"; ';
          $cmd .= 'HF_RPM_STORAGE="'.$tmp_rpm_dir.'"; ';
          $cmd .= 'mkdir -p ${WORK} ; ';                    
          $cmd .= 'RPM_FILES=`ls $BACKUP | grep .rpm`; ';
          $cmd .= 'echo "preparing update dir ${REPO} with rpm files from install repo ${BACKUP}" >> '.$log_file.'; ';
          $cmd .= 'for RPM in ${RPM_FILES}; do ';
          $cmd .= 'ln -v ${BACKUP}${RPM} ${REPO}${RPM} >> '.$log_file.' 2>&1 || exit 1; ';
          $cmd .= 'done; ';
          $cmd .= 'if [ ! -s '.$hf_file.' ]; then ';   
          $cmd .= 'echo "hotfix tar file is empty -> fallback to base FR." >> '.$log_file.'; '; 
          $cmd .= 'rm -rvf ${HF_RPM_STORAGE} >> '.$log_file.' 2>&1 ; ';
          $cmd .= 'else ';                    
          $cmd .= 'echo "extracting hotfix tar to temporary directory" >> '.$log_file.'; ';
          $cmd .= 'nice -n 10 tar --directory=${WORK} -xvf "'.$hf_file.'" ;';                        
          $cmd .= 'RPM_FILES=`ls ${WORK} | grep .rpm`; ';
          $cmd .= 'echo "removing those rpms from the update dir which will be delivered with the hotfix tar" >> '.$log_file.'; ';
          $cmd .= 'for RPM in ${RPM_FILES}; do ';
          $cmd .= 'NAME=`rpm -q --qf %{NAME} -p ${WORK}${RPM}`; ';
          $cmd .= 'rm -rvf ${REPO}${NAME}-[0-9]* >> '.$log_file.' 2>&1 ; ';
          $cmd .= 'done; ';
          $cmd .= 'echo "extracting hotfix tar to update directory" >> '.$log_file.'; ';
          $cmd .= 'rm -rvf ${WORK} ; ';                             
          $cmd .= 'nice -n 10 tar --directory="'.$rpm_dir.'" -xvf '.$hf_file.' >> '.$log_file.' 2>&1 || exit 1; ';
          $cmd .= 'fi; ';          
          if(-e $hf_file) {
            $rc = exec_cmd($cmd);
          } else {
            log_msg_info("stored hotfix file $hf_file not found");
            $rc = $ERROR_HotfixNotFound;
          }
        } else {
          $rc = exec_cmd("nice -n 10 /bin/tar --directory=\"".$rpm_dir."\" -xvf ".$hotfix_dir.$_);
        }
        last if($rc != 0);
      }
    }
    closedir(DIR);
  } else {
    log_msg_info("could not read hotfix directory $hotfix_dir");
    $rc = $ERROR_CanNotReadHotfixDir;    
  }
  
  if ($count == 0) {
    log_msg_info("Error: hotfix directory $hotfix_dir does not contain a valid hotfix");    
    $rc = $ERROR_HotfixDirDoesNotContainValidHotfix;
  }
  if ($count >= 2) {
    log_msg_info("Error: hotfix directory $hotfix_dir contains more than one hotfix tar file");    
    $rc = $ERROR_HotfixDirContainsTooMuchHotfixFiles;
  }    
  function_leave("extract_hotfix", "rc=$rc");
  return($rc);
}

sub rename_rpm_files {
  my $dir_to_check =  $_[0];

  function_enter("rename_rpm_files", "for dir $dir_to_check");
  my $name = "";
  if(opendir(DIR, $dir_to_check)) {
    my @files = readdir(DIR);
    foreach(@files) {
      if($_ ne "." and $_ ne ".." and $_ =~ /\.rpm/) {
        my $cmd = "/bin/rpm -q --qf %{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}.rpm -p ".$dir_to_check.$_;
        $name = exec_cmd_pipe($cmd);
        
        if (!defined($name)) {
          $name = "";
        }
        if(length($name) > 0 and $_ ne $name) {
          rename($dir_to_check.$_, $dir_to_check.$name);
          log_msg_info("renamed file ".$dir_to_check.$_." to ".$dir_to_check.$name);
        }
      }
    }
    closedir(DIR);
  } else {
    log_msg_info("could not read rpm directory $dir_to_check");
  }
  function_leave("rename_rpm_files");
}

sub check_log_file {
  my $log_file_to_use = shift;
  
  my $log_file_size = -s $log_file_to_use;
  my $max_log_file_size = 25*1024*1024;
  my $log_buffer;
  if(defined($log_file_size) && ($log_file_size >= $max_log_file_size)) {
    log_msg_info("log file size $log_file_size greater than $max_log_file_size. Rotating logfile to .1");

    my $bak_file = $log_file_to_use.".1";
    if (-f $bak_file) { removeFile( $bak_file ); }
    my $rc = renameFile($log_file_to_use, $bak_file);
    if ($rc != 0) { log_msg_info("error when renaming file $log_file_to_use to $bak_file, rc=$rc"); }
  } else {
    if (defined($log_file_size)) {
      log_msg_info("log file size: $log_file_size file: $log_file_to_use");
    }
  }
}

sub cleanup_after_upgrade {
  function_enter("cleanup_after_upgrade");
  remove_upgrade_lock_file();    
  remove_remote_lock_file();
  if (-e $skip_quorum_flagfile) {
    removeFile($skip_quorum_flagfile)
  }

  if( $host_reboot == 1 &&  $upgrade_with_completely_stopped_ha == 1 && !$nodaemon) {
    log_msg_info("skipping the insserv of the openais service because the following host reboot should not start the openais automatically (will be done later from the upgrade service)");
    create_call_file_for_reboot();
    create_callstatus_file($Wiederaufsetzpunkt4, $continue_upgrade_iteration, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $upgrade_with_reduce_downtime, $upgrade_via_image);
  } else {
    remove_call_file();
    remove_call_status_file();
    if( $host_reboot == 1 &&  $upgrade_with_completely_stopped_ha == 1) {
      log_msg_info("skipping the insserv of the openais service because the following host reboot should not start the openais automatically (will be done later from the active node)");
    } else {
      if ($is_stmix == 0) {
        log_msg_info("doing the insserv of the openais service");
        execute_ha_do_insserv();
      }
    }
  }
  function_leave("cleanup_after_upgrade");
}

sub interrupt {
  log_msg_info("swupdated: caught signal @_ exiting");
  remove_upgrade_lock_file();    
  remove_remote_lock_file();
  log_msg_info("swupdated: terminated without enabling openais startup to continue after crash");

  die;
}

sub set_openais_state {
  my $state       = shift;
  my $curhosttype = shift;
  my $curhostname = shift;

  my $text = $OPENAIS_STATE_text{$state};

  if ($curhosttype eq "Active") {
    $openais_sysinfo_state_a = $text;
  }
  if ($curhosttype eq "Standby") {
    $openais_sysinfo_state_b = $text;
  }
  if ($curhosttype eq "Quorum") {
    $openais_sysinfo_state_q = $text;
  }

  function_enter("set_openais_state", "state=$state text=$text hostname=$curhostname hosttype=$curhosttype");

  write_sysinfo_status();
  function_leave("set_openais_state");
}

sub set_internal_upgrade_status {
  my $status      = shift;
  my $curhosttype = shift;
  my $curhostname = shift;

  function_enter("set_internal_upgrade_status", "status=$status hostname=$curhostname hosttype=$curhosttype");
  my $text = $STATUS_text{$status};

  if ($curhosttype eq "Active") {
    $upgrade_sysinfo_status_a = $text;
  }
  if ($curhosttype eq "Standby") {
    $upgrade_sysinfo_status_b = $text;
  }
  if ($curhosttype eq "Quorum") {
    $upgrade_sysinfo_status_q = $text;
  }
  
  write_sysinfo_status();
  function_leave("set_internal_upgrade_status", "status=$status text=$text hostname=$curhostname hosttype=$curhosttype");
}


sub set_internal_upgrade_step {
  my $step          = shift;
  my $report_to_SWA = shift;
  my $curhosttype   = shift;
  my $curhostname   = shift;
  my $add_text      = shift;

  my $text = $STEPS_text{$step};
  
  if (defined($add_text)) {
    $internalUpgradeStepValue = $add_text;
  } else {
    $internalUpgradeStepValue = $text;
  }
  function_enter("set_internal_upgrade_step", "step=$step report_to_SWA=$report_to_SWA hostname=$curhostname hosttype=$curhosttype UpgradeRunning=$UpgradeRunning text=$internalUpgradeStepValue");

  $currentHostType = $curhosttype;
  $currentHostName = $curhostname;
  write_sysinfo_status();
  if ($report_to_SWA == 1) {
    write_upgrade_log($internalUpgradeStepValue);
  }
  function_leave("set_internal_upgrade_step", "step=$step");
}

sub set_activation_status_text {
  my $text        = shift;
  function_enter("set_activation_status_text", "text=$text");

  $activation_status_text = $text;
  function_leave("set_activation_status_text");
}

sub get_sysinfo_xml {
    
  
  my $UpgradeTypeValue="RLC";
  if ($hotfix_mode == 1) {
    $UpgradeTypeValue="Hotfix";
  }

  my $MasterUpgradeValue = "true";
  if ($nodaemon) {
    $MasterUpgradeValue = "false";
  }
  
  my $SWURestartValue="no";
  if ($rmx_reboot == 1) {
    $SWURestartValue="yes";
  }

  my $HAUpgradeValue="no";
  if ($upgrade_with_completely_stopped_ha == 1) {
    $HAUpgradeValue="yes";
  }

  my $TelefonyDowntimeValue="yes";
  if ($isMultiNode == 0) {
    # simplex/mono
    if ($rmx_reboot == 0) {
      # on simplex only type1 upgrades have no downtime
      $TelefonyDowntimeValue="no";
    }
  } else {
    # duplex, triplex
    $TelefonyDowntimeValue="no";
    if ($upgrade_with_completely_stopped_ha == 1) {
      $TelefonyDowntimeValue="yes";
      if ($upgrade_with_reduce_downtime == 1) {
        $TelefonyDowntimeValue="no";
      }
    }
  }
    
  my $RebootNecessaryValue="unknown";  
  if ($host_reboot == 1) {
    $RebootNecessaryValue="yes";
  } else {
    $RebootNecessaryValue="no";
  }
  
  # yes|no|n.a.
  my $BIOSUpgradeValue = "no";
  if ($biosupdate != 0) {
    $BIOSUpgradeValue = "yes";
  }
  if ($is_eco == 0) {
    $BIOSUpgradeValue = "n.a.";
  }

  my $activation_status_text_value = $activation_status_text;
  if ($activation_status_text_value eq "")  {
    if ($global_activation_rc != 0) {
      # obviously failure, but I do not have an explicit text
      $activation_status_text_value = "failure";
    }
  }
   
  my $xml = "";
  $xml .= "<upgrade>\n";
  $xml .= "<start_date>\n";  
  $xml .= "<year>$sysinfo_year</year>\n";
  $xml .= "<month>$sysinfo_mon</month>\n";
  $xml .= "<day>$sysinfo_mday</day>\n";      
  $xml .= "<hour>$sysinfo_hour</hour>\n"; 
  $xml .= "<minute>$sysinfo_min</minute>\n"; 
  $xml .= "<second>$sysinfo_sec</second>\n";  
  $xml .= "</start_date>\n";
  $xml .= "<stop_date>\n";  
  $xml .= "<year>$sysinfo2_year</year>\n";
  $xml .= "<month>$sysinfo2_mon</month>\n";
  $xml .= "<day>$sysinfo2_mday</day>\n";      
  $xml .= "<hour>$sysinfo2_hour</hour>\n"; 
  $xml .= "<minute>$sysinfo2_min</minute>\n"; 
  $xml .= "<second>$sysinfo2_sec</second>\n";  
  $xml .= "</stop_date>\n";
  $xml .= "<UpgradeRunning>$UpgradeRunning</UpgradeRunning>\n";
  $xml .= "<UpgradeType>$UpgradeTypeValue</UpgradeType>\n";
  $xml .= "<UpgradeStatus>$global_activation_rc</UpgradeStatus>\n";
  $xml .= "<UpgradeStatusText>$activation_status_text_value</UpgradeStatusText>\n";  
  $xml .= "<SWURestart>$SWURestartValue</SWURestart>\n";
  $xml .= "<HAUpgrade>$HAUpgradeValue</HAUpgrade>\n";
  $xml .= "<MasterUpgrade>$MasterUpgradeValue</MasterUpgrade>\n";
  $xml .= "<RebootNecessary>$RebootNecessaryValue</RebootNecessary>\n";
  $xml .= "<TelefonyDowntime>$TelefonyDowntimeValue</TelefonyDowntime>\n";
  $xml .= "<BIOSUpgrade>$BIOSUpgradeValue</BIOSUpgrade>\n";
  if ($is_eco == 1 and (length($delivered_bios_version) > 0 )) {
  $xml .= "<BIOSVersionDelivered>$delivered_bios_version</BIOSVersionDelivered>\n";
  }
  $xml .= "<currentHostType>$currentHostType</currentHostType>\n";
  $xml .= "<currentHostName>$currentHostName</currentHostName>\n";
  $xml .= "<fromVersion>$installed_version_at_beginning_of_upgrade</fromVersion>\n";
  $xml .= "<toVersion>$upgrade_version</toVersion>\n";
  $xml .= "<runningVersion>$running_version</runningVersion>\n";
  $xml .= "<internalUpgradeStep>$internalUpgradeStepValue</internalUpgradeStep>\n";
  $xml .= "<packageinfo></packageinfo>\n";

  if ($nodaemon == 0) {
    $xml .= "<UpgradeBlockActive>\n";
    $xml .= "<statusText>$upgrade_sysinfo_status_a</statusText>\n";
    $xml .= "<HA_State>$openais_sysinfo_state_a</HA_State>\n";
    $xml .= "<hostname>$hostname</hostname>\n";
    $xml .= "</UpgradeBlockActive>\n";
    if ($isMultiNode == 1) {
      $xml .= "<UpgradeBlockStandby>\n";
      $xml .= "<statusText>$upgrade_sysinfo_status_b</statusText>\n";
      $xml .= "<HA_State>$openais_sysinfo_state_b</HA_State>\n";
      $xml .= "<hostname>$other_node</hostname>\n";
      $xml .= "</UpgradeBlockStandby>\n";
      if (has_quorum($node_q, $multiNodeType) == 1) {
        $xml .= "<UpgradeBlockQuorum>\n";
        $xml .= "<statusText>$upgrade_sysinfo_status_q</statusText>\n";
        $xml .= "<HA_State>$openais_sysinfo_state_q</HA_State>\n";
        $xml .= "<hostname>$node_q</hostname>\n";
        $xml .= "</UpgradeBlockQuorum>\n";
      }
    }
  }

  $xml .= "</upgrade>\n";
  return($xml); 
}


sub write_sysinfo_status {
  function_enter("write_sysinfo_status");
  if ($write_sysinfo_status_enabled == 1) {
    write_file_exclusive($sysinfo_log, get_sysinfo_xml());
  }
  function_leave("write_sysinfo_status", "write_sysinfo_status_enabled=$write_sysinfo_status_enabled");
}

sub write_file_exclusive {
  my $file_name = shift;
  my $file_content = shift;
  if(open(WFEFILE, ">$file_name")) {
    protectFile($file_name);
    flock(WFEFILE, 2); # LOCK_EX
    print WFEFILE $file_content;
    flock(WFEFILE, 8); # LOCK_UN
    close(WFEFILE);
  } else {
    log_msg("could not write file $file_name");
    removeFile($file_name);
  }
}

sub remove_status_files {
  function_enter("remove_status_files");
  my $rc = 0;
  
  # clear status files
  $rc = removeFile($status_file);
  $rc = removeFile($status_file_tmp);
  $rc = removeFile($status_local_file);
  $rc = removeFile($status_remote_file);
  $rc = removeFile($status_quorum_file);
  function_leave("remove_status_files", "rc=$rc");
}

sub copy_upgrade_log {
  my $upgrade_version_to_use = shift;
  function_enter("copy_upgrade_log", "upgrade_version_to_use=$upgrade_version_to_use");
  my $rc = 0;
  if($hotfix_mode > 0) {
    log_msg_info("copy the current upgrade-daemon.log to a version expanded ($upgrade_version_to_use) file to keep it for later analysis");
    my $cmd = "/bin/cp -vfa $log_file $log_file.$upgrade_version_to_use";
    $rc = exec_cmd($cmd);  }

  function_leave("copy_upgrade_log", "rc=$rc");
}

sub transfer_sysinfo_status_to_remote_nodes {
  my $log_on_stdout = shift;
  my $rc = 0;
  my $output = "";
  my $other_node_is_reachable = 0;

  function_enter("transfer_sysinfo_status_to_remote_nodes", "log_on_stdout=$log_on_stdout");

  if ($isMultiNode == 1) {
    log_msg_info("plt-update: copy the sysinfo upgrade_info.xml to remote nodes (this may have failed before the reboot)", $log_on_stdout);
    log_msg_info("intention is that all nodes have the same files with same names. sysinfo will check the hostnames of the entries for correct column correlation");
    
    # synchronizes the sysinfo status, otherwise the sysinfo of the remote side would not get the current status
    execute_remote_duplex_copy($sysinfo_log,         $sysinfo_log,         "put");
    execute_remote_duplex_copy($sysinfo_standby_log, $sysinfo_standby_log, "put");

    if (has_quorum($node_q, $multiNodeType) == 1) {
      # synchronizes the sysinfo status also to the quorum, because it is possible to view the sysinfo page from there directly
      execute_remote_quorum_copy($sysinfo_log, $sysinfo_log, "put");
      execute_remote_quorum_copy($sysinfo_standby_log, $sysinfo_standby_log, "put");
      execute_remote_quorum_copy($sysinfo_quorum_log,  $sysinfo_quorum_log,  "put");

      execute_remote_duplex_copy($sysinfo_quorum_log,  $sysinfo_quorum_log,  "put");
    }
  }

  # for diagnostic purposes print it directly
  dump_sysinfo_status_to_log();
  function_leave("transfer_sysinfo_status_to_remote_nodes",  "write_sysinfo_status_enabled=$write_sysinfo_status_enabled rc=$rc");
  return ($rc);
}

sub dump_sysinfo_status_to_log {
  function_enter("dump_sysinfo_status_to_log");
  exec_cmd("cat $sysinfo_log");
  function_leave("dump_sysinfo_status_to_log",  "write_sysinfo_status_enabled=$write_sysinfo_status_enabled");
}


sub transfer_upgrade_status_to_remote_node {
  my $rc = 0;
  my $output = "";
  my ($other_node_is_reachable, $password_is_expired) = (0, 0);

  function_enter("transfer_upgrade_status_to_remote_node");

  if ($isMultiNode == 1) {
    $rc = execute_remote_duplex_copy($status_file_tmp, $status_file_tmp, "put");
    if ($rc == 0) {
      $rc = execute_remote_duplex_cmd("/bin/mv -v $status_file_tmp $status_file", \$output, $wait_for_ssh_connectivity, \$other_node_is_reachable, \$password_is_expired);
    }
  }
  function_leave("transfer_upgrade_status_to_remote_node", "rc=$rc");  
  return ($rc);
}

sub write_combined_upgrade_status {
  my $specific_status_file = shift;
  my $multiNodeType = shift;
  my $isMultiNode = shift;
  my $other_node = shift;
  my $node_q = shift;

  function_enter("write_combined_upgrade_status", "specific_status_file=$specific_status_file multiNodeType=$multiNodeType isMultiNode=$isMultiNode other_node=$other_node node_q=$node_q");

  my ($rc, $has_a_quorum) = (0, 0);
  my ($local_status, $remote_status, $quorum_status, $summary_status) = (0, 0, 0, 0);
  my ($text, $remote_text, $local_text, $quorum_text, $summary_text) = ("", "", "", "", "");
  my @status_lines = ();

  $has_a_quorum = has_quorum($node_q, $multiNodeType);

  $local_text  = read_file_content($status_local_file);
  @status_lines = split(/\n/, $local_text);
  foreach(@status_lines) {
      if(index($_, 'rc') >= 0) {
        my @parts = split(/<|>/, $_);
        $rc = $parts[2];
        log_msg_info("status rc: $rc");
      }elsif (index($_, 'errortext') >= 0) {
        my @parts = split(/<|>/, $_);
        $text = $parts[2];
        log_msg_info("status error text: $text");        
      }
  }
  $local_status = $rc;
  $local_text = $text;
  if($local_status > 0) { $local_status   = -$local_status;   }

  log_msg("write_combined_upgrade_status: local_status=$local_status");

  if ($local_status == -$ERROR_RemoteUpdateError) {
    $local_status = 1; 
    $local_text = "Active Node has not been upgraded due to failure on the standby/quorum node. You may have a split version situation.";
    log_msg("write_combined_upgrade_status: remote upgrade failure detected");
  }

  if ($isMultiNode == 1) {
    $remote_text = read_file_content($status_remote_file);
    if (length($remote_text) > 0) {
      @status_lines = split(/\n/, $remote_text);
      foreach(@status_lines) {
          if(index($_, 'rc') >= 0) {
            my @parts = split(/<|>/, $_);
            $rc = $parts[2];
            log_msg_info("status rc: $rc");
          }elsif (index($_, 'errortext') >= 0) {
            my @parts = split(/<|>/, $_);
            $text = $parts[2];
            log_msg_info("status error text: $text");
          }
      }
      $remote_status = $rc;
      $remote_text = $text;
      if($remote_status > 0) { $remote_status   = -$remote_status;   }
    } else {
      if ($upgrade_via_image == 1)  {
        log_msg("for NUC its normal that $status_remote_file is missing, no action needed");
      } else {
        log_msg("for non NUC in Duplex scenario its not normal that $status_remote_file is missing, setting partial status to failed, but overall is success");
        # positiv status to indicate this is not an error to assistant
        $remote_status = 1;
        if ($has_a_quorum == 1) {
          $remote_text = "Upgrade on standby node skipped probably due to failure on the quorum node update. You may have a split version situation.";
        } else {
          $remote_text = "Upgrade on standby node skipped or not possible.";
        }
      }
    }


    $quorum_text = read_file_content($status_quorum_file);
    if (length($quorum_text) > 0) {
      @status_lines = split(/\n/, $quorum_text);
      foreach(@status_lines) {
          if(index($_, 'rc') >= 0) {
            my @parts = split(/<|>/, $_);
            $rc = $parts[2];
            log_msg_info("status rc: $rc");
          }elsif (index($_, 'errortext') >= 0) {
            my @parts = split(/<|>/, $_);
            $text = $parts[2];
            log_msg_info("status error text: $text");
          }
      }
      $quorum_status = $rc;
      $quorum_text = $text;
      if($quorum_status > 0) { $quorum_status   = -$quorum_status;  }
    } else {
     if ($upgrade_via_image == 1)  {
        log_msg("for NUC its normal that $status_quorum_file is missing, no action needed");
      } else {
        log_msg("for non NUC in sep. Duplex scenario its not normal that $status_quorum_file is missing, setting partial status to failed, but overall is success");

        # positiv status to indicate this is not an error to assistant
        $quorum_status = 1;
        $quorum_text = "Upgrade on quorum node skipped or not possible";
      }
    }
  }

  # calculate the summary status
  if ($has_a_quorum == 1) {
    if ($quorum_status < 0 )  {
      $summary_status = $quorum_status;
      $summary_text   = $quorum_text;
    }
  }
  
  if ($isMultiNode == 1) {  
    if ($remote_status < 0 )  {
      $summary_status = $remote_status;
      $summary_text   = $remote_text;
    }  
  }
  if ($local_status < 0 )  {
    $summary_status = $local_status;
    $summary_text   = $local_text;
  }  else {
    if ($local_status == 1 )  {
      # remote update error
      $summary_status = - $ERROR_RemoteUpdateError;
      $summary_text   = "Upgrade on standby or quorum node failed. You may have a split version situation.";

    } else {
      # success
      $summary_status = 0;
      $summary_text   = "";
    }
  }
  
  write_extended_status($multiNodeType, $isMultiNode, $has_a_quorum, "", 0, $specific_status_file, $summary_status, $summary_text, $hostname, $local_status, $local_text, "node_a_errortext", $other_node, $remote_status, 1, $remote_text, "node_b_errortext", $node_q, $quorum_status, 1, $quorum_text, "node_q_errortext", $DATA_CURRENTNESS_STATUS_irrelevant);
    
  my $content_xml = read_file_content($specific_status_file);
  print ("$content_xml\n");
  function_leave("write_combined_upgrade_status");
  return($summary_status);
}

sub write_upgrade_status {
  my $rc = shift;
  my $errortext = shift;
  my $specific_status_file = shift;
  function_enter("write_upgrade_status", "rc=$rc errortext=$errortext specific_status_file=$specific_status_file");
  
  my $status = "";
  if($rc != 0) { $rc = -$rc; }
  if(open(WUSFILE, ">$specific_status_file")) {
    protectFile($specific_status_file);    
    $status .= "<date>".datefmt()."</date>\n";
    $status .= "<errortext>$errortext</errortext>\n";
    $status .= "<rc>$rc</rc>\n";
    print WUSFILE $status;
    close WUSFILE;
    chomp($status);
    log_msg("write_upgrade_status:\nfile=$specific_status_file\n$status");
    if(length($errortext) > 0) { log_msg_info($errortext); }
  }
  function_leave("write_upgrade_status");
}

sub write_upgrade_local_status {
  my $rc = shift;
  my $errortext = shift;
  function_enter("write_upgrade_local_status", "rc=$rc errortext=$errortext");

  # write here also the upgrade status for sysinfo for node_A
  if ($rc == 0) {
    set_internal_upgrade_status($STATUS_success, "Active", $hostname);
  } else {
    set_internal_upgrade_status($STATUS_failed,  "Active", $hostname);
  }
  
  write_upgrade_status($rc, $errortext, $status_local_file);
  function_leave("write_upgrade_local_status");
}

sub write_upgrade_remote_status {
  my $rc = shift;
  my $errortext = shift;
  function_enter("write_upgrade_remote_status", "rc=$rc errortext=$errortext");
  write_upgrade_status($rc, $errortext, $status_remote_file);
  function_leave("write_upgrade_remote_status");
}

sub write_upgrade_quorum_status {
  my $rc = shift;
  my $errortext = shift;
  function_enter("write_upgrade_quorum_status", "rc=$rc errortext=$errortext");
  write_upgrade_status($rc, $errortext, $status_quorum_file);
  function_leave("write_upgrade_quorum_status");
}

sub write_status {
  my $rc = shift;
  my $errortext = shift;
  
  log_msg_info("write_status rc=$rc text=$errortext");
  return;
}

sub write_short_status {
  my $rc = shift;
  my $errortext = shift;

  function_enter("write_short_status", "rc=$rc text=$errortext");

  my $status = "";
  if($rc != 0) { $rc = -$rc; }
  if(open(WSFILE, ">$status_file")) {
    protectFile($status_file);
    $status .= "<date>".datefmt()."</date>\n";
    $status .= "<errortext>$errortext</errortext>\n";
    $status .= "<rc>$rc</rc>\n";
    print WSFILE $status;
    close WSFILE;
    chomp($status);
    log_msg("update-status.log:\n$status");
    if(length($errortext) > 0) { log_msg_info($errortext); }
  }
  function_leave("write_short_status");
}


sub get_deployment_as_string {
  my $deployment_as_string_output = "";
  if ($is_manager == 1) {
    $deployment_as_string_output = "Manager";
  } else {
    $deployment_as_string_output = exec_cmd_out("$get_deployment_string_script");
    $deployment_as_string_output = my_trim($deployment_as_string_output);
  }

  log_msg_info("get_deployment_as_string: $deployment_as_string_output");
  return($deployment_as_string_output);
}

sub is_SBC_configured {
  function_enter("is_SBC_configured");
  my $sbc_is_configured = 0;
  my $sbc_configured_entry = retrieve_from_xml("system.sbc-enabled");
  if ($sbc_configured_entry == 1)  {
    log_msg_info("SBC resource is configured in haipsetup.xml");
    $sbc_is_configured = 1;
  }
  function_leave("is_SBC_configured", "sbc_is_configured=$sbc_is_configured");
  return($sbc_is_configured);
}

sub is_DTB_configured {
  function_enter("is_DTB_configured");
  my $dtb_is_configured = 0;
  my $dtb_configured_entry = retrieve_from_xml("system.dtb-configured");
  if ($dtb_configured_entry == 1)  {
    log_msg_info("DTB resource is configured in haipsetup.xml");
    $dtb_is_configured = 1;
  }
  function_leave("is_DTB_configured", "dtb_is_configured=$dtb_is_configured");
  return($dtb_is_configured);
}

sub get_number_of_sg_on_central_hosts {
  my $only_native_SGs = shift;
  function_enter("get_number_of_sg_on_central_hosts, only_native_SGs=$only_native_SGs");
  my ($number_sg, $has_sg, $has_sg1, $has_sg2) = (0, 0, 0, 0);
  $has_sg = retrieve_from_xml("system-hidden.softGate");

  # maximum 2 from iSG@Duplex in Container
  if ($isMultiNode == 1 and $only_native_SGs == 0) {
    $has_sg1 = retrieve_from_xml("system.iSG.node1");
    if ($has_sg1 ne "" and $has_sg1 == 1)  { $number_sg ++; }
    $has_sg2 = retrieve_from_xml("system.iSG.node2");
    if ($has_sg2 ne "" and $has_sg2 == 1)  { $number_sg ++; }
  }

  # maximum additional 1 for iSG@Simplex or iSG@Q
  if ($has_sg == 1)  {
    log_msg_info("softgate configured in haipsetup.xml, assuming number=1");
    $number_sg ++;
  }

  function_leave("get_number_of_sg_on_central_hosts", "number_sg=$number_sg ");
  return($number_sg);
}


sub get_number_of_container_sg_on_network{
  function_enter("get_number_of_container_sg_on_network");
  my $number_of_iSGs = 0;
  my $call_script = "/opt/inventory/inventory";
  my $call_string = $call_script. " appliance_list --search:isgonly 2>/dev/null| wc -l";

  my $rc = is_service_running("inventoryserverd");
  if ($rc == 1) {
    # we are on a central host and can call the inventory query
    if (-x $call_script) {
      $number_of_iSGs = exec_cmd_out($call_string);
    } else {
      log_msg_info("script $call_script does not exist, can not call it");
    }
  } else {
    # we are not on a central host and can not call the inventory query
    log_msg_info("not running on a central host, setting number_of_iSGs to 0");
    $number_of_iSGs = 0;
  }
  
  function_leave("get_number_of_container_sg_on_network", "number_of_iSGs=$number_of_iSGs");
  return($number_of_iSGs);
}


sub get_number_of_os_licenses{
  my $multiNodeType = shift;
  function_enter("get_number_of_os_licenses", "multiNodeType=$multiNodeType");

  my $number_of_os_licenses = 0;

  switch ($multiNodeType) {
    case 0   { $number_of_os_licenses = 1 }
    case 10  { $number_of_os_licenses = 3 }
    case 11  { $number_of_os_licenses = 3 }
    case 20  { $number_of_os_licenses = 1 }
    case 21  { $number_of_os_licenses = 1 }
    case 40  { $number_of_os_licenses = 1 }
    case 41  { $number_of_os_licenses = 1 }
    case 50  { $number_of_os_licenses = 1 }
    case 60  { $number_of_os_licenses = 2 }
    case 61  { $number_of_os_licenses = 2 }
    case 62  { $number_of_os_licenses = 3 }
    case 70  { $number_of_os_licenses = 1 }
    case 71  { $number_of_os_licenses = 1 }
    case 99  { $number_of_os_licenses = 2 }
    case 100 { $number_of_os_licenses = 1 }
    else    {
              log_msg_info("get_number_of_os_licenses unknown multiNodeType=$multiNodeType -> assuming license number 1");
              $number_of_os_licenses = 1 }
  }
  
  # log_msg_info("<<< get_number_of_os_licenses multiNodeType=$multiNodeType number_of_os_licenses=$number_of_os_licenses ");
  function_leave("get_number_of_os_licenses", " multiNodeType=$multiNodeType number_of_os_licenses=$number_of_os_licenses ");
  return($number_of_os_licenses);
}

sub is_survivable_deployment {
  my $multiNodeType = shift;

  function_enter("is_survivable_deployment", "multiNodeType=$multiNodeType");
  my $is_survivable=0;
  switch ($multiNodeType) {
    case 20  {
        log_msg_info("Survivable SG");
        $is_survivable=1;
      }
    case 71  {
        log_msg_info("Survivable Enterprise Gateway");
        $is_survivable=1;
      }      
    else  {
        log_msg_info("not a Survivable");
        $is_survivable=0;
      }
  }
  function_leave("is_survivable_deployment", "is_survivable=$is_survivable");
  return($is_survivable);
}

sub is_survivable_entgw {
  my $multiNodeType = shift;

  function_enter("is_survivable_entgw", "multiNodeType=$multiNodeType");
  my $is_survivable=0;
  switch ($multiNodeType) {
    case 71  {
        log_msg_info("Survivable Enterprise Gateway");
        $is_survivable=1;
      }
    else  {
        log_msg_info("not a Survivable Enterprise Gateway");
        $is_survivable=0;
      }
  }
  function_leave("is_survivable_entgw", "is_survivable=$is_survivable");
  return($is_survivable);
}

sub is_survivable_softgate {
  my $multiNodeType = shift;

  function_enter("is_survivable_softgate", "multiNodeType=$multiNodeType");
  my $is_survivable=0;
  switch ($multiNodeType) {
    case 20  {
        log_msg_info("Survivable SG");
        $is_survivable=1;
      }
    else  {
        log_msg_info("not a Survivable SG");
        $is_survivable=0;
      }
  }
  function_leave("is_survivable_softgate", "is_survivable=$is_survivable");
  return($is_survivable);
}

sub is_integrated_isg_softgate {
  function_enter("is_integrated_isg_softgate");
  my $is_isg=0; # is not iSG

  my $script="/opt/webservice/scripts/is_isg_enabled";
  if (-e $script) {
    my $rc = exec_cmd("$script");
    if ($rc == 0) {
      log_msg_info("not a iSG enabled");
      $is_isg=0;
    } else {
      log_msg_info("an iSG is enabled");
      $is_isg=1;
    }
  } else {
      log_msg_info("script $script does not exist, e.g. STMIX. Assuming not iSG enabled");
      $is_isg=0;
  }

  function_leave("is_integrated_isg_softgate", "is_isg=$is_isg ");
  return($is_isg);
}

sub is_integrated_softgate {
  my $multiNodeType = shift;

  function_enter("is_integrated_softgate", "multiNodeType=$multiNodeType");
  my $is_integrated=0;
  my $number_of_sg_on_central_hosts = get_number_of_sg_on_central_hosts(0);
  
  if ($number_of_sg_on_central_hosts == 0) {
    log_msg_info("no SG configured");
  } else {
    log_msg_info("SG configured");
    $is_integrated=1;
    switch ($multiNodeType) {
      case 20  {
          log_msg_info("Survivable SG");
          $is_integrated=0;
        }
      case 21  {
          log_msg_info("Standalone SG");
          $is_integrated=0;
      }
      case 62  {
          log_msg_info("Sep. DSCXL Duplex");
          if (isQuorumNode() == 1) {
            log_msg_info("SG on Quorum Sep. DSCXL Duplex");            
            $is_integrated=1;           
          } else {
            $is_integrated=0;
          }
      }
      case 10  {
          log_msg_info("Sep. Server Duplex");
          if (isQuorumNode() == 1) {
            log_msg_info("SG on Quorum Sep. Server Duplex");
            $is_integrated=1;
          } else {
            $is_integrated=0;
          }
      }     
      else  {
          log_msg_info("Neither Survivable nor Standalone SG nor Duplex");
          $is_integrated=1;
        }
    }
  }
  function_leave("is_integrated_softgate", "is_integrated=$is_integrated");
  return($is_integrated);
}

# real mere APE (nor Surv. SG) or Surv. EntGW))
sub isMereAPE  {
  my $multiNodeType = shift;

  function_enter("isMereAPE", "multiNodeType=$multiNodeType");
  my $is_ape=0;

  switch ($multiNodeType) {
      case 40  {
          log_msg_info("APE");
          $is_ape=1;
        }
  }
  function_leave("isMereAPE", "is_ape=$is_ape");
  return($is_ape);
}


sub isCentralHost {
  my $multiNodeType = shift;

  function_enter("isCentralHost", "multiNodeType=$multiNodeType");
  my $is_central=1;

  switch ($multiNodeType) {
      case 20  {
          log_msg_info("Survivable SG");
          $is_central=0;
        }
      case 21  {
          log_msg_info("Standalone SG");
          $is_central=0;
      }
      case 40  {
          log_msg_info("APE");
          $is_central=0;
        }
      case 70  {
          log_msg_info("EntGW");
          $is_central=0;
        }
      case 71  {
          log_msg_info("Survivable EntGW");
          $is_central=0;
      }        
      case 100  {
          log_msg_info("Undef deployment");
          $is_central=0;
      }
  }
  function_leave("isCentralHost", "is_central=$is_central");
  return($is_central);
}

sub isQuorumNode {
  function_enter("isQuorumNode", "");
  my $isQuorum = 0;
  my $nameQuorumNode = retrieve_from_xml("system.nameQuorumNode");
  if ($nameQuorumNode ne "" ) {
    if ($nameQuorumNode eq $hostname ) {
      log_msg_info("hostname $hostname matches entry system.nameQuorumNode in haipsetupentry.xml");
      $isQuorum = 1;
    } else {
      log_msg_info("hostname $hostname does not match entry system.nameQuorumNode=$nameQuorumNode in haipsetup.xml");
    }
  }
  function_leave("isQuorumNode", "isQuorum=$isQuorum");
  return($isQuorum);
}

sub is_softgate_standalone {
  my $multiNodeType = shift;
  
  function_enter("is_softgate_standalone", "multiNodeType=$multiNodeType");
  my $is_standalone=0;
  switch ($multiNodeType) {
    case 21  {
        log_msg_info("Standalone SG");
        $is_standalone=1;
      }
    else  {
        log_msg_info("not a SG standalone");
        $is_standalone=0;
      }
  }  
  function_leave("is_softgate_standalone", "is_standalone=$is_standalone");
  return($is_standalone);
}

sub is_entgw_standalone {
  my $multiNodeType = shift;

  function_enter("is_entgw_standalone", "multiNodeType=$multiNodeType");
  my $is_standalone=0;
  switch ($multiNodeType) {
    case 70  {
        log_msg_info("Standalone EntGW");
        $is_standalone=1;
      }
    else  {
        log_msg_info("not a standalone EntGW");
        $is_standalone=0;
      }
  }
  function_leave("is_entgw_standalone", "is_standalone=$is_standalone");
  return($is_standalone);
}

sub is_standalone_deployment {
  my $multiNodeType = shift;

  function_enter("is_standalone_deployment", "multiNodeType=$multiNodeType");
  my $is_standalone=0;
  switch ($multiNodeType) {
    case 21  {
        log_msg_info("Standalone SG");
        $is_standalone=1;
      }
    case 70  {
        log_msg_info("Standalone Enterprise Gateway");
        $is_standalone=1;
      }
    else  {
        log_msg_info("not a Standalone deployment");
        $is_standalone=0;
      }
  }
  function_leave("is_standalone_deployment", "is_standalone=$is_standalone");
  return($is_standalone);
}

sub is_duplex_standalone {
  my $multiNodeType = shift;

  function_enter("is_duplex_standalone", "multiNodeType=$multiNodeType");
  my $is_standalone=0;

  my $rc = exec_cmd("/usr/bin/standalone_operation status");
  if ($rc == 2) {
    log_msg_info("not a Duplex standalone");
    $is_standalone=0;
  } else {
    log_msg_info("it is a Duplex standalone");
    $is_standalone=1;
  }
  function_leave("is_duplex_standalone", "is_standalone=$is_standalone");
  return($is_standalone);
}

sub get_containsCSTA {
  my $multiNodeType = shift;
  my $containsCSTA=1;
  my $is_hpa = hpa_discover();
  function_enter("get_containsCSTA", "multiNodeType=$multiNodeType");
  switch ($multiNodeType) {
    case 21  {
        log_msg_info("Standalone SG -> disabled CSTA ");
        $containsCSTA=1;
      }
    case 41  {
        if ($is_hpa == 1) {
          log_msg_info("RG8300 deployment running on HPA500 -> disabled CSTA ");
          $containsCSTA = 0;
        } else {
          log_msg_info("RG8300 deployment NOT running on HPA500 -> enabled CSTA ");
          $containsCSTA = 1;
        }
      }
    case 70  {
        log_msg_info("Standalone EntGW -> disabled CSTA ");
        $containsCSTA=1;
      }      
    else  {
        if ($is_manager == 1) {
          log_msg_info("Manager does not have CSTA");
          $containsCSTA = 0;
        } else {
          log_msg_info("not a RG8300 or Manager but soco2 deployment -> it contains CSTA in any case");
          $containsCSTA = 1;
        }
      }
  }
  function_leave("get_containsCSTA", "multiNodeType=$multiNodeType containsCSTA=$containsCSTA ");
  return($containsCSTA);
}


sub get_local_mgmt_IP {
  function_enter("check_IP_for_localness","");
  my $rc = 0 ;

  my $hostname = exec_cmd_out("/bin/hostname", "silent");
  chomp($hostname);

  my $node1_name = retrieve_from_xml("system.name-node1");
  my $node2_name = retrieve_from_xml("system.name-node2");

  my $mgmg_ip_node1 = retrieve_from_xml("customer-lan.mgmt-node1-ip-address");
  if ($mgmg_ip_node1 eq "") {
    log_msg_info("There is no Management IP address defined for Node 1 in haipsetup.xml.");
  }
  my $mgmg_ip_node2 = retrieve_from_xml("customer-lan.mgmt-node2-ip-address");
  if ($isMultiNode == 1) {
    if ($mgmg_ip_node2 eq "") {
     log_msg_info("There is no Management IP address defined for Node 2 in haipsetup.xml.");
    }
  }

  my ($mgmg_ip_local, $mgmg_ip_remote) = ("", "");
  if ($hostname eq $node1_name) {
    $mgmg_ip_local=$mgmg_ip_node1;
    $mgmg_ip_remote=$mgmg_ip_node2;
  } elsif ($hostname eq $node2_name) {
    $mgmg_ip_local=$mgmg_ip_node2;
    $mgmg_ip_remote=$mgmg_ip_node1;
  } else {
    log_msg_info("Could not determine which node I am.\n");
  }
  function_leave("get_local_mgmt_IP","status=$rc, mgmg_ip_local=$mgmg_ip_local");
  return($mgmg_ip_local);
}

sub get_local_IPDA_IP {
  function_enter("get_local_IPDA_IP","");
  my $rc = 0 ;

  my $hostname = exec_cmd_out("/bin/hostname", "silent");
  chomp($hostname);

  my $node1_name = retrieve_from_xml("system.name-node1");
  my $node2_name = retrieve_from_xml("system.name-node2");

  my $ip_node1 = retrieve_from_xml("ipda-lan.server-ip-address-node1");
  if ($ip_node1 eq "") {
    log_msg_info("There is no IPDA IP address defined for Node 1 in haipsetup.xml.");
  }
  my $ip_node2 = retrieve_from_xml("ipda-lan.server-ip-address-node2");
  if ($isMultiNode == 1) {
    if ($ip_node2 eq "") {
     log_msg_info("There is no Management IP address defined for Node 2 in haipsetup.xml.");
    }
  }

  my ($ip_local, $ip_remote) = ("", "");
  if ($hostname eq $node1_name) {
    $ip_local=$ip_node1;
    $ip_remote=$ip_node2;
  } elsif ($hostname eq $node2_name) {
    $ip_local=$ip_node2;
    $ip_remote=$ip_node1;
  } else {
    log_msg_info("Could not determine which node I am.\n");
  }
  function_leave("get_local_IPDA_IP","status=$rc, ip_local=$ip_local");
  return($ip_local);
}

sub check_Yast_IPs {
  my $rc = shift ;
  my $msgEN = shift ;
  my $msgDE = shift ;
  my $other_node_is_reachable_ref = shift;
  function_enter("check_Yast_IPs","other_node_is_reachable=$$other_node_is_reachable_ref");

  $$rc = 0;
  $$msgEN = "";
  $$msgDE = "";

  my $hostname = exec_cmd_out("/bin/hostname", "silent");
  chomp($hostname);

  my $node1_name = retrieve_from_xml("system.name-node1");
  my $node2_name = retrieve_from_xml("system.name-node2");

  my $mgmg_ip_node1 = retrieve_from_xml("customer-lan.mgmt-node1-ip-address");
  if ($mgmg_ip_node1 eq "" or $mgmg_ip_node1 eq "0.0.0.0") {
    log_msg_info("There is no Management IP address defined for Node 1 in haipsetup.xml.");
  }
  my $mgmg_ip_node2 = retrieve_from_xml("customer-lan.mgmt-node2-ip-address");
  if ($isMultiNode == 1) {
    if ($mgmg_ip_node2 eq "" or $mgmg_ip_node2 eq "0.0.0.0") {
     log_msg_info("There is no Management IP address defined for Node 2 in haipsetup.xml.");
    }
  }

  my ($mgmg_ip_local, $mgmg_ip_remote) = ("", "");
  if ($hostname eq $node1_name) {
    $mgmg_ip_local=$mgmg_ip_node1;
    $mgmg_ip_remote=$mgmg_ip_node2;
  } elsif ($hostname eq $node2_name) {
    $mgmg_ip_local=$mgmg_ip_node2;
    $mgmg_ip_remote=$mgmg_ip_node1;
  } else {
    log_msg_info("Could not determine which node I am.\n");
  }

  log_msg_info("local node management IP: $mgmg_ip_local");
  log_msg_info("remote node management IP: $mgmg_ip_remote");

  if ($mgmg_ip_local ne "" and $mgmg_ip_local ne "dhcp" and $mgmg_ip_local ne "0.0.0.0") {
    my $addrs = exec_cmd_out("/sbin/ip --one -4 addr");
    if (index($addrs," ".$mgmg_ip_local."/") ==  -1) {
      log_msg_info("did not find  ".$mgmg_ip_local."/ "."in local IPs");
      $$rc = $$rc | 2;
      $$msgEN = "The Management IP $mgmg_ip_local is not present for Node $hostname. A restart of the corresponding Node is recommended via the Portal or Linux to resolve this issue.";
      # $$msgEN = "Active node Yast (Management) IP address $mgmg_ip_local is not present on its network interface.\n";
      $$msgDE = "Die Managementadresse $mgmg_ip_local des aktiven Knoten $hostname ist an der Netzwerkschnittstelle nicht vorhanden. Bitte starten Sie den betroffenen Knoten neu via Portal oder Linux, um diese Fehlersituation zu beheben.";
      # $$msgDE = "Die Yast IP-Adresse (Managementadresse) von aktiven Knoten $mgmg_ip_local ist in der Netzwerkschnittstelle nicht vorhanden.\n";
    } else {
      log_msg_info("found  ".$mgmg_ip_local."/ "."in local IPs");
    }
  }

  if ($isMultiNode == 1 and $$other_node_is_reachable_ref == 1) {
    my $password_is_expired_ref = 0;
    my $rem = 0;
    my $output = "";

    my ($max_try, $wait_for_ssh_lock, $ignore_remote_return_code) = (1, 30, 1);


    if ($mgmg_ip_remote ne "" and $mgmg_ip_remote ne "dhcp" and $mgmg_ip_remote ne "0.0.0.0") {
      $rem = execute_remote_duplex_cmd("/sbin/ip --one -4 addr",\$output,$max_try,$other_node_is_reachable_ref,\$password_is_expired_ref,$ignore_remote_return_code,$wait_for_ssh_lock);
      $output =~ s/remote rc: [0-9]*\n*//;
      chomp($output);
      if (index($output," ".$mgmg_ip_remote."/") == -1) {
        log_msg_info("did not find  ".$mgmg_ip_remote."/ "."in remote IPs: $output");
        $$rc = $$rc | 2;
        $$msgEN = "The Management IP $mgmg_ip_remote is not present for Node $other_node. A restart of the corresponding Node is recommended via the Portal or Linux to resolve this issue.";
        # $$msgEN .= "Standby node Yast (Management) IP address $mgmg_ip_remote is not present on its network interface.\n";
        $$msgDE = "Die Managementadresse $mgmg_ip_remote des Knoten $other_node ist an der Netzwerkschnittstelle nicht vorhanden. Bitte starten Sie den betroffenen Knoten neu via Portal oder Linux, um diese Fehlersituation zu beheben.";
        # $$msgDE = "Die Yast IP-Adresse (Managementadresse) von standby Knoten $mgmg_ip_remote ist in der Netzwerkschnittstelle nicht vorhanden.\n";
      } else {
        log_msg_info("found  ".$mgmg_ip_remote."/ "."in remote IPs");
      }
    }

    if (has_quorum($node_q, $multiNodeType) == 1) {
      my $mgmg_ipQ = retrieve_from_xml("customer-lan.mgmt-nodeQ-ip-address");
      if ($mgmg_ipQ ne "" and $mgmg_ipQ ne "dhcp") {
        my $password_is_expired_ref = 0;
        my $rem = 0;
        my $output = "";
        $rem = execute_remote_quorum_cmd("/sbin/ip --one -4 addr",\$output,$max_try,$other_node_is_reachable_ref,\$password_is_expired_ref,$ignore_remote_return_code );
        $output =~ s/remote rc: [0-9]*\n*//;
        chomp($output);
        if (index($output," ".$mgmg_ipQ."/") == -1) {
          log_msg_info("did not find  ".$mgmg_ipQ."/ "."in remote IPs: $output");
          $$rc = $$rc | 2;
          $$msgEN = "The Management IP $mgmg_ipQ is not present for Quorum node. A restart of the corresponding Node is recommended via the Portal or Linux to resolve this issue.";
          # $$msgEN .= "Quorum Yast (Management) IP address $mgmg_ipQ is not present on its network interface.\n";
          $$msgDE = "Die Managementadresse $mgmg_ipQ des Quorum-Knoten ist an der Netzwerkschnittstelle nicht vorhanden. Bitte starten Sie den betroffenen Knoten neu via Portal oder Linux, um diese Fehlersituation zu beheben.";
          # $$msgDE = "Die Yast IP-Adresse (Managementadresse) von Quorum $mgmg_ipQ ist in der Netzwerkschnittstelle nicht vorhanden.\n";
        } else {
          log_msg_info("found  ".$mgmg_ipQ."/ "."in Q IPs");
        }
      } else {
        log_msg_info( "There is no Management IP address defined for Quorum Node in haipsetup.xml.");
      }
    }
  } else {
    log_msg_info( "There is either no multinode scenario or remote node is not reachable");
  }

  function_leave("check_Yast_IPs","status=$$rc, other_node_is_reachable=$$other_node_is_reachable_ref");
  return(0);
}

sub get_plt_health_status {
  my $multiNodeType = shift;
  my $isMultiNode = shift;
  my $has_a_quorum = shift;
  my $bios_check_status = shift;
  my $data_currentness = shift;
  my $host_b_reachable = shift;
  my $rc_b = shift;
  my $host_q_reachable = shift;
  my $rc_q = shift;
  my $drbd_connect_status = shift;
  my $errortext_a = shift;
  my $errortext_b = shift;
  my $errortext_q = shift;
  my $hostname_a  = shift;
  my $hostname_b  = shift;
  my $hostname_q  = shift;

  function_enter("get_plt_health_status", "multiNodeType=$multiNodeType isMultiNode=$isMultiNode has_a_quorum=$has_a_quorum bios_check_status=$bios_check_status data_currentness=$data_currentness host_b_reachable=$host_b_reachable rc_b=$rc_b host_q_reachable=$host_q_reachable rc_q=$rc_q drbd_connect_status=$drbd_connect_status hostname_a=$hostname_a hostname_b=$hostname_b hostname_q=$hostname_q");
  my $status = "";
  my ($numerical_status, $firstinst_netw_check_status, $nuc_preparation_status) = (0, 0, 0);
  my ($upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync) = ("1", "1");
  my ($description_en, $description_de, $swa_action, $swa_action_rlc, $dashboard_action, $checkername_en, $checkername_de) = ("", "", "warn", "warn", "warn", "SW consistency check: CC/Node Status", "SW-Konsistenz-Check: CC/Node Status");

  my ($local_password_is_expired, $remote_password_is_expired, $other_node_is_reachable) = (0, 0, 1);
  check_local_password_validity(\$local_password_is_expired);
  if ($rc_b == $ERROR_RootPwdExpired or $rc_q == $ERROR_RootPwdExpired) {
    log_msg_info("remote node (B or Q) has a password expire situation, can skip some tests");
    $remote_password_is_expired = 1;
  }
  my ($major_rel, $minor_rel, $fix_rel, $hotfix_rel) = get_current_real_version_as_array();

  $status .= "\n";

  if ($is_manager == 0) {
	  my $specific_wait_for_up = 1;
	  if ($host_b_reachable == 1 and $host_q_reachable == 1 and $local_password_is_expired == 0) {
	    $numerical_status = wait_for_successful_startup(0, $upgrade_with_completely_stopped_ha, $upgrade_with_full_drbd_sync, $multiNodeType, $isMultiNode, "primary", $specific_wait_for_up, 0);
	    if ($numerical_status == 0) {
	      ($swa_action, $dashboard_action) = ("", "");
	    }  else {
	      ($swa_action, $dashboard_action) = ("block", "ignore");
	      $description_en = "at least one of the CCs or the ADP is not reachable or does not have the appropriate state for an upgrade: CC: Active/Standby , ADP: running";
	      $description_de = "mindestens ein CC oder der ADP sind entweder nicht erreichbar oder haben nicht den fuer einen Upgrade notwendigen Zustand: CC: Active/Standby , ADP: running";
	    }
	  } else {
	    log_msg_info("remote node is not reachable or local password expired, I can skip the health check");
	    $numerical_status = $ERROR_CanNotReachRemoteNode;
	    ($swa_action, $dashboard_action) = ("block", "warn");
	    $description_en = "at least one of the platform or quorum nodes is not reachable via corosync network";
	    $description_de = "mindestens einer der Platform- bzw. Quorum-Knoten ist ueber das Corosync-Netzwerk nicht erreichbar";
	    $checkername_en = "Platform Node Connectivity Check";
	    $checkername_de = "Plattform-Knoten Erreichbarkeits-Check";
	    if ($local_password_is_expired == 1) {
	      $description_en = "at least on the active platform node the root password is expired or not initialized";
	      $description_de = "mindestens auf dem aktiven Platform-Knoten ist das root Passwort abgelaufen oder uninitialisiert";
	      $checkername_en = "Root Password Expiry Check on Active Node";
	      $checkername_de = "Gültigkeits-Check des Root-Passworts auf dem aktiven Plattform-Knoten";
	      $numerical_status = $ERROR_RootPwdExpired;
	    } else {
	      if ($isMultiNode == 1) {
	        if ($rc_b == $ERROR_RootPwdExpired or $rc_q == $ERROR_RootPwdExpired) {
	          $description_en = "at least on one of the platform or quorum nodes the root password is expired or not initialized";
	          $description_de = "mindestens auf einem der Platform- bzw. Quorum-Knoten ist das root Passwort abgelaufen oder uninitialisiert";
	          $checkername_en = "Root Password Expiry Check on all Platform Nodes";
	          $checkername_de = "Gültigkeits-Check des Root-Passworts auf allen Plattform-Knoten";
	          $numerical_status = $ERROR_RootPwdExpired;
	        }
	      }
	    }
	  }

	  $status .= " <item>\n";
	  $status .= "  <name_en>$checkername_en</name_en>\n";
	  $status .= "  <name_de>$checkername_de</name_de>\n";
	  $status .= "  <status>$numerical_status</status>\n";
	  $status .= "  <description_en>$description_en</description_en>\n";
	  $status .= "  <description_de>$description_de</description_de>\n";
	  $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
	  $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
	  $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
	  $status .= " </item>\n";
	}

  if ($bios_check_status != 0) {
    ($swa_action, $dashboard_action) = ("block", "warn");
    $description_en = get_bios_error_string_en();
    $description_de = get_bios_error_string_de();
    $status .= " <item>\n";
    $status .= "  <name_en>BIOS version check</name_en>\n";
    $status .= "  <name_de>BIOS-Versions-Check</name_de>\n";
    $status .= "  <status>$bios_check_status</status>\n";
    $status .= "  <description_en>$description_en</description_en>\n";
    $status .= "  <description_de>$description_de</description_de>\n";
    $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
    $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
    $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
    $status .= " </item>\n";
  } else {
    ($swa_action, $dashboard_action) = ("", "");
    $description_en = "";
    $description_de = "";
  }

  # LAN status check
  $description_en = get_firstinst_netw_check_output($multiNodeType, $isMultiNode, $has_a_quorum);
  if (length($description_en) > 0 ) {
    $firstinst_netw_check_status = 1;
    $swa_action = "warn";
    $dashboard_action = "warn"; 
    $description_de = $description_en; # we do not have german firstinstall errors / warnings
  
    $status .= " <item>\n";
    $status .= "  <name_en>Platform network configuration check failed</name_en>\n";
    $status .= "  <name_de>Plattform-Netzwerkkonfigurations-Check schlug fehl</name_de>\n";
    $status .= "  <status>$firstinst_netw_check_status</status>\n";
    $status .= "  <description_en><![CDATA[$description_en]]></description_en>\n";
    $status .= "  <description_de><![CDATA[$description_de]]></description_de>\n";
    $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
    $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
    $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
    $status .= " </item>\n";
  }

  # NUC preparation status check
  $description_en = get_nuc_preparation_status_check_output($multiNodeType, $isMultiNode, $has_a_quorum);
  if (length($description_en) > 0 ) {
    $nuc_preparation_status = 5;
    $swa_action = "warn";
    $dashboard_action = "warn";
    $description_de = $description_en; 

    $status .= " <item>\n";
    $status .= "  <name_en><![CDATA[$description_en]]></name_en>\n";
    $status .= "  <name_de><![CDATA[$description_de]]></name_de>\n";
    $status .= "  <status>$nuc_preparation_status</status>\n";
    $status .= "  <description_en><![CDATA[$description_en]]></description_en>\n";
    $status .= "  <description_de><![CDATA[$description_de]]></description_de>\n";
    $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
    $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
    $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
    $status .= " </item>\n";
  }

  # drbd_connect_status check in case of Duplex (I45717)
  if ($isMultiNode == 1) {  
    if ($drbd_connect_status != 0) {
      log_msg_info("Its a Duplex with reachable node_b but DRBD is not connected -> block");
      $swa_action = "block";
      $dashboard_action = "warn";

      $description_en = "DRBD on platform is in erroneous state and will not recover automatically, complete Host reboot may recover this situation";
      $description_de = "Das Plattform-DRBD befindet sich in einem Fehlerzustand, der sich nicht selbst bereinigen kann. Ein kompletter Reboot des Host könnte u.U. diese Situation bereinigen.";

      $status .= " <item>\n";
      $status .= "  <name_en>Platform DRBD connect check failed</name_en>\n";
      $status .= "  <name_de>Plattform-DRBD-Connect-Check schlug fehl</name_de>\n";
      $status .= "  <status>1</status>\n";
      $status .= "  <description_en><![CDATA[$description_en]]></description_en>\n";
      $status .= "  <description_de><![CDATA[$description_de]]></description_de>\n";
      $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
      $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
      $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
      $status .= " </item>\n";

    }

    if ($remote_password_is_expired == 0) {
      my $corosync_hostnames = 0;
      check_corosync_hostnames(\$corosync_hostnames, \$other_node_is_reachable);
      if ($corosync_hostnames != 0 and $other_node_is_reachable==1) {
        log_msg_info("corosync hostnames do not match haipsetup.xml -> block");
        $swa_action = "block";
        $dashboard_action = "warn";

        $description_en = "Corosync hostnames in /etc/hosts do not match those in haipsetup.xml.";
        $description_de = "Corosync-Hostnamen in /etc/hosts stimmen nicht mit denen in haipsetup.xml überein.";

        $status .= " <item>\n";
        $status .= "  <name_en>Corosync hostnames check failed</name_en>\n";
        $status .= "  <name_de>Corosync hostnames Check schlug fehl</name_de>\n";
        $status .= "  <status>1</status>\n";
        $status .= "  <description_en><![CDATA[$description_en]]></description_en>\n";
        $status .= "  <description_de><![CDATA[$description_de]]></description_de>\n";
        $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
        $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
        $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
        $status .= " </item>\n";
      }
    } else {
      log_msg_info("corosync hostnames skipped due to password expire situation");
    }
  }

  if ($remote_password_is_expired == 0) {
    my $retc=0; my $desc_en=""; my $desc_de="";
    check_Yast_IPs(\$retc,\$desc_en,\$desc_de,\$other_node_is_reachable);
    if ($retc != 0) {
      if ($retc & 1) {
        log_msg_info("Yast IPs check failed -> warning.");
        $swa_action = "warn";
      }
      if ($retc & 2) {
        log_msg_info("Yast IPs check failed -> block.");
        $swa_action = "block";
      }
      $dashboard_action = "warn";

      $status .= " <item>\n";
      $status .= "  <name_en><![CDATA[$desc_en]]></name_en>\n";
      $status .= "  <name_de><![CDATA[$desc_de]]></name_de>\n";
      $status .= "  <status>1</status>\n";
      $status .= "  <description_en></description_en>\n";
      $status .= "  <description_de></description_de>\n";
      $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
      $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
      $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
      $status .= " </item>\n";
    }
  } else {
      log_msg_info("Yast IPs check skipped due to password expire situation");
  }
  


  # softraid check for DSCXL and Eco (I44569) removed with I50535 

  # HW check for COTS server
  if ($is_cots == 1) {
      log_msg_info("Its a COTS server");
      $swa_action = "warn";
      $dashboard_action = "warn";

      $description_en = "From OpenScape 4000 V8 COTS HW is no longer supported.
For further details please see Release Notes.";
      $description_de = "Ab OpenScape 4000 V8 wird Standard-Server(COTS)-HW nicht mehr unterstützt.
Für weitere Details siehe Release Note.";

      $status .= " <item>\n";
      $status .= "  <name_en>COTS (Standard Server) HW IDENTIFIED</name_en>\n";
      $status .= "  <name_de>Standard-Server(COTS)-HW identifiziert</name_de>\n";
      $status .= "  <status>1</status>\n";
      $status .= "  <description_en><![CDATA[$description_en]]></description_en>\n";
      $status .= "  <description_de><![CDATA[$description_de]]></description_de>\n";
      $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
      $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
      $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
      $status .= " </item>\n";
  }

  # HW check for DSCXL2 server
  if ($is_dscxl == 1) {
      my $is_MereAPE = isMereAPE($multiNodeType);
      if ($is_MereAPE == 1) {
        log_msg_info("APE on DSCXL server is no longer allowed on V10 R1 or later");
      } 
      log_msg_info("Its a DSCXL server, check major release");
      if ($major_rel >= 10) {
        log_msg_info("major release is V10 or higher, make DSCXL warning");
        $swa_action = "warn";
        $dashboard_action = "warn";

        $description_en = "From OpenScape 4000 V10 DSCXL2 HW is no longer supported. For further details please see Release Notes.";
        $description_de = "Ab OpenScape 4000 V10 wird DSCXL2-HW nicht mehr unterstützt. Für weitere Details siehe Release Note.";
        # no tooltip at all.
        
        $status .= " <item>\n";
        $status .= "  <name_en><![CDATA[$description_en]]></name_en>\n";
        $status .= "  <name_de><![CDATA[$description_de]]></name_de>\n";
        $status .= "  <status>1</status>\n";
        $status .= "  <description_en></description_en>\n";
        $status .= "  <description_de></description_de>\n";
        $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
        $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
        $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
        $status .= " </item>\n";
      } else {
        log_msg_info("major release is V7 or V8, skip DSCXL warning");
      }
  }

  # check REGEN-GENDB status
  log_msg_info("checking if is_gendb_needed_swupdated API is available");
  my $rc_nrg = exec_cmd("grep is_gendb_needed_swupdated /instmedia/sg_os_rlc/plt-update-new.sh");
  if ($rc_nrg == 0 and $is_manager == 0) {
    $rc_nrg = exec_cmd("/instmedia/sg_os_rlc/plt-update-new.sh is_gendb_needed_swupdated");
    log_msg_info("is_gendb_needed_swupdated rc : $rc_nrg");
    # rc 0 = failed, 1 = running, 2 = ok, 3 = not feasible in this moment
    if ($rc_nrg == 0) {
      log_msg_info("NRG has failed => block SWA and also write to Dashboard");
      $swa_action = "block";
      $dashboard_action = "warn";

      $description_en = exec_cmd_out("/instmedia/sg_os_rlc/plt-update-new.sh is_gendb_needed_swupdated");
      $description_de = $description_en;

      $status .= " <item>\n";
      $status .= "  <name_en>Automatic REGEN-GENDB process has failed</name_en>\n";
      $status .= "  <name_de>Der automatische REGEN-GENDB Prozess schlug fehl.</name_de>\n";
      $status .= "  <status>1</status>\n";
      $status .= "  <description_en><![CDATA[$description_en]]></description_en>\n";
      $status .= "  <description_de><![CDATA[$description_de]]></description_de>\n";
      $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
      $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
      $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
      $status .= " </item>\n";
    }
    if ($rc_nrg == 1) {
      my $get_prep_script   = "$nuc_bin_dir"."get_prep_status.sh";
      my $rc_prep = exec_cmd($get_prep_script);
      if($rc_prep != 1) {
        log_msg_info("NRG is in progress => inform the User");
        $swa_action = "warn";
        $dashboard_action = "warn";

        $description_en = "Automatic REGEN-GENDB process is running right now.";
        $description_de = "Ein automatischer REGEN-GENDB Prozess wurde bereits gestartet.";

        $status .= " <item>\n";
        $status .= "  <name_en>Automatic REGEN-GENDB is running</name_en>\n";
        $status .= "  <name_de>Automatischer REGEN-GENDB laeuft gerade</name_de>\n";
        $status .= "  <status>1</status>\n";
        $status .= "  <description_en><![CDATA[$description_en]]></description_en>\n";
        $status .= "  <description_de><![CDATA[$description_de]]></description_de>\n";
        $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
        $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
        $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
        $status .= " </item>\n";
      } else {
        log_msg_info("NUC Prep is in progress => do not inform the User about REGEN-GENDB status yet");
      }
    }
    if ($rc_nrg == 2) {
      log_msg_info("NRG finished ok => nothing to do.");
    }
    if ($rc_nrg == 3) {
      log_msg_info("NRG status query is not feasible at this moment => nothing to do.");
    }
  } else {
    log_msg_info("is_gendb_needed_swupdated API is NOT available => ignore it");
  }

  # VMware resource checking
  my $vm_checker = "/opt/soco-common/check_vmguest";
  my $rc_vm = 0;
  if ($is_virtual == 1) {
      log_msg_info("Its a virtual/VMWare environment (including Manager), check the resources");
      if (-x $vm_checker) {
        log_msg_info("$vm_checker available. execute check");
        $rc_vm = exec_cmd($vm_checker);
        if ($rc_vm == 0) {
          log_msg_info("check successful, no action");
        } else {
          $swa_action = "warn";
          $dashboard_action = "warn";

          # get the output (have to call the checker again)
          my $check_output = exec_cmd_out($vm_checker);
          
          $description_en = $check_output;
          $description_de = $check_output;

          $status .= " <item>\n";
          $status .= "  <name_en>VM resource allocation error</name_en>\n";
          $status .= "  <name_de>Fehlerhafte VM-Ressourcenallokation</name_de>\n";
          $status .= "  <status>1</status>\n";
          $status .= "  <description_en><![CDATA[$description_en]]></description_en>\n";
          $status .= "  <description_de><![CDATA[$description_de]]></description_de>\n";
          $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
          $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
          $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
          $status .= " </item>\n";
        }
      } else {
        log_msg_info("$vm_checker not available. checks not possible.");
      }

  if (has_quorum($node_q, $multiNodeType) == 1) {
    log_msg_info("Its a sep. Duplex, check the Q for virtual/VMWare environment");
    $rc_vm=check_vmguest_remote($node_q, $vm_checker);
    if ($rc_vm == 0) {
       log_msg_info("check successful, no action");
     } else {
       $swa_action = "warn";
       $dashboard_action = "warn";

       my $check_output = "check_vmguest: Error: Quorum VM is not configured correctly.";

       $description_en = $check_output;
       $description_de = $check_output;

       $status .= " <item>\n";
       $status .= "  <name_en>VM resource allocation error on Quorum</name_en>\n";
       $status .= "  <name_de>Fehlerhafte VM-Ressourcenallokation auf Quorum</name_de>\n";
       $status .= "  <status>1</status>\n";
       $status .= "  <description_en><![CDATA[$description_en]]></description_en>\n";
       $status .= "  <description_de><![CDATA[$description_de]]></description_de>\n";
       $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
       $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
       $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
       $status .= " </item>\n";
     }
    }
  }

  # check if there is an USB stick plugged by parsing the getversionInfo of each node
  log_msg_info("checking for plugged USB stick");
  my ($usb_status_a,$usb_status_b,$usb_status_q,$affected_host) = (0, 0, 0, "");
  
  $usb_status_a    = parse_xml_tag("check_usb_status", $errortext_a);
  if ($usb_status_a eq "") { $usb_status_a = 0; }

  if ($usb_status_a != 0) {
    $affected_host = $hostname_a;
  }

  if ($isMultiNode == 1) {
    if ($host_b_reachable == 1) {
      $usb_status_b    = parse_xml_tag("check_usb_status", $errortext_b);
      if ($usb_status_b eq "") { $usb_status_b = 0; }

      if ($usb_status_b != 0) {
        $affected_host = $hostname_b;
      }
    }

    if (has_quorum($node_q, $multiNodeType) == 1) {
      if ($host_q_reachable == 1) {
        $usb_status_q    = parse_xml_tag("check_usb_status", $errortext_q);
        if ($usb_status_q eq "") { $usb_status_q = 0; }

        if (($host_q_reachable == 1) and ($usb_status_q != 0)) {
          $affected_host = $hostname_q;
        }
      }
    }
  }

  if (($usb_status_a != 0) or ($usb_status_b != 0) or ($usb_status_q != 0)) {
      $swa_action = "warn";
      $dashboard_action = "warn";
      $description_en = "A bootable USB device is detected on $affected_host. The device should be removed to ensure correct boot sequence on next startup.";
      $description_de = "Ein bootfaehiges USB-Device wurde am Knoten $affected_host erkannt. Dieses Device sollte entfernt werden, um einen ordnungsgemaessen Boot beim naechsten Startup sicherzustellen.";

      $status .= " <item>\n";
      $status .= "  <name_en>USB device detected</name_en>\n";
      $status .= "  <name_de>USB-Device erkannt</name_de>\n";
      $status .= "  <status>1</status>\n";
      $status .= "  <description_en><![CDATA[$description_en]]></description_en>\n";
      $status .= "  <description_de><![CDATA[$description_de]]></description_de>\n";
      $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
      $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
      $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
      $status .= " </item>\n";
  }

  # check NTP status by parsing the getversionInfo of each node
  log_msg_info("checking if NTP is in sync");
  my ($ntp_status_a,$ntp_output_a,$ntp_status_b,$ntp_output_b,$ntp_status_q,$ntp_output_q) = (0, "", 0, "",0, "");
  my $check_output = "";
  $ntp_status_a    = parse_xml_tag("ntp_check_status", $errortext_a);
  if (($ntp_status_a != 0)) {
    $check_output = "NTP is not in sync on $hostname";
  }
  $ntp_output_a    = parse_xml_tag("ntp_syncstate", $errortext_a);

  if ($isMultiNode == 1) {
    if ($host_b_reachable == 1) {
      $ntp_status_b    = parse_xml_tag("ntp_check_status", $errortext_b);
      $ntp_output_b    = parse_xml_tag("ntp_syncstate", $errortext_b);
      if ($ntp_status_b != 0) {
        $check_output = "NTP is not in sync on standby node";
      }
    }

    if (has_quorum($node_q, $multiNodeType) == 1) {
      if ($host_q_reachable == 1) {
        $ntp_status_q    = parse_xml_tag("ntp_check_status", $errortext_q);
        $ntp_output_q    = parse_xml_tag("ntp_syncstate", $errortext_q);
        if (($host_q_reachable == 1) and ($ntp_status_q != 0)) {
          $check_output = "NTP is not in sync on quorum node";
        }
      }
    }
  }
  if ($isMultiNode == 1) {
    # NTP sync warning shall be displayed only for Duplex scenarios (OSFOURK-9365)
    if (($ntp_status_a != 0) or ($ntp_status_b != 0) or ($ntp_status_q != 0)) {
      $swa_action = "warn";
      $dashboard_action = "warn";
      $description_en = $check_output;
      $description_de = $check_output;

      $status .= " <item>\n";
      $status .= "  <name_en>NTP synchronization problem</name_en>\n";
      $status .= "  <name_de>NTP-Synchronisation fehlerhaft</name_de>\n";
      $status .= "  <status>1</status>\n";
      $status .= "  <description_en><![CDATA[$description_en]]></description_en>\n";
      $status .= "  <description_de><![CDATA[$description_de]]></description_de>\n";
      $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
      $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
      $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
      $status .= " </item>\n";
    }
  }

  # ec version check for Eco2
  # check ec status by parsing the getversionInfo of each node
  log_msg_info("checking EcoServer2/Branch2 EC version for bad 1.0 or 1.1 versions");
  my ($ec_version_a, $ec_version_b, $ec_version_q, $check_ec_output_en, $check_ec_output_de, $eco2_ectrl_check_status) = ("", "", "", "", "", 0);
  $ec_version_a    = parse_xml_tag("eco2_ectrl_version", $errortext_a);
  if ($ec_version_a ne "") {
    # bad string is: V 1.1 (02.04.2020 09:41)
    $check_output = "EC version on $hostname: $ec_version_a";
    log_msg_info("EcoServer2/Branch2 EC version: $ec_version_a");
    if ($ec_version_a =~ /V 1.[01]/)  {
      log_msg_info("EcoServer2/Branch2 EC version too old");
      $check_ec_output_en .= "Old version 1.0 or 1.1 of EcoServer2/Branch2 embedded controller SW on active node detected.\nThis error may endanger update or replacement operations.\nPlease check SW Release Note for further guidance.\n";
      $check_ec_output_de .= "Veraltete Version 1.0 oder 1.1 der EcoServer2/Branch2 embedded controller SW auf dem aktiven Knoten erkannt.\nDies macht Update und Replacement Operationen riskant.\nBitte SW-Releasenote beachten.";
    }
  }

  if ($isMultiNode == 1) {
    if ($host_b_reachable == 1) {
      $ec_version_b    = parse_xml_tag("eco2_ectrl_version", $errortext_b);
      if ($ec_version_b ne "") {
        if ($ec_version_b =~ /V 1.[01]/) {
          log_msg_info("EcoServer2/Branch2 EC version too old on Standby");
          $check_ec_output_en .= "Old version 1.0 or 1.1 of EcoServer2/Branch2 embedded controller SW on standby node detected.\nThis error may endanger update or replacement operations.\nPlease check SW Release Note for further guidance.\n";
          $check_ec_output_de .= "Veraltete Version 1.0 oder 1.1 der EcoServer2/Branch2 embedded controller SW auf dem Standby Knoten erkannt.\nDies macht Update und Replacement Operationen riskant.\nBitte SW-Releasenote beachten.";
        }
      }
    }

    if (has_quorum($node_q, $multiNodeType) == 1) {
      if ($host_q_reachable == 1) {
        $ec_version_q    = parse_xml_tag("eco2_ectrl_version", $errortext_q);
        if (($host_q_reachable == 1) and ($ec_version_q ne "")) {
          if ($ec_version_q =~ /V 1.[01]/) {
            log_msg_info("EcoServer2/Branch2 EC version too old on Quorum");
            $check_ec_output_en .= "Old version 1.0 or 1.1 of EcoServer2/Branch2 embedded controller SW on Quorum node detected.\nThis error may endanger update or replacement operations.\nPlease check SW Release Note for further guidance.\n";
            $check_ec_output_de .= "Veraltete Version 1.0 oder 1.1 der EcoServer2/Branch2 embedded controller SW auf dem Quorum Knoten erkannt.\nDies macht Update und Replacement Operationen riskant.\nBitte SW-Releasenote beachten.";
          }
        }
      }
    }
  }
  if ($check_ec_output_en ne "") {
    $eco2_ectrl_check_status = 1;
    $swa_action = "warn";
    $swa_action_rlc = "warn";
    $dashboard_action = "warn";
    $description_en = $check_output;

    $status .= " <item>\n";
    $status .= "  <name_en>EcoServer2/Branch2 embedded controller version too old</name_en>\n";
    $status .= "  <name_de>EcoServer2/Branch2 Embedded Controller Version zu alt</name_de>\n";
    $status .= "  <status>$eco2_ectrl_check_status</status>\n";
    $status .= "  <description_en><![CDATA[$check_ec_output_en]]></description_en>\n";
    $status .= "  <description_de><![CDATA[$check_ec_output_de]]></description_de>\n";
    $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
    $status .= "  <swa_action_rlc>$swa_action_rlc</swa_action_rlc>\n";
    $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
    $status .= " </item>\n";
  }

  # root partition size check on quorum
  if ($isMultiNode == 1) {
    if (has_quorum($node_q, $multiNodeType) == 1) {
      log_msg_info("checking root partition size on quorum");
      my ($quorum_rootpartsize, $rootpartsize_check_status) = (0, 0);
      $quorum_rootpartsize = parse_xml_tag("rootpartsize", $errortext_q);
      if (($host_q_reachable == 1) and ($quorum_rootpartsize ne "")) {
        log_msg_info("it is quorum and root partition size is: $quorum_rootpartsize");
        if (looks_like_number($quorum_rootpartsize)) {
          if ($quorum_rootpartsize <= 12321208) {
              log_msg_info("quorum root partition size is smaller than 12321208 (30 GB)");
              $rootpartsize_check_status = 1;
              $swa_action = "warn";
              $swa_action_rlc = "block";
              $dashboard_action = "warn";
              $description_en = "Starting with OS4K V10R1, the Quorum Node must have at least 75GB of storage (like Quorum with Standard SoftGate OVF)";
              $description_de = "Ab OS4K V10R1 muss der Quorum Node mindestens 75GB Storage haben (wie Quorum mit Standard SoftGate OVF)";
              
              $status .= " <item>\n";
              $status .= "  <name_en>Too small layout 30GB of virtual HD on Quorum Node</name_en>\n";
              $status .= "  <name_de>Zu kleines virtuelles HD Layout von 30GB auf Quorum Node</name_de>\n";
              $status .= "  <status>$rootpartsize_check_status</status>\n";
              $status .= "  <description_en><![CDATA[$description_en]]></description_en>\n";
              $status .= "  <description_de><![CDATA[$description_de]]></description_de>\n";
              $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
              $status .= "  <swa_action_rlc>$swa_action_rlc</swa_action_rlc>\n";
              $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
              $status .= " </item>\n";
          }
          else {
            log_msg_info("quorum root partition size is not a number: $quorum_rootpartsize, cannot check the size");
          }
        }
      }
    }
  }

  # check second disk plugged, but not in Raid (OSFOURK-22270)
  log_msg_info("checking second disk plugged, but not in Raid");
  my ($raid_status_a, $raid_status_b, $raid_status_q, $affected_node) = (0, 0, 0, "");
  my $raid_check_output = "";
  $raid_status_a    = parse_xml_tag("disk_raid_inconsistency", $errortext_a);
  if ($raid_status_a eq "") { $raid_status_a = 0; }
  if ($raid_status_a != 0)  { $affected_node = "$hostname_a"; }
  if ($isMultiNode == 1) {
    $raid_status_b    = parse_xml_tag("disk_raid_inconsistency", $errortext_b);
    if ($raid_status_b eq "") { $raid_status_b = 0; }
    if ($raid_status_b != 0)  { $affected_node = "$hostname_b"; }

    if (has_quorum($node_q, $multiNodeType) == 1) {
      $raid_status_q    = parse_xml_tag("disk_raid_inconsistency", $errortext_q);
      if ($raid_status_q eq "") { $raid_status_q = 0; }
      if ($raid_status_q != 0)  { $affected_node = "$hostname_q"; }
    }
  }
  if (($raid_status_a != 0) or ($raid_status_b != 0) or ($raid_status_q != 0)) {
    $swa_action = "warn";
    $dashboard_action = "warn";
    $description_en = "Warning: An inactive disk is found in Node $affected_node. To avoid problems at boot, either the inactive disk should be removed, or both disks returned to active synchronizing via Portal -> Recovery HD.";
    $description_de = "Warnung: Eine inaktive Platte im Knoten $affected_node gefunden. Um Probleme beim Boot zu vermeiden muss entweder die inaktive Platte entfernt oder beide Platten synchronisiert werden via Portal -> Recovery HD.";

    $status .= " <item>\n";
    $status .= "  <name_en>Second disk not in RAID</name_en>\n";
    $status .= "  <name_de>zweite Platte nicht im RAID</name_de>\n";
    $status .= "  <status>1</status>\n";
    $status .= "  <description_en><![CDATA[$description_en]]></description_en>\n";
    $status .= "  <description_de><![CDATA[$description_de]]></description_de>\n";
    $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
    $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
    $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
    $status .= " </item>\n";
  }
  
  # check HD SMART failure
  log_msg_info("checking HD SMART self test assessment");
  my ($hd_status_a, $hd_status_b, $hd_status_q, $hd_output_a, $hd_output_b, $hd_output_q) = (0, 0, 0, "", "", "");
  my $hd_check_output = "";
  $hd_status_a    = parse_xml_tag("hd_check_status", $errortext_a);
  if ($hd_status_a eq "") { $hd_status_a = 0; }
  if ( ($hd_status_a != 0)) {
   $hd_check_output = parse_xml_tag("hd_check_text", $errortext_a);
  }

  if ($isMultiNode == 1) {
    $hd_status_b    = parse_xml_tag("hd_check_status", $errortext_b);
    if ($hd_status_b eq "") { $hd_status_b = 0; }
    if (($hd_status_b != 0)) {
      $hd_check_output = parse_xml_tag("hd_check_text", $errortext_b);
    }

    if (has_quorum($node_q, $multiNodeType) == 1) {
      $hd_status_q    = parse_xml_tag("hd_check_status", $errortext_q);
      if ($hd_status_q eq "") { $hd_status_q = 0; }
      if (($hd_status_q != 0)) {
        $hd_check_output = parse_xml_tag("hd_check_text", $errortext_q);
      }
    }
  }
  if (($hd_status_a != 0) or ($hd_status_b != 0) or ($hd_status_q != 0)) {
    $swa_action = "block";
    $dashboard_action = "warn";
    $description_en = $hd_check_output;
    $description_de = $hd_check_output;

    $status .= " <item>\n";
    $status .= "  <name_en>HD critical error</name_en>\n";
    $status .= "  <name_de>kritisches HD-Problem</name_de>\n";
    $status .= "  <status>1</status>\n";
    $status .= "  <description_en><![CDATA[$description_en]]></description_en>\n";
    $status .= "  <description_de><![CDATA[$description_de]]></description_de>\n";
    $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
    $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
    $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
    $status .= " </item>\n";
  }
  
  #OSFOURK-27603 Display warning in Assistant Dashboard in case Kubernetes is enabled on VM without enough RAM Memory
  log_msg_info("checking for enough RAM on VMs in case Kubernetes is enabled");
  if (-e "/opt/webservice/scripts/is_4kube_feasible") {
    my $rc_script_has_proper_api_version=system("grep -q __SWUPDATED_API_VERSION /opt/webservice/scripts/is_4kube_feasible");
    if ($rc_script_has_proper_api_version == 0) {
        my $output_is_4kube_feasible = exec_cmd_out("bash /opt/webservice/scripts/is_4kube_feasible swupdated");
        chomp($output_is_4kube_feasible);
        log_msg_info( "is_4kube_feasible returned : $output_is_4kube_feasible");
        if ( $output_is_4kube_feasible =~ /4KUBE_CHECK_IGNORE/ ) {
            log_msg_info("Check not applicable.");
        } else {
            if ( $output_is_4kube_feasible =~ /4KUBE_CHECK_OK/ ) {
                log_msg_info("Check returned OK.");
            } else {
                log_msg_info("Check returned NOT OK => show info in Dashboard");
                $swa_action = "warn";
                $dashboard_action = "warn";

                $status .= " <item>\n";
                $status .= "  <name_en>Kubernetes enabled but not enough RAM Memory.</name_en>\n";
                $status .= "  <name_de>Kubernetes ist aktiv aber RAM Speicher ist zu wenig</name_de>\n";
                $status .= "  <status>1</status>\n";
                $status .= "  <description_en><![CDATA[$output_is_4kube_feasible]]></description_en>\n";
                $status .= "  <description_de><![CDATA[$output_is_4kube_feasible]]></description_de>\n";
                $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
                $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
                $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
                $status .= " </item>\n";
            }
        }
    } else {
        log_msg_info("Script /opt/webservice/scripts/is_4kube_feasible found but does not contain the NEW health check code => skipping check");
    }
  }
  
  # check OSA500 with 4GB RAM in cluster
  log_msg_info("checking OSA500 with 4GB RAM on V10 platform");
  if ($major_rel >= 10) {
	  my $osa_script="/opt/webservice/scripts/is_osa500_4GB_in_cluster";
	  if (-f "$osa_script" ) {
	    my $rc_osa5004gb = exec_cmd("$osa_script");
	    if ($rc_osa5004gb == 0) {
	      log_msg_info("No OSA500 with 4GB in cluster found via inventory query, OK");
	    } else {
	      if ($rc_osa5004gb == -1 or $rc_osa5004gb == 255) {
	        log_msg_info("script $osa_script reported general error, no propagation to Assistant");
	      } else {
		      log_msg_info("OSA500 with 4GB in cluster found via inventory query, rc_osa5004gb=$rc_osa5004gb");
			    $swa_action = "warn";
			    $dashboard_action = "warn";

			    $status .= " <item>\n";
			    $status .= "  <name_en>One or more OSA500 with 4GB</name_en>\n";
			    $status .= "  <name_de>Mindestens eine OSA500 mit 4GB</name_de>\n";
			    $status .= "  <status>1</status>\n";
			    $status .= "  <description_en><![CDATA[One or more OSA500 with 4GB RAM detected. Support will end from OS4K V10 R1. Please check Appliance Management to identify HW.]]></description_en>\n";
			    $status .= "  <description_de><![CDATA[Mindestens eine OSA500 mit 4GB RAM entdeckt. Support endet ab V10 R1. Bitte Appliance Management checken, um die HW zu identifizieren.]]></description_de>\n";
			    $status .= "  <swa_action_hf>$swa_action</swa_action_hf>\n";
			    $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
			    $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
			    $status .= " </item>\n";
			  }
	    }
	  } else {
	    log_msg_info("Script $osa_script not found, skipping check");
	  }
  }

  # check HW for Eco1/Eco2 mix mode
  log_msg_info("checking Eco1/Eco2 mix mode on Duplex");
  if ($isMultiNode == 1) {
    my ($hw_a, $hw_b) = ("", "");
    $hw_a    = parse_xml_tag("hw", $errortext_a);
    $hw_b    = parse_xml_tag("hw", $errortext_b);
    if (($hw_a eq "ECOSERVER" and $hw_b eq "ECOSERVER2") or ($hw_a eq "ECOSERVER2" and $hw_b eq "ECOSERVER")) {
      log_msg_info("Eco1/Eco2 mix mode on Duplex detected");
	    $swa_action = "warn";
	    $dashboard_action = "warn";

	    $status .= " <item>\n";
	    $status .= "  <name_en>EcoServer mixed mode is detected!</name_en>\n";
	    $status .= "  <name_de>EcoServer mixed mode erkannt!</name_de>\n";
	    $status .= "  <status>1</status>\n";
	    $status .= "  <description_en><![CDATA[Usage of mixed mode (meaning use of different EcoServer HW revisions in a Duplex/Separated Duplex deployment) is only allowed on Node 1 and Node 2 to temporarily assist customer service transitions.]]></description_en>\n";
	    $status .= "  <description_de><![CDATA[EcoServer Mixed Mode (d.h. Verwendung von unterschiedlichen EcoServer HW-Ständen in einem Duplex/Separated Duplex Deployment) ist für Node 1 und Node 2 nur vorübergehend für Service Aktionen wie Node Replacement erlaubt.]]></description_de>\n";
	    $status .= "  <swa_action_hf></swa_action_hf>\n";
	    $status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
	    $status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
	    $status .= " </item>\n";
		}
	}

	# check for enabled SBC in cluster
	if ( $is_manager == 0 ) {
		log_msg_info("checking enabled SBC");
		my $sbc_script="/opt/webservice/scripts/is_sbc_in_cluster";
		if (-f "$sbc_script") {
			log_msg_info("script $sbc_script found, use it");
			my $rc_active_sbcs = exec_cmd("$sbc_script");
			if ($rc_active_sbcs == 0) {
				log_msg_info("No SBC in cluster found");
			} else {
				if ($rc_active_sbcs == -1 or $rc_active_sbcs == 255) {
					log_msg_info("script $sbc_script reported general error, no propagation to Assistant");
				} else {
					log_msg_info("$rc_active_sbcs SBCs in cluster found, warn SWA");
					$swa_action = "warn";
					$dashboard_action = "";

					$status .= " <item>\n";
					$status .= "  <name_en>Note: hosted SBC/Branch detected on your system. </name_en>\n";
					$status .= "  <name_de>Achtung: konfigurierter Hosted SBC/Branch erkannt!</name_de>\n";
					$status .= "  <status>1</status>\n";
					$status .= "  <description_en><![CDATA[Note: One or more hosted SBC/Branch detected on central host or AP. In order to ensure proper functionality after upgrade, please check compatibility matrix and backup your SBC configuration.]]></description_en>\n";
					$status .= "  <description_de><![CDATA[Achtung: Mindestens ein Hosted SBC/Branch auf dem zentralen Host oder einem AP erkannt! Um korrekte Funktionalität nach dem Upgrade sicherzustellen, bitte die Kompatibilitätsmatrix prüfen und einen SBC Backup erstellen.]]></description_de>\n";
					$status .= "  <swa_action_hf></swa_action_hf>\n";
					$status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
					$status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
					$status .= " </item>\n";
				}
			}
		} else {
			log_msg_info("script $sbc_script not found, check at least locally");
			if ($isMultiNode == 0) {
				my $sbc_status=retrieve_from_xml("system.sbc-enabled");
				if ($sbc_status == 1) {
					log_msg_info("central SBC is enabled, make warn for SWA");
					$swa_action = "warn";
					$dashboard_action = "";

					$status .= " <item>\n";
					$status .= "  <name_en>Note: hosted SBC/Branch detected on your system. </name_en>\n";
					$status .= "  <name_de>Achtung: konfigurierter Hosted SBC/Branch erkannt!</name_de>\n";
					$status .= "  <status>1</status>\n";
					$status .= "  <description_en><![CDATA[Note: Hosted SBC/Branch detected on your system. In order to ensure proper functionality after upgrade, please check compatibility matrix and backup your SBC configuration.]]></description_en>\n";
					$status .= "  <description_de><![CDATA[Achtung: Konfigurierter Hosted SBC/Branch erkannt! Um korrekte Funktionalität nach dem Upgrade sicherzustellen, bitte die Kompatibilitätsmatrix prüfen und einen SBC Backup erstellen.]]></description_de>\n";
					$status .= "  <swa_action_hf></swa_action_hf>\n";
					$status .= "  <swa_action_rlc>$swa_action</swa_action_rlc>\n";
					$status .= "  <dashboard_action>$dashboard_action</dashboard_action>\n";
					$status .= " </item>\n";
				}
			 }
		}
	}

  function_leave("get_plt_health_status", "status=$status");
  return ($status);
}

sub get_nuc_preparation_status_check_output {
  my $multiNodeType = shift;
  my $isMultiNode = shift;
  my $has_a_quorum = shift;

  function_enter("get_nuc_preparation_status_check_output", "multiNodeType=$multiNodeType isMultiNode=$isMultiNode has_a_quorum=$has_a_quorum");
  my ($remote_text, $quorum_text, $local_text, $status, $status_rc) = ("", "", "", "", 0);

  my $remote_log  = $webservice_log_dir."nuc_prep_status_checkoutput_remote.log";
  my $quorum_log  = $webservice_log_dir."nuc_prep_status_checkoutput_quorum.log";
  my $local_log   = $webservice_log_dir."nuc_prep_status_checkoutput_local.log";

  if ($isMultiNode == 1) {
    $remote_text = read_file_content($remote_log);
    if (length($remote_text) > 0) {
      $status_rc=5;
      $status .= "Error in NUC preparation of standby node detected.\nThis error may endanger upgrade or replacement operations.\n$remote_text\n"
    }
    if ($has_a_quorum == 1) {
      $quorum_text = read_file_content($quorum_log);
      if (length($quorum_text) > 0) {
        $status_rc=5;
        $status .= "Error in NUC preparation of quorum node detected.\nThis error may endanger upgrade or replacement operations.\n$quorum_text\n"
      }
    }
  }
  $local_text  = read_file_content($local_log);
  if (length($local_text) > 0) {
    $status_rc=5;
    $status .= "Error in NUC preparation detected.\nThis error may endanger upgrade or replacement operations.\n$local_text"
  }

  # replace /n by <br/>
  $status =~ s/\n/<br\/>/g;

  function_leave("get_nuc_preparation_status_check_output", "status=$status");
  return ($status);
}

sub get_firstinst_netw_check_output {
  my $multiNodeType = shift;
  my $isMultiNode = shift;
  my $has_a_quorum = shift;  
  
  function_enter("get_firstinst_netw_check_output", "multiNodeType=$multiNodeType isMultiNode=$isMultiNode has_a_quorum=$has_a_quorum");
  my ($remote_text, $quorum_text, $local_text, $status) = ("", "", "", "");
  
  my $remote_log  = $webservice_log_dir."firstinst_netw_check_remote.log";
  my $quorum_log  = $webservice_log_dir."firstinst_netw_check_quorum.log";
  my $local_log   = $webservice_log_dir."firstinst_netw_check_local.log";

  if ($isMultiNode == 1) {
    $remote_text = read_file_content($remote_log);
    if (length($remote_text) > 0) {
      $status .= "Error in platform network configuration of standby node detected.\nThis error may endanger upgrade or replacement operations.\n$remote_text\n"
    }
    if ($has_a_quorum == 1) {
      $quorum_text = read_file_content($quorum_log);
      if (length($quorum_text) > 0) {
        $status .= "Error in platform network configuration of quorum node detected.\nThis error may endanger upgrade or replacement operations.\n$quorum_text\n"
      }
    }
  }  
  $local_text  = read_file_content($local_log);
  if (length($local_text) > 0) {
    $status .= "Error in platform network configuration detected.\nThis error may endanger upgrade or replacement operations.\n$local_text"
  }

  # replace /n by <br/>
  $status =~ s/\n/<br\/>/g;
  
  function_leave("get_firstinst_netw_check_output", "status=$status");
  return ($status);  
}

sub get_is_cached {
  my $data_currentness = shift;
  my $rc = 0;
  if ($data_currentness == $DATA_CURRENTNESS_STATUS_irrelevant or
      $data_currentness == $DATA_CURRENTNESS_STATUS_up_to_date) {
    $rc = 0;
  } else {
    $rc = 1;
  }
}

sub write_extended_status {
  my $multiNodeType = shift;
  my $is_duplex = shift;
  my $has_a_quorum = shift;
  my $toplevel_tag = shift;
  my $do_bios_check = shift;
  my $status_file = shift;
  my $rc = shift;
  my $errortext = shift;
  my $hostname_a = shift;
  my $rc_a = shift;
  my $errortext_a = shift;  
  my $errortext_a_tag = shift;    
  my $hostname_b = shift;
  my $rc_b = shift;
  my $host_b_reachable = shift;
  my $errortext_b = shift;  
  my $errortext_b_tag = shift;    
  my $hostname_q = shift;
  my $rc_q = shift;
  my $host_q_reachable = shift;
  my $errortext_q = shift;  
  my $errortext_q_tag = shift;
  my $data_currentness = shift; 
  not defined shift or log_msg("write_extended_status: superfluous param");

  function_enter("write_extended_status", "multiNodeType=$multiNodeType is_duplex=$is_duplex has_a_quorum=$has_a_quorum tag=$toplevel_tag do_bios_check=$do_bios_check file=$status_file rc=$rc text=$errortext hostname_a=$hostname_a rc_a=$rc_a text_a=$errortext_a tag_a=$errortext_a_tag hostname_b=$hostname_b rc_b=$rc_b host_b_reachable=$host_b_reachable text_b=$errortext_b tag_b=$errortext_b_tag hostname_q=$hostname_q rc_q=$rc_q host_q_reachable=$host_q_reachable text_q=$errortext_q tag_q=$errortext_q_tag data_currentness=$data_currentness");

  # as long as we do not have an own flag given, I use bios flag (e.g get_version_info)
  my $do_plt_health_check = $do_bios_check;
  
  my ($status, $node_errortext, $deployment_as_string_output) = ("", "", "");
  my $bios_check_status = 0;
  my ($drbd_sync_status, $drbd_connect_status, $number_of_sg_on_central_hosts, $number_of_container_sg, $containsCSTA) = (0, 0, 0, 0, 0);
  if ($is_stmix == 1) {
    $deployment_as_string_output = "STMIX";
    $do_plt_health_check = 0;
    $do_bios_check = 0;
  } else {
    ($drbd_sync_status, $drbd_connect_status) = get_drbd_status($is_duplex, $host_b_reachable);
    $deployment_as_string_output   = get_deployment_as_string();

    # retrieve only the native SGs on Simplex or Q
    $number_of_sg_on_central_hosts = get_number_of_sg_on_central_hosts(1);

    # EntGW or Duplex
    $number_of_container_sg        = get_number_of_container_sg_on_network();
    # will add the number of container iSGs to the number of "real" central hosts
    # task is to tell the Assistant to reduce the number of needed JeOS licenses from the table of SG shelfes taken from AMO
    # this is a. about integrated Softgates running natively on central hosts or iSGs running on EntGW in container    
    $number_of_sg_on_central_hosts = $number_of_sg_on_central_hosts + $number_of_container_sg;

    $containsCSTA = get_containsCSTA($multiNodeType);
  }
  my $number_of_os_licenses = get_number_of_os_licenses($multiNodeType);
   
  # speciality for upgrade
  if(open(WESFILE, ">$status_file")) {
    
    protectFile($status_file);
    
    if (length($toplevel_tag) > 0) {
      $status .= "<$toplevel_tag>\n";
    }
    $status .= "<date>".datefmt()."</date>\n";
    $status .= "<protocol_version>$status_protocol_version</protocol_version>\n";
    $status .= "<daemon_version>$upd_version</daemon_version>\n";
    $status .= "<multiNodeType>$multiNodeType</multiNodeType>\n";
    $status .= "<deployment>$deployment_as_string_output</deployment>\n";
    $status .= "<containsCSTA>$containsCSTA</containsCSTA>\n";
    $status .= "<number_of_os_licenses>$number_of_os_licenses</number_of_os_licenses>\n";
    $status .= "<number_of_softgates_on_central_hosts>$number_of_sg_on_central_hosts</number_of_softgates_on_central_hosts>\n";
    if ($is_duplex == 1)  {
      $status .= "<drbd_status>$drbd_sync_status</drbd_status>\n";
    }
    if ($do_bios_check  == 1) {
      my $ssh_timeout = 10;
      $bios_check_status = execute_bios_check($is_duplex, $host_b_reachable, $ssh_timeout);
      $status .= "<bios_check_status>$bios_check_status</bios_check_status>\n";
      if ($bios_check_status != 0) {
        my $errstr = get_bios_error_string_de();
        $status .= "<assistant_info_de>\n<message>$errstr</message>\n</assistant_info_de>\n";
        $errstr = get_bios_error_string_en();
        $status .= "<assistant_info_en>\n<message>$errstr</message>\n</assistant_info_en>\n";
      }
    }

    if ($do_plt_health_check  == 1) {
      my $plt_health_status = get_plt_health_status($multiNodeType, $is_duplex, $has_a_quorum, $bios_check_status, $data_currentness, $host_b_reachable, $rc_b, $host_q_reachable, $rc_q, $drbd_connect_status, $errortext_a, $errortext_b, $errortext_q, $hostname_a, $hostname_b, $hostname_q);
      $status .= "<plt_health_status>$plt_health_status</plt_health_status>\n";
      my $is_webservice_running = is_service_running("webservice");
      $status .= "<is_webservice_reachable>$is_webservice_running</is_webservice_reachable>\n";
    }
    
    
    $status .= "<$errortext_a_tag>$errortext_a</$errortext_a_tag>\n";
    $status .= "<node_a_rc>$rc_a</node_a_rc>\n";
    if ($rc_a != 0) {
      if (defined($ERROR_text{$rc_a})) {
        $node_errortext = $ERROR_text{$rc_a};
      }
      $status .= "<node_a_errortext>$node_errortext</node_a_errortext>\n";
    }
    $status .= "<node_a_hostname>$hostname_a</node_a_hostname>\n";
    

    if ($is_duplex == 1) {
      $status .= "<$errortext_b_tag>$errortext_b</$errortext_b_tag>\n";
      $status .= "<node_b_rc>$rc_b</node_b_rc>\n";
      if ($rc_b != 0) {
        if (defined($ERROR_text{$rc_b})) {
          $node_errortext = $ERROR_text{$rc_b};
        }
        $status .= "<node_b_errortext>$node_errortext</node_b_errortext>\n";
      }
      $status .= "<node_b_hostname>$hostname_b</node_b_hostname>\n";
      if ($has_a_quorum == 1) {
        $status .= "<$errortext_q_tag>$errortext_q</$errortext_q_tag>\n";
        $status .= "<node_q_rc>$rc_q</node_q_rc>\n";
        if ($rc_q != 0) {
          if (defined($ERROR_text{$rc_q})) {
            $node_errortext = $ERROR_text{$rc_q};
          }
          $status .= "<node_q_errortext>$node_errortext</node_q_errortext>\n";
        }
        $status .= "<node_q_hostname>$hostname_q</node_q_hostname>\n";
      }
      if (($data_currentness != $DATA_CURRENTNESS_STATUS_irrelevant) and ($host_b_reachable == 1)) {
        my $is_cached = get_is_cached($data_currentness);
        $status .= "<cached>$is_cached</cached>\n";
        $status .= "<reason_for_cache_return_code>$data_currentness</reason_for_cache_return_code>\n";
        $status .= "<reason_for_cache_return_text_en>$DATA_CURRENTNESS_STATUS_text_en{$data_currentness}</reason_for_cache_return_text_en>\n";
        $status .= "<reason_for_cache_return_text_de>$DATA_CURRENTNESS_STATUS_text_de{$data_currentness}</reason_for_cache_return_text_de>\n";
      }
    }

    if ($rc != 0) {
      if ($errortext eq "" and defined($ERROR_text{$rc})) {
        $errortext = $ERROR_text{$rc};
      }
    }
    if ($activation_has_been_avoided != 0) {
    $status .= "<activation_avoidance>intended version already installed</activation_avoidance>\n";
    }
    $status .= "<errortext>$errortext</errortext>\n";
    $status .= "<rc>$rc</rc>\n";
    
    if (length($toplevel_tag) > 0) {
      $status .= "</$toplevel_tag>\n";
    }    
    print WESFILE $status;
    close WESFILE;
    chomp($status);
    log_msg("$status_file:\n$status");
    if(length($errortext) > 0) { log_msg_info($errortext); }
  }
 function_leave("write_extended_status", "");
}

sub read_file_content {
  my $file_to_read = shift;
  my @content = ();
  my $content_string = "";
  if(-e $file_to_read && open(FILE_TO_READ, "<$file_to_read")) {
    @content = <FILE_TO_READ>;
    close (FILE_TO_READ);
  } else {
    log_msg_info("can not open file $file_to_read for reading rc=$!");
  }
  foreach(@content) {
    $content_string .= $_;
  }
  return ($content_string);
}
  
sub print_log_header {
  log_msg_info("****************************************************************");
  log_msg_info("swupdated call: $call_string");
  log_msg_info("called by: $ppid");
  log_msg_info("swupdated version: $upd_version");
  log_msg_info("running on: $hostname ($uname)");
  log_msg_info("");
}

sub protectFile {
  my $file = shift;
  chmod(0640, $file);
}

sub protect_zypper_logs {
  exec_cmd($zypper_protect_script);
  log_msg_info("protecting the zypper logs and test case files with correct file permissions (640)");
}

sub function_enter {
  my $function_name = shift;
  my $info_string   = shift;
  if (!defined($info_string)) { $info_string = ""; }
  log_msg_info(">>> $function_name $info_string");

  $call_stack_depth++;
  switch ($call_stack_depth) {
    case 12{ $log_msg_indent = "                  "; }
    case 11{ $log_msg_indent = "                 "; }
    case 10{ $log_msg_indent = "                "; }
    case 9 { $log_msg_indent = "               "; }
    case 8 { $log_msg_indent = "              "; }
    case 7 { $log_msg_indent = "             "; }
    case 6 { $log_msg_indent = "            "; }
    case 5 { $log_msg_indent = "          "; }
    case 4 { $log_msg_indent = "        "; }
    case 3 { $log_msg_indent = "      "; }
    case 2 { $log_msg_indent = "    "; }
    case 1 { $log_msg_indent = "  "; }
    else   { $log_msg_indent = "                  "; }
  }
}

sub function_leave {
  my $function_name = shift;
  my $info_string   = shift;
  if (!defined($info_string)) { $info_string = ""; }

  $call_stack_depth--;
  switch ($call_stack_depth) {
    case 12{ $log_msg_indent = "                  "; }
    case 11{ $log_msg_indent = "                 "; }
    case 10{ $log_msg_indent = "                "; }
    case 9 { $log_msg_indent = "               "; }
    case 8 { $log_msg_indent = "              "; }
    case 7 { $log_msg_indent = "             "; }
    case 6 { $log_msg_indent = "            "; }
    case 5 { $log_msg_indent = "          "; }
    case 4 { $log_msg_indent = "        "; }
    case 3 { $log_msg_indent = "      "; }
    case 2 { $log_msg_indent = "    "; }
    case 1 { $log_msg_indent = "  "; }
    case 0 { $log_msg_indent = ""; }
    else   { $log_msg_indent = "                  "; }
  }
  log_msg_info("<<< $function_name $info_string");
}


sub log_msg_info {
  my $msg = shift;
  my $use_stdout = shift;
  my $use_syslog = shift;
  my $use_alt_file = shift;
  log_msg($msg, $use_stdout, $use_syslog, $use_alt_file, "INFO ");
}
  
sub log_msg {
  my $msg = shift;
  my $use_stdout = shift;
  my $use_syslog = shift;
  my $use_alt_file = shift;
  my $level = shift;
  if (! defined($level)) {
    $level = "DEBUG";
  }

  my $logfile_to_use = $log_file;
  if (defined($use_alt_file) && $use_alt_file == 1) {
    $logfile_to_use = $log_alt_file;
  }
    
  (my $package, my $file, my $line) = caller();

  if (defined($use_syslog) && $use_syslog == 1) {
    if ($level eq "INFO ") {
      exec_cmd("/bin/logger -t swupdated -p user.info \"$msg\" ");
    } else {
      exec_cmd("/bin/logger -t swupdated \"$msg\" ");
    }
  }
  
  if(open(LMFILE, ">>$logfile_to_use")) {
    print LMFILE (datefmt()." $level [swupdated] $pid $line $log_msg_indent $msg\n");
    close LMFILE;
    protectFile($logfile_to_use);
  }

  if (defined($use_stdout) && $use_stdout == 1) {
   print("$msg\n");
  }
}

sub dump_upgrade_log_to_log {
  function_enter("dump_upgrade_log_to_log");
  exec_cmd("cat $upgrade_log_file");
  function_leave("dump_upgrade_log_to_log");
}

sub dump_file_to_log {
  my $file_to_dump = shift;
  function_enter("dump_file_to_log", "file_to_dump=$file_to_dump");
  exec_cmd("ls -l $file_to_dump");
  exec_cmd("cat $file_to_dump");
  function_leave("dump_file_to_log");
}

sub write_upgrade_log {
  my $msg = shift;
  function_enter("write_upgrade_log", "msg=$msg, write_sysinfo_status_enabled=$write_sysinfo_status_enabled");
  # write the plt-upgrade.log only in case that really an upgrade happens
  my $timestamp = exec_cmd_out("/bin/date +%s");
  chomp($timestamp);
  if(open(UPGLFILE, ">>$upgrade_log_file")) {
    print UPGLFILE ($timestamp.":"."$msg\n");
    close UPGLFILE;
    protectFile($upgrade_log_file);
  }
  function_leave("write_upgrade_log");
}

sub hpa_discover {
  function_enter("hpa_discover");
  my $is_hpa = 0;
  my $rc = exec_cmd("$is_hpa_script");
  if ($rc == 1) {
    $is_hpa = 1;
  }
  function_leave("hpa_discover", "is_hpa=$is_hpa ");
  return($is_hpa);
}

sub dscxl_discover {
  function_enter("dscxl_discover");
  my $is_dscxl = 0;
  if (-x $is_dscxl_script) {
    my $rc = exec_cmd("$is_dscxl_script");
    if ($rc == 1) {
      $is_dscxl = 1;
    }
  } else {
    log_msg_info("dscxl determine script $is_dscxl_script not found, checking lsusb (this may lead to wrong conclusion on Eco)");
    my $rc = exec_cmd("/usr/bin/lsusb -d 045b:2014");
    if ($rc == 0) {
      $is_dscxl = 1;
    }
  }
  function_leave("dscxl_discover", "is_dscxl=$is_dscxl");
  return($is_dscxl);
}

sub manager_discover {
  function_enter("manager_discover");
  my $is_a_manager = 0;

  if (-x $is_manager_script) {
    my $rc = exec_cmd("$is_manager_script");
    if ($rc == 1) {
      $is_a_manager = 1;
    }
  } else {
    log_msg_info("Manager determine script $is_manager_script not found, obviously not a Manager appliance");
  }
  function_leave("manager_discover", "is_a_manager=$is_a_manager");
  return($is_a_manager);
}

sub entgw_discover {
  function_enter("entgw_discover");
  my $is_ecoap = 0;

  if (-x $is_ecoap_script) {
    my $rc = exec_cmd("$is_ecoap_script");
    if ($rc == 1) {
      $is_ecoap = 1;
    }
  } else {
    log_msg_info("Enterprise GW determine script $is_ecoap_script not found, obviously not an Enterprise GW");
  }
  function_leave("entgw_discover", "is_ecoAP=$is_ecoap");
  return($is_ecoap);
}

sub eco_discover {
  function_enter("eco_discover");
  my $is_eco = 0;
  my $eco_script = "/opt/webservice/scripts/is_ecoserver";
  if (-x $eco_script) {
    my $rc = exec_cmd($eco_script);
    if ($rc != 0) {
      $is_eco = 1;
    }
  } else {
    log_msg_info("eco determine script $eco_script not found, assuming not an eco");
    $is_eco = 0;
  }
  function_leave("eco_discover", "is_eco=$is_eco");
  return($is_eco);
}

sub cots_discover {
  function_enter("cots_discover");
  my $is_cots = 0;
  my $cots_script = "/opt/webservice/scripts/is_cots";
  if (-x $cots_script) {
    my $rc = exec_cmd($cots_script);
    if ($rc != 0) {
      $is_cots = 1;
    }
  } else {
    log_msg_info("cots determine script $cots_script not found, assuming not an cots");
    $is_cots = 0;
  }
  function_leave("cots_discover", "is_cots=$is_cots");
  return($is_cots);
}

sub virtual_discover {
  function_enter("virtual_discover");
  my $is_virt = 0;
  my $tool = "/usr/bin/vmxcap";
  my $cmd = "$tool --identify-hypervisor";
  if (-x $tool) {
    my $rc = exec_cmd($cmd);
    if ($rc == 0) {
      $is_virt = 1;
    }
  } else {
    log_msg_info("check tool $tool not available, assuming non virtual");
    $is_virt = 0;
  }

  function_leave("virtual_discover", "is_virt=$is_virt");
  return($is_virt);
}

sub discover_HD_layout {
  function_enter("discover_HD_layout");
  if (-e "/var/opt/soco2/hd_layout_A") {
    $is_HD_layout_A = 1;
    log_msg_info("detected HD layout A");
  }
  if (-e "/var/opt/soco2/hd_layout_B") {
    $is_HD_layout_B = 1;
    log_msg_info("detected HD layout B");
  }
  function_leave("discover_HD_layout", "is_HD_layout_A=$is_HD_layout_A");
  return();
}

sub stmix_discover {
  function_enter("stmix_discover");
  my $is_stmix = 0;
  my $stmix_triggerfile = "/var/opt/soco/stmix";
  if (-f $stmix_triggerfile) {
    log_msg_info("stmix trigger file $stmix_triggerfile detected, STMIX HW");
    $is_stmix = 1;
  } else {
    log_msg_info("stmix trigger file $stmix_triggerfile not detected, not STMIX HW");
    $is_stmix = 0;
  }
  function_leave("stmix_discover", "is_stmix=$is_stmix");
  return($is_stmix);
}

sub get_rpm_version {
  my $package = shift;
  function_enter("get_rpm_version", "package=$package");
  my $current_version = exec_cmd_out("/bin/rpm -q $package");
  $current_version=my_trim($current_version);
  function_leave("get_rpm_version", "current_version=$current_version");
  return($current_version);
}

sub cleanup_sysrestore_files {
  function_enter("cleanup_sysrestore_files");
  my $trigger_file = "$soco2_tmp_dir"."systembackup.installed";
  
  if (-f $trigger_file) {
    log_msg_info("Trigger file from system restore $trigger_file exist. -> removing it to prevent mini zib mode if new webservice evaluates it");
    removeFile($trigger_file);
  }
  function_leave("cleanup_sysrestore_files");
}

sub copy_important_ha_config_files {
  function_enter("copy_important_ha_config_files");
  my $rc = 0;
  my $from_file = $openais_conf_file;
  if (-f $from_file) {
    my $output = exec_cmd_out("ls -l $from_file");
    log_msg_info("file info before upgrade: $output");

    if (! -d $soco2_tmp_dir) {
      makeDir($soco2_tmp_dir);
    }

    my $cmd = "/bin/cp -vfa $from_file $soco2_tmp_dir";
    $rc = exec_cmd($cmd);
  }
  function_leave("copy_important_ha_config_files", "rc=$rc");
  return($rc);
}

sub restore_important_ha_config_files {
  function_enter("restore_important_ha_config_files");
  my $rc = 0;
  my $dest_file = $openais_conf_file;
  my $from_file = $soco2_tmp_dir."openais.conf";

  if (-f $from_file) {
    log_msg_info("temporarily stored config file exist. -> restoring it to the original place");
    my $cmd = "/bin/cp -vfa $from_file $dest_file";
    $rc = exec_cmd($cmd);

    my $output = exec_cmd_out("ls -l $dest_file");
    log_msg_info("file info after restore: $output");

    # I do not need the temp. file anymore
    if ($rc == 0) {
      removeFile($from_file);
    }
  }
  function_leave("restore_important_ha_config_files", "rc=$rc");
  return($rc);
}

sub datefmt {
  my ($sec,$min,$hour,$mday,$mon,$year, $wday,$yday,$iddst) = localtime(time);
  my $datestring;
  $year += 1900;
  $mon++;
  $mon  = insert0($mon);
  $mday = insert0($mday);
  $min  = insert0($min);
  $sec  = insert0($sec);
  $datestring = "$year-$mon-$mday $hour:$min:$sec";
  return($datestring);
}

sub insert0 {
  my $date = shift;
  if($date < 10) {
    return("0$date");
  } 
  return($date);
}

sub renameFile {
  my $file_name_from = shift;
  my $file_name_to = shift;  
  my $rc = 0; 
  $rc = rename($file_name_from, $file_name_to);
  if ($rc) {
    log_msg_info("  renamed $file_name_from to $file_name_to");
    $rc = 0;
  }
  return ($rc);
}

sub removeFile {
  my $file_name = shift;
  my $rc = 0; 
  $rc = unlink($file_name);
  if ($rc != 0) {
    log_msg_info("  removed $file_name");
  }
  return ($rc);
}


sub removeDir {
  my $dir = shift;
  my $rc = 0;  
  $rc = exec_cmd("/bin/rm -rvf $dir");
  return($rc);
}

sub makeDir {
  my $dir = shift;
  my $rc = 0;  
  $rc = exec_cmd("/bin/mkdir -pv $dir");
  return($rc);
}

sub exec_cmd {
  my $cmd  = shift;
  my $mode = shift;
  my $out = "";
  
  if (!defined($mode)) {
    $mode = "";
  }
  my $logfile_to_use = $log_file;
  if ($mode eq "alt") {
    $logfile_to_use = $log_alt_file;
  }
  if ($mode eq "silent") {
    $logfile_to_use = "/dev/null";
  }  

  if ($mode ne "silent") {
    if  ($mode eq "alt") {
      log_msg("cmd: ".$cmd, 0, 0, 1);
    } else {
      log_msg("cmd: ".$cmd);
    }
  }
  
  $out = system($cmd." >> $logfile_to_use 2>&1") >> 8;
  
  if ($mode ne "silent") {
    if  ($mode eq "alt") {
      log_msg("rc: ".$out, 0, 0, 1);
    } else {
      log_msg("rc: ".$out);
    }
  }
  return($out);
}

sub exec_cmd_pipe {
  my $cmd = shift;
  my $out = "";
  log_msg("cmd: ".$cmd);
  if(open(CMD, "$cmd|")) {
    $out = <CMD>;
  } else {
    log_msg("could not execute command");
  }
  if (!defined($out)) {
    log_msg("return string of command pipe undefined");  
  } else {
    log_msg("out: ".$out);
  }
  return($out);
}

sub exec_cmd_out {
  my $cmd = shift;
  my $silent = shift;

  if (!defined($silent)) {
    $silent = "";
  }
  my $ssh_log_trig_file = "/var/log/webservice/ssh-keygen-log.enabled";
  my $out = "";
  if ($silent ne "silent") {
    log_msg("cmd: ".$cmd);
  }
  $out = `$cmd`;
  if(-e $ssh_log_trig_file) {
    if ($silent ne "silent") {    
      log_msg("out: ".$out);
    }
  }
  return($out);
}

sub my_trim($) {
  my $string = shift;
  $string =~ s/^\s+//;
  $string =~ s/\s+$//;
  return($string);
}

sub start_watchdog_daemon {
  function_enter("start_watchdog_daemon");

  if ($is_stmix == 1) {
    # as on STMIX we do not do wd_keepalive -1 anymore, we must not start the wd_keepalive daemon under any circumstances
    log_msg("STMIX detected -> skipping to start wd_keepalive daemon");
    function_leave("start_watchdog_daemon");
    return (0);
  }
  
  my $cmd = '/usr/sbin/wd_keepalive';
  my $rc = 0;
  if (-e "/usr/sbin/wd_keepalive" ) {
    $rc= exec_cmd($cmd);
  }
  function_leave("start_watchdog_daemon");
  return($rc);
}

sub start_watchdog_daemon_remote {
  my $node = shift;

  function_enter("start_watchdog_daemon_remote", "node=$node");
  my ($output, $node_is_reachable, $password_is_expired, $ignore_remote_return_code) = ("", 1, 0, 1);

  my $remote_cmd = '/usr/sbin/wd_keepalive';
  my $rc = 0;

  $rc = execute_remote_cmd($node, \$node_is_reachable, \$password_is_expired, $remote_cmd, \$output, 3, 60, $ignore_remote_return_code);

  function_leave("start_watchdog_daemon_remote", "rc=$rc");
  return($rc);
}

sub remove_remote_pgsql_backup_file {
  function_enter("remove_remote_pgsql_backup_file");
  my ($output, $node_is_reachable, $password_is_expired, $ignore_remote_return_code) = ("", 1, 0, 1);

  my $remote_cmd = "/bin/rm -fv $pgsql_data_zip_file";
  my $rc = 0;

  $rc = execute_remote_duplex_cmd($remote_cmd, \$output, 1, \$node_is_reachable, \$password_is_expired, $ignore_remote_return_code);

  function_leave("remove_remote_pgsql_backup_file", "rc=$rc");
  return($rc);
}

sub trigger_watchdog {
  function_enter("trigger_watchdog", "for additional 10 (eco)/60 (non eco) minutes");
  if ($is_stmix == 1) {
    # on STMIX wd_keepalive must not be called, it can cause resets: OSFOURK-21875
    log_msg("STMIX detected -> skipping wd_keepalive");
    function_leave("trigger_watchdog");
    return (0);
  }
  
  my $cmd = '/usr/sbin/wd_keepalive -1';
  my $rc = 0;
  if (-e "/usr/sbin/wd_keepalive" ) {
    $rc= exec_cmd($cmd, "silent");
  }
  function_leave("trigger_watchdog");
  return($rc);  
}

sub initialize_sysinfo_values {
  ($sysinfo_sec,$sysinfo_min,$sysinfo_hour,$sysinfo_mday,$sysinfo_mon,$sysinfo_year, $sysinfo_wday,$sysinfo_yday,$sysinfo_iddst) = localtime(time);
  $sysinfo_year += 1900;
  $sysinfo_mon++;
  $sysinfo_mon  = insert0($sysinfo_mon);
  $sysinfo_mday = insert0($sysinfo_mday);
  $sysinfo_hour = insert0($sysinfo_hour);
  $sysinfo_min  = insert0($sysinfo_min);
  $sysinfo_sec  = insert0($sysinfo_sec);
}

sub initialize_sysinfo_for_upgrade {
  function_enter("initialize_sysinfo_for_upgrade");
  if (! -e $sysinfo_data_dir."input") {
    makeDir($sysinfo_data_dir."input");
  }
  # remove a remote log from previous update
  removeFile($sysinfo_standby_log);
  removeFile($sysinfo_quorum_log);

  # remove a plt-upgrade.log from previous update
  removeFile($upgrade_log_file);

  $write_sysinfo_status_enabled = 1;
  function_leave("initialize_sysinfo_for_upgrade");
}

sub finalize_sysinfo_values {
  function_enter("finalize_sysinfo_values");
  ($sysinfo2_sec,$sysinfo2_min,$sysinfo2_hour,$sysinfo2_mday,$sysinfo2_mon,$sysinfo2_year, $sysinfo2_wday,$sysinfo2_yday,$sysinfo2_iddst) = localtime(time);
  $sysinfo2_year += 1900;
  $sysinfo2_mon++;
  $sysinfo2_mon  = insert0($sysinfo2_mon);
  $sysinfo2_mday = insert0($sysinfo2_mday);
  $sysinfo2_hour = insert0($sysinfo2_hour);
  $sysinfo2_min  = insert0($sysinfo2_min);
  $sysinfo2_sec  = insert0($sysinfo2_sec);
  function_leave("finalize_sysinfo_values");
}

sub wait_till_hsr_goes_down {
  function_enter("wait_till_hsr_goes_down");
  my $hsr_connection_status = 0;
  my $rc = 0;
  my $is_timeout = 1;
  my $wait_time = 5; # seconds
  my $wait_cycles = 12 * 60 * 24; # one day
  my $log_counter = 0;
  
  for(my $i = 0; $i < $wait_cycles; $i++) {
    $hsr_connection_status = is_sg_connected_via_hsr();
    
    if($hsr_connection_status == 0) {
      log_msg_info("HSR connection went down, terminating wait");
      $rc = 0;
      $is_timeout = 0;
      last;
    } else {
      if ($log_counter >= 5) {
        log_msg_info("HSR connection still up ... waiting $wait_time sec. ($i of 17280)");
        trigger_watchdog();
        $log_counter = 0;
      }
    }

    sleep($wait_time);
    $log_counter++;
  }
  if ($is_timeout == 1) {
    $rc = 1;
  }
  function_leave("wait_till_hsr_goes_down", "rc=$rc");
  return($rc);   
}

sub get_sg_hsr_status_from_sginfo {
  my $hsr_status = "";
  function_enter("get_sg_hsr_status_from_sginfo");

  my $sg_info_file = "/var/opt/soco/sginfo/sysinfo.xml";
  if (-e $sg_info_file) {
    $hsr_status= retrieve_from_xml("HSR_partner_name", $sg_info_file);
  }
  function_leave("get_sg_hsr_status_from_sginfo", "hsr_status=$hsr_status");
  return($hsr_status);  
}

sub is_sg_connected_via_hsr {
  my $is_connected = 0;
  function_enter("is_sg_connected_via_hsr");
  my $sg_hsr_status = get_sg_hsr_status_from_sginfo();
  if ($sg_hsr_status eq "CCA" or $sg_hsr_status eq "CCB" or $sg_hsr_status eq "APE") {
    $is_connected = 1;
  }
  function_leave("is_sg_connected_via_hsr", "is_connected=$is_connected");
  return($is_connected);
}

sub get_node_where_adp_is_running {
  function_enter("get_node_where_adp_is_running");
  my $output = exec_cmd_out("/usr/sbin/crm -D plain resource show rsc4k_adp");
  chomp($output);
  log_msg_info("output=$output");
  
  if ($output eq "") {
    log_msg_info("no ADP host found");
  } else {
    log_msg_info("ADP host found, output=$output");
    my @parts = split(':', $output);
    log_msg_info("parts=@parts");
    if ( !defined($parts[1]) or (defined($parts[1]) && $parts[1] eq "")) {
      log_msg_info("no ADP host found");
      $output = "";
    } else {
      $output=$parts[1];
      chomp($output);
      log_msg_info("ADP host found: $output");
      $output =~ s/^ *//;            # remove leading blanks
      log_msg_info("remove leading blanks, output=$output");
      $output =~ s/[ ].*$//;         # remove all after the first blank
      log_msg_info("remove all after the first blank, output=$output");
      $output =~ s/[\[].*//;         # remove all after a [
      log_msg_info("remove all after a [, output=$output");
      $output =~ s/[\x80-\xFF].*//g;   # remove high non ascii
      log_msg_info("remove all after high non ascii, output=$output");
      $output =~ s/[\x01-\x21].*//g;   # remove low non ascii
      log_msg_info("remove all after low non ascii, output=$output");
    }
  }
    
  function_leave("get_node_where_adp_is_running", "node=$output");
  return ($output);
}

sub get_callstatus_step {
  function_enter("get_callstatus_step");
  my $rc = 0;
  
  my $tag = shift;
  my $value = "";
  if(open(CSFFILE, "<$swupdated_call_status_file")) {
    my @content = <CSFFILE>;
    foreach(@content) {
      my $line = $_;
      if(index($line, "step") >= 0) {
        my @parts = split(/=/, $_);
        if(defined $parts[1]) {
          $value = $parts[1];
          $value =~ s/^\s+//;
          $value =~ s/\s+$//;
          $rc = $value;
        }
      }
      if(index($line, "iteration") >= 0) {
        my @parts = split(/=/, $_);
        if(defined $parts[1]) {
          $value = $parts[1];
          $value =~ s/^\s+//;
          $value =~ s/\s+$//;
          $continue_upgrade_iteration = $value;
          log_msg_info("<<< get_callstatus_step found iteration=$continue_upgrade_iteration");		  
        }
      }
      if(index($line, "upgrade_with_completely_stopped_ha") >= 0) {
        my @parts = split(/=/, $_);
        if(defined $parts[1]) {
          $value = $parts[1];
          $value =~ s/^\s+//;
          $value =~ s/\s+$//;
          $continue_upgrade_with_completely_stopped_ha = $value;
          log_msg_info("<<< get_callstatus_step found upgrade_with_completely_stopped_ha=$continue_upgrade_with_completely_stopped_ha");		  
        }
      }
      if(index($line, "upgrade_with_full_drbd_sync") >= 0) {
        my @parts = split(/=/, $_);
        if(defined $parts[1]) {
          $value = $parts[1];
          $value =~ s/^\s+//;
          $value =~ s/\s+$//;
          $continue_upgrade_with_full_drbd_sync = $value;
          log_msg_info("<<< get_callstatus_step found upgrade_with_full_drbd_sync=$continue_upgrade_with_full_drbd_sync");		  
        }
      }
      if(index($line, "upgrade_with_reduce_downtime") >= 0) {
        my @parts = split(/=/, $_);
        if(defined $parts[1]) {
          $value = $parts[1];
          $value =~ s/^\s+//;
          $value =~ s/\s+$//;
          $continue_upgrade_with_reduce_downtime = $value;
          log_msg_info("<<< get_callstatus_step found upgrade_with_reduce_downtime=$continue_upgrade_with_reduce_downtime");
        }
      }
      if(index($line, "upgrade_via_image") >= 0) {
        my @parts = split(/=/, $_);
        if(defined $parts[1]) {
          $value = $parts[1];
          $value =~ s/^\s+//;
          $value =~ s/\s+$//;
          $continue_upgrade_via_image = $value;
          log_msg_info("<<< get_callstatus_step found upgrade_via_image=$continue_upgrade_via_image");
        }
      }      
    }
    close CSFFILE;
  }
  function_leave("get_callstatus_step", "step=$rc");
  return($rc);
}

sub create_callstatus_file {
  my $step = shift;
  my $iteration = shift;
  my $remember_upgrade_with_completely_stopped_ha = shift;
  my $remember_upgrade_with_full_drbd_sync = shift;
  my $remember_upgrade_with_reduce_downtime = shift;
  my $remember_upgrade_via_image = shift;

  function_enter("create_callstatus_file", "step=$step iteration=$iteration remember_upgrade_with_completely_stopped_ha=$remember_upgrade_with_completely_stopped_ha remember_upgrade_with_full_drbd_sync=$remember_upgrade_with_full_drbd_sync remember_upgrade_with_reduce_downtime=$remember_upgrade_with_reduce_downtime remember_upgrade_via_image=$remember_upgrade_via_image");

  exec_cmd("ls -l $swupdated_call_status_file");

  if($nodaemon and ($hotfix_mode > 0)) {
    log_msg_info("running in hotfix and non-daemon (slave) mode, creating callstatus file not useful, skipped");
  } else {
    log_msg_info("create_callstatus_file increment iteration");
    $iteration += 1;

    # ensure that the dir exists
    if ( ! -d $webservice_data_dir) {
      makeDir($webservice_data_dir);
    }

    # write_status no longer used, it is only informational
    my $content = "step=$step\niteration=$iteration\nupgrade_with_completely_stopped_ha=$remember_upgrade_with_completely_stopped_ha\nupgrade_with_full_drbd_sync=$remember_upgrade_with_full_drbd_sync\nupgrade_with_reduce_downtime=$remember_upgrade_with_reduce_downtime\nupgrade_via_image=$remember_upgrade_via_image\n";
    chomp($content);

    if(open(CCSFWFILE, ">$swupdated_call_status_file")) {
      protectFile($swupdated_call_status_file);
      print CCSFWFILE $content;
      close CCSFWFILE;
      log_msg("$swupdated_call_status_file:\n$content\n");
    } else {
      log_msg_info("error creating callstatus file: $swupdated_call_status_file");
    }
  }
  function_leave("create_callstatus_file");
}

sub remove_call_status_file {
  function_enter("remove_call_status_file");  
  removeFile($swupdated_call_status_file);
  function_leave("remove_call_status_file");
}

sub create_generic_file {
  my $content = shift;
  my $file_to_write = shift;
  function_enter("create_generic_file", "file_to_write=$file_to_write  content=$content");

  if(open(CGFFRWFILE, ">$file_to_write")) {
    protectFile($file_to_write);
    print CGFFRWFILE "$content";
    close CGFFRWFILE;
  }
  function_leave("create_generic_file");
}

sub append_generic_file {
  my $content = shift;
  my $file_to_write = shift;
  function_enter("append_generic_file", "file_to_write=$file_to_write  content=$content");

  if(open(AGFFRWFILE, ">>$file_to_write")) {
    protectFile($file_to_write);
    print AGFFRWFILE "$content";
    close AGFFRWFILE;
  }
  function_leave("append_generic_file");
}

sub create_call_file {
  function_enter("create_call_file");
  exec_cmd("ls -l $swupdated_call_file");
  if($nodaemon and ($hotfix_mode > 0)) {
    log_msg_info("running in non-daemon (slave) mode, creating call file not useful, skipped");
  } else {
    my $content = "";
    if ($continue_upgrade == 0)  {
      $content = "$call_string --continue_upgrade";
    } else {
      $content = "$call_string\n";
    }
    chomp($content);

    # ensure that the dir exists
    if ( ! -d $webservice_data_dir) {
      makeDir($webservice_data_dir);
    }
    if(open(CCFWFILE, ">$swupdated_call_file")) {
      protectFile($swupdated_call_file);
      print CCFWFILE "$content\n";
      close CCFWFILE;
      log_msg("$swupdated_call_file:\n$content\n");
    }
  }
  function_leave("create_call_file");
}

sub create_call_file_for_reboot {  
  function_enter("create_call_file_for_reboot");

  # ensure that the dir exists
  if ( ! -d $webservice_data_dir) {
    makeDir($webservice_data_dir);
  }
  my $script_to_call = "$yaps_self";
  if ($is_stmix == 1) {
    log_msg_info("For STMIX using original $call_script as command for plt_upgrade");
    $script_to_call = "$call_script";
  }
  
  my $content = "$script_to_call configure_cib_local";
  if(open(CCFFRWFILE, ">$swupdated_call_file")) {
    protectFile($swupdated_call_file);
    print CCFFRWFILE "$content\n";
    close CCFFRWFILE;
    log_msg("$swupdated_call_file:\n$content\n");
  }
  function_leave("create_call_file_for_reboot");
}

sub remove_call_file {
  function_enter("remove_call_file");  
  removeFile($swupdated_call_file);
  function_leave("remove_call_file");
}

sub get_number_of_files_in_dir {
  my $dir_to_check = shift;
  function_enter("get_number_of_files_in_dir", "dir=$dir_to_check");
  my @files = <$dir_to_check/*>;
  my $count = @files;
  function_leave("get_number_of_files_in_dir", "count=$count");
  return $count;
}

sub daemonize_me {
  function_enter("daemonize_me", "(to get independant from caller process)");
  my $rc = 0;

  chdir '/'                or $rc = 1;
  if ( $rc == 1) { log_msg_info("Can't chdir to /: $!"); }

  # umask 0;

  open STDIN, '/dev/null'  or  $rc = 2;
  if ( $rc == 2) {
    log_msg_info("Can't read /dev/null: $!");
  }
  open STDOUT, '/dev/null' or $rc = 3;
  if ( $rc == 3) {
    log_msg_info("Can't write to /dev/null: $!");
  }
  open STDERR, '/dev/null' or $rc = 4;
  if ( $rc == 4) {
    log_msg_info("Can't write to /dev/null: $!");
  }
  if (!defined($pid = fork)) {
    $rc = 5;
    log_msg_info("Can't fork: $!");
  }
  if($pid) {
    $pid = $$;
    log_msg("exiting parent process (daemonizing)");
    exit(0);
  } else {
    sleep(1);
    $pid = $$;
    log_msg("proceeding child process (daemonized)");
  }
  setsid                    or $rc = 6;
  if ( $rc == 6) {
    log_msg_info("Can't start a new session: $!");
  }

  $SIG{'INT' } = 'interrupt';
  $SIG{'HUP' } = 'interrupt';
  $SIG{'ABRT'} = 'interrupt';
  $SIG{'QUIT'} = 'interrupt';
  $SIG{'TRAP'} = 'interrupt';
  $SIG{'STOP'} = 'interrupt';
  $SIG{'TERM'} = 'interrupt';
  function_leave("daemonize_me", "rc=$rc");
  return($rc);
}
