#!/bin/bash
NGINX_SERVICE_URL="https://127.0.0.1:8088"
DEVICE_MANAGER_DIR="/OSM/coffer_data/ISM/device-manager"
DM_NGINX_LOADER="${DEVICE_MANAGER_DIR}/kmc/libs/nginx_loader"
BACKUP_KMC_DATA_DIR="/startup_disk/conf/conf_local/dm/kmc"
DEVICE_MANAGER_KMC_DATA_DIR="${DEVICE_MANAGER_DIR}/kmc/data"
DM_KMC_DATA_DIR_NEW="${DEVICE_MANAGER_DIR}/kmc/data_origin_backup"
DEVICE_MANAGER_CERT_DIR="${DEVICE_MANAGER_DIR}/nginx/conf/cert/"
DEVICE_MANAGER_NGINX_DIR="${DEVICE_MANAGER_DIR}/nginx"
DEVICE_MANAGER_SCRIPT_DIR="/ISM/dm"
DEVICE_MANAGER_OWNER="OceanStorMgrUser"
DEVICE_MANAGER_OWNER_GROUP="mgrgroup"
NGINX_LOG_DIR="/OSM/coffer_log/log/ismlog/dmlogs/"
DEPLOY_LOG_PATH="${NGINX_LOG_DIR}/deploy.log"
NGINX_PID_FILE="$DEVICE_MANAGER_NGINX_DIR/conf/nginx.pid"
NGINX_REST_PID_FILE="$DEVICE_MANAGER_NGINX_DIR/conf/nginx_rest.pid"
NGINX_INNER_PID_FILE="$DEVICE_MANAGER_NGINX_DIR/conf/nginx_inner.pid"
DM_CRT_DATA_DIR_NEW="${DEVICE_MANAGER_NGINX_DIR}/conf/data_origin_backup"
DM_REPLACE_DHPARAM_SCRIPT="${DEVICE_MANAGER_DIR}/script/replace_dhparam_pem.sh"
DM_DH_PARAM_TMP="${DEVICE_MANAGER_NGINX_DIR}/conf/cert/device_manager_ssl_dhparam.pem.tmp"
 
source ${DEVICE_MANAGER_SCRIPT_DIR}/log4sh.sh

g_loop_count=0
g_restore_flag=1

logAndEchoInfo "Start nginx daemon..."

function isNotLinkFile()
{
    if [ -L $1 ]; then
        return 1
    else
        return 0
    fi
}

function check_url_once()
{
    status=$(curl -m 20 -ksi ${NGINX_SERVICE_URL} | grep 'HTTP/1.1 200 OK')
    if [ ! -n "${status}" ]; then
        logAndEchoError "Nginx service is abnormal <out 8088>."
        return 1
    fi

    status=$(curl -m 20 -si --unix-socket ${DEVICE_MANAGER_NGINX_DIR}/inner.socket http://127.0.0.1 | grep 'HTTP/1.1 200 OK')
    if [ ! -n "${status}" ]; then
        logAndEchoError "Nginx service is abnormal <inner socket>."
        return 1
    fi

    return 0
}

function check_url()
{
    for((i=1;i<=3;i++));  
    do   
        check_url_once
        if [ $? -eq 0 ]; then
            return 0
        fi
    done

    return 1
}

function get_ip_addr()
{
    local ip_type=$1
    local management_ip_list=`sh /OSM/script/net_get_sys_ip_addr.sh management $ip_type | awk -F '/' '{print $1}'`
    local maintenance_ip_list=""
    if [ $ip_type -eq 4 ]; then
        maintenance_ip_list=`sh /OSM/script/net_get_sys_ip_addr.sh maintenance $ip_type | awk -F '/' '{print $1}'`
    fi
    # 去重，并且返回结果
    echo $management_ip_list $maintenance_ip_list | sed 's/ /\n/g' | sort | uniq | xargs
    return $?
}

function check_web()
{
    local ipv4Addr=$1
    local ipv6Addr=$2

    for var in ${ipv4Addr}
    do
        for i in {1..10}; do
            status=$(curl -m 5 -ksi https://${var}:8088/login | grep 'HTTP/1.1 200 OK')
            if [ -z "${status}" ]; then
                logAndEchoError "Nginx web service ${var}:8088 is abnormal."
                sleep 1
            else
                break
            fi
        done
        if [ -z "${status}" ]; then
            logAndEchoError "Nginx web service ${var}:8088 is abnormal."
            return 1
        fi
    done

    for var in ${ipv6Addr}
    do
        for i in {1..10}; do
            status=$(curl -m 5 -ksi https://[${var}]:8088/login | grep 'HTTP/1.1 200 OK')
            if [ -z "${status}" ]; then
                logAndEchoError "Nginx web service ${var}:8088 is abnormal."
                sleep 1
            else
                break
            fi
        done
        if [ -z "${status}" ]; then
            logAndEchoError "Nginx web service ${var}:8088 is abnormal."
            return 1
        fi
    done

    return 0
}

function set_web_mod_own()
{
    if [ -L ${DEVICE_MANAGER_DIR}/web_lock ];then 
        rm ${DEVICE_MANAGER_DIR}/web_lock
    fi
    touch ${DEVICE_MANAGER_DIR}/web_lock
    isNotLinkFile "${DEVICE_MANAGER_DIR}/web_lock"
    if [ $? == 1 ]; then
        return 1
    fi
    chmod 400 ${DEVICE_MANAGER_DIR}/web_lock
    chown -h ${DEVICE_MANAGER_OWNER}:${DEVICE_MANAGER_OWNER_GROUP} ${DEVICE_MANAGER_DIR}/web_lock

    exec 200<>${DEVICE_MANAGER_DIR}/web_lock
    flock -n 200
    local ret=$?
    if [ $ret -eq 1 ];then
        logAndEchoInfo "the others is setting web file mode and own."
        return 0
    fi

    # 安全整改 nginx web目录权限，目录权限改成700，文件权限改成600
    logAndEchoInfo "Begin to set web file mode."
    if [ -L ${DEVICE_MANAGER_DIR}/web ];then
        rm ${DEVICE_MANAGER_DIR}/web
    fi
    isNotLinkFile "${DEVICE_MANAGER_DIR}/web"
    if [ $? == 1 ]; then
        return 1
    fi
    chmod -Rf 600 ${DEVICE_MANAGER_DIR}/web

    find ${DEVICE_MANAGER_DIR}/web -type l -exec rm {} \; > /dev/null 2>&1

    find ${DEVICE_MANAGER_DIR}/web -type d -exec chmod 700 {} \;
    chown -h -R ${DEVICE_MANAGER_OWNER}:${DEVICE_MANAGER_OWNER_GROUP} ${DEVICE_MANAGER_DIR}/web
    logAndEchoInfo "End to set web file mode."

    flock -u 200
}

function get_config_existed_ip()
{
    local inner_ips=$(python /ISM/dm/get_local_ips.py)

    local missIp=0

    local existedIp=""
    for var in ${inner_ips[@]}
    do
        ip -o -f inet addr | grep -w "${var}" >/dev/null 2>&1
        if [ $? -eq 0 ]; then
            # ip 已存在
            existedIp=`echo "${existedIp} ${var}"`
        else
            missIp=1
        fi
    done

    echo "${existedIp}"
    return ${missIp}
}

G_INIT_NO_IP_10=0
function get_inner_ip_list()
{
    local needRefresh=0

    local inner_ips=""
    inner_ips=$(get_config_existed_ip)
    if [ $? -ne 0 ];then
        # next time 
        needRefresh=1
    fi

    local local_ibc=$(sh /OSM/script/net_get_sys_ip_addr.sh local_ibc  4 | awk -F'/' '{printf $1 " "}')

    if [ -n "${local_ibc}" ]; then
        echo "${inner_ips} ${local_ibc}" | awk '{for(i=1;i<=NF;++i){!arr[$i]++;}}END{for(a in arr) {printf(a " ");}}'
    else
        echo "${inner_ips}" | awk '{for(i=1;i<=NF;++i){!arr[$i]++;}}END{for(a in arr) {printf(a " ");}}'
    fi

    return ${needRefresh}
}

G_WATCH_INNER_IP_CNT=0
function watch_inner_iplist()
{
    ((G_WATCH_INNER_IP_CNT++))
    if [[ ${G_WATCH_INNER_IP_CNT} -ge 10 || ${G_INIT_NO_IP_10} -eq 1 ]]; then
        local inner_iplist=""
        inner_iplist=$(get_inner_ip_list)
        G_INIT_NO_IP_10=$?
        for var in ${inner_iplist[@]}
        do
            netstat -lnt | grep ${var}:8089 >/dev/null 2>&1
            if [ $? -ne 0 ]; then
                logAndEchoInfo "The port 8089 is not listened on this IP address. ip=${var}, cnt=${G_WATCH_INNER_IP_CNT}."
                return 1
            fi
        done

        G_WATCH_INNER_IP_CNT=0
    fi

    return 0
}

function reload_nginx_inner()
{
    logAndEchoInfo "Begin to reload nginx inner."

    reset_inner_ip
    ip vrf exec vrf-inner /ISM/ism_tool "OceanStorMgrUser" -c "env LD_LIBRARY_PATH=${DEVICE_MANAGER_DIR}/kmc/libs PATH=${PATH}:/usr/sbin ${DM_NGINX_LOADER} -r `cat ${DEVICE_MANAGER_DIR}/nginx/conf/nginx_inner.pid`" >> ${DEPLOY_LOG_PATH} 2>&1

    local ret=$?
 
    logAndEchoInfo "Succeed to reload nginx inner, ret=${ret}."
    return ${ret}
}

function bind_inner_ip()
{
    local inner_ip="$1"

    logAndEchoInfo "Begin to bind inner ip. [${inner_ip}]"

    local inner_cfg=${DEVICE_MANAGER_NGINX_DIR}/conf/nginx_inner.conf

    # delete old ip
    sed -i '/ listen .*:8089 ssl;/d;' ${inner_cfg}

    # add new ip
    for var in ${inner_ip[@]}
    do
        sed -i "/ *# Configure the listening IP above this line./i\        listen ${var}:8089 ssl;" ${inner_cfg}
    done

    logAndEchoWarn "End to bind inner ip."
}

function reset_inner_ip()
{
    local inner_iplist=""
    inner_iplist=`get_inner_ip_list`
    G_INIT_NO_IP_10=$?
    bind_inner_ip  "${inner_iplist}"
}

function start_nginx()
{
    web_only=$1

    logAndEchoInfo "Begin to set dm file mode and owner."
    # 清理目录下的软连接，-not -path参数指定的目录会被略过
    find ${DEVICE_MANAGER_DIR} -type l \( -not -path ${DEVICE_MANAGER_NGINX_DIR}/logs \) \( -not -path "${DEVICE_MANAGER_DIR}/luajit/*" \) -exec rm -f {} \;

    cd ${DEVICE_MANAGER_NGINX_DIR}/sbin/
    isNotLinkFile "${DEVICE_MANAGER_DIR}"
    if [ $? == 1 ]; then
        return 1
    fi
    chmod 550 ${DEVICE_MANAGER_DIR}
    chown -h ${DEVICE_MANAGER_OWNER}:${DEVICE_MANAGER_OWNER_GROUP} ${DEVICE_MANAGER_DIR}

    set_web_mod_own &
    cd ${DEVICE_MANAGER_DIR}; ls | grep -vw web | xargs chmod -R 550; cd -
    isNotLinkFile "${DEVICE_MANAGER_DIR}/web_lock"
    if [ $? == 1 ]; then
        return 1
    fi
    chmod 400 ${DEVICE_MANAGER_DIR}/web_lock

    isNotLinkFile "${DEVICE_MANAGER_DIR}/web_rest"
    if [ $? == 1 ]; then
        return 1
    fi
    isNotLinkFile "${DEVICE_MANAGER_DIR}/web_inner"
    if [ $? == 1 ]; then
        return 1
    fi
    test -L ${DEVICE_MANAGER_DIR}/web_rest || chmod 500 ${DEVICE_MANAGER_DIR}/web_rest
    test -L ${DEVICE_MANAGER_DIR}/web_inner || chmod 500 ${DEVICE_MANAGER_DIR}/web_inner
    find ${DEVICE_MANAGER_DIR}/web_rest -type f -exec chmod 400 {} \;
    find ${DEVICE_MANAGER_DIR}/web_inner -type f -exec chmod 400 {} \;

    isNotLinkFile "${DEVICE_MANAGER_NGINX_DIR}/conf/cert"
    if [ $? == 1 ]; then
        return 1
    fi
    chmod 700 ${DEVICE_MANAGER_NGINX_DIR}/conf/cert
    find ${DEVICE_MANAGER_NGINX_DIR}/conf/cert -type f -exec chmod 640 {} \;
    chmod -R 750 ${DEVICE_MANAGER_NGINX_DIR}/*_temp

    isNotLinkFile "${DEVICE_MANAGER_NGINX_DIR}/conf/nginx.conf"
    if [ $? == 1 ]; then
        return 1
    fi
    chmod 600 ${DEVICE_MANAGER_NGINX_DIR}/conf/nginx.conf
    find ${DEVICE_MANAGER_DIR}/script -type f -name "kmc2_*" -exec chmod 500 {} \;
    find ${DEVICE_MANAGER_DIR}/script -type f -name "*_password.sh" -exec chmod 500 {} \;
    
    isNotLinkFile "/OSM/script/tar_dm_log.sh"
    if [ $? == 1 ]; then
        return 1
    fi
    isNotLinkFile "/OSM/script/setup.hook/before/dm_clear.sh"
    if [ $? == 1 ]; then
        return 1
    fi    
    chmod 550 /OSM/script/tar_dm_log.sh
    chmod 550 /OSM/script/setup.hook/before/dm_clear.sh

    rm -rf ${DEVICE_MANAGER_NGINX_DIR}/logs
    test -L ${NGINX_LOG_DIR} && rm -f ${NGINX_LOG_DIR}
    mkdir -p ${NGINX_LOG_DIR}
    ln -sfn ${NGINX_LOG_DIR} ${DEVICE_MANAGER_NGINX_DIR}/logs
    isNotLinkFile "${NGINX_LOG_DIR}"
    if [ $? == 1 ]; then
        return 1
    fi
    chmod 750 ${NGINX_LOG_DIR}
    find ${NGINX_LOG_DIR} -type f -name "*.log" -exec chmod 640 {} \;

    if [ ! -f "$NGINX_PID_FILE" ]; then
        touch $NGINX_PID_FILE
    fi
    if [ ! -f "$NGINX_REST_PID_FILE" ]; then
        touch $NGINX_REST_PID_FILE
    fi
    if [ ! -f "$NGINX_INNER_PID_FILE" ]; then
        touch $NGINX_INNER_PID_FILE
    fi
    
    isNotLinkFile "${DEVICE_MANAGER_NGINX_DIR}/conf/nginx.pid"
    if [ $? == 1 ]; then
        return 1
    fi
    isNotLinkFile "${DEVICE_MANAGER_NGINX_DIR}/conf/nginx_rest.pid"
    if [ $? == 1 ]; then
        return 1
    fi
    isNotLinkFile "${DEVICE_MANAGER_NGINX_DIR}/conf/nginx_inner.pid"
    if [ $? == 1 ]; then
        return 1
    fi
    isNotLinkFile "${DEVICE_MANAGER_DIR}/innerIpInformation.conf"
    if [ $? == 1 ]; then
        return 1
    fi
    chmod 640 ${DEVICE_MANAGER_NGINX_DIR}/conf/nginx.pid
    chmod 640 ${DEVICE_MANAGER_NGINX_DIR}/conf/nginx_rest.pid
    chmod 640 ${DEVICE_MANAGER_NGINX_DIR}/conf/nginx_inner.pid
    chmod 400 ${DEVICE_MANAGER_DIR}/innerIpInformation.conf
    chown -h -R ${DEVICE_MANAGER_OWNER}:${DEVICE_MANAGER_OWNER_GROUP} ${NGINX_LOG_DIR}

    cd ${DEVICE_MANAGER_DIR}; ls | grep -vw web | xargs chown -h -R ${DEVICE_MANAGER_OWNER}:${DEVICE_MANAGER_OWNER_GROUP}; cd -

    isNotLinkFile "${DEVICE_MANAGER_NGINX_DIR}"
    if [ $? == 1 ]; then
        return 1
    fi
    chmod 750 ${DEVICE_MANAGER_NGINX_DIR}

    if [[ $web_only -ne 1 ]]; then
        rm -f ${DEVICE_MANAGER_NGINX_DIR}/inner.socket
        rm -f ${DEVICE_MANAGER_NGINX_DIR}/outter.socket
    fi

    # 为避免老版本的密码文件残留，提前删除
    rm /OSM/coffer_data/ISM/device-manager/nginx/conf/cert/device_manager_plain_password >/dev/null 2>&1
    rm /startup_disk/conf/conf_local/dm/cert/device_manager_plain_password >/dev/null 2>&1

    logAndEchoInfo "End to set dm file mode and owner."

    isNotLinkFile "nginx"
    if [ $? == 1 ]; then
        return 1
    fi
    chmod u+s nginx
    sh /OSM/bin/cgroup_manage.sh -g omm -i $$ start
        su -s /bin/bash "OceanStorMgrUser" -c "env LD_LIBRARY_PATH=/OSM/coffer_data/ISM/device-manager/kmc/libs /OSM/coffer_data/ISM/device-manager/kmc/libs/nginx_loader -s >> ${DEPLOY_LOG_PATH} 2>&1"
        if [[ $web_only -ne 1 ]]; then
            su -s /bin/bash "OceanStorMgrUser" -c "env LD_LIBRARY_PATH=/OSM/coffer_data/ISM/device-manager/kmc/libs /OSM/coffer_data/ISM/device-manager/kmc/libs/nginx_loader -s rest"
            reset_inner_ip
            ip vrf exec vrf-inner /ISM/ism_tool "OceanStorMgrUser" -c "env LD_LIBRARY_PATH=/OSM/coffer_data/ISM/device-manager/kmc/libs PATH=${PATH}:/usr/sbin /OSM/coffer_data/ISM/device-manager/kmc/libs/nginx_loader -s inner"
        fi
    sh /OSM/bin/cgroup_manage.sh -i $$ end
    
    find ${DEVICE_MANAGER_NGINX_DIR}/conf/cert -type f -exec chmod -R 600 {} \;
    isNotLinkFile "${DEVICE_MANAGER_NGINX_DIR}/inner.socket"
    if [ $? == 1 ]; then
        return 1
    fi
    isNotLinkFile "${DEVICE_MANAGER_NGINX_DIR}/outter.socket"
    if [ $? == 1 ]; then
        return 1
    fi
    chmod 600 ${DEVICE_MANAGER_NGINX_DIR}/inner.socket
    chmod 600 ${DEVICE_MANAGER_NGINX_DIR}/outter.socket
    if [ $? -ne 0 ];then
        logAndEchoError "Failed to start nginx service."
    else
        logAndEchoInfo "Succeed to start nginx service."
    fi
    return 0
}

function handle_log()
{
    # 先检查日志处理脚本是否正在运行，如果在运行则不再调用，否则后台调用
    handle_log_path="$DEVICE_MANAGER_SCRIPT_DIR/dm_log_rotate.sh"
    count=$(ps -ef | grep "$handle_log_path" | grep -v "grep" | wc -l)

    if [ $count == 0 ]; then
        nohup sh $handle_log_path > /dev/null 2>&1 &
    fi
    return 0
}

function handle_ip()
{
    local ipv4Addr=$1
    local ipv6Addr=$2
    sh $DEVICE_MANAGER_SCRIPT_DIR/handle_manage_ip.sh "${ipv4Addr}" "${ipv6Addr}" > /dev/null 2>&1
    return $?
}

function handle_oem()
{
    sh $DEVICE_MANAGER_SCRIPT_DIR/oem.sh > /dev/null 2>&1
    return $?
}

function restart_nginx()
{
    killall nginx

    sleep 1
    start_nginx
    sleep 0.1
}


function kill_nginx_by_pid()
{
    local pidFile=$1
    local cmd=$2

    if [ -z ${pidFile} ] || [ -z ${cmd} ];then
        logAndEchoInfo "Failed to kill nginx by '${pidFile}', the file string is null."
        return 1
    fi

    if [ ! -f ${pidFile} ];then
        logAndEchoInfo "Failed to kill nginx by '${pidFile}', it's not a file."
        return 1
    fi

    if [ -L ${pidFile} ]; then
        logAndEchoError "Pid file ${pidFile} is symlink."
        return 1
    fi

    local pid=$(cat ${pidFile})
    if [ -z ${pid} ];then
        logAndEchoInfo "Failed to kill nginx by '${pidFile}', nothing in the file."
        return 1
    fi

    local pidInfo=$(ps -p ${pid} | grep nginx)
    if [ -z "${pidInfo}" ];then
        logAndEchoInfo "the process(${pid}) does not exist."
        return 1
    fi

    logAndEchoInfo "Succeded in kill nginx by '${pidFile}', pid is ${pid}."

    kill ${cmd} ${pid}

    return 0
}

function kill_master_for_worker()
{
    local workerPid=$1
    # first step: to get master pid

    local masterPid=$(ps -p ${workerPid} -o ppid=)
    if [ -z "${masterPid}" ] || [ ${masterPid} -eq 1 ]; then
        return 1
    fi

    local masterName=$(ps -p ${masterPid} -o comm=)
    if [ -z "${masterName}" ];then
        return 1
    fi

    if [ "${masterName}" != "nginx" ];then
        return 1
    fi

    kill -9 ${masterPid}

    logAndEchoInfo "Kill nginx's master. workerPid=${workerPid}, masterPid=${masterPid}"
    return 0
}

function kill_nginx_web()
{
    if [ -L ${NGINX_PID_FILE} ]; then
        logAndEchoError "${NGINX_PID_FILE} is symlink."
    else
        logAndEchoInfo "Begin to kill web nginx pid=$(cat ${NGINX_PID_FILE})."
    fi

    kill_nginx_by_pid $NGINX_PID_FILE -9

    # 杀掉监听127.0.0.2:8088的nginx工作进程
    for i in {1..100}; do
        local pid=$(netstat -lnp | grep 127.0.0.2:8088 | awk '{for(i=1; i<=NF; i++){p=match($i, /\/nginx/); if(p) print substr($i,0,p-1)}}')
        if [ ! -z "${pid}" ]; then
            if [ $i -gt 10 ];then
                # to check if the parent is online, if online, kill
                kill_master_for_worker ${pid}
            fi

            kill -9 $pid
            logAndEchoInfo "Kill web nginx pid=${pid}."
            sleep 0.5
        else
            logAndEchoInfo "Succeed to kill all web nginx."
            return
        fi
    done
}

function restart_nginx_web()
{
    kill_nginx_web

    sleep 1

    start_nginx 1
    sleep 1
}

function reload_nginx_web()
{
    logAndEchoInfo "Begin to reload nginx web."
    # 为避免老版本的密码文件残留，提前删除
    rm /OSM/coffer_data/ISM/device-manager/nginx/conf/cert/device_manager_plain_password >/dev/null 2>&1
    rm /startup_disk/conf/conf_local/dm/cert/device_manager_plain_password >/dev/null 2>&1

    su -s /bin/bash "OceanStorMgrUser" -c "env LD_LIBRARY_PATH=/OSM/coffer_data/ISM/device-manager/kmc/libs /OSM/coffer_data/ISM/device-manager/kmc/libs/nginx_loader -r `cat /OSM/coffer_data/ISM/device-manager/nginx/conf/nginx.pid` >> ${DEPLOY_LOG_PATH} 2>&1"
    local ret=$?
    [ ${ret} -eq 0 ] || logAndEchoInfo "Failed to exe ism tool."
 
    logAndEchoInfo "Succeed to reload nginx web"
    return ${ret}
}

function recovery_from_sa()
{
    # 由于只有 daemon 和 install 的时候调用，不存在并发，不用加锁
    
    diagsh --attach="oam_diagnose_666" --cmd="sa getcertificate 14 0  ${DEVICE_MANAGER_CERT_DIR} OceanStorMgrUser:mgrgroup 600  # # /ISM/dm/saveNginxCertPwd.sh"
}

function recovery_from_pkg()
{
    # 由于只有 daemon 和 install 的时候调用，不存在并发，不用加锁
    if [ -L ${DM_CRT_DATA_DIR_NEW} -o -L ${DEVICE_MANAGER_CERT_DIR} ]; then
        return
    fi
    isNotLinkFile "${DM_CRT_DATA_DIR_NEW}/device_manager.crt"
    if [ $? == 1 ]; then
        return 1
    fi
    isNotLinkFile "${DEVICE_MANAGER_CERT_DIR}/DM_Cert.crt"
    if [ $? == 1 ]; then
        return 1
    fi
    isNotLinkFile "${DM_CRT_DATA_DIR_NEW}/device_manager_cipher.key"
    if [ $? == 1 ]; then
        return 1
    fi
    isNotLinkFile "${DM_CRT_DATA_DIR_NEW}/device_manager_cipher_password"
    if [ $? == 1 ]; then
        return 1
    fi
    isNotLinkFile "${DEVICE_MANAGER_CERT_DIR}/DM_PriKey.key"
    if [ $? == 1 ]; then
        return 1
    fi
    isNotLinkFile "${DEVICE_MANAGER_CERT_DIR}/recencpwd"
    if [ $? == 1 ]; then
        return 1
    fi
    cp -f ${DM_CRT_DATA_DIR_NEW}/device_manager.crt ${DEVICE_MANAGER_CERT_DIR}/DM_Cert.crt
    chown -h OceanStorMgrUser:mgrgroup ${DEVICE_MANAGER_CERT_DIR}/DM_Cert.crt
    chmod 600 ${DEVICE_MANAGER_CERT_DIR}/DM_Cert.crt
    cp -f ${DM_CRT_DATA_DIR_NEW}/device_manager_cipher.key ${DEVICE_MANAGER_CERT_DIR}/DM_PriKey.key
    chown -h OceanStorMgrUser:mgrgroup ${DEVICE_MANAGER_CERT_DIR}/DM_PriKey.key
    chmod 600 ${DEVICE_MANAGER_CERT_DIR}/DM_PriKey.key
    cp -f ${DM_CRT_DATA_DIR_NEW}/device_manager_cipher_password ${DEVICE_MANAGER_CERT_DIR}/recencpwd
    chown -h OceanStorMgrUser:mgrgroup ${DEVICE_MANAGER_CERT_DIR}/recencpwd
    chmod 600 ${DEVICE_MANAGER_CERT_DIR}/recencpwd
}

function recovery_kmc()
{
    su -s /bin/bash "OceanStorMgrUser" -c "env LD_LIBRARY_PATH=${DEVICE_MANAGER_DIR}/kmc/libs PATH=${PATH}:/usr/sbin ${DM_NGINX_LOADER} -check_kmc ${DEVICE_MANAGER_KMC_DATA_DIR}/kmca_primary_file.dat ${DEVICE_MANAGER_KMC_DATA_DIR}/kmca_standby_file.dat"
    local ret=$?
    if [ ${ret} -eq 0 ];then
        return 0
    fi

    if [ ${ret} -ne 1 ];then
        return 1   # 表示不支持 check_kmc 的命令，下面的逻辑无需处理
    fi

    su -s /bin/bash "OceanStorMgrUser" -c "env LD_LIBRARY_PATH=${DEVICE_MANAGER_DIR}/kmc/libs PATH=${PATH}:/usr/sbin ${DM_NGINX_LOADER} -check_kmc ${BACKUP_KMC_DATA_DIR}/kmca_primary_file.dat ${BACKUP_KMC_DATA_DIR}/kmca_standby_file.dat"
    if [ $? -eq 0 ];then
        logAndEchoInfo "succeeded in checking kmc data, now recovery by backup kmc. [Line:${LINENO}, File:${scriptName}]"
        isNotLinkFile "${BACKUP_KMC_DATA_DIR}"
        if [ $? == 1 ]; then
            return 1
        fi
        isNotLinkFile "${DEVICE_MANAGER_KMC_DATA_DIR}"
        if [ $? == 1 ]; then
            return 1
        fi
        cp -f ${BACKUP_KMC_DATA_DIR}/* ${DEVICE_MANAGER_KMC_DATA_DIR}/
        return $?
    fi

    su -s /bin/bash "OceanStorMgrUser" -c "env LD_LIBRARY_PATH=${DEVICE_MANAGER_DIR}/kmc/libs PATH=${PATH}:/usr/sbin ${DM_NGINX_LOADER} -check_kmc ${DM_KMC_DATA_DIR_NEW}/kmca_primary_file.dat ${DM_KMC_DATA_DIR_NEW}/kmca_standby_file.dat"
    if [ $? -eq 0 ];then
        logAndEchoInfo "succeeded in checking origin data, now recovery by origin kmc. [Line:${LINENO}, File:${scriptName}]"
        isNotLinkFile "${DM_KMC_DATA_DIR_NEW}"
        if [ $? == 1 ]; then
            return 1
        fi
        isNotLinkFile "${DEVICE_MANAGER_KMC_DATA_DIR}"
        if [ $? == 1 ]; then
            return 1
        fi
        cp -f ${DM_KMC_DATA_DIR_NEW}/* ${DEVICE_MANAGER_KMC_DATA_DIR}/
        return $?
    fi

    logAndEchoWarn "Failed to recovery kmc, used/backup/origin are all destroyed. [Line:${LINENO}, File:${scriptName}]"
    return 1
}
function reload_nginx_rest()
{
    /ISM/ism_tool "OceanStorMgrUser" -c "env LD_LIBRARY_PATH=/OSM/coffer_data/ISM/device-manager/kmc/libs /OSM/coffer_data/ISM/device-manager/kmc/libs/nginx_loader -r `cat /OSM/coffer_data/ISM/device-manager/nginx/conf/nginx_rest.pid`"
    ret=$?
    logAndEchoInfo "Reload nginx rest, ret=${ret}."
    return ${ret}
}
 
# 1标识已替换过
g_dhparam_replace_flag=0
# 1标识执行过替换脚本
g_dhparam_exec_flag=0
g_dhparam_replace_pid=-1
function check_replace_dhparam_working()
{ 
    cat /proc/${g_dhparam_replace_pid}/cmdline | grep "${DM_REPLACE_DHPARAM_SCRIPT}" | grep -v ism_tool
    if [ $? -eq 0 ]; then
        return 0
    fi
 
    g_dhparam_replace_pid=`ps -wwef | grep ${DM_REPLACE_DHPARAM_SCRIPT} | grep -v grep | grep -v ism_tool | awk '{print $2}'`
    cat /proc/${g_dhparam_replace_pid}/cmdline | grep "${DM_REPLACE_DHPARAM_SCRIPT}"
    if [ $? -eq 0 ]; then
        return 0
    fi
    return 1
}
 
function start_replace_dhparam()
{
    # 可能会耗时较久，在后台运行
    nohup /ISM/ism_tool "OceanStorMgrUser" -c "sh ${DM_REPLACE_DHPARAM_SCRIPT}" >/dev/null 2>&1 &
    logAndEchoInfo "Start ${DM_REPLACE_DHPARAM_SCRIPT}."

    for((i=1;i<=3;i++));  
    do   
        check_replace_dhparam_working
        if [ $? -eq 0 ]; then
            g_dhparam_exec_flag=1
            return 0
        fi
        sleep 1
    done
    return 1
}
 
function reload_after_replace_dhparam()
{
    reload_nginx_web
    if [ $? -ne 0 ];then
        start_replace_dhparam
        return $?
    fi
 
    reload_nginx_inner
    if [ $? -ne 0 ];then
        return $?
    fi
 
    reload_nginx_rest
    if [ $? -ne 0 ];then
        return $?
    fi
 
    g_dhparam_replace_flag=1
    logAndEchoInfo "reload after replace dhparam."
}
 
function check_and_replace_dhparam_pem()
{
    # 替换过，直接返回
    if [ ${g_dhparam_replace_flag} -eq 1 ];then
        return
    fi
 
    # 替换脚本正在执行，则返回
    check_replace_dhparam_working
    if [ $? -eq 0 ];then
        return
    fi
 
    # 未执行过替换脚本
    if [ ${g_dhparam_exec_flag} -eq 0 ];then
        start_replace_dhparam
        return
    fi
 
    # 临时文件还在说明上次替换未完成，重新启动一次
    if [ -f "${DM_DH_PARAM_TMP}" ];then
        logAndEchoWarn "Error: replace file error,restart replace dhparam"
        start_replace_dhparam
        return
    fi
 
    reload_after_replace_dhparam
}
g_restore_cert_cnt=0
function recovery_cert_from_sa()
{
    ((g_restore_cert_cnt++))
    if [ ${g_restore_cert_cnt} -lt 10 ];then
        return 1
    fi
    g_restore_cert_cnt=0

    logAndEchoInfo "Begin to recovery cert from sa."

    recovery_from_sa

    find ${DEVICE_MANAGER_CERT_DIR} -type l -exec rm -f {} \;
    local DM_CERT=${DEVICE_MANAGER_CERT_DIR}/DM_Cert.crt
    local DM_CIPHER_KEY=${DEVICE_MANAGER_CERT_DIR}/DM_PriKey.key
    local DM_CIPHER_PASSWORD=${DEVICE_MANAGER_CERT_DIR}/recencpwd
    su -s /bin/bash "OceanStorMgrUser" -c "env LD_LIBRARY_PATH=${DEVICE_MANAGER_DIR}/kmc/libs PATH=${PATH}:/usr/sbin ${DM_NGINX_LOADER} -check_cert ${DM_CERT} ${DM_CIPHER_KEY} ${DM_CIPHER_PASSWORD}"
    if [ $? -eq 0 ];then
        isNotLinkFile "${DM_CERT}"
        if [ $? == 1 ]; then
            return 1
        fi
        isNotLinkFile "${DEVICE_MANAGER_CERT_DIR}/device_manager.crt"
        if [ $? == 1 ]; then
            return 1
        fi
        isNotLinkFile "${DM_CIPHER_KEY}"
        if [ $? == 1 ]; then
            return 1
        fi
        isNotLinkFile "${DEVICE_MANAGER_CERT_DIR}/device_manager_cipher.key"
        if [ $? == 1 ]; then
            return 1
        fi
        isNotLinkFile "${DM_CIPHER_PASSWORD}"
        if [ $? == 1 ]; then
            return 1
        fi
        isNotLinkFile "${DEVICE_MANAGER_CERT_DIR}/device_manager_cipher_password"
        if [ $? == 1 ]; then
            return 1
        fi

        mv ${DM_CERT} ${DEVICE_MANAGER_CERT_DIR}/device_manager.crt
        local certRet=$?
        mv ${DM_CIPHER_KEY} ${DEVICE_MANAGER_CERT_DIR}/device_manager_cipher.key
        local keyRet=$?
        mv ${DM_CIPHER_PASSWORD} ${DEVICE_MANAGER_CERT_DIR}/device_manager_cipher_password
        local pwdRet=$?

        if [[ ${certRet} -eq 0 && ${keyRet} -eq 0 && ${pwdRet} -eq 0 ]] ;then

            return 0
        fi

        logAndEchoWarn "Info: Succeded in recovery cert from sa, now copy to real use. certRet=${certRet}, keyRet=${keyRet}, pwdRet=${pwdRet}. [Line:${LINENO}, File:${scriptName}]"
    else
        recovery_kmc
    fi

    logAndEchoError "Failed to restore cert from sa."
    return 1
}

g_restore_pkg_cert_cnt=0
function recovery_cert_from_pkg()
{
    ((g_restore_pkg_cert_cnt++))
    if [ ${g_restore_pkg_cert_cnt} -lt 10 ];then
        return 1
    fi
    g_restore_pkg_cert_cnt=0

    logAndEchoInfo "Begin to recovery cert from package."

    recovery_from_pkg

    local DM_CERT=${DEVICE_MANAGER_CERT_DIR}/DM_Cert.crt
    local DM_CIPHER_KEY=${DEVICE_MANAGER_CERT_DIR}/DM_PriKey.key
    local DM_CIPHER_PASSWORD=${DEVICE_MANAGER_CERT_DIR}/recencpwd
    su -s /bin/bash "OceanStorMgrUser" -c "env LD_LIBRARY_PATH=${DEVICE_MANAGER_DIR}/kmc/libs ${DM_NGINX_LOADER} -check_cert ${DM_CERT} ${DM_CIPHER_KEY} ${DM_CIPHER_PASSWORD}"
    if [ $? -eq 0 ];then
        isNotLinkFile "${DM_CERT}"
        if [ $? == 1 ]; then
            return 1
        fi
        isNotLinkFile "${DEVICE_MANAGER_CERT_DIR}/device_manager.crt"
        if [ $? == 1 ]; then
            return 1
        fi
        isNotLinkFile "${DM_CIPHER_KEY}"
        if [ $? == 1 ]; then
            return 1
        fi
        isNotLinkFile "${DEVICE_MANAGER_CERT_DIR}/device_manager_cipher.key"
        if [ $? == 1 ]; then
            return 1
        fi
        isNotLinkFile "${DM_CIPHER_PASSWORD}"
        if [ $? == 1 ]; then
            return 1
        fi
        isNotLinkFile "${DEVICE_MANAGER_CERT_DIR}/device_manager_cipher_password"
        if [ $? == 1 ]; then
            return 1
        fi

        mv ${DM_CERT} ${DEVICE_MANAGER_CERT_DIR}/device_manager.crt
        local certRet=$?
        mv ${DM_CIPHER_KEY} ${DEVICE_MANAGER_CERT_DIR}/device_manager_cipher.key
        local keyRet=$?
        mv ${DM_CIPHER_PASSWORD} ${DEVICE_MANAGER_CERT_DIR}/device_manager_cipher_password
        local pwdRet=$?

        if [[ ${certRet} -eq 0 && ${keyRet} -eq 0 && ${pwdRet} -eq 0 ]] ;then

            return 0
        fi

        logAndEchoWarn "Info: Succeded in recovery cert from pkg, now copy to real use. certRet=${certRet}, keyRet=${keyRet}, pwdRet=${pwdRet}. [Line:${LINENO}, File:${scriptName}]"
    else
        recovery_kmc
    fi

    logAndEchoError "Failed to restore cert from pkg."
    return 1
}

function restore_ngxin_conf()
{
    if [ -f ${DEVICE_MANAGER_CERT_DIR}/need_refresh_cert_from_sa ]; then
        recovery_cert_from_sa
        if [ $? -eq 0 ];then
            rm -f ${DEVICE_MANAGER_CERT_DIR}/need_refresh_cert_from_sa

            # 证书文件恢复后，重新启动
            restart_nginx
            return
        fi
    else
        if [ -f ${DEVICE_MANAGER_CERT_DIR}/need_refresh_cert_from_pkg ]; then
            recovery_cert_from_pkg
            if [ $? -eq 0 ];then
                rm -f ${DEVICE_MANAGER_CERT_DIR}/need_refresh_cert_from_pkg

                # 证书文件恢复后，重新启动
                restart_nginx
                return
            fi
        fi
    fi

    # todo-tx，根据查询控制器状态，判断扩控完成，查一次，不要反复查
    if [ ${g_restore_flag} -eq 1 ]; then
        # 恢复配置
        python $DEVICE_MANAGER_SCRIPT_DIR/handle_nginx_conf.py -restore > /dev/null 2>&1
        if [ $? -ne 0 ]; then
            logAndEchoError "Failed to restore customized nginx configurations."
            return
        fi

        g_restore_flag=0
    fi
}


function check_ipconf()
{
    su -s /bin/bash "OceanStorMgrUser" -c "test -r /OSM/conf/innerIpInformation.conf"
    local ret=$?
    if [ ${ret} -ne 0 ];then
        isNotLinkFile "/OSM/conf/innerIpInformation.conf"
        if [ $? == 1 ]; then
            return 1
        fi
        isNotLinkFile "/OSM/conf"
        if [ $? == 1 ]; then
            return 1
        fi
        isNotLinkFile "/OSM"
        if [ $? == 1 ]; then
            return 1
        fi
        chmod o+r /OSM/conf/innerIpInformation.conf
        chmod o+r /OSM/conf /OSM /
        chmod o+x /OSM/conf /OSM /
    fi

    diff -q /OSM/conf/innerIpInformation.conf ${DEVICE_MANAGER_DIR}/innerIpInformation.conf >/dev/null 2>&1
    ret=$?
    if [ ${ret} -ne 0 ]; then
        install -m 400 -o OceanStorMgrUser -g mgrgroup /OSM/conf/innerIpInformation.conf ${DEVICE_MANAGER_DIR}/innerIpInformation.conf
    fi

    su -s /bin/bash "OceanStorMgrUser" -c "test -r ${DEVICE_MANAGER_DIR}/innerIpInformation.conf"
    ret=$?
    if [ ${ret} -ne 0 ];then
        isNotLinkFile "${DEVICE_MANAGER_DIR}/innerIpInformation.conf"
        if [ $? == 1 ]; then
            return 1
        fi
        isNotLinkFile "/OSM/coffer_data"
        if [ $? == 1 ]; then
            return 1
        fi
        chmod 400 ${DEVICE_MANAGER_DIR}/innerIpInformation.conf
        chmod o+r /OSM/coffer_data
        chmod o+x /OSM/coffer_data
        chown -h OceanStorMgrUser:mgrgroup /OSM/coffer_data/ISM ${DEVICE_MANAGER_DIR} ${DEVICE_MANAGER_DIR}/innerIpInformation.conf
    fi
}

g_cert_check_fail_num=0
function check_cert_if_8088_not_exist()
{
    # 检查是否是证书的问题
    if [ -f ${DEVICE_MANAGER_CERT_DIR}/need_refresh_cert_from_sa ];then 
        g_cert_check_fail_num=0
        return
    fi
    
    test -L ${DEVICE_MANAGER_CERT_DIR}/need_refresh_cert_from_sa && rm -f ${DEVICE_MANAGER_CERT_DIR}/need_refresh_cert_from_sa

    netstat -lnpt | grep 127.0.0.2:8088 >/dev/null
    if [ $? -eq 0 ];then 
        g_cert_check_fail_num=0
        return
    fi

    # 此检查可能与导入证书冲突，导致检查失败，所以需要连续失败三次以后再标志
    local DM_CERT=${DEVICE_MANAGER_CERT_DIR}/device_manager.crt
    local DM_CIPHER_KEY=${DEVICE_MANAGER_CERT_DIR}/device_manager_cipher.key
    local DM_CIPHER_PASSWORD=${DEVICE_MANAGER_CERT_DIR}/device_manager_cipher_password
    su -s /bin/bash "OceanStorMgrUser" -c "env LD_LIBRARY_PATH=${DEVICE_MANAGER_DIR}/kmc/libs ${DM_NGINX_LOADER} -check_cert ${DM_CERT} ${DM_CIPHER_KEY} ${DM_CIPHER_PASSWORD}"
    if [ $? -eq 1 ];then
        ((g_cert_check_fail_num++))
        if [ ${g_cert_check_fail_num} -gt 3 ];then
            touch ${DEVICE_MANAGER_CERT_DIR}/need_refresh_cert_from_sa
            g_restore_cert_cnt=10

            logAndEchoWarn "Failed to check cert, now notify to recovery cert from sa."
        fi
    else
        g_cert_check_fail_num=0
    fi
}

while :
do
    check_ipconf
    handle_log
    handle_oem
    # 升级、扩控、换控等场景，将用户自定义的配置恢复回来
    g_loop_count=$[g_loop_count+1]
    if [ ${g_loop_count} -eq 120 ]; then
        g_loop_count=0
        g_restore_flag=1
    fi
    restore_ngxin_conf

    check_url
    url_rsp=$?
    if [ $url_rsp -eq 1 ]; then
        restart_nginx
    fi

    IPV4_ADDR=$(get_ip_addr 4)
    IPV6_ADDR=$(get_ip_addr 6)
    handle_ip "${IPV4_ADDR}" "${IPV6_ADDR}"
    ip_rsp=$?
    if [ $ip_rsp -eq 1 ]; then
        reload_nginx_web
    fi

    check_web "${IPV4_ADDR}" "${IPV6_ADDR}"
    web_rsp=$?
    if [ $web_rsp -eq 1 ]; then
        restart_nginx_web
    fi

    check_cert_if_8088_not_exist

    watch_inner_iplist
    inner_rsp=$?
    if [ ${inner_rsp} -eq 1 ];then
        reload_nginx_inner
    fi

    check_and_replace_dhparam_pem
    sleep 5
done
