--- ./source/private_module/spc/spc_objmgr/spc_objmgr_objset_sync.c	2018-04-29 08:58:28.000000000 +0000
+++ ./source/private_module/spc/spc_objmgr/spc_objmgr_objset_sync.c.hotpatch	2020-04-26 12:53:39.000000000 +0000
@@ -4142,7 +4142,9 @@
     SPC_OBS_CFG_S           *obsCfg      = NULL;
     SPC_OBJSET_SYNCSTAT_S   *syncStat    = NULL;
     ROWLUN_FLUSH_DATA_MGR_S *flushMgr    = NULL;
-
+    uint64_t                *retryTimes  = NULL;
+    retryTimes                  = (uint64_t *)getContextPriv(context);
+    (*retryTimes)               = 0;
     privateData                 = (SPC_PREPARE_S *)getData(context);
     privateData->syncingObjset  = traverseSpcObjset(privateData->syncingObjset, GET_MOS_MGR(privateData->mosObjset));
     if (NULL == privateData->syncingObjset)
@@ -4211,7 +4213,6 @@
     /* ֵrowlunݹṹеisFlushingΪturespcģԼspcԼˢ */
     /* cacheˢ̡Ŀǰֻеһcacheˢ´rowlunĻԪ */
     flushMgr = getRowlunFlushMgr(objset);
-  
     if (B_TRUE == flushMgr->isFlushing)
     {
         spaceActiveRowlunDestage(objset,  
@@ -4225,7 +4226,6 @@
         SPACE_ACTIVE_CACHE_DESTAGE(privateData->syncingObjset->obsCacheObj, 
                                    privateData->input->tno, callbackObsyncDestage, context);
     }
-        
     return CONTEXT_PENDING;
 }
 
@@ -4240,17 +4240,16 @@
 {
     SPC_PREPARE_S           *privateData = NULL;
     SPC_OBJSET_SYNCSTAT_S   *syncStat    = NULL;
-
+    uint64_t                *retryTimes  = NULL;
     privateData             = (SPC_PREPARE_S *)getData(context);
     privateData->latency    = GET_MILLI_SECOND - privateData->latency;
-
+    retryTimes              = (uint64_t *)getContextPriv(context);
     setSpcStageExit(g_spcSyncDbgStgCnt, SPC_SYNC_DBG_STAGE_PREPARESYNC, SPC_PREPARE_OBS);
     ftdsSpcStatEnd(context, FTDS_OP_NONE, SPC_FTDS_SYNC_OBS, privateData->result);    
     VOLS_COUNTER_BACK_INC(VOL_COUNTER_OBJMGR_SYNC_DESTAGE_OBJSET, VOLS_DEBUG_SPC_OBJMGR,
                           privateData->result);
     VOLS_COUNTER_LATENCY_INC(VOL_COUNTER_OBJMGR_SYNC_DESTAGE_OBJSET, VOLS_DEBUG_SPC_OBJMGR,
                              privateData->latency);
-
     if (unlikely(SPC_DELAY_WARNING < privateData->latency))
     {
         SPC_LOG_INFO(SPC_PRINT_TIMEOUT, DBG_LOG_WARNING,
@@ -4259,7 +4258,6 @@
                     privateData->latency, privateData->syncingObjset->obsId,
                     privateData->input->tno, privateData->result);
     }
-
     if (!IS_OBJSET_DIRTY(privateData->syncingObjset, privateData->input->tno, SPC_OBJSET_STAGE_CACHE))
     {
         if (likely(RETURN_OK == privateData->result))
@@ -4277,14 +4275,11 @@
             SPC_LOG_ERR(SPC_PRINT_SYNC_WRITE, DBG_LOG_ERROR,
                         "SYNC: Sync objset(obsid 0x%llx, timepoint %llu) error(code %d).",
                          privateData->syncingObjset->obsId, privateData->input->tno, privateData->result);
-
             gotoStep(context, SPC_PREPARE_DONE);
             return CONTEXT_CONTINUE;
         }
     }
-
     syncStat = GET_OBS_SYNCSTAT(privateData->syncingObjset, SPC_OBJSET_STAGE_CACHE);
-
     /* 
      * жcacheģdestage̷صִн, ִнRETURN_OKʱ
      * 1)ǰʱеĶѾ·, ȴ
@@ -4298,19 +4293,24 @@
                 "SYNC: Destage obs(obsid 0x%llx, txgNo %llu, dirtySum %d, dirtyCnt %d) data failed(ret %d).",
                 privateData->syncingObjset->obsId, privateData->input->tno,
                 atomic_read(&syncStat->dirtySum), atomic_read(&syncStat->dirtyCnt), privateData->result);
-
+            if (3 > (*retryTimes)){
+                SPC_LOG_ERR(SPC_PRINT_SYNC_WRITE, DBG_LOG_ERROR,
+                "SYNC:Destage obs retry(obsid 0x%llx,txgNo %llu,dirtySum %d,dirtyCnt %d) data failed(ret %d),retry times(%llu).",
+                privateData->syncingObjset->obsId, privateData->input->tno, atomic_read(&syncStat->dirtySum), 
+                atomic_read(&syncStat->dirtyCnt), privateData->result, *retryTimes);
+                (*retryTimes)++;
+                gotoStep(context, SPC_PREPARE_OBS_WAIT);
+                return CONTEXT_CONTINUE_SLOW;
+            }
             gotoStep(context, SPC_PREPARE_DONE);
             return CONTEXT_CONTINUE;
         }
     }
-    
     setSpcStageEnter(g_spcSyncDbgStgCnt, SPC_SYNC_DBG_STAGE_PREPARESYNC, SPC_PREPARE_OBS_WAIT);
     VOLS_COUNTER_START_INC(VOL_COUNTER_OBJMGR_SYNC_WAIT_DESTAGE_OBJSET, VOLS_DEBUG_SPC_OBJMGR);
     privateData->latency = GET_MILLI_SECOND;
-    
     /* ȴ󼯵Ԫˢ */
     asyncWaitForCompletion(&syncStat->comp, context, cbprepareSyncObsChk);
-
     return CONTEXT_PENDING;
 }
 
@@ -4841,7 +4841,9 @@
 {
     SPC_PREPARE_S          *privateData  = NULL;
     ROWLUN_FLUSH_DATA_MGR_S *flushMgr    = NULL;
-    
+    uint64_t                *retryTimes  = NULL;
+    retryTimes                  = (uint64_t *)getContextPriv(context);
+    (*retryTimes)               = 0;
     privateData          = (SPC_PREPARE_S *)getData(context);
     privateData->latency = GET_MILLI_SECOND;
 
@@ -4864,7 +4866,6 @@
     /* ֵrowlunݹṹеisFlushingΪturespcģԼspcԼˢ */
     /* cacheˢ̡Ŀǰֻеһcacheˢ´rowlunĻԪ */
     flushMgr = getRowlunFlushMgr(privateData->mosObjset);
-    
     if (B_TRUE == flushMgr->isFlushing)
     {
         spaceActiveRowlunDestage(privateData->mosObjset,  
@@ -4878,7 +4879,6 @@
         SPACE_ACTIVE_CACHE_DESTAGE(privateData->mosObjset->obsCacheObj, 
                                    privateData->input->tno, callbackObsyncDestage, context);
     }
-    
     return CONTEXT_PENDING;
 }
 
@@ -4893,16 +4893,15 @@
 {
     SPC_PREPARE_S           *privateData = NULL;
     SPC_OBJSET_SYNCSTAT_S   *syncStat    = NULL;
-
+    uint64_t                *retryTimes  = NULL;
     privateData             = (SPC_PREPARE_S *)getData(context);
     privateData->latency    = GET_MILLI_SECOND - privateData->latency;
-
+    retryTimes              = (uint64_t *)getContextPriv(context);
     setSpcStageExit(g_spcSyncDbgStgCnt, SPC_SYNC_DBG_STAGE_PREPARESYNC, SPC_PREPARE_MOS);
     VOLS_COUNTER_BACK_INC(VOL_COUNTER_OBJMGR_SYNC_DESTAGE_OBJSET, VOLS_DEBUG_SPC_OBJMGR, privateData->result);
     VOLS_COUNTER_LATENCY_INC(VOL_COUNTER_OBJMGR_SYNC_DESTAGE_OBJSET, VOLS_DEBUG_SPC_OBJMGR,
                              privateData->latency);
     ftdsSpcStatEnd(context, FTDS_OP_NONE, SPC_FTDS_SYNC_PREPARE_MOS, privateData->result);
-
     if (unlikely(SPC_DELAY_WARNING < privateData->latency))
     {
         SPC_LOG_INFO(SPC_PRINT_TIMEOUT, DBG_LOG_WARNING,
@@ -4911,7 +4910,6 @@
                     privateData->latency, privateData->mosObjset->obsId,
                     privateData->input->tno, privateData->result);
     }
-
     if (!IS_OBJSET_DIRTY(privateData->mosObjset, privateData->input->tno, SPC_OBJSET_STAGE_CACHE))
     {
         if (likely(RETURN_OK == privateData->result))
@@ -4934,12 +4932,10 @@
             return CONTEXT_CONTINUE;
         }
     }
-
     syncStat = GET_OBS_SYNCSTAT(privateData->mosObjset, SPC_OBJSET_STAGE_CACHE);
 
     LVOS_TP_START(PREPARE_SPC_TRANS_2_302_9, syncStat, &privateData->result)
     LVOS_TP_END
-    
     /* 
      * жcacheģdestage̷صִн, ִнRETURN_OKʱ
      * 1)ǰʱеĶѾ·, ȴ
@@ -4953,21 +4949,25 @@
                 "SYNC: Destage obs(obsid 0x%llx, txgNo %llu, dirtySum %d, dirtyCnt %d) data failed(ret %d). syncingObjset addr is [%p].",
                 privateData->mosObjset->obsId, privateData->input->tno,
                 atomic_read(&syncStat->dirtySum), atomic_read(&syncStat->dirtyCnt), privateData->result, privateData->syncingObjset);
-
+            if (3 > (*retryTimes)){
+                SPC_LOG_ERR(SPC_PRINT_SYNC_WRITE, DBG_LOG_ERROR,
+                "SYNC:Destage obs retry(obsid 0x%llx,txgNo %llu,dirtySum %d,dirtyCnt %d) data failed(ret %d),retry times(%llu).",
+                privateData->syncingObjset->obsId, privateData->input->tno, atomic_read(&syncStat->dirtySum), 
+                atomic_read(&syncStat->dirtyCnt), privateData->result, *retryTimes);
+                (*retryTimes)++;
+                gotoStep(context, SPC_PREPARE_OBS_WAIT);
+                return CONTEXT_CONTINUE_SLOW;
+            }
             privateData->result = RETURN_SPACE_ENOMEM;
-            
             gotoStep(context, SPC_PREPARE_DONE);
             return CONTEXT_CONTINUE;
         }
     }
-    
     setSpcStageEnter(g_spcSyncDbgStgCnt, SPC_SYNC_DBG_STAGE_PREPARESYNC, SPC_PREPARE_MOS_WAIT);
     VOLS_COUNTER_START_INC(VOL_COUNTER_OBJMGR_SYNC_WAIT_DESTAGE_OBJSET, VOLS_DEBUG_SPC_OBJMGR);
     privateData->latency = GET_MILLI_SECOND;
-
     /* ȴ󼯵Ԫˢ */
     asyncWaitForCompletion(&syncStat->comp, context, cbprepareSpcMosChk);
-
     return CONTEXT_PENDING;
 }
 
