Add original retries back

pkg/common-controller/snapshot_controller.go

@@ -337,10 +337,16 @@ func (ctrl *csiSnapshotCommonController) syncUnreadySnapshot(snapshot *crdv1.Vol
 		}
 
 		// update snapshot status
+		for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ {
 			klog.V(5).Infof("syncUnreadySnapshot [%s]: trying to update snapshot status", utils.SnapshotKey(snapshot))
 			_, err = ctrl.updateSnapshotStatus(snapshot, newContent)
-		if err != nil {
+			if err == nil {
+				break
+			}
 			klog.V(4).Infof("failed to update snapshot %s status: %v", utils.SnapshotKey(snapshot), err)
+		}
+
+		if err != nil {
 			// update snapshot status failed
 			ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotStatusUpdateFailed", fmt.Sprintf("Snapshot status update failed, %v", err))
 			return err
@@ -393,10 +399,16 @@ func (ctrl *csiSnapshotCommonController) syncUnreadySnapshot(snapshot *crdv1.Vol
 				}
 
 				// Update snapshot status with BoundVolumeSnapshotContentName
+				for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ {
 					klog.V(5).Infof("syncUnreadySnapshot [%s]: trying to update snapshot status", utils.SnapshotKey(snapshot))
 					_, err = ctrl.updateSnapshotStatus(snapshot, content)
-				if err != nil {
+					if err == nil {
+						break
+					}
 					klog.V(4).Infof("failed to update snapshot %s status: %v", utils.SnapshotKey(snapshot), err)
+				}
+
+				if err != nil {
 					// update snapshot status failed
 					ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotStatusUpdateFailed", fmt.Sprintf("Snapshot status update failed, %v", err))
 					return err
@@ -489,7 +501,8 @@ func (ctrl *csiSnapshotCommonController) createSnapshotContent(snapshot *crdv1.V
 
 	var updateContent *crdv1.VolumeSnapshotContent
 	klog.V(3).Infof("volume snapshot content %#v", snapshotContent)
-	// Try to create the VolumeSnapshotContent object
+	// Try to create the VolumeSnapshotContent object several times
+	for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ {
 		klog.V(5).Infof("createSnapshotContent [%s]: trying to save volume snapshot content %s", utils.SnapshotKey(snapshot), snapshotContent.Name)
 		if updateContent, err = ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Create(snapshotContent); err == nil || apierrs.IsAlreadyExists(err) {
 			// Save succeeded.
@@ -500,6 +513,11 @@ func (ctrl *csiSnapshotCommonController) createSnapshotContent(snapshot *crdv1.V
 			} else {
 				klog.V(3).Infof("volume snapshot content %q for snapshot %q saved, %v", snapshotContent.Name, utils.SnapshotKey(snapshot), snapshotContent)
 			}
+			break
+		}
+		// Save failed, try again after a while.
+		klog.V(3).Infof("failed to save volume snapshot content %q for snapshot %q: %v", snapshotContent.Name, utils.SnapshotKey(snapshot), err)
+		time.Sleep(ctrl.createSnapshotContentInterval)
 	}
 
 	if err != nil {
@@ -849,14 +867,17 @@ func (ctrl *csiSnapshotCommonController) bindandUpdateVolumeSnapshot(snapshotCon
 	snapshotCopy := snapshotObj.DeepCopy()
 
 	// update snapshot status
+	for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ {
 		klog.V(5).Infof("bindandUpdateVolumeSnapshot [%s]: trying to update snapshot status", utils.SnapshotKey(snapshotCopy))
 		updateSnapshot, err := ctrl.updateSnapshotStatus(snapshotCopy, snapshotContent)
 		if err == nil {
 			snapshotCopy = updateSnapshot
+			break
+		}
+		klog.V(4).Infof("failed to update snapshot %s status: %v", utils.SnapshotKey(snapshot), err)
 	}
 
 	if err != nil {
-		klog.V(4).Infof("failed to update snapshot %s status: %v", utils.SnapshotKey(snapshot), err)
 		// update snapshot status failed
 		ctrl.updateSnapshotErrorStatusWithEvent(snapshotCopy, v1.EventTypeWarning, "SnapshotStatusUpdateFailed", fmt.Sprintf("Snapshot status update failed, %v", err))
 		return nil, err

pkg/common-controller/snapshot_controller_base.go

@@ -42,6 +42,12 @@ import (
 	"k8s.io/kubernetes/pkg/util/goroutinemap"
 )
 
+// Number of retries when we create a VolumeSnapshotContent object
+const createSnapshotContentRetryCount = 5
+
+// Interval between retries when we create a VolumeSnapshotContent object
+const createSnapshotContentInterval = 10 * time.Second
+
 type csiSnapshotCommonController struct {
 	clientset     clientset.Interface
 	client        kubernetes.Interface
@@ -64,6 +70,8 @@ type csiSnapshotCommonController struct {
 	// Map of scheduled/running operations.
 	runningOperations goroutinemap.GoRoutineMap
 
+	createSnapshotContentRetryCount int
+	createSnapshotContentInterval   time.Duration
 	resyncPeriod                    time.Duration
 }
 
@@ -88,6 +96,8 @@ func NewCSISnapshotCommonController(
 		client:                          client,
 		eventRecorder:                   eventRecorder,
 		runningOperations:               goroutinemap.NewGoRoutineMap(true),
+		createSnapshotContentRetryCount: createSnapshotContentRetryCount,
+		createSnapshotContentInterval:   createSnapshotContentInterval,
 		resyncPeriod:                    resyncPeriod,
 		snapshotStore:                   cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc),
 		contentStore:                    cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc),