alexlebens/external-snapshotter
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add DeleteVolumeGroupSnapshot API to delete group snapshot

Browse Source
This commit is contained in:
Raunak Pradip Shah
2023-07-12 11:37:13 +05:30
parent 49193ef7e7
commit facefbad6e
11 changed files with 647 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
cmd/snapshot-controller/main.go
View File

@@ -81,7 +81,7 @@ var (
var version = "unknown"
// Checks that the VolumeSnapshot v1 CRDs exist.
func ensureCustomResourceDefinitionsExist(client *clientset.Clientset) error {
func ensureCustomResourceDefinitionsExist(client *clientset.Clientset, enableVolumeGroupSnapshots bool) error {
condition := func() (bool, error) {
var err error
@@ -102,6 +102,25 @@ func ensureCustomResourceDefinitionsExist(client *clientset.Clientset) error {
klog.Errorf("Failed to list v1 volumesnapshotcontents with error=%+v", err)
return false, nil
}
if enableVolumeGroupSnapshots {
_, err = client.GroupsnapshotV1alpha1().VolumeGroupSnapshots("").List(context.TODO(), metav1.ListOptions{})
if err != nil {
klog.Errorf("Failed to list v1alpha1 volumegroupsnapshots with error=%+v", err)
return false, nil
}
_, err = client.GroupsnapshotV1alpha1().VolumeGroupSnapshotClasses().List(context.TODO(), metav1.ListOptions{})
if err != nil {
klog.Errorf("Failed to list v1alpha1 volumegroupsnapshotclasses with error=%+v", err)
return false, nil
}
_, err = client.GroupsnapshotV1alpha1().VolumeGroupSnapshotContents().List(context.TODO(), metav1.ListOptions{})
if err != nil {
klog.Errorf("Failed to list v1alpha1 volumegroupsnapshotcontents with error=%+v", err)
return false, nil
}
}
return true, nil
}
@@ -214,7 +233,7 @@ func main() {
*enableVolumeGroupSnapshots,
)
if err := ensureCustomResourceDefinitionsExist(snapClient); err != nil {
if err := ensureCustomResourceDefinitionsExist(snapClient, *enableVolumeGroupSnapshots); err != nil {
klog.Errorf("Exiting due to failure to ensure CRDs exist during startup: %+v", err)
os.Exit(1)
}

12
deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml
View File

@@ -36,11 +36,19 @@ rules:
verbs: ["get", "list", "watch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["get", "list", "watch", "update", "patch"]
verbs: ["get", "list", "watch", "update", "patch", "create"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents/status"]
verbs: ["update", "patch"]
- apiGroups: ["groupsnapshot.storage.k8s.io"]
resources: ["volumegroupsnapshotclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["groupsnapshot.storage.k8s.io"]
resources: ["volumegroupsnapshotcontents"]
verbs: ["get", "list", "watch", "update", "patch"]
- apiGroups: ["groupsnapshot.storage.k8s.io"]
resources: ["volumegroupsnapshotcontents/status"]
verbs: ["update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1

17
deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml
View File

@@ -42,6 +42,23 @@ rules:
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots/status"]
verbs: ["update", "patch"]
- apiGroups: ["groupsnapshot.storage.k8s.io"]
resources: ["volumegroupsnapshotclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["groupsnapshot.storage.k8s.io"]
resources: ["volumegroupsnapshotcontents"]
verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
- apiGroups: ["groupsnapshot.storage.k8s.io"]
resources: ["volumegroupsnapshotcontents/status"]
verbs: ["patch"]
- apiGroups: ["groupsnapshot.storage.k8s.io"]
resources: ["volumegroupsnapshots"]
verbs: ["get", "list", "watch", "update", "patch"]
- apiGroups: ["groupsnapshot.storage.k8s.io"]
resources: ["volumegroupsnapshots/status"]
verbs: ["update", "patch"]
# Enable this RBAC rule only when using distributed snapshotting, i.e. when the enable-distributed-snapshotting flag is set to true
# - apiGroups: [""]
# resources: ["nodes"]

329
pkg/common-controller/groupsnapshot_controller_helper.go
View File

@@ -289,13 +289,26 @@ func (ctrl *csiSnapshotCommonController) syncGroupSnapshot(groupSnapshot *crdv1a
klog.V(5).Infof("syncGroupSnapshot [%s]: check if we should remove finalizer on group snapshot PVC source and remove it if we can", utils.GroupSnapshotKey(groupSnapshot))
/*
TODO:
- Check and remove finalizer if needed.
- Check and set invalid group snapshot label, if needed.
- Process if deletion timestamp is set.
- Check and add group snapshot finalizers.
*/
// Proceed with group snapshot deletion and remove finalizers when needed
if groupSnapshot.ObjectMeta.DeletionTimestamp != nil {
return ctrl.processGroupSnapshotWithDeletionTimestamp(groupSnapshot)
}
// Keep this check in the controller since the validation webhook may not have been deployed.
klog.V(5).Infof("syncGroupSnapshot[%s]: validate group snapshot to make sure source has been correctly specified", utils.GroupSnapshotKey(groupSnapshot))
if (&groupSnapshot.Spec.Source.Selector == nil && groupSnapshot.Spec.Source.VolumeGroupSnapshotContentName == nil) ||
(&groupSnapshot.Spec.Source.Selector != nil && groupSnapshot.Spec.Source.VolumeGroupSnapshotContentName != nil) {
err := fmt.Errorf("Exactly one of Selector and VolumeGroupSnapshotContentName should be specified")
klog.Errorf("syncGroupSnapshot[%s]: validation error, %s", utils.GroupSnapshotKey(groupSnapshot), err.Error())
ctrl.updateGroupSnapshotErrorStatusWithEvent(groupSnapshot, true, v1.EventTypeWarning, "GroupSnapshotValidationError", err.Error())
return err
}
klog.V(5).Infof("syncGroupSnapshot: check if we should add finalizers on group snapshot [%s]", utils.GroupSnapshotKey(groupSnapshot))
if err := ctrl.checkandAddGroupSnapshotFinalizers(groupSnapshot); err != nil {
klog.Errorf("error checkandAddGroupSnapshotFinalizers for group snapshot [%s]: %v", utils.GroupSnapshotKey(groupSnapshot), err)
ctrl.eventRecorder.Event(groupSnapshot, v1.EventTypeWarning, "GroupSnapshotFinalizerError", fmt.Sprintf("Failed to check and update group snapshot: %s", err.Error()))
return err
}
// Need to build or update groupSnapshot.Status in following cases:
// 1) groupSnapshot.Status is nil
@@ -724,10 +737,10 @@ func (ctrl *csiSnapshotCommonController) bindandUpdateVolumeGroupSnapshot(groupS
// createGroupSnapshotContent will only be called for dynamic provisioning
func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) (*crdv1alpha1.VolumeGroupSnapshotContent, error) {
klog.Infof("createSnapshotContent: Creating group snapshot content for groupn snapshot %s through the plugin ...", utils.GroupSnapshotKey(groupSnapshot))
klog.Infof("createSnapshotContent: Creating group snapshot content for group snapshot %s through the plugin ...", utils.GroupSnapshotKey(groupSnapshot))
/*
TODO: Add finalizer to group snapshot
TODO: Add PVC finalizer
*/
groupSnapshotClass, volumes, contentName, err := ctrl.getCreateGroupSnapshotInput(groupSnapshot)
@@ -763,16 +776,16 @@ func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapsho
TODO: Add secret reference details
*/
var updateGroupSnapshoyContent *crdv1alpha1.VolumeGroupSnapshotContent
var updateGroupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent
klog.V(5).Infof("volume group snapshot content %#v", groupSnapshotContent)
// Try to create the VolumeGroupSnapshotContent object
klog.V(5).Infof("createGroupSnapshotContent [%s]: trying to save volume group snapshot content %s", utils.GroupSnapshotKey(groupSnapshot), groupSnapshotContent.Name)
if updateGroupSnapshoyContent, err = ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshotContents().Create(context.TODO(), groupSnapshotContent, metav1.CreateOptions{}); err == nil || apierrs.IsAlreadyExists(err) {
if updateGroupSnapshotContent, err = ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshotContents().Create(context.TODO(), groupSnapshotContent, metav1.CreateOptions{}); err == nil || apierrs.IsAlreadyExists(err) {
// Save succeeded.
if err != nil {
klog.V(3).Infof("volume group snapshot content %q for group snapshot %q already exists, reusing", groupSnapshotContent.Name, utils.GroupSnapshotKey(groupSnapshot))
err = nil
updateGroupSnapshoyContent = groupSnapshotContent
updateGroupSnapshotContent = groupSnapshotContent
} else {
klog.V(3).Infof("volume group snapshot content %q for group snapshot %q saved, %v", groupSnapshotContent.Name, utils.GroupSnapshotKey(groupSnapshot), groupSnapshotContent)
}
@@ -789,12 +802,12 @@ func (ctrl *csiSnapshotCommonController) createGroupSnapshotContent(groupSnapsho
ctrl.eventRecorder.Event(groupSnapshot, v1.EventTypeNormal, "CreatingGroupSnapshot", msg)
// Update group snapshot content in the cache store
_, err = ctrl.storeGroupSnapshotContentUpdate(updateGroupSnapshoyContent)
_, err = ctrl.storeGroupSnapshotContentUpdate(updateGroupSnapshotContent)
if err != nil {
klog.Errorf("failed to update group snapshot content store %v", err)
}
return updateGroupSnapshoyContent, nil
return updateGroupSnapshotContent, nil
}
func (ctrl *csiSnapshotCommonController) getCreateGroupSnapshotInput(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) (*crdv1alpha1.VolumeGroupSnapshotClass, []*v1.PersistentVolume, string, error) {
@@ -849,9 +862,11 @@ func (ctrl *csiSnapshotCommonController) syncGroupSnapshotContent(groupSnapshotC
return nil
}
/*
TODO: Add finalizer to prevent deletion
*/
if utils.NeedToAddGroupSnapshotContentFinalizer(groupSnapshotContent) {
// Group Snapshot Content is not being deleted -> it should have the finalizer.
klog.V(5).Infof("syncGroupSnapshotContent [%s]: Add Finalizer for VolumeGroupSnapshotContent", groupSnapshotContent.Name)
return ctrl.addGroupSnapshotContentFinalizer(groupSnapshotContent)
}
// Check if group snapshot exists in cache store
// If getGroupSnapshotFromStore returns (nil, nil), it means group snapshot not found
@@ -935,3 +950,283 @@ func (ctrl *csiSnapshotCommonController) needsUpdateGroupSnapshotStatus(groupSna
return false
}
// addGroupSnapshotContentFinalizer adds a Finalizer for VolumeGroupSnapshotContent.
func (ctrl *csiSnapshotCommonController) addGroupSnapshotContentFinalizer(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) error {
var patches []utils.PatchOp
if len(groupSnapshotContent.Finalizers) > 0 {
// Add to the end of the finalizers if we have any other finalizers
patches = append(patches, utils.PatchOp{
Op: "add",
Path: "/metadata/finalizers/-",
Value: utils.VolumeGroupSnapshotContentFinalizer,
})
} else {
// Replace finalizers with new array if there are no other finalizers
patches = append(patches, utils.PatchOp{
Op: "add",
Path: "/metadata/finalizers",
Value: []string{utils.VolumeGroupSnapshotContentFinalizer},
})
}
newGroupSnapshotContent, err := utils.PatchVolumeGroupSnapshotContent(groupSnapshotContent, patches, ctrl.clientset)
if err != nil {
return newControllerUpdateError(groupSnapshotContent.Name, err.Error())
}
_, err = ctrl.storeGroupSnapshotContentUpdate(newGroupSnapshotContent)
if err != nil {
klog.Errorf("failed to update group snapshot content store %v", err)
}
klog.V(5).Infof("Added protection finalizer to volume group snapshot content %s", newGroupSnapshotContent.Name)
return nil
}
// checkandAddGroupSnapshotFinalizers checks and adds group snapshot finailzers when needed
func (ctrl *csiSnapshotCommonController) checkandAddGroupSnapshotFinalizers(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) error {
// get the group snapshot content for this group snapshot
var (
groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent
err error
)
if groupSnapshot.Spec.Source.VolumeGroupSnapshotContentName != nil {
groupSnapshotContent, err = ctrl.getPreprovisionedGroupSnapshotContentFromStore(groupSnapshot)
} else {
groupSnapshotContent, err = ctrl.getDynamicallyProvisionedGroupContentFromStore(groupSnapshot)
}
if err != nil {
return err
}
// A bound finalizer is needed ONLY when all following conditions are satisfied:
// 1. the VolumeGroupSnapshot is bound to a VolumeGroupSnapshotContent
// 2. the VolumeGroupSnapshot does not have deletion timestamp set
// 3. the matching VolumeGroupSnapshotContent has a deletion policy to be Delete
// Note that if a matching VolumeGroupSnapshotContent is found, it must point back to the VolumeGroupSnapshot
if groupSnapshotContent != nil && utils.NeedToAddGroupSnapshotBoundFinalizer(groupSnapshot) && (groupSnapshotContent.Spec.DeletionPolicy == crdv1.VolumeSnapshotContentDelete) {
// Snapshot is not being deleted -> it should have the finalizer.
klog.V(5).Infof("checkandAddGroupSnapshotFinalizers: Add Finalizer for VolumeGroupSnapshot[%s]", utils.GroupSnapshotKey(groupSnapshot))
return ctrl.addGroupSnapshotFinalizer(groupSnapshot, true)
}
return nil
}
// addGroupSnapshotFinalizer adds a Finalizer to a VolumeGroupSnapshot.
func (ctrl *csiSnapshotCommonController) addGroupSnapshotFinalizer(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot, addBoundFinalizer bool) error {
var updatedGroupSnapshot *crdv1alpha1.VolumeGroupSnapshot
var err error
// NOTE(ggriffiths): Must perform an update if no finalizers exist.
// Unable to find a patch that correctly updated the finalizers if none currently exist.
if len(groupSnapshot.ObjectMeta.Finalizers) == 0 {
groupSnapshotClone := groupSnapshot.DeepCopy()
if addBoundFinalizer {
groupSnapshotClone.ObjectMeta.Finalizers = append(groupSnapshotClone.ObjectMeta.Finalizers, utils.VolumeGroupSnapshotBoundFinalizer)
}
updatedGroupSnapshot, err = ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshots(groupSnapshotClone.Namespace).Update(context.TODO(), groupSnapshotClone, metav1.UpdateOptions{})
if err != nil {
return newControllerUpdateError(utils.GroupSnapshotKey(groupSnapshot), err.Error())
}
} else {
// Otherwise, perform a patch
var patches []utils.PatchOp
if addBoundFinalizer {
patches = append(patches, utils.PatchOp{
Op: "add",
Path: "/metadata/finalizers/-",
Value: utils.VolumeGroupSnapshotBoundFinalizer,
})
}
updatedGroupSnapshot, err = utils.PatchVolumeGroupSnapshot(groupSnapshot, patches, ctrl.clientset)
if err != nil {
return newControllerUpdateError(utils.GroupSnapshotKey(groupSnapshot), err.Error())
}
}
_, err = ctrl.storeGroupSnapshotUpdate(updatedGroupSnapshot)
if err != nil {
klog.Errorf("failed to update group snapshot store %v", err)
}
klog.V(5).Infof("Added protection finalizer to volume group snapshot %s", utils.GroupSnapshotKey(updatedGroupSnapshot))
return nil
}
// processGroupSnapshotWithDeletionTimestamp processes finalizers and deletes the
// group snapshot content when appropriate. It has the following steps:
// 1. Get the VolumeGroupSnapshotContent which the to-be-deleted VolumeGroupSnapshot
// points to and verifies bi-directional binding.
// 2. Call checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent()
// with information obtained from step 1. This function name is very long but the
// name suggests what it does. It determines whether to remove finalizers on group
// snapshot and whether to delete group snapshot content.
func (ctrl *csiSnapshotCommonController) processGroupSnapshotWithDeletionTimestamp(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) error {
klog.V(5).Infof("processGroupSnapshotWithDeletionTimestamp VolumeGroupSnapshot[%s]: %s", utils.GroupSnapshotKey(groupSnapshot), utils.GetGroupSnapshotStatusForLogging(groupSnapshot))
var groupSnapshotContentName string
if groupSnapshot.Status != nil && groupSnapshot.Status.BoundVolumeGroupSnapshotContentName != nil {
groupSnapshotContentName = *groupSnapshot.Status.BoundVolumeGroupSnapshotContentName
}
// for a dynamically created group snapshot, it's possible that a group snapshot
// content has been created however the Status of the group snapshot has not
// been updated yet, i.e., failed right after group snapshot content creation.
// In this case, use the fixed naming scheme to get the group snapshot content
// name and search
if groupSnapshotContentName == "" && &groupSnapshot.Spec.Source.VolumeGroupSnapshotContentName == nil {
groupSnapshotContentName = utils.GetDynamicSnapshotContentNameForGroupSnapshot(groupSnapshot)
}
// find a group snapshot content from cache store, note that it's completely legit
// that no group snapshot content has been found from group snapshot content
// cache store
groupSnapshotContent, err := ctrl.getGroupSnapshotContentFromStore(groupSnapshotContentName)
if err != nil {
return err
}
// check whether the group snapshot content points back to the passed in group
// snapshot, note that binding should always be bi-directional to trigger the
// deletion on group snapshot content or adding any annotation to the group
// snapshot content
var deleteGroupSnapshotContent bool
if groupSnapshotContent != nil && utils.IsVolumeGroupSnapshotRefSet(groupSnapshot, groupSnapshotContent) {
// group snapshot content points back to group snapshot, whether or not
// to delete a group snapshot content now depends on the deletion policy
// of it.
deleteGroupSnapshotContent = (groupSnapshotContent.Spec.DeletionPolicy == crdv1.VolumeSnapshotContentDelete)
} else {
// the group snapshot content is nil or points to a different group snapshot, reset group snapshot content to nil
// such that there is no operation done on the found group snapshot content in
// checkandRemoveSnapshotFinalizersAndCheckandDeleteContent
groupSnapshotContent = nil
}
klog.V(5).Infof("processGroupSnapshotWithDeletionTimestamp[%s]: delete group snapshot content and remove finalizer from group snapshot if needed", utils.GroupSnapshotKey(groupSnapshot))
return ctrl.checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent(groupSnapshot, groupSnapshotContent, deleteGroupSnapshotContent)
}
// checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent deletes
// the group snapshot content and removes group snapshot finalizers if needed
func (ctrl *csiSnapshotCommonController) checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot, groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent, deleteGroupSnapshotContent bool) error {
klog.V(5).Infof("checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent VolumeGroupSnapshot[%s]: %s", utils.GroupSnapshotKey(groupSnapshot), utils.GetGroupSnapshotStatusForLogging(groupSnapshot))
if !utils.IsGroupSnapshotDeletionCandidate(groupSnapshot) {
return nil
}
// check if an individual snapshot belonging to the group snapshot is being
// used for restore a PVC
// If yes, do nothing and wait until PVC restoration finishes
for _, snapshotRef := range groupSnapshot.Status.VolumeSnapshotRefList {
snapshot, err := ctrl.snapshotLister.VolumeSnapshots(snapshotRef.Namespace).Get(snapshotRef.Name)
if err != nil {
return err
}
if ctrl.isVolumeBeingCreatedFromSnapshot(snapshot) {
msg := fmt.Sprintf("Snapshot %s belonging to VolumeGroupSnapshot %s is being used to restore a PVC", utils.SnapshotKey(snapshot), utils.GroupSnapshotKey(groupSnapshot))
klog.V(4).Info(msg)
ctrl.eventRecorder.Event(groupSnapshot, v1.EventTypeWarning, "SnapshotDeletePending", msg)
// TODO(@xiangqian): should requeue this?
return nil
}
}
// regardless of the deletion policy, set the VolumeSnapshotBeingDeleted on
// content object, this is to allow snapshotter sidecar controller to conduct
// a delete operation whenever the content has deletion timestamp set.
if groupSnapshotContent != nil {
klog.V(5).Infof("checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent[%s]: Set VolumeGroupSnapshotBeingDeleted annotation on the group snapshot content [%s]", utils.GroupSnapshotKey(groupSnapshot), groupSnapshotContent.Name)
updatedGroupSnapshotContent, err := ctrl.setAnnVolumeGroupSnapshotBeingDeleted(groupSnapshotContent)
if err != nil {
klog.V(4).Infof("checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent[%s]: failed to set VolumeGroupSnapshotBeingDeleted annotation on the group snapshot content [%s]", utils.GroupSnapshotKey(groupSnapshot), groupSnapshotContent.Name)
return err
}
groupSnapshotContent = updatedGroupSnapshotContent
}
// VolumeGroupSnapshot should be deleted. Check and remove finalizers
// If group snapshot content exists and has a deletion policy of Delete, set
// DeletionTimeStamp on the content;
// VolumeGroupSnapshotContent won't be deleted immediately due to the VolumeGroupSnapshotContentFinalizer
if groupSnapshotContent != nil && deleteGroupSnapshotContent {
klog.V(5).Infof("checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent: set DeletionTimeStamp on group snapshot content [%s].", groupSnapshotContent.Name)
err := ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshotContents().Delete(context.TODO(), groupSnapshotContent.Name, metav1.DeleteOptions{})
if err != nil {
ctrl.eventRecorder.Event(groupSnapshot, v1.EventTypeWarning, "GroupSnapshotContentObjectDeleteError", "Failed to delete group snapshot content API object")
return fmt.Errorf("failed to delete VolumeGroupSnapshotContent %s from API server: %q", groupSnapshotContent.Name, err)
}
}
klog.V(5).Infof("checkandRemoveGroupSnapshotFinalizersAndCheckandDeleteGroupSnapshotContent: Remove Finalizer for VolumeGroupSnapshot[%s]", utils.GroupSnapshotKey(groupSnapshot))
// remove VolumeSnapshotBoundFinalizer on the VolumeGroupSnapshot object:
// a. If there is no group snapshot content found, remove the finalizer.
// b. If the group snapshot content is being deleted, i.e., with deleteGroupSnapshotContent == true,
// keep this finalizer until the group snapshot content object is removed
// from API server by group snapshot sidecar controller.
// c. If deletion will not cascade to the group snapshot content, remove
// the finalizer on the group snapshot such that it can be removed from
// the API server.
removeBoundFinalizer := !(groupSnapshotContent != nil && deleteGroupSnapshotContent)
return ctrl.removeGroupSnapshotFinalizer(groupSnapshot, removeBoundFinalizer)
}
func (ctrl *csiSnapshotCommonController) setAnnVolumeGroupSnapshotBeingDeleted(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) (*crdv1alpha1.VolumeGroupSnapshotContent, error) {
if groupSnapshotContent == nil {
return groupSnapshotContent, nil
}
// Set AnnVolumeGroupSnapshotBeingDeleted if it is not set yet
if !metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnVolumeGroupSnapshotBeingDeleted) {
klog.V(5).Infof("setAnnVolumeGroupSnapshotBeingDeleted: set annotation [%s] on group snapshot content [%s].", utils.AnnVolumeGroupSnapshotBeingDeleted, groupSnapshotContent.Name)
var patches []utils.PatchOp
metav1.SetMetaDataAnnotation(&groupSnapshotContent.ObjectMeta, utils.AnnVolumeGroupSnapshotBeingDeleted, "yes")
patches = append(patches, utils.PatchOp{
Op: "replace",
Path: "/metadata/annotations",
Value: groupSnapshotContent.ObjectMeta.GetAnnotations(),
})
patchedGroupSnapshotContent, err := utils.PatchVolumeGroupSnapshotContent(groupSnapshotContent, patches, ctrl.clientset)
if err != nil {
return groupSnapshotContent, newControllerUpdateError(groupSnapshotContent.Name, err.Error())
}
// update group snapshot content if update is successful
groupSnapshotContent = patchedGroupSnapshotContent
_, err = ctrl.storeGroupSnapshotContentUpdate(groupSnapshotContent)
if err != nil {
klog.V(4).Infof("setAnnVolumeGroupSnapshotBeingDeleted for group snapshot content [%s]: cannot update internal cache %v", groupSnapshotContent.Name, err)
return groupSnapshotContent, err
}
klog.V(5).Infof("setAnnVolumeGroupSnapshotBeingDeleted: volume group snapshot content %+v", groupSnapshotContent)
}
return groupSnapshotContent, nil
}
// removeGroupSnapshotFinalizer removes a Finalizer for VolumeGroupSnapshot.
func (ctrl *csiSnapshotCommonController) removeGroupSnapshotFinalizer(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot, removeBoundFinalizer bool) error {
if !removeBoundFinalizer {
return nil
}
// TODO: Remove PVC Finalizer
groupSnapshotClone := groupSnapshot.DeepCopy()
groupSnapshotClone.ObjectMeta.Finalizers = utils.RemoveString(groupSnapshotClone.ObjectMeta.Finalizers, utils.VolumeGroupSnapshotBoundFinalizer)
newGroupSnapshot, err := ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshots(groupSnapshotClone.Namespace).Update(context.TODO(), groupSnapshotClone, metav1.UpdateOptions{})
if err != nil {
return newControllerUpdateError(groupSnapshot.Name, err.Error())
}
_, err = ctrl.storeGroupSnapshotUpdate(newGroupSnapshot)
if err != nil {
klog.Errorf("failed to update group snapshot store %v", err)
}
klog.V(5).Infof("Removed protection finalizer from volume group snapshot %s", utils.GroupSnapshotKey(groupSnapshot))
return nil
}

1
pkg/common-controller/snapshot_controller.go
View File

@@ -1165,7 +1165,6 @@ func (ctrl *csiSnapshotCommonController) updateSnapshotStatus(snapshot *crdv1.Vo
updated = true
} else {
newStatus = snapshotObj.Status.DeepCopy()
klog.Infof("Raunak 1 %s", newStatus.VolumeGroupSnapshotName)
if newStatus.BoundVolumeSnapshotContentName == nil {
newStatus.BoundVolumeSnapshotContentName = &boundContentName
updated = true

2
pkg/common-controller/snapshot_controller_base.go
View File

@@ -836,7 +836,7 @@ func (ctrl *csiSnapshotCommonController) updateGroupSnapshotContent(content *crd
// deleteGroupSnapshotContent runs in worker thread and handles "groupsnapshotcontent deleted" event.
func (ctrl *csiSnapshotCommonController) deleteGroupSnapshotContent(content *crdv1alpha1.VolumeGroupSnapshotContent) {
_ = ctrl.contentStore.Delete(content)
_ = ctrl.groupSnapshotContentStore.Delete(content)
klog.V(4).Infof("group snapshot content %q deleted", content.Name)
groupSnapshotName := utils.GroupSnapshotRefKey(&content.Spec.VolumeGroupSnapshotRef)

21
pkg/group_snapshotter/group_snapshotter.go
View File

@@ -18,11 +18,12 @@ package group_snapshotter
import (
"context"
"time"
"github.com/container-storage-interface/spec/lib/go/csi"
csirpc "github.com/kubernetes-csi/csi-lib-utils/rpc"
"google.golang.org/grpc"
klog "k8s.io/klog/v2"
"time"
)
// GroupSnapshotter implements CreateGroupSnapshot/DeleteGroupSnapshot operations against a CSI driver.
@@ -31,7 +32,7 @@ type GroupSnapshotter interface {
CreateGroupSnapshot(ctx context.Context, groupSnapshotName string, volumeIDs []string, parameters map[string]string, snapshotterCredentials map[string]string) (driverName string, groupSnapshotId string, snapshots []*csi.Snapshot, timestamp time.Time, readyToUse bool, err error)
// DeleteGroupSnapshot deletes a group snapshot of multiple volumes
DeleteGroupSnapshot(ctx context.Context, groupSnapshotID string, snapshotterCredentials map[string]string) (err error)
DeleteGroupSnapshot(ctx context.Context, groupSnapshotID string, snapshotIDs []string, snapshotterCredentials map[string]string) (err error)
// GetGroupSnapshotStatus returns if a group snapshot is ready to use, its creation time, etc
GetGroupSnapshotStatus(ctx context.Context, groupSnapshotID string, snapshotterListCredentials map[string]string) (bool, time.Time, error)
@@ -74,8 +75,20 @@ func (gs *groupSnapshot) CreateGroupSnapshot(ctx context.Context, groupSnapshotN
}
func (gs *groupSnapshot) DeleteGroupSnapshot(ctx context.Context, groupSnapshotID string, snapshotterCredentials map[string]string) error {
// TODO: Implement DeleteGroupSnapshot
func (gs *groupSnapshot) DeleteGroupSnapshot(ctx context.Context, groupSnapshotID string, snapshotIds []string, snapshotterCredentials map[string]string) error {
client := csi.NewGroupControllerClient(gs.conn)
req := csi.DeleteVolumeGroupSnapshotRequest{
Secrets: snapshotterCredentials,
GroupSnapshotId: groupSnapshotID,
SnapshotIds: snapshotIds,
}
_, err := client.DeleteVolumeGroupSnapshot(ctx, &req)
if err != nil {
return err
}
return nil
}

27
pkg/sidecar-controller/csi_handler.go
View File

@@ -19,11 +19,12 @@ package sidecar_controller
import (
"context"
"fmt"
"strings"
"time"
"github.com/container-storage-interface/spec/lib/go/csi"
crdv1alpha1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumegroupsnapshot/v1alpha1"
"github.com/kubernetes-csi/external-snapshotter/v6/pkg/group_snapshotter"
"strings"
"time"
crdv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1"
"github.com/kubernetes-csi/external-snapshotter/v6/pkg/snapshotter"
@@ -36,6 +37,7 @@ type Handler interface {
GetSnapshotStatus(content *crdv1.VolumeSnapshotContent, snapshotterListCredentials map[string]string) (bool, time.Time, int64, error)
CreateGroupSnapshot(content *crdv1alpha1.VolumeGroupSnapshotContent, volumeIDs []string, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, []*csi.Snapshot, time.Time, bool, error)
GetGroupSnapshotStatus(content *crdv1alpha1.VolumeGroupSnapshotContent, snapshotterListCredentials map[string]string) (bool, time.Time, error)
DeleteGroupSnapshot(content *crdv1alpha1.VolumeGroupSnapshotContent, SnapshotID []string, snapshotterCredentials map[string]string) error
}
// csiHandler is a handler that calls CSI to create/delete volume snapshot.
@@ -165,6 +167,27 @@ func (handler *csiHandler) CreateGroupSnapshot(content *crdv1alpha1.VolumeGroupS
return handler.groupSnapshotter.CreateGroupSnapshot(ctx, groupSnapshotName, volumeIDs, parameters, snapshotterCredentials)
}
func (handler *csiHandler) DeleteGroupSnapshot(content *crdv1alpha1.VolumeGroupSnapshotContent, snapshotIDs []string, snapshotterCredentials map[string]string) error {
ctx, cancel := context.WithTimeout(context.Background(), handler.timeout)
defer cancel()
// NOTE: snapshotIDs are required for DeleteGroupSnapshot
if len(snapshotIDs) == 0 {
return fmt.Errorf("cannot delete group snapshot content %s. No snapshots found in the group snapshot", content.Name)
}
var groupSnapshotHandle string
if content.Status != nil && content.Status.VolumeGroupSnapshotHandle != nil {
groupSnapshotHandle = *content.Status.VolumeGroupSnapshotHandle
} else if content.Spec.Source.VolumeGroupSnapshotHandle != nil {
groupSnapshotHandle = *content.Spec.Source.VolumeGroupSnapshotHandle
} else {
return fmt.Errorf("failed to delete group snapshot content %s: groupsnapshotHandle is missing", content.Name)
}
return handler.groupSnapshotter.DeleteGroupSnapshot(ctx, groupSnapshotHandle, snapshotIDs, snapshotterCredentials)
}
func (handler *csiHandler) GetGroupSnapshotStatus(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent, snapshotterListCredentials map[string]string) (bool, time.Time, error) {
ctx, cancel := context.WithTimeout(context.Background(), handler.timeout)
defer cancel()

205
pkg/sidecar-controller/groupsnapshot_helper.go
View File

@@ -20,6 +20,7 @@ import (
"context"
"crypto/sha256"
"fmt"
"strings"
"time"
v1 "k8s.io/api/core/v1"
@@ -163,9 +164,23 @@ func (ctrl *csiSnapshotSideCarController) deleteGroupSnapshotContentInCacheStore
func (ctrl *csiSnapshotSideCarController) syncGroupSnapshotContent(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) error {
klog.V(5).Infof("synchronizing VolumeGroupSnapshotContent[%s]", groupSnapshotContent.Name)
/*
TODO: Check if the group snapshot content should be deleted
*/
if ctrl.shouldDeleteGroupSnapshotContent(groupSnapshotContent) {
klog.V(4).Infof("VolumeGroupSnapshotContent[%s]: the policy is %s", groupSnapshotContent.Name, groupSnapshotContent.Spec.DeletionPolicy)
if groupSnapshotContent.Spec.DeletionPolicy == crdv1.VolumeSnapshotContentDelete &&
groupSnapshotContent.Status != nil && groupSnapshotContent.Status.VolumeGroupSnapshotHandle != nil {
// issue a CSI deletion call if the group snapshot has not been deleted
// yet from underlying storage system. Note that the delete group snapshot
// operation will update groups snapshot content's GroupSnapshotHandle
// to nil upon a successful deletion. At this point, the finalizer on
// group snapshot content should NOT be removed to avoid leaking.
return ctrl.deleteCSIGroupSnapshotOperation(groupSnapshotContent)
}
// otherwise, either the snapshot has been deleted from the underlying
// storage system, or the deletion policy is Retain, remove the finalizer
// if there is one so that API server could delete the object if there is
// no other finalizer.
return ctrl.removeGroupSnapshotContentFinalizer(groupSnapshotContent)
}
if len(groupSnapshotContent.Spec.Source.PersistentVolumeNames) != 0 && groupSnapshotContent.Status == nil {
klog.V(5).Infof("syncGroupSnapshotContent: Call CreateGroupSnapshot for group snapshot content %s", groupSnapshotContent.Name)
@@ -184,6 +199,158 @@ func (ctrl *csiSnapshotSideCarController) syncGroupSnapshotContent(groupSnapshot
return ctrl.checkandUpdateGroupSnapshotContentStatus(groupSnapshotContent)
}
// removeGroupSnapshotContentFinalizer removes the VolumeGroupSnapshotContentFinalizer from a
// group snapshot content if there exists one.
func (ctrl csiSnapshotSideCarController) removeGroupSnapshotContentFinalizer(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) error {
if !utils.ContainsString(groupSnapshotContent.ObjectMeta.Finalizers, utils.VolumeGroupSnapshotContentFinalizer) {
// the finalizer does not exit, return directly
return nil
}
var patches []utils.PatchOp
groupSnapshotContentClone := groupSnapshotContent.DeepCopy()
patches = append(patches,
utils.PatchOp{
Op: "replace",
Path: "/metadata/finalizers",
Value: utils.RemoveString(groupSnapshotContentClone.ObjectMeta.Finalizers, utils.VolumeGroupSnapshotContentFinalizer),
})
updatedGroupSnapshotContent, err := utils.PatchVolumeGroupSnapshotContent(groupSnapshotContentClone, patches, ctrl.clientset)
if err != nil {
return newControllerUpdateError(groupSnapshotContent.Name, err.Error())
}
klog.V(5).Infof("Removed protection finalizer from volume group snapshot content %s", updatedGroupSnapshotContent.Name)
_, err = ctrl.storeGroupSnapshotContentUpdate(updatedGroupSnapshotContent)
if err != nil {
klog.Errorf("failed to update group snapshot content store %v", err)
}
return nil
}
// Delete a groupsnapshot: Ask the backend to remove the groupsnapshot device
func (ctrl *csiSnapshotSideCarController) deleteCSIGroupSnapshotOperation(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) error {
klog.V(5).Infof("deleteCSISnapshotOperation [%s] started", groupSnapshotContent.Name)
snapshotterCredentials, err := ctrl.GetCredentialsFromAnnotationForGroupSnapshot(groupSnapshotContent)
if err != nil {
ctrl.eventRecorder.Event(groupSnapshotContent, v1.EventTypeWarning, "SnapshotDeleteError", "Failed to get snapshot credentials")
return fmt.Errorf("failed to get input parameters to delete group snapshot for group snapshot content %s: %q", groupSnapshotContent.Name, err)
}
var snapshotIDs []string
if groupSnapshotContent.Status != nil && len(groupSnapshotContent.Status.VolumeSnapshotContentRefList) != 0 {
for _, contentRef := range groupSnapshotContent.Status.VolumeSnapshotContentRefList {
snapshotContent, err := ctrl.contentLister.Get(contentRef.Name)
if err != nil {
return fmt.Errorf("failed to get snapshot content %s from snapshot content store: %v", contentRef.Name, err)
}
snapshotIDs = append(snapshotIDs, *snapshotContent.Status.SnapshotHandle)
}
}
err = ctrl.handler.DeleteGroupSnapshot(groupSnapshotContent, snapshotIDs, snapshotterCredentials)
if err != nil {
ctrl.eventRecorder.Event(groupSnapshotContent, v1.EventTypeWarning, "GroupSnapshotDeleteError", "Failed to delete group snapshot")
return fmt.Errorf("failed to delete group snapshot %#v, err: %v", groupSnapshotContent.Name, err)
}
// the group snapshot has been deleted from the underlying storage system, update
// group snapshot content status to remove the group snapshot handle etc.
newContent, err := ctrl.clearGroupSnapshotContentStatus(groupSnapshotContent.Name)
if err != nil {
ctrl.eventRecorder.Event(groupSnapshotContent, v1.EventTypeWarning, "GroupSnapshotDeleteError", "Failed to clear content status")
return err
}
// trigger syncGroupSnapshotContent
ctrl.updateGroupSnapshotContentInInformerCache(newContent)
return nil
}
// clearGroupSnapshotContentStatus resets all fields to nil related to a group snapshot
// in groupSnapshotContent.Status. On success, the latest version of the group snapshot
// content object will be returned.
func (ctrl *csiSnapshotSideCarController) clearGroupSnapshotContentStatus(
groupSnapshotContentName string) (*crdv1alpha1.VolumeGroupSnapshotContent, error) {
klog.V(5).Infof("clearGroupSnapshotContentStatus content [%s]", groupSnapshotContentName)
// get the latest version from API server
groupSnapshotContent, err := ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshotContents().Get(context.TODO(), groupSnapshotContentName, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("error get group snapshot content %s from api server: %v", groupSnapshotContentName, err)
}
if groupSnapshotContent.Status != nil {
groupSnapshotContent.Status.VolumeGroupSnapshotHandle = nil
groupSnapshotContent.Status.ReadyToUse = nil
groupSnapshotContent.Status.CreationTime = nil
groupSnapshotContent.Status.Error = nil
groupSnapshotContent.Status.VolumeSnapshotContentRefList = nil
}
newContent, err := ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshotContents().UpdateStatus(context.TODO(), groupSnapshotContent, metav1.UpdateOptions{})
if err != nil {
return groupSnapshotContent, newControllerUpdateError(groupSnapshotContentName, err.Error())
}
return newContent, nil
}
func (ctrl *csiSnapshotSideCarController) GetCredentialsFromAnnotationForGroupSnapshot(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) (map[string]string, error) {
// get secrets if VolumeGroupSnapshotClass specifies it
var snapshotterCredentials map[string]string
var err error
// Check if annotation exists
if metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnDeletionSecretRefName) && metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnDeletionSecretRefNamespace) {
annDeletionSecretName := groupSnapshotContent.Annotations[utils.AnnDeletionSecretRefName]
annDeletionSecretNamespace := groupSnapshotContent.Annotations[utils.AnnDeletionSecretRefNamespace]
snapshotterSecretRef := &v1.SecretReference{}
if annDeletionSecretName == "" || annDeletionSecretNamespace == "" {
return nil, fmt.Errorf("cannot retrieve secrets for group snapshot content %#v, err: secret name or namespace not specified", groupSnapshotContent.Name)
}
snapshotterSecretRef.Name = annDeletionSecretName
snapshotterSecretRef.Namespace = annDeletionSecretNamespace
snapshotterCredentials, err = utils.GetCredentials(ctrl.client, snapshotterSecretRef)
if err != nil {
// Continue with deletion, as the secret may have already been deleted.
klog.Errorf("Failed to get credentials for group snapshot content %s: %s", groupSnapshotContent.Name, err.Error())
return nil, fmt.Errorf("cannot get credentials for group snapshot content %#v", groupSnapshotContent.Name)
}
}
return snapshotterCredentials, nil
}
// shouldDeleteGroupSnapshotContent checks if groupSnapshotContent object should be deleted
// if DeletionTimestamp is set on the groupSnapshotContent
func (ctrl *csiSnapshotSideCarController) shouldDeleteGroupSnapshotContent(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) bool {
klog.V(5).Infof("Check if VolumeGroupSnapshotContent[%s] should be deleted.", groupSnapshotContent.Name)
if groupSnapshotContent.ObjectMeta.DeletionTimestamp == nil {
return false
}
// 1) shouldDeleteGroupSnapshot returns true if a content is not bound
// (VolumeGroupSnapshotRef == "") for pre-provisioned snapshot
if groupSnapshotContent.Spec.Source.VolumeGroupSnapshotHandle != nil && groupSnapshotContent.Spec.VolumeGroupSnapshotRef.UID == "" {
return true
}
// NOTE(xyang): Handle create snapshot timeout
// 2) shouldDeleteGroupSnapshotContent returns false if AnnVolumeGroupSnapshotBeingCreated
// annotation is set. This indicates a CreateGroupSnapshot CSI RPC has
// not responded with success or failure.
// We need to keep waiting for a response from the CSI driver.
if metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnVolumeGroupSnapshotBeingCreated) {
return false
}
// 3) shouldDeleteGroupSnapshotContent returns true if AnnVolumeSnapshotBeingDeleted annotation is set
if metav1.HasAnnotation(groupSnapshotContent.ObjectMeta, utils.AnnVolumeGroupSnapshotBeingDeleted) {
return true
}
return false
}
// createGroupSnapshot starts new asynchronous operation to create group snapshot
func (ctrl *csiSnapshotSideCarController) createGroupSnapshot(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) error {
klog.V(5).Infof("createGroupSnapshot for group snapshot content [%s]: started", groupSnapshotContent.Name)
@@ -289,9 +456,9 @@ func (ctrl *csiSnapshotSideCarController) createGroupSnapshotWrapper(groupSnapsh
VolumeGroupSnapshotContentName: &groupSnapshotContent.Name,
},
}
label := make(map[string]string)
label["volumeGroupSnapshotName"] = groupSnapshotContent.Spec.VolumeGroupSnapshotRef.Name
name := "f"
volumeSnapshot := &crdv1.VolumeSnapshot{
ObjectMeta: metav1.ObjectMeta{
Name: volumeSnapshotName,
@@ -304,29 +471,19 @@ func (ctrl *csiSnapshotSideCarController) createGroupSnapshotWrapper(groupSnapsh
},
},
}
vsc, err := ctrl.clientset.SnapshotV1().VolumeSnapshotContents().Create(context.TODO(), volumeSnapshotContent, metav1.CreateOptions{})
if err != nil {
return groupSnapshotContent, err
}
snapshotContentNames = append(snapshotContentNames, vsc.Name)
klog.Infof("making snapshot %v %s %s", volumeSnapshot.Status, *volumeSnapshot.Status.VolumeGroupSnapshotName, name)
_, err = ctrl.clientset.SnapshotV1().VolumeSnapshots(volumeSnapshotNamespace).Create(context.TODO(), volumeSnapshot, metav1.CreateOptions{})
if err != nil {
return groupSnapshotContent, err
}
// klog.Infof("raunak made snapshot 1 %v", spew.Sdump(sn))
// sn.Status = &crdv1.VolumeSnapshotStatus{
// VolumeGroupSnapshotName: &name,
// }
// sn, err = ctrl.clientset.SnapshotV1().VolumeSnapshots(volumeSnapshotNamespace).UpdateStatus(context.TODO(), sn, metav1.UpdateOptions{})
// if err != nil {
// klog.Infof("failed 2")
// return groupSnapshotContent, err
// }
// klog.Infof("made snapshot 2 %v", spew.Sdump(sn))
}
klog.Infof("raunak 2")
newGroupSnapshotContent, err := ctrl.updateGroupSnapshotContentStatus(groupSnapshotContent, groupSnapshotID, readyToUse, creationTime.UnixNano(), snapshotContentNames)
if err != nil {
klog.Errorf("error updating status for volume group snapshot content %s: %v.", groupSnapshotContent.Name, err)
@@ -452,19 +609,25 @@ func (ctrl csiSnapshotSideCarController) removeAnnVolumeGroupSnapshotBeingCreate
return groupSnapshotContent, nil
}
groupSnapshotContentClone := groupSnapshotContent.DeepCopy()
delete(groupSnapshotContentClone.ObjectMeta.Annotations, utils.AnnVolumeGroupSnapshotBeingCreated)
annotationPatchPath := strings.ReplaceAll(utils.AnnVolumeGroupSnapshotBeingCreated, "/", "~1")
updatedContent, err := ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshotContents().Update(context.TODO(), groupSnapshotContentClone, metav1.UpdateOptions{})
var patches []utils.PatchOp
patches = append(patches, utils.PatchOp{
Op: "remove",
Path: "/metadata/annotations/" + annotationPatchPath,
})
updatedGroupSnapshotContent, err := utils.PatchVolumeGroupSnapshotContent(groupSnapshotContentClone, patches, ctrl.clientset)
if err != nil {
return groupSnapshotContent, newControllerUpdateError(groupSnapshotContent.Name, err.Error())
}
klog.V(5).Infof("Removed VolumeGroupSnapshotBeingCreated annotation from volume group snapshot content %s", groupSnapshotContent.Name)
_, err = ctrl.storeContentUpdate(updatedContent)
_, err = ctrl.storeGroupSnapshotContentUpdate(updatedGroupSnapshotContent)
if err != nil {
klog.Errorf("failed to update groupSnapshotContent store %v", err)
}
return updatedContent, nil
return updatedGroupSnapshotContent, nil
}
func (ctrl *csiSnapshotSideCarController) updateGroupSnapshotContentStatus(
@@ -671,7 +834,7 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateGroupSnapshotContentStat
driverName = groupSnapshotContent.Spec.Driver
groupSnapshotID = *groupSnapshotContent.Spec.Source.VolumeGroupSnapshotHandle
klog.V(5).Infof("checkandUpdateGroupSnapshotContentStatusOperation: driver %s, groupSnapshotId %s, creationTime %v, size %d, readyToUse %t", driverName, groupSnapshotID, creationTime, readyToUse)
klog.V(5).Infof("checkandUpdateGroupSnapshotContentStatusOperation: driver %s, groupSnapshotId %s, creationTime %v, readyToUse %t", driverName, groupSnapshotID, creationTime, readyToUse)
if creationTime.IsZero() {
creationTime = time.Now()

20
pkg/utils/patch.go
View File

@@ -58,6 +58,26 @@ func PatchVolumeSnapshot(
return newSnapshot, nil
}
// PatchVolumeGroupSnapshot patches a volume group snapshot object
func PatchVolumeGroupSnapshot(
existingGroupSnapshot *crdv1alpha1.VolumeGroupSnapshot,
patch []PatchOp,
client clientset.Interface,
subresources ...string,
) (*crdv1alpha1.VolumeGroupSnapshot, error) {
data, err := json.Marshal(patch)
if nil != err {
return existingGroupSnapshot, err
}
newGroupSnapshot, err := client.GroupsnapshotV1alpha1().VolumeGroupSnapshots(existingGroupSnapshot.Namespace).Patch(context.TODO(), existingGroupSnapshot.Name, types.JSONPatchType, data, metav1.PatchOptions{}, subresources...)
if err != nil {
return existingGroupSnapshot, err
}
return newGroupSnapshot, nil
}
// PatchVolumeGroupSnapshotContent patches a volume group snapshot content object
func PatchVolumeGroupSnapshotContent(
existingGroupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent,

40
pkg/utils/util.go
View File

@@ -76,6 +76,10 @@ const (
VolumeSnapshotAsSourceFinalizer = "snapshot.storage.kubernetes.io/volumesnapshot-as-source-protection"
// Name of finalizer on PVCs that is being used as a source to create VolumeSnapshots
PVCFinalizer = "snapshot.storage.kubernetes.io/pvc-as-source-protection"
// Name of finalizer on VolumeGroupSnapshotContents that are bound by VolumeGroupSnapshots
VolumeGroupSnapshotContentFinalizer = "groupsnapshot.storage.kubernetes.io/volumegroupsnapshotcontent-bound-protection"
// Name of finalizer on VolumeGroupSnapshots that are bound to VolumeGroupSnapshotContents
VolumeGroupSnapshotBoundFinalizer = "groupsnapshot.storage.kubernetes.io/volumegroupsnapshot-bound-protection"
IsDefaultSnapshotClassAnnotation = "snapshot.storage.kubernetes.io/is-default-class"
IsDefaultGroupSnapshotClassAnnotation = "groupsnapshot.storage.kubernetes.io/is-default-class"
@@ -114,6 +118,14 @@ const (
// group snapshots.
AnnVolumeGroupSnapshotBeingCreated = "groupsnapshot.storage.kubernetes.io/volumegroupsnapshot-being-created"
// AnnVolumeGroupSnapshotBeingDeleted annotation applies to VolumeGroupSnapshotContents.
// It indicates that the common snapshot controller has verified that volume
// group snapshot has a deletion timestamp and is being deleted.
// Sidecar controller needs to check the deletion policy on the
// VolumeGroupSnapshotContent and decide whether to delete the volume group snapshot
// backing the group snapshot content.
AnnVolumeGroupSnapshotBeingDeleted = "groupsnapshot.storage.kubernetes.io/volumegroupsnapshot-being-deleted"
// Annotation for secret name and namespace will be added to the content
// and used at snapshot content deletion time.
AnnDeletionSecretRefName = "snapshot.storage.kubernetes.io/deletion-secret-name"
@@ -406,12 +418,23 @@ func NeedToAddContentFinalizer(content *crdv1.VolumeSnapshotContent) bool {
return content.ObjectMeta.DeletionTimestamp == nil && !ContainsString(content.ObjectMeta.Finalizers, VolumeSnapshotContentFinalizer)
}
// NeedToAddGroupSnapshotContentFinalizer checks if a Finalizer needs to be added for the volume group snapshot content.
func NeedToAddGroupSnapshotContentFinalizer(groupSnapshotContent *crdv1alpha1.VolumeGroupSnapshotContent) bool {
return groupSnapshotContent.ObjectMeta.DeletionTimestamp == nil && !ContainsString(groupSnapshotContent.ObjectMeta.Finalizers, VolumeGroupSnapshotContentFinalizer)
}
// IsSnapshotDeletionCandidate checks if a volume snapshot deletionTimestamp
// is set and any finalizer is on the snapshot.
func IsSnapshotDeletionCandidate(snapshot *crdv1.VolumeSnapshot) bool {
return snapshot.ObjectMeta.DeletionTimestamp != nil && (ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotAsSourceFinalizer) || ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotBoundFinalizer))
}
// IsGroupSnapshotDeletionCandidate checks if a volume group snapshot deletionTimestamp
// is set and any finalizer is on the group snapshot.
func IsGroupSnapshotDeletionCandidate(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) bool {
return groupSnapshot.ObjectMeta.DeletionTimestamp != nil && ContainsString(groupSnapshot.ObjectMeta.Finalizers, VolumeGroupSnapshotBoundFinalizer)
}
// NeedToAddSnapshotAsSourceFinalizer checks if a Finalizer needs to be added for the volume snapshot as a source for PVC.
func NeedToAddSnapshotAsSourceFinalizer(snapshot *crdv1.VolumeSnapshot) bool {
return snapshot.ObjectMeta.DeletionTimestamp == nil && !ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotAsSourceFinalizer)
@@ -422,6 +445,11 @@ func NeedToAddSnapshotBoundFinalizer(snapshot *crdv1.VolumeSnapshot) bool {
return snapshot.ObjectMeta.DeletionTimestamp == nil && !ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotBoundFinalizer) && IsBoundVolumeSnapshotContentNameSet(snapshot)
}
// NeedToAddGroupSnapshotBoundFinalizer checks if a Finalizer needs to be added for the bound volume group snapshot.
func NeedToAddGroupSnapshotBoundFinalizer(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) bool {
return groupSnapshot.ObjectMeta.DeletionTimestamp == nil && !ContainsString(groupSnapshot.ObjectMeta.Finalizers, VolumeGroupSnapshotBoundFinalizer) && IsBoundVolumeGroupSnapshotContentNameSet(groupSnapshot)
}
func deprecationWarning(deprecatedParam, newParam, removalVersion string) string {
if removalVersion == "" {
removalVersion = "a future release"
@@ -468,6 +496,18 @@ func GetSnapshotStatusForLogging(snapshot *crdv1.VolumeSnapshot) string {
return fmt.Sprintf("bound to: %q, Completed: %v", snapshotContentName, ready)
}
func GetGroupSnapshotStatusForLogging(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) string {
groupSnapshotContentName := ""
if groupSnapshot.Status != nil && groupSnapshot.Status.BoundVolumeGroupSnapshotContentName != nil {
groupSnapshotContentName = *groupSnapshot.Status.BoundVolumeGroupSnapshotContentName
}
ready := false
if groupSnapshot.Status != nil && groupSnapshot.Status.ReadyToUse != nil {
ready = *groupSnapshot.Status.ReadyToUse
}
return fmt.Sprintf("bound to: %q, Completed: %v", groupSnapshotContentName, ready)
}
func IsVolumeSnapshotRefSet(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) bool {
if content.Spec.VolumeSnapshotRef.Name == snapshot.Name &&
content.Spec.VolumeSnapshotRef.Namespace == snapshot.Namespace &&