Files
external-snapshotter/pkg/common-controller/groupsnapshot_controller_helper.go
Raunak Pradip Shah a574b8780c -Introduce volume group snapshot functionality include initializing queues and caches in snapshotter
- Introduce new flag for volume group snapshots and run worker if flag is enabled
- Introduce the main controller for group snapshots in snapshot-controller
2023-05-03 21:21:32 +05:30

357 lines
16 KiB
Go

/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package common_controller
import (
"context"
"fmt"
"time"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
klog "k8s.io/klog/v2"
crdv1alpha1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumegroupsnapshot/v1alpha1"
crdv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1"
"github.com/kubernetes-csi/external-snapshotter/v6/pkg/utils"
)
func (ctrl *csiSnapshotCommonController) storeGroupSnapshotUpdate(groupsnapshot interface{}) (bool, error) {
return utils.StoreObjectUpdate(ctrl.groupSnapshotStore, groupsnapshot, "groupsnapshot")
}
func (ctrl *csiSnapshotCommonController) storeGroupSnapshotContentUpdate(groupsnapshotcontent interface{}) (bool, error) {
return utils.StoreObjectUpdate(ctrl.groupSnapshotContentStore, groupsnapshotcontent, "groupsnapshotcontent")
}
// getGroupSnapshotClass is a helper function to get group snapshot class from the class name.
func (ctrl *csiSnapshotCommonController) getGroupSnapshotClass(className string) (*crdv1alpha1.VolumeGroupSnapshotClass, error) {
klog.V(5).Infof("getGroupSnapshotClass: VolumeGroupSnapshotClassName [%s]", className)
class, err := ctrl.groupSnapshotClassLister.Get(className)
if err != nil {
klog.Errorf("failed to retrieve group snapshot class %s from the informer: %q", className, err)
return nil, err
}
return class, nil
}
// updateGroupSnapshotErrorStatusWithEvent saves new groupsnapshot.Status to API
// server and emits given event on the group snapshot. It saves the status and
// emits the event only when the status has actually changed from the version
// saved in API server.
//
// Parameters:
//
// - groupSnapshot - group snapshot to update
// - setReadyToFalse bool - indicates whether to set the group snapshot's
// ReadyToUse status to false.
// if true, ReadyToUse will be set to false;
// otherwise, ReadyToUse will not be changed.
// - eventtype, reason, message - event to send, see EventRecorder.Event()
func (ctrl *csiSnapshotCommonController) updateGroupSnapshotErrorStatusWithEvent(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot, setReadyToFalse bool, eventtype, reason, message string) error {
klog.V(5).Infof("updateGroupSnapshotErrorStatusWithEvent[%s]", utils.GroupSnapshotKey(groupSnapshot))
if groupSnapshot.Status != nil && groupSnapshot.Status.Error != nil && *groupSnapshot.Status.Error.Message == message {
klog.V(4).Infof("updateGroupSnapshotErrorStatusWithEvent[%s]: the same error %v is already set", groupSnapshot.Name, groupSnapshot.Status.Error)
return nil
}
groupSnapshotClone := groupSnapshot.DeepCopy()
if groupSnapshotClone.Status == nil {
groupSnapshotClone.Status = &crdv1alpha1.VolumeGroupSnapshotStatus{}
}
statusError := &crdv1.VolumeSnapshotError{
Time: &metav1.Time{
Time: time.Now(),
},
Message: &message,
}
groupSnapshotClone.Status.Error = statusError
// Only update ReadyToUse in VolumeSnapshot's Status to false if setReadyToFalse is true.
if setReadyToFalse {
ready := false
groupSnapshotClone.Status.ReadyToUse = &ready
}
newSnapshot, err := ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshots(groupSnapshotClone.Namespace).UpdateStatus(context.TODO(), groupSnapshotClone, metav1.UpdateOptions{})
// Emit the event even if the status update fails so that user can see the error
ctrl.eventRecorder.Event(newSnapshot, eventtype, reason, message)
if err != nil {
klog.V(4).Infof("updating VolumeGroupSnapshot[%s] error status failed %v", utils.GroupSnapshotKey(groupSnapshot), err)
return err
}
_, err = ctrl.storeGroupSnapshotUpdate(newSnapshot)
if err != nil {
klog.V(4).Infof("updating VolumeGroupSnapshot[%s] error status: cannot update internal cache %v", utils.GroupSnapshotKey(groupSnapshot), err)
return err
}
return nil
}
// SetDefaultGroupSnapshotClass is a helper function to figure out the default
// group snapshot class.
// For pre-provisioned case, it's an no-op.
// For dynamic provisioning, it gets the default GroupSnapshotClasses in the
// system if there is any (could be multiple), and finds the one with the same
// CSI Driver as a PV from which a group snapshot will be taken.
func (ctrl *csiSnapshotCommonController) SetDefaultGroupSnapshotClass(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) (*crdv1alpha1.VolumeGroupSnapshotClass, *crdv1alpha1.VolumeGroupSnapshot, error) {
klog.V(5).Infof("SetDefaultGroupSnapshotClass for group snapshot [%s]", groupSnapshot.Name)
if groupSnapshot.Spec.Source.VolumeGroupSnapshotContentName != nil {
// don't return error for pre-provisioned group snapshots
klog.V(5).Infof("Don't need to find GroupSnapshotClass for pre-provisioned group snapshot [%s]", groupSnapshot.Name)
return nil, groupSnapshot, nil
}
// Find default group snapshot class if available
list, err := ctrl.groupSnapshotClassLister.List(labels.Everything())
if err != nil {
return nil, groupSnapshot, err
}
pvDriver, err := ctrl.pvDriverFromGroupSnapshot(groupSnapshot)
if err != nil {
klog.Errorf("failed to get pv csi driver from group snapshot %s/%s: %q", groupSnapshot.Namespace, groupSnapshot.Name, err)
return nil, groupSnapshot, err
}
defaultClasses := []*crdv1alpha1.VolumeGroupSnapshotClass{}
for _, class := range list {
if utils.IsDefaultAnnotation(class.ObjectMeta) && pvDriver == class.Driver {
defaultClasses = append(defaultClasses, class)
klog.V(5).Infof("get defaultGroupClass added: %s, driver: %s", class.Name, pvDriver)
}
}
if len(defaultClasses) == 0 {
return nil, groupSnapshot, fmt.Errorf("cannot find default group snapshot class")
}
if len(defaultClasses) > 1 {
klog.V(4).Infof("get DefaultClass %d defaults found", len(defaultClasses))
return nil, groupSnapshot, fmt.Errorf("%d default snapshot classes were found", len(defaultClasses))
}
klog.V(5).Infof("setDefaultSnapshotClass [%s]: default VolumeSnapshotClassName [%s]", groupSnapshot.Name, defaultClasses[0].Name)
groupSnapshotClone := groupSnapshot.DeepCopy()
groupSnapshotClone.Spec.VolumeGroupSnapshotClassName = &(defaultClasses[0].Name)
newGroupSnapshot, err := ctrl.clientset.GroupsnapshotV1alpha1().VolumeGroupSnapshots(groupSnapshotClone.Namespace).Update(context.TODO(), groupSnapshotClone, metav1.UpdateOptions{})
if err != nil {
klog.V(4).Infof("updating VolumeSnapshot[%s] default class failed %v", utils.GroupSnapshotKey(groupSnapshot), err)
}
_, updateErr := ctrl.storeGroupSnapshotUpdate(newGroupSnapshot)
if updateErr != nil {
// We will get an "snapshot update" event soon, this is not a big error
klog.V(4).Infof("setDefaultSnapshotClass [%s]: cannot update internal cache: %v", utils.GroupSnapshotKey(groupSnapshot), updateErr)
}
return defaultClasses[0], newGroupSnapshot, nil
}
// pvDriverFromGroupSnapshot is a helper function to get the CSI driver name from the targeted PersistentVolume.
// It looks up the PVC from which the snapshot is specified to be created from, and looks for the PVC's corresponding
// PV. Bi-directional binding will be verified between PVC and PV before the PV's CSI driver is returned.
// For an non-CSI volume, it returns an error immediately as it's not supported.
func (ctrl *csiSnapshotCommonController) pvDriverFromGroupSnapshot(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) (string, error) {
pvs, err := ctrl.getVolumesFromVolumeGroupSnapshot(groupSnapshot)
if err != nil {
return "", err
}
// Take any volume to get the driver
if pvs[0].Spec.PersistentVolumeSource.CSI == nil {
return "", fmt.Errorf("snapshotting non-CSI volumes is not supported, snapshot:%s/%s", groupSnapshot.Namespace, groupSnapshot.Name)
}
return pvs[0].Spec.PersistentVolumeSource.CSI.Driver, nil
}
// getVolumesFromVolumeGroupSnapshot returns the list of PersistentVolume from a VolumeGroupSnapshot.
func (ctrl *csiSnapshotCommonController) getVolumesFromVolumeGroupSnapshot(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) ([]*v1.PersistentVolume, error) {
var pvReturnList []*v1.PersistentVolume
pvcs, err := ctrl.getClaimsFromVolumeGroupSnapshot(groupSnapshot)
if err != nil {
return nil, err
}
for _, pvc := range pvcs {
if pvc.Status.Phase != v1.ClaimBound {
return nil, fmt.Errorf("the PVC %s is not yet bound to a PV, will not attempt to take a group snapshot", pvc.Name)
}
pvName := pvc.Spec.VolumeName
pv, err := ctrl.client.CoreV1().PersistentVolumes().Get(context.TODO(), pvName, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("failed to retrieve PV %s from the API server: %q", pvName, err)
}
// Verify binding between PV/PVC is still valid
bound := ctrl.isVolumeBoundToClaim(pv, &pvc)
if bound == false {
klog.Warningf("binding between PV %s and PVC %s is broken", pvName, pvc.Name)
return nil, fmt.Errorf("claim in dataSource not bound or invalid")
}
pvReturnList = append(pvReturnList, pv)
klog.V(5).Infof("getVolumeFromVolumeGroupSnapshot: group snapshot [%s] PV name [%s]", groupSnapshot.Name, pvName)
}
return pvReturnList, nil
}
// getClaimsFromVolumeGroupSnapshot is a helper function to get a list of PVCs from VolumeGroupSnapshot.
func (ctrl *csiSnapshotCommonController) getClaimsFromVolumeGroupSnapshot(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) ([]v1.PersistentVolumeClaim, error) {
labelSelector := groupSnapshot.Spec.Source.Selector
// Get PVC that has group snapshot label applied.
pvcList, err := ctrl.client.CoreV1().PersistentVolumeClaims(groupSnapshot.Namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector.String()})
if err != nil {
return nil, fmt.Errorf("failed to list PVCs with label selector %s: %q", labelSelector.String(), err)
}
if len(pvcList.Items) == 0 {
return nil, fmt.Errorf("label selector %s for group snapshot not applied to any PVC", labelSelector.String())
}
return pvcList.Items, nil
}
// updateGroupSnapshot runs in worker thread and handles "groupsnapshot added",
// "groupsnapshot updated" and "periodic sync" events.
func (ctrl *csiSnapshotCommonController) updateGroupSnapshot(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) error {
// Store the new group snapshot version in the cache and do not process it
// if this is an old version.
klog.V(5).Infof("updateGroupSnapshot %q", utils.GroupSnapshotKey(groupSnapshot))
newGroupSnapshot, err := ctrl.storeGroupSnapshotUpdate(groupSnapshot)
if err != nil {
klog.Errorf("%v", err)
}
if !newGroupSnapshot {
return nil
}
err = ctrl.syncGroupSnapshot(groupSnapshot)
if err != nil {
if errors.IsConflict(err) {
// Version conflict error happens quite often and the controller
// recovers from it easily.
klog.V(3).Infof("could not sync snapshot %q: %+v", utils.GroupSnapshotKey(groupSnapshot), err)
} else {
klog.Errorf("could not sync snapshot %q: %+v", utils.GroupSnapshotKey(groupSnapshot), err)
}
return err
}
return nil
}
// deleteGroupSnapshot runs in worker thread and handles "groupsnapshot deleted" event.
func (ctrl *csiSnapshotCommonController) deleteGroupSnapshot(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) {
_ = ctrl.snapshotStore.Delete(groupSnapshot)
klog.V(4).Infof("snapshot %q deleted", utils.GroupSnapshotKey(groupSnapshot))
groupSnapshotContentName := ""
if groupSnapshot.Status != nil && groupSnapshot.Status.BoundVolumeGroupSnapshotContentName != nil {
groupSnapshotContentName = *groupSnapshot.Status.BoundVolumeGroupSnapshotContentName
}
if groupSnapshotContentName == "" {
klog.V(5).Infof("deleteGroupSnapshot[%q]: group snapshot content not bound", utils.GroupSnapshotKey(groupSnapshot))
return
}
// sync the content when its group snapshot is deleted. Explicitly sync'ing
// the content here in response to group snapshot deletion prevents the content
// from waiting until the next sync period for its release.
klog.V(5).Infof("deleteGroupSnapshot[%q]: scheduling sync of group snapshot content %s", utils.GroupSnapshotKey(groupSnapshot), groupSnapshotContentName)
ctrl.groupSnapshotContentQueue.Add(groupSnapshotContentName)
}
// syncGroupSnapshot is the main controller method to decide what to do with a
// group snapshot. It's invoked by appropriate cache.Controller callbacks when
// a group snapshot is created, updated or periodically synced. We do not
// differentiate between these events.
// For easier readability, it is split into syncUnreadyGroupSnapshot and syncReadyGroupSnapshot
func (ctrl *csiSnapshotCommonController) syncGroupSnapshot(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) error {
klog.V(5).Infof("synchronizing VolumeGroupSnapshot[%s]", utils.GroupSnapshotKey(groupSnapshot))
klog.V(5).Infof("syncGroupSnapshot [%s]: check if we should remove finalizer on group snapshot PVC source and remove it if we can", utils.GroupSnapshotKey(groupSnapshot))
/*
TODO:
- Check and remove finalizer if needed.
- Check and set invalid group snapshot label, if needed.
- Process if deletion timestamp is set.
- Check and add group snapshot finalizers.
*/
// Need to build or update groupSnapshot.Status in following cases:
// 1) groupSnapshot.Status is nil
// 2) groupSnapshot.Status.ReadyToUse is false
// 3) groupSnapshot.Status.BoundVolumeSnapshotContentName is not set
if !utils.IsGroupSnapshotReady(groupSnapshot) || !utils.IsBoundVolumeGroupSnapshotContentNameSet(groupSnapshot) {
//return ctrl.syncUnreadyGroupSnapshot(groupSnapshot)
}
return ctrl.syncReadyGroupSnapshot(groupSnapshot)
}
// syncReadyGroupSnapshot checks the group snapshot which has been bound to group
// snapshot content successfully before.
// If there is any problem with the binding (e.g., group snapshot points to a
// non-existent group snapshot content), update the group snapshot status and emit event.
func (ctrl *csiSnapshotCommonController) syncReadyGroupSnapshot(groupSnapshot *crdv1alpha1.VolumeGroupSnapshot) error {
if !utils.IsBoundVolumeGroupSnapshotContentNameSet(groupSnapshot) {
return fmt.Errorf("group snapshot %s is not bound to a group snapshot content", utils.GroupSnapshotKey(groupSnapshot))
}
content, err := ctrl.getGroupSnapshotContentFromStore(*groupSnapshot.Status.BoundVolumeGroupSnapshotContentName)
if err != nil {
return nil
}
if content == nil {
// this meant there is no matching group snapshot content in cache found
// update status of the group snapshot and return
return ctrl.updateGroupSnapshotErrorStatusWithEvent(groupSnapshot, true, v1.EventTypeWarning, "GroupSnapshotContentMissing", "VolumeGroupSnapshotContent is missing")
}
klog.V(5).Infof("syncReadyGroupSnapshot[%s]: VolumeGroupSnapshotContent %q found", utils.GroupSnapshotKey(groupSnapshot), content.Name)
// check binding from group snapshot content side to make sure the binding is still valid
if !utils.IsVolumeGroupSnapshotRefSet(groupSnapshot, content) {
// group snapshot is bound but group snapshot content is not pointing to the group snapshot
return ctrl.updateGroupSnapshotErrorStatusWithEvent(groupSnapshot, true, v1.EventTypeWarning, "GroupSnapshotMisbound", "VolumeGroupSnapshotContent is not bound to the VolumeGroupSnapshot correctly")
}
// everything is verified, return
return nil
}
// getGroupSnapshotContentFromStore tries to find a VolumeGroupSnapshotContent from group
// snapshot content cache store by name.
// Note that if no VolumeGroupSnapshotContent exists in the cache store and no error
// encountered, it returns (nil, nil)
func (ctrl *csiSnapshotCommonController) getGroupSnapshotContentFromStore(contentName string) (*crdv1alpha1.VolumeGroupSnapshotContent, error) {
obj, exist, err := ctrl.groupSnapshotContentStore.GetByKey(contentName)
if err != nil {
// should never reach here based on implementation at:
// https://github.com/kubernetes/client-go/blob/master/tools/cache/store.go#L226
return nil, err
}
if !exist {
// not able to find a matching content
return nil, nil
}
content, ok := obj.(*crdv1alpha1.VolumeGroupSnapshotContent)
if !ok {
return nil, fmt.Errorf("expected VolumeSnapshotContent, got %+v", obj)
}
return content, nil
}