From 6308420635f1c6f831e7c9d942d15161f13f2e7f Mon Sep 17 00:00:00 2001 From: xing-yang Date: Sat, 19 Oct 2019 14:48:38 +0000 Subject: [PATCH 1/5] Split snapshot controller using beta APIs --- Makefile | 4 +- Dockerfile => cmd/csi-snapshotter/Dockerfile | 2 +- cmd/csi-snapshotter/main.go | 48 +- cmd/snapshot-controller/Dockerfile | 6 + cmd/snapshot-controller/main.go | 159 +++ cmd/snapshot-controller/main_test.go | 161 +++ .../{ => csi-snapshotter}/README.md | 0 .../rbac-csi-snapshotter.yaml} | 18 - .../rbac-external-provisioner.yaml | 0 .../setup-csi-snapshotter.yaml | 16 +- .../rbac-snapshot-controller.yaml | 80 ++ .../setup-snapshot-controller.yaml | 27 + examples/kubernetes/snapshot.yaml | 7 +- examples/kubernetes/snapshotclass.yaml | 5 +- examples/kubernetes/storageclass.yaml | 2 +- .../framework_test.go | 150 +- pkg/common-controller/snapshot_controller.go | 1190 ++++++++++++++++ .../snapshot_controller_base.go | 101 +- .../snapshot_controller_test.go | 11 +- .../snapshot_create_test.go | 173 +-- .../snapshot_delete_test.go | 185 +-- .../snapshot_finalizer_test.go | 14 +- .../snapshot_update_test.go} | 162 ++- pkg/controller/snapshot_controller.go | 1226 ----------------- .../csi_handler.go | 49 +- pkg/sidecar-controller/snapshot_controller.go | 539 ++++++++ .../snapshot_controller_base.go | 280 ++++ pkg/snapshotter/snapshotter.go | 13 +- pkg/snapshotter/snapshotter_test.go | 49 +- pkg/{controller => utils}/util.go | 128 +- pkg/{controller => utils}/util_test.go | 58 +- vendor/modules.txt | 4 +- 32 files changed, 3050 insertions(+), 1817 deletions(-) rename Dockerfile => cmd/csi-snapshotter/Dockerfile (74%) create mode 100644 cmd/snapshot-controller/Dockerfile create mode 100644 cmd/snapshot-controller/main.go create mode 100644 cmd/snapshot-controller/main_test.go rename deploy/kubernetes/{ => csi-snapshotter}/README.md (100%) rename deploy/kubernetes/{rbac.yaml => csi-snapshotter/rbac-csi-snapshotter.yaml} (79%) rename deploy/kubernetes/{ => csi-snapshotter}/rbac-external-provisioner.yaml (100%) rename deploy/kubernetes/{ => csi-snapshotter}/setup-csi-snapshotter.yaml (86%) create mode 100644 deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml create mode 100644 deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml rename pkg/{controller => common-controller}/framework_test.go (92%) create mode 100644 pkg/common-controller/snapshot_controller.go rename pkg/{controller => common-controller}/snapshot_controller_base.go (80%) rename pkg/{controller => common-controller}/snapshot_controller_test.go (93%) rename pkg/{controller => common-controller}/snapshot_create_test.go (63%) rename pkg/{controller => common-controller}/snapshot_delete_test.go (64%) rename pkg/{controller => common-controller}/snapshot_finalizer_test.go (85%) rename pkg/{controller/snapshot_ready_test.go => common-controller/snapshot_update_test.go} (68%) delete mode 100644 pkg/controller/snapshot_controller.go rename pkg/{controller => sidecar-controller}/csi_handler.go (62%) create mode 100644 pkg/sidecar-controller/snapshot_controller.go create mode 100644 pkg/sidecar-controller/snapshot_controller_base.go rename pkg/{controller => utils}/util.go (73%) rename pkg/{controller => utils}/util_test.go (85%) diff --git a/Makefile b/Makefile index 4a81c510..4fca4dde 100644 --- a/Makefile +++ b/Makefile @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -.PHONY: all csi-snapshotter clean test +.PHONY: all snapshot-controller csi-snapshotter clean test -CMDS=csi-snapshotter +CMDS=snapshot-controller csi-snapshotter all: build include release-tools/build.make diff --git a/Dockerfile b/cmd/csi-snapshotter/Dockerfile similarity index 74% rename from Dockerfile rename to cmd/csi-snapshotter/Dockerfile index bf355958..f5221956 100644 --- a/Dockerfile +++ b/cmd/csi-snapshotter/Dockerfile @@ -1,6 +1,6 @@ FROM gcr.io/distroless/static:latest LABEL maintainers="Kubernetes Authors" -LABEL description="CSI External Snapshotter" +LABEL description="CSI External Snapshotter Sidecar" COPY ./bin/csi-snapshotter csi-snapshotter ENTRYPOINT ["/csi-snapshotter"] diff --git a/cmd/csi-snapshotter/main.go b/cmd/csi-snapshotter/main.go index 9d7c4ef5..32ebb412 100644 --- a/cmd/csi-snapshotter/main.go +++ b/cmd/csi-snapshotter/main.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -37,7 +37,7 @@ import ( "github.com/kubernetes-csi/csi-lib-utils/connection" "github.com/kubernetes-csi/csi-lib-utils/leaderelection" csirpc "github.com/kubernetes-csi/csi-lib-utils/rpc" - "github.com/kubernetes-csi/external-snapshotter/pkg/controller" + controller "github.com/kubernetes-csi/external-snapshotter/pkg/sidecar-controller" "github.com/kubernetes-csi/external-snapshotter/pkg/snapshotter" clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" @@ -56,17 +56,13 @@ const ( // Command line flags var ( - snapshotterName = flag.String("snapshotter", "", "This option is deprecated.") - kubeconfig = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Required only when running out of cluster.") - connectionTimeout = flag.Duration("connection-timeout", 0, "The --connection-timeout flag is deprecated") - csiAddress = flag.String("csi-address", "/run/csi/socket", "Address of the CSI driver socket.") - createSnapshotContentRetryCount = flag.Int("create-snapshotcontent-retrycount", 5, "Number of retries when we create a snapshot content object for a snapshot.") - createSnapshotContentInterval = flag.Duration("create-snapshotcontent-interval", 10*time.Second, "Interval between retries when we create a snapshot content object for a snapshot.") - resyncPeriod = flag.Duration("resync-period", 60*time.Second, "Resync interval of the controller.") - snapshotNamePrefix = flag.String("snapshot-name-prefix", "snapshot", "Prefix to apply to the name of a created snapshot") - snapshotNameUUIDLength = flag.Int("snapshot-name-uuid-length", -1, "Length in characters for the generated uuid of a created snapshot. Defaults behavior is to NOT truncate.") - showVersion = flag.Bool("version", false, "Show version.") - csiTimeout = flag.Duration("timeout", defaultCSITimeout, "The timeout for any RPCs to the CSI driver. Default is 1 minute.") + kubeconfig = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Required only when running out of cluster.") + csiAddress = flag.String("csi-address", "/run/csi/socket", "Address of the CSI driver socket.") + resyncPeriod = flag.Duration("resync-period", 60*time.Second, "Resync interval of the controller.") + snapshotNamePrefix = flag.String("snapshot-name-prefix", "snapshot", "Prefix to apply to the name of a created snapshot") + snapshotNameUUIDLength = flag.Int("snapshot-name-uuid-length", -1, "Length in characters for the generated uuid of a created snapshot. Defaults behavior is to NOT truncate.") + showVersion = flag.Bool("version", false, "Show version.") + csiTimeout = flag.Duration("timeout", defaultCSITimeout, "The timeout for any RPCs to the CSI driver. Default is 1 minute.") leaderElection = flag.Bool("leader-election", false, "Enables leader election.") leaderElectionNamespace = flag.String("leader-election-namespace", "", "The namespace where the leader election resource exists. Defaults to the pod namespace if not set.") @@ -88,14 +84,6 @@ func main() { } klog.Infof("Version: %s", version) - if *connectionTimeout != 0 { - klog.Warning("--connection-timeout is deprecated and will have no effect") - } - - if *snapshotterName != "" { - klog.Warning("--snapshotter is deprecated and will have no effect") - } - // Create the client config. Use kubeconfig if given, otherwise assume in-cluster. config, err := buildConfig(*kubeconfig) if err != nil { @@ -133,13 +121,13 @@ func main() { defer cancel() // Find driver name - *snapshotterName, err = csirpc.GetDriverName(ctx, csiConn) + driverName, err := csirpc.GetDriverName(ctx, csiConn) if err != nil { klog.Errorf("error getting CSI driver name: %v", err) os.Exit(1) } - klog.V(2).Infof("CSI driver name: %q", *snapshotterName) + klog.V(2).Infof("CSI driver name: %q", driverName) // Check it's ready if err = csirpc.ProbeForever(csiConn, *csiTimeout); err != nil { @@ -154,7 +142,7 @@ func main() { os.Exit(1) } if !supportsCreateSnapshot { - klog.Errorf("CSI driver %s does not support ControllerCreateSnapshot", *snapshotterName) + klog.Errorf("CSI driver %s does not support ControllerCreateSnapshot", driverName) os.Exit(1) } @@ -163,19 +151,15 @@ func main() { os.Exit(1) } - klog.V(2).Infof("Start NewCSISnapshotController with snapshotter [%s] kubeconfig [%s] csiTimeout [%+v] csiAddress [%s] createSnapshotContentRetryCount [%d] createSnapshotContentInterval [%+v] resyncPeriod [%+v] snapshotNamePrefix [%s] snapshotNameUUIDLength [%d]", *snapshotterName, *kubeconfig, *csiTimeout, *csiAddress, createSnapshotContentRetryCount, *createSnapshotContentInterval, *resyncPeriod, *snapshotNamePrefix, snapshotNameUUIDLength) + klog.V(2).Infof("Start NewCSISnapshotSideCarController with snapshotter [%s] kubeconfig [%s] csiTimeout [%+v] csiAddress [%s] resyncPeriod [%+v] snapshotNamePrefix [%s] snapshotNameUUIDLength [%d]", driverName, *kubeconfig, *csiTimeout, *csiAddress, *resyncPeriod, *snapshotNamePrefix, snapshotNameUUIDLength) snapShotter := snapshotter.NewSnapshotter(csiConn) - ctrl := controller.NewCSISnapshotController( + ctrl := controller.NewCSISnapshotSideCarController( snapClient, kubeClient, - *snapshotterName, - factory.Snapshot().V1beta1().VolumeSnapshots(), + driverName, factory.Snapshot().V1beta1().VolumeSnapshotContents(), factory.Snapshot().V1beta1().VolumeSnapshotClasses(), - coreFactory.Core().V1().PersistentVolumeClaims(), - *createSnapshotContentRetryCount, - *createSnapshotContentInterval, snapShotter, *csiTimeout, *resyncPeriod, @@ -200,7 +184,7 @@ func main() { if !*leaderElection { run(context.TODO()) } else { - lockName := fmt.Sprintf("%s-%s", prefix, strings.Replace(*snapshotterName, "/", "-", -1)) + lockName := fmt.Sprintf("%s-%s", prefix, strings.Replace(driverName, "/", "-", -1)) le := leaderelection.NewLeaderElection(kubeClient, lockName, run) if *leaderElectionNamespace != "" { le.WithNamespace(*leaderElectionNamespace) diff --git a/cmd/snapshot-controller/Dockerfile b/cmd/snapshot-controller/Dockerfile new file mode 100644 index 00000000..b29db098 --- /dev/null +++ b/cmd/snapshot-controller/Dockerfile @@ -0,0 +1,6 @@ +FROM gcr.io/distroless/static:latest +LABEL maintainers="Kubernetes Authors" +LABEL description="Snapshot Controller" + +COPY ./bin/snapshot-controller snapshot-controller +ENTRYPOINT ["/snapshot-controller"] diff --git a/cmd/snapshot-controller/main.go b/cmd/snapshot-controller/main.go new file mode 100644 index 00000000..9895f350 --- /dev/null +++ b/cmd/snapshot-controller/main.go @@ -0,0 +1,159 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "time" + + "google.golang.org/grpc" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/kubernetes-csi/csi-lib-utils/leaderelection" + csirpc "github.com/kubernetes-csi/csi-lib-utils/rpc" + controller "github.com/kubernetes-csi/external-snapshotter/pkg/common-controller" + + clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" + snapshotscheme "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/scheme" + informers "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions" + coreinformers "k8s.io/client-go/informers" +) + +const ( + // Number of worker threads + threads = 10 +) + +// Command line flags +var ( + kubeconfig = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Required only when running out of cluster.") + createSnapshotContentRetryCount = flag.Int("create-snapshotcontent-retrycount", 5, "Number of retries when we create a snapshot content object for a snapshot.") + createSnapshotContentInterval = flag.Duration("create-snapshotcontent-interval", 10*time.Second, "Interval between retries when we create a snapshot content object for a snapshot.") + resyncPeriod = flag.Duration("resync-period", 60*time.Second, "Resync interval of the controller.") + showVersion = flag.Bool("version", false, "Show version.") + + leaderElection = flag.Bool("leader-election", false, "Enables leader election.") + leaderElectionNamespace = flag.String("leader-election-namespace", "", "The namespace where the leader election resource exists. Defaults to the pod namespace if not set.") +) + +var ( + version = "unknown" +) + +func main() { + klog.InitFlags(nil) + flag.Set("logtostderr", "true") + flag.Parse() + + if *showVersion { + fmt.Println(os.Args[0], version) + os.Exit(0) + } + klog.Infof("Version: %s", version) + + // Create the client config. Use kubeconfig if given, otherwise assume in-cluster. + config, err := buildConfig(*kubeconfig) + if err != nil { + klog.Error(err.Error()) + os.Exit(1) + } + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + klog.Error(err.Error()) + os.Exit(1) + } + + snapClient, err := clientset.NewForConfig(config) + if err != nil { + klog.Errorf("Error building snapshot clientset: %s", err.Error()) + os.Exit(1) + } + + factory := informers.NewSharedInformerFactory(snapClient, *resyncPeriod) + coreFactory := coreinformers.NewSharedInformerFactory(kubeClient, *resyncPeriod) + + // Add Snapshot types to the defualt Kubernetes so events can be logged for them + snapshotscheme.AddToScheme(scheme.Scheme) + + klog.V(2).Infof("Start NewCSISnapshotController with kubeconfig [%s] createSnapshotContentRetryCount [%d] createSnapshotContentInterval [%d] resyncPeriod [%+v]", *kubeconfig, *createSnapshotContentRetryCount, *createSnapshotContentInterval, *resyncPeriod) + + ctrl := controller.NewCSISnapshotCommonController( + snapClient, + kubeClient, + factory.Snapshot().V1beta1().VolumeSnapshots(), + factory.Snapshot().V1beta1().VolumeSnapshotContents(), + factory.Snapshot().V1beta1().VolumeSnapshotClasses(), + coreFactory.Core().V1().PersistentVolumeClaims(), + *createSnapshotContentRetryCount, + *createSnapshotContentInterval, + *resyncPeriod, + ) + + run := func(context.Context) { + // run... + stopCh := make(chan struct{}) + factory.Start(stopCh) + coreFactory.Start(stopCh) + go ctrl.Run(threads, stopCh) + + // ...until SIGINT + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + <-c + close(stopCh) + } + + if !*leaderElection { + run(context.TODO()) + } else { + lockName := "snapshot-controller-leader" + le := leaderelection.NewLeaderElection(kubeClient, lockName, run) + if *leaderElectionNamespace != "" { + le.WithNamespace(*leaderElectionNamespace) + } + if err := le.Run(); err != nil { + klog.Fatalf("failed to initialize leader election: %v", err) + } + } +} + +func buildConfig(kubeconfig string) (*rest.Config, error) { + if kubeconfig != "" { + return clientcmd.BuildConfigFromFlags("", kubeconfig) + } + return rest.InClusterConfig() +} + +func supportsControllerCreateSnapshot(ctx context.Context, conn *grpc.ClientConn) (bool, error) { + capabilities, err := csirpc.GetControllerCapabilities(ctx, conn) + if err != nil { + return false, err + } + + return capabilities[csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT], nil +} diff --git a/cmd/snapshot-controller/main_test.go b/cmd/snapshot-controller/main_test.go new file mode 100644 index 00000000..f13aba72 --- /dev/null +++ b/cmd/snapshot-controller/main_test.go @@ -0,0 +1,161 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + "testing" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/mock/gomock" + "github.com/kubernetes-csi/csi-lib-utils/connection" + "github.com/kubernetes-csi/csi-test/driver" + + "google.golang.org/grpc" +) + +func Test_supportsControllerCreateSnapshot(t *testing.T) { + tests := []struct { + name string + output *csi.ControllerGetCapabilitiesResponse + injectError bool + expectError bool + expectResult bool + }{ + { + name: "success", + output: &csi.ControllerGetCapabilitiesResponse{ + Capabilities: []*csi.ControllerServiceCapability{ + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, + }, + }, + }, + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, + }, + }, + }, + }, + }, + expectError: false, + expectResult: true, + }, + { + name: "gRPC error", + output: nil, + injectError: true, + expectError: true, + expectResult: false, + }, + { + name: "no create snapshot", + output: &csi.ControllerGetCapabilitiesResponse{ + Capabilities: []*csi.ControllerServiceCapability{ + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, + }, + }, + }, + }, + }, + expectError: false, + expectResult: false, + }, + { + name: "empty capability", + output: &csi.ControllerGetCapabilitiesResponse{ + Capabilities: []*csi.ControllerServiceCapability{ + { + Type: nil, + }, + }, + }, + expectError: false, + expectResult: false, + }, + { + name: "no capabilities", + output: &csi.ControllerGetCapabilitiesResponse{ + Capabilities: []*csi.ControllerServiceCapability{}, + }, + expectError: false, + expectResult: false, + }, + } + + mockController, driver, _, controllerServer, csiConn, err := createMockServer(t) + if err != nil { + t.Fatal(err) + } + defer mockController.Finish() + defer driver.Stop() + defer csiConn.Close() + + for _, test := range tests { + + in := &csi.ControllerGetCapabilitiesRequest{} + + out := test.output + var injectedErr error + if test.injectError { + injectedErr = fmt.Errorf("mock error") + } + + // Setup expectation + controllerServer.EXPECT().ControllerGetCapabilities(gomock.Any(), in).Return(out, injectedErr).Times(1) + + ok, err := supportsControllerCreateSnapshot(context.Background(), csiConn) + if test.expectError && err == nil { + t.Errorf("test %q: Expected error, got none", test.name) + } + if !test.expectError && err != nil { + t.Errorf("test %q: got error: %v", test.name, err) + } + if err == nil && test.expectResult != ok { + t.Errorf("test fail expected result %t but got %t\n", test.expectResult, ok) + } + } +} + +func createMockServer(t *testing.T) (*gomock.Controller, *driver.MockCSIDriver, *driver.MockIdentityServer, *driver.MockControllerServer, *grpc.ClientConn, error) { + // Start the mock server + mockController := gomock.NewController(t) + identityServer := driver.NewMockIdentityServer(mockController) + controllerServer := driver.NewMockControllerServer(mockController) + drv := driver.NewMockCSIDriver(&driver.MockCSIDriverServers{ + Identity: identityServer, + Controller: controllerServer, + }) + drv.Start() + + // Create a client connection to it + addr := drv.Address() + csiConn, err := connection.Connect(addr) + if err != nil { + return nil, nil, nil, nil, nil, err + } + + return mockController, drv, identityServer, controllerServer, csiConn, nil +} diff --git a/deploy/kubernetes/README.md b/deploy/kubernetes/csi-snapshotter/README.md similarity index 100% rename from deploy/kubernetes/README.md rename to deploy/kubernetes/csi-snapshotter/README.md diff --git a/deploy/kubernetes/rbac.yaml b/deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml similarity index 79% rename from deploy/kubernetes/rbac.yaml rename to deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml index 5bed69e6..9ca4e00a 100644 --- a/deploy/kubernetes/rbac.yaml +++ b/deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml @@ -20,15 +20,6 @@ metadata: # rename if there are conflicts name: external-snapshotter-runner rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] @@ -48,15 +39,6 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents/status"] verbs: ["update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] --- kind: ClusterRoleBinding diff --git a/deploy/kubernetes/rbac-external-provisioner.yaml b/deploy/kubernetes/csi-snapshotter/rbac-external-provisioner.yaml similarity index 100% rename from deploy/kubernetes/rbac-external-provisioner.yaml rename to deploy/kubernetes/csi-snapshotter/rbac-external-provisioner.yaml diff --git a/deploy/kubernetes/setup-csi-snapshotter.yaml b/deploy/kubernetes/csi-snapshotter/setup-csi-snapshotter.yaml similarity index 86% rename from deploy/kubernetes/setup-csi-snapshotter.yaml rename to deploy/kubernetes/csi-snapshotter/setup-csi-snapshotter.yaml index eed194f2..3555b09f 100644 --- a/deploy/kubernetes/setup-csi-snapshotter.yaml +++ b/deploy/kubernetes/csi-snapshotter/setup-csi-snapshotter.yaml @@ -1,6 +1,6 @@ # This YAML file shows how to deploy the CSI snapshotter together # with the hostpath CSI driver. It depends on the RBAC rules -# from rbac.yaml and rbac-external-provisioner.yaml. +# from rbac-csi-snapshotter.yaml and rbac-external-provisioner.yaml. # # Because external-snapshotter and external-provisioner get # deployed in the same pod, we have to merge the permissions @@ -72,11 +72,10 @@ spec: serviceAccount: csi-snapshotter containers: - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:v1.3.0 + image: quay.io/k8scsi/csi-provisioner:v1.5.0-rc1 args: - - "--provisioner=csi-hostpath" + - "--v=5" - "--csi-address=$(ADDRESS)" - - "--connection-timeout=15s" env: - name: ADDRESS value: /csi/csi.sock @@ -85,20 +84,21 @@ spec: - name: socket-dir mountPath: /csi - name: csi-snapshotter - image: quay.io/k8scsi/csi-snapshotter:v1.2.0 + # NOTE: replace with official image when released: quay.io/k8scsi/csi-snapshotter:v2.0.0 + image: csi-snapshotter:testbeta #quay.io/k8scsi/csi-snapshotter:testbeta args: + - "--v=5" - "--csi-address=$(ADDRESS)" - - "--connection-timeout=15s" - "--leader-election=false" env: - name: ADDRESS value: /csi/csi.sock - imagePullPolicy: Always + imagePullPolicy: IfNotPresent #Always volumeMounts: - name: socket-dir mountPath: /csi - name: hostpath - image: quay.io/k8scsi/hostpathplugin:v1.1.0 + image: quay.io/k8scsi/hostpathplugin:v1.2.0 args: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" diff --git a/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml b/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml new file mode 100644 index 00000000..72e0d157 --- /dev/null +++ b/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml @@ -0,0 +1,80 @@ +# RBAC file for the snapshot controller. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: snapshot-controller + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # rename if there are conflicts + name: snapshot-controller-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-role +subjects: + - kind: ServiceAccount + name: snapshot-controller + # replace with non-default namespace name + namespace: default +roleRef: + kind: ClusterRole + # change the name also here if the ClusterRole gets renamed + name: snapshot-controller-runner + apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: default # TODO: replace with the namespace you want for your sidecar + name: snapshot-controller-leaderelection +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-leaderelection + namespace: default # TODO: replace with the namespace you want for your sidecar +subjects: + - kind: ServiceAccount + name: snapshot-controller + namespace: default # TODO: replace with the namespace you want for your sidecar +roleRef: + kind: Role + name: snapshot-controller-leaderelection + apiGroup: rbac.authorization.k8s.io + diff --git a/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml b/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml new file mode 100644 index 00000000..853a35a4 --- /dev/null +++ b/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml @@ -0,0 +1,27 @@ +# This YAML file shows how to deploy the snapshot controller + +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: snapshot-controller +spec: + serviceName: "snapshot-controller" + replicas: 1 + selector: + matchLabels: + app: snapshot-controller + template: + metadata: + labels: + app: snapshot-controller + spec: + serviceAccount: snapshot-controller + containers: + - name: snapshot-controller + # NOTE: replace with official image when released: quay.io/k8scsi/snapshot-controller:v2.0.0 + image: snapshot-controller:testbeta #quay.io/k8scsi/snapshot-controller:testbeta + args: + - "--v=5" + - "--leader-election=false" + imagePullPolicy: IfNotPresent #Always diff --git a/examples/kubernetes/snapshot.yaml b/examples/kubernetes/snapshot.yaml index b7a913f9..86a102b8 100644 --- a/examples/kubernetes/snapshot.yaml +++ b/examples/kubernetes/snapshot.yaml @@ -1,9 +1,8 @@ -apiVersion: snapshot.storage.k8s.io/v1alpha1 +apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshot metadata: name: new-snapshot-demo spec: - snapshotClassName: csi-hostpath-snapclass + volumeSnapshotClassName: csi-hostpath-snapclass source: - name: hpvc - kind: PersistentVolumeClaim + persistentVolumeClaimName: hpvc diff --git a/examples/kubernetes/snapshotclass.yaml b/examples/kubernetes/snapshotclass.yaml index dfa34df5..892dfd0c 100644 --- a/examples/kubernetes/snapshotclass.yaml +++ b/examples/kubernetes/snapshotclass.yaml @@ -1,5 +1,6 @@ -apiVersion: snapshot.storage.k8s.io/v1alpha1 +apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshotClass metadata: name: csi-hostpath-snapclass -snapshotter: csi-hostpath +driver: hostpath.csi.k8s.io #csi-hostpath +deletionPolicy: Delete diff --git a/examples/kubernetes/storageclass.yaml b/examples/kubernetes/storageclass.yaml index c9279716..59999a8c 100644 --- a/examples/kubernetes/storageclass.yaml +++ b/examples/kubernetes/storageclass.yaml @@ -2,6 +2,6 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: csi-hostpath-sc -provisioner: csi-hostpath +provisioner: hostpath.csi.k8s.io #csi-hostpath reclaimPolicy: Delete volumeBindingMode: Immediate diff --git a/pkg/controller/framework_test.go b/pkg/common-controller/framework_test.go similarity index 92% rename from pkg/controller/framework_test.go rename to pkg/common-controller/framework_test.go index b2120843..82531eae 100644 --- a/pkg/controller/framework_test.go +++ b/pkg/common-controller/framework_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package common_controller import ( "context" @@ -35,6 +35,7 @@ import ( snapshotscheme "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/scheme" informers "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions" storagelisters "github.com/kubernetes-csi/external-snapshotter/pkg/client/listers/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/pkg/utils" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -115,7 +116,7 @@ type controllerTest struct { expectSuccess bool } -type testCall func(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest) error +type testCall func(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest) error const testNamespace = "default" const mockDriverName = "csi-mock-plugin" @@ -152,7 +153,7 @@ type snapshotReactor struct { snapshots map[string]*crdv1.VolumeSnapshot changedObjects []interface{} changedSinceLastSync int - ctrl *csiSnapshotController + ctrl *csiSnapshotCommonController fakeContentWatch *watch.FakeWatcher fakeSnapshotWatch *watch.FakeWatcher lock sync.Mutex @@ -170,17 +171,19 @@ type reactorError struct { } func withSnapshotFinalizer(snapshot *crdv1.VolumeSnapshot) *crdv1.VolumeSnapshot { - snapshot.ObjectMeta.Finalizers = append(snapshot.ObjectMeta.Finalizers, VolumeSnapshotFinalizer) + snapshot.ObjectMeta.Finalizers = append(snapshot.ObjectMeta.Finalizers, utils.VolumeSnapshotAsSourceFinalizer) + snapshot.ObjectMeta.Finalizers = append(snapshot.ObjectMeta.Finalizers, utils.VolumeSnapshotBoundFinalizer) return snapshot } func withContentFinalizer(content *crdv1.VolumeSnapshotContent) *crdv1.VolumeSnapshotContent { - content.ObjectMeta.Finalizers = append(content.ObjectMeta.Finalizers, VolumeSnapshotContentFinalizer) + content.ObjectMeta.Finalizers = append(content.ObjectMeta.Finalizers, utils.VolumeSnapshotContentFinalizer) + metav1.SetMetaDataAnnotation(&content.ObjectMeta, utils.AnnVolumeSnapshotBeingDeleted, "yes") return content } func withPVCFinalizer(pvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim { - pvc.ObjectMeta.Finalizers = append(pvc.ObjectMeta.Finalizers, PVCFinalizer) + pvc.ObjectMeta.Finalizers = append(pvc.ObjectMeta.Finalizers, utils.PVCFinalizer) return pvc } @@ -423,7 +426,9 @@ func (r *snapshotReactor) checkContents(expectedContents []*crdv1.VolumeSnapshot v := v.DeepCopy() v.ResourceVersion = "" v.Spec.VolumeSnapshotRef.ResourceVersion = "" - v.Status.CreationTime = nil + if v.Status != nil { + v.Status.CreationTime = nil + } expectedMap[v.Name] = v } for _, v := range r.contents { @@ -432,7 +437,9 @@ func (r *snapshotReactor) checkContents(expectedContents []*crdv1.VolumeSnapshot v := v.DeepCopy() v.ResourceVersion = "" v.Spec.VolumeSnapshotRef.ResourceVersion = "" - v.Status.CreationTime = nil + if v.Status != nil { + v.Status.CreationTime = nil + } gotMap[v.Name] = v } if !reflect.DeepEqual(expectedMap, gotMap) { @@ -480,7 +487,7 @@ func (r *snapshotReactor) checkSnapshots(expectedSnapshots []*crdv1.VolumeSnapsh // checkEvents compares all expectedEvents with events generated during the test // and reports differences. -func checkEvents(t *testing.T, expectedEvents []string, ctrl *csiSnapshotController) error { +func checkEvents(t *testing.T, expectedEvents []string, ctrl *csiSnapshotCommonController) error { var err error // Read recorded events - wait up to 1 minute to get all the expected ones @@ -699,7 +706,7 @@ func (r *snapshotReactor) addSnapshotEvent(snapshot *crdv1.VolumeSnapshot) { } } -func newSnapshotReactor(kubeClient *kubefake.Clientset, client *fake.Clientset, ctrl *csiSnapshotController, fakeVolumeWatch, fakeClaimWatch *watch.FakeWatcher, errors []reactorError) *snapshotReactor { +func newSnapshotReactor(kubeClient *kubefake.Clientset, client *fake.Clientset, ctrl *csiSnapshotCommonController, fakeVolumeWatch, fakeClaimWatch *watch.FakeWatcher, errors []reactorError) *snapshotReactor { reactor := &snapshotReactor{ secrets: make(map[string]*v1.Secret), storageClasses: make(map[string]*storagev1.StorageClass), @@ -732,36 +739,31 @@ func newSnapshotReactor(kubeClient *kubefake.Clientset, client *fake.Clientset, func alwaysReady() bool { return true } func newTestController(kubeClient kubernetes.Interface, clientset clientset.Interface, - informerFactory informers.SharedInformerFactory, t *testing.T, test controllerTest) (*csiSnapshotController, error) { + informerFactory informers.SharedInformerFactory, t *testing.T, test controllerTest) (*csiSnapshotCommonController, error) { if informerFactory == nil { - informerFactory = informers.NewSharedInformerFactory(clientset, NoResyncPeriodFunc()) + informerFactory = informers.NewSharedInformerFactory(clientset, utils.NoResyncPeriodFunc()) } - coreFactory := coreinformers.NewSharedInformerFactory(kubeClient, NoResyncPeriodFunc()) + coreFactory := coreinformers.NewSharedInformerFactory(kubeClient, utils.NoResyncPeriodFunc()) // Construct controller - fakeSnapshot := &fakeSnapshotter{ - t: t, - listCalls: test.expectedListCalls, - createCalls: test.expectedCreateCalls, - deleteCalls: test.expectedDeleteCalls, - } + //fakeSnapshot := &fakeSnapshotter{ + // t: t, + // listCalls: test.expectedListCalls, + // createCalls: test.expectedCreateCalls, + // deleteCalls: test.expectedDeleteCalls, + //} - ctrl := NewCSISnapshotController( + ctrl := NewCSISnapshotCommonController( clientset, kubeClient, - mockDriverName, informerFactory.Snapshot().V1beta1().VolumeSnapshots(), informerFactory.Snapshot().V1beta1().VolumeSnapshotContents(), informerFactory.Snapshot().V1beta1().VolumeSnapshotClasses(), coreFactory.Core().V1().PersistentVolumeClaims(), 3, 5*time.Millisecond, - fakeSnapshot, - 5*time.Millisecond, 60*time.Second, - "snapshot", - -1, ) ctrl.eventRecorder = record.NewFakeRecorder(1000) @@ -776,7 +778,8 @@ func newTestController(kubeClient kubernetes.Interface, clientset clientset.Inte func newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle string, deletionPolicy crdv1.DeletionPolicy, creationTime, size *int64, - withFinalizer bool) *crdv1.VolumeSnapshotContent { + withFinalizer bool, withStatus bool) *crdv1.VolumeSnapshotContent { + ready := true content := crdv1.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: contentName, @@ -786,13 +789,17 @@ func newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHa Driver: mockDriverName, DeletionPolicy: deletionPolicy, }, - Status: &crdv1.VolumeSnapshotContentStatus{ - CreationTime: creationTime, - RestoreSize: size, - }, } - if snapshotHandle != "" { + if withStatus { + content.Status = &crdv1.VolumeSnapshotContentStatus{ + CreationTime: creationTime, + RestoreSize: size, + ReadyToUse: &ready, + } + } + + if withStatus && snapshotHandle != "" { content.Status.SnapshotHandle = &snapshotHandle } @@ -830,14 +837,22 @@ func newContentArray(contentName, boundToSnapshotUID, boundToSnapshotName, snaps deletionPolicy crdv1.DeletionPolicy, size, creationTime *int64, withFinalizer bool) []*crdv1.VolumeSnapshotContent { return []*crdv1.VolumeSnapshotContent{ - newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, creationTime, size, withFinalizer), + newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, creationTime, size, withFinalizer, true), + } +} + +func newContentArrayNoStatus(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle string, + deletionPolicy crdv1.DeletionPolicy, size, creationTime *int64, + withFinalizer bool, withStatus bool) []*crdv1.VolumeSnapshotContent { + return []*crdv1.VolumeSnapshotContent{ + newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, creationTime, size, withFinalizer, withStatus), } } func newContentArrayWithReadyToUse(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle string, deletionPolicy crdv1.DeletionPolicy, creationTime, size *int64, readyToUse *bool, withFinalizer bool) []*crdv1.VolumeSnapshotContent { - content := newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, creationTime, size, withFinalizer) + content := newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, creationTime, size, withFinalizer, true) content.Status.ReadyToUse = readyToUse return []*crdv1.VolumeSnapshotContent{ content, @@ -847,7 +862,7 @@ func newContentArrayWithReadyToUse(contentName, boundToSnapshotUID, boundToSnaps func newContentWithUnmatchDriverArray(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle string, deletionPolicy crdv1.DeletionPolicy, size, creationTime *int64, withFinalizer bool) []*crdv1.VolumeSnapshotContent { - content := newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, size, creationTime, withFinalizer) + content := newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, size, creationTime, withFinalizer, true) content.Spec.Driver = "fake" return []*crdv1.VolumeSnapshotContent{ content, @@ -857,7 +872,7 @@ func newContentWithUnmatchDriverArray(contentName, boundToSnapshotUID, boundToSn func newSnapshot( snapshotName, snapshotUID, pvcName, targetContentName, snapshotClassName, boundContentName string, readyToUse *bool, creationTime *metav1.Time, restoreSize *resource.Quantity, - err *crdv1.VolumeSnapshotError) *crdv1.VolumeSnapshot { + err *crdv1.VolumeSnapshotError, nilStatus bool) *crdv1.VolumeSnapshot { snapshot := crdv1.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: snapshotName, @@ -869,12 +884,15 @@ func newSnapshot( Spec: crdv1.VolumeSnapshotSpec{ VolumeSnapshotClassName: nil, }, - Status: &crdv1.VolumeSnapshotStatus{ + } + + if !nilStatus { + snapshot.Status = &crdv1.VolumeSnapshotStatus{ CreationTime: creationTime, ReadyToUse: readyToUse, Error: err, RestoreSize: restoreSize, - }, + } } if boundContentName != "" { @@ -898,9 +916,9 @@ func newSnapshot( func newSnapshotArray( snapshotName, snapshotUID, pvcName, targetContentName, snapshotClassName, boundContentName string, readyToUse *bool, creationTime *metav1.Time, restoreSize *resource.Quantity, - err *crdv1.VolumeSnapshotError) []*crdv1.VolumeSnapshot { + err *crdv1.VolumeSnapshotError, nilStatus bool) []*crdv1.VolumeSnapshot { return []*crdv1.VolumeSnapshot{ - newSnapshot(snapshotName, snapshotUID, pvcName, targetContentName, snapshotClassName, boundContentName, readyToUse, creationTime, restoreSize, err), + newSnapshot(snapshotName, snapshotUID, pvcName, targetContentName, snapshotClassName, boundContentName, readyToUse, creationTime, restoreSize, err, nilStatus), } } @@ -1015,11 +1033,11 @@ func newVolumeError(message string) *crdv1.VolumeSnapshotError { } } -func testSyncSnapshot(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest) error { +func testSyncSnapshot(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest) error { return ctrl.syncSnapshot(test.initialSnapshots[0]) } -func testSyncSnapshotError(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest) error { +func testSyncSnapshotError(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest) error { err := ctrl.syncSnapshot(test.initialSnapshots[0]) if err != nil { @@ -1028,16 +1046,16 @@ func testSyncSnapshotError(ctrl *csiSnapshotController, reactor *snapshotReactor return fmt.Errorf("syncSnapshot succeeded when failure was expected") } -func testSyncContent(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest) error { +func testSyncContent(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest) error { return ctrl.syncContent(test.initialContents[0]) } -func testAddPVCFinalizer(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest) error { - return ctrl.ensureSnapshotSourceFinalizer(test.initialSnapshots[0]) +func testAddPVCFinalizer(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest) error { + return ctrl.ensurePVCFinalizer(test.initialSnapshots[0]) } -func testRemovePVCFinalizer(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest) error { - return ctrl.checkandRemoveSnapshotSourceFinalizer(test.initialSnapshots[0]) +func testRemovePVCFinalizer(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest) error { + return ctrl.checkandRemovePVCFinalizer(test.initialSnapshots[0]) } var ( @@ -1062,9 +1080,9 @@ var ( // injected function to simulate that something is happening when the // controller waits for the operation lock. Controller is then resumed and we // check how it behaves. -func wrapTestWithInjectedOperation(toWrap testCall, injectBeforeOperation func(ctrl *csiSnapshotController, reactor *snapshotReactor)) testCall { +func wrapTestWithInjectedOperation(toWrap testCall, injectBeforeOperation func(ctrl *csiSnapshotCommonController, reactor *snapshotReactor)) testCall { - return func(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest) error { + return func(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest) error { // Inject a hook before async operation starts klog.V(4).Infof("reactor:injecting call") injectBeforeOperation(ctrl, reactor) @@ -1089,7 +1107,7 @@ func wrapTestWithInjectedOperation(toWrap testCall, injectBeforeOperation func(c } } -func evaluateTestResults(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest, t *testing.T) { +func evaluateTestResults(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest, t *testing.T) { // Evaluate results if err := reactor.checkSnapshots(test.expectedSnapshots); err != nil { t.Errorf("Test %q: %v", test.name, err) @@ -1130,10 +1148,8 @@ func runSyncTests(t *testing.T, tests []controllerTest, snapshotClasses []*crdv1 reactor.snapshots[snapshot.Name] = snapshot } for _, content := range test.initialContents { - if ctrl.isDriverMatch(test.initialContents[0]) { - ctrl.contentStore.Add(content) - reactor.contents[content.Name] = content - } + ctrl.contentStore.Add(content) + reactor.contents[content.Name] = content } pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) @@ -1162,7 +1178,7 @@ func runSyncTests(t *testing.T, tests []controllerTest, snapshotClasses []*crdv1 // Run the tested functions err = test.test(ctrl, reactor, test) - if err != nil { + if test.expectSuccess && err != nil { t.Errorf("Test %q failed: %v", test.name, err) } @@ -1176,7 +1192,7 @@ func runSyncTests(t *testing.T, tests []controllerTest, snapshotClasses []*crdv1 } } -// This tests ensureSnapshotSourceFinalizer and checkandRemoveSnapshotSourceFinalizer +// This tests ensurePVCFinalizer and checkandRemovePVCFinalizer func runPVCFinalizerTests(t *testing.T, tests []controllerTest, snapshotClasses []*crdv1.VolumeSnapshotClass) { snapshotscheme.AddToScheme(scheme.Scheme) for _, test := range tests { @@ -1197,10 +1213,8 @@ func runPVCFinalizerTests(t *testing.T, tests []controllerTest, snapshotClasses reactor.snapshots[snapshot.Name] = snapshot } for _, content := range test.initialContents { - if ctrl.isDriverMatch(test.initialContents[0]) { - ctrl.contentStore.Add(content) - reactor.contents[content.Name] = content - } + ctrl.contentStore.Add(content) + reactor.contents[content.Name] = content } pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) @@ -1239,7 +1253,7 @@ func runPVCFinalizerTests(t *testing.T, tests []controllerTest, snapshotClasses } // Evaluate PVCFinalizer tests results -func evaluatePVCFinalizerTests(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest, t *testing.T) { +func evaluatePVCFinalizerTests(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest, t *testing.T) { // Evaluate results bHasPVCFinalizer := false name := sysruntime.FuncForPC(reflect.ValueOf(test.test).Pointer()).Name() @@ -1255,7 +1269,7 @@ func evaluatePVCFinalizerTests(ctrl *csiSnapshotController, reactor *snapshotRea if funcName == "testAddPVCFinalizer" { for _, pvc := range reactor.claims { if test.initialClaims[0].Name == pvc.Name { - if !slice.ContainsString(test.initialClaims[0].ObjectMeta.Finalizers, PVCFinalizer, nil) && slice.ContainsString(pvc.ObjectMeta.Finalizers, PVCFinalizer, nil) { + if !slice.ContainsString(test.initialClaims[0].ObjectMeta.Finalizers, utils.PVCFinalizer, nil) && slice.ContainsString(pvc.ObjectMeta.Finalizers, utils.PVCFinalizer, nil) { klog.V(4).Infof("test %q succeeded. PVCFinalizer is added to PVC %s", test.name, pvc.Name) bHasPVCFinalizer = true } @@ -1270,7 +1284,7 @@ func evaluatePVCFinalizerTests(ctrl *csiSnapshotController, reactor *snapshotRea if funcName == "testRemovePVCFinalizer" { for _, pvc := range reactor.claims { if test.initialClaims[0].Name == pvc.Name { - if slice.ContainsString(test.initialClaims[0].ObjectMeta.Finalizers, PVCFinalizer, nil) && !slice.ContainsString(pvc.ObjectMeta.Finalizers, PVCFinalizer, nil) { + if slice.ContainsString(test.initialClaims[0].ObjectMeta.Finalizers, utils.PVCFinalizer, nil) && !slice.ContainsString(pvc.ObjectMeta.Finalizers, utils.PVCFinalizer, nil) { klog.V(4).Infof("test %q succeeded. PVCFinalizer is removed from PVC %s", test.name, pvc.Name) bHasPVCFinalizer = false } @@ -1310,23 +1324,23 @@ func secret() *v1.Secret { func secretAnnotations() map[string]string { return map[string]string{ - AnnDeletionSecretRefName: "secret", - AnnDeletionSecretRefNamespace: "default", + utils.AnnDeletionSecretRefName: "secret", + utils.AnnDeletionSecretRefNamespace: "default", } } func emptyNamespaceSecretAnnotations() map[string]string { return map[string]string{ - AnnDeletionSecretRefName: "name", - AnnDeletionSecretRefNamespace: "", + utils.AnnDeletionSecretRefName: "name", + utils.AnnDeletionSecretRefNamespace: "", } } // this refers to emptySecret(), which is missing data. func emptyDataSecretAnnotations() map[string]string { return map[string]string{ - AnnDeletionSecretRefName: "emptysecret", - AnnDeletionSecretRefNamespace: "default", + utils.AnnDeletionSecretRefName: "emptysecret", + utils.AnnDeletionSecretRefNamespace: "default", } } diff --git a/pkg/common-controller/snapshot_controller.go b/pkg/common-controller/snapshot_controller.go new file mode 100644 index 00000000..ec5842df --- /dev/null +++ b/pkg/common-controller/snapshot_controller.go @@ -0,0 +1,1190 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common_controller + +import ( + "fmt" + "strings" + "time" + + crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/pkg/utils" + "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes/scheme" + ref "k8s.io/client-go/tools/reference" + "k8s.io/klog" + "k8s.io/kubernetes/pkg/util/slice" +) + +// ================================================================== +// PLEASE DO NOT ATTEMPT TO SIMPLIFY THIS CODE. +// KEEP THE SPACE SHUTTLE FLYING. +// ================================================================== + +// Design: +// +// The fundamental key to this design is the bi-directional "pointer" between +// VolumeSnapshots and VolumeSnapshotContents, which is represented here +// as snapshot.Status.BoundVolumeSnapshotContentName and content.Spec.VolumeSnapshotRef. +// The bi-directionality is complicated to manage in a transactionless system, but +// without it we can't ensure sane behavior in the face of different forms of +// trouble. For example, a rogue HA controller instance could end up racing +// and making multiple bindings that are indistinguishable, resulting in +// potential data loss. +// +// This controller is designed to work in active-passive high availability +// mode. It *could* work also in active-active HA mode, all the object +// transitions are designed to cope with this, however performance could be +// lower as these two active controllers will step on each other toes +// frequently. +// +// This controller supports both dynamic snapshot creation and pre-bound snapshot. +// In pre-bound mode, objects are created with pre-defined pointers: a VolumeSnapshot +// points to a specific VolumeSnapshotContent and the VolumeSnapshotContent also +// points back for this VolumeSnapshot. +// +// The snapshot controller is split into two controllers in its beta phase: a +// common controller that is deployed on the kubernetes master node and a sidecar +// controller that is deployed with the CSI driver. + +// The dynamic snapshot creation is multi-step process: first common controller +// creates snapshot content object, then the snapshot sidecar triggers snapshot +// creation though csi volume driver and updates snapshot content status with +// snapshotHandle, creationTime, restoreSize, readyToUse, and error fields. The +// common controller updates snapshot status based on content status until +// bi-directional binding is complete and readyToUse becomes true. Error field +// in the snapshot status will be updated accordingly when failure occurrs. + +const snapshotKind = "VolumeSnapshot" +const snapshotAPIGroup = crdv1.GroupName + +const controllerUpdateFailMsg = "snapshot controller failed to update" + +// syncContent deals with one key off the queue. It returns false when it's time to quit. +func (ctrl *csiSnapshotCommonController) syncContent(content *crdv1.VolumeSnapshotContent) error { + klog.V(5).Infof("synchronizing VolumeSnapshotContent[%s]", content.Name) + + snapshotName := utils.SnapshotRefKey(&content.Spec.VolumeSnapshotRef) + + if utils.NeedToAddContentFinalizer(content) { + // Content is not being deleted -> it should have the finalizer. + klog.V(5).Infof("syncContent: Add Finalizer for VolumeSnapshotContent[%s]", content.Name) + return ctrl.addContentFinalizer(content) + } + + klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: content is bound to snapshot %s", content.Name, snapshotName) + // The VolumeSnapshotContent is reserved for a VolumeSnapshot; + // that VolumeSnapshot has not yet been bound to this VolumeSnapshotContent; the VolumeSnapshot sync will handle it. + if content.Spec.VolumeSnapshotRef.UID == "" { + klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: VolumeSnapshotContent is pre-bound to VolumeSnapshot %s", content.Name, snapshotName) + return nil + } + // Get the VolumeSnapshot by _name_ + var snapshot *crdv1.VolumeSnapshot + obj, found, err := ctrl.snapshotStore.GetByKey(snapshotName) + if err != nil { + return err + } + if !found { + klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: snapshot %s not found", content.Name, snapshotName) + // Fall through with snapshot = nil + } else { + var ok bool + snapshot, ok = obj.(*crdv1.VolumeSnapshot) + if !ok { + return fmt.Errorf("cannot convert object from snapshot cache to snapshot %q!?: %#v", content.Name, obj) + } + klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: snapshot %s found", content.Name, snapshotName) + } + if snapshot != nil && snapshot.UID != content.Spec.VolumeSnapshotRef.UID { + // The snapshot that the content was pointing to was deleted, and another + // with the same name created. + klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: content %s has different UID, the old one must have been deleted", content.Name, snapshotName) + // Treat the content as bound to a missing snapshot. + snapshot = nil + } else { + // Check if content status is set to true and update snapshot status if so + if snapshot != nil && content.Status != nil && content.Status.ReadyToUse != nil && *content.Status.ReadyToUse == true { + klog.V(4).Infof("synchronizing VolumeSnapshotContent for snapshot [%s]: update snapshot status to true if needed.", snapshotName) + // Manually trigger a snapshot status update to happen + // right away so that it is in-sync with the content status + ctrl.snapshotQueue.Add(snapshotName) + } + } + + // Trigger content deletion if snapshot has deletion + // timestamp or snapshot does not exist any more + // If snapshot has deletion timestamp and finalizers, set + // AnnVolumeSnapshotBeingDeleted annotation on the content. + // This may trigger the deletion of the content in the + // sidecar controller depending on the deletion policy + // on the content. + // Snapshot won't be deleted until content is deleted + // due to the finalizer + if snapshot == nil || utils.IsSnapshotDeletionCandidate(snapshot) { + // Set AnnVolumeSnapshotBeingDeleted if it is not set yet + if !metav1.HasAnnotation(content.ObjectMeta, utils.AnnVolumeSnapshotBeingDeleted) { + klog.V(5).Infof("syncContent: set annotation [%s] on content [%s].", utils.AnnVolumeSnapshotBeingDeleted, content.Name) + metav1.SetMetaDataAnnotation(&content.ObjectMeta, utils.AnnVolumeSnapshotBeingDeleted, "yes") + + updateContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(content) + if err != nil { + return newControllerUpdateError(content.Name, err.Error()) + } + + _, err = ctrl.storeContentUpdate(updateContent) + if err != nil { + klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status: cannot update internal cache %v", content.Name, err) + return err + } + klog.V(5).Infof("syncContent: volume snapshot content %+v", content) + } + } + + return nil +} + +// syncSnapshot is the main controller method to decide what to do with a snapshot. +// It's invoked by appropriate cache.Controller callbacks when a snapshot is +// created, updated or periodically synced. We do not differentiate between +// these events. +// For easier readability, it is split into syncUnreadySnapshot and syncReadySnapshot +func (ctrl *csiSnapshotCommonController) syncSnapshot(snapshot *crdv1.VolumeSnapshot) error { + klog.V(5).Infof("synchronizing VolumeSnapshot[%s]: %s", utils.SnapshotKey(snapshot), utils.GetSnapshotStatusForLogging(snapshot)) + + err := ctrl.processFinalizersAndCheckandDeleteContent(snapshot) + if err != nil { + return err + } + + if !utils.IsSnapshotReady(snapshot) { + return ctrl.syncUnreadySnapshot(snapshot) + } + return ctrl.syncReadySnapshot(snapshot) +} + +// processFinalizersAndCheckandDeleteContent processes finalizers and deletes the content when appropriate +// It checks if contents exists, it checks if snapshot has bi-directional binding, it checks if +// finalizers should be added or removed, and it checks if content should be deleted and deletes it +// if needed. +func (ctrl *csiSnapshotCommonController) processFinalizersAndCheckandDeleteContent(snapshot *crdv1.VolumeSnapshot) error { + klog.V(5).Infof("processFinalizersAndCheckandDeleteContent VolumeSnapshot[%s]: %s", utils.SnapshotKey(snapshot), utils.GetSnapshotStatusForLogging(snapshot)) + + // If content is deleted already, remove SnapshotBound finalizer + content, err := ctrl.contentExists(snapshot) + if err != nil { + return err + } + deleteContent := false + // It is possible for contentExists to return nil, nil + if content != nil && content.Spec.DeletionPolicy == crdv1.VolumeSnapshotContentDelete { + klog.V(5).Infof("processFinalizersAndCheckandDeleteContent: Content [%s] deletion policy [%s] is delete.", content.Name, content.Spec.DeletionPolicy) + deleteContent = true + } + + snapshotBound := false + // Check if the snapshot content is bound to the snapshot + if content != nil && utils.IsSnapshotBound(snapshot, content) { + klog.Infof("syncSnapshot: VolumeSnapshot %s is bound to volumeSnapshotContent [%s]", snapshot.Name, content.Name) + snapshotBound = true + } + + klog.V(5).Infof("processFinalizersAndCheckandDeleteContent[%s]: delete snapshot content and remove finalizer from snapshot if needed", utils.SnapshotKey(snapshot)) + err = ctrl.checkandRemoveSnapshotFinalizersAndCheckandDeleteContent(snapshot, content, deleteContent) + if err != nil { + return err + } + + klog.V(5).Infof("processFinalizersAndCheckandDeleteContent[%s]: check if we should add finalizers on snapshot", utils.SnapshotKey(snapshot)) + ctrl.checkandAddSnapshotFinalizers(snapshot, snapshotBound, deleteContent) + + klog.V(5).Infof("processFinalizersAndCheckandDeleteContent[%s]: check if we should remove finalizer on snapshot source and remove it if we can", utils.SnapshotKey(snapshot)) + + // Check if we should remove finalizer on PVC and remove it if we can. + errFinalizer := ctrl.checkandRemovePVCFinalizer(snapshot) + if errFinalizer != nil { + klog.Errorf("error check and remove PVC finalizer for snapshot [%s]: %v", snapshot.Name, errFinalizer) + // Log an event and keep the original error from syncUnready/ReadySnapshot + ctrl.eventRecorder.Event(snapshot, v1.EventTypeWarning, "ErrorPVCFinalizer", "Error check and remove PVC Finalizer for VolumeSnapshot") + } + return nil +} + +// checkandRemoveSnapshotFinalizersAndCheckandDeleteContent deletes the content and removes snapshot finalizers if needed +func (ctrl *csiSnapshotCommonController) checkandRemoveSnapshotFinalizersAndCheckandDeleteContent(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent, deleteContent bool) error { + klog.V(5).Infof("deleteContentAndSnapshotFinalizers VolumeSnapshot[%s]: %s", utils.SnapshotKey(snapshot), utils.GetSnapshotStatusForLogging(snapshot)) + + var err error + // Check is snapshot deletionTimestamp is set and any finalizer is on + if utils.IsSnapshotDeletionCandidate(snapshot) { + // Volume snapshot should be deleted. Check if it's used + // and remove finalizer if it's not. + // Check if a volume is being created from snapshot. + inUse := ctrl.isVolumeBeingCreatedFromSnapshot(snapshot) + + klog.V(5).Infof("syncSnapshot[%s]: set DeletionTimeStamp on content.", utils.SnapshotKey(snapshot)) + // If content exists, set DeletionTimeStamp on the content; + // content won't be deleted immediately due to the finalizer + if content != nil && deleteContent && !inUse { + err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Delete(content.Name, &metav1.DeleteOptions{}) + if err != nil { + ctrl.eventRecorder.Event(snapshot, v1.EventTypeWarning, "SnapshotContentObjectDeleteError", "Failed to delete snapshot content API object") + return fmt.Errorf("failed to delete VolumeSnapshotContent %s from API server: %q", content.Name, err) + + } + } + + if !inUse || (content == nil && err == nil) { + klog.V(5).Infof("syncSnapshot: Remove Finalizer for VolumeSnapshot[%s]", utils.SnapshotKey(snapshot)) + doesContentExist := false + if content != nil { + doesContentExist = true + } + return ctrl.removeSnapshotFinalizer(snapshot, !inUse, !doesContentExist) + } + } + return nil +} + +// checkandAddSnapshotFinalizers checks and adds snapshot finailzers when needed +func (ctrl *csiSnapshotCommonController) checkandAddSnapshotFinalizers(snapshot *crdv1.VolumeSnapshot, snapshotBound bool, deleteContent bool) { + addSourceFinalizer := false + addBoundFinalizer := false + if utils.NeedToAddSnapshotAsSourceFinalizer(snapshot) { + addSourceFinalizer = true + } + if utils.NeedToAddSnapshotBoundFinalizer(snapshot) && snapshotBound && deleteContent { + // Add bound finalizer if snapshot is bound to content and deletion policy is delete + addBoundFinalizer = true + } + if addSourceFinalizer || addBoundFinalizer { + // Snapshot is not being deleted -> it should have the finalizer. + klog.V(5).Infof("checkandAddSnapshotFinalizers: Add Finalizer for VolumeSnapshot[%s]", utils.SnapshotKey(snapshot)) + ctrl.addSnapshotFinalizer(snapshot, addSourceFinalizer, addBoundFinalizer) + } +} + +// syncReadySnapshot checks the snapshot which has been bound to snapshot content successfully before. +// If there is any problem with the binding (e.g., snapshot points to a non-exist snapshot content), update the snapshot status and emit event. +func (ctrl *csiSnapshotCommonController) syncReadySnapshot(snapshot *crdv1.VolumeSnapshot) error { + if !utils.IsBoundVolumeSnapshotContentNameSet(snapshot) { + return nil + } + obj, found, err := ctrl.contentStore.GetByKey(*snapshot.Status.BoundVolumeSnapshotContentName) + if err != nil { + return err + } + if !found { + if err = ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotContentMissing", "VolumeSnapshotContent is missing"); err != nil { + return err + } + return nil + } else { + content, ok := obj.(*crdv1.VolumeSnapshotContent) + if !ok { + return fmt.Errorf("Cannot convert object from snapshot content store to VolumeSnapshotContent %q!?: %#v", *snapshot.Status.BoundVolumeSnapshotContentName, obj) + } + + klog.V(5).Infof("syncReadySnapshot[%s]: VolumeSnapshotContent %q found", utils.SnapshotKey(snapshot), content.Name) + if !utils.IsVolumeSnapshotRefSet(snapshot, content) { + // snapshot is bound but content is not bound to snapshot correctly + if err = ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotMisbound", "VolumeSnapshotContent is not bound to the VolumeSnapshot correctly"); err != nil { + return err + } + return nil + } + // Snapshot is correctly bound. + return nil + } +} + +// syncUnreadySnapshot is the main controller method to decide what to do with a snapshot which is not set to ready. +func (ctrl *csiSnapshotCommonController) syncUnreadySnapshot(snapshot *crdv1.VolumeSnapshot) error { + uniqueSnapshotName := utils.SnapshotKey(snapshot) + klog.V(5).Infof("syncUnreadySnapshot %s", uniqueSnapshotName) + + // Pre-provisioned snapshot + if snapshot.Spec.Source.VolumeSnapshotContentName != nil { + content, err := ctrl.findContentfromStore(snapshot) + if err != nil { + return err + } + // Set VolumeSnapshotRef UID + newContent, err := ctrl.checkandBindSnapshotContent(snapshot, content) + if err != nil { + // snapshot is bound but content is not bound to snapshot correctly + ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotBindFailed", fmt.Sprintf("Snapshot failed to bind VolumeSnapshotContent, %v", err)) + return fmt.Errorf("snapshot %s is bound, but VolumeSnapshotContent %s is not bound to the VolumeSnapshot correctly, %v", uniqueSnapshotName, content.Name, err) + } + + // update snapshot status + for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ { + klog.V(5).Infof("syncUnreadySnapshot [%s]: trying to update snapshot status", utils.SnapshotKey(snapshot)) + _, err = ctrl.updateSnapshotStatus(snapshot, newContent) + if err == nil { + break + } + klog.V(4).Infof("failed to update snapshot %s status: %v", utils.SnapshotKey(snapshot), err) + } + + if err != nil { + // update snapshot status failed + ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotStatusUpdateFailed", fmt.Sprintf("Snapshot status update failed, %v", err)) + return err + } + + return nil + } else { // snapshot.Spec.Source.VolumeSnapshotContentName == nil - dynamically creating snapshot + klog.V(5).Infof("before getMatchSnapshotContent for snapshot %s", uniqueSnapshotName) + if contentObj := ctrl.getMatchSnapshotContent(snapshot); contentObj != nil { + klog.V(5).Infof("Found VolumeSnapshotContent object %s for snapshot %s", contentObj.Name, uniqueSnapshotName) + if contentObj.Spec.Source.SnapshotHandle != nil { + ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotHandleNotFound", fmt.Sprintf("Snapshot handle not found in content %s", contentObj.Name)) + return fmt.Errorf("snapshotHandle should not be set in the content for dynamic provisioning for snapshot %s", uniqueSnapshotName) + } + newSnapshot, err := ctrl.bindandUpdateVolumeSnapshot(contentObj, snapshot) + if err != nil { + return err + } + klog.V(5).Infof("bindandUpdateVolumeSnapshot %v", newSnapshot) + return nil + } else if snapshot.Status != nil && snapshot.Status.BoundVolumeSnapshotContentName != nil { + contentObj, found, err := ctrl.contentStore.GetByKey(*snapshot.Status.BoundVolumeSnapshotContentName) + if err != nil { + return err + } + if !found { + if snapshot.ObjectMeta.DeletionTimestamp == nil { + ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotContentNotFound", fmt.Sprintf("Content for snapshot %s not found, but deletion timestamp not set on snapshot", uniqueSnapshotName)) + return fmt.Errorf("content for snapshot %s not found without deletion timestamp on snapshot", uniqueSnapshotName) + } + // NOTE: this is not an error now because we delete content before the snapshot + klog.V(5).Infof("Content for snapshot %s not found. It may be already deleted as expected.", uniqueSnapshotName) + } else { + klog.V(5).Infof("converting content object for snapshot %s", uniqueSnapshotName) + _, ok := contentObj.(*crdv1.VolumeSnapshotContent) + if !ok { + return fmt.Errorf("expected volume snapshot content, got %+v", contentObj) + } + } + } else if snapshot.Status == nil || snapshot.Status.Error == nil || isControllerUpdateFailError(snapshot.Status.Error) { + if snapshot.Spec.Source.PersistentVolumeClaimName == nil { + ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotPVCSourceMissing", fmt.Sprintf("PVC source for snapshot %s is missing", uniqueSnapshotName)) + return fmt.Errorf("expected PVC source for snapshot %s but got nil", uniqueSnapshotName) + } else { + var err error + var content *crdv1.VolumeSnapshotContent + if content, err = ctrl.createSnapshotContent(snapshot); err != nil { + ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotContentCreationFailed", fmt.Sprintf("Failed to create snapshot content with error %v", err)) + return err + } + + // Update snapshot status with BoundVolumeSnapshotContentName + for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ { + klog.V(5).Infof("syncUnreadySnapshot [%s]: trying to update snapshot status", utils.SnapshotKey(snapshot)) + _, err = ctrl.updateSnapshotStatus(snapshot, content) + if err == nil { + break + } + klog.V(4).Infof("failed to update snapshot %s status: %v", utils.SnapshotKey(snapshot), err) + } + + if err != nil { + // update snapshot status failed + ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotStatusUpdateFailed", fmt.Sprintf("Snapshot status update failed, %v", err)) + return err + } + } + } + return nil + } +} + +// findContentfromStore finds content from content cache store +func (ctrl *csiSnapshotCommonController) findContentfromStore(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshotContent, error) { + var contentName string + uniqueSnapshotName := utils.SnapshotKey(snapshot) + if snapshot.Spec.Source.VolumeSnapshotContentName != nil { + contentName = *snapshot.Spec.Source.VolumeSnapshotContentName + } else if snapshot.Status != nil && snapshot.Status.BoundVolumeSnapshotContentName != nil { + contentName = *snapshot.Status.BoundVolumeSnapshotContentName + } + if contentName == "" { + return nil, fmt.Errorf("content name not found for snapshot %s", uniqueSnapshotName) + } + + contentObj, found, err := ctrl.contentStore.GetByKey(contentName) + if err != nil { + return nil, err + } + if !found { + // snapshot is bound to a non-existing content. + ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotContentMissing", "VolumeSnapshotContent is missing") + klog.V(4).Infof("synchronizing unready snapshot[%s]: snapshotcontent %q requested and not found, will try again next time", uniqueSnapshotName, contentName) + return nil, fmt.Errorf("snapshot %s is bound to a non-existing content %s", uniqueSnapshotName, contentName) + } + content, ok := contentObj.(*crdv1.VolumeSnapshotContent) + if !ok { + return nil, fmt.Errorf("expected volume snapshot content, got %+v", contentObj) + } + return content, nil +} + +// createSnapshotContent will only be called for dynamic provisioning +func (ctrl *csiSnapshotCommonController) createSnapshotContent(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshotContent, error) { + klog.Infof("createSnapshotContent: Creating content for snapshot %s through the plugin ...", utils.SnapshotKey(snapshot)) + + // If PVC is not being deleted and finalizer is not added yet, a finalizer should be added to PVC until snapshot is created + klog.V(5).Infof("createSnapshotContent: Check if PVC is not being deleted and add Finalizer for source of snapshot [%s] if needed", snapshot.Name) + err := ctrl.ensurePVCFinalizer(snapshot) + if err != nil { + klog.Errorf("createSnapshotContent failed to add finalizer for source of snapshot %s", err) + return nil, err + } + + class, volume, contentName, snapshotterSecretRef, err := ctrl.getCreateSnapshotInput(snapshot) + if err != nil { + return nil, fmt.Errorf("failed to get input parameters to create snapshot %s: %q", snapshot.Name, err) + } + + // Create VolumeSnapshotContent in the database + if volume.Spec.CSI == nil { + return nil, fmt.Errorf("cannot find CSI PersistentVolumeSource for volume %s", volume.Name) + } + snapshotRef, err := ref.GetReference(scheme.Scheme, snapshot) + if err != nil { + return nil, err + } + + snapshotContent := &crdv1.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: contentName, + }, + Spec: crdv1.VolumeSnapshotContentSpec{ + VolumeSnapshotRef: *snapshotRef, + Source: crdv1.VolumeSnapshotContentSource{ + VolumeHandle: &volume.Spec.CSI.VolumeHandle, + }, + VolumeSnapshotClassName: &(class.Name), + DeletionPolicy: class.DeletionPolicy, + Driver: class.Driver, + }, + } + + // Set AnnDeletionSecretRefName and AnnDeletionSecretRefNamespace + if snapshotterSecretRef != nil { + klog.V(5).Infof("createSnapshotContent: set annotation [%s] on content [%s].", utils.AnnDeletionSecretRefName, snapshotContent.Name) + metav1.SetMetaDataAnnotation(&snapshotContent.ObjectMeta, utils.AnnDeletionSecretRefName, snapshotterSecretRef.Name) + + klog.V(5).Infof("createSnapshotContent: set annotation [%s] on content [%s].", utils.AnnDeletionSecretRefNamespace, snapshotContent.Name) + metav1.SetMetaDataAnnotation(&snapshotContent.ObjectMeta, utils.AnnDeletionSecretRefNamespace, snapshotterSecretRef.Namespace) + } + + var updateContent *crdv1.VolumeSnapshotContent + klog.V(3).Infof("volume snapshot content %#v", snapshotContent) + // Try to create the VolumeSnapshotContent object several times + for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ { + klog.V(5).Infof("createSnapshotContent [%s]: trying to save volume snapshot content %s", utils.SnapshotKey(snapshot), snapshotContent.Name) + if updateContent, err = ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Create(snapshotContent); err == nil || apierrs.IsAlreadyExists(err) { + // Save succeeded. + if err != nil { + klog.V(3).Infof("volume snapshot content %q for snapshot %q already exists, reusing", snapshotContent.Name, utils.SnapshotKey(snapshot)) + err = nil + updateContent = snapshotContent + } else { + klog.V(3).Infof("volume snapshot content %q for snapshot %q saved, %v", snapshotContent.Name, utils.SnapshotKey(snapshot), snapshotContent) + } + break + } + // Save failed, try again after a while. + klog.V(3).Infof("failed to save volume snapshot content %q for snapshot %q: %v", snapshotContent.Name, utils.SnapshotKey(snapshot), err) + time.Sleep(ctrl.createSnapshotContentInterval) + } + + if err != nil { + strerr := fmt.Sprintf("Error creating volume snapshot content object for snapshot %s: %v.", utils.SnapshotKey(snapshot), err) + klog.Error(strerr) + ctrl.eventRecorder.Event(snapshot, v1.EventTypeWarning, "CreateSnapshotContentFailed", strerr) + return nil, newControllerUpdateError(utils.SnapshotKey(snapshot), err.Error()) + } + + // Update content in the cache store + _, err = ctrl.storeContentUpdate(updateContent) + if err != nil { + klog.Errorf("failed to update content store %v", err) + } + + return updateContent, nil +} + +func (ctrl *csiSnapshotCommonController) getCreateSnapshotInput(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshotClass, *v1.PersistentVolume, string, *v1.SecretReference, error) { + className := snapshot.Spec.VolumeSnapshotClassName + klog.V(5).Infof("getCreateSnapshotInput [%s]", snapshot.Name) + var class *crdv1.VolumeSnapshotClass + var err error + if className != nil { + class, err = ctrl.getSnapshotClass(*className) + if err != nil { + klog.Errorf("getCreateSnapshotInput failed to getClassFromVolumeSnapshot %s", err) + return nil, nil, "", nil, err + } + } else { + klog.Errorf("failed to getCreateSnapshotInput %s without a snapshot class", snapshot.Name) + return nil, nil, "", nil, fmt.Errorf("failed to take snapshot %s without a snapshot class", snapshot.Name) + } + + volume, err := ctrl.getVolumeFromVolumeSnapshot(snapshot) + if err != nil { + klog.Errorf("getCreateSnapshotInput failed to get PersistentVolume object [%s]: Error: [%#v]", snapshot.Name, err) + return nil, nil, "", nil, err + } + + // Create VolumeSnapshotContent name + contentName := utils.GetSnapshotContentNameForSnapshot(snapshot) + + // Resolve snapshotting secret credentials. + snapshotterSecretRef, err := utils.GetSecretReference(class.Parameters, contentName, snapshot) + if err != nil { + return nil, nil, "", nil, err + } + + return class, volume, contentName, snapshotterSecretRef, nil +} + +// getMatchSnapshotContent looks up VolumeSnapshotContent for a VolumeSnapshot named snapshotName +func (ctrl *csiSnapshotCommonController) getMatchSnapshotContent(snapshot *crdv1.VolumeSnapshot) *crdv1.VolumeSnapshotContent { + var snapshotContentObj *crdv1.VolumeSnapshotContent + var found bool + + objs := ctrl.contentStore.List() + for _, obj := range objs { + content := obj.(*crdv1.VolumeSnapshotContent) + if content.Spec.VolumeSnapshotRef.Name == snapshot.Name && + content.Spec.VolumeSnapshotRef.Namespace == snapshot.Namespace && + content.Spec.VolumeSnapshotRef.UID == snapshot.UID && + content.Spec.VolumeSnapshotClassName != nil && snapshot.Spec.VolumeSnapshotClassName != nil && + *(content.Spec.VolumeSnapshotClassName) == *(snapshot.Spec.VolumeSnapshotClassName) { + found = true + snapshotContentObj = content + break + } + } + + if !found { + klog.V(4).Infof("No VolumeSnapshotContent for VolumeSnapshot %s found", utils.SnapshotKey(snapshot)) + return nil + } + + return snapshotContentObj +} + +func (ctrl *csiSnapshotCommonController) storeSnapshotUpdate(snapshot interface{}) (bool, error) { + return utils.StoreObjectUpdate(ctrl.snapshotStore, snapshot, "snapshot") +} + +func (ctrl *csiSnapshotCommonController) storeContentUpdate(content interface{}) (bool, error) { + return utils.StoreObjectUpdate(ctrl.contentStore, content, "content") +} + +// updateSnapshotStatusWithEvent saves new snapshot.Status to API server and emits +// given event on the snapshot. It saves the status and emits the event only when +// the status has actually changed from the version saved in API server. +// Parameters: +// snapshot - snapshot to update +// eventtype, reason, message - event to send, see EventRecorder.Event() +func (ctrl *csiSnapshotCommonController) updateSnapshotErrorStatusWithEvent(snapshot *crdv1.VolumeSnapshot, eventtype, reason, message string) error { + klog.V(5).Infof("updateSnapshotStatusWithEvent[%s]", utils.SnapshotKey(snapshot)) + + if snapshot.Status != nil && snapshot.Status.Error != nil && *snapshot.Status.Error.Message == message { + klog.V(4).Infof("updateSnapshotStatusWithEvent[%s]: the same error %v is already set", snapshot.Name, snapshot.Status.Error) + return nil + } + snapshotClone := snapshot.DeepCopy() + if snapshotClone.Status == nil { + snapshotClone.Status = &crdv1.VolumeSnapshotStatus{} + } + statusError := &crdv1.VolumeSnapshotError{ + Time: &metav1.Time{ + Time: time.Now(), + }, + Message: &message, + } + snapshotClone.Status.Error = statusError + ready := false + snapshotClone.Status.ReadyToUse = &ready + newSnapshot, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).UpdateStatus(snapshotClone) + + if err != nil { + klog.V(4).Infof("updating VolumeSnapshot[%s] error status failed %v", utils.SnapshotKey(snapshot), err) + return err + } + + _, err = ctrl.storeSnapshotUpdate(newSnapshot) + if err != nil { + klog.V(4).Infof("updating VolumeSnapshot[%s] error status: cannot update internal cache %v", utils.SnapshotKey(snapshot), err) + return err + } + // Emit the event only when the status change happens + ctrl.eventRecorder.Event(newSnapshot, eventtype, reason, message) + + return nil +} + +// isSnapshotConentBeingUsed checks if snapshot content is bound to snapshot. +func (ctrl *csiSnapshotCommonController) isSnapshotContentBeingUsed(content *crdv1.VolumeSnapshotContent) bool { + if content.Spec.VolumeSnapshotRef.Name != "" && content.Spec.VolumeSnapshotRef.Namespace != "" { + snapshotObj, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(content.Spec.VolumeSnapshotRef.Namespace).Get(content.Spec.VolumeSnapshotRef.Name, metav1.GetOptions{}) + if err != nil { + klog.Infof("isSnapshotContentBeingUsed: Cannot get snapshot %s from api server: [%v]. VolumeSnapshot object may be deleted already.", content.Spec.VolumeSnapshotRef.Name, err) + return false + } + + // Check if the snapshot content is bound to the snapshot + if utils.IsSnapshotBound(snapshotObj, content) { + klog.Infof("isSnapshotContentBeingUsed: VolumeSnapshot %s is bound to volumeSnapshotContent [%s]", snapshotObj.Name, content.Name) + return true + } + } + + klog.V(5).Infof("isSnapshotContentBeingUsed: Snapshot content %s is not being used", content.Name) + return false +} + +// addContentFinalizer adds a Finalizer for VolumeSnapshotContent. +func (ctrl *csiSnapshotCommonController) addContentFinalizer(content *crdv1.VolumeSnapshotContent) error { + contentClone := content.DeepCopy() + contentClone.ObjectMeta.Finalizers = append(contentClone.ObjectMeta.Finalizers, utils.VolumeSnapshotContentFinalizer) + + _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) + if err != nil { + return newControllerUpdateError(content.Name, err.Error()) + } + + _, err = ctrl.storeContentUpdate(contentClone) + if err != nil { + klog.Errorf("failed to update content store %v", err) + } + + klog.V(5).Infof("Added protection finalizer to volume snapshot content %s", content.Name) + return nil +} + +// isVolumeBeingCreatedFromSnapshot checks if an volume is being created from the snapshot. +func (ctrl *csiSnapshotCommonController) isVolumeBeingCreatedFromSnapshot(snapshot *crdv1.VolumeSnapshot) bool { + pvcList, err := ctrl.pvcLister.PersistentVolumeClaims(snapshot.Namespace).List(labels.Everything()) + if err != nil { + klog.Errorf("Failed to retrieve PVCs from the lister to check if volume snapshot %s is being used by a volume: %q", utils.SnapshotKey(snapshot), err) + return false + } + for _, pvc := range pvcList { + if pvc.Spec.DataSource != nil && pvc.Spec.DataSource.Name == snapshot.Name { + if pvc.Spec.DataSource.Kind == snapshotKind && *(pvc.Spec.DataSource.APIGroup) == snapshotAPIGroup { + if pvc.Status.Phase == v1.ClaimPending { + // A volume is being created from the snapshot + klog.Infof("isVolumeBeingCreatedFromSnapshot: volume %s is being created from snapshot %s", pvc.Name, pvc.Spec.DataSource.Name) + return true + } + } + } + } + klog.V(5).Infof("isVolumeBeingCreatedFromSnapshot: no volume is being created from snapshot %s", utils.SnapshotKey(snapshot)) + return false +} + +// ensurePVCFinalizer checks if a Finalizer needs to be added for the snapshot source; +// if true, adds a Finalizer for VolumeSnapshot Source PVC +func (ctrl *csiSnapshotCommonController) ensurePVCFinalizer(snapshot *crdv1.VolumeSnapshot) error { + if snapshot.Spec.Source.PersistentVolumeClaimName == nil { + // PVC finalizer is only needed for dynamic provisioning + return nil + } + + // Get snapshot source which is a PVC + pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) + if err != nil { + klog.Infof("cannot get claim from snapshot [%s]: [%v] Claim may be deleted already.", snapshot.Name, err) + return nil + } + + if pvc.ObjectMeta.DeletionTimestamp != nil { + klog.Errorf("cannot add finalizer on claim [%s] for snapshot [%s]: claim is being deleted", pvc.Name, snapshot.Name) + return newControllerUpdateError(pvc.Name, "cannot add finalizer on claim because it is being deleted") + } + + // If PVC is not being deleted and PVCFinalizer is not added yet, the PVCFinalizer should be added. + if pvc.ObjectMeta.DeletionTimestamp == nil && !slice.ContainsString(pvc.ObjectMeta.Finalizers, utils.PVCFinalizer, nil) { + // Add the finalizer + pvcClone := pvc.DeepCopy() + pvcClone.ObjectMeta.Finalizers = append(pvcClone.ObjectMeta.Finalizers, utils.PVCFinalizer) + _, err = ctrl.client.CoreV1().PersistentVolumeClaims(pvcClone.Namespace).Update(pvcClone) + if err != nil { + klog.Errorf("cannot add finalizer on claim [%s] for snapshot [%s]: [%v]", pvc.Name, snapshot.Name, err) + return newControllerUpdateError(pvcClone.Name, err.Error()) + } + klog.Infof("Added protection finalizer to persistent volume claim %s", pvc.Name) + } + + return nil +} + +// removePVCFinalizer removes a Finalizer for VolumeSnapshot Source PVC. +func (ctrl *csiSnapshotCommonController) removePVCFinalizer(pvc *v1.PersistentVolumeClaim, snapshot *crdv1.VolumeSnapshot) error { + // Get snapshot source which is a PVC + // TODO(xyang): We get PVC from informer but it may be outdated + // Should get it from API server directly before removing finalizer + pvcClone := pvc.DeepCopy() + pvcClone.ObjectMeta.Finalizers = slice.RemoveString(pvcClone.ObjectMeta.Finalizers, utils.PVCFinalizer, nil) + + _, err := ctrl.client.CoreV1().PersistentVolumeClaims(pvcClone.Namespace).Update(pvcClone) + if err != nil { + return newControllerUpdateError(pvcClone.Name, err.Error()) + } + + klog.V(5).Infof("Removed protection finalizer from persistent volume claim %s", pvc.Name) + return nil +} + +// isPVCBeingUsed checks if a PVC is being used as a source to create a snapshot +func (ctrl *csiSnapshotCommonController) isPVCBeingUsed(pvc *v1.PersistentVolumeClaim, snapshot *crdv1.VolumeSnapshot) bool { + klog.V(5).Infof("Checking isPVCBeingUsed for snapshot [%s]", utils.SnapshotKey(snapshot)) + + // Going through snapshots in the cache (snapshotLister). If a snapshot's PVC source + // is the same as the input snapshot's PVC source and snapshot's ReadyToUse status + // is false, the snapshot is still being created from the PVC and the PVC is in-use. + snapshots, err := ctrl.snapshotLister.VolumeSnapshots(snapshot.Namespace).List(labels.Everything()) + if err != nil { + return false + } + for _, snap := range snapshots { + // Skip pre-provisioned snapshot without a PVC source + if snap.Spec.Source.PersistentVolumeClaimName == nil && snap.Spec.Source.VolumeSnapshotContentName != nil { + klog.V(4).Infof("Skipping static bound snapshot %s when checking PVC %s/%s", snap.Name, pvc.Namespace, pvc.Name) + continue + } + if snap.Spec.Source.PersistentVolumeClaimName != nil && pvc.Name == *snap.Spec.Source.PersistentVolumeClaimName && (snap.Status == nil || snap.Status.ReadyToUse == nil || (snap.Status.ReadyToUse != nil && *snap.Status.ReadyToUse == false)) { + klog.V(2).Infof("Keeping PVC %s/%s, it is used by snapshot %s/%s", pvc.Namespace, pvc.Name, snap.Namespace, snap.Name) + return true + } + } + + klog.V(5).Infof("isPVCBeingUsed: no snapshot is being created from PVC %s/%s", pvc.Namespace, pvc.Name) + return false +} + +// checkandRemovePVCFinalizer checks if the snapshot source finalizer should be removed +// and removed it if needed. +func (ctrl *csiSnapshotCommonController) checkandRemovePVCFinalizer(snapshot *crdv1.VolumeSnapshot) error { + if snapshot.Spec.Source.PersistentVolumeClaimName == nil { + // PVC finalizer is only needed for dynamic provisioning + return nil + } + + // Get snapshot source which is a PVC + pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) + if err != nil { + klog.Infof("cannot get claim from snapshot [%s]: [%v] Claim may be deleted already. No need to remove finalizer on the claim.", snapshot.Name, err) + return nil + } + + klog.V(5).Infof("checkandRemovePVCFinalizer for snapshot [%s]: snapshot status [%#v]", snapshot.Name, snapshot.Status) + + // Check if there is a Finalizer on PVC to be removed + if slice.ContainsString(pvc.ObjectMeta.Finalizers, utils.PVCFinalizer, nil) { + // There is a Finalizer on PVC. Check if PVC is used + // and remove finalizer if it's not used. + inUse := ctrl.isPVCBeingUsed(pvc, snapshot) + if !inUse { + klog.Infof("checkandRemovePVCFinalizer[%s]: Remove Finalizer for PVC %s as it is not used by snapshots in creation", snapshot.Name, pvc.Name) + err = ctrl.removePVCFinalizer(pvc, snapshot) + if err != nil { + klog.Errorf("checkandRemovePVCFinalizer [%s]: removePVCFinalizer failed to remove finalizer %v", snapshot.Name, err) + return err + } + } + } + + return nil +} + +// The function checks whether the volumeSnapshotRef in snapshot content matches the given snapshot. If match, it binds the content with the snapshot. This is for static binding where user has specified snapshot name but not UID of the snapshot in content.Spec.VolumeSnapshotRef. +func (ctrl *csiSnapshotCommonController) checkandBindSnapshotContent(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshotContent, error) { + if content.Spec.VolumeSnapshotRef.Name != snapshot.Name { + return nil, fmt.Errorf("Could not bind snapshot %s and content %s, the VolumeSnapshotRef does not match", snapshot.Name, content.Name) + } else if content.Spec.VolumeSnapshotRef.UID != "" && content.Spec.VolumeSnapshotRef.UID != snapshot.UID { + return nil, fmt.Errorf("Could not bind snapshot %s and content %s, the VolumeSnapshotRef does not match", snapshot.Name, content.Name) + } else if content.Spec.VolumeSnapshotRef.UID != "" && content.Spec.VolumeSnapshotClassName != nil { + return content, nil + } + contentClone := content.DeepCopy() + contentClone.Spec.VolumeSnapshotRef.UID = snapshot.UID + if snapshot.Spec.VolumeSnapshotClassName != nil { + className := *(snapshot.Spec.VolumeSnapshotClassName) + contentClone.Spec.VolumeSnapshotClassName = &className + } + newContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) + if err != nil { + klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status failed %v", newContent.Name, err) + return nil, err + } + + _, err = ctrl.storeContentUpdate(newContent) + if err != nil { + klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status: cannot update internal cache %v", newContent.Name, err) + return nil, err + } + return newContent, nil +} + +// This routine sets snapshot.Spec.Source.VolumeSnapshotContentName +func (ctrl *csiSnapshotCommonController) bindandUpdateVolumeSnapshot(snapshotContent *crdv1.VolumeSnapshotContent, snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshot, error) { + klog.V(5).Infof("bindandUpdateVolumeSnapshot for snapshot [%s]: snapshotContent [%s]", snapshot.Name, snapshotContent.Name) + snapshotObj, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshot.Namespace).Get(snapshot.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("error get snapshot %s from api server: %v", utils.SnapshotKey(snapshot), err) + } + + // Copy the snapshot object before updating it + snapshotCopy := snapshotObj.DeepCopy() + + // update snapshot status + for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ { + klog.V(5).Infof("bindandUpdateVolumeSnapshot [%s]: trying to update snapshot status", utils.SnapshotKey(snapshotCopy)) + updateSnapshot, err := ctrl.updateSnapshotStatus(snapshotCopy, snapshotContent) + if err == nil { + snapshotCopy = updateSnapshot + break + } + klog.V(4).Infof("failed to update snapshot %s status: %v", utils.SnapshotKey(snapshot), err) + } + + if err != nil { + // update snapshot status failed + ctrl.updateSnapshotErrorStatusWithEvent(snapshotCopy, v1.EventTypeWarning, "SnapshotStatusUpdateFailed", fmt.Sprintf("Snapshot status update failed, %v", err)) + return nil, err + } + + _, err = ctrl.storeSnapshotUpdate(snapshotCopy) + if err != nil { + klog.Errorf("%v", err) + } + + klog.V(5).Infof("bindandUpdateVolumeSnapshot for snapshot completed [%#v]", snapshotCopy) + return snapshotCopy, nil +} + +// UpdateSnapshotStatus updates snapshot status based on content status +func (ctrl *csiSnapshotCommonController) updateSnapshotStatus(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshot, error) { + klog.V(5).Infof("updateSnapshotStatus[%s]", utils.SnapshotKey(snapshot)) + + boundContentName := content.Name + var createdAt *time.Time + if content.Status != nil && content.Status.CreationTime != nil { + unixTime := time.Unix(0, *content.Status.CreationTime) + createdAt = &unixTime + } + var size *int64 + if content.Status != nil && content.Status.RestoreSize != nil { + size = content.Status.RestoreSize + } + var readyToUse bool + if content.Status != nil && content.Status.ReadyToUse != nil { + readyToUse = *content.Status.ReadyToUse + } + + klog.V(5).Infof("updateSnapshotStatus: updating VolumeSnapshot [%+v] based on VolumeSnapshotContentStatus [%+v]", snapshot, content.Status) + + snapshotObj, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshot.Namespace).Get(snapshot.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("error get snapshot %s from api server: %v", utils.SnapshotKey(snapshot), err) + } + + var newStatus *crdv1.VolumeSnapshotStatus + updated := false + if snapshotObj.Status == nil { + newStatus = &crdv1.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &boundContentName, + ReadyToUse: &readyToUse, + } + if createdAt != nil { + newStatus.CreationTime = &metav1.Time{Time: *createdAt} + } + if size != nil { + newStatus.RestoreSize = resource.NewQuantity(*size, resource.BinarySI) + } + updated = true + } else { + newStatus = snapshotObj.Status.DeepCopy() + if newStatus.BoundVolumeSnapshotContentName == nil { + newStatus.BoundVolumeSnapshotContentName = &boundContentName + updated = true + } + if newStatus.CreationTime == nil && createdAt != nil { + newStatus.CreationTime = &metav1.Time{Time: *createdAt} + updated = true + } + if newStatus.ReadyToUse == nil || *newStatus.ReadyToUse != readyToUse { + newStatus.ReadyToUse = &readyToUse + updated = true + if readyToUse && newStatus.Error != nil { + newStatus.Error = nil + } + } + if (newStatus.RestoreSize == nil && size != nil) || (newStatus.RestoreSize != nil && newStatus.RestoreSize.IsZero() && size != nil && *size > 0) { + newStatus.RestoreSize = resource.NewQuantity(*size, resource.BinarySI) + updated = true + } + } + + if updated { + snapshotClone := snapshotObj.DeepCopy() + snapshotClone.Status = newStatus + newSnapshotObj, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).UpdateStatus(snapshotClone) + if err != nil { + return nil, newControllerUpdateError(utils.SnapshotKey(snapshot), err.Error()) + } + return newSnapshotObj, nil + } + + return snapshotObj, nil +} + +func (ctrl *csiSnapshotCommonController) getVolumeFromVolumeSnapshot(snapshot *crdv1.VolumeSnapshot) (*v1.PersistentVolume, error) { + pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) + if err != nil { + return nil, err + } + + if pvc.Status.Phase != v1.ClaimBound { + return nil, fmt.Errorf("the PVC %s is not yet bound to a PV, will not attempt to take a snapshot", pvc.Name) + } + + pvName := pvc.Spec.VolumeName + pv, err := ctrl.client.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to retrieve PV %s from the API server: %q", pvName, err) + } + + klog.V(5).Infof("getVolumeFromVolumeSnapshot: snapshot [%s] PV name [%s]", snapshot.Name, pvName) + + return pv, nil +} + +func (ctrl *csiSnapshotCommonController) getStorageClassFromVolumeSnapshot(snapshot *crdv1.VolumeSnapshot) (*storagev1.StorageClass, error) { + // Get storage class from PVC or PV + pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) + if err != nil { + return nil, err + } + storageclassName := *pvc.Spec.StorageClassName + if len(storageclassName) == 0 { + volume, err := ctrl.getVolumeFromVolumeSnapshot(snapshot) + if err != nil { + return nil, err + } + storageclassName = volume.Spec.StorageClassName + } + if len(storageclassName) == 0 { + return nil, fmt.Errorf("cannot figure out the snapshot class automatically, please specify one in snapshot spec") + } + storageclass, err := ctrl.client.StorageV1().StorageClasses().Get(storageclassName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return storageclass, nil +} + +// getSnapshotClass is a helper function to get snapshot class from the class name. +func (ctrl *csiSnapshotCommonController) getSnapshotClass(className string) (*crdv1.VolumeSnapshotClass, error) { + klog.V(5).Infof("getSnapshotClass: VolumeSnapshotClassName [%s]", className) + + class, err := ctrl.classLister.Get(className) + if err != nil { + klog.Errorf("failed to retrieve snapshot class %s from the informer: %q", className, err) + return nil, fmt.Errorf("failed to retrieve snapshot class %s from the informer: %q", className, err) + } + + return class, nil +} + +// SetDefaultSnapshotClass is a helper function to figure out the default snapshot class from +// PVC/PV StorageClass and update VolumeSnapshot with this snapshot class name. +func (ctrl *csiSnapshotCommonController) SetDefaultSnapshotClass(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshotClass, *crdv1.VolumeSnapshot, error) { + klog.V(5).Infof("SetDefaultSnapshotClass for snapshot [%s]", snapshot.Name) + + if snapshot.Spec.Source.VolumeSnapshotContentName != nil { + // don't return error for pre-provisioned snapshots + klog.V(5).Infof("Don't need to find SnapshotClass for pre-provisioned snapshot [%s]", snapshot.Name) + return nil, snapshot, nil + } + + storageclass, err := ctrl.getStorageClassFromVolumeSnapshot(snapshot) + if err != nil { + return nil, nil, err + } + // Find default snapshot class if available + list, err := ctrl.classLister.List(labels.Everything()) + if err != nil { + return nil, nil, err + } + defaultClasses := []*crdv1.VolumeSnapshotClass{} + + for _, class := range list { + if utils.IsDefaultAnnotation(class.ObjectMeta) && storageclass.Provisioner == class.Driver { //&& ctrl.snapshotterName == class.Snapshotter { + defaultClasses = append(defaultClasses, class) + klog.V(5).Infof("get defaultClass added: %s", class.Name) + } + } + if len(defaultClasses) == 0 { + return nil, nil, fmt.Errorf("cannot find default snapshot class") + } + if len(defaultClasses) > 1 { + klog.V(4).Infof("get DefaultClass %d defaults found", len(defaultClasses)) + return nil, nil, fmt.Errorf("%d default snapshot classes were found", len(defaultClasses)) + } + klog.V(5).Infof("setDefaultSnapshotClass [%s]: default VolumeSnapshotClassName [%s]", snapshot.Name, defaultClasses[0].Name) + snapshotClone := snapshot.DeepCopy() + snapshotClone.Spec.VolumeSnapshotClassName = &(defaultClasses[0].Name) + newSnapshot, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).Update(snapshotClone) + if err != nil { + klog.V(4).Infof("updating VolumeSnapshot[%s] default class failed %v", utils.SnapshotKey(snapshot), err) + } + _, updateErr := ctrl.storeSnapshotUpdate(newSnapshot) + if updateErr != nil { + // We will get an "snapshot update" event soon, this is not a big error + klog.V(4).Infof("setDefaultSnapshotClass [%s]: cannot update internal cache: %v", utils.SnapshotKey(snapshot), updateErr) + } + + return defaultClasses[0], newSnapshot, nil +} + +// getClaimFromVolumeSnapshot is a helper function to get PVC from VolumeSnapshot. +func (ctrl *csiSnapshotCommonController) getClaimFromVolumeSnapshot(snapshot *crdv1.VolumeSnapshot) (*v1.PersistentVolumeClaim, error) { + if snapshot.Spec.Source.PersistentVolumeClaimName == nil { + return nil, fmt.Errorf("the snapshot source PVC name is not specified") + } + pvcName := *snapshot.Spec.Source.PersistentVolumeClaimName + if pvcName == "" { + return nil, fmt.Errorf("the PVC name is not specified in snapshot %s", utils.SnapshotKey(snapshot)) + } + + pvc, err := ctrl.pvcLister.PersistentVolumeClaims(snapshot.Namespace).Get(pvcName) + if err != nil { + return nil, fmt.Errorf("failed to retrieve PVC %s from the lister: %q", pvcName, err) + } + + return pvc, nil +} + +var _ error = controllerUpdateError{} + +type controllerUpdateError struct { + message string +} + +func newControllerUpdateError(name, message string) error { + return controllerUpdateError{ + message: fmt.Sprintf("%s %s on API server: %s", controllerUpdateFailMsg, name, message), + } +} + +func (e controllerUpdateError) Error() string { + return e.message +} + +func isControllerUpdateFailError(err *crdv1.VolumeSnapshotError) bool { + if err != nil { + if strings.Contains(*err.Message, controllerUpdateFailMsg) { + return true + } + } + return false +} + +// addSnapshotFinalizer adds a Finalizer for VolumeSnapshot. +func (ctrl *csiSnapshotCommonController) addSnapshotFinalizer(snapshot *crdv1.VolumeSnapshot, addSourceFinalizer bool, addBoundFinalizer bool) error { + snapshotClone := snapshot.DeepCopy() + if addSourceFinalizer { + snapshotClone.ObjectMeta.Finalizers = append(snapshotClone.ObjectMeta.Finalizers, utils.VolumeSnapshotAsSourceFinalizer) + } + if addBoundFinalizer { + snapshotClone.ObjectMeta.Finalizers = append(snapshotClone.ObjectMeta.Finalizers, utils.VolumeSnapshotBoundFinalizer) + } + _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).Update(snapshotClone) + if err != nil { + return newControllerUpdateError(snapshot.Name, err.Error()) + } + + _, err = ctrl.storeSnapshotUpdate(snapshotClone) + if err != nil { + klog.Errorf("failed to update snapshot store %v", err) + } + + klog.V(5).Infof("Added protection finalizer to volume snapshot %s", utils.SnapshotKey(snapshot)) + return nil +} + +// removeSnapshotFinalizer removes a Finalizer for VolumeSnapshot. +func (ctrl *csiSnapshotCommonController) removeSnapshotFinalizer(snapshot *crdv1.VolumeSnapshot, removeSourceFinalizer bool, removeBoundFinalizer bool) error { + snapshotClone := snapshot.DeepCopy() + if removeSourceFinalizer { + snapshotClone.ObjectMeta.Finalizers = slice.RemoveString(snapshotClone.ObjectMeta.Finalizers, utils.VolumeSnapshotAsSourceFinalizer, nil) + } + if removeBoundFinalizer { + snapshotClone.ObjectMeta.Finalizers = slice.RemoveString(snapshotClone.ObjectMeta.Finalizers, utils.VolumeSnapshotBoundFinalizer, nil) + } + _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).Update(snapshotClone) + if err != nil { + return newControllerUpdateError(snapshot.Name, err.Error()) + } + + _, err = ctrl.storeSnapshotUpdate(snapshotClone) + if err != nil { + klog.Errorf("failed to update snapshot store %v", err) + } + + klog.V(5).Infof("Removed protection finalizer from volume snapshot %s", utils.SnapshotKey(snapshot)) + return nil +} + +func (ctrl *csiSnapshotCommonController) contentExists(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshotContent, error) { + var contentName string + if snapshot.Status != nil && snapshot.Status.BoundVolumeSnapshotContentName != nil { + contentName = *snapshot.Status.BoundVolumeSnapshotContentName + } else { + contentName = utils.GetSnapshotContentNameForSnapshot(snapshot) + } + obj, found, err := ctrl.contentStore.GetByKey(contentName) + if err != nil { + return nil, err + } + // Not in the content cache store, no error + if !found { + return nil, nil + } + // Found in content cache store + content, ok := obj.(*crdv1.VolumeSnapshotContent) + if !ok { + return content, fmt.Errorf("Cannot convert object from snapshot content store to VolumeSnapshotContent %q!?: %#v", contentName, obj) + } + // Found in content cache store and convert object is successful + return content, nil +} diff --git a/pkg/controller/snapshot_controller_base.go b/pkg/common-controller/snapshot_controller_base.go similarity index 80% rename from pkg/controller/snapshot_controller_base.go rename to pkg/common-controller/snapshot_controller_base.go index 7f58554c..958f03fa 100644 --- a/pkg/controller/snapshot_controller_base.go +++ b/pkg/common-controller/snapshot_controller_base.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package common_controller import ( "fmt" @@ -24,9 +24,9 @@ import ( clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" storageinformers "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions/volumesnapshot/v1beta1" storagelisters "github.com/kubernetes-csi/external-snapshotter/pkg/client/listers/volumesnapshot/v1beta1" - "github.com/kubernetes-csi/external-snapshotter/pkg/snapshotter" + "github.com/kubernetes-csi/external-snapshotter/pkg/utils" - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" @@ -42,10 +42,9 @@ import ( "k8s.io/kubernetes/pkg/util/goroutinemap" ) -type csiSnapshotController struct { +type csiSnapshotCommonController struct { clientset clientset.Interface client kubernetes.Interface - driverName string eventRecorder record.EventRecorder snapshotQueue workqueue.RateLimitingInterface contentQueue workqueue.RateLimitingInterface @@ -62,7 +61,6 @@ type csiSnapshotController struct { snapshotStore cache.Store contentStore cache.Store - handler Handler // Map of scheduled/running operations. runningOperations goroutinemap.GoRoutineMap @@ -71,43 +69,36 @@ type csiSnapshotController struct { resyncPeriod time.Duration } -// NewCSISnapshotController returns a new *csiSnapshotController -func NewCSISnapshotController( +// NewCSISnapshotController returns a new *csiSnapshotCommonController +func NewCSISnapshotCommonController( clientset clientset.Interface, client kubernetes.Interface, - driverName string, volumeSnapshotInformer storageinformers.VolumeSnapshotInformer, volumeSnapshotContentInformer storageinformers.VolumeSnapshotContentInformer, volumeSnapshotClassInformer storageinformers.VolumeSnapshotClassInformer, pvcInformer coreinformers.PersistentVolumeClaimInformer, createSnapshotContentRetryCount int, createSnapshotContentInterval time.Duration, - snapshotter snapshotter.Snapshotter, - timeout time.Duration, resyncPeriod time.Duration, - snapshotNamePrefix string, - snapshotNameUUIDLength int, -) *csiSnapshotController { +) *csiSnapshotCommonController { broadcaster := record.NewBroadcaster() broadcaster.StartLogging(klog.Infof) broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)}) var eventRecorder record.EventRecorder - eventRecorder = broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: fmt.Sprintf("csi-snapshotter %s", driverName)}) + eventRecorder = broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: fmt.Sprintf("snapshot-controller")}) - ctrl := &csiSnapshotController{ + ctrl := &csiSnapshotCommonController{ clientset: clientset, client: client, - driverName: driverName, eventRecorder: eventRecorder, - handler: NewCSIHandler(snapshotter, timeout, snapshotNamePrefix, snapshotNameUUIDLength), runningOperations: goroutinemap.NewGoRoutineMap(true), createSnapshotContentRetryCount: createSnapshotContentRetryCount, createSnapshotContentInterval: createSnapshotContentInterval, resyncPeriod: resyncPeriod, snapshotStore: cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc), contentStore: cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc), - snapshotQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "csi-snapshotter-snapshot"), - contentQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "csi-snapshotter-content"), + snapshotQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "snapshot-controller-snapshot"), + contentQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "snapshot-controller-content"), } ctrl.pvcLister = pvcInformer.Lister() @@ -141,12 +132,12 @@ func NewCSISnapshotController( return ctrl } -func (ctrl *csiSnapshotController) Run(workers int, stopCh <-chan struct{}) { +func (ctrl *csiSnapshotCommonController) Run(workers int, stopCh <-chan struct{}) { defer ctrl.snapshotQueue.ShutDown() defer ctrl.contentQueue.ShutDown() - klog.Infof("Starting CSI snapshotter") - defer klog.Infof("Shutting CSI snapshotter") + klog.Infof("Starting snapshot controller") + defer klog.Infof("Shutting snapshot controller") if !cache.WaitForCacheSync(stopCh, ctrl.snapshotListerSynced, ctrl.contentListerSynced, ctrl.classListerSynced, ctrl.pvcListerSynced) { klog.Errorf("Cannot sync caches") @@ -164,7 +155,7 @@ func (ctrl *csiSnapshotController) Run(workers int, stopCh <-chan struct{}) { } // enqueueSnapshotWork adds snapshot to given work queue. -func (ctrl *csiSnapshotController) enqueueSnapshotWork(obj interface{}) { +func (ctrl *csiSnapshotCommonController) enqueueSnapshotWork(obj interface{}) { // Beware of "xxx deleted" events if unknown, ok := obj.(cache.DeletedFinalStateUnknown); ok && unknown.Obj != nil { obj = unknown.Obj @@ -181,7 +172,7 @@ func (ctrl *csiSnapshotController) enqueueSnapshotWork(obj interface{}) { } // enqueueContentWork adds snapshot content to given work queue. -func (ctrl *csiSnapshotController) enqueueContentWork(obj interface{}) { +func (ctrl *csiSnapshotCommonController) enqueueContentWork(obj interface{}) { // Beware of "xxx deleted" events if unknown, ok := obj.(cache.DeletedFinalStateUnknown); ok && unknown.Obj != nil { obj = unknown.Obj @@ -199,7 +190,7 @@ func (ctrl *csiSnapshotController) enqueueContentWork(obj interface{}) { // snapshotWorker processes items from snapshotQueue. It must run only once, // syncSnapshot is not assured to be reentrant. -func (ctrl *csiSnapshotController) snapshotWorker() { +func (ctrl *csiSnapshotCommonController) snapshotWorker() { workFunc := func() bool { keyObj, quit := ctrl.snapshotQueue.Get() if quit { @@ -264,7 +255,7 @@ func (ctrl *csiSnapshotController) snapshotWorker() { // contentWorker processes items from contentQueue. It must run only once, // syncContent is not assured to be reentrant. -func (ctrl *csiSnapshotController) contentWorker() { +func (ctrl *csiSnapshotCommonController) contentWorker() { workFunc := func() bool { keyObj, quit := ctrl.contentQueue.Get() if quit { @@ -283,9 +274,7 @@ func (ctrl *csiSnapshotController) contentWorker() { // The content still exists in informer cache, the event must have // been add/update/sync if err == nil { - if ctrl.isDriverMatch(content) { - ctrl.updateContent(content) - } + ctrl.updateContent(content) return false } if !errors.IsNotFound(err) { @@ -323,22 +312,16 @@ func (ctrl *csiSnapshotController) contentWorker() { } } -// verify whether the driver specified in VolumeSnapshotContent matches the controller's driver name -func (ctrl *csiSnapshotController) isDriverMatch(content *crdv1.VolumeSnapshotContent) bool { - return content.Spec.Driver == ctrl.driverName -} - // checkAndUpdateSnapshotClass gets the VolumeSnapshotClass from VolumeSnapshot. If it is not set, -// gets it from default VolumeSnapshotClass and sets it. It also detects if snapshotter in the -// VolumeSnapshotClass is the same as the snapshotter in external controller. -func (ctrl *csiSnapshotController) checkAndUpdateSnapshotClass(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshot, error) { +// gets it from default VolumeSnapshotClass and sets it. +func (ctrl *csiSnapshotCommonController) checkAndUpdateSnapshotClass(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshot, error) { className := snapshot.Spec.VolumeSnapshotClassName var class *crdv1.VolumeSnapshotClass var err error newSnapshot := snapshot if className != nil { klog.V(5).Infof("checkAndUpdateSnapshotClass [%s]: VolumeSnapshotClassName [%s]", snapshot.Name, *className) - class, err = ctrl.GetSnapshotClass(*className) + class, err = ctrl.getSnapshotClass(*className) if err != nil { klog.Errorf("checkAndUpdateSnapshotClass failed to getSnapshotClass %v", err) ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "GetSnapshotClassFailed", fmt.Sprintf("Failed to get snapshot class with error %v", err)) @@ -354,20 +337,19 @@ func (ctrl *csiSnapshotController) checkAndUpdateSnapshotClass(snapshot *crdv1.V } } - klog.V(5).Infof("VolumeSnapshotClass Driver [%s] Snapshot Controller driverName [%s]", class.Driver, ctrl.driverName) - if class.Driver != ctrl.driverName { - klog.V(4).Infof("Skipping VolumeSnapshot %s for snapshotter [%s] in VolumeSnapshotClass because it does not match with the snapshotter for controller [%s]", snapshotKey(snapshot), class.Driver, ctrl.driverName) - return nil, fmt.Errorf("volumeSnapshotClass does not match with the snapshotter for controller") + // For pre-provisioned snapshots, we may not have snapshot class + if class != nil { + klog.V(5).Infof("VolumeSnapshotClass [%s] Driver [%s]", class.Name, class.Driver) } return newSnapshot, nil } // updateSnapshot runs in worker thread and handles "snapshot added", // "snapshot updated" and "periodic sync" events. -func (ctrl *csiSnapshotController) updateSnapshot(snapshot *crdv1.VolumeSnapshot) { +func (ctrl *csiSnapshotCommonController) updateSnapshot(snapshot *crdv1.VolumeSnapshot) { // Store the new snapshot version in the cache and do not process it if this is // an old version. - klog.V(5).Infof("updateSnapshot %q", snapshotKey(snapshot)) + klog.V(5).Infof("updateSnapshot %q", utils.SnapshotKey(snapshot)) newSnapshot, err := ctrl.storeSnapshotUpdate(snapshot) if err != nil { klog.Errorf("%v", err) @@ -380,16 +362,16 @@ func (ctrl *csiSnapshotController) updateSnapshot(snapshot *crdv1.VolumeSnapshot if errors.IsConflict(err) { // Version conflict error happens quite often and the controller // recovers from it easily. - klog.V(3).Infof("could not sync claim %q: %+v", snapshotKey(snapshot), err) + klog.V(3).Infof("could not sync claim %q: %+v", utils.SnapshotKey(snapshot), err) } else { - klog.Errorf("could not sync volume %q: %+v", snapshotKey(snapshot), err) + klog.Errorf("could not sync volume %q: %+v", utils.SnapshotKey(snapshot), err) } } } // updateContent runs in worker thread and handles "content added", // "content updated" and "periodic sync" events. -func (ctrl *csiSnapshotController) updateContent(content *crdv1.VolumeSnapshotContent) { +func (ctrl *csiSnapshotCommonController) updateContent(content *crdv1.VolumeSnapshotContent) { // Store the new content version in the cache and do not process it if this is // an old version. new, err := ctrl.storeContentUpdate(content) @@ -412,28 +394,31 @@ func (ctrl *csiSnapshotController) updateContent(content *crdv1.VolumeSnapshotCo } // deleteSnapshot runs in worker thread and handles "snapshot deleted" event. -func (ctrl *csiSnapshotController) deleteSnapshot(snapshot *crdv1.VolumeSnapshot) { +func (ctrl *csiSnapshotCommonController) deleteSnapshot(snapshot *crdv1.VolumeSnapshot) { _ = ctrl.snapshotStore.Delete(snapshot) - klog.V(4).Infof("snapshot %q deleted", snapshotKey(snapshot)) + klog.V(4).Infof("snapshot %q deleted", utils.SnapshotKey(snapshot)) - if snapshot.Status.BoundVolumeSnapshotContentName == nil { - klog.V(5).Infof("deleteSnapshot[%q]: content not bound", snapshotKey(snapshot)) + snapshotContentName := "" + if snapshot.Status != nil && snapshot.Status.BoundVolumeSnapshotContentName != nil { + snapshotContentName = *snapshot.Status.BoundVolumeSnapshotContentName + } + if snapshotContentName == "" { + klog.V(5).Infof("deleteSnapshot[%q]: content not bound", utils.SnapshotKey(snapshot)) return } - snapshotContentName := *snapshot.Status.BoundVolumeSnapshotContentName // sync the content when its snapshot is deleted. Explicitly sync'ing the // content here in response to snapshot deletion prevents the content from // waiting until the next sync period for its Release. - klog.V(5).Infof("deleteSnapshot[%q]: scheduling sync of content %s", snapshotKey(snapshot), snapshotContentName) + klog.V(5).Infof("deleteSnapshot[%q]: scheduling sync of content %s", utils.SnapshotKey(snapshot), snapshotContentName) ctrl.contentQueue.Add(snapshotContentName) } // deleteContent runs in worker thread and handles "content deleted" event. -func (ctrl *csiSnapshotController) deleteContent(content *crdv1.VolumeSnapshotContent) { +func (ctrl *csiSnapshotCommonController) deleteContent(content *crdv1.VolumeSnapshotContent) { _ = ctrl.contentStore.Delete(content) klog.V(4).Infof("content %q deleted", content.Name) - snapshotName := snapshotRefKey(content.Spec.VolumeSnapshotRef) + snapshotName := utils.SnapshotRefKey(&content.Spec.VolumeSnapshotRef) if snapshotName == "" { klog.V(5).Infof("deleteContent[%q]: content not bound", content.Name) return @@ -448,7 +433,7 @@ func (ctrl *csiSnapshotController) deleteContent(content *crdv1.VolumeSnapshotCo // initializeCaches fills all controller caches with initial data from etcd in // order to have the caches already filled when first addSnapshot/addContent to // perform initial synchronization of the controller. -func (ctrl *csiSnapshotController) initializeCaches(snapshotLister storagelisters.VolumeSnapshotLister, contentLister storagelisters.VolumeSnapshotContentLister) { +func (ctrl *csiSnapshotCommonController) initializeCaches(snapshotLister storagelisters.VolumeSnapshotLister, contentLister storagelisters.VolumeSnapshotContentLister) { snapshotList, err := snapshotLister.List(labels.Everything()) if err != nil { klog.Errorf("CSISnapshotController can't initialize caches: %v", err) diff --git a/pkg/controller/snapshot_controller_test.go b/pkg/common-controller/snapshot_controller_test.go similarity index 93% rename from pkg/controller/snapshot_controller_test.go rename to pkg/common-controller/snapshot_controller_test.go index dc89bd25..e1785208 100644 --- a/pkg/controller/snapshot_controller_test.go +++ b/pkg/common-controller/snapshot_controller_test.go @@ -14,21 +14,22 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package common_controller import ( "testing" crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/pkg/utils" "k8s.io/client-go/tools/cache" ) var deletionPolicy = crdv1.VolumeSnapshotContentDelete func storeVersion(t *testing.T, prefix string, c cache.Store, version string, expectedReturn bool) { - content := newContent("contentName", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "pv-handle-1-1", deletionPolicy, nil, nil, false) + content := newContent("contentName", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "pv-handle-1-1", deletionPolicy, nil, nil, false, true) content.ResourceVersion = version - ret, err := storeObjectUpdate(c, content, "content") + ret, err := utils.StoreObjectUpdate(c, content, "content") if err != nil { t.Errorf("%s: expected storeObjectUpdate to succeed, got: %v", prefix, err) } @@ -84,9 +85,9 @@ func TestControllerCacheParsingError(t *testing.T) { c := cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc) // There must be something in the cache to compare with storeVersion(t, "Step1", c, "1", true) - content := newContent("contentName", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "pv-handle-1-1", deletionPolicy, nil, nil, false) + content := newContent("contentName", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "pv-handle-1-1", deletionPolicy, nil, nil, false, true) content.ResourceVersion = "xxx" - _, err := storeObjectUpdate(c, content, "content") + _, err := utils.StoreObjectUpdate(c, content, "content") if err == nil { t.Errorf("Expected parsing error, got nil instead") } diff --git a/pkg/controller/snapshot_create_test.go b/pkg/common-controller/snapshot_create_test.go similarity index 63% rename from pkg/controller/snapshot_create_test.go rename to pkg/common-controller/snapshot_create_test.go index 2d691ecc..21602de8 100644 --- a/pkg/controller/snapshot_create_test.go +++ b/pkg/common-controller/snapshot_create_test.go @@ -14,10 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package common_controller import ( - "errors" + //"errors" "testing" "time" @@ -71,12 +71,12 @@ func TestCreateSnapshotSync(t *testing.T) { { name: "6-1 - successful create snapshot with snapshot class gold", initialContents: nocontents, - expectedContents: newContentArrayWithReadyToUse("snapcontent-snapuid6-1", "snapuid6-1", "snap6-1", "sid6-1", classGold, "", "pv-handle6-1", deletionPolicy, &timeNowStamp, &defaultSize, &True, false), - initialSnapshots: newSnapshotArray("snap6-1", "snapuid6-1", "claim6-1", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap6-1", "snapuid6-1", "claim6-1", "", classGold, "snapcontent-snapuid6-1", &True, metaTimeNowUnix, getSize(defaultSize), nil), + expectedContents: newContentArrayNoStatus("snapcontent-snapuid6-1", "snapuid6-1", "snap6-1", "sid6-1", classGold, "", "pv-handle6-1", deletionPolicy, nil, nil, false, false), + initialSnapshots: newSnapshotArray("snap6-1", "snapuid6-1", "claim6-1", "", classGold, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap6-1", "snapuid6-1", "claim6-1", "", classGold, "snapcontent-snapuid6-1", &False, nil, nil, nil, false), initialClaims: newClaimArray("claim6-1", "pvc-uid6-1", "1Gi", "volume6-1", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume6-1", "pv-uid6-1", "pv-handle6-1", "1Gi", "pvc-uid6-1", "claim6-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - expectedCreateCalls: []createCall{ + /*expectedCreateCalls: []createCall{ { snapshotName: "snapshot-snapuid6-1", volume: newVolume("volume6-1", "pv-uid6-1", "pv-handle6-1", "1Gi", "pvc-uid6-1", "claim6-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), @@ -88,19 +88,19 @@ func TestCreateSnapshotSync(t *testing.T) { creationTime: timeNow, readyToUse: True, }, - }, + },*/ errors: noerrors, test: testSyncSnapshot, }, { name: "6-2 - successful create snapshot with snapshot class silver", initialContents: nocontents, - expectedContents: newContentArrayWithReadyToUse("snapcontent-snapuid6-2", "snapuid6-2", "snap6-2", "sid6-2", classSilver, "", "pv-handle6-2", deletionPolicy, &timeNowStamp, &defaultSize, &True, false), - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "snapcontent-snapuid6-2", &True, metaTimeNowUnix, getSize(defaultSize), nil), + expectedContents: newContentArrayNoStatus("snapcontent-snapuid6-2", "snapuid6-2", "snap6-2", "sid6-2", classSilver, "", "pv-handle6-2", deletionPolicy, nil, nil, false, false), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "snapcontent-snapuid6-2", &False, nil, nil, nil, false), initialClaims: newClaimArray("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume6-2", "pv-uid6-2", "pv-handle6-2", "1Gi", "pvc-uid6-2", "claim6-2", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - expectedCreateCalls: []createCall{ + /*expectedCreateCalls: []createCall{ { snapshotName: "snapshot-snapuid6-2", volume: newVolume("volume6-2", "pv-uid6-2", "pv-handle6-2", "1Gi", "pvc-uid6-2", "claim6-2", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), @@ -112,56 +112,7 @@ func TestCreateSnapshotSync(t *testing.T) { creationTime: timeNow, readyToUse: True, }, - }, - errors: noerrors, - test: testSyncSnapshot, - }, - { - name: "6-5 - successful create snapshot with status uploading", - initialContents: nocontents, - expectedContents: newContentArrayWithReadyToUse("snapcontent-snapuid6-5", "snapuid6-5", "snap6-5", "sid6-5", classGold, "", "pv-handle6-5", deletionPolicy, &timeNowStamp, &defaultSize, &False, false), - initialSnapshots: newSnapshotArray("snap6-5", "snapuid6-5", "claim6-5", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap6-5", "snapuid6-5", "claim6-5", "", classGold, "snapcontent-snapuid6-5", &False, metaTimeNowUnix, getSize(defaultSize), nil), - initialClaims: newClaimArray("claim6-5", "pvc-uid6-5", "1Gi", "volume6-5", v1.ClaimBound, &classEmpty), - initialVolumes: newVolumeArray("volume6-5", "pv-uid6-5", "pv-handle6-5", "1Gi", "pvc-uid6-5", "claim6-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - expectedCreateCalls: []createCall{ - { - snapshotName: "snapshot-snapuid6-5", - volume: newVolume("volume6-5", "pv-uid6-5", "pv-handle6-5", "1Gi", "pvc-uid6-5", "claim6-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - parameters: map[string]string{"param1": "value1"}, - // information to return - driverName: mockDriverName, - size: defaultSize, - snapshotId: "sid6-5", - creationTime: timeNow, - readyToUse: False, - }, - }, - errors: noerrors, - test: testSyncSnapshot, - }, - { - // TODO(xiangqian): this test does not match its name - name: "6-6 - successful create snapshot with status error uploading", - initialContents: nocontents, - expectedContents: newContentArrayWithReadyToUse("snapcontent-snapuid6-6", "snapuid6-6", "snap6-6", "sid6-6", classGold, "", "pv-handle6-6", deletionPolicy, &timeNowStamp, &defaultSize, &False, false), - initialSnapshots: newSnapshotArray("snap6-6", "snapuid6-6", "claim6-6", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap6-6", "snapuid6-6", "claim6-6", "", classGold, "snapcontent-snapuid6-6", &False, metaTimeNowUnix, getSize(defaultSize), nil), - initialClaims: newClaimArray("claim6-6", "pvc-uid6-6", "1Gi", "volume6-6", v1.ClaimBound, &classEmpty), - initialVolumes: newVolumeArray("volume6-6", "pv-uid6-6", "pv-handle6-6", "1Gi", "pvc-uid6-6", "claim6-6", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - expectedCreateCalls: []createCall{ - { - snapshotName: "snapshot-snapuid6-6", - volume: newVolume("volume6-6", "pv-uid6-6", "pv-handle6-6", "1Gi", "pvc-uid6-6", "claim6-6", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - parameters: map[string]string{"param1": "value1"}, - // information to return - driverName: mockDriverName, - size: defaultSize, - snapshotId: "sid6-6", - creationTime: timeNow, - readyToUse: False, - }, - }, + },*/ errors: noerrors, test: testSyncSnapshot, }, @@ -169,15 +120,16 @@ func TestCreateSnapshotSync(t *testing.T) { name: "7-1 - fail to create snapshot with non-existing snapshot class", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, newVolumeError("Failed to create snapshot: failed to get input parameters to create snapshot snap7-1: \"failed to retrieve snapshot class non-existing from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"non-existing\\\\\\\" not found\\\"\"")), + initialSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-1: \"failed to retrieve snapshot class non-existing from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"non-existing\\\\\\\" not found\\\"\""), false), initialClaims: newClaimArray("claim7-1", "pvc-uid7-1", "1Gi", "volume7-1", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume7-1", "pv-uid7-1", "pv-handle7-1", "1Gi", "pvc-uid7-1", "claim7-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - expectedEvents: []string{"Warning SnapshotCreationFailed"}, + expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, errors: noerrors, + expectSuccess: false, test: testSyncSnapshot, }, - { + /*{ name: "7-2 - fail to create snapshot with snapshot class invalid-secret-class", initialContents: nocontents, expectedContents: nocontents, @@ -188,59 +140,63 @@ func TestCreateSnapshotSync(t *testing.T) { expectedEvents: []string{"Warning SnapshotCreationFailed"}, errors: noerrors, test: testSyncSnapshot, - }, + },*/ { name: "7-3 - fail to create snapshot without snapshot class ", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-3", "snapuid7-3", "claim7-3", "", "", "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap7-3", "snapuid7-3", "claim7-3", "", "", "", &False, nil, nil, newVolumeError("Failed to create snapshot: failed to get input parameters to create snapshot snap7-3: \"failed to retrieve snapshot class from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"\\\\\\\" not found\\\"\"")), + initialSnapshots: newSnapshotArray("snap7-3", "snapuid7-3", "claim7-3", "", "", "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap7-3", "snapuid7-3", "claim7-3", "", "", "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-3: \"failed to retrieve snapshot class from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"\\\\\\\" not found\\\"\""), false), initialClaims: newClaimArray("claim7-3", "pvc-uid7-3", "1Gi", "volume7-3", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume7-3", "pv-uid7-3", "pv-handle7-3", "1Gi", "pvc-uid7-3", "claim7-3", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialStorageClasses: []*storage.StorageClass{diffDriverStorageClass}, - expectedEvents: []string{"Warning SnapshotCreationFailed"}, + expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, errors: noerrors, + expectSuccess: false, test: testSyncSnapshot, }, { name: "7-4 - fail create snapshot with no-existing claim", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-4", "snapuid7-4", "claim7-4", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap7-4", "snapuid7-4", "claim7-4", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot: failed to get input parameters to create snapshot snap7-4: \"failed to retrieve PVC claim7-4 from the lister: \\\"persistentvolumeclaim \\\\\\\"claim7-4\\\\\\\" not found\\\"\"")), + initialSnapshots: newSnapshotArray("snap7-4", "snapuid7-4", "claim7-4", "", classGold, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap7-4", "snapuid7-4", "claim7-4", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-4: \"failed to retrieve PVC claim7-4 from the lister: \\\"persistentvolumeclaim \\\\\\\"claim7-4\\\\\\\" not found\\\"\""), false), initialVolumes: newVolumeArray("volume7-4", "pv-uid7-4", "pv-handle7-4", "1Gi", "pvc-uid7-4", "claim7-4", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - expectedEvents: []string{"Warning SnapshotCreationFailed"}, + expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, errors: noerrors, + expectSuccess: false, test: testSyncSnapshot, }, { name: "7-5 - fail create snapshot with no-existing volume", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-5", "snapuid7-5", "claim7-5", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap7-5", "snapuid7-5", "claim7-5", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot: failed to get input parameters to create snapshot snap7-5: \"failed to retrieve PV volume7-5 from the API server: \\\"cannot find volume volume7-5\\\"\"")), + initialSnapshots: newSnapshotArray("snap7-5", "snapuid7-5", "claim7-5", "", classGold, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap7-5", "snapuid7-5", "claim7-5", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-5: \"failed to retrieve PV volume7-5 from the API server: \\\"cannot find volume volume7-5\\\"\""), false), initialClaims: newClaimArray("claim7-5", "pvc-uid7-5", "1Gi", "volume7-5", v1.ClaimBound, &classEmpty), - expectedEvents: []string{"Warning SnapshotCreationFailed"}, + expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, errors: noerrors, + expectSuccess: false, test: testSyncSnapshot, }, { name: "7-6 - fail create snapshot with claim that is not yet bound", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-6", "snapuid7-6", "claim7-6", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap7-6", "snapuid7-6", "claim7-6", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot: failed to get input parameters to create snapshot snap7-6: \"the PVC claim7-6 is not yet bound to a PV, will not attempt to take a snapshot\"")), + initialSnapshots: newSnapshotArray("snap7-6", "snapuid7-6", "claim7-6", "", classGold, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap7-6", "snapuid7-6", "claim7-6", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-6: \"the PVC claim7-6 is not yet bound to a PV, will not attempt to take a snapshot\""), false), initialClaims: newClaimArray("claim7-6", "pvc-uid7-6", "1Gi", "", v1.ClaimPending, &classEmpty), - expectedEvents: []string{"Warning SnapshotCreationFailed"}, + expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, errors: noerrors, + expectSuccess: false, test: testSyncSnapshot, }, - { + /*{ name: "7-7 - fail create snapshot due to csi driver error", initialContents: nocontents, expectedContents: nocontents, initialSnapshots: newSnapshotArray("snap7-7", "snapuid7-7", "claim7-7", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap7-7", "snapuid7-7", "claim7-7", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot: failed to take snapshot of the volume, volume7-7: \"mock create snapshot error\"")), + expectedSnapshots: newSnapshotArray("snap7-7", "snapuid7-7", "claim7-7", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to take snapshot of the volume, volume7-7: \"mock create snapshot error\"")), initialClaims: newClaimArray("claim7-7", "pvc-uid7-7", "1Gi", "volume7-7", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume7-7", "pv-uid7-7", "pv-handle7-7", "1Gi", "pvc-uid7-7", "claim7-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), expectedCreateCalls: []createCall{ @@ -253,41 +209,44 @@ func TestCreateSnapshotSync(t *testing.T) { }, }, errors: noerrors, - expectedEvents: []string{"Warning SnapshotCreationFailed"}, + expectSuccess: false, + expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, test: testSyncSnapshot, - }, - { - name: "7-8 - fail create snapshot due to cannot update snapshot status", - initialContents: nocontents, - expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-8", "snapuid7-8", "claim7-8", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap7-8", "snapuid7-8", "claim7-8", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot: snapshot controller failed to update default/snap7-8 on API server: mock update error")), - initialClaims: newClaimArray("claim7-8", "pvc-uid7-8", "1Gi", "volume7-8", v1.ClaimBound, &classEmpty), - initialVolumes: newVolumeArray("volume7-8", "pv-uid7-8", "pv-handle7-8", "1Gi", "pvc-uid7-8", "claim7-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - expectedCreateCalls: []createCall{ - { - snapshotName: "snapshot-snapuid7-8", - volume: newVolume("volume7-8", "pv-uid7-8", "pv-handle7-8", "1Gi", "pvc-uid7-8", "claim7-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - parameters: map[string]string{"param1": "value1"}, - // information to return - driverName: mockDriverName, - size: defaultSize, - snapshotId: "sid7-8", - creationTime: timeNow, - readyToUse: True, - }, + },*/ + /*{ + name: "7-8 - fail create snapshot due to cannot update snapshot status", + initialContents: nocontents, + expectedContents: nocontents, + initialSnapshots: newSnapshotArray("snap7-8", "snapuid7-8", "claim7-8", "", classGold, "", &False, nil, nil, nil), + expectedSnapshots: newSnapshotArray("snap7-8", "snapuid7-8", "claim7-8", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot: snapshot controller failed to update default/snap7-8 on API server: mock update error")), + initialClaims: newClaimArray("claim7-8", "pvc-uid7-8", "1Gi", "volume7-8", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume7-8", "pv-uid7-8", "pv-handle7-8", "1Gi", "pvc-uid7-8", "claim7-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + /*expectedCreateCalls: []createCall{ + { + snapshotName: "snapshot-snapuid7-8", + volume: newVolume("volume7-8", "pv-uid7-8", "pv-handle7-8", "1Gi", "pvc-uid7-8", "claim7-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + parameters: map[string]string{"param1": "value1"}, + // information to return + driverName: mockDriverName, + size: defaultSize, + snapshotId: "sid7-8", + creationTime: timeNow, + readyToUse: True, }, - errors: []reactorError{ + },*/ + /*errors: []reactorError{ // Inject error to the forth client.VolumesnapshotV1beta1().VolumeSnapshots().Update call. // All other calls will succeed. {"update", "volumesnapshots", errors.New("mock update error")}, {"update", "volumesnapshots", errors.New("mock update error")}, {"update", "volumesnapshots", errors.New("mock update error")}, }, - expectedEvents: []string{"Warning SnapshotCreationFailed"}, + expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, + + expectSuccess: false, test: testSyncSnapshot, - }, - { + },*/ + /*{ // TODO(xiangqian): this test case needs to be revisited the scenario // of VolumeSnapshotContent saving failure. Since there will be no content object // in API server, it could potentially cause leaking issue @@ -330,7 +289,7 @@ func TestCreateSnapshotSync(t *testing.T) { initialSecrets: []*v1.Secret{}, // no initial secret created errors: noerrors, test: testSyncSnapshot, - }, + },*/ } runSyncTests(t, tests, snapshotClasses) } diff --git a/pkg/controller/snapshot_delete_test.go b/pkg/common-controller/snapshot_delete_test.go similarity index 64% rename from pkg/controller/snapshot_delete_test.go rename to pkg/common-controller/snapshot_delete_test.go index f3c157bd..f69cb5c8 100644 --- a/pkg/controller/snapshot_delete_test.go +++ b/pkg/common-controller/snapshot_delete_test.go @@ -14,13 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package common_controller import ( - "errors" + //"errors" "testing" crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/pkg/utils" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -34,18 +35,18 @@ var class2Parameters = map[string]string{ } var class3Parameters = map[string]string{ - "param3": "value3", - snapshotterSecretNameKey: "name", + "param3": "value3", + utils.SnapshotterSecretNameKey: "name", } var class4Parameters = map[string]string{ - snapshotterSecretNameKey: "emptysecret", - snapshotterSecretNamespaceKey: "default", + utils.SnapshotterSecretNameKey: "emptysecret", + utils.SnapshotterSecretNamespaceKey: "default", } var class5Parameters = map[string]string{ - snapshotterSecretNameKey: "secret", - snapshotterSecretNamespaceKey: "default", + utils.SnapshotterSecretNameKey: "secret", + utils.SnapshotterSecretNamespaceKey: "default", } var snapshotClasses = []*crdv1.VolumeSnapshotClass{ @@ -110,7 +111,7 @@ var snapshotClasses = []*crdv1.VolumeSnapshotClass{ }, ObjectMeta: metav1.ObjectMeta{ Name: defaultClass, - Annotations: map[string]string{IsDefaultSnapshotClassAnnotation: "true"}, + Annotations: map[string]string{utils.IsDefaultSnapshotClassAnnotation: "true"}, }, Driver: mockDriverName, DeletionPolicy: crdv1.VolumeSnapshotContentDelete, @@ -124,50 +125,50 @@ var snapshotClasses = []*crdv1.VolumeSnapshotClass{ func TestDeleteSync(t *testing.T) { tests := []controllerTest{ { - name: "1-1 - content with empty snapshot class is deleted if it is bound to a non-exist snapshot and also has a snapshot uid specified", - initialContents: newContentArray("content1-1", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "", deletionPolicy, nil, nil, true), - expectedContents: nocontents, - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - expectedEvents: noevents, - errors: noerrors, - expectedDeleteCalls: []deleteCall{{"sid1-1", nil, nil}}, - test: testSyncContent, + name: "1-1 - content with empty snapshot class is deleted if it is bound to a non-exist snapshot and also has a snapshot uid specified", + initialContents: newContentArray("content1-1", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "", deletionPolicy, nil, nil, true), + expectedContents: newContentArray("content1-1", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "", deletionPolicy, nil, nil, true), + initialSnapshots: nosnapshots, + expectedSnapshots: nosnapshots, + expectedEvents: noevents, + errors: noerrors, + //expectedDeleteCalls: []deleteCall{{"sid1-1", nil, nil}}, + test: testSyncContent, }, { - name: "2-1 - content with empty snapshot class will not be deleted if it is bound to a non-exist snapshot but it does not have a snapshot uid specified", - initialContents: newContentArray("content2-1", "", "snap2-1", "sid2-1", "", "", "", deletionPolicy, nil, nil, true), - expectedContents: newContentArray("content2-1", "", "snap2-1", "sid2-1", "", "", "", deletionPolicy, nil, nil, true), - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - expectedEvents: noevents, - errors: noerrors, - expectedDeleteCalls: []deleteCall{{"sid2-1", nil, nil}}, - test: testSyncContent, + name: "2-1 - content with empty snapshot class will not be deleted if it is bound to a non-exist snapshot but it does not have a snapshot uid specified", + initialContents: newContentArray("content2-1", "", "snap2-1", "sid2-1", "", "", "", deletionPolicy, nil, nil, true), + expectedContents: newContentArray("content2-1", "", "snap2-1", "sid2-1", "", "", "", deletionPolicy, nil, nil, true), + initialSnapshots: nosnapshots, + expectedSnapshots: nosnapshots, + expectedEvents: noevents, + errors: noerrors, + //expectedDeleteCalls: []deleteCall{{"sid2-1", nil, nil}}, + test: testSyncContent, }, { - name: "1-2 - successful delete with snapshot class that has empty secret parameter", - initialContents: newContentArray("content1-2", "sid1-2", "snap1-2", "sid1-2", emptySecretClass, "", "", deletionPolicy, nil, nil, true), - expectedContents: nocontents, - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - initialSecrets: []*v1.Secret{emptySecret()}, - expectedEvents: noevents, - errors: noerrors, - expectedDeleteCalls: []deleteCall{{"sid1-2", map[string]string{}, nil}}, - test: testSyncContent, + name: "1-2 - successful delete with snapshot class that has empty secret parameter", + initialContents: newContentArray("content1-2", "sid1-2", "snap1-2", "sid1-2", emptySecretClass, "", "", deletionPolicy, nil, nil, true), + expectedContents: newContentArray("content1-2", "sid1-2", "snap1-2", "sid1-2", emptySecretClass, "", "", deletionPolicy, nil, nil, true), + initialSnapshots: nosnapshots, + expectedSnapshots: nosnapshots, + initialSecrets: []*v1.Secret{emptySecret()}, + expectedEvents: noevents, + errors: noerrors, + //expectedDeleteCalls: []deleteCall{{"sid1-2", map[string]string{}, nil}}, + test: testSyncContent, }, { - name: "1-3 - successful delete with snapshot class that has valid secret parameter", - initialContents: newContentArray("content1-3", "sid1-3", "snap1-3", "sid1-3", validSecretClass, "", "", deletionPolicy, nil, nil, true), - expectedContents: nocontents, - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - expectedEvents: noevents, - errors: noerrors, - initialSecrets: []*v1.Secret{secret()}, - expectedDeleteCalls: []deleteCall{{"sid1-3", map[string]string{"foo": "bar"}, nil}}, - test: testSyncContent, + name: "1-3 - successful delete with snapshot class that has valid secret parameter", + initialContents: newContentArray("content1-3", "sid1-3", "snap1-3", "sid1-3", validSecretClass, "", "", deletionPolicy, nil, nil, true), + expectedContents: newContentArray("content1-3", "sid1-3", "snap1-3", "sid1-3", validSecretClass, "", "", deletionPolicy, nil, nil, true), + initialSnapshots: nosnapshots, + expectedSnapshots: nosnapshots, + expectedEvents: noevents, + errors: noerrors, + initialSecrets: []*v1.Secret{secret()}, + //expectedDeleteCalls: []deleteCall{{"sid1-3", map[string]string{"foo": "bar"}, nil}}, + test: testSyncContent, }, /*{ name: "1-4 - fail delete with snapshot class that has invalid secret parameter", @@ -179,7 +180,7 @@ func TestDeleteSync(t *testing.T) { errors: noerrors, test: testSyncContent, },*/ - { + /*{ name: "1-5 - csi driver delete snapshot returns error", initialContents: newContentArray("content1-5", "sid1-5", "snap1-5", "sid1-5", validSecretClass, "", "", deletionPolicy, nil, nil, true), expectedContents: newContentArray("content1-5", "sid1-5", "snap1-5", "sid1-5", validSecretClass, "", "", deletionPolicy, nil, nil, true), @@ -190,15 +191,15 @@ func TestDeleteSync(t *testing.T) { expectedEvents: []string{"Warning SnapshotDeleteError"}, errors: noerrors, test: testSyncContent, - }, - { + },*/ + /*{ name: "1-6 - api server delete content returns error", initialContents: newContentArray("content1-6", "sid1-6", "snap1-6", "sid1-6", validSecretClass, "", "", deletionPolicy, nil, nil, true), expectedContents: newContentArray("content1-6", "sid1-6", "snap1-6", "sid1-6", validSecretClass, "", "", deletionPolicy, nil, nil, true), initialSnapshots: nosnapshots, expectedSnapshots: nosnapshots, initialSecrets: []*v1.Secret{secret()}, - expectedDeleteCalls: []deleteCall{{"sid1-6", map[string]string{"foo": "bar"}, nil}}, + //expectedDeleteCalls: []deleteCall{{"sid1-6", map[string]string{"foo": "bar"}, nil}}, expectedEvents: []string{"Warning SnapshotContentObjectDeleteError"}, errors: []reactorError{ // Inject error to the first client.VolumesnapshotV1beta1().VolumeSnapshotContents().Delete call. @@ -206,56 +207,56 @@ func TestDeleteSync(t *testing.T) { {"delete", "volumesnapshotcontents", errors.New("mock delete error")}, }, test: testSyncContent, - }, + },*/ { // delete success - snapshot that the content was pointing to was deleted, and another // with the same name created. - name: "1-7 - prebound content is deleted while the snapshot exists", - initialContents: newContentArray("content1-7", "sid1-7", "snap1-7", "sid1-7", emptySecretClass, "", "", deletionPolicy, nil, nil, true), - expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap1-7", "snapuid1-7-x", "claim1-7", "", validSecretClass, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap1-7", "snapuid1-7-x", "claim1-7", "", validSecretClass, "", &False, nil, nil, nil), - initialSecrets: []*v1.Secret{secret()}, - expectedDeleteCalls: []deleteCall{{"sid1-7", map[string]string{"foo": "bar"}, nil}}, - expectedEvents: noevents, - errors: noerrors, - test: testSyncContent, + name: "1-7 - prebound content is deleted while the snapshot exists", + initialContents: newContentArray("content1-7", "sid1-7", "snap1-7", "sid1-7", emptySecretClass, "", "", deletionPolicy, nil, nil, true), + expectedContents: newContentArray("content1-7", "sid1-7", "snap1-7", "sid1-7", emptySecretClass, "", "", deletionPolicy, nil, nil, true), + initialSnapshots: newSnapshotArray("snap1-7", "snapuid1-7-x", "claim1-7", "", validSecretClass, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap1-7", "snapuid1-7-x", "claim1-7", "", validSecretClass, "", &False, nil, nil, nil, false), + initialSecrets: []*v1.Secret{secret()}, + //expectedDeleteCalls: []deleteCall{{"sid1-7", map[string]string{"foo": "bar"}, nil}}, + expectedEvents: noevents, + errors: noerrors, + test: testSyncContent, }, { // delete success(?) - content is deleted before doDelete() starts - name: "1-8 - content is deleted before deleting", - initialContents: newContentArray("content1-8", "sid1-8", "snap1-8", "sid1-8", validSecretClass, "", "", deletionPolicy, nil, nil, true), - expectedContents: nocontents, - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - initialSecrets: []*v1.Secret{secret()}, - expectedDeleteCalls: []deleteCall{{"sid1-8", map[string]string{"foo": "bar"}, nil}}, - expectedEvents: noevents, - errors: noerrors, - test: wrapTestWithInjectedOperation(testSyncContent, func(ctrl *csiSnapshotController, reactor *snapshotReactor) { + name: "1-8 - content is deleted before deleting", + initialContents: newContentArray("content1-8", "sid1-8", "snap1-8", "sid1-8", validSecretClass, "", "", deletionPolicy, nil, nil, true), + expectedContents: nocontents, + initialSnapshots: nosnapshots, + expectedSnapshots: nosnapshots, + initialSecrets: []*v1.Secret{secret()}, + //expectedDeleteCalls: []deleteCall{{"sid1-8", map[string]string{"foo": "bar"}, nil}}, + expectedEvents: noevents, + errors: noerrors, + test: wrapTestWithInjectedOperation(testSyncContent, func(ctrl *csiSnapshotCommonController, reactor *snapshotReactor) { // Delete the volume before delete operation starts reactor.lock.Lock() delete(reactor.contents, "content1-8") reactor.lock.Unlock() }), }, - { + /*{ name: "1-9 - content will not be deleted if it is bound to a snapshot correctly, snapshot uid is specified", initialContents: newContentArray("content1-9", "snapuid1-9", "snap1-9", "sid1-9", validSecretClass, "", "", deletionPolicy, nil, nil, true), expectedContents: newContentArray("content1-9", "snapuid1-9", "snap1-9", "sid1-9", validSecretClass, "", "", deletionPolicy, nil, nil, true), initialSnapshots: newSnapshotArray("snap1-9", "snapuid1-9", "claim1-9", "", validSecretClass, "content1-9", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap1-9", "snapuid1-9", "claim1-9", "", validSecretClass, "content1-9", &False, nil, nil, nil), + expectedSnapshots: newSnapshotArray("snap1-9", "snapuid1-9", "claim1-9", "", validSecretClass, "content1-9", &True, nil, nil, nil), expectedEvents: noevents, initialSecrets: []*v1.Secret{secret()}, errors: noerrors, test: testSyncContent, - }, + },*/ { name: "1-10 - will not delete content with retain policy set which is bound to a snapshot incorrectly", initialContents: newContentArray("content1-10", "snapuid1-10-x", "snap1-10", "sid1-10", validSecretClass, "", "", retainPolicy, nil, nil, true), expectedContents: newContentArray("content1-10", "snapuid1-10-x", "snap1-10", "sid1-10", validSecretClass, "", "", retainPolicy, nil, nil, true), - initialSnapshots: newSnapshotArray("snap1-10", "snapuid1-10", "claim1-10", "", validSecretClass, "content1-10", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap1-10", "snapuid1-10", "claim1-10", "", validSecretClass, "content1-10", &False, nil, nil, nil), + initialSnapshots: newSnapshotArray("snap1-10", "snapuid1-10", "claim1-10", "", validSecretClass, "content1-10", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap1-10", "snapuid1-10", "claim1-10", "", validSecretClass, "content1-10", &False, nil, nil, nil, false), expectedEvents: noevents, initialSecrets: []*v1.Secret{secret()}, errors: noerrors, @@ -265,8 +266,8 @@ func TestDeleteSync(t *testing.T) { name: "1-11 - content will not be deleted if it is bound to a snapshot correctly, snapsht uid is not specified", initialContents: newContentArray("content1-11", "", "snap1-11", "sid1-11", validSecretClass, "", "", deletePolicy, nil, nil, true), expectedContents: newContentArray("content1-11", "", "snap1-11", "sid1-11", validSecretClass, "", "", deletePolicy, nil, nil, true), - initialSnapshots: newSnapshotArray("snap1-11", "snapuid1-11", "claim1-11", "", validSecretClass, "content1-11", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap1-11", "snapuid1-11", "claim1-11", "", validSecretClass, "content1-11", &False, nil, nil, nil), + initialSnapshots: newSnapshotArray("snap1-11", "snapuid1-11", "claim1-11", "", validSecretClass, "content1-11", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap1-11", "snapuid1-11", "claim1-11", "", validSecretClass, "content1-11", &False, nil, nil, nil, false), expectedEvents: noevents, initialSecrets: []*v1.Secret{secret()}, errors: noerrors, @@ -292,28 +293,28 @@ func TestDeleteSync(t *testing.T) { errors: noerrors, test: testSyncContent, }, - { + /*{ name: "1-14 - content will not be deleted if it is bound to a snapshot correctly, snapshot uid is specified", initialContents: newContentArray("content1-14", "snapuid1-14", "snap1-14", "sid1-14", validSecretClass, "", "", retainPolicy, nil, nil, true), expectedContents: newContentArray("content1-14", "snapuid1-14", "snap1-14", "sid1-14", validSecretClass, "", "", retainPolicy, nil, nil, true), initialSnapshots: newSnapshotArray("snap1-14", "snapuid1-14", "claim1-14", "", validSecretClass, "content1-14", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap1-14", "snapuid1-14", "claim1-14", "", validSecretClass, "content1-14", &False, nil, nil, nil), + expectedSnapshots: newSnapshotArray("snap1-14", "snapuid1-14", "claim1-14", "", validSecretClass, "content1-14", &True, nil, nil, nil), expectedEvents: noevents, initialSecrets: []*v1.Secret{secret()}, errors: noerrors, test: testSyncContent, - }, + },*/ { - name: "1-16 - continue delete with snapshot class that has nonexistent secret", - initialContents: newContentArray("content1-16", "sid1-16", "snap1-16", "sid1-16", emptySecretClass, "", "", deletePolicy, nil, nil, true), - expectedContents: nocontents, - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - expectedEvents: noevents, - errors: noerrors, - initialSecrets: []*v1.Secret{}, // secret does not exist - expectedDeleteCalls: []deleteCall{{"sid1-16", nil, nil}}, - test: testSyncContent, + name: "1-16 - continue delete with snapshot class that has nonexistent secret", + initialContents: newContentArray("content1-16", "sid1-16", "snap1-16", "sid1-16", emptySecretClass, "", "", deletePolicy, nil, nil, true), + expectedContents: newContentArray("content1-16", "sid1-16", "snap1-16", "sid1-16", emptySecretClass, "", "", deletePolicy, nil, nil, true), + initialSnapshots: nosnapshots, + expectedSnapshots: nosnapshots, + expectedEvents: noevents, + errors: noerrors, + initialSecrets: []*v1.Secret{}, // secret does not exist + //expectedDeleteCalls: []deleteCall{{"sid1-16", nil, nil}}, + test: testSyncContent, }, } runSyncTests(t, tests, snapshotClasses) diff --git a/pkg/controller/snapshot_finalizer_test.go b/pkg/common-controller/snapshot_finalizer_test.go similarity index 85% rename from pkg/controller/snapshot_finalizer_test.go rename to pkg/common-controller/snapshot_finalizer_test.go index 296b7875..049262fd 100644 --- a/pkg/controller/snapshot_finalizer_test.go +++ b/pkg/common-controller/snapshot_finalizer_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package common_controller import ( "testing" @@ -22,42 +22,42 @@ import ( v1 "k8s.io/api/core/v1" ) -// Test single call to ensureSnapshotSourceFinalizer and checkandRemoveSnapshotSourceFinalizer, +// Test single call to ensurePVCFinalizer and checkandRemovePVCFinalizer, // expecting PVCFinalizer to be added or removed func TestPVCFinalizer(t *testing.T) { tests := []controllerTest{ { name: "1-1 - successful add PVC finalizer", - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false), initialClaims: newClaimArray("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), test: testAddPVCFinalizer, expectSuccess: true, }, { name: "1-2 - won't add PVC finalizer; already added", - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false), initialClaims: newClaimArrayFinalizer("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), test: testAddPVCFinalizer, expectSuccess: false, }, { name: "1-3 - successful remove PVC finalizer", - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false), initialClaims: newClaimArrayFinalizer("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), test: testRemovePVCFinalizer, expectSuccess: true, }, { name: "1-4 - won't remove PVC finalizer; already removed", - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false), initialClaims: newClaimArray("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), test: testRemovePVCFinalizer, expectSuccess: false, }, { name: "1-5 - won't remove PVC finalizer; PVC in-use", - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false), initialClaims: newClaimArray("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), test: testRemovePVCFinalizer, expectSuccess: false, diff --git a/pkg/controller/snapshot_ready_test.go b/pkg/common-controller/snapshot_update_test.go similarity index 68% rename from pkg/controller/snapshot_ready_test.go rename to pkg/common-controller/snapshot_update_test.go index e39b9f1f..f3a2faea 100644 --- a/pkg/controller/snapshot_ready_test.go +++ b/pkg/common-controller/snapshot_update_test.go @@ -14,10 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package common_controller import ( - "errors" + //"errors" "testing" "time" @@ -41,19 +41,20 @@ var volumeErr = &storagev1beta1.VolumeError{ // controllerTest.testCall *once*. // 3. Compare resulting contents and snapshots with expected contents and snapshots. func TestSync(t *testing.T) { + size := int64(1) tests := []controllerTest{ { // snapshot is bound to a non-existing content name: "2-1 - snapshot is bound to a non-existing content", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap2-1", "snapuid2-1", "claim2-1", "", validSecretClass, "content2-1", &True, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap2-1", "snapuid2-1", "claim2-1", "", validSecretClass, "content2-1", &False, nil, nil, newVolumeError("VolumeSnapshotContent is missing")), + initialSnapshots: newSnapshotArray("snap2-1", "snapuid2-1", "claim2-1", "", validSecretClass, "content2-1", &True, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap2-1", "snapuid2-1", "claim2-1", "", validSecretClass, "content2-1", &False, nil, nil, newVolumeError("VolumeSnapshotContent is missing"), false), expectedEvents: []string{"Warning SnapshotContentMissing"}, errors: noerrors, test: testSyncSnapshot, }, - { + /*{ name: "2-2 - snapshot points to a content but content does not point to snapshot(VolumeSnapshotRef does not match)", initialContents: newContentArray("content2-2", "snapuid2-2-x", "snap2-2", "sid2-2", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content2-2", "snapuid2-2-x", "snap2-2", "sid2-2", validSecretClass, "", "", deletionPolicy, nil, nil, false), @@ -62,17 +63,17 @@ func TestSync(t *testing.T) { expectedEvents: []string{"Warning InvalidSnapshotBinding"}, errors: noerrors, test: testSyncSnapshotError, - }, + },*/ { name: "2-3 - success bind snapshot and content but not ready, no status changed", initialContents: newContentArray("content2-3", "snapuid2-3", "snap2-3", "sid2-3", validSecretClass, "", "", deletionPolicy, nil, nil, false), - expectedContents: newContentArrayWithReadyToUse("content2-3", "snapuid2-3", "snap2-3", "sid2-3", validSecretClass, "", "", deletionPolicy, &timeNowStamp, &defaultSize, &False, false), - initialSnapshots: newSnapshotArray("snap2-3", "snapuid2-3", "claim2-3", "", validSecretClass, "content2-3", &False, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap2-3", "snapuid2-3", "claim2-3", "", validSecretClass, "content2-3", &False, metaTimeNow, getSize(defaultSize), nil), + expectedContents: newContentArrayWithReadyToUse("content2-3", "snapuid2-3", "snap2-3", "sid2-3", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &True, false), + initialSnapshots: newSnapshotArray("snap2-3", "snapuid2-3", "claim2-3", "", validSecretClass, "content2-3", &False, metaTimeNow, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap2-3", "snapuid2-3", "claim2-3", "", validSecretClass, "content2-3", &True, metaTimeNow, nil, nil, false), initialClaims: newClaimArray("claim2-3", "pvc-uid2-3", "1Gi", "volume2-3", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume2-3", "pv-uid2-3", "pv-handle2-3", "1Gi", "pvc-uid2-3", "claim2-3", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialSecrets: []*v1.Secret{secret()}, - expectedCreateCalls: []createCall{ + /*expectedCreateCalls: []createCall{ { snapshotName: "snapshot-snapuid2-3", volume: newVolume("volume2-3", "pv-uid2-3", "pv-handle2-3", "1Gi", "pvc-uid2-3", "claim2-3", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), @@ -85,7 +86,7 @@ func TestSync(t *testing.T) { creationTime: timeNow, readyToUse: False, }, - }, + },*/ errors: noerrors, test: testSyncSnapshot, }, @@ -94,21 +95,21 @@ func TestSync(t *testing.T) { name: "2-4 - noop", initialContents: newContentArray("content2-4", "snapuid2-4", "snap2-4", "sid2-4", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content2-4", "snapuid2-4", "snap2-4", "sid2-4", validSecretClass, "", "", deletionPolicy, nil, nil, false), - initialSnapshots: newSnapshotArray("snap2-4", "snapuid2-4", "claim2-4", "", validSecretClass, "content2-4", &True, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap2-4", "snapuid2-4", "claim2-4", "", validSecretClass, "content2-4", &True, metaTimeNow, nil, nil), + initialSnapshots: newSnapshotArray("snap2-4", "snapuid2-4", "claim2-4", "", validSecretClass, "content2-4", &True, metaTimeNow, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap2-4", "snapuid2-4", "claim2-4", "", validSecretClass, "content2-4", &True, metaTimeNow, nil, nil, false), errors: noerrors, test: testSyncSnapshot, }, { name: "2-5 - snapshot and content bound, status ready false -> true", initialContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), - expectedContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, &timeNowStamp, &defaultSize, &True, false), - initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &False, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &True, metaTimeNow, getSize(defaultSize), nil), + expectedContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), + initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &False, metaTimeNow, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &False, metaTimeNow, nil, nil, false), initialClaims: newClaimArray("claim2-5", "pvc-uid2-5", "1Gi", "volume2-5", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume2-5", "pv-uid2-5", "pv-handle2-5", "1Gi", "pvc-uid2-5", "claim2-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialSecrets: []*v1.Secret{secret()}, - expectedCreateCalls: []createCall{ + /*expectedCreateCalls: []createCall{ { snapshotName: "snapshot-snapuid2-5", volume: newVolume("volume2-5", "pv-uid2-5", "pv-handle2-5", "1Gi", "pvc-uid2-5", "claim2-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), @@ -121,16 +122,16 @@ func TestSync(t *testing.T) { creationTime: timeNow, readyToUse: True, }, - }, + },*/ errors: noerrors, test: testSyncSnapshot, }, { name: "2-6 - snapshot bound to prebound content correctly, status ready false -> true, ref.UID '' -> 'snapuid2-6'", initialContents: newContentArrayWithReadyToUse("content2-6", "snapuid2-6", "snap2-6", "sid2-6", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), - expectedContents: newContentArrayWithReadyToUse("content2-6", "snapuid2-6", "snap2-6", "sid2-6", validSecretClass, "", "", deletionPolicy, &timeNowStamp, &defaultSize, &True, false), - initialSnapshots: newSnapshotArray("snap2-6", "snapuid2-6", "", "content2-6", validSecretClass, "content2-6", &False, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap2-6", "snapuid2-6", "", "content2-6", validSecretClass, "content2-6", &True, metaTimeNow, getSize(defaultSize), nil), + expectedContents: newContentArrayWithReadyToUse("content2-6", "snapuid2-6", "snap2-6", "sid2-6", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), + initialSnapshots: newSnapshotArray("snap2-6", "snapuid2-6", "", "content2-6", validSecretClass, "content2-6", &False, metaTimeNow, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap2-6", "snapuid2-6", "", "content2-6", validSecretClass, "content2-6", &False, metaTimeNow, nil, nil, false), expectedListCalls: []listCall{ { size: defaultSize, @@ -141,7 +142,7 @@ func TestSync(t *testing.T) { errors: noerrors, test: testSyncSnapshot, }, - { + /*{ name: "2-7 - snapshot and content bound, csi driver get status error", initialContents: newContentArrayWithReadyToUse("content2-7", "snapuid2-7", "snap2-7", "sid2-7", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), expectedContents: newContentArrayWithReadyToUse("content2-7", "snapuid2-7", "snap2-7", "sid2-7", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), @@ -163,53 +164,54 @@ func TestSync(t *testing.T) { }, errors: noerrors, test: testSyncSnapshot, - }, - { - name: "2-8 - snapshot and content bound, apiserver update status error", - initialContents: newContentArrayWithReadyToUse("content2-8", "snapuid2-8", "snap2-8", "sid2-8", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), - expectedContents: newContentArrayWithReadyToUse("content2-8", "snapuid2-8", "snap2-8", "sid2-8", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), - initialSnapshots: newSnapshotArray("snap2-8", "snapuid2-8", "claim2-8", "", validSecretClass, "content2-8", &False, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap2-8", "snapuid2-8", "claim2-8", "", validSecretClass, "content2-8", &False, metaTimeNow, nil, newVolumeError("Failed to check and update snapshot: snapshot controller failed to update default/snap2-8 on API server: mock update error")), - expectedEvents: []string{"Warning SnapshotCheckandUpdateFailed"}, - initialClaims: newClaimArray("claim2-8", "pvc-uid2-8", "1Gi", "volume2-8", v1.ClaimBound, &classEmpty), - initialVolumes: newVolumeArray("volume2-8", "pv-uid2-8", "pv-handle2-8", "1Gi", "pvc-uid2-8", "claim2-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - initialSecrets: []*v1.Secret{secret()}, - expectedCreateCalls: []createCall{ - { - snapshotName: "snapshot-snapuid2-8", - volume: newVolume("volume2-8", "pv-uid2-8", "pv-handle2-8", "1Gi", "pvc-uid2-8", "claim2-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - parameters: class5Parameters, - secrets: map[string]string{"foo": "bar"}, - // information to return - driverName: mockDriverName, - size: defaultSize, - snapshotId: "sid2-8", - creationTime: timeNow, - readyToUse: true, - }, + },*/ + /*{ + name: "2-8 - snapshot and content bound, apiserver update status error", + initialContents: newContentArrayWithReadyToUse("content2-8", "snapuid2-8", "snap2-8", "sid2-8", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), + expectedContents: newContentArrayWithReadyToUse("content2-8", "snapuid2-8", "snap2-8", "sid2-8", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), + initialSnapshots: newSnapshotArray("snap2-8", "snapuid2-8", "claim2-8", "", validSecretClass, "content2-8", &False, metaTimeNow, nil, nil), + expectedSnapshots: newSnapshotArray("snap2-8", "snapuid2-8", "claim2-8", "", validSecretClass, "content2-8", &False, metaTimeNow, nil, newVolumeError("Failed to check and update snapshot: snapshot controller failed to update default/snap2-8 on API server: mock update error")), + expectedEvents: []string{"Warning SnapshotCheckandUpdateFailed"}, + initialClaims: newClaimArray("claim2-8", "pvc-uid2-8", "1Gi", "volume2-8", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume2-8", "pv-uid2-8", "pv-handle2-8", "1Gi", "pvc-uid2-8", "claim2-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + initialSecrets: []*v1.Secret{secret()}, + /*expectedCreateCalls: []createCall{ + { + snapshotName: "snapshot-snapuid2-8", + volume: newVolume("volume2-8", "pv-uid2-8", "pv-handle2-8", "1Gi", "pvc-uid2-8", "claim2-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + parameters: class5Parameters, + secrets: map[string]string{"foo": "bar"}, + // information to return + driverName: mockDriverName, + size: defaultSize, + snapshotId: "sid2-8", + creationTime: timeNow, + readyToUse: true, }, + },*/ /* errors: []reactorError{ // Inject error to the first client.VolumesnapshotV1beta1().VolumeSnapshots().Update call. // All other calls will succeed. {"update", "volumesnapshots", errors.New("mock update error")}, }, test: testSyncSnapshot, - }, + },*/ { name: "2-9 - fail on status update as there is not pvc provided", initialContents: newContentArray("content2-9", "snapuid2-9", "snap2-9", "sid2-9", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content2-9", "snapuid2-9", "snap2-9", "sid2-9", validSecretClass, "", "", deletionPolicy, nil, nil, false), - initialSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "", &False, nil, nil, newVolumeError("Failed to check and update snapshot: failed to get input parameters to create snapshot snap2-9: \"failed to retrieve PVC claim2-9 from the lister: \\\"persistentvolumeclaim \\\\\\\"claim2-9\\\\\\\" not found\\\"\"")), - errors: noerrors, - test: testSyncSnapshot, + initialSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "content2-9", &True, nil, nil, nil, false), + //expectedSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "content2-9", &False, nil, nil, newVolumeError("Failed to check and update snapshot: failed to get input parameters to create snapshot snap2-9: \"failed to retrieve PVC claim2-9 from the lister: \\\"persistentvolumeclaim \\\\\\\"claim2-9\\\\\\\" not found\\\"\"")), + errors: noerrors, + test: testSyncSnapshot, }, { name: "2-10 - do not bind when snapshot and content not match", initialContents: newContentArray("content2-10", "snapuid2-10-x", "snap2-10", "sid2-10", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content2-10", "snapuid2-10-x", "snap2-10", "sid2-10", validSecretClass, "", "", deletionPolicy, nil, nil, false), - initialSnapshots: newSnapshotArray("snap2-10", "snapuid2-10", "claim2-10", "", validSecretClass, "", &False, nil, nil, newVolumeError("mock driver error")), - expectedSnapshots: newSnapshotArray("snap2-10", "snapuid2-10", "claim2-10", "", validSecretClass, "", &False, nil, nil, newVolumeError("mock driver error")), + initialSnapshots: newSnapshotArray("snap2-10", "snapuid2-10", "claim2-10", "", validSecretClass, "", &False, nil, nil, newVolumeError("mock driver error"), false), + expectedSnapshots: newSnapshotArray("snap2-10", "snapuid2-10", "claim2-10", "", validSecretClass, "", &False, nil, nil, newVolumeError("mock driver error"), false), errors: noerrors, test: testSyncSnapshot, }, @@ -217,8 +219,8 @@ func TestSync(t *testing.T) { name: "3-1 - ready snapshot lost reference to VolumeSnapshotContent", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &True, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is missing")), + initialSnapshots: newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &True, metaTimeNow, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is missing"), false), errors: noerrors, expectedEvents: []string{"Warning SnapshotContentMissing"}, test: testSyncSnapshot, @@ -227,8 +229,8 @@ func TestSync(t *testing.T) { name: "3-2 - ready snapshot bound to none-exist content", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &True, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is missing")), + initialSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &True, metaTimeNow, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is missing"), false), errors: noerrors, expectedEvents: []string{"Warning SnapshotContentMissing"}, test: testSyncSnapshot, @@ -237,8 +239,8 @@ func TestSync(t *testing.T) { name: "3-3 - ready snapshot(everything is well, do nothing)", initialContents: newContentArray("content3-3", "snapuid3-3", "snap3-3", "sid3-3", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content3-3", "snapuid3-3", "snap3-3", "sid3-3", validSecretClass, "", "", deletionPolicy, nil, nil, false), - initialSnapshots: newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &True, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &True, metaTimeNow, nil, nil), + initialSnapshots: newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &True, metaTimeNow, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &True, metaTimeNow, nil, nil, false), errors: noerrors, test: testSyncSnapshot, }, @@ -246,12 +248,12 @@ func TestSync(t *testing.T) { name: "3-4 - ready snapshot misbound to VolumeSnapshotContent", initialContents: newContentArray("content3-4", "snapuid3-4-x", "snap3-4", "sid3-4", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content3-4", "snapuid3-4-x", "snap3-4", "sid3-4", validSecretClass, "", "", deletionPolicy, nil, nil, false), - initialSnapshots: newSnapshotArray("snap3-4", "snapuid3-4", "claim3-4", "", validSecretClass, "content3-4", &True, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap3-4", "snapuid3-4", "claim3-4", "", validSecretClass, "content3-4", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is not bound to the VolumeSnapshot correctly")), + initialSnapshots: newSnapshotArray("snap3-4", "snapuid3-4", "claim3-4", "", validSecretClass, "content3-4", &True, metaTimeNow, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap3-4", "snapuid3-4", "claim3-4", "", validSecretClass, "content3-4", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is not bound to the VolumeSnapshot correctly"), false), errors: noerrors, test: testSyncSnapshot, }, - { + /*{ name: "3-5 - snapshot bound to content in which the driver does not match", initialContents: newContentWithUnmatchDriverArray("content3-5", "snapuid3-5", "snap3-5", "sid3-5", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: nocontents, @@ -260,6 +262,42 @@ func TestSync(t *testing.T) { expectedEvents: []string{"Warning SnapshotContentMissing"}, errors: noerrors, test: testSyncSnapshotError, + },*/ + { + name: "4-1 - content bound to snapshot, snapshot status missing and rebuilt", + initialContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, &size, &True, false), + expectedContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, &size, &True, false), + initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "", &False, nil, nil, nil, true), + expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &True, nil, getSize(1), nil, false), + initialClaims: newClaimArray("claim2-5", "pvc-uid2-5", "1Gi", "volume2-5", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume2-5", "pv-uid2-5", "pv-handle2-5", "1Gi", "pvc-uid2-5", "claim2-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + initialSecrets: []*v1.Secret{secret()}, + errors: noerrors, + test: testSyncSnapshot, + }, + { + name: "4-2 - snapshot and content bound, ReadyToUse in snapshot status missing and rebuilt", + initialContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, nil, &True, false), + expectedContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, nil, &True, false), + initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &True, nil, nil, nil, false), + initialClaims: newClaimArray("claim2-5", "pvc-uid2-5", "1Gi", "volume2-5", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume2-5", "pv-uid2-5", "pv-handle2-5", "1Gi", "pvc-uid2-5", "claim2-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + initialSecrets: []*v1.Secret{secret()}, + errors: noerrors, + test: testSyncSnapshot, + }, + { + name: "4-3 - content bound to snapshot, fields in snapshot status missing and rebuilt", + initialContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, &size, &True, false), + expectedContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, &size, &True, false), + initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &True, nil, getSize(1), nil, false), + initialClaims: newClaimArray("claim2-5", "pvc-uid2-5", "1Gi", "volume2-5", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume2-5", "pv-uid2-5", "pv-handle2-5", "1Gi", "pvc-uid2-5", "claim2-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + initialSecrets: []*v1.Secret{secret()}, + errors: noerrors, + test: testSyncSnapshot, }, } diff --git a/pkg/controller/snapshot_controller.go b/pkg/controller/snapshot_controller.go deleted file mode 100644 index 88f6ed20..00000000 --- a/pkg/controller/snapshot_controller.go +++ /dev/null @@ -1,1226 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "fmt" - "strings" - "time" - - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - v1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/kubernetes/scheme" - ref "k8s.io/client-go/tools/reference" - "k8s.io/klog" - "k8s.io/kubernetes/pkg/util/goroutinemap" - "k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff" - "k8s.io/kubernetes/pkg/util/slice" -) - -// ================================================================== -// PLEASE DO NOT ATTEMPT TO SIMPLIFY THIS CODE. -// KEEP THE SPACE SHUTTLE FLYING. -// ================================================================== - -// Design: -// -// The fundamental key to this design is the bi-directional "pointer" between -// VolumeSnapshots and VolumeSnapshotContents, which is represented here -// as snapshot.Spec.Source.VolumeSnapshotContentName and content.Spec.VolumeSnapshotRef. -// The bi-directionality is complicated to manage in a transactionless system, but -// without it we can't ensure sane behavior in the face of different forms of -// trouble. For example, a rogue HA controller instance could end up racing -// and making multiple bindings that are indistinguishable, resulting in -// potential data loss. -// -// This controller is designed to work in active-passive high availability -// mode. It *could* work also in active-active HA mode, all the object -// transitions are designed to cope with this, however performance could be -// lower as these two active controllers will step on each other toes -// frequently. -// -// This controller supports both dynamic snapshot creation and pre-bound snapshot. -// In pre-bound mode, objects are created with pre-defined pointers: a VolumeSnapshot -// points to a specific VolumeSnapshotContent and the VolumeSnapshotContent also -// points back for this VolumeSnapshot. -// -// The dynamic snapshot creation is multi-step process: first controller triggers -// snapshot creation though csi volume plugin which should return a snapshot after -// it is created successfully (however, the snapshot might not be ready to use yet if -// there is an uploading phase). The creationTimestamp will be updated according to -// VolumeSnapshot, and then a VolumeSnapshotContent object is created to represent -// this snapshot. After that, the controller will keep checking the snapshot status -// though csi snapshot calls. When the snapshot is ready to use, the controller set -// the status "Bound" to true to indicate the snapshot is bound and ready to use. -// If the createtion failed for any reason, the Error status is set accordingly. -// In alpha version, the controller not retry to create the snapshot after it failed. -// In the future version, a retry policy will be added. - -const pvcKind = "PersistentVolumeClaim" -const apiGroup = "" -const snapshotKind = "VolumeSnapshot" -const snapshotAPIGroup = crdv1.GroupName - -const controllerUpdateFailMsg = "snapshot controller failed to update" - -const IsDefaultSnapshotClassAnnotation = "snapshot.storage.kubernetes.io/is-default-class" - -// syncContent deals with one key off the queue. It returns false when it's time to quit. -func (ctrl *csiSnapshotController) syncContent(content *crdv1.VolumeSnapshotContent) error { - klog.V(5).Infof("synchronizing VolumeSnapshotContent[%s]", content.Name) - - if isContentDeletionCandidate(content) { - // Volume snapshot content should be deleted. Check if it's used - // and remove finalizer if it's not. - // Check if snapshot content is still bound to a snapshot. - isUsed := ctrl.isSnapshotContentBeingUsed(content) - if !isUsed { - klog.V(5).Infof("syncContent: Remove Finalizer for VolumeSnapshotContent[%s]", content.Name) - return ctrl.removeContentFinalizer(content) - } - } - - if needToAddContentFinalizer(content) { - // Content is not being deleted -> it should have the finalizer. - klog.V(5).Infof("syncContent: Add Finalizer for VolumeSnapshotContent[%s]", content.Name) - return ctrl.addContentFinalizer(content) - } - - // VolumeSnapshotContent is not bound to any VolumeSnapshot, in this case we just return err - if content.Spec.VolumeSnapshotRef.Name == "" { - // content is not bound - klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: VolumeSnapshotContent is not bound to any VolumeSnapshot", content.Name) - ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotContentNotBound", "VolumeSnapshotContent is not bound to any VolumeSnapshot") - return fmt.Errorf("volumeSnapshotContent %s is not bound to any VolumeSnapshot", content.Name) - } - klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: content is bound to snapshot %s", content.Name, snapshotRefKey(content.Spec.VolumeSnapshotRef)) - // The VolumeSnapshotContent is reserved for a VolumeSnapshot; - // that VolumeSnapshot has not yet been bound to this VolumeSnapshotContent; the VolumeSnapshot sync will handle it. - if content.Spec.VolumeSnapshotRef.UID == "" { - klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: VolumeSnapshotContent is pre-bound to VolumeSnapshot %s", content.Name, snapshotRefKey(content.Spec.VolumeSnapshotRef)) - return nil - } - // Get the VolumeSnapshot by _name_ - var snapshot *crdv1.VolumeSnapshot - snapshotName := snapshotRefKey(content.Spec.VolumeSnapshotRef) - obj, found, err := ctrl.snapshotStore.GetByKey(snapshotName) - if err != nil { - return err - } - if !found { - klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: snapshot %s not found", content.Name, snapshotRefKey(content.Spec.VolumeSnapshotRef)) - // Fall through with snapshot = nil - } else { - var ok bool - snapshot, ok = obj.(*crdv1.VolumeSnapshot) - if !ok { - return fmt.Errorf("cannot convert object from snapshot cache to snapshot %q!?: %#v", content.Name, obj) - } - klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: snapshot %s found", content.Name, snapshotRefKey(content.Spec.VolumeSnapshotRef)) - } - if snapshot != nil && snapshot.UID != content.Spec.VolumeSnapshotRef.UID { - // The snapshot that the content was pointing to was deleted, and another - // with the same name created. - klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: content %s has different UID, the old one must have been deleted", content.Name, snapshotRefKey(content.Spec.VolumeSnapshotRef)) - // Treat the content as bound to a missing snapshot. - snapshot = nil - } - if snapshot == nil { - switch content.Spec.DeletionPolicy { - case crdv1.VolumeSnapshotContentRetain: - klog.V(4).Infof("VolumeSnapshotContent[%s]: policy is Retain, nothing to do", content.Name) - - case crdv1.VolumeSnapshotContentDelete: - klog.V(4).Infof("VolumeSnapshotContent[%s]: policy is Delete", content.Name) - ctrl.deleteSnapshotContent(content) - default: - // Unknown VolumeSnapshotDeletionolicy - ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotUnknownDeletionPolicy", "Volume Snapshot Content has unrecognized deletion policy") - } - return nil - } - return nil -} - -// syncSnapshot is the main controller method to decide what to do with a snapshot. -// It's invoked by appropriate cache.Controller callbacks when a snapshot is -// created, updated or periodically synced. We do not differentiate between -// these events. -// For easier readability, it is split into syncUnreadySnapshot and syncReadySnapshot -func (ctrl *csiSnapshotController) syncSnapshot(snapshot *crdv1.VolumeSnapshot) error { - klog.V(5).Infof("synchonizing VolumeSnapshot[%s]: %s", snapshotKey(snapshot), getSnapshotStatusForLogging(snapshot)) - - if isSnapshotDeletionCandidate(snapshot) { - // Volume snapshot should be deleted. Check if it's used - // and remove finalizer if it's not. - // Check if a volume is being created from snapshot. - isUsed := ctrl.isVolumeBeingCreatedFromSnapshot(snapshot) - if !isUsed { - klog.V(5).Infof("syncSnapshot: Remove Finalizer for VolumeSnapshot[%s]", snapshotKey(snapshot)) - return ctrl.removeSnapshotFinalizer(snapshot) - } - } - - if needToAddSnapshotFinalizer(snapshot) { - // Snapshot is not being deleted -> it should have the finalizer. - klog.V(5).Infof("syncSnapshot: Add Finalizer for VolumeSnapshot[%s]", snapshotKey(snapshot)) - return ctrl.addSnapshotFinalizer(snapshot) - } - - klog.V(5).Infof("syncSnapshot[%s]: check if we should remove finalizer on snapshot source and remove it if we can", snapshotKey(snapshot)) - // Check if we should remove finalizer on snapshot source and remove it if we can. - errFinalizer := ctrl.checkandRemoveSnapshotSourceFinalizer(snapshot) - if errFinalizer != nil { - klog.Errorf("error check and remove snapshot source finalizer for snapshot [%s]: %v", snapshot.Name, errFinalizer) - // Log an event and keep the original error from syncUnready/ReadySnapshot - ctrl.eventRecorder.Event(snapshot, v1.EventTypeWarning, "ErrorSnapshotSourceFinalizer", "Error check and remove PVC Finalizer for VolumeSnapshot") - } - - if !isSnapshotReadyToUse(snapshot) { - return ctrl.syncUnreadySnapshot(snapshot) - } - return ctrl.syncReadySnapshot(snapshot) -} - -// syncReadySnapshot checks the snapshot which has been bound to snapshot content successfully before. -// If there is any problem with the binding (e.g., snapshot points to a non-exist snapshot content), update the snapshot status and emit event. -func (ctrl *csiSnapshotController) syncReadySnapshot(snapshot *crdv1.VolumeSnapshot) error { - if snapshot.Status.BoundVolumeSnapshotContentName == nil { - if err := ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotLost", "Bound snapshot has lost reference to VolumeSnapshotContent"); err != nil { - return err - } - return nil - } - obj, found, err := ctrl.contentStore.GetByKey(*snapshot.Status.BoundVolumeSnapshotContentName) - if err != nil { - return err - } - if !found { - if err = ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotContentMissing", "VolumeSnapshotContent is missing"); err != nil { - return err - } - return nil - } else { - content, ok := obj.(*crdv1.VolumeSnapshotContent) - if !ok { - return fmt.Errorf("Cannot convert object from snapshot content store to VolumeSnapshotContent %q!?: %#v", *snapshot.Status.BoundVolumeSnapshotContentName, obj) - } - - klog.V(5).Infof("syncReadySnapshot[%s]: VolumeSnapshotContent %q found", snapshotKey(snapshot), content.Name) - if !IsSnapshotBound(snapshot, content) { - // snapshot is bound but content is not bound to snapshot correctly - if err = ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotMisbound", "VolumeSnapshotContent is not bound to the VolumeSnapshot correctly"); err != nil { - return err - } - return nil - } - // Snapshot is correctly bound. - return nil - } -} - -// syncUnreadySnapshot is the main controller method to decide what to do with a snapshot which is not set to ready. -func (ctrl *csiSnapshotController) syncUnreadySnapshot(snapshot *crdv1.VolumeSnapshot) error { - uniqueSnapshotName := snapshotKey(snapshot) - klog.V(5).Infof("syncUnreadySnapshot %s", uniqueSnapshotName) - - if snapshot.Spec.Source.VolumeSnapshotContentName != nil { - contentObj, found, err := ctrl.contentStore.GetByKey(*snapshot.Spec.Source.VolumeSnapshotContentName) - if err != nil { - return err - } - if !found { - // snapshot is bound to a non-existing content. - ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotContentMissing", "VolumeSnapshotContent is missing") - klog.V(4).Infof("synchronizing unready snapshot[%s]: snapshotcontent %q requested and not found, will try again next time", uniqueSnapshotName, *snapshot.Spec.Source.VolumeSnapshotContentName) - return fmt.Errorf("snapshot %s is bound to a non-existing content %s", uniqueSnapshotName, *snapshot.Spec.Source.VolumeSnapshotContentName) - } - content, ok := contentObj.(*crdv1.VolumeSnapshotContent) - if !ok { - return fmt.Errorf("expected volume snapshot content, got %+v", contentObj) - } - contentBound, err := ctrl.checkandBindSnapshotContent(snapshot, content) - if err != nil { - // snapshot is bound but content is not bound to snapshot correctly - ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotBindFailed", fmt.Sprintf("Snapshot failed to bind VolumeSnapshotContent, %v", err)) - return fmt.Errorf("snapshot %s is bound, but VolumeSnapshotContent %s is not bound to the VolumeSnapshot correctly, %v", uniqueSnapshotName, content.Name, err) - } - // snapshot is already bound correctly, check the status and update if it is ready. - klog.V(5).Infof("Check and update snapshot %s status", uniqueSnapshotName) - if err = ctrl.checkandUpdateBoundSnapshotStatus(snapshot, contentBound); err != nil { - return err - } - return nil - } else { // snapshot.Source.Spec.VolumeSnapshotContentName == nil - // find a matching volume snapshot content - if contentObj := ctrl.getMatchSnapshotContent(snapshot); contentObj != nil { - klog.V(5).Infof("Find VolumeSnapshotContent object %s for snapshot %s", contentObj.Name, uniqueSnapshotName) - if err := ctrl.checkandUpdateBoundSnapshotStatus(snapshot, contentObj); err != nil { - return err - } - klog.V(5).Infof("checkandUpdateBoundSnapshotStatus %v", snapshot) - } else if snapshot.Status != nil && snapshot.Status.BoundVolumeSnapshotContentName != nil { - contentObj, found, err := ctrl.contentStore.GetByKey(*snapshot.Status.BoundVolumeSnapshotContentName) - if err != nil { - return err - } - if !found { - ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotContentMissing", "VolumeSnapshotContent is missing") - return fmt.Errorf("snapshot %s is bound to a non-existing content %s", uniqueSnapshotName, *snapshot.Status.BoundVolumeSnapshotContentName) - } else { - content, _ := contentObj.(*crdv1.VolumeSnapshotContent) - ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "InvalidSnapshotBinding", fmt.Sprintf("Snapshot is bound to a VolumeSnapshotContent which is bound to other Snapshot")) - return fmt.Errorf("snapshot %s is bound, but VolumeSnapshotContent %s is not bound to the VolumeSnapshot correctly", uniqueSnapshotName, content.Name) - } - } else if snapshot.Status == nil || snapshot.Status.Error == nil || isControllerUpdateFailError(snapshot.Status.Error) { - if err := ctrl.createSnapshot(snapshot); err != nil { - ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotCreationFailed", fmt.Sprintf("Failed to create snapshot with error %v", err)) - return err - } - } - return nil - } -} - -// getMatchSnapshotContent looks up VolumeSnapshotContent for a VolumeSnapshot named snapshotName -func (ctrl *csiSnapshotController) getMatchSnapshotContent(snapshot *crdv1.VolumeSnapshot) *crdv1.VolumeSnapshotContent { - var snapshotContentObj *crdv1.VolumeSnapshotContent - var found bool - - objs := ctrl.contentStore.List() - for _, obj := range objs { - content := obj.(*crdv1.VolumeSnapshotContent) - if content.Spec.VolumeSnapshotRef.Name == snapshot.Name && - content.Spec.VolumeSnapshotRef.Namespace == snapshot.Namespace && - content.Spec.VolumeSnapshotRef.UID == snapshot.UID { - found = true - snapshotContentObj = content - break - } - } - - if !found { - klog.V(4).Infof("No VolumeSnapshotContent for VolumeSnapshot %s found", snapshotKey(snapshot)) - return nil - } - - return snapshotContentObj -} - -// deleteSnapshotContent starts delete action. -func (ctrl *csiSnapshotController) deleteSnapshotContent(content *crdv1.VolumeSnapshotContent) { - operationName := fmt.Sprintf("delete-%s[%s]", content.Name, string(content.UID)) - klog.V(5).Infof("Snapshotter is about to delete volume snapshot content and the operation named %s", operationName) - ctrl.scheduleOperation(operationName, func() error { - return ctrl.deleteSnapshotContentOperation(content) - }) -} - -// scheduleOperation starts given asynchronous operation on given volume. It -// makes sure the operation is already not running. -func (ctrl *csiSnapshotController) scheduleOperation(operationName string, operation func() error) { - klog.V(5).Infof("scheduleOperation[%s]", operationName) - - err := ctrl.runningOperations.Run(operationName, operation) - if err != nil { - switch { - case goroutinemap.IsAlreadyExists(err): - klog.V(4).Infof("operation %q is already running, skipping", operationName) - case exponentialbackoff.IsExponentialBackoff(err): - klog.V(4).Infof("operation %q postponed due to exponential backoff", operationName) - default: - klog.Errorf("error scheduling operation %q: %v", operationName, err) - } - } -} - -func (ctrl *csiSnapshotController) storeSnapshotUpdate(snapshot interface{}) (bool, error) { - return storeObjectUpdate(ctrl.snapshotStore, snapshot, "snapshot") -} - -func (ctrl *csiSnapshotController) storeContentUpdate(content interface{}) (bool, error) { - return storeObjectUpdate(ctrl.contentStore, content, "content") -} - -// createSnapshot starts new asynchronous operation to create snapshot -func (ctrl *csiSnapshotController) createSnapshot(snapshot *crdv1.VolumeSnapshot) error { - klog.V(5).Infof("createSnapshot[%s]: started", snapshotKey(snapshot)) - opName := fmt.Sprintf("create-%s[%s]", snapshotKey(snapshot), string(snapshot.UID)) - ctrl.scheduleOperation(opName, func() error { - snapshotObj, err := ctrl.createSnapshotOperation(snapshot) - if err != nil { - ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotCreationFailed", fmt.Sprintf("Failed to create snapshot: %v", err)) - klog.Errorf("createSnapshot [%s]: error occurred in createSnapshotOperation: %v", opName, err) - return err - } - _, updateErr := ctrl.storeSnapshotUpdate(snapshotObj) - if updateErr != nil { - // We will get an "snapshot update" event soon, this is not a big error - klog.V(4).Infof("createSnapshot [%s]: cannot update internal cache: %v", snapshotKey(snapshotObj), updateErr) - } - return nil - }) - return nil -} - -func (ctrl *csiSnapshotController) checkandUpdateBoundSnapshotStatus(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) error { - klog.V(5).Infof("checkandUpdateSnapshotStatus[%s] started", snapshotKey(snapshot)) - opName := fmt.Sprintf("check-%s[%s]", snapshotKey(snapshot), string(snapshot.UID)) - ctrl.scheduleOperation(opName, func() error { - snapshotObj, err := ctrl.checkandUpdateBoundSnapshotStatusOperation(snapshot, content) - if err != nil { - ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotCheckandUpdateFailed", fmt.Sprintf("Failed to check and update snapshot: %v", err)) - klog.Errorf("checkandUpdateSnapshotStatus [%s]: error occured %v", snapshotKey(snapshot), err) - return err - } - _, updateErr := ctrl.storeSnapshotUpdate(snapshotObj) - if updateErr != nil { - // We will get an "snapshot update" event soon, this is not a big error - klog.V(4).Infof("checkandUpdateSnapshotStatus [%s]: cannot update internal cache: %v", snapshotKey(snapshotObj), updateErr) - } - - return nil - }) - return nil -} - -// updateSnapshotStatusWithEvent saves new snapshot.Status to API server and emits -// given event on the snapshot. It saves the status and emits the event only when -// the status has actually changed from the version saved in API server. -// Parameters: -// snapshot - snapshot to update -// eventtype, reason, message - event to send, see EventRecorder.Event() -func (ctrl *csiSnapshotController) updateSnapshotErrorStatusWithEvent(snapshot *crdv1.VolumeSnapshot, eventtype, reason, message string) error { - klog.V(5).Infof("updateSnapshotStatusWithEvent[%s]", snapshotKey(snapshot)) - - if snapshot.Status != nil && snapshot.Status.Error != nil && snapshot.Status.Error.Message != nil && *snapshot.Status.Error.Message == message { - klog.V(4).Infof("updateSnapshotStatusWithEvent[%s]: the same error %v is already set", snapshot.Name, snapshot.Status.Error) - return nil - } - snapshotClone := snapshot.DeepCopy() - statusError := &crdv1.VolumeSnapshotError{ - Time: &metav1.Time{ - Time: time.Now(), - }, - Message: &message, - } - if snapshotClone.Status == nil { - snapshotClone.Status = &crdv1.VolumeSnapshotStatus{} - } - snapshotClone.Status.Error = statusError - ready := false - snapshotClone.Status.ReadyToUse = &ready - newSnapshot, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).UpdateStatus(snapshotClone) - - if err != nil { - klog.V(4).Infof("updating VolumeSnapshot[%s] error status failed %v", snapshotKey(snapshot), err) - return err - } - - _, err = ctrl.storeSnapshotUpdate(newSnapshot) - if err != nil { - klog.V(4).Infof("updating VolumeSnapshot[%s] error status: cannot update internal cache %v", snapshotKey(snapshot), err) - return err - } - // Emit the event only when the status change happens - ctrl.eventRecorder.Event(newSnapshot, eventtype, reason, message) - - return nil -} - -// Stateless functions -func getSnapshotStatusForLogging(snapshot *crdv1.VolumeSnapshot) string { - snapshotContentName := "" - readyToUse := isSnapshotReadyToUse(snapshot) - if snapshot.Status != nil && snapshot.Status.BoundVolumeSnapshotContentName != nil { - snapshotContentName = *snapshot.Status.BoundVolumeSnapshotContentName - } - return fmt.Sprintf("bound to: %q, Completed: %v", snapshotContentName, readyToUse) -} - -// IsSnapshotBound returns true/false if snapshot is bound -func IsSnapshotBound(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) bool { - if content.Spec.VolumeSnapshotRef.Name == snapshot.Name && - content.Spec.VolumeSnapshotRef.UID == snapshot.UID { - return true - } - return false -} - -// isSnapshotConentBeingUsed checks if snapshot content is bound to snapshot. -func (ctrl *csiSnapshotController) isSnapshotContentBeingUsed(content *crdv1.VolumeSnapshotContent) bool { - snapshotObj, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(content.Spec.VolumeSnapshotRef.Namespace).Get(content.Spec.VolumeSnapshotRef.Name, metav1.GetOptions{}) - if err != nil { - klog.Infof("isSnapshotContentBeingUsed: Cannot get snapshot %s from api server: [%v]. VolumeSnapshot object may be deleted already.", content.Spec.VolumeSnapshotRef.Name, err) - return false - } - - // Check if the snapshot content is bound to the snapshot - if IsSnapshotBound(snapshotObj, content) && snapshotObj.Status != nil && *snapshotObj.Status.BoundVolumeSnapshotContentName == content.Name { - klog.Infof("isSnapshotContentBeingUsed: VolumeSnapshot %s is bound to volumeSnapshotContent [%s]", snapshotObj.Name, content.Name) - return true - } - - klog.V(5).Infof("isSnapshotContentBeingUsed: Snapshot content %s is not being used", content.Name) - return false -} - -// isVolumeBeingCreatedFromSnapshot checks if an volume is being created from the snapshot. -func (ctrl *csiSnapshotController) isVolumeBeingCreatedFromSnapshot(snapshot *crdv1.VolumeSnapshot) bool { - pvcList, err := ctrl.pvcLister.PersistentVolumeClaims(snapshot.Namespace).List(labels.Everything()) - if err != nil { - klog.Errorf("Failed to retrieve PVCs from the lister to check if volume snapshot %s is being used by a volume: %q", snapshotKey(snapshot), err) - return false - } - for _, pvc := range pvcList { - if pvc.Spec.DataSource != nil && len(pvc.Spec.DataSource.Name) > 0 && pvc.Spec.DataSource.Name == snapshot.Name { - if pvc.Spec.DataSource.Kind == snapshotKind && *(pvc.Spec.DataSource.APIGroup) == snapshotAPIGroup { - if pvc.Status.Phase == v1.ClaimPending { - // A volume is being created from the snapshot - klog.Infof("isVolumeBeingCreatedFromSnapshot: volume %s is being created from snapshot %s", pvc.Name, pvc.Spec.DataSource.Name) - return true - } - } - } - } - klog.V(5).Infof("isVolumeBeingCreatedFromSnapshot: no volume is being created from snapshot %s", snapshotKey(snapshot)) - return false -} - -// The function checks whether the volumeSnapshotRef in snapshot content matches the given snapshot. If match, it binds the content with the snapshot -func (ctrl *csiSnapshotController) checkandBindSnapshotContent(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshotContent, error) { - if content.Spec.VolumeSnapshotRef.Name != snapshot.Name { - return nil, fmt.Errorf("Could not bind snapshot %s and content %s, the VolumeSnapshotRef does not match", snapshot.Name, content.Name) - } else if content.Spec.VolumeSnapshotRef.UID != "" && content.Spec.VolumeSnapshotRef.UID != snapshot.UID { - return nil, fmt.Errorf("Could not bind snapshot %s and content %s, the VolumeSnapshotRef does not match", snapshot.Name, content.Name) - } else if content.Spec.VolumeSnapshotRef.UID != "" { - return content, nil - } - contentClone := content.DeepCopy() - contentClone.Spec.VolumeSnapshotRef.UID = snapshot.UID - newContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) - if err != nil { - klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status failed %v", newContent.Name, err) - return nil, err - } - _, err = ctrl.storeContentUpdate(newContent) - if err != nil { - klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status: cannot update internal cache %v", newContent.Name, err) - return nil, err - } - return newContent, nil -} - -func (ctrl *csiSnapshotController) getCreateSnapshotInput(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshotClass, *v1.PersistentVolume, string, *v1.SecretReference, error) { - className := snapshot.Spec.VolumeSnapshotClassName - var class *crdv1.VolumeSnapshotClass - var err error - if className != nil { - class, err = ctrl.GetSnapshotClass(*className) - if err != nil { - klog.Errorf("getCreateSnapshotInput failed to getClassFromVolumeSnapshot %s", err) - return nil, nil, "", nil, err - } - } else { - klog.Errorf("failed to getCreateSnapshotInput %s without a snapshot class", snapshot.Name) - return nil, nil, "", nil, fmt.Errorf("failed to take snapshot %s without a snapshot class", snapshot.Name) - } - - volume, err := ctrl.getVolumeFromVolumeSnapshot(snapshot) - if err != nil { - klog.Errorf("getCreateSnapshotInput failed to get PersistentVolume object [%s]: Error: [%#v]", snapshot.Name, err) - return nil, nil, "", nil, err - } - - // Create VolumeSnapshotContent name - contentName := GetSnapshotContentNameForSnapshot(snapshot) - - // Resolve snapshotting secret credentials. - snapshotterSecretRef, err := getSecretReference(class.Parameters, contentName, snapshot) - if err != nil { - return nil, nil, "", nil, err - } - - return class, volume, contentName, snapshotterSecretRef, nil -} - -func (ctrl *csiSnapshotController) checkandUpdateBoundSnapshotStatusOperation(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshot, error) { - var err error - var creationTime time.Time - var size int64 - var readyToUse = false - var driverName string - var snapshotID string - - if snapshot.Spec.Source.VolumeSnapshotContentName != nil { - klog.V(5).Infof("checkandUpdateBoundSnapshotStatusOperation: checking whether snapshot [%s] is pre-bound to content [%s]", snapshot.Name, content.Name) - readyToUse, creationTime, size, err = ctrl.handler.GetSnapshotStatus(content) - if err != nil { - klog.Errorf("checkandUpdateBoundSnapshotStatusOperation: failed to call get snapshot status to check whether snapshot is ready to use %q", err) - return nil, err - } - driverName, snapshotID = content.Spec.Driver, *content.Status.SnapshotHandle - } else { - class, volume, _, snapshotterSecretRef, err := ctrl.getCreateSnapshotInput(snapshot) - if err != nil { - return nil, fmt.Errorf("failed to get input parameters to create snapshot %s: %q", snapshot.Name, err) - } - snapshotterCredentials, err := getCredentials(ctrl.client, snapshotterSecretRef) - if err != nil { - return nil, err - } - driverName, snapshotID, creationTime, size, readyToUse, err = ctrl.handler.CreateSnapshot(snapshot, volume, class.Parameters, snapshotterCredentials) - if err != nil { - klog.Errorf("checkandUpdateBoundSnapshotStatusOperation: failed to call create snapshot to check whether the snapshot is ready to use %q", err) - return nil, err - } - } - klog.V(5).Infof("checkandUpdateBoundSnapshotStatusOperation: driver %s, snapshotId %s, creationTime %v, size %d, readyToUse %t", driverName, snapshotID, creationTime, size, readyToUse) - - if creationTime.IsZero() { - creationTime = time.Now() - } - newSnapshot, err := ctrl.updateSnapshotStatus(snapshot, content.Name, readyToUse, creationTime, size) - if err != nil { - return nil, err - } - // get the latest version of snapshot content before update status - latestContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Get(content.Name, metav1.GetOptions{}) - err = ctrl.updateSnapshotContentStatus(latestContent, snapshotID, readyToUse, creationTime.UnixNano(), size) - if err != nil { - return nil, err - } - // store latest content - _, err = ctrl.storeContentUpdate(latestContent) - if err != nil { - return nil, err - } - return newSnapshot, nil -} - -// The function goes through the whole snapshot creation process. -// 1. Trigger the snapshot through csi storage provider. -// 2. Update VolumeSnapshot status with creationtimestamp information -// 3. Create the VolumeSnapshotContent object with the snapshot id information. -// 4. Bind the VolumeSnapshot and VolumeSnapshotContent object -func (ctrl *csiSnapshotController) createSnapshotOperation(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshot, error) { - klog.Infof("createSnapshot: Creating snapshot %s through the plugin ...", snapshotKey(snapshot)) - - if snapshot.Status != nil && snapshot.Status.Error != nil && snapshot.Status.Error.Message != nil && !isControllerUpdateFailError(snapshot.Status.Error) { - klog.V(4).Infof("error is already set in snapshot, do not retry to create: %s", *snapshot.Status.Error.Message) - return snapshot, nil - } - - // If PVC is not being deleted and finalizer is not added yet, a finalizer should be added. - klog.V(5).Infof("createSnapshotOperation: Check if PVC is not being deleted and add Finalizer for source of snapshot [%s] if needed", snapshot.Name) - err := ctrl.ensureSnapshotSourceFinalizer(snapshot) - if err != nil { - klog.Errorf("createSnapshotOperation failed to add finalizer for source of snapshot %s", err) - return nil, err - } - - class, volume, contentName, snapshotterSecretRef, err := ctrl.getCreateSnapshotInput(snapshot) - if err != nil { - return nil, fmt.Errorf("failed to get input parameters to create snapshot %s: %q", snapshot.Name, err) - } - - snapshotterCredentials, err := getCredentials(ctrl.client, snapshotterSecretRef) - if err != nil { - return nil, err - } - - driverName, snapshotID, creationTime, size, readyToUse, err := ctrl.handler.CreateSnapshot(snapshot, volume, class.Parameters, snapshotterCredentials) - if err != nil { - return nil, fmt.Errorf("failed to take snapshot of the volume, %s: %q", volume.Name, err) - } - - klog.V(5).Infof("Created snapshot: driver %s, snapshotId %s, creationTime %v, size %d, readyToUse %t", driverName, snapshotID, creationTime, size, readyToUse) - - var newSnapshot *crdv1.VolumeSnapshot - // Update snapshot status with creationTime - for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ { - klog.V(5).Infof("createSnapshot [%s]: trying to update snapshot creation timestamp", snapshotKey(snapshot)) - newSnapshot, err = ctrl.updateSnapshotStatus(snapshot, contentName, readyToUse, creationTime, size) - if err == nil { - break - } - klog.V(4).Infof("failed to update snapshot %s creation timestamp: %v", snapshotKey(snapshot), err) - } - - if err != nil { - return nil, err - } - // Create VolumeSnapshotContent in the database - snapshotRef, err := ref.GetReference(scheme.Scheme, snapshot) - if err != nil { - return nil, err - } - - timestamp := creationTime.UnixNano() - snapshotContent := &crdv1.VolumeSnapshotContent{ - ObjectMeta: metav1.ObjectMeta{ - Name: contentName, - }, - Spec: crdv1.VolumeSnapshotContentSpec{ - VolumeSnapshotRef: *snapshotRef, - DeletionPolicy: class.DeletionPolicy, - Driver: driverName, - VolumeSnapshotClassName: &class.Name, - Source: crdv1.VolumeSnapshotContentSource{ - VolumeHandle: &volume.Spec.CSI.VolumeHandle, - }, - }, - } - - // Set AnnDeletionSecretRefName and AnnDeletionSecretRefNamespace - if snapshotterSecretRef != nil { - klog.V(5).Infof("createSnapshotOperation: set annotation [%s] on content [%s].", AnnDeletionSecretRefName, snapshotContent.Name) - metav1.SetMetaDataAnnotation(&snapshotContent.ObjectMeta, AnnDeletionSecretRefName, snapshotterSecretRef.Name) - - klog.V(5).Infof("syncContent: set annotation [%s] on content [%s].", AnnDeletionSecretRefNamespace, snapshotContent.Name) - metav1.SetMetaDataAnnotation(&snapshotContent.ObjectMeta, AnnDeletionSecretRefNamespace, snapshotterSecretRef.Namespace) - } - - klog.V(3).Infof("volume snapshot content %v", snapshotContent) - // Try to create the VolumeSnapshotContent object several times - for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ { - klog.V(5).Infof("createSnapshot [%s]: trying to save volume snapshot content %s", snapshotKey(snapshot), snapshotContent.Name) - if _, err = ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Create(snapshotContent); err == nil || apierrs.IsAlreadyExists(err) { - // creation succeeded. - if err != nil { - klog.V(3).Infof("volume snapshot content %q for snapshot %q already exists, reusing", snapshotContent.Name, snapshotKey(snapshot)) - err = nil - } else { - klog.V(3).Infof("volume snapshot content %q for snapshot %q saved, %v", snapshotContent.Name, snapshotKey(snapshot), snapshotContent) - } - newContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Get(snapshotContent.Name, metav1.GetOptions{}) - if err == nil { - err = ctrl.updateSnapshotContentStatus(newContent, snapshotID, readyToUse, timestamp, size) - } - break - } - // Save failed, try again after a while. - klog.V(3).Infof("failed to save volume snapshot content %q for snapshot %q: %v", snapshotContent.Name, snapshotKey(snapshot), err) - time.Sleep(ctrl.createSnapshotContentInterval) - } - - if err != nil { - // Save failed. Now we have a snapshot asset outside of Kubernetes, - // but we don't have appropriate volumesnapshot content object for it. - // Emit some event here and controller should try to create the content in next sync period. - strerr := fmt.Sprintf("Error creating volume snapshot content object for snapshot %s: %v.", snapshotKey(snapshot), err) - klog.Error(strerr) - ctrl.eventRecorder.Event(newSnapshot, v1.EventTypeWarning, "CreateSnapshotContentFailed", strerr) - return nil, newControllerUpdateError(snapshotKey(snapshot), err.Error()) - } - return newSnapshot, nil -} - -// Delete a snapshot -// 1. Find the SnapshotContent corresponding to Snapshot -// 1a: Not found => finish (it's been deleted already) -// 2. Ask the backend to remove the snapshot device -// 3. Delete the SnapshotContent object -// 4. Remove the Snapshot from store -// 5. Finish -func (ctrl *csiSnapshotController) deleteSnapshotContentOperation(content *crdv1.VolumeSnapshotContent) error { - klog.V(5).Infof("deleteSnapshotOperation [%s] started", content.Name) - - // get secrets if VolumeSnapshotClass specifies it - var snapshotterCredentials map[string]string - /* TODO(@Xing-yang): secrete ref retrivial needs to be implemented here - snapshotClassName := content.Spec.VolumeSnapshotClassName - if snapshotClassName != nil { - if snapshotClass, err := ctrl.classLister.Get(*snapshotClassName); err == nil { - // Resolve snapshotting secret credentials. - // No VolumeSnapshot is provided when resolving delete secret names, since the VolumeSnapshot may or may not exist at delete time. - snapshotterSecretRef, err := getSecretReference(snapshotClass.Parameters, content.Name, nil) - if err != nil { - return err - } - snapshotterCredentials, err = getCredentials(ctrl.client, snapshotterSecretRef) - if err != nil { - return err - } - } - }*/ - - err := ctrl.handler.DeleteSnapshot(content, snapshotterCredentials) - if err != nil { - ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotDeleteError", "Failed to delete snapshot") - return fmt.Errorf("failed to delete snapshot %#v, err: %v", content.Name, err) - } - - err = ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Delete(content.Name, &metav1.DeleteOptions{}) - if err != nil { - ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotContentObjectDeleteError", "Failed to delete snapshot content API object") - return fmt.Errorf("failed to delete VolumeSnapshotContent %s from API server: %q", content.Name, err) - } - - return nil -} - -// TODO(xiangqian) This is a temp implementation to make sure the test cases pass -// updateSnapshotContentStatus updates status of a content object -func (ctrl *csiSnapshotController) updateSnapshotContentStatus( - content *crdv1.VolumeSnapshotContent, - snapshotHandle string, - readyToUse bool, - createdAt int64, - size int64) error { - var newStatus *crdv1.VolumeSnapshotContentStatus - updated := false - if content.Status == nil { - newStatus = &crdv1.VolumeSnapshotContentStatus{ - SnapshotHandle: &snapshotHandle, - ReadyToUse: &readyToUse, - CreationTime: &createdAt, - RestoreSize: &size, - } - updated = true - } else { - newStatus = content.Status.DeepCopy() - if newStatus.SnapshotHandle == nil { - newStatus.SnapshotHandle = &snapshotHandle - updated = true - } - if newStatus.ReadyToUse == nil || *newStatus.ReadyToUse != readyToUse { - newStatus.ReadyToUse = &readyToUse - updated = true - if readyToUse && newStatus.Error != nil { - newStatus.Error = nil - } - } - if newStatus.CreationTime == nil { - newStatus.CreationTime = &createdAt - updated = true - } - if newStatus.RestoreSize == nil { - newStatus.RestoreSize = &size - updated = true - } - } - - if updated { - contentClone := content.DeepCopy() - contentClone.Status = newStatus - _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().UpdateStatus(contentClone) - if err != nil { - return newControllerUpdateError(content.Name, err.Error()) - } - } - return nil -} - -// UpdateSnapshotStatus converts snapshot status to crdv1.VolumeSnapshotCondition -func (ctrl *csiSnapshotController) updateSnapshotStatus( - snapshot *crdv1.VolumeSnapshot, - boundContentName string, - readyToUse bool, - createdAt time.Time, - size int64) (*crdv1.VolumeSnapshot, error) { - klog.V(5).Infof("updating VolumeSnapshot[]%s, readyToUse %v, timestamp %v", snapshotKey(snapshot), readyToUse, createdAt) - - var newStatus *crdv1.VolumeSnapshotStatus - updated := false - if snapshot.Status == nil { - newStatus = &crdv1.VolumeSnapshotStatus{ - BoundVolumeSnapshotContentName: &boundContentName, - CreationTime: &metav1.Time{Time: createdAt}, - ReadyToUse: &readyToUse, - RestoreSize: resource.NewQuantity(size, resource.BinarySI), - } - updated = true - } else { - newStatus = snapshot.Status.DeepCopy() - if newStatus.BoundVolumeSnapshotContentName == nil { - newStatus.BoundVolumeSnapshotContentName = &boundContentName - updated = true - } - if newStatus.CreationTime == nil { - newStatus.CreationTime = &metav1.Time{Time: createdAt} - updated = true - } - if newStatus.ReadyToUse == nil || *newStatus.ReadyToUse != readyToUse { - newStatus.ReadyToUse = &readyToUse - updated = true - if readyToUse && newStatus.Error != nil { - newStatus.Error = nil - } - } - if newStatus.RestoreSize == nil { - newStatus.RestoreSize = resource.NewQuantity(size, resource.BinarySI) - updated = true - } - } - - if updated { - snapshotClone := snapshot.DeepCopy() - snapshotClone.Status = newStatus - newSnapshotObj, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).UpdateStatus(snapshotClone) - if err != nil { - return nil, newControllerUpdateError(snapshotKey(snapshot), err.Error()) - } - return newSnapshotObj, nil - } - return snapshot, nil -} - -// getVolumeFromVolumeSnapshot is a helper function to get PV from VolumeSnapshot. -func (ctrl *csiSnapshotController) getVolumeFromVolumeSnapshot(snapshot *crdv1.VolumeSnapshot) (*v1.PersistentVolume, error) { - pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) - if err != nil { - return nil, err - } - - if pvc.Status.Phase != v1.ClaimBound { - return nil, fmt.Errorf("the PVC %s is not yet bound to a PV, will not attempt to take a snapshot", pvc.Name) - } - - pvName := pvc.Spec.VolumeName - pv, err := ctrl.client.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{}) - if err != nil { - return nil, fmt.Errorf("failed to retrieve PV %s from the API server: %q", pvName, err) - } - - klog.V(5).Infof("getVolumeFromVolumeSnapshot: snapshot [%s] PV name [%s]", snapshot.Name, pvName) - - return pv, nil -} - -func (ctrl *csiSnapshotController) getStorageClassFromVolumeSnapshot(snapshot *crdv1.VolumeSnapshot) (*storagev1.StorageClass, error) { - // Get storage class from PVC or PV - pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) - if err != nil { - return nil, err - } - storageclassName := *pvc.Spec.StorageClassName - if len(storageclassName) == 0 { - volume, err := ctrl.getVolumeFromVolumeSnapshot(snapshot) - if err != nil { - return nil, err - } - storageclassName = volume.Spec.StorageClassName - } - if len(storageclassName) == 0 { - return nil, fmt.Errorf("cannot figure out the snapshot class automatically, please specify one in snapshot spec") - } - storageclass, err := ctrl.client.StorageV1().StorageClasses().Get(storageclassName, metav1.GetOptions{}) - if err != nil { - return nil, err - } - return storageclass, nil -} - -// GetSnapshotClass is a helper function to get snapshot class from the class name. -func (ctrl *csiSnapshotController) GetSnapshotClass(className string) (*crdv1.VolumeSnapshotClass, error) { - klog.V(5).Infof("getSnapshotClass: VolumeSnapshotClassName [%s]", className) - - class, err := ctrl.classLister.Get(className) - if err != nil { - klog.Errorf("failed to retrieve snapshot class %s from the informer: %q", className, err) - return nil, fmt.Errorf("failed to retrieve snapshot class %s from the informer: %q", className, err) - } - - return class, nil -} - -// SetDefaultSnapshotClass is a helper function to figure out the default snapshot class from -// PVC/PV StorageClass and update VolumeSnapshot with this snapshot class name. -func (ctrl *csiSnapshotController) SetDefaultSnapshotClass(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshotClass, *crdv1.VolumeSnapshot, error) { - klog.V(5).Infof("SetDefaultSnapshotClass for snapshot [%s]", snapshot.Name) - - storageclass, err := ctrl.getStorageClassFromVolumeSnapshot(snapshot) - if err != nil { - return nil, nil, err - } - // Find default snapshot class if available - list, err := ctrl.classLister.List(labels.Everything()) - if err != nil { - return nil, nil, err - } - defaultClasses := []*crdv1.VolumeSnapshotClass{} - - for _, class := range list { - if IsDefaultAnnotation(class.ObjectMeta) && storageclass.Provisioner == class.Driver && ctrl.driverName == class.Driver { - defaultClasses = append(defaultClasses, class) - klog.V(5).Infof("get defaultClass added: %s", class.Name) - } - } - if len(defaultClasses) == 0 { - return nil, nil, fmt.Errorf("cannot find default snapshot class") - } - if len(defaultClasses) > 1 { - klog.V(4).Infof("get DefaultClass %d defaults found", len(defaultClasses)) - return nil, nil, fmt.Errorf("%d default snapshot classes were found", len(defaultClasses)) - } - klog.V(5).Infof("setDefaultSnapshotClass [%s]: default VolumeSnapshotClassName [%s]", snapshot.Name, defaultClasses[0].Name) - snapshotClone := snapshot.DeepCopy() - snapshotClone.Spec.VolumeSnapshotClassName = &(defaultClasses[0].Name) - newSnapshot, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).Update(snapshotClone) - if err != nil { - klog.V(4).Infof("updating VolumeSnapshot[%s] default class failed %v", snapshotKey(snapshot), err) - } - _, updateErr := ctrl.storeSnapshotUpdate(newSnapshot) - if updateErr != nil { - // We will get an "snapshot update" event soon, this is not a big error - klog.V(4).Infof("setDefaultSnapshotClass [%s]: cannot update internal cache: %v", snapshotKey(snapshot), updateErr) - } - - return defaultClasses[0], newSnapshot, nil -} - -// getClaimFromVolumeSnapshot is a helper function to get PVC from VolumeSnapshot. -func (ctrl *csiSnapshotController) getClaimFromVolumeSnapshot(snapshot *crdv1.VolumeSnapshot) (*v1.PersistentVolumeClaim, error) { - if snapshot.Spec.Source.PersistentVolumeClaimName == nil { - return nil, fmt.Errorf("the snapshot source is not the right type. Expected PersistentVolumeClaimName to be valid") - } - pvcName := *snapshot.Spec.Source.PersistentVolumeClaimName - if pvcName == "" { - return nil, fmt.Errorf("the PVC name is not specified in snapshot %s", snapshotKey(snapshot)) - } - - pvc, err := ctrl.pvcLister.PersistentVolumeClaims(snapshot.Namespace).Get(pvcName) - if err != nil { - return nil, fmt.Errorf("failed to retrieve PVC %s from the lister: %q", pvcName, err) - } - - return pvc, nil -} - -var _ error = controllerUpdateError{} - -type controllerUpdateError struct { - message string -} - -func newControllerUpdateError(name, message string) error { - return controllerUpdateError{ - message: fmt.Sprintf("%s %s on API server: %s", controllerUpdateFailMsg, name, message), - } -} - -func (e controllerUpdateError) Error() string { - return e.message -} - -func isControllerUpdateFailError(err *crdv1.VolumeSnapshotError) bool { - if err != nil && err.Message != nil { - if strings.Contains(*err.Message, controllerUpdateFailMsg) { - return true - } - } - return false -} - -// addContentFinalizer adds a Finalizer for VolumeSnapshotContent. -func (ctrl *csiSnapshotController) addContentFinalizer(content *crdv1.VolumeSnapshotContent) error { - contentClone := content.DeepCopy() - contentClone.ObjectMeta.Finalizers = append(contentClone.ObjectMeta.Finalizers, VolumeSnapshotContentFinalizer) - - _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) - if err != nil { - return newControllerUpdateError(content.Name, err.Error()) - } - - _, err = ctrl.storeContentUpdate(contentClone) - if err != nil { - klog.Errorf("failed to update content store %v", err) - } - - klog.V(5).Infof("Added protection finalizer to volume snapshot content %s", content.Name) - return nil -} - -// removeContentFinalizer removes a Finalizer for VolumeSnapshotContent. -func (ctrl *csiSnapshotController) removeContentFinalizer(content *crdv1.VolumeSnapshotContent) error { - contentClone := content.DeepCopy() - contentClone.ObjectMeta.Finalizers = slice.RemoveString(contentClone.ObjectMeta.Finalizers, VolumeSnapshotContentFinalizer, nil) - - _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) - if err != nil { - return newControllerUpdateError(content.Name, err.Error()) - } - - _, err = ctrl.storeContentUpdate(contentClone) - if err != nil { - klog.Errorf("failed to update content store %v", err) - } - - klog.V(5).Infof("Removed protection finalizer from volume snapshot content %s", content.Name) - return nil -} - -// addSnapshotFinalizer adds a Finalizer for VolumeSnapshot. -func (ctrl *csiSnapshotController) addSnapshotFinalizer(snapshot *crdv1.VolumeSnapshot) error { - snapshotClone := snapshot.DeepCopy() - snapshotClone.ObjectMeta.Finalizers = append(snapshotClone.ObjectMeta.Finalizers, VolumeSnapshotFinalizer) - _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).Update(snapshotClone) - if err != nil { - return newControllerUpdateError(snapshot.Name, err.Error()) - } - - _, err = ctrl.storeSnapshotUpdate(snapshotClone) - if err != nil { - klog.Errorf("failed to update snapshot store %v", err) - } - - klog.V(5).Infof("Added protection finalizer to volume snapshot %s", snapshotKey(snapshot)) - return nil -} - -// removeContentFinalizer removes a Finalizer for VolumeSnapshot. -func (ctrl *csiSnapshotController) removeSnapshotFinalizer(snapshot *crdv1.VolumeSnapshot) error { - snapshotClone := snapshot.DeepCopy() - snapshotClone.ObjectMeta.Finalizers = slice.RemoveString(snapshotClone.ObjectMeta.Finalizers, VolumeSnapshotFinalizer, nil) - - _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).Update(snapshotClone) - if err != nil { - return newControllerUpdateError(snapshot.Name, err.Error()) - } - - _, err = ctrl.storeSnapshotUpdate(snapshotClone) - if err != nil { - klog.Errorf("failed to update snapshot store %v", err) - } - - klog.V(5).Infof("Removed protection finalizer from volume snapshot %s", snapshotKey(snapshot)) - return nil -} - -// ensureSnapshotSourceFinalizer checks if a Finalizer needs to be added for the snapshot source; -// if true, adds a Finalizer for VolumeSnapshot Source PVC -func (ctrl *csiSnapshotController) ensureSnapshotSourceFinalizer(snapshot *crdv1.VolumeSnapshot) error { - // Get snapshot source which is a PVC - pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) - if err != nil { - klog.Infof("cannot get claim from snapshot [%s]: [%v] Claim may be deleted already.", snapshot.Name, err) - return nil - } - - // If PVC is not being deleted and PVCFinalizer is not added yet, the PVCFinalizer should be added. - if pvc.ObjectMeta.DeletionTimestamp == nil && !slice.ContainsString(pvc.ObjectMeta.Finalizers, PVCFinalizer, nil) { - // Add the finalizer - pvcClone := pvc.DeepCopy() - pvcClone.ObjectMeta.Finalizers = append(pvcClone.ObjectMeta.Finalizers, PVCFinalizer) - _, err = ctrl.client.CoreV1().PersistentVolumeClaims(pvcClone.Namespace).Update(pvcClone) - if err != nil { - klog.Errorf("cannot add finalizer on claim [%s] for snapshot [%s]: [%v]", pvc.Name, snapshot.Name, err) - return newControllerUpdateError(pvcClone.Name, err.Error()) - } - klog.Infof("Added protection finalizer to persistent volume claim %s", pvc.Name) - } - - return nil -} - -// removeSnapshotSourceFinalizer removes a Finalizer for VolumeSnapshot Source PVC. -func (ctrl *csiSnapshotController) removeSnapshotSourceFinalizer(snapshot *crdv1.VolumeSnapshot) error { - // Get snapshot source which is a PVC - pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) - if err != nil { - klog.Infof("cannot get claim from snapshot [%s]: [%v] Claim may be deleted already. No need to remove finalizer on the claim.", snapshot.Name, err) - return nil - } - - pvcClone := pvc.DeepCopy() - pvcClone.ObjectMeta.Finalizers = slice.RemoveString(pvcClone.ObjectMeta.Finalizers, PVCFinalizer, nil) - - _, err = ctrl.client.CoreV1().PersistentVolumeClaims(pvcClone.Namespace).Update(pvcClone) - if err != nil { - return newControllerUpdateError(pvcClone.Name, err.Error()) - } - - klog.V(5).Infof("Removed protection finalizer from persistent volume claim %s", pvc.Name) - return nil -} - -// isSnapshotSourceBeingUsed checks if a PVC is being used as a source to create a snapshot -func (ctrl *csiSnapshotController) isSnapshotSourceBeingUsed(snapshot *crdv1.VolumeSnapshot) bool { - klog.V(5).Infof("isSnapshotSourceBeingUsed[%s]: started", snapshotKey(snapshot)) - // Get snapshot source which is a PVC - pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) - if err != nil { - klog.Infof("isSnapshotSourceBeingUsed: cannot to get claim from snapshot: %v", err) - return false - } - - // Going through snapshots in the cache (snapshotLister). If a snapshot's PVC source - // is the same as the input snapshot's PVC source and snapshot's ReadyToUse status - // is false, the snapshot is still being created from the PVC and the PVC is in-use. - snapshots, err := ctrl.snapshotLister.VolumeSnapshots(snapshot.Namespace).List(labels.Everything()) - if err != nil { - return false - } - for _, snap := range snapshots { - // Skip static bound snapshot without a PVC source - if snap.Spec.Source.PersistentVolumeClaimName == nil { - klog.V(4).Infof("Skipping static bound snapshot %s when checking PVC %s/%s", snap.Name, pvc.Namespace, pvc.Name) - continue - } - if pvc.Name == *snap.Spec.Source.PersistentVolumeClaimName && !isSnapshotReadyToUse(snap) { - klog.V(2).Infof("Keeping PVC %s/%s, it is used by snapshot %s/%s", pvc.Namespace, pvc.Name, snap.Namespace, snap.Name) - return true - } - } - - klog.V(5).Infof("isSnapshotSourceBeingUsed: no snapshot is being created from PVC %s/%s", pvc.Namespace, pvc.Name) - return false -} - -// checkandRemoveSnapshotSourceFinalizer checks if the snapshot source finalizer should be removed -// and removed it if needed. -func (ctrl *csiSnapshotController) checkandRemoveSnapshotSourceFinalizer(snapshot *crdv1.VolumeSnapshot) error { - // Get snapshot source which is a PVC - pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) - if err != nil { - klog.Infof("cannot get claim from snapshot [%s]: [%v] Claim may be deleted already. No need to remove finalizer on the claim.", snapshot.Name, err) - return nil - } - - klog.V(5).Infof("checkandRemoveSnapshotSourceFinalizer for snapshot [%s]: snapshot status [%#v]", snapshot.Name, snapshot.Status) - - // Check if there is a Finalizer on PVC to be removed - if slice.ContainsString(pvc.ObjectMeta.Finalizers, PVCFinalizer, nil) { - // There is a Finalizer on PVC. Check if PVC is used - // and remove finalizer if it's not used. - isUsed := ctrl.isSnapshotSourceBeingUsed(snapshot) - if !isUsed { - klog.Infof("checkandRemoveSnapshotSourceFinalizer[%s]: Remove Finalizer for PVC %s as it is not used by snapshots in creation", snapshot.Name, pvc.Name) - err = ctrl.removeSnapshotSourceFinalizer(snapshot) - if err != nil { - klog.Errorf("checkandRemoveSnapshotSourceFinalizer [%s]: removeSnapshotSourceFinalizer failed to remove finalizer %v", snapshot.Name, err) - return err - } - } - } - - return nil -} - -// isSnapshotReadyToUse checks if a snapshot object has ReadyToUse in Status set to true -func isSnapshotReadyToUse(snapshot *crdv1.VolumeSnapshot) bool { - if snapshot.Status == nil || snapshot.Status.ReadyToUse == nil { - return false - } - return *snapshot.Status.ReadyToUse -} diff --git a/pkg/controller/csi_handler.go b/pkg/sidecar-controller/csi_handler.go similarity index 62% rename from pkg/controller/csi_handler.go rename to pkg/sidecar-controller/csi_handler.go index 840ab150..f261c8c9 100644 --- a/pkg/controller/csi_handler.go +++ b/pkg/sidecar-controller/csi_handler.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package sidecar_controller import ( "context" @@ -24,13 +24,12 @@ import ( crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" "github.com/kubernetes-csi/external-snapshotter/pkg/snapshotter" - - v1 "k8s.io/api/core/v1" + "github.com/kubernetes-csi/external-snapshotter/pkg/utils" ) // Handler is responsible for handling VolumeSnapshot events from informer. type Handler interface { - CreateSnapshot(snapshot *crdv1.VolumeSnapshot, volume *v1.PersistentVolume, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, error) + CreateSnapshot(content *crdv1.VolumeSnapshotContent, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, error) DeleteSnapshot(content *crdv1.VolumeSnapshotContent, snapshotterCredentials map[string]string) error GetSnapshotStatus(content *crdv1.VolumeSnapshotContent) (bool, time.Time, int64, error) } @@ -58,27 +57,45 @@ func NewCSIHandler( } } -func (handler *csiHandler) CreateSnapshot(snapshot *crdv1.VolumeSnapshot, volume *v1.PersistentVolume, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, error) { +func (handler *csiHandler) CreateSnapshot(content *crdv1.VolumeSnapshotContent, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, error) { ctx, cancel := context.WithTimeout(context.Background(), handler.timeout) defer cancel() - snapshotName, err := makeSnapshotName(handler.snapshotNamePrefix, string(snapshot.UID), handler.snapshotNameUUIDLength) + if content.Spec.VolumeSnapshotRef.UID == "" { + return "", "", time.Time{}, 0, false, fmt.Errorf("cannot create snapshot. Snapshot content %s not bound to a snapshot", content.Name) + } + + if content.Spec.Source.VolumeHandle == nil { + return "", "", time.Time{}, 0, false, fmt.Errorf("cannot create snapshot. Volume handle not found in snapshot content %s", content.Name) + } + + snapshotName, err := makeSnapshotName(handler.snapshotNamePrefix, string(content.Spec.VolumeSnapshotRef.UID), handler.snapshotNameUUIDLength) if err != nil { return "", "", time.Time{}, 0, false, err } - newParameters, err := removePrefixedParameters(parameters) + newParameters, err := utils.RemovePrefixedParameters(parameters) if err != nil { return "", "", time.Time{}, 0, false, fmt.Errorf("failed to remove CSI Parameters of prefixed keys: %v", err) } - return handler.snapshotter.CreateSnapshot(ctx, snapshotName, volume, newParameters, snapshotterCredentials) + return handler.snapshotter.CreateSnapshot(ctx, snapshotName, *content.Spec.Source.VolumeHandle, newParameters, snapshotterCredentials) } func (handler *csiHandler) DeleteSnapshot(content *crdv1.VolumeSnapshotContent, snapshotterCredentials map[string]string) error { ctx, cancel := context.WithTimeout(context.Background(), handler.timeout) defer cancel() - err := handler.snapshotter.DeleteSnapshot(ctx, *content.Status.SnapshotHandle, snapshotterCredentials) + var snapshotHandle string + var err error + if content.Status != nil && content.Status.SnapshotHandle != nil { + snapshotHandle = *content.Status.SnapshotHandle + } else if content.Spec.Source.SnapshotHandle != nil { + snapshotHandle = *content.Spec.Source.SnapshotHandle + } else { + return fmt.Errorf("failed to delete snapshot content %s: snapshotHandle is missing", content.Name) + } + + err = handler.snapshotter.DeleteSnapshot(ctx, snapshotHandle, snapshotterCredentials) if err != nil { return fmt.Errorf("failed to delete snapshot content %s: %q", content.Name, err) } @@ -90,9 +107,19 @@ func (handler *csiHandler) GetSnapshotStatus(content *crdv1.VolumeSnapshotConten ctx, cancel := context.WithTimeout(context.Background(), handler.timeout) defer cancel() - csiSnapshotStatus, timestamp, size, err := handler.snapshotter.GetSnapshotStatus(ctx, *content.Status.SnapshotHandle) + var snapshotHandle string + var err error + if content.Status != nil && content.Status.SnapshotHandle != nil { + snapshotHandle = *content.Status.SnapshotHandle + } else if content.Spec.Source.SnapshotHandle != nil { + snapshotHandle = *content.Spec.Source.SnapshotHandle + } else { + return false, time.Time{}, 0, fmt.Errorf("failed to list snapshot for content %s: snapshotHandle is missing", content.Name) + } + + csiSnapshotStatus, timestamp, size, err := handler.snapshotter.GetSnapshotStatus(ctx, snapshotHandle) if err != nil { - return false, time.Time{}, 0, fmt.Errorf("failed to list snapshot content %s: %q", content.Name, err) + return false, time.Time{}, 0, fmt.Errorf("failed to list snapshot for content %s: %q", content.Name, err) } return csiSnapshotStatus, timestamp, size, nil diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go new file mode 100644 index 00000000..c3199698 --- /dev/null +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -0,0 +1,539 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sidecar_controller + +import ( + "fmt" + "strings" + "time" + + crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/pkg/utils" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" + "k8s.io/kubernetes/pkg/util/goroutinemap" + "k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff" + "k8s.io/kubernetes/pkg/util/slice" +) + +// Design: +// +// This is the sidecar controller that is responsible for creating and deleting a +// snapshot on the storage infrastructure through a csi volume driver. It watches +// the VolumeSnapshotContent object which is either created/deleted by the +// common snapshot controller in the case of dynamic provisioning or by the admin +// in the case of pre-provisioned snapshots. + +// The snapshot creation through csi volume driver should return a snapshot after +// it is created successfully (however, the snapshot might not be ready to use yet if +// there is an uploading phase). The creationTime will be updated accordingly +// on the status of VolumeSnapshotContent. +// After that, the sidecar controller will keep checking the snapshot status +// through csi snapshot calls. When the snapshot is ready to use, the sidecar +// controller set the status "ReadyToUse" to true on the VolumeSnapshotContent object +// to indicate the snapshot is ready to use. If the creation failed for any reason, +// the Error status is set accordingly. + +const controllerUpdateFailMsg = "snapshot controller failed to update" + +// syncContent deals with one key off the queue. It returns false when it's time to quit. +func (ctrl *csiSnapshotSideCarController) syncContent(content *crdv1.VolumeSnapshotContent) error { + klog.V(5).Infof("synchronizing VolumeSnapshotContent[%s]", content.Name) + + var err error + if ctrl.shouldDelete(content) { + switch content.Spec.DeletionPolicy { + case crdv1.VolumeSnapshotContentRetain: + klog.V(4).Infof("VolumeSnapshotContent[%s]: policy is Retain. Keep physical snapshot and remove content finalizer", content.Name) + // It is a deletion candidate if DeletionTimestamp is not nil and + // VolumeSnapshotContentFinalizer is set. + if utils.IsContentDeletionCandidate(content) { + // Volume snapshot content is a deletion candidate. + // Remove the content finalizer. + klog.V(5).Infof("syncContent: Content [%s] is a deletion candidate. Remove finalizer.", content.Name) + return ctrl.removeContentFinalizer(content) + } + + case crdv1.VolumeSnapshotContentDelete: + klog.V(4).Infof("VolumeSnapshotContent[%s]: policy is Delete. Delete physical snapshot", content.Name) + err = ctrl.deleteCSISnapshot(content) + if err != nil { + return err + } + klog.V(5).Infof("syncContent: check if we should remove Finalizer for VolumeSnapshotContent[%s]", content.Name) + // It is a deletion candidate if DeletionTimestamp is not nil and + // VolumeSnapshotContentFinalizer is set. + if utils.IsContentDeletionCandidate(content) { + // Volume snapshot content is a deletion candidate. + // Remove the content finalizer. + klog.V(5).Infof("syncContent: Content [%s] is a deletion candidate. Remove finalizer.", content.Name) + return ctrl.removeContentFinalizer(content) + } + + default: + // Unknown VolumeSnapshotDeletionPolicy + ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotUnknownDeletionPolicy", "Volume Snapshot Content has unrecognized deletion policy") + } + klog.V(4).Infof("VolumeSnapshotContent[%s]: the policy is %s", content.Name, content.Spec.DeletionPolicy) + } else { + var err error + klog.V(5).Infof("syncContent: Call CreateSnapshot for content %s", content.Name) + if content.Spec.Source.VolumeHandle != nil && content.Status == nil { + if err = ctrl.createSnapshot(content); err != nil { + ctrl.updateContentErrorStatusWithEvent(content, v1.EventTypeWarning, "SnapshotCreationFailed", fmt.Sprintf("Failed to create snapshot with error %v", err)) + return err + } + } else { + + if err = ctrl.checkandUpdateContentStatus(content); err != nil { + ctrl.updateContentErrorStatusWithEvent(content, v1.EventTypeWarning, "SnapshotContentStatusUpdateFailed", fmt.Sprintf("Failed to update snapshot content status with error %v", err)) + return err + } + } + + return nil + } + + return nil +} + +// deleteCSISnapshot starts delete action. +func (ctrl *csiSnapshotSideCarController) deleteCSISnapshot(content *crdv1.VolumeSnapshotContent) error { + operationName := fmt.Sprintf("delete-%s", content.Name) + klog.V(5).Infof("Snapshotter is about to delete volume snapshot content and the operation named %s", operationName) + ctrl.scheduleOperation(operationName, func() error { + return ctrl.deleteCSISnapshotOperation(content) + }) + return nil +} + +// scheduleOperation starts given asynchronous operation on given volume. It +// makes sure the operation is already not running. +func (ctrl *csiSnapshotSideCarController) scheduleOperation(operationName string, operation func() error) { + klog.V(5).Infof("scheduleOperation[%s]", operationName) + + err := ctrl.runningOperations.Run(operationName, operation) + if err != nil { + switch { + case goroutinemap.IsAlreadyExists(err): + klog.V(4).Infof("operation %q is already running, skipping", operationName) + case exponentialbackoff.IsExponentialBackoff(err): + klog.V(4).Infof("operation %q postponed due to exponential backoff", operationName) + default: + klog.Errorf("error scheduling operation %q: %v", operationName, err) + } + } +} + +func (ctrl *csiSnapshotSideCarController) storeContentUpdate(content interface{}) (bool, error) { + return utils.StoreObjectUpdate(ctrl.contentStore, content, "content") +} + +// createSnapshot starts new asynchronous operation to create snapshot +func (ctrl *csiSnapshotSideCarController) createSnapshot(content *crdv1.VolumeSnapshotContent) error { + klog.V(5).Infof("createSnapshot for content [%s]: started", content.Name) + opName := fmt.Sprintf("create-%s", content.Name) + ctrl.scheduleOperation(opName, func() error { + contentObj, err := ctrl.createSnapshotOperation(content) + if err != nil { + ctrl.updateContentErrorStatusWithEvent(content, v1.EventTypeWarning, "SnapshotCreationFailed", fmt.Sprintf("Failed to create snapshot: %v", err)) + klog.Errorf("createSnapshot [%s]: error occurred in createSnapshotOperation: %v", opName, err) + return err + } + + _, updateErr := ctrl.storeContentUpdate(contentObj) + if updateErr != nil { + // We will get an "snapshot update" event soon, this is not a big error + klog.V(4).Infof("createSnapshot [%s]: cannot update internal content cache: %v", content.Name, updateErr) + } + return nil + }) + return nil +} + +func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatus(content *crdv1.VolumeSnapshotContent) error { + klog.V(5).Infof("checkandUpdateContentStatus[%s] started", content.Name) + opName := fmt.Sprintf("check-%s", content.Name) + ctrl.scheduleOperation(opName, func() error { + contentObj, err := ctrl.checkandUpdateContentStatusOperation(content) + if err != nil { + ctrl.updateContentErrorStatusWithEvent(content, v1.EventTypeWarning, "SnapshotContentCheckandUpdateFailed", fmt.Sprintf("Failed to check and update snapshot content: %v", err)) + klog.Errorf("checkandUpdateContentStatus [%s]: error occured %v", content.Name, err) + return err + } + _, updateErr := ctrl.storeContentUpdate(contentObj) + if updateErr != nil { + // We will get an "snapshot update" event soon, this is not a big error + klog.V(4).Infof("checkandUpdateContentStatus [%s]: cannot update internal cache: %v", content.Name, updateErr) + } + + return nil + }) + return nil +} + +// updateContentStatusWithEvent saves new content.Status to API server and emits +// given event on the content. It saves the status and emits the event only when +// the status has actually changed from the version saved in API server. +// Parameters: +// content - content to update +// eventtype, reason, message - event to send, see EventRecorder.Event() +func (ctrl *csiSnapshotSideCarController) updateContentErrorStatusWithEvent(content *crdv1.VolumeSnapshotContent, eventtype, reason, message string) error { + klog.V(5).Infof("updateContentStatusWithEvent[%s]", content.Name) + + if content.Status != nil && content.Status.Error != nil && *content.Status.Error.Message == message { + klog.V(4).Infof("updateContentStatusWithEvent[%s]: the same error %v is already set", content.Name, content.Status.Error) + return nil + } else if content.Status == nil { + content.Status = &crdv1.VolumeSnapshotContentStatus{} + } + contentClone := content.DeepCopy() + statusError := &crdv1.VolumeSnapshotError{ + Time: &metav1.Time{ + Time: time.Now(), + }, + Message: &message, + } + contentClone.Status.Error = statusError + ready := false + contentClone.Status.ReadyToUse = &ready + newContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().UpdateStatus(contentClone) + + if err != nil { + klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status failed %v", content.Name, err) + return err + } + + _, err = ctrl.storeContentUpdate(newContent) + if err != nil { + klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status: cannot update internal cache %v", content.Name, err) + return err + } + // Emit the event only when the status change happens + ctrl.eventRecorder.Event(newContent, eventtype, reason, message) + + return nil +} + +func (ctrl *csiSnapshotSideCarController) getCSISnapshotInput(content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshotClass, map[string]string, error) { + className := content.Spec.VolumeSnapshotClassName + klog.V(5).Infof("getCSISnapshotInput for content [%s]: VolumeSnapshotClassName [%s]", content.Name, *className) + var class *crdv1.VolumeSnapshotClass + var err error + if className != nil { + class, err = ctrl.getSnapshotClass(*className) + if err != nil { + klog.Errorf("getCSISnapshotInput failed to getClassFromVolumeSnapshot %s", err) + return nil, nil, err + } + } else { + // If dynamic provisioning, return failure if no snapshot class + if content.Spec.Source.VolumeHandle != nil { + klog.Errorf("failed to getCSISnapshotInput %s without a snapshot class", content.Name) + return nil, nil, fmt.Errorf("failed to take snapshot %s without a snapshot class", content.Name) + } + // For pre-provisioned snapshot, snapshot class is not required + klog.V(5).Infof("getCSISnapshotInput for content [%s]: no VolumeSnapshotClassName provided for pre-provisioned snapshot", content.Name) + return nil, nil, nil + } + + // Resolve snapshotting secret credentials. + snapshotterCredentials, err := ctrl.GetCredentialsFromAnnotation(content) + if err != nil { + return nil, nil, err + } + + return class, snapshotterCredentials, nil +} + +func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatusOperation(content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshotContent, error) { + var err error + var creationTime time.Time + var size int64 + var readyToUse = false + var driverName string + var snapshotID string + + if content.Spec.Source.SnapshotHandle != nil { + klog.V(5).Infof("checkandUpdateContentStatusOperation: call GetSnapshotStatus for snapshot which is pre-bound to content [%s]", content.Name) + readyToUse, creationTime, size, err = ctrl.handler.GetSnapshotStatus(content) + if err != nil { + klog.Errorf("checkandUpdateContentStatusOperation: failed to call get snapshot status to check whether snapshot is ready to use %q", err) + return nil, err + } + driverName = content.Spec.Driver + if content.Spec.Source.SnapshotHandle != nil { + snapshotID = *content.Spec.Source.SnapshotHandle + } + } else { + class, snapshotterCredentials, err := ctrl.getCSISnapshotInput(content) + if err != nil { + return nil, fmt.Errorf("failed to get input parameters to create snapshot %s: %q", content.Name, err) + } + + driverName, snapshotID, creationTime, size, readyToUse, err = ctrl.handler.CreateSnapshot(content, class.Parameters, snapshotterCredentials) + if err != nil { + klog.Errorf("checkandUpdateContentStatusOperation: failed to call create snapshot to check whether the snapshot is ready to use %q", err) + return nil, err + } + } + klog.V(5).Infof("checkandUpdateContentStatusOperation: driver %s, snapshotId %s, creationTime %v, size %d, readyToUse %t", driverName, snapshotID, creationTime, size, readyToUse) + + if creationTime.IsZero() { + creationTime = time.Now() + } + + updateContent, err := ctrl.updateSnapshotContentStatus(content, snapshotID, readyToUse, creationTime.UnixNano(), size) + if err != nil { + return nil, err + } + return updateContent, nil +} + +// The function goes through the whole snapshot creation process. +// 1. Trigger the snapshot through csi storage provider. +// 2. Update VolumeSnapshot status with creationtimestamp information +// 3. Create the VolumeSnapshotContent object with the snapshot id information. +// 4. Bind the VolumeSnapshot and VolumeSnapshotContent object +func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshotContent, error) { + klog.Infof("createSnapshotOperation: Creating snapshot for content %s through the plugin ...", content.Name) + + // content.Status will be created for the first time after a snapshot + // is created by the CSI driver. If content.Status is not nil, + // we should update content status without creating snapshot again. + if content.Status != nil && content.Status.Error != nil && content.Status.Error.Message != nil && !isControllerUpdateFailError(content.Status.Error) { + klog.V(4).Infof("error is already set in snapshot, do not retry to create: %s", *content.Status.Error.Message) + return content, nil + } + + class, snapshotterCredentials, err := ctrl.getCSISnapshotInput(content) + if err != nil { + return nil, fmt.Errorf("failed to get input parameters to create snapshot for content %s: %q", content.Name, err) + } + + driverName, snapshotID, creationTime, size, readyToUse, err := ctrl.handler.CreateSnapshot(content, class.Parameters, snapshotterCredentials) + if err != nil { + return nil, fmt.Errorf("failed to take snapshot of the volume, %s: %q", *content.Spec.Source.VolumeHandle, err) + } + if driverName != class.Driver { + return nil, fmt.Errorf("failed to take snapshot of the volume, %s: driver name %s returned from the driver is different from driver %s in snapshot class", *content.Spec.Source.VolumeHandle, driverName, class.Driver) + } + + klog.V(5).Infof("Created snapshot: driver %s, snapshotId %s, creationTime %v, size %d, readyToUse %t", driverName, snapshotID, creationTime, size, readyToUse) + + timestamp := creationTime.UnixNano() + newContent, err := ctrl.updateSnapshotContentStatus(content, snapshotID, readyToUse, timestamp, size) + if err != nil { + strerr := fmt.Sprintf("error updating volume snapshot content status for snapshot %s: %v.", content.Name, err) + klog.Error(strerr) + } else { + content = newContent + } + + // Update content in the cache store + _, err = ctrl.storeContentUpdate(content) + if err != nil { + klog.Errorf("failed to update content store %v", err) + } + + return content, nil +} + +// Delete a snapshot: Ask the backend to remove the snapshot device +func (ctrl *csiSnapshotSideCarController) deleteCSISnapshotOperation(content *crdv1.VolumeSnapshotContent) error { + klog.V(5).Infof("deleteCSISnapshotOperation [%s] started", content.Name) + + _, snapshotterCredentials, err := ctrl.getCSISnapshotInput(content) + if err != nil { + return fmt.Errorf("failed to get input parameters to delete snapshot for content %s: %q", content.Name, err) + } + + err = ctrl.handler.DeleteSnapshot(content, snapshotterCredentials) + if err != nil { + ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotDeleteError", "Failed to delete snapshot") + return fmt.Errorf("failed to delete snapshot %#v, err: %v", content.Name, err) + } + + return nil +} + +func (ctrl *csiSnapshotSideCarController) updateSnapshotContentStatus( + content *crdv1.VolumeSnapshotContent, + snapshotHandle string, + readyToUse bool, + createdAt int64, + size int64) (*crdv1.VolumeSnapshotContent, error) { + klog.V(5).Infof("updateSnapshotContentStatus: updating VolumeSnapshotContent [%s], snapshotHandle %s, readyToUse %v, createdAt %v, size %d", content.Name, snapshotHandle, readyToUse, createdAt, size) + + contentObj, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Get(content.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("error get snapshot content %s from api server: %v", content.Name, err) + } + + var newStatus *crdv1.VolumeSnapshotContentStatus + updated := false + if contentObj.Status == nil { + newStatus = &crdv1.VolumeSnapshotContentStatus{ + SnapshotHandle: &snapshotHandle, + ReadyToUse: &readyToUse, + CreationTime: &createdAt, + RestoreSize: &size, + } + updated = true + } else { + newStatus = contentObj.Status.DeepCopy() + if newStatus.SnapshotHandle == nil { + newStatus.SnapshotHandle = &snapshotHandle + updated = true + } + if newStatus.ReadyToUse == nil || *newStatus.ReadyToUse != readyToUse { + newStatus.ReadyToUse = &readyToUse + updated = true + if readyToUse && newStatus.Error != nil { + newStatus.Error = nil + } + } + if newStatus.CreationTime == nil { + newStatus.CreationTime = &createdAt + updated = true + } + if newStatus.RestoreSize == nil { + newStatus.RestoreSize = &size + updated = true + } + } + + if updated { + contentClone := contentObj.DeepCopy() + contentClone.Status = newStatus + newContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().UpdateStatus(contentClone) + if err != nil { + return nil, newControllerUpdateError(content.Name, err.Error()) + } + return newContent, nil + } + + return contentObj, nil +} + +// getSnapshotClass is a helper function to get snapshot class from the class name. +func (ctrl *csiSnapshotSideCarController) getSnapshotClass(className string) (*crdv1.VolumeSnapshotClass, error) { + klog.V(5).Infof("getSnapshotClass: VolumeSnapshotClassName [%s]", className) + + class, err := ctrl.classLister.Get(className) + if err != nil { + klog.Errorf("failed to retrieve snapshot class %s from the informer: %q", className, err) + return nil, fmt.Errorf("failed to retrieve snapshot class %s from the informer: %q", className, err) + } + + return class, nil +} + +var _ error = controllerUpdateError{} + +type controllerUpdateError struct { + message string +} + +func newControllerUpdateError(name, message string) error { + return controllerUpdateError{ + message: fmt.Sprintf("%s %s on API server: %s", controllerUpdateFailMsg, name, message), + } +} + +func (e controllerUpdateError) Error() string { + return e.message +} + +func isControllerUpdateFailError(err *crdv1.VolumeSnapshotError) bool { + if err != nil { + if strings.Contains(*err.Message, controllerUpdateFailMsg) { + return true + } + } + return false +} + +func (ctrl *csiSnapshotSideCarController) GetCredentialsFromAnnotation(content *crdv1.VolumeSnapshotContent) (map[string]string, error) { + // get secrets if VolumeSnapshotClass specifies it + var snapshotterCredentials map[string]string + var err error + + // Check if annotation exists + if metav1.HasAnnotation(content.ObjectMeta, utils.AnnDeletionSecretRefName) && metav1.HasAnnotation(content.ObjectMeta, utils.AnnDeletionSecretRefNamespace) { + annDeletionSecretName := content.Annotations[utils.AnnDeletionSecretRefName] + annDeletionSecretNamespace := content.Annotations[utils.AnnDeletionSecretRefNamespace] + + snapshotterSecretRef := &v1.SecretReference{} + + if annDeletionSecretName == "" || annDeletionSecretNamespace == "" { + return nil, fmt.Errorf("cannot retrieve secrets for snapshot content %#v, err: secret name or namespace not specified", content.Name) + } + + snapshotterSecretRef.Name = annDeletionSecretName + snapshotterSecretRef.Namespace = annDeletionSecretNamespace + + snapshotterCredentials, err = utils.GetCredentials(ctrl.client, snapshotterSecretRef) + if err != nil { + // Continue with deletion, as the secret may have already been deleted. + klog.Errorf("Failed to get credentials for snapshot %s: %s", content.Name, err.Error()) + return nil, fmt.Errorf("cannot get credentials for snapshot content %#v", content.Name) + } + } + + return snapshotterCredentials, nil +} + +// removeContentFinalizer removes a Finalizer for VolumeSnapshotContent. +func (ctrl csiSnapshotSideCarController) removeContentFinalizer(content *crdv1.VolumeSnapshotContent) error { + contentClone := content.DeepCopy() + contentClone.ObjectMeta.Finalizers = slice.RemoveString(contentClone.ObjectMeta.Finalizers, utils.VolumeSnapshotContentFinalizer, nil) + + _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) + if err != nil { + return newControllerUpdateError(content.Name, err.Error()) + } + + _, err = ctrl.storeContentUpdate(contentClone) + if err != nil { + klog.Errorf("failed to update content store %v", err) + } + + klog.V(5).Infof("Removed protection finalizer from volume snapshot content %s", content.Name) + return nil +} + +// shouldDelete checks if content object should be deleted +// if DeletionTimestamp is set on the content +func (ctrl *csiSnapshotSideCarController) shouldDelete(content *crdv1.VolumeSnapshotContent) bool { + klog.V(5).Infof("Check if VolumeSnapshotContent[%s] should be deleted.", content.Name) + + if content.ObjectMeta.DeletionTimestamp == nil { + return false + } + // 1) shouldDelete returns true if content is not bound + // (VolumeSnapshotRef.UID == "") for pre-provisioned snapshot + if content.Spec.Source.SnapshotHandle != nil && content.Spec.VolumeSnapshotRef.UID == "" { + return true + } + // 2) shouldDelete returns true if AnnVolumeSnapshotBeingDeleted annotation is set + if metav1.HasAnnotation(content.ObjectMeta, utils.AnnVolumeSnapshotBeingDeleted) { + return true + } + return false +} diff --git a/pkg/sidecar-controller/snapshot_controller_base.go b/pkg/sidecar-controller/snapshot_controller_base.go new file mode 100644 index 00000000..7612b23c --- /dev/null +++ b/pkg/sidecar-controller/snapshot_controller_base.go @@ -0,0 +1,280 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sidecar_controller + +import ( + "fmt" + "time" + + crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" + storageinformers "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions/volumesnapshot/v1beta1" + storagelisters "github.com/kubernetes-csi/external-snapshotter/pkg/client/listers/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/pkg/snapshotter" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog" + "k8s.io/kubernetes/pkg/util/goroutinemap" +) + +type csiSnapshotSideCarController struct { + clientset clientset.Interface + client kubernetes.Interface + driverName string + eventRecorder record.EventRecorder + contentQueue workqueue.RateLimitingInterface + + contentLister storagelisters.VolumeSnapshotContentLister + contentListerSynced cache.InformerSynced + classLister storagelisters.VolumeSnapshotClassLister + classListerSynced cache.InformerSynced + + contentStore cache.Store + + handler Handler + // Map of scheduled/running operations. + runningOperations goroutinemap.GoRoutineMap + + resyncPeriod time.Duration +} + +// NewCSISnapshotSideCarController returns a new *csiSnapshotSideCarController +func NewCSISnapshotSideCarController( + clientset clientset.Interface, + client kubernetes.Interface, + driverName string, + volumeSnapshotContentInformer storageinformers.VolumeSnapshotContentInformer, + volumeSnapshotClassInformer storageinformers.VolumeSnapshotClassInformer, + snapshotter snapshotter.Snapshotter, + timeout time.Duration, + resyncPeriod time.Duration, + snapshotNamePrefix string, + snapshotNameUUIDLength int, +) *csiSnapshotSideCarController { + broadcaster := record.NewBroadcaster() + broadcaster.StartLogging(klog.Infof) + broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)}) + var eventRecorder record.EventRecorder + eventRecorder = broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: fmt.Sprintf("csi-snapshotter %s", driverName)}) + + ctrl := &csiSnapshotSideCarController{ + clientset: clientset, + client: client, + driverName: driverName, + eventRecorder: eventRecorder, + handler: NewCSIHandler(snapshotter, timeout, snapshotNamePrefix, snapshotNameUUIDLength), + runningOperations: goroutinemap.NewGoRoutineMap(true), + resyncPeriod: resyncPeriod, + contentStore: cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc), + contentQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "csi-snapshotter-content"), + } + + volumeSnapshotContentInformer.Informer().AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { ctrl.enqueueContentWork(obj) }, + UpdateFunc: func(oldObj, newObj interface{}) { ctrl.enqueueContentWork(newObj) }, + DeleteFunc: func(obj interface{}) { ctrl.enqueueContentWork(obj) }, + }, + ctrl.resyncPeriod, + ) + ctrl.contentLister = volumeSnapshotContentInformer.Lister() + ctrl.contentListerSynced = volumeSnapshotContentInformer.Informer().HasSynced + + ctrl.classLister = volumeSnapshotClassInformer.Lister() + ctrl.classListerSynced = volumeSnapshotClassInformer.Informer().HasSynced + + return ctrl +} + +func (ctrl *csiSnapshotSideCarController) Run(workers int, stopCh <-chan struct{}) { + defer ctrl.contentQueue.ShutDown() + + klog.Infof("Starting CSI snapshotter") + defer klog.Infof("Shutting CSI snapshotter") + + if !cache.WaitForCacheSync(stopCh, ctrl.contentListerSynced, ctrl.classListerSynced) { + klog.Errorf("Cannot sync caches") + return + } + + ctrl.initializeCaches(ctrl.contentLister) + + for i := 0; i < workers; i++ { + go wait.Until(ctrl.contentWorker, 0, stopCh) + } + + <-stopCh +} + +// enqueueContentWork adds snapshot content to given work queue. +func (ctrl *csiSnapshotSideCarController) enqueueContentWork(obj interface{}) { + // Beware of "xxx deleted" events + if unknown, ok := obj.(cache.DeletedFinalStateUnknown); ok && unknown.Obj != nil { + obj = unknown.Obj + } + if content, ok := obj.(*crdv1.VolumeSnapshotContent); ok { + objName, err := cache.DeletionHandlingMetaNamespaceKeyFunc(content) + if err != nil { + klog.Errorf("failed to get key from object: %v, %v", err, content) + return + } + klog.V(5).Infof("enqueued %q for sync", objName) + ctrl.contentQueue.Add(objName) + } +} + +// contentWorker processes items from contentQueue. It must run only once, +// syncContent is not assured to be reentrant. +func (ctrl *csiSnapshotSideCarController) contentWorker() { + workFunc := func() bool { + keyObj, quit := ctrl.contentQueue.Get() + if quit { + return true + } + defer ctrl.contentQueue.Done(keyObj) + key := keyObj.(string) + klog.V(5).Infof("contentWorker[%s]", key) + + _, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + klog.V(4).Infof("error getting name of snapshotContent %q to get snapshotContent from informer: %v", key, err) + return false + } + content, err := ctrl.contentLister.Get(name) + // The content still exists in informer cache, the event must have + // been add/update/sync + if err == nil { + if ctrl.isDriverMatch(content) { + ctrl.updateContentInCacheStore(content) + } + return false + } + if !errors.IsNotFound(err) { + klog.V(2).Infof("error getting content %q from informer: %v", key, err) + return false + } + + // The content is not in informer cache, the event must have been + // "delete" + contentObj, found, err := ctrl.contentStore.GetByKey(key) + if err != nil { + klog.V(2).Infof("error getting content %q from cache: %v", key, err) + return false + } + if !found { + // The controller has already processed the delete event and + // deleted the content from its cache + klog.V(2).Infof("deletion of content %q was already processed", key) + return false + } + content, ok := contentObj.(*crdv1.VolumeSnapshotContent) + if !ok { + klog.Errorf("expected content, got %+v", content) + return false + } + ctrl.deleteContentInCacheStore(content) + return false + } + + for { + if quit := workFunc(); quit { + klog.Infof("content worker queue shutting down") + return + } + } +} + +// verify whether the driver specified in VolumeSnapshotContent matches the controller's driver name +func (ctrl *csiSnapshotSideCarController) isDriverMatch(content *crdv1.VolumeSnapshotContent) bool { + if content.Spec.Source.VolumeHandle == nil && content.Spec.Source.SnapshotHandle == nil { + // Skip this snapshot content if it does not have a valid source + return false + } + if content.Spec.Driver != ctrl.driverName { + // Skip this snapshot content if the driver does not match + return false + } + snapshotClassName := content.Spec.VolumeSnapshotClassName + if snapshotClassName != nil { + if snapshotClass, err := ctrl.classLister.Get(*snapshotClassName); err == nil { + if snapshotClass.Driver != ctrl.driverName { + return false + } + } + } + return true +} + +// updateContent runs in worker thread and handles "content added", +// "content updated" and "periodic sync" events. +func (ctrl *csiSnapshotSideCarController) updateContentInCacheStore(content *crdv1.VolumeSnapshotContent) { + // Store the new content version in the cache and do not process it if this is + // an old version. + new, err := ctrl.storeContentUpdate(content) + if err != nil { + klog.Errorf("%v", err) + } + if !new { + return + } + err = ctrl.syncContent(content) + if err != nil { + if errors.IsConflict(err) { + // Version conflict error happens quite often and the controller + // recovers from it easily. + klog.V(3).Infof("could not sync content %q: %+v", content.Name, err) + } else { + klog.Errorf("could not sync content %q: %+v", content.Name, err) + } + } +} + +// deleteContent runs in worker thread and handles "content deleted" event. +func (ctrl *csiSnapshotSideCarController) deleteContentInCacheStore(content *crdv1.VolumeSnapshotContent) { + _ = ctrl.contentStore.Delete(content) + klog.V(4).Infof("content %q deleted", content.Name) +} + +// initializeCaches fills all controller caches with initial data from etcd in +// order to have the caches already filled when first addSnapshot/addContent to +// perform initial synchronization of the controller. +func (ctrl *csiSnapshotSideCarController) initializeCaches(contentLister storagelisters.VolumeSnapshotContentLister) { + contentList, err := contentLister.List(labels.Everything()) + if err != nil { + klog.Errorf("CSISnapshotController can't initialize caches: %v", err) + return + } + for _, content := range contentList { + if ctrl.isDriverMatch(content) { + contentClone := content.DeepCopy() + if _, err = ctrl.storeContentUpdate(contentClone); err != nil { + klog.Errorf("error updating volume snapshot content cache: %v", err) + } + } + } + + klog.V(4).Infof("controller initialized") +} diff --git a/pkg/snapshotter/snapshotter.go b/pkg/snapshotter/snapshotter.go index 157f0897..06814212 100644 --- a/pkg/snapshotter/snapshotter.go +++ b/pkg/snapshotter/snapshotter.go @@ -27,14 +27,13 @@ import ( "google.golang.org/grpc" - "k8s.io/api/core/v1" "k8s.io/klog" ) // Snapshotter implements CreateSnapshot/DeleteSnapshot operations against a remote CSI driver. type Snapshotter interface { // CreateSnapshot creates a snapshot for a volume - CreateSnapshot(ctx context.Context, snapshotName string, volume *v1.PersistentVolume, parameters map[string]string, snapshotterCredentials map[string]string) (driverName string, snapshotId string, timestamp time.Time, size int64, readyToUse bool, err error) + CreateSnapshot(ctx context.Context, snapshotName string, volumeHandle string, parameters map[string]string, snapshotterCredentials map[string]string) (driverName string, snapshotId string, timestamp time.Time, size int64, readyToUse bool, err error) // DeleteSnapshot deletes a snapshot from a volume DeleteSnapshot(ctx context.Context, snapshotID string, snapshotterCredentials map[string]string) (err error) @@ -53,12 +52,8 @@ func NewSnapshotter(conn *grpc.ClientConn) Snapshotter { } } -func (s *snapshot) CreateSnapshot(ctx context.Context, snapshotName string, volume *v1.PersistentVolume, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, error) { +func (s *snapshot) CreateSnapshot(ctx context.Context, snapshotName string, volumeHandle string, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, error) { klog.V(5).Infof("CSI CreateSnapshot: %s", snapshotName) - if volume.Spec.CSI == nil { - return "", "", time.Time{}, 0, false, fmt.Errorf("CSIPersistentVolumeSource not defined in spec") - } - client := csi.NewControllerClient(s.conn) driverName, err := csirpc.GetDriverName(ctx, s.conn) @@ -67,7 +62,7 @@ func (s *snapshot) CreateSnapshot(ctx context.Context, snapshotName string, volu } req := csi.CreateSnapshotRequest{ - SourceVolumeId: volume.Spec.CSI.VolumeHandle, + SourceVolumeId: volumeHandle, Name: snapshotName, Parameters: parameters, Secrets: snapshotterCredentials, @@ -118,6 +113,8 @@ func (s *snapshot) isListSnapshotsSupported(ctx context.Context) (bool, error) { } func (s *snapshot) GetSnapshotStatus(ctx context.Context, snapshotID string) (bool, time.Time, int64, error) { + klog.V(5).Infof("GetSnapshotStatus: %s", snapshotID) + client := csi.NewControllerClient(s.conn) // If the driver does not support ListSnapshots, assume the snapshot ID is valid. diff --git a/pkg/snapshotter/snapshotter_test.go b/pkg/snapshotter/snapshotter_test.go index b0ff50f1..073394b4 100644 --- a/pkg/snapshotter/snapshotter_test.go +++ b/pkg/snapshotter/snapshotter_test.go @@ -79,7 +79,6 @@ func TestCreateSnapshot(t *testing.T) { } csiVolume := FakeCSIVolume() - volumeWithoutCSI := FakeVolume() defaultRequest := &csi.CreateSnapshotRequest{ Name: defaultName, @@ -135,7 +134,7 @@ func TestCreateSnapshot(t *testing.T) { tests := []struct { name string snapshotName string - volume *v1.PersistentVolume + volumeHandle string parameters map[string]string secrets map[string]string input *csi.CreateSnapshotRequest @@ -147,7 +146,7 @@ func TestCreateSnapshot(t *testing.T) { { name: "success", snapshotName: defaultName, - volume: csiVolume, + volumeHandle: csiVolume.Spec.CSI.VolumeHandle, input: defaultRequest, output: defaultResponse, expectError: false, @@ -156,7 +155,7 @@ func TestCreateSnapshot(t *testing.T) { { name: "attributes", snapshotName: defaultName, - volume: csiVolume, + volumeHandle: csiVolume.Spec.CSI.VolumeHandle, parameters: defaultParameter, input: attributesRequest, output: defaultResponse, @@ -166,25 +165,17 @@ func TestCreateSnapshot(t *testing.T) { { name: "secrets", snapshotName: defaultName, - volume: csiVolume, + volumeHandle: csiVolume.Spec.CSI.VolumeHandle, secrets: createSecrets, input: secretsRequest, output: defaultResponse, expectError: false, expectResult: result, }, - { - name: "fail for volume without csi source", - snapshotName: defaultName, - volume: volumeWithoutCSI, - input: nil, - output: nil, - expectError: true, - }, { name: "gRPC transient error", snapshotName: defaultName, - volume: csiVolume, + volumeHandle: csiVolume.Spec.CSI.VolumeHandle, input: defaultRequest, output: nil, injectError: codes.DeadlineExceeded, @@ -193,7 +184,7 @@ func TestCreateSnapshot(t *testing.T) { { name: "gRPC final error", snapshotName: defaultName, - volume: csiVolume, + volumeHandle: csiVolume.Spec.CSI.VolumeHandle, input: defaultRequest, output: nil, injectError: codes.NotFound, @@ -224,7 +215,7 @@ func TestCreateSnapshot(t *testing.T) { } s := NewSnapshotter(csiConn) - driverName, snapshotId, timestamp, size, readyToUse, err := s.CreateSnapshot(context.Background(), test.snapshotName, test.volume, test.parameters, test.secrets) + driverName, snapshotId, timestamp, size, readyToUse, err := s.CreateSnapshot(context.Background(), test.snapshotName, test.volumeHandle, test.parameters, test.secrets) if test.expectError && err == nil { t.Errorf("test %q: Expected error, got none", test.name) } @@ -509,29 +500,3 @@ func FakeCSIVolume() *v1.PersistentVolume { return &volume } - -func FakeVolume() *v1.PersistentVolume { - volume := v1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: "fake-csi-volume", - }, - Spec: v1.PersistentVolumeSpec{ - ClaimRef: &v1.ObjectReference{ - Kind: "PersistentVolumeClaim", - APIVersion: "v1", - UID: types.UID("uid123"), - Namespace: "default", - Name: "test-claim", - }, - PersistentVolumeSource: v1.PersistentVolumeSource{ - GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{}, - }, - StorageClassName: "default", - }, - Status: v1.PersistentVolumeStatus{ - Phase: v1.VolumeBound, - }, - } - - return &volume -} diff --git a/pkg/controller/util.go b/pkg/utils/util.go similarity index 73% rename from pkg/controller/util.go rename to pkg/utils/util.go index 963dd128..50eb5c2d 100644 --- a/pkg/controller/util.go +++ b/pkg/utils/util.go @@ -14,18 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package utils import ( "fmt" "strings" - "os" - "strconv" - "time" - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -34,6 +30,9 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/klog" "k8s.io/kubernetes/pkg/util/slice" + "os" + "strconv" + "time" ) var ( @@ -60,12 +59,27 @@ const ( // [Deprecated] CSI Parameters that are put into fields but // NOT stripped from the parameters passed to CreateSnapshot - snapshotterSecretNameKey = "csiSnapshotterSecretName" - snapshotterSecretNamespaceKey = "csiSnapshotterSecretNamespace" + SnapshotterSecretNameKey = "csiSnapshotterSecretName" + SnapshotterSecretNamespaceKey = "csiSnapshotterSecretNamespace" // Name of finalizer on VolumeSnapshotContents that are bound by VolumeSnapshots - VolumeSnapshotContentFinalizer = "snapshot.storage.kubernetes.io/volumesnapshotcontent-protection" - VolumeSnapshotFinalizer = "snapshot.storage.kubernetes.io/volumesnapshot-protection" + VolumeSnapshotContentFinalizer = "snapshot.storage.kubernetes.io/volumesnapshotcontent-bound-protection" + // Name of finalizer on VolumeSnapshot that is being used as a source to create a PVC + VolumeSnapshotBoundFinalizer = "snapshot.storage.kubernetes.io/volumesnapshot-bound-protection" + // Name of finalizer on VolumeSnapshot that is used as a source to create a PVC + VolumeSnapshotAsSourceFinalizer = "snapshot.storage.kubernetes.io/volumesnapshot-as-source-protection" + // Name of finalizer on PVCs that is being used as a source to create VolumeSnapshots + PVCFinalizer = "snapshot.storage.kubernetes.io/pvc-as-source-protection" + + IsDefaultSnapshotClassAnnotation = "snapshot.storage.kubernetes.io/is-default-class" + + // AnnVolumeSnapshotBeingDeleted annotation applies to VolumeSnapshotContents. + // It indicates that the common snapshot controller has verified that volume + // snapshot has a deletion timestamp and is being deleted. + // Sidecar controller needs to check the deletion policy on the + // VolumeSnapshotContentand and decide whether to delete the volume snapshot + // backing the snapshot content. + AnnVolumeSnapshotBeingDeleted = "snapshot.storage.kubernetes.io/volumesnapshot-being-deleted" // Annotation for secret name and namespace will be added to the content // and used at snapshot content deletion time. @@ -75,20 +89,17 @@ const ( var snapshotterSecretParams = deprecatedSecretParamsMap{ name: "Snapshotter", - deprecatedSecretNameKey: snapshotterSecretNameKey, - deprecatedSecretNamespaceKey: snapshotterSecretNamespaceKey, + deprecatedSecretNameKey: SnapshotterSecretNameKey, + deprecatedSecretNamespaceKey: SnapshotterSecretNamespaceKey, secretNameKey: prefixedSnapshotterSecretNameKey, secretNamespaceKey: prefixedSnapshotterSecretNamespaceKey, } -// Name of finalizer on PVCs that have been used as a source to create VolumeSnapshots -const PVCFinalizer = "snapshot.storage.kubernetes.io/pvc-protection" - -func snapshotKey(vs *crdv1.VolumeSnapshot) string { +func SnapshotKey(vs *crdv1.VolumeSnapshot) string { return fmt.Sprintf("%s/%s", vs.Namespace, vs.Name) } -func snapshotRefKey(vsref v1.ObjectReference) string { +func SnapshotRefKey(vsref *v1.ObjectReference) string { return fmt.Sprintf("%s/%s", vsref.Namespace, vsref.Name) } @@ -96,7 +107,7 @@ func snapshotRefKey(vsref v1.ObjectReference) string { // callback (i.e. with events from etcd) or with an object modified by the // controller itself. Returns "true", if the cache was updated, false if the // object is an old version and should be ignored. -func storeObjectUpdate(store cache.Store, obj interface{}, className string) (bool, error) { +func StoreObjectUpdate(store cache.Store, obj interface{}, className string) (bool, error) { objName, err := keyFunc(obj) if err != nil { return false, fmt.Errorf("Couldn't get key for object %+v: %v", obj, err) @@ -232,7 +243,7 @@ func verifyAndGetSecretNameAndNamespaceTemplate(secret deprecatedSecretParamsMap // - the nameTemplate or namespaceTemplate contains a token that cannot be resolved // - the resolved name is not a valid secret name // - the resolved namespace is not a valid namespace name -func getSecretReference(snapshotClassParams map[string]string, snapContentName string, snapshot *crdv1.VolumeSnapshot) (*v1.SecretReference, error) { +func GetSecretReference(snapshotClassParams map[string]string, snapContentName string, snapshot *crdv1.VolumeSnapshot) (*v1.SecretReference, error) { nameTemplate, namespaceTemplate, err := verifyAndGetSecretNameAndNamespaceTemplate(snapshotterSecretParams, snapshotClassParams) if err != nil { return nil, fmt.Errorf("failed to get name and namespace template from params: %v", err) @@ -244,7 +255,7 @@ func getSecretReference(snapshotClassParams map[string]string, snapContentName s ref := &v1.SecretReference{} - // Secret namespace template can make use of the VolumeSnapshotContent name or the VolumeSnapshot namespace. + // Secret namespace template can make use of the VolumeSnapshotContent name, VolumeSnapshot name or namespace. // Note that neither of those things are under the control of the VolumeSnapshot user. namespaceParams := map[string]string{"volumesnapshotcontent.name": snapContentName} // snapshot may be nil when resolving create/delete snapshot secret names because the @@ -267,7 +278,8 @@ func getSecretReference(snapshotClassParams map[string]string, snapContentName s } ref.Namespace = resolvedNamespace - // Secret name template can make use of the VolumeSnapshotContent name, VolumeSnapshot name or namespace. + // Secret name template can make use of the VolumeSnapshotContent name, VolumeSnapshot name or namespace, + // or a VolumeSnapshot annotation. // Note that VolumeSnapshot name and annotations are under the VolumeSnapshot user's control. nameParams := map[string]string{"volumesnapshotcontent.name": snapContentName} if snapshot != nil { @@ -306,8 +318,8 @@ func resolveTemplate(template string, params map[string]string) (string, error) return resolved, nil } -// getCredentials retrieves credentials stored in v1.SecretReference -func getCredentials(k8s kubernetes.Interface, ref *v1.SecretReference) (map[string]string, error) { +// GetCredentials retrieves credentials stored in v1.SecretReference +func GetCredentials(k8s kubernetes.Interface, ref *v1.SecretReference) (map[string]string, error) { if ref == nil { return nil, nil } @@ -330,23 +342,31 @@ func NoResyncPeriodFunc() time.Duration { } // isContentDeletionCandidate checks if a volume snapshot content is a deletion candidate. -func isContentDeletionCandidate(content *crdv1.VolumeSnapshotContent) bool { +// It is a deletion candidate if DeletionTimestamp is not nil and +// VolumeSnapshotContentFinalizer is set. +func IsContentDeletionCandidate(content *crdv1.VolumeSnapshotContent) bool { return content.ObjectMeta.DeletionTimestamp != nil && slice.ContainsString(content.ObjectMeta.Finalizers, VolumeSnapshotContentFinalizer, nil) } // needToAddContentFinalizer checks if a Finalizer needs to be added for the volume snapshot content. -func needToAddContentFinalizer(content *crdv1.VolumeSnapshotContent) bool { +func NeedToAddContentFinalizer(content *crdv1.VolumeSnapshotContent) bool { return content.ObjectMeta.DeletionTimestamp == nil && !slice.ContainsString(content.ObjectMeta.Finalizers, VolumeSnapshotContentFinalizer, nil) } -// isSnapshotDeletionCandidate checks if a volume snapshot is a deletion candidate. -func isSnapshotDeletionCandidate(snapshot *crdv1.VolumeSnapshot) bool { - return snapshot.ObjectMeta.DeletionTimestamp != nil && slice.ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotFinalizer, nil) +// isSnapshotDeletionCandidate checks if a volume snapshot deletionTimestamp +// is set and any finalizer is on the snapshot. +func IsSnapshotDeletionCandidate(snapshot *crdv1.VolumeSnapshot) bool { + return snapshot.ObjectMeta.DeletionTimestamp != nil && (slice.ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotAsSourceFinalizer, nil) || slice.ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotBoundFinalizer, nil)) } -// needToAddSnapshotFinalizer checks if a Finalizer needs to be added for the volume snapshot. -func needToAddSnapshotFinalizer(snapshot *crdv1.VolumeSnapshot) bool { - return snapshot.ObjectMeta.DeletionTimestamp == nil && !slice.ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotFinalizer, nil) +// needToAddSnapshotAsSourceFinalizer checks if a Finalizer needs to be added for the volume snapshot as a source for PVC. +func NeedToAddSnapshotAsSourceFinalizer(snapshot *crdv1.VolumeSnapshot) bool { + return snapshot.ObjectMeta.DeletionTimestamp == nil && !slice.ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotAsSourceFinalizer, nil) +} + +// needToAddSnapshotBoundFinalizer checks if a Finalizer needs to be added for the bound volume snapshot. +func NeedToAddSnapshotBoundFinalizer(snapshot *crdv1.VolumeSnapshot) bool { + return snapshot.ObjectMeta.DeletionTimestamp == nil && !slice.ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotBoundFinalizer, nil) && snapshot.Status != nil && snapshot.Status.BoundVolumeSnapshotContentName != nil } func deprecationWarning(deprecatedParam, newParam, removalVersion string) string { @@ -360,7 +380,7 @@ func deprecationWarning(deprecatedParam, newParam, removalVersion string) string return fmt.Sprintf("\"%s\" is deprecated and will be removed in %s%s", deprecatedParam, removalVersion, newParamPhrase) } -func removePrefixedParameters(param map[string]string) (map[string]string, error) { +func RemovePrefixedParameters(param map[string]string) (map[string]string, error) { newParam := map[string]string{} for k, v := range param { if strings.HasPrefix(k, csiParameterPrefix) { @@ -379,3 +399,47 @@ func removePrefixedParameters(param map[string]string) (map[string]string, error } return newParam, nil } + +// Stateless functions +func GetSnapshotStatusForLogging(snapshot *crdv1.VolumeSnapshot) string { + snapshotContentName := "" + if snapshot.Status != nil && snapshot.Status.BoundVolumeSnapshotContentName != nil { + snapshotContentName = *snapshot.Status.BoundVolumeSnapshotContentName + } + ready := false + if snapshot.Status != nil && snapshot.Status.ReadyToUse != nil { + ready = *snapshot.Status.ReadyToUse + } + return fmt.Sprintf("bound to: %q, Completed: %v", snapshotContentName, ready) +} + +// IsSnapshotBound returns true/false if snapshot is bound +func IsSnapshotBound(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) bool { + if IsVolumeSnapshotRefSet(snapshot, content) && IsBoundVolumeSnapshotContentNameSet(snapshot) { + return true + } + return false +} + +func IsVolumeSnapshotRefSet(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) bool { + if content.Spec.VolumeSnapshotRef.Name == snapshot.Name && + content.Spec.VolumeSnapshotRef.Namespace == snapshot.Namespace && + content.Spec.VolumeSnapshotRef.UID == snapshot.UID { + return true + } + return false +} + +func IsBoundVolumeSnapshotContentNameSet(snapshot *crdv1.VolumeSnapshot) bool { + if snapshot.Status == nil || snapshot.Status.BoundVolumeSnapshotContentName == nil || *snapshot.Status.BoundVolumeSnapshotContentName == "" { + return false + } + return true +} + +func IsSnapshotReady(snapshot *crdv1.VolumeSnapshot) bool { + if snapshot.Status == nil || snapshot.Status.ReadyToUse == nil || *snapshot.Status.ReadyToUse == false { + return false + } + return true +} diff --git a/pkg/controller/util_test.go b/pkg/utils/util_test.go similarity index 85% rename from pkg/controller/util_test.go rename to pkg/utils/util_test.go index f74fdfaf..bd4b86cb 100644 --- a/pkg/controller/util_test.go +++ b/pkg/utils/util_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package utils import ( crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" @@ -37,11 +37,11 @@ func TestGetSecretReference(t *testing.T) { expectRef: nil, }, "empty err": { - params: map[string]string{snapshotterSecretNameKey: "", snapshotterSecretNamespaceKey: ""}, + params: map[string]string{SnapshotterSecretNameKey: "", SnapshotterSecretNamespaceKey: ""}, expectErr: true, }, "[deprecated] name, no namespace": { - params: map[string]string{snapshotterSecretNameKey: "foo"}, + params: map[string]string{SnapshotterSecretNameKey: "foo"}, expectErr: true, }, "namespace, no name": { @@ -54,7 +54,7 @@ func TestGetSecretReference(t *testing.T) { expectRef: &v1.SecretReference{Name: "name", Namespace: "ns"}, }, "[deprecated] simple - valid, no pvc": { - params: map[string]string{snapshotterSecretNameKey: "name", snapshotterSecretNamespaceKey: "ns"}, + params: map[string]string{SnapshotterSecretNameKey: "name", SnapshotterSecretNamespaceKey: "ns"}, snapshot: nil, expectRef: &v1.SecretReference{Name: "name", Namespace: "ns"}, }, @@ -65,7 +65,25 @@ func TestGetSecretReference(t *testing.T) { expectErr: true, }, "[deprecated] simple - invalid namespace": { - params: map[string]string{snapshotterSecretNameKey: "name", snapshotterSecretNamespaceKey: "bad ns"}, + params: map[string]string{SnapshotterSecretNameKey: "name", SnapshotterSecretNamespaceKey: "bad ns"}, + snapshot: &crdv1.VolumeSnapshot{}, + expectRef: nil, + expectErr: true, + }, + "template - invalid namespace tokens": { + params: map[string]string{ + SnapshotterSecretNameKey: "myname", + SnapshotterSecretNamespaceKey: "mynamespace${bar}", + }, + snapshot: &crdv1.VolumeSnapshot{}, + expectRef: nil, + expectErr: true, + }, + "template - invalid name tokens": { + params: map[string]string{ + SnapshotterSecretNameKey: "myname${foo}", + SnapshotterSecretNamespaceKey: "mynamespace", + }, snapshot: &crdv1.VolumeSnapshot{}, expectRef: nil, expectErr: true, @@ -86,29 +104,11 @@ func TestGetSecretReference(t *testing.T) { expectRef: nil, expectErr: true, }, - "template - invalid namespace tokens": { - params: map[string]string{ - snapshotterSecretNameKey: "myname", - snapshotterSecretNamespaceKey: "mynamespace${bar}", - }, - snapshot: &crdv1.VolumeSnapshot{}, - expectRef: nil, - expectErr: true, - }, - "template - invalid name tokens": { - params: map[string]string{ - snapshotterSecretNameKey: "myname${foo}", - snapshotterSecretNamespaceKey: "mynamespace", - }, - snapshot: &crdv1.VolumeSnapshot{}, - expectRef: nil, - expectErr: true, - }, } for k, tc := range testcases { t.Run(k, func(t *testing.T) { - ref, err := getSecretReference(tc.params, tc.snapContentName, tc.snapshot) + ref, err := GetSecretReference(tc.params, tc.snapContentName, tc.snapshot) if err != nil { if tc.expectErr { return @@ -155,12 +155,12 @@ func TestRemovePrefixedCSIParams(t *testing.T) { { name: "all known deprecated params not stripped", params: map[string]string{ - snapshotterSecretNameKey: "csiBar", - snapshotterSecretNamespaceKey: "csiBar", + SnapshotterSecretNameKey: "csiBar", + SnapshotterSecretNamespaceKey: "csiBar", }, expectedParams: map[string]string{ - snapshotterSecretNameKey: "csiBar", - snapshotterSecretNamespaceKey: "csiBar", + SnapshotterSecretNameKey: "csiBar", + SnapshotterSecretNamespaceKey: "csiBar", }, }, { @@ -176,7 +176,7 @@ func TestRemovePrefixedCSIParams(t *testing.T) { } for _, tc := range testcases { t.Logf("test: %v", tc.name) - newParams, err := removePrefixedParameters(tc.params) + newParams, err := RemovePrefixedParameters(tc.params) if err != nil { if tc.expectErr { continue diff --git a/vendor/modules.txt b/vendor/modules.txt index a92a685a..8a6af7a3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -174,10 +174,10 @@ k8s.io/apimachinery/pkg/watch k8s.io/apimachinery/pkg/types k8s.io/apimachinery/pkg/labels k8s.io/apimachinery/pkg/api/errors +k8s.io/apimachinery/pkg/util/wait k8s.io/apimachinery/pkg/api/meta k8s.io/apimachinery/pkg/util/sets k8s.io/apimachinery/pkg/util/validation -k8s.io/apimachinery/pkg/util/wait k8s.io/apimachinery/pkg/runtime/serializer/streaming k8s.io/apimachinery/pkg/util/net k8s.io/apimachinery/pkg/util/errors @@ -407,8 +407,8 @@ k8s.io/klog k8s.io/kube-openapi/pkg/util/proto # k8s.io/kubernetes v1.14.0 k8s.io/kubernetes/pkg/util/goroutinemap -k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff k8s.io/kubernetes/pkg/util/slice +k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff # k8s.io/utils v0.0.0-20190809000727-6c36bc71fc4a k8s.io/utils/integer k8s.io/utils/buffer From 9000e9c84610e49041c1ed6ea77716ec5596e28d Mon Sep 17 00:00:00 2001 From: xing-yang Date: Fri, 8 Nov 2019 02:46:17 +0000 Subject: [PATCH 2/5] Address review comments --- cmd/csi-snapshotter/main.go | 2 +- cmd/snapshot-controller/main.go | 15 +- cmd/snapshot-controller/main_test.go | 161 ------------------ pkg/common-controller/snapshot_controller.go | 13 +- pkg/common-controller/snapshot_create_test.go | 4 +- pkg/common-controller/snapshot_delete_test.go | 12 +- pkg/sidecar-controller/snapshot_controller.go | 10 +- pkg/utils/util.go | 47 ++--- pkg/utils/util_test.go | 48 ------ 9 files changed, 35 insertions(+), 277 deletions(-) delete mode 100644 cmd/snapshot-controller/main_test.go diff --git a/cmd/csi-snapshotter/main.go b/cmd/csi-snapshotter/main.go index 32ebb412..3fd34d62 100644 --- a/cmd/csi-snapshotter/main.go +++ b/cmd/csi-snapshotter/main.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Kubernetes Authors. +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cmd/snapshot-controller/main.go b/cmd/snapshot-controller/main.go index 9895f350..88d91a7a 100644 --- a/cmd/snapshot-controller/main.go +++ b/cmd/snapshot-controller/main.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Kubernetes Authors. +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -24,17 +24,13 @@ import ( "os/signal" "time" - "google.golang.org/grpc" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/klog" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/kubernetes-csi/csi-lib-utils/leaderelection" - csirpc "github.com/kubernetes-csi/csi-lib-utils/rpc" controller "github.com/kubernetes-csi/external-snapshotter/pkg/common-controller" clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" @@ -148,12 +144,3 @@ func buildConfig(kubeconfig string) (*rest.Config, error) { } return rest.InClusterConfig() } - -func supportsControllerCreateSnapshot(ctx context.Context, conn *grpc.ClientConn) (bool, error) { - capabilities, err := csirpc.GetControllerCapabilities(ctx, conn) - if err != nil { - return false, err - } - - return capabilities[csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT], nil -} diff --git a/cmd/snapshot-controller/main_test.go b/cmd/snapshot-controller/main_test.go deleted file mode 100644 index f13aba72..00000000 --- a/cmd/snapshot-controller/main_test.go +++ /dev/null @@ -1,161 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "context" - "fmt" - "testing" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/golang/mock/gomock" - "github.com/kubernetes-csi/csi-lib-utils/connection" - "github.com/kubernetes-csi/csi-test/driver" - - "google.golang.org/grpc" -) - -func Test_supportsControllerCreateSnapshot(t *testing.T) { - tests := []struct { - name string - output *csi.ControllerGetCapabilitiesResponse - injectError bool - expectError bool - expectResult bool - }{ - { - name: "success", - output: &csi.ControllerGetCapabilitiesResponse{ - Capabilities: []*csi.ControllerServiceCapability{ - { - Type: &csi.ControllerServiceCapability_Rpc{ - Rpc: &csi.ControllerServiceCapability_RPC{ - Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, - }, - }, - }, - { - Type: &csi.ControllerServiceCapability_Rpc{ - Rpc: &csi.ControllerServiceCapability_RPC{ - Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, - }, - }, - }, - }, - }, - expectError: false, - expectResult: true, - }, - { - name: "gRPC error", - output: nil, - injectError: true, - expectError: true, - expectResult: false, - }, - { - name: "no create snapshot", - output: &csi.ControllerGetCapabilitiesResponse{ - Capabilities: []*csi.ControllerServiceCapability{ - { - Type: &csi.ControllerServiceCapability_Rpc{ - Rpc: &csi.ControllerServiceCapability_RPC{ - Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, - }, - }, - }, - }, - }, - expectError: false, - expectResult: false, - }, - { - name: "empty capability", - output: &csi.ControllerGetCapabilitiesResponse{ - Capabilities: []*csi.ControllerServiceCapability{ - { - Type: nil, - }, - }, - }, - expectError: false, - expectResult: false, - }, - { - name: "no capabilities", - output: &csi.ControllerGetCapabilitiesResponse{ - Capabilities: []*csi.ControllerServiceCapability{}, - }, - expectError: false, - expectResult: false, - }, - } - - mockController, driver, _, controllerServer, csiConn, err := createMockServer(t) - if err != nil { - t.Fatal(err) - } - defer mockController.Finish() - defer driver.Stop() - defer csiConn.Close() - - for _, test := range tests { - - in := &csi.ControllerGetCapabilitiesRequest{} - - out := test.output - var injectedErr error - if test.injectError { - injectedErr = fmt.Errorf("mock error") - } - - // Setup expectation - controllerServer.EXPECT().ControllerGetCapabilities(gomock.Any(), in).Return(out, injectedErr).Times(1) - - ok, err := supportsControllerCreateSnapshot(context.Background(), csiConn) - if test.expectError && err == nil { - t.Errorf("test %q: Expected error, got none", test.name) - } - if !test.expectError && err != nil { - t.Errorf("test %q: got error: %v", test.name, err) - } - if err == nil && test.expectResult != ok { - t.Errorf("test fail expected result %t but got %t\n", test.expectResult, ok) - } - } -} - -func createMockServer(t *testing.T) (*gomock.Controller, *driver.MockCSIDriver, *driver.MockIdentityServer, *driver.MockControllerServer, *grpc.ClientConn, error) { - // Start the mock server - mockController := gomock.NewController(t) - identityServer := driver.NewMockIdentityServer(mockController) - controllerServer := driver.NewMockControllerServer(mockController) - drv := driver.NewMockCSIDriver(&driver.MockCSIDriverServers{ - Identity: identityServer, - Controller: controllerServer, - }) - drv.Start() - - // Create a client connection to it - addr := drv.Address() - csiConn, err := connection.Connect(addr) - if err != nil { - return nil, nil, nil, nil, nil, err - } - - return mockController, drv, identityServer, controllerServer, csiConn, nil -} diff --git a/pkg/common-controller/snapshot_controller.go b/pkg/common-controller/snapshot_controller.go index ec5842df..70e059ce 100644 --- a/pkg/common-controller/snapshot_controller.go +++ b/pkg/common-controller/snapshot_controller.go @@ -638,13 +638,14 @@ func (ctrl *csiSnapshotCommonController) updateSnapshotErrorStatusWithEvent(snap return err } + // Emit the event only when the status change happens + ctrl.eventRecorder.Event(newSnapshot, eventtype, reason, message) + _, err = ctrl.storeSnapshotUpdate(newSnapshot) if err != nil { klog.V(4).Infof("updating VolumeSnapshot[%s] error status: cannot update internal cache %v", utils.SnapshotKey(snapshot), err) return err } - // Emit the event only when the status change happens - ctrl.eventRecorder.Event(newSnapshot, eventtype, reason, message) return nil } @@ -722,7 +723,7 @@ func (ctrl *csiSnapshotCommonController) ensurePVCFinalizer(snapshot *crdv1.Volu pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) if err != nil { klog.Infof("cannot get claim from snapshot [%s]: [%v] Claim may be deleted already.", snapshot.Name, err) - return nil + return newControllerUpdateError(snapshot.Name, "cannot get claim from snapshot") } if pvc.ObjectMeta.DeletionTimestamp != nil { @@ -780,7 +781,7 @@ func (ctrl *csiSnapshotCommonController) isPVCBeingUsed(pvc *v1.PersistentVolume klog.V(4).Infof("Skipping static bound snapshot %s when checking PVC %s/%s", snap.Name, pvc.Namespace, pvc.Name) continue } - if snap.Spec.Source.PersistentVolumeClaimName != nil && pvc.Name == *snap.Spec.Source.PersistentVolumeClaimName && (snap.Status == nil || snap.Status.ReadyToUse == nil || (snap.Status.ReadyToUse != nil && *snap.Status.ReadyToUse == false)) { + if snap.Spec.Source.PersistentVolumeClaimName != nil && pvc.Name == *snap.Spec.Source.PersistentVolumeClaimName && !utils.IsSnapshotReady(snap) { klog.V(2).Infof("Keeping PVC %s/%s, it is used by snapshot %s/%s", pvc.Namespace, pvc.Name, snap.Namespace, snap.Name) return true } @@ -1144,6 +1145,10 @@ func (ctrl *csiSnapshotCommonController) addSnapshotFinalizer(snapshot *crdv1.Vo // removeSnapshotFinalizer removes a Finalizer for VolumeSnapshot. func (ctrl *csiSnapshotCommonController) removeSnapshotFinalizer(snapshot *crdv1.VolumeSnapshot, removeSourceFinalizer bool, removeBoundFinalizer bool) error { + if !removeSourceFinalizer && !removeBoundFinalizer { + return nil + } + snapshotClone := snapshot.DeepCopy() if removeSourceFinalizer { snapshotClone.ObjectMeta.Finalizers = slice.RemoveString(snapshotClone.ObjectMeta.Finalizers, utils.VolumeSnapshotAsSourceFinalizer, nil) diff --git a/pkg/common-controller/snapshot_create_test.go b/pkg/common-controller/snapshot_create_test.go index 21602de8..799de7f6 100644 --- a/pkg/common-controller/snapshot_create_test.go +++ b/pkg/common-controller/snapshot_create_test.go @@ -155,7 +155,7 @@ func TestCreateSnapshotSync(t *testing.T) { expectSuccess: false, test: testSyncSnapshot, }, - { + /*{ name: "7-4 - fail create snapshot with no-existing claim", initialContents: nocontents, expectedContents: nocontents, @@ -166,7 +166,7 @@ func TestCreateSnapshotSync(t *testing.T) { errors: noerrors, expectSuccess: false, test: testSyncSnapshot, - }, + },*/ { name: "7-5 - fail create snapshot with no-existing volume", initialContents: nocontents, diff --git a/pkg/common-controller/snapshot_delete_test.go b/pkg/common-controller/snapshot_delete_test.go index f69cb5c8..32f273f6 100644 --- a/pkg/common-controller/snapshot_delete_test.go +++ b/pkg/common-controller/snapshot_delete_test.go @@ -35,18 +35,18 @@ var class2Parameters = map[string]string{ } var class3Parameters = map[string]string{ - "param3": "value3", - utils.SnapshotterSecretNameKey: "name", + "param3": "value3", + //utils.SnapshotterSecretNameKey: "name", } var class4Parameters = map[string]string{ - utils.SnapshotterSecretNameKey: "emptysecret", - utils.SnapshotterSecretNamespaceKey: "default", + //utils.SnapshotterSecretNameKey: "emptysecret", + //utils.SnapshotterSecretNamespaceKey: "default", } var class5Parameters = map[string]string{ - utils.SnapshotterSecretNameKey: "secret", - utils.SnapshotterSecretNamespaceKey: "default", + //utils.SnapshotterSecretNameKey: "secret", + //utils.SnapshotterSecretNamespaceKey: "default", } var snapshotClasses = []*crdv1.VolumeSnapshotClass{ diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index c3199698..d7b2d967 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -219,13 +219,14 @@ func (ctrl *csiSnapshotSideCarController) updateContentErrorStatusWithEvent(cont return err } + // Emit the event only when the status change happens + ctrl.eventRecorder.Event(newContent, eventtype, reason, message) + _, err = ctrl.storeContentUpdate(newContent) if err != nil { klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status: cannot update internal cache %v", content.Name, err) return err } - // Emit the event only when the status change happens - ctrl.eventRecorder.Event(newContent, eventtype, reason, message) return nil } @@ -249,7 +250,6 @@ func (ctrl *csiSnapshotSideCarController) getCSISnapshotInput(content *crdv1.Vol } // For pre-provisioned snapshot, snapshot class is not required klog.V(5).Infof("getCSISnapshotInput for content [%s]: no VolumeSnapshotClassName provided for pre-provisioned snapshot", content.Name) - return nil, nil, nil } // Resolve snapshotting secret credentials. @@ -277,9 +277,7 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatusOperation(c return nil, err } driverName = content.Spec.Driver - if content.Spec.Source.SnapshotHandle != nil { - snapshotID = *content.Spec.Source.SnapshotHandle - } + snapshotID = *content.Spec.Source.SnapshotHandle } else { class, snapshotterCredentials, err := ctrl.getCSISnapshotInput(content) if err != nil { diff --git a/pkg/utils/util.go b/pkg/utils/util.go index 50eb5c2d..b246efc4 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -39,12 +39,10 @@ var ( keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc ) -type deprecatedSecretParamsMap struct { - name string - deprecatedSecretNameKey string - deprecatedSecretNamespaceKey string - secretNameKey string - secretNamespaceKey string +type secretParamsMap struct { + name string + secretNameKey string + secretNamespaceKey string } const ( @@ -57,11 +55,6 @@ const ( prefixedSnapshotterSecretNameKey = csiParameterPrefix + "snapshotter-secret-name" prefixedSnapshotterSecretNamespaceKey = csiParameterPrefix + "snapshotter-secret-namespace" - // [Deprecated] CSI Parameters that are put into fields but - // NOT stripped from the parameters passed to CreateSnapshot - SnapshotterSecretNameKey = "csiSnapshotterSecretName" - SnapshotterSecretNamespaceKey = "csiSnapshotterSecretNamespace" - // Name of finalizer on VolumeSnapshotContents that are bound by VolumeSnapshots VolumeSnapshotContentFinalizer = "snapshot.storage.kubernetes.io/volumesnapshotcontent-bound-protection" // Name of finalizer on VolumeSnapshot that is being used as a source to create a PVC @@ -87,12 +80,10 @@ const ( AnnDeletionSecretRefNamespace = "snapshot.storage.kubernetes.io/deletion-secret-namespace" ) -var snapshotterSecretParams = deprecatedSecretParamsMap{ - name: "Snapshotter", - deprecatedSecretNameKey: SnapshotterSecretNameKey, - deprecatedSecretNamespaceKey: SnapshotterSecretNamespaceKey, - secretNameKey: prefixedSnapshotterSecretNameKey, - secretNamespaceKey: prefixedSnapshotterSecretNamespaceKey, +var snapshotterSecretParams = secretParamsMap{ + name: "Snapshotter", + secretNameKey: prefixedSnapshotterSecretNameKey, + secretNamespaceKey: prefixedSnapshotterSecretNamespaceKey, } func SnapshotKey(vs *crdv1.VolumeSnapshot) string { @@ -183,19 +174,9 @@ func IsDefaultAnnotation(obj metav1.ObjectMeta) bool { // verifyAndGetSecretNameAndNamespaceTemplate gets the values (templates) associated // with the parameters specified in "secret" and verifies that they are specified correctly. -func verifyAndGetSecretNameAndNamespaceTemplate(secret deprecatedSecretParamsMap, snapshotClassParams map[string]string) (nameTemplate, namespaceTemplate string, err error) { +func verifyAndGetSecretNameAndNamespaceTemplate(secret secretParamsMap, snapshotClassParams map[string]string) (nameTemplate, namespaceTemplate string, err error) { numName := 0 numNamespace := 0 - if t, ok := snapshotClassParams[secret.deprecatedSecretNameKey]; ok { - nameTemplate = t - numName++ - klog.Warning(deprecationWarning(secret.deprecatedSecretNameKey, secret.secretNameKey, "")) - } - if t, ok := snapshotClassParams[secret.deprecatedSecretNamespaceKey]; ok { - namespaceTemplate = t - numNamespace++ - klog.Warning(deprecationWarning(secret.deprecatedSecretNamespaceKey, secret.secretNamespaceKey, "")) - } if t, ok := snapshotClassParams[secret.secretNameKey]; ok { nameTemplate = t numName++ @@ -205,10 +186,7 @@ func verifyAndGetSecretNameAndNamespaceTemplate(secret deprecatedSecretParamsMap numNamespace++ } - if numName > 1 || numNamespace > 1 { - // Double specified error - return "", "", fmt.Errorf("%s secrets specified in paramaters with both \"csi\" and \"%s\" keys", secret.name, csiParameterPrefix) - } else if numName != numNamespace { + if numName != numNamespace { // Not both 0 or both 1 return "", "", fmt.Errorf("either name and namespace for %s secrets specified, Both must be specified", secret.name) } else if numName == 1 { @@ -278,9 +256,8 @@ func GetSecretReference(snapshotClassParams map[string]string, snapContentName s } ref.Namespace = resolvedNamespace - // Secret name template can make use of the VolumeSnapshotContent name, VolumeSnapshot name or namespace, - // or a VolumeSnapshot annotation. - // Note that VolumeSnapshot name and annotations are under the VolumeSnapshot user's control. + // Secret name template can make use of the VolumeSnapshotContent name, VolumeSnapshot name or namespace. + // Note that VolumeSnapshot name and namespace are under the VolumeSnapshot user's control. nameParams := map[string]string{"volumesnapshotcontent.name": snapContentName} if snapshot != nil { nameParams["volumesnapshot.name"] = snapshot.Name diff --git a/pkg/utils/util_test.go b/pkg/utils/util_test.go index bd4b86cb..c6bdde88 100644 --- a/pkg/utils/util_test.go +++ b/pkg/utils/util_test.go @@ -36,14 +36,6 @@ func TestGetSecretReference(t *testing.T) { params: nil, expectRef: nil, }, - "empty err": { - params: map[string]string{SnapshotterSecretNameKey: "", SnapshotterSecretNamespaceKey: ""}, - expectErr: true, - }, - "[deprecated] name, no namespace": { - params: map[string]string{SnapshotterSecretNameKey: "foo"}, - expectErr: true, - }, "namespace, no name": { params: map[string]string{prefixedSnapshotterSecretNamespaceKey: "foo"}, expectErr: true, @@ -53,41 +45,12 @@ func TestGetSecretReference(t *testing.T) { snapshot: &crdv1.VolumeSnapshot{}, expectRef: &v1.SecretReference{Name: "name", Namespace: "ns"}, }, - "[deprecated] simple - valid, no pvc": { - params: map[string]string{SnapshotterSecretNameKey: "name", SnapshotterSecretNamespaceKey: "ns"}, - snapshot: nil, - expectRef: &v1.SecretReference{Name: "name", Namespace: "ns"}, - }, "simple - invalid name": { params: map[string]string{prefixedSnapshotterSecretNameKey: "bad name", prefixedSnapshotterSecretNamespaceKey: "ns"}, snapshot: &crdv1.VolumeSnapshot{}, expectRef: nil, expectErr: true, }, - "[deprecated] simple - invalid namespace": { - params: map[string]string{SnapshotterSecretNameKey: "name", SnapshotterSecretNamespaceKey: "bad ns"}, - snapshot: &crdv1.VolumeSnapshot{}, - expectRef: nil, - expectErr: true, - }, - "template - invalid namespace tokens": { - params: map[string]string{ - SnapshotterSecretNameKey: "myname", - SnapshotterSecretNamespaceKey: "mynamespace${bar}", - }, - snapshot: &crdv1.VolumeSnapshot{}, - expectRef: nil, - expectErr: true, - }, - "template - invalid name tokens": { - params: map[string]string{ - SnapshotterSecretNameKey: "myname${foo}", - SnapshotterSecretNamespaceKey: "mynamespace", - }, - snapshot: &crdv1.VolumeSnapshot{}, - expectRef: nil, - expectErr: true, - }, "template - invalid": { params: map[string]string{ prefixedSnapshotterSecretNameKey: "static-${volumesnapshotcontent.name}-${volumesnapshot.namespace}-${volumesnapshot.name}-${volumesnapshot.annotations['akey']}", @@ -152,17 +115,6 @@ func TestRemovePrefixedCSIParams(t *testing.T) { }, expectedParams: map[string]string{}, }, - { - name: "all known deprecated params not stripped", - params: map[string]string{ - SnapshotterSecretNameKey: "csiBar", - SnapshotterSecretNamespaceKey: "csiBar", - }, - expectedParams: map[string]string{ - SnapshotterSecretNameKey: "csiBar", - SnapshotterSecretNamespaceKey: "csiBar", - }, - }, { name: "unknown prefixed var", params: map[string]string{csiParameterPrefix + "bim": "baz"}, From 2ac906d13fbe13fb95607af7f90faceb864b6600 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Thu, 7 Nov 2019 10:35:48 -0800 Subject: [PATCH 3/5] Add sidecar-controller unit tests Signed-off-by: Grant Griffiths --- pkg/sidecar-controller/content_create_test.go | 47 + pkg/sidecar-controller/framework_test.go | 1025 +++++++++++++++++ .../snapshot_controller_test.go | 91 ++ .../snapshot_delete_test.go | 440 +++++++ .../snapshot_finalizer_test.go | 34 + 5 files changed, 1637 insertions(+) create mode 100644 pkg/sidecar-controller/content_create_test.go create mode 100644 pkg/sidecar-controller/framework_test.go create mode 100644 pkg/sidecar-controller/snapshot_controller_test.go create mode 100644 pkg/sidecar-controller/snapshot_delete_test.go create mode 100644 pkg/sidecar-controller/snapshot_finalizer_test.go diff --git a/pkg/sidecar-controller/content_create_test.go b/pkg/sidecar-controller/content_create_test.go new file mode 100644 index 00000000..435fa44f --- /dev/null +++ b/pkg/sidecar-controller/content_create_test.go @@ -0,0 +1,47 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sidecar_controller + +import ( + "testing" + "time" +) + +func TestSyncContent(t *testing.T) { + var tests []controllerTest + + tests = append(tests, controllerTest{ + name: "Basic content create ready to use", + initialContents: newContentArrayWithReadyToUse("content1-1", "snapuid1-1", "snap1-1", "sid1-1", defaultClass, "", "", retainPolicy, nil, &defaultSize, &False, true), + expectedContents: newContentArrayWithReadyToUse("content1-1", "snapuid1-1", "snap1-1", "sid1-1", defaultClass, "", "", retainPolicy, nil, &defaultSize, &True, true), + expectedEvents: noevents, + expectedCreateCalls: []createCall{ + { + snapshotName: "snapshot-snapuid1-1", + driverName: mockDriverName, + snapshotId: "snapuid1-1", + creationTime: timeNow, + readyToUse: true, + }, + }, + expectedListCalls: []listCall{{"sid1-1", true, time.Now(), 1, nil}}, + errors: noerrors, + test: testSyncContent, + }) + + runSyncContentTests(t, tests, snapshotClasses) +} diff --git a/pkg/sidecar-controller/framework_test.go b/pkg/sidecar-controller/framework_test.go new file mode 100644 index 00000000..49ec971e --- /dev/null +++ b/pkg/sidecar-controller/framework_test.go @@ -0,0 +1,1025 @@ +/* +Copyright 2019 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sidecar_controller + +import ( + "context" + "errors" + "fmt" + "reflect" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" + "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/fake" + snapshotscheme "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/scheme" + informers "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions" + storagelisters "github.com/kubernetes-csi/external-snapshotter/pkg/client/listers/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/pkg/utils" + v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + kubefake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/kubernetes/scheme" + core "k8s.io/client-go/testing" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/klog" +) + +// This is a unit test framework for snapshot controller. +// It fills the controller with test snapshots/contents and can simulate these +// scenarios: +// 1) Call syncSnapshot/syncContent once. +// 2) Call syncSnapshot/syncContent several times (both simulating "snapshot/content +// modified" events and periodic sync), until the controller settles down and +// does not modify anything. +// 3) Simulate almost real API server/etcd and call add/update/delete +// content/snapshot. +// In all these scenarios, when the test finishes, the framework can compare +// resulting snapshots/contents with list of expected snapshots/contents and report +// differences. + +// controllerTest contains a single controller test input. +// Each test has initial set of contents and snapshots that are filled into the +// controller before the test starts. The test then contains a reference to +// function to call as the actual test. Available functions are: +// - testSyncSnapshot - calls syncSnapshot on the first snapshot in initialSnapshots. +// - testSyncSnapshotError - calls syncSnapshot on the first snapshot in initialSnapshots +// and expects an error to be returned. +// - testSyncContent - calls syncContent on the first content in initialContents. +// - any custom function for specialized tests. +// The test then contains list of contents/snapshots that are expected at the end +// of the test and list of generated events. +type controllerTest struct { + // Name of the test, for logging + name string + // Initial content of controller content cache. + initialContents []*crdv1.VolumeSnapshotContent + // Expected content of controller content cache at the end of the test. + expectedContents []*crdv1.VolumeSnapshotContent + // Initial content of controller Secret cache. + initialSecrets []*v1.Secret + // Expected events - any event with prefix will pass, we don't check full + // event message. + expectedEvents []string + // Errors to produce on matching action + errors []reactorError + // List of expected CSI Create snapshot calls + expectedCreateCalls []createCall + // List of expected CSI Delete snapshot calls + expectedDeleteCalls []deleteCall + // List of expected CSI list snapshot calls + expectedListCalls []listCall + // Function to call as the test. + test testCall + expectSuccess bool +} + +type testCall func(ctrl *csiSnapshotSideCarController, reactor *snapshotReactor, test controllerTest) error + +const testNamespace = "default" +const mockDriverName = "csi-mock-plugin" + +var errVersionConflict = errors.New("VersionError") +var nocontents []*crdv1.VolumeSnapshotContent +var nosnapshots []*crdv1.VolumeSnapshot +var noevents = []string{} +var noerrors = []reactorError{} + +// snapshotReactor is a core.Reactor that simulates etcd and API server. It +// stores: +// - Latest version of snapshots contents saved by the controller. +// - Queue of all saves (to simulate "content/snapshot updated" events). This queue +// contains all intermediate state of an object - e.g. a snapshot.VolumeName +// is updated first and snapshot.Phase second. This queue will then contain both +// updates as separate entries. +// - Number of changes since the last call to snapshotReactor.syncAll(). +// - Optionally, content and snapshot fake watchers which should be the same ones +// used by the controller. Any time an event function like deleteContentEvent +// is called to simulate an event, the reactor's stores are updated and the +// controller is sent the event via the fake watcher. +// - Optionally, list of error that should be returned by reactor, simulating +// etcd / API server failures. These errors are evaluated in order and every +// error is returned only once. I.e. when the reactor finds matching +// reactorError, it return appropriate error and removes the reactorError from +// the list. +type snapshotReactor struct { + secrets map[string]*v1.Secret + storageClasses map[string]*storagev1.StorageClass + volumes map[string]*v1.PersistentVolume + claims map[string]*v1.PersistentVolumeClaim + contents map[string]*crdv1.VolumeSnapshotContent + snapshots map[string]*crdv1.VolumeSnapshot + changedObjects []interface{} + changedSinceLastSync int + ctrl *csiSnapshotSideCarController + fakeContentWatch *watch.FakeWatcher + fakeSnapshotWatch *watch.FakeWatcher + lock sync.Mutex + errors []reactorError +} + +// reactorError is an error that is returned by test reactor (=simulated +// etcd+/API server) when an action performed by the reactor matches given verb +// ("get", "update", "create", "delete" or "*"") on given resource +// ("volumesnapshotcontents", "volumesnapshots" or "*"). +type reactorError struct { + verb string + resource string + error error +} + +func withContentFinalizer(content *crdv1.VolumeSnapshotContent) *crdv1.VolumeSnapshotContent { + content.ObjectMeta.Finalizers = append(content.ObjectMeta.Finalizers, utils.VolumeSnapshotContentFinalizer) + return content +} + +// React is a callback called by fake kubeClient from the controller. +// In other words, every snapshot/content change performed by the controller ends +// here. +// This callback checks versions of the updated objects and refuse those that +// are too old (simulating real etcd). +// All updated objects are stored locally to keep track of object versions and +// to evaluate test results. +// All updated objects are also inserted into changedObjects queue and +// optionally sent back to the controller via its watchers. +func (r *snapshotReactor) React(action core.Action) (handled bool, ret runtime.Object, err error) { + r.lock.Lock() + defer r.lock.Unlock() + + klog.V(4).Infof("reactor got operation %q on %q", action.GetVerb(), action.GetResource()) + + // Inject error when requested + err = r.injectReactError(action) + if err != nil { + return true, nil, err + } + + // Test did not request to inject an error, continue simulating API server. + switch { + case action.Matches("create", "volumesnapshotcontents"): + obj := action.(core.UpdateAction).GetObject() + content := obj.(*crdv1.VolumeSnapshotContent) + + // check the content does not exist + _, found := r.contents[content.Name] + if found { + return true, nil, fmt.Errorf("cannot create content %s: content already exists", content.Name) + } + + // Store the updated object to appropriate places. + r.contents[content.Name] = content + r.changedObjects = append(r.changedObjects, content) + r.changedSinceLastSync++ + klog.V(5).Infof("created content %s", content.Name) + return true, content, nil + + case action.Matches("update", "volumesnapshotcontents"): + obj := action.(core.UpdateAction).GetObject() + content := obj.(*crdv1.VolumeSnapshotContent) + + // Check and bump object version + storedVolume, found := r.contents[content.Name] + if found { + storedVer, _ := strconv.Atoi(storedVolume.ResourceVersion) + requestedVer, _ := strconv.Atoi(content.ResourceVersion) + if storedVer != requestedVer { + return true, obj, errVersionConflict + } + // Don't modify the existing object + content = content.DeepCopy() + content.ResourceVersion = strconv.Itoa(storedVer + 1) + } else { + return true, nil, fmt.Errorf("cannot update content %s: content not found", content.Name) + } + + // Store the updated object to appropriate places. + r.contents[content.Name] = content + r.changedObjects = append(r.changedObjects, content) + r.changedSinceLastSync++ + klog.V(4).Infof("saved updated content %s", content.Name) + return true, content, nil + + case action.Matches("get", "volumesnapshotcontents"): + name := action.(core.GetAction).GetName() + content, found := r.contents[name] + if found { + klog.V(4).Infof("GetVolume: found %s", content.Name) + return true, content, nil + } + klog.V(4).Infof("GetVolume: content %s not found", name) + return true, nil, fmt.Errorf("cannot find content %s", name) + + case action.Matches("delete", "volumesnapshotcontents"): + name := action.(core.DeleteAction).GetName() + klog.V(4).Infof("deleted content %s", name) + _, found := r.contents[name] + if found { + delete(r.contents, name) + r.changedSinceLastSync++ + return true, nil, nil + } + return true, nil, fmt.Errorf("cannot delete content %s: not found", name) + + case action.Matches("get", "secrets"): + name := action.(core.GetAction).GetName() + secret, found := r.secrets[name] + if found { + klog.V(4).Infof("GetSecret: found %s", secret.Name) + return true, secret, nil + } + klog.V(4).Infof("GetSecret: secret %s not found", name) + return true, nil, fmt.Errorf("cannot find secret %s", name) + + } + + return false, nil, nil +} + +// injectReactError returns an error when the test requested given action to +// fail. nil is returned otherwise. +func (r *snapshotReactor) injectReactError(action core.Action) error { + if len(r.errors) == 0 { + // No more errors to inject, everything should succeed. + return nil + } + + for i, expected := range r.errors { + klog.V(4).Infof("trying to match %q %q with %q %q", expected.verb, expected.resource, action.GetVerb(), action.GetResource()) + if action.Matches(expected.verb, expected.resource) { + // That's the action we're waiting for, remove it from injectedErrors + r.errors = append(r.errors[:i], r.errors[i+1:]...) + klog.V(4).Infof("reactor found matching error at index %d: %q %q, returning %v", i, expected.verb, expected.resource, expected.error) + return expected.error + } + } + return nil +} + +// checkContents compares all expectedContents with set of contents at the end of +// the test and reports differences. +func (r *snapshotReactor) checkContents(expectedContents []*crdv1.VolumeSnapshotContent) error { + r.lock.Lock() + defer r.lock.Unlock() + + expectedMap := make(map[string]*crdv1.VolumeSnapshotContent) + gotMap := make(map[string]*crdv1.VolumeSnapshotContent) + // Clear any ResourceVersion from both sets + for _, v := range expectedContents { + // Don't modify the existing object + v := v.DeepCopy() + v.ResourceVersion = "" + v.Spec.VolumeSnapshotRef.ResourceVersion = "" + v.Status.CreationTime = nil + expectedMap[v.Name] = v + } + for _, v := range r.contents { + // We must clone the content because of golang race check - it was + // written by the controller without any locks on it. + v := v.DeepCopy() + v.ResourceVersion = "" + v.Spec.VolumeSnapshotRef.ResourceVersion = "" + v.Status.CreationTime = nil + gotMap[v.Name] = v + } + if !reflect.DeepEqual(expectedMap, gotMap) { + // Print ugly but useful diff of expected and received objects for + // easier debugging. + return fmt.Errorf("content check failed [A-expected, B-got]: %s", diff.ObjectDiff(expectedMap, gotMap)) + } + return nil +} + +// checkSnapshots compares all expectedSnapshots with set of snapshots at the end of the +// test and reports differences. +func (r *snapshotReactor) checkSnapshots(expectedSnapshots []*crdv1.VolumeSnapshot) error { + r.lock.Lock() + defer r.lock.Unlock() + + expectedMap := make(map[string]*crdv1.VolumeSnapshot) + gotMap := make(map[string]*crdv1.VolumeSnapshot) + for _, c := range expectedSnapshots { + // Don't modify the existing object + c = c.DeepCopy() + c.ResourceVersion = "" + if c.Status.Error != nil { + c.Status.Error.Time = &metav1.Time{} + } + expectedMap[c.Name] = c + } + for _, c := range r.snapshots { + // We must clone the snapshot because of golang race check - it was + // written by the controller without any locks on it. + c = c.DeepCopy() + c.ResourceVersion = "" + if c.Status.Error != nil { + c.Status.Error.Time = &metav1.Time{} + } + gotMap[c.Name] = c + } + if !reflect.DeepEqual(expectedMap, gotMap) { + // Print ugly but useful diff of expected and received objects for + // easier debugging. + return fmt.Errorf("snapshot check failed [A-expected, B-got result]: %s", diff.ObjectDiff(expectedMap, gotMap)) + } + return nil +} + +// checkEvents compares all expectedEvents with events generated during the test +// and reports differences. +func checkEvents(t *testing.T, expectedEvents []string, ctrl *csiSnapshotSideCarController) error { + var err error + + // Read recorded events - wait up to 1 minute to get all the expected ones + // (just in case some goroutines are slower with writing) + timer := time.NewTimer(time.Minute) + defer timer.Stop() + + fakeRecorder := ctrl.eventRecorder.(*record.FakeRecorder) + gotEvents := []string{} + finished := false + for len(gotEvents) < len(expectedEvents) && !finished { + select { + case event, ok := <-fakeRecorder.Events: + if ok { + klog.V(5).Infof("event recorder got event %s", event) + gotEvents = append(gotEvents, event) + } else { + klog.V(5).Infof("event recorder finished") + finished = true + } + case _, _ = <-timer.C: + klog.V(5).Infof("event recorder timeout") + finished = true + } + } + + // Evaluate the events + for i, expected := range expectedEvents { + if len(gotEvents) <= i { + t.Errorf("Event %q not emitted", expected) + err = fmt.Errorf("Events do not match") + continue + } + received := gotEvents[i] + if !strings.HasPrefix(received, expected) { + t.Errorf("Unexpected event received, expected %q, got %q", expected, received) + err = fmt.Errorf("Events do not match") + } + } + for i := len(expectedEvents); i < len(gotEvents); i++ { + t.Errorf("Unexpected event received: %q", gotEvents[i]) + err = fmt.Errorf("Events do not match") + } + return err +} + +// popChange returns one recorded updated object, either *crdv1.VolumeSnapshotContent +// or *crdv1.VolumeSnapshot. Returns nil when there are no changes. +func (r *snapshotReactor) popChange() interface{} { + r.lock.Lock() + defer r.lock.Unlock() + + if len(r.changedObjects) == 0 { + return nil + } + + // For debugging purposes, print the queue + for _, obj := range r.changedObjects { + switch obj.(type) { + case *crdv1.VolumeSnapshotContent: + vol, _ := obj.(*crdv1.VolumeSnapshotContent) + klog.V(4).Infof("reactor queue: %s", vol.Name) + case *crdv1.VolumeSnapshot: + snapshot, _ := obj.(*crdv1.VolumeSnapshot) + klog.V(4).Infof("reactor queue: %s", snapshot.Name) + } + } + + // Pop the first item from the queue and return it + obj := r.changedObjects[0] + r.changedObjects = r.changedObjects[1:] + return obj +} + +// syncAll simulates the controller periodic sync of contents and snapshot. It +// simply adds all these objects to the internal queue of updates. This method +// should be used when the test manually calls syncSnapshot/syncContent. Test that +// use real controller loop (ctrl.Run()) will get periodic sync automatically. +func (r *snapshotReactor) syncAll() { + r.lock.Lock() + defer r.lock.Unlock() + + for _, c := range r.snapshots { + r.changedObjects = append(r.changedObjects, c) + } + for _, v := range r.contents { + r.changedObjects = append(r.changedObjects, v) + } + for _, pvc := range r.claims { + r.changedObjects = append(r.changedObjects, pvc) + } + r.changedSinceLastSync = 0 +} + +func (r *snapshotReactor) getChangeCount() int { + r.lock.Lock() + defer r.lock.Unlock() + return r.changedSinceLastSync +} + +// waitForIdle waits until all tests, controllers and other goroutines do their +// job and no new actions are registered for 10 milliseconds. +func (r *snapshotReactor) waitForIdle() { + r.ctrl.runningOperations.WaitForCompletion() + // Check every 10ms if the controller does something and stop if it's + // idle. + oldChanges := -1 + for { + time.Sleep(10 * time.Millisecond) + changes := r.getChangeCount() + if changes == oldChanges { + // No changes for last 10ms -> controller must be idle. + break + } + oldChanges = changes + } +} + +// waitTest waits until all tests, controllers and other goroutines do their +// job and list of current contents/snapshots is equal to list of expected +// contents/snapshots (with ~10 second timeout). +func (r *snapshotReactor) waitTest(test controllerTest) error { + // start with 10 ms, multiply by 2 each step, 10 steps = 10.23 seconds + backoff := wait.Backoff{ + Duration: 10 * time.Millisecond, + Jitter: 0, + Factor: 2, + Steps: 10, + } + err := wait.ExponentialBackoff(backoff, func() (done bool, err error) { + // Finish all operations that are in progress + r.ctrl.runningOperations.WaitForCompletion() + + // Return 'true' if the reactor reached the expected state + err1 := r.checkContents(test.expectedContents) + if err1 == nil { + return true, nil + } + return false, nil + }) + return err +} + +// deleteContentEvent simulates that a content has been deleted in etcd and +// the controller receives 'content deleted' event. +func (r *snapshotReactor) deleteContentEvent(content *crdv1.VolumeSnapshotContent) { + r.lock.Lock() + defer r.lock.Unlock() + + // Remove the content from list of resulting contents. + delete(r.contents, content.Name) + + // Generate deletion event. Cloned content is needed to prevent races (and we + // would get a clone from etcd too). + if r.fakeContentWatch != nil { + r.fakeContentWatch.Delete(content.DeepCopy()) + } +} + +// deleteSnapshotEvent simulates that a snapshot has been deleted in etcd and the +// controller receives 'snapshot deleted' event. +func (r *snapshotReactor) deleteSnapshotEvent(snapshot *crdv1.VolumeSnapshot) { + r.lock.Lock() + defer r.lock.Unlock() + + // Remove the snapshot from list of resulting snapshots. + delete(r.snapshots, snapshot.Name) + + // Generate deletion event. Cloned content is needed to prevent races (and we + // would get a clone from etcd too). + if r.fakeSnapshotWatch != nil { + r.fakeSnapshotWatch.Delete(snapshot.DeepCopy()) + } +} + +// addContentEvent simulates that a content has been added in etcd and the +// controller receives 'content added' event. +func (r *snapshotReactor) addContentEvent(content *crdv1.VolumeSnapshotContent) { + r.lock.Lock() + defer r.lock.Unlock() + + r.contents[content.Name] = content + // Generate event. No cloning is needed, this snapshot is not stored in the + // controller cache yet. + if r.fakeContentWatch != nil { + r.fakeContentWatch.Add(content) + } +} + +// modifyContentEvent simulates that a content has been modified in etcd and the +// controller receives 'content modified' event. +func (r *snapshotReactor) modifyContentEvent(content *crdv1.VolumeSnapshotContent) { + r.lock.Lock() + defer r.lock.Unlock() + + r.contents[content.Name] = content + // Generate deletion event. Cloned content is needed to prevent races (and we + // would get a clone from etcd too). + if r.fakeContentWatch != nil { + r.fakeContentWatch.Modify(content.DeepCopy()) + } +} + +// addSnapshotEvent simulates that a snapshot has been deleted in etcd and the +// controller receives 'snapshot added' event. +func (r *snapshotReactor) addSnapshotEvent(snapshot *crdv1.VolumeSnapshot) { + r.lock.Lock() + defer r.lock.Unlock() + + r.snapshots[snapshot.Name] = snapshot + // Generate event. No cloning is needed, this snapshot is not stored in the + // controller cache yet. + if r.fakeSnapshotWatch != nil { + r.fakeSnapshotWatch.Add(snapshot) + } +} + +func newSnapshotReactor(kubeClient *kubefake.Clientset, client *fake.Clientset, ctrl *csiSnapshotSideCarController, fakeVolumeWatch, fakeClaimWatch *watch.FakeWatcher, errors []reactorError) *snapshotReactor { + reactor := &snapshotReactor{ + secrets: make(map[string]*v1.Secret), + contents: make(map[string]*crdv1.VolumeSnapshotContent), + ctrl: ctrl, + fakeContentWatch: fakeVolumeWatch, + errors: errors, + } + + client.AddReactor("create", "volumesnapshotcontents", reactor.React) + client.AddReactor("update", "volumesnapshotcontents", reactor.React) + client.AddReactor("update", "volumesnapshots", reactor.React) + client.AddReactor("get", "volumesnapshotcontents", reactor.React) + client.AddReactor("get", "volumesnapshots", reactor.React) + client.AddReactor("delete", "volumesnapshotcontents", reactor.React) + client.AddReactor("delete", "volumesnapshots", reactor.React) + + return reactor +} + +func alwaysReady() bool { return true } + +func newTestController(kubeClient kubernetes.Interface, clientset clientset.Interface, + informerFactory informers.SharedInformerFactory, t *testing.T, test controllerTest) (*csiSnapshotSideCarController, error) { + if informerFactory == nil { + informerFactory = informers.NewSharedInformerFactory(clientset, utils.NoResyncPeriodFunc()) + } + + // Construct controller + fakeSnapshot := &fakeSnapshotter{ + t: t, + listCalls: test.expectedListCalls, + createCalls: test.expectedCreateCalls, + deleteCalls: test.expectedDeleteCalls, + } + + ctrl := NewCSISnapshotSideCarController( + clientset, + kubeClient, + mockDriverName, + informerFactory.Snapshot().V1beta1().VolumeSnapshotContents(), + informerFactory.Snapshot().V1beta1().VolumeSnapshotClasses(), + fakeSnapshot, + 5*time.Millisecond, + 60*time.Second, + "snapshot", + -1, + ) + + ctrl.eventRecorder = record.NewFakeRecorder(1000) + + ctrl.contentListerSynced = alwaysReady + ctrl.classListerSynced = alwaysReady + + return ctrl, nil +} + +func newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle string, + deletionPolicy crdv1.DeletionPolicy, creationTime, size *int64, + withFinalizer bool, deletionTime *metav1.Time) *crdv1.VolumeSnapshotContent { + var annotations map[string]string + + content := crdv1.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: contentName, + ResourceVersion: "1", + DeletionTimestamp: deletionTime, + Annotations: annotations, + }, + Spec: crdv1.VolumeSnapshotContentSpec{ + Driver: mockDriverName, + DeletionPolicy: deletionPolicy, + Source: crdv1.VolumeSnapshotContentSource{ + SnapshotHandle: &snapshotHandle, + VolumeHandle: &volumeHandle, + }, + }, + Status: &crdv1.VolumeSnapshotContentStatus{ + CreationTime: creationTime, + RestoreSize: size, + }, + } + if deletionTime != nil { + metav1.SetMetaDataAnnotation(&content.ObjectMeta, utils.AnnVolumeSnapshotBeingDeleted, "yes") + } + + if snapshotHandle != "" { + content.Status.SnapshotHandle = &snapshotHandle + } + + if snapshotClassName != "" { + content.Spec.VolumeSnapshotClassName = &snapshotClassName + } + + if volumeHandle != "" { + content.Spec.Source = crdv1.VolumeSnapshotContentSource{ + VolumeHandle: &volumeHandle, + } + } else if desiredSnapshotHandle != "" { + content.Spec.Source = crdv1.VolumeSnapshotContentSource{ + SnapshotHandle: &desiredSnapshotHandle, + } + } + + if boundToSnapshotName != "" { + content.Spec.VolumeSnapshotRef = v1.ObjectReference{ + Kind: "VolumeSnapshot", + APIVersion: "snapshot.storage.k8s.io/v1beta1", + UID: types.UID(boundToSnapshotUID), + Namespace: testNamespace, + Name: boundToSnapshotName, + } + } + + if withFinalizer { + return withContentFinalizer(&content) + } + return &content +} + +func newContentArray(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle string, + deletionPolicy crdv1.DeletionPolicy, size, creationTime *int64, + withFinalizer bool) []*crdv1.VolumeSnapshotContent { + return []*crdv1.VolumeSnapshotContent{ + newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, creationTime, size, withFinalizer, nil), + } +} + +func newContentArrayWithReadyToUse(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle string, + deletionPolicy crdv1.DeletionPolicy, creationTime, size *int64, readyToUse *bool, + withFinalizer bool) []*crdv1.VolumeSnapshotContent { + content := newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, creationTime, size, withFinalizer, nil) + content.Status.ReadyToUse = readyToUse + return []*crdv1.VolumeSnapshotContent{ + content, + } +} + +func newContentWithUnmatchDriverArray(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle string, + deletionPolicy crdv1.DeletionPolicy, size, creationTime *int64, + withFinalizer bool) []*crdv1.VolumeSnapshotContent { + content := newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, size, creationTime, withFinalizer, nil) + content.Spec.Driver = "fake" + return []*crdv1.VolumeSnapshotContent{ + content, + } +} + +func newContentArrayWithDeletionTimestamp(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle string, + deletionPolicy crdv1.DeletionPolicy, size, creationTime *int64, + withFinalizer bool, deletionTime *metav1.Time) []*crdv1.VolumeSnapshotContent { + return []*crdv1.VolumeSnapshotContent{ + newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, creationTime, size, withFinalizer, deletionTime), + } +} + +func testSyncContent(ctrl *csiSnapshotSideCarController, reactor *snapshotReactor, test controllerTest) error { + return ctrl.syncContent(test.initialContents[0]) +} +func testSyncContentError(ctrl *csiSnapshotSideCarController, reactor *snapshotReactor, test controllerTest) error { + err := ctrl.syncContent(test.initialContents[0]) + if err != nil { + return nil + } + return fmt.Errorf("syncSnapshotContent succeeded when failure was expected") +} + +var ( + classEmpty string + classGold = "gold" + classSilver = "silver" + classNonExisting = "non-existing" + defaultClass = "default-class" + emptySecretClass = "empty-secret-class" + invalidSecretClass = "invalid-secret-class" + validSecretClass = "valid-secret-class" + sameDriver = "sameDriver" + diffDriver = "diffDriver" + noClaim = "" + noBoundUID = "" + noVolume = "" +) + +// wrapTestWithInjectedOperation returns a testCall that: +// - starts the controller and lets it run original testCall until +// scheduleOperation() call. It blocks the controller there and calls the +// injected function to simulate that something is happening when the +// controller waits for the operation lock. Controller is then resumed and we +// check how it behaves. +func wrapTestWithInjectedOperation(toWrap testCall, injectBeforeOperation func(ctrl *csiSnapshotSideCarController, reactor *snapshotReactor)) testCall { + + return func(ctrl *csiSnapshotSideCarController, reactor *snapshotReactor, test controllerTest) error { + // Inject a hook before async operation starts + klog.V(4).Infof("reactor:injecting call") + injectBeforeOperation(ctrl, reactor) + + // Run the tested function (typically syncSnapshot/syncContent) in a + // separate goroutine. + var testError error + var testFinished int32 + + go func() { + testError = toWrap(ctrl, reactor, test) + // Let the "main" test function know that syncContent has finished. + atomic.StoreInt32(&testFinished, 1) + }() + + // Wait for the controller to finish the test function. + for atomic.LoadInt32(&testFinished) == 0 { + time.Sleep(time.Millisecond * 10) + } + + return testError + } +} + +func evaluateTestResults(ctrl *csiSnapshotSideCarController, reactor *snapshotReactor, test controllerTest, t *testing.T) { + // Evaluate results + if test.expectedContents != nil { + if err := reactor.checkContents(test.expectedContents); err != nil { + t.Errorf("Test %q: %v", test.name, err) + } + } + + if err := checkEvents(t, test.expectedEvents, ctrl); err != nil { + t.Errorf("Test %q: %v", test.name, err) + } +} + +// Test single call to syncSnapshot and syncContent methods. +// For all tests: +// 1. Fill in the controller with initial data +// 2. Call the tested function (syncContent) via +// controllerTest.testCall *once*. +// 3. Compare resulting contents and snapshots with expected contents and snapshots. +func runSyncContentTests(t *testing.T, tests []controllerTest, snapshotClasses []*crdv1.VolumeSnapshotClass) { + snapshotscheme.AddToScheme(scheme.Scheme) + for _, test := range tests { + klog.V(4).Infof("starting test %q", test.name) + + // Initialize the controller + kubeClient := &kubefake.Clientset{} + client := &fake.Clientset{} + + ctrl, err := newTestController(kubeClient, client, nil, t, test) + if err != nil { + t.Fatalf("Test %q construct persistent content failed: %v", test.name, err) + } + + reactor := newSnapshotReactor(kubeClient, client, ctrl, nil, nil, test.errors) + for _, content := range test.initialContents { + if ctrl.isDriverMatch(test.initialContents[0]) { + ctrl.contentStore.Add(content) + reactor.contents[content.Name] = content + } + } + + for _, secret := range test.initialSecrets { + reactor.secrets[secret.Name] = secret + } + + // Inject classes into controller via a custom lister. + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + for _, class := range snapshotClasses { + indexer.Add(class) + } + ctrl.classLister = storagelisters.NewVolumeSnapshotClassLister(indexer) + + // Run the tested functions + err = test.test(ctrl, reactor, test) + if err != nil { + t.Errorf("Test %q failed: %v", test.name, err) + } + + // Wait for the target state + err = reactor.waitTest(test) + if err != nil { + t.Errorf("Test %q failed: %v", test.name, err) + } + + evaluateTestResults(ctrl, reactor, test, t) + } +} + +func getSize(size int64) *resource.Quantity { + return resource.NewQuantity(size, resource.BinarySI) +} + +func emptySecret() *v1.Secret { + return &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "emptysecret", + Namespace: "default", + }, + } +} + +func secret() *v1.Secret { + return &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret", + Namespace: "default", + }, + Data: map[string][]byte{ + "foo": []byte("bar"), + }, + } +} + +func secretAnnotations() map[string]string { + return map[string]string{ + utils.AnnDeletionSecretRefName: "secret", + utils.AnnDeletionSecretRefNamespace: "default", + } +} + +func emptyNamespaceSecretAnnotations() map[string]string { + return map[string]string{ + utils.AnnDeletionSecretRefName: "name", + utils.AnnDeletionSecretRefNamespace: "", + } +} + +// this refers to emptySecret(), which is missing data. +func emptyDataSecretAnnotations() map[string]string { + return map[string]string{ + utils.AnnDeletionSecretRefName: "emptysecret", + utils.AnnDeletionSecretRefNamespace: "default", + } +} + +type listCall struct { + snapshotID string + // information to return + readyToUse bool + createTime time.Time + size int64 + err error +} + +type deleteCall struct { + snapshotID string + secrets map[string]string + err error +} + +type createCall struct { + // expected request parameter + snapshotName string + volumeHandle string + parameters map[string]string + secrets map[string]string + // information to return + driverName string + snapshotId string + creationTime time.Time + size int64 + readyToUse bool + err error +} + +// Fake SnapShotter implementation that check that Attach/Detach is called +// with the right parameters and it returns proper error code and metadata. +type fakeSnapshotter struct { + createCalls []createCall + createCallCounter int + deleteCalls []deleteCall + deleteCallCounter int + listCalls []listCall + listCallCounter int + t *testing.T +} + +func (f *fakeSnapshotter) CreateSnapshot(ctx context.Context, snapshotName string, volumeHandle string, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, error) { + if f.createCallCounter >= len(f.createCalls) { + f.t.Errorf("Unexpected CSI Create Snapshot call: snapshotName=%s, volumeHandle=%v, index: %d, calls: %+v", snapshotName, volumeHandle, f.createCallCounter, f.createCalls) + return "", "", time.Time{}, 0, false, fmt.Errorf("unexpected call") + } + call := f.createCalls[f.createCallCounter] + f.createCallCounter++ + + var err error + if call.snapshotName != snapshotName { + f.t.Errorf("Wrong CSI Create Snapshot call: snapshotName=%s, volumeHandle=%s, expected snapshotName: %s", snapshotName, volumeHandle, call.snapshotName) + err = fmt.Errorf("unexpected create snapshot call") + } + + if call.volumeHandle != volumeHandle { + f.t.Errorf("Wrong CSI Create Snapshot call: snapshotName=%s, volumeHandle=%s, expected volumeHandle: %s", snapshotName, volumeHandle, call.volumeHandle) + err = fmt.Errorf("unexpected create snapshot call") + } + + if !reflect.DeepEqual(call.parameters, parameters) && !(len(call.parameters) == 0 && len(parameters) == 0) { + f.t.Errorf("Wrong CSI Create Snapshot call: snapshotName=%s, volumeHandle=%s, expected parameters %+v, got %+v", snapshotName, volumeHandle, call.parameters, parameters) + err = fmt.Errorf("unexpected create snapshot call") + } + + if !reflect.DeepEqual(call.secrets, snapshotterCredentials) { + f.t.Errorf("Wrong CSI Create Snapshot call: snapshotName=%s, volumeHandle=%s, expected secrets %+v, got %+v", snapshotName, volumeHandle, call.secrets, snapshotterCredentials) + err = fmt.Errorf("unexpected create snapshot call") + } + + if err != nil { + return "", "", time.Time{}, 0, false, fmt.Errorf("unexpected call") + } + return call.driverName, call.snapshotId, call.creationTime, call.size, call.readyToUse, call.err +} + +func (f *fakeSnapshotter) DeleteSnapshot(ctx context.Context, snapshotID string, snapshotterCredentials map[string]string) error { + if f.deleteCallCounter >= len(f.deleteCalls) { + f.t.Errorf("Unexpected CSI Delete Snapshot call: snapshotID=%s, index: %d, calls: %+v", snapshotID, f.createCallCounter, f.createCalls) + return fmt.Errorf("unexpected DeleteSnapshot call") + } + call := f.deleteCalls[f.deleteCallCounter] + f.deleteCallCounter++ + + var err error + if call.snapshotID != snapshotID { + f.t.Errorf("Wrong CSI Create Snapshot call: snapshotID=%s, expected snapshotID: %s", snapshotID, call.snapshotID) + err = fmt.Errorf("unexpected Delete snapshot call") + } + + if !reflect.DeepEqual(call.secrets, snapshotterCredentials) { + f.t.Errorf("Wrong CSI Delete Snapshot call: snapshotID=%s, expected secrets %+v, got %+v", snapshotID, call.secrets, snapshotterCredentials) + err = fmt.Errorf("unexpected Delete Snapshot call") + } + + if err != nil { + return fmt.Errorf("unexpected call") + } + + return call.err +} + +func (f *fakeSnapshotter) GetSnapshotStatus(ctx context.Context, snapshotID string) (bool, time.Time, int64, error) { + if f.listCallCounter >= len(f.listCalls) { + f.t.Errorf("Unexpected CSI list Snapshot call: snapshotID=%s, index: %d, calls: %+v", snapshotID, f.createCallCounter, f.createCalls) + return false, time.Time{}, 0, fmt.Errorf("unexpected call") + } + call := f.listCalls[f.listCallCounter] + f.listCallCounter++ + + var err error + if call.snapshotID != snapshotID { + f.t.Errorf("Wrong CSI List Snapshot call: snapshotID=%s, expected snapshotID: %s", snapshotID, call.snapshotID) + err = fmt.Errorf("unexpected List snapshot call") + } + + if err != nil { + return false, time.Time{}, 0, fmt.Errorf("unexpected call") + } + + return call.readyToUse, call.createTime, call.size, call.err +} diff --git a/pkg/sidecar-controller/snapshot_controller_test.go b/pkg/sidecar-controller/snapshot_controller_test.go new file mode 100644 index 00000000..c4d598e9 --- /dev/null +++ b/pkg/sidecar-controller/snapshot_controller_test.go @@ -0,0 +1,91 @@ +/* +Copyright 2019 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sidecar_controller + +import ( + "testing" + + crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/pkg/utils" + "k8s.io/client-go/tools/cache" +) + +var deletionPolicy = crdv1.VolumeSnapshotContentDelete + +func storeVersion(t *testing.T, prefix string, c cache.Store, version string, expectedReturn bool) { + content := newContent("contentName", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "pv-handle-1-1", deletionPolicy, nil, nil, false, nil) + content.ResourceVersion = version + ret, err := utils.StoreObjectUpdate(c, content, "content") + if err != nil { + t.Errorf("%s: expected storeObjectUpdate to succeed, got: %v", prefix, err) + } + if expectedReturn != ret { + t.Errorf("%s: expected storeObjectUpdate to return %v, got: %v", prefix, expectedReturn, ret) + } + + // find the stored version + + contentObj, found, err := c.GetByKey("contentName") + if err != nil { + t.Errorf("expected content 'contentName' in the cache, got error instead: %v", err) + } + if !found { + t.Errorf("expected content 'contentName' in the cache but it was not found") + } + content, ok := contentObj.(*crdv1.VolumeSnapshotContent) + if !ok { + t.Errorf("expected content in the cache, got different object instead: %#v", contentObj) + } + + if ret { + if content.ResourceVersion != version { + t.Errorf("expected content with version %s in the cache, got %s instead", version, content.ResourceVersion) + } + } else { + if content.ResourceVersion == version { + t.Errorf("expected content with version other than %s in the cache, got %s instead", version, content.ResourceVersion) + } + } +} + +// TestControllerCache tests func storeObjectUpdate() +func TestControllerCache(t *testing.T) { + // Cache under test + c := cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc) + + // Store new PV + storeVersion(t, "Step1", c, "1", true) + // Store the same PV + storeVersion(t, "Step2", c, "1", true) + // Store newer PV + storeVersion(t, "Step3", c, "2", true) + // Store older PV - simulating old "PV updated" event or periodic sync with + // old data + storeVersion(t, "Step4", c, "1", false) + // Store newer PV - test integer parsing ("2" > "10" as string, + // while 2 < 10 as integers) + storeVersion(t, "Step5", c, "10", true) +} + +func TestControllerCacheParsingError(t *testing.T) { + c := cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc) + // There must be something in the cache to compare with + storeVersion(t, "Step1", c, "1", true) + content := newContent("contentName", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "pv-handle-1-1", deletionPolicy, nil, nil, false, nil) + content.ResourceVersion = "xxx" + _, err := utils.StoreObjectUpdate(c, content, "content") + if err == nil { + t.Errorf("Expected parsing error, got nil instead") + } +} diff --git a/pkg/sidecar-controller/snapshot_delete_test.go b/pkg/sidecar-controller/snapshot_delete_test.go new file mode 100644 index 00000000..593b71c2 --- /dev/null +++ b/pkg/sidecar-controller/snapshot_delete_test.go @@ -0,0 +1,440 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sidecar_controller + +import ( + "fmt" + "testing" + "time" + + "errors" + + crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/pkg/utils" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var defaultSize int64 = 1000 +var emptySize int64 = 0 +var deletePolicy = crdv1.VolumeSnapshotContentDelete +var retainPolicy = crdv1.VolumeSnapshotContentRetain +var timeNow = time.Now() +var timeNowMetav1 = metav1.Now() +var False = false +var True = true + +var class1Parameters = map[string]string{ + "param1": "value1", +} + +var class2Parameters = map[string]string{ + "param2": "value2", +} + +var class3Parameters = map[string]string{ + "param3": "value3", + utils.AnnDeletionSecretRefName: "name", +} + +var class4Parameters = map[string]string{ + utils.AnnDeletionSecretRefName: "emptysecret", + utils.AnnDeletionSecretRefNamespace: "default", +} + +var class5Parameters = map[string]string{ + utils.AnnDeletionSecretRefName: "secret", + utils.AnnDeletionSecretRefNamespace: "default", +} + +var snapshotClasses = []*crdv1.VolumeSnapshotClass{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "VolumeSnapshotClass", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: classGold, + }, + Driver: mockDriverName, + Parameters: class1Parameters, + DeletionPolicy: crdv1.VolumeSnapshotContentDelete, + }, + { + TypeMeta: metav1.TypeMeta{ + Kind: "VolumeSnapshotClass", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: classSilver, + }, + Driver: mockDriverName, + Parameters: class2Parameters, + DeletionPolicy: crdv1.VolumeSnapshotContentDelete, + }, + { + TypeMeta: metav1.TypeMeta{ + Kind: "VolumeSnapshotClass", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: emptySecretClass, + }, + Driver: mockDriverName, + Parameters: class4Parameters, + DeletionPolicy: crdv1.VolumeSnapshotContentDelete, + }, + { + TypeMeta: metav1.TypeMeta{ + Kind: "VolumeSnapshotClass", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: invalidSecretClass, + }, + Driver: mockDriverName, + Parameters: class3Parameters, + DeletionPolicy: crdv1.VolumeSnapshotContentDelete, + }, + { + TypeMeta: metav1.TypeMeta{ + Kind: "VolumeSnapshotClass", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: validSecretClass, + }, + Driver: mockDriverName, + Parameters: class5Parameters, + DeletionPolicy: crdv1.VolumeSnapshotContentDelete, + }, + { + TypeMeta: metav1.TypeMeta{ + Kind: "VolumeSnapshotClass", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: defaultClass, + Annotations: map[string]string{utils.IsDefaultSnapshotClassAnnotation: "true"}, + }, + Driver: mockDriverName, + DeletionPolicy: crdv1.VolumeSnapshotContentDelete, + }, +} + +// Test single call to syncContent, expecting deleting to happen. +// 1. Fill in the controller with initial data +// 2. Call the syncContent *once*. +// 3. Compare resulting contents with expected contents. +func TestDeleteSync(t *testing.T) { + + tests := []controllerTest{ + { + name: "1-1 - content non-nil DeletionTimestamp with delete policy will delete snapshot", + initialContents: newContentArrayWithDeletionTimestamp("content1-1", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "snap1-1-volumehandle", deletionPolicy, nil, nil, true, &timeNowMetav1), + expectedContents: newContentArrayWithDeletionTimestamp("content1-1", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "snap1-1-volumehandle", deletionPolicy, nil, nil, false, &timeNowMetav1), + expectedEvents: noevents, + errors: noerrors, + initialSecrets: []*v1.Secret{secret()}, + expectedCreateCalls: []createCall{ + { + snapshotName: "snapshot-snapuid1-1", + volumeHandle: "snap1-1-volumehandle", + parameters: map[string]string{"param1": "value1"}, + driverName: mockDriverName, + size: defaultSize, + snapshotId: "snapuid1-1-deleted", + creationTime: timeNow, + readyToUse: true, + }, + }, + expectedListCalls: []listCall{{"sid1-1", true, time.Now(), 1, nil}}, + expectedDeleteCalls: []deleteCall{{"sid1-1", nil, nil}}, + test: testSyncContent, + }, + { + name: "1-2 - content non-nil DeletionTimestamp with retain policy will not delete snapshot", + initialContents: newContentArrayWithDeletionTimestamp("content1-2", "snapuid1-2", "snap1-2", "sid1-2", classGold, "", "snap1-2-volumehandle", retainPolicy, nil, nil, true, &timeNowMetav1), + expectedContents: newContentArrayWithDeletionTimestamp("content1-2", "snapuid1-2", "snap1-2", "sid1-2", classGold, "", "snap1-2-volumehandle", retainPolicy, nil, nil, false, &timeNowMetav1), + expectedEvents: noevents, + errors: noerrors, + expectedCreateCalls: []createCall{ + { + snapshotName: "snapshot-snapuid1-2", + volumeHandle: "snap1-2-volumehandle", + parameters: map[string]string{"param1": "value1"}, + driverName: mockDriverName, + size: defaultSize, + snapshotId: "snapuid1-2-deleted", + creationTime: timeNow, + readyToUse: true, + }, + }, + expectedListCalls: []listCall{{"sid1-2", true, time.Now(), 1, nil}}, + expectedDeleteCalls: []deleteCall{{"sid1-2", nil, nil}}, + test: testSyncContent, + }, + { + name: "1-3 - delete snapshot error should result in an event", + initialContents: newContentArrayWithDeletionTimestamp("content1-3", "snapuid1-3", "snap1-3", "sid1-3", validSecretClass, "", "snap1-3-volumehandle", deletePolicy, nil, nil, true, &timeNowMetav1), + expectedContents: newContentArrayWithDeletionTimestamp("content1-3", "snapuid1-3", "snap1-3", "sid1-3", validSecretClass, "", "snap1-3-volumehandle", deletePolicy, nil, nil, false, &timeNowMetav1), + errors: noerrors, + expectedCreateCalls: []createCall{ + { + snapshotName: "snapshot-snapuid1-3", + volumeHandle: "snap1-3-volumehandle", + parameters: map[string]string{"foo": "bar"}, + driverName: mockDriverName, + size: defaultSize, + snapshotId: "snapuid1-3-deleted", + creationTime: timeNow, + readyToUse: true, + }, + }, + expectedDeleteCalls: []deleteCall{{"sid1-3", nil, fmt.Errorf("mock csi driver delete error")}}, + expectedEvents: []string{"Warning SnapshotDeleteError"}, + expectedListCalls: []listCall{{"sid1-3", true, time.Now(), 1, nil}}, + test: testSyncContent, + }, + /*{ + name: "1-4 - create snapshot error should result in an event", + initialContents: newContentArrayWithDeletionTimestamp("content1-4", "snapuid1-4", "snap1-4", "sid1-4", classGold, "", "snap1-4-volumehandle", deletePolicy, nil, nil, true, nil), + //expectedContents: newContentArrayWithDeletionTimestamp("content1-4", "snapuid1-4", "snap1-4", "sid1-4", classGold, "", "snap1-4-volumehandle", deletePolicy, nil, nil, true, nil), + errors: []reactorError{}, + expectedCreateCalls: []createCall{ + { + snapshotName: "snapshot-snapuid1-4", + volumeHandle: "snap1-4-volumehandle", + parameters: map[string]string{"param1": "value1"}, + err: fmt.Errorf("Create failed"), + }, + }, + //expectedDeleteCalls: []deleteCall{{"sid1-4", nil, nil}}, + expectedListCalls: []listCall{{"sid1-4", true, time.Now(), 1, nil}}, + expectedEvents: []string{"Warning SnapshotContentCheckandUpdateFailed Failed to check and update snapshot content: Create failed"}, + test: testSyncContent, + },*/ + /*{ + name: "2-1 - content with empty snapshot class will not be deleted if it is bound to a non-exist snapshot but it does not have a snapshot uid specified", + initialContents: newContentArray("content2-1", "", "snap2-1", "sid2-1", "", "", "", deletionPolicy, nil, nil, true), + expectedContents: newContentArray("content2-1", "", "snap2-1", "sid2-1", "", "", "", deletionPolicy, nil, nil, true), + expectedEvents: noevents, + errors: noerrors, + expectedDeleteCalls: []deleteCall{{"sid2-1", nil, nil}}, + test: testSyncContent, + },*/ + /*{ + name: "1-2 - successful delete with snapshot class that has empty secret parameter", + initialContents: newContentArray("content1-2", "sid1-2", "snap1-2", "sid1-2", emptySecretClass, "", "volumeHandle", deletionPolicy, nil, nil, true), + expectedContents: nocontents, + initialSnapshots: nosnapshots, + expectedSnapshots: nosnapshots, + initialSecrets: []*v1.Secret{emptySecret()}, + expectedEvents: noevents, + errors: noerrors, + expectedDeleteCalls: []deleteCall{{"sid1-2", map[string]string{}, nil}}, + test: testSyncContent, + },*/ + /*{ + name: "1-3 - content non-nil DeletionTimestamp with delete policy will delete snapshot", + initialContents: newContentArrayWithDeletionTimestamp("content1-3", "snapuid1-1", "snap1-1", "sid1-3", classGold, "", "snap1-3-volumehandle", deletionPolicy, nil, nil, true, &timeNowMetav1), + expectedContents: newContentArrayWithDeletionTimestamp("content1-3", "snapuid1-1", "snap1-1", "sid1-3", classGold, "", "snap1-3-volumehandle", deletionPolicy, nil, nil, false, &timeNowMetav1), + initialSnapshots: newSnapshotArray("snap1-3", "snapuid1-3", "claim1-3", "", validSecretClass, "", &False, nil, nil, nil), + expectedSnapshots: newSnapshotArray("snap1-3", "snapuid1-3", "claim1-3", "", validSecretClass, "", &False, nil, nil, nil), + expectedEvents: noevents, + errors: noerrors, + initialSecrets: []*v1.Secret{secret()}, + expectedCreateCalls: []createCall{ + { + snapshotName: "snapshot-snapuid1-1", + volumeHandle: "snap1-1-volumehandle", + parameters: map[string]string{"param1": "value1"}, + driverName: mockDriverName, + size: defaultSize, + snapshotId: "snapuid1-1-deleted", + creationTime: timeNow, + readyToUse: true, + }, + }, + expectedListCalls: []listCall{{"sid1-1", true, time.Now(), 1, nil}}, + expectedDeleteCalls: []deleteCall{{"sid1-1", nil, nil}}, + test: testSyncContent, + },*/ + + /* name: "1-3 - successful delete with snapshot class that has valid secret parameter", + initialContents: newContentArray("content1-3", "sid1-3", "snap1-3", "sid1-3", validSecretClass, "", "snap1-3-volumehandle", deletionPolicy, nil, nil, true), + expectedContents: newContentArray("content1-3", "sid1-3", "snap1-3", "sid1-3", validSecretClass, "", "snap1-3-volumehandle", deletionPolicy, nil, nil, false), + initialSnapshots: newSnapshotArray("snapshot-snapuid1-3", "snapuid1-3", "claim1-3", "", validSecretClass, "", &False, nil, nil, nil), + expectedSnapshots: newSnapshotArray("snapshot-snapuid1-3", "snapuid1-3", "claim1-3", "", validSecretClass, "", &False, nil, nil, nil), + expectedEvents: noevents, + errors: noerrors, + initialSecrets: []*v1.Secret{secret()}, + expectedCreateCalls: []createCall{ + { + snapshotName: "snap1-3", + volumeHandle: "snap1-3-volumehandle", + parameters: map[string]string{"param1": "value1"}, + driverName: mockDriverName, + size: defaultSize, + snapshotId: "sid1-3", + creationTime: timeNow, + }, + }, + expectedListCalls: []listCall{{"sid1-3", true, time.Now(), 1, nil}}, + expectedDeleteCalls: []deleteCall{{"sid1-3", map[string]string{"param1": "value1"}, nil}}, + test: testSyncContent, + }, + */ + { + name: "1-4 - fail delete with snapshot class that has invalid secret parameter", + initialContents: newContentArrayWithDeletionTimestamp("content1-1", "snapuid1-1", "snap1-1", "sid1-1", "invalid", "", "snap1-1-volumehandle", deletionPolicy, nil, nil, true, &timeNowMetav1), + expectedContents: newContentArrayWithDeletionTimestamp("content1-1", "snapuid1-1", "snap1-1", "sid1-1", "invalid", "", "snap1-1-volumehandle", deletionPolicy, nil, nil, false, &timeNowMetav1), + expectedEvents: noevents, + errors: noerrors, + test: testSyncContent, + }, + { + name: "1-5 - csi driver delete snapshot returns error", + initialContents: newContentArrayWithDeletionTimestamp("content1-5", "sid1-5", "snap1-5", "sid1-5", validSecretClass, "", "", deletionPolicy, nil, &defaultSize, true, &timeNowMetav1), + expectedContents: newContentArrayWithDeletionTimestamp("content1-5", "sid1-5", "snap1-5", "sid1-5", validSecretClass, "", "", deletionPolicy, nil, &defaultSize, false, &timeNowMetav1), + expectedListCalls: []listCall{{"sid1-5", true, time.Now(), 1000, nil}}, + expectedDeleteCalls: []deleteCall{{"sid1-5", nil, errors.New("mock csi driver delete error")}}, + expectedEvents: []string{"Warning SnapshotDeleteError"}, + errors: noerrors, + test: testSyncContent, + }, + /*{ + name: "1-6 - api server delete content returns error", + initialContents: newContentArray("content1-6", "sid1-6", "snap1-6", "sid1-6", classGold, "", "", deletionPolicy, nil, nil, true), + //expectedContents: newContentArray("content1-6", "sid1-6", "snap1-6", "sid1-6", classGold, "", "", deletionPolicy, nil, nil, true), + expectedDeleteCalls: []deleteCall{{"sid1-6", map[string]string{"foo": "bar"}, nil}}, + expectedListCalls: []listCall{{"sid1-6", false, time.Now(), 0, nil}}, + expectedEvents: []string{"Warning SnapshotContentObjectDeleteError"}, + errors: []reactorError{ + // Inject error to the first client.VolumesnapshotV1beta1().VolumeSnapshotContents().Delete call. + // All other calls will succeed. + {"delete", "volumesnapshotcontents", errors.New("mock delete error")}, + }, + test: testSyncContent, + },*/ + /* + { + // delete success - snapshot that the content was pointing to was deleted, and another + // with the same name created. + name: "1-7 - prebound content is deleted while the snapshot exists", + initialContents: newContentArray("content1-7", "sid1-7", "snap1-7", "sid1-7", emptySecretClass, "", "", deletionPolicy, nil, nil, true), + expectedContents: nocontents, + initialSecrets: []*v1.Secret{secret()}, + expectedDeleteCalls: []deleteCall{{"sid1-7", map[string]string{"foo": "bar"}, nil}}, + expectedEvents: noevents, + errors: noerrors, + test: testSyncContent, + },*/ + { + // delete success(?) - content is deleted before doDelete() starts + name: "1-8 - content is deleted before deleting", + initialContents: newContentArray("content1-8", "sid1-8", "snap1-8", "sid1-8", classGold, "", "", deletionPolicy, nil, nil, true), + expectedContents: nocontents, + expectedListCalls: []listCall{{"sid1-8", false, time.Now(), 0, nil}}, + expectedDeleteCalls: []deleteCall{{"sid1-8", map[string]string{"foo": "bar"}, nil}}, + expectedEvents: noevents, + errors: noerrors, + test: wrapTestWithInjectedOperation(testSyncContent, func(ctrl *csiSnapshotSideCarController, reactor *snapshotReactor) { + // Delete the volume before delete operation starts + reactor.lock.Lock() + delete(reactor.contents, "content1-8") + reactor.lock.Unlock() + }), + }, + /*{ + name: "1-9 - content will not be deleted if it is bound to a snapshot correctly, snapshot uid is specified", + expectedContents: newContentArrayWithDeletionTimestamp("content1-9", "snapuid1-9", "snap1-9", "sid1-9", classGold, "", "snap1-9-volumehandle", retainPolicy, nil, nil, false, &timeNowMetav1), + initialContents: newContentArrayWithDeletionTimestamp("content1-9", "snapuid1-9", "snap1-9", "sid1-9", classGold, "", "snap1-9-volumehandle", retainPolicy, nil, nil, false, &timeNowMetav1), + expectedEvents: noevents, + expectedListCalls: []listCall{{"sid1-9", true, time.Now(), 0, nil}}, + initialSecrets: []*v1.Secret{secret()}, + errors: noerrors, + test: testSyncContent, + },*/ + /* + { + name: "1-10 - will not delete content with retain policy set which is bound to a snapshot incorrectly", + initialContents: newContentArray("content1-10", "snapuid1-10-x", "snap1-10", "sid1-10", validSecretClass, "", "", retainPolicy, nil, nil, true), + expectedContents: newContentArray("content1-10", "snapuid1-10-x", "snap1-10", "sid1-10", validSecretClass, "", "", retainPolicy, nil, nil, true), + initialSnapshots: newSnapshotArray("snap1-10", "snapuid1-10", "claim1-10", "", validSecretClass, "content1-10", &False, nil, nil, nil), + expectedSnapshots: newSnapshotArray("snap1-10", "snapuid1-10", "claim1-10", "", validSecretClass, "content1-10", &False, nil, nil, nil), + expectedEvents: noevents, + initialSecrets: []*v1.Secret{secret()}, + errors: noerrors, + test: testSyncContent, + },*/ + { + name: "1-11 - content will not be deleted if it is bound to a snapshot correctly, snapsht uid is not specified", + initialContents: newContentArrayWithReadyToUse("content1-11", "", "snap1-11", "sid1-11", validSecretClass, "", "", deletePolicy, nil, &defaultSize, &True, true), + expectedContents: newContentArrayWithReadyToUse("content1-11", "", "snap1-11", "sid1-11", validSecretClass, "", "", deletePolicy, nil, &defaultSize, &True, true), + expectedEvents: noevents, + expectedCreateCalls: []createCall{ + { + snapshotName: "snap1-11", + volumeHandle: "snap1-11", + parameters: map[string]string{"param1": "value1"}, + driverName: mockDriverName, + size: defaultSize, + snapshotId: "snapuid1-1-deleted", + creationTime: timeNow, + readyToUse: true, + }, + }, + expectedListCalls: []listCall{{"sid1-11", true, time.Now(), 1000, nil}}, + initialSecrets: []*v1.Secret{secret()}, + errors: noerrors, + test: testSyncContent, + }, + { + name: "1-12 - content with retain policy will not be deleted if it is bound to a non-exist snapshot and also has a snapshot uid specified", + initialContents: newContentArrayWithReadyToUse("content1-12", "sid1-12", "snap1-11", "sid1-11", validSecretClass, "", "", retainPolicy, nil, &defaultSize, &True, true), + expectedContents: newContentArrayWithReadyToUse("content1-12", "sid1-12", "snap1-11", "sid1-11", validSecretClass, "", "", retainPolicy, nil, &defaultSize, &True, true), + expectedEvents: noevents, + expectedListCalls: []listCall{{"sid1-11", true, time.Now(), 0, nil}}, + errors: noerrors, + test: testSyncContent, + }, + /*{ + name: "1-13 - content with empty snapshot class is not deleted when Deletion policy is not set even if it is bound to a non-exist snapshot and also has a snapshot uid specified", + initialContents: newContentArray("content1-13", "sid1-13", "snap1-13", "sid1-13", validSecretClass, "", "", retainPolicy, nil, nil, true), + expectedContents: newContentArray("content1-13", "sid1-13", "snap1-13", "sid1-13", validSecretClass, "", "", retainPolicy, nil, nil, true), + expectedEvents: noevents, + errors: noerrors, + test: testSyncContent, + }, + { + name: "1-14 - content will not be deleted if it is bound to a snapshot correctly, snapshot uid is specified", + initialContents: newContentArray("content1-14", "snapuid1-14", "snap1-14", "sid1-14", validSecretClass, "", "", retainPolicy, nil, nil, true), + expectedContents: newContentArray("content1-14", "snapuid1-14", "snap1-14", "sid1-14", validSecretClass, "", "", retainPolicy, nil, nil, true), + expectedEvents: noevents, + initialSecrets: []*v1.Secret{secret()}, + errors: noerrors, + test: testSyncContent, + },*/ + { + name: "1-16 - continue delete with snapshot class that has nonexistent secret", + initialContents: newContentArrayWithDeletionTimestamp("content1-16", "sid1-16", "snap1-16", "sid1-16", emptySecretClass, "", "", deletePolicy, nil, &defaultSize, true, &timeNowMetav1), + expectedContents: newContentArrayWithDeletionTimestamp("content1-16", "sid1-16", "snap1-16", "sid1-16", emptySecretClass, "", "", deletePolicy, nil, &defaultSize, false, &timeNowMetav1), + expectedEvents: noevents, + expectedListCalls: []listCall{{"sid1-16", true, time.Now(), 0, nil}}, + errors: noerrors, + initialSecrets: []*v1.Secret{}, // secret does not exist + expectedDeleteCalls: []deleteCall{{"sid1-16", nil, nil}}, + test: testSyncContent, + }, + } + runSyncContentTests(t, tests, snapshotClasses) +} diff --git a/pkg/sidecar-controller/snapshot_finalizer_test.go b/pkg/sidecar-controller/snapshot_finalizer_test.go new file mode 100644 index 00000000..a11ea2d4 --- /dev/null +++ b/pkg/sidecar-controller/snapshot_finalizer_test.go @@ -0,0 +1,34 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sidecar_controller + +import ( + "testing" +) + +// Test single call to ensureSnapshotSourceFinalizer and checkandRemoveSnapshotSourceFinalizer, +// expecting PVCFinalizer to be added or removed +func TestContentFinalizer(t *testing.T) { + + // GG TODO - add content finalizer tests + /* + tests := []controllerTest{ + {}, + } + runPVCFinalizerTests(t, tests, snapshotClasses) + */ +} From e4e2f3b44b9be3d1de9dca3f5bc1f26d036ec194 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Fri, 8 Nov 2019 15:55:46 -0800 Subject: [PATCH 4/5] Cleanup framework_test comments and old code Signed-off-by: Grant Griffiths --- pkg/sidecar-controller/framework_test.go | 124 ++++------------------- 1 file changed, 18 insertions(+), 106 deletions(-) diff --git a/pkg/sidecar-controller/framework_test.go b/pkg/sidecar-controller/framework_test.go index 49ec971e..73b8ea35 100644 --- a/pkg/sidecar-controller/framework_test.go +++ b/pkg/sidecar-controller/framework_test.go @@ -33,7 +33,6 @@ import ( storagelisters "github.com/kubernetes-csi/external-snapshotter/pkg/client/listers/volumesnapshot/v1beta1" "github.com/kubernetes-csi/external-snapshotter/pkg/utils" v1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -50,29 +49,26 @@ import ( "k8s.io/klog" ) -// This is a unit test framework for snapshot controller. -// It fills the controller with test snapshots/contents and can simulate these +// This is a unit test framework for snapshot sidecar controller. +// It fills the controller with test contents and can simulate these // scenarios: -// 1) Call syncSnapshot/syncContent once. -// 2) Call syncSnapshot/syncContent several times (both simulating "snapshot/content +// 1) Call syncContent once. +// 2) Call syncContent several times (both simulating "content // modified" events and periodic sync), until the controller settles down and // does not modify anything. // 3) Simulate almost real API server/etcd and call add/update/delete -// content/snapshot. +// content. // In all these scenarios, when the test finishes, the framework can compare -// resulting snapshots/contents with list of expected snapshots/contents and report +// resulting contents with list of expected contents and report // differences. // controllerTest contains a single controller test input. -// Each test has initial set of contents and snapshots that are filled into the +// Each test has initial set of contents that are filled into the // controller before the test starts. The test then contains a reference to // function to call as the actual test. Available functions are: -// - testSyncSnapshot - calls syncSnapshot on the first snapshot in initialSnapshots. -// - testSyncSnapshotError - calls syncSnapshot on the first snapshot in initialSnapshots -// and expects an error to be returned. // - testSyncContent - calls syncContent on the first content in initialContents. // - any custom function for specialized tests. -// The test then contains list of contents/snapshots that are expected at the end +// The test then contains list of contents that are expected at the end // of the test and list of generated events. type controllerTest struct { // Name of the test, for logging @@ -106,19 +102,17 @@ const mockDriverName = "csi-mock-plugin" var errVersionConflict = errors.New("VersionError") var nocontents []*crdv1.VolumeSnapshotContent -var nosnapshots []*crdv1.VolumeSnapshot var noevents = []string{} var noerrors = []reactorError{} // snapshotReactor is a core.Reactor that simulates etcd and API server. It // stores: // - Latest version of snapshots contents saved by the controller. -// - Queue of all saves (to simulate "content/snapshot updated" events). This queue -// contains all intermediate state of an object - e.g. a snapshot.VolumeName -// is updated first and snapshot.Phase second. This queue will then contain both +// - Queue of all saves (to simulate "content updated" events). This queue +// contains all intermediate state of an object. This queue will then contain both // updates as separate entries. // - Number of changes since the last call to snapshotReactor.syncAll(). -// - Optionally, content and snapshot fake watchers which should be the same ones +// - Optionally, content watcher which should be the same ones // used by the controller. Any time an event function like deleteContentEvent // is called to simulate an event, the reactor's stores are updated and the // controller is sent the event via the fake watcher. @@ -129,16 +123,11 @@ var noerrors = []reactorError{} // the list. type snapshotReactor struct { secrets map[string]*v1.Secret - storageClasses map[string]*storagev1.StorageClass - volumes map[string]*v1.PersistentVolume - claims map[string]*v1.PersistentVolumeClaim contents map[string]*crdv1.VolumeSnapshotContent - snapshots map[string]*crdv1.VolumeSnapshot changedObjects []interface{} changedSinceLastSync int ctrl *csiSnapshotSideCarController fakeContentWatch *watch.FakeWatcher - fakeSnapshotWatch *watch.FakeWatcher lock sync.Mutex errors []reactorError } @@ -146,7 +135,7 @@ type snapshotReactor struct { // reactorError is an error that is returned by test reactor (=simulated // etcd+/API server) when an action performed by the reactor matches given verb // ("get", "update", "create", "delete" or "*"") on given resource -// ("volumesnapshotcontents", "volumesnapshots" or "*"). +// ("volumesnapshotcontents" or "*"). type reactorError struct { verb string resource string @@ -203,9 +192,9 @@ func (r *snapshotReactor) React(action core.Action) (handled bool, ret runtime.O content := obj.(*crdv1.VolumeSnapshotContent) // Check and bump object version - storedVolume, found := r.contents[content.Name] + storedContent, found := r.contents[content.Name] if found { - storedVer, _ := strconv.Atoi(storedVolume.ResourceVersion) + storedVer, _ := strconv.Atoi(storedContent.ResourceVersion) requestedVer, _ := strconv.Atoi(content.ResourceVersion) if storedVer != requestedVer { return true, obj, errVersionConflict @@ -314,41 +303,6 @@ func (r *snapshotReactor) checkContents(expectedContents []*crdv1.VolumeSnapshot return nil } -// checkSnapshots compares all expectedSnapshots with set of snapshots at the end of the -// test and reports differences. -func (r *snapshotReactor) checkSnapshots(expectedSnapshots []*crdv1.VolumeSnapshot) error { - r.lock.Lock() - defer r.lock.Unlock() - - expectedMap := make(map[string]*crdv1.VolumeSnapshot) - gotMap := make(map[string]*crdv1.VolumeSnapshot) - for _, c := range expectedSnapshots { - // Don't modify the existing object - c = c.DeepCopy() - c.ResourceVersion = "" - if c.Status.Error != nil { - c.Status.Error.Time = &metav1.Time{} - } - expectedMap[c.Name] = c - } - for _, c := range r.snapshots { - // We must clone the snapshot because of golang race check - it was - // written by the controller without any locks on it. - c = c.DeepCopy() - c.ResourceVersion = "" - if c.Status.Error != nil { - c.Status.Error.Time = &metav1.Time{} - } - gotMap[c.Name] = c - } - if !reflect.DeepEqual(expectedMap, gotMap) { - // Print ugly but useful diff of expected and received objects for - // easier debugging. - return fmt.Errorf("snapshot check failed [A-expected, B-got result]: %s", diff.ObjectDiff(expectedMap, gotMap)) - } - return nil -} - // checkEvents compares all expectedEvents with events generated during the test // and reports differences. func checkEvents(t *testing.T, expectedEvents []string, ctrl *csiSnapshotSideCarController) error { @@ -414,9 +368,6 @@ func (r *snapshotReactor) popChange() interface{} { case *crdv1.VolumeSnapshotContent: vol, _ := obj.(*crdv1.VolumeSnapshotContent) klog.V(4).Infof("reactor queue: %s", vol.Name) - case *crdv1.VolumeSnapshot: - snapshot, _ := obj.(*crdv1.VolumeSnapshot) - klog.V(4).Infof("reactor queue: %s", snapshot.Name) } } @@ -426,23 +377,17 @@ func (r *snapshotReactor) popChange() interface{} { return obj } -// syncAll simulates the controller periodic sync of contents and snapshot. It +// syncAll simulates the controller periodic sync of contents. It // simply adds all these objects to the internal queue of updates. This method -// should be used when the test manually calls syncSnapshot/syncContent. Test that +// should be used when the test manually calls syncContent. Test that // use real controller loop (ctrl.Run()) will get periodic sync automatically. func (r *snapshotReactor) syncAll() { r.lock.Lock() defer r.lock.Unlock() - for _, c := range r.snapshots { - r.changedObjects = append(r.changedObjects, c) - } for _, v := range r.contents { r.changedObjects = append(r.changedObjects, v) } - for _, pvc := range r.claims { - r.changedObjects = append(r.changedObjects, pvc) - } r.changedSinceLastSync = 0 } @@ -511,22 +456,6 @@ func (r *snapshotReactor) deleteContentEvent(content *crdv1.VolumeSnapshotConten } } -// deleteSnapshotEvent simulates that a snapshot has been deleted in etcd and the -// controller receives 'snapshot deleted' event. -func (r *snapshotReactor) deleteSnapshotEvent(snapshot *crdv1.VolumeSnapshot) { - r.lock.Lock() - defer r.lock.Unlock() - - // Remove the snapshot from list of resulting snapshots. - delete(r.snapshots, snapshot.Name) - - // Generate deletion event. Cloned content is needed to prevent races (and we - // would get a clone from etcd too). - if r.fakeSnapshotWatch != nil { - r.fakeSnapshotWatch.Delete(snapshot.DeepCopy()) - } -} - // addContentEvent simulates that a content has been added in etcd and the // controller receives 'content added' event. func (r *snapshotReactor) addContentEvent(content *crdv1.VolumeSnapshotContent) { @@ -555,20 +484,6 @@ func (r *snapshotReactor) modifyContentEvent(content *crdv1.VolumeSnapshotConten } } -// addSnapshotEvent simulates that a snapshot has been deleted in etcd and the -// controller receives 'snapshot added' event. -func (r *snapshotReactor) addSnapshotEvent(snapshot *crdv1.VolumeSnapshot) { - r.lock.Lock() - defer r.lock.Unlock() - - r.snapshots[snapshot.Name] = snapshot - // Generate event. No cloning is needed, this snapshot is not stored in the - // controller cache yet. - if r.fakeSnapshotWatch != nil { - r.fakeSnapshotWatch.Add(snapshot) - } -} - func newSnapshotReactor(kubeClient *kubefake.Clientset, client *fake.Clientset, ctrl *csiSnapshotSideCarController, fakeVolumeWatch, fakeClaimWatch *watch.FakeWatcher, errors []reactorError) *snapshotReactor { reactor := &snapshotReactor{ secrets: make(map[string]*v1.Secret), @@ -580,11 +495,8 @@ func newSnapshotReactor(kubeClient *kubefake.Clientset, client *fake.Clientset, client.AddReactor("create", "volumesnapshotcontents", reactor.React) client.AddReactor("update", "volumesnapshotcontents", reactor.React) - client.AddReactor("update", "volumesnapshots", reactor.React) client.AddReactor("get", "volumesnapshotcontents", reactor.React) - client.AddReactor("get", "volumesnapshots", reactor.React) client.AddReactor("delete", "volumesnapshotcontents", reactor.React) - client.AddReactor("delete", "volumesnapshots", reactor.React) return reactor } @@ -765,7 +677,7 @@ func wrapTestWithInjectedOperation(toWrap testCall, injectBeforeOperation func(c klog.V(4).Infof("reactor:injecting call") injectBeforeOperation(ctrl, reactor) - // Run the tested function (typically syncSnapshot/syncContent) in a + // Run the tested function (typically syncContent) in a // separate goroutine. var testError error var testFinished int32 @@ -798,7 +710,7 @@ func evaluateTestResults(ctrl *csiSnapshotSideCarController, reactor *snapshotRe } } -// Test single call to syncSnapshot and syncContent methods. +// Test single call to syncContent methods. // For all tests: // 1. Fill in the controller with initial data // 2. Call the tested function (syncContent) via From 7f0ecc9b8a70a054e0c1117005e99639ea8a45f2 Mon Sep 17 00:00:00 2001 From: xing-yang Date: Sat, 9 Nov 2019 04:05:43 +0000 Subject: [PATCH 5/5] Update snapshot-controller and csi-snapshotter images to v2.0.0-rc2 --- deploy/kubernetes/csi-snapshotter/setup-csi-snapshotter.yaml | 2 +- .../snapshot-controller/setup-snapshot-controller.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/csi-snapshotter/setup-csi-snapshotter.yaml b/deploy/kubernetes/csi-snapshotter/setup-csi-snapshotter.yaml index 3555b09f..11d1bcc3 100644 --- a/deploy/kubernetes/csi-snapshotter/setup-csi-snapshotter.yaml +++ b/deploy/kubernetes/csi-snapshotter/setup-csi-snapshotter.yaml @@ -85,7 +85,7 @@ spec: mountPath: /csi - name: csi-snapshotter # NOTE: replace with official image when released: quay.io/k8scsi/csi-snapshotter:v2.0.0 - image: csi-snapshotter:testbeta #quay.io/k8scsi/csi-snapshotter:testbeta + image: quay.io/k8scsi/csi-snapshotter:v2.0.0-rc2 args: - "--v=5" - "--csi-address=$(ADDRESS)" diff --git a/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml b/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml index 853a35a4..714af766 100644 --- a/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml +++ b/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml @@ -20,7 +20,7 @@ spec: containers: - name: snapshot-controller # NOTE: replace with official image when released: quay.io/k8scsi/snapshot-controller:v2.0.0 - image: snapshot-controller:testbeta #quay.io/k8scsi/snapshot-controller:testbeta + image: quay.io/k8scsi/snapshot-controller:v2.0.0-rc2 args: - "--v=5" - "--leader-election=false"