Merge commit '91bc0c8f98c19aa5551969afc8621afeb48911a8' into prow-update-master

# Conflicts:
#	release-tools/build.make
#	release-tools/prow.sh

release-tools/build.make

@@ -69,12 +69,17 @@ endif
 # toolchain.
 BUILD_PLATFORMS =
 
+# Add go ldflags using LDFLAGS at the time of compilation.
+IMPORTPATH_LDFLAGS = -X main.version=$(REV) 
+EXT_LDFLAGS = -extldflags "-static"
+LDFLAGS = 
+FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS)
 # This builds each command (= the sub-directories of ./cmd) for the target platform(s)
 # defined by BUILD_PLATFORMS.
 $(CMDS:%=build-%): build-%: check-go-version-go
 	mkdir -p bin
 	echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \
-		if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o "./bin/$*$$suffix" ./cmd/$*); then \
+		if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \
 			echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \
 			exit 1; \
 		fi; \

release-tools/prow.sh

@@ -33,9 +33,9 @@
 # The expected environment is:
 # - $GOPATH/src/<import path> for the repository that is to be tested,
 #   with PR branch merged (when testing a PR)
+# - optional: bazel installed (when testing against Kubernetes master),
+#   must be recent enough for Kubernetes master
 # - running on linux-amd64
-# - bazel installed (when testing against Kubernetes master), must be recent
-#   enough for Kubernetes master
 # - kind (https://github.com/kubernetes-sigs/kind) installed
 # - optional: Go already installed
 
@@ -52,26 +52,6 @@ configvar () {
     eval echo "\$3:" "$1=\${$1}"
 }
 
-# Takes the minor version of $CSI_PROW_KUBERNETES_VERSION and overrides it to
-# $1 if they are equal minor versions. Ignores versions that begin with
-# "release-".
-override_k8s_version () {
-    local current_minor_version
-    local override_minor_version
-
-    # Ignore: See if you can use ${variable//search/replace} instead.
-    # shellcheck disable=SC2001
-    current_minor_version="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1\.\2/')"
-
-    # Ignore: See if you can use ${variable//search/replace} instead.
-    # shellcheck disable=SC2001
-    override_minor_version="$(echo "${1}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1\.\2/')"
-    if [ "${current_minor_version}" == "${override_minor_version}" ]; then
-      CSI_PROW_KUBERNETES_VERSION="$1"
-      echo "Overriding CSI_PROW_KUBERNETES_VERSION with $1: $CSI_PROW_KUBERNETES_VERSION"
-    fi
-}
-
 # Prints the value of a variable + version suffix, falling back to variable + "LATEST".
 get_versioned_variable () {
     local var="$1"
@@ -107,9 +87,22 @@ configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version fo
 configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building ginkgo" # depends on CSI_PROW_GINKGO_VERSION below
 
 # kind version to use. If the pre-installed version is different,
-# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/
+# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases
 # (if available), otherwise it is built from source.
-configvar CSI_PROW_KIND_VERSION "v0.6.0" "kind"
+configvar CSI_PROW_KIND_VERSION "v0.9.0" "kind"
+
+# kind images to use. Must match the kind version.
+# The release notes of each kind release list the supported images.
+configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.19.1@sha256:98cf5288864662e37115e362b23e4369c8c4a408f99cbc06e58ac30ddc721600
+kindest/node:v1.18.8@sha256:f4bcc97a0ad6e7abaf3f643d890add7efe6ee4ab90baeb374b4f41a4c95567eb
+kindest/node:v1.17.11@sha256:5240a7a2c34bf241afb54ac05669f8a46661912eab05705d660971eeb12f6555
+kindest/node:v1.16.15@sha256:a89c771f7de234e6547d43695c7ab047809ffc71a0c3b65aa54eda051c45ed20
+kindest/node:v1.15.12@sha256:d9b939055c1e852fe3d86955ee24976cab46cba518abcb8b13ba70917e6547a6
+kindest/node:v1.14.10@sha256:ce4355398a704fca68006f8a29f37aafb49f8fc2f64ede3ccd0d9198da910146
+kindest/node:v1.13.12@sha256:1c1a48c2bfcbae4d5f4fa4310b5ed10756facad0b7a2ca93c7a4b5bae5db29f5" "kind images"
+
+# Use kind node-image --type=bazel by default, but allow to disable that.
+configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation"
 
 # ginkgo test runner version to use. If the pre-installed version is
 # different, the desired version is built from source.
@@ -124,10 +117,13 @@ configvar CSI_PROW_GINKO_PARALLEL "-p" "Ginko parallelism parameter(s)"
 configvar CSI_PROW_BUILD_JOB true "building code in repo enabled"
 
 # Kubernetes version to test against. This must be a version number
-# (like 1.13.3) for which there is a pre-built kind image (see
+# (like 1.13.3), "latest" (builds Kubernetes from the master branch)
-# https://hub.docker.com/r/kindest/node/tags), "latest" (builds
+# or "release-x.yy" (builds Kubernetes from a release branch).
-# Kubernetes from the master branch) or "release-x.yy" (builds
+#
-# Kubernetes from a release branch).
+# The patch version is only relevant for picking the E2E test suite
+# that is used for testing. The script automatically picks
+# the kind images for the major/minor version of Kubernetes
+# that the kind release supports.
 #
 # This can also be a version that was not released yet at the time
 # that the settings below were chose. The script will then
@@ -136,16 +132,6 @@ configvar CSI_PROW_BUILD_JOB true "building code in repo enabled"
 # deprecating or changing the implementation of an alpha feature.
 configvar CSI_PROW_KUBERNETES_VERSION 1.17.0 "Kubernetes"
 
-# This is a hack to workaround the issue that each version
-# of kind currently only supports specific patch versions of
-# Kubernetes. We need to override CSI_PROW_KUBERNETES_VERSION
-# passed in by our CI/pull jobs to the versions that
-# kind v0.5.0 supports.
-#
-# If the version is prefixed with "release-", then nothing
-# is overridden.
-override_k8s_version "1.15.3"
-
 # CSI_PROW_KUBERNETES_VERSION reduced to first two version numbers and
 # with underscore (1_13 instead of 1.13.3) and in uppercase (LATEST
 # instead of latest).
@@ -262,11 +248,16 @@ configvar CSI_PROW_DEP_VERSION v0.5.1 "golang dep version to be used for vendor
 #
 # Unknown or unsupported entries are ignored.
 #
+# Testing of alpha features is only supported for CSI_PROW_KUBERNETES_VERSION=latest
+# because CSI_PROW_E2E_ALPHA and CSI_PROW_E2E_ALPHA_GATES are not set for
+# older Kubernetes releases. The script supports that, it just isn't done because
+# it is not needed and would cause additional maintenance effort.
+#
 # Sanity testing with csi-sanity only covers the CSI driver itself and
 # thus only makes sense in repos which provide their own CSI
 # driver. Repos can enable sanity testing by setting
 # CSI_PROW_TESTS_SANITY=sanity.
-configvar CSI_PROW_TESTS "unit parallel serial parallel-alpha serial-alpha sanity" "tests to run"
+configvar CSI_PROW_TESTS "unit parallel serial $(if [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ]; then echo parallel-alpha serial-alpha; fi) sanity" "tests to run"
 tests_enabled () {
     local t1 t2
     # We want word-splitting here, so ignore: Quote to prevent word splitting, or split robustly with mapfile or read -a.
@@ -298,11 +289,7 @@ tests_need_alpha_cluster () {
 
 # Regex for non-alpha, feature-tagged tests that should be run.
 #
-# Starting with 1.17, snapshots is beta, but the E2E tests still have the
+configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for latest Kubernetes version"
-# [Feature:] tag. They need to be explicitly enabled.
-configvar CSI_PROW_E2E_FOCUS_1_15 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.15" # no tests to run, match nothing
-configvar CSI_PROW_E2E_FOCUS_1_16 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.16" # no tests to run, match nothing
-configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for Kubernetes >= 1.17"
 configvar CSI_PROW_E2E_FOCUS "$(get_versioned_variable CSI_PROW_E2E_FOCUS "${csi_prow_kubernetes_version_suffix}")" "non-alpha, feature-tagged tests"
 
 # Serial vs. parallel is always determined by these regular expressions.
@@ -324,7 +311,7 @@ regex_join () {
 # alpha in previous Kubernetes releases. This was considered too
 # error prone. Therefore we use E2E tests that match the Kubernetes
 # version that is getting tested.
-configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for Kubernetes >= 1.14" # there's no need to update this, adding a new case for CSI_PROW_E2E for a new Kubernetes is enough
+configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for latest Kubernetes version" # there's no need to update this, adding a new case for CSI_PROW_E2E for a new Kubernetes is enough
 configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi_prow_kubernetes_version_suffix}")" "alpha tests"
 
 # After the parallel E2E test without alpha features, a test cluster
@@ -339,15 +326,11 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi
 # kubernetes-csi components must be updated, either by disabling
 # the failing test for "latest" or by updating the test and not running
 # it anymore for older releases.
-configvar CSI_PROW_E2E_ALPHA_GATES_1_15 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.15"
+configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'GenericEphemeralVolume=true,CSIStorageCapacity=true' "alpha feature gates for latest Kubernetes"
-configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha feature gates for Kubernetes 1.16"
-# TODO: add new CSI_PROW_ALPHA_GATES_xxx entry for future Kubernetes releases and
-# add new gates to CSI_PROW_E2E_ALPHA_GATES_LATEST.
-configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kubernetes"
 configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates"
 
 # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment
-configvar CSI_SNAPSHOTTER_VERSION 'v2.0.1' "external-snapshotter version tag"
+configvar CSI_SNAPSHOTTER_VERSION 'v3.0.0' "external-snapshotter version tag"
 
 # Some tests are known to be unusable in a KinD cluster. For example,
 # stopping kubelet with "ssh <node IP> systemctl stop kubelet" simply
@@ -507,6 +490,22 @@ list_gates () (
     done
 )
 
+# Turn feature gates in the format foo=true,bar=false into
+# a YAML map with the corresponding API groups for use
+# with https://kind.sigs.k8s.io/docs/user/configuration/#runtime-config
+list_api_groups () (
+    set -f; IFS=','
+    # Ignore: Double quote to prevent globbing and word splitting.
+    # shellcheck disable=SC2086
+    set -- $1
+    while [ "$1" ]; do
+        if [ "$1" = 'CSIStorageCapacity=true' ]; then
+            echo '   "storage.k8s.io/v1alpha1": "true"'
+        fi
+        shift
+    done
+)
+
 go_version_for_kubernetes () (
     local path="$1"
     local version="$2"
@@ -536,77 +535,52 @@ start_cluster () {
         run kind delete cluster --name=csi-prow || die "kind delete failed"
     fi
 
-    # Build from source?
+    # Try to find a pre-built kind image if asked to use a specific version.
-    if [[ "${CSI_PROW_KUBERNETES_VERSION}" =~ ^release-|^latest$ ]]; then
+    if ! [[ "${CSI_PROW_KUBERNETES_VERSION}" =~ ^release-|^latest$ ]]; then
+        # Ignore: See if you can use ${variable//search/replace} instead.
+        # shellcheck disable=SC2001
+        major_minor=$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/^\([0-9]*\)\.\([0-9]*\).*/\1.\2/')
+        for i in ${CSI_PROW_KIND_IMAGES}; do
+            if echo "$i" | grep -q "kindest/node:v${major_minor}"; then
+                image="$i"
+                break
+            fi
+        done
+    fi
+
+    # Need to build from source?
+    if ! [ "$image" ]; then
         if ! ${csi_prow_kind_have_kubernetes}; then
             local version="${CSI_PROW_KUBERNETES_VERSION}"
             if [ "$version" = "latest" ]; then
                 version=master
             fi
+            if ${CSI_PROW_USE_BAZEL}; then
+                type="bazel"
+            else
+                type="docker"
+            fi
             git_clone_branch https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version" || die "checking out Kubernetes $version failed"
 
             go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes"
-            run_with_go "$go_version" kind build node-image --type bazel --image csiprow/node:latest --kube-root "${CSI_PROW_WORK}/src/kubernetes" || die "'kind build node-image' failed"
+            # Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910
+            (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest --type="$type" --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed"
             csi_prow_kind_have_kubernetes=true
         fi
         image="csiprow/node:latest"
-    else
-        image="kindest/node:v${CSI_PROW_KUBERNETES_VERSION}"
     fi
     cat >"${CSI_PROW_WORK}/kind-config.yaml" <<EOF
 kind: Cluster
-apiVersion: kind.sigs.k8s.io/v1alpha3
+apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
 - role: control-plane
 - role: worker
 - role: worker
-EOF
+featureGates:
-
-    # kubeadm has API dependencies between apiVersion and Kubernetes version
-    # 1.15+ requires kubeadm.k8s.io/v1beta2
-    # We only run alpha tests against master so we don't need to maintain
-    # different patches for different Kubernetes releases.
-    if [[ -n "$gates" ]]; then
-        cat >>"${CSI_PROW_WORK}/kind-config.yaml" <<EOF
-kubeadmConfigPatches:
-- |
-  apiVersion: kubeadm.k8s.io/v1beta2
-  kind: ClusterConfiguration
-  metadata:
-    name: config
-  apiServer:
-    extraArgs:
-      "feature-gates": "$gates"
-  controllerManager:
-    extraArgs:
-      "feature-gates": "$gates"
-  scheduler:
-    extraArgs:
-      "feature-gates": "$gates"
-- |
-  apiVersion: kubeadm.k8s.io/v1beta2
-  kind: InitConfiguration
-  metadata:
-    name: config
-  nodeRegistration:
-    kubeletExtraArgs:
-      "feature-gates": "$gates"
-- |
-  apiVersion: kubelet.config.k8s.io/v1beta1
-  kind: KubeletConfiguration
-  metadata:
-    name: config
-  featureGates:
-$(list_gates "$gates")
-- |
-  apiVersion: kubeproxy.config.k8s.io/v1alpha1
-  kind: KubeProxyConfiguration
-  metadata:
-    name: config
-  featureGates:
 $(list_gates "$gates")
+runtimeConfig:
+$(list_api_groups "$gates")
 EOF
-    fi
 
     info "kind-config.yaml:"
     cat "${CSI_PROW_WORK}/kind-config.yaml"
@@ -718,7 +692,7 @@ install_csi_driver () {
 # Installs all nessesary snapshotter CRDs  
 install_snapshot_crds() {
   # Wait until volumesnapshot CRDs are in place.
-  CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/config/crd"
+  CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd"
   kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotclasses.yaml" --validate=false
   kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshots.yaml" --validate=false
   kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotcontents.yaml" --validate=false
@@ -1101,14 +1075,8 @@ main () {
             start_cluster || die "starting the non-alpha cluster failed"
 
             # Install necessary snapshot CRDs and snapshot controller
-            # For Kubernetes 1.17+, we will install the CRDs and snapshot controller.
+            install_snapshot_crds
-            if version_gt "${CSI_PROW_KUBERNETES_VERSION}" "1.16.255" || "${CSI_PROW_KUBERNETES_VERSION}" == "latest"; then
+            install_snapshot_controller
-                info "Version ${CSI_PROW_KUBERNETES_VERSION}, installing CRDs and snapshot controller"
-                install_snapshot_crds
-                install_snapshot_controller
-            else
-                info "Version ${CSI_PROW_KUBERNETES_VERSION}, skipping CRDs and snapshot controller"
-            fi
 
             # Installing the driver might be disabled.
             if ${CSI_PROW_DRIVER_INSTALL} "$images"; then
@@ -1158,14 +1126,8 @@ main () {
             start_cluster "${CSI_PROW_E2E_ALPHA_GATES}" || die "starting alpha cluster failed"
 
             # Install necessary snapshot CRDs and snapshot controller
-            # For Kubernetes 1.17+, we will install the CRDs and snapshot controller.
+            install_snapshot_crds
-            if version_gt "${CSI_PROW_KUBERNETES_VERSION}" "1.16.255" || "${CSI_PROW_KUBERNETES_VERSION}" == "latest"; then
+            install_snapshot_controller
-                info "Version ${CSI_PROW_KUBERNETES_VERSION}, installing CRDs and snapshot controller"
-                install_snapshot_crds
-                install_snapshot_controller
-            else
-                info "Version ${CSI_PROW_KUBERNETES_VERSION}, skipping CRDs and snapshot controller"
-            fi
 
             # Installing the driver might be disabled.
             if ${CSI_PROW_DRIVER_INSTALL} "$images"; then