Squashed 'release-tools/' changes from 3041b8a4..4aff857d

4aff857d Merge pull request #109 from pohly/alpha-test-defaults 0427289d Merge pull request #110 from pohly/kind-0.9-bazel-build-workaround 9a370ab9 prow.sh: work around "kind build node-image" failure 522361ec prow.sh: only run alpha tests for latest Kubernetes release 22c0395c Merge pull request #108 from bnrjee/master b5b447b5 Add go ldflags using LDFLAGS at the time of compilation 16f4afbd Merge pull request #107 from pohly/kind-update 7bcee13d prow.sh: update to kind 0.9, support Kubernetes 1.19 df518fbd prow.sh: usage of Bazel optional c3afd427 Merge pull request #104 from xing-yang/snapshot dde93b22 Update to snapshot-controller v3.0.0 a0f195cc Merge pull request #106 from msau42/fix-canary 7100c120 Only set staging registry when running canary job b3c65f9c Merge pull request #99 from msau42/add-release-process e53f3e85 Merge pull request #103 from msau42/fix-canary d1294628 Document new method for adding CI jobs are new K8s versions e73c2ce5 Use staging registry for canary tests 2c098465 Add cleanup instructions to release-notes generation 60e1cd3d Merge pull request #98 from pohly/kubernetes-1-19-fixes 0979c091 prow.sh: fix E2E suite for Kubernetes >= 1.18 3b4a2f1d prow.sh: fix installing Go for Kubernetes 1.19.0 1fbb636c Merge pull request #97 from pohly/go-1.15 82d108ac switch to Go 1.15 d8a25300 Merge pull request #95 from msau42/add-release-process 843bddca Add steps on promoting release images 0345a835 Merge pull request #94 from linux-on-ibm-z/bump-timeout 1fdf2d53 cloud build: bump timeout in Prow job 41ec6d15 Merge pull request #93 from animeshk08/patch-1 5a54e67d filter-junit: Fix gofmt error 0676fcbd Merge pull request #92 from animeshk08/patch-1 36ea4ffa filter-junit: Fix golint error f5a42037 Merge pull request #91 from cyb70289/arm64 43e50d6f prow.sh: enable building arm64 image 0d5bd843 Merge pull request #90 from pohly/k8s-staging-sig-storage 3df86b7d cloud build: k8s-staging-sig-storage c5fd9610 Merge pull request #89 from pohly/cloud-build-binfmt db0c2a7d cloud build: initialize support for running commands in Dockerfile be902f40 Merge pull request #88 from pohly/multiarch-windows-fix 340e082f build.make: optional inclusion of Windows in multiarch images 5231f05d build.make: properly declare push-multiarch 4569f27a build.make: fix push-multiarch ambiguity 17dde9ef Merge pull request #87 from pohly/cloud-build bd416901 cloud build: initial set of shared files 9084fecb Merge pull request #81 from msau42/add-release-process 6f2322e8 Update patch release notes generation command 0fcc3b1b Merge pull request #78 from ggriffiths/fix_csi_snapshotter_rbac_version_set d8c76fee Support local snapshot RBAC for pull jobs c1bdf5bf Merge pull request #80 from msau42/add-release-process ea1f94aa update release tools instructions 152396e2 Merge pull request #77 from ggriffiths/snapshotter201_update 7edc1461 Update snapshotter to version 2.0.1 4cf843f6 Merge pull request #76 from pohly/build-targets 3863a0f6 build for multiple platforms only in CI, add s390x 8322a7d0 Merge pull request #72 from pohly/hostpath-update 7c5a89c8 prow.sh: use 1.3.0 hostpath driver for testing b8587b2b Merge pull request #71 from wozniakjan/test-vet fdb32183 Change 'make test-vet' to call 'go vet' d717c8c4 Merge pull request #69 from pohly/test-driver-config a1432bc9 Merge pull request #70 from pohly/kubelet-feature-gates 5f74333a prow.sh: also configure feature gates for kubelet 84f78b12 prow.sh: generic driver installation 3c34b4f2 Merge pull request #67 from windayski/fix-link fa90abd0 fix incorrect link ff3cc3f1 Merge pull request #54 from msau42/add-release-process ac8a0212 Document the process for releasing a new sidecar 23be6525 Merge pull request #65 from msau42/update-hostpath 6582f2ff Update hostpath driver version to get fix for connection-timeout 4cc91745 Merge pull request #64 from ggriffiths/snapshotter_2_version_update 8191eab6 Update snapshotter to version v2.0.0 3c463fb1 Merge pull request #61 from msau42/enable-snapshots 8b0316c7 Fix overriding of junit results by using unique names for each e2e run 5f444b80 Merge pull request #60 from saad-ali/updateHostpathVersion af9549b5 Update prow hostpath driver version to 1.3.0-rc2 f6c74b30 Merge pull request #57 from ggriffiths/version_gt_kubernetes_fix fc809759 Fix version_gt to work with kubernetes prefix 9f1f3dd8 Merge pull request #56 from msau42/enable-snapshots b98b2aed Enable snapshot tests in 1.17 to be run in non-alpha jobs. 9ace0204 Merge pull request #52 from msau42/update-readme 540599ba Merge pull request #53 from msau42/fix-make a4e62996 fix syntax for ppc64le build 771ca6f2 Merge pull request #49 from ggriffiths/prowsh_improve_version_gt d7c69d2f Merge pull request #51 from msau42/enable-multinode 4ad69492 Improve snapshot pod running checks and improve version_gt 53888ae7 Improve README by adding an explicit Kubernetes dependency section 9a7a685e Create a kind cluster with two worker nodes so that the topology feature can be tested. Test cases that test accessing volumes from multiple nodes need to be skipped 4ff2f5f0 Merge pull request #50 from darkowlzz/kind-0.6.0 80bba1fe Use kind v0.6.0 6d674a7f Merge pull request #47 from Pensu/multi-arch 8adde494 Merge pull request #45 from ggriffiths/snapshot_beta_crds 003c14b2 Add snapshotter CRDs after cluster setup a41f3860 Merge pull request #46 from mucahitkurt/kind-cluster-cleanup 1eaaaa1c Delete kind cluster after tests run. 83a4ef15 Adding build for ppc64le 4fcafece Merge pull request #43 from pohly/system-pod-logging f41c1351 prow.sh: also log output of system containers ee22a9ca Merge pull request #42 from pohly/use-vendor-dir 80678456 travis.yml: also use vendor directory 23df4aef prow.sh: use vendor directory if available a53bd4c4 Merge pull request #41 from pohly/go-version c8a1c4af better handling of Go version 5e773d2d update CI to use Go 1.13.3 f419d745 Merge pull request #40 from msau42/add-1.16 e0fde8c4 Add new variables for 1.16 and remove 1.13 adf00fea Merge pull request #36 from msau42/full-clone f1697d2c Do full git clones in travis. Shallow clones are causing test-subtree errors when the depth is exactly 50. 2c819198 Merge pull request #34 from pohly/go-mod-tidy 518d6af6 Merge pull request #35 from ddebroy/winbld2 2d6b3ce8 Build Windows only for amd64 c1078a65 go-get-kubernetes.sh: automate Kubernetes dependency handling 194289aa update Go mod support 0affdf95 Merge pull request #33 from gnufied/enable-hostpath-expansion 6208f6ab Enable hostpath expansion 6ecaa76e Merge pull request #30 from msau42/fix-windows ea2f1b52 build windows binaries with .exe suffix 2d335506 Merge pull request #29 from mucahitkurt/create-2-node-kind-cluster a8ea8bcc create 2-node kind cluster since topology support is added to hostpath driver df8530d9 Merge pull request #27 from pohly/dep-vendor-check 35ceaedc prow.sh: install dep if needed f85ab5af Merge pull request #26 from ddebroy/windows1 9fba09b4 Add rule for building Windows binaries 04008676 Merge pull request #25 from msau42/fix-master-jobs dc0a5d83 Update kind to v0.5.0 aa85b82c Merge pull request #23 from msau42/fix-master-jobs f46191d9 Kubernetes master changed the way that releases are tagged, which needed changes to kind. There are 3 changes made to prow.sh: 1cac3af3 Merge pull request #22 from msau42/add-1.15-jobs 0c0dc300 prow.sh: tag master images with a large version number f4f73cef Merge pull request #21 from msau42/add-1.15-jobs 4e31f078 Change default hostpath driver name to hostpath.csi.k8s.io 4b6fa4a0 Update hostpath version for sidecar testing to v1.2.0-rc2 ecc79187 Update kind to v0.4.0. This requires overriding Kubernetes versions with specific patch versions that kind 0.4.0 supports. Also, feature gate setting is only supported on 1.15+ due to kind.sigs.k8s.io/v1alpha3 and kubeadm.k8s.io/v1beta2 dependencies. a6f21d40 Add variables for 1.15 db8abb6e Merge pull request #20 from pohly/test-driver-config b2f4e051 prow.sh: flexible test driver config 03999882 Merge pull request #19 from pohly/go-mod-vendor 066143d1 build.make: allow repos to use 'go mod' for vendoring 0bee7493 Merge pull request #18 from pohly/go-version e157b6b5 update to Go 1.12.4 88dc9a47 Merge pull request #17 from pohly/prow 0fafc663 prow.sh: skip sanity testing if component doesn't support it bcac1c1f Merge pull request #16 from pohly/prow 0b10f6a4 prow.sh: update csi-driver-host-path 0c2677e8 Merge pull request #15 from pengzhisun/master ff9bce4a Replace 'return' to 'exit' to fix shellcheck error c60f3823 Merge pull request #14 from pohly/prow 7aaac225 prow.sh: remove AllAlpha=all, part II 66177736 Merge pull request #13 from pohly/prow cda2fc58 prow.sh: avoid AllAlpha=true 546d5504 prow.sh: debug failing KinD cluster creation 9b0d9cd7 build.make: skip shellcheck if Docker is not available aa45a1cd prow.sh: more efficient execution of individual tests f3d1d2df prow.sh: fix hostpath driver version check 31dfaf31 prow.sh: fix running of just "alpha" tests f5014439 prow.sh: AllAlpha=true for unknown Kubernetes versions 95ae9de9 Merge pull request #9 from pohly/prow d87eccb4 prow.sh: switch back to upstream csi-driver-host-path 6602d38b prow.sh: different E2E suite depending on Kubernetes version 741319bd prow.sh: improve building Kubernetes from source 29545bb0 prow.sh: take Go version from Kubernetes source 429581c5 prow.sh: pull Go version from travis.yml 0a0fd49b prow.sh: comment clarification 2069a0af Merge pull request #11 from pohly/verify-shellcheck 55212ff2 initial Prow test job 6c7ba1be build.make: integrate shellcheck into "make test" b2d25d4f verify-shellcheck.sh: make it usable in csi-release-tools 3b6af7b1 Merge pull request #12 from pohly/local-e2e-suite 104a1ac9 build.make: avoid unit-testing E2E test suite 34010e75 Merge pull request #10 from pohly/vendor-check e6db50df check vendor directory fb13c519 verify-shellcheck.sh: import from Kubernetes 94fc1e31 build.make: avoid unit-testing E2E test suite 849db0ad Merge pull request #8 from pohly/subtree-check-relax cc564f92 verify-subtree.sh: relax check and ignore old content 33d58fdc Merge pull request #5 from pohly/test-enhancements be8a4400 Merge pull request #4 from pohly/canary-fix b0336b55 build.make: more readable "make test" output 09436b9f build.make: fix pushing of "canary" image from master branch 147892c9 build.make: support suppressing checks 154e33d4 build.make: clarify usage of "make V=1" git-subtree-dir: release-tools git-subtree-split: 4aff857d88149e07951fcd1322f462f765401a86
2020-10-27 18:21:34 +01:00
parent 3041b8a425
commit 91bc0c8f98
16 changed files with 2313 additions and 18 deletions
--- a/.prow.sh
+++ b/.prow.sh
@@ -0,0 +1,7 @@
+#! /bin/bash -e
+#
+# This is for testing csi-release-tools itself in Prow. All other
+# repos use prow.sh for that, but as csi-release-tools isn't a normal
+# repo with some Go code in it, it has a custom Prow test script.
+
+./verify-shellcheck.sh "$(pwd)"
--- a/README.md
+++ b/README.md
@@ -49,3 +49,118 @@ Cheat sheet:
 - `git subtree add --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - add release tools to a repo which does not have them yet (only once)
 - `git subtree pull --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - update local copy to latest upstream (whenever upstream changes)
 - edit, `git commit`, `git subtree push --prefix=release-tools git@github.com:<user>/csi-release-tools.git <my-new-or-existing-branch>` - push to a new branch before submitting a PR
+
+verify-shellcheck.sh
+--------------------
+
+The [verify-shellcheck.sh](./verify-shellcheck.sh) script in this repo
+is a stripped down copy of the [corresponding
+script](https://github.com/kubernetes/kubernetes/blob/release-1.14/hack/verify-shellcheck.sh)
+in the Kubernetes repository. It can be used to check for certain
+errors shell scripts, like missing quotation marks. The default
+`test-shellcheck` target in [build.make](./build.make) only checks the
+scripts in this directory. Components can add more directories to
+`TEST_SHELLCHECK_DIRS` to check also other scripts.
+
+End-to-end testing
+------------------
+
+A repo that wants to opt into testing via Prow must set up a top-level
+`.prow.sh`. Typically that will source `prow.sh` and then transfer
+control to it:
+
+``` bash
+#! /bin/bash -e
+
+. release-tools/prow.sh
+main
+```
+
+All Kubernetes-CSI repos are expected to switch to Prow. For details
+on what is enabled in Prow, see
+https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-csi
+
+Test results for periodic jobs are visible in
+https://testgrid.k8s.io/sig-storage-csi-ci
+
+It is possible to reproduce the Prow testing locally on a suitable machine:
+- Linux host
+- Docker installed
+- code to be tested checkout out in `$GOPATH/src/<import path>`
+- `cd $GOPATH/src/<import path> && ./.prow.sh`
+
+Beware that the script intentionally doesn't clean up after itself and
+modifies the content of `$GOPATH`, in particular the `kubernetes` and
+`kind` repositories there. Better run it in an empty, disposable
+`$GOPATH`.
+
+When it terminates, the following command can be used to get access to
+the Kubernetes cluster that was brought up for testing (assuming that
+this step succeeded):
+
+    export KUBECONFIG="$(kind get kubeconfig-path --name="csi-prow")"
+
+It is possible to control the execution via environment variables. See
+`prow.sh` for details. Particularly useful is testing against different
+Kubernetes releases:
+
+    CSI_PROW_KUBERNETES_VERSION=1.13.3 ./.prow.sh
+    CSI_PROW_KUBERNETES_VERSION=latest ./.prow.sh
+
+Dependencies and vendoring
+--------------------------
+
+Most projects will (eventually) use `go mod` to manage
+dependencies. `dep` is also still supported by `csi-release-tools`,
+but not documented here because it's not recommended anymore.
+
+The usual instructions for using [go
+modules](https://github.com/golang/go/wiki/Modules) apply. Here's a cheat sheet
+for some of the relevant commands:
+- list available updates: `GO111MODULE=on go list -u -m all`
+- update or add a single dependency: `GO111MODULE=on go get <package>`
+- update all dependencies to their next minor or patch release:
+  `GO111MODULE=on go get ./...` (add `-u=patch` to limit to patch
+  releases)
+- lock onto a specific version: `GO111MODULE=on go get <package>@<version>`
+- clean up `go.mod`: `GO111MODULE=on go mod tidy`
+- update vendor directory: `GO111MODULE=on go mod vendor`
+
+`GO111MODULE=on` can be left out when using Go >= 1.13 or when the
+source is checked out outside of `$GOPATH`.
+
+`go mod tidy` must be used to ensure that the listed dependencies are
+really still needed. Changing import statements or a tentative `go
+get` can result in stale dependencies.
+
+The `test-vendor` verifies that it was used when run locally or in a
+pre-merge CI job. If a `vendor` directory is present, it will also
+verify that it's content is up-to-date.
+
+The `vendor` directory is optional. It is still present in projects
+because it avoids downloading sources during CI builds. If this is no
+longer deemed necessary, then a project can also remove the directory.
+
+Conversion of a repository that uses `dep` to `go mod` can be done with:
+
+    GO111MODULE=on go mod init
+    release-tools/go-get-kubernetes.sh <current Kubernetes version from Gopkg.toml>
+    GO111MODULE=on go mod tidy
+    GO111MODULE=on go mod vendor
+    git rm -f Gopkg.toml Gopkg.lock
+    git add go.mod go.sum vendor
+
+### Updating Kubernetes dependencies
+
+When using packages that are part of the Kubernetes source code, the
+commands above are not enough because the [lack of semantic
+versioning](https://github.com/kubernetes/kubernetes/issues/72638)
+prevents `go mod` from finding newer releases. Importing directly from
+`kubernetes/kubernetes` also needs `replace` statements to override
+the fake `v0.0.0` versions
+(https://github.com/kubernetes/kubernetes/issues/79384). The
+`go-get-kubernetes.sh` script can be used to update all packages in
+lockstep to a different Kubernetes version. Example usage:
+```
+$ ./release-tools/go-get-kubernetes.sh 1.16.4
+```
--- a/SIDECAR_RELEASE_PROCESS.md
+++ b/SIDECAR_RELEASE_PROCESS.md
@@ -0,0 +1,106 @@
+# Sidecar Release Process
+
+This page describes the process for releasing a kubernetes-csi sidecar.
+
+## Prerequisites
+
+The release manager must:
+
+* Be a member of the kubernetes-csi organization. Open an
+  [issue](https://github.com/kubernetes/org/issues/new?assignees=&labels=area%2Fgithub-membership&template=membership.md&title=REQUEST%3A+New+membership+for+%3Cyour-GH-handle%3E) in
+  kubernetes/org to request membership
+* Be a top level approver for the repository. To become a top level approver,
+  the candidate must demonstrate ownership and deep knowledge of the repository
+  through active maintainence, responding to and fixing issues, reviewing PRs,
+  test triage.
+* Be part of the maintainers or admin group for the repository. admin is a
+  superset of maintainers, only maintainers level is required for cutting a
+  release.  Membership can be requested by submitting a PR to kubernetes/org.
+  [Example](https://github.com/kubernetes/org/pull/1467)
+
+## Updating CI Jobs
+Whenever a new Kubernetes minor version is released, our kubernetes-csi CI jobs
+must be updated.
+
+[Our CI jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) have the
+naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
+
+1. Jobs should be actively monitored to find and fix failures in sidecars and
+   infrastructure changes early in the development cycle. Test failures are sent
+   to kubernetes-sig-storage-test-failures@googlegroups.com.
+1. "-on-master" jobs are the closest reflection to the new Kubernetes version.
+1. Fixes to our prow.sh CI script can be tested in the [CSI hostpath
+   repo](https://github.com/kubernetes-csi/csi-driver-host-path) by modifying
+   [prow.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/master/release-tools/prow.sh)
+   along with any overrides in
+   [.prow.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/master/.prow.sh)
+   to mirror the failing environment. Once e2e tests are passing (verify-unit tests
+   will fail), then the prow.sh changes can be submitted to [csi-release-tools](https://github.com/kubernetes-csi/csi-release-tools).
+1. Changes can then be updated in all the sidecar repos and hostpath driver repo
+   by following the [update
+   instructions](https://github.com/kubernetes-csi/csi-release-tools/blob/master/README.md#sharing-and-updating).
+1. New pull and CI jobs are configured by adding new K8s versions to the top of
+   [gen-jobs.sh](https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-csi/gen-jobs.sh).
+   New pull jobs that have been unverified should be initially made optional by
+   setting the new K8s version as
+   [experimental](https://github.com/kubernetes/test-infra/blob/a1858f46d6014480b130789df58b230a49203a64/config/jobs/kubernetes-csi/gen-jobs.sh#L40).
+1. Once new pull and CI jobs have been verified, and the new Kubernetes version
+   is released, we can make the optional jobs required, and also remove the
+   Kubernetes versions that are no longer supported.
+
+## Release Process
+1. Identify all issues and ongoing PRs that should go into the release, and
+  drive them to resolution.
+1. Download v2.8+ [K8s release notes
+  generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes)
+1. Generate release notes for the release. Replace arguments with the relevant
+  information.
+    * Clean up old cached information (also needed if you are generating release
+      notes for multiple repos)
+      ```bash
+      rm -rf /tmp/k8s-repo
+      ```
+    * For new minor releases on master:
+        ```bash
+        GITHUB_TOKEN=<token> release-notes --discover=mergebase-to-latest
+        --github-org=kubernetes-csi --github-repo=external-provisioner
+        --required-author="" --output out.md
+        ```
+    * For new patch releases on a release branch:
+        ```bash
+        GITHUB_TOKEN=<token> release-notes --discover=patch-to-latest --branch=release-1.1
+        --github-org=kubernetes-csi --github-repo=external-provisioner
+        --required-author="" --output out.md
+        ```
+1. Compare the generated output to the new commits for the release to check if
+   any notable change missed a release note.
+1. Reword release notes as needed. Make sure to check notes for breaking
+   changes and deprecations.
+1. If release is a new major/minor version, create a new `CHANGELOG-<major>.<minor>.md`
+   file. Otherwise, add the release notes to the top of the existing CHANGELOG
+   file for that minor version.
+1. Submit a PR for the CHANGELOG changes.
+1. Submit a PR for README changes, in particular, Compatibility, Feature status,
+   and any other sections that may need updating.
+1. Check that all [canary CI
+  jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) are passing,
+  and that test coverage is adequate for the changes that are going into the release.
+1. Make sure that no new PRs have merged in the meantime, and no PRs are in
+   flight and soon to be merged.
+1. Create a new release following a previous release as a template. Be sure to select the correct
+   branch. This requires Github release permissions as required by the prerequisites.
+   [external-provisioner example](https://github.com/kubernetes-csi/external-provisioner/releases/new)
+1. If release was a new major/minor version, create a new `release-<minor>`
+   branch at that commit.
+1. Check [image build status](https://k8s-testgrid.appspot.com/sig-storage-image-build).
+1. Promote images from k8s-staging-sig-storage to k8s.gcr.io/sig-storage. From
+   the [k8s image
+   repo](https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage),
+   run `./generate.sh > images.yaml`, and send a PR with the updated images.
+   Once merged, the image promoter will copy the images from staging to prod.
+1. Update [kubernetes-csi/docs](https://github.com/kubernetes-csi/docs) sidecar
+   and feature pages with the new released version.
+1. After all the sidecars have been released, update
+   CSI hostpath driver with the new sidecars in the [CSI repo](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy)
+   and [k/k
+   in-tree](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/testing-manifests/storage-csi/hostpath/hostpath)
--- a/build.make
+++ b/build.make
@@ -22,6 +22,9 @@
 # including build.make.
 REGISTRY_NAME=quay.io/k8scsi

+# Can be set to -mod=vendor to ensure that the "vendor" directory is used.
+GOFLAGS_VENDOR=
+
 # Revision that gets built into each binary via the main.version
 # string. Uses the `git describe` output based on the most recent
 # version tag with a short revision suffix or, if nothing has been
@@ -51,26 +54,48 @@ IMAGE_TAGS+=$(shell tagged="$$(git describe --tags --match='v*' --abbrev=0)"; if
 IMAGE_NAME=$(REGISTRY_NAME)/$*

 ifdef V
+# Adding "-alsologtostderr" assumes that all test binaries contain glog. This is not guaranteed.
 TESTARGS = -v -args -alsologtostderr -v 5
 else
 TESTARGS =
 endif

-build-%:
-	mkdir -p bin
-	CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$*
+# Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables
+# to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below.

-container-%: build-%
+# BUILD_PLATFORMS contains a set of <os> <arch> <suffix> triplets,
+# separated by semicolon. An empty variable or empty entry (= just a
+# semicolon) builds for the default platform of the current Go
+# toolchain.
+BUILD_PLATFORMS =
+
+# Add go ldflags using LDFLAGS at the time of compilation.
+IMPORTPATH_LDFLAGS = -X main.version=$(REV) 
+EXT_LDFLAGS = -extldflags "-static"
+LDFLAGS = 
+FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS)
+# This builds each command (= the sub-directories of ./cmd) for the target platform(s)
+# defined by BUILD_PLATFORMS.
+$(CMDS:%=build-%): build-%: check-go-version-go
+	mkdir -p bin
+	echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \
+		if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \
+			echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \
+			exit 1; \
+		fi; \
+	done
+
+$(CMDS:%=container-%): container-%: build-%
 	docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) .

-push-%: container-%
+$(CMDS:%=push-%): push-%: container-%
 	set -ex; \
 	push_image () { \
 		docker tag $*:latest $(IMAGE_NAME):$$tag; \
 		docker push $(IMAGE_NAME):$$tag; \
 	}; \
 	for tag in $(IMAGE_TAGS); do \
-		if echo $$tag | grep -q -e '-canary$$'; then \
+		if [ "$$tag" = "canary" ] || echo "$$tag" | grep -q -e '-canary$$'; then \
 			: "creating or overwriting canary image"; \
 			push_image; \
 		elif docker pull $(IMAGE_NAME):$$tag 2>&1 | tee /dev/stderr | grep -q "manifest for $(IMAGE_NAME):$$tag not found"; then \
@@ -85,36 +110,168 @@ build: $(CMDS:%=build-%)
 container: $(CMDS:%=container-%)
 push: $(CMDS:%=push-%)

+# Additional parameters are needed when pushing to a local registry,
+# see https://github.com/docker/buildx/issues/94.
+# However, that then runs into https://github.com/docker/cli/issues/2396.
+#
+# What works for local testing is:
+# make push-multiarch PULL_BASE_REF=master REGISTRY_NAME=<your account on dockerhub.io> BUILD_PLATFORMS="linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x"
+DOCKER_BUILDX_CREATE_ARGS ?=
+
+# This target builds a multiarch image for one command using Moby BuildKit builder toolkit.
+# Docker Buildx is included in Docker 19.03.
+#
+# ./cmd/<command>/Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows].
+# It is currently optional: if no such file exists, Windows images are not included,
+# even when Windows is listed in BUILD_PLATFORMS. That way, projects can test that
+# Windows binaries can be built before adding a Dockerfile for it.
+#
+# BUILD_PLATFORMS determines which individual images are included in the multiarch image.
+# PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines
+# the tag for the resulting multiarch image.
+$(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
+	set -ex; \
+	DOCKER_CLI_EXPERIMENTAL=enabled; \
+	export DOCKER_CLI_EXPERIMENTAL; \
+	docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \
+	trap "docker buildx rm multiarchimage-buildertest" EXIT; \
+	dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \
+	dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \
+	if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \
+	if ! [ -f "$$dockerfile_windows" ]; then \
+		build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \
+	fi; \
+	pushMultiArch () { \
+		tag=$$1; \
+		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \
+			docker buildx build --push \
+				--tag $(IMAGE_NAME):$$arch-$$os-$$tag \
+				--platform=$$os/$$arch \
+				--file $$(eval echo \$${dockerfile_$$os}) \
+				--build-arg binary=./bin/$*$$suffix \
+				--label revision=$(REV) \
+				.; \
+		done; \
+		images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \
+		docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \
+		docker manifest push -p $(IMAGE_NAME):$$tag; \
+	}; \
+	if [ $(PULL_BASE_REF) = "master" ]; then \
+			: "creating or overwriting canary image"; \
+			pushMultiArch canary; \
+	elif echo $(PULL_BASE_REF) | grep -q -e 'release-*' ; then \
+			: "creating or overwriting canary image for release branch"; \
+			release_canary_tag=$$(echo $(PULL_BASE_REF) | cut -f2 -d '-')-canary; \
+			pushMultiArch $$release_canary_tag; \
+	elif docker pull $(IMAGE_NAME):$(PULL_BASE_REF) 2>&1 | tee /dev/stderr | grep -q "manifest for $(IMAGE_NAME):$(PULL_BASE_REF) not found"; then \
+			: "creating release image"; \
+			pushMultiArch $(PULL_BASE_REF); \
+	else \
+			: "ERROR: release image $(IMAGE_NAME):$(PULL_BASE_REF) already exists: a new tag is required!"; \
+			exit 1; \
+	fi
+
+.PHONY: check-pull-base-ref
+check-pull-base-ref:
+	if ! [ "$(PULL_BASE_REF)" ]; then \
+		echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name."; \
+		exit 1; \
+	fi
+
+.PHONY: push-multiarch
+push-multiarch: $(CMDS:%=push-multiarch-%)
+
 clean:
 	-rm -rf bin

-test:
+test: check-go-version-go

 .PHONY: test-go
 test: test-go
 test-go:
-	@ echo; echo $@
-	go test `go list ./... | grep -v 'vendor'` $(TESTARGS)
+	@ echo; echo "### $@:"
+	go test $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v -e 'vendor' -e '/test/e2e$$' $(TEST_GO_FILTER_CMD)` $(TESTARGS)

 .PHONY: test-vet
 test: test-vet
 test-vet:
-	@ echo; echo $@
-	go vet `go list ./... | grep -v vendor`
+	@ echo; echo "### $@:"
+	go vet $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v vendor $(TEST_VET_FILTER_CMD)`

 .PHONY: test-fmt
 test: test-fmt
 test-fmt:
-	@ echo; echo $@
-	files=$$(find . -name '*.go' | grep -v './vendor'); \
+	@ echo; echo "### $@:"
+	files=$$(find . -name '*.go' | grep -v './vendor' $(TEST_FMT_FILTER_CMD)); \
 	if [ $$(gofmt -d $$files | wc -l) -ne 0 ]; then \
 		echo "formatting errors:"; \
 		gofmt -d $$files; \
 		false; \
 	fi

+# This test only runs when dep >= 0.5 is installed, which is the case for the CI setup.
+# When using 'go mod', we allow the test to be skipped in the Prow CI under some special
+# circumstances, because it depends on accessing all remote repos and thus
+# running it all the time would defeat the purpose of vendoring:
+# - not handling a PR or
+# - the fabricated merge commit leaves go.mod, go.sum and vendor dir unchanged
+# - release-tools also didn't change (changing rules or Go version might lead to
+#   a different result and thus must be tested)
+# - import statements not changed (because if they change, go.mod might have to be updated)
+#
+# "git diff" is intelligent enough to annotate changes inside the "import" block in
+# the start of the diff hunk:
+#
+# diff --git a/rpc/common.go b/rpc/common.go
+# index bb4a5c4..5fa4271 100644
+# --- a/rpc/common.go
+# +++ b/rpc/common.go
+# @@ -21,7 +21,6 @@ import (
+#         "fmt"
+#         "time"
+#
+# -       "google.golang.org/grpc"
+#         "google.golang.org/grpc/codes"
+#         "google.golang.org/grpc/status"
+#
+# We rely on that to find such changes.
+#
+# Vendoring is optional when using go.mod.
+.PHONY: test-vendor
+test: test-vendor
+test-vendor:
+	@ echo; echo "### $@:"
+	@ ./release-tools/verify-vendor.sh
+
 .PHONY: test-subtree
 test: test-subtree
 test-subtree:
-	@ echo; echo $@
+	@ echo; echo "### $@:"
 	./release-tools/verify-subtree.sh release-tools
+
+# Components can extend the set of directories which must pass shellcheck.
+# The default is to check only the release-tools directory itself.
+TEST_SHELLCHECK_DIRS=release-tools
+.PHONY: test-shellcheck
+test: test-shellcheck
+test-shellcheck:
+	@ echo; echo "### $@:"
+	@ ret=0; \
+	if ! command -v docker; then \
+		echo "skipped, no Docker"; \
+		exit 0; \
+        fi; \
+	for dir in $(abspath $(TEST_SHELLCHECK_DIRS)); do \
+		echo; \
+		echo "$$dir:"; \
+		./release-tools/verify-shellcheck.sh "$$dir" || ret=1; \
+	done; \
+	exit $$ret
+
+# Targets in the makefile can depend on check-go-version-<path to go binary>
+# to trigger a warning if the x.y version of that binary does not match
+# what the project uses. Make ensures that this is only checked once per
+# invocation.
+.PHONY: check-go-version-%
+check-go-version-%:
+	./release-tools/verify-go-version.sh "$*"
--- a/cloudbuild.sh
+++ b/cloudbuild.sh
@@ -0,0 +1,6 @@
+#! /bin/bash
+
+# shellcheck disable=SC1091
+. release-tools/prow.sh
+
+gcr_cloud_build
--- a/cloudbuild.yaml
+++ b/cloudbuild.yaml
@@ -0,0 +1,46 @@
+# A configuration file for multi-arch image building with the Google cloud build service.
+#
+# Repos using this file must:
+# - import csi-release-tools
+# - add a symlink cloudbuild.yaml -> release-tools/cloudbuild.yaml
+# - add a .cloudbuild.sh which can be a custom file or a symlink
+#   to release-tools/cloudbuild.sh
+# - accept "binary" as build argument in their Dockerfile(s) (see
+#   https://github.com/pohly/node-driver-registrar/blob/3018101987b0bb6da2a2657de607174d6e3728f7/Dockerfile#L4-L6)
+#   because binaries will get built for different architectures and then
+#   get copied from the built host into the container image
+#
+# See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md
+# for more details on image pushing process in Kubernetes.
+#
+# To promote release images, see https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage.
+
+# This must be specified in seconds. If omitted, defaults to 600s (10 mins).
+timeout: 1800s
+# This prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF,
+# or any new substitutions added in the future.
+options:
+  substitution_option: ALLOW_LOOSE
+steps:
+  # The image must contain bash and curl. Ideally it should also contain
+  # the desired version of Go (currently defined in release-tools/travis.yml),
+  # but that just speeds up the build and is not required.
+  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8'
+    entrypoint: ./.cloudbuild.sh
+    env:
+    - GIT_TAG=${_GIT_TAG}
+    - PULL_BASE_REF=${_PULL_BASE_REF}
+    - REGISTRY_NAME=gcr.io/${_STAGING_PROJECT}
+    - HOME=/root
+substitutions:
+  # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and
+  # can be used as a substitution.
+  _GIT_TAG: '12345'
+  # _PULL_BASE_REF will contain the ref that was pushed to trigger this build -
+  # a branch like 'master' or 'release-0.2', or a tag like 'v0.2'.
+  _PULL_BASE_REF: 'master'
+  # The default gcr.io staging project for Kubernetes-CSI
+  # (=> https://console.cloud.google.com/gcr/images/k8s-staging-sig-storage/GLOBAL).
+  # Might be overridden in the Prow build job for a repo which wants
+  # images elsewhere.
+  _STAGING_PROJECT: 'k8s-staging-sig-storage'
--- a/filter-junit.go
+++ b/filter-junit.go
@@ -0,0 +1,133 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+This command filters a JUnit file such that only tests with a name
+matching a regular expression are passed through. By concatenating
+multiple input files it is possible to merge them into a single file.
+*/
+package main
+
+import (
+	"encoding/xml"
+	"flag"
+	"io/ioutil"
+	"os"
+	"regexp"
+)
+
+var (
+	output = flag.String("o", "-", "junit file to write, - for stdout")
+	tests  = flag.String("t", "", "regular expression matching the test names that are to be included in the output")
+)
+
+/*
+ * TestSuite represents a JUnit file. Due to how encoding/xml works, we have
+ * represent all fields that we want to be passed through. It's therefore
+ * not a complete solution, but good enough for Ginkgo + Spyglass.
+ */
+type TestSuite struct {
+	XMLName   string     `xml:"testsuite"`
+	TestCases []TestCase `xml:"testcase"`
+}
+
+type TestCase struct {
+	Name      string     `xml:"name,attr"`
+	Time      string     `xml:"time,attr"`
+	SystemOut string     `xml:"system-out,omitempty"`
+	Failure   string     `xml:"failure,omitempty"`
+	Skipped   SkipReason `xml:"skipped,omitempty"`
+}
+
+// SkipReason deals with the special <skipped></skipped>:
+// if present, we must re-encode it, even if empty.
+type SkipReason string
+
+func (s *SkipReason) UnmarshalText(text []byte) error {
+	*s = SkipReason(text)
+	if *s == "" {
+		*s = " "
+	}
+	return nil
+}
+
+func (s SkipReason) MarshalText() ([]byte, error) {
+	if s == " " {
+		return []byte{}, nil
+	}
+	return []byte(s), nil
+}
+
+func main() {
+	var junit TestSuite
+	var data []byte
+
+	flag.Parse()
+
+	re := regexp.MustCompile(*tests)
+
+	// Read all input files.
+	for _, input := range flag.Args() {
+		if input == "-" {
+			if _, err := os.Stdin.Read(data); err != nil {
+				panic(err)
+			}
+		} else {
+			var err error
+			data, err = ioutil.ReadFile(input)
+			if err != nil {
+				panic(err)
+			}
+		}
+		if err := xml.Unmarshal(data, &junit); err != nil {
+			panic(err)
+		}
+	}
+
+	// Keep only matching testcases. Testcases skipped in all test runs are only stored once.
+	filtered := map[string]TestCase{}
+	for _, testcase := range junit.TestCases {
+		if !re.MatchString(testcase.Name) {
+			continue
+		}
+		entry, ok := filtered[testcase.Name]
+		if !ok || // not present yet
+			entry.Skipped != "" && testcase.Skipped == "" { // replaced skipped test with real test run
+			filtered[testcase.Name] = testcase
+		}
+	}
+	junit.TestCases = nil
+	for _, testcase := range filtered {
+		junit.TestCases = append(junit.TestCases, testcase)
+	}
+
+	// Re-encode.
+	data, err := xml.MarshalIndent(junit, "", "  ")
+	if err != nil {
+		panic(err)
+	}
+
+	// Write to output.
+	if *output == "-" {
+		if _, err := os.Stdout.Write(data); err != nil {
+			panic(err)
+		}
+	} else {
+		if err := ioutil.WriteFile(*output, data, 0644); err != nil {
+			panic(err)
+		}
+	}
+}
--- a/go-get-kubernetes.sh
+++ b/go-get-kubernetes.sh
@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This script can be used while converting a repo from "dep" to "go mod"
+# by calling it after "go mod init" or to update the Kubernetes packages
+# in a repo that has already been converted. Only packages that are
+# part of kubernetes/kubernetes and thus part of a Kubernetes release
+# are modified. Other k8.io packages (like k8s.io/klog, k8s.io/utils)
+# need to be updated separately.
+
+set -o pipefail
+
+cmd=$0
+
+function help () {
+    echo "$cmd <kubernetes version = x.y.z> - update all components from kubernetes/kubernetes to that version"
+}
+
+if [ $# -ne 1 ]; then
+    help
+    exit 1
+fi
+case "$1" in -h|--help|help) help; exit 0;; esac
+
+die () {
+    echo >&2 "$@"
+    exit 1
+}
+
+k8s="$1"
+
+# If the repo imports k8s.io/kubernetes (directly or indirectly), then
+# "go mod" will try to find "v0.0.0" versions because
+# k8s.io/kubernetes has those in it's go.mod file
+# (https://github.com/kubernetes/kubernetes/blob/2bd9643cee5b3b3a5ecbd3af49d09018f0773c77/go.mod#L146-L157).
+# (https://github.com/kubernetes/kubernetes/issues/79384).
+#
+# We need to replicate the replace statements to override those fake
+# versions also in our go.mod file (idea and some code from
+# https://github.com/kubernetes/kubernetes/issues/79384#issuecomment-521493597).
+mods=$( (set -x; curl --silent --show-error --fail "https://raw.githubusercontent.com/kubernetes/kubernetes/v${k8s}/go.mod") |
+          sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p'
+   ) || die "failed to determine Kubernetes staging modules"
+for mod in $mods; do
+    # The presence of a potentially incomplete go.mod file affects this command,
+    # so move elsewhere.
+    modinfo=$(set -x; cd /; env GO111MODULE=on go mod download -json "$mod@kubernetes-${k8s}") ||
+        die "failed to determine version of $mod: $modinfo"
+    v=$(echo "$modinfo" | sed -n 's|.*"Version": "\(.*\)".*|\1|p')
+    (set -x; env GO111MODULE=on go mod edit "-replace=$mod=$mod@$v") || die "'go mod edit' failed"
+done
+
+packages=
+
+# Beware that we have to work with packages, not modules (i.e. no -m
+# flag), because some modules trigger a "no Go code except tests"
+# error.  Getting their packages works.
+if ! packages=$( (set -x; env GO111MODULE=on go list all) | grep ^k8s.io/ | sed -e 's; *;;'); then
+    cat >&2 <<EOF
+
+Warning: "GO111MODULE=on go list all" failed, trying individual packages instead.
+
+EOF
+    if ! packages=$( (set -x; env GO111MODULE=on go list -f '{{ join .Deps "\n" }}' ./...) | grep ^k8s.io/); then
+        cat >&2 <<EOF
+
+ERROR: could not obtain package list, both of these commands failed:
+       GO111MODULE=on go list all
+       GO111MODULE=on go list -f '{{ join .Deps "\n" }}' ./pkg/...
+EOF
+        exit 1
+    fi
+fi
+
+deps=
+for package in $packages; do
+    # Some k8s.io packages do not come from Kubernetes staging and
+    # thus have different versioning (or none at all...). We need to
+    # skip those.  We know what packages are from staging because we
+    # now have "replace" statements for them in go.mod.
+    #
+    # shellcheck disable=SC2001
+    module=$(echo "$package" | sed -e 's;k8s.io/\([^/]*\)/.*;k8s.io/\1;')
+    if grep -q -w "$module *=>" go.mod; then
+        deps="$deps $(echo "$package" | sed -e "s;\$;@kubernetes-$k8s;" -e 's;^k8s.io/kubernetes\(/.*\)@kubernetes-;k8s.io/kubernetes\1@v;')"
+    fi
+done
+
+# shellcheck disable=SC2086
+(set -x; env GO111MODULE=on go get $deps 2>&1) || die "go get failed"
+echo "SUCCESS"
--- a/prow.sh
+++ b/prow.sh
--- a/travis.yml
+++ b/travis.yml
@@ -2,13 +2,20 @@ language: go
 sudo: required
 services:
  - docker
+git:
+  depth: false
 matrix:
  include:
-  - go: 1.11.1
+  - go: 1.15
+before_script:
+- mkdir -p bin
+- wget https://github.com/golang/dep/releases/download/v0.5.1/dep-linux-amd64 -O bin/dep
+- chmod u+x bin/dep
+- export PATH=$PWD/bin:$PATH
 script:
- make -k all test
+- make -k all test GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' )
 after_success:
  - if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then
      docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}" quay.io;
-      make push;
+      make push GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' );
    fi
--- a/update-vendor.sh
+++ b/update-vendor.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ -f Gopkg.toml ]; then
+    echo "Repo uses 'dep' for vendoring."
+    (set -x; dep ensure)
+elif [ -f go.mod ]; then
+    release-tools/verify-go-version.sh "go"
+    (set -x; env GO111MODULE=on go mod tidy && env GO111MODULE=on go mod vendor)
+fi
--- a/util.sh
+++ b/util.sh
@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+
+# Copyright 2014 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function kube::util::sourced_variable {
+  # Call this function to tell shellcheck that a variable is supposed to
+  # be used from other calling context. This helps quiet an "unused
+  # variable" warning from shellcheck and also document your code.
+  true
+}
+
+kube::util::sortable_date() {
+  date "+%Y%m%d-%H%M%S"
+}
+
+# arguments: target, item1, item2, item3, ...
+# returns 0 if target is in the given items, 1 otherwise.
+kube::util::array_contains() {
+  local search="$1"
+  local element
+  shift
+  for element; do
+    if [[ "${element}" == "${search}" ]]; then
+      return 0
+     fi
+  done
+  return 1
+}
+
+# Example:  kube::util::trap_add 'echo "in trap DEBUG"' DEBUG
+# See: http://stackoverflow.com/questions/3338030/multiple-bash-traps-for-the-same-signal
+kube::util::trap_add() {
+  local trap_add_cmd
+  trap_add_cmd=$1
+  shift
+
+  for trap_add_name in "$@"; do
+    local existing_cmd
+    local new_cmd
+
+    # Grab the currently defined trap commands for this trap
+    existing_cmd=$(trap -p "${trap_add_name}" |  awk -F"'" '{print $2}')
+
+    if [[ -z "${existing_cmd}" ]]; then
+      new_cmd="${trap_add_cmd}"
+    else
+      new_cmd="${trap_add_cmd};${existing_cmd}"
+    fi
+
+    # Assign the test. Disable the shellcheck warning telling that trap
+    # commands should be single quoted to avoid evaluating them at this
+    # point instead evaluating them at run time. The logic of adding new
+    # commands to a single trap requires them to be evaluated right away.
+    # shellcheck disable=SC2064
+    trap "${new_cmd}" "${trap_add_name}"
+  done
+}
+
+kube::util::download_file() {
+  local -r url=$1
+  local -r destination_file=$2
+
+  rm "${destination_file}" 2&> /dev/null || true
+
+  for i in $(seq 5)
+  do
+    if ! curl -fsSL --retry 3 --keepalive-time 2 "${url}" -o "${destination_file}"; then
+      echo "Downloading ${url} failed. $((5-i)) retries left."
+      sleep 1
+    else
+      echo "Downloading ${url} succeed"
+      return 0
+    fi
+  done
+  return 1
+}
+
+# Wait for background jobs to finish. Return with
+# an error status if any of the jobs failed.
+kube::util::wait-for-jobs() {
+  local fail=0
+  local job
+  for job in $(jobs -p); do
+    wait "${job}" || fail=$((fail + 1))
+  done
+  return ${fail}
+}
+
+# kube::util::join <delim> <list...>
+# Concatenates the list elements with the delimiter passed as first parameter
+#
+# Ex: kube::util::join , a b c
+#  -> a,b,c
+function kube::util::join {
+  local IFS="$1"
+  shift
+  echo "$*"
+}
+
+# kube::util::check-file-in-alphabetical-order <file>
+# Check that the file is in alphabetical order
+#
+function kube::util::check-file-in-alphabetical-order {
+  local failure_file="$1"
+  if ! diff -u "${failure_file}" <(LC_ALL=C sort "${failure_file}"); then
+    {
+      echo
+      echo "${failure_file} is not in alphabetical order. Please sort it:"
+      echo
+      echo "  LC_ALL=C sort -o ${failure_file} ${failure_file}"
+      echo
+    } >&2
+    false
+  fi
+}
+
+# Some useful colors.
+if [[ -z "${color_start-}" ]]; then
+  declare -r color_start="\033["
+  declare -r color_red="${color_start}0;31m"
+  declare -r color_yellow="${color_start}0;33m"
+  declare -r color_green="${color_start}0;32m"
+  declare -r color_blue="${color_start}1;34m"
+  declare -r color_cyan="${color_start}1;36m"
+  declare -r color_norm="${color_start}0m"
+
+  kube::util::sourced_variable "${color_start}"
+  kube::util::sourced_variable "${color_red}"
+  kube::util::sourced_variable "${color_yellow}"
+  kube::util::sourced_variable "${color_green}"
+  kube::util::sourced_variable "${color_blue}"
+  kube::util::sourced_variable "${color_cyan}"
+  kube::util::sourced_variable "${color_norm}"
+fi
+
+# ex: ts=2 sw=2 et filetype=sh
--- a/verify-go-version.sh
+++ b/verify-go-version.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+GO="$1"
+
+if [ ! "$GO" ]; then
+    echo >&2 "usage: $0 <path to go binary>"
+    exit 1
+fi
+
+die () {
+    echo "ERROR: $*"
+    exit 1
+}
+
+version=$("$GO" version) || die "determining version of $GO failed"
+# shellcheck disable=SC2001
+majorminor=$(echo "$version" | sed -e 's/.*go\([0-9]*\)\.\([0-9]*\).*/\1.\2/')
+# shellcheck disable=SC2001
+expected=$(grep "^ *- go:" "release-tools/travis.yml" | sed -e 's/.*go: *\([0-9]*\)\.\([0-9]*\).*/\1.\2/')
+
+if [ "$majorminor" != "$expected" ]; then
+    cat >&2 <<EOF
+
+======================================================
+                  WARNING
+
+  This projects is tested with Go v$expected.
+  Your current Go version is v$majorminor.
+  This may or may not be close enough.
+
+  In particular test-gofmt and test-vendor
+  are known to be sensitive to the version of
+  Go.
+======================================================
+
+EOF
+fi
--- a/verify-shellcheck.sh
+++ b/verify-shellcheck.sh
@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# The csi-release-tools directory.
+TOOLS="$(dirname "${BASH_SOURCE[0]}")"
+. "${TOOLS}/util.sh"
+
+# Directory to check. Default is the parent of the tools themselves.
+ROOT="${1:-${TOOLS}/..}"
+
+# required version for this script, if not installed on the host we will
+# use the official docker image instead. keep this in sync with SHELLCHECK_IMAGE
+SHELLCHECK_VERSION="0.6.0"
+# upstream shellcheck latest stable image as of January 10th, 2019
+SHELLCHECK_IMAGE="koalaman/shellcheck-alpine:v0.6.0@sha256:7d4d712a2686da99d37580b4e2f45eb658b74e4b01caf67c1099adc294b96b52"
+
+# fixed name for the shellcheck docker container so we can reliably clean it up
+SHELLCHECK_CONTAINER="k8s-shellcheck"
+
+# disabled lints
+disabled=(
+  # this lint disallows non-constant source, which we use extensively without
+  # any known bugs
+  1090
+  # this lint prefers command -v to which, they are not the same
+  2230
+)
+# comma separate for passing to shellcheck
+join_by() {
+  local IFS="$1";
+  shift;
+  echo "$*";
+}
+SHELLCHECK_DISABLED="$(join_by , "${disabled[@]}")"
+readonly SHELLCHECK_DISABLED
+
+# creates the shellcheck container for later use
+create_container () {
+  # TODO(bentheelder): this is a performance hack, we create the container with
+  # a sleep MAX_INT32 so that it is effectively paused.
+  # We then repeatedly exec to it to run each shellcheck, and later rm it when
+  # we're done.
+  # This is incredibly much faster than creating a container for each shellcheck
+  # call ...
+  docker run --name "${SHELLCHECK_CONTAINER}" -d --rm -v "${ROOT}:${ROOT}" -w "${ROOT}" --entrypoint="sleep" "${SHELLCHECK_IMAGE}" 2147483647
+}
+# removes the shellcheck container
+remove_container () {
+  docker rm -f "${SHELLCHECK_CONTAINER}" &> /dev/null || true
+}
+
+# ensure we're linting the source tree
+cd "${ROOT}"
+
+# find all shell scripts excluding ./_*, ./.git/*, ./vendor*,
+# and anything git-ignored
+all_shell_scripts=()
+while IFS=$'\n' read -r script;
+  do git check-ignore -q "$script" || all_shell_scripts+=("$script");
+done < <(find . -name "*.sh" \
+  -not \( \
+    -path ./_\*      -o \
+    -path ./.git\*   -o \
+    -path ./vendor\*    \
+  \))
+
+# detect if the host machine has the required shellcheck version installed
+# if so, we will use that instead.
+HAVE_SHELLCHECK=false
+if which shellcheck &>/dev/null; then
+  detected_version="$(shellcheck --version | grep 'version: .*')"
+  if [[ "${detected_version}" = "version: ${SHELLCHECK_VERSION}" ]]; then
+    HAVE_SHELLCHECK=true
+  fi
+fi
+
+# tell the user which we've selected and possibly set up the container
+if ${HAVE_SHELLCHECK}; then
+  echo "Using host shellcheck ${SHELLCHECK_VERSION} binary."
+else
+  echo "Using shellcheck ${SHELLCHECK_VERSION} docker image."
+  # remove any previous container, ensure we will attempt to cleanup on exit,
+  # and create the container
+  remove_container
+  kube::util::trap_add 'remove_container' EXIT
+  if ! output="$(create_container 2>&1)"; then
+      {
+        echo "Failed to create shellcheck container with output: "
+        echo ""
+        echo "${output}"
+      } >&2
+      exit 1
+  fi
+fi
+
+# lint each script, tracking failures
+errors=()
+for f in "${all_shell_scripts[@]}"; do
+  set +o errexit
+  if ${HAVE_SHELLCHECK}; then
+    failedLint=$(shellcheck --exclude="${SHELLCHECK_DISABLED}" "${f}")
+  else
+    failedLint=$(docker exec -t ${SHELLCHECK_CONTAINER} \
+                 shellcheck --exclude="${SHELLCHECK_DISABLED}" "${f}")
+  fi
+  set -o errexit
+  if [[ -n "${failedLint}" ]]; then
+      errors+=( "${failedLint}" )
+  fi
+done
+
+# Check to be sure all the packages that should pass lint are.
+if [ ${#errors[@]} -eq 0 ]; then
+  echo 'Congratulations! All shell files are passing lint.'
+else
+  {
+    echo "Errors from shellcheck:"
+    for err in "${errors[@]}"; do
+      echo "$err"
+    done
+    echo
+    echo 'Please review the above warnings. You can test via "./hack/verify-shellcheck"'
+    echo 'If the above warnings do not make sense, you can exempt them from shellcheck'
+    echo 'checking by adding the "shellcheck disable" directive'
+    echo '(https://github.com/koalaman/shellcheck/wiki/Directive#disable).'
+    echo
+  } >&2
+  false
+fi
--- a/verify-subtree.sh
+++ b/verify-subtree.sh
@@ -30,7 +30,7 @@ if [ ! "$DIR" ]; then
    exit 1
 fi

-REV=$(git log -n1 --format=format:%H --no-merges -- "$DIR")
+REV=$(git log -n1 --remove-empty --format=format:%H --no-merges -- "$DIR")
 if [ "$REV" ]; then
    echo "Directory '$DIR' contains non-upstream changes:"
    echo
--- a/verify-vendor.sh
+++ b/verify-vendor.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ -f Gopkg.toml ]; then
+    echo "Repo uses 'dep' for vendoring."
+    case "$(dep version 2>/dev/null | grep 'version *:')" in
+	*v0.[56789]*)
+            if dep check; then
+                echo "vendor up-to-date"
+            else
+                exit 1
+            fi
+            ;;
+	*) echo "skipping check, dep >= 0.5 required";;
+    esac
+elif [ -f go.mod ]; then
+    echo "Repo uses 'go mod'."
+    # shellcheck disable=SC2235
+    if [ "${JOB_NAME}" ] &&
+           ( [ "${JOB_TYPE}" != "presubmit" ] ||
+                 [ "$( (git diff "${PULL_BASE_SHA}..HEAD" -- go.mod go.sum vendor release-tools;
+                        git diff "${PULL_BASE_SHA}..HEAD" | grep -e '^@@.*@@ import (' -e '^[+-]import') |
+		           wc -l)" -eq 0 ] ); then
+	echo "Skipping vendor check because the Prow pre-submit job does not affect dependencies."
+    elif ! (set -x; env GO111MODULE=on go mod tidy); then
+	echo "ERROR: vendor check failed."
+	exit 1
+    elif [ "$(git status --porcelain -- go.mod go.sum | wc -l)" -gt 0 ]; then
+	echo "ERROR: go module files *not* up-to-date, they did get modified by 'GO111MODULE=on go mod tidy':";
+	git diff -- go.mod go.sum
+	exit 1
+    elif [ -d vendor ]; then
+	if ! (set -x; env GO111MODULE=on go mod vendor); then
+	    echo "ERROR: vendor check failed."
+	    exit 1
+	elif [ "$(git status --porcelain -- vendor | wc -l)" -gt 0 ]; then
+	    echo "ERROR: vendor directory *not* up-to-date, it did get modified by 'GO111MODULE=on go mod vendor':"
+	    git status -- vendor
+	    git diff -- vendor
+	    exit 1
+	else
+	    echo "Go dependencies and vendor directory up-to-date."
+	fi
+    else
+	echo "Go dependencies up-to-date."
+    fi
+fi