Bumping k8s dependencies to 1.13

This commit is contained in:
Cheng Xing
2018-11-16 14:08:25 -08:00
parent 305407125c
commit b4c0b68ec7
8002 changed files with 884099 additions and 276228 deletions

View File

@@ -16,7 +16,7 @@ go_library(
visibility = ["//visibility:public"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha3:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/features:go_default_library",
"//cmd/kubeadm/app/images:go_default_library",
@@ -25,6 +25,7 @@ go_library(
"//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:go_default_library",
"//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library",
"//cmd/kubeadm/app/phases/certs:go_default_library",
"//cmd/kubeadm/app/phases/certs/renewal:go_default_library",
"//cmd/kubeadm/app/phases/controlplane:go_default_library",
"//cmd/kubeadm/app/phases/etcd:go_default_library",
"//cmd/kubeadm/app/phases/kubelet:go_default_library",
@@ -38,15 +39,15 @@ go_library(
"//cmd/kubeadm/app/util/etcd:go_default_library",
"//pkg/util/version:go_default_library",
"//pkg/version:go_default_library",
"//vendor/k8s.io/api/apps/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/util/cert:go_default_library",
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
],
)
@@ -76,23 +77,23 @@ go_test(
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/phases/certs:go_default_library",
"//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
"//cmd/kubeadm/app/phases/controlplane:go_default_library",
"//cmd/kubeadm/app/phases/etcd:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//cmd/kubeadm/app/util/config:go_default_library",
"//cmd/kubeadm/app/util/etcd:go_default_library",
"//cmd/kubeadm/test:go_default_library",
"//cmd/kubeadm/test/certs:go_default_library",
"//pkg/util/version:go_default_library",
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/github.com/coreos/etcd/clientv3:go_default_library",
"//vendor/github.com/coreos/etcd/pkg/transport:go_default_library",
"//vendor/k8s.io/api/apps/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
],
)

View File

@@ -168,7 +168,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.10.3",
KubeadmVersion: "v1.10.3",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.1.12",
},
},
@@ -207,7 +207,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.10.3",
KubeadmVersion: "v1.10.3",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.1.12",
},
},
@@ -246,7 +246,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.11.0",
KubeadmVersion: "v1.11.0",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.18",
},
},
@@ -285,7 +285,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.10.5",
KubeadmVersion: "v1.10.5", // Note: The kubeadm version mustn't be "downgraded" here
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.1.12",
},
},
@@ -305,7 +305,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.11.1",
KubeadmVersion: "v1.11.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.18",
},
},
@@ -364,7 +364,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.11.0-alpha.2",
KubeadmVersion: "v1.11.0-alpha.2",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.18",
},
},
@@ -404,7 +404,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.11.0-alpha.2",
KubeadmVersion: "v1.11.0-alpha.2",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.18",
},
},
@@ -445,7 +445,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.11.0-beta.1",
KubeadmVersion: "v1.11.0-beta.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.18",
},
},
@@ -486,7 +486,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.11.0-rc.1",
KubeadmVersion: "v1.11.0-rc.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.18",
},
},
@@ -527,7 +527,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.11.6-rc.1",
KubeadmVersion: "v1.11.6-rc.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.18",
},
},
@@ -568,7 +568,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.11.0-rc.1",
KubeadmVersion: "v1.11.0-rc.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.18",
},
},
@@ -588,8 +588,8 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.12.0-alpha.2",
KubeadmVersion: "v1.12.0-alpha.2",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.2.18",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.24",
},
},
},
@@ -641,8 +641,8 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.12.1",
KubeadmVersion: "v1.12.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.2.18",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.24",
},
},
},
@@ -678,8 +678,8 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.12.0",
KubeadmVersion: "v1.12.0",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.2.18",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.24",
},
},
},
@@ -715,8 +715,8 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.12.0",
KubeadmVersion: "v1.12.0",
DNSType: "kube-dns",
DNSVersion: "1.14.10",
EtcdVersion: "3.2.18",
DNSVersion: "1.14.13",
EtcdVersion: "3.2.24",
},
},
},

View File

@@ -34,38 +34,38 @@ func TestEnforceVersionPolicies(t *testing.T) {
{
name: "minor upgrade",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.5",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.11.5",
},
newK8sVersion: "v1.10.5",
newK8sVersion: "v1.11.5",
},
{
name: "major upgrade",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.2",
kubeadmVersion: "v1.11.1",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.2",
kubeadmVersion: "v1.12.1",
},
newK8sVersion: "v1.11.0",
newK8sVersion: "v1.12.0",
},
{
name: "downgrade",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.3",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.11.3",
},
newK8sVersion: "v1.10.2",
newK8sVersion: "v1.11.2",
},
{
name: "same version upgrade",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.3",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.11.3",
},
newK8sVersion: "v1.10.3",
newK8sVersion: "v1.11.3",
},
{
name: "new version must be higher than v1.10.0",
@@ -92,103 +92,103 @@ func TestEnforceVersionPolicies(t *testing.T) {
{
name: "downgrading two minor versions in one go is not supported",
vg: &fakeVersionGetter{
clusterVersion: "v1.12.3",
kubeletVersion: "v1.12.3",
kubeadmVersion: "v1.12.0",
clusterVersion: "v1.13.3",
kubeletVersion: "v1.13.3",
kubeadmVersion: "v1.13.0",
},
newK8sVersion: "v1.10.3",
newK8sVersion: "v1.11.3",
expectedMandatoryErrs: 1, // can't downgrade two minor versions
expectedSkippableErrs: 1, // can't upgrade old k8s with newer kubeadm
},
{
name: "kubeadm version must be higher than the new kube version. However, patch version skews may be forced",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.3",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.11.3",
},
newK8sVersion: "v1.10.5",
newK8sVersion: "v1.11.5",
expectedSkippableErrs: 1,
},
{
name: "kubeadm version must be higher than the new kube version. Trying to upgrade k8s to a higher minor version than kubeadm itself should never be supported",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.3",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.11.3",
},
newK8sVersion: "v1.11.0",
newK8sVersion: "v1.12.0",
expectedMandatoryErrs: 1,
},
{
name: "the maximum skew between the cluster version and the kubelet versions should be one minor version. This may be forced through though.",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.9.8",
kubeadmVersion: "v1.11.0",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.10.8",
kubeadmVersion: "v1.12.0",
},
newK8sVersion: "v1.11.0",
newK8sVersion: "v1.12.0",
expectedSkippableErrs: 1,
},
{
name: "experimental upgrades supported if the flag is set",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0-beta.1",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.12.0-beta.1",
},
newK8sVersion: "v1.11.0-beta.1",
newK8sVersion: "v1.12.0-beta.1",
allowExperimental: true,
},
{
name: "release candidate upgrades supported if the flag is set",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0-rc.1",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.12.0-rc.1",
},
newK8sVersion: "v1.11.0-rc.1",
newK8sVersion: "v1.12.0-rc.1",
allowRCs: true,
},
{
name: "release candidate upgrades supported if the flag is set",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0-rc.1",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.12.0-rc.1",
},
newK8sVersion: "v1.11.0-rc.1",
newK8sVersion: "v1.12.0-rc.1",
allowExperimental: true,
},
{
name: "the user should not be able to upgrade to an experimental version if they haven't opted into that",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0-beta.1",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.12.0-beta.1",
},
newK8sVersion: "v1.11.0-beta.1",
newK8sVersion: "v1.12.0-beta.1",
allowRCs: true,
expectedSkippableErrs: 1,
},
{
name: "the user should not be able to upgrade to an release candidate version if they haven't opted into that",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0-rc.1",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.12.0-rc.1",
},
newK8sVersion: "v1.11.0-rc.1",
newK8sVersion: "v1.12.0-rc.1",
expectedSkippableErrs: 1,
},
{
name: "the user can't use a newer minor version of kubeadm to upgrade an older version of kubeadm",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.12.0",
},
newK8sVersion: "v1.10.6",
newK8sVersion: "v1.11.6",
expectedSkippableErrs: 1, // can't upgrade old k8s with newer kubeadm
},
}

View File

@@ -29,7 +29,7 @@ import (
clientset "k8s.io/client-go/kubernetes"
certutil "k8s.io/client-go/util/cert"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
@@ -50,7 +50,7 @@ var expiry = 180 * 24 * time.Hour
// PerformPostUpgradeTasks runs nearly the same functions as 'kubeadm init' would do
// Note that the markmaster phase is left out, not needed, and no token is created as that doesn't belong to the upgrade
func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterConfiguration, newK8sVer *version.Version, dryRun bool) error {
func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, newK8sVer *version.Version, dryRun bool) error {
errs := []error{}
// Upload currently used configuration to the cluster
@@ -70,7 +70,7 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterC
errs = append(errs, err)
}
// Annotate the node with the crisocket information, sourced either from the MasterConfiguration struct or
// Annotate the node with the crisocket information, sourced either from the InitConfiguration struct or
// --cri-socket.
// TODO: In the future we want to use something more official like NodeStatus or similar for detecting this properly
if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil {
@@ -108,7 +108,7 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterC
}
// Rotate the kube-apiserver cert and key if needed
if err := backupAPIServerCertIfNeeded(cfg, dryRun); err != nil {
if err := BackupAPIServerCertIfNeeded(cfg, dryRun); err != nil {
errs = append(errs, err)
}
@@ -127,7 +127,7 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterC
return errors.NewAggregate(errs)
}
func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface, dryRun bool) error {
func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.InitConfiguration, client clientset.Interface, dryRun bool) error {
return apiclient.TryRunCommand(func() error {
installedDeploymentName := kubeadmconstants.KubeDNS
deploymentToDelete := kubeadmconstants.CoreDNS
@@ -158,7 +158,7 @@ func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.MasterConfiguratio
}, 10)
}
func upgradeToSelfHosting(client clientset.Interface, cfg *kubeadmapi.MasterConfiguration, dryRun bool) error {
func upgradeToSelfHosting(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, dryRun bool) error {
if features.Enabled(cfg.FeatureGates, features.SelfHosting) && !IsControlPlaneSelfHosted(client) {
waiter := getWaiter(dryRun, client)
@@ -172,8 +172,9 @@ func upgradeToSelfHosting(client clientset.Interface, cfg *kubeadmapi.MasterConf
return nil
}
func backupAPIServerCertIfNeeded(cfg *kubeadmapi.MasterConfiguration, dryRun bool) error {
certAndKeyDir := kubeadmapiv1alpha2.DefaultCertificatesDir
// BackupAPIServerCertIfNeeded rotates the kube-apiserver certificate if older than 180 days
func BackupAPIServerCertIfNeeded(cfg *kubeadmapi.InitConfiguration, dryRun bool) error {
certAndKeyDir := kubeadmapiv1alpha3.DefaultCertificatesDir
shouldBackup, err := shouldBackupAPIServerCertAndKey(certAndKeyDir)
if err != nil {
// Don't fail the upgrade phase if failing to determine to backup kube-apiserver cert and key.
@@ -195,10 +196,14 @@ func backupAPIServerCertIfNeeded(cfg *kubeadmapi.MasterConfiguration, dryRun boo
if err := backupAPIServerCertAndKey(certAndKeyDir); err != nil {
fmt.Printf("[postupgrade] WARNING: failed to backup kube-apiserver cert and key: %v", err)
}
return certsphase.CreateAPIServerCertAndKeyFiles(cfg)
return certsphase.CreateCertAndKeyFilesWithCA(
&certsphase.KubeadmCertAPIServer,
&certsphase.KubeadmCertRootCA,
cfg,
)
}
func writeKubeletConfigFiles(client clientset.Interface, cfg *kubeadmapi.MasterConfiguration, newK8sVer *version.Version, dryRun bool) error {
func writeKubeletConfigFiles(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, newK8sVer *version.Version, dryRun bool) error {
kubeletDir, err := getKubeletDir(dryRun)
if err != nil {
// The error here should never occur in reality, would only be thrown if /tmp doesn't exist on the machine.

View File

@@ -27,7 +27,6 @@ import (
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
)
@@ -130,9 +129,11 @@ func TestRollbackFiles(t *testing.T) {
}
func TestShouldBackupAPIServerCertAndKey(t *testing.T) {
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
cfg := &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "test-node"},
}
@@ -148,23 +149,21 @@ func TestShouldBackupAPIServerCertAndKey(t *testing.T) {
expected: true,
},
} {
caCert, caKey, err := certsphase.NewCACertAndKey()
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
cfg.CertificatesDir = tmpdir
caCert, caKey, err := certsphase.KubeadmCertRootCA.CreateAsCA(cfg)
if err != nil {
t.Fatalf("failed creation of ca cert and key: %v", err)
}
caCert.NotBefore = caCert.NotBefore.Add(-test.adjustedExpiry).UTC()
apiCert, apiKey, err := certsphase.NewAPIServerCertAndKey(cfg, caCert, caKey)
err = certsphase.KubeadmCertAPIServer.CreateFromCA(cfg, caCert, caKey)
if err != nil {
t.Fatalf("Test %s: failed creation of cert and key: %v", desc, err)
}
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
if err := pkiutil.WriteCertAndKey(tmpdir, constants.APIServerCertAndKeyBaseName, apiCert, apiKey); err != nil {
t.Fatalf("Test %s: failure while saving %s certificate and key: %v", desc, constants.APIServerCertAndKeyBaseName, err)
}
certAndKey := []string{filepath.Join(tmpdir, constants.APIServerCertName), filepath.Join(tmpdir, constants.APIServerKeyName)}
for _, path := range certAndKey {
if _, err := os.Stat(path); os.IsNotExist(err) {

View File

@@ -44,12 +44,12 @@ type Prepuller interface {
// DaemonSetPrepuller makes sure the control plane images are available on all masters
type DaemonSetPrepuller struct {
client clientset.Interface
cfg *kubeadmapi.MasterConfiguration
cfg *kubeadmapi.ClusterConfiguration
waiter apiclient.Waiter
}
// NewDaemonSetPrepuller creates a new instance of the DaemonSetPrepuller struct
func NewDaemonSetPrepuller(client clientset.Interface, waiter apiclient.Waiter, cfg *kubeadmapi.MasterConfiguration) *DaemonSetPrepuller {
func NewDaemonSetPrepuller(client clientset.Interface, waiter apiclient.Waiter, cfg *kubeadmapi.ClusterConfiguration) *DaemonSetPrepuller {
return &DaemonSetPrepuller{
client: client,
cfg: cfg,
@@ -59,7 +59,12 @@ func NewDaemonSetPrepuller(client clientset.Interface, waiter apiclient.Waiter,
// CreateFunc creates a DaemonSet for making the image available on every relevant node
func (d *DaemonSetPrepuller) CreateFunc(component string) error {
image := images.GetCoreImage(component, d.cfg.GetControlPlaneImageRepository(), d.cfg.KubernetesVersion, d.cfg.UnifiedControlPlaneImage)
var image string
if component == constants.Etcd {
image = images.GetEtcdImage(d.cfg)
} else {
image = images.GetKubeControlPlaneImage(component, d.cfg)
}
ds := buildPrePullDaemonSet(component, image)
// Create the DaemonSet in the API Server
@@ -154,6 +159,11 @@ func buildPrePullDaemonSet(component, image string) *apps.DaemonSet {
Namespace: metav1.NamespaceSystem,
},
Spec: apps.DaemonSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"k8s-app": addPrepullPrefix(component),
},
},
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{

View File

@@ -81,7 +81,7 @@ type controlPlaneComponentResources struct {
// -> Backup component v1 is Deleted
// 5. Wait for Self-Hosted component v2 Running to become active
// 6. Repeat for all control plane components
func SelfHostedControlPlane(client clientset.Interface, waiter apiclient.Waiter, cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) error {
func SelfHostedControlPlane(client clientset.Interface, waiter apiclient.Waiter, cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Version) error {
// Adjust the timeout slightly to something self-hosting specific
waiter.SetTimeout(selfHostingWaitTimeout)
@@ -158,7 +158,7 @@ func SelfHostedControlPlane(client clientset.Interface, waiter apiclient.Waiter,
}
// BuildUpgradedDaemonSetsFromConfig takes a config object and the current version and returns the DaemonSet objects to post to the master
func BuildUpgradedDaemonSetsFromConfig(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) map[string]*apps.DaemonSet {
func BuildUpgradedDaemonSetsFromConfig(cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Version) map[string]*apps.DaemonSet {
// Here the map of different mutators to use for the control plane's podspec is stored
mutators := selfhosting.GetMutatorsFromFeatureGates(cfg.FeatureGates)
// Get the new PodSpecs to use

View File

@@ -25,6 +25,7 @@ import (
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/renewal"
controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
"k8s.io/kubernetes/cmd/kubeadm/app/util"
@@ -33,6 +34,11 @@ import (
"k8s.io/kubernetes/pkg/util/version"
)
const (
// UpgradeManifestTimeout is timeout of upgrading the static pod manifest
UpgradeManifestTimeout = 5 * time.Minute
)
// StaticPodPathManager is responsible for tracking the directories used in the static pod upgrade transition
type StaticPodPathManager interface {
// MoveFile should move a file from oldPath to newPath
@@ -156,7 +162,7 @@ func (spm *KubeStaticPodPathManager) CleanupDirs() error {
return nil
}
func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.MasterConfiguration, beforePodHash string, recoverManifests map[string]string, isTLSUpgrade bool) error {
func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, beforePodHash string, recoverManifests map[string]string, isTLSUpgrade bool) error {
// Special treatment is required for etcd case, when rollbackOldManifests should roll back etcd
// manifests only for the case when component is Etcd
recoverEtcd := false
@@ -180,27 +186,8 @@ func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticP
}
}
// ensure etcd certs are generated for etcd and kube-apiserver
if component == constants.Etcd || component == constants.KubeAPIServer {
if err := certsphase.CreateEtcdCACertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s CA certificate and key: %v", constants.Etcd, err)
}
}
if component == constants.Etcd {
if err := certsphase.CreateEtcdServerCertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s certificate and key: %v", constants.Etcd, err)
}
if err := certsphase.CreateEtcdPeerCertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s peer certificate and key: %v", constants.Etcd, err)
}
if err := certsphase.CreateEtcdHealthcheckClientCertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s healthcheck certificate and key: %v", constants.Etcd, err)
}
}
if component == constants.KubeAPIServer {
if err := certsphase.CreateAPIServerEtcdClientCertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s %s-client certificate and key: %v", constants.KubeAPIServer, constants.Etcd, err)
}
if err := renewCerts(cfg, component); err != nil {
return fmt.Errorf("failed to renew certificates for component %q: %v", component, err)
}
// The old manifest is here; in the /etc/kubernetes/manifests/
@@ -228,6 +215,7 @@ func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticP
if waitForComponentRestart {
fmt.Println("[upgrade/staticpods] Waiting for the kubelet to restart the component")
fmt.Printf("[upgrade/staticpods] This might take a minute or longer depending on the component/version gap (timeout %v\n", UpgradeManifestTimeout)
// Wait for the mirror Pod hash to change; otherwise we'll run into race conditions here when the kubelet hasn't had time to
// notice the removal of the Static Pod, leading to a false positive below where we check that the API endpoint is healthy
@@ -251,7 +239,7 @@ func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticP
}
// performEtcdStaticPodUpgrade performs upgrade of etcd, it returns bool which indicates fatal error or not and the actual error.
func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.MasterConfiguration, recoverManifests map[string]string, isTLSUpgrade bool, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) (bool, error) {
func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, recoverManifests map[string]string, isTLSUpgrade bool, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) (bool, error) {
// Add etcd static pod spec only if external etcd is not configured
if cfg.Etcd.External != nil {
return false, fmt.Errorf("external etcd detected, won't try to change any etcd state")
@@ -398,7 +386,7 @@ func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathM
}
// StaticPodControlPlane upgrades a static pod-hosted control plane
func StaticPodControlPlane(waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.MasterConfiguration, etcdUpgrade bool, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) error {
func StaticPodControlPlane(waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, etcdUpgrade bool, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) error {
recoverManifests := map[string]string{}
var isTLSUpgrade bool
var isExternalEtcd bool
@@ -503,7 +491,7 @@ func rollbackOldManifests(oldManifests map[string]string, origErr error, pathMgr
// rollbackEtcdData rolls back the the content of etcd folder if something went wrong.
// When the folder contents are successfully rolled back, nil is returned, otherwise an error is returned.
func rollbackEtcdData(cfg *kubeadmapi.MasterConfiguration, pathMgr StaticPodPathManager) error {
func rollbackEtcdData(cfg *kubeadmapi.InitConfiguration, pathMgr StaticPodPathManager) error {
backupEtcdDir := pathMgr.BackupEtcdDir()
runningEtcdDir := cfg.Etcd.Local.DataDir
@@ -514,3 +502,35 @@ func rollbackEtcdData(cfg *kubeadmapi.MasterConfiguration, pathMgr StaticPodPath
return nil
}
func renewCerts(cfg *kubeadmapi.InitConfiguration, component string) error {
if cfg.Etcd.Local != nil {
// ensure etcd certs are loaded for etcd and kube-apiserver
if component == constants.Etcd || component == constants.KubeAPIServer {
caCert, caKey, err := certsphase.LoadCertificateAuthority(cfg.CertificatesDir, certsphase.KubeadmCertEtcdCA.BaseName)
if err != nil {
return fmt.Errorf("failed to upgrade the %s CA certificate and key: %v", constants.Etcd, err)
}
renewer := renewal.NewFileRenewal(caCert, caKey)
if component == constants.Etcd {
for _, cert := range []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdServer,
&certsphase.KubeadmCertEtcdPeer,
&certsphase.KubeadmCertEtcdHealthcheck,
} {
if err := renewal.RenewExistingCert(cfg.CertificatesDir, cert.BaseName, renewer); err != nil {
return fmt.Errorf("failed to renew %s certificate and key: %v", cert.Name, err)
}
}
}
if component == constants.KubeAPIServer {
cert := certsphase.KubeadmCertEtcdAPIClient
if err := renewal.RenewExistingCert(cfg.CertificatesDir, cert.BaseName, renewer); err != nil {
return fmt.Errorf("failed to renew %s certificate and key: %v", cert.Name, err)
}
}
}
}
return nil
}

View File

@@ -20,6 +20,7 @@ import (
"crypto/sha256"
"fmt"
"io/ioutil"
"math/big"
"os"
"path/filepath"
"strings"
@@ -28,16 +29,18 @@ import (
"github.com/coreos/etcd/clientv3"
"github.com/coreos/etcd/pkg/transport"
"k8s.io/apimachinery/pkg/runtime"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
certstestutil "k8s.io/kubernetes/cmd/kubeadm/test/certs"
)
const (
@@ -46,8 +49,14 @@ const (
waitForPodsWithLabel = "wait-for-pods-with-label"
testConfiguration = `
apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
apiVersion: kubeadm.k8s.io/v1alpha3
kind: InitConfiguration
nodeRegistration:
name: foo
criSocket: ""
---
apiVersion: kubeadm.k8s.io/v1alpha3
kind: ClusterConfiguration
api:
advertiseAddress: 1.2.3.4
bindPort: 6443
@@ -415,32 +424,21 @@ func TestStaticPodControlPlane(t *testing.T) {
}
defer os.RemoveAll(tmpEtcdDataDir)
oldcfg, err := getConfig("v1.9.0", tempCertsDir, tmpEtcdDataDir)
oldcfg, err := getConfig("v1.12.0", tempCertsDir, tmpEtcdDataDir)
if err != nil {
t.Fatalf("couldn't create config: %v", err)
}
// Initialize PKI minus any etcd certificates to simulate etcd PKI upgrade
certActions := []func(cfg *kubeadmapi.MasterConfiguration) error{
certsphase.CreateCACertAndKeyFiles,
certsphase.CreateAPIServerCertAndKeyFiles,
certsphase.CreateAPIServerKubeletClientCertAndKeyFiles,
// certsphase.CreateEtcdCACertAndKeyFiles,
// certsphase.CreateEtcdServerCertAndKeyFiles,
// certsphase.CreateEtcdPeerCertAndKeyFiles,
// certsphase.CreateEtcdHealthcheckClientCertAndKeyFiles,
// certsphase.CreateAPIServerEtcdClientCertAndKeyFiles,
certsphase.CreateServiceAccountKeyAndPublicKeyFiles,
certsphase.CreateFrontProxyCACertAndKeyFiles,
certsphase.CreateFrontProxyClientCertAndKeyFiles,
tree, err := certsphase.GetCertsWithoutEtcd().AsMap().CertTree()
if err != nil {
t.Fatalf("couldn't get cert tree: %v", err)
}
for _, action := range certActions {
err := action(oldcfg)
if err != nil {
t.Fatalf("couldn't initialize pre-upgrade certificate: %v", err)
}
if err := tree.CreateTree(oldcfg); err != nil {
t.Fatalf("couldn't get create cert tree: %v", err)
}
fmt.Printf("Wrote certs to %s\n", oldcfg.CertificatesDir)
t.Logf("Wrote certs to %s\n", oldcfg.CertificatesDir)
// Initialize the directory with v1.7 manifests; should then be upgraded to v1.8 using the method
err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), oldcfg)
@@ -457,11 +455,27 @@ func TestStaticPodControlPlane(t *testing.T) {
t.Fatalf("couldn't read temp file: %v", err)
}
newcfg, err := getConfig("v1.10.0", tempCertsDir, tmpEtcdDataDir)
newcfg, err := getConfig("v1.11.0", tempCertsDir, tmpEtcdDataDir)
if err != nil {
t.Fatalf("couldn't create config: %v", err)
}
// create the kubeadm etcd certs
caCert, caKey, err := certsphase.KubeadmCertEtcdCA.CreateAsCA(newcfg)
if err != nil {
t.Fatalf("couldn't create new CA certificate: %v", err)
}
for _, cert := range []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdServer,
&certsphase.KubeadmCertEtcdPeer,
&certsphase.KubeadmCertEtcdHealthcheck,
&certsphase.KubeadmCertEtcdAPIClient,
} {
if err := cert.CreateFromCA(newcfg, caCert, caKey); err != nil {
t.Fatalf("couldn't create certificate %s: %v", cert.Name, err)
}
}
actualErr := StaticPodControlPlane(
waiter,
pathMgr,
@@ -513,15 +527,26 @@ func getAPIServerHash(dir string) (string, error) {
return fmt.Sprintf("%x", sha256.Sum256(fileBytes)), nil
}
// TODO: Make this test function use the rest of the "official" API machinery helper funcs we have inside of kubeadm
func getConfig(version, certsDir, etcdDataDir string) (*kubeadmapi.MasterConfiguration, error) {
externalcfg := &kubeadmapiv1alpha2.MasterConfiguration{}
internalcfg := &kubeadmapi.MasterConfiguration{}
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(fmt.Sprintf(testConfiguration, certsDir, etcdDataDir, version)), externalcfg); err != nil {
return nil, fmt.Errorf("unable to decode config: %v", err)
func getConfig(version, certsDir, etcdDataDir string) (*kubeadmapi.InitConfiguration, error) {
configBytes := []byte(fmt.Sprintf(testConfiguration, certsDir, etcdDataDir, version))
// Unmarshal the config
cfg, err := configutil.BytesToInternalConfig(configBytes)
if err != nil {
return nil, err
}
kubeadmscheme.Scheme.Convert(externalcfg, internalcfg, nil)
return internalcfg, nil
// Applies dynamic defaults to settings not provided with flags
if err = configutil.SetInitDynamicDefaults(cfg); err != nil {
return nil, err
}
// Validates cfg (flags/configs + defaults + dynamic defaults)
if err = validation.ValidateInitConfiguration(cfg).ToAggregate(); err != nil {
return nil, err
}
return cfg, nil
}
func getTempDir(t *testing.T, name string) (string, func()) {
@@ -601,3 +626,113 @@ func TestCleanupDirs(t *testing.T) {
})
}
}
func TestRenewCerts(t *testing.T) {
caCert, caKey := certstestutil.SetupCertificateAuthorithy(t)
t.Run("all certs exist, should be rotated", func(t *testing.T) {
})
tests := []struct {
name string
component string
skipCreateCA bool
shouldErrorOnRenew bool
certsShouldExist []*certsphase.KubeadmCert
}{
{
name: "all certs exist, should be rotated",
component: constants.Etcd,
certsShouldExist: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdServer,
&certsphase.KubeadmCertEtcdPeer,
&certsphase.KubeadmCertEtcdHealthcheck,
},
},
{
name: "just renew API cert",
component: constants.KubeAPIServer,
certsShouldExist: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdAPIClient,
},
},
{
name: "ignores other compnonents",
skipCreateCA: true,
component: constants.KubeScheduler,
},
{
name: "missing a cert to renew",
component: constants.Etcd,
shouldErrorOnRenew: true,
certsShouldExist: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdServer,
&certsphase.KubeadmCertEtcdPeer,
},
},
{
name: "no CA, cannot continue",
component: constants.Etcd,
skipCreateCA: true,
shouldErrorOnRenew: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
// Setup up basic requities
tmpDir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpDir)
cfg := testutil.GetDefaultInternalConfig(t)
cfg.CertificatesDir = tmpDir
if !test.skipCreateCA {
if err := pkiutil.WriteCertAndKey(tmpDir, constants.EtcdCACertAndKeyBaseName, caCert, caKey); err != nil {
t.Fatalf("couldn't write out CA: %v", err)
}
}
// Create expected certs
for _, kubeCert := range test.certsShouldExist {
if err := kubeCert.CreateFromCA(cfg, caCert, caKey); err != nil {
t.Fatalf("couldn't renew certificate %q: %v", kubeCert.Name, err)
}
}
// Load expected certs to check if serial numbers changes
certMaps := make(map[*certsphase.KubeadmCert]big.Int)
for _, kubeCert := range test.certsShouldExist {
cert, err := pkiutil.TryLoadCertFromDisk(tmpDir, kubeCert.BaseName)
if err != nil {
t.Fatalf("couldn't load certificate %q: %v", kubeCert.Name, err)
}
certMaps[kubeCert] = *cert.SerialNumber
}
// Renew everything
err := renewCerts(cfg, test.component)
if test.shouldErrorOnRenew {
if err == nil {
t.Fatal("expected renewal error, got nothing")
}
// expected error, got error
return
}
if err != nil {
t.Fatalf("couldn't renew certificates: %v", err)
}
// See if the certificate serial numbers change
for kubeCert, cert := range certMaps {
newCert, err := pkiutil.TryLoadCertFromDisk(tmpDir, kubeCert.BaseName)
if err != nil {
t.Errorf("couldn't load new certificate %q: %v", kubeCert.Name, err)
continue
}
if cert.Cmp(newCert.SerialNumber) == 0 {
t.Errorf("certifitate %v was not reissued", kubeCert.Name)
}
}
})
}
}