Add generated file

This PR adds generated files under pkg/client and vendor folder.
This commit is contained in:
xing-yang
2018-07-12 10:55:15 -07:00
parent 36b1de0341
commit e213d1890d
17729 changed files with 5090889 additions and 0 deletions

View File

@@ -0,0 +1,62 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = ["dns_test.go"],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//pkg/apis/core:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/client-go/testing:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"dns.go",
"manifests.go",
],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/features:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//vendor/github.com/mholt/caddy/caddyfile:go_default_library",
"//vendor/k8s.io/api/apps/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/rbac/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View File

@@ -0,0 +1,404 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package dns
import (
"encoding/json"
"fmt"
"runtime"
"strings"
"github.com/mholt/caddy/caddyfile"
apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kuberuntime "k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
clientsetscheme "k8s.io/client-go/kubernetes/scheme"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
)
const (
// KubeDNSServiceAccountName describes the name of the ServiceAccount for the kube-dns addon
KubeDNSServiceAccountName = "kube-dns"
kubeDNSStubDomain = "stubDomains"
kubeDNSUpstreamNameservers = "upstreamNameservers"
kubeDNSFederation = "federations"
)
// DeployedDNSAddon returns the type of DNS addon currently deployed
func DeployedDNSAddon(client clientset.Interface) (string, string, error) {
deploymentsClient := client.AppsV1().Deployments(metav1.NamespaceSystem)
deployments, err := deploymentsClient.List(metav1.ListOptions{LabelSelector: "k8s-app=kube-dns"})
if err != nil {
return "", "", fmt.Errorf("couldn't retrieve DNS addon deployments: %v", err)
}
switch len(deployments.Items) {
case 0:
return "", "", nil
case 1:
addonName := deployments.Items[0].Name
addonImage := deployments.Items[0].Spec.Template.Spec.Containers[0].Image
addonImageParts := strings.Split(addonImage, ":")
addonVersion := addonImageParts[len(addonImageParts)-1]
return addonName, addonVersion, nil
default:
return "", "", fmt.Errorf("multiple DNS addon deployments found: %v", deployments.Items)
}
}
// EnsureDNSAddon creates the kube-dns or CoreDNS addon
func EnsureDNSAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
if features.Enabled(cfg.FeatureGates, features.CoreDNS) {
return coreDNSAddon(cfg, client)
}
return kubeDNSAddon(cfg, client)
}
func kubeDNSAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
if err := CreateServiceAccount(client); err != nil {
return err
}
dnsip, err := kubeadmconstants.GetDNSIP(cfg.Networking.ServiceSubnet)
if err != nil {
return err
}
var dnsBindAddr, dnsProbeAddr string
if dnsip.To4() == nil {
dnsBindAddr = "::1"
dnsProbeAddr = "[" + dnsBindAddr + "]"
} else {
dnsBindAddr = "127.0.0.1"
dnsProbeAddr = dnsBindAddr
}
dnsDeploymentBytes, err := kubeadmutil.ParseTemplate(KubeDNSDeployment,
struct{ ImageRepository, Arch, Version, DNSBindAddr, DNSProbeAddr, DNSDomain, MasterTaintKey string }{
ImageRepository: cfg.ImageRepository,
Arch: runtime.GOARCH,
Version: kubeadmconstants.KubeDNSVersion,
DNSBindAddr: dnsBindAddr,
DNSProbeAddr: dnsProbeAddr,
DNSDomain: cfg.Networking.DNSDomain,
MasterTaintKey: kubeadmconstants.LabelNodeRoleMaster,
})
if err != nil {
return fmt.Errorf("error when parsing kube-dns deployment template: %v", err)
}
dnsServiceBytes, err := kubeadmutil.ParseTemplate(KubeDNSService, struct{ DNSIP string }{
DNSIP: dnsip.String(),
})
if err != nil {
return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err)
}
if err := createKubeDNSAddon(dnsDeploymentBytes, dnsServiceBytes, client); err != nil {
return err
}
fmt.Println("[addons] Applied essential addon: kube-dns")
return nil
}
// CreateServiceAccount creates the necessary serviceaccounts that kubeadm uses/might use, if they don't already exist.
func CreateServiceAccount(client clientset.Interface) error {
return apiclient.CreateOrUpdateServiceAccount(client, &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: KubeDNSServiceAccountName,
Namespace: metav1.NamespaceSystem,
},
})
}
func createKubeDNSAddon(deploymentBytes, serviceBytes []byte, client clientset.Interface) error {
kubednsDeployment := &apps.Deployment{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), deploymentBytes, kubednsDeployment); err != nil {
return fmt.Errorf("unable to decode kube-dns deployment %v", err)
}
// Create the Deployment for kube-dns or update it in case it already exists
if err := apiclient.CreateOrUpdateDeployment(client, kubednsDeployment); err != nil {
return err
}
kubednsService := &v1.Service{}
return createDNSService(kubednsService, serviceBytes, client)
}
func coreDNSAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
// Get the YAML manifest
coreDNSDeploymentBytes, err := kubeadmutil.ParseTemplate(CoreDNSDeployment, struct{ ImageRepository, MasterTaintKey, Version string }{
ImageRepository: cfg.ImageRepository,
MasterTaintKey: kubeadmconstants.LabelNodeRoleMaster,
Version: kubeadmconstants.CoreDNSVersion,
})
if err != nil {
return fmt.Errorf("error when parsing CoreDNS deployment template: %v", err)
}
// Get the kube-dns ConfigMap for translation to equivalent CoreDNS Config.
kubeDNSConfigMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeDNS, metav1.GetOptions{})
if err != nil && !apierrors.IsNotFound(err) {
return err
}
stubDomain, err := translateStubDomainOfKubeDNSToProxyCoreDNS(kubeDNSStubDomain, kubeDNSConfigMap)
if err != nil {
return err
}
upstreamNameserver, err := translateUpstreamNameServerOfKubeDNSToUpstreamProxyCoreDNS(kubeDNSUpstreamNameservers, kubeDNSConfigMap)
if err != nil {
return err
}
coreDNSDomain := cfg.Networking.DNSDomain
federations, err := translateFederationsofKubeDNSToCoreDNS(kubeDNSFederation, coreDNSDomain, kubeDNSConfigMap)
if err != nil {
return err
}
// Get the config file for CoreDNS
coreDNSConfigMapBytes, err := kubeadmutil.ParseTemplate(CoreDNSConfigMap, struct{ DNSDomain, UpstreamNameserver, Federation, StubDomain string }{
DNSDomain: coreDNSDomain,
UpstreamNameserver: upstreamNameserver,
Federation: federations,
StubDomain: stubDomain,
})
if err != nil {
return fmt.Errorf("error when parsing CoreDNS configMap template: %v", err)
}
dnsip, err := kubeadmconstants.GetDNSIP(cfg.Networking.ServiceSubnet)
if err != nil {
return err
}
coreDNSServiceBytes, err := kubeadmutil.ParseTemplate(KubeDNSService, struct{ DNSIP string }{
DNSIP: dnsip.String(),
})
if err != nil {
return fmt.Errorf("error when parsing CoreDNS service template: %v", err)
}
if err := createCoreDNSAddon(coreDNSDeploymentBytes, coreDNSServiceBytes, coreDNSConfigMapBytes, client); err != nil {
return err
}
fmt.Println("[addons] Applied essential addon: CoreDNS")
return nil
}
func createCoreDNSAddon(deploymentBytes, serviceBytes, configBytes []byte, client clientset.Interface) error {
coreDNSConfigMap := &v1.ConfigMap{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), configBytes, coreDNSConfigMap); err != nil {
return fmt.Errorf("unable to decode CoreDNS configmap %v", err)
}
// Create the ConfigMap for CoreDNS or retain it in case it already exists
if err := apiclient.CreateOrRetainConfigMap(client, coreDNSConfigMap, kubeadmconstants.CoreDNS); err != nil {
return err
}
coreDNSClusterRoles := &rbac.ClusterRole{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), []byte(CoreDNSClusterRole), coreDNSClusterRoles); err != nil {
return fmt.Errorf("unable to decode CoreDNS clusterroles %v", err)
}
// Create the Clusterroles for CoreDNS or update it in case it already exists
if err := apiclient.CreateOrUpdateClusterRole(client, coreDNSClusterRoles); err != nil {
return err
}
coreDNSClusterRolesBinding := &rbac.ClusterRoleBinding{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), []byte(CoreDNSClusterRoleBinding), coreDNSClusterRolesBinding); err != nil {
return fmt.Errorf("unable to decode CoreDNS clusterrolebindings %v", err)
}
// Create the Clusterrolebindings for CoreDNS or update it in case it already exists
if err := apiclient.CreateOrUpdateClusterRoleBinding(client, coreDNSClusterRolesBinding); err != nil {
return err
}
coreDNSServiceAccount := &v1.ServiceAccount{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), []byte(CoreDNSServiceAccount), coreDNSServiceAccount); err != nil {
return fmt.Errorf("unable to decode CoreDNS serviceaccount %v", err)
}
// Create the ConfigMap for CoreDNS or update it in case it already exists
if err := apiclient.CreateOrUpdateServiceAccount(client, coreDNSServiceAccount); err != nil {
return err
}
coreDNSDeployment := &apps.Deployment{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), deploymentBytes, coreDNSDeployment); err != nil {
return fmt.Errorf("unable to decode CoreDNS deployment %v", err)
}
// Create the Deployment for CoreDNS or update it in case it already exists
if err := apiclient.CreateOrUpdateDeployment(client, coreDNSDeployment); err != nil {
return err
}
coreDNSService := &v1.Service{}
return createDNSService(coreDNSService, serviceBytes, client)
}
func createDNSService(dnsService *v1.Service, serviceBytes []byte, client clientset.Interface) error {
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), serviceBytes, dnsService); err != nil {
return fmt.Errorf("unable to decode the DNS service %v", err)
}
// Can't use a generic apiclient helper func here as we have to tolerate more than AlreadyExists.
if _, err := client.CoreV1().Services(metav1.NamespaceSystem).Create(dnsService); err != nil {
// Ignore if the Service is invalid with this error message:
// Service "kube-dns" is invalid: spec.clusterIP: Invalid value: "10.96.0.10": provided IP is already allocated
if !apierrors.IsAlreadyExists(err) && !apierrors.IsInvalid(err) {
return fmt.Errorf("unable to create a new DNS service: %v", err)
}
if _, err := client.CoreV1().Services(metav1.NamespaceSystem).Update(dnsService); err != nil {
return fmt.Errorf("unable to create/update the DNS service: %v", err)
}
}
return nil
}
// translateStubDomainOfKubeDNSToProxyCoreDNS translates StubDomain Data in kube-dns ConfigMap
// in the form of Proxy for the CoreDNS Corefile.
func translateStubDomainOfKubeDNSToProxyCoreDNS(dataField string, kubeDNSConfigMap *v1.ConfigMap) (string, error) {
if kubeDNSConfigMap == nil {
return "", nil
}
if proxy, ok := kubeDNSConfigMap.Data[dataField]; ok {
stubDomainData := make(map[string][]string)
err := json.Unmarshal([]byte(proxy), &stubDomainData)
if err != nil {
return "", fmt.Errorf("failed to parse JSON from 'kube-dns ConfigMap: %v", err)
}
var proxyStanza []interface{}
for domain, proxyIP := range stubDomainData {
pStanza := map[string]interface{}{}
pStanza["keys"] = []string{domain + ":53"}
pStanza["body"] = [][]string{
{"errors"},
{"cache", "30"},
append([]string{"proxy", "."}, proxyIP...),
}
proxyStanza = append(proxyStanza, pStanza)
}
stanzasBytes, err := json.Marshal(proxyStanza)
if err != nil {
return "", err
}
corefileStanza, err := caddyfile.FromJSON(stanzasBytes)
if err != nil {
return "", err
}
return prepCorefileFormat(string(corefileStanza), 4), nil
}
return "", nil
}
// translateUpstreamNameServerOfKubeDNSToUpstreamProxyCoreDNS translates UpstreamNameServer Data in kube-dns ConfigMap
// in the form of Proxy for the CoreDNS Corefile.
func translateUpstreamNameServerOfKubeDNSToUpstreamProxyCoreDNS(dataField string, kubeDNSConfigMap *v1.ConfigMap) (string, error) {
if kubeDNSConfigMap == nil {
return "", nil
}
if upstreamValues, ok := kubeDNSConfigMap.Data[dataField]; ok {
var upstreamProxyIP []string
err := json.Unmarshal([]byte(upstreamValues), &upstreamProxyIP)
if err != nil {
return "", fmt.Errorf("failed to parse JSON from 'kube-dns ConfigMap: %v", err)
}
coreDNSProxyStanzaList := strings.Join(upstreamProxyIP, " ")
return coreDNSProxyStanzaList, nil
}
return "/etc/resolv.conf", nil
}
// translateFederationsofKubeDNSToCoreDNS translates Federations Data in kube-dns ConfigMap
// to Federation for CoreDNS Corefile.
func translateFederationsofKubeDNSToCoreDNS(dataField, coreDNSDomain string, kubeDNSConfigMap *v1.ConfigMap) (string, error) {
if kubeDNSConfigMap == nil {
return "", nil
}
if federation, ok := kubeDNSConfigMap.Data[dataField]; ok {
var (
federationStanza []interface{}
body [][]string
)
federationData := make(map[string]string)
err := json.Unmarshal([]byte(federation), &federationData)
if err != nil {
return "", fmt.Errorf("failed to parse JSON from kube-dns ConfigMap: %v", err)
}
fStanza := map[string]interface{}{}
for name, domain := range federationData {
body = append(body, []string{name, domain})
}
federationStanza = append(federationStanza, fStanza)
fStanza["keys"] = []string{"federation " + coreDNSDomain}
fStanza["body"] = body
stanzasBytes, err := json.Marshal(federationStanza)
if err != nil {
return "", err
}
corefileStanza, err := caddyfile.FromJSON(stanzasBytes)
if err != nil {
return "", err
}
return prepCorefileFormat(string(corefileStanza), 8), nil
}
return "", nil
}
// prepCorefileFormat indents the output of the Corefile caddytext and replaces tabs with spaces
// to neatly format the configmap, making it readable.
func prepCorefileFormat(s string, indentation int) string {
r := []string{}
for _, line := range strings.Split(s, "\n") {
indented := strings.Repeat(" ", indentation) + line
r = append(r, indented)
}
corefile := strings.Join(r, "\n")
return "\n" + strings.Replace(corefile, "\t", " ", -1)
}

View File

@@ -0,0 +1,418 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package dns
import (
"strings"
"testing"
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientsetfake "k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
api "k8s.io/kubernetes/pkg/apis/core"
)
func TestCreateServiceAccount(t *testing.T) {
tests := []struct {
name string
createErr error
expectErr bool
}{
{
"error-free case",
nil,
false,
},
{
"duplication errors should be ignored",
apierrors.NewAlreadyExists(api.Resource(""), ""),
false,
},
{
"unexpected errors should be returned",
apierrors.NewUnauthorized(""),
true,
},
}
for _, tc := range tests {
client := clientsetfake.NewSimpleClientset()
if tc.createErr != nil {
client.PrependReactor("create", "serviceaccounts", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, tc.createErr
})
}
err := CreateServiceAccount(client)
if tc.expectErr {
if err == nil {
t.Errorf("CreateServiceAccounts(%s) wanted err, got nil", tc.name)
}
continue
} else if !tc.expectErr && err != nil {
t.Errorf("CreateServiceAccounts(%s) returned unexpected err: %v", tc.name, err)
}
wantResourcesCreated := 1
if len(client.Actions()) != wantResourcesCreated {
t.Errorf("CreateServiceAccounts(%s) should have made %d actions, but made %d", tc.name, wantResourcesCreated, len(client.Actions()))
}
for _, action := range client.Actions() {
if action.GetVerb() != "create" || action.GetResource().Resource != "serviceaccounts" {
t.Errorf("CreateServiceAccounts(%s) called [%v %v], but wanted [create serviceaccounts]",
tc.name, action.GetVerb(), action.GetResource().Resource)
}
}
}
}
func TestCompileManifests(t *testing.T) {
var tests = []struct {
manifest string
data interface{}
expected bool
}{
{
manifest: KubeDNSDeployment,
data: struct{ ImageRepository, Arch, Version, DNSBindAddr, DNSProbeAddr, DNSDomain, MasterTaintKey string }{
ImageRepository: "foo",
Arch: "foo",
Version: "foo",
DNSBindAddr: "foo",
DNSProbeAddr: "foo",
DNSDomain: "foo",
MasterTaintKey: "foo",
},
expected: true,
},
{
manifest: KubeDNSService,
data: struct{ DNSIP string }{
DNSIP: "foo",
},
expected: true,
},
{
manifest: CoreDNSDeployment,
data: struct{ ImageRepository, MasterTaintKey, Version string }{
ImageRepository: "foo",
MasterTaintKey: "foo",
Version: "foo",
},
expected: true,
},
{
manifest: KubeDNSService,
data: struct{ DNSIP string }{
DNSIP: "foo",
},
expected: true,
},
{
manifest: CoreDNSConfigMap,
data: struct{ DNSDomain, Federation, UpstreamNameserver, StubDomain string }{
DNSDomain: "foo",
Federation: "foo",
UpstreamNameserver: "foo",
StubDomain: "foo",
},
expected: true,
},
}
for _, rt := range tests {
_, actual := kubeadmutil.ParseTemplate(rt.manifest, rt.data)
if (actual == nil) != rt.expected {
t.Errorf(
"failed CompileManifests:\n\texpected: %t\n\t actual: %t",
rt.expected,
(actual == nil),
)
}
}
}
func TestGetDNSIP(t *testing.T) {
var tests = []struct {
svcSubnet, expectedDNSIP string
}{
{
svcSubnet: "10.96.0.0/12",
expectedDNSIP: "10.96.0.10",
},
{
svcSubnet: "10.87.116.64/26",
expectedDNSIP: "10.87.116.74",
},
}
for _, rt := range tests {
dnsIP, err := kubeadmconstants.GetDNSIP(rt.svcSubnet)
if err != nil {
t.Fatalf("couldn't get dnsIP : %v", err)
}
actualDNSIP := dnsIP.String()
if actualDNSIP != rt.expectedDNSIP {
t.Errorf(
"failed GetDNSIP\n\texpected: %s\n\t actual: %s",
rt.expectedDNSIP,
actualDNSIP,
)
}
}
}
func TestTranslateStubDomainKubeDNSToCoreDNS(t *testing.T) {
testCases := []struct {
configMap *v1.ConfigMap
expectOne string
expectTwo string
}{
{
configMap: &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "kube-dns",
Namespace: "kube-system",
},
Data: map[string]string{
"stubDomains": `{"foo.com" : ["1.2.3.4:5300","3.3.3.3"], "my.cluster.local" : ["2.3.4.5"]}`,
"upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`,
},
},
expectOne: `
foo.com:53 {
errors
cache 30
proxy . 1.2.3.4:5300 3.3.3.3
}
my.cluster.local:53 {
errors
cache 30
proxy . 2.3.4.5
}`,
expectTwo: `
my.cluster.local:53 {
errors
cache 30
proxy . 2.3.4.5
}
foo.com:53 {
errors
cache 30
proxy . 1.2.3.4:5300 3.3.3.3
}`,
},
{
configMap: &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "kubedns",
Namespace: "kube-system",
},
},
expectOne: "",
},
{
configMap: &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "kube-dns",
Namespace: "kube-system",
},
Data: map[string]string{
"stubDomains": `{"foo.com" : ["1.2.3.4:5300"], "my.cluster.local" : ["2.3.4.5"]}`,
"upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`,
},
},
expectOne: `
foo.com:53 {
errors
cache 30
proxy . 1.2.3.4:5300
}
my.cluster.local:53 {
errors
cache 30
proxy . 2.3.4.5
}`,
expectTwo: `
my.cluster.local:53 {
errors
cache 30
proxy . 2.3.4.5
}
foo.com:53 {
errors
cache 30
proxy . 1.2.3.4:5300
}`,
},
{
configMap: &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "kube-dns",
Namespace: "kube-system",
},
Data: map[string]string{
"upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`,
},
},
expectOne: "",
},
}
for _, testCase := range testCases {
out, err := translateStubDomainOfKubeDNSToProxyCoreDNS(kubeDNSStubDomain, testCase.configMap)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if !strings.Contains(out, testCase.expectOne) && !strings.Contains(out, testCase.expectTwo) {
t.Errorf("expected to find %q or %q in output: %q", testCase.expectOne, testCase.expectTwo, out)
}
}
}
func TestTranslateUpstreamKubeDNSToCoreDNS(t *testing.T) {
testCases := []struct {
configMap *v1.ConfigMap
expect string
}{
{
configMap: &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "kube-dns",
Namespace: "kube-system",
},
},
expect: "/etc/resolv.conf",
},
{
configMap: &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "kubedns",
Namespace: "kube-system",
},
Data: map[string]string{
"stubDomains": ` {"foo.com" : ["1.2.3.4:5300"], "my.cluster.local" : ["2.3.4.5"]}`,
"upstreamNameservers": `["8.8.8.8", "8.8.4.4", "4.4.4.4"]`,
},
},
expect: "8.8.8.8 8.8.4.4 4.4.4.4",
},
{
configMap: &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "kubedns",
Namespace: "kube-system",
},
Data: map[string]string{
"upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`,
},
},
expect: "8.8.8.8 8.8.4.4",
},
}
for _, testCase := range testCases {
out, err := translateUpstreamNameServerOfKubeDNSToUpstreamProxyCoreDNS(kubeDNSUpstreamNameservers, testCase.configMap)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if !strings.Contains(out, testCase.expect) {
t.Errorf("expected to find %q in output: %q", testCase.expect, out)
}
}
}
func TestTranslateFederationKubeDNSToCoreDNS(t *testing.T) {
testCases := []struct {
configMap *v1.ConfigMap
expectOne string
expectTwo string
}{
{
configMap: &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "kube-dns",
Namespace: "kube-system",
},
Data: map[string]string{
"federations": `{"foo" : "foo.feddomain.com", "bar" : "bar.feddomain.com"}`,
"stubDomains": `{"foo.com" : ["1.2.3.4:5300","3.3.3.3"], "my.cluster.local" : ["2.3.4.5"]}`,
"upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`,
},
},
expectOne: `
federation cluster.local {
foo foo.feddomain.com
bar bar.feddomain.com
}`,
expectTwo: `
federation cluster.local {
bar bar.feddomain.com
foo foo.feddomain.com
}`,
},
{
configMap: &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "kubedns",
Namespace: "kube-system",
},
},
expectOne: "",
},
{
configMap: &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "kube-dns",
Namespace: "kube-system",
},
Data: map[string]string{
"stubDomains": `{"foo.com" : ["1.2.3.4:5300"], "my.cluster.local" : ["2.3.4.5"]}`,
"upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`,
},
},
expectOne: "",
},
}
for _, testCase := range testCases {
out, err := translateFederationsofKubeDNSToCoreDNS(kubeDNSFederation, "cluster.local", testCase.configMap)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
if !strings.Contains(out, testCase.expectOne) && !strings.Contains(out, testCase.expectTwo) {
t.Errorf("expected to find %q or %q in output: %q", testCase.expectOne, testCase.expectTwo, out)
}
}
}

View File

@@ -0,0 +1,356 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package dns
const (
// KubeDNSDeployment is the kube-dns Deployment manifest for the kube-dns manifest for v1.7+
KubeDNSDeployment = `
apiVersion: apps/v1
kind: Deployment
metadata:
name: kube-dns
namespace: kube-system
labels:
k8s-app: kube-dns
spec:
# replicas: not specified here:
# 1. In order to make Addon Manager do not reconcile this replicas parameter.
# 2. Default is 1.
# 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
strategy:
rollingUpdate:
maxSurge: 10%
maxUnavailable: 0
selector:
matchLabels:
k8s-app: kube-dns
template:
metadata:
labels:
k8s-app: kube-dns
spec:
volumes:
- name: kube-dns-config
configMap:
name: kube-dns
optional: true
containers:
- name: kubedns
image: {{ .ImageRepository }}/k8s-dns-kube-dns-{{ .Arch }}:{{ .Version }}
imagePullPolicy: IfNotPresent
resources:
# TODO: Set memory limits when we've profiled the container for large
# clusters, then set request = limit to keep this container in
# guaranteed class. Currently, this container falls into the
# "burstable" category so the kubelet doesn't backoff from restarting it.
limits:
memory: 170Mi
requests:
cpu: 100m
memory: 70Mi
livenessProbe:
httpGet:
path: /healthcheck/kubedns
port: 10054
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
readinessProbe:
httpGet:
path: /readiness
port: 8081
scheme: HTTP
# we poll on pod startup for the Kubernetes master service and
# only setup the /readiness HTTP server once that's available.
initialDelaySeconds: 3
timeoutSeconds: 5
args:
- --domain={{ .DNSDomain }}.
- --dns-port=10053
- --config-dir=/kube-dns-config
- --v=2
env:
- name: PROMETHEUS_PORT
value: "10055"
ports:
- containerPort: 10053
name: dns-local
protocol: UDP
- containerPort: 10053
name: dns-tcp-local
protocol: TCP
- containerPort: 10055
name: metrics
protocol: TCP
volumeMounts:
- name: kube-dns-config
mountPath: /kube-dns-config
- name: dnsmasq
image: {{ .ImageRepository }}/k8s-dns-dnsmasq-nanny-{{ .Arch }}:{{ .Version }}
imagePullPolicy: IfNotPresent
livenessProbe:
httpGet:
path: /healthcheck/dnsmasq
port: 10054
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
args:
- -v=2
- -logtostderr
- -configDir=/etc/k8s/dns/dnsmasq-nanny
- -restartDnsmasq=true
- --
- -k
- --cache-size=1000
- --no-negcache
- --log-facility=-
- --server=/{{ .DNSDomain }}/{{ .DNSBindAddr }}#10053
- --server=/in-addr.arpa/{{ .DNSBindAddr }}#10053
- --server=/ip6.arpa/{{ .DNSBindAddr }}#10053
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
# see: https://github.com/kubernetes/kubernetes/issues/29055 for details
resources:
requests:
cpu: 150m
memory: 20Mi
volumeMounts:
- name: kube-dns-config
mountPath: /etc/k8s/dns/dnsmasq-nanny
- name: sidecar
image: {{ .ImageRepository }}/k8s-dns-sidecar-{{ .Arch }}:{{ .Version }}
imagePullPolicy: IfNotPresent
livenessProbe:
httpGet:
path: /metrics
port: 10054
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
args:
- --v=2
- --logtostderr
- --probe=kubedns,{{ .DNSProbeAddr }}:10053,kubernetes.default.svc.{{ .DNSDomain }},5,SRV
- --probe=dnsmasq,{{ .DNSProbeAddr }}:53,kubernetes.default.svc.{{ .DNSDomain }},5,SRV
ports:
- containerPort: 10054
name: metrics
protocol: TCP
resources:
requests:
memory: 20Mi
cpu: 10m
dnsPolicy: Default # Don't use cluster DNS.
serviceAccountName: kube-dns
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- key: {{ .MasterTaintKey }}
effect: NoSchedule
nodeSelector:
beta.kubernetes.io/arch: {{ .Arch }}
`
// KubeDNSService is the kube-dns Service manifest
KubeDNSService = `
apiVersion: v1
kind: Service
metadata:
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "KubeDNS"
name: kube-dns
namespace: kube-system
annotations:
prometheus.io/scrape: "true"
# Without this resourceVersion value, an update of the Service between versions will yield:
# Service "kube-dns" is invalid: metadata.resourceVersion: Invalid value: "": must be specified for an update
resourceVersion: "0"
spec:
clusterIP: {{ .DNSIP }}
ports:
- name: dns
port: 53
protocol: UDP
targetPort: 53
- name: dns-tcp
port: 53
protocol: TCP
targetPort: 53
selector:
k8s-app: kube-dns
`
// CoreDNSDeployment is the CoreDNS Deployment manifest
CoreDNSDeployment = `
apiVersion: apps/v1
kind: Deployment
metadata:
name: coredns
namespace: kube-system
labels:
k8s-app: kube-dns
spec:
replicas: 2
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
selector:
matchLabels:
k8s-app: kube-dns
template:
metadata:
labels:
k8s-app: kube-dns
spec:
serviceAccountName: coredns
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- key: {{ .MasterTaintKey }}
effect: NoSchedule
containers:
- name: coredns
image: {{ .ImageRepository }}/coredns:{{ .Version }}
imagePullPolicy: IfNotPresent
resources:
limits:
memory: 170Mi
requests:
cpu: 100m
memory: 70Mi
args: [ "-conf", "/etc/coredns/Corefile" ]
volumeMounts:
- name: config-volume
mountPath: /etc/coredns
readOnly: true
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
- containerPort: 9153
name: metrics
protocol: TCP
livenessProbe:
httpGet:
path: /health
port: 8080
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_BIND_SERVICE
drop:
- all
readOnlyRootFilesystem: true
dnsPolicy: Default
volumes:
- name: config-volume
configMap:
name: coredns
items:
- key: Corefile
path: Corefile
`
// CoreDNSConfigMap is the CoreDNS ConfigMap manifest
CoreDNSConfigMap = `
apiVersion: v1
kind: ConfigMap
metadata:
name: coredns
namespace: kube-system
data:
Corefile: |
.:53 {
errors
health
kubernetes {{ .DNSDomain }} in-addr.arpa ip6.arpa {
pods insecure
upstream
fallthrough in-addr.arpa ip6.arpa
}{{ .Federation }}
prometheus :9153
proxy . {{ .UpstreamNameserver }}
cache 30
reload
}{{ .StubDomain }}
`
// CoreDNSClusterRole is the CoreDNS ClusterRole manifest
CoreDNSClusterRole = `
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:coredns
rules:
- apiGroups:
- ""
resources:
- endpoints
- services
- pods
- namespaces
verbs:
- list
- watch
`
// CoreDNSClusterRoleBinding is the CoreDNS Clusterrolebinding manifest
CoreDNSClusterRoleBinding = `
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:coredns
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:coredns
subjects:
- kind: ServiceAccount
name: coredns
namespace: kube-system
`
// CoreDNSServiceAccount is the CoreDNS ServiceAccount manifest
CoreDNSServiceAccount = `
apiVersion: v1
kind: ServiceAccount
metadata:
name: coredns
namespace: kube-system
`
)

View File

@@ -0,0 +1,63 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = ["proxy_test.go"],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/config:go_default_library",
"//pkg/apis/core:go_default_library",
"//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
"//pkg/util/pointer:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/client-go/testing:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"manifests.go",
"proxy.go",
],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//pkg/proxy/apis/kubeproxyconfig/scheme:go_default_library",
"//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
"//vendor/k8s.io/api/apps/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/rbac/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View File

@@ -0,0 +1,112 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package proxy
const (
// KubeProxyConfigMap19 is the proxy ConfigMap manifest for Kubernetes 1.9 and above
KubeProxyConfigMap19 = `
kind: ConfigMap
apiVersion: v1
metadata:
name: kube-proxy
namespace: kube-system
labels:
app: kube-proxy
data:
kubeconfig.conf: |-
apiVersion: v1
kind: Config
clusters:
- cluster:
certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
server: {{ .MasterEndpoint }}
name: default
contexts:
- context:
cluster: default
namespace: default
user: default
name: default
current-context: default
users:
- name: default
user:
tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
config.conf: |-
{{ .ProxyConfig}}
`
// KubeProxyDaemonSet19 is the proxy DaemonSet manifest for Kubernetes 1.9 and above
KubeProxyDaemonSet19 = `
apiVersion: apps/v1
kind: DaemonSet
metadata:
labels:
k8s-app: kube-proxy
name: kube-proxy
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: kube-proxy
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
k8s-app: kube-proxy
spec:
containers:
- name: kube-proxy
image: {{ if .ImageOverride }}{{ .ImageOverride }}{{ else }}{{ .ImageRepository }}/kube-proxy-{{ .Arch }}:{{ .Version }}{{ end }}
imagePullPolicy: IfNotPresent
command:
- /usr/local/bin/kube-proxy
- --config=/var/lib/kube-proxy/config.conf
securityContext:
privileged: true
volumeMounts:
- mountPath: /var/lib/kube-proxy
name: kube-proxy
- mountPath: /run/xtables.lock
name: xtables-lock
readOnly: false
- mountPath: /lib/modules
name: lib-modules
readOnly: true
hostNetwork: true
serviceAccountName: kube-proxy
volumes:
- name: kube-proxy
configMap:
name: kube-proxy
- name: xtables-lock
hostPath:
path: /run/xtables.lock
type: FileOrCreate
- name: lib-modules
hostPath:
path: /lib/modules
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- key: {{ .MasterTaintKey }}
effect: NoSchedule
nodeSelector:
beta.kubernetes.io/arch: {{ .Arch }}
`
)

View File

@@ -0,0 +1,154 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package proxy
import (
"bytes"
"fmt"
"runtime"
apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kuberuntime "k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
clientsetscheme "k8s.io/client-go/kubernetes/scheme"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
kubeproxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"
kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
)
const (
// KubeProxyClusterRoleName sets the name for the kube-proxy ClusterRole
// TODO: This k8s-generic, well-known constant should be fetchable from another source, not be in this package
KubeProxyClusterRoleName = "system:node-proxier"
// KubeProxyServiceAccountName describes the name of the ServiceAccount for the kube-proxy addon
KubeProxyServiceAccountName = "kube-proxy"
)
// EnsureProxyAddon creates the kube-proxy addons
func EnsureProxyAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
if err := CreateServiceAccount(client); err != nil {
return fmt.Errorf("error when creating kube-proxy service account: %v", err)
}
// Generate Master Enpoint kubeconfig file
masterEndpoint, err := kubeadmutil.GetMasterEndpoint(&cfg.API)
if err != nil {
return err
}
proxyBytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg.KubeProxy.Config, kubeproxyconfigv1alpha1.SchemeGroupVersion,
kubeproxyconfigscheme.Codecs)
if err != nil {
return fmt.Errorf("error when marshaling: %v", err)
}
var prefixBytes bytes.Buffer
apiclient.PrintBytesWithLinePrefix(&prefixBytes, proxyBytes, " ")
var proxyConfigMapBytes, proxyDaemonSetBytes []byte
proxyConfigMapBytes, err = kubeadmutil.ParseTemplate(KubeProxyConfigMap19,
struct {
MasterEndpoint string
ProxyConfig string
}{
MasterEndpoint: masterEndpoint,
ProxyConfig: prefixBytes.String(),
})
if err != nil {
return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err)
}
proxyDaemonSetBytes, err = kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ ImageRepository, Arch, Version, ImageOverride, MasterTaintKey string }{
ImageRepository: cfg.GetControlPlaneImageRepository(),
Arch: runtime.GOARCH,
Version: kubeadmutil.KubernetesVersionToImageTag(cfg.KubernetesVersion),
ImageOverride: cfg.UnifiedControlPlaneImage,
MasterTaintKey: kubeadmconstants.LabelNodeRoleMaster,
})
if err != nil {
return fmt.Errorf("error when parsing kube-proxy daemonset template: %v", err)
}
if err := createKubeProxyAddon(proxyConfigMapBytes, proxyDaemonSetBytes, client); err != nil {
return err
}
if err := CreateRBACRules(client); err != nil {
return fmt.Errorf("error when creating kube-proxy RBAC rules: %v", err)
}
fmt.Println("[addons] Applied essential addon: kube-proxy")
return nil
}
// CreateServiceAccount creates the necessary serviceaccounts that kubeadm uses/might use, if they don't already exist.
func CreateServiceAccount(client clientset.Interface) error {
return apiclient.CreateOrUpdateServiceAccount(client, &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: KubeProxyServiceAccountName,
Namespace: metav1.NamespaceSystem,
},
})
}
// CreateRBACRules creates the essential RBAC rules for a minimally set-up cluster
func CreateRBACRules(client clientset.Interface) error {
return createClusterRoleBindings(client)
}
func createKubeProxyAddon(configMapBytes, daemonSetbytes []byte, client clientset.Interface) error {
kubeproxyConfigMap := &v1.ConfigMap{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), configMapBytes, kubeproxyConfigMap); err != nil {
return fmt.Errorf("unable to decode kube-proxy configmap %v", err)
}
// Create the ConfigMap for kube-proxy or update it in case it already exists
if err := apiclient.CreateOrUpdateConfigMap(client, kubeproxyConfigMap); err != nil {
return err
}
kubeproxyDaemonSet := &apps.DaemonSet{}
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), daemonSetbytes, kubeproxyDaemonSet); err != nil {
return fmt.Errorf("unable to decode kube-proxy daemonset %v", err)
}
// Create the DaemonSet for kube-proxy or update it in case it already exists
return apiclient.CreateOrUpdateDaemonSet(client, kubeproxyDaemonSet)
}
func createClusterRoleBindings(client clientset.Interface) error {
return apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "kubeadm:node-proxier",
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "ClusterRole",
Name: KubeProxyClusterRoleName,
},
Subjects: []rbac.Subject{
{
Kind: rbac.ServiceAccountKind,
Name: KubeProxyServiceAccountName,
Namespace: metav1.NamespaceSystem,
},
},
})
}

View File

@@ -0,0 +1,254 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package proxy
import (
"strings"
"testing"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientsetfake "k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
api "k8s.io/kubernetes/pkg/apis/core"
kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
"k8s.io/kubernetes/pkg/util/pointer"
)
func TestCreateServiceAccount(t *testing.T) {
tests := []struct {
name string
createErr error
expectErr bool
}{
{
"error-free case",
nil,
false,
},
{
"duplication errors should be ignored",
apierrors.NewAlreadyExists(api.Resource(""), ""),
false,
},
{
"unexpected errors should be returned",
apierrors.NewUnauthorized(""),
true,
},
}
for _, tc := range tests {
client := clientsetfake.NewSimpleClientset()
if tc.createErr != nil {
client.PrependReactor("create", "serviceaccounts", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, tc.createErr
})
}
err := CreateServiceAccount(client)
if tc.expectErr {
if err == nil {
t.Errorf("CreateServiceAccounts(%s) wanted err, got nil", tc.name)
}
continue
} else if !tc.expectErr && err != nil {
t.Errorf("CreateServiceAccounts(%s) returned unexpected err: %v", tc.name, err)
}
wantResourcesCreated := 1
if len(client.Actions()) != wantResourcesCreated {
t.Errorf("CreateServiceAccounts(%s) should have made %d actions, but made %d", tc.name, wantResourcesCreated, len(client.Actions()))
}
for _, action := range client.Actions() {
if action.GetVerb() != "create" || action.GetResource().Resource != "serviceaccounts" {
t.Errorf("CreateServiceAccounts(%s) called [%v %v], but wanted [create serviceaccounts]",
tc.name, action.GetVerb(), action.GetResource().Resource)
}
}
}
}
func TestCompileManifests(t *testing.T) {
var tests = []struct {
manifest string
data interface{}
expected bool
}{
{
manifest: KubeProxyConfigMap19,
data: struct {
MasterEndpoint, ProxyConfig string
}{
MasterEndpoint: "foo",
ProxyConfig: " bindAddress: 0.0.0.0\n clusterCIDR: 192.168.1.1\n enableProfiling: false",
},
expected: true,
},
{
manifest: KubeProxyDaemonSet19,
data: struct{ ImageRepository, Arch, Version, ImageOverride, MasterTaintKey, CloudTaintKey string }{
ImageRepository: "foo",
Arch: "foo",
Version: "foo",
ImageOverride: "foo",
MasterTaintKey: "foo",
CloudTaintKey: "foo",
},
expected: true,
},
}
for _, rt := range tests {
_, actual := kubeadmutil.ParseTemplate(rt.manifest, rt.data)
if (actual == nil) != rt.expected {
t.Errorf(
"failed to compile %s manifest:\n\texpected: %t\n\t actual: %t",
rt.manifest,
rt.expected,
(actual == nil),
)
}
}
}
func TestEnsureProxyAddon(t *testing.T) {
type SimulatedError int
const (
NoError SimulatedError = iota
ServiceAccountError
InvalidMasterEndpoint
IPv6SetBindAddress
)
var testCases = []struct {
name string
simError SimulatedError
expErrString string
expBindAddr string
expClusterCIDR string
}{
{
name: "Successful proxy addon",
simError: NoError,
expErrString: "",
expBindAddr: "0.0.0.0",
expClusterCIDR: "5.6.7.8/24",
}, {
name: "Simulated service account error",
simError: ServiceAccountError,
expErrString: "error when creating kube-proxy service account",
expBindAddr: "0.0.0.0",
expClusterCIDR: "5.6.7.8/24",
}, {
name: "IPv6 AdvertiseAddress address",
simError: IPv6SetBindAddress,
expErrString: "",
expBindAddr: "::",
expClusterCIDR: "2001:101::/96",
},
}
for _, tc := range testCases {
// Create a fake client and set up default test configuration
client := clientsetfake.NewSimpleClientset()
masterConfig := &kubeadmapiv1alpha2.MasterConfiguration{
API: kubeadmapiv1alpha2.API{
AdvertiseAddress: "1.2.3.4",
BindPort: 1234,
},
KubeProxy: kubeadmapiv1alpha2.KubeProxy{
Config: &kubeproxyconfigv1alpha1.KubeProxyConfiguration{
BindAddress: "",
HealthzBindAddress: "0.0.0.0:10256",
MetricsBindAddress: "127.0.0.1:10249",
Conntrack: kubeproxyconfigv1alpha1.KubeProxyConntrackConfiguration{
Max: pointer.Int32Ptr(2),
MaxPerCore: pointer.Int32Ptr(1),
Min: pointer.Int32Ptr(1),
TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second},
TCPCloseWaitTimeout: &metav1.Duration{Duration: 5 * time.Second},
},
},
},
Networking: kubeadmapiv1alpha2.Networking{
PodSubnet: "5.6.7.8/24",
},
ImageRepository: "someRepo",
KubernetesVersion: "v1.10.0",
UnifiedControlPlaneImage: "someImage",
}
// Simulate an error if necessary
switch tc.simError {
case ServiceAccountError:
client.PrependReactor("create", "serviceaccounts", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, apierrors.NewUnauthorized("")
})
case InvalidMasterEndpoint:
masterConfig.API.AdvertiseAddress = "1.2.3"
case IPv6SetBindAddress:
masterConfig.API.AdvertiseAddress = "1:2::3:4"
masterConfig.Networking.PodSubnet = "2001:101::/96"
}
kubeadmapiv1alpha2.SetDefaults_MasterConfiguration(masterConfig)
intMaster, err := cmdutil.ConfigFileAndDefaultsToInternalConfig("", masterConfig)
if err != nil {
t.Errorf(" test failed to convert v1alpha1 to internal version")
break
}
err = EnsureProxyAddon(intMaster, client)
// Compare actual to expected errors
actErr := "No error"
if err != nil {
actErr = err.Error()
}
expErr := "No error"
if tc.expErrString != "" {
expErr = tc.expErrString
}
if !strings.Contains(actErr, expErr) {
t.Errorf(
"%s test failed, expected: %s, got: %s",
tc.name,
expErr,
actErr)
}
if intMaster.KubeProxy.Config.BindAddress != tc.expBindAddr {
t.Errorf("%s test failed, expected: %s, got: %s",
tc.name,
tc.expBindAddr,
intMaster.KubeProxy.Config.BindAddress)
}
if intMaster.KubeProxy.Config.ClusterCIDR != tc.expClusterCIDR {
t.Errorf("%s test failed, expected: %s, got: %s",
tc.name,
tc.expClusterCIDR,
intMaster.KubeProxy.Config.ClusterCIDR)
}
}
}

View File

@@ -0,0 +1,52 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = ["clusterinfo_test.go"],
embed = [":go_default_library"],
deps = [
"//pkg/apis/core:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/client-go/testing:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = ["clusterinfo.go"],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo",
deps = [
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//pkg/apis/rbac/v1:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/rbac/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/tools/bootstrap/token/api:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View File

@@ -0,0 +1,110 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package clusterinfo
import (
"fmt"
"github.com/golang/glog"
"k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/authentication/user"
clientset "k8s.io/client-go/kubernetes"
bootstrapapi "k8s.io/client-go/tools/bootstrap/token/api"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
)
const (
// BootstrapSignerClusterRoleName sets the name for the ClusterRole that allows access to ConfigMaps in the kube-public ns
BootstrapSignerClusterRoleName = "kubeadm:bootstrap-signer-clusterinfo"
)
// CreateBootstrapConfigMapIfNotExists creates the kube-public ConfigMap if it doesn't exist already
func CreateBootstrapConfigMapIfNotExists(client clientset.Interface, file string) error {
fmt.Printf("[bootstraptoken] creating the %q ConfigMap in the %q namespace\n", bootstrapapi.ConfigMapClusterInfo, metav1.NamespacePublic)
glog.V(1).Infoln("[bootstraptoken] loading admin kubeconfig")
adminConfig, err := clientcmd.LoadFromFile(file)
if err != nil {
return fmt.Errorf("failed to load admin kubeconfig [%v]", err)
}
adminCluster := adminConfig.Contexts[adminConfig.CurrentContext].Cluster
// Copy the cluster from admin.conf to the bootstrap kubeconfig, contains the CA cert and the server URL
glog.V(1).Infoln("[bootstraptoken] copying the cluster from admin.conf to the bootstrap kubeconfig")
bootstrapConfig := &clientcmdapi.Config{
Clusters: map[string]*clientcmdapi.Cluster{
"": adminConfig.Clusters[adminCluster],
},
}
bootstrapBytes, err := clientcmd.Write(*bootstrapConfig)
if err != nil {
return err
}
// Create or update the ConfigMap in the kube-public namespace
glog.V(1).Infoln("[bootstraptoken] creating/updating ConfigMap in kube-public namespace")
return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: bootstrapapi.ConfigMapClusterInfo,
Namespace: metav1.NamespacePublic,
},
Data: map[string]string{
bootstrapapi.KubeConfigKey: string(bootstrapBytes),
},
})
}
// CreateClusterInfoRBACRules creates the RBAC rules for exposing the cluster-info ConfigMap in the kube-public namespace to unauthenticated users
func CreateClusterInfoRBACRules(client clientset.Interface) error {
glog.V(1).Infoln("creating the RBAC rules for exposing the cluster-info ConfigMap in the kube-public namespace")
err := apiclient.CreateOrUpdateRole(client, &rbac.Role{
ObjectMeta: metav1.ObjectMeta{
Name: BootstrapSignerClusterRoleName,
Namespace: metav1.NamespacePublic,
},
Rules: []rbac.PolicyRule{
rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(bootstrapapi.ConfigMapClusterInfo).RuleOrDie(),
},
})
if err != nil {
return err
}
return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: BootstrapSignerClusterRoleName,
Namespace: metav1.NamespacePublic,
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "Role",
Name: BootstrapSignerClusterRoleName,
},
Subjects: []rbac.Subject{
{
Kind: rbac.UserKind,
Name: user.Anonymous,
},
},
})
}

View File

@@ -0,0 +1,113 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package clusterinfo
import (
"io/ioutil"
"os"
"testing"
"text/template"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
clientsetfake "k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
api "k8s.io/kubernetes/pkg/apis/core"
)
var testConfigTempl = template.Must(template.New("test").Parse(`apiVersion: v1
clusters:
- cluster:
server: {{.Server}}
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: kubernetes-admin
name: kubernetes-admin@kubernetes
current-context: kubernetes-admin@kubernetes
kind: Config
preferences: {}
users:
- name: kubernetes-admin`))
func TestCreateBootstrapConfigMapIfNotExists(t *testing.T) {
tests := []struct {
name string
createErr error
updateErr error
expectErr bool
}{
{
"successful case should have no error",
nil,
nil,
false,
},
{
"if both create and update errors, return error",
apierrors.NewAlreadyExists(api.Resource("configmaps"), "test"),
apierrors.NewUnauthorized("go away!"),
true,
},
{
"unexpected error should be returned",
apierrors.NewUnauthorized("go away!"),
nil,
true,
},
}
servers := []struct {
Server string
}{
{Server: "https://10.128.0.6:6443"},
{Server: "https://[2001:db8::6]:3446"},
}
for _, server := range servers {
file, err := ioutil.TempFile("", "")
if err != nil {
t.Fatalf("could not create tempfile: %v", err)
}
defer os.Remove(file.Name())
if err := testConfigTempl.Execute(file, server); err != nil {
t.Fatalf("could not write to tempfile: %v", err)
}
if err := file.Close(); err != nil {
t.Fatalf("could not close tempfile: %v", err)
}
for _, tc := range tests {
client := clientsetfake.NewSimpleClientset()
if tc.createErr != nil {
client.PrependReactor("create", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, tc.createErr
})
}
err := CreateBootstrapConfigMapIfNotExists(client, file.Name())
if tc.expectErr && err == nil {
t.Errorf("CreateBootstrapConfigMapIfNotExists(%s) wanted error, got nil", tc.name)
} else if !tc.expectErr && err != nil {
t.Errorf("CreateBootstrapConfigMapIfNotExists(%s) returned unexpected error: %v", tc.name, err)
}
}
}
}

View File

@@ -0,0 +1,37 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"tlsbootstrap.go",
"token.go",
],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//vendor/k8s.io/api/rbac/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/tools/bootstrap/token/util:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View File

@@ -0,0 +1,113 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package node
import (
"fmt"
rbac "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
)
const (
// NodeBootstrapperClusterRoleName defines the name of the auto-bootstrapped ClusterRole for letting someone post a CSR
// TODO: This value should be defined in an other, generic authz package instead of here
NodeBootstrapperClusterRoleName = "system:node-bootstrapper"
// NodeKubeletBootstrap defines the name of the ClusterRoleBinding that lets kubelets post CSRs
NodeKubeletBootstrap = "kubeadm:kubelet-bootstrap"
// CSRAutoApprovalClusterRoleName defines the name of the auto-bootstrapped ClusterRole for making the csrapprover controller auto-approve the CSR
// TODO: This value should be defined in an other, generic authz package instead of here
// Starting from v1.8, CSRAutoApprovalClusterRoleName is automatically created by the API server on startup
CSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:nodeclient"
// NodeSelfCSRAutoApprovalClusterRoleName is a role defined in default 1.8 RBAC policies for automatic CSR approvals for automatically rotated node certificates
NodeSelfCSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:selfnodeclient"
// NodeAutoApproveBootstrapClusterRoleBinding defines the name of the ClusterRoleBinding that makes the csrapprover approve node CSRs
NodeAutoApproveBootstrapClusterRoleBinding = "kubeadm:node-autoapprove-bootstrap"
// NodeAutoApproveCertificateRotationClusterRoleBinding defines name of the ClusterRoleBinding that makes the csrapprover approve node auto rotated CSRs
NodeAutoApproveCertificateRotationClusterRoleBinding = "kubeadm:node-autoapprove-certificate-rotation"
)
// AllowBootstrapTokensToPostCSRs creates RBAC rules in a way the makes Node Bootstrap Tokens able to post CSRs
func AllowBootstrapTokensToPostCSRs(client clientset.Interface) error {
fmt.Println("[bootstraptoken] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials")
return apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: NodeKubeletBootstrap,
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "ClusterRole",
Name: NodeBootstrapperClusterRoleName,
},
Subjects: []rbac.Subject{
{
Kind: rbac.GroupKind,
Name: constants.NodeBootstrapTokenAuthGroup,
},
},
})
}
// AutoApproveNodeBootstrapTokens creates RBAC rules in a way that makes Node Bootstrap Tokens' CSR auto-approved by the csrapprover controller
func AutoApproveNodeBootstrapTokens(client clientset.Interface) error {
fmt.Println("[bootstraptoken] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token")
// Always create this kubeadm-specific binding though
return apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: NodeAutoApproveBootstrapClusterRoleBinding,
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "ClusterRole",
Name: CSRAutoApprovalClusterRoleName,
},
Subjects: []rbac.Subject{
{
Kind: "Group",
Name: constants.NodeBootstrapTokenAuthGroup,
},
},
})
}
// AutoApproveNodeCertificateRotation creates RBAC rules in a way that makes Node certificate rotation CSR auto-approved by the csrapprover controller
func AutoApproveNodeCertificateRotation(client clientset.Interface) error {
fmt.Println("[bootstraptoken] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster")
return apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: NodeAutoApproveCertificateRotationClusterRoleBinding,
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "ClusterRole",
Name: NodeSelfCSRAutoApprovalClusterRoleName,
},
Subjects: []rbac.Subject{
{
Kind: "Group",
Name: constants.NodesGroup,
},
},
})
}

View File

@@ -0,0 +1,60 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package node
import (
"fmt"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
bootstraputil "k8s.io/client-go/tools/bootstrap/token/util"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
)
// TODO(mattmoyer): Move CreateNewTokens, UpdateOrCreateTokens out of this package to client-go for a generic abstraction and client for a Bootstrap Token
// CreateNewTokens tries to create a token and fails if one with the same ID already exists
func CreateNewTokens(client clientset.Interface, tokens []kubeadmapi.BootstrapToken) error {
return UpdateOrCreateTokens(client, true, tokens)
}
// UpdateOrCreateTokens attempts to update a token with the given ID, or create if it does not already exist.
func UpdateOrCreateTokens(client clientset.Interface, failIfExists bool, tokens []kubeadmapi.BootstrapToken) error {
for _, token := range tokens {
secretName := bootstraputil.BootstrapTokenSecretName(token.Token.ID)
secret, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Get(secretName, metav1.GetOptions{})
if secret != nil && err == nil && failIfExists {
return fmt.Errorf("a token with id %q already exists", token.Token.ID)
}
updatedOrNewSecret := token.ToSecret()
// Try to create or update the token with an exponential backoff
err = apiclient.TryRunCommand(func() error {
if err := apiclient.CreateOrUpdateSecret(client, updatedOrNewSecret); err != nil {
return fmt.Errorf("failed to create or update bootstrap token with name %s: %v", secretName, err)
}
return nil
}, 5)
if err != nil {
return err
}
}
return nil
}

View File

@@ -0,0 +1,52 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = ["certs_test.go"],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
"//cmd/kubeadm/test:go_default_library",
"//cmd/kubeadm/test/certs:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"certs.go",
"doc.go",
],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/client-go/util/cert:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//cmd/kubeadm/app/phases/certs/pkiutil:all-srcs",
],
tags = ["automanaged"],
)

View File

@@ -0,0 +1,721 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package certs
import (
"crypto/rsa"
"crypto/x509"
"fmt"
"os"
"path/filepath"
"github.com/golang/glog"
certutil "k8s.io/client-go/util/cert"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
)
// CreatePKIAssets will create and write to disk all PKI assets necessary to establish the control plane.
// If the PKI assets already exists in the target folder, they are used only if evaluated equal; otherwise an error is returned.
func CreatePKIAssets(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating PKI assets")
certActions := []func(cfg *kubeadmapi.MasterConfiguration) error{
CreateCACertAndKeyFiles,
CreateAPIServerCertAndKeyFiles,
CreateAPIServerKubeletClientCertAndKeyFiles,
CreateServiceAccountKeyAndPublicKeyFiles,
CreateFrontProxyCACertAndKeyFiles,
CreateFrontProxyClientCertAndKeyFiles,
}
etcdCertActions := []func(cfg *kubeadmapi.MasterConfiguration) error{
CreateEtcdCACertAndKeyFiles,
CreateEtcdServerCertAndKeyFiles,
CreateEtcdPeerCertAndKeyFiles,
CreateEtcdHealthcheckClientCertAndKeyFiles,
CreateAPIServerEtcdClientCertAndKeyFiles,
}
if cfg.Etcd.Local != nil {
certActions = append(certActions, etcdCertActions...)
}
for _, action := range certActions {
err := action(cfg)
if err != nil {
return err
}
}
fmt.Printf("[certificates] valid certificates and keys now exist in %q\n", cfg.CertificatesDir)
return nil
}
// CreateCACertAndKeyFiles create a new self signed cluster CA certificate and key files.
// If the CA certificate and key files already exists in the target folder, they are used only if evaluated equal; otherwise an error is returned.
func CreateCACertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("create a new self signed cluster CA certificate and key files")
caCert, caKey, err := NewCACertAndKey()
if err != nil {
return err
}
return writeCertificateAuthorithyFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.CACertAndKeyBaseName,
caCert,
caKey,
)
}
// CreateAPIServerCertAndKeyFiles create a new certificate and key files for the apiserver.
// If the apiserver certificate and key files already exists in the target folder, they are used only if evaluated equal; otherwise an error is returned.
// It assumes the cluster CA certificate and key files exist in the CertificatesDir.
func CreateAPIServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a new certificate and key files for the apiserver")
caCert, caKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
if err != nil {
return err
}
apiCert, apiKey, err := NewAPIServerCertAndKey(cfg, caCert, caKey)
if err != nil {
return err
}
return writeCertificateFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.APIServerCertAndKeyBaseName,
caCert,
apiCert,
apiKey,
)
}
// CreateAPIServerKubeletClientCertAndKeyFiles create a new certificate for kubelets calling apiserver.
// If the apiserver-kubelet-client certificate and key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned.
// It assumes the cluster CA certificate and key files exist in the CertificatesDir.
func CreateAPIServerKubeletClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a new certificate for kubelets calling apiserver")
caCert, caKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
if err != nil {
return err
}
apiKubeletClientCert, apiKubeletClientKey, err := NewAPIServerKubeletClientCertAndKey(caCert, caKey)
if err != nil {
return err
}
return writeCertificateFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName,
caCert,
apiKubeletClientCert,
apiKubeletClientKey,
)
}
// CreateEtcdCACertAndKeyFiles create a self signed etcd CA certificate and key files.
// The etcd CA and client certs are used to secure communication between etcd peers and connections to etcd from the API server.
// This is a separate CA, so that kubernetes client identities cannot connect to etcd directly or peer with the etcd cluster.
// If the etcd CA certificate and key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned.
func CreateEtcdCACertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a self signed etcd CA certificate and key files")
etcdCACert, etcdCAKey, err := NewEtcdCACertAndKey()
if err != nil {
return err
}
return writeCertificateAuthorithyFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.EtcdCACertAndKeyBaseName,
etcdCACert,
etcdCAKey,
)
}
// CreateEtcdServerCertAndKeyFiles create a new certificate and key file for etcd.
// If the etcd serving certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned.
// It assumes the etcd CA certificate and key file exist in the CertificatesDir
func CreateEtcdServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a new server certificate and key files for etcd")
etcdCACert, etcdCAKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.EtcdCACertAndKeyBaseName)
if err != nil {
return err
}
etcdServerCert, etcdServerKey, err := NewEtcdServerCertAndKey(cfg, etcdCACert, etcdCAKey)
if err != nil {
return err
}
return writeCertificateFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.EtcdServerCertAndKeyBaseName,
etcdCACert,
etcdServerCert,
etcdServerKey,
)
}
// CreateEtcdPeerCertAndKeyFiles create a new certificate and key file for etcd peering.
// If the etcd peer certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned.
// It assumes the etcd CA certificate and key file exist in the CertificatesDir
func CreateEtcdPeerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a new certificate and key files for etcd peering")
etcdCACert, etcdCAKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.EtcdCACertAndKeyBaseName)
if err != nil {
return err
}
etcdPeerCert, etcdPeerKey, err := NewEtcdPeerCertAndKey(cfg, etcdCACert, etcdCAKey)
if err != nil {
return err
}
return writeCertificateFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.EtcdPeerCertAndKeyBaseName,
etcdCACert,
etcdPeerCert,
etcdPeerKey,
)
}
// CreateEtcdHealthcheckClientCertAndKeyFiles create a new client certificate for liveness probes to healthcheck etcd
// If the etcd-healthcheck-client certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned.
// It assumes the etcd CA certificate and key file exist in the CertificatesDir
func CreateEtcdHealthcheckClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
etcdCACert, etcdCAKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.EtcdCACertAndKeyBaseName)
if err != nil {
return err
}
etcdHealthcheckClientCert, etcdHealthcheckClientKey, err := NewEtcdHealthcheckClientCertAndKey(etcdCACert, etcdCAKey)
if err != nil {
return err
}
return writeCertificateFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.EtcdHealthcheckClientCertAndKeyBaseName,
etcdCACert,
etcdHealthcheckClientCert,
etcdHealthcheckClientKey,
)
}
// CreateAPIServerEtcdClientCertAndKeyFiles create a new client certificate for the apiserver calling etcd
// If the apiserver-etcd-client certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned.
// It assumes the etcd CA certificate and key file exist in the CertificatesDir
func CreateAPIServerEtcdClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a new client certificate for the apiserver calling etcd")
etcdCACert, etcdCAKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.EtcdCACertAndKeyBaseName)
if err != nil {
return err
}
apiEtcdClientCert, apiEtcdClientKey, err := NewAPIServerEtcdClientCertAndKey(etcdCACert, etcdCAKey)
if err != nil {
return err
}
return writeCertificateFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName,
etcdCACert,
apiEtcdClientCert,
apiEtcdClientKey,
)
}
// CreateServiceAccountKeyAndPublicKeyFiles create a new public/private key files for signing service account users.
// If the sa public/private key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned.
func CreateServiceAccountKeyAndPublicKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a new public/private key files for signing service account users")
saSigningKey, err := NewServiceAccountSigningKey()
if err != nil {
return err
}
return writeKeyFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.ServiceAccountKeyBaseName,
saSigningKey,
)
}
// CreateFrontProxyCACertAndKeyFiles create a self signed front proxy CA certificate and key files.
// Front proxy CA and client certs are used to secure a front proxy authenticator which is used to assert identity
// without the client cert; This is a separate CA, so that front proxy identities cannot hit the API and normal client certs cannot be used
// as front proxies.
// If the front proxy CA certificate and key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned.
func CreateFrontProxyCACertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a self signed front proxy CA certificate and key files")
frontProxyCACert, frontProxyCAKey, err := NewFrontProxyCACertAndKey()
if err != nil {
return err
}
return writeCertificateAuthorithyFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.FrontProxyCACertAndKeyBaseName,
frontProxyCACert,
frontProxyCAKey,
)
}
// CreateFrontProxyClientCertAndKeyFiles create a new certificate for proxy server client.
// If the front-proxy-client certificate and key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned.
// It assumes the front proxy CA certificate and key files exist in the CertificatesDir.
func CreateFrontProxyClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a new certificate for proxy server client")
frontProxyCACert, frontProxyCAKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName)
if err != nil {
return err
}
frontProxyClientCert, frontProxyClientKey, err := NewFrontProxyClientCertAndKey(frontProxyCACert, frontProxyCAKey)
if err != nil {
return err
}
return writeCertificateFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.FrontProxyClientCertAndKeyBaseName,
frontProxyCACert,
frontProxyClientCert,
frontProxyClientKey,
)
}
// NewCACertAndKey will generate a self signed CA.
func NewCACertAndKey() (*x509.Certificate, *rsa.PrivateKey, error) {
caCert, caKey, err := pkiutil.NewCertificateAuthority()
if err != nil {
return nil, nil, fmt.Errorf("failure while generating CA certificate and key: %v", err)
}
return caCert, caKey, nil
}
// NewAPIServerCertAndKey generate certificate for apiserver, signed by the given CA.
func NewAPIServerCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
altNames, err := pkiutil.GetAPIServerAltNames(cfg)
if err != nil {
return nil, nil, fmt.Errorf("failure while composing altnames for API server: %v", err)
}
config := certutil.Config{
CommonName: kubeadmconstants.APIServerCertCommonName,
AltNames: *altNames,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
}
apiCert, apiKey, err := pkiutil.NewCertAndKey(caCert, caKey, config)
if err != nil {
return nil, nil, fmt.Errorf("failure while creating API server key and certificate: %v", err)
}
return apiCert, apiKey, nil
}
// NewAPIServerKubeletClientCertAndKey generate certificate for the apiservers to connect to the kubelets securely, signed by the given CA.
func NewAPIServerKubeletClientCertAndKey(caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
config := certutil.Config{
CommonName: kubeadmconstants.APIServerKubeletClientCertCommonName,
Organization: []string{kubeadmconstants.MastersGroup},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
apiClientCert, apiClientKey, err := pkiutil.NewCertAndKey(caCert, caKey, config)
if err != nil {
return nil, nil, fmt.Errorf("failure while creating API server kubelet client key and certificate: %v", err)
}
return apiClientCert, apiClientKey, nil
}
// NewEtcdCACertAndKey generate a self signed etcd CA.
func NewEtcdCACertAndKey() (*x509.Certificate, *rsa.PrivateKey, error) {
etcdCACert, etcdCAKey, err := pkiutil.NewCertificateAuthority()
if err != nil {
return nil, nil, fmt.Errorf("failure while generating etcd CA certificate and key: %v", err)
}
return etcdCACert, etcdCAKey, nil
}
// NewEtcdServerCertAndKey generate certificate for etcd, signed by the given CA.
func NewEtcdServerCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
altNames, err := pkiutil.GetEtcdAltNames(cfg)
if err != nil {
return nil, nil, fmt.Errorf("failure while composing altnames for etcd: %v", err)
}
config := certutil.Config{
CommonName: cfg.NodeRegistration.Name,
AltNames: *altNames,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
}
etcdServerCert, etcdServerKey, err := pkiutil.NewCertAndKey(caCert, caKey, config)
if err != nil {
return nil, nil, fmt.Errorf("failure while creating etcd key and certificate: %v", err)
}
return etcdServerCert, etcdServerKey, nil
}
// NewEtcdPeerCertAndKey generate certificate for etcd peering, signed by the given CA.
func NewEtcdPeerCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
altNames, err := pkiutil.GetEtcdPeerAltNames(cfg)
if err != nil {
return nil, nil, fmt.Errorf("failure while composing altnames for etcd peering: %v", err)
}
config := certutil.Config{
CommonName: cfg.NodeRegistration.Name,
AltNames: *altNames,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
}
etcdPeerCert, etcdPeerKey, err := pkiutil.NewCertAndKey(caCert, caKey, config)
if err != nil {
return nil, nil, fmt.Errorf("failure while creating etcd peer key and certificate: %v", err)
}
return etcdPeerCert, etcdPeerKey, nil
}
// NewEtcdHealthcheckClientCertAndKey generate certificate for liveness probes to healthcheck etcd, signed by the given CA.
func NewEtcdHealthcheckClientCertAndKey(caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
config := certutil.Config{
CommonName: kubeadmconstants.EtcdHealthcheckClientCertCommonName,
Organization: []string{kubeadmconstants.MastersGroup},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
etcdHealcheckClientCert, etcdHealcheckClientKey, err := pkiutil.NewCertAndKey(caCert, caKey, config)
if err != nil {
return nil, nil, fmt.Errorf("failure while creating etcd healthcheck client key and certificate: %v", err)
}
return etcdHealcheckClientCert, etcdHealcheckClientKey, nil
}
// NewAPIServerEtcdClientCertAndKey generate certificate for the apiservers to connect to etcd securely, signed by the given CA.
func NewAPIServerEtcdClientCertAndKey(caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
config := certutil.Config{
CommonName: kubeadmconstants.APIServerEtcdClientCertCommonName,
Organization: []string{kubeadmconstants.MastersGroup},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
apiClientCert, apiClientKey, err := pkiutil.NewCertAndKey(caCert, caKey, config)
if err != nil {
return nil, nil, fmt.Errorf("failure while creating API server etcd client key and certificate: %v", err)
}
return apiClientCert, apiClientKey, nil
}
// NewServiceAccountSigningKey generate public/private key pairs for signing service account tokens.
func NewServiceAccountSigningKey() (*rsa.PrivateKey, error) {
// The key does NOT exist, let's generate it now
saSigningKey, err := certutil.NewPrivateKey()
if err != nil {
return nil, fmt.Errorf("failure while creating service account token signing key: %v", err)
}
return saSigningKey, nil
}
// NewFrontProxyCACertAndKey generate a self signed front proxy CA.
func NewFrontProxyCACertAndKey() (*x509.Certificate, *rsa.PrivateKey, error) {
frontProxyCACert, frontProxyCAKey, err := pkiutil.NewCertificateAuthority()
if err != nil {
return nil, nil, fmt.Errorf("failure while generating front-proxy CA certificate and key: %v", err)
}
return frontProxyCACert, frontProxyCAKey, nil
}
// NewFrontProxyClientCertAndKey generate certificate for proxy server client, signed by the given front proxy CA.
func NewFrontProxyClientCertAndKey(frontProxyCACert *x509.Certificate, frontProxyCAKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
config := certutil.Config{
CommonName: kubeadmconstants.FrontProxyClientCertCommonName,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
frontProxyClientCert, frontProxyClientKey, err := pkiutil.NewCertAndKey(frontProxyCACert, frontProxyCAKey, config)
if err != nil {
return nil, nil, fmt.Errorf("failure while creating front-proxy client key and certificate: %v", err)
}
return frontProxyClientCert, frontProxyClientKey, nil
}
// loadCertificateAuthority loads certificate authority
func loadCertificateAuthority(pkiDir string, baseName string) (*x509.Certificate, *rsa.PrivateKey, error) {
// Checks if certificate authority exists in the PKI directory
if !pkiutil.CertOrKeyExist(pkiDir, baseName) {
return nil, nil, fmt.Errorf("couldn't load %s certificate authority from %s", baseName, pkiDir)
}
// Try to load certificate authority .crt and .key from the PKI directory
caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(pkiDir, baseName)
if err != nil {
return nil, nil, fmt.Errorf("failure loading %s certificate authority: %v", baseName, err)
}
// Make sure the loaded CA cert actually is a CA
if !caCert.IsCA {
return nil, nil, fmt.Errorf("%s certificate is not a certificate authority", baseName)
}
return caCert, caKey, nil
}
// writeCertificateAuthorithyFilesIfNotExist write a new certificate Authority to the given path.
// If there already is a certificate file at the given path; kubeadm tries to load it and check if the values in the
// existing and the expected certificate equals. If they do; kubeadm will just skip writing the file as it's up-to-date,
// otherwise this function returns an error.
func writeCertificateAuthorithyFilesIfNotExist(pkiDir string, baseName string, caCert *x509.Certificate, caKey *rsa.PrivateKey) error {
// If cert or key exists, we should try to load them
if pkiutil.CertOrKeyExist(pkiDir, baseName) {
// Try to load .crt and .key from the PKI directory
caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkiDir, baseName)
if err != nil {
return fmt.Errorf("failure loading %s certificate: %v", baseName, err)
}
// Check if the existing cert is a CA
if !caCert.IsCA {
return fmt.Errorf("certificate %s is not a CA", baseName)
}
// kubeadm doesn't validate the existing certificate Authority more than this;
// Basically, if we find a certificate file with the same path; and it is a CA
// kubeadm thinks those files are equal and doesn't bother writing a new file
fmt.Printf("[certificates] Using the existing %s certificate and key.\n", baseName)
} else {
// Write .crt and .key files to disk
if err := pkiutil.WriteCertAndKey(pkiDir, baseName, caCert, caKey); err != nil {
return fmt.Errorf("failure while saving %s certificate and key: %v", baseName, err)
}
fmt.Printf("[certificates] Generated %s certificate and key.\n", baseName)
}
return nil
}
// writeCertificateFilesIfNotExist write a new certificate to the given path.
// If there already is a certificate file at the given path; kubeadm tries to load it and check if the values in the
// existing and the expected certificate equals. If they do; kubeadm will just skip writing the file as it's up-to-date,
// otherwise this function returns an error.
func writeCertificateFilesIfNotExist(pkiDir string, baseName string, signingCert *x509.Certificate, cert *x509.Certificate, key *rsa.PrivateKey) error {
// Checks if the signed certificate exists in the PKI directory
if pkiutil.CertOrKeyExist(pkiDir, baseName) {
// Try to load signed certificate .crt and .key from the PKI directory
signedCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkiDir, baseName)
if err != nil {
return fmt.Errorf("failure loading %s certificate: %v", baseName, err)
}
// Check if the existing cert is signed by the given CA
if err := signedCert.CheckSignatureFrom(signingCert); err != nil {
return fmt.Errorf("certificate %s is not signed by corresponding CA", baseName)
}
// kubeadm doesn't validate the existing certificate more than this;
// Basically, if we find a certificate file with the same path; and it is signed by
// the expected certificate authority, kubeadm thinks those files are equal and
// doesn't bother writing a new file
fmt.Printf("[certificates] Using the existing %s certificate and key.\n", baseName)
} else {
// Write .crt and .key files to disk
if err := pkiutil.WriteCertAndKey(pkiDir, baseName, cert, key); err != nil {
return fmt.Errorf("failure while saving %s certificate and key: %v", baseName, err)
}
fmt.Printf("[certificates] Generated %s certificate and key.\n", baseName)
if pkiutil.HasServerAuth(cert) {
fmt.Printf("[certificates] %s serving cert is signed for DNS names %v and IPs %v\n", baseName, cert.DNSNames, cert.IPAddresses)
}
}
return nil
}
// writeKeyFilesIfNotExist write a new key to the given path.
// If there already is a key file at the given path; kubeadm tries to load it and check if the values in the
// existing and the expected key equals. If they do; kubeadm will just skip writing the file as it's up-to-date,
// otherwise this function returns an error.
func writeKeyFilesIfNotExist(pkiDir string, baseName string, key *rsa.PrivateKey) error {
// Checks if the key exists in the PKI directory
if pkiutil.CertOrKeyExist(pkiDir, baseName) {
// Try to load .key from the PKI directory
_, err := pkiutil.TryLoadKeyFromDisk(pkiDir, baseName)
if err != nil {
return fmt.Errorf("%s key existed but it could not be loaded properly: %v", baseName, err)
}
// kubeadm doesn't validate the existing certificate key more than this;
// Basically, if we find a key file with the same path kubeadm thinks those files
// are equal and doesn't bother writing a new file
fmt.Printf("[certificates] Using the existing %s key.\n", baseName)
} else {
// Write .key and .pub files to disk
if err := pkiutil.WriteKey(pkiDir, baseName, key); err != nil {
return fmt.Errorf("failure while saving %s key: %v", baseName, err)
}
if err := pkiutil.WritePublicKey(pkiDir, baseName, &key.PublicKey); err != nil {
return fmt.Errorf("failure while saving %s public key: %v", baseName, err)
}
fmt.Printf("[certificates] Generated %s key and public key.\n", baseName)
}
return nil
}
type certKeyLocation struct {
pkiDir string
caBaseName string
baseName string
uxName string
}
// UsingExternalCA determines whether the user is relying on an external CA. We currently implicitly determine this is the case
// when both the CA Cert and the front proxy CA Cert are present but the CA Key and front proxy CA Key are not.
// This allows us to, e.g., skip generating certs or not start the csr signing controller.
func UsingExternalCA(cfg *kubeadmapi.MasterConfiguration) (bool, error) {
if err := validateCACert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, "", "CA"}); err != nil {
return false, err
}
caKeyPath := filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName)
if _, err := os.Stat(caKeyPath); !os.IsNotExist(err) {
return false, fmt.Errorf("%s exists", kubeadmconstants.CAKeyName)
}
if err := validateSignedCert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, kubeadmconstants.APIServerCertAndKeyBaseName, "API server"}); err != nil {
return false, err
}
if err := validateSignedCert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, "API server kubelet client"}); err != nil {
return false, err
}
if err := validatePrivatePublicKey(certKeyLocation{cfg.CertificatesDir, "", kubeadmconstants.ServiceAccountKeyBaseName, "service account"}); err != nil {
return false, err
}
if err := validateCACert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName, "", "front-proxy CA"}); err != nil {
return false, err
}
frontProxyCAKeyPath := filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCAKeyName)
if _, err := os.Stat(frontProxyCAKeyPath); !os.IsNotExist(err) {
return false, fmt.Errorf("%s exists", kubeadmconstants.FrontProxyCAKeyName)
}
if err := validateSignedCert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName, kubeadmconstants.FrontProxyClientCertAndKeyBaseName, "front-proxy client"}); err != nil {
return false, err
}
return true, nil
}
// validateCACert tries to load a x509 certificate from pkiDir and validates that it is a CA
func validateCACert(l certKeyLocation) error {
// Check CA Cert
caCert, err := pkiutil.TryLoadCertFromDisk(l.pkiDir, l.caBaseName)
if err != nil {
return fmt.Errorf("failure loading certificate for %s: %v", l.uxName, err)
}
// Check if cert is a CA
if !caCert.IsCA {
return fmt.Errorf("certificate %s is not a CA", l.uxName)
}
return nil
}
// validateCACertAndKey tries to load a x509 certificate and private key from pkiDir,
// and validates that the cert is a CA
func validateCACertAndKey(l certKeyLocation) error {
if err := validateCACert(l); err != nil {
return err
}
_, err := pkiutil.TryLoadKeyFromDisk(l.pkiDir, l.caBaseName)
if err != nil {
return fmt.Errorf("failure loading key for %s: %v", l.uxName, err)
}
return nil
}
// validateSignedCert tries to load a x509 certificate and private key from pkiDir and validates
// that the cert is signed by a given CA
func validateSignedCert(l certKeyLocation) error {
// Try to load CA
caCert, err := pkiutil.TryLoadCertFromDisk(l.pkiDir, l.caBaseName)
if err != nil {
return fmt.Errorf("failure loading certificate authority for %s: %v", l.uxName, err)
}
// Try to load key and signed certificate
signedCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(l.pkiDir, l.baseName)
if err != nil {
return fmt.Errorf("failure loading certificate for %s: %v", l.uxName, err)
}
// Check if the cert is signed by the CA
if err := signedCert.CheckSignatureFrom(caCert); err != nil {
return fmt.Errorf("certificate %s is not signed by corresponding CA", l.uxName)
}
return nil
}
// validatePrivatePublicKey tries to load a private key from pkiDir
func validatePrivatePublicKey(l certKeyLocation) error {
// Try to load key
_, _, err := pkiutil.TryLoadPrivatePublicKeyFromDisk(l.pkiDir, l.baseName)
if err != nil {
return fmt.Errorf("failure loading key for %s: %v", l.uxName, err)
}
return nil
}

View File

@@ -0,0 +1,734 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package certs
import (
"crypto/rsa"
"crypto/x509"
"fmt"
"net"
"os"
"path/filepath"
"testing"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
certstestutil "k8s.io/kubernetes/cmd/kubeadm/test/certs"
)
func TestWriteCertificateAuthorithyFilesIfNotExist(t *testing.T) {
setupCert, setupKey, _ := NewCACertAndKey()
caCert, caKey, _ := NewCACertAndKey()
var tests = []struct {
setupFunc func(pkiDir string) error
expectedError bool
expectedCa *x509.Certificate
}{
{ // ca cert does not exists > ca written
expectedCa: caCert,
},
{ // ca cert exists, is ca > existing ca used
setupFunc: func(pkiDir string) error {
return writeCertificateAuthorithyFilesIfNotExist(pkiDir, "dummy", setupCert, setupKey)
},
expectedCa: setupCert,
},
{ // some file exists, but it is not a valid ca cert > err
setupFunc: func(pkiDir string) error {
testutil.SetupEmptyFiles(t, pkiDir, "dummy.crt")
return nil
},
expectedError: true,
},
{ // cert exists, but it is not a ca > err
setupFunc: func(pkiDir string) error {
cert, key, _ := NewFrontProxyClientCertAndKey(setupCert, setupKey)
return writeCertificateFilesIfNotExist(pkiDir, "dummy", setupCert, cert, key)
},
expectedError: true,
},
}
for _, test := range tests {
// Create temp folder for the test case
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
// executes setup func (if necessary)
if test.setupFunc != nil {
if err := test.setupFunc(tmpdir); err != nil {
t.Errorf("error executing setupFunc: %v", err)
continue
}
}
// executes create func
err := writeCertificateAuthorithyFilesIfNotExist(tmpdir, "dummy", caCert, caKey)
if !test.expectedError && err != nil {
t.Errorf("error writeCertificateAuthorithyFilesIfNotExist failed when not expected to fail: %v", err)
continue
} else if test.expectedError && err == nil {
t.Error("error writeCertificateAuthorithyFilesIfNotExist didn't failed when expected")
continue
} else if test.expectedError {
continue
}
// asserts expected files are there
testutil.AssertFileExists(t, tmpdir, "dummy.key", "dummy.crt")
// check created cert
resultingCaCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(tmpdir, "dummy")
if err != nil {
t.Errorf("failure reading created cert: %v", err)
continue
}
if !resultingCaCert.Equal(test.expectedCa) {
t.Error("created ca cert does not match expected ca cert")
}
}
}
func TestWriteCertificateFilesIfNotExist(t *testing.T) {
caCert, caKey, _ := NewFrontProxyCACertAndKey()
setupCert, setupKey, _ := NewFrontProxyClientCertAndKey(caCert, caKey)
cert, key, _ := NewFrontProxyClientCertAndKey(caCert, caKey)
var tests = []struct {
setupFunc func(pkiDir string) error
expectedError bool
expectedCert *x509.Certificate
}{
{ // cert does not exists > cert written
expectedCert: cert,
},
{ // cert exists, is signed by the same ca > existing cert used
setupFunc: func(pkiDir string) error {
return writeCertificateFilesIfNotExist(pkiDir, "dummy", caCert, setupCert, setupKey)
},
expectedCert: setupCert,
},
{ // some file exists, but it is not a valid cert > err
setupFunc: func(pkiDir string) error {
testutil.SetupEmptyFiles(t, pkiDir, "dummy.crt")
return nil
},
expectedError: true,
},
{ // cert exists, is signed by another ca > err
setupFunc: func(pkiDir string) error {
anotherCaCert, anotherCaKey, _ := NewFrontProxyCACertAndKey()
anotherCert, anotherKey, _ := NewFrontProxyClientCertAndKey(anotherCaCert, anotherCaKey)
return writeCertificateFilesIfNotExist(pkiDir, "dummy", anotherCaCert, anotherCert, anotherKey)
},
expectedError: true,
},
}
for _, test := range tests {
// Create temp folder for the test case
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
// executes setup func (if necessary)
if test.setupFunc != nil {
if err := test.setupFunc(tmpdir); err != nil {
t.Errorf("error executing setupFunc: %v", err)
continue
}
}
// executes create func
err := writeCertificateFilesIfNotExist(tmpdir, "dummy", caCert, cert, key)
if !test.expectedError && err != nil {
t.Errorf("error writeCertificateFilesIfNotExist failed when not expected to fail: %v", err)
continue
} else if test.expectedError && err == nil {
t.Error("error writeCertificateFilesIfNotExist didn't failed when expected")
continue
} else if test.expectedError {
continue
}
// asserts expected files are there
testutil.AssertFileExists(t, tmpdir, "dummy.key", "dummy.crt")
// check created cert
resultingCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(tmpdir, "dummy")
if err != nil {
t.Errorf("failure reading created cert: %v", err)
continue
}
if !resultingCert.Equal(test.expectedCert) {
t.Error("created cert does not match expected cert")
}
}
}
func TestWriteKeyFilesIfNotExist(t *testing.T) {
setupKey, _ := NewServiceAccountSigningKey()
key, _ := NewServiceAccountSigningKey()
var tests = []struct {
setupFunc func(pkiDir string) error
expectedError bool
expectedKey *rsa.PrivateKey
}{
{ // key does not exists > key written
expectedKey: key,
},
{ // key exists > existing key used
setupFunc: func(pkiDir string) error {
return writeKeyFilesIfNotExist(pkiDir, "dummy", setupKey)
},
expectedKey: setupKey,
},
{ // some file exists, but it is not a valid key > err
setupFunc: func(pkiDir string) error {
testutil.SetupEmptyFiles(t, pkiDir, "dummy.key")
return nil
},
expectedError: true,
},
}
for _, test := range tests {
// Create temp folder for the test case
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
// executes setup func (if necessary)
if test.setupFunc != nil {
if err := test.setupFunc(tmpdir); err != nil {
t.Errorf("error executing setupFunc: %v", err)
continue
}
}
// executes create func
err := writeKeyFilesIfNotExist(tmpdir, "dummy", key)
if !test.expectedError && err != nil {
t.Errorf("error writeKeyFilesIfNotExist failed when not expected to fail: %v", err)
continue
} else if test.expectedError && err == nil {
t.Error("error writeKeyFilesIfNotExist didn't failed when expected")
continue
} else if test.expectedError {
continue
}
// asserts expected files are there
testutil.AssertFileExists(t, tmpdir, "dummy.key", "dummy.pub")
// check created key
resultingKey, err := pkiutil.TryLoadKeyFromDisk(tmpdir, "dummy")
if err != nil {
t.Errorf("failure reading created key: %v", err)
continue
}
//TODO: check if there is a better method to compare keys
if resultingKey.D == key.D {
t.Error("created key does not match expected key")
}
}
}
func TestNewCACertAndKey(t *testing.T) {
caCert, _, err := NewCACertAndKey()
if err != nil {
t.Fatalf("failed call NewCACertAndKey: %v", err)
}
certstestutil.AssertCertificateIsCa(t, caCert)
}
func TestNewAPIServerCertAndKey(t *testing.T) {
hostname := "valid-hostname"
advertiseAddresses := []string{"1.2.3.4", "1:2:3::4"}
for _, addr := range advertiseAddresses {
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: addr},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: hostname},
}
caCert, caKey, err := NewCACertAndKey()
if err != nil {
t.Fatalf("failed creation of ca cert and key: %v", err)
}
apiServerCert, _, err := NewAPIServerCertAndKey(cfg, caCert, caKey)
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
certstestutil.AssertCertificateIsSignedByCa(t, apiServerCert, caCert)
certstestutil.AssertCertificateHasServerAuthUsage(t, apiServerCert)
certstestutil.AssertCertificateHasDNSNames(t, apiServerCert, hostname, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local")
certstestutil.AssertCertificateHasIPAddresses(t, apiServerCert, net.ParseIP("10.96.0.1"), net.ParseIP(addr))
}
}
func TestNewAPIServerKubeletClientCertAndKey(t *testing.T) {
caCert, caKey, err := NewCACertAndKey()
if err != nil {
t.Fatalf("failed creation of ca cert and key: %v", err)
}
apiKubeletClientCert, _, err := NewAPIServerKubeletClientCertAndKey(caCert, caKey)
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
certstestutil.AssertCertificateIsSignedByCa(t, apiKubeletClientCert, caCert)
certstestutil.AssertCertificateHasClientAuthUsage(t, apiKubeletClientCert)
certstestutil.AssertCertificateHasOrganizations(t, apiKubeletClientCert, kubeadmconstants.MastersGroup)
}
func TestNewEtcdCACertAndKey(t *testing.T) {
etcdCACert, _, err := NewEtcdCACertAndKey()
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
certstestutil.AssertCertificateIsCa(t, etcdCACert)
}
func TestNewEtcdServerCertAndKey(t *testing.T) {
proxy := "user-etcd-proxy"
proxyIP := "10.10.10.100"
cfg := &kubeadmapi.MasterConfiguration{
NodeRegistration: kubeadmapi.NodeRegistrationOptions{
Name: "etcd-server-cert",
},
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
ServerCertSANs: []string{
proxy,
proxyIP,
},
},
},
}
caCert, caKey, err := NewCACertAndKey()
if err != nil {
t.Fatalf("failed creation of ca cert and key: %v", err)
}
etcdServerCert, _, err := NewEtcdServerCertAndKey(cfg, caCert, caKey)
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
certstestutil.AssertCertificateIsSignedByCa(t, etcdServerCert, caCert)
certstestutil.AssertCertificateHasServerAuthUsage(t, etcdServerCert)
certstestutil.AssertCertificateHasDNSNames(t, etcdServerCert, "localhost", proxy)
certstestutil.AssertCertificateHasIPAddresses(t, etcdServerCert, net.ParseIP("127.0.0.1"), net.ParseIP(proxyIP))
}
func TestNewEtcdPeerCertAndKey(t *testing.T) {
hostname := "valid-hostname"
proxy := "user-etcd-proxy"
proxyIP := "10.10.10.100"
advertiseAddresses := []string{"1.2.3.4", "1:2:3::4"}
for _, addr := range advertiseAddresses {
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: addr},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: hostname},
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
PeerCertSANs: []string{
proxy,
proxyIP,
},
},
},
}
caCert, caKey, err := NewCACertAndKey()
if err != nil {
t.Fatalf("failed creation of ca cert and key: %v", err)
}
etcdPeerCert, _, err := NewEtcdPeerCertAndKey(cfg, caCert, caKey)
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
certstestutil.AssertCertificateIsSignedByCa(t, etcdPeerCert, caCert)
certstestutil.AssertCertificateHasServerAuthUsage(t, etcdPeerCert)
certstestutil.AssertCertificateHasClientAuthUsage(t, etcdPeerCert)
certstestutil.AssertCertificateHasDNSNames(t, etcdPeerCert, hostname, proxy)
certstestutil.AssertCertificateHasIPAddresses(t, etcdPeerCert, net.ParseIP(addr), net.ParseIP(proxyIP))
}
}
func TestNewEtcdHealthcheckClientCertAndKey(t *testing.T) {
caCert, caKey, err := NewCACertAndKey()
if err != nil {
t.Fatalf("failed creation of ca cert and key: %v", err)
}
etcdHealthcheckClientCert, _, err := NewEtcdHealthcheckClientCertAndKey(caCert, caKey)
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
certstestutil.AssertCertificateIsSignedByCa(t, etcdHealthcheckClientCert, caCert)
certstestutil.AssertCertificateHasClientAuthUsage(t, etcdHealthcheckClientCert)
certstestutil.AssertCertificateHasOrganizations(t, etcdHealthcheckClientCert, kubeadmconstants.MastersGroup)
}
func TestNewAPIServerEtcdClientCertAndKey(t *testing.T) {
caCert, caKey, err := NewCACertAndKey()
if err != nil {
t.Fatalf("failed creation of ca cert and key: %v", err)
}
apiEtcdClientCert, _, err := NewAPIServerEtcdClientCertAndKey(caCert, caKey)
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
certstestutil.AssertCertificateIsSignedByCa(t, apiEtcdClientCert, caCert)
certstestutil.AssertCertificateHasClientAuthUsage(t, apiEtcdClientCert)
certstestutil.AssertCertificateHasOrganizations(t, apiEtcdClientCert, kubeadmconstants.MastersGroup)
}
func TestNewNewServiceAccountSigningKey(t *testing.T) {
key, err := NewServiceAccountSigningKey()
if err != nil {
t.Fatalf("failed creation of key: %v", err)
}
if key.N.BitLen() < 2048 {
t.Error("Service account signing key has less than 2048 bits size")
}
}
func TestNewFrontProxyCACertAndKey(t *testing.T) {
frontProxyCACert, _, err := NewFrontProxyCACertAndKey()
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
certstestutil.AssertCertificateIsCa(t, frontProxyCACert)
}
func TestNewFrontProxyClientCertAndKey(t *testing.T) {
frontProxyCACert, frontProxyCAKey, err := NewFrontProxyCACertAndKey()
if err != nil {
t.Fatalf("failed creation of ca cert and key: %v", err)
}
frontProxyClientCert, _, err := NewFrontProxyClientCertAndKey(frontProxyCACert, frontProxyCAKey)
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
certstestutil.AssertCertificateIsSignedByCa(t, frontProxyClientCert, frontProxyCACert)
certstestutil.AssertCertificateHasClientAuthUsage(t, frontProxyClientCert)
}
func TestUsingExternalCA(t *testing.T) {
tests := []struct {
setupFuncs []func(cfg *kubeadmapi.MasterConfiguration) error
expected bool
}{
{
setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
CreatePKIAssets,
},
expected: false,
},
{
setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
CreatePKIAssets,
deleteCAKey,
deleteFrontProxyCAKey,
},
expected: true,
},
}
for _, test := range tests {
dir := testutil.SetupTempDir(t)
defer os.RemoveAll(dir)
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"},
CertificatesDir: dir,
}
for _, f := range test.setupFuncs {
if err := f(cfg); err != nil {
t.Errorf("error executing setup function: %v", err)
}
}
if val, _ := UsingExternalCA(cfg); val != test.expected {
t.Errorf("UsingExternalCA did not match expected: %v", test.expected)
}
}
}
func TestValidateMethods(t *testing.T) {
tests := []struct {
name string
setupFuncs []func(cfg *kubeadmapi.MasterConfiguration) error
validateFunc func(l certKeyLocation) error
loc certKeyLocation
expectedSuccess bool
}{
{
name: "validateCACert",
setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
CreateCACertAndKeyFiles,
},
validateFunc: validateCACert,
loc: certKeyLocation{caBaseName: "ca", baseName: "", uxName: "CA"},
expectedSuccess: true,
},
{
name: "validateCACertAndKey (files present)",
setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
CreateCACertAndKeyFiles,
},
validateFunc: validateCACertAndKey,
loc: certKeyLocation{caBaseName: "ca", baseName: "", uxName: "CA"},
expectedSuccess: true,
},
{
name: "validateCACertAndKey (key missing)",
setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
CreatePKIAssets,
deleteCAKey,
},
validateFunc: validateCACertAndKey,
loc: certKeyLocation{caBaseName: "ca", baseName: "", uxName: "CA"},
expectedSuccess: false,
},
{
name: "validateSignedCert",
setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
CreateCACertAndKeyFiles,
CreateAPIServerCertAndKeyFiles,
},
validateFunc: validateSignedCert,
loc: certKeyLocation{caBaseName: "ca", baseName: "apiserver", uxName: "apiserver"},
expectedSuccess: true,
},
{
name: "validatePrivatePublicKey",
setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
CreateServiceAccountKeyAndPublicKeyFiles,
},
validateFunc: validatePrivatePublicKey,
loc: certKeyLocation{baseName: "sa", uxName: "service account"},
expectedSuccess: true,
},
}
for _, test := range tests {
dir := testutil.SetupTempDir(t)
defer os.RemoveAll(dir)
test.loc.pkiDir = dir
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"},
CertificatesDir: dir,
}
fmt.Println("Testing", test.name)
for _, f := range test.setupFuncs {
if err := f(cfg); err != nil {
t.Errorf("error executing setup function: %v", err)
}
}
err := test.validateFunc(test.loc)
if test.expectedSuccess && err != nil {
t.Errorf("expected success, error executing validateFunc: %v, %v", test.name, err)
} else if !test.expectedSuccess && err == nil {
t.Errorf("expected failure, no error executing validateFunc: %v", test.name)
}
}
}
func deleteCAKey(cfg *kubeadmapi.MasterConfiguration) error {
if err := os.Remove(filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName)); err != nil {
return fmt.Errorf("failed removing %s: %v", kubeadmconstants.CAKeyName, err)
}
return nil
}
func deleteFrontProxyCAKey(cfg *kubeadmapi.MasterConfiguration) error {
if err := os.Remove(filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCAKeyName)); err != nil {
return fmt.Errorf("failed removing %s: %v", kubeadmconstants.FrontProxyCAKeyName, err)
}
return nil
}
func TestCreateCertificateFilesMethods(t *testing.T) {
var tests = []struct {
setupFunc func(cfg *kubeadmapi.MasterConfiguration) error
createFunc func(cfg *kubeadmapi.MasterConfiguration) error
expectedFiles []string
externalEtcd bool
}{
{
createFunc: CreatePKIAssets,
expectedFiles: []string{
kubeadmconstants.CACertName, kubeadmconstants.CAKeyName,
kubeadmconstants.APIServerCertName, kubeadmconstants.APIServerKeyName,
kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName,
kubeadmconstants.EtcdCACertName, kubeadmconstants.EtcdCAKeyName,
kubeadmconstants.EtcdServerCertName, kubeadmconstants.EtcdServerKeyName,
kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName,
kubeadmconstants.EtcdHealthcheckClientCertName, kubeadmconstants.EtcdHealthcheckClientKeyName,
kubeadmconstants.APIServerEtcdClientCertName, kubeadmconstants.APIServerEtcdClientKeyName,
kubeadmconstants.ServiceAccountPrivateKeyName, kubeadmconstants.ServiceAccountPublicKeyName,
kubeadmconstants.FrontProxyCACertName, kubeadmconstants.FrontProxyCAKeyName,
kubeadmconstants.FrontProxyClientCertName, kubeadmconstants.FrontProxyClientKeyName,
},
},
{
createFunc: CreatePKIAssets,
externalEtcd: true,
expectedFiles: []string{
kubeadmconstants.CACertName, kubeadmconstants.CAKeyName,
kubeadmconstants.APIServerCertName, kubeadmconstants.APIServerKeyName,
kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName,
kubeadmconstants.ServiceAccountPrivateKeyName, kubeadmconstants.ServiceAccountPublicKeyName,
kubeadmconstants.FrontProxyCACertName, kubeadmconstants.FrontProxyCAKeyName,
kubeadmconstants.FrontProxyClientCertName, kubeadmconstants.FrontProxyClientKeyName,
},
},
{
createFunc: CreateCACertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.CACertName, kubeadmconstants.CAKeyName},
},
{
setupFunc: CreateCACertAndKeyFiles,
createFunc: CreateAPIServerCertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.APIServerCertName, kubeadmconstants.APIServerKeyName},
},
{
setupFunc: CreateCACertAndKeyFiles,
createFunc: CreateAPIServerKubeletClientCertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName},
},
{
createFunc: CreateEtcdCACertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.EtcdCACertName, kubeadmconstants.EtcdCAKeyName},
},
{
setupFunc: CreateEtcdCACertAndKeyFiles,
createFunc: CreateEtcdServerCertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.EtcdServerCertName, kubeadmconstants.EtcdServerKeyName},
},
{
setupFunc: CreateEtcdCACertAndKeyFiles,
createFunc: CreateEtcdPeerCertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName},
},
{
setupFunc: CreateEtcdCACertAndKeyFiles,
createFunc: CreateEtcdHealthcheckClientCertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.EtcdHealthcheckClientCertName, kubeadmconstants.EtcdHealthcheckClientKeyName},
},
{
setupFunc: CreateEtcdCACertAndKeyFiles,
createFunc: CreateAPIServerEtcdClientCertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.APIServerEtcdClientCertName, kubeadmconstants.APIServerEtcdClientKeyName},
},
{
createFunc: CreateServiceAccountKeyAndPublicKeyFiles,
expectedFiles: []string{kubeadmconstants.ServiceAccountPrivateKeyName, kubeadmconstants.ServiceAccountPublicKeyName},
},
{
createFunc: CreateFrontProxyCACertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.FrontProxyCACertName, kubeadmconstants.FrontProxyCAKeyName},
},
{
setupFunc: CreateFrontProxyCACertAndKeyFiles,
createFunc: CreateFrontProxyClientCertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.FrontProxyCACertName, kubeadmconstants.FrontProxyCAKeyName},
},
}
for _, test := range tests {
// Create temp folder for the test case
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
Etcd: kubeadmapi.Etcd{Local: &kubeadmapi.LocalEtcd{}},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"},
CertificatesDir: tmpdir,
}
if test.externalEtcd {
if cfg.Etcd.External == nil {
cfg.Etcd.External = &kubeadmapi.ExternalEtcd{}
}
cfg.Etcd.Local = nil
cfg.Etcd.External.Endpoints = []string{"192.168.1.1:2379"}
}
// executes setup func (if necessary)
if test.setupFunc != nil {
if err := test.setupFunc(cfg); err != nil {
t.Errorf("error executing setupFunc: %v", err)
continue
}
}
// executes create func
if err := test.createFunc(cfg); err != nil {
t.Errorf("error executing createFunc: %v", err)
continue
}
// asserts expected files are there
testutil.AssertFileExists(t, tmpdir, test.expectedFiles...)
}
}

View File

@@ -0,0 +1,58 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package certs
/*
PHASE: CERTIFICATES
INPUTS:
From MasterConfiguration
.API.AdvertiseAddress is an optional parameter that can be passed for an extra addition to the SAN IPs
.APIServerCertSANs is an optional parameter for adding DNS names and IPs to the API Server serving cert SAN
.Etcd.Local.ServerCertSANs is an optional parameter for adding DNS names and IPs to the etcd serving cert SAN
.Etcd.Local.PeerCertSANs is an optional parameter for adding DNS names and IPs to the etcd peer cert SAN
.Networking.DNSDomain is needed for knowing which DNS name the internal kubernetes service has
.Networking.ServiceSubnet is needed for knowing which IP the internal kubernetes service is going to point to
.CertificatesDir is required for knowing where all certificates should be stored
OUTPUTS:
Files to .CertificatesDir (default /etc/kubernetes/pki):
- ca.crt
- ca.key
- apiserver.crt
- apiserver.key
- apiserver-kubelet-client.crt
- apiserver-kubelet-client.key
- apiserver-etcd-client.crt
- apiserver-etcd-client.key
- etcd/ca.crt
- etcd/ca.key
- etcd/server.crt
- etcd/server.key
- etcd/peer.crt
- etcd/peer.key
- etcd/healthcheck-client.crt
- etcd/healthcheck-client.key
- sa.pub
- sa.key
- front-proxy-ca.crt
- front-proxy-ca.key
- front-proxy-client.crt
- front-proxy-client.key
*/

View File

@@ -0,0 +1,44 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = ["pki_helpers_test.go"],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//vendor/k8s.io/client-go/util/cert:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = ["pki_helpers.go"],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//pkg/registry/core/service/ipallocator:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
"//vendor/k8s.io/client-go/util/cert:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View File

@@ -0,0 +1,369 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package pkiutil
import (
"crypto/rsa"
"crypto/x509"
"fmt"
"net"
"os"
"path/filepath"
"time"
"k8s.io/apimachinery/pkg/util/validation"
certutil "k8s.io/client-go/util/cert"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
)
// NewCertificateAuthority creates new certificate and private key for the certificate authority
func NewCertificateAuthority() (*x509.Certificate, *rsa.PrivateKey, error) {
key, err := certutil.NewPrivateKey()
if err != nil {
return nil, nil, fmt.Errorf("unable to create private key [%v]", err)
}
config := certutil.Config{
CommonName: "kubernetes",
}
cert, err := certutil.NewSelfSignedCACert(config, key)
if err != nil {
return nil, nil, fmt.Errorf("unable to create self-signed certificate [%v]", err)
}
return cert, key, nil
}
// NewCertAndKey creates new certificate and key by passing the certificate authority certificate and key
func NewCertAndKey(caCert *x509.Certificate, caKey *rsa.PrivateKey, config certutil.Config) (*x509.Certificate, *rsa.PrivateKey, error) {
key, err := certutil.NewPrivateKey()
if err != nil {
return nil, nil, fmt.Errorf("unable to create private key [%v]", err)
}
cert, err := certutil.NewSignedCert(config, key, caCert, caKey)
if err != nil {
return nil, nil, fmt.Errorf("unable to sign certificate [%v]", err)
}
return cert, key, nil
}
// HasServerAuth returns true if the given certificate is a ServerAuth
func HasServerAuth(cert *x509.Certificate) bool {
for i := range cert.ExtKeyUsage {
if cert.ExtKeyUsage[i] == x509.ExtKeyUsageServerAuth {
return true
}
}
return false
}
// WriteCertAndKey stores certificate and key at the specified location
func WriteCertAndKey(pkiPath string, name string, cert *x509.Certificate, key *rsa.PrivateKey) error {
if err := WriteKey(pkiPath, name, key); err != nil {
return err
}
return WriteCert(pkiPath, name, cert)
}
// WriteCert stores the given certificate at the given location
func WriteCert(pkiPath, name string, cert *x509.Certificate) error {
if cert == nil {
return fmt.Errorf("certificate cannot be nil when writing to file")
}
certificatePath := pathForCert(pkiPath, name)
if err := certutil.WriteCert(certificatePath, certutil.EncodeCertPEM(cert)); err != nil {
return fmt.Errorf("unable to write certificate to file %q: [%v]", certificatePath, err)
}
return nil
}
// WriteKey stores the given key at the given location
func WriteKey(pkiPath, name string, key *rsa.PrivateKey) error {
if key == nil {
return fmt.Errorf("private key cannot be nil when writing to file")
}
privateKeyPath := pathForKey(pkiPath, name)
if err := certutil.WriteKey(privateKeyPath, certutil.EncodePrivateKeyPEM(key)); err != nil {
return fmt.Errorf("unable to write private key to file %q: [%v]", privateKeyPath, err)
}
return nil
}
// WritePublicKey stores the given public key at the given location
func WritePublicKey(pkiPath, name string, key *rsa.PublicKey) error {
if key == nil {
return fmt.Errorf("public key cannot be nil when writing to file")
}
publicKeyBytes, err := certutil.EncodePublicKeyPEM(key)
if err != nil {
return err
}
publicKeyPath := pathForPublicKey(pkiPath, name)
if err := certutil.WriteKey(publicKeyPath, publicKeyBytes); err != nil {
return fmt.Errorf("unable to write public key to file %q: [%v]", publicKeyPath, err)
}
return nil
}
// CertOrKeyExist returns a boolean whether the cert or the key exists
func CertOrKeyExist(pkiPath, name string) bool {
certificatePath, privateKeyPath := pathsForCertAndKey(pkiPath, name)
_, certErr := os.Stat(certificatePath)
_, keyErr := os.Stat(privateKeyPath)
if os.IsNotExist(certErr) && os.IsNotExist(keyErr) {
// The cert or the key did not exist
return false
}
// Both files exist or one of them
return true
}
// TryLoadCertAndKeyFromDisk tries to load a cert and a key from the disk and validates that they are valid
func TryLoadCertAndKeyFromDisk(pkiPath, name string) (*x509.Certificate, *rsa.PrivateKey, error) {
cert, err := TryLoadCertFromDisk(pkiPath, name)
if err != nil {
return nil, nil, err
}
key, err := TryLoadKeyFromDisk(pkiPath, name)
if err != nil {
return nil, nil, err
}
return cert, key, nil
}
// TryLoadCertFromDisk tries to load the cert from the disk and validates that it is valid
func TryLoadCertFromDisk(pkiPath, name string) (*x509.Certificate, error) {
certificatePath := pathForCert(pkiPath, name)
certs, err := certutil.CertsFromFile(certificatePath)
if err != nil {
return nil, fmt.Errorf("couldn't load the certificate file %s: %v", certificatePath, err)
}
// We are only putting one certificate in the certificate pem file, so it's safe to just pick the first one
// TODO: Support multiple certs here in order to be able to rotate certs
cert := certs[0]
// Check so that the certificate is valid now
now := time.Now()
if now.Before(cert.NotBefore) {
return nil, fmt.Errorf("the certificate is not valid yet")
}
if now.After(cert.NotAfter) {
return nil, fmt.Errorf("the certificate has expired")
}
return cert, nil
}
// TryLoadKeyFromDisk tries to load the key from the disk and validates that it is valid
func TryLoadKeyFromDisk(pkiPath, name string) (*rsa.PrivateKey, error) {
privateKeyPath := pathForKey(pkiPath, name)
// Parse the private key from a file
privKey, err := certutil.PrivateKeyFromFile(privateKeyPath)
if err != nil {
return nil, fmt.Errorf("couldn't load the private key file %s: %v", privateKeyPath, err)
}
// Allow RSA format only
var key *rsa.PrivateKey
switch k := privKey.(type) {
case *rsa.PrivateKey:
key = k
default:
return nil, fmt.Errorf("the private key file %s isn't in RSA format", privateKeyPath)
}
return key, nil
}
// TryLoadPrivatePublicKeyFromDisk tries to load the key from the disk and validates that it is valid
func TryLoadPrivatePublicKeyFromDisk(pkiPath, name string) (*rsa.PrivateKey, *rsa.PublicKey, error) {
privateKeyPath := pathForKey(pkiPath, name)
// Parse the private key from a file
privKey, err := certutil.PrivateKeyFromFile(privateKeyPath)
if err != nil {
return nil, nil, fmt.Errorf("couldn't load the private key file %s: %v", privateKeyPath, err)
}
publicKeyPath := pathForPublicKey(pkiPath, name)
// Parse the public key from a file
pubKeys, err := certutil.PublicKeysFromFile(publicKeyPath)
if err != nil {
return nil, nil, fmt.Errorf("couldn't load the public key file %s: %v", publicKeyPath, err)
}
// Allow RSA format only
k, ok := privKey.(*rsa.PrivateKey)
if !ok {
return nil, nil, fmt.Errorf("the private key file %s isn't in RSA format", privateKeyPath)
}
p := pubKeys[0].(*rsa.PublicKey)
return k, p, nil
}
func pathsForCertAndKey(pkiPath, name string) (string, string) {
return pathForCert(pkiPath, name), pathForKey(pkiPath, name)
}
func pathForCert(pkiPath, name string) string {
return filepath.Join(pkiPath, fmt.Sprintf("%s.crt", name))
}
func pathForKey(pkiPath, name string) string {
return filepath.Join(pkiPath, fmt.Sprintf("%s.key", name))
}
func pathForPublicKey(pkiPath, name string) string {
return filepath.Join(pkiPath, fmt.Sprintf("%s.pub", name))
}
// GetAPIServerAltNames builds an AltNames object for to be used when generating apiserver certificate
func GetAPIServerAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, error) {
// advertise address
advertiseAddress := net.ParseIP(cfg.API.AdvertiseAddress)
if advertiseAddress == nil {
return nil, fmt.Errorf("error parsing API AdvertiseAddress %v: is not a valid textual representation of an IP address", cfg.API.AdvertiseAddress)
}
// internal IP address for the API server
_, svcSubnet, err := net.ParseCIDR(cfg.Networking.ServiceSubnet)
if err != nil {
return nil, fmt.Errorf("error parsing CIDR %q: %v", cfg.Networking.ServiceSubnet, err)
}
internalAPIServerVirtualIP, err := ipallocator.GetIndexedIP(svcSubnet, 1)
if err != nil {
return nil, fmt.Errorf("unable to get first IP address from the given CIDR (%s): %v", svcSubnet.String(), err)
}
// create AltNames with defaults DNSNames/IPs
altNames := &certutil.AltNames{
DNSNames: []string{
cfg.NodeRegistration.Name,
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
fmt.Sprintf("kubernetes.default.svc.%s", cfg.Networking.DNSDomain),
},
IPs: []net.IP{
internalAPIServerVirtualIP,
advertiseAddress,
},
}
// add api server controlPlaneEndpoint if present (dns or ip)
if len(cfg.API.ControlPlaneEndpoint) > 0 {
if host, _, err := kubeadmutil.ParseHostPort(cfg.API.ControlPlaneEndpoint); err == nil {
if ip := net.ParseIP(host); ip != nil {
altNames.IPs = append(altNames.IPs, ip)
} else {
altNames.DNSNames = append(altNames.DNSNames, host)
}
} else {
return nil, fmt.Errorf("error parsing API api.controlPlaneEndpoint %q: %s", cfg.API.ControlPlaneEndpoint, err)
}
}
appendSANsToAltNames(altNames, cfg.APIServerCertSANs, kubeadmconstants.APIServerCertName)
return altNames, nil
}
// GetEtcdAltNames builds an AltNames object for generating the etcd server certificate.
// `localhost` is included in the SAN since this is the interface the etcd static pod listens on.
// Hostname and `API.AdvertiseAddress` are excluded since etcd does not listen on this interface by default.
// The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.ServerCertSANs`.
func GetEtcdAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, error) {
// create AltNames with defaults DNSNames/IPs
altNames := &certutil.AltNames{
DNSNames: []string{cfg.NodeRegistration.Name, "localhost"},
IPs: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
}
if cfg.Etcd.Local != nil {
appendSANsToAltNames(altNames, cfg.Etcd.Local.ServerCertSANs, kubeadmconstants.EtcdServerCertName)
}
return altNames, nil
}
// GetEtcdPeerAltNames builds an AltNames object for generating the etcd peer certificate.
// `localhost` is excluded from the SAN since etcd will not refer to itself as a peer.
// Hostname and `API.AdvertiseAddress` are included if the user chooses to promote the single node etcd cluster into a multi-node one.
// The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.PeerCertSANs`.
func GetEtcdPeerAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, error) {
// advertise address
advertiseAddress := net.ParseIP(cfg.API.AdvertiseAddress)
if advertiseAddress == nil {
return nil, fmt.Errorf("error parsing API AdvertiseAddress %v: is not a valid textual representation of an IP address", cfg.API.AdvertiseAddress)
}
// create AltNames with defaults DNSNames/IPs
altNames := &certutil.AltNames{
DNSNames: []string{cfg.NodeRegistration.Name, "localhost"},
IPs: []net.IP{advertiseAddress, net.IPv4(127, 0, 0, 1), net.IPv6loopback},
}
if cfg.Etcd.Local != nil {
appendSANsToAltNames(altNames, cfg.Etcd.Local.PeerCertSANs, kubeadmconstants.EtcdPeerCertName)
}
return altNames, nil
}
// appendSANsToAltNames parses SANs from as list of strings and adds them to altNames for use on a specific cert
// altNames is passed in with a pointer, and the struct is modified
// valid IP address strings are parsed and added to altNames.IPs as net.IP's
// RFC-1123 compliant DNS strings are added to altNames.DNSNames as strings
// certNames is used to print user facing warnings and should be the name of the cert the altNames will be used for
func appendSANsToAltNames(altNames *certutil.AltNames, SANs []string, certName string) {
for _, altname := range SANs {
if ip := net.ParseIP(altname); ip != nil {
altNames.IPs = append(altNames.IPs, ip)
} else if len(validation.IsDNS1123Subdomain(altname)) == 0 {
altNames.DNSNames = append(altNames.DNSNames, altname)
} else {
fmt.Printf(
"[certificates] WARNING: '%s' was not added to the '%s' SAN, because it is not a valid IP or RFC-1123 compliant DNS entry\n",
altname,
certName,
)
}
}
}

View File

@@ -0,0 +1,612 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package pkiutil
import (
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"io/ioutil"
"net"
"os"
"testing"
certutil "k8s.io/client-go/util/cert"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
)
func TestNewCertificateAuthority(t *testing.T) {
cert, key, err := NewCertificateAuthority()
if cert == nil {
t.Errorf(
"failed NewCertificateAuthority, cert == nil",
)
}
if key == nil {
t.Errorf(
"failed NewCertificateAuthority, key == nil",
)
}
if err != nil {
t.Errorf(
"failed NewCertificateAuthority with an error: %v",
err,
)
}
}
func TestNewCertAndKey(t *testing.T) {
var tests = []struct {
caKeySize int
expected bool
}{
{
// RSA key too small
caKeySize: 128,
expected: false,
},
{
// Should succeed
caKeySize: 2048,
expected: true,
},
}
for _, rt := range tests {
caKey, err := rsa.GenerateKey(rand.Reader, rt.caKeySize)
if err != nil {
t.Fatalf("Couldn't create rsa Private Key")
}
caCert := &x509.Certificate{}
config := certutil.Config{
CommonName: "test",
Organization: []string{"test"},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
_, _, actual := NewCertAndKey(caCert, caKey, config)
if (actual == nil) != rt.expected {
t.Errorf(
"failed NewCertAndKey:\n\texpected: %t\n\t actual: %t",
rt.expected,
(actual == nil),
)
}
}
}
func TestHasServerAuth(t *testing.T) {
caCert, caKey, _ := NewCertificateAuthority()
var tests = []struct {
config certutil.Config
expected bool
}{
{
config: certutil.Config{
CommonName: "test",
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
},
expected: true,
},
{
config: certutil.Config{
CommonName: "test",
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
},
expected: false,
},
}
for _, rt := range tests {
cert, _, err := NewCertAndKey(caCert, caKey, rt.config)
if err != nil {
t.Fatalf("Couldn't create cert: %v", err)
}
actual := HasServerAuth(cert)
if actual != rt.expected {
t.Errorf(
"failed HasServerAuth:\n\texpected: %t\n\t actual: %t",
rt.expected,
actual,
)
}
}
}
func TestWriteCertAndKey(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Couldn't create tmpdir")
}
defer os.RemoveAll(tmpdir)
caKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatalf("Couldn't create rsa Private Key")
}
caCert := &x509.Certificate{}
actual := WriteCertAndKey(tmpdir, "foo", caCert, caKey)
if actual != nil {
t.Errorf(
"failed WriteCertAndKey with an error: %v",
actual,
)
}
}
func TestWriteCert(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Couldn't create tmpdir")
}
defer os.RemoveAll(tmpdir)
caCert := &x509.Certificate{}
actual := WriteCert(tmpdir, "foo", caCert)
if actual != nil {
t.Errorf(
"failed WriteCertAndKey with an error: %v",
actual,
)
}
}
func TestWriteKey(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Couldn't create tmpdir")
}
defer os.RemoveAll(tmpdir)
caKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatalf("Couldn't create rsa Private Key")
}
actual := WriteKey(tmpdir, "foo", caKey)
if actual != nil {
t.Errorf(
"failed WriteCertAndKey with an error: %v",
actual,
)
}
}
func TestWritePublicKey(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Couldn't create tmpdir")
}
defer os.RemoveAll(tmpdir)
caKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatalf("Couldn't create rsa Private Key")
}
actual := WritePublicKey(tmpdir, "foo", &caKey.PublicKey)
if actual != nil {
t.Errorf(
"failed WriteCertAndKey with an error: %v",
actual,
)
}
}
func TestCertOrKeyExist(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Couldn't create tmpdir")
}
defer os.RemoveAll(tmpdir)
caKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatalf("Couldn't create rsa Private Key")
}
caCert := &x509.Certificate{}
actual := WriteCertAndKey(tmpdir, "foo", caCert, caKey)
if actual != nil {
t.Errorf(
"failed WriteCertAndKey with an error: %v",
actual,
)
}
var tests = []struct {
path string
name string
expected bool
}{
{
path: "",
name: "",
expected: false,
},
{
path: tmpdir,
name: "foo",
expected: true,
},
}
for _, rt := range tests {
actual := CertOrKeyExist(rt.path, rt.name)
if actual != rt.expected {
t.Errorf(
"failed CertOrKeyExist:\n\texpected: %t\n\t actual: %t",
rt.expected,
actual,
)
}
}
}
func TestTryLoadCertAndKeyFromDisk(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Couldn't create tmpdir")
}
defer os.RemoveAll(tmpdir)
caCert, caKey, err := NewCertificateAuthority()
if err != nil {
t.Errorf(
"failed to create cert and key with an error: %v",
err,
)
}
err = WriteCertAndKey(tmpdir, "foo", caCert, caKey)
if err != nil {
t.Errorf(
"failed to write cert and key with an error: %v",
err,
)
}
var tests = []struct {
path string
name string
expected bool
}{
{
path: "",
name: "",
expected: false,
},
{
path: tmpdir,
name: "foo",
expected: true,
},
}
for _, rt := range tests {
_, _, actual := TryLoadCertAndKeyFromDisk(rt.path, rt.name)
if (actual == nil) != rt.expected {
t.Errorf(
"failed TryLoadCertAndKeyFromDisk:\n\texpected: %t\n\t actual: %t",
rt.expected,
(actual == nil),
)
}
}
}
func TestTryLoadCertFromDisk(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Couldn't create tmpdir")
}
defer os.RemoveAll(tmpdir)
caCert, _, err := NewCertificateAuthority()
if err != nil {
t.Errorf(
"failed to create cert and key with an error: %v",
err,
)
}
err = WriteCert(tmpdir, "foo", caCert)
if err != nil {
t.Errorf(
"failed to write cert and key with an error: %v",
err,
)
}
var tests = []struct {
path string
name string
expected bool
}{
{
path: "",
name: "",
expected: false,
},
{
path: tmpdir,
name: "foo",
expected: true,
},
}
for _, rt := range tests {
_, actual := TryLoadCertFromDisk(rt.path, rt.name)
if (actual == nil) != rt.expected {
t.Errorf(
"failed TryLoadCertAndKeyFromDisk:\n\texpected: %t\n\t actual: %t",
rt.expected,
(actual == nil),
)
}
}
}
func TestTryLoadKeyFromDisk(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Couldn't create tmpdir")
}
defer os.RemoveAll(tmpdir)
_, caKey, err := NewCertificateAuthority()
if err != nil {
t.Errorf(
"failed to create cert and key with an error: %v",
err,
)
}
err = WriteKey(tmpdir, "foo", caKey)
if err != nil {
t.Errorf(
"failed to write cert and key with an error: %v",
err,
)
}
var tests = []struct {
path string
name string
expected bool
}{
{
path: "",
name: "",
expected: false,
},
{
path: tmpdir,
name: "foo",
expected: true,
},
}
for _, rt := range tests {
_, actual := TryLoadKeyFromDisk(rt.path, rt.name)
if (actual == nil) != rt.expected {
t.Errorf(
"failed TryLoadCertAndKeyFromDisk:\n\texpected: %t\n\t actual: %t",
rt.expected,
(actual == nil),
)
}
}
}
func TestPathsForCertAndKey(t *testing.T) {
crtPath, keyPath := pathsForCertAndKey("/foo", "bar")
if crtPath != "/foo/bar.crt" {
t.Errorf("unexpected certificate path: %s", crtPath)
}
if keyPath != "/foo/bar.key" {
t.Errorf("unexpected key path: %s", keyPath)
}
}
func TestPathForCert(t *testing.T) {
crtPath := pathForCert("/foo", "bar")
if crtPath != "/foo/bar.crt" {
t.Errorf("unexpected certificate path: %s", crtPath)
}
}
func TestPathForKey(t *testing.T) {
keyPath := pathForKey("/foo", "bar")
if keyPath != "/foo/bar.key" {
t.Errorf("unexpected certificate path: %s", keyPath)
}
}
func TestPathForPublicKey(t *testing.T) {
pubPath := pathForPublicKey("/foo", "bar")
if pubPath != "/foo/bar.pub" {
t.Errorf("unexpected certificate path: %s", pubPath)
}
}
func TestGetAPIServerAltNames(t *testing.T) {
var tests = []struct {
name string
cfg *kubeadmapi.MasterConfiguration
expectedDNSNames []string
expectedIPAddresses []string
}{
{
name: "ControlPlaneEndpoint DNS",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", ControlPlaneEndpoint: "api.k8s.io:6443"},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"},
APIServerCertSANs: []string{"10.1.245.94", "10.1.245.95", "1.2.3.L", "invalid,commas,in,DNS"},
},
expectedDNSNames: []string{"valid-hostname", "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local", "api.k8s.io"},
expectedIPAddresses: []string{"10.96.0.1", "1.2.3.4", "10.1.245.94", "10.1.245.95"},
},
{
name: "ControlPlaneEndpoint IP",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", ControlPlaneEndpoint: "4.5.6.7:6443"},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"},
APIServerCertSANs: []string{"10.1.245.94", "10.1.245.95", "1.2.3.L", "invalid,commas,in,DNS"},
},
expectedDNSNames: []string{"valid-hostname", "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local"},
expectedIPAddresses: []string{"10.96.0.1", "1.2.3.4", "10.1.245.94", "10.1.245.95", "4.5.6.7"},
},
}
for _, rt := range tests {
altNames, err := GetAPIServerAltNames(rt.cfg)
if err != nil {
t.Fatalf("failed calling GetAPIServerAltNames: %s: %v", rt.name, err)
}
for _, DNSName := range rt.expectedDNSNames {
found := false
for _, val := range altNames.DNSNames {
if val == DNSName {
found = true
break
}
}
if !found {
t.Errorf("%s: altNames does not contain DNSName %s but %v", rt.name, DNSName, altNames.DNSNames)
}
}
for _, IPAddress := range rt.expectedIPAddresses {
found := false
for _, val := range altNames.IPs {
if val.Equal(net.ParseIP(IPAddress)) {
found = true
break
}
}
if !found {
t.Errorf("%s: altNames does not contain IPAddress %s but %v", rt.name, IPAddress, altNames.IPs)
}
}
}
}
func TestGetEtcdAltNames(t *testing.T) {
proxy := "user-etcd-proxy"
proxyIP := "10.10.10.100"
cfg := &kubeadmapi.MasterConfiguration{
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
ServerCertSANs: []string{
proxy,
proxyIP,
"1.2.3.L",
"invalid,commas,in,DNS",
},
},
},
}
altNames, err := GetEtcdAltNames(cfg)
if err != nil {
t.Fatalf("failed calling GetEtcdAltNames: %v", err)
}
expectedDNSNames := []string{"localhost", proxy}
for _, DNSName := range expectedDNSNames {
found := false
for _, val := range altNames.DNSNames {
if val == DNSName {
found = true
break
}
}
if !found {
t.Errorf("altNames does not contain DNSName %s", DNSName)
}
}
expectedIPAddresses := []string{"127.0.0.1", net.IPv6loopback.String(), proxyIP}
for _, IPAddress := range expectedIPAddresses {
found := false
for _, val := range altNames.IPs {
if val.Equal(net.ParseIP(IPAddress)) {
found = true
break
}
}
if !found {
t.Errorf("altNames does not contain IPAddress %s", IPAddress)
}
}
}
func TestGetEtcdPeerAltNames(t *testing.T) {
hostname := "valid-hostname"
proxy := "user-etcd-proxy"
proxyIP := "10.10.10.100"
advertiseIP := "1.2.3.4"
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: advertiseIP},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: hostname},
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
PeerCertSANs: []string{
proxy,
proxyIP,
"1.2.3.L",
"invalid,commas,in,DNS",
},
},
},
}
altNames, err := GetEtcdPeerAltNames(cfg)
if err != nil {
t.Fatalf("failed calling GetEtcdPeerAltNames: %v", err)
}
expectedDNSNames := []string{hostname, proxy}
for _, DNSName := range expectedDNSNames {
found := false
for _, val := range altNames.DNSNames {
if val == DNSName {
found = true
break
}
}
if !found {
t.Errorf("altNames does not contain DNSName %s", DNSName)
}
}
expectedIPAddresses := []string{advertiseIP, proxyIP}
for _, IPAddress := range expectedIPAddresses {
found := false
for _, val := range altNames.IPs {
if val.Equal(net.ParseIP(IPAddress)) {
found = true
break
}
}
if !found {
t.Errorf("altNames does not contain IPAddress %s", IPAddress)
}
}
}

View File

@@ -0,0 +1,65 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = [
"manifests_test.go",
"volumes_test.go",
],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/features:go_default_library",
"//cmd/kubeadm/app/phases/certs:go_default_library",
"//cmd/kubeadm/test:go_default_library",
"//pkg/kubeapiserver/authorizer/modes:go_default_library",
"//pkg/util/pointer:go_default_library",
"//pkg/util/version:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"manifests.go",
"volumes.go",
],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/features:go_default_library",
"//cmd/kubeadm/app/images:go_default_library",
"//cmd/kubeadm/app/phases/certs:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/staticpod:go_default_library",
"//pkg/kubeapiserver/authorizer/modes:go_default_library",
"//pkg/util/version:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View File

@@ -0,0 +1,347 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controlplane
import (
"fmt"
"net"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/golang/glog"
"k8s.io/api/core/v1"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/images"
certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
"k8s.io/kubernetes/pkg/util/version"
)
// CreateInitStaticPodManifestFiles will write all static pod manifest files needed to bring up the control plane.
func CreateInitStaticPodManifestFiles(manifestDir string, cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("[controlplane] creating static pod files")
return createStaticPodFiles(manifestDir, cfg, kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler)
}
// CreateAPIServerStaticPodManifestFile will write APIserver static pod manifest file.
func CreateAPIServerStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating APIserver static pod files")
return createStaticPodFiles(manifestDir, cfg, kubeadmconstants.KubeAPIServer)
}
// CreateControllerManagerStaticPodManifestFile will write controller manager static pod manifest file.
func CreateControllerManagerStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating controller manager static pod files")
return createStaticPodFiles(manifestDir, cfg, kubeadmconstants.KubeControllerManager)
}
// CreateSchedulerStaticPodManifestFile will write scheduler static pod manifest file.
func CreateSchedulerStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating scheduler static pod files")
return createStaticPodFiles(manifestDir, cfg, kubeadmconstants.KubeScheduler)
}
// GetStaticPodSpecs returns all staticPodSpecs actualized to the context of the current MasterConfiguration
// NB. this methods holds the information about how kubeadm creates static pod manifests.
func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) map[string]v1.Pod {
// Get the required hostpath mounts
mounts := getHostPathVolumesForTheControlPlane(cfg)
// Prepare static pod specs
staticPodSpecs := map[string]v1.Pod{
kubeadmconstants.KubeAPIServer: staticpodutil.ComponentPod(v1.Container{
Name: kubeadmconstants.KubeAPIServer,
Image: images.GetCoreImage(kubeadmconstants.KubeAPIServer, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
ImagePullPolicy: v1.PullIfNotPresent,
Command: getAPIServerCommand(cfg),
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeAPIServer)),
LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeAPIServer, int(cfg.API.BindPort), "/healthz", v1.URISchemeHTTPS),
Resources: staticpodutil.ComponentResources("250m"),
Env: getProxyEnvVars(),
}, mounts.GetVolumes(kubeadmconstants.KubeAPIServer)),
kubeadmconstants.KubeControllerManager: staticpodutil.ComponentPod(v1.Container{
Name: kubeadmconstants.KubeControllerManager,
Image: images.GetCoreImage(kubeadmconstants.KubeControllerManager, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
ImagePullPolicy: v1.PullIfNotPresent,
Command: getControllerManagerCommand(cfg, k8sVersion),
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)),
LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeControllerManager, 10252, "/healthz", v1.URISchemeHTTP),
Resources: staticpodutil.ComponentResources("200m"),
Env: getProxyEnvVars(),
}, mounts.GetVolumes(kubeadmconstants.KubeControllerManager)),
kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{
Name: kubeadmconstants.KubeScheduler,
Image: images.GetCoreImage(kubeadmconstants.KubeScheduler, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
ImagePullPolicy: v1.PullIfNotPresent,
Command: getSchedulerCommand(cfg),
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)),
LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeScheduler, 10251, "/healthz", v1.URISchemeHTTP),
Resources: staticpodutil.ComponentResources("100m"),
Env: getProxyEnvVars(),
}, mounts.GetVolumes(kubeadmconstants.KubeScheduler)),
}
return staticPodSpecs
}
// createStaticPodFiles creates all the requested static pod files.
func createStaticPodFiles(manifestDir string, cfg *kubeadmapi.MasterConfiguration, componentNames ...string) error {
// TODO: Move the "pkg/util/version".Version object into the internal API instead of always parsing the string
k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
if err != nil {
return err
}
// gets the StaticPodSpecs, actualized for the current MasterConfiguration
glog.V(1).Infoln("[controlplane] getting StaticPodSpecs")
specs := GetStaticPodSpecs(cfg, k8sVersion)
// creates required static pod specs
for _, componentName := range componentNames {
// retrives the StaticPodSpec for given component
spec, exists := specs[componentName]
if !exists {
return fmt.Errorf("couldn't retrive StaticPodSpec for %s", componentName)
}
// writes the StaticPodSpec to disk
if err := staticpodutil.WriteStaticPodToDisk(componentName, manifestDir, spec); err != nil {
return fmt.Errorf("failed to create static pod manifest file for %q: %v", componentName, err)
}
fmt.Printf("[controlplane] wrote Static Pod manifest for component %s to %q\n", componentName, kubeadmconstants.GetStaticPodFilepath(componentName, manifestDir))
}
return nil
}
// getAPIServerCommand builds the right API server command from the given config object and version
func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration) []string {
defaultArguments := map[string]string{
"advertise-address": cfg.API.AdvertiseAddress,
"insecure-port": "0",
"enable-admission-plugins": "NodeRestriction",
// TODO: remove `PersistentVolumeLabel` in kubeadm v1.11, as it's automatically disabled in v1.11.
// ref: https://github.com/kubernetes/kubernetes/pull/64326
// we can't skip it now as we support v1.10 clusters still.
// remove it from the unit tests too.
"disable-admission-plugins": "PersistentVolumeLabel",
"service-cluster-ip-range": cfg.Networking.ServiceSubnet,
"service-account-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName),
"client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName),
"tls-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerCertName),
"tls-private-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKeyName),
"kubelet-client-certificate": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientCertName),
"kubelet-client-key": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientKeyName),
"enable-bootstrap-token-auth": "true",
"secure-port": fmt.Sprintf("%d", cfg.API.BindPort),
"allow-privileged": "true",
"kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname",
// add options to configure the front proxy. Without the generated client cert, this will never be useable
// so add it unconditionally with recommended values
"requestheader-username-headers": "X-Remote-User",
"requestheader-group-headers": "X-Remote-Group",
"requestheader-extra-headers-prefix": "X-Remote-Extra-",
"requestheader-client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName),
"requestheader-allowed-names": "front-proxy-client",
"proxy-client-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientCertName),
"proxy-client-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName),
}
command := []string{"kube-apiserver"}
// If the user set endpoints for an external etcd cluster
if cfg.Etcd.External != nil {
defaultArguments["etcd-servers"] = strings.Join(cfg.Etcd.External.Endpoints, ",")
// Use any user supplied etcd certificates
if cfg.Etcd.External.CAFile != "" {
defaultArguments["etcd-cafile"] = cfg.Etcd.External.CAFile
}
if cfg.Etcd.External.CertFile != "" && cfg.Etcd.External.KeyFile != "" {
defaultArguments["etcd-certfile"] = cfg.Etcd.External.CertFile
defaultArguments["etcd-keyfile"] = cfg.Etcd.External.KeyFile
}
} else {
// Default to etcd static pod on localhost
defaultArguments["etcd-servers"] = "https://127.0.0.1:2379"
defaultArguments["etcd-cafile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName)
defaultArguments["etcd-certfile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientCertName)
defaultArguments["etcd-keyfile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName)
}
if features.Enabled(cfg.FeatureGates, features.HighAvailability) {
defaultArguments["endpoint-reconciler-type"] = kubeadmconstants.LeaseEndpointReconcilerType
}
if features.Enabled(cfg.FeatureGates, features.DynamicKubeletConfig) {
defaultArguments["feature-gates"] = "DynamicKubeletConfig=true"
}
if features.Enabled(cfg.FeatureGates, features.Auditing) {
defaultArguments["audit-policy-file"] = kubeadmconstants.GetStaticPodAuditPolicyFile()
defaultArguments["audit-log-path"] = filepath.Join(kubeadmconstants.StaticPodAuditPolicyLogDir, kubeadmconstants.AuditPolicyLogFile)
if cfg.AuditPolicyConfiguration.LogMaxAge == nil {
defaultArguments["audit-log-maxage"] = fmt.Sprintf("%d", kubeadmapiv1alpha2.DefaultAuditPolicyLogMaxAge)
} else {
defaultArguments["audit-log-maxage"] = fmt.Sprintf("%d", *cfg.AuditPolicyConfiguration.LogMaxAge)
}
}
if cfg.APIServerExtraArgs == nil {
cfg.APIServerExtraArgs = map[string]string{}
}
cfg.APIServerExtraArgs["authorization-mode"] = getAuthzModes(cfg.APIServerExtraArgs["authorization-mode"])
command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.APIServerExtraArgs)...)
return command
}
// getAuthzModes gets the authorization-related parameters to the api server
// Node,RBAC should be fixed in this order at the beginning
// AlwaysAllow and AlwaysDeny is ignored as they are only for testing
func getAuthzModes(authzModeExtraArgs string) string {
modes := []string{
authzmodes.ModeNode,
authzmodes.ModeRBAC,
}
if strings.Contains(authzModeExtraArgs, authzmodes.ModeABAC) {
modes = append(modes, authzmodes.ModeABAC)
}
if strings.Contains(authzModeExtraArgs, authzmodes.ModeWebhook) {
modes = append(modes, authzmodes.ModeWebhook)
}
return strings.Join(modes, ",")
}
// calcNodeCidrSize determines the size of the subnets used on each node, based
// on the pod subnet provided. For IPv4, we assume that the pod subnet will
// be /16 and use /24. If the pod subnet cannot be parsed, the IPv4 value will
// be used (/24).
//
// For IPv6, the algorithm will do two three. First, the node CIDR will be set
// to a multiple of 8, using the available bits for easier readability by user.
// Second, the number of nodes will be 512 to 64K to attempt to maximize the
// number of nodes (see NOTE below). Third, pod networks of /113 and larger will
// be rejected, as the amount of bits available is too small.
//
// A special case is when the pod network size is /112, where /120 will be used,
// only allowing 256 nodes and 256 pods.
//
// If the pod network size is /113 or larger, the node CIDR will be set to the same
// size and this will be rejected later in validation.
//
// NOTE: Currently, the design allows a maximum of 64K nodes. This algorithm splits
// the available bits to maximize the number used for nodes, but still have the node
// CIDR be a multiple of eight.
//
func calcNodeCidrSize(podSubnet string) string {
maskSize := "24"
if ip, podCidr, err := net.ParseCIDR(podSubnet); err == nil {
if ip.To4() == nil {
var nodeCidrSize int
podNetSize, totalBits := podCidr.Mask.Size()
switch {
case podNetSize == 112:
// Special case, allows 256 nodes, 256 pods/node
nodeCidrSize = 120
case podNetSize < 112:
// Use multiple of 8 for node CIDR, with 512 to 64K nodes
nodeCidrSize = totalBits - ((totalBits-podNetSize-1)/8-1)*8
default:
// Not enough bits, will fail later, when validate
nodeCidrSize = podNetSize
}
maskSize = strconv.Itoa(nodeCidrSize)
}
}
return maskSize
}
// getControllerManagerCommand builds the right controller manager command from the given config object and version
func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) []string {
defaultArguments := map[string]string{
"address": "127.0.0.1",
"leader-elect": "true",
"kubeconfig": filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName),
"root-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName),
"service-account-private-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName),
"cluster-signing-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName),
"cluster-signing-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName),
"use-service-account-credentials": "true",
"controllers": "*,bootstrapsigner,tokencleaner",
}
// If using external CA, pass empty string to controller manager instead of ca.key/ca.crt path,
// so that the csrsigning controller fails to start
if res, _ := certphase.UsingExternalCA(cfg); res {
defaultArguments["cluster-signing-key-file"] = ""
defaultArguments["cluster-signing-cert-file"] = ""
}
// Let the controller-manager allocate Node CIDRs for the Pod network.
// Each node will get a subspace of the address CIDR provided with --pod-network-cidr.
if cfg.Networking.PodSubnet != "" {
maskSize := calcNodeCidrSize(cfg.Networking.PodSubnet)
defaultArguments["allocate-node-cidrs"] = "true"
defaultArguments["cluster-cidr"] = cfg.Networking.PodSubnet
defaultArguments["node-cidr-mask-size"] = maskSize
}
command := []string{"kube-controller-manager"}
command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.ControllerManagerExtraArgs)...)
return command
}
// getSchedulerCommand builds the right scheduler command from the given config object and version
func getSchedulerCommand(cfg *kubeadmapi.MasterConfiguration) []string {
defaultArguments := map[string]string{
"address": "127.0.0.1",
"leader-elect": "true",
"kubeconfig": filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName),
}
command := []string{"kube-scheduler"}
command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.SchedulerExtraArgs)...)
return command
}
// getProxyEnvVars builds a list of environment variables to use in the control plane containers in order to use the right proxy
func getProxyEnvVars() []v1.EnvVar {
envs := []v1.EnvVar{}
for _, env := range os.Environ() {
pos := strings.Index(env, "=")
if pos == -1 {
// malformed environment variable, skip it.
continue
}
name := env[:pos]
value := env[pos+1:]
if strings.HasSuffix(strings.ToLower(name), "_proxy") && value != "" {
envVar := v1.EnvVar{Name: name, Value: value}
envs = append(envs, envVar)
}
}
return envs
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,235 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controlplane
import (
"fmt"
"os"
"path/filepath"
"strings"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/sets"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
)
const (
caCertsVolumeName = "ca-certs"
caCertsVolumePath = "/etc/ssl/certs"
flexvolumeDirVolumeName = "flexvolume-dir"
flexvolumeDirVolumePath = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec"
)
// caCertsExtraVolumePaths specifies the paths that can be conditionally mounted into the apiserver and controller-manager containers
// as /etc/ssl/certs might be or contain a symlink to them. It's a variable since it may be changed in unit testing. This var MUST
// NOT be changed in normal codepaths during runtime.
var caCertsExtraVolumePaths = []string{"/etc/pki", "/usr/share/ca-certificates", "/usr/local/share/ca-certificates", "/etc/ca-certificates"}
// getHostPathVolumesForTheControlPlane gets the required hostPath volumes and mounts for the control plane
func getHostPathVolumesForTheControlPlane(cfg *kubeadmapi.MasterConfiguration) controlPlaneHostPathMounts {
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
hostPathFileOrCreate := v1.HostPathFileOrCreate
hostPathFile := v1.HostPathFile
mounts := newControlPlaneHostPathMounts()
// HostPath volumes for the API Server
// Read-only mount for the certificates directory
// TODO: Always mount the K8s Certificates directory to a static path inside of the container
mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeCertificatesVolumeName, cfg.CertificatesDir, cfg.CertificatesDir, true, &hostPathDirectoryOrCreate)
// Read-only mount for the ca certs (/etc/ssl/certs) directory
mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, caCertsVolumeName, caCertsVolumePath, caCertsVolumePath, true, &hostPathDirectoryOrCreate)
if features.Enabled(cfg.FeatureGates, features.Auditing) {
// Read-only mount for the audit policy file.
mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeAuditPolicyVolumeName, cfg.AuditPolicyConfiguration.Path, kubeadmconstants.GetStaticPodAuditPolicyFile(), true, &hostPathFile)
// Write mount for the audit logs.
mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeAuditPolicyLogVolumeName, cfg.AuditPolicyConfiguration.LogDir, kubeadmconstants.StaticPodAuditPolicyLogDir, false, &hostPathDirectoryOrCreate)
}
// If external etcd is specified, mount the directories needed for accessing the CA/serving certs and the private key
if cfg.Etcd.External != nil {
etcdVols, etcdVolMounts := getEtcdCertVolumes(cfg.Etcd.External, cfg.CertificatesDir)
mounts.AddHostPathMounts(kubeadmconstants.KubeAPIServer, etcdVols, etcdVolMounts)
}
// HostPath volumes for the controller manager
// Read-only mount for the certificates directory
// TODO: Always mount the K8s Certificates directory to a static path inside of the container
mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeCertificatesVolumeName, cfg.CertificatesDir, cfg.CertificatesDir, true, &hostPathDirectoryOrCreate)
// Read-only mount for the ca certs (/etc/ssl/certs) directory
mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, caCertsVolumeName, caCertsVolumePath, caCertsVolumePath, true, &hostPathDirectoryOrCreate)
// Read-only mount for the controller manager kubeconfig file
controllerManagerKubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName)
mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeConfigVolumeName, controllerManagerKubeConfigFile, controllerManagerKubeConfigFile, true, &hostPathFileOrCreate)
// Mount for the flexvolume directory (/usr/libexec/kubernetes/kubelet-plugins/volume/exec) directory
// Flexvolume dir must NOT be readonly as it is used for third-party plugins to integrate with their storage backends via unix domain socket.
if stat, err := os.Stat(flexvolumeDirVolumePath); err == nil && stat.IsDir() {
mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, flexvolumeDirVolumeName, flexvolumeDirVolumePath, flexvolumeDirVolumePath, false, &hostPathDirectoryOrCreate)
}
// HostPath volumes for the scheduler
// Read-only mount for the scheduler kubeconfig file
schedulerKubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName)
mounts.NewHostPathMount(kubeadmconstants.KubeScheduler, kubeadmconstants.KubeConfigVolumeName, schedulerKubeConfigFile, schedulerKubeConfigFile, true, &hostPathFileOrCreate)
// On some systems were we host-mount /etc/ssl/certs, it is also required to mount additional directories.
// This is needed due to symlinks pointing from files in /etc/ssl/certs to these directories.
for _, caCertsExtraVolumePath := range caCertsExtraVolumePaths {
if isExtraVolumeMountNeeded(caCertsExtraVolumePath) {
caCertsExtraVolumeName := strings.Replace(caCertsExtraVolumePath, "/", "-", -1)[1:]
mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, caCertsExtraVolumeName, caCertsExtraVolumePath, caCertsExtraVolumePath, true, &hostPathDirectoryOrCreate)
mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, caCertsExtraVolumeName, caCertsExtraVolumePath, caCertsExtraVolumePath, true, &hostPathDirectoryOrCreate)
}
}
// Merge user defined mounts and ensure unique volume and volume mount
// names
mounts.AddExtraHostPathMounts(kubeadmconstants.KubeAPIServer, cfg.APIServerExtraVolumes)
mounts.AddExtraHostPathMounts(kubeadmconstants.KubeControllerManager, cfg.ControllerManagerExtraVolumes)
mounts.AddExtraHostPathMounts(kubeadmconstants.KubeScheduler, cfg.SchedulerExtraVolumes)
return mounts
}
// controlPlaneHostPathMounts is a helper struct for handling all the control plane's hostPath mounts in an easy way
type controlPlaneHostPathMounts struct {
// volumes is a nested map that forces a unique volumes. The outer map's
// keys are a string that should specify the target component to add the
// volume to. The values (inner map) of the outer map are maps with string
// keys and v1.Volume values. The inner map's key should specify the volume
// name.
volumes map[string]map[string]v1.Volume
// volumeMounts is a nested map that forces a unique volume mounts. The
// outer map's keys are a string that should specify the target component
// to add the volume mount to. The values (inner map) of the outer map are
// maps with string keys and v1.VolumeMount values. The inner map's key
// should specify the volume mount name.
volumeMounts map[string]map[string]v1.VolumeMount
}
func newControlPlaneHostPathMounts() controlPlaneHostPathMounts {
return controlPlaneHostPathMounts{
volumes: map[string]map[string]v1.Volume{},
volumeMounts: map[string]map[string]v1.VolumeMount{},
}
}
func (c *controlPlaneHostPathMounts) NewHostPathMount(component, mountName, hostPath, containerPath string, readOnly bool, hostPathType *v1.HostPathType) {
vol := staticpodutil.NewVolume(mountName, hostPath, hostPathType)
c.addComponentVolume(component, vol)
volMount := staticpodutil.NewVolumeMount(mountName, containerPath, readOnly)
c.addComponentVolumeMount(component, volMount)
}
func (c *controlPlaneHostPathMounts) AddHostPathMounts(component string, vols []v1.Volume, volMounts []v1.VolumeMount) {
for _, v := range vols {
c.addComponentVolume(component, v)
}
for _, v := range volMounts {
c.addComponentVolumeMount(component, v)
}
}
// AddExtraHostPathMounts adds host path mounts and overwrites the default
// paths in the case that a user specifies the same volume/volume mount name.
func (c *controlPlaneHostPathMounts) AddExtraHostPathMounts(component string, extraVols []kubeadmapi.HostPathMount) {
for _, extraVol := range extraVols {
fmt.Printf("[controlplane] Adding extra host path mount %q to %q\n", extraVol.Name, component)
hostPathType := extraVol.PathType
c.NewHostPathMount(component, extraVol.Name, extraVol.HostPath, extraVol.MountPath, !extraVol.Writable, &hostPathType)
}
}
func (c *controlPlaneHostPathMounts) GetVolumes(component string) map[string]v1.Volume {
return c.volumes[component]
}
func (c *controlPlaneHostPathMounts) GetVolumeMounts(component string) map[string]v1.VolumeMount {
return c.volumeMounts[component]
}
func (c *controlPlaneHostPathMounts) addComponentVolume(component string, vol v1.Volume) {
if _, ok := c.volumes[component]; !ok {
c.volumes[component] = map[string]v1.Volume{}
}
c.volumes[component][vol.Name] = vol
}
func (c *controlPlaneHostPathMounts) addComponentVolumeMount(component string, volMount v1.VolumeMount) {
if _, ok := c.volumeMounts[component]; !ok {
c.volumeMounts[component] = map[string]v1.VolumeMount{}
}
c.volumeMounts[component][volMount.Name] = volMount
}
// getEtcdCertVolumes returns the volumes/volumemounts needed for talking to an external etcd cluster
func getEtcdCertVolumes(etcdCfg *kubeadmapi.ExternalEtcd, k8sCertificatesDir string) ([]v1.Volume, []v1.VolumeMount) {
certPaths := []string{etcdCfg.CAFile, etcdCfg.CertFile, etcdCfg.KeyFile}
certDirs := sets.NewString()
for _, certPath := range certPaths {
certDir := filepath.Dir(certPath)
// Ignore ".", which is the result of passing an empty path.
// Also ignore the cert directories that already may be mounted; /etc/ssl/certs, /etc/pki or Kubernetes CertificatesDir
// If the etcd certs are in there, it's okay, we don't have to do anything
extraVolumePath := false
for _, caCertsExtraVolumePath := range caCertsExtraVolumePaths {
if strings.HasPrefix(certDir, caCertsExtraVolumePath) {
extraVolumePath = true
break
}
}
if certDir == "." || extraVolumePath || strings.HasPrefix(certDir, caCertsVolumePath) || strings.HasPrefix(certDir, k8sCertificatesDir) {
continue
}
// Filter out any existing hostpath mounts in the list that contains a subset of the path
alreadyExists := false
for _, existingCertDir := range certDirs.List() {
// If the current directory is a parent of an existing one, remove the already existing one
if strings.HasPrefix(existingCertDir, certDir) {
certDirs.Delete(existingCertDir)
} else if strings.HasPrefix(certDir, existingCertDir) {
// If an existing directory is a parent of the current one, don't add the current one
alreadyExists = true
}
}
if alreadyExists {
continue
}
certDirs.Insert(certDir)
}
volumes := []v1.Volume{}
volumeMounts := []v1.VolumeMount{}
pathType := v1.HostPathDirectoryOrCreate
for i, certDir := range certDirs.List() {
name := fmt.Sprintf("etcd-certs-%d", i)
volumes = append(volumes, staticpodutil.NewVolume(name, certDir, &pathType))
volumeMounts = append(volumeMounts, staticpodutil.NewVolumeMount(name, certDir, true))
}
return volumes, volumeMounts
}
// isExtraVolumeMountNeeded specifies whether /etc/pki should be host-mounted into the containers
// On some systems were we host-mount /etc/ssl/certs, it is also required to mount /etc/pki. This is needed
// due to symlinks pointing from files in /etc/ssl/certs into /etc/pki/
func isExtraVolumeMountNeeded(caCertsExtraVolumePath string) bool {
if _, err := os.Stat(caCertsExtraVolumePath); err == nil {
return true
}
return false
}

View File

@@ -0,0 +1,675 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controlplane
import (
"fmt"
"io/ioutil"
"os"
"reflect"
"testing"
"k8s.io/api/core/v1"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
)
func TestGetEtcdCertVolumes(t *testing.T) {
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
k8sCertifcatesDir := "/etc/kubernetes/pki"
var tests = []struct {
ca, cert, key string
vol []v1.Volume
volMount []v1.VolumeMount
}{
{
// Should ignore files in /etc/ssl/certs
ca: "/etc/ssl/certs/my-etcd-ca.crt",
cert: "/etc/ssl/certs/my-etcd.crt",
key: "/etc/ssl/certs/my-etcd.key",
vol: []v1.Volume{},
volMount: []v1.VolumeMount{},
},
{
// Should ignore files in subdirs of /etc/ssl/certs
ca: "/etc/ssl/certs/etcd/my-etcd-ca.crt",
cert: "/etc/ssl/certs/etcd/my-etcd.crt",
key: "/etc/ssl/certs/etcd/my-etcd.key",
vol: []v1.Volume{},
volMount: []v1.VolumeMount{},
},
{
// Should ignore files in /etc/pki
ca: "/etc/pki/my-etcd-ca.crt",
cert: "/etc/pki/my-etcd.crt",
key: "/etc/pki/my-etcd.key",
vol: []v1.Volume{},
volMount: []v1.VolumeMount{},
},
{
// Should ignore files in Kubernetes PKI directory (and subdirs)
ca: k8sCertifcatesDir + "/ca/my-etcd-ca.crt",
cert: k8sCertifcatesDir + "/my-etcd.crt",
key: k8sCertifcatesDir + "/my-etcd.key",
vol: []v1.Volume{},
volMount: []v1.VolumeMount{},
},
{
// All in the same dir
ca: "/var/lib/certs/etcd/my-etcd-ca.crt",
cert: "/var/lib/certs/etcd/my-etcd.crt",
key: "/var/lib/certs/etcd/my-etcd.key",
vol: []v1.Volume{
{
Name: "etcd-certs-0",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/var/lib/certs/etcd",
Type: &hostPathDirectoryOrCreate,
},
},
},
},
volMount: []v1.VolumeMount{
{
Name: "etcd-certs-0",
MountPath: "/var/lib/certs/etcd",
ReadOnly: true,
},
},
},
{
// One file + two files in separate dirs
ca: "/etc/certs/etcd/my-etcd-ca.crt",
cert: "/var/lib/certs/etcd/my-etcd.crt",
key: "/var/lib/certs/etcd/my-etcd.key",
vol: []v1.Volume{
{
Name: "etcd-certs-0",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/certs/etcd",
Type: &hostPathDirectoryOrCreate,
},
},
},
{
Name: "etcd-certs-1",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/var/lib/certs/etcd",
Type: &hostPathDirectoryOrCreate,
},
},
},
},
volMount: []v1.VolumeMount{
{
Name: "etcd-certs-0",
MountPath: "/etc/certs/etcd",
ReadOnly: true,
},
{
Name: "etcd-certs-1",
MountPath: "/var/lib/certs/etcd",
ReadOnly: true,
},
},
},
{
// All three files in different directories
ca: "/etc/certs/etcd/my-etcd-ca.crt",
cert: "/var/lib/certs/etcd/my-etcd.crt",
key: "/var/lib/certs/private/my-etcd.key",
vol: []v1.Volume{
{
Name: "etcd-certs-0",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/certs/etcd",
Type: &hostPathDirectoryOrCreate,
},
},
},
{
Name: "etcd-certs-1",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/var/lib/certs/etcd",
Type: &hostPathDirectoryOrCreate,
},
},
},
{
Name: "etcd-certs-2",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/var/lib/certs/private",
Type: &hostPathDirectoryOrCreate,
},
},
},
},
volMount: []v1.VolumeMount{
{
Name: "etcd-certs-0",
MountPath: "/etc/certs/etcd",
ReadOnly: true,
},
{
Name: "etcd-certs-1",
MountPath: "/var/lib/certs/etcd",
ReadOnly: true,
},
{
Name: "etcd-certs-2",
MountPath: "/var/lib/certs/private",
ReadOnly: true,
},
},
},
{
// The most top-level dir should be used
ca: "/etc/certs/etcd/my-etcd-ca.crt",
cert: "/etc/certs/etcd/serving/my-etcd.crt",
key: "/etc/certs/etcd/serving/my-etcd.key",
vol: []v1.Volume{
{
Name: "etcd-certs-0",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/certs/etcd",
Type: &hostPathDirectoryOrCreate,
},
},
},
},
volMount: []v1.VolumeMount{
{
Name: "etcd-certs-0",
MountPath: "/etc/certs/etcd",
ReadOnly: true,
},
},
},
{
// The most top-level dir should be used, regardless of order
ca: "/etc/certs/etcd/ca/my-etcd-ca.crt",
cert: "/etc/certs/etcd/my-etcd.crt",
key: "/etc/certs/etcd/my-etcd.key",
vol: []v1.Volume{
{
Name: "etcd-certs-0",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/certs/etcd",
Type: &hostPathDirectoryOrCreate,
},
},
},
},
volMount: []v1.VolumeMount{
{
Name: "etcd-certs-0",
MountPath: "/etc/certs/etcd",
ReadOnly: true,
},
},
},
}
for _, rt := range tests {
actualVol, actualVolMount := getEtcdCertVolumes(&kubeadmapi.ExternalEtcd{
CAFile: rt.ca,
CertFile: rt.cert,
KeyFile: rt.key,
}, k8sCertifcatesDir)
if !reflect.DeepEqual(actualVol, rt.vol) {
t.Errorf(
"failed getEtcdCertVolumes:\n\texpected: %v\n\t actual: %v",
rt.vol,
actualVol,
)
}
if !reflect.DeepEqual(actualVolMount, rt.volMount) {
t.Errorf(
"failed getEtcdCertVolumes:\n\texpected: %v\n\t actual: %v",
rt.volMount,
actualVolMount,
)
}
}
}
func TestGetHostPathVolumesForTheControlPlane(t *testing.T) {
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
hostPathFileOrCreate := v1.HostPathFileOrCreate
hostPathFile := v1.HostPathFile
volMap := make(map[string]map[string]v1.Volume)
volMap[kubeadmconstants.KubeAPIServer] = map[string]v1.Volume{}
volMap[kubeadmconstants.KubeAPIServer]["k8s-certs"] = v1.Volume{
Name: "k8s-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: testCertsDir,
Type: &hostPathDirectoryOrCreate,
},
},
}
volMap[kubeadmconstants.KubeAPIServer]["ca-certs"] = v1.Volume{
Name: "ca-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/ssl/certs",
Type: &hostPathDirectoryOrCreate,
},
},
}
volMap[kubeadmconstants.KubeAPIServer]["audit"] = v1.Volume{
Name: "audit",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/foo/bar/baz.yaml",
Type: &hostPathFile,
},
},
}
volMap[kubeadmconstants.KubeAPIServer]["audit-log"] = v1.Volume{
Name: "audit-log",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/bar/foo",
Type: &hostPathDirectoryOrCreate,
},
},
}
volMap[kubeadmconstants.KubeControllerManager] = map[string]v1.Volume{}
volMap[kubeadmconstants.KubeControllerManager]["k8s-certs"] = v1.Volume{
Name: "k8s-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: testCertsDir,
Type: &hostPathDirectoryOrCreate,
},
},
}
volMap[kubeadmconstants.KubeControllerManager]["ca-certs"] = v1.Volume{
Name: "ca-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/ssl/certs",
Type: &hostPathDirectoryOrCreate,
},
},
}
volMap[kubeadmconstants.KubeControllerManager]["kubeconfig"] = v1.Volume{
Name: "kubeconfig",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/kubernetes/controller-manager.conf",
Type: &hostPathFileOrCreate,
},
},
}
volMap[kubeadmconstants.KubeScheduler] = map[string]v1.Volume{}
volMap[kubeadmconstants.KubeScheduler]["kubeconfig"] = v1.Volume{
Name: "kubeconfig",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/kubernetes/scheduler.conf",
Type: &hostPathFileOrCreate,
},
},
}
volMountMap := make(map[string]map[string]v1.VolumeMount)
volMountMap[kubeadmconstants.KubeAPIServer] = map[string]v1.VolumeMount{}
volMountMap[kubeadmconstants.KubeAPIServer]["k8s-certs"] = v1.VolumeMount{
Name: "k8s-certs",
MountPath: testCertsDir,
ReadOnly: true,
}
volMountMap[kubeadmconstants.KubeAPIServer]["ca-certs"] = v1.VolumeMount{
Name: "ca-certs",
MountPath: "/etc/ssl/certs",
ReadOnly: true,
}
volMountMap[kubeadmconstants.KubeAPIServer]["audit"] = v1.VolumeMount{
Name: "audit",
MountPath: "/etc/kubernetes/audit/audit.yaml",
ReadOnly: true,
}
volMountMap[kubeadmconstants.KubeAPIServer]["audit-log"] = v1.VolumeMount{
Name: "audit-log",
MountPath: "/var/log/kubernetes/audit",
ReadOnly: false,
}
volMountMap[kubeadmconstants.KubeControllerManager] = map[string]v1.VolumeMount{}
volMountMap[kubeadmconstants.KubeControllerManager]["k8s-certs"] = v1.VolumeMount{
Name: "k8s-certs",
MountPath: testCertsDir,
ReadOnly: true,
}
volMountMap[kubeadmconstants.KubeControllerManager]["ca-certs"] = v1.VolumeMount{
Name: "ca-certs",
MountPath: "/etc/ssl/certs",
ReadOnly: true,
}
volMountMap[kubeadmconstants.KubeControllerManager]["kubeconfig"] = v1.VolumeMount{
Name: "kubeconfig",
MountPath: "/etc/kubernetes/controller-manager.conf",
ReadOnly: true,
}
volMountMap[kubeadmconstants.KubeScheduler] = map[string]v1.VolumeMount{}
volMountMap[kubeadmconstants.KubeScheduler]["kubeconfig"] = v1.VolumeMount{
Name: "kubeconfig",
MountPath: "/etc/kubernetes/scheduler.conf",
ReadOnly: true,
}
volMap2 := make(map[string]map[string]v1.Volume)
volMap2[kubeadmconstants.KubeAPIServer] = map[string]v1.Volume{}
volMap2[kubeadmconstants.KubeAPIServer]["k8s-certs"] = v1.Volume{
Name: "k8s-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: testCertsDir,
Type: &hostPathDirectoryOrCreate,
},
},
}
volMap2[kubeadmconstants.KubeAPIServer]["ca-certs"] = v1.Volume{
Name: "ca-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/ssl/certs",
Type: &hostPathDirectoryOrCreate,
},
},
}
volMap2[kubeadmconstants.KubeAPIServer]["etcd-certs-0"] = v1.Volume{
Name: "etcd-certs-0",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/certs/etcd",
Type: &hostPathDirectoryOrCreate,
},
},
}
volMap2[kubeadmconstants.KubeAPIServer]["etcd-certs-1"] = v1.Volume{
Name: "etcd-certs-1",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/var/lib/etcd/certs",
Type: &hostPathDirectoryOrCreate,
},
},
}
volMap2[kubeadmconstants.KubeControllerManager] = map[string]v1.Volume{}
volMap2[kubeadmconstants.KubeControllerManager]["k8s-certs"] = v1.Volume{
Name: "k8s-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: testCertsDir,
Type: &hostPathDirectoryOrCreate,
},
},
}
volMap2[kubeadmconstants.KubeControllerManager]["ca-certs"] = v1.Volume{
Name: "ca-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/ssl/certs",
Type: &hostPathDirectoryOrCreate,
},
},
}
volMap2[kubeadmconstants.KubeControllerManager]["kubeconfig"] = v1.Volume{
Name: "kubeconfig",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/kubernetes/controller-manager.conf",
Type: &hostPathFileOrCreate,
},
},
}
volMap2[kubeadmconstants.KubeScheduler] = map[string]v1.Volume{}
volMap2[kubeadmconstants.KubeScheduler]["kubeconfig"] = v1.Volume{
Name: "kubeconfig",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/kubernetes/scheduler.conf",
Type: &hostPathFileOrCreate,
},
},
}
volMountMap2 := make(map[string]map[string]v1.VolumeMount)
volMountMap2[kubeadmconstants.KubeAPIServer] = map[string]v1.VolumeMount{}
volMountMap2[kubeadmconstants.KubeAPIServer]["k8s-certs"] = v1.VolumeMount{
Name: "k8s-certs",
MountPath: testCertsDir,
ReadOnly: true,
}
volMountMap2[kubeadmconstants.KubeAPIServer]["ca-certs"] = v1.VolumeMount{
Name: "ca-certs",
MountPath: "/etc/ssl/certs",
ReadOnly: true,
}
volMountMap2[kubeadmconstants.KubeAPIServer]["etcd-certs-0"] = v1.VolumeMount{
Name: "etcd-certs-0",
MountPath: "/etc/certs/etcd",
ReadOnly: true,
}
volMountMap2[kubeadmconstants.KubeAPIServer]["etcd-certs-1"] = v1.VolumeMount{
Name: "etcd-certs-1",
MountPath: "/var/lib/etcd/certs",
ReadOnly: true,
}
volMountMap2[kubeadmconstants.KubeControllerManager] = map[string]v1.VolumeMount{}
volMountMap2[kubeadmconstants.KubeControllerManager]["k8s-certs"] = v1.VolumeMount{
Name: "k8s-certs",
MountPath: testCertsDir,
ReadOnly: true,
}
volMountMap2[kubeadmconstants.KubeControllerManager]["ca-certs"] = v1.VolumeMount{
Name: "ca-certs",
MountPath: "/etc/ssl/certs",
ReadOnly: true,
}
volMountMap2[kubeadmconstants.KubeControllerManager]["kubeconfig"] = v1.VolumeMount{
Name: "kubeconfig",
MountPath: "/etc/kubernetes/controller-manager.conf",
ReadOnly: true,
}
volMountMap2[kubeadmconstants.KubeScheduler] = map[string]v1.VolumeMount{}
volMountMap2[kubeadmconstants.KubeScheduler]["kubeconfig"] = v1.VolumeMount{
Name: "kubeconfig",
MountPath: "/etc/kubernetes/scheduler.conf",
ReadOnly: true,
}
var tests = []struct {
cfg *kubeadmapi.MasterConfiguration
vol map[string]map[string]v1.Volume
volMount map[string]map[string]v1.VolumeMount
}{
{
// Should ignore files in /etc/ssl/certs
cfg: &kubeadmapi.MasterConfiguration{
CertificatesDir: testCertsDir,
Etcd: kubeadmapi.Etcd{},
FeatureGates: map[string]bool{features.Auditing: true},
AuditPolicyConfiguration: kubeadmapi.AuditPolicyConfiguration{
Path: "/foo/bar/baz.yaml",
LogDir: "/bar/foo",
},
},
vol: volMap,
volMount: volMountMap,
},
{
// Should ignore files in /etc/ssl/certs and in CertificatesDir
cfg: &kubeadmapi.MasterConfiguration{
CertificatesDir: testCertsDir,
Etcd: kubeadmapi.Etcd{
External: &kubeadmapi.ExternalEtcd{
Endpoints: []string{"foo"},
CAFile: "/etc/certs/etcd/my-etcd-ca.crt",
CertFile: testCertsDir + "/etcd/my-etcd.crt",
KeyFile: "/var/lib/etcd/certs/my-etcd.key",
},
},
},
vol: volMap2,
volMount: volMountMap2,
},
}
tmpdir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Couldn't create tmpdir")
}
defer os.RemoveAll(tmpdir)
// set up tmp caCertsExtraVolumePaths for testing
caCertsExtraVolumePaths = []string{fmt.Sprintf("%s/etc/pki", tmpdir), fmt.Sprintf("%s/usr/share/ca-certificates", tmpdir)}
defer func() { caCertsExtraVolumePaths = []string{"/etc/pki", "/usr/share/ca-certificates"} }()
for _, rt := range tests {
mounts := getHostPathVolumesForTheControlPlane(rt.cfg)
// Avoid unit test errors when the flexvolume is mounted
if _, ok := mounts.volumes[kubeadmconstants.KubeControllerManager][flexvolumeDirVolumeName]; ok {
delete(mounts.volumes[kubeadmconstants.KubeControllerManager], flexvolumeDirVolumeName)
}
if _, ok := mounts.volumeMounts[kubeadmconstants.KubeControllerManager][flexvolumeDirVolumeName]; ok {
delete(mounts.volumeMounts[kubeadmconstants.KubeControllerManager], flexvolumeDirVolumeName)
}
if !reflect.DeepEqual(mounts.volumes, rt.vol) {
t.Errorf(
"failed getHostPathVolumesForTheControlPlane:\n\texpected: %v\n\t actual: %v",
rt.vol,
mounts.volumes,
)
}
if !reflect.DeepEqual(mounts.volumeMounts, rt.volMount) {
t.Errorf(
"failed getHostPathVolumesForTheControlPlane:\n\texpected: %v\n\t actual: %v",
rt.volMount,
mounts.volumeMounts,
)
}
}
}
func TestAddExtraHostPathMounts(t *testing.T) {
mounts := newControlPlaneHostPathMounts()
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
hostPathFileOrCreate := v1.HostPathFileOrCreate
vols := []v1.Volume{
{
Name: "foo",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/tmp/foo",
Type: &hostPathDirectoryOrCreate,
},
},
},
{
Name: "bar",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/tmp/bar",
Type: &hostPathFileOrCreate,
},
},
},
}
volMounts := []v1.VolumeMount{
{
Name: "foo",
MountPath: "/tmp/foo",
ReadOnly: true,
},
{
Name: "bar",
MountPath: "/tmp/bar",
ReadOnly: true,
},
}
mounts.AddHostPathMounts("component", vols, volMounts)
hostPathMounts := []kubeadmapi.HostPathMount{
{
Name: "foo-0",
HostPath: "/tmp/qux-0",
MountPath: "/tmp/qux-0",
Writable: false,
PathType: v1.HostPathFile,
},
{
Name: "bar-0",
HostPath: "/tmp/asd-0",
MountPath: "/tmp/asd-0",
Writable: true,
PathType: v1.HostPathDirectory,
},
{
Name: "foo-1",
HostPath: "/tmp/qux-1",
MountPath: "/tmp/qux-1",
Writable: false,
PathType: v1.HostPathFileOrCreate,
},
{
Name: "bar-1",
HostPath: "/tmp/asd-1",
MountPath: "/tmp/asd-1",
Writable: true,
PathType: v1.HostPathDirectoryOrCreate,
},
}
mounts.AddExtraHostPathMounts("component", hostPathMounts)
for _, hostMount := range hostPathMounts {
volumeName := hostMount.Name
if _, ok := mounts.volumes["component"][volumeName]; !ok {
t.Errorf("Expected to find volume %q", volumeName)
}
vol := mounts.volumes["component"][volumeName]
if vol.Name != volumeName {
t.Errorf("Expected volume name %q", volumeName)
}
if vol.HostPath.Path != hostMount.HostPath {
t.Errorf("Expected host path %q", hostMount.HostPath)
}
if _, ok := mounts.volumeMounts["component"][volumeName]; !ok {
t.Errorf("Expected to find volume mount %q", volumeName)
}
if *vol.HostPath.Type != v1.HostPathType(hostMount.PathType) {
t.Errorf("Expected to host path type %q", hostMount.PathType)
}
volMount, _ := mounts.volumeMounts["component"][volumeName]
if volMount.Name != volumeName {
t.Errorf("Expected volume mount name %q", volumeName)
}
if volMount.MountPath != hostMount.MountPath {
t.Errorf("Expected container path %q", hostMount.MountPath)
}
if volMount.ReadOnly != !hostMount.Writable {
t.Errorf("Expected volume writable setting %t", hostMount.Writable)
}
}
}

View File

@@ -0,0 +1,46 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = ["local_test.go"],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/test:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = ["local.go"],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/images:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/staticpod:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View File

@@ -0,0 +1,101 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package etcd
import (
"fmt"
"path/filepath"
"github.com/golang/glog"
"k8s.io/api/core/v1"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/images"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
)
const (
etcdVolumeName = "etcd-data"
certsVolumeName = "etcd-certs"
)
// CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file.
func CreateLocalEtcdStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating local etcd static pod manifest file")
// gets etcd StaticPodSpec, actualized for the current MasterConfiguration
spec := GetEtcdPodSpec(cfg)
// writes etcd StaticPod to disk
if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
return err
}
fmt.Printf("[etcd] Wrote Static Pod manifest for a local etcd instance to %q\n", kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.Etcd, manifestDir))
return nil
}
// GetEtcdPodSpec returns the etcd static Pod actualized to the context of the current MasterConfiguration
// NB. GetEtcdPodSpec methods holds the information about how kubeadm creates etcd static pod manifests.
func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod {
pathType := v1.HostPathDirectoryOrCreate
etcdMounts := map[string]v1.Volume{
etcdVolumeName: staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.Local.DataDir, &pathType),
certsVolumeName: staticpodutil.NewVolume(certsVolumeName, cfg.CertificatesDir+"/etcd", &pathType),
}
return staticpodutil.ComponentPod(v1.Container{
Name: kubeadmconstants.Etcd,
Command: getEtcdCommand(cfg),
Image: images.GetCoreImage(kubeadmconstants.Etcd, cfg.ImageRepository, cfg.KubernetesVersion, cfg.Etcd.Local.Image),
ImagePullPolicy: v1.PullIfNotPresent,
// Mount the etcd datadir path read-write so etcd can store data in a more persistent manner
VolumeMounts: []v1.VolumeMount{
staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.Local.DataDir, false),
staticpodutil.NewVolumeMount(certsVolumeName, cfg.CertificatesDir+"/etcd", false),
},
LivenessProbe: staticpodutil.EtcdProbe(
cfg, kubeadmconstants.Etcd, 2379, cfg.CertificatesDir,
kubeadmconstants.EtcdCACertName, kubeadmconstants.EtcdHealthcheckClientCertName, kubeadmconstants.EtcdHealthcheckClientKeyName,
),
}, etcdMounts)
}
// getEtcdCommand builds the right etcd command from the given config object
func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string {
defaultArguments := map[string]string{
"name": cfg.GetNodeName(),
"listen-client-urls": "https://127.0.0.1:2379",
"advertise-client-urls": "https://127.0.0.1:2379",
"listen-peer-urls": "https://127.0.0.1:2380",
"initial-advertise-peer-urls": "https://127.0.0.1:2380",
"data-dir": cfg.Etcd.Local.DataDir,
"cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerCertName),
"key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName),
"trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName),
"client-cert-auth": "true",
"peer-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerCertName),
"peer-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName),
"peer-trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName),
"peer-client-cert-auth": "true",
"snapshot-count": "10000",
"initial-cluster": fmt.Sprintf("%s=https://127.0.0.1:2380", cfg.GetNodeName()),
}
command := []string{"etcd"}
command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.Etcd.Local.ExtraArgs)...)
return command
}

View File

@@ -0,0 +1,195 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package etcd
import (
"os"
"path/filepath"
"reflect"
"sort"
"testing"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
)
func TestGetEtcdPodSpec(t *testing.T) {
// Creates a Master Configuration
cfg := &kubeadmapi.MasterConfiguration{
KubernetesVersion: "v1.7.0",
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
DataDir: "/var/lib/etcd",
Image: "",
},
},
}
// Executes GetEtcdPodSpec
spec := GetEtcdPodSpec(cfg)
// Assert each specs refers to the right pod
if spec.Spec.Containers[0].Name != kubeadmconstants.Etcd {
t.Errorf("getKubeConfigSpecs spec for etcd contains pod %s, expects %s", spec.Spec.Containers[0].Name, kubeadmconstants.Etcd)
}
}
func TestCreateLocalEtcdStaticPodManifestFile(t *testing.T) {
// Create temp folder for the test case
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
// Creates a Master Configuration
cfg := &kubeadmapi.MasterConfiguration{
KubernetesVersion: "v1.7.0",
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
DataDir: "/var/lib/etcd",
Image: "k8s.gcr.io/etcd",
},
},
}
// Execute createStaticPodFunction
manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName)
err := CreateLocalEtcdStaticPodManifestFile(manifestPath, cfg)
if err != nil {
t.Errorf("Error executing CreateEtcdStaticPodManifestFile: %v", err)
}
// Assert expected files are there
testutil.AssertFilesCount(t, manifestPath, 1)
testutil.AssertFileExists(t, manifestPath, kubeadmconstants.Etcd+".yaml")
}
func TestGetEtcdCommand(t *testing.T) {
var tests = []struct {
cfg *kubeadmapi.MasterConfiguration
expected []string
}{
{
cfg: &kubeadmapi.MasterConfiguration{
NodeRegistration: kubeadmapi.NodeRegistrationOptions{
Name: "foo",
},
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
DataDir: "/var/lib/etcd",
},
},
},
expected: []string{
"etcd",
"--name=foo",
"--listen-client-urls=https://127.0.0.1:2379",
"--advertise-client-urls=https://127.0.0.1:2379",
"--listen-peer-urls=https://127.0.0.1:2380",
"--initial-advertise-peer-urls=https://127.0.0.1:2380",
"--data-dir=/var/lib/etcd",
"--cert-file=" + kubeadmconstants.EtcdServerCertName,
"--key-file=" + kubeadmconstants.EtcdServerKeyName,
"--trusted-ca-file=" + kubeadmconstants.EtcdCACertName,
"--client-cert-auth=true",
"--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName,
"--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName,
"--peer-trusted-ca-file=" + kubeadmconstants.EtcdCACertName,
"--snapshot-count=10000",
"--peer-client-cert-auth=true",
"--initial-cluster=foo=https://127.0.0.1:2380",
},
},
{
cfg: &kubeadmapi.MasterConfiguration{
NodeRegistration: kubeadmapi.NodeRegistrationOptions{
Name: "bar",
},
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
DataDir: "/var/lib/etcd",
ExtraArgs: map[string]string{
"listen-client-urls": "https://10.0.1.10:2379",
"advertise-client-urls": "https://10.0.1.10:2379",
},
},
},
},
expected: []string{
"etcd",
"--name=bar",
"--listen-client-urls=https://10.0.1.10:2379",
"--advertise-client-urls=https://10.0.1.10:2379",
"--listen-peer-urls=https://127.0.0.1:2380",
"--initial-advertise-peer-urls=https://127.0.0.1:2380",
"--data-dir=/var/lib/etcd",
"--cert-file=" + kubeadmconstants.EtcdServerCertName,
"--key-file=" + kubeadmconstants.EtcdServerKeyName,
"--trusted-ca-file=" + kubeadmconstants.EtcdCACertName,
"--client-cert-auth=true",
"--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName,
"--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName,
"--peer-trusted-ca-file=" + kubeadmconstants.EtcdCACertName,
"--snapshot-count=10000",
"--peer-client-cert-auth=true",
"--initial-cluster=bar=https://127.0.0.1:2380",
},
},
{
cfg: &kubeadmapi.MasterConfiguration{
NodeRegistration: kubeadmapi.NodeRegistrationOptions{
Name: "wombat",
},
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
DataDir: "/etc/foo",
},
},
},
expected: []string{
"etcd",
"--name=wombat",
"--listen-client-urls=https://127.0.0.1:2379",
"--advertise-client-urls=https://127.0.0.1:2379",
"--listen-peer-urls=https://127.0.0.1:2380",
"--initial-advertise-peer-urls=https://127.0.0.1:2380",
"--data-dir=/etc/foo",
"--cert-file=" + kubeadmconstants.EtcdServerCertName,
"--key-file=" + kubeadmconstants.EtcdServerKeyName,
"--trusted-ca-file=" + kubeadmconstants.EtcdCACertName,
"--client-cert-auth=true",
"--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName,
"--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName,
"--peer-trusted-ca-file=" + kubeadmconstants.EtcdCACertName,
"--snapshot-count=10000",
"--peer-client-cert-auth=true",
"--initial-cluster=wombat=https://127.0.0.1:2380",
},
},
}
for _, rt := range tests {
actual := getEtcdCommand(rt.cfg)
sort.Strings(actual)
sort.Strings(rt.expected)
if !reflect.DeepEqual(actual, rt.expected) {
t.Errorf("failed getEtcdCommand:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual)
}
}
}

View File

@@ -0,0 +1,57 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = [
"doc.go",
"kubeconfig.go",
],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/kubeconfig:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
"//vendor/k8s.io/client-go/util/cert:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)
go_test(
name = "go_default_test",
srcs = ["kubeconfig_test.go"],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/test:go_default_library",
"//cmd/kubeadm/test/certs:go_default_library",
"//cmd/kubeadm/test/kubeconfig:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
],
)

View File

@@ -0,0 +1,35 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubeconfig
/*
PHASE: KUBECONFIG
INPUTS:
From MasterConfiguration
The Master API Server endpoint (AdvertiseAddress + BindPort) is required so the KubeConfig file knows where to find the master
The KubernetesDir path is required for knowing where to put the KubeConfig files
The PKIPath is required for knowing where all certificates should be stored
OUTPUTS:
Files to KubernetesDir (default /etc/kubernetes):
- admin.conf
- kubelet.conf
- scheduler.conf
- controller-manager.conf
*/

View File

@@ -0,0 +1,343 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubeconfig
import (
"bytes"
"crypto/x509"
"fmt"
"io"
"os"
"path/filepath"
"crypto/rsa"
"github.com/golang/glog"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
certutil "k8s.io/client-go/util/cert"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
)
// clientCertAuth struct holds info required to build a client certificate to provide authentication info in a kubeconfig object
type clientCertAuth struct {
CAKey *rsa.PrivateKey
Organizations []string
}
// tokenAuth struct holds info required to use a token to provide authentication info in a kubeconfig object
type tokenAuth struct {
Token string
}
// kubeConfigSpec struct holds info required to build a KubeConfig object
type kubeConfigSpec struct {
CACert *x509.Certificate
APIServer string
ClientName string
TokenAuth *tokenAuth
ClientCertAuth *clientCertAuth
}
// CreateInitKubeConfigFiles will create and write to disk all kubeconfig files necessary in the kubeadm init phase
// to establish the control plane, including also the admin kubeconfig file.
// If kubeconfig files already exists, they are used only if evaluated equal; otherwise an error is returned.
func CreateInitKubeConfigFiles(outDir string, cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating all kubeconfig files")
return createKubeConfigFiles(
outDir,
cfg,
kubeadmconstants.AdminKubeConfigFileName,
kubeadmconstants.KubeletKubeConfigFileName,
kubeadmconstants.ControllerManagerKubeConfigFileName,
kubeadmconstants.SchedulerKubeConfigFileName,
)
}
// CreateAdminKubeConfigFile create a kubeconfig file for the admin to use and for kubeadm itself.
// If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned.
func CreateAdminKubeConfigFile(outDir string, cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("create a kubeconfig file for the admin and for kubeadm itself")
return createKubeConfigFiles(outDir, cfg, kubeadmconstants.AdminKubeConfigFileName)
}
// CreateKubeletKubeConfigFile create a kubeconfig file for the Kubelet to use.
// If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned.
func CreateKubeletKubeConfigFile(outDir string, cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a kubeconfig file for the Kubelet")
return createKubeConfigFiles(outDir, cfg, kubeadmconstants.KubeletKubeConfigFileName)
}
// CreateControllerManagerKubeConfigFile create a kubeconfig file for the ControllerManager to use.
// If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned.
func CreateControllerManagerKubeConfigFile(outDir string, cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating kubeconfig file for the ControllerManager")
return createKubeConfigFiles(outDir, cfg, kubeadmconstants.ControllerManagerKubeConfigFileName)
}
// CreateSchedulerKubeConfigFile create a create a kubeconfig file for the Scheduler to use.
// If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned.
func CreateSchedulerKubeConfigFile(outDir string, cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating kubeconfig file for Scheduler")
return createKubeConfigFiles(outDir, cfg, kubeadmconstants.SchedulerKubeConfigFileName)
}
// createKubeConfigFiles creates all the requested kubeconfig files.
// If kubeconfig files already exists, they are used only if evaluated equal; otherwise an error is returned.
func createKubeConfigFiles(outDir string, cfg *kubeadmapi.MasterConfiguration, kubeConfigFileNames ...string) error {
// gets the KubeConfigSpecs, actualized for the current MasterConfiguration
specs, err := getKubeConfigSpecs(cfg)
if err != nil {
return err
}
for _, kubeConfigFileName := range kubeConfigFileNames {
// retrives the KubeConfigSpec for given kubeConfigFileName
spec, exists := specs[kubeConfigFileName]
if !exists {
return fmt.Errorf("couldn't retrive KubeConfigSpec for %s", kubeConfigFileName)
}
// builds the KubeConfig object
config, err := buildKubeConfigFromSpec(spec, cfg.ClusterName)
if err != nil {
return err
}
// writes the KubeConfig to disk if it not exists
if err = createKubeConfigFileIfNotExists(outDir, kubeConfigFileName, config); err != nil {
return err
}
}
return nil
}
// getKubeConfigSpecs returns all KubeConfigSpecs actualized to the context of the current MasterConfiguration
// NB. this methods holds the information about how kubeadm creates kubeconfig files.
func getKubeConfigSpecs(cfg *kubeadmapi.MasterConfiguration) (map[string]*kubeConfigSpec, error) {
caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
if err != nil {
return nil, fmt.Errorf("couldn't create a kubeconfig; the CA files couldn't be loaded: %v", err)
}
masterEndpoint, err := kubeadmutil.GetMasterEndpoint(&cfg.API)
if err != nil {
return nil, err
}
var kubeConfigSpec = map[string]*kubeConfigSpec{
kubeadmconstants.AdminKubeConfigFileName: {
CACert: caCert,
APIServer: masterEndpoint,
ClientName: "kubernetes-admin",
ClientCertAuth: &clientCertAuth{
CAKey: caKey,
Organizations: []string{kubeadmconstants.MastersGroup},
},
},
kubeadmconstants.KubeletKubeConfigFileName: {
CACert: caCert,
APIServer: masterEndpoint,
ClientName: fmt.Sprintf("system:node:%s", cfg.NodeRegistration.Name),
ClientCertAuth: &clientCertAuth{
CAKey: caKey,
Organizations: []string{kubeadmconstants.NodesGroup},
},
},
kubeadmconstants.ControllerManagerKubeConfigFileName: {
CACert: caCert,
APIServer: masterEndpoint,
ClientName: kubeadmconstants.ControllerManagerUser,
ClientCertAuth: &clientCertAuth{
CAKey: caKey,
},
},
kubeadmconstants.SchedulerKubeConfigFileName: {
CACert: caCert,
APIServer: masterEndpoint,
ClientName: kubeadmconstants.SchedulerUser,
ClientCertAuth: &clientCertAuth{
CAKey: caKey,
},
},
}
return kubeConfigSpec, nil
}
// buildKubeConfigFromSpec creates a kubeconfig object for the given kubeConfigSpec
func buildKubeConfigFromSpec(spec *kubeConfigSpec, clustername string) (*clientcmdapi.Config, error) {
// If this kubeconfig should use token
if spec.TokenAuth != nil {
// create a kubeconfig with a token
return kubeconfigutil.CreateWithToken(
spec.APIServer,
clustername,
spec.ClientName,
certutil.EncodeCertPEM(spec.CACert),
spec.TokenAuth.Token,
), nil
}
// otherwise, create a client certs
clientCertConfig := certutil.Config{
CommonName: spec.ClientName,
Organization: spec.ClientCertAuth.Organizations,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
clientCert, clientKey, err := pkiutil.NewCertAndKey(spec.CACert, spec.ClientCertAuth.CAKey, clientCertConfig)
if err != nil {
return nil, fmt.Errorf("failure while creating %s client certificate: %v", spec.ClientName, err)
}
// create a kubeconfig with the client certs
return kubeconfigutil.CreateWithCerts(
spec.APIServer,
clustername,
spec.ClientName,
certutil.EncodeCertPEM(spec.CACert),
certutil.EncodePrivateKeyPEM(clientKey),
certutil.EncodeCertPEM(clientCert),
), nil
}
// createKubeConfigFileIfNotExists saves the KubeConfig object into a file if there isn't any file at the given path.
// If there already is a KubeConfig file at the given path; kubeadm tries to load it and check if the values in the
// existing and the expected config equals. If they do; kubeadm will just skip writing the file as it's up-to-date,
// but if a file exists but has old content or isn't a kubeconfig file, this function returns an error.
func createKubeConfigFileIfNotExists(outDir, filename string, config *clientcmdapi.Config) error {
kubeConfigFilePath := filepath.Join(outDir, filename)
// Check if the file exist, and if it doesn't, just write it to disk
if _, err := os.Stat(kubeConfigFilePath); os.IsNotExist(err) {
err = kubeconfigutil.WriteToDisk(kubeConfigFilePath, config)
if err != nil {
return fmt.Errorf("failed to save kubeconfig file %s on disk: %v", kubeConfigFilePath, err)
}
fmt.Printf("[kubeconfig] Wrote KubeConfig file to disk: %q\n", kubeConfigFilePath)
return nil
}
// The kubeconfig already exists, let's check if it has got the same CA and server URL
currentConfig, err := clientcmd.LoadFromFile(kubeConfigFilePath)
if err != nil {
return fmt.Errorf("failed to load kubeconfig file %s that already exists on disk: %v", kubeConfigFilePath, err)
}
expectedCtx := config.CurrentContext
expectedCluster := config.Contexts[expectedCtx].Cluster
currentCtx := currentConfig.CurrentContext
currentCluster := currentConfig.Contexts[currentCtx].Cluster
// If the current CA cert on disk doesn't match the expected CA cert, error out because we have a file, but it's stale
if !bytes.Equal(currentConfig.Clusters[currentCluster].CertificateAuthorityData, config.Clusters[expectedCluster].CertificateAuthorityData) {
return fmt.Errorf("a kubeconfig file %q exists already but has got the wrong CA cert", kubeConfigFilePath)
}
// If the current API Server location on disk doesn't match the expected API server, error out because we have a file, but it's stale
if currentConfig.Clusters[currentCluster].Server != config.Clusters[expectedCluster].Server {
return fmt.Errorf("a kubeconfig file %q exists already but has got the wrong API Server URL", kubeConfigFilePath)
}
// kubeadm doesn't validate the existing kubeconfig file more than this (kubeadm trusts the client certs to be valid)
// Basically, if we find a kubeconfig file with the same path; the same CA cert and the same server URL;
// kubeadm thinks those files are equal and doesn't bother writing a new file
fmt.Printf("[kubeconfig] Using existing up-to-date KubeConfig file: %q\n", kubeConfigFilePath)
return nil
}
// WriteKubeConfigWithClientCert writes a kubeconfig file - with a client certificate as authentication info - to the given writer.
func WriteKubeConfigWithClientCert(out io.Writer, cfg *kubeadmapi.MasterConfiguration, clientName string, organizations []string) error {
// creates the KubeConfigSpecs, actualized for the current MasterConfiguration
caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
if err != nil {
return fmt.Errorf("couldn't create a kubeconfig; the CA files couldn't be loaded: %v", err)
}
masterEndpoint, err := kubeadmutil.GetMasterEndpoint(&cfg.API)
if err != nil {
return err
}
spec := &kubeConfigSpec{
ClientName: clientName,
APIServer: masterEndpoint,
CACert: caCert,
ClientCertAuth: &clientCertAuth{
CAKey: caKey,
Organizations: organizations,
},
}
return writeKubeConfigFromSpec(out, spec, cfg.ClusterName)
}
// WriteKubeConfigWithToken writes a kubeconfig file - with a token as client authentication info - to the given writer.
func WriteKubeConfigWithToken(out io.Writer, cfg *kubeadmapi.MasterConfiguration, clientName, token string) error {
// creates the KubeConfigSpecs, actualized for the current MasterConfiguration
caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
if err != nil {
return fmt.Errorf("couldn't create a kubeconfig; the CA files couldn't be loaded: %v", err)
}
masterEndpoint, err := kubeadmutil.GetMasterEndpoint(&cfg.API)
if err != nil {
return err
}
spec := &kubeConfigSpec{
ClientName: clientName,
APIServer: masterEndpoint,
CACert: caCert,
TokenAuth: &tokenAuth{
Token: token,
},
}
return writeKubeConfigFromSpec(out, spec, cfg.ClusterName)
}
// writeKubeConfigFromSpec creates a kubeconfig object from a kubeConfigSpec and writes it to the given writer.
func writeKubeConfigFromSpec(out io.Writer, spec *kubeConfigSpec, clustername string) error {
// builds the KubeConfig object
config, err := buildKubeConfigFromSpec(spec, clustername)
if err != nil {
return err
}
// writes the KubeConfig to disk if it not exists
configBytes, err := clientcmd.Write(*config)
if err != nil {
return fmt.Errorf("failure while serializing admin kubeconfig: %v", err)
}
fmt.Fprintln(out, string(configBytes))
return nil
}

View File

@@ -0,0 +1,466 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubeconfig
import (
"bytes"
"crypto/rsa"
"crypto/x509"
"fmt"
"io"
"os"
"reflect"
"testing"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
pkiutil "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
certstestutil "k8s.io/kubernetes/cmd/kubeadm/test/certs"
kubeconfigtestutil "k8s.io/kubernetes/cmd/kubeadm/test/kubeconfig"
)
func TestGetKubeConfigSpecsFailsIfCADoesntExists(t *testing.T) {
// Create temp folder for the test case (without a CA)
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
// Creates a Master Configuration pointing to the pkidir folder
cfg := &kubeadmapi.MasterConfiguration{
CertificatesDir: tmpdir,
}
// Executes getKubeConfigSpecs
if _, err := getKubeConfigSpecs(cfg); err == nil {
t.Error("getKubeConfigSpecs didnt failed when expected")
}
}
func TestGetKubeConfigSpecs(t *testing.T) {
// Create temp folder for the test case
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
// Adds a pki folder with a ca certs to the temp folder
pkidir := testutil.SetupPkiDirWithCertificateAuthorithy(t, tmpdir)
// Creates Master Configurations pointing to the pkidir folder
cfgs := []*kubeadmapi.MasterConfiguration{
{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", BindPort: 1234},
CertificatesDir: pkidir,
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"},
},
{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", ControlPlaneEndpoint: "api.k8s.io", BindPort: 1234},
CertificatesDir: pkidir,
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"},
},
{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", ControlPlaneEndpoint: "api.k8s.io:4321", BindPort: 1234},
CertificatesDir: pkidir,
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"},
},
{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", ControlPlaneEndpoint: "api.k8s.io", BindPort: 1234},
CertificatesDir: pkidir,
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"},
},
{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", ControlPlaneEndpoint: "api.k8s.io:4321", BindPort: 1234},
CertificatesDir: pkidir,
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"},
},
}
for _, cfg := range cfgs {
var assertions = []struct {
kubeConfigFile string
clientName string
organizations []string
}{
{
kubeConfigFile: kubeadmconstants.AdminKubeConfigFileName,
clientName: "kubernetes-admin",
organizations: []string{kubeadmconstants.MastersGroup},
},
{
kubeConfigFile: kubeadmconstants.KubeletKubeConfigFileName,
clientName: fmt.Sprintf("system:node:%s", cfg.NodeRegistration.Name),
organizations: []string{kubeadmconstants.NodesGroup},
},
{
kubeConfigFile: kubeadmconstants.ControllerManagerKubeConfigFileName,
clientName: kubeadmconstants.ControllerManagerUser,
},
{
kubeConfigFile: kubeadmconstants.SchedulerKubeConfigFileName,
clientName: kubeadmconstants.SchedulerUser,
},
}
for _, assertion := range assertions {
// Executes getKubeConfigSpecs
specs, err := getKubeConfigSpecs(cfg)
if err != nil {
t.Fatal("getKubeConfigSpecs failed!")
}
var spec *kubeConfigSpec
var ok bool
// assert the spec for the kubeConfigFile exists
if spec, ok = specs[assertion.kubeConfigFile]; !ok {
t.Errorf("getKubeConfigSpecs didn't create spec for %s ", assertion.kubeConfigFile)
continue
}
// Assert clientName
if spec.ClientName != assertion.clientName {
t.Errorf("getKubeConfigSpecs for %s clientName is %s, expected %s", assertion.kubeConfigFile, spec.ClientName, assertion.clientName)
}
// Assert Organizations
if spec.ClientCertAuth == nil || !reflect.DeepEqual(spec.ClientCertAuth.Organizations, assertion.organizations) {
t.Errorf("getKubeConfigSpecs for %s Organizations is %v, expected %v", assertion.kubeConfigFile, spec.ClientCertAuth.Organizations, assertion.organizations)
}
// Asserts MasterConfiguration values injected into spec
masterEndpoint, err := kubeadmutil.GetMasterEndpoint(&cfg.API)
if err != nil {
t.Error(err)
}
if spec.APIServer != masterEndpoint {
t.Errorf("getKubeConfigSpecs didn't injected cfg.APIServer endpoint into spec for %s", assertion.kubeConfigFile)
}
// Asserts CA certs and CA keys loaded into specs
if spec.CACert == nil {
t.Errorf("getKubeConfigSpecs didn't loaded CACert into spec for %s!", assertion.kubeConfigFile)
}
if spec.ClientCertAuth == nil || spec.ClientCertAuth.CAKey == nil {
t.Errorf("getKubeConfigSpecs didn't loaded CAKey into spec for %s!", assertion.kubeConfigFile)
}
}
}
}
func TestBuildKubeConfigFromSpecWithClientAuth(t *testing.T) {
// Creates a CA
caCert, caKey := certstestutil.SetupCertificateAuthorithy(t)
// Executes buildKubeConfigFromSpec passing a KubeConfigSpec with a ClientAuth
config := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://1.2.3.4:1234", "myClientName", "test-cluster", "myOrg1", "myOrg2")
// Asserts spec data are propagated to the kubeconfig
kubeconfigtestutil.AssertKubeConfigCurrentCluster(t, config, "https://1.2.3.4:1234", caCert)
kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithClientCert(t, config, caCert, "myClientName", "myOrg1", "myOrg2")
}
func TestBuildKubeConfigFromSpecWithTokenAuth(t *testing.T) {
// Creates a CA
caCert, _ := certstestutil.SetupCertificateAuthorithy(t)
// Executes buildKubeConfigFromSpec passing a KubeConfigSpec with a Token
config := setupdKubeConfigWithTokenAuth(t, caCert, "https://1.2.3.4:1234", "myClientName", "123456", "test-cluster")
// Asserts spec data are propagated to the kubeconfig
kubeconfigtestutil.AssertKubeConfigCurrentCluster(t, config, "https://1.2.3.4:1234", caCert)
kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithToken(t, config, "myClientName", "123456")
}
func TestCreateKubeConfigFileIfNotExists(t *testing.T) {
// Creates a CAs
caCert, caKey := certstestutil.SetupCertificateAuthorithy(t)
anotherCaCert, anotherCaKey := certstestutil.SetupCertificateAuthorithy(t)
// build kubeconfigs (to be used to test kubeconfigs equality/not equality)
config := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://1.2.3.4:1234", "test-cluster", "myOrg1", "myOrg2")
configWithAnotherClusterCa := setupdKubeConfigWithClientAuth(t, anotherCaCert, anotherCaKey, "https://1.2.3.4:1234", "test-cluster", "myOrg1", "myOrg2")
configWithAnotherClusterAddress := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://3.4.5.6:3456", "myOrg1", "test-cluster", "myOrg2")
var tests = []struct {
existingKubeConfig *clientcmdapi.Config
kubeConfig *clientcmdapi.Config
expectedError bool
}{
{ // if there is no existing KubeConfig, creates the kubeconfig
kubeConfig: config,
},
{ // if KubeConfig is equal to the existingKubeConfig - refers to the same cluster -, use the existing (Test idempotency)
existingKubeConfig: config,
kubeConfig: config,
},
{ // if KubeConfig is not equal to the existingKubeConfig - refers to the another cluster (a cluster with another Ca) -, raise error
existingKubeConfig: config,
kubeConfig: configWithAnotherClusterCa,
expectedError: true,
},
{ // if KubeConfig is not equal to the existingKubeConfig - refers to the another cluster (a cluster with another address) -, raise error
existingKubeConfig: config,
kubeConfig: configWithAnotherClusterAddress,
expectedError: true,
},
}
for _, test := range tests {
// Create temp folder for the test case
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
// Writes the existing kubeconfig file to disk
if test.existingKubeConfig != nil {
if err := createKubeConfigFileIfNotExists(tmpdir, "test.conf", test.existingKubeConfig); err != nil {
t.Errorf("createKubeConfigFileIfNotExists failed")
}
}
// Writes the KubeConfig file to disk
err := createKubeConfigFileIfNotExists(tmpdir, "test.conf", test.kubeConfig)
if test.expectedError && err == nil {
t.Errorf("createKubeConfigFileIfNotExists didn't failed when expected to fail")
}
if !test.expectedError && err != nil {
t.Errorf("createKubeConfigFileIfNotExists failed")
}
// Assert that the created file is there
testutil.AssertFileExists(t, tmpdir, "test.conf")
}
}
func TestCreateKubeconfigFilesAndWrappers(t *testing.T) {
var tests = []struct {
createKubeConfigFunction func(outDir string, cfg *kubeadmapi.MasterConfiguration) error
expectedFiles []string
expectedError bool
}{
{ // Test createKubeConfigFiles fails for unknown kubeconfig is requested
createKubeConfigFunction: func(outDir string, cfg *kubeadmapi.MasterConfiguration) error {
return createKubeConfigFiles(outDir, cfg, "unknown.conf")
},
expectedError: true,
},
{ // Test CreateInitKubeConfigFiles (wrapper to createKubeConfigFile)
createKubeConfigFunction: CreateInitKubeConfigFiles,
expectedFiles: []string{
kubeadmconstants.AdminKubeConfigFileName,
kubeadmconstants.KubeletKubeConfigFileName,
kubeadmconstants.ControllerManagerKubeConfigFileName,
kubeadmconstants.SchedulerKubeConfigFileName,
},
},
{ // Test CreateAdminKubeConfigFile (wrapper to createKubeConfigFile)
createKubeConfigFunction: CreateAdminKubeConfigFile,
expectedFiles: []string{kubeadmconstants.AdminKubeConfigFileName},
},
{ // Test CreateKubeletKubeConfigFile (wrapper to createKubeConfigFile)
createKubeConfigFunction: CreateKubeletKubeConfigFile,
expectedFiles: []string{kubeadmconstants.KubeletKubeConfigFileName},
},
{ // Test CreateControllerManagerKubeConfigFile (wrapper to createKubeConfigFile)
createKubeConfigFunction: CreateControllerManagerKubeConfigFile,
expectedFiles: []string{kubeadmconstants.ControllerManagerKubeConfigFileName},
},
{ // Test createKubeConfigFile (wrapper to createKubeConfigFile)
createKubeConfigFunction: CreateSchedulerKubeConfigFile,
expectedFiles: []string{kubeadmconstants.SchedulerKubeConfigFileName},
},
}
for _, test := range tests {
// Create temp folder for the test case
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
// Adds a pki folder with a ca certs to the temp folder
pkidir := testutil.SetupPkiDirWithCertificateAuthorithy(t, tmpdir)
// Creates a Master Configuration pointing to the pkidir folder
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", BindPort: 1234},
CertificatesDir: pkidir,
}
// Execs the createKubeConfigFunction
err := test.createKubeConfigFunction(tmpdir, cfg)
if test.expectedError && err == nil {
t.Errorf("createKubeConfigFunction didn't failed when expected to fail")
continue
}
if !test.expectedError && err != nil {
t.Errorf("createKubeConfigFunction failed")
continue
}
// Assert expected files are there
testutil.AssertFileExists(t, tmpdir, test.expectedFiles...)
}
}
func TestWriteKubeConfigFailsIfCADoesntExists(t *testing.T) {
// Temporary folders for the test case (without a CA)
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
// Creates a Master Configuration pointing to the tmpdir folder
cfg := &kubeadmapi.MasterConfiguration{
CertificatesDir: tmpdir,
}
var tests = []struct {
writeKubeConfigFunction func(out io.Writer) error
}{
{ // Test WriteKubeConfigWithClientCert
writeKubeConfigFunction: func(out io.Writer) error {
return WriteKubeConfigWithClientCert(out, cfg, "myUser", []string{"myOrg"})
},
},
{ // Test WriteKubeConfigWithToken
writeKubeConfigFunction: func(out io.Writer) error {
return WriteKubeConfigWithToken(out, cfg, "myUser", "12345")
},
},
}
for _, test := range tests {
buf := new(bytes.Buffer)
// executes writeKubeConfigFunction
if err := test.writeKubeConfigFunction(buf); err == nil {
t.Error("writeKubeConfigFunction didnt failed when expected")
}
}
}
func TestWriteKubeConfig(t *testing.T) {
// Temporary folders for the test case
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
// Adds a pki folder with a ca cert to the temp folder
pkidir := testutil.SetupPkiDirWithCertificateAuthorithy(t, tmpdir)
// Retrieves ca cert for assertions
caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkidir, kubeadmconstants.CACertAndKeyBaseName)
if err != nil {
t.Fatalf("couldn't retrieve ca cert: %v", err)
}
// Creates a Master Configuration pointing to the pkidir folder
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", BindPort: 1234},
CertificatesDir: pkidir,
}
var tests = []struct {
writeKubeConfigFunction func(out io.Writer) error
withClientCert bool
withToken bool
}{
{ // Test WriteKubeConfigWithClientCert
writeKubeConfigFunction: func(out io.Writer) error {
return WriteKubeConfigWithClientCert(out, cfg, "myUser", []string{"myOrg"})
},
withClientCert: true,
},
{ // Test WriteKubeConfigWithToken
writeKubeConfigFunction: func(out io.Writer) error {
return WriteKubeConfigWithToken(out, cfg, "myUser", "12345")
},
withToken: true,
},
}
for _, test := range tests {
buf := new(bytes.Buffer)
// executes writeKubeConfigFunction
if err := test.writeKubeConfigFunction(buf); err != nil {
t.Error("writeKubeConfigFunction failed")
continue
}
// reads kubeconfig written to stdout
config, err := clientcmd.Load(buf.Bytes())
if err != nil {
t.Errorf("Couldn't read kubeconfig file from buffer: %v", err)
continue
}
// checks that CLI flags are properly propagated
kubeconfigtestutil.AssertKubeConfigCurrentCluster(t, config, "https://1.2.3.4:1234", caCert)
if test.withClientCert {
// checks that kubeconfig files have expected client cert
kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithClientCert(t, config, caCert, "myUser")
}
if test.withToken {
// checks that kubeconfig files have expected token
kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithToken(t, config, "myUser", "12345")
}
}
}
// setupdKubeConfigWithClientAuth is a test utility function that wraps buildKubeConfigFromSpec for building a KubeConfig object With ClientAuth
func setupdKubeConfigWithClientAuth(t *testing.T, caCert *x509.Certificate, caKey *rsa.PrivateKey, APIServer, clientName, clustername string, organizations ...string) *clientcmdapi.Config {
spec := &kubeConfigSpec{
CACert: caCert,
APIServer: APIServer,
ClientName: clientName,
ClientCertAuth: &clientCertAuth{
CAKey: caKey,
Organizations: organizations,
},
}
config, err := buildKubeConfigFromSpec(spec, clustername)
if err != nil {
t.Fatal("buildKubeConfigFromSpec failed!")
}
return config
}
// setupdKubeConfigWithClientAuth is a test utility function that wraps buildKubeConfigFromSpec for building a KubeConfig object With Token
func setupdKubeConfigWithTokenAuth(t *testing.T, caCert *x509.Certificate, APIServer, clientName, token, clustername string) *clientcmdapi.Config {
spec := &kubeConfigSpec{
CACert: caCert,
APIServer: APIServer,
ClientName: clientName,
TokenAuth: &tokenAuth{
Token: token,
},
}
config, err := buildKubeConfigFromSpec(spec, clustername)
if err != nil {
t.Fatal("buildKubeConfigFromSpec failed!")
}
return config
}

View File

@@ -0,0 +1,70 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"config.go",
"dynamic.go",
"flags.go",
],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet",
visibility = ["//visibility:public"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/features:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//pkg/apis/rbac/v1:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library",
"//pkg/util/node:go_default_library",
"//pkg/util/procfs:go_default_library",
"//pkg/util/version:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/rbac/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = [
"config_test.go",
"dynamic_test.go",
"flags_test.go",
],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//pkg/kubelet/apis:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library",
"//pkg/util/version:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/client-go/testing:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,180 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
"k8s.io/kubernetes/pkg/util/version"
)
// WriteConfigToDisk writes the kubelet config object down to a file
// Used at "kubeadm init" and "kubeadm upgrade" time
func WriteConfigToDisk(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration, kubeletDir string) error {
kubeletBytes, err := getConfigBytes(kubeletConfig)
if err != nil {
return err
}
return writeConfigBytesToDisk(kubeletBytes, kubeletDir)
}
// CreateConfigMap creates a ConfigMap with the generic kubelet configuration.
// Used at "kubeadm init" and "kubeadm upgrade" time
func CreateConfigMap(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
if err != nil {
return err
}
configMapName := configMapName(k8sVersion)
fmt.Printf("[kubelet] Creating a ConfigMap %q in namespace %s with the configuration for the kubelets in the cluster\n", configMapName, metav1.NamespaceSystem)
kubeletBytes, err := getConfigBytes(cfg.KubeletConfiguration.BaseConfig)
if err != nil {
return err
}
if err := apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: configMapName,
Namespace: metav1.NamespaceSystem,
},
Data: map[string]string{
kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(kubeletBytes),
},
}); err != nil {
return err
}
if err := createConfigMapRBACRules(client, k8sVersion); err != nil {
return fmt.Errorf("error creating kubelet configuration configmap RBAC rules: %v", err)
}
return nil
}
// createConfigMapRBACRules creates the RBAC rules for exposing the base kubelet ConfigMap in the kube-system namespace to unauthenticated users
func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Version) error {
if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{
ObjectMeta: metav1.ObjectMeta{
Name: configMapRBACName(k8sVersion),
Namespace: metav1.NamespaceSystem,
},
Rules: []rbac.PolicyRule{
rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(configMapName(k8sVersion)).RuleOrDie(),
},
}); err != nil {
return err
}
return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: configMapRBACName(k8sVersion),
Namespace: metav1.NamespaceSystem,
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "Role",
Name: configMapRBACName(k8sVersion),
},
Subjects: []rbac.Subject{
{
Kind: rbac.GroupKind,
Name: kubeadmconstants.NodesGroup,
},
{
Kind: rbac.GroupKind,
Name: kubeadmconstants.NodeBootstrapTokenAuthGroup,
},
},
})
}
// DownloadConfig downloads the kubelet configuration from a ConfigMap and writes it to disk.
// Used at "kubeadm join" time
func DownloadConfig(client clientset.Interface, kubeletVersion *version.Version, kubeletDir string) error {
// Download the ConfigMap from the cluster based on what version the kubelet is
configMapName := configMapName(kubeletVersion)
fmt.Printf("[kubelet] Downloading configuration for the kubelet from the %q ConfigMap in the %s namespace\n",
configMapName, metav1.NamespaceSystem)
kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{})
// If the ConfigMap wasn't found and the kubelet version is v1.10.x, where we didn't support the config file yet
// just return, don't error out
if apierrors.IsNotFound(err) && kubeletVersion.Minor() == 10 {
return nil
}
if err != nil {
return err
}
return writeConfigBytesToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey]), kubeletDir)
}
// configMapName returns the right ConfigMap name for the right branch of k8s
func configMapName(k8sVersion *version.Version) string {
return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigurationConfigMapPrefix, k8sVersion.Major(), k8sVersion.Minor())
}
// configMapRBACName returns the name for the Role/RoleBinding for the kubelet config configmap for the right branch of k8s
func configMapRBACName(k8sVersion *version.Version) string {
return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigMapRolePrefix, k8sVersion.Major(), k8sVersion.Minor())
}
// getConfigBytes marshals a kubeletconfiguration object to bytes
func getConfigBytes(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) ([]byte, error) {
_, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs()
if err != nil {
return []byte{}, err
}
return kubeadmutil.MarshalToYamlForCodecs(kubeletConfig, kubeletconfigv1beta1.SchemeGroupVersion, *kubeletCodecs)
}
// writeConfigBytesToDisk writes a byte slice down to disk at the specific location of the kubelet config file
func writeConfigBytesToDisk(b []byte, kubeletDir string) error {
configFile := filepath.Join(kubeletDir, kubeadmconstants.KubeletConfigurationFileName)
fmt.Printf("[kubelet] Writing kubelet configuration to file %q\n", configFile)
// creates target folder if not already exists
if err := os.MkdirAll(kubeletDir, 0700); err != nil {
return fmt.Errorf("failed to create directory %q: %v", kubeletDir, err)
}
if err := ioutil.WriteFile(configFile, b, 0644); err != nil {
return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", configFile, err)
}
return nil
}

View File

@@ -0,0 +1,78 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"testing"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
"k8s.io/kubernetes/pkg/util/version"
)
func TestCreateConfigMap(t *testing.T) {
nodeName := "fake-node"
client := fake.NewSimpleClientset()
cfg := &kubeadmapi.MasterConfiguration{
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: nodeName},
KubernetesVersion: "v1.11.0",
KubeletConfiguration: kubeadmapi.KubeletConfiguration{
BaseConfig: &kubeletconfigv1beta1.KubeletConfiguration{},
},
}
client.PrependReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) {
return true, &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: nodeName,
},
Spec: v1.NodeSpec{},
}, nil
})
client.PrependReactor("create", "roles", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
client.PrependReactor("create", "rolebindings", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
client.PrependReactor("create", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
if err := CreateConfigMap(cfg, client); err != nil {
t.Errorf("CreateConfigMap: unexpected error %v", err)
}
}
func TestCreateConfigMapRBACRules(t *testing.T) {
client := fake.NewSimpleClientset()
client.PrependReactor("create", "roles", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
client.PrependReactor("create", "rolebindings", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
if err := createConfigMapRBACRules(client, version.MustParseSemantic("v1.11.0")); err != nil {
t.Errorf("createConfigMapRBACRules: unexpected error %v", err)
}
}

View File

@@ -0,0 +1,61 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"fmt"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
clientset "k8s.io/client-go/kubernetes"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
"k8s.io/kubernetes/pkg/util/version"
)
// EnableDynamicConfigForNode updates the Node's ConfigSource to enable Dynamic Kubelet Configuration, depending on what version the kubelet is
// Used at "kubeadm init", "kubeadm join" and "kubeadm upgrade" time
// This func is ONLY run if the user enables the `DynamicKubeletConfig` feature gate, which is by default off
func EnableDynamicConfigForNode(client clientset.Interface, nodeName string, kubeletVersion *version.Version) error {
configMapName := configMapName(kubeletVersion)
fmt.Printf("[kubelet] Enabling Dynamic Kubelet Config for Node %q; config sourced from ConfigMap %q in namespace %s\n",
nodeName, configMapName, metav1.NamespaceSystem)
fmt.Println("[kubelet] WARNING: The Dynamic Kubelet Config feature is alpha and off by default. It hasn't been well-tested yet at this stage, use with caution.")
kubeletConfigMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("couldn't get the kubelet configuration ConfigMap: %v", err)
}
// Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned.
return apiclient.PatchNode(client, nodeName, func(n *v1.Node) {
patchNodeForDynamicConfig(n, configMapName, kubeletConfigMap.UID)
})
}
func patchNodeForDynamicConfig(n *v1.Node, configMapName string, configMapUID types.UID) {
n.Spec.ConfigSource = &v1.NodeConfigSource{
ConfigMap: &v1.ConfigMapNodeConfigSource{
Name: configMapName,
Namespace: metav1.NamespaceSystem,
UID: configMapUID,
KubeletConfigKey: kubeadmconstants.KubeletBaseConfigurationConfigMapKey,
},
}
}

View File

@@ -0,0 +1,65 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"testing"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
"k8s.io/kubernetes/pkg/util/version"
)
func TestEnableDynamicConfigForNode(t *testing.T) {
nodeName := "fake-node"
client := fake.NewSimpleClientset()
client.PrependReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) {
return true, &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: nodeName,
Labels: map[string]string{kubeletapis.LabelHostname: nodeName},
},
Spec: v1.NodeSpec{
ConfigSource: &v1.NodeConfigSource{
ConfigMap: &v1.ConfigMapNodeConfigSource{
UID: "",
},
},
},
}, nil
})
client.PrependReactor("get", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
return true, &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "kubelet-config-1.11",
Namespace: metav1.NamespaceSystem,
UID: "fake-uid",
},
}, nil
})
client.PrependReactor("patch", "nodes", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, nil
})
if err := EnableDynamicConfigForNode(client, nodeName, version.MustParseSemantic("v1.11.0")); err != nil {
t.Errorf("UpdateNodeWithConfigMap: unexpected error %v", err)
}
}

View File

@@ -0,0 +1,130 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/golang/glog"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
nodeutil "k8s.io/kubernetes/pkg/util/node"
"k8s.io/kubernetes/pkg/util/procfs"
utilsexec "k8s.io/utils/exec"
)
type kubeletFlagsOpts struct {
nodeRegOpts *kubeadmapi.NodeRegistrationOptions
featureGates map[string]bool
registerTaintsUsingFlags bool
execer utilsexec.Interface
pidOfFunc func(string) ([]int, error)
defaultHostname string
}
// WriteKubeletDynamicEnvFile writes a environment file with dynamic flags to the kubelet.
// Used at "kubeadm init" and "kubeadm join" time.
func WriteKubeletDynamicEnvFile(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, featureGates map[string]bool, registerTaintsUsingFlags bool, kubeletDir string) error {
flagOpts := kubeletFlagsOpts{
nodeRegOpts: nodeRegOpts,
featureGates: featureGates,
registerTaintsUsingFlags: registerTaintsUsingFlags,
execer: utilsexec.New(),
pidOfFunc: procfs.PidOf,
defaultHostname: nodeutil.GetHostname(""),
}
stringMap := buildKubeletArgMap(flagOpts)
argList := kubeadmutil.BuildArgumentListFromMap(stringMap, nodeRegOpts.KubeletExtraArgs)
envFileContent := fmt.Sprintf("%s=%s\n", constants.KubeletEnvFileVariableName, strings.Join(argList, " "))
return writeKubeletFlagBytesToDisk([]byte(envFileContent), kubeletDir)
}
// buildKubeletArgMap takes a MasterConfiguration object and builds based on that a string-string map with flags
// that should be given to the local kubelet daemon.
func buildKubeletArgMap(opts kubeletFlagsOpts) map[string]string {
kubeletFlags := map[string]string{}
if opts.nodeRegOpts.CRISocket == kubeadmapiv1alpha2.DefaultCRISocket {
// These flags should only be set when running docker
kubeletFlags["network-plugin"] = "cni"
kubeletFlags["cni-conf-dir"] = "/etc/cni/net.d"
kubeletFlags["cni-bin-dir"] = "/opt/cni/bin"
driver, err := kubeadmutil.GetCgroupDriverDocker(opts.execer)
if err != nil {
glog.Warningf("cannot automatically assign a '--cgroup-driver' value when starting the Kubelet: %v\n", err)
} else {
kubeletFlags["cgroup-driver"] = driver
}
} else {
kubeletFlags["container-runtime"] = "remote"
kubeletFlags["container-runtime-endpoint"] = opts.nodeRegOpts.CRISocket
}
if opts.registerTaintsUsingFlags && opts.nodeRegOpts.Taints != nil && len(opts.nodeRegOpts.Taints) > 0 {
taintStrs := []string{}
for _, taint := range opts.nodeRegOpts.Taints {
taintStrs = append(taintStrs, taint.ToString())
}
kubeletFlags["register-with-taints"] = strings.Join(taintStrs, ",")
}
if pids, _ := opts.pidOfFunc("systemd-resolved"); len(pids) > 0 {
// procfs.PidOf only returns an error if the regex is empty or doesn't compile, so we can ignore it
kubeletFlags["resolv-conf"] = "/run/systemd/resolve/resolv.conf"
}
// Make sure the node name we're passed will work with Kubelet
if opts.nodeRegOpts.Name != "" && opts.nodeRegOpts.Name != opts.defaultHostname {
glog.V(1).Info("setting kubelet hostname-override to %q", opts.nodeRegOpts.Name)
kubeletFlags["hostname-override"] = opts.nodeRegOpts.Name
}
// If the user enabled Dynamic Kubelet Configuration (which is disabled by default), set the directory
// in the CLI flags so that the feature actually gets enabled
if features.Enabled(opts.featureGates, features.DynamicKubeletConfig) {
kubeletFlags["dynamic-config-dir"] = filepath.Join(constants.KubeletRunDirectory, constants.DynamicKubeletConfigurationDirectoryName)
}
// TODO: Conditionally set `--cgroup-driver` to either `systemd` or `cgroupfs` for CRI other than Docker
return kubeletFlags
}
// writeKubeletFlagBytesToDisk writes a byte slice down to disk at the specific location of the kubelet flag overrides file
func writeKubeletFlagBytesToDisk(b []byte, kubeletDir string) error {
kubeletEnvFilePath := filepath.Join(kubeletDir, constants.KubeletEnvFileName)
fmt.Printf("[kubelet] Writing kubelet environment file with flags to file %q\n", kubeletEnvFilePath)
// creates target folder if not already exists
if err := os.MkdirAll(kubeletDir, 0700); err != nil {
return fmt.Errorf("failed to create directory %q: %v", kubeletDir, err)
}
if err := ioutil.WriteFile(kubeletEnvFilePath, b, 0644); err != nil {
return fmt.Errorf("failed to write kubelet configuration to the file %q: %v", kubeletEnvFilePath, err)
}
return nil
}

View File

@@ -0,0 +1,277 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"context"
"errors"
"fmt"
"io"
"reflect"
"strings"
"testing"
"k8s.io/api/core/v1"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/utils/exec"
)
type fakeCmd struct {
b []byte
err error
}
func (f fakeCmd) Run() error { return f.err }
func (f fakeCmd) CombinedOutput() ([]byte, error) { return f.b, f.err }
func (f fakeCmd) Output() ([]byte, error) { return f.b, f.err }
func (f fakeCmd) SetDir(dir string) {}
func (f fakeCmd) SetStdin(in io.Reader) {}
func (f fakeCmd) SetStdout(out io.Writer) {}
func (f fakeCmd) SetStderr(out io.Writer) {}
func (f fakeCmd) Stop() {}
type fakeExecer struct {
ioMap map[string]fakeCmd
}
func (f fakeExecer) Command(cmd string, args ...string) exec.Cmd {
cmds := []string{cmd}
cmds = append(cmds, args...)
return f.ioMap[strings.Join(cmds, " ")]
}
func (f fakeExecer) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
return f.Command(cmd, args...)
}
func (f fakeExecer) LookPath(file string) (string, error) { return "", errors.New("unknown binary") }
var (
systemdCgroupExecer = fakeExecer{
ioMap: map[string]fakeCmd{
"docker info": {
b: []byte(`Cgroup Driver: systemd`),
},
},
}
cgroupfsCgroupExecer = fakeExecer{
ioMap: map[string]fakeCmd{
"docker info": {
b: []byte(`Cgroup Driver: cgroupfs`),
},
},
}
errCgroupExecer = fakeExecer{
ioMap: map[string]fakeCmd{
"docker info": {
err: fmt.Errorf("no such binary: docker"),
},
},
}
)
func binaryRunningPidOfFunc(_ string) ([]int, error) {
return []int{1, 2, 3}, nil
}
func binaryNotRunningPidOfFunc(_ string) ([]int, error) {
return []int{}, nil
}
func TestBuildKubeletArgMap(t *testing.T) {
tests := []struct {
name string
opts kubeletFlagsOpts
expected map[string]string
}{
{
name: "the simplest case",
opts: kubeletFlagsOpts{
nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{
CRISocket: "/var/run/dockershim.sock",
Name: "foo",
Taints: []v1.Taint{ // This should be ignored as registerTaintsUsingFlags is false
{
Key: "foo",
Value: "bar",
Effect: "baz",
},
},
},
execer: errCgroupExecer,
pidOfFunc: binaryNotRunningPidOfFunc,
defaultHostname: "foo",
},
expected: map[string]string{
"network-plugin": "cni",
"cni-conf-dir": "/etc/cni/net.d",
"cni-bin-dir": "/opt/cni/bin",
},
},
{
name: "nodeRegOpts.Name != default hostname",
opts: kubeletFlagsOpts{
nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{
CRISocket: "/var/run/dockershim.sock",
Name: "override-name",
},
execer: errCgroupExecer,
pidOfFunc: binaryNotRunningPidOfFunc,
defaultHostname: "default",
},
expected: map[string]string{
"network-plugin": "cni",
"cni-conf-dir": "/etc/cni/net.d",
"cni-bin-dir": "/opt/cni/bin",
"hostname-override": "override-name",
},
},
{
name: "systemd cgroup driver",
opts: kubeletFlagsOpts{
nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{
CRISocket: "/var/run/dockershim.sock",
Name: "foo",
},
execer: systemdCgroupExecer,
pidOfFunc: binaryNotRunningPidOfFunc,
defaultHostname: "foo",
},
expected: map[string]string{
"network-plugin": "cni",
"cni-conf-dir": "/etc/cni/net.d",
"cni-bin-dir": "/opt/cni/bin",
"cgroup-driver": "systemd",
},
},
{
name: "cgroupfs cgroup driver",
opts: kubeletFlagsOpts{
nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{
CRISocket: "/var/run/dockershim.sock",
Name: "foo",
},
execer: cgroupfsCgroupExecer,
pidOfFunc: binaryNotRunningPidOfFunc,
defaultHostname: "foo",
},
expected: map[string]string{
"network-plugin": "cni",
"cni-conf-dir": "/etc/cni/net.d",
"cni-bin-dir": "/opt/cni/bin",
"cgroup-driver": "cgroupfs",
},
},
{
name: "external CRI runtime",
opts: kubeletFlagsOpts{
nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{
CRISocket: "/var/run/containerd.sock",
Name: "foo",
},
execer: cgroupfsCgroupExecer,
pidOfFunc: binaryNotRunningPidOfFunc,
defaultHostname: "foo",
},
expected: map[string]string{
"container-runtime": "remote",
"container-runtime-endpoint": "/var/run/containerd.sock",
},
},
{
name: "register with taints",
opts: kubeletFlagsOpts{
nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{
CRISocket: "/var/run/containerd.sock",
Name: "foo",
Taints: []v1.Taint{
{
Key: "foo",
Value: "bar",
Effect: "baz",
},
{
Key: "key",
Value: "val",
Effect: "eff",
},
},
},
registerTaintsUsingFlags: true,
execer: cgroupfsCgroupExecer,
pidOfFunc: binaryNotRunningPidOfFunc,
defaultHostname: "foo",
},
expected: map[string]string{
"container-runtime": "remote",
"container-runtime-endpoint": "/var/run/containerd.sock",
"register-with-taints": "foo=bar:baz,key=val:eff",
},
},
{
name: "systemd-resolved running",
opts: kubeletFlagsOpts{
nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{
CRISocket: "/var/run/containerd.sock",
Name: "foo",
},
execer: cgroupfsCgroupExecer,
pidOfFunc: binaryRunningPidOfFunc,
defaultHostname: "foo",
},
expected: map[string]string{
"container-runtime": "remote",
"container-runtime-endpoint": "/var/run/containerd.sock",
"resolv-conf": "/run/systemd/resolve/resolv.conf",
},
},
{
name: "dynamic kubelet config enabled",
opts: kubeletFlagsOpts{
nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{
CRISocket: "/var/run/containerd.sock",
Name: "foo",
},
featureGates: map[string]bool{
"DynamicKubeletConfig": true,
},
execer: cgroupfsCgroupExecer,
pidOfFunc: binaryNotRunningPidOfFunc,
defaultHostname: "foo",
},
expected: map[string]string{
"container-runtime": "remote",
"container-runtime-endpoint": "/var/run/containerd.sock",
"dynamic-config-dir": "/var/lib/kubelet/dynamic-config",
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
actual := buildKubeletArgMap(test.opts)
if !reflect.DeepEqual(actual, test.expected) {
t.Errorf(
"failed buildKubeletArgMap:\n\texpected: %v\n\t actual: %v",
test.expected,
actual,
)
}
})
}
}

View File

@@ -0,0 +1,47 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = ["markmaster_test.go"],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/constants:go_default_library",
"//pkg/kubelet/apis:go_default_library",
"//pkg/util/node:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/rest:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = ["markmaster.go"],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/markmaster",
deps = [
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View File

@@ -0,0 +1,66 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package markmaster
import (
"fmt"
"k8s.io/api/core/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
)
// MarkMaster taints the master and sets the master label
func MarkMaster(client clientset.Interface, masterName string, taints []v1.Taint) error {
fmt.Printf("[markmaster] Marking the node %s as master by adding the label \"%s=''\"\n", masterName, constants.LabelNodeRoleMaster)
if taints != nil && len(taints) > 0 {
taintStrs := []string{}
for _, taint := range taints {
taintStrs = append(taintStrs, taint.ToString())
}
fmt.Printf("[markmaster] Marking the node %s as master by adding the taints %v\n", masterName, taintStrs)
}
return apiclient.PatchNode(client, masterName, func(n *v1.Node) {
markMasterNode(n, taints)
})
}
func taintExists(taint v1.Taint, taints []v1.Taint) bool {
for _, t := range taints {
if t == taint {
return true
}
}
return false
}
func markMasterNode(n *v1.Node, taints []v1.Taint) {
n.ObjectMeta.Labels[constants.LabelNodeRoleMaster] = ""
for _, nt := range n.Spec.Taints {
if !taintExists(nt, taints) {
taints = append(taints, nt)
}
}
n.Spec.Taints = taints
}

View File

@@ -0,0 +1,178 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package markmaster
import (
"bytes"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"testing"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
"k8s.io/kubernetes/pkg/util/node"
)
func TestMarkMaster(t *testing.T) {
// Note: this test takes advantage of the deterministic marshalling of
// JSON provided by strategicpatch so that "expectedPatch" can use a
// string equality test instead of a logical JSON equality test. That
// will need to change if strategicpatch's behavior changes in the
// future.
tests := []struct {
name string
existingLabel string
existingTaints []v1.Taint
newTaints []v1.Taint
expectedPatch string
}{
{
"master label and taint missing",
"",
nil,
[]v1.Taint{kubeadmconstants.MasterTaint},
"{\"metadata\":{\"labels\":{\"node-role.kubernetes.io/master\":\"\"}},\"spec\":{\"taints\":[{\"effect\":\"NoSchedule\",\"key\":\"node-role.kubernetes.io/master\"}]}}",
},
{
"master label and taint missing but taint not wanted",
"",
nil,
nil,
"{\"metadata\":{\"labels\":{\"node-role.kubernetes.io/master\":\"\"}}}",
},
{
"master label missing",
"",
[]v1.Taint{kubeadmconstants.MasterTaint},
[]v1.Taint{kubeadmconstants.MasterTaint},
"{\"metadata\":{\"labels\":{\"node-role.kubernetes.io/master\":\"\"}}}",
},
{
"master taint missing",
kubeadmconstants.LabelNodeRoleMaster,
nil,
[]v1.Taint{kubeadmconstants.MasterTaint},
"{\"spec\":{\"taints\":[{\"effect\":\"NoSchedule\",\"key\":\"node-role.kubernetes.io/master\"}]}}",
},
{
"nothing missing",
kubeadmconstants.LabelNodeRoleMaster,
[]v1.Taint{kubeadmconstants.MasterTaint},
[]v1.Taint{kubeadmconstants.MasterTaint},
"{}",
},
{
"has taint and no new taints wanted",
kubeadmconstants.LabelNodeRoleMaster,
[]v1.Taint{
{
Key: "node.cloudprovider.kubernetes.io/uninitialized",
Effect: v1.TaintEffectNoSchedule,
},
},
nil,
"{}",
},
{
"has taint and should merge with wanted taint",
kubeadmconstants.LabelNodeRoleMaster,
[]v1.Taint{
{
Key: "node.cloudprovider.kubernetes.io/uninitialized",
Effect: v1.TaintEffectNoSchedule,
},
},
[]v1.Taint{kubeadmconstants.MasterTaint},
"{\"spec\":{\"taints\":[{\"effect\":\"NoSchedule\",\"key\":\"node-role.kubernetes.io/master\"},{\"effect\":\"NoSchedule\",\"key\":\"node.cloudprovider.kubernetes.io/uninitialized\"}]}}",
},
}
for _, tc := range tests {
hostname := node.GetHostname("")
masterNode := &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: hostname,
Labels: map[string]string{
kubeletapis.LabelHostname: hostname,
},
},
}
if tc.existingLabel != "" {
masterNode.ObjectMeta.Labels[tc.existingLabel] = ""
}
if tc.existingTaints != nil {
masterNode.Spec.Taints = tc.existingTaints
}
jsonNode, err := json.Marshal(masterNode)
if err != nil {
t.Fatalf("MarkMaster(%s): unexpected encoding error: %v", tc.name, err)
}
var patchRequest string
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.Header().Set("Content-Type", "application/json")
if req.URL.Path != "/api/v1/nodes/"+hostname {
t.Errorf("MarkMaster(%s): request for unexpected HTTP resource: %v", tc.name, req.URL.Path)
w.WriteHeader(http.StatusNotFound)
return
}
switch req.Method {
case "GET":
case "PATCH":
patchRequest = toString(req.Body)
default:
t.Errorf("MarkMaster(%s): request for unexpected HTTP verb: %v", tc.name, req.Method)
w.WriteHeader(http.StatusNotFound)
return
}
w.WriteHeader(http.StatusOK)
w.Write(jsonNode)
}))
defer s.Close()
cs, err := clientset.NewForConfig(&restclient.Config{Host: s.URL})
if err != nil {
t.Fatalf("MarkMaster(%s): unexpected error building clientset: %v", tc.name, err)
}
if err := MarkMaster(cs, hostname, tc.newTaints); err != nil {
t.Errorf("MarkMaster(%s) returned unexpected error: %v", tc.name, err)
}
if tc.expectedPatch != patchRequest {
t.Errorf("MarkMaster(%s) wanted patch %v, got %v", tc.name, tc.expectedPatch, patchRequest)
}
}
}
func toString(r io.Reader) string {
buf := new(bytes.Buffer)
buf.ReadFrom(r)
return buf.String()
}

View File

@@ -0,0 +1,28 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["patchnode.go"],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode",
visibility = ["//visibility:public"],
deps = [
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,40 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package patchnode
import (
"fmt"
"k8s.io/api/core/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
)
// AnnotateCRISocket annotates the node with the given crisocket
func AnnotateCRISocket(client clientset.Interface, nodeName string, criSocket string) error {
fmt.Printf("[patchnode] Uploading the CRI Socket information %q to the Node API object %q as an annotation\n", criSocket, nodeName)
return apiclient.PatchNode(client, nodeName, func(n *v1.Node) {
annotateNodeWithCRISocket(n, criSocket)
})
}
func annotateNodeWithCRISocket(n *v1.Node, criSocket string) {
n.ObjectMeta.Annotations[constants.AnnotationKubeadmCRISocket] = criSocket
}

View File

@@ -0,0 +1,60 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = [
"podspec_mutation_test.go",
"selfhosting_test.go",
"selfhosting_volumes_test.go",
],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//vendor/k8s.io/api/apps/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"podspec_mutation.go",
"selfhosting.go",
"selfhosting_volumes.go",
],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/features:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/apps/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View File

@@ -0,0 +1,191 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package selfhosting
import (
"path/filepath"
"strings"
"k8s.io/api/core/v1"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
)
const (
// selfHostedKubeConfigDir sets the directory where kubeconfig files for the scheduler and controller-manager should be mounted
// Due to how the projected volume mount works (can only be a full directory, not mount individual files), we must change this from
// the default as mounts cannot be nested (/etc/kubernetes would override /etc/kubernetes/pki)
selfHostedKubeConfigDir = "/etc/kubernetes/kubeconfig"
)
// PodSpecMutatorFunc is a function capable of mutating a PodSpec
type PodSpecMutatorFunc func(*v1.PodSpec)
// GetDefaultMutators gets the mutator functions that always should be used
func GetDefaultMutators() map[string][]PodSpecMutatorFunc {
return map[string][]PodSpecMutatorFunc{
kubeadmconstants.KubeAPIServer: {
addNodeSelectorToPodSpec,
setMasterTolerationOnPodSpec,
setRightDNSPolicyOnPodSpec,
setHostIPOnPodSpec,
},
kubeadmconstants.KubeControllerManager: {
addNodeSelectorToPodSpec,
setMasterTolerationOnPodSpec,
setRightDNSPolicyOnPodSpec,
},
kubeadmconstants.KubeScheduler: {
addNodeSelectorToPodSpec,
setMasterTolerationOnPodSpec,
setRightDNSPolicyOnPodSpec,
},
}
}
// GetMutatorsFromFeatureGates returns all mutators needed based on the feature gates passed
func GetMutatorsFromFeatureGates(featureGates map[string]bool) map[string][]PodSpecMutatorFunc {
// Here the map of different mutators to use for the control plane's podspec is stored
mutators := GetDefaultMutators()
// Some extra work to be done if we should store the control plane certificates in Secrets
if features.Enabled(featureGates, features.StoreCertsInSecrets) {
// Add the store-certs-in-secrets-specific mutators here so that the self-hosted component starts using them
mutators[kubeadmconstants.KubeAPIServer] = append(mutators[kubeadmconstants.KubeAPIServer], setSelfHostedVolumesForAPIServer)
mutators[kubeadmconstants.KubeControllerManager] = append(mutators[kubeadmconstants.KubeControllerManager], setSelfHostedVolumesForControllerManager)
mutators[kubeadmconstants.KubeScheduler] = append(mutators[kubeadmconstants.KubeScheduler], setSelfHostedVolumesForScheduler)
}
return mutators
}
// mutatePodSpec makes a Static Pod-hosted PodSpec suitable for self-hosting
func mutatePodSpec(mutators map[string][]PodSpecMutatorFunc, name string, podSpec *v1.PodSpec) {
// Get the mutator functions for the component in question, then loop through and execute them
mutatorsForComponent := mutators[name]
for _, mutateFunc := range mutatorsForComponent {
mutateFunc(podSpec)
}
}
// addNodeSelectorToPodSpec makes Pod require to be scheduled on a node marked with the master label
func addNodeSelectorToPodSpec(podSpec *v1.PodSpec) {
if podSpec.NodeSelector == nil {
podSpec.NodeSelector = map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""}
return
}
podSpec.NodeSelector[kubeadmconstants.LabelNodeRoleMaster] = ""
}
// setMasterTolerationOnPodSpec makes the Pod tolerate the master taint
func setMasterTolerationOnPodSpec(podSpec *v1.PodSpec) {
if podSpec.Tolerations == nil {
podSpec.Tolerations = []v1.Toleration{kubeadmconstants.MasterToleration}
return
}
podSpec.Tolerations = append(podSpec.Tolerations, kubeadmconstants.MasterToleration)
}
// setHostIPOnPodSpec sets the environment variable HOST_IP using downward API
func setHostIPOnPodSpec(podSpec *v1.PodSpec) {
envVar := v1.EnvVar{
Name: "HOST_IP",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "status.hostIP",
},
},
}
podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, envVar)
for i := range podSpec.Containers[0].Command {
if strings.Contains(podSpec.Containers[0].Command[i], "advertise-address") {
podSpec.Containers[0].Command[i] = "--advertise-address=$(HOST_IP)"
}
}
}
// setRightDNSPolicyOnPodSpec makes sure the self-hosted components can look up things via kube-dns if necessary
func setRightDNSPolicyOnPodSpec(podSpec *v1.PodSpec) {
podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
}
// setSelfHostedVolumesForAPIServer makes sure the self-hosted api server has the right volume source coming from a self-hosted cluster
func setSelfHostedVolumesForAPIServer(podSpec *v1.PodSpec) {
for i, v := range podSpec.Volumes {
// If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted
if v.Name == kubeadmconstants.KubeCertificatesVolumeName {
podSpec.Volumes[i].VolumeSource = apiServerCertificatesVolumeSource()
}
}
}
// setSelfHostedVolumesForControllerManager makes sure the self-hosted controller manager has the right volume source coming from a self-hosted cluster
func setSelfHostedVolumesForControllerManager(podSpec *v1.PodSpec) {
for i, v := range podSpec.Volumes {
// If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted
if v.Name == kubeadmconstants.KubeCertificatesVolumeName {
podSpec.Volumes[i].VolumeSource = controllerManagerCertificatesVolumeSource()
} else if v.Name == kubeadmconstants.KubeConfigVolumeName {
podSpec.Volumes[i].VolumeSource = kubeConfigVolumeSource(kubeadmconstants.ControllerManagerKubeConfigFileName)
}
}
// Change directory for the kubeconfig directory to selfHostedKubeConfigDir
for i, vm := range podSpec.Containers[0].VolumeMounts {
if vm.Name == kubeadmconstants.KubeConfigVolumeName {
podSpec.Containers[0].VolumeMounts[i].MountPath = selfHostedKubeConfigDir
}
}
// Rewrite the --kubeconfig path as the volume mount path may not overlap with certs dir, which it does by default (/etc/kubernetes and /etc/kubernetes/pki)
// This is not a problem with hostPath mounts as hostPath supports mounting one file only, instead of always a full directory. Secrets and Projected Volumes
// don't support that.
podSpec.Containers[0].Command = kubeadmutil.ReplaceArgument(podSpec.Containers[0].Command, func(argMap map[string]string) map[string]string {
argMap["kubeconfig"] = filepath.Join(selfHostedKubeConfigDir, kubeadmconstants.ControllerManagerKubeConfigFileName)
return argMap
})
}
// setSelfHostedVolumesForScheduler makes sure the self-hosted scheduler has the right volume source coming from a self-hosted cluster
func setSelfHostedVolumesForScheduler(podSpec *v1.PodSpec) {
for i, v := range podSpec.Volumes {
// If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted
if v.Name == kubeadmconstants.KubeConfigVolumeName {
podSpec.Volumes[i].VolumeSource = kubeConfigVolumeSource(kubeadmconstants.SchedulerKubeConfigFileName)
}
}
// Change directory for the kubeconfig directory to selfHostedKubeConfigDir
for i, vm := range podSpec.Containers[0].VolumeMounts {
if vm.Name == kubeadmconstants.KubeConfigVolumeName {
podSpec.Containers[0].VolumeMounts[i].MountPath = selfHostedKubeConfigDir
}
}
// Rewrite the --kubeconfig path as the volume mount path may not overlap with certs dir, which it does by default (/etc/kubernetes and /etc/kubernetes/pki)
// This is not a problem with hostPath mounts as hostPath supports mounting one file only, instead of always a full directory. Secrets and Projected Volumes
// don't support that.
podSpec.Containers[0].Command = kubeadmutil.ReplaceArgument(podSpec.Containers[0].Command, func(argMap map[string]string) map[string]string {
argMap["kubeconfig"] = filepath.Join(selfHostedKubeConfigDir, kubeadmconstants.SchedulerKubeConfigFileName)
return argMap
})
}

View File

@@ -0,0 +1,545 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package selfhosting
import (
"reflect"
"sort"
"testing"
"k8s.io/api/core/v1"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
)
func TestMutatePodSpec(t *testing.T) {
var tests = []struct {
component string
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
component: kubeadmconstants.KubeAPIServer,
podSpec: &v1.PodSpec{
Containers: []v1.Container{
{
Name: "kube-apiserver",
Command: []string{
"--advertise-address=10.0.0.1",
},
},
},
},
expected: v1.PodSpec{
Containers: []v1.Container{
{
Name: "kube-apiserver",
Command: []string{
"--advertise-address=$(HOST_IP)",
},
Env: []v1.EnvVar{
{
Name: "HOST_IP",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "status.hostIP",
},
},
},
},
},
},
NodeSelector: map[string]string{
kubeadmconstants.LabelNodeRoleMaster: "",
},
Tolerations: []v1.Toleration{
kubeadmconstants.MasterToleration,
},
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
{
component: kubeadmconstants.KubeControllerManager,
podSpec: &v1.PodSpec{},
expected: v1.PodSpec{
NodeSelector: map[string]string{
kubeadmconstants.LabelNodeRoleMaster: "",
},
Tolerations: []v1.Toleration{
kubeadmconstants.MasterToleration,
},
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
{
component: kubeadmconstants.KubeScheduler,
podSpec: &v1.PodSpec{},
expected: v1.PodSpec{
NodeSelector: map[string]string{
kubeadmconstants.LabelNodeRoleMaster: "",
},
Tolerations: []v1.Toleration{
kubeadmconstants.MasterToleration,
},
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
}
for _, rt := range tests {
mutatePodSpec(GetDefaultMutators(), rt.component, rt.podSpec)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed mutatePodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
}
}
func TestAddNodeSelectorToPodSpec(t *testing.T) {
var tests = []struct {
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
podSpec: &v1.PodSpec{},
expected: v1.PodSpec{
NodeSelector: map[string]string{
kubeadmconstants.LabelNodeRoleMaster: "",
},
},
},
{
podSpec: &v1.PodSpec{
NodeSelector: map[string]string{
"foo": "bar",
},
},
expected: v1.PodSpec{
NodeSelector: map[string]string{
"foo": "bar",
kubeadmconstants.LabelNodeRoleMaster: "",
},
},
},
}
for _, rt := range tests {
addNodeSelectorToPodSpec(rt.podSpec)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed addNodeSelectorToPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
}
}
func TestSetMasterTolerationOnPodSpec(t *testing.T) {
var tests = []struct {
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
podSpec: &v1.PodSpec{},
expected: v1.PodSpec{
Tolerations: []v1.Toleration{
kubeadmconstants.MasterToleration,
},
},
},
{
podSpec: &v1.PodSpec{
Tolerations: []v1.Toleration{
{Key: "foo", Value: "bar"},
},
},
expected: v1.PodSpec{
Tolerations: []v1.Toleration{
{Key: "foo", Value: "bar"},
kubeadmconstants.MasterToleration,
},
},
},
}
for _, rt := range tests {
setMasterTolerationOnPodSpec(rt.podSpec)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed setMasterTolerationOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
}
}
func TestSetRightDNSPolicyOnPodSpec(t *testing.T) {
var tests = []struct {
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
podSpec: &v1.PodSpec{},
expected: v1.PodSpec{
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
{
podSpec: &v1.PodSpec{
DNSPolicy: v1.DNSClusterFirst,
},
expected: v1.PodSpec{
DNSPolicy: v1.DNSClusterFirstWithHostNet,
},
},
}
for _, rt := range tests {
setRightDNSPolicyOnPodSpec(rt.podSpec)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed setRightDNSPolicyOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
}
}
func TestSetHostIPOnPodSpec(t *testing.T) {
var tests = []struct {
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
podSpec: &v1.PodSpec{
Containers: []v1.Container{
{
Name: "kube-apiserver",
Command: []string{
"--advertise-address=10.0.0.1",
},
Env: []v1.EnvVar{},
},
},
},
expected: v1.PodSpec{
Containers: []v1.Container{
{
Name: "kube-apiserver",
Command: []string{
"--advertise-address=$(HOST_IP)",
},
Env: []v1.EnvVar{
{
Name: "HOST_IP",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "status.hostIP",
},
},
},
},
},
},
},
},
}
for _, rt := range tests {
setHostIPOnPodSpec(rt.podSpec)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed setHostIPOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
}
}
func TestSetSelfHostedVolumesForAPIServer(t *testing.T) {
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
var tests = []struct {
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
podSpec: &v1.PodSpec{
Containers: []v1.Container{
{
VolumeMounts: []v1.VolumeMount{
{
Name: "ca-certs",
MountPath: "/etc/ssl/certs",
},
{
Name: "k8s-certs",
MountPath: "/etc/kubernetes/pki",
},
},
Command: []string{
"--foo=bar",
},
},
},
Volumes: []v1.Volume{
{
Name: "ca-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/ssl/certs",
Type: &hostPathDirectoryOrCreate,
},
},
},
{
Name: "k8s-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/kubernetes/pki",
Type: &hostPathDirectoryOrCreate,
},
},
},
},
},
expected: v1.PodSpec{
Containers: []v1.Container{
{
VolumeMounts: []v1.VolumeMount{
{
Name: "ca-certs",
MountPath: "/etc/ssl/certs",
},
{
Name: "k8s-certs",
MountPath: "/etc/kubernetes/pki",
},
},
Command: []string{
"--foo=bar",
},
},
},
Volumes: []v1.Volume{
{
Name: "ca-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/ssl/certs",
Type: &hostPathDirectoryOrCreate,
},
},
},
{
Name: "k8s-certs",
VolumeSource: apiServerCertificatesVolumeSource(),
},
},
},
},
}
for _, rt := range tests {
setSelfHostedVolumesForAPIServer(rt.podSpec)
sort.Strings(rt.podSpec.Containers[0].Command)
sort.Strings(rt.expected.Containers[0].Command)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed setSelfHostedVolumesForAPIServer:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
}
}
func TestSetSelfHostedVolumesForControllerManager(t *testing.T) {
hostPathFileOrCreate := v1.HostPathFileOrCreate
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
var tests = []struct {
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
podSpec: &v1.PodSpec{
Containers: []v1.Container{
{
VolumeMounts: []v1.VolumeMount{
{
Name: "ca-certs",
MountPath: "/etc/ssl/certs",
},
{
Name: "k8s-certs",
MountPath: "/etc/kubernetes/pki",
},
{
Name: "kubeconfig",
MountPath: "/etc/kubernetes/controller-manager.conf",
},
},
Command: []string{
"--kubeconfig=/etc/kubernetes/controller-manager.conf",
"--foo=bar",
},
},
},
Volumes: []v1.Volume{
{
Name: "ca-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/ssl/certs",
Type: &hostPathDirectoryOrCreate,
},
},
},
{
Name: "k8s-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/kubernetes/pki",
Type: &hostPathDirectoryOrCreate,
},
},
},
{
Name: "kubeconfig",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/kubernetes/controller-manager.conf",
Type: &hostPathFileOrCreate,
},
},
},
},
},
expected: v1.PodSpec{
Containers: []v1.Container{
{
VolumeMounts: []v1.VolumeMount{
{
Name: "ca-certs",
MountPath: "/etc/ssl/certs",
},
{
Name: "k8s-certs",
MountPath: "/etc/kubernetes/pki",
},
{
Name: "kubeconfig",
MountPath: "/etc/kubernetes/kubeconfig",
},
},
Command: []string{
"--kubeconfig=/etc/kubernetes/kubeconfig/controller-manager.conf",
"--foo=bar",
},
},
},
Volumes: []v1.Volume{
{
Name: "ca-certs",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/ssl/certs",
Type: &hostPathDirectoryOrCreate,
},
},
},
{
Name: "k8s-certs",
VolumeSource: controllerManagerCertificatesVolumeSource(),
},
{
Name: "kubeconfig",
VolumeSource: kubeConfigVolumeSource(kubeadmconstants.ControllerManagerKubeConfigFileName),
},
},
},
},
}
for _, rt := range tests {
setSelfHostedVolumesForControllerManager(rt.podSpec)
sort.Strings(rt.podSpec.Containers[0].Command)
sort.Strings(rt.expected.Containers[0].Command)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed setSelfHostedVolumesForControllerManager:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
}
}
func TestSetSelfHostedVolumesForScheduler(t *testing.T) {
hostPathFileOrCreate := v1.HostPathFileOrCreate
var tests = []struct {
podSpec *v1.PodSpec
expected v1.PodSpec
}{
{
podSpec: &v1.PodSpec{
Containers: []v1.Container{
{
VolumeMounts: []v1.VolumeMount{
{
Name: "kubeconfig",
MountPath: "/etc/kubernetes/scheduler.conf",
},
},
Command: []string{
"--kubeconfig=/etc/kubernetes/scheduler.conf",
"--foo=bar",
},
},
},
Volumes: []v1.Volume{
{
Name: "kubeconfig",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/etc/kubernetes/scheduler.conf",
Type: &hostPathFileOrCreate,
},
},
},
},
},
expected: v1.PodSpec{
Containers: []v1.Container{
{
VolumeMounts: []v1.VolumeMount{
{
Name: "kubeconfig",
MountPath: "/etc/kubernetes/kubeconfig",
},
},
Command: []string{
"--kubeconfig=/etc/kubernetes/kubeconfig/scheduler.conf",
"--foo=bar",
},
},
},
Volumes: []v1.Volume{
{
Name: "kubeconfig",
VolumeSource: kubeConfigVolumeSource(kubeadmconstants.SchedulerKubeConfigFileName),
},
},
},
},
}
for _, rt := range tests {
setSelfHostedVolumesForScheduler(rt.podSpec)
sort.Strings(rt.podSpec.Containers[0].Command)
sort.Strings(rt.expected.Containers[0].Command)
if !reflect.DeepEqual(*rt.podSpec, rt.expected) {
t.Errorf("failed setSelfHostedVolumesForScheduler:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec)
}
}
}

View File

@@ -0,0 +1,197 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package selfhosting
import (
"fmt"
"io/ioutil"
"os"
"time"
"github.com/golang/glog"
apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
clientscheme "k8s.io/client-go/kubernetes/scheme"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
)
const (
// selfHostingWaitTimeout describes the maximum amount of time a self-hosting wait process should wait before timing out
selfHostingWaitTimeout = 2 * time.Minute
// selfHostingFailureThreshold describes how many times kubeadm will retry creating the DaemonSets
selfHostingFailureThreshold int = 5
)
// CreateSelfHostedControlPlane is responsible for turning a Static Pod-hosted control plane to a self-hosted one
// It achieves that task this way:
// 1. Load the Static Pod specification from disk (from /etc/kubernetes/manifests)
// 2. Extract the PodSpec from that Static Pod specification
// 3. Mutate the PodSpec to be compatible with self-hosting (add the right labels, taints, etc. so it can schedule correctly)
// 4. Build a new DaemonSet object for the self-hosted component in question. Use the above mentioned PodSpec
// 5. Create the DaemonSet resource. Wait until the Pods are running.
// 6. Remove the Static Pod manifest file. The kubelet will stop the original Static Pod-hosted component that was running.
// 7. The self-hosted containers should now step up and take over.
// 8. In order to avoid race conditions, we have to make sure that static pod is deleted correctly before we continue
// Otherwise, there is a race condition when we proceed without kubelet having restarted the API server correctly and the next .Create call flakes
// 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop
func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubeadmapi.MasterConfiguration, client clientset.Interface, waiter apiclient.Waiter, dryRun bool) error {
glog.V(1).Infoln("creating self hosted control plane")
// Adjust the timeout slightly to something self-hosting specific
waiter.SetTimeout(selfHostingWaitTimeout)
// Here the map of different mutators to use for the control plane's PodSpec is stored
glog.V(1).Infoln("getting mutators")
mutators := GetMutatorsFromFeatureGates(cfg.FeatureGates)
// Some extra work to be done if we should store the control plane certificates in Secrets
if features.Enabled(cfg.FeatureGates, features.StoreCertsInSecrets) {
// Upload the certificates and kubeconfig files from disk to the cluster as Secrets
if err := uploadTLSSecrets(client, cfg.CertificatesDir); err != nil {
return err
}
if err := uploadKubeConfigSecrets(client, kubeConfigDir); err != nil {
return err
}
}
for _, componentName := range kubeadmconstants.MasterComponents {
start := time.Now()
manifestPath := kubeadmconstants.GetStaticPodFilepath(componentName, manifestsDir)
// Since we want this function to be idempotent; just continue and try the next component if this file doesn't exist
if _, err := os.Stat(manifestPath); err != nil {
fmt.Printf("[self-hosted] The Static Pod for the component %q doesn't seem to be on the disk; trying the next one\n", componentName)
continue
}
// Load the Static Pod spec in order to be able to create a self-hosted variant of that file
podSpec, err := loadPodSpecFromFile(manifestPath)
if err != nil {
return err
}
// Build a DaemonSet object from the loaded PodSpec
ds := BuildDaemonSet(componentName, podSpec, mutators)
// Create or update the DaemonSet in the API Server, and retry selfHostingFailureThreshold times if it errors out
if err := apiclient.TryRunCommand(func() error {
return apiclient.CreateOrUpdateDaemonSet(client, ds)
}, selfHostingFailureThreshold); err != nil {
return err
}
// Wait for the self-hosted component to come up
if err := waiter.WaitForPodsWithLabel(BuildSelfHostedComponentLabelQuery(componentName)); err != nil {
return err
}
// Remove the old Static Pod manifest if not dryrunning
if !dryRun {
if err := os.RemoveAll(manifestPath); err != nil {
return fmt.Errorf("unable to delete static pod manifest for %s [%v]", componentName, err)
}
}
// Wait for the mirror Pod hash to be removed; otherwise we'll run into race conditions here when the kubelet hasn't had time to
// remove the Static Pod (or the mirror Pod respectively). This implicitly also tests that the API server endpoint is healthy,
// because this blocks until the API server returns a 404 Not Found when getting the Static Pod
staticPodName := fmt.Sprintf("%s-%s", componentName, cfg.NodeRegistration.Name)
if err := waiter.WaitForPodToDisappear(staticPodName); err != nil {
return err
}
// Just as an extra safety check; make sure the API server is returning ok at the /healthz endpoint (although we know it could return a GET answer for a Pod above)
if err := waiter.WaitForAPI(); err != nil {
return err
}
fmt.Printf("[self-hosted] self-hosted %s ready after %f seconds\n", componentName, time.Since(start).Seconds())
}
return nil
}
// BuildDaemonSet is responsible for mutating the PodSpec and returns a DaemonSet which is suitable for self-hosting
func BuildDaemonSet(name string, podSpec *v1.PodSpec, mutators map[string][]PodSpecMutatorFunc) *apps.DaemonSet {
// Mutate the PodSpec so it's suitable for self-hosting
mutatePodSpec(mutators, name, podSpec)
// Return a DaemonSet based on that Spec
return &apps.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Name: kubeadmconstants.AddSelfHostedPrefix(name),
Namespace: metav1.NamespaceSystem,
Labels: BuildSelfhostedComponentLabels(name),
},
Spec: apps.DaemonSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: BuildSelfhostedComponentLabels(name),
},
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: BuildSelfhostedComponentLabels(name),
},
Spec: *podSpec,
},
UpdateStrategy: apps.DaemonSetUpdateStrategy{
// Make the DaemonSet utilize the RollingUpdate rollout strategy
Type: apps.RollingUpdateDaemonSetStrategyType,
},
},
}
}
// BuildSelfhostedComponentLabels returns the labels for a self-hosted component
func BuildSelfhostedComponentLabels(component string) map[string]string {
return map[string]string{
"k8s-app": kubeadmconstants.AddSelfHostedPrefix(component),
}
}
// BuildSelfHostedComponentLabelQuery creates the right query for matching a self-hosted Pod
func BuildSelfHostedComponentLabelQuery(componentName string) string {
return fmt.Sprintf("k8s-app=%s", kubeadmconstants.AddSelfHostedPrefix(componentName))
}
func loadPodSpecFromFile(filePath string) (*v1.PodSpec, error) {
podDef, err := ioutil.ReadFile(filePath)
if err != nil {
return nil, fmt.Errorf("failed to read file path %s: %+v", filePath, err)
}
if len(podDef) == 0 {
return nil, fmt.Errorf("file was empty: %s", filePath)
}
codec := clientscheme.Codecs.UniversalDecoder()
pod := &v1.Pod{}
if err = runtime.DecodeInto(codec, podDef, pod); err != nil {
return nil, fmt.Errorf("failed decoding pod: %v", err)
}
return &pod.Spec, nil
}

View File

@@ -0,0 +1,600 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package selfhosting
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"testing"
apps "k8s.io/api/apps/v1"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util"
)
const (
testAPIServerPod = `
apiVersion: v1
kind: Pod
metadata:
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ""
creationTimestamp: null
name: kube-apiserver
namespace: kube-system
spec:
containers:
- command:
- kube-apiserver
- --service-account-key-file=/etc/kubernetes/pki/sa.pub
- --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
- --secure-port=6443
- --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --requestheader-group-headers=X-Remote-Group
- --service-cluster-ip-range=10.96.0.0/12
- --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
- --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
- --advertise-address=192.168.1.115
- --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
- --insecure-port=0
- --experimental-bootstrap-token-auth=true
- --requestheader-username-headers=X-Remote-User
- --requestheader-extra-headers-prefix=X-Remote-Extra-
- --requestheader-allowed-names=front-proxy-client
- --admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota
- --allow-privileged=true
- --client-ca-file=/etc/kubernetes/pki/ca.crt
- --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
- --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
- --authorization-mode=Node,RBAC
- --etcd-servers=http://127.0.0.1:2379
image: k8s.gcr.io/kube-apiserver-amd64:v1.7.4
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 6443
scheme: HTTPS
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-apiserver
resources:
requests:
cpu: 250m
volumeMounts:
- mountPath: /etc/kubernetes/pki
name: k8s-certs
readOnly: true
- mountPath: /etc/ssl/certs
name: ca-certs
readOnly: true
- mountPath: /etc/pki
name: ca-certs-etc-pki
readOnly: true
hostNetwork: true
volumes:
- hostPath:
path: /etc/kubernetes/pki
name: k8s-certs
- hostPath:
path: /etc/ssl/certs
name: ca-certs
- hostPath:
path: /etc/pki
name: ca-certs-etc-pki
status: {}
`
testAPIServerDaemonSet = `apiVersion: apps/v1
kind: DaemonSet
metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-apiserver
name: self-hosted-kube-apiserver
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: self-hosted-kube-apiserver
template:
metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-apiserver
spec:
containers:
- command:
- kube-apiserver
- --service-account-key-file=/etc/kubernetes/pki/sa.pub
- --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
- --secure-port=6443
- --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --requestheader-group-headers=X-Remote-Group
- --service-cluster-ip-range=10.96.0.0/12
- --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
- --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
- --advertise-address=$(HOST_IP)
- --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
- --insecure-port=0
- --experimental-bootstrap-token-auth=true
- --requestheader-username-headers=X-Remote-User
- --requestheader-extra-headers-prefix=X-Remote-Extra-
- --requestheader-allowed-names=front-proxy-client
- --admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota
- --allow-privileged=true
- --client-ca-file=/etc/kubernetes/pki/ca.crt
- --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
- --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
- --authorization-mode=Node,RBAC
- --etcd-servers=http://127.0.0.1:2379
env:
- name: HOST_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
image: k8s.gcr.io/kube-apiserver-amd64:v1.7.4
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 6443
scheme: HTTPS
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-apiserver
resources:
requests:
cpu: 250m
volumeMounts:
- mountPath: /etc/kubernetes/pki
name: k8s-certs
readOnly: true
- mountPath: /etc/ssl/certs
name: ca-certs
readOnly: true
- mountPath: /etc/pki
name: ca-certs-etc-pki
readOnly: true
dnsPolicy: ClusterFirstWithHostNet
hostNetwork: true
nodeSelector:
node-role.kubernetes.io/master: ""
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
volumes:
- hostPath:
path: /etc/kubernetes/pki
name: k8s-certs
- hostPath:
path: /etc/ssl/certs
name: ca-certs
- hostPath:
path: /etc/pki
name: ca-certs-etc-pki
updateStrategy:
type: RollingUpdate
status:
currentNumberScheduled: 0
desiredNumberScheduled: 0
numberMisscheduled: 0
numberReady: 0
`
testControllerManagerPod = `
apiVersion: v1
kind: Pod
metadata:
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ""
creationTimestamp: null
name: kube-controller-manager
namespace: kube-system
spec:
containers:
- command:
- kube-controller-manager
- --leader-elect=true
- --controllers=*,bootstrapsigner,tokencleaner
- --kubeconfig=/etc/kubernetes/controller-manager.conf
- --root-ca-file=/etc/kubernetes/pki/ca.crt
- --service-account-private-key-file=/etc/kubernetes/pki/sa.key
- --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
- --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
- --address=127.0.0.1
- --use-service-account-credentials=true
image: k8s.gcr.io/kube-controller-manager-amd64:v1.7.4
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 10252
scheme: HTTP
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-controller-manager
resources:
requests:
cpu: 200m
volumeMounts:
- mountPath: /etc/kubernetes/pki
name: k8s-certs
readOnly: true
- mountPath: /etc/ssl/certs
name: ca-certs
readOnly: true
- mountPath: /etc/kubernetes/controller-manager.conf
name: kubeconfig
readOnly: true
- mountPath: /etc/pki
name: ca-certs-etc-pki
readOnly: true
hostNetwork: true
volumes:
- hostPath:
path: /etc/kubernetes/pki
name: k8s-certs
- hostPath:
path: /etc/ssl/certs
name: ca-certs
- hostPath:
path: /etc/kubernetes/controller-manager.conf
type: FileOrCreate
name: kubeconfig
- hostPath:
path: /etc/pki
name: ca-certs-etc-pki
status: {}
`
testControllerManagerDaemonSet = `apiVersion: apps/v1
kind: DaemonSet
metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-controller-manager
name: self-hosted-kube-controller-manager
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: self-hosted-kube-controller-manager
template:
metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-controller-manager
spec:
containers:
- command:
- kube-controller-manager
- --leader-elect=true
- --controllers=*,bootstrapsigner,tokencleaner
- --kubeconfig=/etc/kubernetes/controller-manager.conf
- --root-ca-file=/etc/kubernetes/pki/ca.crt
- --service-account-private-key-file=/etc/kubernetes/pki/sa.key
- --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
- --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
- --address=127.0.0.1
- --use-service-account-credentials=true
image: k8s.gcr.io/kube-controller-manager-amd64:v1.7.4
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 10252
scheme: HTTP
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-controller-manager
resources:
requests:
cpu: 200m
volumeMounts:
- mountPath: /etc/kubernetes/pki
name: k8s-certs
readOnly: true
- mountPath: /etc/ssl/certs
name: ca-certs
readOnly: true
- mountPath: /etc/kubernetes/controller-manager.conf
name: kubeconfig
readOnly: true
- mountPath: /etc/pki
name: ca-certs-etc-pki
readOnly: true
dnsPolicy: ClusterFirstWithHostNet
hostNetwork: true
nodeSelector:
node-role.kubernetes.io/master: ""
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
volumes:
- hostPath:
path: /etc/kubernetes/pki
name: k8s-certs
- hostPath:
path: /etc/ssl/certs
name: ca-certs
- hostPath:
path: /etc/kubernetes/controller-manager.conf
type: FileOrCreate
name: kubeconfig
- hostPath:
path: /etc/pki
name: ca-certs-etc-pki
updateStrategy:
type: RollingUpdate
status:
currentNumberScheduled: 0
desiredNumberScheduled: 0
numberMisscheduled: 0
numberReady: 0
`
testSchedulerPod = `
apiVersion: v1
kind: Pod
metadata:
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ""
creationTimestamp: null
name: kube-scheduler
namespace: kube-system
spec:
containers:
- command:
- kube-scheduler
- --leader-elect=true
- --kubeconfig=/etc/kubernetes/scheduler.conf
- --address=127.0.0.1
image: k8s.gcr.io/kube-scheduler-amd64:v1.7.4
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 10251
scheme: HTTP
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-scheduler
resources:
requests:
cpu: 100m
volumeMounts:
- mountPath: /etc/kubernetes/scheduler.conf
name: kubeconfig
readOnly: true
hostNetwork: true
volumes:
- hostPath:
path: /etc/kubernetes/scheduler.conf
type: FileOrCreate
name: kubeconfig
status: {}
`
testSchedulerDaemonSet = `apiVersion: apps/v1
kind: DaemonSet
metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-scheduler
name: self-hosted-kube-scheduler
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: self-hosted-kube-scheduler
template:
metadata:
creationTimestamp: null
labels:
k8s-app: self-hosted-kube-scheduler
spec:
containers:
- command:
- kube-scheduler
- --leader-elect=true
- --kubeconfig=/etc/kubernetes/scheduler.conf
- --address=127.0.0.1
image: k8s.gcr.io/kube-scheduler-amd64:v1.7.4
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 10251
scheme: HTTP
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-scheduler
resources:
requests:
cpu: 100m
volumeMounts:
- mountPath: /etc/kubernetes/scheduler.conf
name: kubeconfig
readOnly: true
dnsPolicy: ClusterFirstWithHostNet
hostNetwork: true
nodeSelector:
node-role.kubernetes.io/master: ""
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
volumes:
- hostPath:
path: /etc/kubernetes/scheduler.conf
type: FileOrCreate
name: kubeconfig
updateStrategy:
type: RollingUpdate
status:
currentNumberScheduled: 0
desiredNumberScheduled: 0
numberMisscheduled: 0
numberReady: 0
`
)
func TestBuildDaemonSet(t *testing.T) {
var tests = []struct {
component string
podBytes []byte
dsBytes []byte
}{
{
component: constants.KubeAPIServer,
podBytes: []byte(testAPIServerPod),
dsBytes: []byte(testAPIServerDaemonSet),
},
{
component: constants.KubeControllerManager,
podBytes: []byte(testControllerManagerPod),
dsBytes: []byte(testControllerManagerDaemonSet),
},
{
component: constants.KubeScheduler,
podBytes: []byte(testSchedulerPod),
dsBytes: []byte(testSchedulerDaemonSet),
},
}
for _, rt := range tests {
tempFile, err := createTempFileWithContent(rt.podBytes)
if err != nil {
t.Errorf("error creating tempfile with content:%v", err)
}
defer os.Remove(tempFile)
podSpec, err := loadPodSpecFromFile(tempFile)
if err != nil {
t.Fatalf("couldn't load the specified Pod Spec")
}
ds := BuildDaemonSet(rt.component, podSpec, GetDefaultMutators())
dsBytes, err := util.MarshalToYaml(ds, apps.SchemeGroupVersion)
if err != nil {
t.Fatalf("failed to marshal daemonset to YAML: %v", err)
}
if !bytes.Equal(dsBytes, rt.dsBytes) {
t.Errorf("failed TestBuildDaemonSet:\nexpected:\n%s\nsaw:\n%s", rt.dsBytes, dsBytes)
}
}
}
func TestLoadPodSpecFromFile(t *testing.T) {
tests := []struct {
content string
expectError bool
}{
{
// No content
content: "",
expectError: true,
},
{
// Good YAML
content: `
apiVersion: v1
kind: Pod
metadata:
name: testpod
spec:
containers:
- image: k8s.gcr.io/busybox
`,
expectError: false,
},
{
// Good JSON
content: `
{
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": "testpod"
},
"spec": {
"containers": [
{
"image": "k8s.gcr.io/busybox"
}
]
}
}`,
expectError: false,
},
{
// Bad PodSpec
content: `
apiVersion: v1
kind: Pod
metadata:
name: testpod
spec:
- image: k8s.gcr.io/busybox
`,
expectError: true,
},
}
for _, rt := range tests {
tempFile, err := createTempFileWithContent([]byte(rt.content))
if err != nil {
t.Errorf("error creating tempfile with content:%v", err)
}
defer os.Remove(tempFile)
_, err = loadPodSpecFromFile(tempFile)
if (err != nil) != rt.expectError {
t.Errorf("failed TestLoadPodSpecFromFile:\nexpected error:\n%t\nsaw:\n%v", rt.expectError, err)
}
}
_, err := loadPodSpecFromFile("")
if err == nil {
t.Error("unexpected success: loadPodSpecFromFile should return error when no file is given")
}
}
func createTempFileWithContent(content []byte) (string, error) {
tempFile, err := ioutil.TempFile("", "")
if err != nil {
return "", fmt.Errorf("cannot create temporary file: %v", err)
}
if _, err = tempFile.Write([]byte(content)); err != nil {
return "", fmt.Errorf("cannot save temporary file: %v", err)
}
if err = tempFile.Close(); err != nil {
return "", fmt.Errorf("cannot close temporary file: %v", err)
}
return tempFile.Name(), nil
}

View File

@@ -0,0 +1,298 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package selfhosting
import (
"fmt"
"io/ioutil"
"path/filepath"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
)
type tlsKeyPair struct {
name string
cert string
key string
}
func apiServerCertificatesVolumeSource() v1.VolumeSource {
return v1.VolumeSource{
Projected: &v1.ProjectedVolumeSource{
Sources: []v1.VolumeProjection{
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.CACertAndKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSCertKey,
Path: kubeadmconstants.CACertName,
},
},
},
},
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.APIServerCertAndKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSCertKey,
Path: kubeadmconstants.APIServerCertName,
},
{
Key: v1.TLSPrivateKeyKey,
Path: kubeadmconstants.APIServerKeyName,
},
},
},
},
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSCertKey,
Path: kubeadmconstants.APIServerKubeletClientCertName,
},
{
Key: v1.TLSPrivateKeyKey,
Path: kubeadmconstants.APIServerKubeletClientKeyName,
},
},
},
},
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.ServiceAccountKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSCertKey,
Path: kubeadmconstants.ServiceAccountPublicKeyName,
},
},
},
},
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.FrontProxyCACertAndKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSCertKey,
Path: kubeadmconstants.FrontProxyCACertName,
},
},
},
},
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.FrontProxyClientCertAndKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSCertKey,
Path: kubeadmconstants.FrontProxyClientCertName,
},
{
Key: v1.TLSPrivateKeyKey,
Path: kubeadmconstants.FrontProxyClientKeyName,
},
},
},
},
},
},
}
}
func controllerManagerCertificatesVolumeSource() v1.VolumeSource {
return v1.VolumeSource{
Projected: &v1.ProjectedVolumeSource{
Sources: []v1.VolumeProjection{
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.CACertAndKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSCertKey,
Path: kubeadmconstants.CACertName,
},
{
Key: v1.TLSPrivateKeyKey,
Path: kubeadmconstants.CAKeyName,
},
},
},
},
{
Secret: &v1.SecretProjection{
LocalObjectReference: v1.LocalObjectReference{
Name: kubeadmconstants.ServiceAccountKeyBaseName,
},
Items: []v1.KeyToPath{
{
Key: v1.TLSPrivateKeyKey,
Path: kubeadmconstants.ServiceAccountPrivateKeyName,
},
},
},
},
},
},
}
}
func kubeConfigVolumeSource(kubeconfigSecretName string) v1.VolumeSource {
return v1.VolumeSource{
Secret: &v1.SecretVolumeSource{
SecretName: kubeconfigSecretName,
},
}
}
func uploadTLSSecrets(client clientset.Interface, certDir string) error {
for _, tlsKeyPair := range getTLSKeyPairs() {
secret, err := createTLSSecretFromFiles(
tlsKeyPair.name,
filepath.Join(certDir, tlsKeyPair.cert),
filepath.Join(certDir, tlsKeyPair.key),
)
if err != nil {
return err
}
if err := apiclient.CreateOrUpdateSecret(client, secret); err != nil {
return err
}
fmt.Printf("[self-hosted] Created TLS secret %q from %s and %s\n", tlsKeyPair.name, tlsKeyPair.cert, tlsKeyPair.key)
}
return nil
}
func uploadKubeConfigSecrets(client clientset.Interface, kubeConfigDir string) error {
files := []string{
kubeadmconstants.SchedulerKubeConfigFileName,
kubeadmconstants.ControllerManagerKubeConfigFileName,
}
for _, file := range files {
kubeConfigPath := filepath.Join(kubeConfigDir, file)
secret, err := createOpaqueSecretFromFile(file, kubeConfigPath)
if err != nil {
return err
}
if err := apiclient.CreateOrUpdateSecret(client, secret); err != nil {
return err
}
fmt.Printf("[self-hosted] Created secret for kubeconfig file %q\n", file)
}
return nil
}
func createTLSSecretFromFiles(secretName, crt, key string) (*v1.Secret, error) {
crtBytes, err := ioutil.ReadFile(crt)
if err != nil {
return nil, err
}
keyBytes, err := ioutil.ReadFile(key)
if err != nil {
return nil, err
}
return &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: metav1.NamespaceSystem,
},
Type: v1.SecretTypeTLS,
Data: map[string][]byte{
v1.TLSCertKey: crtBytes,
v1.TLSPrivateKeyKey: keyBytes,
},
}, nil
}
func createOpaqueSecretFromFile(secretName, file string) (*v1.Secret, error) {
fileBytes, err := ioutil.ReadFile(file)
if err != nil {
return nil, err
}
return &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: metav1.NamespaceSystem,
},
Type: v1.SecretTypeOpaque,
Data: map[string][]byte{
filepath.Base(file): fileBytes,
},
}, nil
}
func getTLSKeyPairs() []*tlsKeyPair {
return []*tlsKeyPair{
{
name: kubeadmconstants.CACertAndKeyBaseName,
cert: kubeadmconstants.CACertName,
key: kubeadmconstants.CAKeyName,
},
{
name: kubeadmconstants.APIServerCertAndKeyBaseName,
cert: kubeadmconstants.APIServerCertName,
key: kubeadmconstants.APIServerKeyName,
},
{
name: kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName,
cert: kubeadmconstants.APIServerKubeletClientCertName,
key: kubeadmconstants.APIServerKubeletClientKeyName,
},
{
name: kubeadmconstants.ServiceAccountKeyBaseName,
cert: kubeadmconstants.ServiceAccountPublicKeyName,
key: kubeadmconstants.ServiceAccountPrivateKeyName,
},
{
name: kubeadmconstants.FrontProxyCACertAndKeyBaseName,
cert: kubeadmconstants.FrontProxyCACertName,
key: kubeadmconstants.FrontProxyCAKeyName,
},
{
name: kubeadmconstants.FrontProxyClientCertAndKeyBaseName,
cert: kubeadmconstants.FrontProxyClientCertName,
key: kubeadmconstants.FrontProxyClientKeyName,
},
}
}

View File

@@ -0,0 +1,72 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package selfhosting
import (
"io/ioutil"
"log"
"os"
"testing"
)
func createTemporaryFile(name string) *os.File {
content := []byte("foo")
tmpfile, err := ioutil.TempFile("", name)
if err != nil {
log.Fatal(err)
}
if _, err := tmpfile.Write(content); err != nil {
log.Fatal(err)
}
return tmpfile
}
func TestCreateTLSSecretFromFile(t *testing.T) {
tmpCert := createTemporaryFile("foo.crt")
defer os.Remove(tmpCert.Name())
tmpKey := createTemporaryFile("foo.key")
defer os.Remove(tmpKey.Name())
_, err := createTLSSecretFromFiles("foo", tmpCert.Name(), tmpKey.Name())
if err != nil {
log.Fatal(err)
}
if err := tmpCert.Close(); err != nil {
log.Fatal(err)
}
if err := tmpKey.Close(); err != nil {
log.Fatal(err)
}
}
func TestCreateOpaqueSecretFromFile(t *testing.T) {
tmpFile := createTemporaryFile("foo")
defer os.Remove(tmpFile.Name())
_, err := createOpaqueSecretFromFile("foo", tmpFile.Name())
if err != nil {
log.Fatal(err)
}
if err := tmpFile.Close(); err != nil {
log.Fatal(err)
}
}

View File

@@ -0,0 +1,98 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"compute.go",
"health.go",
"policy.go",
"postupgrade.go",
"prepull.go",
"selfhosted.go",
"staticpods.go",
"versiongetter.go",
],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade",
visibility = ["//visibility:public"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/features:go_default_library",
"//cmd/kubeadm/app/images:go_default_library",
"//cmd/kubeadm/app/phases/addons/dns:go_default_library",
"//cmd/kubeadm/app/phases/addons/proxy:go_default_library",
"//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:go_default_library",
"//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library",
"//cmd/kubeadm/app/phases/certs:go_default_library",
"//cmd/kubeadm/app/phases/controlplane:go_default_library",
"//cmd/kubeadm/app/phases/etcd:go_default_library",
"//cmd/kubeadm/app/phases/kubelet:go_default_library",
"//cmd/kubeadm/app/phases/patchnode:go_default_library",
"//cmd/kubeadm/app/phases/selfhosting:go_default_library",
"//cmd/kubeadm/app/phases/uploadconfig:go_default_library",
"//cmd/kubeadm/app/preflight:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//cmd/kubeadm/app/util/dryrun:go_default_library",
"//cmd/kubeadm/app/util/etcd:go_default_library",
"//pkg/util/version:go_default_library",
"//pkg/version:go_default_library",
"//vendor/k8s.io/api/apps/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/util/cert:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_test",
srcs = [
"compute_test.go",
"policy_test.go",
"postupgrade_test.go",
"prepull_test.go",
"staticpods_test.go",
],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/phases/certs:go_default_library",
"//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
"//cmd/kubeadm/app/phases/controlplane:go_default_library",
"//cmd/kubeadm/app/phases/etcd:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//cmd/kubeadm/app/util/etcd:go_default_library",
"//cmd/kubeadm/test:go_default_library",
"//pkg/util/version:go_default_library",
"//vendor/github.com/coreos/etcd/clientv3:go_default_library",
"//vendor/github.com/coreos/etcd/pkg/transport:go_default_library",
"//vendor/k8s.io/api/apps/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
],
)

View File

@@ -0,0 +1,309 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrade
import (
"fmt"
"strings"
clientset "k8s.io/client-go/kubernetes"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
versionutil "k8s.io/kubernetes/pkg/util/version"
)
// Upgrade defines an upgrade possibility to upgrade from a current version to a new one
type Upgrade struct {
Description string
Before ClusterState
After ClusterState
}
// CanUpgradeKubelets returns whether an upgrade of any kubelet in the cluster is possible
func (u *Upgrade) CanUpgradeKubelets() bool {
// If there are multiple different versions now, an upgrade is possible (even if only for a subset of the nodes)
if len(u.Before.KubeletVersions) > 1 {
return true
}
// Don't report something available for upgrade if we don't know the current state
if len(u.Before.KubeletVersions) == 0 {
return false
}
// if the same version number existed both before and after, we don't have to upgrade it
_, sameVersionFound := u.Before.KubeletVersions[u.After.KubeVersion]
return !sameVersionFound
}
// CanUpgradeEtcd returns whether an upgrade of etcd is possible
func (u *Upgrade) CanUpgradeEtcd() bool {
return u.Before.EtcdVersion != u.After.EtcdVersion
}
// ActiveDNSAddon returns the version of CoreDNS or kube-dns
func ActiveDNSAddon(featureGates map[string]bool) string {
if features.Enabled(featureGates, features.CoreDNS) {
return kubeadmconstants.CoreDNS
}
return kubeadmconstants.KubeDNS
}
// ClusterState describes the state of certain versions for a cluster
type ClusterState struct {
// KubeVersion describes the version of the Kubernetes API Server, Controller Manager, Scheduler and Proxy.
KubeVersion string
// DNSType
DNSType string
// DNSVersion describes the version of the kube-dns images used and manifest version
DNSVersion string
// KubeadmVersion describes the version of the kubeadm CLI
KubeadmVersion string
// KubeletVersions is a map with a version number linked to the amount of kubelets running that version in the cluster
KubeletVersions map[string]uint16
// EtcdVersion represents the version of etcd used in the cluster
EtcdVersion string
}
// GetAvailableUpgrades fetches all versions from the specified VersionGetter and computes which
// kinds of upgrades can be performed
func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesAllowed, rcUpgradesAllowed bool, etcdClient etcdutil.ClusterInterrogator, featureGates map[string]bool, client clientset.Interface) ([]Upgrade, error) {
fmt.Println("[upgrade] Fetching available versions to upgrade to")
// Collect the upgrades kubeadm can do in this list
upgrades := []Upgrade{}
// Get the cluster version
clusterVersionStr, clusterVersion, err := versionGetterImpl.ClusterVersion()
if err != nil {
return upgrades, err
}
// Get current kubeadm CLI version
kubeadmVersionStr, kubeadmVersion, err := versionGetterImpl.KubeadmVersion()
if err != nil {
return upgrades, err
}
// Get and output the current latest stable version
stableVersionStr, stableVersion, err := versionGetterImpl.VersionFromCILabel("stable", "stable version")
if err != nil {
fmt.Printf("[upgrade/versions] WARNING: %v\n", err)
fmt.Println("[upgrade/versions] WARNING: Falling back to current kubeadm version as latest stable version")
stableVersionStr, stableVersion = kubeadmVersionStr, kubeadmVersion
}
// Get the kubelet versions in the cluster
kubeletVersions, err := versionGetterImpl.KubeletVersions()
if err != nil {
return upgrades, err
}
// Get current etcd version
etcdVersion, err := etcdClient.GetVersion()
if err != nil {
return upgrades, err
}
dnsType, dnsVersion, err := dns.DeployedDNSAddon(client)
if err != nil {
return nil, err
}
// Construct a descriptor for the current state of the world
beforeState := ClusterState{
KubeVersion: clusterVersionStr,
DNSType: dnsType,
DNSVersion: dnsVersion,
KubeadmVersion: kubeadmVersionStr,
KubeletVersions: kubeletVersions,
EtcdVersion: etcdVersion,
}
// Do a "dumb guess" that a new minor upgrade is available just because the latest stable version is higher than the cluster version
// This guess will be corrected once we know if there is a patch version available
canDoMinorUpgrade := clusterVersion.LessThan(stableVersion)
// A patch version doesn't exist if the cluster version is higher than or equal to the current stable version
// in the case that a user is trying to upgrade from, let's say, v1.8.0-beta.2 to v1.8.0-rc.1 (given we support such upgrades experimentally)
// a stable-1.8 branch doesn't exist yet. Hence this check.
if patchVersionBranchExists(clusterVersion, stableVersion) {
currentBranch := getBranchFromVersion(clusterVersionStr)
versionLabel := fmt.Sprintf("stable-%s", currentBranch)
description := fmt.Sprintf("version in the v%s series", currentBranch)
// Get and output the latest patch version for the cluster branch
patchVersionStr, patchVersion, err := versionGetterImpl.VersionFromCILabel(versionLabel, description)
if err != nil {
fmt.Printf("[upgrade/versions] WARNING: %v\n", err)
} else {
// Check if a minor version upgrade is possible when a patch release exists
// It's only possible if the latest patch version is higher than the current patch version
// If that's the case, they must be on different branches => a newer minor version can be upgraded to
canDoMinorUpgrade = minorUpgradePossibleWithPatchRelease(stableVersion, patchVersion)
// If the cluster version is lower than the newest patch version, we should inform about the possible upgrade
if patchUpgradePossible(clusterVersion, patchVersion) {
// The kubeadm version has to be upgraded to the latest patch version
newKubeadmVer := patchVersionStr
if kubeadmVersion.AtLeast(patchVersion) {
// In this case, the kubeadm CLI version is new enough. Don't display an update suggestion for kubeadm by making .NewKubeadmVersion equal .CurrentKubeadmVersion
newKubeadmVer = kubeadmVersionStr
}
upgrades = append(upgrades, Upgrade{
Description: description,
Before: beforeState,
After: ClusterState{
KubeVersion: patchVersionStr,
DNSType: ActiveDNSAddon(featureGates),
DNSVersion: kubeadmconstants.GetDNSVersion(ActiveDNSAddon(featureGates)),
KubeadmVersion: newKubeadmVer,
EtcdVersion: getSuggestedEtcdVersion(patchVersionStr),
// KubeletVersions is unset here as it is not used anywhere in .After
},
})
}
}
}
if canDoMinorUpgrade {
upgrades = append(upgrades, Upgrade{
Description: "stable version",
Before: beforeState,
After: ClusterState{
KubeVersion: stableVersionStr,
DNSType: ActiveDNSAddon(featureGates),
DNSVersion: kubeadmconstants.GetDNSVersion(ActiveDNSAddon(featureGates)),
KubeadmVersion: stableVersionStr,
EtcdVersion: getSuggestedEtcdVersion(stableVersionStr),
// KubeletVersions is unset here as it is not used anywhere in .After
},
})
}
if experimentalUpgradesAllowed || rcUpgradesAllowed {
// dl.k8s.io/release/latest.txt is ALWAYS an alpha.X version
// dl.k8s.io/release/latest-1.X.txt is first v1.X.0-alpha.0 -> v1.X.0-alpha.Y, then v1.X.0-beta.0 to v1.X.0-beta.Z, then v1.X.0-rc.1 to v1.X.0-rc.W.
// After the v1.X.0 release, latest-1.X.txt is always a beta.0 version. Let's say the latest stable version on the v1.7 branch is v1.7.3, then the
// latest-1.7 version is v1.7.4-beta.0
// Worth noticing is that when the release-1.X branch is cut; there are two versions tagged: v1.X.0-beta.0 AND v1.(X+1).alpha.0
// The v1.(X+1).alpha.0 is pretty much useless and should just be ignored, as more betas may be released that have more features than the initial v1.(X+1).alpha.0
// So what we do below is getting the latest overall version, always an v1.X.0-alpha.Y version. Then we get latest-1.(X-1) version. This version may be anything
// between v1.(X-1).0-beta.0 and v1.(X-1).Z-beta.0. At some point in time, latest-1.(X-1) will point to v1.(X-1).0-rc.1. Then we should show it.
// The flow looks like this (with time on the X axis):
// v1.8.0-alpha.1 -> v1.8.0-alpha.2 -> v1.8.0-alpha.3 | release-1.8 branch | v1.8.0-beta.0 -> v1.8.0-beta.1 -> v1.8.0-beta.2 -> v1.8.0-rc.1 -> v1.8.0 -> v1.8.1
// v1.9.0-alpha.0 -> v1.9.0-alpha.1 -> v1.9.0-alpha.2
// Get and output the current latest unstable version
latestVersionStr, latestVersion, err := versionGetterImpl.VersionFromCILabel("latest", "experimental version")
if err != nil {
return upgrades, err
}
minorUnstable := latestVersion.Components()[1]
// Get and output the current latest unstable version
previousBranch := fmt.Sprintf("latest-1.%d", minorUnstable-1)
previousBranchLatestVersionStr, previousBranchLatestVersion, err := versionGetterImpl.VersionFromCILabel(previousBranch, "")
if err != nil {
return upgrades, err
}
// If that previous latest version is an RC, RCs are allowed and the cluster version is lower than the RC version, show the upgrade
if rcUpgradesAllowed && rcUpgradePossible(clusterVersion, previousBranchLatestVersion) {
upgrades = append(upgrades, Upgrade{
Description: "release candidate version",
Before: beforeState,
After: ClusterState{
KubeVersion: previousBranchLatestVersionStr,
DNSType: ActiveDNSAddon(featureGates),
DNSVersion: kubeadmconstants.GetDNSVersion(ActiveDNSAddon(featureGates)),
KubeadmVersion: previousBranchLatestVersionStr,
EtcdVersion: getSuggestedEtcdVersion(previousBranchLatestVersionStr),
// KubeletVersions is unset here as it is not used anywhere in .After
},
})
}
// Show the possibility if experimental upgrades are allowed
if experimentalUpgradesAllowed && clusterVersion.LessThan(latestVersion) {
// Default to assume that the experimental version to show is the unstable one
unstableKubeVersion := latestVersionStr
unstableKubeDNSVersion := kubeadmconstants.GetDNSVersion(ActiveDNSAddon(featureGates))
// Ẃe should not display alpha.0. The previous branch's beta/rc versions are more relevant due how the kube branching process works.
if latestVersion.PreRelease() == "alpha.0" {
unstableKubeVersion = previousBranchLatestVersionStr
unstableKubeDNSVersion = kubeadmconstants.GetDNSVersion(ActiveDNSAddon(featureGates))
}
upgrades = append(upgrades, Upgrade{
Description: "experimental version",
Before: beforeState,
After: ClusterState{
KubeVersion: unstableKubeVersion,
DNSType: ActiveDNSAddon(featureGates),
DNSVersion: unstableKubeDNSVersion,
KubeadmVersion: unstableKubeVersion,
EtcdVersion: getSuggestedEtcdVersion(unstableKubeVersion),
// KubeletVersions is unset here as it is not used anywhere in .After
},
})
}
}
// Add a newline in the end of this output to leave some space to the next output section
fmt.Println("")
return upgrades, nil
}
func getBranchFromVersion(version string) string {
v := versionutil.MustParseGeneric(version)
return fmt.Sprintf("%d.%d", v.Major(), v.Minor())
}
func patchVersionBranchExists(clusterVersion, stableVersion *versionutil.Version) bool {
return stableVersion.AtLeast(clusterVersion)
}
func patchUpgradePossible(clusterVersion, patchVersion *versionutil.Version) bool {
return clusterVersion.LessThan(patchVersion)
}
func rcUpgradePossible(clusterVersion, previousBranchLatestVersion *versionutil.Version) bool {
return strings.HasPrefix(previousBranchLatestVersion.PreRelease(), "rc") && clusterVersion.LessThan(previousBranchLatestVersion)
}
func minorUpgradePossibleWithPatchRelease(stableVersion, patchVersion *versionutil.Version) bool {
return patchVersion.LessThan(stableVersion)
}
func getSuggestedEtcdVersion(kubernetesVersion string) string {
etcdVersion, err := kubeadmconstants.EtcdSupportedVersion(kubernetesVersion)
if err != nil {
fmt.Printf("[upgrade/versions] WARNING: No recommended etcd for requested kubernetes version (%s)\n", kubernetesVersion)
return "N/A"
}
return etcdVersion.String()
}

View File

@@ -0,0 +1,881 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrade
import (
"fmt"
"reflect"
"testing"
"time"
"github.com/coreos/etcd/clientv3"
apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientsetfake "k8s.io/client-go/kubernetes/fake"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
versionutil "k8s.io/kubernetes/pkg/util/version"
)
type fakeVersionGetter struct {
clusterVersion, kubeadmVersion, stableVersion, latestVersion, latestDevBranchVersion, stablePatchVersion, kubeletVersion string
}
var _ VersionGetter = &fakeVersionGetter{}
// ClusterVersion gets a fake API server version
func (f *fakeVersionGetter) ClusterVersion() (string, *versionutil.Version, error) {
return f.clusterVersion, versionutil.MustParseSemantic(f.clusterVersion), nil
}
// KubeadmVersion gets a fake kubeadm version
func (f *fakeVersionGetter) KubeadmVersion() (string, *versionutil.Version, error) {
return f.kubeadmVersion, versionutil.MustParseSemantic(f.kubeadmVersion), nil
}
// VersionFromCILabel gets fake latest versions from CI
func (f *fakeVersionGetter) VersionFromCILabel(ciVersionLabel, _ string) (string, *versionutil.Version, error) {
if ciVersionLabel == "stable" {
return f.stableVersion, versionutil.MustParseSemantic(f.stableVersion), nil
}
if ciVersionLabel == "latest" {
return f.latestVersion, versionutil.MustParseSemantic(f.latestVersion), nil
}
if ciVersionLabel == "latest-1.11" {
return f.latestDevBranchVersion, versionutil.MustParseSemantic(f.latestDevBranchVersion), nil
}
return f.stablePatchVersion, versionutil.MustParseSemantic(f.stablePatchVersion), nil
}
// KubeletVersions gets the versions of the kubelets in the cluster
func (f *fakeVersionGetter) KubeletVersions() (map[string]uint16, error) {
return map[string]uint16{
f.kubeletVersion: 1,
}, nil
}
type fakeEtcdClient struct {
TLS bool
mismatchedVersions bool
}
func (f fakeEtcdClient) HasTLS() bool { return f.TLS }
func (f fakeEtcdClient) ClusterAvailable() (bool, error) { return true, nil }
func (f fakeEtcdClient) WaitForClusterAvailable(delay time.Duration, retries int, retryInterval time.Duration) (bool, error) {
return true, nil
}
func (f fakeEtcdClient) GetClusterStatus() (map[string]*clientv3.StatusResponse, error) {
return make(map[string]*clientv3.StatusResponse), nil
}
func (f fakeEtcdClient) GetVersion() (string, error) {
versions, _ := f.GetClusterVersions()
if f.mismatchedVersions {
return "", fmt.Errorf("etcd cluster contains endpoints with mismatched versions: %v", versions)
}
return "3.1.12", nil
}
func (f fakeEtcdClient) GetClusterVersions() (map[string]string, error) {
if f.mismatchedVersions {
return map[string]string{
"foo": "3.1.12",
"bar": "3.2.0",
}, nil
}
return map[string]string{
"foo": "3.1.12",
"bar": "3.1.12",
}, nil
}
func TestGetAvailableUpgrades(t *testing.T) {
etcdClient := fakeEtcdClient{}
tests := []struct {
name string
vg VersionGetter
expectedUpgrades []Upgrade
allowExperimental, allowRCs bool
errExpected bool
etcdClient etcdutil.ClusterInterrogator
beforeDNSType string
beforeDNSVersion string
featureGates map[string]bool
}{
{
name: "no action needed, already up-to-date",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.3",
stablePatchVersion: "v1.10.3",
stableVersion: "v1.10.3",
},
beforeDNSType: constants.CoreDNS,
beforeDNSVersion: "v1.0.6",
featureGates: make(map[string]bool),
expectedUpgrades: []Upgrade{},
allowExperimental: false,
errExpected: false,
etcdClient: etcdClient,
},
{
name: "simple patch version upgrade",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.1",
kubeletVersion: "v1.10.1", // the kubelet are on the same version as the control plane
kubeadmVersion: "v1.10.2",
stablePatchVersion: "v1.10.3",
stableVersion: "v1.10.3",
},
beforeDNSType: constants.CoreDNS,
beforeDNSVersion: "1.0.6",
featureGates: make(map[string]bool),
expectedUpgrades: []Upgrade{
{
Description: "version in the v1.10 series",
Before: ClusterState{
KubeVersion: "v1.10.1",
KubeletVersions: map[string]uint16{
"v1.10.1": 1,
},
KubeadmVersion: "v1.10.2",
DNSType: "coredns",
DNSVersion: "1.0.6",
EtcdVersion: "3.1.12",
},
After: ClusterState{
KubeVersion: "v1.10.3",
KubeadmVersion: "v1.10.3",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.1.12",
},
},
},
allowExperimental: false,
errExpected: false,
etcdClient: etcdClient,
},
{
name: "no version provided to offline version getter does not change behavior",
vg: NewOfflineVersionGetter(&fakeVersionGetter{
clusterVersion: "v1.10.1",
kubeletVersion: "v1.10.1", // the kubelet are on the same version as the control plane
kubeadmVersion: "v1.10.2",
stablePatchVersion: "v1.10.3",
stableVersion: "v1.10.3",
}, ""),
beforeDNSType: constants.CoreDNS,
beforeDNSVersion: "1.0.6",
featureGates: make(map[string]bool),
expectedUpgrades: []Upgrade{
{
Description: "version in the v1.10 series",
Before: ClusterState{
KubeVersion: "v1.10.1",
KubeletVersions: map[string]uint16{
"v1.10.1": 1,
},
KubeadmVersion: "v1.10.2",
DNSType: "coredns",
DNSVersion: "1.0.6",
EtcdVersion: "3.1.12",
},
After: ClusterState{
KubeVersion: "v1.10.3",
KubeadmVersion: "v1.10.3",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.1.12",
},
},
},
allowExperimental: false,
errExpected: false,
etcdClient: etcdClient,
},
{
name: "minor version upgrade only",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.1",
kubeletVersion: "v1.10.1", // the kubelet are on the same version as the control plane
kubeadmVersion: "v1.11.0",
stablePatchVersion: "v1.10.1",
stableVersion: "v1.11.0",
},
beforeDNSType: constants.CoreDNS,
beforeDNSVersion: "1.0.6",
featureGates: make(map[string]bool),
expectedUpgrades: []Upgrade{
{
Description: "stable version",
Before: ClusterState{
KubeVersion: "v1.10.1",
KubeletVersions: map[string]uint16{
"v1.10.1": 1,
},
KubeadmVersion: "v1.11.0",
DNSType: "coredns",
DNSVersion: "1.0.6",
EtcdVersion: "3.1.12",
},
After: ClusterState{
KubeVersion: "v1.11.0",
KubeadmVersion: "v1.11.0",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.2.18",
},
},
},
allowExperimental: false,
errExpected: false,
etcdClient: etcdClient,
},
{
name: "both minor version upgrade and patch version upgrade available",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3", // the kubelet are on the same version as the control plane
kubeadmVersion: "v1.10.5",
stablePatchVersion: "v1.10.5",
stableVersion: "v1.11.1",
},
beforeDNSType: constants.CoreDNS,
beforeDNSVersion: "1.0.6",
featureGates: make(map[string]bool),
expectedUpgrades: []Upgrade{
{
Description: "version in the v1.10 series",
Before: ClusterState{
KubeVersion: "v1.10.3",
KubeletVersions: map[string]uint16{
"v1.10.3": 1,
},
KubeadmVersion: "v1.10.5",
DNSType: "coredns",
DNSVersion: "1.0.6",
EtcdVersion: "3.1.12",
},
After: ClusterState{
KubeVersion: "v1.10.5",
KubeadmVersion: "v1.10.5", // Note: The kubeadm version mustn't be "downgraded" here
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.1.12",
},
},
{
Description: "stable version",
Before: ClusterState{
KubeVersion: "v1.10.3",
KubeletVersions: map[string]uint16{
"v1.10.3": 1,
},
KubeadmVersion: "v1.10.5",
DNSType: "coredns",
DNSVersion: "1.0.6",
EtcdVersion: "3.1.12",
},
After: ClusterState{
KubeVersion: "v1.11.1",
KubeadmVersion: "v1.11.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.2.18",
},
},
},
allowExperimental: false,
errExpected: false,
etcdClient: etcdClient,
},
{
name: "allow experimental upgrades, but no upgrade available",
vg: &fakeVersionGetter{
clusterVersion: "v1.11.0-alpha.2",
kubeletVersion: "v1.10.5",
kubeadmVersion: "v1.10.5",
stablePatchVersion: "v1.10.5",
stableVersion: "v1.10.5",
latestVersion: "v1.11.0-alpha.2",
},
beforeDNSType: constants.CoreDNS,
beforeDNSVersion: "v1.0.6",
featureGates: make(map[string]bool),
expectedUpgrades: []Upgrade{},
allowExperimental: true,
errExpected: false,
etcdClient: etcdClient,
},
{
name: "upgrade to an unstable version should be supported",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.5",
kubeletVersion: "v1.10.5",
kubeadmVersion: "v1.10.5",
stablePatchVersion: "v1.10.5",
stableVersion: "v1.10.5",
latestVersion: "v1.11.0-alpha.2",
},
beforeDNSType: constants.CoreDNS,
beforeDNSVersion: "1.0.6",
featureGates: make(map[string]bool),
expectedUpgrades: []Upgrade{
{
Description: "experimental version",
Before: ClusterState{
KubeVersion: "v1.10.5",
KubeletVersions: map[string]uint16{
"v1.10.5": 1,
},
KubeadmVersion: "v1.10.5",
DNSType: "coredns",
DNSVersion: "1.0.6",
EtcdVersion: "3.1.12",
},
After: ClusterState{
KubeVersion: "v1.11.0-alpha.2",
KubeadmVersion: "v1.11.0-alpha.2",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.2.18",
},
},
},
allowExperimental: true,
errExpected: false,
etcdClient: etcdClient,
},
{
name: "upgrade from an unstable version to an unstable version should be supported",
vg: &fakeVersionGetter{
clusterVersion: "v1.11.0-alpha.1",
kubeletVersion: "v1.10.5",
kubeadmVersion: "v1.10.5",
stablePatchVersion: "v1.10.5",
stableVersion: "v1.10.5",
latestVersion: "v1.11.0-alpha.2",
},
beforeDNSType: constants.CoreDNS,
beforeDNSVersion: "1.0.6",
featureGates: make(map[string]bool),
expectedUpgrades: []Upgrade{
{
Description: "experimental version",
Before: ClusterState{
KubeVersion: "v1.11.0-alpha.1",
KubeletVersions: map[string]uint16{
"v1.10.5": 1,
},
KubeadmVersion: "v1.10.5",
DNSType: "coredns",
DNSVersion: "1.0.6",
EtcdVersion: "3.1.12",
},
After: ClusterState{
KubeVersion: "v1.11.0-alpha.2",
KubeadmVersion: "v1.11.0-alpha.2",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.2.18",
},
},
},
allowExperimental: true,
errExpected: false,
etcdClient: etcdClient,
},
{
name: "v1.X.0-alpha.0 should be ignored",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.5",
kubeletVersion: "v1.10.5",
kubeadmVersion: "v1.10.5",
stablePatchVersion: "v1.10.5",
stableVersion: "v1.10.5",
latestDevBranchVersion: "v1.11.0-beta.1",
latestVersion: "v1.12.0-alpha.0",
},
beforeDNSType: constants.CoreDNS,
beforeDNSVersion: "1.0.6",
featureGates: make(map[string]bool),
expectedUpgrades: []Upgrade{
{
Description: "experimental version",
Before: ClusterState{
KubeVersion: "v1.10.5",
KubeletVersions: map[string]uint16{
"v1.10.5": 1,
},
KubeadmVersion: "v1.10.5",
DNSType: "coredns",
DNSVersion: "1.0.6",
EtcdVersion: "3.1.12",
},
After: ClusterState{
KubeVersion: "v1.11.0-beta.1",
KubeadmVersion: "v1.11.0-beta.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.2.18",
},
},
},
allowExperimental: true,
errExpected: false,
etcdClient: etcdClient,
},
{
name: "upgrade to an RC version should be supported",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.5",
kubeletVersion: "v1.10.5",
kubeadmVersion: "v1.10.5",
stablePatchVersion: "v1.10.5",
stableVersion: "v1.10.5",
latestDevBranchVersion: "v1.11.0-rc.1",
latestVersion: "v1.12.0-alpha.1",
},
beforeDNSType: constants.CoreDNS,
beforeDNSVersion: "1.0.6",
featureGates: make(map[string]bool),
expectedUpgrades: []Upgrade{
{
Description: "release candidate version",
Before: ClusterState{
KubeVersion: "v1.10.5",
KubeletVersions: map[string]uint16{
"v1.10.5": 1,
},
KubeadmVersion: "v1.10.5",
DNSType: "coredns",
DNSVersion: "1.0.6",
EtcdVersion: "3.1.12",
},
After: ClusterState{
KubeVersion: "v1.11.0-rc.1",
KubeadmVersion: "v1.11.0-rc.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.2.18",
},
},
},
allowRCs: true,
errExpected: false,
etcdClient: etcdClient,
},
{
name: "it is possible (but very uncommon) that the latest version from the previous branch is an rc and the current latest version is alpha.0. In that case, show the RC",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.5",
kubeletVersion: "v1.10.5",
kubeadmVersion: "v1.10.5",
stablePatchVersion: "v1.10.5",
stableVersion: "v1.10.5",
latestDevBranchVersion: "v1.11.6-rc.1",
latestVersion: "v1.12.1-alpha.0",
},
beforeDNSType: constants.CoreDNS,
beforeDNSVersion: "1.0.6",
featureGates: make(map[string]bool),
expectedUpgrades: []Upgrade{
{
Description: "experimental version", // Note that this is considered an experimental version in this uncommon scenario
Before: ClusterState{
KubeVersion: "v1.10.5",
KubeletVersions: map[string]uint16{
"v1.10.5": 1,
},
KubeadmVersion: "v1.10.5",
DNSType: "coredns",
DNSVersion: "1.0.6",
EtcdVersion: "3.1.12",
},
After: ClusterState{
KubeVersion: "v1.11.6-rc.1",
KubeadmVersion: "v1.11.6-rc.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.2.18",
},
},
},
allowExperimental: true,
errExpected: false,
etcdClient: etcdClient,
},
{
name: "upgrade to an RC version should be supported. There may also be an even newer unstable version.",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.5",
kubeletVersion: "v1.10.5",
kubeadmVersion: "v1.10.5",
stablePatchVersion: "v1.10.5",
stableVersion: "v1.10.5",
latestDevBranchVersion: "v1.11.0-rc.1",
latestVersion: "v1.12.0-alpha.2",
},
beforeDNSType: constants.CoreDNS,
beforeDNSVersion: "1.0.6",
featureGates: make(map[string]bool),
expectedUpgrades: []Upgrade{
{
Description: "release candidate version",
Before: ClusterState{
KubeVersion: "v1.10.5",
KubeletVersions: map[string]uint16{
"v1.10.5": 1,
},
KubeadmVersion: "v1.10.5",
DNSType: "coredns",
DNSVersion: "1.0.6",
EtcdVersion: "3.1.12",
},
After: ClusterState{
KubeVersion: "v1.11.0-rc.1",
KubeadmVersion: "v1.11.0-rc.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.2.18",
},
},
{
Description: "experimental version",
Before: ClusterState{
KubeVersion: "v1.10.5",
KubeletVersions: map[string]uint16{
"v1.10.5": 1,
},
KubeadmVersion: "v1.10.5",
DNSType: "coredns",
DNSVersion: "1.0.6",
EtcdVersion: "3.1.12",
},
After: ClusterState{
KubeVersion: "v1.12.0-alpha.2",
KubeadmVersion: "v1.12.0-alpha.2",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.2.18",
},
},
},
allowRCs: true,
allowExperimental: true,
errExpected: false,
etcdClient: etcdClient,
},
{
name: "Upgrades with external etcd with mismatched versions should not be allowed.",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.3",
stablePatchVersion: "v1.10.3",
stableVersion: "v1.10.3",
},
allowRCs: false,
allowExperimental: false,
etcdClient: fakeEtcdClient{mismatchedVersions: true},
expectedUpgrades: []Upgrade{},
errExpected: true,
},
{
name: "offline version getter",
vg: NewOfflineVersionGetter(&fakeVersionGetter{
clusterVersion: "v1.11.1",
kubeletVersion: "v1.11.0",
kubeadmVersion: "v1.11.1",
}, "v1.12.1"),
etcdClient: etcdClient,
beforeDNSType: constants.CoreDNS,
beforeDNSVersion: "1.0.6",
featureGates: make(map[string]bool),
expectedUpgrades: []Upgrade{
{
Description: "version in the v1.11 series",
Before: ClusterState{
KubeVersion: "v1.11.1",
KubeletVersions: map[string]uint16{
"v1.11.0": 1,
},
KubeadmVersion: "v1.11.1",
DNSType: "coredns",
DNSVersion: "1.0.6",
EtcdVersion: "3.1.12",
},
After: ClusterState{
KubeVersion: "v1.12.1",
KubeadmVersion: "v1.12.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.2.18",
},
},
},
},
{
name: "kubedns to coredns",
vg: &fakeVersionGetter{
clusterVersion: "v1.11.2",
kubeletVersion: "v1.11.2", // the kubelet are on the same version as the control plane
kubeadmVersion: "v1.12.0",
stablePatchVersion: "v1.12.0",
stableVersion: "v1.12.0",
},
etcdClient: etcdClient,
beforeDNSType: constants.KubeDNS,
beforeDNSVersion: "1.14.7",
featureGates: make(map[string]bool),
expectedUpgrades: []Upgrade{
{
Description: "version in the v1.11 series",
Before: ClusterState{
KubeVersion: "v1.11.2",
KubeletVersions: map[string]uint16{
"v1.11.2": 1,
},
KubeadmVersion: "v1.12.0",
DNSType: "kube-dns",
DNSVersion: "1.14.7",
EtcdVersion: "3.1.12",
},
After: ClusterState{
KubeVersion: "v1.12.0",
KubeadmVersion: "v1.12.0",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.2.18",
},
},
},
},
{
name: "keep coredns",
vg: &fakeVersionGetter{
clusterVersion: "v1.11.2",
kubeletVersion: "v1.11.2", // the kubelet are on the same version as the control plane
kubeadmVersion: "v1.12.0",
stablePatchVersion: "v1.12.0",
stableVersion: "v1.12.0",
},
etcdClient: etcdClient,
beforeDNSType: constants.KubeDNS,
beforeDNSVersion: "1.14.7",
featureGates: map[string]bool{"CoreDNS": false},
expectedUpgrades: []Upgrade{
{
Description: "version in the v1.11 series",
Before: ClusterState{
KubeVersion: "v1.11.2",
KubeletVersions: map[string]uint16{
"v1.11.2": 1,
},
KubeadmVersion: "v1.12.0",
DNSType: "kube-dns",
DNSVersion: "1.14.7",
EtcdVersion: "3.1.12",
},
After: ClusterState{
KubeVersion: "v1.12.0",
KubeadmVersion: "v1.12.0",
DNSType: "kube-dns",
DNSVersion: "1.14.10",
EtcdVersion: "3.2.18",
},
},
},
},
}
// Instantiating a fake etcd cluster for being able to get etcd version for a corresponding
// kubernetes release.
for _, rt := range tests {
t.Run(rt.name, func(t *testing.T) {
client := clientsetfake.NewSimpleClientset(&apps.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: rt.beforeDNSType,
Namespace: "kube-system",
Labels: map[string]string{
"k8s-app": "kube-dns",
},
},
Spec: apps.DeploymentSpec{
Template: v1.PodTemplateSpec{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Image: "test:" + rt.beforeDNSVersion,
},
},
},
},
},
})
actualUpgrades, actualErr := GetAvailableUpgrades(rt.vg, rt.allowExperimental, rt.allowRCs, rt.etcdClient, rt.featureGates, client)
if !reflect.DeepEqual(actualUpgrades, rt.expectedUpgrades) {
t.Errorf("failed TestGetAvailableUpgrades\n\texpected upgrades: %v\n\tgot: %v", rt.expectedUpgrades, actualUpgrades)
}
if (actualErr != nil) != rt.errExpected {
fmt.Printf("Hello error")
t.Errorf("failed TestGetAvailableUpgrades\n\texpected error: %t\n\tgot error: %t", rt.errExpected, (actualErr != nil))
}
if !reflect.DeepEqual(actualUpgrades, rt.expectedUpgrades) {
t.Errorf("failed TestGetAvailableUpgrades\n\texpected upgrades: %v\n\tgot: %v", rt.expectedUpgrades, actualUpgrades)
}
})
}
}
func TestKubeletUpgrade(t *testing.T) {
tests := []struct {
before map[string]uint16
after string
expected bool
}{
{ // upgrade available
before: map[string]uint16{
"v1.10.1": 1,
},
after: "v1.10.3",
expected: true,
},
{ // upgrade available
before: map[string]uint16{
"v1.10.1": 1,
"v1.10.3": 100,
},
after: "v1.10.3",
expected: true,
},
{ // upgrade not available
before: map[string]uint16{
"v1.10.3": 1,
},
after: "v1.10.3",
expected: false,
},
{ // upgrade not available
before: map[string]uint16{
"v1.10.3": 100,
},
after: "v1.10.3",
expected: false,
},
{ // upgrade not available if we don't know anything about the earlier state
before: map[string]uint16{},
after: "v1.10.3",
expected: false,
},
}
for _, rt := range tests {
upgrade := Upgrade{
Before: ClusterState{
KubeletVersions: rt.before,
},
After: ClusterState{
KubeVersion: rt.after,
},
}
actual := upgrade.CanUpgradeKubelets()
if actual != rt.expected {
t.Errorf("failed TestKubeletUpgrade\n\texpected: %t\n\tgot: %t\n\ttest object: %v", rt.expected, actual, upgrade)
}
}
}
func TestGetBranchFromVersion(t *testing.T) {
testCases := []struct {
version string
expectedVersion string
}{
{
version: "v1.9.5",
expectedVersion: "1.9",
},
{
version: "v1.9.0-alpha.2",
expectedVersion: "1.9",
},
{
version: "v1.9.0-beta.0",
expectedVersion: "1.9",
},
{
version: "v1.9.0-rc.1",
expectedVersion: "1.9",
},
{
version: "v1.12.5",
expectedVersion: "1.12",
},
{
version: "v1.11.0-alpha.0",
expectedVersion: "1.11",
},
{
version: "v1.11.0-beta.1",
expectedVersion: "1.11",
},
{
version: "v1.11.0-rc.0",
expectedVersion: "1.11",
},
{
version: "1.12.5",
expectedVersion: "1.12",
},
}
for _, tc := range testCases {
v := getBranchFromVersion(tc.version)
if v != tc.expectedVersion {
t.Errorf("expected version %s, got %s", tc.expectedVersion, v)
}
}
}

View File

@@ -0,0 +1,214 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrade
import (
"fmt"
"net/http"
"os"
apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/sets"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
)
// healthCheck is a helper struct for easily performing healthchecks against the cluster and printing the output
type healthCheck struct {
name string
client clientset.Interface
// f is invoked with a k8s client passed to it. Should return an optional error
f func(clientset.Interface) error
}
// Check is part of the preflight.Checker interface
func (c *healthCheck) Check() (warnings, errors []error) {
if err := c.f(c.client); err != nil {
return nil, []error{err}
}
return nil, nil
}
// Name is part of the preflight.Checker interface
func (c *healthCheck) Name() string {
return c.name
}
// CheckClusterHealth makes sure:
// - the API /healthz endpoint is healthy
// - all master Nodes are Ready
// - (if self-hosted) that there are DaemonSets with at least one Pod for all control plane components
// - (if static pod-hosted) that all required Static Pod manifests exist on disk
func CheckClusterHealth(client clientset.Interface, ignoreChecksErrors sets.String) error {
fmt.Println("[upgrade] Making sure the cluster is healthy:")
healthChecks := []preflight.Checker{
&healthCheck{
name: "APIServerHealth",
client: client,
f: apiServerHealthy,
},
&healthCheck{
name: "MasterNodesReady",
client: client,
f: masterNodesReady,
},
// TODO: Add a check for ComponentStatuses here?
}
// Run slightly different health checks depending on control plane hosting type
if IsControlPlaneSelfHosted(client) {
healthChecks = append(healthChecks, &healthCheck{
name: "ControlPlaneHealth",
client: client,
f: controlPlaneHealth,
})
} else {
healthChecks = append(healthChecks, &healthCheck{
name: "StaticPodManifest",
client: client,
f: staticPodManifestHealth,
})
}
return preflight.RunChecks(healthChecks, os.Stderr, ignoreChecksErrors)
}
// apiServerHealthy checks whether the API server's /healthz endpoint is healthy
func apiServerHealthy(client clientset.Interface) error {
healthStatus := 0
// If client.Discovery().RESTClient() is nil, the fake client is used, and that means we are dry-running. Just proceed
if client.Discovery().RESTClient() == nil {
return nil
}
client.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus)
if healthStatus != http.StatusOK {
return fmt.Errorf("the API Server is unhealthy; /healthz didn't return %q", "ok")
}
return nil
}
// masterNodesReady checks whether all master Nodes in the cluster are in the Running state
func masterNodesReady(client clientset.Interface) error {
selector := labels.SelectorFromSet(labels.Set(map[string]string{
constants.LabelNodeRoleMaster: "",
}))
masters, err := client.CoreV1().Nodes().List(metav1.ListOptions{
LabelSelector: selector.String(),
})
if err != nil {
return fmt.Errorf("couldn't list masters in cluster: %v", err)
}
if len(masters.Items) == 0 {
return fmt.Errorf("failed to find any nodes with master role")
}
notReadyMasters := getNotReadyNodes(masters.Items)
if len(notReadyMasters) != 0 {
return fmt.Errorf("there are NotReady masters in the cluster: %v", notReadyMasters)
}
return nil
}
// controlPlaneHealth ensures all control plane DaemonSets are healthy
func controlPlaneHealth(client clientset.Interface) error {
notReadyDaemonSets, err := getNotReadyDaemonSets(client)
if err != nil {
return err
}
if len(notReadyDaemonSets) != 0 {
return fmt.Errorf("there are control plane DaemonSets in the cluster that are not ready: %v", notReadyDaemonSets)
}
return nil
}
// staticPodManifestHealth makes sure the required static pods are presents
func staticPodManifestHealth(_ clientset.Interface) error {
nonExistentManifests := []string{}
for _, component := range constants.MasterComponents {
manifestFile := constants.GetStaticPodFilepath(component, constants.GetStaticPodDirectory())
if _, err := os.Stat(manifestFile); os.IsNotExist(err) {
nonExistentManifests = append(nonExistentManifests, manifestFile)
}
}
if len(nonExistentManifests) == 0 {
return nil
}
return fmt.Errorf("The control plane seems to be Static Pod-hosted, but some of the manifests don't seem to exist on disk. This probably means you're running 'kubeadm upgrade' on a remote machine, which is not supported for a Static Pod-hosted cluster. Manifest files not found: %v", nonExistentManifests)
}
// IsControlPlaneSelfHosted returns whether the control plane is self hosted or not
func IsControlPlaneSelfHosted(client clientset.Interface) bool {
notReadyDaemonSets, err := getNotReadyDaemonSets(client)
if err != nil {
return false
}
// If there are no NotReady DaemonSets, we are using self-hosting
return len(notReadyDaemonSets) == 0
}
// getNotReadyDaemonSets gets the amount of Ready control plane DaemonSets
func getNotReadyDaemonSets(client clientset.Interface) ([]error, error) {
notReadyDaemonSets := []error{}
for _, component := range constants.MasterComponents {
dsName := constants.AddSelfHostedPrefix(component)
ds, err := client.AppsV1().DaemonSets(metav1.NamespaceSystem).Get(dsName, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("couldn't get daemonset %q in the %s namespace", dsName, metav1.NamespaceSystem)
}
if err := daemonSetHealth(&ds.Status); err != nil {
notReadyDaemonSets = append(notReadyDaemonSets, fmt.Errorf("DaemonSet %q not healthy: %v", dsName, err))
}
}
return notReadyDaemonSets, nil
}
// daemonSetHealth is a helper function for getting the health of a DaemonSet's status
func daemonSetHealth(dsStatus *apps.DaemonSetStatus) error {
if dsStatus.CurrentNumberScheduled != dsStatus.DesiredNumberScheduled {
return fmt.Errorf("current number of scheduled Pods ('%d') doesn't match the amount of desired Pods ('%d')", dsStatus.CurrentNumberScheduled, dsStatus.DesiredNumberScheduled)
}
if dsStatus.NumberAvailable == 0 {
return fmt.Errorf("no available Pods for DaemonSet")
}
if dsStatus.NumberReady == 0 {
return fmt.Errorf("no ready Pods for DaemonSet")
}
return nil
}
// getNotReadyNodes returns a string slice of nodes in the cluster that are NotReady
func getNotReadyNodes(nodes []v1.Node) []string {
notReadyNodes := []string{}
for _, node := range nodes {
for _, condition := range node.Status.Conditions {
if condition.Type == v1.NodeReady && condition.Status != v1.ConditionTrue {
notReadyNodes = append(notReadyNodes, node.ObjectMeta.Name)
}
}
}
return notReadyNodes
}

View File

@@ -0,0 +1,184 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrade
import (
"fmt"
"strings"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/pkg/util/version"
)
const (
// MaximumAllowedMinorVersionUpgradeSkew describes how many minor versions kubeadm can upgrade the control plane version in one go
MaximumAllowedMinorVersionUpgradeSkew = 1
// MaximumAllowedMinorVersionDowngradeSkew describes how many minor versions kubeadm can upgrade the control plane version in one go
MaximumAllowedMinorVersionDowngradeSkew = 1
// MaximumAllowedMinorVersionKubeletSkew describes how many minor versions the control plane version and the kubelet can skew in a kubeadm cluster
MaximumAllowedMinorVersionKubeletSkew = 1
)
// VersionSkewPolicyErrors describes version skew errors that might be seen during the validation process in EnforceVersionPolicies
type VersionSkewPolicyErrors struct {
Mandatory []error
Skippable []error
}
// EnforceVersionPolicies enforces that the proposed new version is compatible with all the different version skew policies
func EnforceVersionPolicies(versionGetter VersionGetter, newK8sVersionStr string, newK8sVersion *version.Version, allowExperimentalUpgrades, allowRCUpgrades bool) *VersionSkewPolicyErrors {
skewErrors := &VersionSkewPolicyErrors{
Mandatory: []error{},
Skippable: []error{},
}
clusterVersionStr, clusterVersion, err := versionGetter.ClusterVersion()
if err != nil {
// This case can't be forced: kubeadm has to be able to lookup cluster version for upgrades to work
skewErrors.Mandatory = append(skewErrors.Mandatory, fmt.Errorf("Unable to fetch cluster version: %v", err))
return skewErrors
}
kubeadmVersionStr, kubeadmVersion, err := versionGetter.KubeadmVersion()
if err != nil {
// This case can't be forced: kubeadm has to be able to lookup its version for upgrades to work
skewErrors.Mandatory = append(skewErrors.Mandatory, fmt.Errorf("Unable to fetch kubeadm version: %v", err))
return skewErrors
}
kubeletVersions, err := versionGetter.KubeletVersions()
if err != nil {
// This is a non-critical error; continue although kubeadm couldn't look this up
skewErrors.Skippable = append(skewErrors.Skippable, fmt.Errorf("Unable to fetch kubelet version: %v", err))
}
// Make sure the new version is a supported version (higher than the minimum one supported)
if constants.MinimumControlPlaneVersion.AtLeast(newK8sVersion) {
// This must not happen, kubeadm always supports a minimum version; and we can't go below that
skewErrors.Mandatory = append(skewErrors.Mandatory, fmt.Errorf("Specified version to upgrade to %q is equal to or lower than the minimum supported version %q. Please specify a higher version to upgrade to", newK8sVersionStr, clusterVersionStr))
}
// kubeadm doesn't support upgrades between two minor versions; e.g. a v1.7 -> v1.9 upgrade is not supported right away
if newK8sVersion.Minor() > clusterVersion.Minor()+MaximumAllowedMinorVersionUpgradeSkew {
tooLargeUpgradeSkewErr := fmt.Errorf("Specified version to upgrade to %q is too high; kubeadm can upgrade only %d minor version at a time", newK8sVersionStr, MaximumAllowedMinorVersionUpgradeSkew)
// If the version that we're about to upgrade to is a released version, we should fully enforce this policy
// If the version is a CI/dev/experimental version, it's okay to jump two minor version steps, but then require the -f flag
if len(newK8sVersion.PreRelease()) == 0 {
skewErrors.Mandatory = append(skewErrors.Mandatory, tooLargeUpgradeSkewErr)
} else {
skewErrors.Skippable = append(skewErrors.Skippable, tooLargeUpgradeSkewErr)
}
}
// kubeadm doesn't support downgrades between two minor versions; e.g. a v1.9 -> v1.7 downgrade is not supported right away
if newK8sVersion.Minor() < clusterVersion.Minor()-MaximumAllowedMinorVersionDowngradeSkew {
tooLargeDowngradeSkewErr := fmt.Errorf("Specified version to downgrade to %q is too low; kubeadm can downgrade only %d minor version at a time", newK8sVersionStr, MaximumAllowedMinorVersionDowngradeSkew)
// If the version that we're about to downgrade to is a released version, we should fully enforce this policy
// If the version is a CI/dev/experimental version, it's okay to jump two minor version steps, but then require the -f flag
if len(newK8sVersion.PreRelease()) == 0 {
skewErrors.Mandatory = append(skewErrors.Mandatory, tooLargeDowngradeSkewErr)
} else {
skewErrors.Skippable = append(skewErrors.Skippable, tooLargeDowngradeSkewErr)
}
}
// If the kubeadm version is lower than what we want to upgrade to; error
if kubeadmVersion.LessThan(newK8sVersion) {
if newK8sVersion.Minor() > kubeadmVersion.Minor() {
tooLargeKubeadmSkew := fmt.Errorf("Specified version to upgrade to %q is at least one minor release higher than the kubeadm minor release (%d > %d). Such an upgrade is not supported", newK8sVersionStr, newK8sVersion.Minor(), kubeadmVersion.Minor())
// This is unsupported; kubeadm has no idea how it should handle a newer minor release than itself
// If the version is a CI/dev/experimental version though, lower the severity of this check, but then require the -f flag
if len(newK8sVersion.PreRelease()) == 0 {
skewErrors.Mandatory = append(skewErrors.Mandatory, tooLargeKubeadmSkew)
} else {
skewErrors.Skippable = append(skewErrors.Skippable, tooLargeKubeadmSkew)
}
} else {
// Upgrading to a higher patch version than kubeadm is ok if the user specifies --force. Not recommended, but possible.
skewErrors.Skippable = append(skewErrors.Skippable, fmt.Errorf("Specified version to upgrade to %q is higher than the kubeadm version %q. Upgrade kubeadm first using the tool you used to install kubeadm", newK8sVersionStr, kubeadmVersionStr))
}
}
if kubeadmVersion.Major() > newK8sVersion.Major() ||
kubeadmVersion.Minor() > newK8sVersion.Minor() {
skewErrors.Skippable = append(skewErrors.Skippable, fmt.Errorf("Kubeadm version %s can only be used to upgrade to Kubernetes version %d.%d", kubeadmVersionStr, kubeadmVersion.Major(), kubeadmVersion.Minor()))
}
// Detect if the version is unstable and the user didn't allow that
if err = detectUnstableVersionError(newK8sVersion, newK8sVersionStr, allowExperimentalUpgrades, allowRCUpgrades); err != nil {
skewErrors.Skippable = append(skewErrors.Skippable, err)
}
// Detect if there are too old kubelets in the cluster
// Check for nil here since this is the only case where kubeletVersions can be nil; if KubeletVersions() returned an error
// However, it's okay to skip that check
if kubeletVersions != nil {
if err = detectTooOldKubelets(newK8sVersion, kubeletVersions); err != nil {
skewErrors.Skippable = append(skewErrors.Skippable, err)
}
}
// If we did not see any errors, return nil
if len(skewErrors.Skippable) == 0 && len(skewErrors.Mandatory) == 0 {
return nil
}
// Uh oh, we encountered one or more errors, return them
return skewErrors
}
// detectUnstableVersionError is a helper function for detecting if the unstable version (if specified) is allowed to be used
func detectUnstableVersionError(newK8sVersion *version.Version, newK8sVersionStr string, allowExperimentalUpgrades, allowRCUpgrades bool) error {
// Short-circuit quickly if this is not an unstable version
if len(newK8sVersion.PreRelease()) == 0 {
return nil
}
// If the user has specified that unstable versions are fine, then no error should be returned
if allowExperimentalUpgrades {
return nil
}
// If this is a release candidate and we allow such ones, everything's fine
if strings.HasPrefix(newK8sVersion.PreRelease(), "rc") && allowRCUpgrades {
return nil
}
return fmt.Errorf("Specified version to upgrade to %q is an unstable version and such upgrades weren't allowed via setting the --allow-*-upgrades flags", newK8sVersionStr)
}
// detectTooOldKubelets errors out if the kubelet versions are so old that an unsupported skew would happen if the cluster was upgraded
func detectTooOldKubelets(newK8sVersion *version.Version, kubeletVersions map[string]uint16) error {
tooOldKubeletVersions := []string{}
for versionStr := range kubeletVersions {
kubeletVersion, err := version.ParseSemantic(versionStr)
if err != nil {
return fmt.Errorf("couldn't parse kubelet version %s", versionStr)
}
if newK8sVersion.Minor() > kubeletVersion.Minor()+MaximumAllowedMinorVersionKubeletSkew {
tooOldKubeletVersions = append(tooOldKubeletVersions, versionStr)
}
}
if len(tooOldKubeletVersions) == 0 {
return nil
}
return fmt.Errorf("There are kubelets in this cluster that are too old that have these versions %v", tooOldKubeletVersions)
}

View File

@@ -0,0 +1,222 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrade
import (
"testing"
"k8s.io/kubernetes/pkg/util/version"
)
func TestEnforceVersionPolicies(t *testing.T) {
tests := []struct {
name string
vg *fakeVersionGetter
expectedMandatoryErrs int
expectedSkippableErrs int
allowExperimental, allowRCs bool
newK8sVersion string
}{
{
name: "minor upgrade",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.5",
},
newK8sVersion: "v1.10.5",
},
{
name: "major upgrade",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.2",
kubeadmVersion: "v1.11.1",
},
newK8sVersion: "v1.11.0",
},
{
name: "downgrade",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.3",
},
newK8sVersion: "v1.10.2",
},
{
name: "same version upgrade",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.3",
},
newK8sVersion: "v1.10.3",
},
{
name: "new version must be higher than v1.10.0",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.3",
},
newK8sVersion: "v1.9.10",
expectedMandatoryErrs: 1, // version must be higher than v1.10.0
expectedSkippableErrs: 1, // can't upgrade old k8s with newer kubeadm
},
{
name: "upgrading two minor versions in one go is not supported",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.12.0",
},
newK8sVersion: "v1.12.0",
expectedMandatoryErrs: 1, // can't upgrade two minor versions
expectedSkippableErrs: 1, // kubelet <-> apiserver skew too large
},
{
name: "downgrading two minor versions in one go is not supported",
vg: &fakeVersionGetter{
clusterVersion: "v1.12.3",
kubeletVersion: "v1.12.3",
kubeadmVersion: "v1.12.0",
},
newK8sVersion: "v1.10.3",
expectedMandatoryErrs: 1, // can't downgrade two minor versions
expectedSkippableErrs: 1, // can't upgrade old k8s with newer kubeadm
},
{
name: "kubeadm version must be higher than the new kube version. However, patch version skews may be forced",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.3",
},
newK8sVersion: "v1.10.5",
expectedSkippableErrs: 1,
},
{
name: "kubeadm version must be higher than the new kube version. Trying to upgrade k8s to a higher minor version than kubeadm itself should never be supported",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.3",
},
newK8sVersion: "v1.11.0",
expectedMandatoryErrs: 1,
},
{
name: "the maximum skew between the cluster version and the kubelet versions should be one minor version. This may be forced through though.",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.9.8",
kubeadmVersion: "v1.11.0",
},
newK8sVersion: "v1.11.0",
expectedSkippableErrs: 1,
},
{
name: "experimental upgrades supported if the flag is set",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0-beta.1",
},
newK8sVersion: "v1.11.0-beta.1",
allowExperimental: true,
},
{
name: "release candidate upgrades supported if the flag is set",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0-rc.1",
},
newK8sVersion: "v1.11.0-rc.1",
allowRCs: true,
},
{
name: "release candidate upgrades supported if the flag is set",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0-rc.1",
},
newK8sVersion: "v1.11.0-rc.1",
allowExperimental: true,
},
{
name: "the user should not be able to upgrade to an experimental version if they haven't opted into that",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0-beta.1",
},
newK8sVersion: "v1.11.0-beta.1",
allowRCs: true,
expectedSkippableErrs: 1,
},
{
name: "the user should not be able to upgrade to an release candidate version if they haven't opted into that",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0-rc.1",
},
newK8sVersion: "v1.11.0-rc.1",
expectedSkippableErrs: 1,
},
{
name: "the user can't use a newer minor version of kubeadm to upgrade an older version of kubeadm",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0",
},
newK8sVersion: "v1.10.6",
expectedSkippableErrs: 1, // can't upgrade old k8s with newer kubeadm
},
}
for _, rt := range tests {
t.Run(rt.name, func(t *testing.T) {
newK8sVer, err := version.ParseSemantic(rt.newK8sVersion)
if err != nil {
t.Fatalf("couldn't parse version %s: %v", rt.newK8sVersion, err)
}
actualSkewErrs := EnforceVersionPolicies(rt.vg, rt.newK8sVersion, newK8sVer, rt.allowExperimental, rt.allowRCs)
if actualSkewErrs == nil {
// No errors were seen. Report unit test failure if we expected to see errors
if rt.expectedMandatoryErrs+rt.expectedSkippableErrs > 0 {
t.Errorf("failed TestEnforceVersionPolicies\n\texpected errors but got none")
}
// Otherwise, just move on with the next test
return
}
if len(actualSkewErrs.Skippable) != rt.expectedSkippableErrs {
t.Errorf("failed TestEnforceVersionPolicies\n\texpected skippable errors: %d\n\tgot skippable errors: %d %v", rt.expectedSkippableErrs, len(actualSkewErrs.Skippable), *rt.vg)
}
if len(actualSkewErrs.Mandatory) != rt.expectedMandatoryErrs {
t.Errorf("failed TestEnforceVersionPolicies\n\texpected mandatory errors: %d\n\tgot mandatory errors: %d %v", rt.expectedMandatoryErrs, len(actualSkewErrs.Mandatory), *rt.vg)
}
})
}
}

View File

@@ -0,0 +1,309 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrade
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/errors"
clientset "k8s.io/client-go/kubernetes"
certutil "k8s.io/client-go/util/cert"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
"k8s.io/kubernetes/pkg/util/version"
)
var expiry = 180 * 24 * time.Hour
// PerformPostUpgradeTasks runs nearly the same functions as 'kubeadm init' would do
// Note that the markmaster phase is left out, not needed, and no token is created as that doesn't belong to the upgrade
func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterConfiguration, newK8sVer *version.Version, dryRun bool) error {
errs := []error{}
// Upload currently used configuration to the cluster
// Note: This is done right in the beginning of cluster initialization; as we might want to make other phases
// depend on centralized information from this source in the future
if err := uploadconfig.UploadConfiguration(cfg, client); err != nil {
errs = append(errs, err)
}
// Create the new, version-branched kubelet ComponentConfig ConfigMap
if err := kubeletphase.CreateConfigMap(cfg, client); err != nil {
errs = append(errs, fmt.Errorf("error creating kubelet configuration ConfigMap: %v", err))
}
// Write the new kubelet config down to disk and the env file if needed
if err := writeKubeletConfigFiles(client, cfg, newK8sVer, dryRun); err != nil {
errs = append(errs, err)
}
// Annotate the node with the crisocket information, sourced either from the MasterConfiguration struct or
// --cri-socket.
// TODO: In the future we want to use something more official like NodeStatus or similar for detecting this properly
if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil {
errs = append(errs, fmt.Errorf("error uploading crisocket: %v", err))
}
// Create/update RBAC rules that makes the bootstrap tokens able to post CSRs
if err := nodebootstraptoken.AllowBootstrapTokensToPostCSRs(client); err != nil {
errs = append(errs, err)
}
// Create/update RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically
if err := nodebootstraptoken.AutoApproveNodeBootstrapTokens(client); err != nil {
errs = append(errs, err)
}
// Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically
if err := nodebootstraptoken.AutoApproveNodeCertificateRotation(client); err != nil {
errs = append(errs, err)
}
// Upgrade to a self-hosted control plane if possible
if err := upgradeToSelfHosting(client, cfg, dryRun); err != nil {
errs = append(errs, err)
}
// TODO: Is this needed to do here? I think that updating cluster info should probably be separate from a normal upgrade
// Create the cluster-info ConfigMap with the associated RBAC rules
// if err := clusterinfo.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil {
// return err
//}
// Create/update RBAC rules that makes the cluster-info ConfigMap reachable
if err := clusterinfo.CreateClusterInfoRBACRules(client); err != nil {
errs = append(errs, err)
}
// Rotate the kube-apiserver cert and key if needed
if err := backupAPIServerCertIfNeeded(cfg, dryRun); err != nil {
errs = append(errs, err)
}
// Upgrade kube-dns/CoreDNS and kube-proxy
if err := dns.EnsureDNSAddon(cfg, client); err != nil {
errs = append(errs, err)
}
// Remove the old DNS deployment if a new DNS service is now used (kube-dns to CoreDNS or vice versa)
if err := removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg, client, dryRun); err != nil {
errs = append(errs, err)
}
if err := proxy.EnsureProxyAddon(cfg, client); err != nil {
errs = append(errs, err)
}
return errors.NewAggregate(errs)
}
func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface, dryRun bool) error {
return apiclient.TryRunCommand(func() error {
installedDeploymentName := kubeadmconstants.KubeDNS
deploymentToDelete := kubeadmconstants.CoreDNS
if features.Enabled(cfg.FeatureGates, features.CoreDNS) {
installedDeploymentName = kubeadmconstants.CoreDNS
deploymentToDelete = kubeadmconstants.KubeDNS
}
// If we're dry-running, we don't need to wait for the new DNS addon to become ready
if !dryRun {
dnsDeployment, err := client.AppsV1().Deployments(metav1.NamespaceSystem).Get(installedDeploymentName, metav1.GetOptions{})
if err != nil {
return err
}
if dnsDeployment.Status.ReadyReplicas == 0 {
return fmt.Errorf("the DNS deployment isn't ready yet")
}
}
// We don't want to wait for the DNS deployment above to become ready when dryrunning (as it never will)
// but here we should execute the DELETE command against the dryrun clientset, as it will only be logged
err := apiclient.DeleteDeploymentForeground(client, metav1.NamespaceSystem, deploymentToDelete)
if err != nil && !apierrors.IsNotFound(err) {
return err
}
return nil
}, 10)
}
func upgradeToSelfHosting(client clientset.Interface, cfg *kubeadmapi.MasterConfiguration, dryRun bool) error {
if features.Enabled(cfg.FeatureGates, features.SelfHosting) && !IsControlPlaneSelfHosted(client) {
waiter := getWaiter(dryRun, client)
// kubeadm will now convert the static Pod-hosted control plane into a self-hosted one
fmt.Println("[self-hosted] Creating self-hosted control plane.")
if err := selfhosting.CreateSelfHostedControlPlane(kubeadmconstants.GetStaticPodDirectory(), kubeadmconstants.KubernetesDir, cfg, client, waiter, dryRun); err != nil {
return fmt.Errorf("error creating self hosted control plane: %v", err)
}
}
return nil
}
func backupAPIServerCertIfNeeded(cfg *kubeadmapi.MasterConfiguration, dryRun bool) error {
certAndKeyDir := kubeadmapiv1alpha2.DefaultCertificatesDir
shouldBackup, err := shouldBackupAPIServerCertAndKey(certAndKeyDir)
if err != nil {
// Don't fail the upgrade phase if failing to determine to backup kube-apiserver cert and key.
return fmt.Errorf("[postupgrade] WARNING: failed to determine to backup kube-apiserver cert and key: %v", err)
}
if !shouldBackup {
return nil
}
// If dry-running, just say that this would happen to the user and exit
if dryRun {
fmt.Println("[postupgrade] Would rotate the API server certificate and key.")
return nil
}
// Don't fail the upgrade phase if failing to backup kube-apiserver cert and key, just continue rotating the cert
// TODO: We might want to reconsider this choice.
if err := backupAPIServerCertAndKey(certAndKeyDir); err != nil {
fmt.Printf("[postupgrade] WARNING: failed to backup kube-apiserver cert and key: %v", err)
}
return certsphase.CreateAPIServerCertAndKeyFiles(cfg)
}
func writeKubeletConfigFiles(client clientset.Interface, cfg *kubeadmapi.MasterConfiguration, newK8sVer *version.Version, dryRun bool) error {
kubeletDir, err := getKubeletDir(dryRun)
if err != nil {
// The error here should never occur in reality, would only be thrown if /tmp doesn't exist on the machine.
return err
}
errs := []error{}
// Write the configuration for the kubelet down to disk so the upgraded kubelet can start with fresh config
if err := kubeletphase.DownloadConfig(client, newK8sVer, kubeletDir); err != nil {
// Tolerate the error being NotFound when dryrunning, as there is a pretty common scenario: the dryrun process
// *would* post the new kubelet-config-1.X configmap that doesn't exist now when we're trying to download it
// again.
if !(apierrors.IsNotFound(err) && dryRun) {
errs = append(errs, fmt.Errorf("error downloading kubelet configuration from the ConfigMap: %v", err))
}
}
if dryRun { // Print what contents would be written
dryrunutil.PrintDryRunFile(kubeadmconstants.KubeletConfigurationFileName, kubeletDir, kubeadmconstants.KubeletRunDirectory, os.Stdout)
}
envFilePath := filepath.Join(kubeadmconstants.KubeletRunDirectory, kubeadmconstants.KubeletEnvFileName)
if _, err := os.Stat(envFilePath); os.IsNotExist(err) {
// Write env file with flags for the kubelet to use. We do not need to write the --register-with-taints for the master,
// as we handle that ourselves in the markmaster phase
// TODO: Maybe we want to do that some time in the future, in order to remove some logic from the markmaster phase?
if err := kubeletphase.WriteKubeletDynamicEnvFile(&cfg.NodeRegistration, cfg.FeatureGates, false, kubeletDir); err != nil {
errs = append(errs, fmt.Errorf("error writing a dynamic environment file for the kubelet: %v", err))
}
if dryRun { // Print what contents would be written
dryrunutil.PrintDryRunFile(kubeadmconstants.KubeletEnvFileName, kubeletDir, kubeadmconstants.KubeletRunDirectory, os.Stdout)
}
}
return errors.NewAggregate(errs)
}
// getWaiter gets the right waiter implementation for the right occasion
// TODO: Consolidate this with what's in init.go?
func getWaiter(dryRun bool, client clientset.Interface) apiclient.Waiter {
if dryRun {
return dryrunutil.NewWaiter()
}
return apiclient.NewKubeWaiter(client, 30*time.Minute, os.Stdout)
}
// getKubeletDir gets the kubelet directory based on whether the user is dry-running this command or not.
// TODO: Consolidate this with similar funcs?
func getKubeletDir(dryRun bool) (string, error) {
if dryRun {
return ioutil.TempDir("", "kubeadm-upgrade-dryrun")
}
return kubeadmconstants.KubeletRunDirectory, nil
}
// backupAPIServerCertAndKey backups the old cert and key of kube-apiserver to a specified directory.
func backupAPIServerCertAndKey(certAndKeyDir string) error {
subDir := filepath.Join(certAndKeyDir, "expired")
if err := os.Mkdir(subDir, 0766); err != nil {
return fmt.Errorf("failed to created backup directory %s: %v", subDir, err)
}
filesToMove := map[string]string{
filepath.Join(certAndKeyDir, kubeadmconstants.APIServerCertName): filepath.Join(subDir, kubeadmconstants.APIServerCertName),
filepath.Join(certAndKeyDir, kubeadmconstants.APIServerKeyName): filepath.Join(subDir, kubeadmconstants.APIServerKeyName),
}
return moveFiles(filesToMove)
}
// moveFiles moves files from one directory to another.
func moveFiles(files map[string]string) error {
filesToRecover := map[string]string{}
for from, to := range files {
if err := os.Rename(from, to); err != nil {
return rollbackFiles(filesToRecover, err)
}
filesToRecover[to] = from
}
return nil
}
// rollbackFiles moves the files back to the original directory.
func rollbackFiles(files map[string]string, originalErr error) error {
errs := []error{originalErr}
for from, to := range files {
if err := os.Rename(from, to); err != nil {
errs = append(errs, err)
}
}
return fmt.Errorf("couldn't move these files: %v. Got errors: %v", files, errors.NewAggregate(errs))
}
// shouldBackupAPIServerCertAndKey checks if the cert of kube-apiserver will be expired in 180 days.
func shouldBackupAPIServerCertAndKey(certAndKeyDir string) (bool, error) {
apiServerCert := filepath.Join(certAndKeyDir, kubeadmconstants.APIServerCertName)
certs, err := certutil.CertsFromFile(apiServerCert)
if err != nil {
return false, fmt.Errorf("couldn't load the certificate file %s: %v", apiServerCert, err)
}
if len(certs) == 0 {
return false, fmt.Errorf("no certificate data found")
}
if time.Now().Sub(certs[0].NotBefore) > expiry {
return true, nil
}
return false, nil
}

View File

@@ -0,0 +1,184 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrade
import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
)
func TestBackupAPIServerCertAndKey(t *testing.T) {
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
os.Chmod(tmpdir, 0766)
certPath := filepath.Join(tmpdir, constants.APIServerCertName)
certFile, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
if err != nil {
t.Fatalf("Failed to create cert file %s: %v", certPath, err)
}
defer certFile.Close()
keyPath := filepath.Join(tmpdir, constants.APIServerKeyName)
keyFile, err := os.OpenFile(keyPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
if err != nil {
t.Fatalf("Failed to create key file %s: %v", keyPath, err)
}
defer keyFile.Close()
if err := backupAPIServerCertAndKey(tmpdir); err != nil {
t.Fatalf("Failed to backup cert and key in dir %s: %v", tmpdir, err)
}
}
func TestMoveFiles(t *testing.T) {
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
os.Chmod(tmpdir, 0766)
certPath := filepath.Join(tmpdir, constants.APIServerCertName)
certFile, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
if err != nil {
t.Fatalf("Failed to create cert file %s: %v", certPath, err)
}
defer certFile.Close()
keyPath := filepath.Join(tmpdir, constants.APIServerKeyName)
keyFile, err := os.OpenFile(keyPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
if err != nil {
t.Fatalf("Failed to create key file %s: %v", keyPath, err)
}
defer keyFile.Close()
subDir := filepath.Join(tmpdir, "expired")
if err := os.Mkdir(subDir, 0766); err != nil {
t.Fatalf("Failed to create backup directory %s: %v", subDir, err)
}
filesToMove := map[string]string{
filepath.Join(tmpdir, constants.APIServerCertName): filepath.Join(subDir, constants.APIServerCertName),
filepath.Join(tmpdir, constants.APIServerKeyName): filepath.Join(subDir, constants.APIServerKeyName),
}
if err := moveFiles(filesToMove); err != nil {
t.Fatalf("Failed to move files %v: %v", filesToMove, err)
}
}
func TestRollbackFiles(t *testing.T) {
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
os.Chmod(tmpdir, 0766)
subDir := filepath.Join(tmpdir, "expired")
if err := os.Mkdir(subDir, 0766); err != nil {
t.Fatalf("Failed to create backup directory %s: %v", subDir, err)
}
certPath := filepath.Join(subDir, constants.APIServerCertName)
certFile, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
if err != nil {
t.Fatalf("Failed to create cert file %s: %v", certPath, err)
}
defer certFile.Close()
keyPath := filepath.Join(subDir, constants.APIServerKeyName)
keyFile, err := os.OpenFile(keyPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
if err != nil {
t.Fatalf("Failed to create key file %s: %v", keyPath, err)
}
defer keyFile.Close()
filesToRollBack := map[string]string{
filepath.Join(subDir, constants.APIServerCertName): filepath.Join(tmpdir, constants.APIServerCertName),
filepath.Join(subDir, constants.APIServerKeyName): filepath.Join(tmpdir, constants.APIServerKeyName),
}
errString := "there are files need roll back"
originalErr := errors.New(errString)
err = rollbackFiles(filesToRollBack, originalErr)
if err == nil {
t.Fatalf("Expected error contains %q, got nil", errString)
}
if !strings.Contains(err.Error(), errString) {
t.Fatalf("Expected error contains %q, got %v", errString, err)
}
}
func TestShouldBackupAPIServerCertAndKey(t *testing.T) {
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "test-node"},
}
for desc, test := range map[string]struct {
adjustedExpiry time.Duration
expected bool
}{
"default: cert not older than 180 days doesn't needs to backup": {
expected: false,
},
"cert older than 180 days need to backup": {
adjustedExpiry: expiry + 100*time.Hour,
expected: true,
},
} {
caCert, caKey, err := certsphase.NewCACertAndKey()
if err != nil {
t.Fatalf("failed creation of ca cert and key: %v", err)
}
caCert.NotBefore = caCert.NotBefore.Add(-test.adjustedExpiry).UTC()
apiCert, apiKey, err := certsphase.NewAPIServerCertAndKey(cfg, caCert, caKey)
if err != nil {
t.Fatalf("Test %s: failed creation of cert and key: %v", desc, err)
}
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
if err := pkiutil.WriteCertAndKey(tmpdir, constants.APIServerCertAndKeyBaseName, apiCert, apiKey); err != nil {
t.Fatalf("Test %s: failure while saving %s certificate and key: %v", desc, constants.APIServerCertAndKeyBaseName, err)
}
certAndKey := []string{filepath.Join(tmpdir, constants.APIServerCertName), filepath.Join(tmpdir, constants.APIServerKeyName)}
for _, path := range certAndKey {
if _, err := os.Stat(path); os.IsNotExist(err) {
t.Fatalf("Test %s: %s not exist: %v", desc, path, err)
}
}
shouldBackup, err := shouldBackupAPIServerCertAndKey(tmpdir)
if err != nil {
t.Fatalf("Test %s: failed to check shouldBackupAPIServerCertAndKey: %v", desc, err)
}
if shouldBackup != test.expected {
t.Fatalf("Test %s: shouldBackupAPIServerCertAndKey expected %v, got %v", desc, test.expected, shouldBackup)
}
}
}

View File

@@ -0,0 +1,180 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrade
import (
"fmt"
"time"
apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/images"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
)
const (
prepullPrefix = "upgrade-prepull-"
)
// Prepuller defines an interface for performing a prepull operation in a create-wait-delete fashion in parallel
type Prepuller interface {
CreateFunc(string) error
WaitFunc(string)
DeleteFunc(string) error
}
// DaemonSetPrepuller makes sure the control plane images are available on all masters
type DaemonSetPrepuller struct {
client clientset.Interface
cfg *kubeadmapi.MasterConfiguration
waiter apiclient.Waiter
}
// NewDaemonSetPrepuller creates a new instance of the DaemonSetPrepuller struct
func NewDaemonSetPrepuller(client clientset.Interface, waiter apiclient.Waiter, cfg *kubeadmapi.MasterConfiguration) *DaemonSetPrepuller {
return &DaemonSetPrepuller{
client: client,
cfg: cfg,
waiter: waiter,
}
}
// CreateFunc creates a DaemonSet for making the image available on every relevant node
func (d *DaemonSetPrepuller) CreateFunc(component string) error {
image := images.GetCoreImage(component, d.cfg.GetControlPlaneImageRepository(), d.cfg.KubernetesVersion, d.cfg.UnifiedControlPlaneImage)
ds := buildPrePullDaemonSet(component, image)
// Create the DaemonSet in the API Server
if err := apiclient.CreateOrUpdateDaemonSet(d.client, ds); err != nil {
return fmt.Errorf("unable to create a DaemonSet for prepulling the component %q: %v", component, err)
}
return nil
}
// WaitFunc waits for all Pods in the specified DaemonSet to be in the Running state
func (d *DaemonSetPrepuller) WaitFunc(component string) {
fmt.Printf("[upgrade/prepull] Prepulling image for component %s.\n", component)
d.waiter.WaitForPodsWithLabel("k8s-app=upgrade-prepull-" + component)
}
// DeleteFunc deletes the DaemonSet used for making the image available on every relevant node
func (d *DaemonSetPrepuller) DeleteFunc(component string) error {
dsName := addPrepullPrefix(component)
if err := apiclient.DeleteDaemonSetForeground(d.client, metav1.NamespaceSystem, dsName); err != nil {
return fmt.Errorf("unable to cleanup the DaemonSet used for prepulling %s: %v", component, err)
}
fmt.Printf("[upgrade/prepull] Prepulled image for component %s.\n", component)
return nil
}
// PrepullImagesInParallel creates DaemonSets synchronously but waits in parallel for the images to pull
func PrepullImagesInParallel(kubePrepuller Prepuller, timeout time.Duration) error {
componentsToPrepull := append(constants.MasterComponents, constants.Etcd)
fmt.Printf("[upgrade/prepull] Will prepull images for components %v\n", componentsToPrepull)
timeoutChan := time.After(timeout)
// Synchronously create the DaemonSets
for _, component := range componentsToPrepull {
if err := kubePrepuller.CreateFunc(component); err != nil {
return err
}
}
// Create a channel for streaming data from goroutines that run in parallel to a blocking for loop that cleans up
prePulledChan := make(chan string, len(componentsToPrepull))
for _, component := range componentsToPrepull {
go func(c string) {
// Wait as long as needed. This WaitFunc call should be blocking until completion
kubePrepuller.WaitFunc(c)
// When the task is done, go ahead and cleanup by sending the name to the channel
prePulledChan <- c
}(component)
}
// This call blocks until all expected messages are received from the channel or errors out if timeoutChan fires.
// For every successful wait, kubePrepuller.DeleteFunc is executed
if err := waitForItemsFromChan(timeoutChan, prePulledChan, len(componentsToPrepull), kubePrepuller.DeleteFunc); err != nil {
return err
}
fmt.Println("[upgrade/prepull] Successfully prepulled the images for all the control plane components")
return nil
}
// waitForItemsFromChan waits for n elements from stringChan with a timeout. For every item received from stringChan, cleanupFunc is executed
func waitForItemsFromChan(timeoutChan <-chan time.Time, stringChan chan string, n int, cleanupFunc func(string) error) error {
i := 0
for {
select {
case <-timeoutChan:
return fmt.Errorf("The prepull operation timed out")
case result := <-stringChan:
i++
// If the cleanup function errors; error here as well
if err := cleanupFunc(result); err != nil {
return err
}
if i == n {
return nil
}
}
}
}
// addPrepullPrefix adds the prepull prefix for this functionality; can be used in names, labels, etc.
func addPrepullPrefix(component string) string {
return fmt.Sprintf("%s%s", prepullPrefix, component)
}
// buildPrePullDaemonSet builds the DaemonSet that ensures the control plane image is available
func buildPrePullDaemonSet(component, image string) *apps.DaemonSet {
var gracePeriodSecs int64
return &apps.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Name: addPrepullPrefix(component),
Namespace: metav1.NamespaceSystem,
},
Spec: apps.DaemonSetSpec{
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"k8s-app": addPrepullPrefix(component),
},
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: component,
Image: image,
Command: []string{"/bin/sleep", "3600"},
},
},
NodeSelector: map[string]string{
constants.LabelNodeRoleMaster: "",
},
Tolerations: []v1.Toleration{constants.MasterToleration},
TerminationGracePeriodSeconds: &gracePeriodSecs,
},
},
},
}
}

View File

@@ -0,0 +1,145 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrade
import (
"fmt"
"testing"
"time"
//"k8s.io/kubernetes/pkg/util/version"
)
// failedCreatePrepuller is a fake prepuller that errors for kube-controller-manager in the CreateFunc call
type failedCreatePrepuller struct{}
func NewFailedCreatePrepuller() Prepuller {
return &failedCreatePrepuller{}
}
func (p *failedCreatePrepuller) CreateFunc(component string) error {
if component == "kube-controller-manager" {
return fmt.Errorf("boo")
}
return nil
}
func (p *failedCreatePrepuller) WaitFunc(component string) {}
func (p *failedCreatePrepuller) DeleteFunc(component string) error {
return nil
}
// foreverWaitPrepuller is a fake prepuller that basically waits "forever" (10 mins, but longer than the 10sec timeout)
type foreverWaitPrepuller struct{}
func NewForeverWaitPrepuller() Prepuller {
return &foreverWaitPrepuller{}
}
func (p *foreverWaitPrepuller) CreateFunc(component string) error {
return nil
}
func (p *foreverWaitPrepuller) WaitFunc(component string) {
time.Sleep(10 * time.Minute)
}
func (p *foreverWaitPrepuller) DeleteFunc(component string) error {
return nil
}
// failedDeletePrepuller is a fake prepuller that errors for kube-scheduler in the DeleteFunc call
type failedDeletePrepuller struct{}
func NewFailedDeletePrepuller() Prepuller {
return &failedDeletePrepuller{}
}
func (p *failedDeletePrepuller) CreateFunc(component string) error {
return nil
}
func (p *failedDeletePrepuller) WaitFunc(component string) {}
func (p *failedDeletePrepuller) DeleteFunc(component string) error {
if component == "kube-scheduler" {
return fmt.Errorf("boo")
}
return nil
}
// goodPrepuller is a fake prepuller that works as expected
type goodPrepuller struct{}
func NewGoodPrepuller() Prepuller {
return &goodPrepuller{}
}
func (p *goodPrepuller) CreateFunc(component string) error {
time.Sleep(300 * time.Millisecond)
return nil
}
func (p *goodPrepuller) WaitFunc(component string) {
time.Sleep(300 * time.Millisecond)
}
func (p *goodPrepuller) DeleteFunc(component string) error {
time.Sleep(300 * time.Millisecond)
return nil
}
func TestPrepullImagesInParallel(t *testing.T) {
tests := []struct {
p Prepuller
timeout time.Duration
expectedErr bool
}{
{ // should error out; create failed
p: NewFailedCreatePrepuller(),
timeout: 10 * time.Second,
expectedErr: true,
},
{ // should error out; timeout exceeded
p: NewForeverWaitPrepuller(),
timeout: 10 * time.Second,
expectedErr: true,
},
{ // should error out; delete failed
p: NewFailedDeletePrepuller(),
timeout: 10 * time.Second,
expectedErr: true,
},
{ // should work just fine
p: NewGoodPrepuller(),
timeout: 10 * time.Second,
expectedErr: false,
},
}
for _, rt := range tests {
actualErr := PrepullImagesInParallel(rt.p, rt.timeout)
if (actualErr != nil) != rt.expectedErr {
t.Errorf(
"failed TestPrepullImagesInParallel\n\texpected error: %t\n\tgot: %t",
rt.expectedErr,
(actualErr != nil),
)
}
}
}

View File

@@ -0,0 +1,272 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrade
import (
"fmt"
"time"
apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
"k8s.io/kubernetes/pkg/util/version"
)
const (
// upgradeTempDSPrefix is the prefix added to the temporary DaemonSet's name used during the upgrade
upgradeTempDSPrefix = "temp-upgrade-"
// upgradeTempLabel is the label key used for identifying the temporary component's DaemonSet
upgradeTempLabel = "temp-upgrade-component"
// selfHostingWaitTimeout describes the maximum amount of time a self-hosting wait process should wait before timing out
selfHostingWaitTimeout = 2 * time.Minute
// selfHostingFailureThreshold describes how many times kubeadm will retry creating the DaemonSets
selfHostingFailureThreshold int = 10
)
// controlPlaneComponentResources holds the relevant Pod and DaemonSet associated with a control plane component
type controlPlaneComponentResources struct {
pod *v1.Pod
daemonSet *apps.DaemonSet
}
// SelfHostedControlPlane upgrades a self-hosted control plane
// It works as follows:
// - The client gets the currently running DaemonSets and their associated Pods used for self-hosting the control plane
// - A temporary DaemonSet for the component in question is created; but nearly identical to the DaemonSet for the self-hosted component running right now
// - Why use this temporary DaemonSet? Because, the RollingUpdate strategy for upgrading DaemonSets first kills the old Pod, and then adds the new one
// - This doesn't work for self-hosted upgrades, as if you remove the only API server for instance you have in the cluster, the cluster essentially goes down
// - So instead, a nearly identical copy of the pre-upgrade DaemonSet is created and applied to the cluster. In the beginning, this duplicate DS is just idle
// - kubeadm waits for the temporary DaemonSet's Pod to become Running
// - kubeadm updates the real, self-hosted component. This will result in the pre-upgrade component Pod being removed from the cluster
// - Luckily, the temporary, backup DaemonSet now kicks in and takes over and acts as the control plane. It recognizes that a new Pod should be created,
// - as the "real" DaemonSet is being updated.
// - kubeadm waits for the pre-upgrade Pod to become deleted. It now takes advantage of the backup/temporary component
// - kubeadm waits for the new, upgraded DaemonSet to become Running.
// - Now that the new, upgraded DaemonSet is Running, we can delete the backup/temporary DaemonSet
// - Lastly, make sure the API /healthz endpoint still is reachable
//
// TL;DR; This is what the flow looks like in pseudo-code:
// for [kube-apiserver, kube-controller-manager, kube-scheduler], do:
// 1. Self-Hosted component v1 Running
// -> Duplicate the DaemonSet manifest
// 2. Self-Hosted component v1 Running (active). Backup component v1 Running (passive)
// -> Upgrade the Self-Hosted component v1 to v2.
// -> Self-Hosted component v1 is Deleted from the cluster
// 3. Backup component v1 Running becomes active and completes the upgrade by creating the Self-Hosted component v2 Pod (passive)
// -> Wait for Self-Hosted component v2 to become Running
// 4. Backup component v1 Running (active). Self-Hosted component v2 Running (passive)
// -> Backup component v1 is Deleted
// 5. Wait for Self-Hosted component v2 Running to become active
// 6. Repeat for all control plane components
func SelfHostedControlPlane(client clientset.Interface, waiter apiclient.Waiter, cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) error {
// Adjust the timeout slightly to something self-hosting specific
waiter.SetTimeout(selfHostingWaitTimeout)
// This function returns a map of DaemonSet objects ready to post to the API server
newControlPlaneDaemonSets := BuildUpgradedDaemonSetsFromConfig(cfg, k8sVersion)
controlPlaneResources, err := getCurrentControlPlaneComponentResources(client)
if err != nil {
return err
}
for _, component := range constants.MasterComponents {
// Make a shallow copy of the current DaemonSet in order to create a new, temporary one
tempDS := *controlPlaneResources[component].daemonSet
// Mutate the temp daemonset a little to be suitable for this usage (change label selectors, etc)
mutateTempDaemonSet(&tempDS, component)
// Create or update the DaemonSet in the API Server, and retry selfHostingFailureThreshold times if it errors out
if err := apiclient.TryRunCommand(func() error {
return apiclient.CreateOrUpdateDaemonSet(client, &tempDS)
}, selfHostingFailureThreshold); err != nil {
return err
}
// Wait for the temporary/backup self-hosted component to come up
if err := waiter.WaitForPodsWithLabel(buildTempUpgradeDSLabelQuery(component)); err != nil {
return err
}
newDS := newControlPlaneDaemonSets[component]
// Upgrade the component's self-hosted resource
// During this upgrade; the temporary/backup component will take over
if err := apiclient.TryRunCommand(func() error {
if _, err := client.AppsV1().DaemonSets(newDS.ObjectMeta.Namespace).Update(newDS); err != nil {
return fmt.Errorf("couldn't update self-hosted component's DaemonSet: %v", err)
}
return nil
}, selfHostingFailureThreshold); err != nil {
return err
}
// Wait for the component's old Pod to disappear
oldPod := controlPlaneResources[component].pod
if err := waiter.WaitForPodToDisappear(oldPod.ObjectMeta.Name); err != nil {
return err
}
// Wait for the main, upgraded self-hosted component to come up
// Here we're talking to the temporary/backup component; the upgraded component is in the process of starting up
if err := waiter.WaitForPodsWithLabel(selfhosting.BuildSelfHostedComponentLabelQuery(component)); err != nil {
return err
}
// Delete the temporary DaemonSet, and retry selfHostingFailureThreshold times if it errors out
// In order to pivot back to the upgraded API server, we kill the temporary/backup component
if err := apiclient.TryRunCommand(func() error {
return apiclient.DeleteDaemonSetForeground(client, tempDS.ObjectMeta.Namespace, tempDS.ObjectMeta.Name)
}, selfHostingFailureThreshold); err != nil {
return err
}
// Just as an extra safety check; make sure the API server is returning ok at the /healthz endpoint
if err := waiter.WaitForAPI(); err != nil {
return err
}
fmt.Printf("[upgrade/apply] Self-hosted component %q upgraded successfully!\n", component)
}
return nil
}
// BuildUpgradedDaemonSetsFromConfig takes a config object and the current version and returns the DaemonSet objects to post to the master
func BuildUpgradedDaemonSetsFromConfig(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) map[string]*apps.DaemonSet {
// Here the map of different mutators to use for the control plane's podspec is stored
mutators := selfhosting.GetMutatorsFromFeatureGates(cfg.FeatureGates)
// Get the new PodSpecs to use
controlPlanePods := controlplane.GetStaticPodSpecs(cfg, k8sVersion)
// Store the created DaemonSets in this map
controlPlaneDaemonSets := map[string]*apps.DaemonSet{}
for _, component := range constants.MasterComponents {
podSpec := controlPlanePods[component].Spec
// Build the full DaemonSet object from the PodSpec generated from the control plane phase and
// using the self-hosting mutators available from the selfhosting phase
ds := selfhosting.BuildDaemonSet(component, &podSpec, mutators)
controlPlaneDaemonSets[component] = ds
}
return controlPlaneDaemonSets
}
// addTempUpgradeDSPrefix adds the upgradeTempDSPrefix to the specified DaemonSet name
func addTempUpgradeDSPrefix(currentName string) string {
return fmt.Sprintf("%s%s", upgradeTempDSPrefix, currentName)
}
// buildTempUpgradeLabels returns the label string-string map for identifying the temporary
func buildTempUpgradeLabels(component string) map[string]string {
return map[string]string{
upgradeTempLabel: component,
}
}
// buildTempUpgradeDSLabelQuery creates the right query for matching
func buildTempUpgradeDSLabelQuery(component string) string {
return fmt.Sprintf("%s=%s", upgradeTempLabel, component)
}
// mutateTempDaemonSet mutates the specified self-hosted DaemonSet for the specified component
// in a way that makes it possible to post a nearly identical, temporary DaemonSet as a backup
func mutateTempDaemonSet(tempDS *apps.DaemonSet, component string) {
// Prefix the name of the temporary DaemonSet with upgradeTempDSPrefix
tempDS.ObjectMeta.Name = addTempUpgradeDSPrefix(tempDS.ObjectMeta.Name)
// Set .Labels to something else than the "real" self-hosted components have
tempDS.ObjectMeta.Labels = buildTempUpgradeLabels(component)
tempDS.Spec.Selector.MatchLabels = buildTempUpgradeLabels(component)
tempDS.Spec.Template.ObjectMeta.Labels = buildTempUpgradeLabels(component)
// Clean all unnecessary ObjectMeta fields
tempDS.ObjectMeta = extractRelevantObjectMeta(tempDS.ObjectMeta)
// Reset .Status as we're posting a new object
tempDS.Status = apps.DaemonSetStatus{}
}
// extractRelevantObjectMeta returns only the relevant parts of ObjectMeta required when creating
// a new, identical resource. We should not POST ResourceVersion, UUIDs, etc., only the name, labels,
// namespace and annotations should be preserved.
func extractRelevantObjectMeta(ob metav1.ObjectMeta) metav1.ObjectMeta {
return metav1.ObjectMeta{
Name: ob.Name,
Namespace: ob.Namespace,
Labels: ob.Labels,
Annotations: ob.Annotations,
}
}
// listPodsWithLabelSelector returns the relevant Pods for the given LabelSelector
func listPodsWithLabelSelector(client clientset.Interface, kvLabel string) (*v1.PodList, error) {
return client.CoreV1().Pods(metav1.NamespaceSystem).List(metav1.ListOptions{
LabelSelector: kvLabel,
})
}
// getCurrentControlPlaneComponentResources returns a string-(Pod|DaemonSet) map for later use
func getCurrentControlPlaneComponentResources(client clientset.Interface) (map[string]controlPlaneComponentResources, error) {
controlPlaneResources := map[string]controlPlaneComponentResources{}
for _, component := range constants.MasterComponents {
var podList *v1.PodList
var currentDS *apps.DaemonSet
// Get the self-hosted pod associated with the component
podLabelSelector := selfhosting.BuildSelfHostedComponentLabelQuery(component)
if err := apiclient.TryRunCommand(func() error {
var tryrunerr error
podList, tryrunerr = listPodsWithLabelSelector(client, podLabelSelector)
return tryrunerr // note that tryrunerr is most likely nil here (in successful cases)
}, selfHostingFailureThreshold); err != nil {
return nil, err
}
// Make sure that there are only one Pod with this label selector; otherwise unexpected things can happen
if len(podList.Items) > 1 {
return nil, fmt.Errorf("too many pods with label selector %q found in the %s namespace", podLabelSelector, metav1.NamespaceSystem)
}
// Get the component's DaemonSet object
dsName := constants.AddSelfHostedPrefix(component)
if err := apiclient.TryRunCommand(func() error {
var tryrunerr error
// Try to get the current self-hosted component
currentDS, tryrunerr = client.AppsV1().DaemonSets(metav1.NamespaceSystem).Get(dsName, metav1.GetOptions{})
return tryrunerr // note that tryrunerr is most likely nil here (in successful cases)
}, selfHostingFailureThreshold); err != nil {
return nil, err
}
// Add the associated resources to the map to return later
controlPlaneResources[component] = controlPlaneComponentResources{
pod: &podList.Items[0],
daemonSet: currentDS,
}
}
return controlPlaneResources, nil
}

View File

@@ -0,0 +1,516 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrade
import (
"fmt"
"os"
"strings"
"time"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
"k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
"k8s.io/kubernetes/pkg/util/version"
)
// StaticPodPathManager is responsible for tracking the directories used in the static pod upgrade transition
type StaticPodPathManager interface {
// MoveFile should move a file from oldPath to newPath
MoveFile(oldPath, newPath string) error
// RealManifestPath gets the file path for the component in the "real" static pod manifest directory used by the kubelet
RealManifestPath(component string) string
// RealManifestDir should point to the static pod manifest directory used by the kubelet
RealManifestDir() string
// TempManifestPath gets the file path for the component in the temporary directory created for generating new manifests for the upgrade
TempManifestPath(component string) string
// TempManifestDir should point to the temporary directory created for generating new manifests for the upgrade
TempManifestDir() string
// BackupManifestPath gets the file path for the component in the backup directory used for backuping manifests during the transition
BackupManifestPath(component string) string
// BackupManifestDir should point to the backup directory used for backuping manifests during the transition
BackupManifestDir() string
// BackupEtcdDir should point to the backup directory used for backuping manifests during the transition
BackupEtcdDir() string
// CleanupDirs cleans up all temporary directories
CleanupDirs() error
}
// KubeStaticPodPathManager is a real implementation of StaticPodPathManager that is used when upgrading a static pod cluster
type KubeStaticPodPathManager struct {
realManifestDir string
tempManifestDir string
backupManifestDir string
backupEtcdDir string
keepManifestDir bool
keepEtcdDir bool
}
// NewKubeStaticPodPathManager creates a new instance of KubeStaticPodPathManager
func NewKubeStaticPodPathManager(realDir, tempDir, backupDir, backupEtcdDir string, keepManifestDir, keepEtcdDir bool) StaticPodPathManager {
return &KubeStaticPodPathManager{
realManifestDir: realDir,
tempManifestDir: tempDir,
backupManifestDir: backupDir,
backupEtcdDir: backupEtcdDir,
keepManifestDir: keepManifestDir,
keepEtcdDir: keepEtcdDir,
}
}
// NewKubeStaticPodPathManagerUsingTempDirs creates a new instance of KubeStaticPodPathManager with temporary directories backing it
func NewKubeStaticPodPathManagerUsingTempDirs(realManifestDir string, saveManifestsDir, saveEtcdDir bool) (StaticPodPathManager, error) {
upgradedManifestsDir, err := constants.CreateTempDirForKubeadm("kubeadm-upgraded-manifests")
if err != nil {
return nil, err
}
backupManifestsDir, err := constants.CreateTimestampDirForKubeadm("kubeadm-backup-manifests")
if err != nil {
return nil, err
}
backupEtcdDir, err := constants.CreateTimestampDirForKubeadm("kubeadm-backup-etcd")
if err != nil {
return nil, err
}
return NewKubeStaticPodPathManager(realManifestDir, upgradedManifestsDir, backupManifestsDir, backupEtcdDir, saveManifestsDir, saveEtcdDir), nil
}
// MoveFile should move a file from oldPath to newPath
func (spm *KubeStaticPodPathManager) MoveFile(oldPath, newPath string) error {
return os.Rename(oldPath, newPath)
}
// RealManifestPath gets the file path for the component in the "real" static pod manifest directory used by the kubelet
func (spm *KubeStaticPodPathManager) RealManifestPath(component string) string {
return constants.GetStaticPodFilepath(component, spm.realManifestDir)
}
// RealManifestDir should point to the static pod manifest directory used by the kubelet
func (spm *KubeStaticPodPathManager) RealManifestDir() string {
return spm.realManifestDir
}
// TempManifestPath gets the file path for the component in the temporary directory created for generating new manifests for the upgrade
func (spm *KubeStaticPodPathManager) TempManifestPath(component string) string {
return constants.GetStaticPodFilepath(component, spm.tempManifestDir)
}
// TempManifestDir should point to the temporary directory created for generating new manifests for the upgrade
func (spm *KubeStaticPodPathManager) TempManifestDir() string {
return spm.tempManifestDir
}
// BackupManifestPath gets the file path for the component in the backup directory used for backuping manifests during the transition
func (spm *KubeStaticPodPathManager) BackupManifestPath(component string) string {
return constants.GetStaticPodFilepath(component, spm.backupManifestDir)
}
// BackupManifestDir should point to the backup directory used for backuping manifests during the transition
func (spm *KubeStaticPodPathManager) BackupManifestDir() string {
return spm.backupManifestDir
}
// BackupEtcdDir should point to the backup directory used for backuping manifests during the transition
func (spm *KubeStaticPodPathManager) BackupEtcdDir() string {
return spm.backupEtcdDir
}
// CleanupDirs cleans up all temporary directories except those the user has requested to keep around
func (spm *KubeStaticPodPathManager) CleanupDirs() error {
if err := os.RemoveAll(spm.TempManifestDir()); err != nil {
return err
}
if !spm.keepManifestDir {
if err := os.RemoveAll(spm.BackupManifestDir()); err != nil {
return err
}
}
if !spm.keepEtcdDir {
if err := os.RemoveAll(spm.BackupEtcdDir()); err != nil {
return err
}
}
return nil
}
func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.MasterConfiguration, beforePodHash string, recoverManifests map[string]string, isTLSUpgrade bool) error {
// Special treatment is required for etcd case, when rollbackOldManifests should roll back etcd
// manifests only for the case when component is Etcd
recoverEtcd := false
waitForComponentRestart := true
if component == constants.Etcd {
recoverEtcd = true
}
if isTLSUpgrade {
// We currently depend on getting the Etcd mirror Pod hash from the KubeAPIServer;
// Upgrading the Etcd protocol takes down the apiserver, so we can't verify component restarts if we restart Etcd independently.
// Skip waiting for Etcd to restart and immediately move on to updating the apiserver.
if component == constants.Etcd {
waitForComponentRestart = false
}
// Normally, if an Etcd upgrade is successful, but the apiserver upgrade fails, Etcd is not rolled back.
// In the case of a TLS upgrade, the old KubeAPIServer config is incompatible with the new Etcd confg, so we rollback Etcd
// if the APIServer upgrade fails.
if component == constants.KubeAPIServer {
recoverEtcd = true
fmt.Printf("[upgrade/staticpods] The %s manifest will be restored if component %q fails to upgrade\n", constants.Etcd, component)
}
}
// ensure etcd certs are generated for etcd and kube-apiserver
if component == constants.Etcd || component == constants.KubeAPIServer {
if err := certsphase.CreateEtcdCACertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s CA certificate and key: %v", constants.Etcd, err)
}
}
if component == constants.Etcd {
if err := certsphase.CreateEtcdServerCertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s certificate and key: %v", constants.Etcd, err)
}
if err := certsphase.CreateEtcdPeerCertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s peer certificate and key: %v", constants.Etcd, err)
}
if err := certsphase.CreateEtcdHealthcheckClientCertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s healthcheck certificate and key: %v", constants.Etcd, err)
}
}
if component == constants.KubeAPIServer {
if err := certsphase.CreateAPIServerEtcdClientCertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s %s-client certificate and key: %v", constants.KubeAPIServer, constants.Etcd, err)
}
}
// The old manifest is here; in the /etc/kubernetes/manifests/
currentManifestPath := pathMgr.RealManifestPath(component)
// The new, upgraded manifest will be written here
newManifestPath := pathMgr.TempManifestPath(component)
// The old manifest will be moved here; into a subfolder of the temporary directory
// If a rollback is needed, these manifests will be put back to where they where initially
backupManifestPath := pathMgr.BackupManifestPath(component)
// Store the backup path in the recover list. If something goes wrong now, this component will be rolled back.
recoverManifests[component] = backupManifestPath
// Move the old manifest into the old-manifests directory
if err := pathMgr.MoveFile(currentManifestPath, backupManifestPath); err != nil {
return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd)
}
// Move the new manifest into the manifests directory
if err := pathMgr.MoveFile(newManifestPath, currentManifestPath); err != nil {
return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd)
}
fmt.Printf("[upgrade/staticpods] Moved new manifest to %q and backed up old manifest to %q\n", currentManifestPath, backupManifestPath)
if waitForComponentRestart {
fmt.Println("[upgrade/staticpods] Waiting for the kubelet to restart the component")
// Wait for the mirror Pod hash to change; otherwise we'll run into race conditions here when the kubelet hasn't had time to
// notice the removal of the Static Pod, leading to a false positive below where we check that the API endpoint is healthy
// If we don't do this, there is a case where we remove the Static Pod manifest, kubelet is slow to react, kubeadm checks the
// API endpoint below of the OLD Static Pod component and proceeds quickly enough, which might lead to unexpected results.
if err := waiter.WaitForStaticPodHashChange(cfg.NodeRegistration.Name, component, beforePodHash); err != nil {
return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd)
}
// Wait for the static pod component to come up and register itself as a mirror pod
if err := waiter.WaitForPodsWithLabel("component=" + component); err != nil {
return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd)
}
fmt.Printf("[upgrade/staticpods] Component %q upgraded successfully!\n", component)
} else {
fmt.Printf("[upgrade/staticpods] Not waiting for pod-hash change for component %q\n", component)
}
return nil
}
// performEtcdStaticPodUpgrade performs upgrade of etcd, it returns bool which indicates fatal error or not and the actual error.
func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.MasterConfiguration, recoverManifests map[string]string, isTLSUpgrade bool, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) (bool, error) {
// Add etcd static pod spec only if external etcd is not configured
if cfg.Etcd.External != nil {
return false, fmt.Errorf("external etcd detected, won't try to change any etcd state")
}
// Checking health state of etcd before proceeding with the upgrade
_, err := oldEtcdClient.GetClusterStatus()
if err != nil {
return true, fmt.Errorf("etcd cluster is not healthy: %v", err)
}
// Backing up etcd data store
backupEtcdDir := pathMgr.BackupEtcdDir()
runningEtcdDir := cfg.Etcd.Local.DataDir
if err := util.CopyDir(runningEtcdDir, backupEtcdDir); err != nil {
return true, fmt.Errorf("failed to back up etcd data: %v", err)
}
// Need to check currently used version and version from constants, if differs then upgrade
desiredEtcdVersion, err := constants.EtcdSupportedVersion(cfg.KubernetesVersion)
if err != nil {
return true, fmt.Errorf("failed to retrieve an etcd version for the target kubernetes version: %v", err)
}
currentEtcdVersionStr, err := oldEtcdClient.GetVersion()
if err != nil {
return true, fmt.Errorf("failed to retrieve the current etcd version: %v", err)
}
currentEtcdVersion, err := version.ParseSemantic(currentEtcdVersionStr)
if err != nil {
return true, fmt.Errorf("failed to parse the current etcd version(%s): %v", currentEtcdVersionStr, err)
}
// Comparing current etcd version with desired to catch the same version or downgrade condition and fail on them.
if desiredEtcdVersion.LessThan(currentEtcdVersion) {
return false, fmt.Errorf("the desired etcd version for this Kubernetes version %q is %q, but the current etcd version is %q. Won't downgrade etcd, instead just continue", cfg.KubernetesVersion, desiredEtcdVersion.String(), currentEtcdVersion.String())
}
// For the case when desired etcd version is the same as current etcd version
if strings.Compare(desiredEtcdVersion.String(), currentEtcdVersion.String()) == 0 {
return false, nil
}
beforeEtcdPodHash, err := waiter.WaitForStaticPodSingleHash(cfg.NodeRegistration.Name, constants.Etcd)
if err != nil {
return true, fmt.Errorf("failed to get etcd pod's hash: %v", err)
}
// Write the updated etcd static Pod manifest into the temporary directory, at this point no etcd change
// has occurred in any aspects.
if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.TempManifestDir(), cfg); err != nil {
return true, fmt.Errorf("error creating local etcd static pod manifest file: %v", err)
}
// Waiter configurations for checking etcd status
noDelay := 0 * time.Second
podRestartDelay := noDelay
if isTLSUpgrade {
// If we are upgrading TLS we need to wait for old static pod to be removed.
// This is needed because we are not able to currently verify that the static pod
// has been updated through the apiserver across an etcd TLS upgrade.
// This value is arbitrary but seems to be long enough in manual testing.
podRestartDelay = 30 * time.Second
}
retries := 10
retryInterval := 15 * time.Second
// Perform etcd upgrade using common to all control plane components function
if err := upgradeComponent(constants.Etcd, waiter, pathMgr, cfg, beforeEtcdPodHash, recoverManifests, isTLSUpgrade); err != nil {
fmt.Printf("[upgrade/etcd] Failed to upgrade etcd: %v\n", err)
// Since upgrade component failed, the old etcd manifest has either been restored or was never touched
// Now we need to check the health of etcd cluster if it is up with old manifest
fmt.Println("[upgrade/etcd] Waiting for previous etcd to become available")
if _, err := oldEtcdClient.WaitForClusterAvailable(noDelay, retries, retryInterval); err != nil {
fmt.Printf("[upgrade/etcd] Failed to healthcheck previous etcd: %v\n", err)
// At this point we know that etcd cluster is dead and it is safe to copy backup datastore and to rollback old etcd manifest
fmt.Println("[upgrade/etcd] Rolling back etcd data")
if err := rollbackEtcdData(cfg, pathMgr); err != nil {
// Even copying back datastore failed, no options for recovery left, bailing out
return true, fmt.Errorf("fatal error rolling back local etcd cluster datadir: %v, the backup of etcd database is stored here:(%s)", err, backupEtcdDir)
}
fmt.Println("[upgrade/etcd] Etcd data rollback successful")
// Now that we've rolled back the data, let's check if the cluster comes up
fmt.Println("[upgrade/etcd] Waiting for previous etcd to become available")
if _, err := oldEtcdClient.WaitForClusterAvailable(noDelay, retries, retryInterval); err != nil {
fmt.Printf("[upgrade/etcd] Failed to healthcheck previous etcd: %v\n", err)
// Nothing else left to try to recover etcd cluster
return true, fmt.Errorf("fatal error rolling back local etcd cluster manifest: %v, the backup of etcd database is stored here:(%s)", err, backupEtcdDir)
}
// We've recovered to the previous etcd from this case
}
fmt.Println("[upgrade/etcd] Etcd was rolled back and is now available")
// Since etcd cluster came back up with the old manifest
return true, fmt.Errorf("fatal error when trying to upgrade the etcd cluster: %v, rolled the state back to pre-upgrade state", err)
}
// Initialize the new etcd client if it wasn't pre-initialized
if newEtcdClient == nil {
client, err := etcdutil.NewFromStaticPod(
[]string{"localhost:2379"},
constants.GetStaticPodDirectory(),
cfg.CertificatesDir,
)
if err != nil {
return true, fmt.Errorf("fatal error creating etcd client: %v", err)
}
newEtcdClient = client
}
// Checking health state of etcd after the upgrade
fmt.Println("[upgrade/etcd] Waiting for etcd to become available")
if _, err = newEtcdClient.WaitForClusterAvailable(podRestartDelay, retries, retryInterval); err != nil {
fmt.Printf("[upgrade/etcd] Failed to healthcheck etcd: %v\n", err)
// Despite the fact that upgradeComponent was successful, there is something wrong with the etcd cluster
// First step is to restore back up of datastore
fmt.Println("[upgrade/etcd] Rolling back etcd data")
if err := rollbackEtcdData(cfg, pathMgr); err != nil {
// Even copying back datastore failed, no options for recovery left, bailing out
return true, fmt.Errorf("fatal error rolling back local etcd cluster datadir: %v, the backup of etcd database is stored here:(%s)", err, backupEtcdDir)
}
fmt.Println("[upgrade/etcd] Etcd data rollback successful")
// Old datastore has been copied, rolling back old manifests
fmt.Println("[upgrade/etcd] Rolling back etcd manifest")
rollbackOldManifests(recoverManifests, err, pathMgr, true)
// rollbackOldManifests() always returns an error -- ignore it and continue
// Assuming rollback of the old etcd manifest was successful, check the status of etcd cluster again
fmt.Println("[upgrade/etcd] Waiting for previous etcd to become available")
if _, err := oldEtcdClient.WaitForClusterAvailable(noDelay, retries, retryInterval); err != nil {
fmt.Printf("[upgrade/etcd] Failed to healthcheck previous etcd: %v\n", err)
// Nothing else left to try to recover etcd cluster
return true, fmt.Errorf("fatal error rolling back local etcd cluster manifest: %v, the backup of etcd database is stored here:(%s)", err, backupEtcdDir)
}
fmt.Println("[upgrade/etcd] Etcd was rolled back and is now available")
// We've successfully rolled back etcd, and now return an error describing that the upgrade failed
return true, fmt.Errorf("fatal error upgrading local etcd cluster: %v, rolled the state back to pre-upgrade state", err)
}
return false, nil
}
// StaticPodControlPlane upgrades a static pod-hosted control plane
func StaticPodControlPlane(waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.MasterConfiguration, etcdUpgrade bool, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) error {
recoverManifests := map[string]string{}
var isTLSUpgrade bool
var isExternalEtcd bool
beforePodHashMap, err := waiter.WaitForStaticPodControlPlaneHashes(cfg.NodeRegistration.Name)
if err != nil {
return err
}
if oldEtcdClient == nil {
if cfg.Etcd.External != nil {
// External etcd
isExternalEtcd = true
client, err := etcdutil.New(
cfg.Etcd.External.Endpoints,
cfg.Etcd.External.CAFile,
cfg.Etcd.External.CertFile,
cfg.Etcd.External.KeyFile,
)
if err != nil {
return fmt.Errorf("failed to create etcd client for external etcd: %v", err)
}
oldEtcdClient = client
// Since etcd is managed externally, the new etcd client will be the same as the old client
if newEtcdClient == nil {
newEtcdClient = client
}
} else {
// etcd Static Pod
client, err := etcdutil.NewFromStaticPod(
[]string{"localhost:2379"},
constants.GetStaticPodDirectory(),
cfg.CertificatesDir,
)
if err != nil {
return fmt.Errorf("failed to create etcd client: %v", err)
}
oldEtcdClient = client
}
}
// etcd upgrade is done prior to other control plane components
if !isExternalEtcd && etcdUpgrade {
previousEtcdHasTLS := oldEtcdClient.HasTLS()
// set the TLS upgrade flag for all components
isTLSUpgrade = !previousEtcdHasTLS
if isTLSUpgrade {
fmt.Printf("[upgrade/etcd] Upgrading to TLS for %s\n", constants.Etcd)
}
// Perform etcd upgrade using common to all control plane components function
fatal, err := performEtcdStaticPodUpgrade(waiter, pathMgr, cfg, recoverManifests, isTLSUpgrade, oldEtcdClient, newEtcdClient)
if err != nil {
if fatal {
return err
}
fmt.Printf("[upgrade/etcd] non fatal issue encountered during upgrade: %v\n", err)
}
}
// Write the updated static Pod manifests into the temporary directory
fmt.Printf("[upgrade/staticpods] Writing new Static Pod manifests to %q\n", pathMgr.TempManifestDir())
err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), cfg)
if err != nil {
return fmt.Errorf("error creating init static pod manifest files: %v", err)
}
for _, component := range constants.MasterComponents {
if err = upgradeComponent(component, waiter, pathMgr, cfg, beforePodHashMap[component], recoverManifests, isTLSUpgrade); err != nil {
return err
}
}
// Remove the temporary directories used on a best-effort (don't fail if the calls error out)
// The calls are set here by design; we should _not_ use "defer" above as that would remove the directories
// even in the "fail and rollback" case, where we want the directories preserved for the user.
return pathMgr.CleanupDirs()
}
// rollbackOldManifests rolls back the backed-up manifests if something went wrong.
// It always returns an error to the caller.
func rollbackOldManifests(oldManifests map[string]string, origErr error, pathMgr StaticPodPathManager, restoreEtcd bool) error {
errs := []error{origErr}
for component, backupPath := range oldManifests {
// Will restore etcd manifest only if it was explicitly requested by setting restoreEtcd to True
if component == constants.Etcd && !restoreEtcd {
continue
}
// Where we should put back the backed up manifest
realManifestPath := pathMgr.RealManifestPath(component)
// Move the backup manifest back into the manifests directory
err := pathMgr.MoveFile(backupPath, realManifestPath)
if err != nil {
errs = append(errs, err)
}
}
// Let the user know there were problems, but we tried to recover
return fmt.Errorf("couldn't upgrade control plane. kubeadm has tried to recover everything into the earlier state. Errors faced: %v", errs)
}
// rollbackEtcdData rolls back the the content of etcd folder if something went wrong.
// When the folder contents are successfully rolled back, nil is returned, otherwise an error is returned.
func rollbackEtcdData(cfg *kubeadmapi.MasterConfiguration, pathMgr StaticPodPathManager) error {
backupEtcdDir := pathMgr.BackupEtcdDir()
runningEtcdDir := cfg.Etcd.Local.DataDir
if err := util.CopyDir(backupEtcdDir, runningEtcdDir); err != nil {
// Let the user know there we're problems, but we tried to reçover
return fmt.Errorf("couldn't recover etcd database with error: %v, the location of etcd backup: %s ", err, backupEtcdDir)
}
return nil
}

View File

@@ -0,0 +1,603 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrade
import (
"crypto/sha256"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/coreos/etcd/clientv3"
"github.com/coreos/etcd/pkg/transport"
"k8s.io/apimachinery/pkg/runtime"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
)
const (
waitForHashes = "wait-for-hashes"
waitForHashChange = "wait-for-hash-change"
waitForPodsWithLabel = "wait-for-pods-with-label"
testConfiguration = `
apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
api:
advertiseAddress: 1.2.3.4
bindPort: 6443
apiServerCertSANs: null
apiServerExtraArgs: null
certificatesDir: %s
controllerManagerExtraArgs: null
etcd:
local:
dataDir: %s
image: ""
featureFlags: null
imageRepository: k8s.gcr.io
kubernetesVersion: %s
networking:
dnsDomain: cluster.local
podSubnet: ""
serviceSubnet: 10.96.0.0/12
nodeRegistration:
name: foo
criSocket: ""
schedulerExtraArgs: null
token: ce3aa5.5ec8455bb76b379f
tokenTTL: 24h
unifiedControlPlaneImage: ""
`
)
// fakeWaiter is a fake apiclient.Waiter that returns errors it was initialized with
type fakeWaiter struct {
errsToReturn map[string]error
}
func NewFakeStaticPodWaiter(errsToReturn map[string]error) apiclient.Waiter {
return &fakeWaiter{
errsToReturn: errsToReturn,
}
}
// WaitForAPI just returns a dummy nil, to indicate that the program should just proceed
func (w *fakeWaiter) WaitForAPI() error {
return nil
}
// WaitForPodsWithLabel just returns an error if set from errsToReturn
func (w *fakeWaiter) WaitForPodsWithLabel(kvLabel string) error {
return w.errsToReturn[waitForPodsWithLabel]
}
// WaitForPodToDisappear just returns a dummy nil, to indicate that the program should just proceed
func (w *fakeWaiter) WaitForPodToDisappear(podName string) error {
return nil
}
// SetTimeout is a no-op; we don't use it in this implementation
func (w *fakeWaiter) SetTimeout(_ time.Duration) {}
// WaitForStaticPodControlPlaneHashes returns an error if set from errsToReturn
func (w *fakeWaiter) WaitForStaticPodControlPlaneHashes(_ string) (map[string]string, error) {
return map[string]string{}, w.errsToReturn[waitForHashes]
}
// WaitForStaticPodSingleHash returns an error if set from errsToReturn
func (w *fakeWaiter) WaitForStaticPodSingleHash(_ string, _ string) (string, error) {
return "", w.errsToReturn[waitForHashes]
}
// WaitForStaticPodHashChange returns an error if set from errsToReturn
func (w *fakeWaiter) WaitForStaticPodHashChange(_, _, _ string) error {
return w.errsToReturn[waitForHashChange]
}
// WaitForHealthyKubelet returns a dummy nil just to implement the interface
func (w *fakeWaiter) WaitForHealthyKubelet(_ time.Duration, _ string) error {
return nil
}
type fakeStaticPodPathManager struct {
kubernetesDir string
realManifestDir string
tempManifestDir string
backupManifestDir string
backupEtcdDir string
MoveFileFunc func(string, string) error
}
func NewFakeStaticPodPathManager(moveFileFunc func(string, string) error) (StaticPodPathManager, error) {
kubernetesDir, err := ioutil.TempDir("", "kubeadm-pathmanager-")
if err != nil {
return nil, fmt.Errorf("couldn't create a temporary directory for the upgrade: %v", err)
}
realManifestDir := filepath.Join(kubernetesDir, constants.ManifestsSubDirName)
if err := os.Mkdir(realManifestDir, 0700); err != nil {
return nil, fmt.Errorf("couldn't create a realManifestDir for the upgrade: %v", err)
}
upgradedManifestDir := filepath.Join(kubernetesDir, "upgraded-manifests")
if err := os.Mkdir(upgradedManifestDir, 0700); err != nil {
return nil, fmt.Errorf("couldn't create a upgradedManifestDir for the upgrade: %v", err)
}
backupManifestDir := filepath.Join(kubernetesDir, "backup-manifests")
if err := os.Mkdir(backupManifestDir, 0700); err != nil {
return nil, fmt.Errorf("couldn't create a backupManifestDir for the upgrade: %v", err)
}
backupEtcdDir := filepath.Join(kubernetesDir, "kubeadm-backup-etcd")
if err := os.Mkdir(backupEtcdDir, 0700); err != nil {
return nil, err
}
return &fakeStaticPodPathManager{
kubernetesDir: kubernetesDir,
realManifestDir: realManifestDir,
tempManifestDir: upgradedManifestDir,
backupManifestDir: backupManifestDir,
backupEtcdDir: backupEtcdDir,
MoveFileFunc: moveFileFunc,
}, nil
}
func (spm *fakeStaticPodPathManager) MoveFile(oldPath, newPath string) error {
return spm.MoveFileFunc(oldPath, newPath)
}
func (spm *fakeStaticPodPathManager) KubernetesDir() string {
return spm.kubernetesDir
}
func (spm *fakeStaticPodPathManager) RealManifestPath(component string) string {
return constants.GetStaticPodFilepath(component, spm.realManifestDir)
}
func (spm *fakeStaticPodPathManager) RealManifestDir() string {
return spm.realManifestDir
}
func (spm *fakeStaticPodPathManager) TempManifestPath(component string) string {
return constants.GetStaticPodFilepath(component, spm.tempManifestDir)
}
func (spm *fakeStaticPodPathManager) TempManifestDir() string {
return spm.tempManifestDir
}
func (spm *fakeStaticPodPathManager) BackupManifestPath(component string) string {
return constants.GetStaticPodFilepath(component, spm.backupManifestDir)
}
func (spm *fakeStaticPodPathManager) BackupManifestDir() string {
return spm.backupManifestDir
}
func (spm *fakeStaticPodPathManager) BackupEtcdDir() string {
return spm.backupEtcdDir
}
func (spm *fakeStaticPodPathManager) CleanupDirs() error {
if err := os.RemoveAll(spm.TempManifestDir()); err != nil {
return err
}
if err := os.RemoveAll(spm.BackupManifestDir()); err != nil {
return err
}
return os.RemoveAll(spm.BackupEtcdDir())
}
type fakeTLSEtcdClient struct{ TLS bool }
func (c fakeTLSEtcdClient) HasTLS() bool {
return c.TLS
}
func (c fakeTLSEtcdClient) ClusterAvailable() (bool, error) { return true, nil }
func (c fakeTLSEtcdClient) WaitForClusterAvailable(delay time.Duration, retries int, retryInterval time.Duration) (bool, error) {
return true, nil
}
func (c fakeTLSEtcdClient) GetClusterStatus() (map[string]*clientv3.StatusResponse, error) {
return map[string]*clientv3.StatusResponse{
"foo": {
Version: "3.1.12",
}}, nil
}
func (c fakeTLSEtcdClient) GetClusterVersions() (map[string]string, error) {
return map[string]string{
"foo": "3.1.12",
}, nil
}
func (c fakeTLSEtcdClient) GetVersion() (string, error) {
return "3.1.12", nil
}
type fakePodManifestEtcdClient struct{ ManifestDir, CertificatesDir string }
func (c fakePodManifestEtcdClient) HasTLS() bool {
hasTLS, _ := etcdutil.PodManifestsHaveTLS(c.ManifestDir)
return hasTLS
}
func (c fakePodManifestEtcdClient) ClusterAvailable() (bool, error) { return true, nil }
func (c fakePodManifestEtcdClient) WaitForClusterAvailable(delay time.Duration, retries int, retryInterval time.Duration) (bool, error) {
return true, nil
}
func (c fakePodManifestEtcdClient) GetClusterStatus() (map[string]*clientv3.StatusResponse, error) {
// Make sure the certificates generated from the upgrade are readable from disk
tlsInfo := transport.TLSInfo{
CertFile: filepath.Join(c.CertificatesDir, constants.EtcdCACertName),
KeyFile: filepath.Join(c.CertificatesDir, constants.EtcdHealthcheckClientCertName),
TrustedCAFile: filepath.Join(c.CertificatesDir, constants.EtcdHealthcheckClientKeyName),
}
_, err := tlsInfo.ClientConfig()
if err != nil {
return nil, err
}
return map[string]*clientv3.StatusResponse{
"foo": {Version: "3.1.12"},
}, nil
}
func (c fakePodManifestEtcdClient) GetClusterVersions() (map[string]string, error) {
return map[string]string{
"foo": "3.1.12",
}, nil
}
func (c fakePodManifestEtcdClient) GetVersion() (string, error) {
return "3.1.12", nil
}
func TestStaticPodControlPlane(t *testing.T) {
tests := []struct {
description string
waitErrsToReturn map[string]error
moveFileFunc func(string, string) error
expectedErr bool
manifestShouldChange bool
}{
{
description: "error-free case should succeed",
waitErrsToReturn: map[string]error{
waitForHashes: nil,
waitForHashChange: nil,
waitForPodsWithLabel: nil,
},
moveFileFunc: func(oldPath, newPath string) error {
return os.Rename(oldPath, newPath)
},
expectedErr: false,
manifestShouldChange: true,
},
{
description: "any wait error should result in a rollback and an abort",
waitErrsToReturn: map[string]error{
waitForHashes: fmt.Errorf("boo! failed"),
waitForHashChange: nil,
waitForPodsWithLabel: nil,
},
moveFileFunc: func(oldPath, newPath string) error {
return os.Rename(oldPath, newPath)
},
expectedErr: true,
manifestShouldChange: false,
},
{
description: "any wait error should result in a rollback and an abort",
waitErrsToReturn: map[string]error{
waitForHashes: nil,
waitForHashChange: fmt.Errorf("boo! failed"),
waitForPodsWithLabel: nil,
},
moveFileFunc: func(oldPath, newPath string) error {
return os.Rename(oldPath, newPath)
},
expectedErr: true,
manifestShouldChange: false,
},
{
description: "any wait error should result in a rollback and an abort",
waitErrsToReturn: map[string]error{
waitForHashes: nil,
waitForHashChange: nil,
waitForPodsWithLabel: fmt.Errorf("boo! failed"),
},
moveFileFunc: func(oldPath, newPath string) error {
return os.Rename(oldPath, newPath)
},
expectedErr: true,
manifestShouldChange: false,
},
{
description: "any path-moving error should result in a rollback and an abort",
waitErrsToReturn: map[string]error{
waitForHashes: nil,
waitForHashChange: nil,
waitForPodsWithLabel: nil,
},
moveFileFunc: func(oldPath, newPath string) error {
// fail for kube-apiserver move
if strings.Contains(newPath, "kube-apiserver") {
return fmt.Errorf("moving the kube-apiserver file failed")
}
return os.Rename(oldPath, newPath)
},
expectedErr: true,
manifestShouldChange: false,
},
{
description: "any path-moving error should result in a rollback and an abort",
waitErrsToReturn: map[string]error{
waitForHashes: nil,
waitForHashChange: nil,
waitForPodsWithLabel: nil,
},
moveFileFunc: func(oldPath, newPath string) error {
// fail for kube-controller-manager move
if strings.Contains(newPath, "kube-controller-manager") {
return fmt.Errorf("moving the kube-apiserver file failed")
}
return os.Rename(oldPath, newPath)
},
expectedErr: true,
manifestShouldChange: false,
},
{
description: "any path-moving error should result in a rollback and an abort; even though this is the last component (kube-apiserver and kube-controller-manager healthy)",
waitErrsToReturn: map[string]error{
waitForHashes: nil,
waitForHashChange: nil,
waitForPodsWithLabel: nil,
},
moveFileFunc: func(oldPath, newPath string) error {
// fail for kube-scheduler move
if strings.Contains(newPath, "kube-scheduler") {
return fmt.Errorf("moving the kube-apiserver file failed")
}
return os.Rename(oldPath, newPath)
},
expectedErr: true,
manifestShouldChange: false,
},
}
for _, rt := range tests {
waiter := NewFakeStaticPodWaiter(rt.waitErrsToReturn)
pathMgr, err := NewFakeStaticPodPathManager(rt.moveFileFunc)
if err != nil {
t.Fatalf("couldn't run NewFakeStaticPodPathManager: %v", err)
}
defer os.RemoveAll(pathMgr.(*fakeStaticPodPathManager).KubernetesDir())
constants.KubernetesDir = pathMgr.(*fakeStaticPodPathManager).KubernetesDir()
tempCertsDir, err := ioutil.TempDir("", "kubeadm-certs")
if err != nil {
t.Fatalf("couldn't create temporary certificates directory: %v", err)
}
defer os.RemoveAll(tempCertsDir)
tmpEtcdDataDir, err := ioutil.TempDir("", "kubeadm-etcd-data")
if err != nil {
t.Fatalf("couldn't create temporary etcd data directory: %v", err)
}
defer os.RemoveAll(tmpEtcdDataDir)
oldcfg, err := getConfig("v1.9.0", tempCertsDir, tmpEtcdDataDir)
if err != nil {
t.Fatalf("couldn't create config: %v", err)
}
// Initialize PKI minus any etcd certificates to simulate etcd PKI upgrade
certActions := []func(cfg *kubeadmapi.MasterConfiguration) error{
certsphase.CreateCACertAndKeyFiles,
certsphase.CreateAPIServerCertAndKeyFiles,
certsphase.CreateAPIServerKubeletClientCertAndKeyFiles,
// certsphase.CreateEtcdCACertAndKeyFiles,
// certsphase.CreateEtcdServerCertAndKeyFiles,
// certsphase.CreateEtcdPeerCertAndKeyFiles,
// certsphase.CreateEtcdHealthcheckClientCertAndKeyFiles,
// certsphase.CreateAPIServerEtcdClientCertAndKeyFiles,
certsphase.CreateServiceAccountKeyAndPublicKeyFiles,
certsphase.CreateFrontProxyCACertAndKeyFiles,
certsphase.CreateFrontProxyClientCertAndKeyFiles,
}
for _, action := range certActions {
err := action(oldcfg)
if err != nil {
t.Fatalf("couldn't initialize pre-upgrade certificate: %v", err)
}
}
fmt.Printf("Wrote certs to %s\n", oldcfg.CertificatesDir)
// Initialize the directory with v1.7 manifests; should then be upgraded to v1.8 using the method
err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), oldcfg)
if err != nil {
t.Fatalf("couldn't run CreateInitStaticPodManifestFiles: %v", err)
}
err = etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.RealManifestDir(), oldcfg)
if err != nil {
t.Fatalf("couldn't run CreateLocalEtcdStaticPodManifestFile: %v", err)
}
// Get a hash of the v1.7 API server manifest to compare later (was the file re-written)
oldHash, err := getAPIServerHash(pathMgr.RealManifestDir())
if err != nil {
t.Fatalf("couldn't read temp file: %v", err)
}
newcfg, err := getConfig("v1.10.0", tempCertsDir, tmpEtcdDataDir)
if err != nil {
t.Fatalf("couldn't create config: %v", err)
}
actualErr := StaticPodControlPlane(
waiter,
pathMgr,
newcfg,
true,
fakeTLSEtcdClient{
TLS: false,
},
fakePodManifestEtcdClient{
ManifestDir: pathMgr.RealManifestDir(),
CertificatesDir: newcfg.CertificatesDir,
},
)
if (actualErr != nil) != rt.expectedErr {
t.Errorf(
"failed UpgradeStaticPodControlPlane\n%s\n\texpected error: %t\n\tgot: %t\n\tactual error: %v",
rt.description,
rt.expectedErr,
(actualErr != nil),
actualErr,
)
}
newHash, err := getAPIServerHash(pathMgr.RealManifestDir())
if err != nil {
t.Fatalf("couldn't read temp file: %v", err)
}
if (oldHash != newHash) != rt.manifestShouldChange {
t.Errorf(
"failed StaticPodControlPlane\n%s\n\texpected manifest change: %t\n\tgot: %t",
rt.description,
rt.manifestShouldChange,
(oldHash != newHash),
)
}
return
}
}
func getAPIServerHash(dir string) (string, error) {
manifestPath := constants.GetStaticPodFilepath(constants.KubeAPIServer, dir)
fileBytes, err := ioutil.ReadFile(manifestPath)
if err != nil {
return "", err
}
return fmt.Sprintf("%x", sha256.Sum256(fileBytes)), nil
}
// TODO: Make this test function use the rest of the "official" API machinery helper funcs we have inside of kubeadm
func getConfig(version, certsDir, etcdDataDir string) (*kubeadmapi.MasterConfiguration, error) {
externalcfg := &kubeadmapiv1alpha2.MasterConfiguration{}
internalcfg := &kubeadmapi.MasterConfiguration{}
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(fmt.Sprintf(testConfiguration, certsDir, etcdDataDir, version)), externalcfg); err != nil {
return nil, fmt.Errorf("unable to decode config: %v", err)
}
kubeadmscheme.Scheme.Convert(externalcfg, internalcfg, nil)
return internalcfg, nil
}
func getTempDir(t *testing.T, name string) (string, func()) {
dir, err := ioutil.TempDir(os.TempDir(), name)
if err != nil {
t.Fatalf("couldn't make temporary directory: %v", err)
}
return dir, func() {
os.RemoveAll(dir)
}
}
func TestCleanupDirs(t *testing.T) {
tests := []struct {
name string
keepManifest, keepEtcd bool
}{
{
name: "save manifest backup",
keepManifest: true,
},
{
name: "save both etcd and manifest",
keepManifest: true,
keepEtcd: true,
},
{
name: "save nothing",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
realManifestDir, cleanup := getTempDir(t, "realManifestDir")
defer cleanup()
tempManifestDir, cleanup := getTempDir(t, "tempManifestDir")
defer cleanup()
backupManifestDir, cleanup := getTempDir(t, "backupManifestDir")
defer cleanup()
backupEtcdDir, cleanup := getTempDir(t, "backupEtcdDir")
defer cleanup()
mgr := NewKubeStaticPodPathManager(realManifestDir, tempManifestDir, backupManifestDir, backupEtcdDir, test.keepManifest, test.keepEtcd)
err := mgr.CleanupDirs()
if err != nil {
t.Errorf("unexpected error cleaning up: %v", err)
}
if _, err := os.Stat(tempManifestDir); !os.IsNotExist(err) {
t.Errorf("%q should not have existed", tempManifestDir)
}
_, err = os.Stat(backupManifestDir)
if test.keepManifest {
if err != nil {
t.Errorf("unexpected error getting backup manifest dir")
}
} else {
if !os.IsNotExist(err) {
t.Error("expected backup manifest to not exist")
}
}
_, err = os.Stat(backupEtcdDir)
if test.keepEtcd {
if err != nil {
t.Errorf("unexpected error getting backup etcd dir")
}
} else {
if !os.IsNotExist(err) {
t.Error("expected backup etcd dir to not exist")
}
}
})
}
}

View File

@@ -0,0 +1,151 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package upgrade
import (
"fmt"
"io"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
versionutil "k8s.io/kubernetes/pkg/util/version"
"k8s.io/kubernetes/pkg/version"
)
// VersionGetter defines an interface for fetching different versions.
// Easy to implement a fake variant of this interface for unit testing
type VersionGetter interface {
// ClusterVersion should return the version of the cluster i.e. the API Server version
ClusterVersion() (string, *versionutil.Version, error)
// KubeadmVersion should return the version of the kubeadm CLI
KubeadmVersion() (string, *versionutil.Version, error)
// VersionFromCILabel should resolve CI labels like `latest`, `stable`, `stable-1.8`, etc. to real versions
VersionFromCILabel(string, string) (string, *versionutil.Version, error)
// KubeletVersions should return a map with a version and a number that describes how many kubelets there are for that version
KubeletVersions() (map[string]uint16, error)
}
// KubeVersionGetter handles the version-fetching mechanism from external sources
type KubeVersionGetter struct {
client clientset.Interface
w io.Writer
}
// NewKubeVersionGetter returns a new instance of KubeVersionGetter
func NewKubeVersionGetter(client clientset.Interface, writer io.Writer) VersionGetter {
return &KubeVersionGetter{
client: client,
w: writer,
}
}
// ClusterVersion gets API server version
func (g *KubeVersionGetter) ClusterVersion() (string, *versionutil.Version, error) {
clusterVersionInfo, err := g.client.Discovery().ServerVersion()
if err != nil {
return "", nil, fmt.Errorf("Couldn't fetch cluster version from the API Server: %v", err)
}
fmt.Fprintf(g.w, "[upgrade/versions] Cluster version: %s\n", clusterVersionInfo.String())
clusterVersion, err := versionutil.ParseSemantic(clusterVersionInfo.String())
if err != nil {
return "", nil, fmt.Errorf("Couldn't parse cluster version: %v", err)
}
return clusterVersionInfo.String(), clusterVersion, nil
}
// KubeadmVersion gets kubeadm version
func (g *KubeVersionGetter) KubeadmVersion() (string, *versionutil.Version, error) {
kubeadmVersionInfo := version.Get()
fmt.Fprintf(g.w, "[upgrade/versions] kubeadm version: %s\n", kubeadmVersionInfo.String())
kubeadmVersion, err := versionutil.ParseSemantic(kubeadmVersionInfo.String())
if err != nil {
return "", nil, fmt.Errorf("Couldn't parse kubeadm version: %v", err)
}
return kubeadmVersionInfo.String(), kubeadmVersion, nil
}
// VersionFromCILabel resolves a version label like "latest" or "stable" to an actual version using the public Kubernetes CI uploads
func (g *KubeVersionGetter) VersionFromCILabel(ciVersionLabel, description string) (string, *versionutil.Version, error) {
versionStr, err := kubeadmutil.KubernetesReleaseVersion(ciVersionLabel)
if err != nil {
return "", nil, fmt.Errorf("Couldn't fetch latest %s from the internet: %v", description, err)
}
if description != "" {
fmt.Fprintf(g.w, "[upgrade/versions] Latest %s: %s\n", description, versionStr)
}
ver, err := versionutil.ParseSemantic(versionStr)
if err != nil {
return "", nil, fmt.Errorf("Couldn't parse latest %s: %v", description, err)
}
return versionStr, ver, nil
}
// KubeletVersions gets the versions of the kubelets in the cluster
func (g *KubeVersionGetter) KubeletVersions() (map[string]uint16, error) {
nodes, err := g.client.CoreV1().Nodes().List(metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("couldn't list all nodes in cluster")
}
return computeKubeletVersions(nodes.Items), nil
}
// computeKubeletVersions returns a string-int map that describes how many nodes are of a specific version
func computeKubeletVersions(nodes []v1.Node) map[string]uint16 {
kubeletVersions := map[string]uint16{}
for _, node := range nodes {
kver := node.Status.NodeInfo.KubeletVersion
if _, found := kubeletVersions[kver]; !found {
kubeletVersions[kver] = 1
continue
}
kubeletVersions[kver]++
}
return kubeletVersions
}
// OfflineVersionGetter will use the version provided or
type OfflineVersionGetter struct {
VersionGetter
version string
}
// NewOfflineVersionGetter wraps a VersionGetter and skips online communication if default information is supplied.
// Version can be "" and the behavior will be identical to the versionGetter passed in.
func NewOfflineVersionGetter(versionGetter VersionGetter, version string) VersionGetter {
return &OfflineVersionGetter{
VersionGetter: versionGetter,
version: version,
}
}
// VersionFromCILabel will return the version that was passed into the struct
func (o *OfflineVersionGetter) VersionFromCILabel(ciVersionLabel, description string) (string, *versionutil.Version, error) {
if o.version == "" {
return o.VersionGetter.VersionFromCILabel(ciVersionLabel, description)
}
ver, err := versionutil.ParseSemantic(o.version)
if err != nil {
return "", nil, fmt.Errorf("Couldn't parse version %s: %v", description, err)
}
return o.version, ver, nil
}

View File

@@ -0,0 +1,54 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = ["uploadconfig.go"],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)
go_test(
name = "go_default_test",
srcs = ["uploadconfig_test.go"],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/client-go/testing:go_default_library",
],
)

View File

@@ -0,0 +1,62 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package uploadconfig
import (
"fmt"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
)
// UploadConfiguration saves the MasterConfiguration used for later reference (when upgrading for instance)
func UploadConfiguration(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
fmt.Printf("[uploadconfig] storing the configuration used in ConfigMap %q in the %q Namespace\n", kubeadmconstants.MasterConfigurationConfigMap, metav1.NamespaceSystem)
// Convert cfg to the external version as that's the only version of the API that can be deserialized later
externalcfg := &kubeadmapiv1alpha2.MasterConfiguration{}
kubeadmscheme.Scheme.Convert(cfg, externalcfg, nil)
// Removes sensitive info from the data that will be stored in the config map
externalcfg.BootstrapTokens = nil
// Clear the NodeRegistration object.
externalcfg.NodeRegistration = kubeadmapiv1alpha2.NodeRegistrationOptions{}
cfgYaml, err := util.MarshalToYamlForCodecs(externalcfg, kubeadmapiv1alpha2.SchemeGroupVersion, scheme.Codecs)
if err != nil {
return err
}
return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: kubeadmconstants.MasterConfigurationConfigMap,
Namespace: metav1.NamespaceSystem,
},
Data: map[string]string{
kubeadmconstants.MasterConfigurationConfigMapKey: string(cfgYaml),
},
})
}

View File

@@ -0,0 +1,144 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package uploadconfig
import (
"testing"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientsetfake "k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
)
func TestUploadConfiguration(t *testing.T) {
tests := []struct {
name string
errOnCreate error
errOnUpdate error
updateExisting bool
errExpected bool
verifyResult bool
}{
{
name: "basic validation with correct key",
verifyResult: true,
},
{
name: "update existing should report no error",
updateExisting: true,
verifyResult: true,
},
{
name: "unexpected errors for create should be returned",
errOnCreate: apierrors.NewUnauthorized(""),
errExpected: true,
},
{
name: "update existing show report error if unexpected error for update is returned",
errOnUpdate: apierrors.NewUnauthorized(""),
updateExisting: true,
errExpected: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := &kubeadmapi.MasterConfiguration{
KubernetesVersion: "v1.10.3",
BootstrapTokens: []kubeadmapi.BootstrapToken{
{
Token: &kubeadmapi.BootstrapTokenString{
ID: "abcdef",
Secret: "abcdef0123456789",
},
},
},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{
Name: "node-foo",
CRISocket: "/var/run/custom-cri.sock",
},
}
client := clientsetfake.NewSimpleClientset()
if tt.errOnCreate != nil {
client.PrependReactor("create", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, tt.errOnCreate
})
}
// For idempotent test, we check the result of the second call.
if err := UploadConfiguration(cfg, client); !tt.updateExisting && (err != nil) != tt.errExpected {
t.Errorf("UploadConfiguration() error = %v, wantErr %v", err, tt.errExpected)
}
if tt.updateExisting {
if tt.errOnUpdate != nil {
client.PrependReactor("update", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
return true, nil, tt.errOnUpdate
})
}
if err := UploadConfiguration(cfg, client); (err != nil) != tt.errExpected {
t.Errorf("UploadConfiguration() error = %v", err)
}
}
if tt.verifyResult {
masterCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.MasterConfigurationConfigMap, metav1.GetOptions{})
if err != nil {
t.Errorf("Fail to query ConfigMap error = %v", err)
}
configData := masterCfg.Data[kubeadmconstants.MasterConfigurationConfigMapKey]
if configData == "" {
t.Errorf("Fail to find ConfigMap key")
}
decodedExtCfg := &kubeadmapiv1alpha2.MasterConfiguration{}
decodedCfg := &kubeadmapi.MasterConfiguration{}
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(configData), decodedExtCfg); err != nil {
t.Errorf("unable to decode config from bytes: %v", err)
}
// Default and convert to the internal version
kubeadmscheme.Scheme.Default(decodedExtCfg)
kubeadmscheme.Scheme.Convert(decodedExtCfg, decodedCfg, nil)
if decodedCfg.KubernetesVersion != cfg.KubernetesVersion {
t.Errorf("Decoded value doesn't match, decoded = %#v, expected = %#v", decodedCfg.KubernetesVersion, cfg.KubernetesVersion)
}
// If the decoded cfg has a BootstrapTokens array, verify the sensitive information we had isn't still there.
if len(decodedCfg.BootstrapTokens) > 0 && decodedCfg.BootstrapTokens[0].Token != nil && decodedCfg.BootstrapTokens[0].Token.String() == cfg.BootstrapTokens[0].Token.String() {
t.Errorf("Decoded value contains .BootstrapTokens (sensitive info), decoded = %#v, expected = empty", decodedCfg.BootstrapTokens)
}
// Make sure no information from NodeRegistrationOptions was uploaded.
if decodedCfg.NodeRegistration.Name == cfg.NodeRegistration.Name || decodedCfg.NodeRegistration.CRISocket != kubeadmapiv1alpha2.DefaultCRISocket {
t.Errorf("Decoded value contains .NodeRegistration (node-specific info shouldn't be uploaded), decoded = %#v, expected = empty", decodedCfg.NodeRegistration)
}
if decodedExtCfg.Kind != "MasterConfiguration" {
t.Errorf("Expected kind MasterConfiguration, got %v", decodedExtCfg.Kind)
}
if decodedExtCfg.APIVersion != "kubeadm.k8s.io/v1alpha2" {
t.Errorf("Expected apiVersion kubeadm.k8s.io/v1alpha2, got %v", decodedExtCfg.APIVersion)
}
}
})
}
}