Bumping k8s dependencies to 1.13

This commit is contained in:
Cheng Xing
2018-11-16 14:08:25 -08:00
parent 305407125c
commit b4c0b68ec7
8002 changed files with 884099 additions and 276228 deletions

View File

@@ -14,12 +14,12 @@ go_test(
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//pkg/apis/core:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/client-go/testing:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
"//staging/src/k8s.io/client-go/testing:go_default_library",
],
)
@@ -36,15 +36,15 @@ go_library(
"//cmd/kubeadm/app/features:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/rbac/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//vendor/github.com/mholt/caddy/caddyfile:go_default_library",
"//vendor/k8s.io/api/apps/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/rbac/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
],
)

View File

@@ -19,7 +19,6 @@ package dns
import (
"encoding/json"
"fmt"
"runtime"
"strings"
"github.com/mholt/caddy/caddyfile"
@@ -70,14 +69,14 @@ func DeployedDNSAddon(client clientset.Interface) (string, string, error) {
}
// EnsureDNSAddon creates the kube-dns or CoreDNS addon
func EnsureDNSAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
func EnsureDNSAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error {
if features.Enabled(cfg.FeatureGates, features.CoreDNS) {
return coreDNSAddon(cfg, client)
}
return kubeDNSAddon(cfg, client)
}
func kubeDNSAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
func kubeDNSAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error {
if err := CreateServiceAccount(client); err != nil {
return err
}
@@ -97,9 +96,8 @@ func kubeDNSAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Interfac
}
dnsDeploymentBytes, err := kubeadmutil.ParseTemplate(KubeDNSDeployment,
struct{ ImageRepository, Arch, Version, DNSBindAddr, DNSProbeAddr, DNSDomain, MasterTaintKey string }{
struct{ ImageRepository, Version, DNSBindAddr, DNSProbeAddr, DNSDomain, MasterTaintKey string }{
ImageRepository: cfg.ImageRepository,
Arch: runtime.GOARCH,
Version: kubeadmconstants.KubeDNSVersion,
DNSBindAddr: dnsBindAddr,
DNSProbeAddr: dnsProbeAddr,
@@ -150,7 +148,7 @@ func createKubeDNSAddon(deploymentBytes, serviceBytes []byte, client clientset.I
return createDNSService(kubednsService, serviceBytes, client)
}
func coreDNSAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
func coreDNSAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error {
// Get the YAML manifest
coreDNSDeploymentBytes, err := kubeadmutil.ParseTemplate(CoreDNSDeployment, struct{ ImageRepository, MasterTaintKey, Version string }{
ImageRepository: cfg.ImageRepository,
@@ -310,6 +308,7 @@ func translateStubDomainOfKubeDNSToProxyCoreDNS(dataField string, kubeDNSConfigM
pStanza["body"] = [][]string{
{"errors"},
{"cache", "30"},
{"loop"},
append([]string{"proxy", "."}, proxyIP...),
}
proxyStanza = append(proxyStanza, pStanza)

View File

@@ -95,9 +95,8 @@ func TestCompileManifests(t *testing.T) {
}{
{
manifest: KubeDNSDeployment,
data: struct{ ImageRepository, Arch, Version, DNSBindAddr, DNSProbeAddr, DNSDomain, MasterTaintKey string }{
data: struct{ ImageRepository, Version, DNSBindAddr, DNSProbeAddr, DNSDomain, MasterTaintKey string }{
ImageRepository: "foo",
Arch: "foo",
Version: "foo",
DNSBindAddr: "foo",
DNSProbeAddr: "foo",
@@ -204,24 +203,28 @@ func TestTranslateStubDomainKubeDNSToCoreDNS(t *testing.T) {
foo.com:53 {
errors
cache 30
loop
proxy . 1.2.3.4:5300 3.3.3.3
}
my.cluster.local:53 {
errors
cache 30
loop
proxy . 2.3.4.5
}`,
expectTwo: `
my.cluster.local:53 {
errors
cache 30
loop
proxy . 2.3.4.5
}
foo.com:53 {
errors
cache 30
loop
proxy . 1.2.3.4:5300 3.3.3.3
}`,
},
@@ -251,24 +254,28 @@ func TestTranslateStubDomainKubeDNSToCoreDNS(t *testing.T) {
foo.com:53 {
errors
cache 30
loop
proxy . 1.2.3.4:5300
}
my.cluster.local:53 {
errors
cache 30
loop
proxy . 2.3.4.5
}`,
expectTwo: `
my.cluster.local:53 {
errors
cache 30
loop
proxy . 2.3.4.5
}
foo.com:53 {
errors
cache 30
loop
proxy . 1.2.3.4:5300
}`,
},

View File

@@ -50,7 +50,7 @@ spec:
optional: true
containers:
- name: kubedns
image: {{ .ImageRepository }}/k8s-dns-kube-dns-{{ .Arch }}:{{ .Version }}
image: {{ .ImageRepository }}/k8s-dns-kube-dns:{{ .Version }}
imagePullPolicy: IfNotPresent
resources:
# TODO: Set memory limits when we've profiled the container for large
@@ -102,7 +102,7 @@ spec:
- name: kube-dns-config
mountPath: /kube-dns-config
- name: dnsmasq
image: {{ .ImageRepository }}/k8s-dns-dnsmasq-nanny-{{ .Arch }}:{{ .Version }}
image: {{ .ImageRepository }}/k8s-dns-dnsmasq-nanny:{{ .Version }}
imagePullPolicy: IfNotPresent
livenessProbe:
httpGet:
@@ -122,6 +122,7 @@ spec:
- -k
- --cache-size=1000
- --no-negcache
- --dns-loop-detect
- --log-facility=-
- --server=/{{ .DNSDomain }}/{{ .DNSBindAddr }}#10053
- --server=/in-addr.arpa/{{ .DNSBindAddr }}#10053
@@ -142,7 +143,7 @@ spec:
- name: kube-dns-config
mountPath: /etc/k8s/dns/dnsmasq-nanny
- name: sidecar
image: {{ .ImageRepository }}/k8s-dns-sidecar-{{ .Arch }}:{{ .Version }}
image: {{ .ImageRepository }}/k8s-dns-sidecar:{{ .Version }}
imagePullPolicy: IfNotPresent
livenessProbe:
httpGet:
@@ -173,8 +174,6 @@ spec:
operator: Exists
- key: {{ .MasterTaintKey }}
effect: NoSchedule
nodeSelector:
beta.kubernetes.io/arch: {{ .Arch }}
`
// KubeDNSService is the kube-dns Service manifest
@@ -189,6 +188,7 @@ metadata:
name: kube-dns
namespace: kube-system
annotations:
prometheus.io/port: "9153"
prometheus.io/scrape: "true"
# Without this resourceVersion value, an update of the Service between versions will yield:
# Service "kube-dns" is invalid: metadata.resourceVersion: Invalid value: "": must be specified for an update
@@ -309,7 +309,9 @@ data:
prometheus :9153
proxy . {{ .UpstreamNameserver }}
cache 30
loop
reload
loadbalance
}{{ .StubDomain }}
`
// CoreDNSClusterRole is the CoreDNS ClusterRole manifest

View File

@@ -11,17 +11,17 @@ go_test(
srcs = ["proxy_test.go"],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha3:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/config:go_default_library",
"//pkg/apis/core:go_default_library",
"//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
"//pkg/util/pointer:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/client-go/testing:go_default_library",
"//pkg/proxy/apis/config:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
"//staging/src/k8s.io/client-go/testing:go_default_library",
"//vendor/k8s.io/utils/pointer:go_default_library",
],
)
@@ -34,18 +34,19 @@ go_library(
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/componentconfigs:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/images:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//pkg/proxy/apis/kubeproxyconfig/scheme:go_default_library",
"//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
"//vendor/k8s.io/api/apps/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/rbac/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//pkg/apis/rbac/v1:go_default_library",
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/rbac/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
],
)

View File

@@ -22,7 +22,7 @@ const (
kind: ConfigMap
apiVersion: v1
metadata:
name: kube-proxy
name: {{ .ProxyConfigMap }}
namespace: kube-system
labels:
app: kube-proxy
@@ -46,7 +46,7 @@ data:
- name: default
user:
tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
config.conf: |-
{{ .ProxyConfigMapKey }}: |-
{{ .ProxyConfig}}
`
@@ -69,14 +69,17 @@ spec:
metadata:
labels:
k8s-app: kube-proxy
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ""
spec:
priorityClassName: system-node-critical
containers:
- name: kube-proxy
image: {{ if .ImageOverride }}{{ .ImageOverride }}{{ else }}{{ .ImageRepository }}/kube-proxy-{{ .Arch }}:{{ .Version }}{{ end }}
image: {{ .Image }}
imagePullPolicy: IfNotPresent
command:
- /usr/local/bin/kube-proxy
- --config=/var/lib/kube-proxy/config.conf
- --config=/var/lib/kube-proxy/{{ .ProxyConfigMapKey }}
securityContext:
privileged: true
volumeMounts:
@@ -93,7 +96,7 @@ spec:
volumes:
- name: kube-proxy
configMap:
name: kube-proxy
name: {{ .ProxyConfigMap }}
- name: xtables-lock
hostPath:
path: /run/xtables.lock
@@ -104,9 +107,6 @@ spec:
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- key: {{ .MasterTaintKey }}
effect: NoSchedule
nodeSelector:
beta.kubernetes.io/arch: {{ .Arch }}
- operator: Exists
`
)

View File

@@ -19,7 +19,6 @@ package proxy
import (
"bytes"
"fmt"
"runtime"
apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
@@ -29,11 +28,12 @@ import (
clientset "k8s.io/client-go/kubernetes"
clientsetscheme "k8s.io/client-go/kubernetes/scheme"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/images"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
kubeproxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"
kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
)
const (
@@ -46,19 +46,18 @@ const (
)
// EnsureProxyAddon creates the kube-proxy addons
func EnsureProxyAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
func EnsureProxyAddon(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error {
if err := CreateServiceAccount(client); err != nil {
return fmt.Errorf("error when creating kube-proxy service account: %v", err)
}
// Generate Master Enpoint kubeconfig file
masterEndpoint, err := kubeadmutil.GetMasterEndpoint(&cfg.API)
masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg)
if err != nil {
return err
}
proxyBytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg.KubeProxy.Config, kubeproxyconfigv1alpha1.SchemeGroupVersion,
kubeproxyconfigscheme.Codecs)
proxyBytes, err := componentconfigs.Known[componentconfigs.KubeProxyConfigurationKind].Marshal(cfg.ComponentConfigs.KubeProxy)
if err != nil {
return fmt.Errorf("error when marshaling: %v", err)
}
@@ -67,21 +66,23 @@ func EnsureProxyAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Inte
var proxyConfigMapBytes, proxyDaemonSetBytes []byte
proxyConfigMapBytes, err = kubeadmutil.ParseTemplate(KubeProxyConfigMap19,
struct {
MasterEndpoint string
ProxyConfig string
MasterEndpoint string
ProxyConfig string
ProxyConfigMap string
ProxyConfigMapKey string
}{
MasterEndpoint: masterEndpoint,
ProxyConfig: prefixBytes.String(),
MasterEndpoint: masterEndpoint,
ProxyConfig: prefixBytes.String(),
ProxyConfigMap: constants.KubeProxyConfigMap,
ProxyConfigMapKey: constants.KubeProxyConfigMapKey,
})
if err != nil {
return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err)
}
proxyDaemonSetBytes, err = kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ ImageRepository, Arch, Version, ImageOverride, MasterTaintKey string }{
ImageRepository: cfg.GetControlPlaneImageRepository(),
Arch: runtime.GOARCH,
Version: kubeadmutil.KubernetesVersionToImageTag(cfg.KubernetesVersion),
ImageOverride: cfg.UnifiedControlPlaneImage,
MasterTaintKey: kubeadmconstants.LabelNodeRoleMaster,
proxyDaemonSetBytes, err = kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ Image, ProxyConfigMap, ProxyConfigMapKey string }{
Image: images.GetKubeControlPlaneImage(constants.KubeProxy, &cfg.ClusterConfiguration),
ProxyConfigMap: constants.KubeProxyConfigMap,
ProxyConfigMapKey: constants.KubeProxyConfigMapKey,
})
if err != nil {
return fmt.Errorf("error when parsing kube-proxy daemonset template: %v", err)
@@ -134,7 +135,7 @@ func createKubeProxyAddon(configMapBytes, daemonSetbytes []byte, client clientse
}
func createClusterRoleBindings(client clientset.Interface) error {
return apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{
if err := apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "kubeadm:node-proxier",
},
@@ -150,5 +151,39 @@ func createClusterRoleBindings(client clientset.Interface) error {
Namespace: metav1.NamespaceSystem,
},
},
}); err != nil {
return err
}
// Create a role for granting read only access to the kube-proxy component config ConfigMap
if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{
ObjectMeta: metav1.ObjectMeta{
Name: constants.KubeProxyConfigMap,
Namespace: metav1.NamespaceSystem,
},
Rules: []rbac.PolicyRule{
rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(constants.KubeProxyConfigMap).RuleOrDie(),
},
}); err != nil {
return err
}
// Bind the role to bootstrap tokens for allowing fetchConfiguration during join
return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: constants.KubeProxyConfigMap,
Namespace: metav1.NamespaceSystem,
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "Role",
Name: constants.KubeProxyConfigMap,
},
Subjects: []rbac.Subject{
{
Kind: rbac.GroupKind,
Name: constants.NodeBootstrapTokenAuthGroup,
},
},
})
}

View File

@@ -26,12 +26,12 @@ import (
"k8s.io/apimachinery/pkg/runtime"
clientsetfake "k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
api "k8s.io/kubernetes/pkg/apis/core"
kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
"k8s.io/kubernetes/pkg/util/pointer"
kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config"
"k8s.io/utils/pointer"
)
func TestCreateServiceAccount(t *testing.T) {
@@ -99,22 +99,21 @@ func TestCompileManifests(t *testing.T) {
{
manifest: KubeProxyConfigMap19,
data: struct {
MasterEndpoint, ProxyConfig string
MasterEndpoint, ProxyConfig, ProxyConfigMap, ProxyConfigMapKey string
}{
MasterEndpoint: "foo",
ProxyConfig: " bindAddress: 0.0.0.0\n clusterCIDR: 192.168.1.1\n enableProfiling: false",
MasterEndpoint: "foo",
ProxyConfig: " bindAddress: 0.0.0.0\n clusterCIDR: 192.168.1.1\n enableProfiling: false",
ProxyConfigMap: "bar",
ProxyConfigMapKey: "baz",
},
expected: true,
},
{
manifest: KubeProxyDaemonSet19,
data: struct{ ImageRepository, Arch, Version, ImageOverride, MasterTaintKey, CloudTaintKey string }{
ImageRepository: "foo",
Arch: "foo",
Version: "foo",
ImageOverride: "foo",
MasterTaintKey: "foo",
CloudTaintKey: "foo",
data: struct{ Image, ProxyConfigMap, ProxyConfigMapKey string }{
Image: "foo",
ProxyConfigMap: "bar",
ProxyConfigMapKey: "baz",
},
expected: true,
},
@@ -173,32 +172,19 @@ func TestEnsureProxyAddon(t *testing.T) {
// Create a fake client and set up default test configuration
client := clientsetfake.NewSimpleClientset()
masterConfig := &kubeadmapiv1alpha2.MasterConfiguration{
API: kubeadmapiv1alpha2.API{
// TODO: Consider using a YAML file instead for this that makes it possible to specify YAML documents for the ComponentConfigs
masterConfig := &kubeadmapiv1alpha3.InitConfiguration{
APIEndpoint: kubeadmapiv1alpha3.APIEndpoint{
AdvertiseAddress: "1.2.3.4",
BindPort: 1234,
},
KubeProxy: kubeadmapiv1alpha2.KubeProxy{
Config: &kubeproxyconfigv1alpha1.KubeProxyConfiguration{
BindAddress: "",
HealthzBindAddress: "0.0.0.0:10256",
MetricsBindAddress: "127.0.0.1:10249",
Conntrack: kubeproxyconfigv1alpha1.KubeProxyConntrackConfiguration{
Max: pointer.Int32Ptr(2),
MaxPerCore: pointer.Int32Ptr(1),
Min: pointer.Int32Ptr(1),
TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second},
TCPCloseWaitTimeout: &metav1.Duration{Duration: 5 * time.Second},
},
ClusterConfiguration: kubeadmapiv1alpha3.ClusterConfiguration{
Networking: kubeadmapiv1alpha3.Networking{
PodSubnet: "5.6.7.8/24",
},
ImageRepository: "someRepo",
KubernetesVersion: "v1.11.0",
},
Networking: kubeadmapiv1alpha2.Networking{
PodSubnet: "5.6.7.8/24",
},
ImageRepository: "someRepo",
KubernetesVersion: "v1.10.0",
UnifiedControlPlaneImage: "someImage",
}
// Simulate an error if necessary
@@ -208,16 +194,32 @@ func TestEnsureProxyAddon(t *testing.T) {
return true, nil, apierrors.NewUnauthorized("")
})
case InvalidMasterEndpoint:
masterConfig.API.AdvertiseAddress = "1.2.3"
masterConfig.APIEndpoint.AdvertiseAddress = "1.2.3"
case IPv6SetBindAddress:
masterConfig.API.AdvertiseAddress = "1:2::3:4"
masterConfig.APIEndpoint.AdvertiseAddress = "1:2::3:4"
masterConfig.Networking.PodSubnet = "2001:101::/96"
}
kubeadmapiv1alpha2.SetDefaults_MasterConfiguration(masterConfig)
intMaster, err := cmdutil.ConfigFileAndDefaultsToInternalConfig("", masterConfig)
intMaster, err := configutil.ConfigFileAndDefaultsToInternalConfig("", masterConfig)
if err != nil {
t.Errorf(" test failed to convert v1alpha1 to internal version")
t.Errorf("test failed to convert external to internal version")
break
}
intMaster.ComponentConfigs.KubeProxy = &kubeproxyconfig.KubeProxyConfiguration{
BindAddress: "",
HealthzBindAddress: "0.0.0.0:10256",
MetricsBindAddress: "127.0.0.1:10249",
Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{
Max: pointer.Int32Ptr(2),
MaxPerCore: pointer.Int32Ptr(1),
Min: pointer.Int32Ptr(1),
TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second},
TCPCloseWaitTimeout: &metav1.Duration{Duration: 5 * time.Second},
},
}
// Run dynamic defaulting again as we changed the internal cfg
if err := configutil.SetInitDynamicDefaults(intMaster); err != nil {
t.Errorf("test failed to set dynamic defaults: %v", err)
break
}
err = EnsureProxyAddon(intMaster, client)
@@ -238,17 +240,17 @@ func TestEnsureProxyAddon(t *testing.T) {
expErr,
actErr)
}
if intMaster.KubeProxy.Config.BindAddress != tc.expBindAddr {
if intMaster.ComponentConfigs.KubeProxy.BindAddress != tc.expBindAddr {
t.Errorf("%s test failed, expected: %s, got: %s",
tc.name,
tc.expBindAddr,
intMaster.KubeProxy.Config.BindAddress)
intMaster.ComponentConfigs.KubeProxy.BindAddress)
}
if intMaster.KubeProxy.Config.ClusterCIDR != tc.expClusterCIDR {
if intMaster.ComponentConfigs.KubeProxy.ClusterCIDR != tc.expClusterCIDR {
t.Errorf("%s test failed, expected: %s, got: %s",
tc.name,
tc.expClusterCIDR,
intMaster.KubeProxy.Config.ClusterCIDR)
intMaster.ComponentConfigs.KubeProxy.ClusterCIDR)
}
}
}

View File

@@ -12,10 +12,10 @@ go_test(
embed = [":go_default_library"],
deps = [
"//pkg/apis/core:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/client-go/testing:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
"//staging/src/k8s.io/client-go/testing:go_default_library",
],
)
@@ -26,15 +26,15 @@ go_library(
deps = [
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//pkg/apis/rbac/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/rbac/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/tools/bootstrap/token/api:go_default_library",
"//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
"//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/rbac/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/tools/bootstrap/token/api:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
],
)

View File

@@ -16,10 +16,10 @@ go_library(
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//vendor/k8s.io/api/rbac/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/tools/bootstrap/token/util:go_default_library",
"//staging/src/k8s.io/api/rbac/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/tools/bootstrap/token/util:go_default_library",
],
)

View File

@@ -8,7 +8,10 @@ load(
go_test(
name = "go_default_test",
srcs = ["certs_test.go"],
srcs = [
"certlist_test.go",
"certs_test.go",
],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
@@ -16,12 +19,14 @@ go_test(
"//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
"//cmd/kubeadm/test:go_default_library",
"//cmd/kubeadm/test/certs:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"certlist.go",
"certs.go",
"doc.go",
],
@@ -30,8 +35,8 @@ go_library(
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/client-go/util/cert:go_default_library",
],
)
@@ -47,6 +52,7 @@ filegroup(
srcs = [
":package-srcs",
"//cmd/kubeadm/app/phases/certs/pkiutil:all-srcs",
"//cmd/kubeadm/app/phases/certs/renewal:all-srcs",
],
tags = ["automanaged"],
)

View File

@@ -0,0 +1,377 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package certs
import (
"crypto/rsa"
"crypto/x509"
"fmt"
certutil "k8s.io/client-go/util/cert"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
)
type configMutatorsFunc func(*kubeadmapi.InitConfiguration, *certutil.Config) error
// KubeadmCert represents a certificate that Kubeadm will create to function properly.
type KubeadmCert struct {
Name string
LongName string
BaseName string
CAName string
// Some attributes will depend on the InitConfiguration, only known at runtime.
// These functions will be run in series, passed both the InitConfiguration and a cert Config.
configMutators []configMutatorsFunc
config certutil.Config
}
// GetConfig returns the definition for the given cert given the provided InitConfiguration
func (k *KubeadmCert) GetConfig(ic *kubeadmapi.InitConfiguration) (*certutil.Config, error) {
for _, f := range k.configMutators {
if err := f(ic, &k.config); err != nil {
return nil, err
}
}
return &k.config, nil
}
// CreateFromCA makes and writes a certificate using the given CA cert and key.
func (k *KubeadmCert) CreateFromCA(ic *kubeadmapi.InitConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) error {
cfg, err := k.GetConfig(ic)
if err != nil {
return fmt.Errorf("couldn't create %q certificate: %v", k.Name, err)
}
cert, key, err := pkiutil.NewCertAndKey(caCert, caKey, cfg)
if err != nil {
return err
}
err = writeCertificateFilesIfNotExist(
ic.CertificatesDir,
k.BaseName,
caCert,
cert,
key,
)
if err != nil {
return fmt.Errorf("failed to write certificate %q: %v", k.Name, err)
}
return nil
}
// CreateAsCA creates a certificate authority, writing the files to disk and also returning the created CA so it can be used to sign child certs.
func (k *KubeadmCert) CreateAsCA(ic *kubeadmapi.InitConfiguration) (*x509.Certificate, *rsa.PrivateKey, error) {
cfg, err := k.GetConfig(ic)
if err != nil {
return nil, nil, fmt.Errorf("couldn't get configuration for %q CA certificate: %v", k.Name, err)
}
caCert, caKey, err := NewCACertAndKey(cfg)
if err != nil {
return nil, nil, fmt.Errorf("couldn't generate %q CA certificate: %v", k.Name, err)
}
err = writeCertificateAuthorithyFilesIfNotExist(
ic.CertificatesDir,
k.BaseName,
caCert,
caKey,
)
if err != nil {
return nil, nil, fmt.Errorf("couldn't write out %q CA certificate: %v", k.Name, err)
}
return caCert, caKey, nil
}
// CertificateTree is represents a one-level-deep tree, mapping a CA to the certs that depend on it.
type CertificateTree map[*KubeadmCert]Certificates
// CreateTree creates the CAs, certs signed by the CAs, and writes them all to disk.
func (t CertificateTree) CreateTree(ic *kubeadmapi.InitConfiguration) error {
for ca, leaves := range t {
cfg, err := ca.GetConfig(ic)
if err != nil {
return err
}
var caKey *rsa.PrivateKey
caCert, err := pkiutil.TryLoadCertFromDisk(ic.CertificatesDir, ca.BaseName)
if err == nil {
// Cert exists already, make sure it's valid
if !caCert.IsCA {
return fmt.Errorf("certificate %q is not a CA", ca.Name)
}
// Try and load a CA Key
caKey, err = pkiutil.TryLoadKeyFromDisk(ic.CertificatesDir, ca.BaseName)
if err != nil {
// If there's no CA key, make sure every certificate exists.
for _, leaf := range leaves {
cl := certKeyLocation{
pkiDir: ic.CertificatesDir,
baseName: leaf.BaseName,
uxName: leaf.Name,
}
if err := validateSignedCertWithCA(cl, caCert); err != nil {
return fmt.Errorf("could not load expected certificate %q or validate the existence of key %q for it: %v", leaf.Name, ca.Name, err)
}
}
// CACert exists and all clients exist, continue to next CA.
continue
}
// CA key exists; just use that to create new certificates.
} else {
// CACert doesn't already exist, create a new cert and key.
caCert, caKey, err = NewCACertAndKey(cfg)
if err != nil {
return err
}
err = writeCertificateAuthorithyFilesIfNotExist(
ic.CertificatesDir,
ca.BaseName,
caCert,
caKey,
)
if err != nil {
return err
}
}
for _, leaf := range leaves {
if err := leaf.CreateFromCA(ic, caCert, caKey); err != nil {
return err
}
}
}
return nil
}
// CertificateMap is a flat map of certificates, keyed by Name.
type CertificateMap map[string]*KubeadmCert
// CertTree returns a one-level-deep tree, mapping a CA cert to an array of certificates that should be signed by it.
func (m CertificateMap) CertTree() (CertificateTree, error) {
caMap := make(CertificateTree)
for _, cert := range m {
if cert.CAName == "" {
if _, ok := caMap[cert]; !ok {
caMap[cert] = []*KubeadmCert{}
}
} else {
ca, ok := m[cert.CAName]
if !ok {
return nil, fmt.Errorf("Certificate %q references unknown CA %q", cert.Name, cert.CAName)
}
caMap[ca] = append(caMap[ca], cert)
}
}
return caMap, nil
}
// Certificates is a list of Certificates that Kubeadm should create.
type Certificates []*KubeadmCert
// AsMap returns the list of certificates as a map, keyed by name.
func (c Certificates) AsMap() CertificateMap {
certMap := make(map[string]*KubeadmCert)
for _, cert := range c {
certMap[cert.Name] = cert
}
return certMap
}
// GetDefaultCertList returns all of the certificates kubeadm requires to function.
func GetDefaultCertList() Certificates {
return Certificates{
&KubeadmCertRootCA,
&KubeadmCertAPIServer,
&KubeadmCertKubeletClient,
// Front Proxy certs
&KubeadmCertFrontProxyCA,
&KubeadmCertFrontProxyClient,
// etcd certs
&KubeadmCertEtcdCA,
&KubeadmCertEtcdServer,
&KubeadmCertEtcdPeer,
&KubeadmCertEtcdHealthcheck,
&KubeadmCertEtcdAPIClient,
}
}
// GetCertsWithoutEtcd returns all of the certificates kubeadm needs when etcd is hosted externally.
func GetCertsWithoutEtcd() Certificates {
return Certificates{
&KubeadmCertRootCA,
&KubeadmCertAPIServer,
&KubeadmCertKubeletClient,
// Front Proxy certs
&KubeadmCertFrontProxyCA,
&KubeadmCertFrontProxyClient,
}
}
var (
// KubeadmCertRootCA is the definition of the Kubernetes Root CA for the API Server and kubelet.
KubeadmCertRootCA = KubeadmCert{
Name: "ca",
LongName: "self-signed kubernetes CA to provision identities for other kuberenets components",
BaseName: kubeadmconstants.CACertAndKeyBaseName,
config: certutil.Config{
CommonName: "kubernetes",
},
}
// KubeadmCertAPIServer is the definition of the cert used to serve the kubernetes API.
KubeadmCertAPIServer = KubeadmCert{
Name: "apiserver",
LongName: "certificate for serving the kubernetes API",
BaseName: kubeadmconstants.APIServerCertAndKeyBaseName,
CAName: "ca",
config: certutil.Config{
CommonName: kubeadmconstants.APIServerCertCommonName,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
},
configMutators: []configMutatorsFunc{
makeAltNamesMutator(pkiutil.GetAPIServerAltNames),
},
}
// KubeadmCertKubeletClient is the definition of the cert used by the API server to access the kubelet.
KubeadmCertKubeletClient = KubeadmCert{
Name: "apiserver-kubelet-client",
LongName: "Client certificate for the API server to connect to kubelet",
BaseName: kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName,
CAName: "ca",
config: certutil.Config{
CommonName: kubeadmconstants.APIServerKubeletClientCertCommonName,
Organization: []string{kubeadmconstants.MastersGroup},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
},
}
// KubeadmCertFrontProxyCA is the definition of the CA used for the front end proxy.
KubeadmCertFrontProxyCA = KubeadmCert{
Name: "front-proxy-ca",
LongName: "self-signed CA to provision identities for front proxy",
BaseName: kubeadmconstants.FrontProxyCACertAndKeyBaseName,
config: certutil.Config{
CommonName: "front-proxy-ca",
},
}
// KubeadmCertFrontProxyClient is the definition of the cert used by the API server to access the front proxy.
KubeadmCertFrontProxyClient = KubeadmCert{
Name: "front-proxy-client",
BaseName: kubeadmconstants.FrontProxyClientCertAndKeyBaseName,
LongName: "client for the front proxy",
CAName: "front-proxy-ca",
config: certutil.Config{
CommonName: kubeadmconstants.FrontProxyClientCertCommonName,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
},
}
// KubeadmCertEtcdCA is the definition of the root CA used by the hosted etcd server.
KubeadmCertEtcdCA = KubeadmCert{
Name: "etcd-ca",
LongName: "self-signed CA to provision identities for etcd",
BaseName: kubeadmconstants.EtcdCACertAndKeyBaseName,
config: certutil.Config{
CommonName: "etcd-ca",
},
}
// KubeadmCertEtcdServer is the definition of the cert used to serve etcd to clients.
KubeadmCertEtcdServer = KubeadmCert{
Name: "etcd-server",
LongName: "certificate for serving etcd",
BaseName: kubeadmconstants.EtcdServerCertAndKeyBaseName,
CAName: "etcd-ca",
config: certutil.Config{
// TODO: etcd 3.2 introduced an undocumented requirement for ClientAuth usage on the
// server cert: https://github.com/coreos/etcd/issues/9785#issuecomment-396715692
// Once the upstream issue is resolved, this should be returned to only allowing
// ServerAuth usage.
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
},
configMutators: []configMutatorsFunc{
makeAltNamesMutator(pkiutil.GetEtcdAltNames),
setCommonNameToNodeName(),
},
}
// KubeadmCertEtcdPeer is the definition of the cert used by etcd peers to access each other.
KubeadmCertEtcdPeer = KubeadmCert{
Name: "etcd-peer",
LongName: "credentials for etcd nodes to communicate with each other",
BaseName: kubeadmconstants.EtcdPeerCertAndKeyBaseName,
CAName: "etcd-ca",
config: certutil.Config{
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
},
configMutators: []configMutatorsFunc{
makeAltNamesMutator(pkiutil.GetEtcdPeerAltNames),
setCommonNameToNodeName(),
},
}
// KubeadmCertEtcdHealthcheck is the definition of the cert used by Kubernetes to check the health of the etcd server.
KubeadmCertEtcdHealthcheck = KubeadmCert{
Name: "etcd-healthcheck-client",
LongName: "client certificate for liveness probes to healtcheck etcd",
BaseName: kubeadmconstants.EtcdHealthcheckClientCertAndKeyBaseName,
CAName: "etcd-ca",
config: certutil.Config{
CommonName: kubeadmconstants.EtcdHealthcheckClientCertCommonName,
Organization: []string{kubeadmconstants.MastersGroup},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
},
}
// KubeadmCertEtcdAPIClient is the definition of the cert used by the API server to access etcd.
KubeadmCertEtcdAPIClient = KubeadmCert{
Name: "apiserver-etcd-client",
LongName: "client apiserver uses to access etcd",
BaseName: kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName,
CAName: "etcd-ca",
config: certutil.Config{
CommonName: kubeadmconstants.APIServerEtcdClientCertCommonName,
Organization: []string{kubeadmconstants.MastersGroup},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
},
}
)
func makeAltNamesMutator(f func(*kubeadmapi.InitConfiguration) (*certutil.AltNames, error)) configMutatorsFunc {
return func(mc *kubeadmapi.InitConfiguration, cc *certutil.Config) error {
altNames, err := f(mc)
if err != nil {
return err
}
cc.AltNames = *altNames
return nil
}
}
func setCommonNameToNodeName() configMutatorsFunc {
return func(mc *kubeadmapi.InitConfiguration, cc *certutil.Config) error {
cc.CommonName = mc.NodeRegistration.Name
return nil
}
}

View File

@@ -0,0 +1,187 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package certs
import (
"crypto"
"crypto/tls"
"crypto/x509"
"io/ioutil"
"os"
"path"
"testing"
certutil "k8s.io/client-go/util/cert"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
)
func TestCAPointersValid(t *testing.T) {
tests := []struct {
certs Certificates
name string
}{
{
name: "Default Certificate List",
certs: GetDefaultCertList(),
},
{
name: "Cert list less etcd",
certs: GetCertsWithoutEtcd(),
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
certMap := test.certs.AsMap()
for _, cert := range test.certs {
if cert.CAName != "" && certMap[cert.CAName] == nil {
t.Errorf("Certificate %q references nonexistent CA %q", cert.Name, cert.CAName)
}
}
})
}
}
func TestMakeCertTree(t *testing.T) {
rootCert := &KubeadmCert{
Name: "root",
}
leaf0 := &KubeadmCert{
Name: "leaf0",
CAName: "root",
}
leaf1 := &KubeadmCert{
Name: "leaf1",
CAName: "root",
}
selfSigned := &KubeadmCert{
Name: "self-signed",
}
certMap := CertificateMap{
"root": rootCert,
"leaf0": leaf0,
"leaf1": leaf1,
"self-signed": selfSigned,
}
orphanCertMap := CertificateMap{
"leaf0": leaf0,
}
if _, err := orphanCertMap.CertTree(); err == nil {
t.Error("expected orphan cert map to error, but got nil")
}
certTree, err := certMap.CertTree()
t.Logf("cert tree: %v", certTree)
if err != nil {
t.Errorf("expected no error, but got %v", err)
}
if len(certTree) != 2 {
t.Errorf("Expected tree to have 2 roots, got %d", len(certTree))
}
if len(certTree[rootCert]) != 2 {
t.Errorf("Expected root to have 2 leaves, got %d", len(certTree[rootCert]))
}
if _, ok := certTree[selfSigned]; !ok {
t.Error("Expected selfSigned to be present in tree, but missing")
}
}
func TestCreateCertificateChain(t *testing.T) {
dir, err := ioutil.TempDir("", t.Name())
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
ic := &kubeadmapi.InitConfiguration{
NodeRegistration: kubeadmapi.NodeRegistrationOptions{
Name: "test-node",
},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
CertificatesDir: dir,
},
}
caCfg := Certificates{
{
config: certutil.Config{},
Name: "test-ca",
BaseName: "test-ca",
},
{
config: certutil.Config{
AltNames: certutil.AltNames{
DNSNames: []string{"test-domain.space"},
},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
},
configMutators: []configMutatorsFunc{
setCommonNameToNodeName(),
},
CAName: "test-ca",
Name: "test-daughter",
BaseName: "test-daughter",
},
}
certTree, err := caCfg.AsMap().CertTree()
if err != nil {
t.Fatalf("unexpected error getting tree: %v", err)
}
if certTree.CreateTree(ic); err != nil {
t.Fatal(err)
}
caCert, _ := parseCertAndKey(path.Join(dir, "test-ca"), t)
daughterCert, _ := parseCertAndKey(path.Join(dir, "test-daughter"), t)
pool := x509.NewCertPool()
pool.AddCert(caCert)
_, err = daughterCert.Verify(x509.VerifyOptions{
DNSName: "test-domain.space",
Roots: pool,
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
})
if err != nil {
t.Errorf("couldn't verify daughter cert: %v", err)
}
}
func parseCertAndKey(basePath string, t *testing.T) (*x509.Certificate, crypto.PrivateKey) {
certPair, err := tls.LoadX509KeyPair(basePath+".crt", basePath+".key")
if err != nil {
t.Fatalf("couldn't parse certificate and key: %v", err)
}
parsedCert, err := x509.ParseCertificate(certPair.Certificate[0])
if err != nil {
t.Fatalf("couldn't parse certificate: %v", err)
}
return parsedCert, certPair.PrivateKey
}

View File

@@ -33,223 +33,42 @@ import (
// CreatePKIAssets will create and write to disk all PKI assets necessary to establish the control plane.
// If the PKI assets already exists in the target folder, they are used only if evaluated equal; otherwise an error is returned.
func CreatePKIAssets(cfg *kubeadmapi.MasterConfiguration) error {
func CreatePKIAssets(cfg *kubeadmapi.InitConfiguration) error {
glog.V(1).Infoln("creating PKI assets")
certActions := []func(cfg *kubeadmapi.MasterConfiguration) error{
CreateCACertAndKeyFiles,
CreateAPIServerCertAndKeyFiles,
CreateAPIServerKubeletClientCertAndKeyFiles,
CreateServiceAccountKeyAndPublicKeyFiles,
CreateFrontProxyCACertAndKeyFiles,
CreateFrontProxyClientCertAndKeyFiles,
}
etcdCertActions := []func(cfg *kubeadmapi.MasterConfiguration) error{
CreateEtcdCACertAndKeyFiles,
CreateEtcdServerCertAndKeyFiles,
CreateEtcdPeerCertAndKeyFiles,
CreateEtcdHealthcheckClientCertAndKeyFiles,
CreateAPIServerEtcdClientCertAndKeyFiles,
// This structure cannot handle multilevel CA hierarchies.
// This isn't a problem right now, but may become one in the future.
var certList Certificates
if cfg.Etcd.Local == nil {
certList = GetCertsWithoutEtcd()
} else {
certList = GetDefaultCertList()
}
if cfg.Etcd.Local != nil {
certActions = append(certActions, etcdCertActions...)
certTree, err := certList.AsMap().CertTree()
if err != nil {
return err
}
for _, action := range certActions {
err := action(cfg)
if err != nil {
return err
}
if err := certTree.CreateTree(cfg); err != nil {
return fmt.Errorf("Error creating PKI assets: %v", err)
}
fmt.Printf("[certificates] valid certificates and keys now exist in %q\n", cfg.CertificatesDir)
// Service accounts are not x509 certs, so handled separately
if err := CreateServiceAccountKeyAndPublicKeyFiles(cfg); err != nil {
return err
}
return nil
}
// CreateCACertAndKeyFiles create a new self signed cluster CA certificate and key files.
// If the CA certificate and key files already exists in the target folder, they are used only if evaluated equal; otherwise an error is returned.
func CreateCACertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("create a new self signed cluster CA certificate and key files")
caCert, caKey, err := NewCACertAndKey()
if err != nil {
return err
}
return writeCertificateAuthorithyFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.CACertAndKeyBaseName,
caCert,
caKey,
)
}
// CreateAPIServerCertAndKeyFiles create a new certificate and key files for the apiserver.
// If the apiserver certificate and key files already exists in the target folder, they are used only if evaluated equal; otherwise an error is returned.
// It assumes the cluster CA certificate and key files exist in the CertificatesDir.
func CreateAPIServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a new certificate and key files for the apiserver")
caCert, caKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
if err != nil {
return err
}
apiCert, apiKey, err := NewAPIServerCertAndKey(cfg, caCert, caKey)
if err != nil {
return err
}
return writeCertificateFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.APIServerCertAndKeyBaseName,
caCert,
apiCert,
apiKey,
)
}
// CreateAPIServerKubeletClientCertAndKeyFiles create a new certificate for kubelets calling apiserver.
// If the apiserver-kubelet-client certificate and key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned.
// It assumes the cluster CA certificate and key files exist in the CertificatesDir.
func CreateAPIServerKubeletClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a new certificate for kubelets calling apiserver")
caCert, caKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
if err != nil {
return err
}
apiKubeletClientCert, apiKubeletClientKey, err := NewAPIServerKubeletClientCertAndKey(caCert, caKey)
if err != nil {
return err
}
return writeCertificateFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName,
caCert,
apiKubeletClientCert,
apiKubeletClientKey,
)
}
// CreateEtcdCACertAndKeyFiles create a self signed etcd CA certificate and key files.
// The etcd CA and client certs are used to secure communication between etcd peers and connections to etcd from the API server.
// This is a separate CA, so that kubernetes client identities cannot connect to etcd directly or peer with the etcd cluster.
// If the etcd CA certificate and key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned.
func CreateEtcdCACertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a self signed etcd CA certificate and key files")
etcdCACert, etcdCAKey, err := NewEtcdCACertAndKey()
if err != nil {
return err
}
return writeCertificateAuthorithyFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.EtcdCACertAndKeyBaseName,
etcdCACert,
etcdCAKey,
)
}
// CreateEtcdServerCertAndKeyFiles create a new certificate and key file for etcd.
// If the etcd serving certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned.
// It assumes the etcd CA certificate and key file exist in the CertificatesDir
func CreateEtcdServerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a new server certificate and key files for etcd")
etcdCACert, etcdCAKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.EtcdCACertAndKeyBaseName)
if err != nil {
return err
}
etcdServerCert, etcdServerKey, err := NewEtcdServerCertAndKey(cfg, etcdCACert, etcdCAKey)
if err != nil {
return err
}
return writeCertificateFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.EtcdServerCertAndKeyBaseName,
etcdCACert,
etcdServerCert,
etcdServerKey,
)
}
// CreateEtcdPeerCertAndKeyFiles create a new certificate and key file for etcd peering.
// If the etcd peer certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned.
// It assumes the etcd CA certificate and key file exist in the CertificatesDir
func CreateEtcdPeerCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a new certificate and key files for etcd peering")
etcdCACert, etcdCAKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.EtcdCACertAndKeyBaseName)
if err != nil {
return err
}
etcdPeerCert, etcdPeerKey, err := NewEtcdPeerCertAndKey(cfg, etcdCACert, etcdCAKey)
if err != nil {
return err
}
return writeCertificateFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.EtcdPeerCertAndKeyBaseName,
etcdCACert,
etcdPeerCert,
etcdPeerKey,
)
}
// CreateEtcdHealthcheckClientCertAndKeyFiles create a new client certificate for liveness probes to healthcheck etcd
// If the etcd-healthcheck-client certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned.
// It assumes the etcd CA certificate and key file exist in the CertificatesDir
func CreateEtcdHealthcheckClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
etcdCACert, etcdCAKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.EtcdCACertAndKeyBaseName)
if err != nil {
return err
}
etcdHealthcheckClientCert, etcdHealthcheckClientKey, err := NewEtcdHealthcheckClientCertAndKey(etcdCACert, etcdCAKey)
if err != nil {
return err
}
return writeCertificateFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.EtcdHealthcheckClientCertAndKeyBaseName,
etcdCACert,
etcdHealthcheckClientCert,
etcdHealthcheckClientKey,
)
}
// CreateAPIServerEtcdClientCertAndKeyFiles create a new client certificate for the apiserver calling etcd
// If the apiserver-etcd-client certificate and key file already exist in the target folder, they are used only if evaluated equal; otherwise an error is returned.
// It assumes the etcd CA certificate and key file exist in the CertificatesDir
func CreateAPIServerEtcdClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a new client certificate for the apiserver calling etcd")
etcdCACert, etcdCAKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.EtcdCACertAndKeyBaseName)
if err != nil {
return err
}
apiEtcdClientCert, apiEtcdClientKey, err := NewAPIServerEtcdClientCertAndKey(etcdCACert, etcdCAKey)
if err != nil {
return err
}
return writeCertificateFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName,
etcdCACert,
apiEtcdClientCert,
apiEtcdClientKey,
)
}
// CreateServiceAccountKeyAndPublicKeyFiles create a new public/private key files for signing service account users.
// If the sa public/private key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned.
func CreateServiceAccountKeyAndPublicKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
func CreateServiceAccountKeyAndPublicKeyFiles(cfg *kubeadmapi.InitConfiguration) error {
glog.V(1).Infoln("creating a new public/private key files for signing service account users")
saSigningKey, err := NewServiceAccountSigningKey()
if err != nil {
@@ -263,186 +82,8 @@ func CreateServiceAccountKeyAndPublicKeyFiles(cfg *kubeadmapi.MasterConfiguratio
)
}
// CreateFrontProxyCACertAndKeyFiles create a self signed front proxy CA certificate and key files.
// Front proxy CA and client certs are used to secure a front proxy authenticator which is used to assert identity
// without the client cert; This is a separate CA, so that front proxy identities cannot hit the API and normal client certs cannot be used
// as front proxies.
// If the front proxy CA certificate and key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned.
func CreateFrontProxyCACertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a self signed front proxy CA certificate and key files")
frontProxyCACert, frontProxyCAKey, err := NewFrontProxyCACertAndKey()
if err != nil {
return err
}
return writeCertificateAuthorithyFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.FrontProxyCACertAndKeyBaseName,
frontProxyCACert,
frontProxyCAKey,
)
}
// CreateFrontProxyClientCertAndKeyFiles create a new certificate for proxy server client.
// If the front-proxy-client certificate and key files already exists in the target folder, they are used only if evaluated equals; otherwise an error is returned.
// It assumes the front proxy CA certificate and key files exist in the CertificatesDir.
func CreateFrontProxyClientCertAndKeyFiles(cfg *kubeadmapi.MasterConfiguration) error {
glog.V(1).Infoln("creating a new certificate for proxy server client")
frontProxyCACert, frontProxyCAKey, err := loadCertificateAuthority(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName)
if err != nil {
return err
}
frontProxyClientCert, frontProxyClientKey, err := NewFrontProxyClientCertAndKey(frontProxyCACert, frontProxyCAKey)
if err != nil {
return err
}
return writeCertificateFilesIfNotExist(
cfg.CertificatesDir,
kubeadmconstants.FrontProxyClientCertAndKeyBaseName,
frontProxyCACert,
frontProxyClientCert,
frontProxyClientKey,
)
}
// NewCACertAndKey will generate a self signed CA.
func NewCACertAndKey() (*x509.Certificate, *rsa.PrivateKey, error) {
caCert, caKey, err := pkiutil.NewCertificateAuthority()
if err != nil {
return nil, nil, fmt.Errorf("failure while generating CA certificate and key: %v", err)
}
return caCert, caKey, nil
}
// NewAPIServerCertAndKey generate certificate for apiserver, signed by the given CA.
func NewAPIServerCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
altNames, err := pkiutil.GetAPIServerAltNames(cfg)
if err != nil {
return nil, nil, fmt.Errorf("failure while composing altnames for API server: %v", err)
}
config := certutil.Config{
CommonName: kubeadmconstants.APIServerCertCommonName,
AltNames: *altNames,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
}
apiCert, apiKey, err := pkiutil.NewCertAndKey(caCert, caKey, config)
if err != nil {
return nil, nil, fmt.Errorf("failure while creating API server key and certificate: %v", err)
}
return apiCert, apiKey, nil
}
// NewAPIServerKubeletClientCertAndKey generate certificate for the apiservers to connect to the kubelets securely, signed by the given CA.
func NewAPIServerKubeletClientCertAndKey(caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
config := certutil.Config{
CommonName: kubeadmconstants.APIServerKubeletClientCertCommonName,
Organization: []string{kubeadmconstants.MastersGroup},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
apiClientCert, apiClientKey, err := pkiutil.NewCertAndKey(caCert, caKey, config)
if err != nil {
return nil, nil, fmt.Errorf("failure while creating API server kubelet client key and certificate: %v", err)
}
return apiClientCert, apiClientKey, nil
}
// NewEtcdCACertAndKey generate a self signed etcd CA.
func NewEtcdCACertAndKey() (*x509.Certificate, *rsa.PrivateKey, error) {
etcdCACert, etcdCAKey, err := pkiutil.NewCertificateAuthority()
if err != nil {
return nil, nil, fmt.Errorf("failure while generating etcd CA certificate and key: %v", err)
}
return etcdCACert, etcdCAKey, nil
}
// NewEtcdServerCertAndKey generate certificate for etcd, signed by the given CA.
func NewEtcdServerCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
altNames, err := pkiutil.GetEtcdAltNames(cfg)
if err != nil {
return nil, nil, fmt.Errorf("failure while composing altnames for etcd: %v", err)
}
config := certutil.Config{
CommonName: cfg.NodeRegistration.Name,
AltNames: *altNames,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
}
etcdServerCert, etcdServerKey, err := pkiutil.NewCertAndKey(caCert, caKey, config)
if err != nil {
return nil, nil, fmt.Errorf("failure while creating etcd key and certificate: %v", err)
}
return etcdServerCert, etcdServerKey, nil
}
// NewEtcdPeerCertAndKey generate certificate for etcd peering, signed by the given CA.
func NewEtcdPeerCertAndKey(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
altNames, err := pkiutil.GetEtcdPeerAltNames(cfg)
if err != nil {
return nil, nil, fmt.Errorf("failure while composing altnames for etcd peering: %v", err)
}
config := certutil.Config{
CommonName: cfg.NodeRegistration.Name,
AltNames: *altNames,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
}
etcdPeerCert, etcdPeerKey, err := pkiutil.NewCertAndKey(caCert, caKey, config)
if err != nil {
return nil, nil, fmt.Errorf("failure while creating etcd peer key and certificate: %v", err)
}
return etcdPeerCert, etcdPeerKey, nil
}
// NewEtcdHealthcheckClientCertAndKey generate certificate for liveness probes to healthcheck etcd, signed by the given CA.
func NewEtcdHealthcheckClientCertAndKey(caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
config := certutil.Config{
CommonName: kubeadmconstants.EtcdHealthcheckClientCertCommonName,
Organization: []string{kubeadmconstants.MastersGroup},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
etcdHealcheckClientCert, etcdHealcheckClientKey, err := pkiutil.NewCertAndKey(caCert, caKey, config)
if err != nil {
return nil, nil, fmt.Errorf("failure while creating etcd healthcheck client key and certificate: %v", err)
}
return etcdHealcheckClientCert, etcdHealcheckClientKey, nil
}
// NewAPIServerEtcdClientCertAndKey generate certificate for the apiservers to connect to etcd securely, signed by the given CA.
func NewAPIServerEtcdClientCertAndKey(caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
config := certutil.Config{
CommonName: kubeadmconstants.APIServerEtcdClientCertCommonName,
Organization: []string{kubeadmconstants.MastersGroup},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
apiClientCert, apiClientKey, err := pkiutil.NewCertAndKey(caCert, caKey, config)
if err != nil {
return nil, nil, fmt.Errorf("failure while creating API server etcd client key and certificate: %v", err)
}
return apiClientCert, apiClientKey, nil
}
// NewServiceAccountSigningKey generate public/private key pairs for signing service account tokens.
func NewServiceAccountSigningKey() (*rsa.PrivateKey, error) {
// The key does NOT exist, let's generate it now
saSigningKey, err := certutil.NewPrivateKey()
if err != nil {
@@ -452,34 +93,73 @@ func NewServiceAccountSigningKey() (*rsa.PrivateKey, error) {
return saSigningKey, nil
}
// NewFrontProxyCACertAndKey generate a self signed front proxy CA.
func NewFrontProxyCACertAndKey() (*x509.Certificate, *rsa.PrivateKey, error) {
// NewCACertAndKey will generate a self signed CA.
func NewCACertAndKey(certSpec *certutil.Config) (*x509.Certificate, *rsa.PrivateKey, error) {
frontProxyCACert, frontProxyCAKey, err := pkiutil.NewCertificateAuthority()
caCert, caKey, err := pkiutil.NewCertificateAuthority(certSpec)
if err != nil {
return nil, nil, fmt.Errorf("failure while generating front-proxy CA certificate and key: %v", err)
return nil, nil, fmt.Errorf("failure while generating CA certificate and key: %v", err)
}
return frontProxyCACert, frontProxyCAKey, nil
return caCert, caKey, nil
}
// NewFrontProxyClientCertAndKey generate certificate for proxy server client, signed by the given front proxy CA.
func NewFrontProxyClientCertAndKey(frontProxyCACert *x509.Certificate, frontProxyCAKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
config := certutil.Config{
CommonName: kubeadmconstants.FrontProxyClientCertCommonName,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
// CreateCACertAndKeyFiles generates and writes out a given certificate authority.
// The certSpec should be one of the variables from this package.
func CreateCACertAndKeyFiles(certSpec *KubeadmCert, cfg *kubeadmapi.InitConfiguration) error {
if certSpec.CAName != "" {
return fmt.Errorf("This function should only be used for CAs, but cert %s has CA %s", certSpec.Name, certSpec.CAName)
}
frontProxyClientCert, frontProxyClientKey, err := pkiutil.NewCertAndKey(frontProxyCACert, frontProxyCAKey, config)
glog.V(1).Infoln("creating a new certificate authority for %s", certSpec.Name)
certConfig, err := certSpec.GetConfig(cfg)
if err != nil {
return nil, nil, fmt.Errorf("failure while creating front-proxy client key and certificate: %v", err)
return err
}
return frontProxyClientCert, frontProxyClientKey, nil
caCert, caKey, err := NewCACertAndKey(certConfig)
if err != nil {
return err
}
return writeCertificateAuthorithyFilesIfNotExist(
cfg.CertificatesDir,
certSpec.BaseName,
caCert,
caKey,
)
}
// loadCertificateAuthority loads certificate authority
func loadCertificateAuthority(pkiDir string, baseName string) (*x509.Certificate, *rsa.PrivateKey, error) {
// CreateCertAndKeyFilesWithCA loads the given certificate authority from disk, then generates and writes out the given certificate and key.
// The certSpec and caCertSpec should both be one of the variables from this package.
func CreateCertAndKeyFilesWithCA(certSpec *KubeadmCert, caCertSpec *KubeadmCert, cfg *kubeadmapi.InitConfiguration) error {
if certSpec.CAName != caCertSpec.Name {
return fmt.Errorf("Expected CAname for %s to be %q, but was %s", certSpec.Name, certSpec.CAName, caCertSpec.Name)
}
caCert, caKey, err := LoadCertificateAuthority(cfg.CertificatesDir, caCertSpec.BaseName)
if err != nil {
return fmt.Errorf("Couldn't load CA certificate %s: %v", caCertSpec.Name, err)
}
return certSpec.CreateFromCA(cfg, caCert, caKey)
}
func newCertAndKeyFromSpec(certSpec *KubeadmCert, cfg *kubeadmapi.InitConfiguration, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey, error) {
certConfig, err := certSpec.GetConfig(cfg)
if err != nil {
return nil, nil, fmt.Errorf("failure while creating certificate %s: %v", certSpec.Name, err)
}
cert, key, err := pkiutil.NewCertAndKey(caCert, caKey, certConfig)
if err != nil {
return nil, nil, fmt.Errorf("failure while creating %s key and certificate: %v", certSpec.Name, err)
}
return cert, key, err
}
// LoadCertificateAuthority tries to load a CA in the given directory with the given name.
func LoadCertificateAuthority(pkiDir string, baseName string) (*x509.Certificate, *rsa.PrivateKey, error) {
// Checks if certificate authority exists in the PKI directory
if !pkiutil.CertOrKeyExist(pkiDir, baseName) {
return nil, nil, fmt.Errorf("couldn't load %s certificate authority from %s", baseName, pkiDir)
@@ -501,7 +181,7 @@ func loadCertificateAuthority(pkiDir string, baseName string) (*x509.Certificate
// writeCertificateAuthorithyFilesIfNotExist write a new certificate Authority to the given path.
// If there already is a certificate file at the given path; kubeadm tries to load it and check if the values in the
// existing and the expected certificate equals. If they do; kubeadm will just skip writing the file as it's up-to-date,
// existing and the eexpected certificate equals. If they do; kubeadm will just skip writing the file as it's up-to-date,
// otherwise this function returns an error.
func writeCertificateAuthorithyFilesIfNotExist(pkiDir string, baseName string, caCert *x509.Certificate, caKey *rsa.PrivateKey) error {
@@ -617,10 +297,29 @@ type certKeyLocation struct {
uxName string
}
// SharedCertificateExists verifies if the shared certificates - the certificates that must be
// equal across masters: ca.key, ca.crt, sa.key, sa.pub
func SharedCertificateExists(cfg *kubeadmapi.InitConfiguration) (bool, error) {
if err := validateCACertAndKey(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, "", "CA"}); err != nil {
return false, err
}
if err := validatePrivatePublicKey(certKeyLocation{cfg.CertificatesDir, "", kubeadmconstants.ServiceAccountKeyBaseName, "service account"}); err != nil {
return false, err
}
if err := validateCACertAndKey(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName, "", "front-proxy CA"}); err != nil {
return false, err
}
return true, nil
}
// UsingExternalCA determines whether the user is relying on an external CA. We currently implicitly determine this is the case
// when both the CA Cert and the front proxy CA Cert are present but the CA Key and front proxy CA Key are not.
// This allows us to, e.g., skip generating certs or not start the csr signing controller.
func UsingExternalCA(cfg *kubeadmapi.MasterConfiguration) (bool, error) {
func UsingExternalCA(cfg *kubeadmapi.InitConfiguration) (bool, error) {
if err := validateCACert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, "", "CA"}); err != nil {
return false, err
@@ -697,6 +396,11 @@ func validateSignedCert(l certKeyLocation) error {
return fmt.Errorf("failure loading certificate authority for %s: %v", l.uxName, err)
}
return validateSignedCertWithCA(l, caCert)
}
// validateSignedCertWithCA tries to load a certificate and validate it with the given caCert
func validateSignedCertWithCA(l certKeyLocation, caCert *x509.Certificate) error {
// Try to load key and signed certificate
signedCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(l.pkiDir, l.baseName)
if err != nil {

View File

@@ -20,11 +20,12 @@ import (
"crypto/rsa"
"crypto/x509"
"fmt"
"net"
"os"
"path"
"path/filepath"
"testing"
certutil "k8s.io/client-go/util/cert"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
@@ -32,10 +33,30 @@ import (
certstestutil "k8s.io/kubernetes/cmd/kubeadm/test/certs"
)
func TestWriteCertificateAuthorithyFilesIfNotExist(t *testing.T) {
func createCACert(t *testing.T) (*x509.Certificate, *rsa.PrivateKey) {
certCfg := &certutil.Config{CommonName: "kubernetes"}
cert, key, err := NewCACertAndKey(certCfg)
if err != nil {
t.Fatalf("couldn't create CA: %v", err)
}
return cert, key
}
setupCert, setupKey, _ := NewCACertAndKey()
caCert, caKey, _ := NewCACertAndKey()
func createTestCert(t *testing.T, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, *rsa.PrivateKey) {
cert, key, err := pkiutil.NewCertAndKey(caCert, caKey,
&certutil.Config{
CommonName: "testCert",
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny},
})
if err != nil {
t.Fatalf("couldn't create test cert: %v", err)
}
return cert, key
}
func TestWriteCertificateAuthorithyFilesIfNotExist(t *testing.T) {
setupCert, setupKey := createCACert(t)
caCert, caKey := createCACert(t)
var tests = []struct {
setupFunc func(pkiDir string) error
@@ -60,7 +81,7 @@ func TestWriteCertificateAuthorithyFilesIfNotExist(t *testing.T) {
},
{ // cert exists, but it is not a ca > err
setupFunc: func(pkiDir string) error {
cert, key, _ := NewFrontProxyClientCertAndKey(setupCert, setupKey)
cert, key := createTestCert(t, setupCert, setupKey)
return writeCertificateFilesIfNotExist(pkiDir, "dummy", setupCert, cert, key)
},
expectedError: true,
@@ -110,9 +131,9 @@ func TestWriteCertificateAuthorithyFilesIfNotExist(t *testing.T) {
func TestWriteCertificateFilesIfNotExist(t *testing.T) {
caCert, caKey, _ := NewFrontProxyCACertAndKey()
setupCert, setupKey, _ := NewFrontProxyClientCertAndKey(caCert, caKey)
cert, key, _ := NewFrontProxyClientCertAndKey(caCert, caKey)
caCert, caKey := createCACert(t)
setupCert, setupKey := createTestCert(t, caCert, caKey)
cert, key := createTestCert(t, caCert, caKey)
var tests = []struct {
setupFunc func(pkiDir string) error
@@ -137,8 +158,8 @@ func TestWriteCertificateFilesIfNotExist(t *testing.T) {
},
{ // cert exists, is signed by another ca > err
setupFunc: func(pkiDir string) error {
anotherCaCert, anotherCaKey, _ := NewFrontProxyCACertAndKey()
anotherCert, anotherKey, _ := NewFrontProxyClientCertAndKey(anotherCaCert, anotherCaKey)
anotherCaCert, anotherCaKey := createCACert(t)
anotherCert, anotherKey := createTestCert(t, anotherCaCert, anotherCaKey)
return writeCertificateFilesIfNotExist(pkiDir, "dummy", anotherCaCert, anotherCert, anotherKey)
},
@@ -259,7 +280,8 @@ func TestWriteKeyFilesIfNotExist(t *testing.T) {
}
func TestNewCACertAndKey(t *testing.T) {
caCert, _, err := NewCACertAndKey()
certCfg := &certutil.Config{CommonName: "kubernetes"}
caCert, _, err := NewCACertAndKey(certCfg)
if err != nil {
t.Fatalf("failed call NewCACertAndKey: %v", err)
}
@@ -267,210 +289,187 @@ func TestNewCACertAndKey(t *testing.T) {
certstestutil.AssertCertificateIsCa(t, caCert)
}
func TestNewAPIServerCertAndKey(t *testing.T) {
hostname := "valid-hostname"
func TestSharedCertificateExists(t *testing.T) {
caCert, caKey := createCACert(t)
_, key := createTestCert(t, caCert, caKey)
publicKey := &key.PublicKey
advertiseAddresses := []string{"1.2.3.4", "1:2:3::4"}
for _, addr := range advertiseAddresses {
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: addr},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: hostname},
}
caCert, caKey, err := NewCACertAndKey()
if err != nil {
t.Fatalf("failed creation of ca cert and key: %v", err)
}
apiServerCert, _, err := NewAPIServerCertAndKey(cfg, caCert, caKey)
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
certstestutil.AssertCertificateIsSignedByCa(t, apiServerCert, caCert)
certstestutil.AssertCertificateHasServerAuthUsage(t, apiServerCert)
certstestutil.AssertCertificateHasDNSNames(t, apiServerCert, hostname, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local")
certstestutil.AssertCertificateHasIPAddresses(t, apiServerCert, net.ParseIP("10.96.0.1"), net.ParseIP(addr))
}
}
func TestNewAPIServerKubeletClientCertAndKey(t *testing.T) {
caCert, caKey, err := NewCACertAndKey()
if err != nil {
t.Fatalf("failed creation of ca cert and key: %v", err)
}
apiKubeletClientCert, _, err := NewAPIServerKubeletClientCertAndKey(caCert, caKey)
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
certstestutil.AssertCertificateIsSignedByCa(t, apiKubeletClientCert, caCert)
certstestutil.AssertCertificateHasClientAuthUsage(t, apiKubeletClientCert)
certstestutil.AssertCertificateHasOrganizations(t, apiKubeletClientCert, kubeadmconstants.MastersGroup)
}
func TestNewEtcdCACertAndKey(t *testing.T) {
etcdCACert, _, err := NewEtcdCACertAndKey()
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
certstestutil.AssertCertificateIsCa(t, etcdCACert)
}
func TestNewEtcdServerCertAndKey(t *testing.T) {
proxy := "user-etcd-proxy"
proxyIP := "10.10.10.100"
cfg := &kubeadmapi.MasterConfiguration{
NodeRegistration: kubeadmapi.NodeRegistrationOptions{
Name: "etcd-server-cert",
},
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
ServerCertSANs: []string{
proxy,
proxyIP,
},
var tests = []struct {
name string
files pkiFiles
expectedError bool
}{
{
name: "success",
files: pkiFiles{
"ca.crt": caCert,
"ca.key": caKey,
"front-proxy-ca.crt": caCert,
"front-proxy-ca.key": caKey,
"sa.pub": publicKey,
"sa.key": key,
},
},
}
caCert, caKey, err := NewCACertAndKey()
if err != nil {
t.Fatalf("failed creation of ca cert and key: %v", err)
}
etcdServerCert, _, err := NewEtcdServerCertAndKey(cfg, caCert, caKey)
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
certstestutil.AssertCertificateIsSignedByCa(t, etcdServerCert, caCert)
certstestutil.AssertCertificateHasServerAuthUsage(t, etcdServerCert)
certstestutil.AssertCertificateHasDNSNames(t, etcdServerCert, "localhost", proxy)
certstestutil.AssertCertificateHasIPAddresses(t, etcdServerCert, net.ParseIP("127.0.0.1"), net.ParseIP(proxyIP))
}
func TestNewEtcdPeerCertAndKey(t *testing.T) {
hostname := "valid-hostname"
proxy := "user-etcd-proxy"
proxyIP := "10.10.10.100"
advertiseAddresses := []string{"1.2.3.4", "1:2:3::4"}
for _, addr := range advertiseAddresses {
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: addr},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: hostname},
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
PeerCertSANs: []string{
proxy,
proxyIP,
},
},
{
name: "missing ca.crt",
files: pkiFiles{
"ca.key": caKey,
"front-proxy-ca.crt": caCert,
"front-proxy-ca.key": caKey,
"sa.pub": publicKey,
"sa.key": key,
},
}
caCert, caKey, err := NewCACertAndKey()
if err != nil {
t.Fatalf("failed creation of ca cert and key: %v", err)
}
expectedError: true,
},
{
name: "missing sa.key",
files: pkiFiles{
"ca.crt": caCert,
"ca.key": caKey,
"front-proxy-ca.crt": caCert,
"front-proxy-ca.key": caKey,
"sa.pub": publicKey,
},
expectedError: true,
},
{
name: "expected front-proxy.crt missing",
files: pkiFiles{
"ca.crt": caCert,
"ca.key": caKey,
"front-proxy-ca.key": caKey,
"sa.pub": publicKey,
"sa.key": key,
},
expectedError: true,
},
}
etcdPeerCert, _, err := NewEtcdPeerCertAndKey(cfg, caCert, caKey)
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
certstestutil.AssertCertificateIsSignedByCa(t, etcdPeerCert, caCert)
certstestutil.AssertCertificateHasServerAuthUsage(t, etcdPeerCert)
certstestutil.AssertCertificateHasClientAuthUsage(t, etcdPeerCert)
certstestutil.AssertCertificateHasDNSNames(t, etcdPeerCert, hostname, proxy)
certstestutil.AssertCertificateHasIPAddresses(t, etcdPeerCert, net.ParseIP(addr), net.ParseIP(proxyIP))
cfg := &kubeadmapi.InitConfiguration{
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
CertificatesDir: tmpdir,
},
}
// created expected keys
writePKIFiles(t, tmpdir, test.files)
// executes create func
ret, err := SharedCertificateExists(cfg)
switch {
case !test.expectedError && err != nil:
t.Errorf("error SharedCertificateExists failed when not expected to fail: %v", err)
case test.expectedError && err == nil:
t.Errorf("error SharedCertificateExists didn't failed when expected")
case ret != (err == nil):
t.Errorf("error SharedCertificateExists returned %v when expected to return %v", ret, err == nil)
}
})
}
}
func TestNewEtcdHealthcheckClientCertAndKey(t *testing.T) {
caCert, caKey, err := NewCACertAndKey()
if err != nil {
t.Fatalf("failed creation of ca cert and key: %v", err)
func TestCreatePKIAssetsWithSparseCerts(t *testing.T) {
caCert, caKey := createCACert(t)
fpCACert, fpCAKey := createCACert(t)
etcdCACert, etcdCAKey := createCACert(t)
fpCert, fpKey := createTestCert(t, fpCACert, fpCAKey)
tests := []struct {
name string
files pkiFiles
expectError bool
}{
{
name: "nothing present",
},
{
name: "CAs already exist",
files: pkiFiles{
"ca.crt": caCert,
"ca.key": caKey,
"front-proxy-ca.crt": fpCACert,
"front-proxy-ca.key": fpCAKey,
"etcd/ca.crt": etcdCACert,
"etcd/ca.key": etcdCAKey,
},
},
{
name: "CA certs only",
files: pkiFiles{
"ca.crt": caCert,
"front-proxy-ca.crt": fpCACert,
"etcd/ca.crt": etcdCACert,
},
expectError: true,
},
{
name: "FrontProxyCA with certs",
files: pkiFiles{
"ca.crt": caCert,
"ca.key": caKey,
"front-proxy-ca.crt": fpCACert,
"front-proxy-client.crt": fpCert,
"front-proxy-client.key": fpKey,
"etcd/ca.crt": etcdCACert,
"etcd/ca.key": etcdCAKey,
},
},
{
name: "FrontProxy certs missing CA",
files: pkiFiles{
"front-proxy-client.crt": fpCert,
"front-proxy-client.key": fpKey,
},
expectError: true,
},
}
etcdHealthcheckClientCert, _, err := NewEtcdHealthcheckClientCertAndKey(caCert, caKey)
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
cfg := testutil.GetDefaultInternalConfig(t)
cfg.ClusterConfiguration.CertificatesDir = tmpdir
writePKIFiles(t, tmpdir, test.files)
err := CreatePKIAssets(cfg)
if err != nil {
if test.expectError {
return
}
t.Fatalf("Unexpected error: %v", err)
}
if test.expectError {
t.Fatal("Expected error from CreatePKIAssets, got none")
}
assertCertsExist(t, tmpdir)
})
}
certstestutil.AssertCertificateIsSignedByCa(t, etcdHealthcheckClientCert, caCert)
certstestutil.AssertCertificateHasClientAuthUsage(t, etcdHealthcheckClientCert)
certstestutil.AssertCertificateHasOrganizations(t, etcdHealthcheckClientCert, kubeadmconstants.MastersGroup)
}
func TestNewAPIServerEtcdClientCertAndKey(t *testing.T) {
caCert, caKey, err := NewCACertAndKey()
if err != nil {
t.Fatalf("failed creation of ca cert and key: %v", err)
}
apiEtcdClientCert, _, err := NewAPIServerEtcdClientCertAndKey(caCert, caKey)
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
certstestutil.AssertCertificateIsSignedByCa(t, apiEtcdClientCert, caCert)
certstestutil.AssertCertificateHasClientAuthUsage(t, apiEtcdClientCert)
certstestutil.AssertCertificateHasOrganizations(t, apiEtcdClientCert, kubeadmconstants.MastersGroup)
}
func TestNewNewServiceAccountSigningKey(t *testing.T) {
key, err := NewServiceAccountSigningKey()
if err != nil {
t.Fatalf("failed creation of key: %v", err)
}
if key.N.BitLen() < 2048 {
t.Error("Service account signing key has less than 2048 bits size")
}
}
func TestNewFrontProxyCACertAndKey(t *testing.T) {
frontProxyCACert, _, err := NewFrontProxyCACertAndKey()
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
certstestutil.AssertCertificateIsCa(t, frontProxyCACert)
}
func TestNewFrontProxyClientCertAndKey(t *testing.T) {
frontProxyCACert, frontProxyCAKey, err := NewFrontProxyCACertAndKey()
if err != nil {
t.Fatalf("failed creation of ca cert and key: %v", err)
}
frontProxyClientCert, _, err := NewFrontProxyClientCertAndKey(frontProxyCACert, frontProxyCAKey)
if err != nil {
t.Fatalf("failed creation of cert and key: %v", err)
}
certstestutil.AssertCertificateIsSignedByCa(t, frontProxyClientCert, frontProxyCACert)
certstestutil.AssertCertificateHasClientAuthUsage(t, frontProxyClientCert)
}
func TestUsingExternalCA(t *testing.T) {
tests := []struct {
setupFuncs []func(cfg *kubeadmapi.MasterConfiguration) error
setupFuncs []func(cfg *kubeadmapi.InitConfiguration) error
expected bool
}{
{
setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
setupFuncs: []func(cfg *kubeadmapi.InitConfiguration) error{
CreatePKIAssets,
},
expected: false,
},
{
setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
setupFuncs: []func(cfg *kubeadmapi.InitConfiguration) error{
CreatePKIAssets,
deleteCAKey,
deleteFrontProxyCAKey,
@@ -483,11 +482,13 @@ func TestUsingExternalCA(t *testing.T) {
dir := testutil.SetupTempDir(t)
defer os.RemoveAll(dir)
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
cfg := &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
CertificatesDir: dir,
},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"},
CertificatesDir: dir,
}
for _, f := range test.setupFuncs {
@@ -504,17 +505,20 @@ func TestUsingExternalCA(t *testing.T) {
func TestValidateMethods(t *testing.T) {
caCert, caKey := createCACert(t)
cert, key := createTestCert(t, caCert, caKey)
tests := []struct {
name string
setupFuncs []func(cfg *kubeadmapi.MasterConfiguration) error
files pkiFiles
validateFunc func(l certKeyLocation) error
loc certKeyLocation
expectedSuccess bool
}{
{
name: "validateCACert",
setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
CreateCACertAndKeyFiles,
files: pkiFiles{
"ca.crt": caCert,
},
validateFunc: validateCACert,
loc: certKeyLocation{caBaseName: "ca", baseName: "", uxName: "CA"},
@@ -522,28 +526,30 @@ func TestValidateMethods(t *testing.T) {
},
{
name: "validateCACertAndKey (files present)",
setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
CreateCACertAndKeyFiles,
files: pkiFiles{
"ca.crt": caCert,
"ca.key": caKey,
},
validateFunc: validateCACertAndKey,
loc: certKeyLocation{caBaseName: "ca", baseName: "", uxName: "CA"},
expectedSuccess: true,
},
{
name: "validateCACertAndKey (key missing)",
setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
CreatePKIAssets,
deleteCAKey,
files: pkiFiles{
"ca.crt": caCert,
},
name: "validateCACertAndKey (key missing)",
validateFunc: validateCACertAndKey,
loc: certKeyLocation{caBaseName: "ca", baseName: "", uxName: "CA"},
expectedSuccess: false,
},
{
name: "validateSignedCert",
setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
CreateCACertAndKeyFiles,
CreateAPIServerCertAndKeyFiles,
files: pkiFiles{
"ca.crt": caCert,
"ca.key": caKey,
"apiserver.crt": cert,
"apiserver.key": key,
},
validateFunc: validateSignedCert,
loc: certKeyLocation{caBaseName: "ca", baseName: "apiserver", uxName: "apiserver"},
@@ -551,8 +557,9 @@ func TestValidateMethods(t *testing.T) {
},
{
name: "validatePrivatePublicKey",
setupFuncs: []func(cfg *kubeadmapi.MasterConfiguration) error{
CreateServiceAccountKeyAndPublicKeyFiles,
files: pkiFiles{
"sa.pub": &key.PublicKey,
"sa.key": key,
},
validateFunc: validatePrivatePublicKey,
loc: certKeyLocation{baseName: "sa", uxName: "service account"},
@@ -561,25 +568,11 @@ func TestValidateMethods(t *testing.T) {
}
for _, test := range tests {
dir := testutil.SetupTempDir(t)
defer os.RemoveAll(dir)
test.loc.pkiDir = dir
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"},
CertificatesDir: dir,
}
fmt.Println("Testing", test.name)
for _, f := range test.setupFuncs {
if err := f(cfg); err != nil {
t.Errorf("error executing setup function: %v", err)
}
}
writePKIFiles(t, dir, test.files)
err := test.validateFunc(test.loc)
if test.expectedSuccess && err != nil {
@@ -590,25 +583,36 @@ func TestValidateMethods(t *testing.T) {
}
}
func deleteCAKey(cfg *kubeadmapi.MasterConfiguration) error {
if err := os.Remove(filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName)); err != nil {
return fmt.Errorf("failed removing %s: %v", kubeadmconstants.CAKeyName, err)
}
return nil
}
type pkiFiles map[string]interface{}
func deleteFrontProxyCAKey(cfg *kubeadmapi.MasterConfiguration) error {
if err := os.Remove(filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCAKeyName)); err != nil {
return fmt.Errorf("failed removing %s: %v", kubeadmconstants.FrontProxyCAKeyName, err)
func writePKIFiles(t *testing.T, dir string, files pkiFiles) {
for filename, body := range files {
switch body := body.(type) {
case *x509.Certificate:
if err := certutil.WriteCert(path.Join(dir, filename), certutil.EncodeCertPEM(body)); err != nil {
t.Errorf("unable to write certificate to file %q: [%v]", dir, err)
}
case *rsa.PublicKey:
publicKeyBytes, err := certutil.EncodePublicKeyPEM(body)
if err != nil {
t.Errorf("unable to write public key to file %q: [%v]", filename, err)
}
if err := certutil.WriteKey(path.Join(dir, filename), publicKeyBytes); err != nil {
t.Errorf("unable to write public key to file %q: [%v]", filename, err)
}
case *rsa.PrivateKey:
if err := certutil.WriteKey(path.Join(dir, filename), certutil.EncodePrivateKeyPEM(body)); err != nil {
t.Errorf("unable to write private key to file %q: [%v]", filename, err)
}
}
}
return nil
}
func TestCreateCertificateFilesMethods(t *testing.T) {
var tests = []struct {
setupFunc func(cfg *kubeadmapi.MasterConfiguration) error
createFunc func(cfg *kubeadmapi.MasterConfiguration) error
setupFunc func(cfg *kubeadmapi.InitConfiguration) error
createFunc func(cfg *kubeadmapi.InitConfiguration) error
expectedFiles []string
externalEtcd bool
}{
@@ -640,57 +644,6 @@ func TestCreateCertificateFilesMethods(t *testing.T) {
kubeadmconstants.FrontProxyClientCertName, kubeadmconstants.FrontProxyClientKeyName,
},
},
{
createFunc: CreateCACertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.CACertName, kubeadmconstants.CAKeyName},
},
{
setupFunc: CreateCACertAndKeyFiles,
createFunc: CreateAPIServerCertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.APIServerCertName, kubeadmconstants.APIServerKeyName},
},
{
setupFunc: CreateCACertAndKeyFiles,
createFunc: CreateAPIServerKubeletClientCertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName},
},
{
createFunc: CreateEtcdCACertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.EtcdCACertName, kubeadmconstants.EtcdCAKeyName},
},
{
setupFunc: CreateEtcdCACertAndKeyFiles,
createFunc: CreateEtcdServerCertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.EtcdServerCertName, kubeadmconstants.EtcdServerKeyName},
},
{
setupFunc: CreateEtcdCACertAndKeyFiles,
createFunc: CreateEtcdPeerCertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName},
},
{
setupFunc: CreateEtcdCACertAndKeyFiles,
createFunc: CreateEtcdHealthcheckClientCertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.EtcdHealthcheckClientCertName, kubeadmconstants.EtcdHealthcheckClientKeyName},
},
{
setupFunc: CreateEtcdCACertAndKeyFiles,
createFunc: CreateAPIServerEtcdClientCertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.APIServerEtcdClientCertName, kubeadmconstants.APIServerEtcdClientKeyName},
},
{
createFunc: CreateServiceAccountKeyAndPublicKeyFiles,
expectedFiles: []string{kubeadmconstants.ServiceAccountPrivateKeyName, kubeadmconstants.ServiceAccountPublicKeyName},
},
{
createFunc: CreateFrontProxyCACertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.FrontProxyCACertName, kubeadmconstants.FrontProxyCAKeyName},
},
{
setupFunc: CreateFrontProxyCACertAndKeyFiles,
createFunc: CreateFrontProxyClientCertAndKeyFiles,
expectedFiles: []string{kubeadmconstants.FrontProxyCACertName, kubeadmconstants.FrontProxyCAKeyName},
},
}
for _, test := range tests {
@@ -698,12 +651,14 @@ func TestCreateCertificateFilesMethods(t *testing.T) {
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
Etcd: kubeadmapi.Etcd{Local: &kubeadmapi.LocalEtcd{}},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
cfg := &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Etcd: kubeadmapi.Etcd{Local: &kubeadmapi.LocalEtcd{}},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
CertificatesDir: tmpdir,
},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"},
CertificatesDir: tmpdir,
}
if test.externalEtcd {
@@ -714,14 +669,6 @@ func TestCreateCertificateFilesMethods(t *testing.T) {
cfg.Etcd.External.Endpoints = []string{"192.168.1.1:2379"}
}
// executes setup func (if necessary)
if test.setupFunc != nil {
if err := test.setupFunc(cfg); err != nil {
t.Errorf("error executing setupFunc: %v", err)
continue
}
}
// executes create func
if err := test.createFunc(cfg); err != nil {
t.Errorf("error executing createFunc: %v", err)
@@ -732,3 +679,38 @@ func TestCreateCertificateFilesMethods(t *testing.T) {
testutil.AssertFileExists(t, tmpdir, test.expectedFiles...)
}
}
func deleteCAKey(cfg *kubeadmapi.InitConfiguration) error {
if err := os.Remove(filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName)); err != nil {
return fmt.Errorf("failed removing %s: %v", kubeadmconstants.CAKeyName, err)
}
return nil
}
func deleteFrontProxyCAKey(cfg *kubeadmapi.InitConfiguration) error {
if err := os.Remove(filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCAKeyName)); err != nil {
return fmt.Errorf("failed removing %s: %v", kubeadmconstants.FrontProxyCAKeyName, err)
}
return nil
}
func assertCertsExist(t *testing.T, dir string) {
tree, err := GetDefaultCertList().AsMap().CertTree()
if err != nil {
t.Fatalf("unexpected error getting certificates: %v", err)
}
for caCert, certs := range tree {
if err := validateCACert(certKeyLocation{dir, caCert.BaseName, "", caCert.Name}); err != nil {
t.Errorf("couldn't validate CA certificate %v: %v", caCert.Name, err)
// Don't bother validating child certs, but do try the other CAs
continue
}
for _, cert := range certs {
if err := validateSignedCert(certKeyLocation{dir, caCert.BaseName, cert.BaseName, cert.Name}); err != nil {
t.Errorf("couldn't validate certificate %v: %v", cert.Name, err)
}
}
}
}

View File

@@ -21,7 +21,7 @@ package certs
PHASE: CERTIFICATES
INPUTS:
From MasterConfiguration
From InitConfiguration
.API.AdvertiseAddress is an optional parameter that can be passed for an extra addition to the SAN IPs
.APIServerCertSANs is an optional parameter for adding DNS names and IPs to the API Server serving cert SAN
.Etcd.Local.ServerCertSANs is an optional parameter for adding DNS names and IPs to the etcd serving cert SAN

View File

@@ -12,7 +12,7 @@ go_test(
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//vendor/k8s.io/client-go/util/cert:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
],
)
@@ -25,8 +25,8 @@ go_library(
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//pkg/registry/core/service/ipallocator:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
"//vendor/k8s.io/client-go/util/cert:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
],
)

View File

@@ -34,16 +34,13 @@ import (
)
// NewCertificateAuthority creates new certificate and private key for the certificate authority
func NewCertificateAuthority() (*x509.Certificate, *rsa.PrivateKey, error) {
func NewCertificateAuthority(config *certutil.Config) (*x509.Certificate, *rsa.PrivateKey, error) {
key, err := certutil.NewPrivateKey()
if err != nil {
return nil, nil, fmt.Errorf("unable to create private key [%v]", err)
}
config := certutil.Config{
CommonName: "kubernetes",
}
cert, err := certutil.NewSelfSignedCACert(config, key)
cert, err := certutil.NewSelfSignedCACert(*config, key)
if err != nil {
return nil, nil, fmt.Errorf("unable to create self-signed certificate [%v]", err)
}
@@ -52,13 +49,13 @@ func NewCertificateAuthority() (*x509.Certificate, *rsa.PrivateKey, error) {
}
// NewCertAndKey creates new certificate and key by passing the certificate authority certificate and key
func NewCertAndKey(caCert *x509.Certificate, caKey *rsa.PrivateKey, config certutil.Config) (*x509.Certificate, *rsa.PrivateKey, error) {
func NewCertAndKey(caCert *x509.Certificate, caKey *rsa.PrivateKey, config *certutil.Config) (*x509.Certificate, *rsa.PrivateKey, error) {
key, err := certutil.NewPrivateKey()
if err != nil {
return nil, nil, fmt.Errorf("unable to create private key [%v]", err)
}
cert, err := certutil.NewSignedCert(config, key, caCert, caKey)
cert, err := certutil.NewSignedCert(*config, key, caCert, caKey)
if err != nil {
return nil, nil, fmt.Errorf("unable to sign certificate [%v]", err)
}
@@ -133,7 +130,7 @@ func WritePublicKey(pkiPath, name string, key *rsa.PublicKey) error {
// CertOrKeyExist returns a boolean whether the cert or the key exists
func CertOrKeyExist(pkiPath, name string) bool {
certificatePath, privateKeyPath := pathsForCertAndKey(pkiPath, name)
certificatePath, privateKeyPath := PathsForCertAndKey(pkiPath, name)
_, certErr := os.Stat(certificatePath)
_, keyErr := os.Stat(privateKeyPath)
@@ -237,7 +234,8 @@ func TryLoadPrivatePublicKeyFromDisk(pkiPath, name string) (*rsa.PrivateKey, *rs
return k, p, nil
}
func pathsForCertAndKey(pkiPath, name string) (string, string) {
// PathsForCertAndKey returns the paths for the certificate and key given the path and basename.
func PathsForCertAndKey(pkiPath, name string) (string, string) {
return pathForCert(pkiPath, name), pathForKey(pkiPath, name)
}
@@ -254,11 +252,11 @@ func pathForPublicKey(pkiPath, name string) string {
}
// GetAPIServerAltNames builds an AltNames object for to be used when generating apiserver certificate
func GetAPIServerAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, error) {
func GetAPIServerAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) {
// advertise address
advertiseAddress := net.ParseIP(cfg.API.AdvertiseAddress)
advertiseAddress := net.ParseIP(cfg.APIEndpoint.AdvertiseAddress)
if advertiseAddress == nil {
return nil, fmt.Errorf("error parsing API AdvertiseAddress %v: is not a valid textual representation of an IP address", cfg.API.AdvertiseAddress)
return nil, fmt.Errorf("error parsing APIEndpoint AdvertiseAddress %v: is not a valid textual representation of an IP address", cfg.APIEndpoint.AdvertiseAddress)
}
// internal IP address for the API server
@@ -287,16 +285,16 @@ func GetAPIServerAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNam
},
}
// add api server controlPlaneEndpoint if present (dns or ip)
if len(cfg.API.ControlPlaneEndpoint) > 0 {
if host, _, err := kubeadmutil.ParseHostPort(cfg.API.ControlPlaneEndpoint); err == nil {
// add cluster controlPlaneEndpoint if present (dns or ip)
if len(cfg.ControlPlaneEndpoint) > 0 {
if host, _, err := kubeadmutil.ParseHostPort(cfg.ControlPlaneEndpoint); err == nil {
if ip := net.ParseIP(host); ip != nil {
altNames.IPs = append(altNames.IPs, ip)
} else {
altNames.DNSNames = append(altNames.DNSNames, host)
}
} else {
return nil, fmt.Errorf("error parsing API api.controlPlaneEndpoint %q: %s", cfg.API.ControlPlaneEndpoint, err)
return nil, fmt.Errorf("error parsing cluster controlPlaneEndpoint %q: %s", cfg.ControlPlaneEndpoint, err)
}
}
@@ -309,7 +307,7 @@ func GetAPIServerAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNam
// `localhost` is included in the SAN since this is the interface the etcd static pod listens on.
// Hostname and `API.AdvertiseAddress` are excluded since etcd does not listen on this interface by default.
// The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.ServerCertSANs`.
func GetEtcdAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, error) {
func GetEtcdAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) {
// create AltNames with defaults DNSNames/IPs
altNames := &certutil.AltNames{
DNSNames: []string{cfg.NodeRegistration.Name, "localhost"},
@@ -327,11 +325,11 @@ func GetEtcdAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, e
// `localhost` is excluded from the SAN since etcd will not refer to itself as a peer.
// Hostname and `API.AdvertiseAddress` are included if the user chooses to promote the single node etcd cluster into a multi-node one.
// The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.PeerCertSANs`.
func GetEtcdPeerAltNames(cfg *kubeadmapi.MasterConfiguration) (*certutil.AltNames, error) {
func GetEtcdPeerAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) {
// advertise address
advertiseAddress := net.ParseIP(cfg.API.AdvertiseAddress)
advertiseAddress := net.ParseIP(cfg.APIEndpoint.AdvertiseAddress)
if advertiseAddress == nil {
return nil, fmt.Errorf("error parsing API AdvertiseAddress %v: is not a valid textual representation of an IP address", cfg.API.AdvertiseAddress)
return nil, fmt.Errorf("error parsing APIEndpoint AdvertiseAddress %v: is not a valid textual representation of an IP address", cfg.APIEndpoint.AdvertiseAddress)
}
// create AltNames with defaults DNSNames/IPs

View File

@@ -30,7 +30,7 @@ import (
)
func TestNewCertificateAuthority(t *testing.T) {
cert, key, err := NewCertificateAuthority()
cert, key, err := NewCertificateAuthority(&certutil.Config{CommonName: "kubernetes"})
if cert == nil {
t.Errorf(
@@ -73,7 +73,7 @@ func TestNewCertAndKey(t *testing.T) {
t.Fatalf("Couldn't create rsa Private Key")
}
caCert := &x509.Certificate{}
config := certutil.Config{
config := &certutil.Config{
CommonName: "test",
Organization: []string{"test"},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
@@ -90,7 +90,7 @@ func TestNewCertAndKey(t *testing.T) {
}
func TestHasServerAuth(t *testing.T) {
caCert, caKey, _ := NewCertificateAuthority()
caCert, caKey, _ := NewCertificateAuthority(&certutil.Config{CommonName: "kubernetes"})
var tests = []struct {
config certutil.Config
@@ -113,7 +113,7 @@ func TestHasServerAuth(t *testing.T) {
}
for _, rt := range tests {
cert, _, err := NewCertAndKey(caCert, caKey, rt.config)
cert, _, err := NewCertAndKey(caCert, caKey, &rt.config)
if err != nil {
t.Fatalf("Couldn't create cert: %v", err)
}
@@ -261,7 +261,7 @@ func TestTryLoadCertAndKeyFromDisk(t *testing.T) {
}
defer os.RemoveAll(tmpdir)
caCert, caKey, err := NewCertificateAuthority()
caCert, caKey, err := NewCertificateAuthority(&certutil.Config{CommonName: "kubernetes"})
if err != nil {
t.Errorf(
"failed to create cert and key with an error: %v",
@@ -311,7 +311,7 @@ func TestTryLoadCertFromDisk(t *testing.T) {
}
defer os.RemoveAll(tmpdir)
caCert, _, err := NewCertificateAuthority()
caCert, _, err := NewCertificateAuthority(&certutil.Config{CommonName: "kubernetes"})
if err != nil {
t.Errorf(
"failed to create cert and key with an error: %v",
@@ -361,7 +361,7 @@ func TestTryLoadKeyFromDisk(t *testing.T) {
}
defer os.RemoveAll(tmpdir)
_, caKey, err := NewCertificateAuthority()
_, caKey, err := NewCertificateAuthority(&certutil.Config{CommonName: "kubernetes"})
if err != nil {
t.Errorf(
"failed to create cert and key with an error: %v",
@@ -405,7 +405,7 @@ func TestTryLoadKeyFromDisk(t *testing.T) {
}
func TestPathsForCertAndKey(t *testing.T) {
crtPath, keyPath := pathsForCertAndKey("/foo", "bar")
crtPath, keyPath := PathsForCertAndKey("/foo", "bar")
if crtPath != "/foo/bar.crt" {
t.Errorf("unexpected certificate path: %s", crtPath)
}
@@ -439,28 +439,34 @@ func TestGetAPIServerAltNames(t *testing.T) {
var tests = []struct {
name string
cfg *kubeadmapi.MasterConfiguration
cfg *kubeadmapi.InitConfiguration
expectedDNSNames []string
expectedIPAddresses []string
}{
{
name: "ControlPlaneEndpoint DNS",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", ControlPlaneEndpoint: "api.k8s.io:6443"},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"},
APIServerCertSANs: []string{"10.1.245.94", "10.1.245.95", "1.2.3.L", "invalid,commas,in,DNS"},
cfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
ControlPlaneEndpoint: "api.k8s.io:6443",
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
APIServerCertSANs: []string{"10.1.245.94", "10.1.245.95", "1.2.3.L", "invalid,commas,in,DNS"},
},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"},
},
expectedDNSNames: []string{"valid-hostname", "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local", "api.k8s.io"},
expectedIPAddresses: []string{"10.96.0.1", "1.2.3.4", "10.1.245.94", "10.1.245.95"},
},
{
name: "ControlPlaneEndpoint IP",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", ControlPlaneEndpoint: "4.5.6.7:6443"},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"},
APIServerCertSANs: []string{"10.1.245.94", "10.1.245.95", "1.2.3.L", "invalid,commas,in,DNS"},
cfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
ControlPlaneEndpoint: "4.5.6.7:6443",
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
APIServerCertSANs: []string{"10.1.245.94", "10.1.245.95", "1.2.3.L", "invalid,commas,in,DNS"},
},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"},
},
expectedDNSNames: []string{"valid-hostname", "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local"},
expectedIPAddresses: []string{"10.96.0.1", "1.2.3.4", "10.1.245.94", "10.1.245.95", "4.5.6.7"},
@@ -506,14 +512,16 @@ func TestGetAPIServerAltNames(t *testing.T) {
func TestGetEtcdAltNames(t *testing.T) {
proxy := "user-etcd-proxy"
proxyIP := "10.10.10.100"
cfg := &kubeadmapi.MasterConfiguration{
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
ServerCertSANs: []string{
proxy,
proxyIP,
"1.2.3.L",
"invalid,commas,in,DNS",
cfg := &kubeadmapi.InitConfiguration{
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
ServerCertSANs: []string{
proxy,
proxyIP,
"1.2.3.L",
"invalid,commas,in,DNS",
},
},
},
},
@@ -560,19 +568,21 @@ func TestGetEtcdPeerAltNames(t *testing.T) {
proxy := "user-etcd-proxy"
proxyIP := "10.10.10.100"
advertiseIP := "1.2.3.4"
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: advertiseIP},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: hostname},
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
PeerCertSANs: []string{
proxy,
proxyIP,
"1.2.3.L",
"invalid,commas,in,DNS",
cfg := &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: advertiseIP},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
PeerCertSANs: []string{
proxy,
proxyIP,
"1.2.3.L",
"invalid,commas,in,DNS",
},
},
},
},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: hostname},
}
altNames, err := GetEtcdPeerAltNames(cfg)

View File

@@ -0,0 +1,60 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"certsapi.go",
"filerenewal.go",
"interface.go",
"renewal.go",
],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/renewal",
visibility = ["//visibility:public"],
deps = [
"//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
"//staging/src/k8s.io/api/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = [
"filerenewal_test.go",
"renewal_test.go",
],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/phases/certs:go_default_library",
"//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
"//cmd/kubeadm/test:go_default_library",
"//cmd/kubeadm/test/certs:go_default_library",
"//staging/src/k8s.io/api/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1/fake:go_default_library",
"//staging/src/k8s.io/client-go/testing:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,152 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package renewal
import (
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"fmt"
"time"
"github.com/pkg/errors"
certsapi "k8s.io/api/certificates/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
certstype "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
certutil "k8s.io/client-go/util/cert"
)
const (
certAPIPrefixName = "kubeadm-cert"
)
var (
watchTimeout = 5 * time.Minute
)
// CertsAPIRenewal creates new certificates using the certs API
type CertsAPIRenewal struct {
client certstype.CertificatesV1beta1Interface
}
// NewCertsAPIRenawal takes a Kubernetes interface and returns a renewal Interface.
func NewCertsAPIRenawal(client kubernetes.Interface) Interface {
return &CertsAPIRenewal{
client: client.CertificatesV1beta1(),
}
}
// Renew takes a certificate using the cert and key.
func (r *CertsAPIRenewal) Renew(cfg *certutil.Config) (*x509.Certificate, *rsa.PrivateKey, error) {
reqTmp := &x509.CertificateRequest{
Subject: pkix.Name{
CommonName: cfg.CommonName,
Organization: cfg.Organization,
},
DNSNames: cfg.AltNames.DNSNames,
IPAddresses: cfg.AltNames.IPs,
}
key, err := certutil.NewPrivateKey()
if err != nil {
return nil, nil, errors.Wrap(err, "couldn't create new private key")
}
csr, err := x509.CreateCertificateRequest(rand.Reader, reqTmp, key)
if err != nil {
return nil, nil, errors.Wrap(err, "couldn't create certificate signing request")
}
usages := make([]certsapi.KeyUsage, len(cfg.Usages))
for i, usage := range cfg.Usages {
certsAPIUsage, ok := usageMap[usage]
if !ok {
return nil, nil, fmt.Errorf("unknown key usage: %v", usage)
}
usages[i] = certsAPIUsage
}
k8sCSR := &certsapi.CertificateSigningRequest{
ObjectMeta: metav1.ObjectMeta{
GenerateName: certAPIPrefixName,
},
Spec: certsapi.CertificateSigningRequestSpec{
Request: csr,
Usages: usages,
},
}
req, err := r.client.CertificateSigningRequests().Create(k8sCSR)
if err != nil {
return nil, nil, errors.Wrap(err, "couldn't create certificate signing request")
}
watcher, err := r.client.CertificateSigningRequests().Watch(metav1.ListOptions{
Watch: true,
FieldSelector: fields.Set{"metadata.name": req.Name}.String(),
})
if err != nil {
return nil, nil, errors.Wrap(err, "couldn't watch for certificate creation")
}
defer watcher.Stop()
select {
case ev := <-watcher.ResultChan():
if ev.Type != watch.Modified {
return nil, nil, fmt.Errorf("unexpected event received: %q", ev.Type)
}
case <-time.After(watchTimeout):
return nil, nil, errors.New("timeout trying to sign certificate")
}
req, err = r.client.CertificateSigningRequests().Get(req.Name, metav1.GetOptions{})
if len(req.Status.Conditions) < 1 {
return nil, nil, errors.New("certificate signing request has no statuses")
}
// TODO: under what circumstances are there more than one?
if status := req.Status.Conditions[0].Type; status != certsapi.CertificateApproved {
return nil, nil, fmt.Errorf("unexpected certificate status: %v", status)
}
cert, err := x509.ParseCertificate(req.Status.Certificate)
if err != nil {
return nil, nil, errors.Wrap(err, "couldn't parse issued certificate")
}
return cert, key, nil
}
var usageMap = map[x509.ExtKeyUsage]certsapi.KeyUsage{
x509.ExtKeyUsageAny: certsapi.UsageAny,
x509.ExtKeyUsageServerAuth: certsapi.UsageServerAuth,
x509.ExtKeyUsageClientAuth: certsapi.UsageClientAuth,
x509.ExtKeyUsageCodeSigning: certsapi.UsageCodeSigning,
x509.ExtKeyUsageEmailProtection: certsapi.UsageEmailProtection,
x509.ExtKeyUsageIPSECEndSystem: certsapi.UsageIPsecEndSystem,
x509.ExtKeyUsageIPSECTunnel: certsapi.UsageIPsecTunnel,
x509.ExtKeyUsageIPSECUser: certsapi.UsageIPsecUser,
x509.ExtKeyUsageTimeStamping: certsapi.UsageTimestamping,
x509.ExtKeyUsageOCSPSigning: certsapi.UsageOCSPSigning,
x509.ExtKeyUsageMicrosoftServerGatedCrypto: certsapi.UsageMicrosoftSGC,
x509.ExtKeyUsageNetscapeServerGatedCrypto: certsapi.UsageNetscapSGC,
}

View File

@@ -0,0 +1,44 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package renewal
import (
"crypto/rsa"
"crypto/x509"
certutil "k8s.io/client-go/util/cert"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
)
// FileRenewal renews a certificate using local certs
type FileRenewal struct {
caCert *x509.Certificate
caKey *rsa.PrivateKey
}
// NewFileRenewal takes a certificate pair to construct the Interface.
func NewFileRenewal(caCert *x509.Certificate, caKey *rsa.PrivateKey) Interface {
return &FileRenewal{
caCert: caCert,
caKey: caKey,
}
}
// Renew takes a certificate using the cert and key
func (r *FileRenewal) Renew(cfg *certutil.Config) (*x509.Certificate, *rsa.PrivateKey, error) {
return pkiutil.NewCertAndKey(r.caCert, r.caKey, cfg)
}

View File

@@ -0,0 +1,61 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package renewal
import (
"crypto/x509"
"testing"
certutil "k8s.io/client-go/util/cert"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
)
func TestFileRenew(t *testing.T) {
caCertCfg := &certutil.Config{CommonName: "kubernetes"}
caCert, caKey, err := certs.NewCACertAndKey(caCertCfg)
if err != nil {
t.Fatalf("couldn't create CA: %v", err)
}
fr := NewFileRenewal(caCert, caKey)
certCfg := &certutil.Config{
CommonName: "test-certs",
AltNames: certutil.AltNames{
DNSNames: []string{"test-domain.space"},
},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
cert, _, err := fr.Renew(certCfg)
if err != nil {
t.Fatalf("unexpected error renewing cert: %v", err)
}
pool := x509.NewCertPool()
pool.AddCert(caCert)
_, err = cert.Verify(x509.VerifyOptions{
DNSName: "test-domain.space",
Roots: pool,
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
})
if err != nil {
t.Errorf("couldn't verify new cert: %v", err)
}
}

View File

@@ -0,0 +1,29 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package renewal
import (
"crypto/rsa"
"crypto/x509"
certutil "k8s.io/client-go/util/cert"
)
// Interface represents a standard way to renew a certificate.
type Interface interface {
Renew(*certutil.Config) (*x509.Certificate, *rsa.PrivateKey, error)
}

View File

@@ -0,0 +1,63 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package renewal
import (
"crypto/x509"
"fmt"
"github.com/pkg/errors"
certutil "k8s.io/client-go/util/cert"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
)
// RenewExistingCert loads a certificate file, uses the renew interface to renew it,
// and saves the resulting certificate and key over the old one.
func RenewExistingCert(certsDir, baseName string, impl Interface) error {
certificatePath, _ := pkiutil.PathsForCertAndKey(certsDir, baseName)
certs, err := certutil.CertsFromFile(certificatePath)
if err != nil {
return fmt.Errorf("failed to load existing certificate %s: %v", baseName, err)
}
if len(certs) != 1 {
return fmt.Errorf("wanted exactly one certificate, got %d", len(certs))
}
cfg := certToConfig(certs[0])
newCert, newKey, err := impl.Renew(cfg)
if err != nil {
return errors.Wrapf(err, "failed to renew certificate %s", baseName)
}
if err := pkiutil.WriteCertAndKey(certsDir, baseName, newCert, newKey); err != nil {
return errors.Wrapf(err, "failed to write new certificate %s", baseName)
}
return nil
}
func certToConfig(cert *x509.Certificate) *certutil.Config {
return &certutil.Config{
CommonName: cert.Subject.CommonName,
Organization: cert.Subject.Organization,
AltNames: certutil.AltNames{
IPs: cert.IPAddresses,
DNSNames: cert.DNSNames,
},
Usages: cert.ExtKeyUsage,
}
}

View File

@@ -0,0 +1,235 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package renewal
import (
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"net"
"os"
"testing"
"time"
certsapi "k8s.io/api/certificates/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
fakecerts "k8s.io/client-go/kubernetes/typed/certificates/v1beta1/fake"
k8stesting "k8s.io/client-go/testing"
certutil "k8s.io/client-go/util/cert"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
certtestutil "k8s.io/kubernetes/cmd/kubeadm/test/certs"
)
func TestRenewImplementations(t *testing.T) {
caCertCfg := &certutil.Config{CommonName: "kubernetes"}
caCert, caKey, err := certs.NewCACertAndKey(caCertCfg)
if err != nil {
t.Fatalf("couldn't create CA: %v", err)
}
client := &fakecerts.FakeCertificatesV1beta1{
Fake: &k8stesting.Fake{},
}
certReq := getCertReq(t, caCert, caKey)
client.AddReactor("get", "certificatesigningrequests", defaultReactionFunc(certReq))
watcher := watch.NewFakeWithChanSize(1, false)
watcher.Modify(certReq)
client.AddWatchReactor("certificatesigningrequests", k8stesting.DefaultWatchReactor(watcher, nil))
// override the timeout so tests are faster
watchTimeout = time.Second
tests := []struct {
name string
impl Interface
}{
{
name: "filerenewal",
impl: NewFileRenewal(caCert, caKey),
},
{
name: "certs api",
impl: &CertsAPIRenewal{
client: client,
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
certCfg := &certutil.Config{
CommonName: "test-certs",
AltNames: certutil.AltNames{
DNSNames: []string{"test-domain.space"},
},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
cert, _, err := test.impl.Renew(certCfg)
if err != nil {
t.Fatalf("unexpected error renewing cert: %v", err)
}
pool := x509.NewCertPool()
pool.AddCert(caCert)
_, err = cert.Verify(x509.VerifyOptions{
DNSName: "test-domain.space",
Roots: pool,
KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
})
if err != nil {
t.Errorf("couldn't verify new cert: %v", err)
}
})
}
}
func defaultReactionFunc(obj runtime.Object) k8stesting.ReactionFunc {
return func(act k8stesting.Action) (bool, runtime.Object, error) {
return true, obj, nil
}
}
func getCertReq(t *testing.T, caCert *x509.Certificate, caKey *rsa.PrivateKey) *certsapi.CertificateSigningRequest {
cert, _, err := pkiutil.NewCertAndKey(caCert, caKey, &certutil.Config{
CommonName: "testcert",
AltNames: certutil.AltNames{
DNSNames: []string{"test-domain.space"},
},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
})
if err != nil {
t.Fatalf("couldn't generate cert: %v", err)
}
return &certsapi.CertificateSigningRequest{
ObjectMeta: metav1.ObjectMeta{
Name: "testcert",
},
Status: certsapi.CertificateSigningRequestStatus{
Conditions: []certsapi.CertificateSigningRequestCondition{
{
Type: certsapi.CertificateApproved,
},
},
Certificate: cert.Raw,
},
}
}
func TestCertToConfig(t *testing.T) {
expectedConfig := &certutil.Config{
CommonName: "test-common-name",
Organization: []string{"sig-cluster-lifecycle"},
AltNames: certutil.AltNames{
IPs: []net.IP{net.ParseIP("10.100.0.1")},
DNSNames: []string{"test-domain.space"},
},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
cert := &x509.Certificate{
Subject: pkix.Name{
CommonName: "test-common-name",
Organization: []string{"sig-cluster-lifecycle"},
},
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
DNSNames: []string{"test-domain.space"},
IPAddresses: []net.IP{net.ParseIP("10.100.0.1")},
}
cfg := certToConfig(cert)
if cfg.CommonName != expectedConfig.CommonName {
t.Errorf("expected common name %q, got %q", expectedConfig.CommonName, cfg.CommonName)
}
if len(cfg.Organization) != 1 || cfg.Organization[0] != expectedConfig.Organization[0] {
t.Errorf("expected organization %v, got %v", expectedConfig.Organization, cfg.Organization)
}
if len(cfg.Usages) != 1 || cfg.Usages[0] != expectedConfig.Usages[0] {
t.Errorf("expected ext key usage %v, got %v", expectedConfig.Usages, cfg.Usages)
}
if len(cfg.AltNames.IPs) != 1 || cfg.AltNames.IPs[0].String() != expectedConfig.AltNames.IPs[0].String() {
t.Errorf("expected SAN IPs %v, got %v", expectedConfig.AltNames.IPs, cfg.AltNames.IPs)
}
if len(cfg.AltNames.DNSNames) != 1 || cfg.AltNames.DNSNames[0] != expectedConfig.AltNames.DNSNames[0] {
t.Errorf("expected SAN DNSNames %v, got %v", expectedConfig.AltNames.DNSNames, cfg.AltNames.DNSNames)
}
}
func TestRenewExistingCert(t *testing.T) {
cfg := &certutil.Config{
CommonName: "test-common-name",
Organization: []string{"sig-cluster-lifecycle"},
AltNames: certutil.AltNames{
IPs: []net.IP{net.ParseIP("10.100.0.1")},
DNSNames: []string{"test-domain.space"},
},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
caCertCfg := &certutil.Config{CommonName: "kubernetes"}
caCert, caKey, err := certs.NewCACertAndKey(caCertCfg)
if err != nil {
t.Fatalf("couldn't create CA: %v", err)
}
cert, key, err := pkiutil.NewCertAndKey(caCert, caKey, cfg)
if err != nil {
t.Fatalf("couldn't generate certificate: %v", err)
}
dir := testutil.SetupTempDir(t)
defer os.RemoveAll(dir)
if err := pkiutil.WriteCertAndKey(dir, "server", cert, key); err != nil {
t.Fatalf("couldn't write out certificate")
}
renewer := NewFileRenewal(caCert, caKey)
if err := RenewExistingCert(dir, "server", renewer); err != nil {
t.Fatalf("couldn't renew certificate: %v", err)
}
newCert, err := pkiutil.TryLoadCertFromDisk(dir, "server")
if err != nil {
t.Fatalf("couldn't load created certificate: %v", err)
}
if newCert.SerialNumber.Cmp(cert.SerialNumber) == 0 {
t.Fatal("expected new certificate, but renewed certificate has same serial number")
}
certtestutil.AssertCertificateIsSignedByCa(t, newCert, caCert)
certtestutil.AssertCertificateHasClientAuthUsage(t, newCert)
certtestutil.AssertCertificateHasOrganizations(t, newCert, cfg.Organization...)
certtestutil.AssertCertificateHasCommonName(t, newCert, cfg.CommonName)
certtestutil.AssertCertificateHasDNSNames(t, newCert, cfg.AltNames.DNSNames...)
certtestutil.AssertCertificateHasIPAddresses(t, newCert, cfg.AltNames.IPs...)
}

View File

@@ -20,10 +20,10 @@ go_test(
"//cmd/kubeadm/app/phases/certs:go_default_library",
"//cmd/kubeadm/test:go_default_library",
"//pkg/kubeapiserver/authorizer/modes:go_default_library",
"//pkg/util/pointer:go_default_library",
"//pkg/util/version:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/utils/pointer:go_default_library",
],
)
@@ -36,7 +36,7 @@ go_library(
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane",
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha3:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/features:go_default_library",
"//cmd/kubeadm/app/images:go_default_library",
@@ -45,9 +45,9 @@ go_library(
"//cmd/kubeadm/app/util/staticpod:go_default_library",
"//pkg/kubeapiserver/authorizer/modes:go_default_library",
"//pkg/util/version:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
],
)

View File

@@ -28,7 +28,7 @@ import (
"k8s.io/api/core/v1"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/images"
@@ -40,32 +40,32 @@ import (
)
// CreateInitStaticPodManifestFiles will write all static pod manifest files needed to bring up the control plane.
func CreateInitStaticPodManifestFiles(manifestDir string, cfg *kubeadmapi.MasterConfiguration) error {
func CreateInitStaticPodManifestFiles(manifestDir string, cfg *kubeadmapi.InitConfiguration) error {
glog.V(1).Infoln("[controlplane] creating static pod files")
return createStaticPodFiles(manifestDir, cfg, kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler)
}
// CreateAPIServerStaticPodManifestFile will write APIserver static pod manifest file.
func CreateAPIServerStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.MasterConfiguration) error {
func CreateAPIServerStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.InitConfiguration) error {
glog.V(1).Infoln("creating APIserver static pod files")
return createStaticPodFiles(manifestDir, cfg, kubeadmconstants.KubeAPIServer)
}
// CreateControllerManagerStaticPodManifestFile will write controller manager static pod manifest file.
func CreateControllerManagerStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.MasterConfiguration) error {
func CreateControllerManagerStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.InitConfiguration) error {
glog.V(1).Infoln("creating controller manager static pod files")
return createStaticPodFiles(manifestDir, cfg, kubeadmconstants.KubeControllerManager)
}
// CreateSchedulerStaticPodManifestFile will write scheduler static pod manifest file.
func CreateSchedulerStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.MasterConfiguration) error {
func CreateSchedulerStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.InitConfiguration) error {
glog.V(1).Infoln("creating scheduler static pod files")
return createStaticPodFiles(manifestDir, cfg, kubeadmconstants.KubeScheduler)
}
// GetStaticPodSpecs returns all staticPodSpecs actualized to the context of the current MasterConfiguration
// GetStaticPodSpecs returns all staticPodSpecs actualized to the context of the current InitConfiguration
// NB. this methods holds the information about how kubeadm creates static pod manifests.
func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) map[string]v1.Pod {
func GetStaticPodSpecs(cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Version) map[string]v1.Pod {
// Get the required hostpath mounts
mounts := getHostPathVolumesForTheControlPlane(cfg)
@@ -73,17 +73,17 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.
staticPodSpecs := map[string]v1.Pod{
kubeadmconstants.KubeAPIServer: staticpodutil.ComponentPod(v1.Container{
Name: kubeadmconstants.KubeAPIServer,
Image: images.GetCoreImage(kubeadmconstants.KubeAPIServer, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
Image: images.GetKubeControlPlaneImage(kubeadmconstants.KubeAPIServer, &cfg.ClusterConfiguration),
ImagePullPolicy: v1.PullIfNotPresent,
Command: getAPIServerCommand(cfg),
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeAPIServer)),
LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeAPIServer, int(cfg.API.BindPort), "/healthz", v1.URISchemeHTTPS),
LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeAPIServer, int(cfg.APIEndpoint.BindPort), "/healthz", v1.URISchemeHTTPS),
Resources: staticpodutil.ComponentResources("250m"),
Env: getProxyEnvVars(),
}, mounts.GetVolumes(kubeadmconstants.KubeAPIServer)),
kubeadmconstants.KubeControllerManager: staticpodutil.ComponentPod(v1.Container{
Name: kubeadmconstants.KubeControllerManager,
Image: images.GetCoreImage(kubeadmconstants.KubeControllerManager, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
Image: images.GetKubeControlPlaneImage(kubeadmconstants.KubeControllerManager, &cfg.ClusterConfiguration),
ImagePullPolicy: v1.PullIfNotPresent,
Command: getControllerManagerCommand(cfg, k8sVersion),
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)),
@@ -93,7 +93,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.
}, mounts.GetVolumes(kubeadmconstants.KubeControllerManager)),
kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{
Name: kubeadmconstants.KubeScheduler,
Image: images.GetCoreImage(kubeadmconstants.KubeScheduler, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage),
Image: images.GetKubeControlPlaneImage(kubeadmconstants.KubeScheduler, &cfg.ClusterConfiguration),
ImagePullPolicy: v1.PullIfNotPresent,
Command: getSchedulerCommand(cfg),
VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)),
@@ -106,14 +106,14 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.
}
// createStaticPodFiles creates all the requested static pod files.
func createStaticPodFiles(manifestDir string, cfg *kubeadmapi.MasterConfiguration, componentNames ...string) error {
func createStaticPodFiles(manifestDir string, cfg *kubeadmapi.InitConfiguration, componentNames ...string) error {
// TODO: Move the "pkg/util/version".Version object into the internal API instead of always parsing the string
k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
if err != nil {
return err
}
// gets the StaticPodSpecs, actualized for the current MasterConfiguration
// gets the StaticPodSpecs, actualized for the current InitConfiguration
glog.V(1).Infoln("[controlplane] getting StaticPodSpecs")
specs := GetStaticPodSpecs(cfg, k8sVersion)
@@ -137,16 +137,11 @@ func createStaticPodFiles(manifestDir string, cfg *kubeadmapi.MasterConfiguratio
}
// getAPIServerCommand builds the right API server command from the given config object and version
func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration) []string {
func getAPIServerCommand(cfg *kubeadmapi.InitConfiguration) []string {
defaultArguments := map[string]string{
"advertise-address": cfg.API.AdvertiseAddress,
"insecure-port": "0",
"enable-admission-plugins": "NodeRestriction",
// TODO: remove `PersistentVolumeLabel` in kubeadm v1.11, as it's automatically disabled in v1.11.
// ref: https://github.com/kubernetes/kubernetes/pull/64326
// we can't skip it now as we support v1.10 clusters still.
// remove it from the unit tests too.
"disable-admission-plugins": "PersistentVolumeLabel",
"advertise-address": cfg.APIEndpoint.AdvertiseAddress,
"insecure-port": "0",
"enable-admission-plugins": "NodeRestriction",
"service-cluster-ip-range": cfg.Networking.ServiceSubnet,
"service-account-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName),
"client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName),
@@ -155,7 +150,7 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration) []string {
"kubelet-client-certificate": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientCertName),
"kubelet-client-key": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientKeyName),
"enable-bootstrap-token-auth": "true",
"secure-port": fmt.Sprintf("%d", cfg.API.BindPort),
"secure-port": fmt.Sprintf("%d", cfg.APIEndpoint.BindPort),
"allow-privileged": "true",
"kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname",
// add options to configure the front proxy. Without the generated client cert, this will never be useable
@@ -189,6 +184,13 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration) []string {
defaultArguments["etcd-cafile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName)
defaultArguments["etcd-certfile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientCertName)
defaultArguments["etcd-keyfile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName)
// Apply user configurations for local etcd
if cfg.Etcd.Local != nil {
if value, ok := cfg.Etcd.Local.ExtraArgs["listen-client-urls"]; ok {
defaultArguments["etcd-servers"] = value
}
}
}
if features.Enabled(cfg.FeatureGates, features.HighAvailability) {
@@ -203,7 +205,7 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration) []string {
defaultArguments["audit-policy-file"] = kubeadmconstants.GetStaticPodAuditPolicyFile()
defaultArguments["audit-log-path"] = filepath.Join(kubeadmconstants.StaticPodAuditPolicyLogDir, kubeadmconstants.AuditPolicyLogFile)
if cfg.AuditPolicyConfiguration.LogMaxAge == nil {
defaultArguments["audit-log-maxage"] = fmt.Sprintf("%d", kubeadmapiv1alpha2.DefaultAuditPolicyLogMaxAge)
defaultArguments["audit-log-maxage"] = fmt.Sprintf("%d", kubeadmapiv1alpha3.DefaultAuditPolicyLogMaxAge)
} else {
defaultArguments["audit-log-maxage"] = fmt.Sprintf("%d", *cfg.AuditPolicyConfiguration.LogMaxAge)
}
@@ -279,7 +281,7 @@ func calcNodeCidrSize(podSubnet string) string {
}
// getControllerManagerCommand builds the right controller manager command from the given config object and version
func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) []string {
func getControllerManagerCommand(cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Version) []string {
defaultArguments := map[string]string{
"address": "127.0.0.1",
"leader-elect": "true",
@@ -292,6 +294,14 @@ func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion
"controllers": "*,bootstrapsigner,tokencleaner",
}
//add the extra arguments for v1.12+
if k8sVersion.Major() >= 1 && k8sVersion.Minor() >= 12 {
defaultArguments["authentication-kubeconfig"] = filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName)
defaultArguments["authorization-kubeconfig"] = filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName)
defaultArguments["client-ca-file"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName)
defaultArguments["requestheader-client-ca-file"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName)
}
// If using external CA, pass empty string to controller manager instead of ca.key/ca.crt path,
// so that the csrsigning controller fails to start
if res, _ := certphase.UsingExternalCA(cfg); res {
@@ -315,7 +325,7 @@ func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion
}
// getSchedulerCommand builds the right scheduler command from the given config object and version
func getSchedulerCommand(cfg *kubeadmapi.MasterConfiguration) []string {
func getSchedulerCommand(cfg *kubeadmapi.InitConfiguration) []string {
defaultArguments := map[string]string{
"address": "127.0.0.1",
"leader-elect": "true",

View File

@@ -34,7 +34,7 @@ import (
"k8s.io/kubernetes/pkg/util/version"
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
utilpointer "k8s.io/kubernetes/pkg/util/pointer"
utilpointer "k8s.io/utils/pointer"
)
const (
@@ -45,8 +45,10 @@ const (
func TestGetStaticPodSpecs(t *testing.T) {
// Creates a Master Configuration
cfg := &kubeadmapi.MasterConfiguration{
KubernetesVersion: "v1.9.0",
cfg := &kubeadmapi.InitConfiguration{
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
KubernetesVersion: "v1.9.0",
},
}
// Executes GetStaticPodSpecs
@@ -89,7 +91,7 @@ func TestGetStaticPodSpecs(t *testing.T) {
func TestCreateStaticPodFilesAndWrappers(t *testing.T) {
var tests = []struct {
createStaticPodFunction func(outDir string, cfg *kubeadmapi.MasterConfiguration) error
createStaticPodFunction func(outDir string, cfg *kubeadmapi.InitConfiguration) error
expectedFiles []string
}{
{ // CreateInitStaticPodManifestFiles
@@ -117,8 +119,10 @@ func TestCreateStaticPodFilesAndWrappers(t *testing.T) {
defer os.RemoveAll(tmpdir)
// Creates a Master Configuration
cfg := &kubeadmapi.MasterConfiguration{
KubernetesVersion: "v1.9.0",
cfg := &kubeadmapi.InitConfiguration{
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
KubernetesVersion: "v1.9.0",
},
}
// Execute createStaticPodFunction
@@ -141,21 +145,22 @@ func TestCreateStaticPodFilesAndWrappers(t *testing.T) {
func TestGetAPIServerCommand(t *testing.T) {
var tests = []struct {
name string
cfg *kubeadmapi.MasterConfiguration
cfg *kubeadmapi.InitConfiguration
expected []string
}{
{
name: "testing defaults",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
CertificatesDir: testCertsDir,
cfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
CertificatesDir: testCertsDir,
},
},
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--enable-admission-plugins=NodeRestriction",
"--disable-admission-plugins=PersistentVolumeLabel",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@@ -184,21 +189,22 @@ func TestGetAPIServerCommand(t *testing.T) {
},
{
name: "ignores the audit policy if the feature gate is not enabled",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"},
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
CertificatesDir: testCertsDir,
AuditPolicyConfiguration: kubeadmapi.AuditPolicyConfiguration{
Path: "/foo/bar",
LogDir: "/foo/baz",
LogMaxAge: utilpointer.Int32Ptr(10),
cfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "4.3.2.1"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
CertificatesDir: testCertsDir,
AuditPolicyConfiguration: kubeadmapi.AuditPolicyConfiguration{
Path: "/foo/bar",
LogDir: "/foo/baz",
LogMaxAge: utilpointer.Int32Ptr(10),
},
},
},
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--enable-admission-plugins=NodeRestriction",
"--disable-admission-plugins=PersistentVolumeLabel",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@@ -227,16 +233,17 @@ func TestGetAPIServerCommand(t *testing.T) {
},
{
name: "ipv6 advertise address",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"},
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
CertificatesDir: testCertsDir,
cfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
CertificatesDir: testCertsDir,
},
},
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--enable-admission-plugins=NodeRestriction",
"--disable-admission-plugins=PersistentVolumeLabel",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@@ -265,25 +272,26 @@ func TestGetAPIServerCommand(t *testing.T) {
},
{
name: "an external etcd with custom ca, certs and keys",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"},
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
FeatureGates: map[string]bool{features.HighAvailability: true},
Etcd: kubeadmapi.Etcd{
External: &kubeadmapi.ExternalEtcd{
Endpoints: []string{"https://8.6.4.1:2379", "https://8.6.4.2:2379"},
CAFile: "fuz",
CertFile: "fiz",
KeyFile: "faz",
cfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
FeatureGates: map[string]bool{features.HighAvailability: true},
Etcd: kubeadmapi.Etcd{
External: &kubeadmapi.ExternalEtcd{
Endpoints: []string{"https://8.6.4.1:2379", "https://8.6.4.2:2379"},
CAFile: "fuz",
CertFile: "fiz",
KeyFile: "faz",
},
},
CertificatesDir: testCertsDir,
},
CertificatesDir: testCertsDir,
},
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--enable-admission-plugins=NodeRestriction",
"--disable-admission-plugins=PersistentVolumeLabel",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@@ -313,21 +321,22 @@ func TestGetAPIServerCommand(t *testing.T) {
},
{
name: "an insecure etcd",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"},
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
Etcd: kubeadmapi.Etcd{
External: &kubeadmapi.ExternalEtcd{
Endpoints: []string{"http://127.0.0.1:2379", "http://127.0.0.1:2380"},
cfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
Etcd: kubeadmapi.Etcd{
External: &kubeadmapi.ExternalEtcd{
Endpoints: []string{"http://127.0.0.1:2379", "http://127.0.0.1:2380"},
},
},
CertificatesDir: testCertsDir,
},
CertificatesDir: testCertsDir,
},
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--enable-admission-plugins=NodeRestriction",
"--disable-admission-plugins=PersistentVolumeLabel",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@@ -353,20 +362,21 @@ func TestGetAPIServerCommand(t *testing.T) {
},
{
name: "auditing and HA are enabled with a custom log max age of 0",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "2001:db8::1"},
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
FeatureGates: map[string]bool{features.HighAvailability: true, features.Auditing: true},
CertificatesDir: testCertsDir,
AuditPolicyConfiguration: kubeadmapi.AuditPolicyConfiguration{
LogMaxAge: utilpointer.Int32Ptr(0),
cfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
FeatureGates: map[string]bool{features.HighAvailability: true, features.Auditing: true},
CertificatesDir: testCertsDir,
AuditPolicyConfiguration: kubeadmapi.AuditPolicyConfiguration{
LogMaxAge: utilpointer.Int32Ptr(0),
},
},
},
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--enable-admission-plugins=NodeRestriction",
"--disable-admission-plugins=PersistentVolumeLabel",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@@ -399,17 +409,18 @@ func TestGetAPIServerCommand(t *testing.T) {
},
{
name: "ensure the DynamicKubelet flag gets passed through",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
CertificatesDir: testCertsDir,
FeatureGates: map[string]bool{features.DynamicKubeletConfig: true},
cfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
CertificatesDir: testCertsDir,
FeatureGates: map[string]bool{features.DynamicKubeletConfig: true},
},
},
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--enable-admission-plugins=NodeRestriction",
"--disable-admission-plugins=PersistentVolumeLabel",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@@ -439,23 +450,24 @@ func TestGetAPIServerCommand(t *testing.T) {
},
{
name: "test APIServerExtraArgs works as expected",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
CertificatesDir: testCertsDir,
FeatureGates: map[string]bool{features.DynamicKubeletConfig: true, features.Auditing: true},
APIServerExtraArgs: map[string]string{
"service-cluster-ip-range": "baz",
"advertise-address": "9.9.9.9",
"audit-policy-file": "/etc/config/audit.yaml",
"audit-log-path": "/var/log/kubernetes",
cfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
CertificatesDir: testCertsDir,
FeatureGates: map[string]bool{features.DynamicKubeletConfig: true, features.Auditing: true},
APIServerExtraArgs: map[string]string{
"service-cluster-ip-range": "baz",
"advertise-address": "9.9.9.9",
"audit-policy-file": "/etc/config/audit.yaml",
"audit-log-path": "/var/log/kubernetes",
},
},
},
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--enable-admission-plugins=NodeRestriction",
"--disable-admission-plugins=PersistentVolumeLabel",
"--service-cluster-ip-range=baz",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@@ -488,19 +500,20 @@ func TestGetAPIServerCommand(t *testing.T) {
},
{
name: "authorization-mode extra-args ABAC",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
CertificatesDir: testCertsDir,
APIServerExtraArgs: map[string]string{
"authorization-mode": authzmodes.ModeABAC,
cfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
CertificatesDir: testCertsDir,
APIServerExtraArgs: map[string]string{
"authorization-mode": authzmodes.ModeABAC,
},
},
},
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--enable-admission-plugins=NodeRestriction",
"--disable-admission-plugins=PersistentVolumeLabel",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@@ -529,19 +542,20 @@ func TestGetAPIServerCommand(t *testing.T) {
},
{
name: "insecure-port extra-args",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
CertificatesDir: testCertsDir,
APIServerExtraArgs: map[string]string{
"insecure-port": "1234",
cfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
CertificatesDir: testCertsDir,
APIServerExtraArgs: map[string]string{
"insecure-port": "1234",
},
},
},
expected: []string{
"kube-apiserver",
"--insecure-port=1234",
"--enable-admission-plugins=NodeRestriction",
"--disable-admission-plugins=PersistentVolumeLabel",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@@ -570,19 +584,20 @@ func TestGetAPIServerCommand(t *testing.T) {
},
{
name: "authorization-mode extra-args Webhook",
cfg: &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
CertificatesDir: testCertsDir,
APIServerExtraArgs: map[string]string{
"authorization-mode": authzmodes.ModeWebhook,
cfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{ServiceSubnet: "bar"},
CertificatesDir: testCertsDir,
APIServerExtraArgs: map[string]string{
"authorization-mode": authzmodes.ModeWebhook,
},
},
},
expected: []string{
"kube-apiserver",
"--insecure-port=0",
"--enable-admission-plugins=NodeRestriction",
"--disable-admission-plugins=PersistentVolumeLabel",
"--service-cluster-ip-range=bar",
"--service-account-key-file=" + testCertsDir + "/sa.pub",
"--client-ca-file=" + testCertsDir + "/ca.crt",
@@ -644,14 +659,119 @@ func removeCommon(left, right []string) []string {
func TestGetControllerManagerCommand(t *testing.T) {
var tests = []struct {
name string
cfg *kubeadmapi.MasterConfiguration
cfg *kubeadmapi.ClusterConfiguration
expected []string
}{
{
name: "custom certs dir",
cfg: &kubeadmapi.MasterConfiguration{
name: "custom certs dir for v1.12.0-beta.2",
cfg: &kubeadmapi.ClusterConfiguration{
CertificatesDir: testCertsDir,
KubernetesVersion: "v1.7.0",
KubernetesVersion: "v1.12.0-beta.2",
},
expected: []string{
"kube-controller-manager",
"--address=127.0.0.1",
"--leader-elect=true",
"--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--root-ca-file=" + testCertsDir + "/ca.crt",
"--service-account-private-key-file=" + testCertsDir + "/sa.key",
"--cluster-signing-cert-file=" + testCertsDir + "/ca.crt",
"--cluster-signing-key-file=" + testCertsDir + "/ca.key",
"--use-service-account-credentials=true",
"--controllers=*,bootstrapsigner,tokencleaner",
"--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--client-ca-file=" + testCertsDir + "/ca.crt",
"--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
},
},
{
name: "custom cloudprovider for v1.12.0-beta.2",
cfg: &kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{PodSubnet: "10.0.1.15/16"},
CertificatesDir: testCertsDir,
KubernetesVersion: "v1.12.0-beta.2",
},
expected: []string{
"kube-controller-manager",
"--address=127.0.0.1",
"--leader-elect=true",
"--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--root-ca-file=" + testCertsDir + "/ca.crt",
"--service-account-private-key-file=" + testCertsDir + "/sa.key",
"--cluster-signing-cert-file=" + testCertsDir + "/ca.crt",
"--cluster-signing-key-file=" + testCertsDir + "/ca.key",
"--use-service-account-credentials=true",
"--controllers=*,bootstrapsigner,tokencleaner",
"--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--client-ca-file=" + testCertsDir + "/ca.crt",
"--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
"--allocate-node-cidrs=true",
"--cluster-cidr=10.0.1.15/16",
"--node-cidr-mask-size=24",
},
},
{
name: "custom extra-args for v1.12.0-beta.2",
cfg: &kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{PodSubnet: "10.0.1.15/16"},
ControllerManagerExtraArgs: map[string]string{"node-cidr-mask-size": "20"},
CertificatesDir: testCertsDir,
KubernetesVersion: "v1.12.0-beta.2",
},
expected: []string{
"kube-controller-manager",
"--address=127.0.0.1",
"--leader-elect=true",
"--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--root-ca-file=" + testCertsDir + "/ca.crt",
"--service-account-private-key-file=" + testCertsDir + "/sa.key",
"--cluster-signing-cert-file=" + testCertsDir + "/ca.crt",
"--cluster-signing-key-file=" + testCertsDir + "/ca.key",
"--use-service-account-credentials=true",
"--controllers=*,bootstrapsigner,tokencleaner",
"--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--client-ca-file=" + testCertsDir + "/ca.crt",
"--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
"--allocate-node-cidrs=true",
"--cluster-cidr=10.0.1.15/16",
"--node-cidr-mask-size=20",
},
},
{
name: "custom IPv6 networking for v1.12.0-beta.2",
cfg: &kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{PodSubnet: "2001:db8::/64"},
CertificatesDir: testCertsDir,
KubernetesVersion: "v1.12.0-beta.2",
},
expected: []string{
"kube-controller-manager",
"--address=127.0.0.1",
"--leader-elect=true",
"--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--root-ca-file=" + testCertsDir + "/ca.crt",
"--service-account-private-key-file=" + testCertsDir + "/sa.key",
"--cluster-signing-cert-file=" + testCertsDir + "/ca.crt",
"--cluster-signing-key-file=" + testCertsDir + "/ca.key",
"--use-service-account-credentials=true",
"--controllers=*,bootstrapsigner,tokencleaner",
"--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--client-ca-file=" + testCertsDir + "/ca.crt",
"--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt",
"--allocate-node-cidrs=true",
"--cluster-cidr=2001:db8::/64",
"--node-cidr-mask-size=80",
},
},
{
name: "custom certs dir for v1.11.3",
cfg: &kubeadmapi.ClusterConfiguration{
CertificatesDir: testCertsDir,
KubernetesVersion: "v1.11.3",
},
expected: []string{
"kube-controller-manager",
@@ -667,11 +787,11 @@ func TestGetControllerManagerCommand(t *testing.T) {
},
},
{
name: "custom cloudprovider",
cfg: &kubeadmapi.MasterConfiguration{
name: "custom cloudprovider for v1.11.3",
cfg: &kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{PodSubnet: "10.0.1.15/16"},
CertificatesDir: testCertsDir,
KubernetesVersion: "v1.7.0",
KubernetesVersion: "v1.11.3",
},
expected: []string{
"kube-controller-manager",
@@ -690,12 +810,12 @@ func TestGetControllerManagerCommand(t *testing.T) {
},
},
{
name: "custom extra-args",
cfg: &kubeadmapi.MasterConfiguration{
name: "custom extra-args for v1.11.3",
cfg: &kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{PodSubnet: "10.0.1.15/16"},
ControllerManagerExtraArgs: map[string]string{"node-cidr-mask-size": "20"},
CertificatesDir: testCertsDir,
KubernetesVersion: "v1.7.0",
KubernetesVersion: "v1.11.3",
},
expected: []string{
"kube-controller-manager",
@@ -714,11 +834,11 @@ func TestGetControllerManagerCommand(t *testing.T) {
},
},
{
name: "custom IPv6 networking",
cfg: &kubeadmapi.MasterConfiguration{
name: "custom IPv6 networking for v1.11.3",
cfg: &kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{PodSubnet: "2001:db8::/64"},
CertificatesDir: testCertsDir,
KubernetesVersion: "v1.7.0",
KubernetesVersion: "v1.11.3",
},
expected: []string{
"kube-controller-manager",
@@ -739,7 +859,11 @@ func TestGetControllerManagerCommand(t *testing.T) {
}
for _, rt := range tests {
actual := getControllerManagerCommand(rt.cfg, version.MustParseSemantic(rt.cfg.KubernetesVersion))
// TODO: Make getControllerManagerCommand accept a ClusterConfiguration object instead of InitConfiguration
initcfg := &kubeadmapi.InitConfiguration{
ClusterConfiguration: *rt.cfg,
}
actual := getControllerManagerCommand(initcfg, version.MustParseSemantic(rt.cfg.KubernetesVersion))
sort.Strings(actual)
sort.Strings(rt.expected)
if !reflect.DeepEqual(actual, rt.expected) {
@@ -828,17 +952,76 @@ func TestGetControllerManagerCommandExternalCA(t *testing.T) {
tests := []struct {
name string
cfg *kubeadmapi.MasterConfiguration
cfg *kubeadmapi.InitConfiguration
caKeyPresent bool
expectedArgFunc func(dir string) []string
}{
{
name: "caKeyPresent-false",
cfg: &kubeadmapi.MasterConfiguration{
KubernetesVersion: "v1.7.0",
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"},
name: "caKeyPresent-false for v1.12.0-beta.2",
cfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
KubernetesVersion: "v1.12.0-beta.2",
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
},
},
caKeyPresent: false,
expectedArgFunc: func(tmpdir string) []string {
return []string{
"kube-controller-manager",
"--address=127.0.0.1",
"--leader-elect=true",
"--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--root-ca-file=" + tmpdir + "/ca.crt",
"--service-account-private-key-file=" + tmpdir + "/sa.key",
"--cluster-signing-cert-file=",
"--cluster-signing-key-file=",
"--use-service-account-credentials=true",
"--controllers=*,bootstrapsigner,tokencleaner",
"--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--client-ca-file=" + tmpdir + "/ca.crt",
"--requestheader-client-ca-file=" + tmpdir + "/front-proxy-ca.crt",
}
},
},
{
name: "caKeyPresent true for v1.12.0-beta.2",
cfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
KubernetesVersion: "v1.12.0-beta.2",
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
},
},
caKeyPresent: true,
expectedArgFunc: func(tmpdir string) []string {
return []string{
"kube-controller-manager",
"--address=127.0.0.1",
"--leader-elect=true",
"--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--root-ca-file=" + tmpdir + "/ca.crt",
"--service-account-private-key-file=" + tmpdir + "/sa.key",
"--cluster-signing-cert-file=" + tmpdir + "/ca.crt",
"--cluster-signing-key-file=" + tmpdir + "/ca.key",
"--use-service-account-credentials=true",
"--controllers=*,bootstrapsigner,tokencleaner",
"--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf",
"--client-ca-file=" + tmpdir + "/ca.crt",
"--requestheader-client-ca-file=" + tmpdir + "/front-proxy-ca.crt",
}
},
},
{
name: "caKeyPresent-false for v1.11.3",
cfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
KubernetesVersion: "v1.11.3",
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
},
},
caKeyPresent: false,
expectedArgFunc: func(tmpdir string) []string {
@@ -857,12 +1040,13 @@ func TestGetControllerManagerCommandExternalCA(t *testing.T) {
},
},
{
name: "caKeyPresent true",
cfg: &kubeadmapi.MasterConfiguration{
KubernetesVersion: "v1.7.0",
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"},
name: "caKeyPresent true for v1.11.3",
cfg: &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
KubernetesVersion: "v1.11.3",
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
},
},
caKeyPresent: true,
expectedArgFunc: func(tmpdir string) []string {
@@ -915,12 +1099,12 @@ func TestGetControllerManagerCommandExternalCA(t *testing.T) {
func TestGetSchedulerCommand(t *testing.T) {
var tests = []struct {
name string
cfg *kubeadmapi.MasterConfiguration
cfg *kubeadmapi.ClusterConfiguration
expected []string
}{
{
name: "scheduler defaults",
cfg: &kubeadmapi.MasterConfiguration{},
cfg: &kubeadmapi.ClusterConfiguration{},
expected: []string{
"kube-scheduler",
"--address=127.0.0.1",
@@ -931,7 +1115,11 @@ func TestGetSchedulerCommand(t *testing.T) {
}
for _, rt := range tests {
actual := getSchedulerCommand(rt.cfg)
// TODO: Make getSchedulerCommand accept a ClusterConfiguration object instead of InitConfiguration
initcfg := &kubeadmapi.InitConfiguration{
ClusterConfiguration: *rt.cfg,
}
actual := getSchedulerCommand(initcfg)
sort.Strings(actual)
sort.Strings(rt.expected)
if !reflect.DeepEqual(actual, rt.expected) {

View File

@@ -43,7 +43,7 @@ const (
var caCertsExtraVolumePaths = []string{"/etc/pki", "/usr/share/ca-certificates", "/usr/local/share/ca-certificates", "/etc/ca-certificates"}
// getHostPathVolumesForTheControlPlane gets the required hostPath volumes and mounts for the control plane
func getHostPathVolumesForTheControlPlane(cfg *kubeadmapi.MasterConfiguration) controlPlaneHostPathMounts {
func getHostPathVolumesForTheControlPlane(cfg *kubeadmapi.InitConfiguration) controlPlaneHostPathMounts {
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
hostPathFileOrCreate := v1.HostPathFileOrCreate
hostPathFile := v1.HostPathFile

View File

@@ -502,13 +502,13 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) {
ReadOnly: true,
}
var tests = []struct {
cfg *kubeadmapi.MasterConfiguration
cfg *kubeadmapi.ClusterConfiguration
vol map[string]map[string]v1.Volume
volMount map[string]map[string]v1.VolumeMount
}{
{
// Should ignore files in /etc/ssl/certs
cfg: &kubeadmapi.MasterConfiguration{
cfg: &kubeadmapi.ClusterConfiguration{
CertificatesDir: testCertsDir,
Etcd: kubeadmapi.Etcd{},
FeatureGates: map[string]bool{features.Auditing: true},
@@ -522,7 +522,7 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) {
},
{
// Should ignore files in /etc/ssl/certs and in CertificatesDir
cfg: &kubeadmapi.MasterConfiguration{
cfg: &kubeadmapi.ClusterConfiguration{
CertificatesDir: testCertsDir,
Etcd: kubeadmapi.Etcd{
External: &kubeadmapi.ExternalEtcd{
@@ -549,7 +549,11 @@ func TestGetHostPathVolumesForTheControlPlane(t *testing.T) {
defer func() { caCertsExtraVolumePaths = []string{"/etc/pki", "/usr/share/ca-certificates"} }()
for _, rt := range tests {
mounts := getHostPathVolumesForTheControlPlane(rt.cfg)
// TODO: Make getHostPathVolumesForTheControlPlane accept a ClusterConfiguration object instead of InitConfiguration
initcfg := &kubeadmapi.InitConfiguration{
ClusterConfiguration: *rt.cfg,
}
mounts := getHostPathVolumesForTheControlPlane(initcfg)
// Avoid unit test errors when the flexvolume is mounted
if _, ok := mounts.volumes[kubeadmconstants.KubeControllerManager][flexvolumeDirVolumeName]; ok {

View File

@@ -27,8 +27,8 @@ go_library(
"//cmd/kubeadm/app/images:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/staticpod:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
],
)

View File

@@ -36,9 +36,9 @@ const (
)
// CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file.
func CreateLocalEtcdStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.MasterConfiguration) error {
func CreateLocalEtcdStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.InitConfiguration) error {
glog.V(1).Infoln("creating local etcd static pod manifest file")
// gets etcd StaticPodSpec, actualized for the current MasterConfiguration
// gets etcd StaticPodSpec, actualized for the current InitConfiguration
spec := GetEtcdPodSpec(cfg)
// writes etcd StaticPod to disk
if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
@@ -49,9 +49,9 @@ func CreateLocalEtcdStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.Ma
return nil
}
// GetEtcdPodSpec returns the etcd static Pod actualized to the context of the current MasterConfiguration
// GetEtcdPodSpec returns the etcd static Pod actualized to the context of the current InitConfiguration
// NB. GetEtcdPodSpec methods holds the information about how kubeadm creates etcd static pod manifests.
func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod {
func GetEtcdPodSpec(cfg *kubeadmapi.InitConfiguration) v1.Pod {
pathType := v1.HostPathDirectoryOrCreate
etcdMounts := map[string]v1.Volume{
etcdVolumeName: staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.Local.DataDir, &pathType),
@@ -60,7 +60,7 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod {
return staticpodutil.ComponentPod(v1.Container{
Name: kubeadmconstants.Etcd,
Command: getEtcdCommand(cfg),
Image: images.GetCoreImage(kubeadmconstants.Etcd, cfg.ImageRepository, cfg.KubernetesVersion, cfg.Etcd.Local.Image),
Image: images.GetEtcdImage(&cfg.ClusterConfiguration),
ImagePullPolicy: v1.PullIfNotPresent,
// Mount the etcd datadir path read-write so etcd can store data in a more persistent manner
VolumeMounts: []v1.VolumeMount{
@@ -75,7 +75,7 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod {
}
// getEtcdCommand builds the right etcd command from the given config object
func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string {
func getEtcdCommand(cfg *kubeadmapi.InitConfiguration) []string {
defaultArguments := map[string]string{
"name": cfg.GetNodeName(),
"listen-client-urls": "https://127.0.0.1:2379",

View File

@@ -32,12 +32,14 @@ import (
func TestGetEtcdPodSpec(t *testing.T) {
// Creates a Master Configuration
cfg := &kubeadmapi.MasterConfiguration{
KubernetesVersion: "v1.7.0",
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
DataDir: "/var/lib/etcd",
Image: "",
cfg := &kubeadmapi.InitConfiguration{
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
KubernetesVersion: "v1.7.0",
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
DataDir: "/var/lib/etcd",
Image: "",
},
},
},
}
@@ -58,12 +60,14 @@ func TestCreateLocalEtcdStaticPodManifestFile(t *testing.T) {
defer os.RemoveAll(tmpdir)
// Creates a Master Configuration
cfg := &kubeadmapi.MasterConfiguration{
KubernetesVersion: "v1.7.0",
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
DataDir: "/var/lib/etcd",
Image: "k8s.gcr.io/etcd",
cfg := &kubeadmapi.InitConfiguration{
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
KubernetesVersion: "v1.7.0",
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
DataDir: "/var/lib/etcd",
Image: "k8s.gcr.io/etcd",
},
},
},
}
@@ -82,17 +86,19 @@ func TestCreateLocalEtcdStaticPodManifestFile(t *testing.T) {
func TestGetEtcdCommand(t *testing.T) {
var tests = []struct {
cfg *kubeadmapi.MasterConfiguration
cfg *kubeadmapi.InitConfiguration
expected []string
}{
{
cfg: &kubeadmapi.MasterConfiguration{
cfg: &kubeadmapi.InitConfiguration{
NodeRegistration: kubeadmapi.NodeRegistrationOptions{
Name: "foo",
},
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
DataDir: "/var/lib/etcd",
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
DataDir: "/var/lib/etcd",
},
},
},
},
@@ -117,16 +123,18 @@ func TestGetEtcdCommand(t *testing.T) {
},
},
{
cfg: &kubeadmapi.MasterConfiguration{
cfg: &kubeadmapi.InitConfiguration{
NodeRegistration: kubeadmapi.NodeRegistrationOptions{
Name: "bar",
},
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
DataDir: "/var/lib/etcd",
ExtraArgs: map[string]string{
"listen-client-urls": "https://10.0.1.10:2379",
"advertise-client-urls": "https://10.0.1.10:2379",
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
DataDir: "/var/lib/etcd",
ExtraArgs: map[string]string{
"listen-client-urls": "https://10.0.1.10:2379",
"advertise-client-urls": "https://10.0.1.10:2379",
},
},
},
},
@@ -152,13 +160,15 @@ func TestGetEtcdCommand(t *testing.T) {
},
},
{
cfg: &kubeadmapi.MasterConfiguration{
cfg: &kubeadmapi.InitConfiguration{
NodeRegistration: kubeadmapi.NodeRegistrationOptions{
Name: "wombat",
},
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
DataDir: "/etc/foo",
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Etcd: kubeadmapi.Etcd{
Local: &kubeadmapi.LocalEtcd{
DataDir: "/etc/foo",
},
},
},
},

View File

@@ -19,10 +19,10 @@ go_library(
"//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/kubeconfig:go_default_library",
"//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
"//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
"//vendor/k8s.io/client-go/util/cert:go_default_library",
],
)
@@ -51,7 +51,7 @@ go_test(
"//cmd/kubeadm/test:go_default_library",
"//cmd/kubeadm/test/certs:go_default_library",
"//cmd/kubeadm/test/kubeconfig:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
"//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
"//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
"//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library",
],
)

View File

@@ -21,7 +21,7 @@ package kubeconfig
PHASE: KUBECONFIG
INPUTS:
From MasterConfiguration
From InitConfiguration
The Master API Server endpoint (AdvertiseAddress + BindPort) is required so the KubeConfig file knows where to find the master
The KubernetesDir path is required for knowing where to put the KubeConfig files
The PKIPath is required for knowing where all certificates should be stored

View File

@@ -61,7 +61,7 @@ type kubeConfigSpec struct {
// CreateInitKubeConfigFiles will create and write to disk all kubeconfig files necessary in the kubeadm init phase
// to establish the control plane, including also the admin kubeconfig file.
// If kubeconfig files already exists, they are used only if evaluated equal; otherwise an error is returned.
func CreateInitKubeConfigFiles(outDir string, cfg *kubeadmapi.MasterConfiguration) error {
func CreateInitKubeConfigFiles(outDir string, cfg *kubeadmapi.InitConfiguration) error {
glog.V(1).Infoln("creating all kubeconfig files")
return createKubeConfigFiles(
outDir,
@@ -73,39 +73,53 @@ func CreateInitKubeConfigFiles(outDir string, cfg *kubeadmapi.MasterConfiguratio
)
}
// CreateJoinControlPlaneKubeConfigFiles will create and write to disk the kubeconfig files required by kubeadm
// join --control-plane workflow, plus the admin kubeconfig file used by the administrator and kubeadm itself; the
// kubelet.conf file must not be created because it will be created and signed by the kubelet TLS bootstrap process.
// If any kubeconfig files already exists, it used only if evaluated equal; otherwise an error is returned.
func CreateJoinControlPlaneKubeConfigFiles(outDir string, cfg *kubeadmapi.InitConfiguration) error {
return createKubeConfigFiles(
outDir,
cfg,
kubeadmconstants.AdminKubeConfigFileName,
kubeadmconstants.ControllerManagerKubeConfigFileName,
kubeadmconstants.SchedulerKubeConfigFileName,
)
}
// CreateAdminKubeConfigFile create a kubeconfig file for the admin to use and for kubeadm itself.
// If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned.
func CreateAdminKubeConfigFile(outDir string, cfg *kubeadmapi.MasterConfiguration) error {
func CreateAdminKubeConfigFile(outDir string, cfg *kubeadmapi.InitConfiguration) error {
glog.V(1).Infoln("create a kubeconfig file for the admin and for kubeadm itself")
return createKubeConfigFiles(outDir, cfg, kubeadmconstants.AdminKubeConfigFileName)
}
// CreateKubeletKubeConfigFile create a kubeconfig file for the Kubelet to use.
// If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned.
func CreateKubeletKubeConfigFile(outDir string, cfg *kubeadmapi.MasterConfiguration) error {
func CreateKubeletKubeConfigFile(outDir string, cfg *kubeadmapi.InitConfiguration) error {
glog.V(1).Infoln("creating a kubeconfig file for the Kubelet")
return createKubeConfigFiles(outDir, cfg, kubeadmconstants.KubeletKubeConfigFileName)
}
// CreateControllerManagerKubeConfigFile create a kubeconfig file for the ControllerManager to use.
// If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned.
func CreateControllerManagerKubeConfigFile(outDir string, cfg *kubeadmapi.MasterConfiguration) error {
func CreateControllerManagerKubeConfigFile(outDir string, cfg *kubeadmapi.InitConfiguration) error {
glog.V(1).Infoln("creating kubeconfig file for the ControllerManager")
return createKubeConfigFiles(outDir, cfg, kubeadmconstants.ControllerManagerKubeConfigFileName)
}
// CreateSchedulerKubeConfigFile create a create a kubeconfig file for the Scheduler to use.
// If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned.
func CreateSchedulerKubeConfigFile(outDir string, cfg *kubeadmapi.MasterConfiguration) error {
func CreateSchedulerKubeConfigFile(outDir string, cfg *kubeadmapi.InitConfiguration) error {
glog.V(1).Infoln("creating kubeconfig file for Scheduler")
return createKubeConfigFiles(outDir, cfg, kubeadmconstants.SchedulerKubeConfigFileName)
}
// createKubeConfigFiles creates all the requested kubeconfig files.
// If kubeconfig files already exists, they are used only if evaluated equal; otherwise an error is returned.
func createKubeConfigFiles(outDir string, cfg *kubeadmapi.MasterConfiguration, kubeConfigFileNames ...string) error {
func createKubeConfigFiles(outDir string, cfg *kubeadmapi.InitConfiguration, kubeConfigFileNames ...string) error {
// gets the KubeConfigSpecs, actualized for the current MasterConfiguration
// gets the KubeConfigSpecs, actualized for the current InitConfiguration
specs, err := getKubeConfigSpecs(cfg)
if err != nil {
return err
@@ -133,16 +147,16 @@ func createKubeConfigFiles(outDir string, cfg *kubeadmapi.MasterConfiguration, k
return nil
}
// getKubeConfigSpecs returns all KubeConfigSpecs actualized to the context of the current MasterConfiguration
// getKubeConfigSpecs returns all KubeConfigSpecs actualized to the context of the current InitConfiguration
// NB. this methods holds the information about how kubeadm creates kubeconfig files.
func getKubeConfigSpecs(cfg *kubeadmapi.MasterConfiguration) (map[string]*kubeConfigSpec, error) {
func getKubeConfigSpecs(cfg *kubeadmapi.InitConfiguration) (map[string]*kubeConfigSpec, error) {
caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
if err != nil {
return nil, fmt.Errorf("couldn't create a kubeconfig; the CA files couldn't be loaded: %v", err)
}
masterEndpoint, err := kubeadmutil.GetMasterEndpoint(&cfg.API)
masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg)
if err != nil {
return nil, err
}
@@ -160,7 +174,7 @@ func getKubeConfigSpecs(cfg *kubeadmapi.MasterConfiguration) (map[string]*kubeCo
kubeadmconstants.KubeletKubeConfigFileName: {
CACert: caCert,
APIServer: masterEndpoint,
ClientName: fmt.Sprintf("system:node:%s", cfg.NodeRegistration.Name),
ClientName: fmt.Sprintf("%s%s", kubeadmconstants.NodesUserPrefix, cfg.NodeRegistration.Name),
ClientCertAuth: &clientCertAuth{
CAKey: caKey,
Organizations: []string{kubeadmconstants.NodesGroup},
@@ -208,7 +222,7 @@ func buildKubeConfigFromSpec(spec *kubeConfigSpec, clustername string) (*clientc
Organization: spec.ClientCertAuth.Organizations,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}
clientCert, clientKey, err := pkiutil.NewCertAndKey(spec.CACert, spec.ClientCertAuth.CAKey, clientCertConfig)
clientCert, clientKey, err := pkiutil.NewCertAndKey(spec.CACert, spec.ClientCertAuth.CAKey, &clientCertConfig)
if err != nil {
return nil, fmt.Errorf("failure while creating %s client certificate: %v", spec.ClientName, err)
}
@@ -271,15 +285,15 @@ func createKubeConfigFileIfNotExists(outDir, filename string, config *clientcmda
}
// WriteKubeConfigWithClientCert writes a kubeconfig file - with a client certificate as authentication info - to the given writer.
func WriteKubeConfigWithClientCert(out io.Writer, cfg *kubeadmapi.MasterConfiguration, clientName string, organizations []string) error {
func WriteKubeConfigWithClientCert(out io.Writer, cfg *kubeadmapi.InitConfiguration, clientName string, organizations []string) error {
// creates the KubeConfigSpecs, actualized for the current MasterConfiguration
// creates the KubeConfigSpecs, actualized for the current InitConfiguration
caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
if err != nil {
return fmt.Errorf("couldn't create a kubeconfig; the CA files couldn't be loaded: %v", err)
}
masterEndpoint, err := kubeadmutil.GetMasterEndpoint(&cfg.API)
masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg)
if err != nil {
return err
}
@@ -298,15 +312,15 @@ func WriteKubeConfigWithClientCert(out io.Writer, cfg *kubeadmapi.MasterConfigur
}
// WriteKubeConfigWithToken writes a kubeconfig file - with a token as client authentication info - to the given writer.
func WriteKubeConfigWithToken(out io.Writer, cfg *kubeadmapi.MasterConfiguration, clientName, token string) error {
func WriteKubeConfigWithToken(out io.Writer, cfg *kubeadmapi.InitConfiguration, clientName, token string) error {
// creates the KubeConfigSpecs, actualized for the current MasterConfiguration
// creates the KubeConfigSpecs, actualized for the current InitConfiguration
caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
if err != nil {
return fmt.Errorf("couldn't create a kubeconfig; the CA files couldn't be loaded: %v", err)
}
masterEndpoint, err := kubeadmutil.GetMasterEndpoint(&cfg.API)
masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg)
if err != nil {
return err
}

View File

@@ -26,14 +26,11 @@ import (
"reflect"
"testing"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
pkiutil "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
certstestutil "k8s.io/kubernetes/cmd/kubeadm/test/certs"
@@ -46,8 +43,10 @@ func TestGetKubeConfigSpecsFailsIfCADoesntExists(t *testing.T) {
defer os.RemoveAll(tmpdir)
// Creates a Master Configuration pointing to the pkidir folder
cfg := &kubeadmapi.MasterConfiguration{
CertificatesDir: tmpdir,
cfg := &kubeadmapi.InitConfiguration{
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
CertificatesDir: tmpdir,
},
}
// Executes getKubeConfigSpecs
@@ -65,30 +64,44 @@ func TestGetKubeConfigSpecs(t *testing.T) {
pkidir := testutil.SetupPkiDirWithCertificateAuthorithy(t, tmpdir)
// Creates Master Configurations pointing to the pkidir folder
cfgs := []*kubeadmapi.MasterConfiguration{
cfgs := []*kubeadmapi.InitConfiguration{
{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", BindPort: 1234},
CertificatesDir: pkidir,
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
CertificatesDir: pkidir,
},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"},
},
{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", ControlPlaneEndpoint: "api.k8s.io", BindPort: 1234},
CertificatesDir: pkidir,
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
ControlPlaneEndpoint: "api.k8s.io",
CertificatesDir: pkidir,
},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"},
},
{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", ControlPlaneEndpoint: "api.k8s.io:4321", BindPort: 1234},
CertificatesDir: pkidir,
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
ControlPlaneEndpoint: "api.k8s.io:4321",
CertificatesDir: pkidir,
},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"},
},
{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", ControlPlaneEndpoint: "api.k8s.io", BindPort: 1234},
CertificatesDir: pkidir,
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
ControlPlaneEndpoint: "api.k8s.io",
CertificatesDir: pkidir,
},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"},
},
{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", ControlPlaneEndpoint: "api.k8s.io:4321", BindPort: 1234},
CertificatesDir: pkidir,
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
ControlPlaneEndpoint: "api.k8s.io:4321",
CertificatesDir: pkidir,
},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"},
},
}
@@ -106,7 +119,7 @@ func TestGetKubeConfigSpecs(t *testing.T) {
},
{
kubeConfigFile: kubeadmconstants.KubeletKubeConfigFileName,
clientName: fmt.Sprintf("system:node:%s", cfg.NodeRegistration.Name),
clientName: fmt.Sprintf("%s%s", kubeadmconstants.NodesUserPrefix, cfg.NodeRegistration.Name),
organizations: []string{kubeadmconstants.NodesGroup},
},
{
@@ -145,8 +158,8 @@ func TestGetKubeConfigSpecs(t *testing.T) {
t.Errorf("getKubeConfigSpecs for %s Organizations is %v, expected %v", assertion.kubeConfigFile, spec.ClientCertAuth.Organizations, assertion.organizations)
}
// Asserts MasterConfiguration values injected into spec
masterEndpoint, err := kubeadmutil.GetMasterEndpoint(&cfg.API)
// Asserts InitConfiguration values injected into spec
masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg)
if err != nil {
t.Error(err)
}
@@ -252,12 +265,12 @@ func TestCreateKubeConfigFileIfNotExists(t *testing.T) {
func TestCreateKubeconfigFilesAndWrappers(t *testing.T) {
var tests = []struct {
createKubeConfigFunction func(outDir string, cfg *kubeadmapi.MasterConfiguration) error
createKubeConfigFunction func(outDir string, cfg *kubeadmapi.InitConfiguration) error
expectedFiles []string
expectedError bool
}{
{ // Test createKubeConfigFiles fails for unknown kubeconfig is requested
createKubeConfigFunction: func(outDir string, cfg *kubeadmapi.MasterConfiguration) error {
createKubeConfigFunction: func(outDir string, cfg *kubeadmapi.InitConfiguration) error {
return createKubeConfigFiles(outDir, cfg, "unknown.conf")
},
expectedError: true,
@@ -271,6 +284,14 @@ func TestCreateKubeconfigFilesAndWrappers(t *testing.T) {
kubeadmconstants.SchedulerKubeConfigFileName,
},
},
{ // Test CreateJoinControlPlaneKubeConfigFiles (wrapper to createKubeConfigFile)
createKubeConfigFunction: CreateJoinControlPlaneKubeConfigFiles,
expectedFiles: []string{
kubeadmconstants.AdminKubeConfigFileName,
kubeadmconstants.ControllerManagerKubeConfigFileName,
kubeadmconstants.SchedulerKubeConfigFileName,
},
},
{ // Test CreateAdminKubeConfigFile (wrapper to createKubeConfigFile)
createKubeConfigFunction: CreateAdminKubeConfigFile,
expectedFiles: []string{kubeadmconstants.AdminKubeConfigFileName},
@@ -298,9 +319,11 @@ func TestCreateKubeconfigFilesAndWrappers(t *testing.T) {
pkidir := testutil.SetupPkiDirWithCertificateAuthorithy(t, tmpdir)
// Creates a Master Configuration pointing to the pkidir folder
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", BindPort: 1234},
CertificatesDir: pkidir,
cfg := &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
CertificatesDir: pkidir,
},
}
// Execs the createKubeConfigFunction
@@ -326,8 +349,10 @@ func TestWriteKubeConfigFailsIfCADoesntExists(t *testing.T) {
defer os.RemoveAll(tmpdir)
// Creates a Master Configuration pointing to the tmpdir folder
cfg := &kubeadmapi.MasterConfiguration{
CertificatesDir: tmpdir,
cfg := &kubeadmapi.InitConfiguration{
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
CertificatesDir: tmpdir,
},
}
var tests = []struct {
@@ -371,9 +396,11 @@ func TestWriteKubeConfig(t *testing.T) {
}
// Creates a Master Configuration pointing to the pkidir folder
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4", BindPort: 1234},
CertificatesDir: pkidir,
cfg := &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
CertificatesDir: pkidir,
},
}
var tests = []struct {

View File

@@ -6,29 +6,30 @@ go_library(
"config.go",
"dynamic.go",
"flags.go",
"kubelet.go",
],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet",
visibility = ["//visibility:public"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha3:go_default_library",
"//cmd/kubeadm/app/componentconfigs:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/features:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//pkg/apis/rbac/v1:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library",
"//pkg/kubelet/apis/config:go_default_library",
"//pkg/util/initsystem:go_default_library",
"//pkg/util/node:go_default_library",
"//pkg/util/procfs:go_default_library",
"//pkg/util/version:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/rbac/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/rbac/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
],
)
@@ -43,14 +44,15 @@ go_test(
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//pkg/kubelet/apis:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library",
"//pkg/kubelet/apis/config:go_default_library",
"//pkg/util/version:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/client-go/testing:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
"//staging/src/k8s.io/client-go/testing:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
],
)

View File

@@ -28,18 +28,17 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
"k8s.io/kubernetes/pkg/util/version"
)
// WriteConfigToDisk writes the kubelet config object down to a file
// Used at "kubeadm init" and "kubeadm upgrade" time
func WriteConfigToDisk(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration, kubeletDir string) error {
func WriteConfigToDisk(kubeletConfig *kubeletconfig.KubeletConfiguration, kubeletDir string) error {
kubeletBytes, err := getConfigBytes(kubeletConfig)
if err != nil {
@@ -50,17 +49,17 @@ func WriteConfigToDisk(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration,
// CreateConfigMap creates a ConfigMap with the generic kubelet configuration.
// Used at "kubeadm init" and "kubeadm upgrade" time
func CreateConfigMap(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
func CreateConfigMap(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error {
k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
if err != nil {
return err
}
configMapName := configMapName(k8sVersion)
configMapName := kubeadmconstants.GetKubeletConfigMapName(k8sVersion)
fmt.Printf("[kubelet] Creating a ConfigMap %q in namespace %s with the configuration for the kubelets in the cluster\n", configMapName, metav1.NamespaceSystem)
kubeletBytes, err := getConfigBytes(cfg.KubeletConfiguration.BaseConfig)
kubeletBytes, err := getConfigBytes(cfg.ComponentConfigs.Kubelet)
if err != nil {
return err
}
@@ -91,7 +90,7 @@ func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Ve
Namespace: metav1.NamespaceSystem,
},
Rules: []rbac.PolicyRule{
rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(configMapName(k8sVersion)).RuleOrDie(),
rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(kubeadmconstants.GetKubeletConfigMapName(k8sVersion)).RuleOrDie(),
},
}); err != nil {
return err
@@ -125,7 +124,7 @@ func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Ve
func DownloadConfig(client clientset.Interface, kubeletVersion *version.Version, kubeletDir string) error {
// Download the ConfigMap from the cluster based on what version the kubelet is
configMapName := configMapName(kubeletVersion)
configMapName := kubeadmconstants.GetKubeletConfigMapName(kubeletVersion)
fmt.Printf("[kubelet] Downloading configuration for the kubelet from the %q ConfigMap in the %s namespace\n",
configMapName, metav1.NamespaceSystem)
@@ -143,24 +142,14 @@ func DownloadConfig(client clientset.Interface, kubeletVersion *version.Version,
return writeConfigBytesToDisk([]byte(kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey]), kubeletDir)
}
// configMapName returns the right ConfigMap name for the right branch of k8s
func configMapName(k8sVersion *version.Version) string {
return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigurationConfigMapPrefix, k8sVersion.Major(), k8sVersion.Minor())
}
// configMapRBACName returns the name for the Role/RoleBinding for the kubelet config configmap for the right branch of k8s
func configMapRBACName(k8sVersion *version.Version) string {
return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigMapRolePrefix, k8sVersion.Major(), k8sVersion.Minor())
}
// getConfigBytes marshals a kubeletconfiguration object to bytes
func getConfigBytes(kubeletConfig *kubeletconfigv1beta1.KubeletConfiguration) ([]byte, error) {
_, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs()
if err != nil {
return []byte{}, err
}
return kubeadmutil.MarshalToYamlForCodecs(kubeletConfig, kubeletconfigv1beta1.SchemeGroupVersion, *kubeletCodecs)
// getConfigBytes marshals a KubeletConfiguration object to bytes
func getConfigBytes(kubeletConfig *kubeletconfig.KubeletConfiguration) ([]byte, error) {
return componentconfigs.Known[componentconfigs.KubeletConfigurationKind].Marshal(kubeletConfig)
}
// writeConfigBytesToDisk writes a byte slice down to disk at the specific location of the kubelet config file

View File

@@ -25,18 +25,20 @@ import (
"k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
"k8s.io/kubernetes/pkg/util/version"
)
func TestCreateConfigMap(t *testing.T) {
nodeName := "fake-node"
client := fake.NewSimpleClientset()
cfg := &kubeadmapi.MasterConfiguration{
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: nodeName},
KubernetesVersion: "v1.11.0",
KubeletConfiguration: kubeadmapi.KubeletConfiguration{
BaseConfig: &kubeletconfigv1beta1.KubeletConfiguration{},
cfg := &kubeadmapi.InitConfiguration{
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: nodeName},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
KubernetesVersion: "v1.12.0",
ComponentConfigs: kubeadmapi.ComponentConfigs{
Kubelet: &kubeletconfig.KubeletConfiguration{},
},
},
}

View File

@@ -21,7 +21,6 @@ import (
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
clientset "k8s.io/client-go/kubernetes"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
@@ -33,28 +32,27 @@ import (
// This func is ONLY run if the user enables the `DynamicKubeletConfig` feature gate, which is by default off
func EnableDynamicConfigForNode(client clientset.Interface, nodeName string, kubeletVersion *version.Version) error {
configMapName := configMapName(kubeletVersion)
configMapName := kubeadmconstants.GetKubeletConfigMapName(kubeletVersion)
fmt.Printf("[kubelet] Enabling Dynamic Kubelet Config for Node %q; config sourced from ConfigMap %q in namespace %s\n",
nodeName, configMapName, metav1.NamespaceSystem)
fmt.Println("[kubelet] WARNING: The Dynamic Kubelet Config feature is alpha and off by default. It hasn't been well-tested yet at this stage, use with caution.")
fmt.Println("[kubelet] WARNING: The Dynamic Kubelet Config feature is beta, but off by default. It hasn't been well-tested yet at this stage, use with caution.")
kubeletConfigMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{})
_, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("couldn't get the kubelet configuration ConfigMap: %v", err)
}
// Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned.
return apiclient.PatchNode(client, nodeName, func(n *v1.Node) {
patchNodeForDynamicConfig(n, configMapName, kubeletConfigMap.UID)
patchNodeForDynamicConfig(n, configMapName)
})
}
func patchNodeForDynamicConfig(n *v1.Node, configMapName string, configMapUID types.UID) {
func patchNodeForDynamicConfig(n *v1.Node, configMapName string) {
n.Spec.ConfigSource = &v1.NodeConfigSource{
ConfigMap: &v1.ConfigMapNodeConfigSource{
Name: configMapName,
Namespace: metav1.NamespaceSystem,
UID: configMapUID,
KubeletConfigKey: kubeadmconstants.KubeletBaseConfigurationConfigMapKey,
},
}

View File

@@ -25,7 +25,7 @@ import (
"github.com/golang/glog"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
@@ -46,6 +46,10 @@ type kubeletFlagsOpts struct {
// WriteKubeletDynamicEnvFile writes a environment file with dynamic flags to the kubelet.
// Used at "kubeadm init" and "kubeadm join" time.
func WriteKubeletDynamicEnvFile(nodeRegOpts *kubeadmapi.NodeRegistrationOptions, featureGates map[string]bool, registerTaintsUsingFlags bool, kubeletDir string) error {
hostName, err := nodeutil.GetHostname("")
if err != nil {
return err
}
flagOpts := kubeletFlagsOpts{
nodeRegOpts: nodeRegOpts,
@@ -53,7 +57,7 @@ func WriteKubeletDynamicEnvFile(nodeRegOpts *kubeadmapi.NodeRegistrationOptions,
registerTaintsUsingFlags: registerTaintsUsingFlags,
execer: utilsexec.New(),
pidOfFunc: procfs.PidOf,
defaultHostname: nodeutil.GetHostname(""),
defaultHostname: hostName,
}
stringMap := buildKubeletArgMap(flagOpts)
argList := kubeadmutil.BuildArgumentListFromMap(stringMap, nodeRegOpts.KubeletExtraArgs)
@@ -62,16 +66,14 @@ func WriteKubeletDynamicEnvFile(nodeRegOpts *kubeadmapi.NodeRegistrationOptions,
return writeKubeletFlagBytesToDisk([]byte(envFileContent), kubeletDir)
}
// buildKubeletArgMap takes a MasterConfiguration object and builds based on that a string-string map with flags
// buildKubeletArgMap takes a InitConfiguration object and builds based on that a string-string map with flags
// that should be given to the local kubelet daemon.
func buildKubeletArgMap(opts kubeletFlagsOpts) map[string]string {
kubeletFlags := map[string]string{}
if opts.nodeRegOpts.CRISocket == kubeadmapiv1alpha2.DefaultCRISocket {
if opts.nodeRegOpts.CRISocket == kubeadmapiv1alpha3.DefaultCRISocket {
// These flags should only be set when running docker
kubeletFlags["network-plugin"] = "cni"
kubeletFlags["cni-conf-dir"] = "/etc/cni/net.d"
kubeletFlags["cni-bin-dir"] = "/opt/cni/bin"
driver, err := kubeadmutil.GetCgroupDriverDocker(opts.execer)
if err != nil {
glog.Warningf("cannot automatically assign a '--cgroup-driver' value when starting the Kubelet: %v\n", err)
@@ -99,7 +101,7 @@ func buildKubeletArgMap(opts kubeletFlagsOpts) map[string]string {
// Make sure the node name we're passed will work with Kubelet
if opts.nodeRegOpts.Name != "" && opts.nodeRegOpts.Name != opts.defaultHostname {
glog.V(1).Info("setting kubelet hostname-override to %q", opts.nodeRegOpts.Name)
glog.V(1).Infof("setting kubelet hostname-override to %q", opts.nodeRegOpts.Name)
kubeletFlags["hostname-override"] = opts.nodeRegOpts.Name
}

View File

@@ -27,6 +27,7 @@ import (
"k8s.io/api/core/v1"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/utils/exec"
)
@@ -119,8 +120,6 @@ func TestBuildKubeletArgMap(t *testing.T) {
},
expected: map[string]string{
"network-plugin": "cni",
"cni-conf-dir": "/etc/cni/net.d",
"cni-bin-dir": "/opt/cni/bin",
},
},
{
@@ -136,8 +135,6 @@ func TestBuildKubeletArgMap(t *testing.T) {
},
expected: map[string]string{
"network-plugin": "cni",
"cni-conf-dir": "/etc/cni/net.d",
"cni-bin-dir": "/opt/cni/bin",
"hostname-override": "override-name",
},
},
@@ -154,8 +151,6 @@ func TestBuildKubeletArgMap(t *testing.T) {
},
expected: map[string]string{
"network-plugin": "cni",
"cni-conf-dir": "/etc/cni/net.d",
"cni-bin-dir": "/opt/cni/bin",
"cgroup-driver": "systemd",
},
},
@@ -172,8 +167,6 @@ func TestBuildKubeletArgMap(t *testing.T) {
},
expected: map[string]string{
"network-plugin": "cni",
"cni-conf-dir": "/etc/cni/net.d",
"cni-bin-dir": "/opt/cni/bin",
"cgroup-driver": "cgroupfs",
},
},
@@ -257,7 +250,7 @@ func TestBuildKubeletArgMap(t *testing.T) {
expected: map[string]string{
"container-runtime": "remote",
"container-runtime-endpoint": "/var/run/containerd.sock",
"dynamic-config-dir": "/var/lib/kubelet/dynamic-config",
"dynamic-config-dir": fmt.Sprintf("%s/dynamic-config", kubeadmconstants.KubeletRunDirectory),
},
},
}

View File

@@ -0,0 +1,63 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"fmt"
"k8s.io/kubernetes/pkg/util/initsystem"
)
// TryStartKubelet attempts to bring up kubelet service
func TryStartKubelet() {
// If we notice that the kubelet service is inactive, try to start it
initSystem, err := initsystem.GetInitSystem()
if err != nil {
fmt.Println("[preflight] no supported init system detected, won't make sure the kubelet is running properly.")
return
}
if !initSystem.ServiceExists("kubelet") {
fmt.Println("[preflight] couldn't detect a kubelet service, can't make sure the kubelet is running properly.")
}
fmt.Println("[preflight] Activating the kubelet service")
// This runs "systemctl daemon-reload && systemctl restart kubelet"
if err := initSystem.ServiceRestart("kubelet"); err != nil {
fmt.Printf("[preflight] WARNING: unable to start the kubelet service: [%v]\n", err)
fmt.Printf("[preflight] please ensure kubelet is reloaded and running manually.\n")
}
}
// TryStopKubelet attempts to bring down the kubelet service momentarily
func TryStopKubelet() {
// If we notice that the kubelet service is inactive, try to start it
initSystem, err := initsystem.GetInitSystem()
if err != nil {
fmt.Println("[preflight] no supported init system detected, won't make sure the kubelet not running for a short period of time while setting up configuration for it.")
return
}
if !initSystem.ServiceExists("kubelet") {
fmt.Println("[preflight] couldn't detect a kubelet service, can't make sure the kubelet not running for a short period of time while setting up configuration for it.")
}
// This runs "systemctl daemon-reload && systemctl stop kubelet"
if err := initSystem.ServiceStop("kubelet"); err != nil {
fmt.Printf("[preflight] WARNING: unable to stop the kubelet service momentarily: [%v]\n", err)
}
}

View File

@@ -14,10 +14,10 @@ go_test(
"//cmd/kubeadm/app/constants:go_default_library",
"//pkg/kubelet/apis:go_default_library",
"//pkg/util/node:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/rest:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/rest:go_default_library",
],
)
@@ -28,8 +28,8 @@ go_library(
deps = [
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
],
)

View File

@@ -108,7 +108,10 @@ func TestMarkMaster(t *testing.T) {
}
for _, tc := range tests {
hostname := node.GetHostname("")
hostname, err := node.GetHostname("")
if err != nil {
t.Fatalf("MarkMaster(%s): unexpected error: %v", tc.name, err)
}
masterNode := &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: hostname,

View File

@@ -8,8 +8,8 @@ go_library(
deps = [
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
],
)

View File

@@ -36,5 +36,8 @@ func AnnotateCRISocket(client clientset.Interface, nodeName string, criSocket st
}
func annotateNodeWithCRISocket(n *v1.Node, criSocket string) {
if n.ObjectMeta.Annotations == nil {
n.ObjectMeta.Annotations = make(map[string]string)
}
n.ObjectMeta.Annotations[constants.AnnotationKubeadmCRISocket] = criSocket
}

View File

@@ -17,8 +17,8 @@ go_test(
deps = [
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//vendor/k8s.io/api/apps/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
],
)
@@ -36,13 +36,13 @@ go_library(
"//cmd/kubeadm/app/features:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/apps/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
],
)

View File

@@ -56,7 +56,7 @@ const (
// 8. In order to avoid race conditions, we have to make sure that static pod is deleted correctly before we continue
// Otherwise, there is a race condition when we proceed without kubelet having restarted the API server correctly and the next .Create call flakes
// 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop
func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubeadmapi.MasterConfiguration, client clientset.Interface, waiter apiclient.Waiter, dryRun bool) error {
func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubeadmapi.InitConfiguration, client clientset.Interface, waiter apiclient.Waiter, dryRun bool) error {
glog.V(1).Infoln("creating self hosted control plane")
// Adjust the timeout slightly to something self-hosting specific
waiter.SetTimeout(selfHostingWaitTimeout)

View File

@@ -16,7 +16,7 @@ go_library(
visibility = ["//visibility:public"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha3:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/features:go_default_library",
"//cmd/kubeadm/app/images:go_default_library",
@@ -25,6 +25,7 @@ go_library(
"//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:go_default_library",
"//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library",
"//cmd/kubeadm/app/phases/certs:go_default_library",
"//cmd/kubeadm/app/phases/certs/renewal:go_default_library",
"//cmd/kubeadm/app/phases/controlplane:go_default_library",
"//cmd/kubeadm/app/phases/etcd:go_default_library",
"//cmd/kubeadm/app/phases/kubelet:go_default_library",
@@ -38,15 +39,15 @@ go_library(
"//cmd/kubeadm/app/util/etcd:go_default_library",
"//pkg/util/version:go_default_library",
"//pkg/version:go_default_library",
"//vendor/k8s.io/api/apps/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/util/cert:go_default_library",
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
],
)
@@ -76,23 +77,23 @@ go_test(
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/phases/certs:go_default_library",
"//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
"//cmd/kubeadm/app/phases/controlplane:go_default_library",
"//cmd/kubeadm/app/phases/etcd:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//cmd/kubeadm/app/util/config:go_default_library",
"//cmd/kubeadm/app/util/etcd:go_default_library",
"//cmd/kubeadm/test:go_default_library",
"//cmd/kubeadm/test/certs:go_default_library",
"//pkg/util/version:go_default_library",
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/github.com/coreos/etcd/clientv3:go_default_library",
"//vendor/github.com/coreos/etcd/pkg/transport:go_default_library",
"//vendor/k8s.io/api/apps/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
],
)

View File

@@ -168,7 +168,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.10.3",
KubeadmVersion: "v1.10.3",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.1.12",
},
},
@@ -207,7 +207,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.10.3",
KubeadmVersion: "v1.10.3",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.1.12",
},
},
@@ -246,7 +246,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.11.0",
KubeadmVersion: "v1.11.0",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.18",
},
},
@@ -285,7 +285,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.10.5",
KubeadmVersion: "v1.10.5", // Note: The kubeadm version mustn't be "downgraded" here
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.1.12",
},
},
@@ -305,7 +305,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.11.1",
KubeadmVersion: "v1.11.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.18",
},
},
@@ -364,7 +364,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.11.0-alpha.2",
KubeadmVersion: "v1.11.0-alpha.2",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.18",
},
},
@@ -404,7 +404,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.11.0-alpha.2",
KubeadmVersion: "v1.11.0-alpha.2",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.18",
},
},
@@ -445,7 +445,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.11.0-beta.1",
KubeadmVersion: "v1.11.0-beta.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.18",
},
},
@@ -486,7 +486,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.11.0-rc.1",
KubeadmVersion: "v1.11.0-rc.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.18",
},
},
@@ -527,7 +527,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.11.6-rc.1",
KubeadmVersion: "v1.11.6-rc.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.18",
},
},
@@ -568,7 +568,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.11.0-rc.1",
KubeadmVersion: "v1.11.0-rc.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.18",
},
},
@@ -588,8 +588,8 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.12.0-alpha.2",
KubeadmVersion: "v1.12.0-alpha.2",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.2.18",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.24",
},
},
},
@@ -641,8 +641,8 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.12.1",
KubeadmVersion: "v1.12.1",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.2.18",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.24",
},
},
},
@@ -678,8 +678,8 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.12.0",
KubeadmVersion: "v1.12.0",
DNSType: "coredns",
DNSVersion: "1.1.3",
EtcdVersion: "3.2.18",
DNSVersion: "1.2.2",
EtcdVersion: "3.2.24",
},
},
},
@@ -715,8 +715,8 @@ func TestGetAvailableUpgrades(t *testing.T) {
KubeVersion: "v1.12.0",
KubeadmVersion: "v1.12.0",
DNSType: "kube-dns",
DNSVersion: "1.14.10",
EtcdVersion: "3.2.18",
DNSVersion: "1.14.13",
EtcdVersion: "3.2.24",
},
},
},

View File

@@ -34,38 +34,38 @@ func TestEnforceVersionPolicies(t *testing.T) {
{
name: "minor upgrade",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.5",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.11.5",
},
newK8sVersion: "v1.10.5",
newK8sVersion: "v1.11.5",
},
{
name: "major upgrade",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.2",
kubeadmVersion: "v1.11.1",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.2",
kubeadmVersion: "v1.12.1",
},
newK8sVersion: "v1.11.0",
newK8sVersion: "v1.12.0",
},
{
name: "downgrade",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.3",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.11.3",
},
newK8sVersion: "v1.10.2",
newK8sVersion: "v1.11.2",
},
{
name: "same version upgrade",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.3",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.11.3",
},
newK8sVersion: "v1.10.3",
newK8sVersion: "v1.11.3",
},
{
name: "new version must be higher than v1.10.0",
@@ -92,103 +92,103 @@ func TestEnforceVersionPolicies(t *testing.T) {
{
name: "downgrading two minor versions in one go is not supported",
vg: &fakeVersionGetter{
clusterVersion: "v1.12.3",
kubeletVersion: "v1.12.3",
kubeadmVersion: "v1.12.0",
clusterVersion: "v1.13.3",
kubeletVersion: "v1.13.3",
kubeadmVersion: "v1.13.0",
},
newK8sVersion: "v1.10.3",
newK8sVersion: "v1.11.3",
expectedMandatoryErrs: 1, // can't downgrade two minor versions
expectedSkippableErrs: 1, // can't upgrade old k8s with newer kubeadm
},
{
name: "kubeadm version must be higher than the new kube version. However, patch version skews may be forced",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.3",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.11.3",
},
newK8sVersion: "v1.10.5",
newK8sVersion: "v1.11.5",
expectedSkippableErrs: 1,
},
{
name: "kubeadm version must be higher than the new kube version. Trying to upgrade k8s to a higher minor version than kubeadm itself should never be supported",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.10.3",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.11.3",
},
newK8sVersion: "v1.11.0",
newK8sVersion: "v1.12.0",
expectedMandatoryErrs: 1,
},
{
name: "the maximum skew between the cluster version and the kubelet versions should be one minor version. This may be forced through though.",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.9.8",
kubeadmVersion: "v1.11.0",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.10.8",
kubeadmVersion: "v1.12.0",
},
newK8sVersion: "v1.11.0",
newK8sVersion: "v1.12.0",
expectedSkippableErrs: 1,
},
{
name: "experimental upgrades supported if the flag is set",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0-beta.1",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.12.0-beta.1",
},
newK8sVersion: "v1.11.0-beta.1",
newK8sVersion: "v1.12.0-beta.1",
allowExperimental: true,
},
{
name: "release candidate upgrades supported if the flag is set",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0-rc.1",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.12.0-rc.1",
},
newK8sVersion: "v1.11.0-rc.1",
newK8sVersion: "v1.12.0-rc.1",
allowRCs: true,
},
{
name: "release candidate upgrades supported if the flag is set",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0-rc.1",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.12.0-rc.1",
},
newK8sVersion: "v1.11.0-rc.1",
newK8sVersion: "v1.12.0-rc.1",
allowExperimental: true,
},
{
name: "the user should not be able to upgrade to an experimental version if they haven't opted into that",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0-beta.1",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.12.0-beta.1",
},
newK8sVersion: "v1.11.0-beta.1",
newK8sVersion: "v1.12.0-beta.1",
allowRCs: true,
expectedSkippableErrs: 1,
},
{
name: "the user should not be able to upgrade to an release candidate version if they haven't opted into that",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0-rc.1",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.12.0-rc.1",
},
newK8sVersion: "v1.11.0-rc.1",
newK8sVersion: "v1.12.0-rc.1",
expectedSkippableErrs: 1,
},
{
name: "the user can't use a newer minor version of kubeadm to upgrade an older version of kubeadm",
vg: &fakeVersionGetter{
clusterVersion: "v1.10.3",
kubeletVersion: "v1.10.3",
kubeadmVersion: "v1.11.0",
clusterVersion: "v1.11.3",
kubeletVersion: "v1.11.3",
kubeadmVersion: "v1.12.0",
},
newK8sVersion: "v1.10.6",
newK8sVersion: "v1.11.6",
expectedSkippableErrs: 1, // can't upgrade old k8s with newer kubeadm
},
}

View File

@@ -29,7 +29,7 @@ import (
clientset "k8s.io/client-go/kubernetes"
certutil "k8s.io/client-go/util/cert"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
@@ -50,7 +50,7 @@ var expiry = 180 * 24 * time.Hour
// PerformPostUpgradeTasks runs nearly the same functions as 'kubeadm init' would do
// Note that the markmaster phase is left out, not needed, and no token is created as that doesn't belong to the upgrade
func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterConfiguration, newK8sVer *version.Version, dryRun bool) error {
func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, newK8sVer *version.Version, dryRun bool) error {
errs := []error{}
// Upload currently used configuration to the cluster
@@ -70,7 +70,7 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterC
errs = append(errs, err)
}
// Annotate the node with the crisocket information, sourced either from the MasterConfiguration struct or
// Annotate the node with the crisocket information, sourced either from the InitConfiguration struct or
// --cri-socket.
// TODO: In the future we want to use something more official like NodeStatus or similar for detecting this properly
if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil {
@@ -108,7 +108,7 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterC
}
// Rotate the kube-apiserver cert and key if needed
if err := backupAPIServerCertIfNeeded(cfg, dryRun); err != nil {
if err := BackupAPIServerCertIfNeeded(cfg, dryRun); err != nil {
errs = append(errs, err)
}
@@ -127,7 +127,7 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.MasterC
return errors.NewAggregate(errs)
}
func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface, dryRun bool) error {
func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.InitConfiguration, client clientset.Interface, dryRun bool) error {
return apiclient.TryRunCommand(func() error {
installedDeploymentName := kubeadmconstants.KubeDNS
deploymentToDelete := kubeadmconstants.CoreDNS
@@ -158,7 +158,7 @@ func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.MasterConfiguratio
}, 10)
}
func upgradeToSelfHosting(client clientset.Interface, cfg *kubeadmapi.MasterConfiguration, dryRun bool) error {
func upgradeToSelfHosting(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, dryRun bool) error {
if features.Enabled(cfg.FeatureGates, features.SelfHosting) && !IsControlPlaneSelfHosted(client) {
waiter := getWaiter(dryRun, client)
@@ -172,8 +172,9 @@ func upgradeToSelfHosting(client clientset.Interface, cfg *kubeadmapi.MasterConf
return nil
}
func backupAPIServerCertIfNeeded(cfg *kubeadmapi.MasterConfiguration, dryRun bool) error {
certAndKeyDir := kubeadmapiv1alpha2.DefaultCertificatesDir
// BackupAPIServerCertIfNeeded rotates the kube-apiserver certificate if older than 180 days
func BackupAPIServerCertIfNeeded(cfg *kubeadmapi.InitConfiguration, dryRun bool) error {
certAndKeyDir := kubeadmapiv1alpha3.DefaultCertificatesDir
shouldBackup, err := shouldBackupAPIServerCertAndKey(certAndKeyDir)
if err != nil {
// Don't fail the upgrade phase if failing to determine to backup kube-apiserver cert and key.
@@ -195,10 +196,14 @@ func backupAPIServerCertIfNeeded(cfg *kubeadmapi.MasterConfiguration, dryRun boo
if err := backupAPIServerCertAndKey(certAndKeyDir); err != nil {
fmt.Printf("[postupgrade] WARNING: failed to backup kube-apiserver cert and key: %v", err)
}
return certsphase.CreateAPIServerCertAndKeyFiles(cfg)
return certsphase.CreateCertAndKeyFilesWithCA(
&certsphase.KubeadmCertAPIServer,
&certsphase.KubeadmCertRootCA,
cfg,
)
}
func writeKubeletConfigFiles(client clientset.Interface, cfg *kubeadmapi.MasterConfiguration, newK8sVer *version.Version, dryRun bool) error {
func writeKubeletConfigFiles(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, newK8sVer *version.Version, dryRun bool) error {
kubeletDir, err := getKubeletDir(dryRun)
if err != nil {
// The error here should never occur in reality, would only be thrown if /tmp doesn't exist on the machine.

View File

@@ -27,7 +27,6 @@ import (
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
)
@@ -130,9 +129,11 @@ func TestRollbackFiles(t *testing.T) {
}
func TestShouldBackupAPIServerCertAndKey(t *testing.T) {
cfg := &kubeadmapi.MasterConfiguration{
API: kubeadmapi.API{AdvertiseAddress: "1.2.3.4"},
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
cfg := &kubeadmapi.InitConfiguration{
APIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"},
ClusterConfiguration: kubeadmapi.ClusterConfiguration{
Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"},
},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "test-node"},
}
@@ -148,23 +149,21 @@ func TestShouldBackupAPIServerCertAndKey(t *testing.T) {
expected: true,
},
} {
caCert, caKey, err := certsphase.NewCACertAndKey()
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
cfg.CertificatesDir = tmpdir
caCert, caKey, err := certsphase.KubeadmCertRootCA.CreateAsCA(cfg)
if err != nil {
t.Fatalf("failed creation of ca cert and key: %v", err)
}
caCert.NotBefore = caCert.NotBefore.Add(-test.adjustedExpiry).UTC()
apiCert, apiKey, err := certsphase.NewAPIServerCertAndKey(cfg, caCert, caKey)
err = certsphase.KubeadmCertAPIServer.CreateFromCA(cfg, caCert, caKey)
if err != nil {
t.Fatalf("Test %s: failed creation of cert and key: %v", desc, err)
}
tmpdir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpdir)
if err := pkiutil.WriteCertAndKey(tmpdir, constants.APIServerCertAndKeyBaseName, apiCert, apiKey); err != nil {
t.Fatalf("Test %s: failure while saving %s certificate and key: %v", desc, constants.APIServerCertAndKeyBaseName, err)
}
certAndKey := []string{filepath.Join(tmpdir, constants.APIServerCertName), filepath.Join(tmpdir, constants.APIServerKeyName)}
for _, path := range certAndKey {
if _, err := os.Stat(path); os.IsNotExist(err) {

View File

@@ -44,12 +44,12 @@ type Prepuller interface {
// DaemonSetPrepuller makes sure the control plane images are available on all masters
type DaemonSetPrepuller struct {
client clientset.Interface
cfg *kubeadmapi.MasterConfiguration
cfg *kubeadmapi.ClusterConfiguration
waiter apiclient.Waiter
}
// NewDaemonSetPrepuller creates a new instance of the DaemonSetPrepuller struct
func NewDaemonSetPrepuller(client clientset.Interface, waiter apiclient.Waiter, cfg *kubeadmapi.MasterConfiguration) *DaemonSetPrepuller {
func NewDaemonSetPrepuller(client clientset.Interface, waiter apiclient.Waiter, cfg *kubeadmapi.ClusterConfiguration) *DaemonSetPrepuller {
return &DaemonSetPrepuller{
client: client,
cfg: cfg,
@@ -59,7 +59,12 @@ func NewDaemonSetPrepuller(client clientset.Interface, waiter apiclient.Waiter,
// CreateFunc creates a DaemonSet for making the image available on every relevant node
func (d *DaemonSetPrepuller) CreateFunc(component string) error {
image := images.GetCoreImage(component, d.cfg.GetControlPlaneImageRepository(), d.cfg.KubernetesVersion, d.cfg.UnifiedControlPlaneImage)
var image string
if component == constants.Etcd {
image = images.GetEtcdImage(d.cfg)
} else {
image = images.GetKubeControlPlaneImage(component, d.cfg)
}
ds := buildPrePullDaemonSet(component, image)
// Create the DaemonSet in the API Server
@@ -154,6 +159,11 @@ func buildPrePullDaemonSet(component, image string) *apps.DaemonSet {
Namespace: metav1.NamespaceSystem,
},
Spec: apps.DaemonSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"k8s-app": addPrepullPrefix(component),
},
},
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{

View File

@@ -81,7 +81,7 @@ type controlPlaneComponentResources struct {
// -> Backup component v1 is Deleted
// 5. Wait for Self-Hosted component v2 Running to become active
// 6. Repeat for all control plane components
func SelfHostedControlPlane(client clientset.Interface, waiter apiclient.Waiter, cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) error {
func SelfHostedControlPlane(client clientset.Interface, waiter apiclient.Waiter, cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Version) error {
// Adjust the timeout slightly to something self-hosting specific
waiter.SetTimeout(selfHostingWaitTimeout)
@@ -158,7 +158,7 @@ func SelfHostedControlPlane(client clientset.Interface, waiter apiclient.Waiter,
}
// BuildUpgradedDaemonSetsFromConfig takes a config object and the current version and returns the DaemonSet objects to post to the master
func BuildUpgradedDaemonSetsFromConfig(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) map[string]*apps.DaemonSet {
func BuildUpgradedDaemonSetsFromConfig(cfg *kubeadmapi.InitConfiguration, k8sVersion *version.Version) map[string]*apps.DaemonSet {
// Here the map of different mutators to use for the control plane's podspec is stored
mutators := selfhosting.GetMutatorsFromFeatureGates(cfg.FeatureGates)
// Get the new PodSpecs to use

View File

@@ -25,6 +25,7 @@ import (
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/renewal"
controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
"k8s.io/kubernetes/cmd/kubeadm/app/util"
@@ -33,6 +34,11 @@ import (
"k8s.io/kubernetes/pkg/util/version"
)
const (
// UpgradeManifestTimeout is timeout of upgrading the static pod manifest
UpgradeManifestTimeout = 5 * time.Minute
)
// StaticPodPathManager is responsible for tracking the directories used in the static pod upgrade transition
type StaticPodPathManager interface {
// MoveFile should move a file from oldPath to newPath
@@ -156,7 +162,7 @@ func (spm *KubeStaticPodPathManager) CleanupDirs() error {
return nil
}
func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.MasterConfiguration, beforePodHash string, recoverManifests map[string]string, isTLSUpgrade bool) error {
func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, beforePodHash string, recoverManifests map[string]string, isTLSUpgrade bool) error {
// Special treatment is required for etcd case, when rollbackOldManifests should roll back etcd
// manifests only for the case when component is Etcd
recoverEtcd := false
@@ -180,27 +186,8 @@ func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticP
}
}
// ensure etcd certs are generated for etcd and kube-apiserver
if component == constants.Etcd || component == constants.KubeAPIServer {
if err := certsphase.CreateEtcdCACertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s CA certificate and key: %v", constants.Etcd, err)
}
}
if component == constants.Etcd {
if err := certsphase.CreateEtcdServerCertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s certificate and key: %v", constants.Etcd, err)
}
if err := certsphase.CreateEtcdPeerCertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s peer certificate and key: %v", constants.Etcd, err)
}
if err := certsphase.CreateEtcdHealthcheckClientCertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s healthcheck certificate and key: %v", constants.Etcd, err)
}
}
if component == constants.KubeAPIServer {
if err := certsphase.CreateAPIServerEtcdClientCertAndKeyFiles(cfg); err != nil {
return fmt.Errorf("failed to upgrade the %s %s-client certificate and key: %v", constants.KubeAPIServer, constants.Etcd, err)
}
if err := renewCerts(cfg, component); err != nil {
return fmt.Errorf("failed to renew certificates for component %q: %v", component, err)
}
// The old manifest is here; in the /etc/kubernetes/manifests/
@@ -228,6 +215,7 @@ func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticP
if waitForComponentRestart {
fmt.Println("[upgrade/staticpods] Waiting for the kubelet to restart the component")
fmt.Printf("[upgrade/staticpods] This might take a minute or longer depending on the component/version gap (timeout %v\n", UpgradeManifestTimeout)
// Wait for the mirror Pod hash to change; otherwise we'll run into race conditions here when the kubelet hasn't had time to
// notice the removal of the Static Pod, leading to a false positive below where we check that the API endpoint is healthy
@@ -251,7 +239,7 @@ func upgradeComponent(component string, waiter apiclient.Waiter, pathMgr StaticP
}
// performEtcdStaticPodUpgrade performs upgrade of etcd, it returns bool which indicates fatal error or not and the actual error.
func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.MasterConfiguration, recoverManifests map[string]string, isTLSUpgrade bool, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) (bool, error) {
func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, recoverManifests map[string]string, isTLSUpgrade bool, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) (bool, error) {
// Add etcd static pod spec only if external etcd is not configured
if cfg.Etcd.External != nil {
return false, fmt.Errorf("external etcd detected, won't try to change any etcd state")
@@ -398,7 +386,7 @@ func performEtcdStaticPodUpgrade(waiter apiclient.Waiter, pathMgr StaticPodPathM
}
// StaticPodControlPlane upgrades a static pod-hosted control plane
func StaticPodControlPlane(waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.MasterConfiguration, etcdUpgrade bool, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) error {
func StaticPodControlPlane(waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, etcdUpgrade bool, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) error {
recoverManifests := map[string]string{}
var isTLSUpgrade bool
var isExternalEtcd bool
@@ -503,7 +491,7 @@ func rollbackOldManifests(oldManifests map[string]string, origErr error, pathMgr
// rollbackEtcdData rolls back the the content of etcd folder if something went wrong.
// When the folder contents are successfully rolled back, nil is returned, otherwise an error is returned.
func rollbackEtcdData(cfg *kubeadmapi.MasterConfiguration, pathMgr StaticPodPathManager) error {
func rollbackEtcdData(cfg *kubeadmapi.InitConfiguration, pathMgr StaticPodPathManager) error {
backupEtcdDir := pathMgr.BackupEtcdDir()
runningEtcdDir := cfg.Etcd.Local.DataDir
@@ -514,3 +502,35 @@ func rollbackEtcdData(cfg *kubeadmapi.MasterConfiguration, pathMgr StaticPodPath
return nil
}
func renewCerts(cfg *kubeadmapi.InitConfiguration, component string) error {
if cfg.Etcd.Local != nil {
// ensure etcd certs are loaded for etcd and kube-apiserver
if component == constants.Etcd || component == constants.KubeAPIServer {
caCert, caKey, err := certsphase.LoadCertificateAuthority(cfg.CertificatesDir, certsphase.KubeadmCertEtcdCA.BaseName)
if err != nil {
return fmt.Errorf("failed to upgrade the %s CA certificate and key: %v", constants.Etcd, err)
}
renewer := renewal.NewFileRenewal(caCert, caKey)
if component == constants.Etcd {
for _, cert := range []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdServer,
&certsphase.KubeadmCertEtcdPeer,
&certsphase.KubeadmCertEtcdHealthcheck,
} {
if err := renewal.RenewExistingCert(cfg.CertificatesDir, cert.BaseName, renewer); err != nil {
return fmt.Errorf("failed to renew %s certificate and key: %v", cert.Name, err)
}
}
}
if component == constants.KubeAPIServer {
cert := certsphase.KubeadmCertEtcdAPIClient
if err := renewal.RenewExistingCert(cfg.CertificatesDir, cert.BaseName, renewer); err != nil {
return fmt.Errorf("failed to renew %s certificate and key: %v", cert.Name, err)
}
}
}
}
return nil
}

View File

@@ -20,6 +20,7 @@ import (
"crypto/sha256"
"fmt"
"io/ioutil"
"math/big"
"os"
"path/filepath"
"strings"
@@ -28,16 +29,18 @@ import (
"github.com/coreos/etcd/clientv3"
"github.com/coreos/etcd/pkg/transport"
"k8s.io/apimachinery/pkg/runtime"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
testutil "k8s.io/kubernetes/cmd/kubeadm/test"
certstestutil "k8s.io/kubernetes/cmd/kubeadm/test/certs"
)
const (
@@ -46,8 +49,14 @@ const (
waitForPodsWithLabel = "wait-for-pods-with-label"
testConfiguration = `
apiVersion: kubeadm.k8s.io/v1alpha2
kind: MasterConfiguration
apiVersion: kubeadm.k8s.io/v1alpha3
kind: InitConfiguration
nodeRegistration:
name: foo
criSocket: ""
---
apiVersion: kubeadm.k8s.io/v1alpha3
kind: ClusterConfiguration
api:
advertiseAddress: 1.2.3.4
bindPort: 6443
@@ -415,32 +424,21 @@ func TestStaticPodControlPlane(t *testing.T) {
}
defer os.RemoveAll(tmpEtcdDataDir)
oldcfg, err := getConfig("v1.9.0", tempCertsDir, tmpEtcdDataDir)
oldcfg, err := getConfig("v1.12.0", tempCertsDir, tmpEtcdDataDir)
if err != nil {
t.Fatalf("couldn't create config: %v", err)
}
// Initialize PKI minus any etcd certificates to simulate etcd PKI upgrade
certActions := []func(cfg *kubeadmapi.MasterConfiguration) error{
certsphase.CreateCACertAndKeyFiles,
certsphase.CreateAPIServerCertAndKeyFiles,
certsphase.CreateAPIServerKubeletClientCertAndKeyFiles,
// certsphase.CreateEtcdCACertAndKeyFiles,
// certsphase.CreateEtcdServerCertAndKeyFiles,
// certsphase.CreateEtcdPeerCertAndKeyFiles,
// certsphase.CreateEtcdHealthcheckClientCertAndKeyFiles,
// certsphase.CreateAPIServerEtcdClientCertAndKeyFiles,
certsphase.CreateServiceAccountKeyAndPublicKeyFiles,
certsphase.CreateFrontProxyCACertAndKeyFiles,
certsphase.CreateFrontProxyClientCertAndKeyFiles,
tree, err := certsphase.GetCertsWithoutEtcd().AsMap().CertTree()
if err != nil {
t.Fatalf("couldn't get cert tree: %v", err)
}
for _, action := range certActions {
err := action(oldcfg)
if err != nil {
t.Fatalf("couldn't initialize pre-upgrade certificate: %v", err)
}
if err := tree.CreateTree(oldcfg); err != nil {
t.Fatalf("couldn't get create cert tree: %v", err)
}
fmt.Printf("Wrote certs to %s\n", oldcfg.CertificatesDir)
t.Logf("Wrote certs to %s\n", oldcfg.CertificatesDir)
// Initialize the directory with v1.7 manifests; should then be upgraded to v1.8 using the method
err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), oldcfg)
@@ -457,11 +455,27 @@ func TestStaticPodControlPlane(t *testing.T) {
t.Fatalf("couldn't read temp file: %v", err)
}
newcfg, err := getConfig("v1.10.0", tempCertsDir, tmpEtcdDataDir)
newcfg, err := getConfig("v1.11.0", tempCertsDir, tmpEtcdDataDir)
if err != nil {
t.Fatalf("couldn't create config: %v", err)
}
// create the kubeadm etcd certs
caCert, caKey, err := certsphase.KubeadmCertEtcdCA.CreateAsCA(newcfg)
if err != nil {
t.Fatalf("couldn't create new CA certificate: %v", err)
}
for _, cert := range []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdServer,
&certsphase.KubeadmCertEtcdPeer,
&certsphase.KubeadmCertEtcdHealthcheck,
&certsphase.KubeadmCertEtcdAPIClient,
} {
if err := cert.CreateFromCA(newcfg, caCert, caKey); err != nil {
t.Fatalf("couldn't create certificate %s: %v", cert.Name, err)
}
}
actualErr := StaticPodControlPlane(
waiter,
pathMgr,
@@ -513,15 +527,26 @@ func getAPIServerHash(dir string) (string, error) {
return fmt.Sprintf("%x", sha256.Sum256(fileBytes)), nil
}
// TODO: Make this test function use the rest of the "official" API machinery helper funcs we have inside of kubeadm
func getConfig(version, certsDir, etcdDataDir string) (*kubeadmapi.MasterConfiguration, error) {
externalcfg := &kubeadmapiv1alpha2.MasterConfiguration{}
internalcfg := &kubeadmapi.MasterConfiguration{}
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(fmt.Sprintf(testConfiguration, certsDir, etcdDataDir, version)), externalcfg); err != nil {
return nil, fmt.Errorf("unable to decode config: %v", err)
func getConfig(version, certsDir, etcdDataDir string) (*kubeadmapi.InitConfiguration, error) {
configBytes := []byte(fmt.Sprintf(testConfiguration, certsDir, etcdDataDir, version))
// Unmarshal the config
cfg, err := configutil.BytesToInternalConfig(configBytes)
if err != nil {
return nil, err
}
kubeadmscheme.Scheme.Convert(externalcfg, internalcfg, nil)
return internalcfg, nil
// Applies dynamic defaults to settings not provided with flags
if err = configutil.SetInitDynamicDefaults(cfg); err != nil {
return nil, err
}
// Validates cfg (flags/configs + defaults + dynamic defaults)
if err = validation.ValidateInitConfiguration(cfg).ToAggregate(); err != nil {
return nil, err
}
return cfg, nil
}
func getTempDir(t *testing.T, name string) (string, func()) {
@@ -601,3 +626,113 @@ func TestCleanupDirs(t *testing.T) {
})
}
}
func TestRenewCerts(t *testing.T) {
caCert, caKey := certstestutil.SetupCertificateAuthorithy(t)
t.Run("all certs exist, should be rotated", func(t *testing.T) {
})
tests := []struct {
name string
component string
skipCreateCA bool
shouldErrorOnRenew bool
certsShouldExist []*certsphase.KubeadmCert
}{
{
name: "all certs exist, should be rotated",
component: constants.Etcd,
certsShouldExist: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdServer,
&certsphase.KubeadmCertEtcdPeer,
&certsphase.KubeadmCertEtcdHealthcheck,
},
},
{
name: "just renew API cert",
component: constants.KubeAPIServer,
certsShouldExist: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdAPIClient,
},
},
{
name: "ignores other compnonents",
skipCreateCA: true,
component: constants.KubeScheduler,
},
{
name: "missing a cert to renew",
component: constants.Etcd,
shouldErrorOnRenew: true,
certsShouldExist: []*certsphase.KubeadmCert{
&certsphase.KubeadmCertEtcdServer,
&certsphase.KubeadmCertEtcdPeer,
},
},
{
name: "no CA, cannot continue",
component: constants.Etcd,
skipCreateCA: true,
shouldErrorOnRenew: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
// Setup up basic requities
tmpDir := testutil.SetupTempDir(t)
defer os.RemoveAll(tmpDir)
cfg := testutil.GetDefaultInternalConfig(t)
cfg.CertificatesDir = tmpDir
if !test.skipCreateCA {
if err := pkiutil.WriteCertAndKey(tmpDir, constants.EtcdCACertAndKeyBaseName, caCert, caKey); err != nil {
t.Fatalf("couldn't write out CA: %v", err)
}
}
// Create expected certs
for _, kubeCert := range test.certsShouldExist {
if err := kubeCert.CreateFromCA(cfg, caCert, caKey); err != nil {
t.Fatalf("couldn't renew certificate %q: %v", kubeCert.Name, err)
}
}
// Load expected certs to check if serial numbers changes
certMaps := make(map[*certsphase.KubeadmCert]big.Int)
for _, kubeCert := range test.certsShouldExist {
cert, err := pkiutil.TryLoadCertFromDisk(tmpDir, kubeCert.BaseName)
if err != nil {
t.Fatalf("couldn't load certificate %q: %v", kubeCert.Name, err)
}
certMaps[kubeCert] = *cert.SerialNumber
}
// Renew everything
err := renewCerts(cfg, test.component)
if test.shouldErrorOnRenew {
if err == nil {
t.Fatal("expected renewal error, got nothing")
}
// expected error, got error
return
}
if err != nil {
t.Fatalf("couldn't renew certificates: %v", err)
}
// See if the certificate serial numbers change
for kubeCert, cert := range certMaps {
newCert, err := pkiutil.TryLoadCertFromDisk(tmpDir, kubeCert.BaseName)
if err != nil {
t.Errorf("couldn't load new certificate %q: %v", kubeCert.Name, err)
continue
}
if cert.Cmp(newCert.SerialNumber) == 0 {
t.Errorf("certifitate %v was not reissued", kubeCert.Name)
}
}
})
}
}

View File

@@ -13,13 +13,16 @@ go_library(
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//cmd/kubeadm/app/util/config:go_default_library",
"//pkg/apis/rbac/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/rbac/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
],
)
@@ -43,12 +46,16 @@ go_test(
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha2:go_default_library",
"//cmd/kubeadm/app/apis/kubeadm/v1alpha3:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/client-go/testing:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//cmd/kubeadm/app/util/config:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
"//staging/src/k8s.io/client-go/testing:go_default_library",
],
)

View File

@@ -20,43 +20,131 @@ import (
"fmt"
"k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
)
// UploadConfiguration saves the MasterConfiguration used for later reference (when upgrading for instance)
func UploadConfiguration(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
const (
// NodesKubeadmConfigClusterRoleName sets the name for the ClusterRole that allows
// the bootstrap tokens to access the kubeadm-config ConfigMap during the node bootstrap/discovery
// or during upgrade nodes
NodesKubeadmConfigClusterRoleName = "kubeadm:nodes-kubeadm-config"
)
fmt.Printf("[uploadconfig] storing the configuration used in ConfigMap %q in the %q Namespace\n", kubeadmconstants.MasterConfigurationConfigMap, metav1.NamespaceSystem)
// UploadConfiguration saves the InitConfiguration used for later reference (when upgrading for instance)
func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error {
fmt.Printf("[uploadconfig] storing the configuration used in ConfigMap %q in the %q Namespace\n", kubeadmconstants.InitConfigurationConfigMap, metav1.NamespaceSystem)
// Convert cfg to the external version as that's the only version of the API that can be deserialized later
externalcfg := &kubeadmapiv1alpha2.MasterConfiguration{}
kubeadmscheme.Scheme.Convert(cfg, externalcfg, nil)
// Prepare the ClusterConfiguration for upload
// The components store their config in their own ConfigMaps, then reset the .ComponentConfig struct;
// We don't want to mutate the cfg itself, so create a copy of it using .DeepCopy of it first
clusterConfigurationToUpload := cfg.ClusterConfiguration.DeepCopy()
clusterConfigurationToUpload.ComponentConfigs = kubeadmapi.ComponentConfigs{}
// Removes sensitive info from the data that will be stored in the config map
externalcfg.BootstrapTokens = nil
// Clear the NodeRegistration object.
externalcfg.NodeRegistration = kubeadmapiv1alpha2.NodeRegistrationOptions{}
cfgYaml, err := util.MarshalToYamlForCodecs(externalcfg, kubeadmapiv1alpha2.SchemeGroupVersion, scheme.Codecs)
// Marshal the ClusterConfiguration into YAML
clusterConfigurationYaml, err := configutil.MarshalKubeadmConfigObject(clusterConfigurationToUpload)
if err != nil {
return err
}
return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
// Prepare the ClusterStatus for upload
// Gets the current cluster status
// TODO: use configmap locks on this object on the get before the update.
clusterStatus, err := getClusterStatus(client)
if err != nil {
return err
}
// Updates the ClusterStatus with the current control plane instance
if clusterStatus.APIEndpoints == nil {
clusterStatus.APIEndpoints = map[string]kubeadmapi.APIEndpoint{}
}
clusterStatus.APIEndpoints[cfg.NodeRegistration.Name] = cfg.APIEndpoint
// Marshal the ClusterStatus back into into YAML
clusterStatusYaml, err := configutil.MarshalKubeadmConfigObject(clusterStatus)
if err != nil {
return err
}
err = apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: kubeadmconstants.MasterConfigurationConfigMap,
Name: kubeadmconstants.InitConfigurationConfigMap,
Namespace: metav1.NamespaceSystem,
},
Data: map[string]string{
kubeadmconstants.MasterConfigurationConfigMapKey: string(cfgYaml),
kubeadmconstants.ClusterConfigurationConfigMapKey: string(clusterConfigurationYaml),
kubeadmconstants.ClusterStatusConfigMapKey: string(clusterStatusYaml),
},
})
if err != nil {
return err
}
// Ensure that the NodesKubeadmConfigClusterRoleName exists
err = apiclient.CreateOrUpdateRole(client, &rbac.Role{
ObjectMeta: metav1.ObjectMeta{
Name: NodesKubeadmConfigClusterRoleName,
Namespace: metav1.NamespaceSystem,
},
Rules: []rbac.PolicyRule{
rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(kubeadmconstants.InitConfigurationConfigMap).RuleOrDie(),
},
})
if err != nil {
return err
}
// Binds the NodesKubeadmConfigClusterRoleName to all the bootstrap tokens
// that are members of the system:bootstrappers:kubeadm:default-node-token group
// and to all nodes
return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: NodesKubeadmConfigClusterRoleName,
Namespace: metav1.NamespaceSystem,
},
RoleRef: rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "Role",
Name: NodesKubeadmConfigClusterRoleName,
},
Subjects: []rbac.Subject{
{
Kind: rbac.GroupKind,
Name: kubeadmconstants.NodeBootstrapTokenAuthGroup,
},
{
Kind: rbac.GroupKind,
Name: kubeadmconstants.NodesGroup,
},
},
})
}
func getClusterStatus(client clientset.Interface) (*kubeadmapi.ClusterStatus, error) {
obj := &kubeadmapi.ClusterStatus{}
// Read the ConfigMap from the cluster
configMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.InitConfigurationConfigMap, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
return obj, nil
}
if err != nil {
return nil, err
}
// Decode the file content using the componentconfig Codecs that knows about all APIs
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(configMap.Data[kubeadmconstants.ClusterStatusConfigMapKey]), obj); err != nil {
return nil, err
}
return obj, nil
}

View File

@@ -17,17 +17,22 @@ limitations under the License.
package uploadconfig
import (
"reflect"
"testing"
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
clientsetfake "k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha2"
kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
)
func TestUploadConfiguration(t *testing.T) {
@@ -61,22 +66,38 @@ func TestUploadConfiguration(t *testing.T) {
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := &kubeadmapi.MasterConfiguration{
KubernetesVersion: "v1.10.3",
BootstrapTokens: []kubeadmapi.BootstrapToken{
t.Run(tt.name, func(t2 *testing.T) {
initialcfg := &kubeadmapiv1alpha3.InitConfiguration{
APIEndpoint: kubeadmapiv1alpha3.APIEndpoint{
AdvertiseAddress: "1.2.3.4",
},
ClusterConfiguration: kubeadmapiv1alpha3.ClusterConfiguration{
KubernetesVersion: "v1.11.10",
},
BootstrapTokens: []kubeadmapiv1alpha3.BootstrapToken{
{
Token: &kubeadmapi.BootstrapTokenString{
Token: &kubeadmapiv1alpha3.BootstrapTokenString{
ID: "abcdef",
Secret: "abcdef0123456789",
},
},
},
NodeRegistration: kubeadmapi.NodeRegistrationOptions{
NodeRegistration: kubeadmapiv1alpha3.NodeRegistrationOptions{
Name: "node-foo",
CRISocket: "/var/run/custom-cri.sock",
},
}
cfg, err := configutil.ConfigFileAndDefaultsToInternalConfig("", initialcfg)
if err != nil {
t2.Fatalf("UploadConfiguration() error = %v", err)
}
status := &kubeadmapi.ClusterStatus{
APIEndpoints: map[string]kubeadmapi.APIEndpoint{
"node-foo": cfg.APIEndpoint,
},
}
client := clientsetfake.NewSimpleClientset()
if tt.errOnCreate != nil {
client.PrependReactor("create", "configmaps", func(action core.Action) (bool, runtime.Object, error) {
@@ -85,7 +106,7 @@ func TestUploadConfiguration(t *testing.T) {
}
// For idempotent test, we check the result of the second call.
if err := UploadConfiguration(cfg, client); !tt.updateExisting && (err != nil) != tt.errExpected {
t.Errorf("UploadConfiguration() error = %v, wantErr %v", err, tt.errExpected)
t2.Fatalf("UploadConfiguration() error = %v, wantErr %v", err, tt.errExpected)
}
if tt.updateExisting {
if tt.errOnUpdate != nil {
@@ -94,51 +115,101 @@ func TestUploadConfiguration(t *testing.T) {
})
}
if err := UploadConfiguration(cfg, client); (err != nil) != tt.errExpected {
t.Errorf("UploadConfiguration() error = %v", err)
t2.Fatalf("UploadConfiguration() error = %v", err)
}
}
if tt.verifyResult {
masterCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.MasterConfigurationConfigMap, metav1.GetOptions{})
masterCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.InitConfigurationConfigMap, metav1.GetOptions{})
if err != nil {
t.Errorf("Fail to query ConfigMap error = %v", err)
t2.Fatalf("Fail to query ConfigMap error = %v", err)
}
configData := masterCfg.Data[kubeadmconstants.MasterConfigurationConfigMapKey]
configData := masterCfg.Data[kubeadmconstants.ClusterConfigurationConfigMapKey]
if configData == "" {
t.Errorf("Fail to find ConfigMap key")
t2.Fatal("Fail to find ClusterConfigurationConfigMapKey key")
}
decodedExtCfg := &kubeadmapiv1alpha2.MasterConfiguration{}
decodedCfg := &kubeadmapi.MasterConfiguration{}
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(configData), decodedExtCfg); err != nil {
t.Errorf("unable to decode config from bytes: %v", err)
}
// Default and convert to the internal version
kubeadmscheme.Scheme.Default(decodedExtCfg)
kubeadmscheme.Scheme.Convert(decodedExtCfg, decodedCfg, nil)
if decodedCfg.KubernetesVersion != cfg.KubernetesVersion {
t.Errorf("Decoded value doesn't match, decoded = %#v, expected = %#v", decodedCfg.KubernetesVersion, cfg.KubernetesVersion)
decodedCfg := &kubeadmapi.ClusterConfiguration{}
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(configData), decodedCfg); err != nil {
t2.Fatalf("unable to decode config from bytes: %v", err)
}
// If the decoded cfg has a BootstrapTokens array, verify the sensitive information we had isn't still there.
if len(decodedCfg.BootstrapTokens) > 0 && decodedCfg.BootstrapTokens[0].Token != nil && decodedCfg.BootstrapTokens[0].Token.String() == cfg.BootstrapTokens[0].Token.String() {
t.Errorf("Decoded value contains .BootstrapTokens (sensitive info), decoded = %#v, expected = empty", decodedCfg.BootstrapTokens)
if !reflect.DeepEqual(decodedCfg, &cfg.ClusterConfiguration) {
t2.Errorf("the initial and decoded ClusterConfiguration didn't match")
}
// Make sure no information from NodeRegistrationOptions was uploaded.
if decodedCfg.NodeRegistration.Name == cfg.NodeRegistration.Name || decodedCfg.NodeRegistration.CRISocket != kubeadmapiv1alpha2.DefaultCRISocket {
t.Errorf("Decoded value contains .NodeRegistration (node-specific info shouldn't be uploaded), decoded = %#v, expected = empty", decodedCfg.NodeRegistration)
statusData := masterCfg.Data[kubeadmconstants.ClusterStatusConfigMapKey]
if statusData == "" {
t2.Fatal("failed to find ClusterStatusConfigMapKey key")
}
if decodedExtCfg.Kind != "MasterConfiguration" {
t.Errorf("Expected kind MasterConfiguration, got %v", decodedExtCfg.Kind)
decodedStatus := &kubeadmapi.ClusterStatus{}
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(statusData), decodedStatus); err != nil {
t2.Fatalf("unable to decode status from bytes: %v", err)
}
if decodedExtCfg.APIVersion != "kubeadm.k8s.io/v1alpha2" {
t.Errorf("Expected apiVersion kubeadm.k8s.io/v1alpha2, got %v", decodedExtCfg.APIVersion)
if !reflect.DeepEqual(decodedStatus, status) {
t2.Error("the initial and decoded ClusterStatus didn't match")
}
}
})
}
}
func TestGetClusterStatus(t *testing.T) {
var tests = []struct {
name string
clusterStatus *kubeadmapi.ClusterStatus
expectedClusterEndpoints int
}{
{
name: "return empty ClusterStatus if cluster kubeadm-config doesn't exist (e.g init)",
expectedClusterEndpoints: 0,
},
{
name: "return ClusterStatus if cluster kubeadm-config exist (e.g upgrade)",
clusterStatus: &kubeadmapi.ClusterStatus{
APIEndpoints: map[string]kubeadmapi.APIEndpoint{
"dummy": {AdvertiseAddress: "1.2.3.4", BindPort: 1234},
},
},
expectedClusterEndpoints: 1,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := clientsetfake.NewSimpleClientset()
if tt.clusterStatus != nil {
createConfigMapWithStatus(tt.clusterStatus, client)
}
actual, err := getClusterStatus(client)
if err != nil {
t.Error("GetClusterStatus returned unexpected error")
return
}
if tt.expectedClusterEndpoints != len(actual.APIEndpoints) {
t.Error("actual ClusterStatus doesn't return expected endpoints")
}
})
}
}
// createConfigMapWithStatus create a ConfigMap with ClusterStatus for TestGetClusterStatus
func createConfigMapWithStatus(statusToCreate *kubeadmapi.ClusterStatus, client clientset.Interface) error {
statusYaml, err := configutil.MarshalKubeadmConfigObject(statusToCreate)
if err != nil {
return err
}
return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: kubeadmconstants.InitConfigurationConfigMap,
Namespace: metav1.NamespaceSystem,
},
Data: map[string]string{
kubeadmconstants.ClusterStatusConfigMapKey: string(statusYaml),
},
})
}