Bumping k8s dependencies to 1.13

This commit is contained in:
Cheng Xing
2018-11-16 14:08:25 -08:00
parent 305407125c
commit b4c0b68ec7
8002 changed files with 884099 additions and 276228 deletions

View File

@@ -5,38 +5,39 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"container.go",
"benchmark_util.go",
"device_plugin.go",
"doc.go",
"docker_util.go",
"framework.go",
"gpu_device_plugin.go",
"image_list.go",
"node_problem_detector_linux.go",
"resource_collector.go",
"util.go",
] + select({
"@io_bazel_rules_go//go/platform:linux": [
"benchmark_util.go",
"node_problem_detector_linux.go",
"resource_collector.go",
],
"//conditions:default": [],
}),
],
importpath = "k8s.io/kubernetes/test/e2e_node",
visibility = ["//visibility:public"],
deps = [
"//pkg/api/v1/pod:go_default_library",
"//pkg/features:go_default_library",
"//pkg/kubelet/apis/config:go_default_library",
"//pkg/kubelet/apis/cri:go_default_library",
"//pkg/kubelet/apis/cri/runtime/v1alpha2:go_default_library",
"//pkg/kubelet/apis/deviceplugin/v1beta1:go_default_library",
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library",
"//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
"//pkg/kubelet/cm:go_default_library",
"//pkg/kubelet/cm/devicemanager:go_default_library",
"//pkg/kubelet/kubeletconfig/util/codec:go_default_library",
"//pkg/kubelet/metrics:go_default_library",
"//pkg/kubelet/remote:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library",
"//test/e2e/common:go_default_library",
"//test/e2e/framework:go_default_library",
"//test/e2e/framework/metrics:go_default_library",
@@ -49,30 +50,22 @@ go_library(
"//vendor/github.com/onsi/ginkgo:go_default_library",
"//vendor/github.com/onsi/gomega:go_default_library",
"//vendor/github.com/prometheus/common/model:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:linux": [
"//pkg/api/v1/node:go_default_library",
"//pkg/util/procfs:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
"//test/e2e/perftype:go_default_library",
"//test/e2e_node/perftype:go_default_library",
"//vendor/github.com/google/cadvisor/client/v2:go_default_library",
"//vendor/github.com/google/cadvisor/info/v2:go_default_library",
"//vendor/github.com/opencontainers/runc/libcontainer/cgroups:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
],
"//conditions:default": [],
}),
@@ -83,36 +76,32 @@ go_test(
srcs = [
"apparmor_test.go",
"container_log_rotation_test.go",
"container_manager_test.go",
"cpu_manager_test.go",
"critical_pod_test.go",
"density_test.go",
"docker_test.go",
"dockershim_checkpoint_test.go",
"dynamic_kubelet_config_test.go",
"e2e_node_suite_test.go",
"eviction_test.go",
"garbage_collector_test.go",
"gke_environment_test.go",
"hugepages_test.go",
"image_id_test.go",
"kubelet_test.go",
"lifecycle_hook_test.go",
"log_path_test.go",
"mirror_pod_test.go",
"node_container_manager_test.go",
"node_perf_test.go",
"pods_container_manager_test.go",
"resource_usage_test.go",
"restart_test.go",
"runtime_conformance_test.go",
"security_context_test.go",
"summary_test.go",
"volume_manager_test.go",
] + select({
"@io_bazel_rules_go//go/platform:linux": [
"container_manager_test.go",
"density_test.go",
"e2e_node_suite_test.go",
"node_container_manager_test.go",
"resource_usage_test.go",
"restart_test.go",
],
"//conditions:default": [],
}),
],
embed = [":go_default_library"],
tags = ["e2e"],
deps = [
@@ -120,9 +109,9 @@ go_test(
"//pkg/apis/core:go_default_library",
"//pkg/features:go_default_library",
"//pkg/kubelet:go_default_library",
"//pkg/kubelet/apis/config:go_default_library",
"//pkg/kubelet/apis/cri:go_default_library",
"//pkg/kubelet/apis/cri/runtime/v1alpha2:go_default_library",
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
"//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
"//pkg/kubelet/cm:go_default_library",
"//pkg/kubelet/cm/cpumanager:go_default_library",
@@ -136,8 +125,26 @@ go_test(
"//pkg/kubelet/metrics:go_default_library",
"//pkg/kubelet/types:go_default_library",
"//pkg/security/apparmor:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/scheduling/v1beta1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
"//test/e2e/common:go_default_library",
"//test/e2e/framework:go_default_library",
"//test/e2e/framework/metrics:go_default_library",
"//test/e2e_node/perf/workloads:go_default_library",
"//test/e2e_node/services:go_default_library",
"//test/utils/image:go_default_library",
"//vendor/github.com/blang/semver:go_default_library",
@@ -149,34 +156,18 @@ go_test(
"//vendor/github.com/onsi/gomega/gstruct:go_default_library",
"//vendor/github.com/onsi/gomega/types:go_default_library",
"//vendor/github.com/prometheus/common/model:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:linux": [
"//test/e2e/common:go_default_library",
"//test/e2e_node/system:go_default_library",
"//cmd/kubeadm/app/util/system:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
"//test/utils:go_default_library",
"//vendor/github.com/kardianos/osext:go_default_library",
"//vendor/github.com/onsi/ginkgo/config:go_default_library",
"//vendor/github.com/onsi/ginkgo/reporters:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
],
"//conditions:default": [],
}),
@@ -204,12 +195,12 @@ filegroup(
":package-srcs",
"//test/e2e_node/builder:all-srcs",
"//test/e2e_node/environment:all-srcs",
"//test/e2e_node/perf/workloads:all-srcs",
"//test/e2e_node/perftype:all-srcs",
"//test/e2e_node/remote:all-srcs",
"//test/e2e_node/runner/local:all-srcs",
"//test/e2e_node/runner/remote:all-srcs",
"//test/e2e_node/services:all-srcs",
"//test/e2e_node/system:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],

View File

@@ -18,6 +18,7 @@ package e2e_node
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"os"
@@ -31,6 +32,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/watch"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/kubernetes/pkg/security/apparmor"
"k8s.io/kubernetes/test/e2e/framework"
@@ -151,7 +153,9 @@ func runAppArmorTest(f *framework.Framework, shouldRun bool, profile string) v1.
// Pod should remain in the pending state. Wait for the Reason to be set to "AppArmor".
w, err := f.PodClient().Watch(metav1.SingleObject(metav1.ObjectMeta{Name: pod.Name}))
framework.ExpectNoError(err)
_, err = watch.Until(framework.PodStartTimeout, w, func(e watch.Event) (bool, error) {
ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), framework.PodStartTimeout)
defer cancel()
_, err = watchtools.UntilWithoutRetry(ctx, w, func(e watch.Event) (bool, error) {
switch e.Type {
case watch.Deleted:
return false, errors.NewNotFound(schema.GroupResource{Resource: "pods"}, pod.Name)

View File

@@ -1,128 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e_node
import (
"fmt"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
"k8s.io/kubernetes/test/e2e/framework"
)
// One pod one container
type ConformanceContainer struct {
Container v1.Container
RestartPolicy v1.RestartPolicy
Volumes []v1.Volume
ImagePullSecrets []string
PodClient *framework.PodClient
podName string
PodSecurityContext *v1.PodSecurityContext
}
func (cc *ConformanceContainer) Create() {
cc.podName = cc.Container.Name + string(uuid.NewUUID())
imagePullSecrets := []v1.LocalObjectReference{}
for _, s := range cc.ImagePullSecrets {
imagePullSecrets = append(imagePullSecrets, v1.LocalObjectReference{Name: s})
}
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: cc.podName,
},
Spec: v1.PodSpec{
RestartPolicy: cc.RestartPolicy,
Containers: []v1.Container{
cc.Container,
},
SecurityContext: cc.PodSecurityContext,
Volumes: cc.Volumes,
ImagePullSecrets: imagePullSecrets,
},
}
cc.PodClient.Create(pod)
}
func (cc *ConformanceContainer) Delete() error {
return cc.PodClient.Delete(cc.podName, metav1.NewDeleteOptions(0))
}
func (cc *ConformanceContainer) IsReady() (bool, error) {
pod, err := cc.PodClient.Get(cc.podName, metav1.GetOptions{})
if err != nil {
return false, err
}
return podutil.IsPodReady(pod), nil
}
func (cc *ConformanceContainer) GetPhase() (v1.PodPhase, error) {
pod, err := cc.PodClient.Get(cc.podName, metav1.GetOptions{})
if err != nil {
return v1.PodUnknown, err
}
return pod.Status.Phase, nil
}
func (cc *ConformanceContainer) GetStatus() (v1.ContainerStatus, error) {
pod, err := cc.PodClient.Get(cc.podName, metav1.GetOptions{})
if err != nil {
return v1.ContainerStatus{}, err
}
statuses := pod.Status.ContainerStatuses
if len(statuses) != 1 || statuses[0].Name != cc.Container.Name {
return v1.ContainerStatus{}, fmt.Errorf("unexpected container statuses %v", statuses)
}
return statuses[0], nil
}
func (cc *ConformanceContainer) Present() (bool, error) {
_, err := cc.PodClient.Get(cc.podName, metav1.GetOptions{})
if err == nil {
return true, nil
}
if errors.IsNotFound(err) {
return false, nil
}
return false, err
}
type ContainerState string
const (
ContainerStateWaiting ContainerState = "Waiting"
ContainerStateRunning ContainerState = "Running"
ContainerStateTerminated ContainerState = "Terminated"
ContainerStateUnknown ContainerState = "Unknown"
)
func GetContainerState(state v1.ContainerState) ContainerState {
if state.Waiting != nil {
return ContainerStateWaiting
}
if state.Running != nil {
return ContainerStateRunning
}
if state.Terminated != nil {
return ContainerStateTerminated
}
return ContainerStateUnknown
}

View File

@@ -22,7 +22,7 @@ import (
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
kubelogs "k8s.io/kubernetes/pkg/kubelet/logs"
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"

View File

@@ -178,7 +178,7 @@ var _ = framework.KubeDescribe("Container Manager Misc [Serial]", func() {
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Image: imageutils.GetE2EImage(imageutils.NginxSlim),
Image: imageutils.GetE2EImage(imageutils.Nginx),
Name: podName,
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{

View File

@@ -26,8 +26,8 @@ import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager"
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
"k8s.io/kubernetes/pkg/kubelet/types"

View File

@@ -24,7 +24,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeapi "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
"k8s.io/kubernetes/test/e2e/framework"
imageutils "k8s.io/kubernetes/test/utils/image"
@@ -45,6 +45,9 @@ var _ = framework.KubeDescribe("CriticalPod [Serial] [Disruptive] [NodeFeature:C
Context("when we need to admit a critical pod", func() {
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
if initialConfig.FeatureGates == nil {
initialConfig.FeatureGates = make(map[string]bool)
}
initialConfig.FeatureGates[string(features.ExperimentalCriticalPodAnnotation)] = true
})
@@ -142,9 +145,9 @@ func getTestPod(critical bool, name string, resources v1.ResourceRequirements) *
pod.ObjectMeta.Annotations = map[string]string{
kubelettypes.CriticalPodAnnotationKey: "",
}
Expect(kubelettypes.IsCriticalPod(pod)).To(BeTrue(), "pod should be a critical pod")
Expect(kubelettypes.IsCritical(pod.Namespace, pod.Annotations)).To(BeTrue(), "pod should be a critical pod")
} else {
Expect(kubelettypes.IsCriticalPod(pod)).To(BeFalse(), "pod should not be a critical pod")
Expect(kubelettypes.IsCritical(pod.Namespace, pod.Annotations)).To(BeFalse(), "pod should not be a critical pod")
}
return pod
}

View File

@@ -31,7 +31,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
kubemetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
"k8s.io/kubernetes/test/e2e/framework"

View File

@@ -26,9 +26,10 @@ import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/util/uuid"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/pkg/features"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
"k8s.io/kubernetes/test/e2e/framework"
pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
@@ -40,14 +41,28 @@ import (
const (
// fake resource name
resourceName = "fake.com/resource"
resourceName = "fake.com/resource"
resourceNameWithProbeSupport = "fake.com/resource2"
)
// Serial because the test restarts Kubelet
var _ = framework.KubeDescribe("Device Plugin [Feature:DevicePlugin][NodeFeature:DevicePlugin][Serial]", func() {
f := framework.NewDefaultFramework("device-plugin-errors")
testDevicePlugin(f, false, pluginapi.DevicePluginPath)
})
var _ = framework.KubeDescribe("Device Plugin [Feature:DevicePluginProbe][NodeFeature:DevicePluginProbe][Serial]", func() {
f := framework.NewDefaultFramework("device-plugin-errors")
testDevicePlugin(f, true, "/var/lib/kubelet/plugins/")
})
func testDevicePlugin(f *framework.Framework, enablePluginWatcher bool, pluginSockDir string) {
Context("DevicePlugin", func() {
By("Enabling support for Kubelet Plugins Watcher")
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
initialConfig.FeatureGates[string(features.KubeletPluginsWatcher)] = enablePluginWatcher
})
//devicePluginSockPaths := []string{pluginapi.DevicePluginPath}
It("Verifies the Kubelet device plugin functionality.", func() {
By("Start stub device plugin")
// fake devices for e2e test
@@ -56,15 +71,16 @@ var _ = framework.KubeDescribe("Device Plugin [Feature:DevicePlugin][NodeFeature
{ID: "Dev-2", Health: pluginapi.Healthy},
}
socketPath := pluginapi.DevicePluginPath + "dp." + fmt.Sprintf("%d", time.Now().Unix())
socketPath := pluginSockDir + "dp." + fmt.Sprintf("%d", time.Now().Unix())
framework.Logf("socketPath %v", socketPath)
dp1 := dm.NewDevicePluginStub(devs, socketPath)
dp1 := dm.NewDevicePluginStub(devs, socketPath, resourceName, false)
dp1.SetAllocFunc(stubAllocFunc)
err := dp1.Start()
framework.ExpectNoError(err)
By("Register resources")
err = dp1.Register(pluginapi.KubeletSocket, resourceName, false)
err = dp1.Register(pluginapi.KubeletSocket, resourceName, pluginapi.DevicePluginPath)
framework.ExpectNoError(err)
By("Waiting for the resource exported by the stub device plugin to become available on the local node")
@@ -103,13 +119,13 @@ var _ = framework.KubeDescribe("Device Plugin [Feature:DevicePlugin][NodeFeature
By("Wait for node is ready")
framework.WaitForAllNodesSchedulable(f.ClientSet, framework.TestContext.NodeSchedulableTimeout)
By("Re-Register resources after kubelet restart")
dp1 = dm.NewDevicePluginStub(devs, socketPath)
By("Re-Register resources")
dp1 = dm.NewDevicePluginStub(devs, socketPath, resourceName, false)
dp1.SetAllocFunc(stubAllocFunc)
err = dp1.Start()
framework.ExpectNoError(err)
err = dp1.Register(pluginapi.KubeletSocket, resourceName, false)
err = dp1.Register(pluginapi.KubeletSocket, resourceName, pluginSockDir)
framework.ExpectNoError(err)
By("Waiting for resource to become available on the local node after re-registration")
@@ -149,12 +165,12 @@ var _ = framework.KubeDescribe("Device Plugin [Feature:DevicePlugin][NodeFeature
Expect(devIdRestart2).To(Equal(devId2))
By("Re-register resources")
dp1 = dm.NewDevicePluginStub(devs, socketPath)
dp1 = dm.NewDevicePluginStub(devs, socketPath, resourceName, false)
dp1.SetAllocFunc(stubAllocFunc)
err = dp1.Start()
framework.ExpectNoError(err)
err = dp1.Register(pluginapi.KubeletSocket, resourceName, false)
err = dp1.Register(pluginapi.KubeletSocket, resourceName, pluginSockDir)
framework.ExpectNoError(err)
By("Waiting for the resource exported by the stub device plugin to become healthy on the local node")
@@ -192,7 +208,7 @@ var _ = framework.KubeDescribe("Device Plugin [Feature:DevicePlugin][NodeFeature
f.PodClient().DeleteSync(pod2.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
})
})
})
}
// makeBusyboxPod returns a simple Pod spec with a busybox container
// that requests resourceName and runs the specified command.
@@ -235,7 +251,7 @@ func ensurePodContainerRestart(f *framework.Framework, podName string, contName
currentCount = p.Status.ContainerStatuses[0].RestartCount
framework.Logf("initial %v, current %v", initialCount, currentCount)
return currentCount > initialCount
}, 2*time.Minute, framework.Poll).Should(BeTrue())
}, 5*time.Minute, framework.Poll).Should(BeTrue())
}
// parseLog returns the matching string for the specified regular expression parsed from the container logs.

View File

@@ -61,7 +61,7 @@ var _ = framework.KubeDescribe("Docker features [Feature:Docker][Legacy:Docker]"
Spec: v1.PodSpec{
Containers: []v1.Container{{
Name: containerName,
Image: imageutils.GetE2EImage(imageutils.NginxSlim),
Image: imageutils.GetE2EImage(imageutils.Nginx),
}},
},
})

View File

@@ -29,7 +29,7 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
controller "k8s.io/kubernetes/pkg/kubelet/kubeletconfig"
"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/status"
"k8s.io/kubernetes/pkg/kubelet/metrics"

View File

@@ -38,11 +38,11 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilyaml "k8s.io/apimachinery/pkg/util/yaml"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/cmd/kubeadm/app/util/system"
nodeutil "k8s.io/kubernetes/pkg/api/v1/node"
commontest "k8s.io/kubernetes/test/e2e/common"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e_node/services"
"k8s.io/kubernetes/test/e2e_node/system"
"github.com/golang/glog"
"github.com/kardianos/osext"

View File

@@ -99,7 +99,7 @@ func containerRuntime() error {
}
// Setup cadvisor to check the container environment
c, err := cadvisor.New("", 0 /*don't start the http server*/, cadvisor.NewImageFsInfoProvider("docker", ""), "/var/lib/kubelet", false)
c, err := cadvisor.New(cadvisor.NewImageFsInfoProvider("docker", ""), "/var/lib/kubelet", false)
if err != nil {
return printError("Container Runtime Check: %s Could not start cadvisor %v", failed, err)
}

View File

@@ -24,12 +24,13 @@ import (
"time"
"k8s.io/api/core/v1"
schedulerapi "k8s.io/api/scheduling/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
nodeutil "k8s.io/kubernetes/pkg/api/v1/node"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
"k8s.io/kubernetes/pkg/kubelet/eviction"
kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
@@ -42,7 +43,7 @@ import (
)
// Eviction Policy is described here:
// https://github.com/kubernetes/community/blob/master/contributors/design-proposals/kubelet-eviction.md
// https://github.com/kubernetes/community/blob/master/contributors/design-proposals/node/kubelet-eviction.md
const (
postTestConditionMonitoringPeriod = 1 * time.Minute
@@ -232,7 +233,6 @@ var _ = framework.KubeDescribe("LocalStorageCapacityIsolationEviction [Slow] [Se
evictionTestTimeout := 10 * time.Minute
Context(fmt.Sprintf(testContextFmt, "evictions due to pod local storage violations"), func() {
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
initialConfig.FeatureGates[string(features.LocalStorageCapacityIsolation)] = true
// setting a threshold to 0% disables; non-empty map overrides default value (necessary due to omitempty)
initialConfig.EvictionHard = map[string]string{"memory.available": "0%"}
})
@@ -285,9 +285,12 @@ var _ = framework.KubeDescribe("PriorityMemoryEvictionOrdering [Slow] [Serial] [
expectedNodeCondition := v1.NodeMemoryPressure
expectedStarvedResource := v1.ResourceMemory
pressureTimeout := 10 * time.Minute
highPriorityClassName := f.BaseName + "-high-priority"
highPriority := int32(999999999)
Context(fmt.Sprintf(testContextFmt, expectedNodeCondition), func() {
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
initialConfig.FeatureGates[string(features.PodPriority)] = true
memoryConsumed := resource.MustParse("600Mi")
summary := eventuallyGetSummary()
availableBytes := *(summary.Node.Memory.AvailableBytes)
@@ -297,6 +300,14 @@ var _ = framework.KubeDescribe("PriorityMemoryEvictionOrdering [Slow] [Serial] [
initialConfig.EvictionHard = map[string]string{"memory.available": fmt.Sprintf("%d", availableBytes-uint64(memoryConsumed.Value()))}
initialConfig.EvictionMinimumReclaim = map[string]string{}
})
BeforeEach(func() {
_, err := f.ClientSet.SchedulingV1beta1().PriorityClasses().Create(&schedulerapi.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: highPriorityClassName}, Value: highPriority})
Expect(err == nil || errors.IsAlreadyExists(err)).To(BeTrue())
})
AfterEach(func() {
err := f.ClientSet.SchedulingV1beta1().PriorityClasses().Delete(highPriorityClassName, &metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
})
specs := []podEvictSpec{
{
evictionPriority: 2,
@@ -318,8 +329,7 @@ var _ = framework.KubeDescribe("PriorityMemoryEvictionOrdering [Slow] [Serial] [
}),
},
}
systemPriority := int32(2147483647)
specs[1].pod.Spec.Priority = &systemPriority
specs[1].pod.Spec.PriorityClassName = highPriorityClassName
runEvictionTest(f, pressureTimeout, expectedNodeCondition, expectedStarvedResource, logMemoryMetrics, specs)
})
})
@@ -332,10 +342,12 @@ var _ = framework.KubeDescribe("PriorityLocalStorageEvictionOrdering [Slow] [Ser
expectedNodeCondition := v1.NodeDiskPressure
expectedStarvedResource := v1.ResourceEphemeralStorage
pressureTimeout := 10 * time.Minute
highPriorityClassName := f.BaseName + "-high-priority"
highPriority := int32(999999999)
Context(fmt.Sprintf(testContextFmt, expectedNodeCondition), func() {
tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
initialConfig.FeatureGates[string(features.PodPriority)] = true
initialConfig.FeatureGates[string(features.LocalStorageCapacityIsolation)] = true
diskConsumed := resource.MustParse("350Mi")
summary := eventuallyGetSummary()
availableBytes := *(summary.Node.Fs.AvailableBytes)
@@ -345,6 +357,14 @@ var _ = framework.KubeDescribe("PriorityLocalStorageEvictionOrdering [Slow] [Ser
initialConfig.EvictionHard = map[string]string{"nodefs.available": fmt.Sprintf("%d", availableBytes-uint64(diskConsumed.Value()))}
initialConfig.EvictionMinimumReclaim = map[string]string{}
})
BeforeEach(func() {
_, err := f.ClientSet.SchedulingV1beta1().PriorityClasses().Create(&schedulerapi.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: highPriorityClassName}, Value: highPriority})
Expect(err == nil || errors.IsAlreadyExists(err)).To(BeTrue())
})
AfterEach(func() {
err := f.ClientSet.SchedulingV1beta1().PriorityClasses().Delete(highPriorityClassName, &metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
})
specs := []podEvictSpec{
{
evictionPriority: 2,
@@ -367,8 +387,7 @@ var _ = framework.KubeDescribe("PriorityLocalStorageEvictionOrdering [Slow] [Ser
}),
},
}
systemPriority := int32(2147483647)
specs[1].pod.Spec.Priority = &systemPriority
specs[1].pod.Spec.PriorityClassName = highPriorityClassName
runEvictionTest(f, pressureTimeout, expectedNodeCondition, expectedStarvedResource, logDiskMetrics, specs)
})
})

View File

@@ -27,6 +27,7 @@ import (
"strings"
"k8s.io/kubernetes/test/e2e/framework"
imageutils "k8s.io/kubernetes/test/utils/image"
"github.com/blang/semver"
. "github.com/onsi/ginkgo"
@@ -170,7 +171,7 @@ func checkDockerConfig() error {
// checkDockerNetworkClient checks client networking by pinging an external IP
// address from a container.
func checkDockerNetworkClient() error {
const imageName = "k8s.gcr.io/busybox"
imageName := imageutils.GetE2EImage(imageutils.BusyBox)
output, err := runCommand("docker", "run", "--rm", imageName, "sh", "-c", "ping -w 5 -q google.com")
if err != nil {
return err
@@ -310,7 +311,7 @@ func checkDockerStorageDriver() error {
return fmt.Errorf("failed to find storage driver")
}
var _ = framework.KubeDescribe("GKE system requirements [Conformance][NodeConformance][Feature:GKEEnv][NodeFeature:GKEEnv]", func() {
var _ = framework.KubeDescribe("GKE system requirements [NodeConformance][Feature:GKEEnv][NodeFeature:GKEEnv]", func() {
BeforeEach(func() {
framework.RunIfSystemSpecNameIs("gke")
})

View File

@@ -42,6 +42,7 @@ var _ = framework.KubeDescribe("NVIDIA GPU Device Plugin [Feature:GPUDevicePlugi
Context("DevicePlugin", func() {
var devicePluginPod *v1.Pod
var err error
BeforeEach(func() {
By("Ensuring that Nvidia GPUs exists on the node")
if !checkIfNvidiaGPUsExistOnNode() {
@@ -49,12 +50,13 @@ var _ = framework.KubeDescribe("NVIDIA GPU Device Plugin [Feature:GPUDevicePlugi
}
By("Creating the Google Device Plugin pod for NVIDIA GPU in GKE")
devicePluginPod = f.PodClient().CreateSync(framework.NVIDIADevicePlugin(f.Namespace.Name))
devicePluginPod, err = f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Create(framework.NVIDIADevicePlugin())
framework.ExpectNoError(err)
By("Waiting for GPUs to become available on the local node")
Eventually(func() bool {
return framework.NumberOfNVIDIAGPUs(getLocalNode(f)) > 0
}, 10*time.Second, framework.Poll).Should(BeTrue())
}, 5*time.Minute, framework.Poll).Should(BeTrue())
if framework.NumberOfNVIDIAGPUs(getLocalNode(f)) < 2 {
Skip("Not enough GPUs to execute this test (at least two needed)")
@@ -97,7 +99,7 @@ var _ = framework.KubeDescribe("NVIDIA GPU Device Plugin [Feature:GPUDevicePlugi
framework.WaitForAllNodesSchedulable(f.ClientSet, framework.TestContext.NodeSchedulableTimeout)
Eventually(func() bool {
return framework.NumberOfNVIDIAGPUs(getLocalNode(f)) > 0
}, 10*time.Second, framework.Poll).Should(BeTrue())
}, 5*time.Minute, framework.Poll).Should(BeTrue())
p2 := f.PodClient().CreateSync(makeBusyboxPod(framework.NVIDIAGPUResourceName, podRECMD))
By("Checking that pods got a different GPU")
@@ -106,7 +108,7 @@ var _ = framework.KubeDescribe("NVIDIA GPU Device Plugin [Feature:GPUDevicePlugi
Expect(devId1).To(Not(Equal(devId2)))
By("Deleting device plugin.")
f.PodClient().Delete(devicePluginPod.Name, &metav1.DeleteOptions{})
f.ClientSet.CoreV1().Pods(metav1.NamespaceSystem).Delete(devicePluginPod.Name, &metav1.DeleteOptions{})
By("Waiting for GPUs to become unavailable on the local node")
Eventually(func() bool {
node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})

View File

@@ -28,9 +28,10 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
"k8s.io/kubernetes/pkg/kubelet/cm"
"k8s.io/kubernetes/test/e2e/framework"
imageutils "k8s.io/kubernetes/test/utils/image"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@@ -168,7 +169,7 @@ func runHugePagesTests(f *framework.Framework) {
Spec: apiv1.PodSpec{
Containers: []apiv1.Container{
{
Image: framework.GetPauseImageName(f.ClientSet),
Image: imageutils.GetPauseImageName(),
Name: "container" + string(uuid.NewUUID()),
Resources: apiv1.ResourceRequirements{
Limits: apiv1.ResourceList{

View File

@@ -47,12 +47,15 @@ var NodeImageWhiteList = sets.NewString(
busyboxImage,
"k8s.gcr.io/busybox@sha256:4bdd623e848417d96127e16037743f0cd8b528c026e9175e22a84f639eca58ff",
"k8s.gcr.io/node-problem-detector:v0.4.1",
imageutils.GetE2EImage(imageutils.NginxSlim),
imageutils.GetE2EImage(imageutils.Nginx),
imageutils.GetE2EImage(imageutils.ServeHostname),
imageutils.GetE2EImage(imageutils.Netexec),
imageutils.GetE2EImage(imageutils.Nonewprivs),
imageutils.GetPauseImageName(),
framework.GetGPUDevicePluginImage(),
"gcr.io/kubernetes-e2e-test-images/node-perf/npb-is-amd64:1.0",
"gcr.io/kubernetes-e2e-test-images/node-perf/npb-ep-amd64:1.0",
"gcr.io/kubernetes-e2e-test-images/node-perf/tf-wide-deep-amd64:1.0",
)
func init() {

View File

@@ -39,7 +39,12 @@ var _ = framework.KubeDescribe("Kubelet", func() {
})
Context("when scheduling a busybox command in a pod", func() {
podName := "busybox-scheduling-" + string(uuid.NewUUID())
framework.ConformanceIt("it should print the output to logs [NodeConformance]", func() {
/*
Release : v1.9
Testname: Kubelet, log output, default
Description: By default the stdout and stderr from the process being executed in a pod MUST be sent to the pod's logs.
*/
It("it should print the output to logs [NodeConformance]", func() {
podClient.CreateSync(&v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
@@ -164,7 +169,12 @@ var _ = framework.KubeDescribe("Kubelet", func() {
})
Context("when scheduling a read only busybox container", func() {
podName := "busybox-readonly-fs" + string(uuid.NewUUID())
framework.ConformanceIt("it should not write to root filesystem [NodeConformance]", func() {
/*
Release : v1.9
Testname: Kubelet, Pod with read only root file system
Description: Create a Pod with security context set with ReadOnlyRootFileSystem set to true. The Pod then tries to write to the /file on the root, write operation to the root filesystem MUST fail as expected.
*/
It("it should not write to root filesystem [NodeConformance]", func() {
isReadOnly := true
podClient.CreateSync(&v1.Pod{
ObjectMeta: metav1.ObjectMeta{

View File

@@ -1,153 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e_node
import (
"time"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/kubernetes/test/e2e/framework"
imageutils "k8s.io/kubernetes/test/utils/image"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = framework.KubeDescribe("Container Lifecycle Hook", func() {
f := framework.NewDefaultFramework("container-lifecycle-hook")
var podClient *framework.PodClient
const (
podCheckInterval = 1 * time.Second
postStartWaitTimeout = 2 * time.Minute
preStopWaitTimeout = 30 * time.Second
)
Context("when create a pod with lifecycle hook", func() {
var targetIP string
podHandleHookRequest := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod-handle-http-request",
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "pod-handle-http-request",
Image: imageutils.GetE2EImage(imageutils.Netexec),
Ports: []v1.ContainerPort{
{
ContainerPort: 8080,
Protocol: v1.ProtocolTCP,
},
},
},
},
},
}
BeforeEach(func() {
podClient = f.PodClient()
By("create the container to handle the HTTPGet hook request.")
newPod := podClient.CreateSync(podHandleHookRequest)
targetIP = newPod.Status.PodIP
})
testPodWithHook := func(podWithHook *v1.Pod) {
By("create the pod with lifecycle hook")
podClient.CreateSync(podWithHook)
if podWithHook.Spec.Containers[0].Lifecycle.PostStart != nil {
By("check poststart hook")
Eventually(func() error {
return podClient.MatchContainerOutput(podHandleHookRequest.Name, podHandleHookRequest.Spec.Containers[0].Name,
`GET /echo\?msg=poststart`)
}, postStartWaitTimeout, podCheckInterval).Should(BeNil())
}
By("delete the pod with lifecycle hook")
podClient.DeleteSync(podWithHook.Name, metav1.NewDeleteOptions(15), framework.DefaultPodDeletionTimeout)
if podWithHook.Spec.Containers[0].Lifecycle.PreStop != nil {
By("check prestop hook")
Eventually(func() error {
return podClient.MatchContainerOutput(podHandleHookRequest.Name, podHandleHookRequest.Spec.Containers[0].Name,
`GET /echo\?msg=prestop`)
}, preStopWaitTimeout, podCheckInterval).Should(BeNil())
}
}
framework.ConformanceIt("should execute poststart exec hook properly [NodeConformance]", func() {
lifecycle := &v1.Lifecycle{
PostStart: &v1.Handler{
Exec: &v1.ExecAction{
Command: []string{"sh", "-c", "curl http://" + targetIP + ":8080/echo?msg=poststart"},
},
},
}
podWithHook := getPodWithHook("pod-with-poststart-exec-hook", imageutils.GetE2EImage(imageutils.Hostexec), lifecycle)
testPodWithHook(podWithHook)
})
framework.ConformanceIt("should execute prestop exec hook properly [NodeConformance]", func() {
lifecycle := &v1.Lifecycle{
PreStop: &v1.Handler{
Exec: &v1.ExecAction{
Command: []string{"sh", "-c", "curl http://" + targetIP + ":8080/echo?msg=prestop"},
},
},
}
podWithHook := getPodWithHook("pod-with-prestop-exec-hook", imageutils.GetE2EImage(imageutils.Hostexec), lifecycle)
testPodWithHook(podWithHook)
})
framework.ConformanceIt("should execute poststart http hook properly [NodeConformance]", func() {
lifecycle := &v1.Lifecycle{
PostStart: &v1.Handler{
HTTPGet: &v1.HTTPGetAction{
Path: "/echo?msg=poststart",
Host: targetIP,
Port: intstr.FromInt(8080),
},
},
}
podWithHook := getPodWithHook("pod-with-poststart-http-hook", imageutils.GetPauseImageName(), lifecycle)
testPodWithHook(podWithHook)
})
framework.ConformanceIt("should execute prestop http hook properly [NodeConformance]", func() {
lifecycle := &v1.Lifecycle{
PreStop: &v1.Handler{
HTTPGet: &v1.HTTPGetAction{
Path: "/echo?msg=prestop",
Host: targetIP,
Port: intstr.FromInt(8080),
},
},
}
podWithHook := getPodWithHook("pod-with-prestop-http-hook", imageutils.GetPauseImageName(), lifecycle)
testPodWithHook(podWithHook)
})
})
})
func getPodWithHook(name string, image string, lifecycle *v1.Lifecycle) *v1.Pod {
return &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: name,
Image: image,
Lifecycle: lifecycle,
},
},
},
}
}

View File

@@ -49,7 +49,7 @@ var _ = framework.KubeDescribe("MirrorPod", func() {
By("create the static pod")
err := createStaticPod(podPath, staticPodName, ns,
imageutils.GetE2EImage(imageutils.NginxSlim), v1.RestartPolicyAlways)
imageutils.GetE2EImage(imageutils.Nginx), v1.RestartPolicyAlways)
Expect(err).ShouldNot(HaveOccurred())
By("wait for the mirror pod to be running")
@@ -57,7 +57,12 @@ var _ = framework.KubeDescribe("MirrorPod", func() {
return checkMirrorPodRunning(f.ClientSet, mirrorPodName, ns)
}, 2*time.Minute, time.Second*4).Should(BeNil())
})
framework.ConformanceIt("should be updated when static pod updated [NodeConformance]", func() {
/*
Release : v1.9
Testname: Mirror Pod, update
Description: Updating a static Pod MUST recreate an updated mirror Pod. Create a static pod, verify that a mirror pod is created. Update the static pod by changing the container image, the mirror pod MUST be re-created and updated with the new image.
*/
It("should be updated when static pod updated [NodeConformance]", func() {
By("get mirror pod uid")
pod, err := f.ClientSet.CoreV1().Pods(ns).Get(mirrorPodName, metav1.GetOptions{})
Expect(err).ShouldNot(HaveOccurred())
@@ -79,7 +84,12 @@ var _ = framework.KubeDescribe("MirrorPod", func() {
Expect(len(pod.Spec.Containers)).Should(Equal(1))
Expect(pod.Spec.Containers[0].Image).Should(Equal(image))
})
framework.ConformanceIt("should be recreated when mirror pod gracefully deleted [NodeConformance]", func() {
/*
Release : v1.9
Testname: Mirror Pod, delete
Description: When a mirror-Pod is deleted then the mirror pod MUST be re-created. Create a static pod, verify that a mirror pod is created. Delete the mirror pod, the mirror pod MUST be re-created and running.
*/
It("should be recreated when mirror pod gracefully deleted [NodeConformance]", func() {
By("get mirror pod uid")
pod, err := f.ClientSet.CoreV1().Pods(ns).Get(mirrorPodName, metav1.GetOptions{})
Expect(err).ShouldNot(HaveOccurred())
@@ -94,7 +104,12 @@ var _ = framework.KubeDescribe("MirrorPod", func() {
return checkMirrorPodRecreatedAndRunnig(f.ClientSet, mirrorPodName, ns, uid)
}, 2*time.Minute, time.Second*4).Should(BeNil())
})
framework.ConformanceIt("should be recreated when mirror pod forcibly deleted [NodeConformance]", func() {
/*
Release : v1.9
Testname: Mirror Pod, force delete
Description: When a mirror-Pod is deleted, forcibly, then the mirror pod MUST be re-created. Create a static pod, verify that a mirror pod is created. Delete the mirror pod with delete wait time set to zero forcing immediate deletion, the mirror pod MUST be re-created and running.
*/
It("should be recreated when mirror pod forcibly deleted [NodeConformance]", func() {
By("get mirror pod uid")
pod, err := f.ClientSet.CoreV1().Pods(ns).Get(mirrorPodName, metav1.GetOptions{})
Expect(err).ShouldNot(HaveOccurred())

View File

@@ -29,7 +29,7 @@ import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
"k8s.io/kubernetes/pkg/kubelet/cm"
"k8s.io/kubernetes/test/e2e/framework"

View File

@@ -0,0 +1,108 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e_node
import (
"fmt"
"time"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e_node/perf/workloads"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
// makeNodePerfPod returns a pod with the information provided from the workload.
func makeNodePerfPod(w workloads.NodePerfWorkload) *corev1.Pod {
return &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-pod", w.Name()),
},
Spec: w.PodSpec(),
}
}
func setKubeletConfig(f *framework.Framework, cfg *kubeletconfig.KubeletConfiguration) {
if cfg != nil {
framework.ExpectNoError(setKubeletConfiguration(f, cfg))
}
// Wait for the Kubelet to be ready.
Eventually(func() bool {
nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
return len(nodeList.Items) == 1
}, time.Minute, time.Second).Should(BeTrue())
}
// Serial because the test updates kubelet configuration.
// Slow by design.
var _ = SIGDescribe("Node Performance Testing [Serial] [Slow]", func() {
f := framework.NewDefaultFramework("node-performance-testing")
Context("Run node performance testing with pre-defined workloads", func() {
It("run each pre-defined workload", func() {
By("running the workloads")
for _, workload := range workloads.NodePerfWorkloads {
By("running the pre test exec from the workload")
err := workload.PreTestExec()
framework.ExpectNoError(err)
By("restarting kubelet with required configuration")
// Get the Kubelet config required for this workload.
oldCfg, err := getCurrentKubeletConfig()
framework.ExpectNoError(err)
newCfg, err := workload.KubeletConfig(oldCfg)
framework.ExpectNoError(err)
// Set the Kubelet config required for this workload.
setKubeletConfig(f, newCfg)
By("running the workload and waiting for success")
// Make the pod for the workload.
pod := makeNodePerfPod(workload)
// Create the pod.
pod = f.PodClient().CreateSync(pod)
// Wait for pod success.
f.PodClient().WaitForSuccess(pod.Name, workload.Timeout())
podLogs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, pod.Spec.Containers[0].Name)
framework.ExpectNoError(err)
perf, err := workload.ExtractPerformanceFromLogs(podLogs)
framework.ExpectNoError(err)
framework.Logf("Time to complete workload %s: %v", workload.Name(), perf)
// Delete the pod.
gp := int64(0)
delOpts := metav1.DeleteOptions{
GracePeriodSeconds: &gp,
}
f.PodClient().DeleteSync(pod.Name, &delOpts, framework.DefaultPodDeletionTimeout)
By("running the post test exec from the workload")
err = workload.PostTestExec()
framework.ExpectNoError(err)
// Set the Kubelet config back to the old one.
setKubeletConfig(f, oldCfg)
}
})
})
})

View File

@@ -0,0 +1,35 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = [
"npb_ep.go",
"npb_is.go",
"tf_wide_deep.go",
"utils.go",
"workloads.go",
],
importpath = "k8s.io/kubernetes/test/e2e_node/perf/workloads",
visibility = ["//visibility:public"],
deps = [
"//pkg/kubelet/apis/config:go_default_library",
"//pkg/kubelet/cm/cpumanager:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,7 @@
approvers:
- vishh
- derekwaynecarr
- balajismaniam
- ConnorDoyle
reviewers:
- sig-node-reviewers

View File

@@ -0,0 +1,120 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package workloads
import (
"fmt"
"strings"
"time"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager"
)
// npbEPWorkload defines a workload to run the Embarrassingly Parallel (EP) workload
// from NAS parallel benchmark (NPB) suite.
type npbEPWorkload struct{}
// Ensure npbEPWorkload implemets NodePerfWorkload interface.
var _ NodePerfWorkload = &npbEPWorkload{}
func (w npbEPWorkload) Name() string {
return "npb-ep"
}
func (w npbEPWorkload) PodSpec() corev1.PodSpec {
var containers []corev1.Container
ctn := corev1.Container{
Name: fmt.Sprintf("%s-ctn", w.Name()),
Image: "gcr.io/kubernetes-e2e-test-images/node-perf/npb-ep-amd64:1.0",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("15000m"),
corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("48Gi"),
},
Limits: corev1.ResourceList{
corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("15000m"),
corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("48Gi"),
},
},
Command: []string{"/bin/sh"},
Args: []string{"-c", "/ep.D.x"},
}
containers = append(containers, ctn)
return corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyNever,
Containers: containers,
}
}
func (w npbEPWorkload) Timeout() time.Duration {
return 10 * time.Minute
}
func (w npbEPWorkload) KubeletConfig(oldCfg *kubeletconfig.KubeletConfiguration) (newCfg *kubeletconfig.KubeletConfiguration, err error) {
// Enable CPU Manager in Kubelet with static policy.
newCfg = oldCfg.DeepCopy()
// Set the CPU Manager policy to static.
newCfg.CPUManagerPolicy = string(cpumanager.PolicyStatic)
// Set the CPU Manager reconcile period to 10 second.
newCfg.CPUManagerReconcilePeriod = metav1.Duration{Duration: 10 * time.Second}
// The Kubelet panics if either kube-reserved or system-reserved is not set
// when static CPU Manager is enabled. Set cpu in kube-reserved > 0 so that
// kubelet doesn't panic.
if newCfg.KubeReserved == nil {
newCfg.KubeReserved = map[string]string{}
}
if _, ok := newCfg.KubeReserved["cpu"]; !ok {
newCfg.KubeReserved["cpu"] = "200m"
}
return newCfg, nil
}
func (w npbEPWorkload) PreTestExec() error {
cmd := "/bin/sh"
args := []string{"-c", "rm -f /var/lib/kubelet/cpu_manager_state"}
err := runCmd(cmd, args)
return err
}
func (w npbEPWorkload) PostTestExec() error {
cmd := "/bin/sh"
args := []string{"-c", "rm -f /var/lib/kubelet/cpu_manager_state"}
err := runCmd(cmd, args)
return err
}
func (w npbEPWorkload) ExtractPerformanceFromLogs(logs string) (perf time.Duration, err error) {
perfLine, err := getMatchingLineFromLog(logs, "Time in seconds =")
if err != nil {
return perf, err
}
perfStrings := strings.Split(perfLine, "=")
perfString := fmt.Sprintf("%ss", strings.TrimSpace(perfStrings[1]))
perf, err = time.ParseDuration(perfString)
return perf, err
}

View File

@@ -0,0 +1,92 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package workloads
import (
"fmt"
"strings"
"time"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
)
// npbISWorkload defines a workload to run the integer sort (IS) workload
// from NAS parallel benchmark (NPB) suite.
type npbISWorkload struct{}
// Ensure npbISWorkload implemets NodePerfWorkload interface.
var _ NodePerfWorkload = &npbISWorkload{}
func (w npbISWorkload) Name() string {
return "npb-is"
}
func (w npbISWorkload) PodSpec() corev1.PodSpec {
var containers []corev1.Container
ctn := corev1.Container{
Name: fmt.Sprintf("%s-ctn", w.Name()),
Image: "gcr.io/kubernetes-e2e-test-images/node-perf/npb-is-amd64:1.0",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("16000m"),
corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("48Gi"),
},
Limits: corev1.ResourceList{
corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("16000m"),
corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("48Gi"),
},
},
Command: []string{"/bin/sh"},
Args: []string{"-c", "/is.D.x"},
}
containers = append(containers, ctn)
return corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyNever,
Containers: containers,
}
}
func (w npbISWorkload) Timeout() time.Duration {
return 4 * time.Minute
}
func (w npbISWorkload) KubeletConfig(oldCfg *kubeletconfig.KubeletConfiguration) (newCfg *kubeletconfig.KubeletConfiguration, err error) {
return oldCfg, nil
}
func (w npbISWorkload) PreTestExec() error {
return nil
}
func (w npbISWorkload) PostTestExec() error {
return nil
}
func (w npbISWorkload) ExtractPerformanceFromLogs(logs string) (perf time.Duration, err error) {
perfLine, err := getMatchingLineFromLog(logs, "Time in seconds =")
if err != nil {
return perf, err
}
perfStrings := strings.Split(perfLine, "=")
perfString := fmt.Sprintf("%ss", strings.TrimSpace(perfStrings[1]))
perf, err = time.ParseDuration(perfString)
return perf, err
}

View File

@@ -0,0 +1,119 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package workloads
import (
"fmt"
"strings"
"time"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager"
)
// tfWideDeepWorkload defines a workload to run
// https://github.com/tensorflow/models/tree/master/official/wide_deep.
type tfWideDeepWorkload struct{}
// Ensure tfWideDeepWorkload implemets NodePerfWorkload interface.
var _ NodePerfWorkload = &tfWideDeepWorkload{}
func (w tfWideDeepWorkload) Name() string {
return "tensorflow-wide-deep"
}
func (w tfWideDeepWorkload) PodSpec() corev1.PodSpec {
var containers []corev1.Container
ctn := corev1.Container{
Name: fmt.Sprintf("%s-ctn", w.Name()),
Image: "gcr.io/kubernetes-e2e-test-images/node-perf/tf-wide-deep-amd64:1.0",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("15000m"),
corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("16Gi"),
},
Limits: corev1.ResourceList{
corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("15000m"),
corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("16Gi"),
},
},
Command: []string{"/bin/sh"},
Args: []string{"-c", "python ./data_download.py && time -p python ./wide_deep.py --model_type=wide_deep --train_epochs=300 --epochs_between_evals=300 --batch_size=32561"},
}
containers = append(containers, ctn)
return corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyNever,
Containers: containers,
}
}
func (w tfWideDeepWorkload) Timeout() time.Duration {
return 15 * time.Minute
}
func (w tfWideDeepWorkload) KubeletConfig(oldCfg *kubeletconfig.KubeletConfiguration) (newCfg *kubeletconfig.KubeletConfiguration, err error) {
// Enable CPU Manager in Kubelet with static policy.
newCfg = oldCfg.DeepCopy()
// Set the CPU Manager policy to static.
newCfg.CPUManagerPolicy = string(cpumanager.PolicyStatic)
// Set the CPU Manager reconcile period to 10 second.
newCfg.CPUManagerReconcilePeriod = metav1.Duration{Duration: 10 * time.Second}
// The Kubelet panics if either kube-reserved or system-reserved is not set
// when static CPU Manager is enabled. Set cpu in kube-reserved > 0 so that
// kubelet doesn't panic.
if newCfg.KubeReserved == nil {
newCfg.KubeReserved = map[string]string{}
}
if _, ok := newCfg.KubeReserved["cpu"]; !ok {
newCfg.KubeReserved["cpu"] = "200m"
}
return newCfg, nil
}
func (w tfWideDeepWorkload) PreTestExec() error {
cmd := "/bin/sh"
args := []string{"-c", "rm -f /var/lib/kubelet/cpu_manager_state"}
err := runCmd(cmd, args)
return err
}
func (w tfWideDeepWorkload) PostTestExec() error {
cmd := "/bin/sh"
args := []string{"-c", "rm -f /var/lib/kubelet/cpu_manager_state"}
err := runCmd(cmd, args)
return err
}
func (w tfWideDeepWorkload) ExtractPerformanceFromLogs(logs string) (perf time.Duration, err error) {
perfLine, err := getMatchingLineFromLog(logs, "real")
if err != nil {
return perf, err
}
perfString := fmt.Sprintf("%ss", strings.TrimSpace(strings.TrimPrefix(perfLine, "real")))
perf, err = time.ParseDuration(perfString)
return perf, err
}

View File

@@ -0,0 +1,45 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package workloads
import (
"fmt"
"os/exec"
"regexp"
"strings"
)
func runCmd(cmd string, args []string) error {
err := exec.Command(cmd, args...).Run()
return err
}
func getMatchingLineFromLog(log string, pattern string) (line string, err error) {
regex, err := regexp.Compile(pattern)
if err != nil {
return line, fmt.Errorf("failed to compile regexp %v: %v", pattern, err)
}
logLines := strings.Split(log, "\n")
for _, line := range logLines {
if regex.MatchString(line) {
return line, nil
}
}
return line, fmt.Errorf("line with pattern %v not found in log", pattern)
}

View File

@@ -0,0 +1,53 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package workloads
import (
"time"
corev1 "k8s.io/api/core/v1"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
)
// NodePerfWorkload provides the necessary information to run a workload for
// node performance testing.
type NodePerfWorkload interface {
// Name of the workload.
Name() string
// PodSpec used to run this workload.
PodSpec() corev1.PodSpec
// Timeout provides the expected time to completion
// for this workload.
Timeout() time.Duration
// KubeletConfig specifies the Kubelet configuration
// required for this workload.
KubeletConfig(old *kubeletconfig.KubeletConfiguration) (new *kubeletconfig.KubeletConfiguration, err error)
// PreTestExec is used for defining logic that needs
// to be run before restarting the Kubelet with the new Kubelet
// configuration required for the workload.
PreTestExec() error
// PostTestExec is used for defining logic that needs
// to be run after the workload has completed.
PostTestExec() error
// ExtractPerformanceFromLogs is used get the performance of the workload
// from pod logs. Currently, we support only performance reported in
// time.Duration format.
ExtractPerformanceFromLogs(logs string) (perf time.Duration, err error)
}
// NodePerfWorkloads is the collection of all node performance testing workloads.
var NodePerfWorkloads = []NodePerfWorkload{npbISWorkload{}, npbEPWorkload{}, tfWideDeepWorkload{}}

View File

@@ -18,10 +18,10 @@ go_library(
],
importpath = "k8s.io/kubernetes/test/e2e_node/remote",
deps = [
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//test/e2e_node/builder:go_default_library",
"//test/utils:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
],
)

View File

@@ -31,7 +31,7 @@ import (
)
const (
systemSpecPath = "test/e2e_node/system/specs"
systemSpecPath = "k8s.io/kubernetes/cmd/kubeadm/app/util/system/specs"
)
// NodeE2ERemote contains the specific functions in the node e2e test suite.

View File

@@ -86,9 +86,9 @@ func NewResourceCollector(interval time.Duration) *ResourceCollector {
// then repeatedly runs collectStats.
func (r *ResourceCollector) Start() {
// Get the cgroup container names for kubelet and runtime
kubeletContainer, err := getContainerNameForProcess(kubeletProcessName, "")
runtimeContainer, err := getContainerNameForProcess(framework.TestContext.ContainerRuntimeProcessName, framework.TestContext.ContainerRuntimePidFile)
if err == nil {
kubeletContainer, err1 := getContainerNameForProcess(kubeletProcessName, "")
runtimeContainer, err2 := getContainerNameForProcess(framework.TestContext.ContainerRuntimeProcessName, framework.TestContext.ContainerRuntimePidFile)
if err1 == nil && err2 == nil {
systemContainers = map[string]string{
stats.SystemContainerKubelet: kubeletContainer,
stats.SystemContainerRuntime: runtimeContainer,
@@ -165,9 +165,6 @@ func (r *ResourceCollector) collectStats(oldStatsMap map[string]*cadvisorapiv2.C
newStats := cStats.Stats[0]
if oldStats, ok := oldStatsMap[name]; ok && oldStats.Timestamp.Before(newStats.Timestamp) {
if oldStats.Timestamp.Equal(newStats.Timestamp) {
continue
}
r.buffers[name] = append(r.buffers[name], computeContainerResourceUsage(name, oldStats, newStats))
}
oldStatsMap[name] = newStats

View File

@@ -33,10 +33,10 @@ import (
var buildDependencies = flag.Bool("build-dependencies", true, "If true, build all dependencies.")
var ginkgoFlags = flag.String("ginkgo-flags", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
var testFlags = flag.String("test-flags", "", "Space-separated list of arguments to pass to node e2e test.")
var systemSpecName = flag.String("system-spec-name", "", "The name of the system spec used for validating the image in the node conformance test. The specs are at test/e2e_node/system/specs/. If unspecified, the default built-in spec (system.DefaultSpec) will be used.")
var systemSpecName = flag.String("system-spec-name", "", "The name of the system spec used for validating the image in the node conformance test. The specs are at k8s.io/kubernetes/cmd/kubeadm/app/util/system/specs/. If unspecified, the default built-in spec (system.DefaultSpec) will be used.")
const (
systemSpecPath = "test/e2e_node/system/specs"
systemSpecPath = "k8s.io/kubernetes/cmd/kubeadm/app/util/system/specs"
)
func main() {

View File

@@ -61,7 +61,7 @@ var buildOnly = flag.Bool("build-only", false, "If true, build e2e_node_test.tar
var instanceMetadata = flag.String("instance-metadata", "", "key/value metadata for instances separated by '=' or '<', 'k=v' means the key is 'k' and the value is 'v'; 'k<p' means the key is 'k' and the value is extracted from the local path 'p', e.g. k1=v1,k2<p2")
var gubernator = flag.Bool("gubernator", false, "If true, output Gubernator link to view logs")
var ginkgoFlags = flag.String("ginkgo-flags", "", "Passed to ginkgo to specify additional flags such as --skip=.")
var systemSpecName = flag.String("system-spec-name", "", "The name of the system spec used for validating the image in the node conformance test. The specs are at test/e2e_node/system/specs/. If unspecified, the default built-in spec (system.DefaultSpec) will be used.")
var systemSpecName = flag.String("system-spec-name", "", "The name of the system spec used for validating the image in the node conformance test. The specs are at k8s.io/kubernetes/cmd/kubeadm/app/util/system/specs/. If unspecified, the default built-in spec (system.DefaultSpec) will be used.")
// envs is the type used to collect all node envs. The key is the env name,
// and the value is the env value

View File

@@ -20,226 +20,23 @@ import (
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"time"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/kubernetes/pkg/kubelet/images"
"k8s.io/kubernetes/test/e2e/common"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e_node/services"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
gomegatypes "github.com/onsi/gomega/types"
)
const (
consistentCheckTimeout = time.Second * 5
retryTimeout = time.Minute * 5
pollInterval = time.Second * 1
)
var _ = framework.KubeDescribe("Container Runtime Conformance Test", func() {
f := framework.NewDefaultFramework("runtime-conformance")
Describe("container runtime conformance blackbox test", func() {
Context("when starting a container that exits", func() {
framework.ConformanceIt("it should run with the expected status [NodeConformance]", func() {
restartCountVolumeName := "restart-count"
restartCountVolumePath := "/restart-count"
testContainer := v1.Container{
Image: busyboxImage,
VolumeMounts: []v1.VolumeMount{
{
MountPath: restartCountVolumePath,
Name: restartCountVolumeName,
},
},
}
testVolumes := []v1.Volume{
{
Name: restartCountVolumeName,
VolumeSource: v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{Medium: v1.StorageMediumMemory},
},
},
}
testCases := []struct {
Name string
RestartPolicy v1.RestartPolicy
Phase v1.PodPhase
State ContainerState
RestartCount int32
Ready bool
}{
{"terminate-cmd-rpa", v1.RestartPolicyAlways, v1.PodRunning, ContainerStateRunning, 2, true},
{"terminate-cmd-rpof", v1.RestartPolicyOnFailure, v1.PodSucceeded, ContainerStateTerminated, 1, false},
{"terminate-cmd-rpn", v1.RestartPolicyNever, v1.PodFailed, ContainerStateTerminated, 0, false},
}
for _, testCase := range testCases {
// It failed at the 1st run, then succeeded at 2nd run, then run forever
cmdScripts := `
f=%s
count=$(echo 'hello' >> $f ; wc -l $f | awk {'print $1'})
if [ $count -eq 1 ]; then
exit 1
fi
if [ $count -eq 2 ]; then
exit 0
fi
while true; do sleep 1; done
`
tmpCmd := fmt.Sprintf(cmdScripts, path.Join(restartCountVolumePath, "restartCount"))
testContainer.Name = testCase.Name
testContainer.Command = []string{"sh", "-c", tmpCmd}
terminateContainer := ConformanceContainer{
PodClient: f.PodClient(),
Container: testContainer,
RestartPolicy: testCase.RestartPolicy,
Volumes: testVolumes,
PodSecurityContext: &v1.PodSecurityContext{
SELinuxOptions: &v1.SELinuxOptions{
Level: "s0",
},
},
}
terminateContainer.Create()
defer terminateContainer.Delete()
By("it should get the expected 'RestartCount'")
Eventually(func() (int32, error) {
status, err := terminateContainer.GetStatus()
return status.RestartCount, err
}, retryTimeout, pollInterval).Should(Equal(testCase.RestartCount))
By("it should get the expected 'Phase'")
Eventually(terminateContainer.GetPhase, retryTimeout, pollInterval).Should(Equal(testCase.Phase))
By("it should get the expected 'Ready' condition")
Expect(terminateContainer.IsReady()).Should(Equal(testCase.Ready))
status, err := terminateContainer.GetStatus()
Expect(err).ShouldNot(HaveOccurred())
By("it should get the expected 'State'")
Expect(GetContainerState(status.State)).To(Equal(testCase.State))
By("it should be possible to delete [Conformance][NodeConformance]")
Expect(terminateContainer.Delete()).To(Succeed())
Eventually(terminateContainer.Present, retryTimeout, pollInterval).Should(BeFalse())
}
})
rootUser := int64(0)
nonRootUser := int64(10000)
for _, testCase := range []struct {
name string
container v1.Container
phase v1.PodPhase
message gomegatypes.GomegaMatcher
}{
{
name: "if TerminationMessagePath is set [Conformance][NodeConformance]",
container: v1.Container{
Image: busyboxImage,
Command: []string{"/bin/sh", "-c"},
Args: []string{"/bin/echo -n DONE > /dev/termination-log"},
TerminationMessagePath: "/dev/termination-log",
SecurityContext: &v1.SecurityContext{
RunAsUser: &rootUser,
},
},
phase: v1.PodSucceeded,
message: Equal("DONE"),
},
{
name: "if TerminationMessagePath is set as non-root user and at a non-default path [Conformance][NodeConformance]",
container: v1.Container{
Image: busyboxImage,
Command: []string{"/bin/sh", "-c"},
Args: []string{"/bin/echo -n DONE > /dev/termination-custom-log"},
TerminationMessagePath: "/dev/termination-custom-log",
SecurityContext: &v1.SecurityContext{
RunAsUser: &nonRootUser,
},
},
phase: v1.PodSucceeded,
message: Equal("DONE"),
},
{
name: "from log output if TerminationMessagePolicy FallbackToLogOnError is set [Conformance][NodeConformance]",
container: v1.Container{
Image: busyboxImage,
Command: []string{"/bin/sh", "-c"},
Args: []string{"/bin/echo -n DONE; /bin/false"},
TerminationMessagePath: "/dev/termination-log",
TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError,
},
phase: v1.PodFailed,
message: Equal("DONE\n"),
},
{
name: "as empty when pod succeeds and TerminationMessagePolicy FallbackToLogOnError is set [NodeConformance]",
container: v1.Container{
Image: busyboxImage,
Command: []string{"/bin/sh", "-c"},
Args: []string{"/bin/echo DONE; /bin/true"},
TerminationMessagePath: "/dev/termination-log",
TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError,
},
phase: v1.PodSucceeded,
message: Equal(""),
},
{
name: "from file when pod succeeds and TerminationMessagePolicy FallbackToLogOnError is set [Conformance][NodeConformance]",
container: v1.Container{
Image: busyboxImage,
Command: []string{"/bin/sh", "-c"},
Args: []string{"/bin/echo -n OK > /dev/termination-log; /bin/echo DONE; /bin/true"},
TerminationMessagePath: "/dev/termination-log",
TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError,
},
phase: v1.PodSucceeded,
message: Equal("OK"),
},
} {
It(fmt.Sprintf("should report termination message %s", testCase.name), func() {
testCase.container.Name = "termination-message-container"
c := ConformanceContainer{
PodClient: f.PodClient(),
Container: testCase.container,
RestartPolicy: v1.RestartPolicyNever,
}
By("create the container")
c.Create()
defer c.Delete()
By(fmt.Sprintf("wait for the container to reach %s", testCase.phase))
Eventually(c.GetPhase, retryTimeout, pollInterval).Should(Equal(testCase.phase))
By("get the container status")
status, err := c.GetStatus()
Expect(err).NotTo(HaveOccurred())
By("the container should be terminated")
Expect(GetContainerState(status.State)).To(Equal(ContainerStateTerminated))
By("the termination message should be set")
Expect(status.State.Terminated.Message).Should(testCase.message)
By("delete the container")
Expect(c.Delete()).To(Succeed())
})
}
})
Context("when running a container with a new image", func() {
// The service account only has pull permission
@@ -252,71 +49,27 @@ while true; do sleep 1; done
}
}
}`
secret := &v1.Secret{
Data: map[string][]byte{v1.DockerConfigJsonKey: []byte(auth)},
Type: v1.SecretTypeDockerConfigJson,
}
// The following images are not added into NodeImageWhiteList, because this test is
// testing image pulling, these images don't need to be prepulled. The ImagePullPolicy
// is v1.PullAlways, so it won't be blocked by framework image white list check.
for _, testCase := range []struct {
description string
image string
secret bool
credentialProvider bool
phase v1.PodPhase
waiting bool
description string
image string
phase v1.PodPhase
waiting bool
}{
{
description: "should not be able to pull image from invalid registry",
image: "invalid.com/invalid/alpine:3.1",
phase: v1.PodPending,
waiting: true,
},
{
description: "should not be able to pull non-existing image from gcr.io",
image: "k8s.gcr.io/invalid-image:invalid-tag",
phase: v1.PodPending,
waiting: true,
},
{
description: "should be able to pull image from gcr.io",
image: "k8s.gcr.io/alpine-with-bash:1.0",
phase: v1.PodRunning,
waiting: false,
},
{
description: "should be able to pull image from docker hub",
image: "alpine:3.1",
phase: v1.PodRunning,
waiting: false,
},
{
description: "should not be able to pull from private registry without secret",
description: "should be able to pull from private registry with credential provider",
image: "gcr.io/authenticated-image-pulling/alpine:3.1",
phase: v1.PodPending,
waiting: true,
},
{
description: "should be able to pull from private registry with secret",
image: "gcr.io/authenticated-image-pulling/alpine:3.1",
secret: true,
phase: v1.PodRunning,
waiting: false,
},
{
description: "should be able to pull from private registry with credential provider",
image: "gcr.io/authenticated-image-pulling/alpine:3.1",
credentialProvider: true,
phase: v1.PodRunning,
waiting: false,
},
} {
testCase := testCase
It(testCase.description+" [Conformance][NodeConformance]", func() {
It(testCase.description+" [NodeConformance]", func() {
name := "image-pull-test"
command := []string{"/bin/sh", "-c", "while true; do sleep 1; done"}
container := ConformanceContainer{
container := common.ConformanceContainer{
PodClient: f.PodClient(),
Container: v1.Container{
Name: name,
@@ -327,20 +80,12 @@ while true; do sleep 1; done
},
RestartPolicy: v1.RestartPolicyNever,
}
if testCase.secret {
secret.Name = "image-pull-secret-" + string(uuid.NewUUID())
By("create image pull secret")
_, err := f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(secret)
Expect(err).NotTo(HaveOccurred())
defer f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Delete(secret.Name, nil)
container.ImagePullSecrets = []string{secret.Name}
}
if testCase.credentialProvider {
configFile := filepath.Join(services.KubeletRootDirectory, "config.json")
err := ioutil.WriteFile(configFile, []byte(auth), 0644)
Expect(err).NotTo(HaveOccurred())
defer os.Remove(configFile)
}
configFile := filepath.Join(services.KubeletRootDirectory, "config.json")
err := ioutil.WriteFile(configFile, []byte(auth), 0644)
Expect(err).NotTo(HaveOccurred())
defer os.Remove(configFile)
// checkContainerStatus checks whether the container status matches expectation.
checkContainerStatus := func() error {
status, err := container.GetStatus()
@@ -354,13 +99,13 @@ while true; do sleep 1; done
if !testCase.waiting {
if status.State.Running == nil {
return fmt.Errorf("expected container state: Running, got: %q",
GetContainerState(status.State))
common.GetContainerState(status.State))
}
}
if testCase.waiting {
if status.State.Waiting == nil {
return fmt.Errorf("expected container state: Waiting, got: %q",
GetContainerState(status.State))
common.GetContainerState(status.State))
}
reason := status.State.Waiting.Reason
if reason != images.ErrImagePull.Error() &&
@@ -386,7 +131,7 @@ while true; do sleep 1; done
By("create the container")
container.Create()
By("check the container status")
for start := time.Now(); time.Since(start) < retryTimeout; time.Sleep(pollInterval) {
for start := time.Now(); time.Since(start) < common.ContainerStatusRetryTimeout; time.Sleep(common.ContainerStatusPollInterval) {
if err = checkContainerStatus(); err == nil {
break
}

View File

@@ -50,12 +50,12 @@ var _ = framework.KubeDescribe("Security Context", func() {
Containers: []v1.Container{
{
Name: "test-container-1",
Image: "busybox",
Image: imageutils.GetE2EImage(imageutils.BusyBox),
Command: []string{"/bin/top"},
},
{
Name: "test-container-2",
Image: "busybox",
Image: imageutils.GetE2EImage(imageutils.BusyBox),
Command: []string{"/bin/sleep"},
Args: []string{"10000"},
},
@@ -91,12 +91,12 @@ var _ = framework.KubeDescribe("Security Context", func() {
Containers: []v1.Container{
{
Name: "test-container-1",
Image: "busybox",
Image: imageutils.GetE2EImage(imageutils.BusyBox),
Command: []string{"/bin/top"},
},
{
Name: "test-container-2",
Image: "busybox",
Image: imageutils.GetE2EImage(imageutils.BusyBox),
Command: []string{"/bin/sleep"},
Args: []string{"10000"},
},
@@ -146,7 +146,7 @@ var _ = framework.KubeDescribe("Security Context", func() {
BeforeEach(func() {
nginxPodName := "nginx-hostpid-" + string(uuid.NewUUID())
podClient.CreateSync(makeHostPidPod(nginxPodName,
imageutils.GetE2EImage(imageutils.NginxSlim),
imageutils.GetE2EImage(imageutils.Nginx),
nil,
true,
))
@@ -350,177 +350,6 @@ var _ = framework.KubeDescribe("Security Context", func() {
})
})
Context("When creating a container with runAsUser", func() {
makeUserPod := func(podName, image string, command []string, userid int64) *v1.Pod {
return &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
},
Spec: v1.PodSpec{
RestartPolicy: v1.RestartPolicyNever,
Containers: []v1.Container{
{
Image: image,
Name: podName,
Command: command,
SecurityContext: &v1.SecurityContext{
RunAsUser: &userid,
},
},
},
},
}
}
createAndWaitUserPod := func(userid int64) {
podName := fmt.Sprintf("busybox-user-%d-%s", userid, uuid.NewUUID())
podClient.Create(makeUserPod(podName,
busyboxImage,
[]string{"sh", "-c", fmt.Sprintf("test $(id -u) -eq %d", userid)},
userid,
))
podClient.WaitForSuccess(podName, framework.PodStartTimeout)
}
It("should run the container with uid 65534 [NodeConformance]", func() {
createAndWaitUserPod(65534)
})
It("should run the container with uid 0 [NodeConformance]", func() {
createAndWaitUserPod(0)
})
})
Context("When creating a pod with readOnlyRootFilesystem", func() {
makeUserPod := func(podName, image string, command []string, readOnlyRootFilesystem bool) *v1.Pod {
return &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
},
Spec: v1.PodSpec{
RestartPolicy: v1.RestartPolicyNever,
Containers: []v1.Container{
{
Image: image,
Name: podName,
Command: command,
SecurityContext: &v1.SecurityContext{
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
},
},
},
},
}
}
createAndWaitUserPod := func(readOnlyRootFilesystem bool) string {
podName := fmt.Sprintf("busybox-readonly-%v-%s", readOnlyRootFilesystem, uuid.NewUUID())
podClient.Create(makeUserPod(podName,
"busybox",
[]string{"sh", "-c", "touch checkfile"},
readOnlyRootFilesystem,
))
if readOnlyRootFilesystem {
podClient.WaitForFailure(podName, framework.PodStartTimeout)
} else {
podClient.WaitForSuccess(podName, framework.PodStartTimeout)
}
return podName
}
It("should run the container with readonly rootfs when readOnlyRootFilesystem=true [NodeConformance]", func() {
createAndWaitUserPod(true)
})
It("should run the container with writable rootfs when readOnlyRootFilesystem=false [NodeConformance]", func() {
createAndWaitUserPod(false)
})
})
Context("when creating containers with AllowPrivilegeEscalation", func() {
BeforeEach(func() {
if framework.TestContext.ContainerRuntime == "docker" {
isSupported, err := isDockerNoNewPrivilegesSupported()
framework.ExpectNoError(err)
if !isSupported {
framework.Skipf("Skipping because no_new_privs is not supported in this docker")
}
// It turns out SELinux policy in RHEL 7 does not play well with
// the "NoNewPrivileges" flag. So let's skip this test when running
// with SELinux support enabled.
//
// TODO(filbranden): Remove this after the fix for
// https://github.com/projectatomic/container-selinux/issues/45
// has been backported to RHEL 7 (expected on RHEL 7.5)
selinuxEnabled, err := isDockerSELinuxSupportEnabled()
framework.ExpectNoError(err)
if selinuxEnabled {
framework.Skipf("Skipping because Docker daemon is running with SELinux support enabled")
}
}
})
makeAllowPrivilegeEscalationPod := func(podName string, allowPrivilegeEscalation *bool, uid int64) *v1.Pod {
return &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
},
Spec: v1.PodSpec{
RestartPolicy: v1.RestartPolicyNever,
Containers: []v1.Container{
{
Image: imageutils.GetE2EImage(imageutils.Nonewprivs),
Name: podName,
SecurityContext: &v1.SecurityContext{
AllowPrivilegeEscalation: allowPrivilegeEscalation,
RunAsUser: &uid,
},
},
},
},
}
}
createAndMatchOutput := func(podName, output string, allowPrivilegeEscalation *bool, uid int64) error {
podClient.Create(makeAllowPrivilegeEscalationPod(podName,
allowPrivilegeEscalation,
uid,
))
podClient.WaitForSuccess(podName, framework.PodStartTimeout)
if err := podClient.MatchContainerOutput(podName, podName, output); err != nil {
return err
}
return nil
}
It("should allow privilege escalation when not explicitly set and uid != 0 [NodeConformance]", func() {
podName := "alpine-nnp-nil-" + string(uuid.NewUUID())
if err := createAndMatchOutput(podName, "Effective uid: 0", nil, 1000); err != nil {
framework.Failf("Match output for pod %q failed: %v", podName, err)
}
})
It("should not allow privilege escalation when false [NodeConformance]", func() {
podName := "alpine-nnp-false-" + string(uuid.NewUUID())
apeFalse := false
if err := createAndMatchOutput(podName, "Effective uid: 1000", &apeFalse, 1000); err != nil {
framework.Failf("Match output for pod %q failed: %v", podName, err)
}
})
It("should allow privilege escalation when true [NodeConformance]", func() {
podName := "alpine-nnp-true-" + string(uuid.NewUUID())
apeTrue := true
if err := createAndMatchOutput(podName, "Effective uid: 0", &apeTrue, 1000); err != nil {
framework.Failf("Match output for pod %q failed: %v", podName, err)
}
})
})
Context("When creating a pod with privileged", func() {
makeUserPod := func(podName, image string, command []string, privileged bool) *v1.Pod {
return &v1.Pod{
@@ -549,9 +378,7 @@ var _ = framework.KubeDescribe("Security Context", func() {
[]string{"sh", "-c", "ip link add dummy0 type dummy || true"},
privileged,
))
podClient.WaitForSuccess(podName, framework.PodStartTimeout)
return podName
}
@@ -567,19 +394,5 @@ var _ = framework.KubeDescribe("Security Context", func() {
framework.Failf("privileged container should be able to create dummy device")
}
})
It("should run the container as unprivileged when false [NodeConformance]", func() {
podName := createAndWaitUserPod(false)
logs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, podName, podName)
if err != nil {
framework.Failf("GetPodLogs for pod %q failed: %v", podName, err)
}
framework.Logf("Got logs for pod %q: %q", podName, logs)
if !strings.Contains(logs, "Operation not permitted") {
framework.Failf("unprivileged container shouldn't be able to create dummy device")
}
})
})
})

View File

@@ -25,9 +25,17 @@ go_library(
"//cmd/kubelet/app/options:go_default_library",
"//pkg/controller/namespace:go_default_library",
"//pkg/features:go_default_library",
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library",
"//pkg/kubelet/apis/config:go_default_library",
"//pkg/kubelet/kubeletconfig/util/codec:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/util/flag:go_default_library",
"//staging/src/k8s.io/client-go/dynamic:go_default_library",
"//staging/src/k8s.io/client-go/informers:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/rest:go_default_library",
"//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library",
"//test/e2e/framework:go_default_library",
"//test/e2e_node/builder:go_default_library",
"//test/e2e_node/remote:go_default_library",
@@ -38,14 +46,6 @@ go_library(
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/kardianos/osext:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
"//vendor/k8s.io/client-go/dynamic:go_default_library",
"//vendor/k8s.io/client-go/informers:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/rest:go_default_library",
],
)

View File

@@ -32,10 +32,10 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilflag "k8s.io/apiserver/pkg/util/flag"
kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
"k8s.io/kubernetes/cmd/kubelet/app/options"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
kubeletconfigcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e_node/builder"

View File

@@ -1,95 +0,0 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = [
"cgroup_validator.go",
"docker_validator.go",
"kernel_validator.go",
"kernel_validator_helper.go",
"os_validator.go",
"package_validator.go",
"report.go",
"types.go",
"validators.go",
] + select({
"@io_bazel_rules_go//go/platform:android": [
"types_unix.go",
],
"@io_bazel_rules_go//go/platform:darwin": [
"types_unix.go",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"types_unix.go",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"types_unix.go",
],
"@io_bazel_rules_go//go/platform:linux": [
"types_unix.go",
],
"@io_bazel_rules_go//go/platform:nacl": [
"types_unix.go",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"types_unix.go",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"types_unix.go",
],
"@io_bazel_rules_go//go/platform:plan9": [
"types_unix.go",
],
"@io_bazel_rules_go//go/platform:solaris": [
"types_unix.go",
],
"@io_bazel_rules_go//go/platform:windows": [
"types_windows.go",
],
"//conditions:default": [],
}),
importpath = "k8s.io/kubernetes/test/e2e_node/system",
deps = [
"//vendor/github.com/blang/semver:go_default_library",
"//vendor/github.com/docker/docker/api/types:go_default_library",
"//vendor/github.com/docker/docker/client:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = [
"cgroup_validator_test.go",
"docker_validator_test.go",
"kernel_validator_test.go",
"os_validator_test.go",
"package_validator_test.go",
],
embed = [":go_default_library"],
tags = ["e2e"],
deps = [
"//vendor/github.com/docker/docker/api/types:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View File

@@ -1,95 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"bufio"
"fmt"
"os"
"strings"
)
var _ Validator = &CgroupsValidator{}
type CgroupsValidator struct {
Reporter Reporter
}
func (c *CgroupsValidator) Name() string {
return "cgroups"
}
const (
cgroupsConfigPrefix = "CGROUPS_"
)
func (c *CgroupsValidator) Validate(spec SysSpec) (error, error) {
subsystems, err := c.getCgroupSubsystems()
if err != nil {
return nil, fmt.Errorf("failed to get cgroup subsystems: %v", err)
}
return nil, c.validateCgroupSubsystems(spec.Cgroups, subsystems)
}
func (c *CgroupsValidator) validateCgroupSubsystems(cgroupSpec, subsystems []string) error {
missing := []string{}
for _, cgroup := range cgroupSpec {
found := false
for _, subsystem := range subsystems {
if cgroup == subsystem {
found = true
break
}
}
item := cgroupsConfigPrefix + strings.ToUpper(cgroup)
if found {
c.Reporter.Report(item, "enabled", good)
} else {
c.Reporter.Report(item, "missing", bad)
missing = append(missing, cgroup)
}
}
if len(missing) > 0 {
return fmt.Errorf("missing cgroups: %s", strings.Join(missing, " "))
}
return nil
}
func (c *CgroupsValidator) getCgroupSubsystems() ([]string, error) {
f, err := os.Open("/proc/cgroups")
if err != nil {
return nil, err
}
defer f.Close()
subsystems := []string{}
s := bufio.NewScanner(f)
for s.Scan() {
if err := s.Err(); err != nil {
return nil, err
}
text := s.Text()
if text[0] != '#' {
parts := strings.Fields(text)
if len(parts) >= 4 && parts[3] != "0" {
subsystems = append(subsystems, parts[0])
}
}
}
return subsystems, nil
}

View File

@@ -1,56 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestValidateCgroupSubsystem(t *testing.T) {
v := &CgroupsValidator{
Reporter: DefaultReporter,
}
cgroupSpec := []string{"system1", "system2"}
for desc, test := range map[string]struct {
cgroupSpec []string
subsystems []string
err bool
}{
"missing cgroup subsystem should report error": {
subsystems: []string{"system1"},
err: true,
},
"extra cgroup subsystems should not report error": {
subsystems: []string{"system1", "system2", "system3"},
err: false,
},
"subsystems the same with spec should not report error": {
subsystems: []string{"system1", "system2"},
err: false,
},
} {
err := v.validateCgroupSubsystems(cgroupSpec, test.subsystems)
if !test.err {
assert.Nil(t, err, "%q: Expect error not to occur with cgroup", desc)
} else {
assert.NotNil(t, err, "%q: Expect error to occur with docker info", desc)
}
}
}

View File

@@ -1,100 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"context"
"fmt"
"regexp"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
)
var _ Validator = &DockerValidator{}
// DockerValidator validates docker configuration.
type DockerValidator struct {
Reporter Reporter
}
func (d *DockerValidator) Name() string {
return "docker"
}
const (
dockerConfigPrefix = "DOCKER_"
maxDockerValidatedVersion = "17.03"
)
// TODO(random-liu): Add more validating items.
func (d *DockerValidator) Validate(spec SysSpec) (error, error) {
if spec.RuntimeSpec.DockerSpec == nil {
// If DockerSpec is not specified, assume current runtime is not
// docker, skip the docker configuration validation.
return nil, nil
}
c, err := client.NewClient(dockerEndpoint, "", nil, nil)
if err != nil {
return nil, fmt.Errorf("failed to create docker client: %v", err)
}
info, err := c.Info(context.Background())
if err != nil {
return nil, fmt.Errorf("failed to get docker info: %v", err)
}
return d.validateDockerInfo(spec.RuntimeSpec.DockerSpec, info)
}
func (d *DockerValidator) validateDockerInfo(spec *DockerSpec, info types.Info) (error, error) {
// Validate docker version.
matched := false
for _, v := range spec.Version {
r := regexp.MustCompile(v)
if r.MatchString(info.ServerVersion) {
d.Reporter.Report(dockerConfigPrefix+"VERSION", info.ServerVersion, good)
matched = true
}
}
if !matched {
// If it's of the new Docker version scheme but didn't match above, it
// must be a newer version than the most recently validated one.
ver := `\d{2}\.\d+\.\d+-[a-z]{2}`
r := regexp.MustCompile(ver)
if r.MatchString(info.ServerVersion) {
d.Reporter.Report(dockerConfigPrefix+"VERSION", info.ServerVersion, good)
w := fmt.Errorf(
"docker version is greater than the most recently validated version. Docker version: %s. Max validated version: %s",
info.ServerVersion,
maxDockerValidatedVersion,
)
return w, nil
}
d.Reporter.Report(dockerConfigPrefix+"VERSION", info.ServerVersion, bad)
return nil, fmt.Errorf("unsupported docker version: %s", info.ServerVersion)
}
// Validate graph driver.
item := dockerConfigPrefix + "GRAPH_DRIVER"
for _, gd := range spec.GraphDriver {
if info.Driver == gd {
d.Reporter.Report(item, info.Driver, good)
return nil, nil
}
}
d.Reporter.Report(item, info.Driver, bad)
return nil, fmt.Errorf("unsupported graph driver: %s", info.Driver)
}

View File

@@ -1,88 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"testing"
"github.com/docker/docker/api/types"
"github.com/stretchr/testify/assert"
)
func TestValidateDockerInfo(t *testing.T) {
v := &DockerValidator{
Reporter: DefaultReporter,
}
spec := &DockerSpec{
Version: []string{`1\.1[1-3]\..*`, `17\.03\..*`}, // Requires [1.11, 17.03].
GraphDriver: []string{"driver_1", "driver_2"},
}
for _, test := range []struct {
info types.Info
err bool
warn bool
}{
{
info: types.Info{Driver: "driver_1", ServerVersion: "1.10.1"},
err: true,
warn: false,
},
{
info: types.Info{Driver: "bad_driver", ServerVersion: "1.11.1"},
err: true,
warn: false,
},
{
info: types.Info{Driver: "driver_1", ServerVersion: "1.11.1"},
err: false,
warn: false,
},
{
info: types.Info{Driver: "driver_2", ServerVersion: "1.12.1"},
err: false,
warn: false,
},
{
info: types.Info{Driver: "driver_2", ServerVersion: "1.13.1"},
err: false,
warn: false,
},
{
info: types.Info{Driver: "driver_2", ServerVersion: "17.03.0-ce"},
err: false,
warn: false,
},
{
info: types.Info{Driver: "driver_2", ServerVersion: "17.06.0-ce"},
err: false,
warn: true,
},
} {
warn, err := v.validateDockerInfo(spec, test.info)
if !test.err {
assert.Nil(t, err, "Expect error not to occur with docker info %+v", test.info)
} else {
assert.NotNil(t, err, "Expect error to occur with docker info %+v", test.info)
}
if !test.warn {
assert.Nil(t, warn, "Expect error not to occur with docker info %+v", test.info)
} else {
assert.NotNil(t, warn, "Expect error to occur with docker info %+v", test.info)
}
}
}

View File

@@ -1,259 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"bufio"
"bytes"
"compress/gzip"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"github.com/golang/glog"
"k8s.io/apimachinery/pkg/util/errors"
)
var _ Validator = &KernelValidator{}
// KernelValidator validates kernel. Currently only validate kernel version
// and kernel configuration.
type KernelValidator struct {
kernelRelease string
Reporter Reporter
}
func (k *KernelValidator) Name() string {
return "kernel"
}
// kConfigOption is the possible kernel config option.
type kConfigOption string
const (
builtIn kConfigOption = "y"
asModule kConfigOption = "m"
leftOut kConfigOption = "n"
// validKConfigRegex is the regex matching kernel configuration line.
validKConfigRegex = "^CONFIG_[A-Z0-9_]+=[myn]"
// kConfigPrefix is the prefix of kernel configuration.
kConfigPrefix = "CONFIG_"
)
func (k *KernelValidator) Validate(spec SysSpec) (error, error) {
helper := KernelValidatorHelperImpl{}
release, err := helper.GetKernelReleaseVersion()
if err != nil {
return nil, fmt.Errorf("failed to get kernel release: %v", err)
}
k.kernelRelease = release
var errs []error
errs = append(errs, k.validateKernelVersion(spec.KernelSpec))
// only validate kernel config when necessary (currently no kernel config for windows)
if len(spec.KernelSpec.Required) > 0 || len(spec.KernelSpec.Forbidden) > 0 || len(spec.KernelSpec.Optional) > 0 {
errs = append(errs, k.validateKernelConfig(spec.KernelSpec))
}
return nil, errors.NewAggregate(errs)
}
// validateKernelVersion validates the kernel version.
func (k *KernelValidator) validateKernelVersion(kSpec KernelSpec) error {
glog.Infof("Validating kernel version")
versionRegexps := kSpec.Versions
for _, versionRegexp := range versionRegexps {
r := regexp.MustCompile(versionRegexp)
if r.MatchString(k.kernelRelease) {
k.Reporter.Report("KERNEL_VERSION", k.kernelRelease, good)
return nil
}
}
k.Reporter.Report("KERNEL_VERSION", k.kernelRelease, bad)
return fmt.Errorf("unsupported kernel release: %s", k.kernelRelease)
}
// validateKernelConfig validates the kernel configurations.
func (k *KernelValidator) validateKernelConfig(kSpec KernelSpec) error {
glog.Infof("Validating kernel config")
allConfig, err := k.getKernelConfig()
if err != nil {
return fmt.Errorf("failed to parse kernel config: %v", err)
}
return k.validateCachedKernelConfig(allConfig, kSpec)
}
// validateCachedKernelConfig validates the kernel confgiurations cached in internal data type.
func (k *KernelValidator) validateCachedKernelConfig(allConfig map[string]kConfigOption, kSpec KernelSpec) error {
badConfigs := []string{}
// reportAndRecord is a helper function to record bad config when
// report.
reportAndRecord := func(name, msg, desc string, result ValidationResultType) {
if result == bad {
badConfigs = append(badConfigs, name)
}
// report description when the config is bad or warn.
if result != good && desc != "" {
msg = msg + " - " + desc
}
k.Reporter.Report(name, msg, result)
}
const (
required = iota
optional
forbidden
)
validateOpt := func(config KernelConfig, expect int) {
var found, missing ValidationResultType
switch expect {
case required:
found, missing = good, bad
case optional:
found, missing = good, warn
case forbidden:
found, missing = bad, good
}
var name string
var opt kConfigOption
var ok bool
for _, name = range append([]string{config.Name}, config.Aliases...) {
name = kConfigPrefix + name
if opt, ok = allConfig[name]; ok {
break
}
}
if !ok {
reportAndRecord(name, "not set", config.Description, missing)
return
}
switch opt {
case builtIn:
reportAndRecord(name, "enabled", config.Description, found)
case asModule:
reportAndRecord(name, "enabled (as module)", config.Description, found)
case leftOut:
reportAndRecord(name, "disabled", config.Description, missing)
default:
reportAndRecord(name, fmt.Sprintf("unknown option: %s", opt), config.Description, missing)
}
}
for _, config := range kSpec.Required {
validateOpt(config, required)
}
for _, config := range kSpec.Optional {
validateOpt(config, optional)
}
for _, config := range kSpec.Forbidden {
validateOpt(config, forbidden)
}
if len(badConfigs) > 0 {
return fmt.Errorf("unexpected kernel config: %s", strings.Join(badConfigs, " "))
}
return nil
}
// getKernelConfigReader search kernel config file in a predefined list. Once the kernel config
// file is found it will read the configurations into a byte buffer and return. If the kernel
// config file is not found, it will try to load kernel config module and retry again.
func (k *KernelValidator) getKernelConfigReader() (io.Reader, error) {
possibePaths := []string{
"/proc/config.gz",
"/boot/config-" + k.kernelRelease,
"/usr/src/linux-" + k.kernelRelease + "/.config",
"/usr/src/linux/.config",
}
configsModule := "configs"
modprobeCmd := "modprobe"
// loadModule indicates whether we've tried to load kernel config module ourselves.
loadModule := false
for {
for _, path := range possibePaths {
_, err := os.Stat(path)
if err != nil {
continue
}
// Buffer the whole file, so that we can close the file and unload
// kernel config module in this function.
b, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
var r io.Reader
r = bytes.NewReader(b)
// This is a gzip file (config.gz), unzip it.
if filepath.Ext(path) == ".gz" {
r, err = gzip.NewReader(r)
if err != nil {
return nil, err
}
}
return r, nil
}
// If we've tried to load kernel config module, break and return error.
if loadModule {
break
}
// If the kernel config file is not found, try to load the kernel
// config module and check again.
output, err := exec.Command(modprobeCmd, configsModule).CombinedOutput()
if err != nil {
return nil, fmt.Errorf("unable to load kernel module %q: output - %q, err - %v",
configsModule, output, err)
}
// Unload the kernel config module to make sure the validation have no side effect.
defer exec.Command(modprobeCmd, "-r", configsModule).Run()
loadModule = true
}
return nil, fmt.Errorf("no config path in %v is available", possibePaths)
}
// getKernelConfig gets kernel config from kernel config file and convert kernel config to internal type.
func (k *KernelValidator) getKernelConfig() (map[string]kConfigOption, error) {
r, err := k.getKernelConfigReader()
if err != nil {
return nil, err
}
return k.parseKernelConfig(r)
}
// parseKernelConfig converts kernel config to internal type.
func (k *KernelValidator) parseKernelConfig(r io.Reader) (map[string]kConfigOption, error) {
config := map[string]kConfigOption{}
regex := regexp.MustCompile(validKConfigRegex)
s := bufio.NewScanner(r)
for s.Scan() {
if err := s.Err(); err != nil {
return nil, err
}
line := strings.TrimSpace(s.Text())
if !regex.MatchString(line) {
continue
}
fields := strings.Split(line, "=")
if len(fields) != 2 {
glog.Errorf("Unexpected fields number in config %q", line)
continue
}
config[fields[0]] = kConfigOption(fields[1])
}
return config, nil
}

View File

@@ -1,23 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
// KernelValidatorHelper is an interface intended to help with os specific kernel validation
type KernelValidatorHelper interface {
// GetKernelReleaseVersion gets the current kernel release version of the system
GetKernelReleaseVersion() (string, error)
}

View File

@@ -1,197 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"bytes"
"testing"
"github.com/stretchr/testify/assert"
)
func TestValidateKernelVersion(t *testing.T) {
v := &KernelValidator{
Reporter: DefaultReporter,
}
// Currently, testRegex is align with DefaultSysSpec.KernelVersion, but in the future
// they may be different.
// This is fine, because the test mainly tests the kernel version validation logic,
// not the DefaultSysSpec. The DefaultSysSpec should be tested with node e2e.
testRegex := []string{`3\.[1-9][0-9].*`, `4\..*`}
for _, test := range []struct {
version string
err bool
}{
// first version regex matches
{
version: "3.19.9-99-test",
err: false,
},
// one of version regexes matches
{
version: "4.4.14+",
err: false,
},
// no version regex matches
{
version: "2.0.0",
err: true,
},
{
version: "5.0.0",
err: true,
},
{
version: "3.9.0",
err: true,
},
} {
v.kernelRelease = test.version
err := v.validateKernelVersion(KernelSpec{Versions: testRegex})
if !test.err {
assert.Nil(t, err, "Expect error not to occur with kernel version %q", test.version)
} else {
assert.NotNil(t, err, "Expect error to occur with kenrel version %q", test.version)
}
}
}
func TestValidateCachedKernelConfig(t *testing.T) {
v := &KernelValidator{
Reporter: DefaultReporter,
}
testKernelSpec := KernelSpec{
Required: []KernelConfig{{Name: "REQUIRED_1"}, {Name: "REQUIRED_2", Aliases: []string{"ALIASE_REQUIRED_2"}}},
Optional: []KernelConfig{{Name: "OPTIONAL_1"}, {Name: "OPTIONAL_2"}},
Forbidden: []KernelConfig{
{Name: "FORBIDDEN_1", Description: "TEST FORBIDDEN"},
{Name: "FORBIDDEN_2", Aliases: []string{"ALIASE_FORBIDDEN_2"}},
},
}
for c, test := range []struct {
desc string
config map[string]kConfigOption
err bool
}{
{
desc: "meet all required configurations should not report error.",
config: map[string]kConfigOption{
"REQUIRED_1": builtIn,
"REQUIRED_2": asModule,
},
err: false,
},
{
desc: "one required configuration disabled should report error.",
config: map[string]kConfigOption{
"REQUIRED_1": leftOut,
"REQUIRED_2": builtIn,
},
err: true,
},
{
desc: "one required configuration missing should report error.",
config: map[string]kConfigOption{
"REQUIRED_1": builtIn,
},
err: true,
},
{
desc: "alias of required configuration should not report error.",
config: map[string]kConfigOption{
"REQUIRED_1": builtIn,
"ALIASE_REQUIRED_2": asModule,
},
err: false,
},
{
desc: "optional configuration set or not should not report error.",
config: map[string]kConfigOption{
"REQUIRED_1": builtIn,
"REQUIRED_2": asModule,
"OPTIONAL_1": builtIn,
},
err: false,
},
{
desc: "forbidden configuration disabled should not report error.",
config: map[string]kConfigOption{
"REQUIRED_1": builtIn,
"REQUIRED_2": asModule,
"FORBIDDEN_1": leftOut,
},
err: false,
},
{
desc: "forbidden configuration built-in should report error.",
config: map[string]kConfigOption{
"REQUIRED_1": builtIn,
"REQUIRED_2": asModule,
"FORBIDDEN_1": builtIn,
},
err: true,
},
{
desc: "forbidden configuration built as module should report error.",
config: map[string]kConfigOption{
"REQUIRED_1": builtIn,
"REQUIRED_2": asModule,
"FORBIDDEN_1": asModule,
},
err: true,
},
{
desc: "alias of forbidden configuration should report error.",
config: map[string]kConfigOption{
"REQUIRED_1": builtIn,
"REQUIRED_2": asModule,
"ALIASE_FORBIDDEN_2": asModule,
},
err: true,
},
} {
t.Logf("TestCase #%d %s", c, test.desc)
// Add kernel config prefix.
for k, v := range test.config {
delete(test.config, k)
test.config[kConfigPrefix+k] = v
}
err := v.validateCachedKernelConfig(test.config, testKernelSpec)
if !test.err {
assert.Nil(t, err, "Expect error not to occur with kernel config %q", test.config)
} else {
assert.NotNil(t, err, "Expect error to occur with kenrel config %q", test.config)
}
}
}
func TestValidateParseKernelConfig(t *testing.T) {
config := `CONFIG_1=y
CONFIG_2=m
CONFIG_3=n`
expected := map[string]kConfigOption{
"CONFIG_1": builtIn,
"CONFIG_2": asModule,
"CONFIG_3": leftOut,
}
v := &KernelValidator{
Reporter: DefaultReporter,
}
got, err := v.parseKernelConfig(bytes.NewReader([]byte(config)))
assert.Nil(t, err, "Expect error not to occur when parse kernel configuration %q", config)
assert.Equal(t, expected, got)
}

View File

@@ -1,50 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"fmt"
"os/exec"
"strings"
)
var _ Validator = &OSValidator{}
type OSValidator struct {
Reporter Reporter
}
func (o *OSValidator) Name() string {
return "os"
}
func (o *OSValidator) Validate(spec SysSpec) (error, error) {
os, err := exec.Command("uname").CombinedOutput()
if err != nil {
return nil, fmt.Errorf("failed to get os name: %v", err)
}
return nil, o.validateOS(strings.TrimSpace(string(os)), spec.OS)
}
func (o *OSValidator) validateOS(os, specOS string) error {
if os != specOS {
o.Reporter.Report("OS", os, bad)
return fmt.Errorf("unsupported operating system: %s", os)
}
o.Reporter.Report("OS", os, good)
return nil
}

View File

@@ -1,54 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestValidateOS(t *testing.T) {
v := &OSValidator{
Reporter: DefaultReporter,
}
specOS := "Linux"
for _, test := range []struct {
os string
err bool
}{
{
os: "Linux",
err: false,
},
{
os: "Windows",
err: true,
},
{
os: "Darwin",
err: true,
},
} {
err := v.validateOS(test.os, specOS)
if !test.err {
assert.Nil(t, err, "Expect error not to occur with os %q", test.os)
} else {
assert.NotNil(t, err, "Expect error to occur with os %q", test.os)
}
}
}

View File

@@ -1,325 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"fmt"
"io/ioutil"
"os/exec"
"strings"
"k8s.io/apimachinery/pkg/util/errors"
"github.com/blang/semver"
"github.com/golang/glog"
)
// semVerDotsCount is the number of dots in a valid semantic version.
const semVerDotsCount int = 2
// packageManager is an interface that abstracts the basic operations of a
// package manager.
type packageManager interface {
// getPackageVersion returns the version of the package given the
// packageName, or an error if no such package exists.
getPackageVersion(packageName string) (string, error)
}
// newPackageManager returns the package manager on the running machine, and an
// error if no package managers is available.
func newPackageManager() (packageManager, error) {
if m, ok := newDPKG(); ok {
return m, nil
}
return nil, fmt.Errorf("failed to find package manager")
}
// dpkg implements packageManager. It uses "dpkg-query" to retrieve package
// information.
type dpkg struct{}
// newDPKG returns a Debian package manager. It returns (nil, false) if no such
// package manager exists on the running machine.
func newDPKG() (packageManager, bool) {
_, err := exec.LookPath("dpkg-query")
if err != nil {
return nil, false
}
return dpkg{}, true
}
// getPackageVersion returns the upstream package version for the package given
// the packageName, and an error if no such package exists.
func (_ dpkg) getPackageVersion(packageName string) (string, error) {
output, err := exec.Command("dpkg-query", "--show", "--showformat='${Version}'", packageName).Output()
if err != nil {
return "", fmt.Errorf("dpkg-query failed: %s", err)
}
version := extractUpstreamVersion(string(output))
if version == "" {
return "", fmt.Errorf("no version information")
}
return version, nil
}
// packageValidator implements the Validator interface. It validates packages
// and their versions.
type packageValidator struct {
reporter Reporter
kernelRelease string
osDistro string
}
// Name returns the name of the package validator.
func (self *packageValidator) Name() string {
return "package"
}
// Validate checks packages and their versions against the spec using the
// package manager on the running machine, and returns an error on any
// package/version mismatch.
func (self *packageValidator) Validate(spec SysSpec) (error, error) {
if len(spec.PackageSpecs) == 0 {
return nil, nil
}
var err error
if self.kernelRelease, err = getKernelRelease(); err != nil {
return nil, err
}
if self.osDistro, err = getOSDistro(); err != nil {
return nil, err
}
manager, err := newPackageManager()
if err != nil {
return nil, err
}
specs := applyPackageSpecOverride(spec.PackageSpecs, spec.PackageSpecOverrides, self.osDistro)
return self.validate(specs, manager)
}
// Validate checks packages and their versions against the packageSpecs using
// the packageManager, and returns an error on any package/version mismatch.
func (self *packageValidator) validate(packageSpecs []PackageSpec, manager packageManager) (error, error) {
var errs []error
for _, spec := range packageSpecs {
// Substitute variables in package name.
packageName := resolvePackageName(spec.Name, self.kernelRelease)
nameWithVerRange := fmt.Sprintf("%s (%s)", packageName, spec.VersionRange)
// Get the version of the package on the running machine.
version, err := manager.getPackageVersion(packageName)
if err != nil {
glog.V(1).Infof("Failed to get the version for the package %q: %s\n", packageName, err)
errs = append(errs, err)
self.reporter.Report(nameWithVerRange, "not installed", bad)
continue
}
// Version requirement will not be enforced if version range is
// not specified in the spec.
if spec.VersionRange == "" {
self.reporter.Report(packageName, version, good)
continue
}
// Convert both the version range in the spec and the version returned
// from package manager to semantic version format, and then check if
// the version is in the range.
sv, err := semver.Make(toSemVer(version))
if err != nil {
glog.Errorf("Failed to convert %q to semantic version: %s\n", version, err)
errs = append(errs, err)
self.reporter.Report(nameWithVerRange, "internal error", bad)
continue
}
versionRange := semver.MustParseRange(toSemVerRange(spec.VersionRange))
if versionRange(sv) {
self.reporter.Report(nameWithVerRange, version, good)
} else {
errs = append(errs, fmt.Errorf("package \"%s %s\" does not meet the spec \"%s (%s)\"", packageName, sv, packageName, spec.VersionRange))
self.reporter.Report(nameWithVerRange, version, bad)
}
}
return nil, errors.NewAggregate(errs)
}
// getKernelRelease returns the kernel release of the local machine.
func getKernelRelease() (string, error) {
output, err := exec.Command("uname", "-r").Output()
if err != nil {
return "", fmt.Errorf("failed to get kernel release: %s", err)
}
return strings.TrimSpace(string(output)), nil
}
// getOSDistro returns the OS distro of the local machine.
func getOSDistro() (string, error) {
f := "/etc/lsb-release"
b, err := ioutil.ReadFile(f)
if err != nil {
return "", fmt.Errorf("failed to read %q: %s", f, err)
}
content := string(b)
switch {
case strings.Contains(content, "Ubuntu"):
return "ubuntu", nil
case strings.Contains(content, "Chrome OS"):
return "cos", nil
case strings.Contains(content, "CoreOS"):
return "coreos", nil
default:
return "", fmt.Errorf("failed to get OS distro: %s", content)
}
}
// resolvePackageName substitutes the variables in the packageName with the
// local information.
// E.g., "linux-headers-${KERNEL_RELEASE}" -> "linux-headers-4.4.0-75-generic".
func resolvePackageName(packageName string, kernelRelease string) string {
packageName = strings.Replace(packageName, "${KERNEL_RELEASE}", kernelRelease, -1)
return packageName
}
// applyPackageSpecOverride applies the package spec overrides for the given
// osDistro to the packageSpecs and returns the applied result.
func applyPackageSpecOverride(packageSpecs []PackageSpec, overrides []PackageSpecOverride, osDistro string) []PackageSpec {
var override *PackageSpecOverride
for _, o := range overrides {
if o.OSDistro == osDistro {
override = &o
break
}
}
if override == nil {
return packageSpecs
}
// Remove packages in the spec that matches the overrides in
// Subtractions.
var out []PackageSpec
subtractions := make(map[string]bool)
for _, spec := range override.Subtractions {
subtractions[spec.Name] = true
}
for _, spec := range packageSpecs {
if _, ok := subtractions[spec.Name]; !ok {
out = append(out, spec)
}
}
// Add packages in the spec that matches the overrides in Additions.
return append(out, override.Additions...)
}
// extractUpstreamVersion returns the upstream version of the given full
// version in dpkg format. E.g., "1:1.0.6-2ubuntu2.1" -> "1.0.6".
func extractUpstreamVersion(version string) string {
// The full version is in the format of
// "[epoch:]upstream_version[-debian_revision]". See
// https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version.
version = strings.Trim(version, " '")
if i := strings.Index(version, ":"); i != -1 {
version = version[i+1:]
}
if i := strings.Index(version, "-"); i != -1 {
version = version[:i]
}
return version
}
// toSemVerRange converts the input to a semantic version range.
// E.g., ">=1.0" -> ">=1.0.x"
// ">=1" -> ">=1.x"
// ">=1 <=2.3" -> ">=1.x <=2.3.x"
// ">1 || >3.1.0 !4.2" -> ">1.x || >3.1.0 !4.2.x"
func toSemVerRange(input string) string {
var output []string
fields := strings.Fields(input)
for _, f := range fields {
numDots, hasDigits := 0, false
for _, c := range f {
switch {
case c == '.':
numDots++
case c >= '0' && c <= '9':
hasDigits = true
}
}
if hasDigits && numDots < semVerDotsCount {
f = strings.TrimRight(f, " ")
f += ".x"
}
output = append(output, f)
}
return strings.Join(output, " ")
}
// toSemVer converts the input to a semantic version, and an empty string on
// error.
func toSemVer(version string) string {
// Remove the first non-digit and non-dot character as well as the ones
// following it.
// E.g., "1.8.19p1" -> "1.8.19".
if i := strings.IndexFunc(version, func(c rune) bool {
if (c < '0' || c > '9') && c != '.' {
return true
}
return false
}); i != -1 {
version = version[:i]
}
// Remove the trailing dots if there's any, and then returns an empty
// string if nothing left.
version = strings.TrimRight(version, ".")
if version == "" {
return ""
}
numDots := strings.Count(version, ".")
switch {
case numDots < semVerDotsCount:
// Add minor version and patch version.
// E.g. "1.18" -> "1.18.0" and "481" -> "481.0.0".
version += strings.Repeat(".0", semVerDotsCount-numDots)
case numDots > semVerDotsCount:
// Remove anything beyond the patch version
// E.g. "2.0.10.4" -> "2.0.10".
for numDots != semVerDotsCount {
if i := strings.LastIndex(version, "."); i != -1 {
version = version[:i]
numDots--
}
}
}
// Remove leading zeros in major/minor/patch version.
// E.g., "2.02" -> "2.2"
// "8.0.0095" -> "8.0.95"
var subs []string
for _, s := range strings.Split(version, ".") {
s := strings.TrimLeft(s, "0")
if s == "" {
s = "0"
}
subs = append(subs, s)
}
return strings.Join(subs, ".")
}

View File

@@ -1,266 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"errors"
"fmt"
"reflect"
"testing"
)
func TestExtractUpstreamVersion(t *testing.T) {
for _, test := range []struct {
input string
expected string
}{
{
input: "",
expected: "",
},
{
input: "1.0.6",
expected: "1.0.6",
},
{
input: "1:1.0.6",
expected: "1.0.6",
},
{
input: "1.0.6-2ubuntu2.1",
expected: "1.0.6",
},
{
input: "1:1.0.6-2ubuntu2.1",
expected: "1.0.6",
},
} {
got := extractUpstreamVersion(test.input)
if test.expected != got {
t.Errorf("extractUpstreamVersion(%q) = %q, want %q", test.input, got, test.expected)
}
}
}
func TestToSemVer(t *testing.T) {
for _, test := range []struct {
input string
expected string
}{
{
input: "",
expected: "",
},
{
input: "1.2.3",
expected: "1.2.3",
},
{
input: "1.8.19p1",
expected: "1.8.19",
},
{
input: "1.8.19.p1",
expected: "1.8.19",
},
{
input: "p1",
expected: "",
},
{
input: "1.18",
expected: "1.18.0",
},
{
input: "481",
expected: "481.0.0",
},
{
input: "2.0.10.4",
expected: "2.0.10",
},
{
input: "03",
expected: "3.0.0",
},
{
input: "2.02",
expected: "2.2.0",
},
{
input: "8.0.0095",
expected: "8.0.95",
},
} {
got := toSemVer(test.input)
if test.expected != got {
t.Errorf("toSemVer(%q) = %q, want %q", test.input, got, test.expected)
}
}
}
func TestToSemVerRange(t *testing.T) {
for _, test := range []struct {
input string
expected string
}{
{
input: "",
expected: "",
},
{
input: ">=1.0.0",
expected: ">=1.0.0",
},
{
input: ">=1.0",
expected: ">=1.0.x",
},
{
input: ">=1",
expected: ">=1.x",
},
{
input: ">=1 || !2.3",
expected: ">=1.x || !2.3.x",
},
{
input: ">1 || >3.1.0 !4.2",
expected: ">1.x || >3.1.0 !4.2.x",
},
} {
got := toSemVerRange(test.input)
if test.expected != got {
t.Errorf("toSemVerRange(%q) = %q, want %q", test.input, got, test.expected)
}
}
}
// testPackageManager implements the packageManager interface.
type testPackageManager struct {
packageVersions map[string]string
}
func (m testPackageManager) getPackageVersion(packageName string) (string, error) {
if v, ok := m.packageVersions[packageName]; ok {
return v, nil
}
return "", fmt.Errorf("package %q does not exist", packageName)
}
func TestValidatePackageVersion(t *testing.T) {
testKernelRelease := "test-kernel-release"
manager := testPackageManager{
packageVersions: map[string]string{
"foo": "1.0.0",
"bar": "2.1.0",
"bar-" + testKernelRelease: "3.0.0",
},
}
v := &packageValidator{
reporter: DefaultReporter,
kernelRelease: testKernelRelease,
}
for _, test := range []struct {
desc string
specs []PackageSpec
err error
}{
{
desc: "all packages meet the spec",
specs: []PackageSpec{
{Name: "foo", VersionRange: ">=1.0"},
{Name: "bar", VersionRange: ">=2.0 <= 3.0"},
},
},
{
desc: "package version does not meet the spec",
specs: []PackageSpec{
{Name: "foo", VersionRange: ">=1.0"},
{Name: "bar", VersionRange: ">=3.0"},
},
err: errors.New("package \"bar 2.1.0\" does not meet the spec \"bar (>=3.0)\""),
},
{
desc: "package not installed",
specs: []PackageSpec{
{Name: "baz"},
},
err: errors.New("package \"baz\" does not exist"),
},
{
desc: "use variable in package name",
specs: []PackageSpec{
{Name: "bar-${KERNEL_RELEASE}", VersionRange: ">=3.0"},
},
},
} {
_, err := v.validate(test.specs, manager)
if test.err == nil && err != nil {
t.Errorf("%s: v.validate(): err = %s", test.desc, err)
}
if test.err != nil {
if err == nil {
t.Errorf("%s: v.validate() is expected to fail.", test.desc)
} else if test.err.Error() != err.Error() {
t.Errorf("%s: v.validate(): err = %q, want = %q", test.desc, err, test.err)
}
}
}
}
func TestApplyPackageOverride(t *testing.T) {
for _, test := range []struct {
overrides []PackageSpecOverride
osDistro string
specs []PackageSpec
expected []PackageSpec
}{
{
specs: []PackageSpec{{Name: "foo", VersionRange: ">=1.0"}},
expected: []PackageSpec{{Name: "foo", VersionRange: ">=1.0"}},
},
{
osDistro: "ubuntu",
overrides: []PackageSpecOverride{
{
OSDistro: "ubuntu",
Subtractions: []PackageSpec{{Name: "foo"}},
Additions: []PackageSpec{{Name: "bar", VersionRange: ">=2.0"}},
},
},
specs: []PackageSpec{{Name: "foo", VersionRange: ">=1.0"}},
expected: []PackageSpec{{Name: "bar", VersionRange: ">=2.0"}},
},
{
osDistro: "ubuntu",
overrides: []PackageSpecOverride{
{
OSDistro: "debian",
Subtractions: []PackageSpec{{Name: "foo"}},
},
},
specs: []PackageSpec{{Name: "foo", VersionRange: ">=1.0"}},
expected: []PackageSpec{{Name: "foo", VersionRange: ">=1.0"}},
},
} {
got := applyPackageSpecOverride(test.specs, test.overrides, test.osDistro)
if !reflect.DeepEqual(test.expected, got) {
t.Errorf("applyPackageSpecOverride(%+v, %+v, %s) = %+v, want = %+v", test.specs, test.overrides, test.osDistro, got, test.expected)
}
}
}

View File

@@ -1,78 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"fmt"
"io"
"os"
)
// ValidationResultType is type of the validation result. Different validation results
// corresponds to different colors.
type ValidationResultType int32
const (
good ValidationResultType = iota
bad
warn
)
// color is the color of the message.
type color int32
const (
red color = 31
green = 32
yellow = 33
white = 37
)
func colorize(s string, c color) string {
return fmt.Sprintf("\033[0;%dm%s\033[0m", c, s)
}
// The default reporter for the system verification test
type StreamReporter struct {
// The stream that this reporter is writing to
WriteStream io.Writer
}
func (dr *StreamReporter) Report(key, value string, resultType ValidationResultType) error {
var c color
switch resultType {
case good:
c = green
case bad:
c = red
case warn:
c = yellow
default:
c = white
}
if dr.WriteStream == nil {
return fmt.Errorf("WriteStream has to be defined for this reporter")
}
fmt.Fprintf(dr.WriteStream, "%s: %s\n", colorize(key, white), colorize(value, c))
return nil
}
// DefaultReporter is the default Reporter
var DefaultReporter = &StreamReporter{
WriteStream: os.Stdout,
}

View File

@@ -1,269 +0,0 @@
# This is the system spec that must be satisfied by the images running on GKE.
os: Linux
kernelSpec:
versions:
# GKE requires kernel version 4.4+.
- '4\.[4-9].*'
- '4\.[1-9][0-9].*'
- '[5-9].*'
# Required kernel configurations -- the configuration must be set to "y" or
# "m".
required:
# The configurations required by virtual machine or cloud provider.
- name: BOOTPARAM_HARDLOCKUP_PANIC
description: 'Enable the kernel to panic on "hard lockups".'
- name: BOOTPARAM_SOFTLOCKUP_PANIC
description: 'Enable the kernel to panic on "soft lockups".'
- name: PANIC_ON_OOPS
description: 'Enable the kernel to panic when it oops.'
- name: PVPANIC
description: 'Enable the VM (guest) to communicate panic events with the
host.'
- name: DMIID
description: 'Make sure /sys/class/dmi is exported - cAdvisor currently
uses this to determine which the cloud provider it is: aws, azure, or
gce, etc'
- name: ACPI_BUTTON
description: 'Enable the software-controlled power management, and required
by reset or stop button of GCE console.'
# The configurations required by network.
- name: INET
description: 'Enable TCP/IP networking.'
- name: VXLAN
description: 'Required by the overlay networking in Kubernetes.'
- name: IP_SET
description: 'Required by Kubernetes network policy.'
- name: IP_SET_HASH_IP
description: 'This introduces hash:ip set type support, which is required
by Kubernetes Calico networking.'
- name: IPVLAN
description: 'Required by IPVLAN feature.'
- name: IPV6
description: 'Required by IPVLAN feature.'
- name: IP6_NF_IPTABLES
description: 'Required by kube-proxy.'
- name: IP_NF_TARGET_REDIRECT
alias:
- NETFILTER_XT_TARGET_REDIRECT
description: 'Enabled REDIRECT: all incoming connections are mapped onto
the incoming interface''s address, causing the packets to come to the
local machine instead of passing through. This is required by
kube-proxy.'
- name: NETFILTER_XT_MATCH_COMMENT
description: 'This option adds a "comment" dummy-match, which allows you to
put comments in your iptables ruleset. Today''s kube-proxy implementation
depends on this feature.'
# This is not critical, but debian-based container-vm kernel module study
# shows that many customers' nodes have loaded those kernel modules. We
# suspect sysdig module depends on these set of kernel modules for
# monitoring.
- name: PACKET_DIAG
description: 'Required by ss (similar to netstat) tools to display Linux
TCP / UDP network and socket information.'
- name: UNIX_DIAG
description: 'Required by ss (similar to netstat) tools to display Linux
TCP / UDP network and socket information.'
- name: INET_DIAG
description: 'Required by ss (similar to netstat) tools to display Linux
TCP / UDP network and socket information.'
- name: INET_TCP_DIAG
description: 'Required by ss (similar to netstat) tools to display Linux
TCP / UDP network and socket information.'
- name: INET_UDP_DIAG
description: 'Required by ss (similar to netstat) tools to display Linux
TCP / UDP network and socket information.'
- name: NETLINK_DIAG
description: 'Required by ss (similar to netstat) tools to display Linux
TCP / UDP network and socket information.'
# The configurations are required by filesystem.
- name: EXT4_FS
- name: DEBUG_FS
- name: PROC_FS
- name: XFS_FS
- name: SCSI_PROC_FS
# Currently Kubelet supports three docker graph drivers: overlay, aufs, and
# devicemapper due to the legacy reason. But for GKE, we plan to only support
# overlayfs.
- name: OVERLAY_FS
description: 'Enable OverlayFS, which will be the only docker graph driver
supported on GKE.'
- name: NFS_FS
description: 'Required by NFS support.'
- name: AUTOFS4_FS
description: 'Required by NFS support.'
- name: NFS_FSCACHE
description: 'Required by NFS support.'
- name: FSCACHE
description: 'Required by NFS support.'
- name: CACHEFILES
description: 'Required by NFS support.'
- name: FUSE_FS
description: 'Required by GlusterFS support.'
- name: BCACHE
# TODO(yguo0905): Add a description for BCACHE.
# The configuration required by the resource isolation, accounting, and
# management.
- name: NAMESPACES
description: 'Required by kubelet and docker. Enabling it allows the
processes within a pod or a container to have their own view of the
system.'
- name: IPC_NS
description: 'Required by kubelet and docker. Enabling it allows the
processes within a pod or a container to have their own view of the
system.'
- name: NET_NS
description: 'Required by kubelet and docker. Enabling it allows the
processes within a pod or a container to have their own view of the
system.'
- name: PID_NS
description: 'Required by kubelet and docker. Enabling it allows the
processes within a pod or a container to have their own view of the
system.'
- name: UTS_NS
description: 'Required by kubelet and docker. Enabling it allows the
processes within a pod or a container to have their own view of the
system.'
- name: CGROUPS
description: 'Required by kubelet and docker. The resource usage of the
processes within a pod or a container can be monitored, accounted, and
controlled.'
- name: CGROUP_CPUACCT
description: 'Required by kubelet and docker. The resource usage of the
processes within a pod or a container can be monitored, accounted, and
controlled.'
- name: CGROUP_DEVICE
description: 'Required by kubelet and docker. The resource usage of the
processes within a pod or a container can be monitored, accounted, and
controlled.'
- name: CGROUP_SCHED
description: 'Required by kubelet and docker. The resource usage of the
processes within a pod or a container can be monitored, accounted, and
controlled.'
- name: CPUSETS
description: 'Required by kubelet and docker. The resource usage of the
processes within a pod or a container can be monitored, accounted, and
controlled.'
- name: MEMCG
description: 'Required by kubelet and docker. The resource usage of the
processes within a pod or a container can be monitored, accounted, and
controlled.'
- name: QUOTA
description: 'Required by kubelet to have an accurate and efficient disk
space and inode accounting, and eventually to limit the usage.'
# The security-related configurations
- name: SECCOMP
description: 'Enabled the SECCOMP application API.'
- name: SECURITY_APPARMOR
description: 'Enable for AppArmor support.'
- name: CC_STACKPROTECTOR_STRONG
alias:
- CONFIG_CC_STACKPROTECTOR_REGULAR
CONFIG_CC_STACKPROTECTOR_ALL
description: 'Add the stack buffer overflow protections.'
- name: STRICT_DEVMEM
description: 'Required for blocking the direct physical memory access.'
- name: IMA
description: 'Required for security-related logging and auditing.'
- name: AUDIT
description: 'Required for security-related logging and auditing.'
- name: AUDITSYSCALL
description: 'Required for security-related logging and auditing.'
# Misc. configurations
- name: MODULES
description: 'Required for loadable module support.'
- name: PRINTK
description: 'Required for kernel logging message.'
- name: MMU
description: 'Required for memory management hardware and mmap() system
call.'
packageSpecs:
- name: apparmor
versionRange: '>=2.10.1'
- name: apparmor-profiles
versionRange: '>=2.10.1'
- name: audit
versionRange: '>=2.5.0'
- name: autofs
versionRange: '>=5.0.7'
- name: bash
versionRange: '>=4.3'
- name: bridge-utils
versionRange: '>=1.5'
- name: cloud-init
versionRange: '>=0.7.6'
- name: coreutils
versionRange: '>=8.24'
- name: dbus
versionRange: '>=1.6.8'
- name: e2fsprogs
versionRange: '>=1.4.3'
- name: ebtables
versionRange: '>=2.0.10'
- name: ethtool
versionRange: '>=3.18'
- name: iproute2
versionRange: '>=4.2.0'
- name: less
versionRange: '>=481'
- name: netcat-openbsd
versionRange: '>=1.10'
- name: python
versionRange: '>=2.7.10'
- name: pv
versionRange: '>=1.3.4'
- name: sudo
versionRange: '>=1.8.12'
- name: systemd
versionRange: '>=225'
- name: tar
versionRange: '>=1.28'
- name: util-linux
versionRange: '>=2.27.1'
- name: wget
versionRange: '>=1.18'
- name: gce-compute-image-packages
versionRange: '>=20170227'
# TODO(yguo0905): Figure out whether watchdog is required.
# packageSpecOverrides contains the OS distro specific package requirements.
packageSpecOverrides:
# The following overrides apply to all Ubuntu images.
- osDistro: ubuntu
subtractions:
- name: apparmor-profiles
description: 'On Ubuntu the apparmor profiles are shipped with individual
application package, so the "apparmor-profiles" package is not required.'
- name: audit
description: 'On Ubuntu the equivalent package is called "auditd", so the
"audit" package is not required and "auditd" exists in the additions.'
- name: wget
description: 'The Ubuntu 1604-xenial image includes wget 1.17.1, which does
not satisfy the spec (>=1.18), but meets the functionality requirements.
Therefore, it is removed from the base spec. See wget in the additions.'
additions:
- name: auditd
versionRange: '>=2.4.5'
description: 'auditd 2.4.5 currently satisfies the requirements because the
GKE features that require auditd 2.5 are not yet available.'
- name: grub-common
versionRange: '>=2.2'
description: 'grub is the bootloader on Ubuntu.'
- name: wget
versionRange: '>=1.17.1'
description: 'wget 1.17.1 satisfies the functionality requirements but does
not meet the spec, which is fine'

View File

@@ -1,124 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
// KernelConfig defines one kernel configration item.
type KernelConfig struct {
// Name is the general name of the kernel configuration. It is used to
// match kernel configuration.
Name string `json:"name,omitempty"`
// TODO(yguo0905): Support the "or" operation, which will be the same
// as the "aliases".
//
// Aliases are aliases of the kernel configuration. Some configuration
// has different names in different kernel version. Names of different
// versions will be treated as aliases.
Aliases []string `json:"aliases,omitempty"`
// Description is the description of the kernel configuration, for example:
// * What is it used for?
// * Why is it needed?
// * Who needs it?
Description string `json:"description,omitempty"`
}
// KernelSpec defines the specification for the kernel. Currently, it contains
// specification for:
// * Kernel Version
// * Kernel Configuration
type KernelSpec struct {
// Versions define supported kernel version. It is a group of regexps.
Versions []string `json:"versions,omitempty"`
// Required contains all kernel configurations required to be enabled
// (built in or as module).
Required []KernelConfig `json:"required,omitempty"`
// Optional contains all kernel configurations are required for optional
// features.
Optional []KernelConfig `json:"optional,omitempty"`
// Forbidden contains all kernel configurations which areforbidden (disabled
// or not set)
Forbidden []KernelConfig `json:"forbidden,omitempty"`
}
// DockerSpec defines the requirement configuration for docker. Currently, it only
// contains spec for graph driver.
type DockerSpec struct {
// Version is a group of regex matching supported docker versions.
Version []string `json:"version,omitempty"`
// GraphDriver is the graph drivers supported by kubelet.
GraphDriver []string `json:"graphDriver,omitempty"`
}
// RuntimeSpec is the abstract layer for different runtimes. Different runtimes
// should put their spec inside the RuntimeSpec.
type RuntimeSpec struct {
*DockerSpec `json:",inline"`
}
// PackageSpec defines the required packages and their versions.
// PackageSpec is only supported on OS distro with Debian package manager.
//
// TODO(yguo0905): Support operator OR of multiple packages for the case where
// either "foo (>=1.0)" or "bar (>=2.0)" is required.
type PackageSpec struct {
// Name is the name of the package to be checked.
Name string `json:"name,omitempty"`
// VersionRange represents a range of versions that the package must
// satisfy. Note that the version requirement will not be enforced if
// the version range is empty. For example,
// - "" would match any versions but the package must be installed.
// - ">=1" would match "1.0.0", "1.0.1", "1.1.0", and "2.0".
// - ">1.0 <2.0" would match between both ranges, so "1.1.1" and "1.8.7"
// but not "1.0.0" or "2.0.0".
// - "<2.0.0 || >=3.0.0" would match "1.0.0" and "3.0.0" but not "2.0.0".
VersionRange string `json:"versionRange,omitempty"`
// Description explains the reason behind this package requirements.
//
// TODO(yguo0905): Print the description where necessary.
Description string `json:"description,omitempty"`
}
// PackageSpecOverride defines the overrides on the PackageSpec for an OS
// distro.
type PackageSpecOverride struct {
// OSDistro identifies to which OS distro this override applies.
// Must be "ubuntu", "cos" or "coreos".
OSDistro string `json:"osDistro,omitempty"`
// Subtractions is a list of package names that are excluded from the
// package spec.
Subtractions []PackageSpec `json:"subtractions,omitempty"`
// Additions is a list of additional package requirements included the
// package spec.
Additions []PackageSpec `json:"additions,omitempty"`
}
// SysSpec defines the requirement of supported system. Currently, it only contains
// spec for OS, Kernel and Cgroups.
type SysSpec struct {
// OS is the operating system of the SysSpec.
OS string `json:"os,omitempty"`
// KernelConfig defines the spec for kernel.
KernelSpec KernelSpec `json:"kernelSpec,omitempty"`
// Cgroups is the required cgroups.
Cgroups []string `json:"cgroups,omitempty"`
// RuntimeSpec defines the spec for runtime.
RuntimeSpec RuntimeSpec `json:"runtimeSpec,omitempty"`
// PackageSpec defines the required packages and their versions.
PackageSpecs []PackageSpec `json:"packageSpecs,omitempty"`
// PackageSpec defines the overrides of the required packages and their
// versions for an OS distro.
PackageSpecOverrides []PackageSpecOverride `json:"packageSpecOverrides,omitempty"`
}

View File

@@ -1,83 +0,0 @@
// +build !windows
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"os/exec"
"strings"
)
// dockerEndpoint is the os specific endpoint for docker communication
const dockerEndpoint = "unix:///var/run/docker.sock"
// DefaultSysSpec is the default SysSpec for Linux
var DefaultSysSpec = SysSpec{
OS: "Linux",
KernelSpec: KernelSpec{
Versions: []string{`3\.[1-9][0-9].*`, `4\..*`}, // Requires 3.10+ or 4+
// TODO(random-liu): Add more config
// TODO(random-liu): Add description for each kernel configuration:
Required: []KernelConfig{
{Name: "NAMESPACES"},
{Name: "NET_NS"},
{Name: "PID_NS"},
{Name: "IPC_NS"},
{Name: "UTS_NS"},
{Name: "CGROUPS"},
{Name: "CGROUP_CPUACCT"},
{Name: "CGROUP_DEVICE"},
{Name: "CGROUP_FREEZER"},
{Name: "CGROUP_SCHED"},
{Name: "CPUSETS"},
{Name: "MEMCG"},
{Name: "INET"},
{Name: "EXT4_FS"},
{Name: "PROC_FS"},
{Name: "NETFILTER_XT_TARGET_REDIRECT", Aliases: []string{"IP_NF_TARGET_REDIRECT"}},
{Name: "NETFILTER_XT_MATCH_COMMENT"},
},
Optional: []KernelConfig{
{Name: "OVERLAY_FS", Aliases: []string{"OVERLAYFS_FS"}, Description: "Required for overlayfs."},
{Name: "AUFS_FS", Description: "Required for aufs."},
{Name: "BLK_DEV_DM", Description: "Required for devicemapper."},
},
Forbidden: []KernelConfig{},
},
Cgroups: []string{"cpu", "cpuacct", "cpuset", "devices", "freezer", "memory"},
RuntimeSpec: RuntimeSpec{
DockerSpec: &DockerSpec{
Version: []string{`1\.1[1-3]\..*`, `17\.03\..*`}, // Requires [1.11, 17.03]
GraphDriver: []string{"aufs", "overlay", "overlay2", "devicemapper"},
},
},
}
// KernelValidatorHelperImpl is the 'linux' implementation of KernelValidatorHelper
type KernelValidatorHelperImpl struct{}
var _ KernelValidatorHelper = &KernelValidatorHelperImpl{}
// GetKernelReleaseVersion returns the kernel release version (ex. 4.4.0-96-generic) as a string
func (o *KernelValidatorHelperImpl) GetKernelReleaseVersion() (string, error) {
releaseVersion, err := exec.Command("uname", "-r").CombinedOutput()
if err != nil {
return "", err
}
return strings.TrimSpace(string(releaseVersion)), nil
}

View File

@@ -1,60 +0,0 @@
// +build windows
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"os/exec"
"strings"
)
// dockerEndpoint is the os specific endpoint for docker communication
const dockerEndpoint = "npipe:////./pipe/docker_engine"
// DefaultSysSpec is the default SysSpec for Windows
var DefaultSysSpec = SysSpec{
OS: "Microsoft Windows Server 2016",
KernelSpec: KernelSpec{
Versions: []string{`10\.0\.1439[3-9]`, `10\.0\.14[4-9][0-9]{2}`, `10\.0\.1[5-9][0-9]{3}`, `10\.0\.[2-9][0-9]{4}`, `10\.[1-9]+\.[0-9]+`}, //requires >= '10.0.14393'
Required: []KernelConfig{},
Optional: []KernelConfig{},
Forbidden: []KernelConfig{},
},
Cgroups: []string{},
RuntimeSpec: RuntimeSpec{
DockerSpec: &DockerSpec{
Version: []string{`17\.03\..*`}, //Requires [17.03] or later
GraphDriver: []string{"windowsfilter"},
},
},
}
// KernelValidatorHelperImpl is the 'windows' implementation of KernelValidatorHelper
type KernelValidatorHelperImpl struct{}
var _ KernelValidatorHelper = &KernelValidatorHelperImpl{}
// GetKernelRelease returns the windows release version (ex. 10.0.14393) as a string
func (o *KernelValidatorHelperImpl) GetKernelReleaseVersion() (string, error) {
args := []string{"(Get-CimInstance Win32_OperatingSystem).Version"}
releaseVersion, err := exec.Command("powershell", args...).Output()
if err != nil {
return "", err
}
return strings.TrimSpace(string(releaseVersion)), nil
}

View File

@@ -1,72 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package system
import (
"github.com/golang/glog"
"k8s.io/apimachinery/pkg/util/errors"
)
// Validator is the interface for all validators.
type Validator interface {
// Name is the name of the validator.
Name() string
// Validate is the validate function.
Validate(SysSpec) (error, error)
}
// Reporter is the interface for the reporters for the validators.
type Reporter interface {
// Report reports the results of the system verification
Report(string, string, ValidationResultType) error
}
// Validate uses validators to validate the system and returns a warning or error.
func Validate(spec SysSpec, validators []Validator) (error, error) {
var errs []error
var warns []error
for _, v := range validators {
glog.Infof("Validating %s...", v.Name())
warn, err := v.Validate(spec)
errs = append(errs, err)
warns = append(warns, warn)
}
return errors.NewAggregate(warns), errors.NewAggregate(errs)
}
// ValidateSpec uses all default validators to validate the system and writes to stdout.
func ValidateSpec(spec SysSpec, runtime string) (error, error) {
// OS-level validators.
var osValidators = []Validator{
&OSValidator{Reporter: DefaultReporter},
&KernelValidator{Reporter: DefaultReporter},
&CgroupsValidator{Reporter: DefaultReporter},
&packageValidator{reporter: DefaultReporter},
}
// Docker-specific validators.
var dockerValidators = []Validator{
&DockerValidator{Reporter: DefaultReporter},
}
validators := osValidators
switch runtime {
case "docker":
validators = append(validators, dockerValidators...)
}
return Validate(spec, validators)
}

View File

@@ -34,10 +34,10 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/kubernetes/scheme"
kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
"k8s.io/kubernetes/pkg/features"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
"k8s.io/kubernetes/pkg/kubelet/cm"
kubeletconfigcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec"
@@ -46,6 +46,7 @@ import (
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/framework/metrics"
frameworkmetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
imageutils "k8s.io/kubernetes/test/utils/image"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@@ -56,7 +57,7 @@ var kubeletAddress = flag.String("kubelet-address", "http://127.0.0.1:10255", "H
var startServices = flag.Bool("start-services", true, "If true, start local node services")
var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tests")
var busyboxImage = "busybox"
var busyboxImage = imageutils.GetE2EImage(imageutils.BusyBox)
const (
// Kubelet internal cgroup name for node allocatable cgroup.