Add generated file
This PR adds generated files under pkg/client and vendor folder.
This commit is contained in:
835
vendor/k8s.io/kubernetes/pkg/kubelet/kuberuntime/kuberuntime_container.go
generated
vendored
Normal file
835
vendor/k8s.io/kubernetes/pkg/kubelet/kuberuntime/kuberuntime_container.go
generated
vendored
Normal file
@@ -0,0 +1,835 @@
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package kuberuntime
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/armon/circbuf"
|
||||
"github.com/golang/glog"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
kubetypes "k8s.io/apimachinery/pkg/types"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
|
||||
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
||||
"k8s.io/kubernetes/pkg/kubelet/events"
|
||||
"k8s.io/kubernetes/pkg/kubelet/types"
|
||||
"k8s.io/kubernetes/pkg/kubelet/util/format"
|
||||
"k8s.io/kubernetes/pkg/util/selinux"
|
||||
"k8s.io/kubernetes/pkg/util/tail"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrCreateContainerConfig = errors.New("CreateContainerConfigError")
|
||||
ErrCreateContainer = errors.New("CreateContainerError")
|
||||
ErrPostStartHook = errors.New("PostStartHookError")
|
||||
)
|
||||
|
||||
// recordContainerEvent should be used by the runtime manager for all container related events.
|
||||
// it has sanity checks to ensure that we do not write events that can abuse our masters.
|
||||
// in particular, it ensures that a containerID never appears in an event message as that
|
||||
// is prone to causing a lot of distinct events that do not count well.
|
||||
// it replaces any reference to a containerID with the containerName which is stable, and is what users know.
|
||||
func (m *kubeGenericRuntimeManager) recordContainerEvent(pod *v1.Pod, container *v1.Container, containerID, eventType, reason, message string, args ...interface{}) {
|
||||
ref, err := kubecontainer.GenerateContainerRef(pod, container)
|
||||
if err != nil {
|
||||
glog.Errorf("Can't make a ref to pod %q, container %v: %v", format.Pod(pod), container.Name, err)
|
||||
return
|
||||
}
|
||||
eventMessage := message
|
||||
if len(args) > 0 {
|
||||
eventMessage = fmt.Sprintf(message, args...)
|
||||
}
|
||||
// this is a hack, but often the error from the runtime includes the containerID
|
||||
// which kills our ability to deduplicate events. this protection makes a huge
|
||||
// difference in the number of unique events
|
||||
if containerID != "" {
|
||||
eventMessage = strings.Replace(eventMessage, containerID, container.Name, -1)
|
||||
}
|
||||
m.recorder.Event(ref, eventType, reason, eventMessage)
|
||||
}
|
||||
|
||||
// startContainer starts a container and returns a message indicates why it is failed on error.
|
||||
// It starts the container through the following steps:
|
||||
// * pull the image
|
||||
// * create the container
|
||||
// * start the container
|
||||
// * run the post start lifecycle hooks (if applicable)
|
||||
func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandboxConfig *runtimeapi.PodSandboxConfig, container *v1.Container, pod *v1.Pod, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, podIP string, containerType kubecontainer.ContainerType) (string, error) {
|
||||
// Step 1: pull the image.
|
||||
imageRef, msg, err := m.imagePuller.EnsureImageExists(pod, container, pullSecrets)
|
||||
if err != nil {
|
||||
m.recordContainerEvent(pod, container, "", v1.EventTypeWarning, events.FailedToCreateContainer, "Error: %v", grpc.ErrorDesc(err))
|
||||
return msg, err
|
||||
}
|
||||
|
||||
// Step 2: create the container.
|
||||
ref, err := kubecontainer.GenerateContainerRef(pod, container)
|
||||
if err != nil {
|
||||
glog.Errorf("Can't make a ref to pod %q, container %v: %v", format.Pod(pod), container.Name, err)
|
||||
}
|
||||
glog.V(4).Infof("Generating ref for container %s: %#v", container.Name, ref)
|
||||
|
||||
// For a new container, the RestartCount should be 0
|
||||
restartCount := 0
|
||||
containerStatus := podStatus.FindContainerStatusByName(container.Name)
|
||||
if containerStatus != nil {
|
||||
restartCount = containerStatus.RestartCount + 1
|
||||
}
|
||||
|
||||
containerConfig, cleanupAction, err := m.generateContainerConfig(container, pod, restartCount, podIP, imageRef, containerType)
|
||||
if cleanupAction != nil {
|
||||
defer cleanupAction()
|
||||
}
|
||||
if err != nil {
|
||||
m.recordContainerEvent(pod, container, "", v1.EventTypeWarning, events.FailedToCreateContainer, "Error: %v", grpc.ErrorDesc(err))
|
||||
return grpc.ErrorDesc(err), ErrCreateContainerConfig
|
||||
}
|
||||
|
||||
containerID, err := m.runtimeService.CreateContainer(podSandboxID, containerConfig, podSandboxConfig)
|
||||
if err != nil {
|
||||
m.recordContainerEvent(pod, container, containerID, v1.EventTypeWarning, events.FailedToCreateContainer, "Error: %v", grpc.ErrorDesc(err))
|
||||
return grpc.ErrorDesc(err), ErrCreateContainer
|
||||
}
|
||||
err = m.internalLifecycle.PreStartContainer(pod, container, containerID)
|
||||
if err != nil {
|
||||
m.recorder.Eventf(ref, v1.EventTypeWarning, events.FailedToStartContainer, "Internal PreStartContainer hook failed: %v", err)
|
||||
return "Internal PreStartContainer hook failed", err
|
||||
}
|
||||
m.recordContainerEvent(pod, container, containerID, v1.EventTypeNormal, events.CreatedContainer, "Created container")
|
||||
|
||||
if ref != nil {
|
||||
m.containerRefManager.SetRef(kubecontainer.ContainerID{
|
||||
Type: m.runtimeName,
|
||||
ID: containerID,
|
||||
}, ref)
|
||||
}
|
||||
|
||||
// Step 3: start the container.
|
||||
err = m.runtimeService.StartContainer(containerID)
|
||||
if err != nil {
|
||||
m.recordContainerEvent(pod, container, containerID, v1.EventTypeWarning, events.FailedToStartContainer, "Error: %v", grpc.ErrorDesc(err))
|
||||
return grpc.ErrorDesc(err), kubecontainer.ErrRunContainer
|
||||
}
|
||||
m.recordContainerEvent(pod, container, containerID, v1.EventTypeNormal, events.StartedContainer, "Started container")
|
||||
|
||||
// Symlink container logs to the legacy container log location for cluster logging
|
||||
// support.
|
||||
// TODO(random-liu): Remove this after cluster logging supports CRI container log path.
|
||||
containerMeta := containerConfig.GetMetadata()
|
||||
sandboxMeta := podSandboxConfig.GetMetadata()
|
||||
legacySymlink := legacyLogSymlink(containerID, containerMeta.Name, sandboxMeta.Name,
|
||||
sandboxMeta.Namespace)
|
||||
containerLog := filepath.Join(podSandboxConfig.LogDirectory, containerConfig.LogPath)
|
||||
// only create legacy symlink if containerLog path exists (or the error is not IsNotExist).
|
||||
// Because if containerLog path does not exist, only dandling legacySymlink is created.
|
||||
// This dangling legacySymlink is later removed by container gc, so it does not make sense
|
||||
// to create it in the first place. it happens when journald logging driver is used with docker.
|
||||
if _, err := m.osInterface.Stat(containerLog); !os.IsNotExist(err) {
|
||||
if err := m.osInterface.Symlink(containerLog, legacySymlink); err != nil {
|
||||
glog.Errorf("Failed to create legacy symbolic link %q to container %q log %q: %v",
|
||||
legacySymlink, containerID, containerLog, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Step 4: execute the post start hook.
|
||||
if container.Lifecycle != nil && container.Lifecycle.PostStart != nil {
|
||||
kubeContainerID := kubecontainer.ContainerID{
|
||||
Type: m.runtimeName,
|
||||
ID: containerID,
|
||||
}
|
||||
msg, handlerErr := m.runner.Run(kubeContainerID, pod, container, container.Lifecycle.PostStart)
|
||||
if handlerErr != nil {
|
||||
m.recordContainerEvent(pod, container, kubeContainerID.ID, v1.EventTypeWarning, events.FailedPostStartHook, msg)
|
||||
if err := m.killContainer(pod, kubeContainerID, container.Name, "FailedPostStartHook", nil); err != nil {
|
||||
glog.Errorf("Failed to kill container %q(id=%q) in pod %q: %v, %v",
|
||||
container.Name, kubeContainerID.String(), format.Pod(pod), ErrPostStartHook, err)
|
||||
}
|
||||
return msg, fmt.Errorf("%s: %v", ErrPostStartHook, handlerErr)
|
||||
}
|
||||
}
|
||||
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// generateContainerConfig generates container config for kubelet runtime v1.
|
||||
func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Container, pod *v1.Pod, restartCount int, podIP, imageRef string, containerType kubecontainer.ContainerType) (*runtimeapi.ContainerConfig, func(), error) {
|
||||
opts, cleanupAction, err := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
uid, username, err := m.getImageUser(container.Image)
|
||||
if err != nil {
|
||||
return nil, cleanupAction, err
|
||||
}
|
||||
|
||||
// Verify RunAsNonRoot. Non-root verification only supports numeric user.
|
||||
if err := verifyRunAsNonRoot(pod, container, uid, username); err != nil {
|
||||
return nil, cleanupAction, err
|
||||
}
|
||||
|
||||
command, args := kubecontainer.ExpandContainerCommandAndArgs(container, opts.Envs)
|
||||
logDir := BuildContainerLogsDirectory(kubetypes.UID(pod.UID), container.Name)
|
||||
err = m.osInterface.MkdirAll(logDir, 0755)
|
||||
if err != nil {
|
||||
return nil, cleanupAction, fmt.Errorf("create container log directory for container %s failed: %v", container.Name, err)
|
||||
}
|
||||
containerLogsPath := buildContainerLogsPath(container.Name, restartCount)
|
||||
restartCountUint32 := uint32(restartCount)
|
||||
config := &runtimeapi.ContainerConfig{
|
||||
Metadata: &runtimeapi.ContainerMetadata{
|
||||
Name: container.Name,
|
||||
Attempt: restartCountUint32,
|
||||
},
|
||||
Image: &runtimeapi.ImageSpec{Image: imageRef},
|
||||
Command: command,
|
||||
Args: args,
|
||||
WorkingDir: container.WorkingDir,
|
||||
Labels: newContainerLabels(container, pod, containerType),
|
||||
Annotations: newContainerAnnotations(container, pod, restartCount, opts),
|
||||
Devices: makeDevices(opts),
|
||||
Mounts: m.makeMounts(opts, container),
|
||||
LogPath: containerLogsPath,
|
||||
Stdin: container.Stdin,
|
||||
StdinOnce: container.StdinOnce,
|
||||
Tty: container.TTY,
|
||||
}
|
||||
|
||||
// set platform specific configurations.
|
||||
if err := m.applyPlatformSpecificContainerConfig(config, container, pod, uid, username); err != nil {
|
||||
return nil, cleanupAction, err
|
||||
}
|
||||
|
||||
// set environment variables
|
||||
envs := make([]*runtimeapi.KeyValue, len(opts.Envs))
|
||||
for idx := range opts.Envs {
|
||||
e := opts.Envs[idx]
|
||||
envs[idx] = &runtimeapi.KeyValue{
|
||||
Key: e.Name,
|
||||
Value: e.Value,
|
||||
}
|
||||
}
|
||||
config.Envs = envs
|
||||
|
||||
return config, cleanupAction, nil
|
||||
}
|
||||
|
||||
// makeDevices generates container devices for kubelet runtime v1.
|
||||
func makeDevices(opts *kubecontainer.RunContainerOptions) []*runtimeapi.Device {
|
||||
devices := make([]*runtimeapi.Device, len(opts.Devices))
|
||||
|
||||
for idx := range opts.Devices {
|
||||
device := opts.Devices[idx]
|
||||
devices[idx] = &runtimeapi.Device{
|
||||
HostPath: device.PathOnHost,
|
||||
ContainerPath: device.PathInContainer,
|
||||
Permissions: device.Permissions,
|
||||
}
|
||||
}
|
||||
|
||||
return devices
|
||||
}
|
||||
|
||||
// makeMounts generates container volume mounts for kubelet runtime v1.
|
||||
func (m *kubeGenericRuntimeManager) makeMounts(opts *kubecontainer.RunContainerOptions, container *v1.Container) []*runtimeapi.Mount {
|
||||
volumeMounts := []*runtimeapi.Mount{}
|
||||
|
||||
for idx := range opts.Mounts {
|
||||
v := opts.Mounts[idx]
|
||||
selinuxRelabel := v.SELinuxRelabel && selinux.SELinuxEnabled()
|
||||
mount := &runtimeapi.Mount{
|
||||
HostPath: v.HostPath,
|
||||
ContainerPath: v.ContainerPath,
|
||||
Readonly: v.ReadOnly,
|
||||
SelinuxRelabel: selinuxRelabel,
|
||||
Propagation: v.Propagation,
|
||||
}
|
||||
|
||||
volumeMounts = append(volumeMounts, mount)
|
||||
}
|
||||
|
||||
// The reason we create and mount the log file in here (not in kubelet) is because
|
||||
// the file's location depends on the ID of the container, and we need to create and
|
||||
// mount the file before actually starting the container.
|
||||
if opts.PodContainerDir != "" && len(container.TerminationMessagePath) != 0 {
|
||||
// Because the PodContainerDir contains pod uid and container name which is unique enough,
|
||||
// here we just add a random id to make the path unique for different instances
|
||||
// of the same container.
|
||||
cid := makeUID()
|
||||
containerLogPath := filepath.Join(opts.PodContainerDir, cid)
|
||||
fs, err := m.osInterface.Create(containerLogPath)
|
||||
if err != nil {
|
||||
utilruntime.HandleError(fmt.Errorf("error on creating termination-log file %q: %v", containerLogPath, err))
|
||||
} else {
|
||||
fs.Close()
|
||||
|
||||
// Chmod is needed because ioutil.WriteFile() ends up calling
|
||||
// open(2) to create the file, so the final mode used is "mode &
|
||||
// ~umask". But we want to make sure the specified mode is used
|
||||
// in the file no matter what the umask is.
|
||||
if err := m.osInterface.Chmod(containerLogPath, 0666); err != nil {
|
||||
utilruntime.HandleError(fmt.Errorf("unable to set termination-log file permissions %q: %v", containerLogPath, err))
|
||||
}
|
||||
|
||||
selinuxRelabel := selinux.SELinuxEnabled()
|
||||
volumeMounts = append(volumeMounts, &runtimeapi.Mount{
|
||||
HostPath: containerLogPath,
|
||||
ContainerPath: container.TerminationMessagePath,
|
||||
SelinuxRelabel: selinuxRelabel,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return volumeMounts
|
||||
}
|
||||
|
||||
// getKubeletContainers lists containers managed by kubelet.
|
||||
// The boolean parameter specifies whether returns all containers including
|
||||
// those already exited and dead containers (used for garbage collection).
|
||||
func (m *kubeGenericRuntimeManager) getKubeletContainers(allContainers bool) ([]*runtimeapi.Container, error) {
|
||||
filter := &runtimeapi.ContainerFilter{}
|
||||
if !allContainers {
|
||||
filter.State = &runtimeapi.ContainerStateValue{
|
||||
State: runtimeapi.ContainerState_CONTAINER_RUNNING,
|
||||
}
|
||||
}
|
||||
|
||||
containers, err := m.runtimeService.ListContainers(filter)
|
||||
if err != nil {
|
||||
glog.Errorf("getKubeletContainers failed: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return containers, nil
|
||||
}
|
||||
|
||||
// makeUID returns a randomly generated string.
|
||||
func makeUID() string {
|
||||
return fmt.Sprintf("%08x", rand.Uint32())
|
||||
}
|
||||
|
||||
// getTerminationMessage looks on the filesystem for the provided termination message path, returning a limited
|
||||
// amount of those bytes, or returns true if the logs should be checked.
|
||||
func getTerminationMessage(status *runtimeapi.ContainerStatus, terminationMessagePath string, fallbackToLogs bool) (string, bool) {
|
||||
if len(terminationMessagePath) == 0 {
|
||||
return "", fallbackToLogs
|
||||
}
|
||||
for _, mount := range status.Mounts {
|
||||
if mount.ContainerPath != terminationMessagePath {
|
||||
continue
|
||||
}
|
||||
path := mount.HostPath
|
||||
data, _, err := tail.ReadAtMost(path, kubecontainer.MaxContainerTerminationMessageLength)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return "", fallbackToLogs
|
||||
}
|
||||
return fmt.Sprintf("Error on reading termination log %s: %v", path, err), false
|
||||
}
|
||||
return string(data), (fallbackToLogs && len(data) == 0)
|
||||
}
|
||||
return "", fallbackToLogs
|
||||
}
|
||||
|
||||
// readLastStringFromContainerLogs attempts to read up to the max log length from the end of the CRI log represented
|
||||
// by path. It reads up to max log lines.
|
||||
func (m *kubeGenericRuntimeManager) readLastStringFromContainerLogs(path string) string {
|
||||
value := int64(kubecontainer.MaxContainerTerminationMessageLogLines)
|
||||
buf, _ := circbuf.NewBuffer(kubecontainer.MaxContainerTerminationMessageLogLength)
|
||||
if err := m.ReadLogs(path, "", &v1.PodLogOptions{TailLines: &value}, buf, buf); err != nil {
|
||||
return fmt.Sprintf("Error on reading termination message from logs: %v", err)
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// getPodContainerStatuses gets all containers' statuses for the pod.
|
||||
func (m *kubeGenericRuntimeManager) getPodContainerStatuses(uid kubetypes.UID, name, namespace string) ([]*kubecontainer.ContainerStatus, error) {
|
||||
// Select all containers of the given pod.
|
||||
containers, err := m.runtimeService.ListContainers(&runtimeapi.ContainerFilter{
|
||||
LabelSelector: map[string]string{types.KubernetesPodUIDLabel: string(uid)},
|
||||
})
|
||||
if err != nil {
|
||||
glog.Errorf("ListContainers error: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
statuses := make([]*kubecontainer.ContainerStatus, len(containers))
|
||||
// TODO: optimization: set maximum number of containers per container name to examine.
|
||||
for i, c := range containers {
|
||||
status, err := m.runtimeService.ContainerStatus(c.Id)
|
||||
if err != nil {
|
||||
glog.Errorf("ContainerStatus for %s error: %v", c.Id, err)
|
||||
return nil, err
|
||||
}
|
||||
cStatus := toKubeContainerStatus(status, m.runtimeName)
|
||||
if status.State == runtimeapi.ContainerState_CONTAINER_EXITED {
|
||||
// Populate the termination message if needed.
|
||||
annotatedInfo := getContainerInfoFromAnnotations(status.Annotations)
|
||||
labeledInfo := getContainerInfoFromLabels(status.Labels)
|
||||
fallbackToLogs := annotatedInfo.TerminationMessagePolicy == v1.TerminationMessageFallbackToLogsOnError && cStatus.ExitCode != 0
|
||||
tMessage, checkLogs := getTerminationMessage(status, annotatedInfo.TerminationMessagePath, fallbackToLogs)
|
||||
if checkLogs {
|
||||
// if dockerLegacyService is populated, we're supposed to use it to fetch logs
|
||||
if m.legacyLogProvider != nil {
|
||||
tMessage, err = m.legacyLogProvider.GetContainerLogTail(uid, name, namespace, kubecontainer.ContainerID{Type: m.runtimeName, ID: c.Id})
|
||||
if err != nil {
|
||||
tMessage = fmt.Sprintf("Error reading termination message from logs: %v", err)
|
||||
}
|
||||
} else {
|
||||
path := buildFullContainerLogsPath(uid, labeledInfo.ContainerName, annotatedInfo.RestartCount)
|
||||
tMessage = m.readLastStringFromContainerLogs(path)
|
||||
}
|
||||
}
|
||||
// Use the termination message written by the application is not empty
|
||||
if len(tMessage) != 0 {
|
||||
cStatus.Message = tMessage
|
||||
}
|
||||
}
|
||||
statuses[i] = cStatus
|
||||
}
|
||||
|
||||
sort.Sort(containerStatusByCreated(statuses))
|
||||
return statuses, nil
|
||||
}
|
||||
|
||||
func toKubeContainerStatus(status *runtimeapi.ContainerStatus, runtimeName string) *kubecontainer.ContainerStatus {
|
||||
annotatedInfo := getContainerInfoFromAnnotations(status.Annotations)
|
||||
labeledInfo := getContainerInfoFromLabels(status.Labels)
|
||||
cStatus := &kubecontainer.ContainerStatus{
|
||||
ID: kubecontainer.ContainerID{
|
||||
Type: runtimeName,
|
||||
ID: status.Id,
|
||||
},
|
||||
Name: labeledInfo.ContainerName,
|
||||
Image: status.Image.Image,
|
||||
ImageID: status.ImageRef,
|
||||
Hash: annotatedInfo.Hash,
|
||||
RestartCount: annotatedInfo.RestartCount,
|
||||
State: toKubeContainerState(status.State),
|
||||
CreatedAt: time.Unix(0, status.CreatedAt),
|
||||
}
|
||||
|
||||
if status.State != runtimeapi.ContainerState_CONTAINER_CREATED {
|
||||
// If container is not in the created state, we have tried and
|
||||
// started the container. Set the StartedAt time.
|
||||
cStatus.StartedAt = time.Unix(0, status.StartedAt)
|
||||
}
|
||||
if status.State == runtimeapi.ContainerState_CONTAINER_EXITED {
|
||||
cStatus.Reason = status.Reason
|
||||
cStatus.Message = status.Message
|
||||
cStatus.ExitCode = int(status.ExitCode)
|
||||
cStatus.FinishedAt = time.Unix(0, status.FinishedAt)
|
||||
}
|
||||
return cStatus
|
||||
}
|
||||
|
||||
// executePreStopHook runs the pre-stop lifecycle hooks if applicable and returns the duration it takes.
|
||||
func (m *kubeGenericRuntimeManager) executePreStopHook(pod *v1.Pod, containerID kubecontainer.ContainerID, containerSpec *v1.Container, gracePeriod int64) int64 {
|
||||
glog.V(3).Infof("Running preStop hook for container %q", containerID.String())
|
||||
|
||||
start := metav1.Now()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
defer utilruntime.HandleCrash()
|
||||
if msg, err := m.runner.Run(containerID, pod, containerSpec, containerSpec.Lifecycle.PreStop); err != nil {
|
||||
glog.Errorf("preStop hook for container %q failed: %v", containerSpec.Name, err)
|
||||
m.recordContainerEvent(pod, containerSpec, containerID.ID, v1.EventTypeWarning, events.FailedPreStopHook, msg)
|
||||
}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-time.After(time.Duration(gracePeriod) * time.Second):
|
||||
glog.V(2).Infof("preStop hook for container %q did not complete in %d seconds", containerID, gracePeriod)
|
||||
case <-done:
|
||||
glog.V(3).Infof("preStop hook for container %q completed", containerID)
|
||||
}
|
||||
|
||||
return int64(metav1.Now().Sub(start.Time).Seconds())
|
||||
}
|
||||
|
||||
// restoreSpecsFromContainerLabels restores all information needed for killing a container. In some
|
||||
// case we may not have pod and container spec when killing a container, e.g. pod is deleted during
|
||||
// kubelet restart.
|
||||
// To solve this problem, we've already written necessary information into container labels. Here we
|
||||
// just need to retrieve them from container labels and restore the specs.
|
||||
// TODO(random-liu): Add a node e2e test to test this behaviour.
|
||||
// TODO(random-liu): Change the lifecycle handler to just accept information needed, so that we can
|
||||
// just pass the needed function not create the fake object.
|
||||
func (m *kubeGenericRuntimeManager) restoreSpecsFromContainerLabels(containerID kubecontainer.ContainerID) (*v1.Pod, *v1.Container, error) {
|
||||
var pod *v1.Pod
|
||||
var container *v1.Container
|
||||
s, err := m.runtimeService.ContainerStatus(containerID.ID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
l := getContainerInfoFromLabels(s.Labels)
|
||||
a := getContainerInfoFromAnnotations(s.Annotations)
|
||||
// Notice that the followings are not full spec. The container killing code should not use
|
||||
// un-restored fields.
|
||||
pod = &v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
UID: l.PodUID,
|
||||
Name: l.PodName,
|
||||
Namespace: l.PodNamespace,
|
||||
DeletionGracePeriodSeconds: a.PodDeletionGracePeriod,
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
TerminationGracePeriodSeconds: a.PodTerminationGracePeriod,
|
||||
},
|
||||
}
|
||||
container = &v1.Container{
|
||||
Name: l.ContainerName,
|
||||
Ports: a.ContainerPorts,
|
||||
TerminationMessagePath: a.TerminationMessagePath,
|
||||
}
|
||||
if a.PreStopHandler != nil {
|
||||
container.Lifecycle = &v1.Lifecycle{
|
||||
PreStop: a.PreStopHandler,
|
||||
}
|
||||
}
|
||||
return pod, container, nil
|
||||
}
|
||||
|
||||
// killContainer kills a container through the following steps:
|
||||
// * Run the pre-stop lifecycle hooks (if applicable).
|
||||
// * Stop the container.
|
||||
func (m *kubeGenericRuntimeManager) killContainer(pod *v1.Pod, containerID kubecontainer.ContainerID, containerName string, reason string, gracePeriodOverride *int64) error {
|
||||
var containerSpec *v1.Container
|
||||
if pod != nil {
|
||||
if containerSpec = kubecontainer.GetContainerSpec(pod, containerName); containerSpec == nil {
|
||||
return fmt.Errorf("failed to get containerSpec %q(id=%q) in pod %q when killing container for reason %q",
|
||||
containerName, containerID.String(), format.Pod(pod), reason)
|
||||
}
|
||||
} else {
|
||||
// Restore necessary information if one of the specs is nil.
|
||||
restoredPod, restoredContainer, err := m.restoreSpecsFromContainerLabels(containerID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pod, containerSpec = restoredPod, restoredContainer
|
||||
}
|
||||
|
||||
// From this point , pod and container must be non-nil.
|
||||
gracePeriod := int64(minimumGracePeriodInSeconds)
|
||||
switch {
|
||||
case pod.DeletionGracePeriodSeconds != nil:
|
||||
gracePeriod = *pod.DeletionGracePeriodSeconds
|
||||
case pod.Spec.TerminationGracePeriodSeconds != nil:
|
||||
gracePeriod = *pod.Spec.TerminationGracePeriodSeconds
|
||||
}
|
||||
|
||||
glog.V(2).Infof("Killing container %q with %d second grace period", containerID.String(), gracePeriod)
|
||||
|
||||
// Run internal pre-stop lifecycle hook
|
||||
if err := m.internalLifecycle.PreStopContainer(containerID.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Run the pre-stop lifecycle hooks if applicable and if there is enough time to run it
|
||||
if containerSpec.Lifecycle != nil && containerSpec.Lifecycle.PreStop != nil && gracePeriod > 0 {
|
||||
gracePeriod = gracePeriod - m.executePreStopHook(pod, containerID, containerSpec, gracePeriod)
|
||||
}
|
||||
// always give containers a minimal shutdown window to avoid unnecessary SIGKILLs
|
||||
if gracePeriod < minimumGracePeriodInSeconds {
|
||||
gracePeriod = minimumGracePeriodInSeconds
|
||||
}
|
||||
if gracePeriodOverride != nil {
|
||||
gracePeriod = *gracePeriodOverride
|
||||
glog.V(3).Infof("Killing container %q, but using %d second grace period override", containerID, gracePeriod)
|
||||
}
|
||||
|
||||
err := m.runtimeService.StopContainer(containerID.ID, gracePeriod)
|
||||
if err != nil {
|
||||
glog.Errorf("Container %q termination failed with gracePeriod %d: %v", containerID.String(), gracePeriod, err)
|
||||
} else {
|
||||
glog.V(3).Infof("Container %q exited normally", containerID.String())
|
||||
}
|
||||
|
||||
message := fmt.Sprintf("Killing container with id %s", containerID.String())
|
||||
if reason != "" {
|
||||
message = fmt.Sprint(message, ":", reason)
|
||||
}
|
||||
m.recordContainerEvent(pod, containerSpec, containerID.ID, v1.EventTypeNormal, events.KillingContainer, message)
|
||||
m.containerRefManager.ClearRef(containerID)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// killContainersWithSyncResult kills all pod's containers with sync results.
|
||||
func (m *kubeGenericRuntimeManager) killContainersWithSyncResult(pod *v1.Pod, runningPod kubecontainer.Pod, gracePeriodOverride *int64) (syncResults []*kubecontainer.SyncResult) {
|
||||
containerResults := make(chan *kubecontainer.SyncResult, len(runningPod.Containers))
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
wg.Add(len(runningPod.Containers))
|
||||
for _, container := range runningPod.Containers {
|
||||
go func(container *kubecontainer.Container) {
|
||||
defer utilruntime.HandleCrash()
|
||||
defer wg.Done()
|
||||
|
||||
killContainerResult := kubecontainer.NewSyncResult(kubecontainer.KillContainer, container.Name)
|
||||
if err := m.killContainer(pod, container.ID, container.Name, "Need to kill Pod", gracePeriodOverride); err != nil {
|
||||
killContainerResult.Fail(kubecontainer.ErrKillContainer, err.Error())
|
||||
}
|
||||
containerResults <- killContainerResult
|
||||
}(container)
|
||||
}
|
||||
wg.Wait()
|
||||
close(containerResults)
|
||||
|
||||
for containerResult := range containerResults {
|
||||
syncResults = append(syncResults, containerResult)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// pruneInitContainersBeforeStart ensures that before we begin creating init
|
||||
// containers, we have reduced the number of outstanding init containers still
|
||||
// present. This reduces load on the container garbage collector by only
|
||||
// preserving the most recent terminated init container.
|
||||
func (m *kubeGenericRuntimeManager) pruneInitContainersBeforeStart(pod *v1.Pod, podStatus *kubecontainer.PodStatus) {
|
||||
// only the last execution of each init container should be preserved, and only preserve it if it is in the
|
||||
// list of init containers to keep.
|
||||
initContainerNames := sets.NewString()
|
||||
for _, container := range pod.Spec.InitContainers {
|
||||
initContainerNames.Insert(container.Name)
|
||||
}
|
||||
for name := range initContainerNames {
|
||||
count := 0
|
||||
for _, status := range podStatus.ContainerStatuses {
|
||||
if status.Name != name || !initContainerNames.Has(status.Name) || status.State != kubecontainer.ContainerStateExited {
|
||||
continue
|
||||
}
|
||||
count++
|
||||
// keep the first init container for this name
|
||||
if count == 1 {
|
||||
continue
|
||||
}
|
||||
// prune all other init containers that match this container name
|
||||
glog.V(4).Infof("Removing init container %q instance %q %d", status.Name, status.ID.ID, count)
|
||||
if err := m.removeContainer(status.ID.ID); err != nil {
|
||||
utilruntime.HandleError(fmt.Errorf("failed to remove pod init container %q: %v; Skipping pod %q", status.Name, err, format.Pod(pod)))
|
||||
continue
|
||||
}
|
||||
|
||||
// remove any references to this container
|
||||
if _, ok := m.containerRefManager.GetRef(status.ID); ok {
|
||||
m.containerRefManager.ClearRef(status.ID)
|
||||
} else {
|
||||
glog.Warningf("No ref for container %q", status.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove all init containres. Note that this function does not check the state
|
||||
// of the container because it assumes all init containers have been stopped
|
||||
// before the call happens.
|
||||
func (m *kubeGenericRuntimeManager) purgeInitContainers(pod *v1.Pod, podStatus *kubecontainer.PodStatus) {
|
||||
initContainerNames := sets.NewString()
|
||||
for _, container := range pod.Spec.InitContainers {
|
||||
initContainerNames.Insert(container.Name)
|
||||
}
|
||||
for name := range initContainerNames {
|
||||
count := 0
|
||||
for _, status := range podStatus.ContainerStatuses {
|
||||
if status.Name != name || !initContainerNames.Has(status.Name) {
|
||||
continue
|
||||
}
|
||||
count++
|
||||
// Purge all init containers that match this container name
|
||||
glog.V(4).Infof("Removing init container %q instance %q %d", status.Name, status.ID.ID, count)
|
||||
if err := m.removeContainer(status.ID.ID); err != nil {
|
||||
utilruntime.HandleError(fmt.Errorf("failed to remove pod init container %q: %v; Skipping pod %q", status.Name, err, format.Pod(pod)))
|
||||
continue
|
||||
}
|
||||
// Remove any references to this container
|
||||
if _, ok := m.containerRefManager.GetRef(status.ID); ok {
|
||||
m.containerRefManager.ClearRef(status.ID)
|
||||
} else {
|
||||
glog.Warningf("No ref for container %q", status.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// findNextInitContainerToRun returns the status of the last failed container, the
|
||||
// next init container to start, or done if there are no further init containers.
|
||||
// Status is only returned if an init container is failed, in which case next will
|
||||
// point to the current container.
|
||||
func findNextInitContainerToRun(pod *v1.Pod, podStatus *kubecontainer.PodStatus) (status *kubecontainer.ContainerStatus, next *v1.Container, done bool) {
|
||||
if len(pod.Spec.InitContainers) == 0 {
|
||||
return nil, nil, true
|
||||
}
|
||||
|
||||
// If there are failed containers, return the status of the last failed one.
|
||||
for i := len(pod.Spec.InitContainers) - 1; i >= 0; i-- {
|
||||
container := &pod.Spec.InitContainers[i]
|
||||
status := podStatus.FindContainerStatusByName(container.Name)
|
||||
if status != nil && isContainerFailed(status) {
|
||||
return status, container, false
|
||||
}
|
||||
}
|
||||
|
||||
// There are no failed containers now.
|
||||
for i := len(pod.Spec.InitContainers) - 1; i >= 0; i-- {
|
||||
container := &pod.Spec.InitContainers[i]
|
||||
status := podStatus.FindContainerStatusByName(container.Name)
|
||||
if status == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// container is still running, return not done.
|
||||
if status.State == kubecontainer.ContainerStateRunning {
|
||||
return nil, nil, false
|
||||
}
|
||||
|
||||
if status.State == kubecontainer.ContainerStateExited {
|
||||
// all init containers successful
|
||||
if i == (len(pod.Spec.InitContainers) - 1) {
|
||||
return nil, nil, true
|
||||
}
|
||||
|
||||
// all containers up to i successful, go to i+1
|
||||
return nil, &pod.Spec.InitContainers[i+1], false
|
||||
}
|
||||
}
|
||||
|
||||
return nil, &pod.Spec.InitContainers[0], false
|
||||
}
|
||||
|
||||
// GetContainerLogs returns logs of a specific container.
|
||||
func (m *kubeGenericRuntimeManager) GetContainerLogs(pod *v1.Pod, containerID kubecontainer.ContainerID, logOptions *v1.PodLogOptions, stdout, stderr io.Writer) (err error) {
|
||||
status, err := m.runtimeService.ContainerStatus(containerID.ID)
|
||||
if err != nil {
|
||||
glog.V(4).Infof("failed to get container status for %v: %v", containerID.String(), err)
|
||||
return fmt.Errorf("Unable to retrieve container logs for %v", containerID.String())
|
||||
}
|
||||
return m.ReadLogs(status.GetLogPath(), containerID.ID, logOptions, stdout, stderr)
|
||||
}
|
||||
|
||||
// GetExec gets the endpoint the runtime will serve the exec request from.
|
||||
func (m *kubeGenericRuntimeManager) GetExec(id kubecontainer.ContainerID, cmd []string, stdin, stdout, stderr, tty bool) (*url.URL, error) {
|
||||
req := &runtimeapi.ExecRequest{
|
||||
ContainerId: id.ID,
|
||||
Cmd: cmd,
|
||||
Tty: tty,
|
||||
Stdin: stdin,
|
||||
Stdout: stdout,
|
||||
Stderr: stderr,
|
||||
}
|
||||
resp, err := m.runtimeService.Exec(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return url.Parse(resp.Url)
|
||||
}
|
||||
|
||||
// GetAttach gets the endpoint the runtime will serve the attach request from.
|
||||
func (m *kubeGenericRuntimeManager) GetAttach(id kubecontainer.ContainerID, stdin, stdout, stderr, tty bool) (*url.URL, error) {
|
||||
req := &runtimeapi.AttachRequest{
|
||||
ContainerId: id.ID,
|
||||
Stdin: stdin,
|
||||
Stdout: stdout,
|
||||
Stderr: stderr,
|
||||
Tty: tty,
|
||||
}
|
||||
resp, err := m.runtimeService.Attach(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return url.Parse(resp.Url)
|
||||
}
|
||||
|
||||
// RunInContainer synchronously executes the command in the container, and returns the output.
|
||||
func (m *kubeGenericRuntimeManager) RunInContainer(id kubecontainer.ContainerID, cmd []string, timeout time.Duration) ([]byte, error) {
|
||||
stdout, stderr, err := m.runtimeService.ExecSync(id.ID, cmd, timeout)
|
||||
// NOTE(tallclair): This does not correctly interleave stdout & stderr, but should be sufficient
|
||||
// for logging purposes. A combined output option will need to be added to the ExecSyncRequest
|
||||
// if more precise output ordering is ever required.
|
||||
return append(stdout, stderr...), err
|
||||
}
|
||||
|
||||
// removeContainer removes the container and the container logs.
|
||||
// Notice that we remove the container logs first, so that container will not be removed if
|
||||
// container logs are failed to be removed, and kubelet will retry this later. This guarantees
|
||||
// that container logs to be removed with the container.
|
||||
// Notice that we assume that the container should only be removed in non-running state, and
|
||||
// it will not write container logs anymore in that state.
|
||||
func (m *kubeGenericRuntimeManager) removeContainer(containerID string) error {
|
||||
glog.V(4).Infof("Removing container %q", containerID)
|
||||
// Call internal container post-stop lifecycle hook.
|
||||
if err := m.internalLifecycle.PostStopContainer(containerID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove the container log.
|
||||
// TODO: Separate log and container lifecycle management.
|
||||
if err := m.removeContainerLog(containerID); err != nil {
|
||||
return err
|
||||
}
|
||||
// Remove the container.
|
||||
return m.runtimeService.RemoveContainer(containerID)
|
||||
}
|
||||
|
||||
// removeContainerLog removes the container log.
|
||||
func (m *kubeGenericRuntimeManager) removeContainerLog(containerID string) error {
|
||||
// Remove the container log.
|
||||
status, err := m.runtimeService.ContainerStatus(containerID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get container status %q: %v", containerID, err)
|
||||
}
|
||||
labeledInfo := getContainerInfoFromLabels(status.Labels)
|
||||
annotatedInfo := getContainerInfoFromAnnotations(status.Annotations)
|
||||
path := buildFullContainerLogsPath(labeledInfo.PodUID, labeledInfo.ContainerName, annotatedInfo.RestartCount)
|
||||
if err := m.osInterface.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("failed to remove container %q log %q: %v", containerID, path, err)
|
||||
}
|
||||
|
||||
// Remove the legacy container log symlink.
|
||||
// TODO(random-liu): Remove this after cluster logging supports CRI container log path.
|
||||
legacySymlink := legacyLogSymlink(containerID, labeledInfo.ContainerName, labeledInfo.PodName,
|
||||
labeledInfo.PodNamespace)
|
||||
if err := m.osInterface.Remove(legacySymlink); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("failed to remove container %q log legacy symbolic link %q: %v",
|
||||
containerID, legacySymlink, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteContainer removes a container.
|
||||
func (m *kubeGenericRuntimeManager) DeleteContainer(containerID kubecontainer.ContainerID) error {
|
||||
return m.removeContainer(containerID.ID)
|
||||
}
|
Reference in New Issue
Block a user