Add generated file

This PR adds generated files under pkg/client and vendor folder.

vendor/k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/BUILD

@@ -0,0 +1,71 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "device_plugin_stub.go",
+        "endpoint.go",
+        "manager.go",
+        "manager_stub.go",
+        "pod_devices.go",
+        "types.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/apis/core/v1/helper:go_default_library",
+        "//pkg/kubelet/apis/deviceplugin/v1beta1:go_default_library",
+        "//pkg/kubelet/checkpointmanager:go_default_library",
+        "//pkg/kubelet/checkpointmanager/errors:go_default_library",
+        "//pkg/kubelet/cm/devicemanager/checkpoint:go_default_library",
+        "//pkg/kubelet/config:go_default_library",
+        "//pkg/kubelet/container:go_default_library",
+        "//pkg/kubelet/lifecycle:go_default_library",
+        "//pkg/kubelet/metrics:go_default_library",
+        "//pkg/scheduler/cache:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/google.golang.org/grpc:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "endpoint_test.go",
+        "manager_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = [
+        "//pkg/kubelet/apis/deviceplugin/v1beta1:go_default_library",
+        "//pkg/kubelet/checkpointmanager:go_default_library",
+        "//pkg/kubelet/lifecycle:go_default_library",
+        "//pkg/scheduler/cache:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/github.com/stretchr/testify/require:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/kubelet/cm/devicemanager/checkpoint:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/OWNERS

@@ -0,0 +1,7 @@
+approvers:
+- jiayingz
+- vishh
+reviewers:
+- mindprince
+- RenaudWasTaken
+- vikaschoudhary16

vendor/k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint/BUILD

@@ -0,0 +1,26 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["checkpoint.go"],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/kubelet/checkpointmanager:go_default_library",
+        "//pkg/kubelet/checkpointmanager/checksum:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint/checkpoint.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package checkpoint
+
+import (
+	"encoding/json"
+
+	"k8s.io/kubernetes/pkg/kubelet/checkpointmanager"
+	"k8s.io/kubernetes/pkg/kubelet/checkpointmanager/checksum"
+)
+
+type DeviceManagerCheckpoint interface {
+	checkpointmanager.Checkpoint
+	GetData() ([]PodDevicesEntry, map[string][]string)
+}
+
+type PodDevicesEntry struct {
+	PodUID        string
+	ContainerName string
+	ResourceName  string
+	DeviceIDs     []string
+	AllocResp     []byte
+}
+
+// checkpointData struct is used to store pod to device allocation information
+// in a checkpoint file.
+// TODO: add version control when we need to change checkpoint format.
+type checkpointData struct {
+	PodDeviceEntries  []PodDevicesEntry
+	RegisteredDevices map[string][]string
+}
+
+type Data struct {
+	Data     checkpointData
+	Checksum checksum.Checksum
+}
+
+// NewDeviceManagerCheckpoint returns an instance of Checkpoint
+func New(devEntries []PodDevicesEntry,
+	devices map[string][]string) DeviceManagerCheckpoint {
+	return &Data{
+		Data: checkpointData{
+			PodDeviceEntries:  devEntries,
+			RegisteredDevices: devices,
+		},
+	}
+}
+
+// MarshalCheckpoint returns marshalled data
+func (cp *Data) MarshalCheckpoint() ([]byte, error) {
+	cp.Checksum = checksum.New(cp.Data)
+	return json.Marshal(*cp)
+}
+
+// UnmarshalCheckpoint returns unmarshalled data
+func (cp *Data) UnmarshalCheckpoint(blob []byte) error {
+	return json.Unmarshal(blob, cp)
+}
+
+// VerifyChecksum verifies that passed checksum is same as calculated checksum
+func (cp *Data) VerifyChecksum() error {
+	return cp.Checksum.Verify(cp.Data)
+}
+
+func (cp *Data) GetData() ([]PodDevicesEntry, map[string][]string) {
+	return cp.Data.PodDeviceEntries, cp.Data.RegisteredDevices
+}

vendor/k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/device_plugin_stub.go

@@ -0,0 +1,200 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package devicemanager
+
+import (
+	"context"
+	"log"
+	"net"
+	"os"
+	"path"
+	"sync"
+	"time"
+
+	"google.golang.org/grpc"
+
+	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
+)
+
+// Stub implementation for DevicePlugin.
+type Stub struct {
+	devs   []*pluginapi.Device
+	socket string
+
+	stop   chan interface{}
+	wg     sync.WaitGroup
+	update chan []*pluginapi.Device
+
+	server *grpc.Server
+
+	// allocFunc is used for handling allocation request
+	allocFunc stubAllocFunc
+}
+
+// stubAllocFunc is the function called when receive an allocation request from Kubelet
+type stubAllocFunc func(r *pluginapi.AllocateRequest, devs map[string]pluginapi.Device) (*pluginapi.AllocateResponse, error)
+
+func defaultAllocFunc(r *pluginapi.AllocateRequest, devs map[string]pluginapi.Device) (*pluginapi.AllocateResponse, error) {
+	var response pluginapi.AllocateResponse
+
+	return &response, nil
+}
+
+// NewDevicePluginStub returns an initialized DevicePlugin Stub.
+func NewDevicePluginStub(devs []*pluginapi.Device, socket string) *Stub {
+	return &Stub{
+		devs:   devs,
+		socket: socket,
+
+		stop:   make(chan interface{}),
+		update: make(chan []*pluginapi.Device),
+
+		allocFunc: defaultAllocFunc,
+	}
+}
+
+// SetAllocFunc sets allocFunc of the device plugin
+func (m *Stub) SetAllocFunc(f stubAllocFunc) {
+	m.allocFunc = f
+}
+
+// Start starts the gRPC server of the device plugin. Can only
+// be called once.
+func (m *Stub) Start() error {
+	err := m.cleanup()
+	if err != nil {
+		return err
+	}
+
+	sock, err := net.Listen("unix", m.socket)
+	if err != nil {
+		return err
+	}
+
+	m.wg.Add(1)
+	m.server = grpc.NewServer([]grpc.ServerOption{}...)
+	pluginapi.RegisterDevicePluginServer(m.server, m)
+
+	go func() {
+		defer m.wg.Done()
+		m.server.Serve(sock)
+	}()
+	_, conn, err := dial(m.socket)
+	if err != nil {
+		return err
+	}
+	conn.Close()
+	log.Println("Starting to serve on", m.socket)
+
+	return nil
+}
+
+// Stop stops the gRPC server. Can be called without a prior Start
+// and more than once. Not safe to be called concurrently by different
+// goroutines!
+func (m *Stub) Stop() error {
+	if m.server == nil {
+		return nil
+	}
+	m.server.Stop()
+	m.wg.Wait()
+	m.server = nil
+	close(m.stop) // This prevents re-starting the server.
+
+	return m.cleanup()
+}
+
+// Register registers the device plugin for the given resourceName with Kubelet.
+func (m *Stub) Register(kubeletEndpoint, resourceName string, preStartContainerFlag bool) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	conn, err := grpc.DialContext(ctx, kubeletEndpoint, grpc.WithInsecure(), grpc.WithBlock(),
+		grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
+			return net.DialTimeout("unix", addr, timeout)
+		}))
+	defer conn.Close()
+	if err != nil {
+		return err
+	}
+	client := pluginapi.NewRegistrationClient(conn)
+	reqt := &pluginapi.RegisterRequest{
+		Version:      pluginapi.Version,
+		Endpoint:     path.Base(m.socket),
+		ResourceName: resourceName,
+		Options:      &pluginapi.DevicePluginOptions{PreStartRequired: preStartContainerFlag},
+	}
+
+	_, err = client.Register(context.Background(), reqt)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// GetDevicePluginOptions returns DevicePluginOptions settings for the device plugin.
+func (m *Stub) GetDevicePluginOptions(ctx context.Context, e *pluginapi.Empty) (*pluginapi.DevicePluginOptions, error) {
+	return &pluginapi.DevicePluginOptions{}, nil
+}
+
+// PreStartContainer resets the devices received
+func (m *Stub) PreStartContainer(ctx context.Context, r *pluginapi.PreStartContainerRequest) (*pluginapi.PreStartContainerResponse, error) {
+	log.Printf("PreStartContainer, %+v", r)
+	return &pluginapi.PreStartContainerResponse{}, nil
+}
+
+// ListAndWatch lists devices and update that list according to the Update call
+func (m *Stub) ListAndWatch(e *pluginapi.Empty, s pluginapi.DevicePlugin_ListAndWatchServer) error {
+	log.Println("ListAndWatch")
+
+	s.Send(&pluginapi.ListAndWatchResponse{Devices: m.devs})
+
+	for {
+		select {
+		case <-m.stop:
+			return nil
+		case updated := <-m.update:
+			s.Send(&pluginapi.ListAndWatchResponse{Devices: updated})
+		}
+	}
+}
+
+// Update allows the device plugin to send new devices through ListAndWatch
+func (m *Stub) Update(devs []*pluginapi.Device) {
+	m.update <- devs
+}
+
+// Allocate does a mock allocation
+func (m *Stub) Allocate(ctx context.Context, r *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {
+	log.Printf("Allocate, %+v", r)
+
+	devs := make(map[string]pluginapi.Device)
+
+	for _, dev := range m.devs {
+		devs[dev.ID] = *dev
+	}
+
+	return m.allocFunc(r, devs)
+}
+
+func (m *Stub) cleanup() error {
+	if err := os.Remove(m.socket); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/endpoint.go

@@ -0,0 +1,261 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package devicemanager
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/golang/glog"
+	"google.golang.org/grpc"
+
+	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
+)
+
+// endpoint maps to a single registered device plugin. It is responsible
+// for managing gRPC communications with the device plugin and caching
+// device states reported by the device plugin.
+type endpoint interface {
+	run()
+	stop()
+	allocate(devs []string) (*pluginapi.AllocateResponse, error)
+	preStartContainer(devs []string) (*pluginapi.PreStartContainerResponse, error)
+	getDevices() []pluginapi.Device
+	callback(resourceName string, added, updated, deleted []pluginapi.Device)
+	isStopped() bool
+	stopGracePeriodExpired() bool
+}
+
+type endpointImpl struct {
+	client     pluginapi.DevicePluginClient
+	clientConn *grpc.ClientConn
+
+	socketPath   string
+	resourceName string
+	stopTime     time.Time
+
+	devices map[string]pluginapi.Device
+	mutex   sync.Mutex
+
+	cb monitorCallback
+}
+
+// newEndpoint creates a new endpoint for the given resourceName.
+// This is to be used during normal device plugin registration.
+func newEndpointImpl(socketPath, resourceName string, devices map[string]pluginapi.Device, callback monitorCallback) (*endpointImpl, error) {
+	client, c, err := dial(socketPath)
+	if err != nil {
+		glog.Errorf("Can't create new endpoint with path %s err %v", socketPath, err)
+		return nil, err
+	}
+
+	return &endpointImpl{
+		client:     client,
+		clientConn: c,
+
+		socketPath:   socketPath,
+		resourceName: resourceName,
+
+		devices: devices,
+		cb:      callback,
+	}, nil
+}
+
+// newStoppedEndpointImpl creates a new endpoint for the given resourceName with stopTime set.
+// This is to be used during Kubelet restart, before the actual device plugin re-registers.
+func newStoppedEndpointImpl(resourceName string, devices map[string]pluginapi.Device) *endpointImpl {
+	return &endpointImpl{
+		resourceName: resourceName,
+		devices:      devices,
+		stopTime:     time.Now(),
+	}
+}
+
+func (e *endpointImpl) callback(resourceName string, added, updated, deleted []pluginapi.Device) {
+	e.cb(resourceName, added, updated, deleted)
+}
+
+func (e *endpointImpl) getDevices() []pluginapi.Device {
+	e.mutex.Lock()
+	defer e.mutex.Unlock()
+	var devs []pluginapi.Device
+
+	for _, d := range e.devices {
+		devs = append(devs, d)
+	}
+
+	return devs
+}
+
+// run initializes ListAndWatch gRPC call for the device plugin and
+// blocks on receiving ListAndWatch gRPC stream updates. Each ListAndWatch
+// stream update contains a new list of device states. listAndWatch compares the new
+// device states with its cached states to get list of new, updated, and deleted devices.
+// It then issues a callback to pass this information to the device manager which
+// will adjust the resource available information accordingly.
+func (e *endpointImpl) run() {
+	stream, err := e.client.ListAndWatch(context.Background(), &pluginapi.Empty{})
+	if err != nil {
+		glog.Errorf(errListAndWatch, e.resourceName, err)
+
+		return
+	}
+
+	devices := make(map[string]pluginapi.Device)
+
+	e.mutex.Lock()
+	for _, d := range e.devices {
+		devices[d.ID] = d
+	}
+	e.mutex.Unlock()
+
+	for {
+		response, err := stream.Recv()
+		if err != nil {
+			glog.Errorf(errListAndWatch, e.resourceName, err)
+			return
+		}
+
+		devs := response.Devices
+		glog.V(2).Infof("State pushed for device plugin %s", e.resourceName)
+
+		newDevs := make(map[string]*pluginapi.Device)
+		var added, updated []pluginapi.Device
+
+		for _, d := range devs {
+			dOld, ok := devices[d.ID]
+			newDevs[d.ID] = d
+
+			if !ok {
+				glog.V(2).Infof("New device for Endpoint %s: %v", e.resourceName, d)
+
+				devices[d.ID] = *d
+				added = append(added, *d)
+
+				continue
+			}
+
+			if d.Health == dOld.Health {
+				continue
+			}
+
+			if d.Health == pluginapi.Unhealthy {
+				glog.Errorf("Device %s is now Unhealthy", d.ID)
+			} else if d.Health == pluginapi.Healthy {
+				glog.V(2).Infof("Device %s is now Healthy", d.ID)
+			}
+
+			devices[d.ID] = *d
+			updated = append(updated, *d)
+		}
+
+		var deleted []pluginapi.Device
+		for id, d := range devices {
+			if _, ok := newDevs[id]; ok {
+				continue
+			}
+
+			glog.Errorf("Device %s was deleted", d.ID)
+
+			deleted = append(deleted, d)
+			delete(devices, id)
+		}
+
+		e.mutex.Lock()
+		// NOTE: Return a copy of 'devices' instead of returning a direct reference to local 'devices'
+		e.devices = make(map[string]pluginapi.Device)
+		for _, d := range devices {
+			e.devices[d.ID] = d
+		}
+		e.mutex.Unlock()
+
+		e.callback(e.resourceName, added, updated, deleted)
+	}
+}
+
+func (e *endpointImpl) isStopped() bool {
+	e.mutex.Lock()
+	defer e.mutex.Unlock()
+	return !e.stopTime.IsZero()
+}
+
+func (e *endpointImpl) stopGracePeriodExpired() bool {
+	e.mutex.Lock()
+	defer e.mutex.Unlock()
+	return !e.stopTime.IsZero() && time.Since(e.stopTime) > endpointStopGracePeriod
+}
+
+// used for testing only
+func (e *endpointImpl) setStopTime(t time.Time) {
+	e.mutex.Lock()
+	defer e.mutex.Unlock()
+	e.stopTime = t
+}
+
+// allocate issues Allocate gRPC call to the device plugin.
+func (e *endpointImpl) allocate(devs []string) (*pluginapi.AllocateResponse, error) {
+	if e.isStopped() {
+		return nil, fmt.Errorf(errEndpointStopped, e)
+	}
+	return e.client.Allocate(context.Background(), &pluginapi.AllocateRequest{
+		ContainerRequests: []*pluginapi.ContainerAllocateRequest{
+			{DevicesIDs: devs},
+		},
+	})
+}
+
+// preStartContainer issues PreStartContainer gRPC call to the device plugin.
+func (e *endpointImpl) preStartContainer(devs []string) (*pluginapi.PreStartContainerResponse, error) {
+	if e.isStopped() {
+		return nil, fmt.Errorf(errEndpointStopped, e)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), pluginapi.KubeletPreStartContainerRPCTimeoutInSecs*time.Second)
+	defer cancel()
+	return e.client.PreStartContainer(ctx, &pluginapi.PreStartContainerRequest{
+		DevicesIDs: devs,
+	})
+}
+
+func (e *endpointImpl) stop() {
+	e.mutex.Lock()
+	defer e.mutex.Unlock()
+	if e.clientConn != nil {
+		e.clientConn.Close()
+	}
+	e.stopTime = time.Now()
+}
+
+// dial establishes the gRPC communication with the registered device plugin. https://godoc.org/google.golang.org/grpc#Dial
+func dial(unixSocketPath string) (pluginapi.DevicePluginClient, *grpc.ClientConn, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	c, err := grpc.DialContext(ctx, unixSocketPath, grpc.WithInsecure(), grpc.WithBlock(),
+		grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
+			return net.DialTimeout("unix", addr, timeout)
+		}),
+	)
+
+	if err != nil {
+		return nil, nil, fmt.Errorf(errFailedToDialDevicePlugin+" %v", err)
+	}
+
+	return pluginapi.NewDevicePluginClient(c), c, nil
+}

vendor/k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/endpoint_test.go

@@ -0,0 +1,197 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package devicemanager
+
+import (
+	"path"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
+)
+
+var (
+	esocketName = "mock.sock"
+)
+
+func TestNewEndpoint(t *testing.T) {
+	socket := path.Join("/tmp", esocketName)
+
+	devs := []*pluginapi.Device{
+		{ID: "ADeviceId", Health: pluginapi.Healthy},
+	}
+
+	p, e := esetup(t, devs, socket, "mock", func(n string, a, u, r []pluginapi.Device) {})
+	defer ecleanup(t, p, e)
+}
+
+func TestRun(t *testing.T) {
+	socket := path.Join("/tmp", esocketName)
+
+	devs := []*pluginapi.Device{
+		{ID: "ADeviceId", Health: pluginapi.Healthy},
+		{ID: "AnotherDeviceId", Health: pluginapi.Healthy},
+		{ID: "AThirdDeviceId", Health: pluginapi.Unhealthy},
+	}
+
+	updated := []*pluginapi.Device{
+		{ID: "ADeviceId", Health: pluginapi.Unhealthy},
+		{ID: "AThirdDeviceId", Health: pluginapi.Healthy},
+		{ID: "AFourthDeviceId", Health: pluginapi.Healthy},
+	}
+
+	callbackCount := 0
+	callbackChan := make(chan int)
+	callback := func(n string, a, u, r []pluginapi.Device) {
+		// Should be called twice:
+		// one for plugin registration, one for plugin update.
+		if callbackCount > 2 {
+			t.FailNow()
+		}
+
+		// Check plugin registration
+		if callbackCount == 0 {
+			require.Len(t, a, 3)
+			require.Len(t, u, 0)
+			require.Len(t, r, 0)
+		}
+
+		// Check plugin update
+		if callbackCount == 1 {
+			require.Len(t, a, 1)
+			require.Len(t, u, 2)
+			require.Len(t, r, 1)
+
+			require.Equal(t, a[0].ID, updated[2].ID)
+			require.Equal(t, u[0].ID, updated[0].ID)
+			require.Equal(t, u[0].Health, updated[0].Health)
+			require.Equal(t, u[1].ID, updated[1].ID)
+			require.Equal(t, u[1].Health, updated[1].Health)
+			require.Equal(t, r[0].ID, devs[1].ID)
+		}
+
+		callbackCount++
+		callbackChan <- callbackCount
+	}
+
+	p, e := esetup(t, devs, socket, "mock", callback)
+	defer ecleanup(t, p, e)
+
+	go e.run()
+	// Wait for the first callback to be issued.
+	<-callbackChan
+
+	p.Update(updated)
+
+	// Wait for the second callback to be issued.
+	<-callbackChan
+
+	e.mutex.Lock()
+	defer e.mutex.Unlock()
+
+	require.Len(t, e.devices, 3)
+	for _, dref := range updated {
+		d, ok := e.devices[dref.ID]
+
+		require.True(t, ok)
+		require.Equal(t, d.ID, dref.ID)
+		require.Equal(t, d.Health, dref.Health)
+	}
+
+}
+
+func TestAllocate(t *testing.T) {
+	socket := path.Join("/tmp", esocketName)
+	devs := []*pluginapi.Device{
+		{ID: "ADeviceId", Health: pluginapi.Healthy},
+	}
+	callbackCount := 0
+	callbackChan := make(chan int)
+	p, e := esetup(t, devs, socket, "mock", func(n string, a, u, r []pluginapi.Device) {
+		callbackCount++
+		callbackChan <- callbackCount
+	})
+	defer ecleanup(t, p, e)
+
+	resp := new(pluginapi.AllocateResponse)
+	contResp := new(pluginapi.ContainerAllocateResponse)
+	contResp.Devices = append(contResp.Devices, &pluginapi.DeviceSpec{
+		ContainerPath: "/dev/aaa",
+		HostPath:      "/dev/aaa",
+		Permissions:   "mrw",
+	})
+
+	contResp.Devices = append(contResp.Devices, &pluginapi.DeviceSpec{
+		ContainerPath: "/dev/bbb",
+		HostPath:      "/dev/bbb",
+		Permissions:   "mrw",
+	})
+
+	contResp.Mounts = append(contResp.Mounts, &pluginapi.Mount{
+		ContainerPath: "/container_dir1/file1",
+		HostPath:      "host_dir1/file1",
+		ReadOnly:      true,
+	})
+
+	resp.ContainerResponses = append(resp.ContainerResponses, contResp)
+
+	p.SetAllocFunc(func(r *pluginapi.AllocateRequest, devs map[string]pluginapi.Device) (*pluginapi.AllocateResponse, error) {
+		return resp, nil
+	})
+
+	go e.run()
+	// Wait for the callback to be issued.
+	select {
+	case <-callbackChan:
+		break
+	case <-time.After(time.Second):
+		t.FailNow()
+	}
+
+	respOut, err := e.allocate([]string{"ADeviceId"})
+	require.NoError(t, err)
+	require.Equal(t, resp, respOut)
+}
+
+func TestGetDevices(t *testing.T) {
+	e := endpointImpl{
+		devices: map[string]pluginapi.Device{
+			"ADeviceId": {ID: "ADeviceId", Health: pluginapi.Healthy},
+		},
+	}
+	devs := e.getDevices()
+	require.Len(t, devs, 1)
+}
+
+func esetup(t *testing.T, devs []*pluginapi.Device, socket, resourceName string, callback monitorCallback) (*Stub, *endpointImpl) {
+	p := NewDevicePluginStub(devs, socket)
+
+	err := p.Start()
+	require.NoError(t, err)
+
+	e, err := newEndpointImpl(socket, resourceName, make(map[string]pluginapi.Device), callback)
+	require.NoError(t, err)
+
+	return p, e
+}
+
+func ecleanup(t *testing.T, p *Stub, e *endpointImpl) {
+	p.Stop()
+	e.stop()
+}

vendor/k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/manager.go

@@ -0,0 +1,776 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package devicemanager
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/golang/glog"
+	"google.golang.org/grpc"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/sets"
+	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
+	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
+	"k8s.io/kubernetes/pkg/kubelet/checkpointmanager"
+	"k8s.io/kubernetes/pkg/kubelet/checkpointmanager/errors"
+	"k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint"
+	"k8s.io/kubernetes/pkg/kubelet/config"
+	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
+	"k8s.io/kubernetes/pkg/kubelet/metrics"
+	schedulercache "k8s.io/kubernetes/pkg/scheduler/cache"
+)
+
+// ActivePodsFunc is a function that returns a list of pods to reconcile.
+type ActivePodsFunc func() []*v1.Pod
+
+// monitorCallback is the function called when a device's health state changes,
+// or new devices are reported, or old devices are deleted.
+// Updated contains the most recent state of the Device.
+type monitorCallback func(resourceName string, added, updated, deleted []pluginapi.Device)
+
+// ManagerImpl is the structure in charge of managing Device Plugins.
+type ManagerImpl struct {
+	socketname string
+	socketdir  string
+
+	endpoints map[string]endpoint // Key is ResourceName
+	mutex     sync.Mutex
+
+	server *grpc.Server
+	wg     sync.WaitGroup
+
+	// activePods is a method for listing active pods on the node
+	// so the amount of pluginResources requested by existing pods
+	// could be counted when updating allocated devices
+	activePods ActivePodsFunc
+
+	// sourcesReady provides the readiness of kubelet configuration sources such as apiserver update readiness.
+	// We use it to determine when we can purge inactive pods from checkpointed state.
+	sourcesReady config.SourcesReady
+
+	// callback is used for updating devices' states in one time call.
+	// e.g. a new device is advertised, two old devices are deleted and a running device fails.
+	callback monitorCallback
+
+	// healthyDevices contains all of the registered healthy resourceNames and their exported device IDs.
+	healthyDevices map[string]sets.String
+
+	// unhealthyDevices contains all of the unhealthy devices and their exported device IDs.
+	unhealthyDevices map[string]sets.String
+
+	// allocatedDevices contains allocated deviceIds, keyed by resourceName.
+	allocatedDevices map[string]sets.String
+
+	// podDevices contains pod to allocated device mapping.
+	podDevices        podDevices
+	pluginOpts        map[string]*pluginapi.DevicePluginOptions
+	checkpointManager checkpointmanager.CheckpointManager
+}
+
+type sourcesReadyStub struct{}
+
+func (s *sourcesReadyStub) AddSource(source string) {}
+func (s *sourcesReadyStub) AllReady() bool          { return true }
+
+// NewManagerImpl creates a new manager.
+func NewManagerImpl() (*ManagerImpl, error) {
+	return newManagerImpl(pluginapi.KubeletSocket)
+}
+
+func newManagerImpl(socketPath string) (*ManagerImpl, error) {
+	glog.V(2).Infof("Creating Device Plugin manager at %s", socketPath)
+
+	if socketPath == "" || !filepath.IsAbs(socketPath) {
+		return nil, fmt.Errorf(errBadSocket+" %v", socketPath)
+	}
+
+	dir, file := filepath.Split(socketPath)
+	manager := &ManagerImpl{
+		endpoints:        make(map[string]endpoint),
+		socketname:       file,
+		socketdir:        dir,
+		healthyDevices:   make(map[string]sets.String),
+		unhealthyDevices: make(map[string]sets.String),
+		allocatedDevices: make(map[string]sets.String),
+		pluginOpts:       make(map[string]*pluginapi.DevicePluginOptions),
+		podDevices:       make(podDevices),
+	}
+	manager.callback = manager.genericDeviceUpdateCallback
+
+	// The following structs are populated with real implementations in manager.Start()
+	// Before that, initializes them to perform no-op operations.
+	manager.activePods = func() []*v1.Pod { return []*v1.Pod{} }
+	manager.sourcesReady = &sourcesReadyStub{}
+	checkpointManager, err := checkpointmanager.NewCheckpointManager(dir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize checkpoint manager: %+v", err)
+	}
+	manager.checkpointManager = checkpointManager
+
+	return manager, nil
+}
+
+func (m *ManagerImpl) genericDeviceUpdateCallback(resourceName string, added, updated, deleted []pluginapi.Device) {
+	kept := append(updated, added...)
+	m.mutex.Lock()
+	if _, ok := m.healthyDevices[resourceName]; !ok {
+		m.healthyDevices[resourceName] = sets.NewString()
+	}
+	if _, ok := m.unhealthyDevices[resourceName]; !ok {
+		m.unhealthyDevices[resourceName] = sets.NewString()
+	}
+	for _, dev := range kept {
+		if dev.Health == pluginapi.Healthy {
+			m.healthyDevices[resourceName].Insert(dev.ID)
+			m.unhealthyDevices[resourceName].Delete(dev.ID)
+		} else {
+			m.unhealthyDevices[resourceName].Insert(dev.ID)
+			m.healthyDevices[resourceName].Delete(dev.ID)
+		}
+	}
+	for _, dev := range deleted {
+		m.healthyDevices[resourceName].Delete(dev.ID)
+		m.unhealthyDevices[resourceName].Delete(dev.ID)
+	}
+	m.mutex.Unlock()
+	m.writeCheckpoint()
+}
+
+func (m *ManagerImpl) removeContents(dir string) error {
+	d, err := os.Open(dir)
+	if err != nil {
+		return err
+	}
+	defer d.Close()
+	names, err := d.Readdirnames(-1)
+	if err != nil {
+		return err
+	}
+	for _, name := range names {
+		filePath := filepath.Join(dir, name)
+		if filePath == m.checkpointFile() {
+			continue
+		}
+		stat, err := os.Stat(filePath)
+		if err != nil {
+			glog.Errorf("Failed to stat file %v: %v", filePath, err)
+			continue
+		}
+		if stat.IsDir() {
+			continue
+		}
+		err = os.RemoveAll(filePath)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// checkpointFile returns device plugin checkpoint file path.
+func (m *ManagerImpl) checkpointFile() string {
+	return filepath.Join(m.socketdir, kubeletDeviceManagerCheckpoint)
+}
+
+// Start starts the Device Plugin Manager amd start initialization of
+// podDevices and allocatedDevices information from checkpoint-ed state and
+// starts device plugin registration service.
+func (m *ManagerImpl) Start(activePods ActivePodsFunc, sourcesReady config.SourcesReady) error {
+	glog.V(2).Infof("Starting Device Plugin manager")
+	fmt.Println("Starting Device Plugin manager")
+
+	m.activePods = activePods
+	m.sourcesReady = sourcesReady
+
+	// Loads in allocatedDevices information from disk.
+	err := m.readCheckpoint()
+	if err != nil {
+		glog.Warningf("Continue after failing to read checkpoint file. Device allocation info may NOT be up-to-date. Err: %v", err)
+	}
+
+	socketPath := filepath.Join(m.socketdir, m.socketname)
+	os.MkdirAll(m.socketdir, 0755)
+
+	// Removes all stale sockets in m.socketdir. Device plugins can monitor
+	// this and use it as a signal to re-register with the new Kubelet.
+	if err := m.removeContents(m.socketdir); err != nil {
+		glog.Errorf("Fail to clean up stale contents under %s: %+v", m.socketdir, err)
+	}
+
+	s, err := net.Listen("unix", socketPath)
+	if err != nil {
+		glog.Errorf(errListenSocket+" %+v", err)
+		return err
+	}
+
+	m.wg.Add(1)
+	m.server = grpc.NewServer([]grpc.ServerOption{}...)
+
+	pluginapi.RegisterRegistrationServer(m.server, m)
+	go func() {
+		defer m.wg.Done()
+		m.server.Serve(s)
+	}()
+
+	glog.V(2).Infof("Serving device plugin registration server on %q", socketPath)
+
+	return nil
+}
+
+// Devices is the map of devices that are known by the Device
+// Plugin manager with the kind of the devices as key
+func (m *ManagerImpl) Devices() map[string][]pluginapi.Device {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	devs := make(map[string][]pluginapi.Device)
+	for k, e := range m.endpoints {
+		glog.V(3).Infof("Endpoint: %+v: %p", k, e)
+		devs[k] = e.getDevices()
+	}
+
+	return devs
+}
+
+// Allocate is the call that you can use to allocate a set of devices
+// from the registered device plugins.
+func (m *ManagerImpl) Allocate(node *schedulercache.NodeInfo, attrs *lifecycle.PodAdmitAttributes) error {
+	pod := attrs.Pod
+	devicesToReuse := make(map[string]sets.String)
+	// TODO: Reuse devices between init containers and regular containers.
+	for _, container := range pod.Spec.InitContainers {
+		if err := m.allocateContainerResources(pod, &container, devicesToReuse); err != nil {
+			return err
+		}
+		m.podDevices.addContainerAllocatedResources(string(pod.UID), container.Name, devicesToReuse)
+	}
+	for _, container := range pod.Spec.Containers {
+		if err := m.allocateContainerResources(pod, &container, devicesToReuse); err != nil {
+			return err
+		}
+		m.podDevices.removeContainerAllocatedResources(string(pod.UID), container.Name, devicesToReuse)
+	}
+
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	// quick return if no pluginResources requested
+	if _, podRequireDevicePluginResource := m.podDevices[string(pod.UID)]; !podRequireDevicePluginResource {
+		return nil
+	}
+
+	m.sanitizeNodeAllocatable(node)
+	return nil
+}
+
+// Register registers a device plugin.
+func (m *ManagerImpl) Register(ctx context.Context, r *pluginapi.RegisterRequest) (*pluginapi.Empty, error) {
+	glog.Infof("Got registration request from device plugin with resource name %q", r.ResourceName)
+	metrics.DevicePluginRegistrationCount.WithLabelValues(r.ResourceName).Inc()
+	var versionCompatible bool
+	for _, v := range pluginapi.SupportedVersions {
+		if r.Version == v {
+			versionCompatible = true
+			break
+		}
+	}
+	if !versionCompatible {
+		errorString := fmt.Sprintf(errUnsupportedVersion, r.Version, pluginapi.SupportedVersions)
+		glog.Infof("Bad registration request from device plugin with resource name %q: %v", r.ResourceName, errorString)
+		return &pluginapi.Empty{}, fmt.Errorf(errorString)
+	}
+
+	if !v1helper.IsExtendedResourceName(v1.ResourceName(r.ResourceName)) {
+		errorString := fmt.Sprintf(errInvalidResourceName, r.ResourceName)
+		glog.Infof("Bad registration request from device plugin: %v", errorString)
+		return &pluginapi.Empty{}, fmt.Errorf(errorString)
+	}
+
+	// TODO: for now, always accepts newest device plugin. Later may consider to
+	// add some policies here, e.g., verify whether an old device plugin with the
+	// same resource name is still alive to determine whether we want to accept
+	// the new registration.
+	go m.addEndpoint(r)
+
+	return &pluginapi.Empty{}, nil
+}
+
+// Stop is the function that can stop the gRPC server.
+// Can be called concurrently, more than once, and is safe to call
+// without a prior Start.
+func (m *ManagerImpl) Stop() error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	for _, e := range m.endpoints {
+		e.stop()
+	}
+
+	if m.server == nil {
+		return nil
+	}
+	m.server.Stop()
+	m.wg.Wait()
+	m.server = nil
+	return nil
+}
+
+func (m *ManagerImpl) addEndpoint(r *pluginapi.RegisterRequest) {
+	existingDevs := make(map[string]pluginapi.Device)
+	m.mutex.Lock()
+	old, ok := m.endpoints[r.ResourceName]
+	if ok && old != nil {
+		// Pass devices of previous endpoint into re-registered one,
+		// to avoid potential orphaned devices upon re-registration
+		devices := make(map[string]pluginapi.Device)
+		for _, device := range old.getDevices() {
+			device.Health = pluginapi.Unhealthy
+			devices[device.ID] = device
+		}
+		existingDevs = devices
+	}
+	m.mutex.Unlock()
+
+	socketPath := filepath.Join(m.socketdir, r.Endpoint)
+	e, err := newEndpointImpl(socketPath, r.ResourceName, existingDevs, m.callback)
+	if err != nil {
+		glog.Errorf("Failed to dial device plugin with request %v: %v", r, err)
+		return
+	}
+	m.mutex.Lock()
+	if r.Options != nil {
+		m.pluginOpts[r.ResourceName] = r.Options
+	}
+	// Check for potential re-registration during the initialization of new endpoint,
+	// and skip updating if re-registration happens.
+	// TODO: simplify the part once we have a better way to handle registered devices
+	ext := m.endpoints[r.ResourceName]
+	if ext != old {
+		glog.Warningf("Some other endpoint %v is added while endpoint %v is initialized", ext, e)
+		m.mutex.Unlock()
+		e.stop()
+		return
+	}
+	// Associates the newly created endpoint with the corresponding resource name.
+	// Stops existing endpoint if there is any.
+	m.endpoints[r.ResourceName] = e
+	glog.V(2).Infof("Registered endpoint %v", e)
+	m.mutex.Unlock()
+
+	if old != nil {
+		old.stop()
+	}
+
+	go func() {
+		e.run()
+		e.stop()
+		m.mutex.Lock()
+		if old, ok := m.endpoints[r.ResourceName]; ok && old == e {
+			m.markResourceUnhealthy(r.ResourceName)
+		}
+		glog.V(2).Infof("Unregistered endpoint %v", e)
+		m.mutex.Unlock()
+	}()
+}
+
+func (m *ManagerImpl) markResourceUnhealthy(resourceName string) {
+	glog.V(2).Infof("Mark all resources Unhealthy for resource %s", resourceName)
+	healthyDevices := sets.NewString()
+	if _, ok := m.healthyDevices[resourceName]; ok {
+		healthyDevices = m.healthyDevices[resourceName]
+		m.healthyDevices[resourceName] = sets.NewString()
+	}
+	if _, ok := m.unhealthyDevices[resourceName]; !ok {
+		m.unhealthyDevices[resourceName] = sets.NewString()
+	}
+	m.unhealthyDevices[resourceName] = m.unhealthyDevices[resourceName].Union(healthyDevices)
+}
+
+// GetCapacity is expected to be called when Kubelet updates its node status.
+// The first returned variable contains the registered device plugin resource capacity.
+// The second returned variable contains the registered device plugin resource allocatable.
+// The third returned variable contains previously registered resources that are no longer active.
+// Kubelet uses this information to update resource capacity/allocatable in its node status.
+// After the call, device plugin can remove the inactive resources from its internal list as the
+// change is already reflected in Kubelet node status.
+// Note in the special case after Kubelet restarts, device plugin resource capacities can
+// temporarily drop to zero till corresponding device plugins re-register. This is OK because
+// cm.UpdatePluginResource() run during predicate Admit guarantees we adjust nodeinfo
+// capacity for already allocated pods so that they can continue to run. However, new pods
+// requiring device plugin resources will not be scheduled till device plugin re-registers.
+func (m *ManagerImpl) GetCapacity() (v1.ResourceList, v1.ResourceList, []string) {
+	needsUpdateCheckpoint := false
+	var capacity = v1.ResourceList{}
+	var allocatable = v1.ResourceList{}
+	deletedResources := sets.NewString()
+	m.mutex.Lock()
+	for resourceName, devices := range m.healthyDevices {
+		e, ok := m.endpoints[resourceName]
+		if (ok && e.stopGracePeriodExpired()) || !ok {
+			// The resources contained in endpoints and (un)healthyDevices
+			// should always be consistent. Otherwise, we run with the risk
+			// of failing to garbage collect non-existing resources or devices.
+			if !ok {
+				glog.Errorf("unexpected: healthyDevices and endpoints are out of sync")
+			}
+			delete(m.endpoints, resourceName)
+			delete(m.healthyDevices, resourceName)
+			deletedResources.Insert(resourceName)
+			needsUpdateCheckpoint = true
+		} else {
+			capacity[v1.ResourceName(resourceName)] = *resource.NewQuantity(int64(devices.Len()), resource.DecimalSI)
+			allocatable[v1.ResourceName(resourceName)] = *resource.NewQuantity(int64(devices.Len()), resource.DecimalSI)
+		}
+	}
+	for resourceName, devices := range m.unhealthyDevices {
+		e, ok := m.endpoints[resourceName]
+		if (ok && e.stopGracePeriodExpired()) || !ok {
+			if !ok {
+				glog.Errorf("unexpected: unhealthyDevices and endpoints are out of sync")
+			}
+			delete(m.endpoints, resourceName)
+			delete(m.unhealthyDevices, resourceName)
+			deletedResources.Insert(resourceName)
+			needsUpdateCheckpoint = true
+		} else {
+			capacityCount := capacity[v1.ResourceName(resourceName)]
+			unhealthyCount := *resource.NewQuantity(int64(devices.Len()), resource.DecimalSI)
+			capacityCount.Add(unhealthyCount)
+			capacity[v1.ResourceName(resourceName)] = capacityCount
+		}
+	}
+	m.mutex.Unlock()
+	if needsUpdateCheckpoint {
+		m.writeCheckpoint()
+	}
+	return capacity, allocatable, deletedResources.UnsortedList()
+}
+
+// Checkpoints device to container allocation information to disk.
+func (m *ManagerImpl) writeCheckpoint() error {
+	m.mutex.Lock()
+	registeredDevs := make(map[string][]string)
+	for resource, devices := range m.healthyDevices {
+		registeredDevs[resource] = devices.UnsortedList()
+	}
+	data := checkpoint.New(m.podDevices.toCheckpointData(),
+		registeredDevs)
+	m.mutex.Unlock()
+	err := m.checkpointManager.CreateCheckpoint(kubeletDeviceManagerCheckpoint, data)
+	if err != nil {
+		return fmt.Errorf("failed to write checkpoint file %q: %v", kubeletDeviceManagerCheckpoint, err)
+	}
+	return nil
+}
+
+// Reads device to container allocation information from disk, and populates
+// m.allocatedDevices accordingly.
+func (m *ManagerImpl) readCheckpoint() error {
+	registeredDevs := make(map[string][]string)
+	devEntries := make([]checkpoint.PodDevicesEntry, 0)
+	cp := checkpoint.New(devEntries, registeredDevs)
+	err := m.checkpointManager.GetCheckpoint(kubeletDeviceManagerCheckpoint, cp)
+	if err != nil {
+		if err == errors.ErrCheckpointNotFound {
+			glog.Warningf("Failed to retrieve checkpoint for %q: %v", kubeletDeviceManagerCheckpoint, err)
+			return nil
+		}
+		return err
+	}
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	podDevices, registeredDevs := cp.GetData()
+	m.podDevices.fromCheckpointData(podDevices)
+	m.allocatedDevices = m.podDevices.devices()
+	for resource := range registeredDevs {
+		// During start up, creates empty healthyDevices list so that the resource capacity
+		// will stay zero till the corresponding device plugin re-registers.
+		m.healthyDevices[resource] = sets.NewString()
+		m.unhealthyDevices[resource] = sets.NewString()
+		m.endpoints[resource] = newStoppedEndpointImpl(resource, make(map[string]pluginapi.Device))
+	}
+	return nil
+}
+
+// updateAllocatedDevices gets a list of active pods and then frees any Devices that are bound to
+// terminated pods. Returns error on failure.
+func (m *ManagerImpl) updateAllocatedDevices(activePods []*v1.Pod) {
+	if !m.sourcesReady.AllReady() {
+		return
+	}
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	activePodUids := sets.NewString()
+	for _, pod := range activePods {
+		activePodUids.Insert(string(pod.UID))
+	}
+	allocatedPodUids := m.podDevices.pods()
+	podsToBeRemoved := allocatedPodUids.Difference(activePodUids)
+	if len(podsToBeRemoved) <= 0 {
+		return
+	}
+	glog.V(3).Infof("pods to be removed: %v", podsToBeRemoved.List())
+	m.podDevices.delete(podsToBeRemoved.List())
+	// Regenerated allocatedDevices after we update pod allocation information.
+	m.allocatedDevices = m.podDevices.devices()
+}
+
+// Returns list of device Ids we need to allocate with Allocate rpc call.
+// Returns empty list in case we don't need to issue the Allocate rpc call.
+func (m *ManagerImpl) devicesToAllocate(podUID, contName, resource string, required int, reusableDevices sets.String) (sets.String, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	needed := required
+	// Gets list of devices that have already been allocated.
+	// This can happen if a container restarts for example.
+	devices := m.podDevices.containerDevices(podUID, contName, resource)
+	if devices != nil {
+		glog.V(3).Infof("Found pre-allocated devices for resource %s container %q in Pod %q: %v", resource, contName, podUID, devices.List())
+		needed = needed - devices.Len()
+		// A pod's resource is not expected to change once admitted by the API server,
+		// so just fail loudly here. We can revisit this part if this no longer holds.
+		if needed != 0 {
+			return nil, fmt.Errorf("pod %v container %v changed request for resource %v from %v to %v", podUID, contName, resource, devices.Len(), required)
+		}
+	}
+	if needed == 0 {
+		// No change, no work.
+		return nil, nil
+	}
+	glog.V(3).Infof("Needs to allocate %v %v for pod %q container %q", needed, resource, podUID, contName)
+	// Needs to allocate additional devices.
+	if _, ok := m.healthyDevices[resource]; !ok {
+		return nil, fmt.Errorf("can't allocate unregistered device %v", resource)
+	}
+	devices = sets.NewString()
+	// Allocates from reusableDevices list first.
+	for device := range reusableDevices {
+		devices.Insert(device)
+		needed--
+		if needed == 0 {
+			return devices, nil
+		}
+	}
+	// Needs to allocate additional devices.
+	if m.allocatedDevices[resource] == nil {
+		m.allocatedDevices[resource] = sets.NewString()
+	}
+	// Gets Devices in use.
+	devicesInUse := m.allocatedDevices[resource]
+	// Gets a list of available devices.
+	available := m.healthyDevices[resource].Difference(devicesInUse)
+	if int(available.Len()) < needed {
+		return nil, fmt.Errorf("requested number of devices unavailable for %s. Requested: %d, Available: %d", resource, needed, available.Len())
+	}
+	allocated := available.UnsortedList()[:needed]
+	// Updates m.allocatedDevices with allocated devices to prevent them
+	// from being allocated to other pods/containers, given that we are
+	// not holding lock during the rpc call.
+	for _, device := range allocated {
+		m.allocatedDevices[resource].Insert(device)
+		devices.Insert(device)
+	}
+	return devices, nil
+}
+
+// allocateContainerResources attempts to allocate all of required device
+// plugin resources for the input container, issues an Allocate rpc request
+// for each new device resource requirement, processes their AllocateResponses,
+// and updates the cached containerDevices on success.
+func (m *ManagerImpl) allocateContainerResources(pod *v1.Pod, container *v1.Container, devicesToReuse map[string]sets.String) error {
+	podUID := string(pod.UID)
+	contName := container.Name
+	allocatedDevicesUpdated := false
+	// Extended resources are not allowed to be overcommitted.
+	// Since device plugin advertises extended resources,
+	// therefore Requests must be equal to Limits and iterating
+	// over the Limits should be sufficient.
+	for k, v := range container.Resources.Limits {
+		resource := string(k)
+		needed := int(v.Value())
+		glog.V(3).Infof("needs %d %s", needed, resource)
+		if !m.isDevicePluginResource(resource) {
+			continue
+		}
+		// Updates allocatedDevices to garbage collect any stranded resources
+		// before doing the device plugin allocation.
+		if !allocatedDevicesUpdated {
+			m.updateAllocatedDevices(m.activePods())
+			allocatedDevicesUpdated = true
+		}
+		allocDevices, err := m.devicesToAllocate(podUID, contName, resource, needed, devicesToReuse[resource])
+		if err != nil {
+			return err
+		}
+		if allocDevices == nil || len(allocDevices) <= 0 {
+			continue
+		}
+
+		startRPCTime := time.Now()
+		// Manager.Allocate involves RPC calls to device plugin, which
+		// could be heavy-weight. Therefore we want to perform this operation outside
+		// mutex lock. Note if Allocate call fails, we may leave container resources
+		// partially allocated for the failed container. We rely on updateAllocatedDevices()
+		// to garbage collect these resources later. Another side effect is that if
+		// we have X resource A and Y resource B in total, and two containers, container1
+		// and container2 both require X resource A and Y resource B. Both allocation
+		// requests may fail if we serve them in mixed order.
+		// TODO: may revisit this part later if we see inefficient resource allocation
+		// in real use as the result of this. Should also consider to parallize device
+		// plugin Allocate grpc calls if it becomes common that a container may require
+		// resources from multiple device plugins.
+		m.mutex.Lock()
+		e, ok := m.endpoints[resource]
+		m.mutex.Unlock()
+		if !ok {
+			m.mutex.Lock()
+			m.allocatedDevices = m.podDevices.devices()
+			m.mutex.Unlock()
+			return fmt.Errorf("Unknown Device Plugin %s", resource)
+		}
+
+		devs := allocDevices.UnsortedList()
+		// TODO: refactor this part of code to just append a ContainerAllocationRequest
+		// in a passed in AllocateRequest pointer, and issues a single Allocate call per pod.
+		glog.V(3).Infof("Making allocation request for devices %v for device plugin %s", devs, resource)
+		resp, err := e.allocate(devs)
+		metrics.DevicePluginAllocationLatency.WithLabelValues(resource).Observe(metrics.SinceInMicroseconds(startRPCTime))
+		if err != nil {
+			// In case of allocation failure, we want to restore m.allocatedDevices
+			// to the actual allocated state from m.podDevices.
+			m.mutex.Lock()
+			m.allocatedDevices = m.podDevices.devices()
+			m.mutex.Unlock()
+			return err
+		}
+
+		// Update internal cached podDevices state.
+		m.mutex.Lock()
+		m.podDevices.insert(podUID, contName, resource, allocDevices, resp.ContainerResponses[0])
+		m.mutex.Unlock()
+	}
+
+	// Checkpoints device to container allocation information.
+	return m.writeCheckpoint()
+}
+
+// GetDeviceRunContainerOptions checks whether we have cached containerDevices
+// for the passed-in <pod, container> and returns its DeviceRunContainerOptions
+// for the found one. An empty struct is returned in case no cached state is found.
+func (m *ManagerImpl) GetDeviceRunContainerOptions(pod *v1.Pod, container *v1.Container) (*DeviceRunContainerOptions, error) {
+	podUID := string(pod.UID)
+	contName := container.Name
+	for k := range container.Resources.Limits {
+		resource := string(k)
+		if !m.isDevicePluginResource(resource) {
+			continue
+		}
+		err := m.callPreStartContainerIfNeeded(podUID, contName, resource)
+		if err != nil {
+			return nil, err
+		}
+	}
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	return m.podDevices.deviceRunContainerOptions(string(pod.UID), container.Name), nil
+}
+
+// callPreStartContainerIfNeeded issues PreStartContainer grpc call for device plugin resource
+// with PreStartRequired option set.
+func (m *ManagerImpl) callPreStartContainerIfNeeded(podUID, contName, resource string) error {
+	m.mutex.Lock()
+	opts, ok := m.pluginOpts[resource]
+	if !ok {
+		m.mutex.Unlock()
+		glog.V(4).Infof("Plugin options not found in cache for resource: %s. Skip PreStartContainer", resource)
+		return nil
+	}
+
+	if !opts.PreStartRequired {
+		m.mutex.Unlock()
+		glog.V(4).Infof("Plugin options indicate to skip PreStartContainer for resource, %v", resource)
+		return nil
+	}
+
+	devices := m.podDevices.containerDevices(podUID, contName, resource)
+	if devices == nil {
+		m.mutex.Unlock()
+		return fmt.Errorf("no devices found allocated in local cache for pod %s, container %s, resource %s", podUID, contName, resource)
+	}
+
+	e, ok := m.endpoints[resource]
+	if !ok {
+		m.mutex.Unlock()
+		return fmt.Errorf("endpoint not found in cache for a registered resource: %s", resource)
+	}
+
+	m.mutex.Unlock()
+	devs := devices.UnsortedList()
+	glog.V(4).Infof("Issuing an PreStartContainer call for container, %s, of pod %s", contName, podUID)
+	_, err := e.preStartContainer(devs)
+	if err != nil {
+		return fmt.Errorf("device plugin PreStartContainer rpc failed with err: %v", err)
+	}
+	// TODO: Add metrics support for init RPC
+	return nil
+}
+
+// sanitizeNodeAllocatable scans through allocatedDevices in the device manager
+// and if necessary, updates allocatableResource in nodeInfo to at least equal to
+// the allocated capacity. This allows pods that have already been scheduled on
+// the node to pass GeneralPredicates admission checking even upon device plugin failure.
+func (m *ManagerImpl) sanitizeNodeAllocatable(node *schedulercache.NodeInfo) {
+	var newAllocatableResource *schedulercache.Resource
+	allocatableResource := node.AllocatableResource()
+	if allocatableResource.ScalarResources == nil {
+		allocatableResource.ScalarResources = make(map[v1.ResourceName]int64)
+	}
+	for resource, devices := range m.allocatedDevices {
+		needed := devices.Len()
+		quant, ok := allocatableResource.ScalarResources[v1.ResourceName(resource)]
+		if ok && int(quant) >= needed {
+			continue
+		}
+		// Needs to update nodeInfo.AllocatableResource to make sure
+		// NodeInfo.allocatableResource at least equal to the capacity already allocated.
+		if newAllocatableResource == nil {
+			newAllocatableResource = allocatableResource.Clone()
+		}
+		newAllocatableResource.ScalarResources[v1.ResourceName(resource)] = int64(needed)
+	}
+	if newAllocatableResource != nil {
+		node.SetAllocatableResource(newAllocatableResource)
+	}
+}
+
+func (m *ManagerImpl) isDevicePluginResource(resource string) bool {
+	_, registeredResource := m.healthyDevices[resource]
+	_, allocatedResource := m.allocatedDevices[resource]
+	// Return true if this is either an active device plugin resource or
+	// a resource we have previously allocated.
+	if registeredResource || allocatedResource {
+		return true
+	}
+	return false
+}

vendor/k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/manager_stub.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package devicemanager
+
+import (
+	"k8s.io/api/core/v1"
+	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
+	"k8s.io/kubernetes/pkg/kubelet/config"
+	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
+	schedulercache "k8s.io/kubernetes/pkg/scheduler/cache"
+)
+
+// ManagerStub provides a simple stub implementation for the Device Manager.
+type ManagerStub struct{}
+
+// NewManagerStub creates a ManagerStub.
+func NewManagerStub() (*ManagerStub, error) {
+	return &ManagerStub{}, nil
+}
+
+// Start simply returns nil.
+func (h *ManagerStub) Start(activePods ActivePodsFunc, sourcesReady config.SourcesReady) error {
+	return nil
+}
+
+// Stop simply returns nil.
+func (h *ManagerStub) Stop() error {
+	return nil
+}
+
+// Devices returns an empty map.
+func (h *ManagerStub) Devices() map[string][]pluginapi.Device {
+	return make(map[string][]pluginapi.Device)
+}
+
+// Allocate simply returns nil.
+func (h *ManagerStub) Allocate(node *schedulercache.NodeInfo, attrs *lifecycle.PodAdmitAttributes) error {
+	return nil
+}
+
+// GetDeviceRunContainerOptions simply returns nil.
+func (h *ManagerStub) GetDeviceRunContainerOptions(pod *v1.Pod, container *v1.Container) (*DeviceRunContainerOptions, error) {
+	return nil, nil
+}
+
+// GetCapacity simply returns nil capacity and empty removed resource list.
+func (h *ManagerStub) GetCapacity() (v1.ResourceList, v1.ResourceList, []string) {
+	return nil, nil, []string{}
+}

vendor/k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/manager_test.go

@@ -0,0 +1,912 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package devicemanager
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"reflect"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
+	"k8s.io/kubernetes/pkg/kubelet/checkpointmanager"
+	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
+	schedulercache "k8s.io/kubernetes/pkg/scheduler/cache"
+)
+
+const (
+	testResourceName = "fake-domain/resource"
+)
+
+func tmpSocketDir() (socketDir, socketName, pluginSocketName string, err error) {
+	socketDir, err = ioutil.TempDir("", "device_plugin")
+	if err != nil {
+		return
+	}
+	socketName = socketDir + "/server.sock"
+	pluginSocketName = socketDir + "/device-plugin.sock"
+	return
+}
+
+func TestNewManagerImpl(t *testing.T) {
+	socketDir, socketName, _, err := tmpSocketDir()
+	require.NoError(t, err)
+	defer os.RemoveAll(socketDir)
+	_, err = newManagerImpl(socketName)
+	require.NoError(t, err)
+	os.RemoveAll(socketDir)
+}
+
+func TestNewManagerImplStart(t *testing.T) {
+	socketDir, socketName, pluginSocketName, err := tmpSocketDir()
+	require.NoError(t, err)
+	defer os.RemoveAll(socketDir)
+	m, p := setup(t, []*pluginapi.Device{}, func(n string, a, u, r []pluginapi.Device) {}, socketName, pluginSocketName)
+	cleanup(t, m, p)
+	// Stop should tolerate being called more than once.
+	cleanup(t, m, p)
+}
+
+func TestNewManagerImplStop(t *testing.T) {
+	socketDir, socketName, pluginSocketName, err := tmpSocketDir()
+	require.NoError(t, err)
+	defer os.RemoveAll(socketDir)
+
+	m, err := newManagerImpl(socketName)
+	require.NoError(t, err)
+	// No prior Start, but that should be okay.
+	err = m.Stop()
+	require.NoError(t, err)
+
+	devs := []*pluginapi.Device{
+		{ID: "Dev1", Health: pluginapi.Healthy},
+		{ID: "Dev2", Health: pluginapi.Healthy},
+	}
+	p := NewDevicePluginStub(devs, pluginSocketName)
+	// Same here.
+	err = p.Stop()
+	require.NoError(t, err)
+}
+
+// Tests that the device plugin manager correctly handles registration and re-registration by
+// making sure that after registration, devices are correctly updated and if a re-registration
+// happens, we will NOT delete devices; and no orphaned devices left.
+func TestDevicePluginReRegistration(t *testing.T) {
+	socketDir, socketName, pluginSocketName, err := tmpSocketDir()
+	require.NoError(t, err)
+	defer os.RemoveAll(socketDir)
+	devs := []*pluginapi.Device{
+		{ID: "Dev1", Health: pluginapi.Healthy},
+		{ID: "Dev2", Health: pluginapi.Healthy},
+	}
+	devsForRegistration := []*pluginapi.Device{
+		{ID: "Dev3", Health: pluginapi.Healthy},
+	}
+	for _, preStartContainerFlag := range []bool{false, true} {
+
+		expCallbackCount := int32(0)
+		callbackCount := int32(0)
+		callbackChan := make(chan int32)
+		callback := func(n string, a, u, r []pluginapi.Device) {
+			callbackCount++
+			if callbackCount > atomic.LoadInt32(&expCallbackCount) {
+				t.FailNow()
+			}
+			callbackChan <- callbackCount
+		}
+		m, p1 := setup(t, devs, callback, socketName, pluginSocketName)
+		atomic.StoreInt32(&expCallbackCount, 1)
+		p1.Register(socketName, testResourceName, preStartContainerFlag)
+		// Wait for the first callback to be issued.
+
+		select {
+		case <-callbackChan:
+			break
+		case <-time.After(time.Second):
+			t.FailNow()
+		}
+		devices := m.Devices()
+		require.Equal(t, 2, len(devices[testResourceName]), "Devices are not updated.")
+
+		p2 := NewDevicePluginStub(devs, pluginSocketName+".new")
+		err = p2.Start()
+		require.NoError(t, err)
+		atomic.StoreInt32(&expCallbackCount, 2)
+		p2.Register(socketName, testResourceName, preStartContainerFlag)
+		// Wait for the second callback to be issued.
+		select {
+		case <-callbackChan:
+			break
+		case <-time.After(time.Second):
+			t.FailNow()
+		}
+
+		devices2 := m.Devices()
+		require.Equal(t, 2, len(devices2[testResourceName]), "Devices shouldn't change.")
+
+		// Test the scenario that a plugin re-registers with different devices.
+		p3 := NewDevicePluginStub(devsForRegistration, pluginSocketName+".third")
+		err = p3.Start()
+		require.NoError(t, err)
+		atomic.StoreInt32(&expCallbackCount, 3)
+		p3.Register(socketName, testResourceName, preStartContainerFlag)
+		// Wait for the second callback to be issued.
+		select {
+		case <-callbackChan:
+			break
+		case <-time.After(time.Second):
+			t.FailNow()
+		}
+		devices3 := m.Devices()
+		require.Equal(t, 1, len(devices3[testResourceName]), "Devices of plugin previously registered should be removed.")
+		p2.Stop()
+		p3.Stop()
+		cleanup(t, m, p1)
+		close(callbackChan)
+	}
+}
+
+func setup(t *testing.T, devs []*pluginapi.Device, callback monitorCallback, socketName string, pluginSocketName string) (Manager, *Stub) {
+	m, err := newManagerImpl(socketName)
+	require.NoError(t, err)
+
+	m.callback = callback
+
+	activePods := func() []*v1.Pod {
+		return []*v1.Pod{}
+	}
+	err = m.Start(activePods, &sourcesReadyStub{})
+	require.NoError(t, err)
+
+	p := NewDevicePluginStub(devs, pluginSocketName)
+	err = p.Start()
+	require.NoError(t, err)
+
+	return m, p
+}
+
+func cleanup(t *testing.T, m Manager, p *Stub) {
+	p.Stop()
+	m.Stop()
+}
+
+func TestUpdateCapacityAllocatable(t *testing.T) {
+	socketDir, socketName, _, err := tmpSocketDir()
+	require.NoError(t, err)
+	defer os.RemoveAll(socketDir)
+	testManager, err := newManagerImpl(socketName)
+	as := assert.New(t)
+	as.NotNil(testManager)
+	as.Nil(err)
+
+	devs := []pluginapi.Device{
+		{ID: "Device1", Health: pluginapi.Healthy},
+		{ID: "Device2", Health: pluginapi.Healthy},
+		{ID: "Device3", Health: pluginapi.Unhealthy},
+	}
+	callback := testManager.genericDeviceUpdateCallback
+
+	// Adds three devices for resource1, two healthy and one unhealthy.
+	// Expects capacity for resource1 to be 2.
+	resourceName1 := "domain1.com/resource1"
+	e1 := &endpointImpl{devices: make(map[string]pluginapi.Device)}
+	testManager.endpoints[resourceName1] = e1
+	callback(resourceName1, devs, []pluginapi.Device{}, []pluginapi.Device{})
+	capacity, allocatable, removedResources := testManager.GetCapacity()
+	resource1Capacity, ok := capacity[v1.ResourceName(resourceName1)]
+	as.True(ok)
+	resource1Allocatable, ok := allocatable[v1.ResourceName(resourceName1)]
+	as.True(ok)
+	as.Equal(int64(3), resource1Capacity.Value())
+	as.Equal(int64(2), resource1Allocatable.Value())
+	as.Equal(0, len(removedResources))
+
+	// Deletes an unhealthy device should NOT change allocatable but change capacity.
+	callback(resourceName1, []pluginapi.Device{}, []pluginapi.Device{}, []pluginapi.Device{devs[2]})
+	capacity, allocatable, removedResources = testManager.GetCapacity()
+	resource1Capacity, ok = capacity[v1.ResourceName(resourceName1)]
+	as.True(ok)
+	resource1Allocatable, ok = allocatable[v1.ResourceName(resourceName1)]
+	as.True(ok)
+	as.Equal(int64(2), resource1Capacity.Value())
+	as.Equal(int64(2), resource1Allocatable.Value())
+	as.Equal(0, len(removedResources))
+
+	// Updates a healthy device to unhealthy should reduce allocatable by 1.
+	dev2 := devs[1]
+	dev2.Health = pluginapi.Unhealthy
+	callback(resourceName1, []pluginapi.Device{}, []pluginapi.Device{dev2}, []pluginapi.Device{})
+	capacity, allocatable, removedResources = testManager.GetCapacity()
+	resource1Capacity, ok = capacity[v1.ResourceName(resourceName1)]
+	as.True(ok)
+	resource1Allocatable, ok = allocatable[v1.ResourceName(resourceName1)]
+	as.True(ok)
+	as.Equal(int64(2), resource1Capacity.Value())
+	as.Equal(int64(1), resource1Allocatable.Value())
+	as.Equal(0, len(removedResources))
+
+	// Deletes a healthy device should reduce capacity and allocatable by 1.
+	callback(resourceName1, []pluginapi.Device{}, []pluginapi.Device{}, []pluginapi.Device{devs[0]})
+	capacity, allocatable, removedResources = testManager.GetCapacity()
+	resource1Capacity, ok = capacity[v1.ResourceName(resourceName1)]
+	as.True(ok)
+	resource1Allocatable, ok = allocatable[v1.ResourceName(resourceName1)]
+	as.True(ok)
+	as.Equal(int64(0), resource1Allocatable.Value())
+	as.Equal(int64(1), resource1Capacity.Value())
+	as.Equal(0, len(removedResources))
+
+	// Tests adding another resource.
+	resourceName2 := "resource2"
+	e2 := &endpointImpl{devices: make(map[string]pluginapi.Device)}
+	testManager.endpoints[resourceName2] = e2
+	callback(resourceName2, devs, []pluginapi.Device{}, []pluginapi.Device{})
+	capacity, allocatable, removedResources = testManager.GetCapacity()
+	as.Equal(2, len(capacity))
+	resource2Capacity, ok := capacity[v1.ResourceName(resourceName2)]
+	as.True(ok)
+	resource2Allocatable, ok := allocatable[v1.ResourceName(resourceName2)]
+	as.True(ok)
+	as.Equal(int64(3), resource2Capacity.Value())
+	as.Equal(int64(2), resource2Allocatable.Value())
+	as.Equal(0, len(removedResources))
+
+	// Expires resourceName1 endpoint. Verifies testManager.GetCapacity() reports that resourceName1
+	// is removed from capacity and it no longer exists in healthyDevices after the call.
+	e1.setStopTime(time.Now().Add(-1*endpointStopGracePeriod - time.Duration(10)*time.Second))
+	capacity, allocatable, removed := testManager.GetCapacity()
+	as.Equal([]string{resourceName1}, removed)
+	_, ok = capacity[v1.ResourceName(resourceName1)]
+	as.False(ok)
+	val, ok := capacity[v1.ResourceName(resourceName2)]
+	as.True(ok)
+	as.Equal(int64(3), val.Value())
+	_, ok = testManager.healthyDevices[resourceName1]
+	as.False(ok)
+	_, ok = testManager.unhealthyDevices[resourceName1]
+	as.False(ok)
+	_, ok = testManager.endpoints[resourceName1]
+	as.False(ok)
+	as.Equal(1, len(testManager.endpoints))
+
+	// Stops resourceName2 endpoint. Verifies its stopTime is set, allocate and
+	// preStartContainer calls return errors.
+	e2.stop()
+	as.False(e2.stopTime.IsZero())
+	_, err = e2.allocate([]string{"Device1"})
+	reflect.DeepEqual(err, fmt.Errorf(errEndpointStopped, e2))
+	_, err = e2.preStartContainer([]string{"Device1"})
+	reflect.DeepEqual(err, fmt.Errorf(errEndpointStopped, e2))
+	// Marks resourceName2 unhealthy and verifies its capacity/allocatable are
+	// correctly updated.
+	testManager.markResourceUnhealthy(resourceName2)
+	capacity, allocatable, removed = testManager.GetCapacity()
+	val, ok = capacity[v1.ResourceName(resourceName2)]
+	as.True(ok)
+	as.Equal(int64(3), val.Value())
+	val, ok = allocatable[v1.ResourceName(resourceName2)]
+	as.True(ok)
+	as.Equal(int64(0), val.Value())
+	as.Empty(removed)
+	// Writes and re-reads checkpoints. Verifies we create a stopped endpoint
+	// for resourceName2, its capacity is set to zero, and we still consider
+	// it as a DevicePlugin resource. This makes sure any pod that was scheduled
+	// during the time of propagating capacity change to the scheduler will be
+	// properly rejected instead of being incorrectly started.
+	err = testManager.writeCheckpoint()
+	as.Nil(err)
+	testManager.healthyDevices = make(map[string]sets.String)
+	testManager.unhealthyDevices = make(map[string]sets.String)
+	err = testManager.readCheckpoint()
+	as.Nil(err)
+	as.Equal(1, len(testManager.endpoints))
+	_, ok = testManager.endpoints[resourceName2]
+	as.True(ok)
+	capacity, allocatable, removed = testManager.GetCapacity()
+	val, ok = capacity[v1.ResourceName(resourceName2)]
+	as.True(ok)
+	as.Equal(int64(0), val.Value())
+	as.Empty(removed)
+	as.True(testManager.isDevicePluginResource(resourceName2))
+}
+
+func constructDevices(devices []string) sets.String {
+	ret := sets.NewString()
+	for _, dev := range devices {
+		ret.Insert(dev)
+	}
+	return ret
+}
+
+func constructAllocResp(devices, mounts, envs map[string]string) *pluginapi.ContainerAllocateResponse {
+	resp := &pluginapi.ContainerAllocateResponse{}
+	for k, v := range devices {
+		resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
+			HostPath:      k,
+			ContainerPath: v,
+			Permissions:   "mrw",
+		})
+	}
+	for k, v := range mounts {
+		resp.Mounts = append(resp.Mounts, &pluginapi.Mount{
+			ContainerPath: k,
+			HostPath:      v,
+			ReadOnly:      true,
+		})
+	}
+	resp.Envs = make(map[string]string)
+	for k, v := range envs {
+		resp.Envs[k] = v
+	}
+	return resp
+}
+
+func TestCheckpoint(t *testing.T) {
+	resourceName1 := "domain1.com/resource1"
+	resourceName2 := "domain2.com/resource2"
+	as := assert.New(t)
+	tmpDir, err := ioutil.TempDir("", "checkpoint")
+	as.Nil(err)
+	ckm, err := checkpointmanager.NewCheckpointManager(tmpDir)
+	as.Nil(err)
+	testManager := &ManagerImpl{
+		endpoints:         make(map[string]endpoint),
+		healthyDevices:    make(map[string]sets.String),
+		unhealthyDevices:  make(map[string]sets.String),
+		allocatedDevices:  make(map[string]sets.String),
+		podDevices:        make(podDevices),
+		checkpointManager: ckm,
+	}
+
+	testManager.podDevices.insert("pod1", "con1", resourceName1,
+		constructDevices([]string{"dev1", "dev2"}),
+		constructAllocResp(map[string]string{"/dev/r1dev1": "/dev/r1dev1", "/dev/r1dev2": "/dev/r1dev2"},
+			map[string]string{"/home/r1lib1": "/usr/r1lib1"}, map[string]string{}))
+	testManager.podDevices.insert("pod1", "con1", resourceName2,
+		constructDevices([]string{"dev1", "dev2"}),
+		constructAllocResp(map[string]string{"/dev/r2dev1": "/dev/r2dev1", "/dev/r2dev2": "/dev/r2dev2"},
+			map[string]string{"/home/r2lib1": "/usr/r2lib1"},
+			map[string]string{"r2devices": "dev1 dev2"}))
+	testManager.podDevices.insert("pod1", "con2", resourceName1,
+		constructDevices([]string{"dev3"}),
+		constructAllocResp(map[string]string{"/dev/r1dev3": "/dev/r1dev3"},
+			map[string]string{"/home/r1lib1": "/usr/r1lib1"}, map[string]string{}))
+	testManager.podDevices.insert("pod2", "con1", resourceName1,
+		constructDevices([]string{"dev4"}),
+		constructAllocResp(map[string]string{"/dev/r1dev4": "/dev/r1dev4"},
+			map[string]string{"/home/r1lib1": "/usr/r1lib1"}, map[string]string{}))
+
+	testManager.healthyDevices[resourceName1] = sets.NewString()
+	testManager.healthyDevices[resourceName1].Insert("dev1")
+	testManager.healthyDevices[resourceName1].Insert("dev2")
+	testManager.healthyDevices[resourceName1].Insert("dev3")
+	testManager.healthyDevices[resourceName1].Insert("dev4")
+	testManager.healthyDevices[resourceName1].Insert("dev5")
+	testManager.healthyDevices[resourceName2] = sets.NewString()
+	testManager.healthyDevices[resourceName2].Insert("dev1")
+	testManager.healthyDevices[resourceName2].Insert("dev2")
+
+	expectedPodDevices := testManager.podDevices
+	expectedAllocatedDevices := testManager.podDevices.devices()
+	expectedAllDevices := testManager.healthyDevices
+
+	err = testManager.writeCheckpoint()
+
+	as.Nil(err)
+	testManager.podDevices = make(podDevices)
+	err = testManager.readCheckpoint()
+	as.Nil(err)
+
+	as.Equal(len(expectedPodDevices), len(testManager.podDevices))
+	for podUID, containerDevices := range expectedPodDevices {
+		for conName, resources := range containerDevices {
+			for resource := range resources {
+				expDevices := expectedPodDevices.containerDevices(podUID, conName, resource)
+				testDevices := testManager.podDevices.containerDevices(podUID, conName, resource)
+				as.True(reflect.DeepEqual(expDevices, testDevices))
+				opts1 := expectedPodDevices.deviceRunContainerOptions(podUID, conName)
+				opts2 := testManager.podDevices.deviceRunContainerOptions(podUID, conName)
+				as.Equal(len(opts1.Envs), len(opts2.Envs))
+				as.Equal(len(opts1.Mounts), len(opts2.Mounts))
+				as.Equal(len(opts1.Devices), len(opts2.Devices))
+			}
+		}
+	}
+	as.True(reflect.DeepEqual(expectedAllocatedDevices, testManager.allocatedDevices))
+	as.True(reflect.DeepEqual(expectedAllDevices, testManager.healthyDevices))
+}
+
+type activePodsStub struct {
+	activePods []*v1.Pod
+}
+
+func (a *activePodsStub) getActivePods() []*v1.Pod {
+	return a.activePods
+}
+
+func (a *activePodsStub) updateActivePods(newPods []*v1.Pod) {
+	a.activePods = newPods
+}
+
+type MockEndpoint struct {
+	allocateFunc func(devs []string) (*pluginapi.AllocateResponse, error)
+	initChan     chan []string
+}
+
+func (m *MockEndpoint) stop() {}
+func (m *MockEndpoint) run()  {}
+
+func (m *MockEndpoint) getDevices() []pluginapi.Device {
+	return []pluginapi.Device{}
+}
+
+func (m *MockEndpoint) callback(resourceName string, added, updated, deleted []pluginapi.Device) {}
+
+func (m *MockEndpoint) preStartContainer(devs []string) (*pluginapi.PreStartContainerResponse, error) {
+	m.initChan <- devs
+	return &pluginapi.PreStartContainerResponse{}, nil
+}
+
+func (m *MockEndpoint) allocate(devs []string) (*pluginapi.AllocateResponse, error) {
+	if m.allocateFunc != nil {
+		return m.allocateFunc(devs)
+	}
+	return nil, nil
+}
+
+func (m *MockEndpoint) isStopped() bool { return false }
+
+func (m *MockEndpoint) stopGracePeriodExpired() bool { return false }
+
+func makePod(limits v1.ResourceList) *v1.Pod {
+	return &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			UID: uuid.NewUUID(),
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Resources: v1.ResourceRequirements{
+						Limits: limits,
+					},
+				},
+			},
+		},
+	}
+}
+
+func getTestManager(tmpDir string, activePods ActivePodsFunc, testRes []TestResource, opts map[string]*pluginapi.DevicePluginOptions) (*ManagerImpl, error) {
+	monitorCallback := func(resourceName string, added, updated, deleted []pluginapi.Device) {}
+	ckm, err := checkpointmanager.NewCheckpointManager(tmpDir)
+	if err != nil {
+		return nil, err
+	}
+	testManager := &ManagerImpl{
+		socketdir:         tmpDir,
+		callback:          monitorCallback,
+		healthyDevices:    make(map[string]sets.String),
+		unhealthyDevices:  make(map[string]sets.String),
+		allocatedDevices:  make(map[string]sets.String),
+		endpoints:         make(map[string]endpoint),
+		pluginOpts:        opts,
+		podDevices:        make(podDevices),
+		activePods:        activePods,
+		sourcesReady:      &sourcesReadyStub{},
+		checkpointManager: ckm,
+	}
+	for _, res := range testRes {
+		testManager.healthyDevices[res.resourceName] = sets.NewString()
+		for _, dev := range res.devs {
+			testManager.healthyDevices[res.resourceName].Insert(dev)
+		}
+		if res.resourceName == "domain1.com/resource1" {
+			testManager.endpoints[res.resourceName] = &MockEndpoint{
+				allocateFunc: allocateStubFunc(),
+			}
+		}
+		if res.resourceName == "domain2.com/resource2" {
+			testManager.endpoints[res.resourceName] = &MockEndpoint{
+				allocateFunc: func(devs []string) (*pluginapi.AllocateResponse, error) {
+					resp := new(pluginapi.ContainerAllocateResponse)
+					resp.Envs = make(map[string]string)
+					for _, dev := range devs {
+						switch dev {
+						case "dev3":
+							resp.Envs["key2"] = "val2"
+
+						case "dev4":
+							resp.Envs["key2"] = "val3"
+						}
+					}
+					resps := new(pluginapi.AllocateResponse)
+					resps.ContainerResponses = append(resps.ContainerResponses, resp)
+					return resps, nil
+				},
+			}
+		}
+	}
+	return testManager, nil
+}
+
+func getTestNodeInfo(allocatable v1.ResourceList) *schedulercache.NodeInfo {
+	cachedNode := &v1.Node{
+		Status: v1.NodeStatus{
+			Allocatable: allocatable,
+		},
+	}
+	nodeInfo := &schedulercache.NodeInfo{}
+	nodeInfo.SetNode(cachedNode)
+	return nodeInfo
+}
+
+type TestResource struct {
+	resourceName     string
+	resourceQuantity resource.Quantity
+	devs             []string
+}
+
+func TestPodContainerDeviceAllocation(t *testing.T) {
+	res1 := TestResource{
+		resourceName:     "domain1.com/resource1",
+		resourceQuantity: *resource.NewQuantity(int64(2), resource.DecimalSI),
+		devs:             []string{"dev1", "dev2"},
+	}
+	res2 := TestResource{
+		resourceName:     "domain2.com/resource2",
+		resourceQuantity: *resource.NewQuantity(int64(1), resource.DecimalSI),
+		devs:             []string{"dev3", "dev4"},
+	}
+	testResources := make([]TestResource, 2)
+	testResources = append(testResources, res1)
+	testResources = append(testResources, res2)
+	as := require.New(t)
+	podsStub := activePodsStub{
+		activePods: []*v1.Pod{},
+	}
+	tmpDir, err := ioutil.TempDir("", "checkpoint")
+	as.Nil(err)
+	defer os.RemoveAll(tmpDir)
+	nodeInfo := getTestNodeInfo(v1.ResourceList{})
+	pluginOpts := make(map[string]*pluginapi.DevicePluginOptions)
+	testManager, err := getTestManager(tmpDir, podsStub.getActivePods, testResources, pluginOpts)
+	as.Nil(err)
+
+	testPods := []*v1.Pod{
+		makePod(v1.ResourceList{
+			v1.ResourceName(res1.resourceName): res1.resourceQuantity,
+			v1.ResourceName("cpu"):             res1.resourceQuantity,
+			v1.ResourceName(res2.resourceName): res2.resourceQuantity}),
+		makePod(v1.ResourceList{
+			v1.ResourceName(res1.resourceName): res2.resourceQuantity}),
+		makePod(v1.ResourceList{
+			v1.ResourceName(res2.resourceName): res2.resourceQuantity}),
+	}
+	testCases := []struct {
+		description               string
+		testPod                   *v1.Pod
+		expectedContainerOptsLen  []int
+		expectedAllocatedResName1 int
+		expectedAllocatedResName2 int
+		expErr                    error
+	}{
+		{
+			description:               "Successful allocation of two Res1 resources and one Res2 resource",
+			testPod:                   testPods[0],
+			expectedContainerOptsLen:  []int{3, 2, 2},
+			expectedAllocatedResName1: 2,
+			expectedAllocatedResName2: 1,
+			expErr: nil,
+		},
+		{
+			description:               "Requesting to create a pod without enough resources should fail",
+			testPod:                   testPods[1],
+			expectedContainerOptsLen:  nil,
+			expectedAllocatedResName1: 2,
+			expectedAllocatedResName2: 1,
+			expErr: fmt.Errorf("requested number of devices unavailable for domain1.com/resource1. Requested: 1, Available: 0"),
+		},
+		{
+			description:               "Successful allocation of all available Res1 resources and Res2 resources",
+			testPod:                   testPods[2],
+			expectedContainerOptsLen:  []int{0, 0, 1},
+			expectedAllocatedResName1: 2,
+			expectedAllocatedResName2: 2,
+			expErr: nil,
+		},
+	}
+	activePods := []*v1.Pod{}
+	for _, testCase := range testCases {
+		pod := testCase.testPod
+		activePods = append(activePods, pod)
+		podsStub.updateActivePods(activePods)
+		err := testManager.Allocate(nodeInfo, &lifecycle.PodAdmitAttributes{Pod: pod})
+		if !reflect.DeepEqual(err, testCase.expErr) {
+			t.Errorf("DevicePluginManager error (%v). expected error: %v but got: %v",
+				testCase.description, testCase.expErr, err)
+		}
+		runContainerOpts, err := testManager.GetDeviceRunContainerOptions(pod, &pod.Spec.Containers[0])
+		as.Nil(err)
+		if testCase.expectedContainerOptsLen == nil {
+			as.Nil(runContainerOpts)
+		} else {
+			as.Equal(len(runContainerOpts.Devices), testCase.expectedContainerOptsLen[0])
+			as.Equal(len(runContainerOpts.Mounts), testCase.expectedContainerOptsLen[1])
+			as.Equal(len(runContainerOpts.Envs), testCase.expectedContainerOptsLen[2])
+		}
+		as.Equal(testCase.expectedAllocatedResName1, testManager.allocatedDevices[res1.resourceName].Len())
+		as.Equal(testCase.expectedAllocatedResName2, testManager.allocatedDevices[res2.resourceName].Len())
+	}
+
+}
+
+func TestInitContainerDeviceAllocation(t *testing.T) {
+	// Requesting to create a pod that requests resourceName1 in init containers and normal containers
+	// should succeed with devices allocated to init containers reallocated to normal containers.
+	res1 := TestResource{
+		resourceName:     "domain1.com/resource1",
+		resourceQuantity: *resource.NewQuantity(int64(2), resource.DecimalSI),
+		devs:             []string{"dev1", "dev2"},
+	}
+	res2 := TestResource{
+		resourceName:     "domain2.com/resource2",
+		resourceQuantity: *resource.NewQuantity(int64(1), resource.DecimalSI),
+		devs:             []string{"dev3", "dev4"},
+	}
+	testResources := make([]TestResource, 2)
+	testResources = append(testResources, res1)
+	testResources = append(testResources, res2)
+	as := require.New(t)
+	podsStub := activePodsStub{
+		activePods: []*v1.Pod{},
+	}
+	nodeInfo := getTestNodeInfo(v1.ResourceList{})
+	tmpDir, err := ioutil.TempDir("", "checkpoint")
+	as.Nil(err)
+	defer os.RemoveAll(tmpDir)
+	pluginOpts := make(map[string]*pluginapi.DevicePluginOptions)
+	testManager, err := getTestManager(tmpDir, podsStub.getActivePods, testResources, pluginOpts)
+	as.Nil(err)
+
+	podWithPluginResourcesInInitContainers := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			UID: uuid.NewUUID(),
+		},
+		Spec: v1.PodSpec{
+			InitContainers: []v1.Container{
+				{
+					Name: string(uuid.NewUUID()),
+					Resources: v1.ResourceRequirements{
+						Limits: v1.ResourceList{
+							v1.ResourceName(res1.resourceName): res2.resourceQuantity,
+						},
+					},
+				},
+				{
+					Name: string(uuid.NewUUID()),
+					Resources: v1.ResourceRequirements{
+						Limits: v1.ResourceList{
+							v1.ResourceName(res1.resourceName): res1.resourceQuantity,
+						},
+					},
+				},
+			},
+			Containers: []v1.Container{
+				{
+					Name: string(uuid.NewUUID()),
+					Resources: v1.ResourceRequirements{
+						Limits: v1.ResourceList{
+							v1.ResourceName(res1.resourceName): res2.resourceQuantity,
+							v1.ResourceName(res2.resourceName): res2.resourceQuantity,
+						},
+					},
+				},
+				{
+					Name: string(uuid.NewUUID()),
+					Resources: v1.ResourceRequirements{
+						Limits: v1.ResourceList{
+							v1.ResourceName(res1.resourceName): res2.resourceQuantity,
+							v1.ResourceName(res2.resourceName): res2.resourceQuantity,
+						},
+					},
+				},
+			},
+		},
+	}
+	podsStub.updateActivePods([]*v1.Pod{podWithPluginResourcesInInitContainers})
+	err = testManager.Allocate(nodeInfo, &lifecycle.PodAdmitAttributes{Pod: podWithPluginResourcesInInitContainers})
+	as.Nil(err)
+	podUID := string(podWithPluginResourcesInInitContainers.UID)
+	initCont1 := podWithPluginResourcesInInitContainers.Spec.InitContainers[0].Name
+	initCont2 := podWithPluginResourcesInInitContainers.Spec.InitContainers[1].Name
+	normalCont1 := podWithPluginResourcesInInitContainers.Spec.Containers[0].Name
+	normalCont2 := podWithPluginResourcesInInitContainers.Spec.Containers[1].Name
+	initCont1Devices := testManager.podDevices.containerDevices(podUID, initCont1, res1.resourceName)
+	initCont2Devices := testManager.podDevices.containerDevices(podUID, initCont2, res1.resourceName)
+	normalCont1Devices := testManager.podDevices.containerDevices(podUID, normalCont1, res1.resourceName)
+	normalCont2Devices := testManager.podDevices.containerDevices(podUID, normalCont2, res1.resourceName)
+	as.Equal(1, initCont1Devices.Len())
+	as.Equal(2, initCont2Devices.Len())
+	as.Equal(1, normalCont1Devices.Len())
+	as.Equal(1, normalCont2Devices.Len())
+	as.True(initCont2Devices.IsSuperset(initCont1Devices))
+	as.True(initCont2Devices.IsSuperset(normalCont1Devices))
+	as.True(initCont2Devices.IsSuperset(normalCont2Devices))
+	as.Equal(0, normalCont1Devices.Intersection(normalCont2Devices).Len())
+}
+
+func TestSanitizeNodeAllocatable(t *testing.T) {
+	resourceName1 := "domain1.com/resource1"
+	devID1 := "dev1"
+
+	resourceName2 := "domain2.com/resource2"
+	devID2 := "dev2"
+
+	as := assert.New(t)
+	monitorCallback := func(resourceName string, added, updated, deleted []pluginapi.Device) {}
+	tmpDir, err := ioutil.TempDir("", "checkpoint")
+	as.Nil(err)
+
+	ckm, err := checkpointmanager.NewCheckpointManager(tmpDir)
+	as.Nil(err)
+	testManager := &ManagerImpl{
+		callback:          monitorCallback,
+		allocatedDevices:  make(map[string]sets.String),
+		healthyDevices:    make(map[string]sets.String),
+		podDevices:        make(podDevices),
+		checkpointManager: ckm,
+	}
+	// require one of resource1 and one of resource2
+	testManager.allocatedDevices[resourceName1] = sets.NewString()
+	testManager.allocatedDevices[resourceName1].Insert(devID1)
+	testManager.allocatedDevices[resourceName2] = sets.NewString()
+	testManager.allocatedDevices[resourceName2].Insert(devID2)
+
+	cachedNode := &v1.Node{
+		Status: v1.NodeStatus{
+			Allocatable: v1.ResourceList{
+				// has no resource1 and two of resource2
+				v1.ResourceName(resourceName2): *resource.NewQuantity(int64(2), resource.DecimalSI),
+			},
+		},
+	}
+	nodeInfo := &schedulercache.NodeInfo{}
+	nodeInfo.SetNode(cachedNode)
+
+	testManager.sanitizeNodeAllocatable(nodeInfo)
+
+	allocatableScalarResources := nodeInfo.AllocatableResource().ScalarResources
+	// allocatable in nodeInfo is less than needed, should update
+	as.Equal(1, int(allocatableScalarResources[v1.ResourceName(resourceName1)]))
+	// allocatable in nodeInfo is more than needed, should skip updating
+	as.Equal(2, int(allocatableScalarResources[v1.ResourceName(resourceName2)]))
+}
+
+func TestDevicePreStartContainer(t *testing.T) {
+	// Ensures that if device manager is indicated to invoke `PreStartContainer` RPC
+	// by device plugin, then device manager invokes PreStartContainer at endpoint interface.
+	// Also verifies that final allocation of mounts, envs etc is same as expected.
+	res1 := TestResource{
+		resourceName:     "domain1.com/resource1",
+		resourceQuantity: *resource.NewQuantity(int64(2), resource.DecimalSI),
+		devs:             []string{"dev1", "dev2"},
+	}
+	as := require.New(t)
+	podsStub := activePodsStub{
+		activePods: []*v1.Pod{},
+	}
+	tmpDir, err := ioutil.TempDir("", "checkpoint")
+	as.Nil(err)
+	defer os.RemoveAll(tmpDir)
+	nodeInfo := getTestNodeInfo(v1.ResourceList{})
+	pluginOpts := make(map[string]*pluginapi.DevicePluginOptions)
+	pluginOpts[res1.resourceName] = &pluginapi.DevicePluginOptions{PreStartRequired: true}
+
+	testManager, err := getTestManager(tmpDir, podsStub.getActivePods, []TestResource{res1}, pluginOpts)
+	as.Nil(err)
+
+	ch := make(chan []string, 1)
+	testManager.endpoints[res1.resourceName] = &MockEndpoint{
+		initChan:     ch,
+		allocateFunc: allocateStubFunc(),
+	}
+
+	pod := makePod(v1.ResourceList{
+		v1.ResourceName(res1.resourceName): res1.resourceQuantity})
+	activePods := []*v1.Pod{}
+	activePods = append(activePods, pod)
+	podsStub.updateActivePods(activePods)
+	err = testManager.Allocate(nodeInfo, &lifecycle.PodAdmitAttributes{Pod: pod})
+	as.Nil(err)
+	runContainerOpts, err := testManager.GetDeviceRunContainerOptions(pod, &pod.Spec.Containers[0])
+	as.Nil(err)
+	var initializedDevs []string
+	select {
+	case <-time.After(time.Second):
+		t.Fatalf("Timed out while waiting on channel for response from PreStartContainer RPC stub")
+	case initializedDevs = <-ch:
+		break
+	}
+
+	as.Contains(initializedDevs, "dev1")
+	as.Contains(initializedDevs, "dev2")
+	as.Equal(len(initializedDevs), len(res1.devs))
+
+	expectedResps, err := allocateStubFunc()([]string{"dev1", "dev2"})
+	as.Nil(err)
+	as.Equal(1, len(expectedResps.ContainerResponses))
+	expectedResp := expectedResps.ContainerResponses[0]
+	as.Equal(len(runContainerOpts.Devices), len(expectedResp.Devices))
+	as.Equal(len(runContainerOpts.Mounts), len(expectedResp.Mounts))
+	as.Equal(len(runContainerOpts.Envs), len(expectedResp.Envs))
+}
+
+func allocateStubFunc() func(devs []string) (*pluginapi.AllocateResponse, error) {
+	return func(devs []string) (*pluginapi.AllocateResponse, error) {
+		resp := new(pluginapi.ContainerAllocateResponse)
+		resp.Envs = make(map[string]string)
+		for _, dev := range devs {
+			switch dev {
+			case "dev1":
+				resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
+					ContainerPath: "/dev/aaa",
+					HostPath:      "/dev/aaa",
+					Permissions:   "mrw",
+				})
+
+				resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
+					ContainerPath: "/dev/bbb",
+					HostPath:      "/dev/bbb",
+					Permissions:   "mrw",
+				})
+
+				resp.Mounts = append(resp.Mounts, &pluginapi.Mount{
+					ContainerPath: "/container_dir1/file1",
+					HostPath:      "host_dir1/file1",
+					ReadOnly:      true,
+				})
+
+			case "dev2":
+				resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
+					ContainerPath: "/dev/ccc",
+					HostPath:      "/dev/ccc",
+					Permissions:   "mrw",
+				})
+
+				resp.Mounts = append(resp.Mounts, &pluginapi.Mount{
+					ContainerPath: "/container_dir1/file2",
+					HostPath:      "host_dir1/file2",
+					ReadOnly:      true,
+				})
+
+				resp.Envs["key1"] = "val1"
+			}
+		}
+		resps := new(pluginapi.AllocateResponse)
+		resps.ContainerResponses = append(resps.ContainerResponses, resp)
+		return resps, nil
+	}
+}

vendor/k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/pod_devices.go

@@ -0,0 +1,273 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package devicemanager
+
+import (
+	"github.com/golang/glog"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
+	"k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+)
+
+type deviceAllocateInfo struct {
+	// deviceIds contains device Ids allocated to this container for the given resourceName.
+	deviceIds sets.String
+	// allocResp contains cached rpc AllocateResponse.
+	allocResp *pluginapi.ContainerAllocateResponse
+}
+
+type resourceAllocateInfo map[string]deviceAllocateInfo // Keyed by resourceName.
+type containerDevices map[string]resourceAllocateInfo   // Keyed by containerName.
+type podDevices map[string]containerDevices             // Keyed by podUID.
+
+func (pdev podDevices) pods() sets.String {
+	ret := sets.NewString()
+	for k := range pdev {
+		ret.Insert(k)
+	}
+	return ret
+}
+
+func (pdev podDevices) insert(podUID, contName, resource string, devices sets.String, resp *pluginapi.ContainerAllocateResponse) {
+	if _, podExists := pdev[podUID]; !podExists {
+		pdev[podUID] = make(containerDevices)
+	}
+	if _, contExists := pdev[podUID][contName]; !contExists {
+		pdev[podUID][contName] = make(resourceAllocateInfo)
+	}
+	pdev[podUID][contName][resource] = deviceAllocateInfo{
+		deviceIds: devices,
+		allocResp: resp,
+	}
+}
+
+func (pdev podDevices) delete(pods []string) {
+	for _, uid := range pods {
+		delete(pdev, uid)
+	}
+}
+
+// Returns list of device Ids allocated to the given container for the given resource.
+// Returns nil if we don't have cached state for the given <podUID, contName, resource>.
+func (pdev podDevices) containerDevices(podUID, contName, resource string) sets.String {
+	if _, podExists := pdev[podUID]; !podExists {
+		return nil
+	}
+	if _, contExists := pdev[podUID][contName]; !contExists {
+		return nil
+	}
+	devs, resourceExists := pdev[podUID][contName][resource]
+	if !resourceExists {
+		return nil
+	}
+	return devs.deviceIds
+}
+
+// Populates allocatedResources with the device resources allocated to the specified <podUID, contName>.
+func (pdev podDevices) addContainerAllocatedResources(podUID, contName string, allocatedResources map[string]sets.String) {
+	containers, exists := pdev[podUID]
+	if !exists {
+		return
+	}
+	resources, exists := containers[contName]
+	if !exists {
+		return
+	}
+	for resource, devices := range resources {
+		allocatedResources[resource] = allocatedResources[resource].Union(devices.deviceIds)
+	}
+}
+
+// Removes the device resources allocated to the specified <podUID, contName> from allocatedResources.
+func (pdev podDevices) removeContainerAllocatedResources(podUID, contName string, allocatedResources map[string]sets.String) {
+	containers, exists := pdev[podUID]
+	if !exists {
+		return
+	}
+	resources, exists := containers[contName]
+	if !exists {
+		return
+	}
+	for resource, devices := range resources {
+		allocatedResources[resource] = allocatedResources[resource].Difference(devices.deviceIds)
+	}
+}
+
+// Returns all of devices allocated to the pods being tracked, keyed by resourceName.
+func (pdev podDevices) devices() map[string]sets.String {
+	ret := make(map[string]sets.String)
+	for _, containerDevices := range pdev {
+		for _, resources := range containerDevices {
+			for resource, devices := range resources {
+				if _, exists := ret[resource]; !exists {
+					ret[resource] = sets.NewString()
+				}
+				if devices.allocResp != nil {
+					ret[resource] = ret[resource].Union(devices.deviceIds)
+				}
+			}
+		}
+	}
+	return ret
+}
+
+// Turns podDevices to checkpointData.
+func (pdev podDevices) toCheckpointData() []checkpoint.PodDevicesEntry {
+	var data []checkpoint.PodDevicesEntry
+	for podUID, containerDevices := range pdev {
+		for conName, resources := range containerDevices {
+			for resource, devices := range resources {
+				devIds := devices.deviceIds.UnsortedList()
+				if devices.allocResp == nil {
+					glog.Errorf("Can't marshal allocResp for %v %v %v: allocation response is missing", podUID, conName, resource)
+					continue
+				}
+
+				allocResp, err := devices.allocResp.Marshal()
+				if err != nil {
+					glog.Errorf("Can't marshal allocResp for %v %v %v: %v", podUID, conName, resource, err)
+					continue
+				}
+				data = append(data, checkpoint.PodDevicesEntry{
+					PodUID:        podUID,
+					ContainerName: conName,
+					ResourceName:  resource,
+					DeviceIDs:     devIds,
+					AllocResp:     allocResp})
+			}
+		}
+	}
+	return data
+}
+
+// Populates podDevices from the passed in checkpointData.
+func (pdev podDevices) fromCheckpointData(data []checkpoint.PodDevicesEntry) {
+	for _, entry := range data {
+		glog.V(2).Infof("Get checkpoint entry: %v %v %v %v %v\n",
+			entry.PodUID, entry.ContainerName, entry.ResourceName, entry.DeviceIDs, entry.AllocResp)
+		devIDs := sets.NewString()
+		for _, devID := range entry.DeviceIDs {
+			devIDs.Insert(devID)
+		}
+		allocResp := &pluginapi.ContainerAllocateResponse{}
+		err := allocResp.Unmarshal(entry.AllocResp)
+		if err != nil {
+			glog.Errorf("Can't unmarshal allocResp for %v %v %v: %v", entry.PodUID, entry.ContainerName, entry.ResourceName, err)
+			continue
+		}
+		pdev.insert(entry.PodUID, entry.ContainerName, entry.ResourceName, devIDs, allocResp)
+	}
+}
+
+// Returns combined container runtime settings to consume the container's allocated devices.
+func (pdev podDevices) deviceRunContainerOptions(podUID, contName string) *DeviceRunContainerOptions {
+	containers, exists := pdev[podUID]
+	if !exists {
+		return nil
+	}
+	resources, exists := containers[contName]
+	if !exists {
+		return nil
+	}
+	opts := &DeviceRunContainerOptions{}
+	// Maps to detect duplicate settings.
+	devsMap := make(map[string]string)
+	mountsMap := make(map[string]string)
+	envsMap := make(map[string]string)
+	annotationsMap := make(map[string]string)
+	// Loops through AllocationResponses of all cached device resources.
+	for _, devices := range resources {
+		resp := devices.allocResp
+		// Each Allocate response has the following artifacts.
+		// Environment variables
+		// Mount points
+		// Device files
+		// Container annotations
+		// These artifacts are per resource per container.
+		// Updates RunContainerOptions.Envs.
+		for k, v := range resp.Envs {
+			if e, ok := envsMap[k]; ok {
+				glog.V(4).Infof("Skip existing env %s %s", k, v)
+				if e != v {
+					glog.Errorf("Environment variable %s has conflicting setting: %s and %s", k, e, v)
+				}
+				continue
+			}
+			glog.V(4).Infof("Add env %s %s", k, v)
+			envsMap[k] = v
+			opts.Envs = append(opts.Envs, kubecontainer.EnvVar{Name: k, Value: v})
+		}
+
+		// Updates RunContainerOptions.Devices.
+		for _, dev := range resp.Devices {
+			if d, ok := devsMap[dev.ContainerPath]; ok {
+				glog.V(4).Infof("Skip existing device %s %s", dev.ContainerPath, dev.HostPath)
+				if d != dev.HostPath {
+					glog.Errorf("Container device %s has conflicting mapping host devices: %s and %s",
+						dev.ContainerPath, d, dev.HostPath)
+				}
+				continue
+			}
+			glog.V(4).Infof("Add device %s %s", dev.ContainerPath, dev.HostPath)
+			devsMap[dev.ContainerPath] = dev.HostPath
+			opts.Devices = append(opts.Devices, kubecontainer.DeviceInfo{
+				PathOnHost:      dev.HostPath,
+				PathInContainer: dev.ContainerPath,
+				Permissions:     dev.Permissions,
+			})
+		}
+
+		// Updates RunContainerOptions.Mounts.
+		for _, mount := range resp.Mounts {
+			if m, ok := mountsMap[mount.ContainerPath]; ok {
+				glog.V(4).Infof("Skip existing mount %s %s", mount.ContainerPath, mount.HostPath)
+				if m != mount.HostPath {
+					glog.Errorf("Container mount %s has conflicting mapping host mounts: %s and %s",
+						mount.ContainerPath, m, mount.HostPath)
+				}
+				continue
+			}
+			glog.V(4).Infof("Add mount %s %s", mount.ContainerPath, mount.HostPath)
+			mountsMap[mount.ContainerPath] = mount.HostPath
+			opts.Mounts = append(opts.Mounts, kubecontainer.Mount{
+				Name:          mount.ContainerPath,
+				ContainerPath: mount.ContainerPath,
+				HostPath:      mount.HostPath,
+				ReadOnly:      mount.ReadOnly,
+				// TODO: This may need to be part of Device plugin API.
+				SELinuxRelabel: false,
+			})
+		}
+
+		// Updates for Annotations
+		for k, v := range resp.Annotations {
+			if e, ok := annotationsMap[k]; ok {
+				glog.V(4).Infof("Skip existing annotation %s %s", k, v)
+				if e != v {
+					glog.Errorf("Annotation %s has conflicting setting: %s and %s", k, e, v)
+				}
+				continue
+			}
+			glog.V(4).Infof("Add annotation %s %s", k, v)
+			annotationsMap[k] = v
+			opts.Annotations = append(opts.Annotations, kubecontainer.Annotation{Name: k, Value: v})
+		}
+	}
+	return opts
+}

vendor/k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/types.go

@@ -0,0 +1,111 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package devicemanager
+
+import (
+	"time"
+
+	"k8s.io/api/core/v1"
+	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
+	"k8s.io/kubernetes/pkg/kubelet/config"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
+	schedulercache "k8s.io/kubernetes/pkg/scheduler/cache"
+)
+
+// Manager manages all the Device Plugins running on a node.
+type Manager interface {
+	// Start starts device plugin registration service.
+	Start(activePods ActivePodsFunc, sourcesReady config.SourcesReady) error
+
+	// Devices is the map of devices that have registered themselves
+	// against the manager.
+	// The map key is the ResourceName of the device plugins.
+	Devices() map[string][]pluginapi.Device
+
+	// Allocate configures and assigns devices to pods. The pods are provided
+	// through the pod admission attributes in the attrs argument. From the
+	// requested device resources, Allocate will communicate with the owning
+	// device plugin to allow setup procedures to take place, and for the
+	// device plugin to provide runtime settings to use the device (environment
+	// variables, mount points and device files). The node object is provided
+	// for the device manager to update the node capacity to reflect the
+	// currently available devices.
+	Allocate(node *schedulercache.NodeInfo, attrs *lifecycle.PodAdmitAttributes) error
+
+	// Stop stops the manager.
+	Stop() error
+
+	// GetDeviceRunContainerOptions checks whether we have cached containerDevices
+	// for the passed-in <pod, container> and returns its DeviceRunContainerOptions
+	// for the found one. An empty struct is returned in case no cached state is found.
+	GetDeviceRunContainerOptions(pod *v1.Pod, container *v1.Container) (*DeviceRunContainerOptions, error)
+
+	// GetCapacity returns the amount of available device plugin resource capacity, resource allocatable
+	// and inactive device plugin resources previously registered on the node.
+	GetCapacity() (v1.ResourceList, v1.ResourceList, []string)
+}
+
+// DeviceRunContainerOptions contains the combined container runtime settings to consume its allocated devices.
+type DeviceRunContainerOptions struct {
+	// The environment variables list.
+	Envs []kubecontainer.EnvVar
+	// The mounts for the container.
+	Mounts []kubecontainer.Mount
+	// The host devices mapped into the container.
+	Devices []kubecontainer.DeviceInfo
+	// The Annotations for the container
+	Annotations []kubecontainer.Annotation
+}
+
+// TODO: evaluate whether we need these error definitions.
+const (
+	// errFailedToDialDevicePlugin is the error raised when the device plugin could not be
+	// reached on the registered socket
+	errFailedToDialDevicePlugin = "failed to dial device plugin:"
+	// errUnsupportedVersion is the error raised when the device plugin uses an API version not
+	// supported by the Kubelet registry
+	errUnsupportedVersion = "requested API version %q is not supported by kubelet. Supported versions are %q"
+	// errDevicePluginAlreadyExists is the error raised when a device plugin with the
+	// same Resource Name tries to register itself
+	errDevicePluginAlreadyExists = "another device plugin already registered this Resource Name"
+	// errInvalidResourceName is the error raised when a device plugin is registering
+	// itself with an invalid ResourceName
+	errInvalidResourceName = "the ResourceName %q is invalid"
+	// errEmptyResourceName is the error raised when the resource name field is empty
+	errEmptyResourceName = "invalid Empty ResourceName"
+	// errEndpointStopped indicates that the endpoint has been stopped
+	errEndpointStopped = "endpoint %v has been stopped"
+
+	// errBadSocket is the error raised when the registry socket path is not absolute
+	errBadSocket = "bad socketPath, must be an absolute path:"
+	// errRemoveSocket is the error raised when the registry could not remove the existing socket
+	errRemoveSocket = "failed to remove socket while starting device plugin registry, with error"
+	// errListenSocket is the error raised when the registry could not listen on the socket
+	errListenSocket = "failed to listen to socket while starting device plugin registry, with error"
+	// errListAndWatch is the error raised when ListAndWatch ended unsuccessfully
+	errListAndWatch = "listAndWatch ended unexpectedly for device plugin %s with error %v"
+)
+
+// endpointStopGracePeriod indicates the grace period after an endpoint is stopped
+// because its device plugin fails. DeviceManager keeps the stopped endpoint in its
+// cache during this grace period to cover the time gap for the capacity change to
+// take effect.
+const endpointStopGracePeriod = time.Duration(5) * time.Minute
+
+// kubeletDeviceManagerCheckpoint is the file name of device plugin checkpoint
+const kubeletDeviceManagerCheckpoint = "kubelet_internal_checkpoint"