Bumping k8s dependencies to 1.13

vendor/k8s.io/kubernetes/test/e2e/scalability/density.go

@@ -56,6 +56,7 @@ const (
 	MinSaturationThreshold     = 2 * time.Minute
 	MinPodsPerSecondThroughput = 8
 	DensityPollInterval        = 10 * time.Second
+	MinPodStartupMeasurements  = 500
 )
 
 // Maximum container failures this test tolerates before failing.
@@ -64,6 +65,9 @@ var MaxContainerFailures = 0
 // Maximum no. of missing measurements related to pod-startup that the test tolerates.
 var MaxMissingPodStartupMeasurements = 0
 
+// Number of nodes in the cluster (computed inside BeforeEach).
+var nodeCount = 0
+
 type DensityTestConfig struct {
 	Configs            []testutils.RunObjectConfig
 	ClientSets         []clientset.Interface
@@ -168,9 +172,9 @@ func density30AddonResourceVerifier(numNodes int) map[string]framework.ResourceC
 		}
 	} else {
 		if numNodes <= 100 {
-			apiserverCPU = 1.8
+			apiserverCPU = 2.2
 			apiserverMem = 1700 * (1024 * 1024)
-			controllerCPU = 0.6
+			controllerCPU = 0.8
 			controllerMem = 530 * (1024 * 1024)
 			schedulerCPU = 0.4
 			schedulerMem = 180 * (1024 * 1024)
@@ -285,6 +289,11 @@ func runDensityTest(dtc DensityTestConfig, testPhaseDurations *timer.TestPhaseTi
 
 	replicationCtrlStartupPhase := testPhaseDurations.StartPhase(300, "saturation pods creation")
 	defer replicationCtrlStartupPhase.End()
+
+	// Start scheduler CPU profile-gatherer before we begin cluster saturation.
+	profileGatheringDelay := time.Duration(1+nodeCount/100) * time.Minute
+	schedulerProfilingStopCh := framework.StartCPUProfileGatherer("kube-scheduler", "density", profileGatheringDelay)
+
 	// Start all replication controllers.
 	startTime := time.Now()
 	wg := sync.WaitGroup{}
@@ -304,10 +313,16 @@ func runDensityTest(dtc DensityTestConfig, testPhaseDurations *timer.TestPhaseTi
 	wg.Wait()
 	startupTime := time.Since(startTime)
 	close(logStopCh)
+	close(schedulerProfilingStopCh)
 	framework.Logf("E2E startup time for %d pods: %v", dtc.PodCount, startupTime)
 	framework.Logf("Throughput (pods/s) during cluster saturation phase: %v", float32(dtc.PodCount)/float32(startupTime/time.Second))
 	replicationCtrlStartupPhase.End()
 
+	// Grabbing scheduler memory profile after cluster saturation finished.
+	wg.Add(1)
+	framework.GatherMemoryProfile("kube-scheduler", "density", &wg)
+	wg.Wait()
+
 	printPodAllocationPhase := testPhaseDurations.StartPhase(400, "printing pod allocation")
 	defer printPodAllocationPhase.End()
 	// Print some data about Pod to Node allocation
@@ -366,7 +381,6 @@ func cleanupDensityTest(dtc DensityTestConfig, testPhaseDurations *timer.TestPha
 // limits on Docker's concurrent container startup.
 var _ = SIGDescribe("Density", func() {
 	var c clientset.Interface
-	var nodeCount int
 	var additionalPodsPrefix string
 	var ns string
 	var uuid string
@@ -381,6 +395,7 @@ var _ = SIGDescribe("Density", func() {
 	missingMeasurements := 0
 	var testPhaseDurations *timer.TestPhaseTimer
 	var profileGathererStopCh chan struct{}
+	var etcdMetricsCollector *framework.EtcdMetricsCollector
 
 	// Gathers data prior to framework namespace teardown
 	AfterEach(func() {
@@ -388,7 +403,7 @@ var _ = SIGDescribe("Density", func() {
 		close(profileGathererStopCh)
 		wg := sync.WaitGroup{}
 		wg.Add(1)
-		framework.GatherApiserverMemoryProfile(&wg, "density")
+		framework.GatherMemoryProfile("kube-apiserver", "density", &wg)
 		wg.Wait()
 
 		saturationThreshold := time.Duration((totalPods / MinPodsPerSecondThroughput)) * time.Second
@@ -412,7 +427,7 @@ var _ = SIGDescribe("Density", func() {
 			summaries = append(summaries, metrics)
 		}
 
-		// Verify scheduler metrics.
+		// Summarize scheduler metrics.
 		latency, err := framework.VerifySchedulerLatency(c)
 		framework.ExpectNoError(err)
 		if err == nil {
@@ -428,6 +443,14 @@ var _ = SIGDescribe("Density", func() {
 			}
 			summaries = append(summaries, latency)
 		}
+
+		// Summarize etcd metrics.
+		err = etcdMetricsCollector.StopAndSummarize()
+		framework.ExpectNoError(err)
+		if err == nil {
+			summaries = append(summaries, etcdMetricsCollector.GetMetrics())
+		}
+
 		summaries = append(summaries, testPhaseDurations)
 
 		framework.PrintSummaries(summaries, testCaseBaseName)
@@ -487,7 +510,11 @@ var _ = SIGDescribe("Density", func() {
 
 		// Start apiserver CPU profile gatherer with frequency based on cluster size.
 		profileGatheringDelay := time.Duration(5+nodeCount/100) * time.Minute
-		profileGathererStopCh = framework.StartApiserverCPUProfileGatherer(profileGatheringDelay)
+		profileGathererStopCh = framework.StartCPUProfileGatherer("kube-apiserver", "density", profileGatheringDelay)
+
+		// Start etcs metrics collection.
+		etcdMetricsCollector = framework.NewEtcdMetricsCollector()
+		etcdMetricsCollector.StartCollecting(time.Minute)
 	})
 
 	type Density struct {
@@ -588,6 +615,7 @@ var _ = SIGDescribe("Density", func() {
 			timeout := time.Duration(totalPods/podThroughput)*time.Second + 3*time.Minute
 			// createClients is defined in load.go
 			clients, internalClients, scalesClients, err := createClients(numberOfCollections)
+			framework.ExpectNoError(err)
 			for i := 0; i < numberOfCollections; i++ {
 				nsName := namespaces[i].Name
 				secretNames := []string{}
@@ -651,7 +679,7 @@ var _ = SIGDescribe("Density", func() {
 
 			// Single client is running out of http2 connections in delete phase, hence we need more.
 			clients, internalClients, scalesClients, err = createClients(2)
-
+			framework.ExpectNoError(err)
 			dConfig := DensityTestConfig{
 				ClientSets:         clients,
 				InternalClientsets: internalClients,
@@ -675,7 +703,10 @@ var _ = SIGDescribe("Density", func() {
 			}
 			e2eStartupTime = runDensityTest(dConfig, testPhaseDurations, &scheduleThroughputs)
 			if itArg.runLatencyTest {
-				By("Scheduling additional Pods to measure startup latencies")
+				// Pick latencyPodsIterations so that:
+				// latencyPodsIterations * nodeCount >= MinPodStartupMeasurements.
+				latencyPodsIterations := (MinPodStartupMeasurements + nodeCount - 1) / nodeCount
+				By(fmt.Sprintf("Scheduling additional %d Pods to measure startup latencies", latencyPodsIterations*nodeCount))
 
 				createTimes := make(map[string]metav1.Time, 0)
 				nodeNames := make(map[string]string, 0)
@@ -754,58 +785,76 @@ var _ = SIGDescribe("Density", func() {
 
 					go controller.Run(stopCh)
 				}
+				for latencyPodsIteration := 0; latencyPodsIteration < latencyPodsIterations; latencyPodsIteration++ {
+					podIndexOffset := latencyPodsIteration * nodeCount
+					framework.Logf("Creating %d latency pods in range [%d, %d]", nodeCount, podIndexOffset+1, podIndexOffset+nodeCount)
 
-				// Create some additional pods with throughput ~5 pods/sec.
-				latencyPodStartupPhase := testPhaseDurations.StartPhase(800, "latency pods creation")
-				defer latencyPodStartupPhase.End()
-				var wg sync.WaitGroup
-				wg.Add(nodeCount)
-				// Explicitly set requests here.
-				// Thanks to it we trigger increasing priority function by scheduling
-				// a pod to a node, which in turn will result in spreading latency pods
-				// more evenly between nodes.
-				cpuRequest := *resource.NewMilliQuantity(nodeCpuCapacity/5, resource.DecimalSI)
-				memRequest := *resource.NewQuantity(nodeMemCapacity/5, resource.DecimalSI)
-				if podsPerNode > 30 {
-					// This is to make them schedulable on high-density tests
-					// (e.g. 100 pods/node kubemark).
-					cpuRequest = *resource.NewMilliQuantity(0, resource.DecimalSI)
-					memRequest = *resource.NewQuantity(0, resource.DecimalSI)
-				}
-				rcNameToNsMap := map[string]string{}
-				for i := 1; i <= nodeCount; i++ {
-					name := additionalPodsPrefix + "-" + strconv.Itoa(i)
-					nsName := namespaces[i%len(namespaces)].Name
-					rcNameToNsMap[name] = nsName
-					go createRunningPodFromRC(&wg, c, name, nsName, imageutils.GetPauseImageName(), additionalPodsPrefix, cpuRequest, memRequest)
-					time.Sleep(200 * time.Millisecond)
-				}
-				wg.Wait()
-				latencyPodStartupPhase.End()
+					watchTimesLen := len(watchTimes)
 
-				latencyMeasurementPhase := testPhaseDurations.StartPhase(810, "pod startup latencies measurement")
-				defer latencyMeasurementPhase.End()
-				By("Waiting for all Pods begin observed by the watch...")
-				waitTimeout := 10 * time.Minute
-				for start := time.Now(); len(watchTimes) < nodeCount; time.Sleep(10 * time.Second) {
-					if time.Since(start) < waitTimeout {
-						framework.Failf("Timeout reached waiting for all Pods being observed by the watch.")
+					// Create some additional pods with throughput ~5 pods/sec.
+					latencyPodStartupPhase := testPhaseDurations.StartPhase(800+latencyPodsIteration*10, "latency pods creation")
+					defer latencyPodStartupPhase.End()
+					var wg sync.WaitGroup
+					wg.Add(nodeCount)
+					// Explicitly set requests here.
+					// Thanks to it we trigger increasing priority function by scheduling
+					// a pod to a node, which in turn will result in spreading latency pods
+					// more evenly between nodes.
+					cpuRequest := *resource.NewMilliQuantity(nodeCpuCapacity/5, resource.DecimalSI)
+					memRequest := *resource.NewQuantity(nodeMemCapacity/5, resource.DecimalSI)
+					if podsPerNode > 30 {
+						// This is to make them schedulable on high-density tests
+						// (e.g. 100 pods/node kubemark).
+						cpuRequest = *resource.NewMilliQuantity(0, resource.DecimalSI)
+						memRequest = *resource.NewQuantity(0, resource.DecimalSI)
 					}
-				}
-				close(stopCh)
-
-				nodeToLatencyPods := make(map[string]int)
-				for i := range latencyPodStores {
-					for _, item := range latencyPodStores[i].List() {
-						pod := item.(*v1.Pod)
-						nodeToLatencyPods[pod.Spec.NodeName]++
+					rcNameToNsMap := map[string]string{}
+					for i := 1; i <= nodeCount; i++ {
+						name := additionalPodsPrefix + "-" + strconv.Itoa(podIndexOffset+i)
+						nsName := namespaces[i%len(namespaces)].Name
+						rcNameToNsMap[name] = nsName
+						go createRunningPodFromRC(&wg, c, name, nsName, imageutils.GetPauseImageName(), additionalPodsPrefix, cpuRequest, memRequest)
+						time.Sleep(200 * time.Millisecond)
 					}
-					for node, count := range nodeToLatencyPods {
-						if count > 1 {
-							framework.Logf("%d latency pods scheduled on %s", count, node)
+					wg.Wait()
+					latencyPodStartupPhase.End()
+
+					latencyMeasurementPhase := testPhaseDurations.StartPhase(801+latencyPodsIteration*10, "pod startup latencies measurement")
+					defer latencyMeasurementPhase.End()
+					By("Waiting for all Pods begin observed by the watch...")
+					waitTimeout := 10 * time.Minute
+					for start := time.Now(); len(watchTimes) < watchTimesLen+nodeCount; time.Sleep(10 * time.Second) {
+						if time.Since(start) < waitTimeout {
+							framework.Failf("Timeout reached waiting for all Pods being observed by the watch.")
 						}
 					}
+
+					nodeToLatencyPods := make(map[string]int)
+					for i := range latencyPodStores {
+						for _, item := range latencyPodStores[i].List() {
+							pod := item.(*v1.Pod)
+							nodeToLatencyPods[pod.Spec.NodeName]++
+						}
+						for node, count := range nodeToLatencyPods {
+							if count > 1 {
+								framework.Logf("%d latency pods scheduled on %s", count, node)
+							}
+						}
+					}
+					latencyMeasurementPhase.End()
+
+					By("Removing additional replication controllers")
+					podDeletionPhase := testPhaseDurations.StartPhase(802+latencyPodsIteration*10, "latency pods deletion")
+					defer podDeletionPhase.End()
+					deleteRC := func(i int) {
+						defer GinkgoRecover()
+						name := additionalPodsPrefix + "-" + strconv.Itoa(podIndexOffset+i+1)
+						framework.ExpectNoError(framework.DeleteRCAndWaitForGC(c, rcNameToNsMap[name], name))
+					}
+					workqueue.Parallelize(25, nodeCount, deleteRC)
+					podDeletionPhase.End()
 				}
+				close(stopCh)
 
 				for i := 0; i < len(namespaces); i++ {
 					nsName := namespaces[i].Name
@@ -893,18 +942,6 @@ var _ = SIGDescribe("Density", func() {
 				framework.ExpectNoError(framework.VerifyLatencyWithinThreshold(podStartupLatencyThreshold, podStartupLatency.E2ELatency, "pod startup"))
 
 				framework.LogSuspiciousLatency(startupLag, e2eLag, nodeCount, c)
-				latencyMeasurementPhase.End()
-
-				By("Removing additional replication controllers")
-				podDeletionPhase := testPhaseDurations.StartPhase(820, "latency pods deletion")
-				defer podDeletionPhase.End()
-				deleteRC := func(i int) {
-					defer GinkgoRecover()
-					name := additionalPodsPrefix + "-" + strconv.Itoa(i+1)
-					framework.ExpectNoError(framework.DeleteRCAndWaitForGC(c, rcNameToNsMap[name], name))
-				}
-				workqueue.Parallelize(25, nodeCount, deleteRC)
-				podDeletionPhase.End()
 			}
 			cleanupDensityTest(dConfig, testPhaseDurations)
 		})