Replace godep with dep

vendor/k8s.io/kubernetes/test/e2e/node/BUILD

@@ -0,0 +1,43 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "apparmor.go",
+        "framework.go",
+        "kubelet.go",
+        "kubelet_perf.go",
+        "security_context.go",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/api/testapi:go_default_library",
+        "//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
+        "//test/e2e/common:go_default_library",
+        "//test/e2e/framework:go_default_library",
+        "//test/utils:go_default_library",
+        "//test/utils/image:go_default_library",
+        "//vendor/github.com/onsi/ginkgo:go_default_library",
+        "//vendor/github.com/onsi/gomega:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/test/e2e/node/OWNERS

@@ -0,0 +1,9 @@
+approvers:
+- Random-Liu
+- dchen1107
+- derekwaynecarr
+- tallclair
+- vishh
+- yujuhong
+reviewers:
+- sig-node-reviewers

vendor/k8s.io/kubernetes/test/e2e/node/apparmor.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package node
+
+import (
+	"k8s.io/kubernetes/test/e2e/common"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+)
+
+var _ = SIGDescribe("AppArmor", func() {
+	f := framework.NewDefaultFramework("apparmor")
+
+	Context("load AppArmor profiles", func() {
+		BeforeEach(func() {
+			common.SkipIfAppArmorNotSupported()
+			common.LoadAppArmorProfiles(f)
+		})
+		AfterEach(func() {
+			if !CurrentGinkgoTestDescription().Failed {
+				return
+			}
+			framework.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf)
+		})
+
+		It("should enforce an AppArmor profile", func() {
+			common.CreateAppArmorTestPod(f, true)
+		})
+	})
+})

vendor/k8s.io/kubernetes/test/e2e/node/framework.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package node
+
+import "k8s.io/kubernetes/test/e2e/framework"
+
+func SIGDescribe(text string, body func()) bool {
+	return framework.KubeDescribe("[sig-node] "+text, body)
+}

vendor/k8s.io/kubernetes/test/e2e/node/kubelet.go

@@ -0,0 +1,454 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package node
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/pkg/api/testapi"
+	"k8s.io/kubernetes/test/e2e/framework"
+	testutils "k8s.io/kubernetes/test/utils"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+)
+
+const (
+	// Interval to framework.Poll /runningpods on a node
+	pollInterval = 1 * time.Second
+	// Interval to framework.Poll /stats/container on a node
+	containerStatsPollingInterval = 5 * time.Second
+	// Maximum number of nodes that we constraint to
+	maxNodesToCheck = 10
+)
+
+// getPodMatches returns a set of pod names on the given node that matches the
+// podNamePrefix and namespace.
+func getPodMatches(c clientset.Interface, nodeName string, podNamePrefix string, namespace string) sets.String {
+	matches := sets.NewString()
+	framework.Logf("Checking pods on node %v via /runningpods endpoint", nodeName)
+	runningPods, err := framework.GetKubeletPods(c, nodeName)
+	if err != nil {
+		framework.Logf("Error checking running pods on %v: %v", nodeName, err)
+		return matches
+	}
+	for _, pod := range runningPods.Items {
+		if pod.Namespace == namespace && strings.HasPrefix(pod.Name, podNamePrefix) {
+			matches.Insert(pod.Name)
+		}
+	}
+	return matches
+}
+
+// waitTillNPodsRunningOnNodes polls the /runningpods endpoint on kubelet until
+// it finds targetNumPods pods that match the given criteria (namespace and
+// podNamePrefix). Note that we usually use label selector to filter pods that
+// belong to the same RC. However, we use podNamePrefix with namespace here
+// because pods returned from /runningpods do not contain the original label
+// information; they are reconstructed by examining the container runtime. In
+// the scope of this test, we do not expect pod naming conflicts so
+// podNamePrefix should be sufficient to identify the pods.
+func waitTillNPodsRunningOnNodes(c clientset.Interface, nodeNames sets.String, podNamePrefix string, namespace string, targetNumPods int, timeout time.Duration) error {
+	return wait.Poll(pollInterval, timeout, func() (bool, error) {
+		matchCh := make(chan sets.String, len(nodeNames))
+		for _, item := range nodeNames.List() {
+			// Launch a goroutine per node to check the pods running on the nodes.
+			nodeName := item
+			go func() {
+				matchCh <- getPodMatches(c, nodeName, podNamePrefix, namespace)
+			}()
+		}
+
+		seen := sets.NewString()
+		for i := 0; i < len(nodeNames.List()); i++ {
+			seen = seen.Union(<-matchCh)
+		}
+		if seen.Len() == targetNumPods {
+			return true, nil
+		}
+		framework.Logf("Waiting for %d pods to be running on the node; %d are currently running;", targetNumPods, seen.Len())
+		return false, nil
+	})
+}
+
+// updates labels of nodes given by nodeNames.
+// In case a given label already exists, it overwrites it. If label to remove doesn't exist
+// it silently ignores it.
+// TODO: migrate to use framework.AddOrUpdateLabelOnNode/framework.RemoveLabelOffNode
+func updateNodeLabels(c clientset.Interface, nodeNames sets.String, toAdd, toRemove map[string]string) {
+	const maxRetries = 5
+	for nodeName := range nodeNames {
+		var node *v1.Node
+		var err error
+		for i := 0; i < maxRetries; i++ {
+			node, err = c.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
+			if err != nil {
+				framework.Logf("Error getting node %s: %v", nodeName, err)
+				continue
+			}
+			if toAdd != nil {
+				for k, v := range toAdd {
+					node.ObjectMeta.Labels[k] = v
+				}
+			}
+			if toRemove != nil {
+				for k := range toRemove {
+					delete(node.ObjectMeta.Labels, k)
+				}
+			}
+			_, err = c.CoreV1().Nodes().Update(node)
+			if err != nil {
+				framework.Logf("Error updating node %s: %v", nodeName, err)
+			} else {
+				break
+			}
+		}
+		Expect(err).NotTo(HaveOccurred())
+	}
+}
+
+// Restart the passed-in nfs-server by issuing a `/usr/sbin/rpc.nfsd 1` command in the
+// pod's (only) container. This command changes the number of nfs server threads from
+// (presumably) zero back to 1, and therefore allows nfs to open connections again.
+func restartNfsServer(serverPod *v1.Pod) {
+	const startcmd = "/usr/sbin/rpc.nfsd 1"
+	ns := fmt.Sprintf("--namespace=%v", serverPod.Namespace)
+	framework.RunKubectlOrDie("exec", ns, serverPod.Name, "--", "/bin/sh", "-c", startcmd)
+}
+
+// Stop the passed-in nfs-server by issuing a `/usr/sbin/rpc.nfsd 0` command in the
+// pod's (only) container. This command changes the number of nfs server threads to 0,
+// thus closing all open nfs connections.
+func stopNfsServer(serverPod *v1.Pod) {
+	const stopcmd = "/usr/sbin/rpc.nfsd 0"
+	ns := fmt.Sprintf("--namespace=%v", serverPod.Namespace)
+	framework.RunKubectlOrDie("exec", ns, serverPod.Name, "--", "/bin/sh", "-c", stopcmd)
+}
+
+// Creates a pod that mounts an nfs volume that is served by the nfs-server pod. The container
+// will execute the passed in shell cmd. Waits for the pod to start.
+// Note: the nfs plugin is defined inline, no PV or PVC.
+func createPodUsingNfs(f *framework.Framework, c clientset.Interface, ns, nfsIP, cmd string) *v1.Pod {
+	By("create pod using nfs volume")
+
+	isPrivileged := true
+	cmdLine := []string{"-c", cmd}
+	pod := &v1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: testapi.Groups[v1.GroupName].GroupVersion().String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "pod-nfs-vol-",
+			Namespace:    ns,
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:    "pod-nfs-vol",
+					Image:   imageutils.GetBusyBoxImage(),
+					Command: []string{"/bin/sh"},
+					Args:    cmdLine,
+					VolumeMounts: []v1.VolumeMount{
+						{
+							Name:      "nfs-vol",
+							MountPath: "/mnt",
+						},
+					},
+					SecurityContext: &v1.SecurityContext{
+						Privileged: &isPrivileged,
+					},
+				},
+			},
+			RestartPolicy: v1.RestartPolicyNever, //don't restart pod
+			Volumes: []v1.Volume{
+				{
+					Name: "nfs-vol",
+					VolumeSource: v1.VolumeSource{
+						NFS: &v1.NFSVolumeSource{
+							Server:   nfsIP,
+							Path:     "/exports",
+							ReadOnly: false,
+						},
+					},
+				},
+			},
+		},
+	}
+	rtnPod, err := c.CoreV1().Pods(ns).Create(pod)
+	Expect(err).NotTo(HaveOccurred())
+
+	err = f.WaitForPodReady(rtnPod.Name) // running & ready
+	Expect(err).NotTo(HaveOccurred())
+
+	rtnPod, err = c.CoreV1().Pods(ns).Get(rtnPod.Name, metav1.GetOptions{}) // return fresh pod
+	Expect(err).NotTo(HaveOccurred())
+	return rtnPod
+}
+
+// Checks for a lingering nfs mount and/or uid directory on the pod's host. The host IP is used
+// so that this test runs in GCE, where it appears that SSH cannot resolve the hostname.
+// If expectClean is true then we expect the node to be cleaned up and thus commands like
+// `ls <uid-dir>` should fail (since that dir was removed). If expectClean is false then we expect
+// the node is not cleaned up, and thus cmds like `ls <uid-dir>` should succeed. We wait for the
+// kubelet to be cleaned up, afterwhich an error is reported.
+func checkPodCleanup(c clientset.Interface, pod *v1.Pod, expectClean bool) {
+	timeout := 5 * time.Minute
+	poll := 20 * time.Second
+	podDir := filepath.Join("/var/lib/kubelet/pods", string(pod.UID))
+	mountDir := filepath.Join(podDir, "volumes", "kubernetes.io~nfs")
+	// use ip rather than hostname in GCE
+	nodeIP, err := framework.GetHostExternalAddress(c, pod)
+	Expect(err).NotTo(HaveOccurred())
+
+	condMsg := "deleted"
+	if !expectClean {
+		condMsg = "present"
+	}
+
+	// table of host tests to perform (order may matter so not using a map)
+	type testT struct {
+		feature string // feature to test
+		cmd     string // remote command to execute on node
+	}
+	tests := []testT{
+		{
+			feature: "pod UID directory",
+			cmd:     fmt.Sprintf("sudo ls %v", podDir),
+		},
+		{
+			feature: "pod nfs mount",
+			cmd:     fmt.Sprintf("sudo mount | grep %v", mountDir),
+		},
+	}
+
+	for _, test := range tests {
+		framework.Logf("Wait up to %v for host's (%v) %q to be %v", timeout, nodeIP, test.feature, condMsg)
+		err = wait.Poll(poll, timeout, func() (bool, error) {
+			result, err := framework.NodeExec(nodeIP, test.cmd)
+			Expect(err).NotTo(HaveOccurred())
+			framework.LogSSHResult(result)
+			ok := (result.Code == 0 && len(result.Stdout) > 0 && len(result.Stderr) == 0)
+			if expectClean && ok { // keep trying
+				return false, nil
+			}
+			if !expectClean && !ok { // stop wait loop
+				return true, fmt.Errorf("%v is gone but expected to exist", test.feature)
+			}
+			return true, nil // done, host is as expected
+		})
+		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Host (%v) cleanup error: %v. Expected %q to be %v", nodeIP, err, test.feature, condMsg))
+	}
+
+	if expectClean {
+		framework.Logf("Pod's host has been cleaned up")
+	} else {
+		framework.Logf("Pod's host has not been cleaned up (per expectation)")
+	}
+}
+
+var _ = SIGDescribe("kubelet", func() {
+	var (
+		c  clientset.Interface
+		ns string
+	)
+	f := framework.NewDefaultFramework("kubelet")
+
+	BeforeEach(func() {
+		c = f.ClientSet
+		ns = f.Namespace.Name
+	})
+
+	SIGDescribe("Clean up pods on node", func() {
+		var (
+			numNodes        int
+			nodeNames       sets.String
+			nodeLabels      map[string]string
+			resourceMonitor *framework.ResourceMonitor
+		)
+		type DeleteTest struct {
+			podsPerNode int
+			timeout     time.Duration
+		}
+
+		deleteTests := []DeleteTest{
+			{podsPerNode: 10, timeout: 1 * time.Minute},
+		}
+
+		BeforeEach(func() {
+			// Use node labels to restrict the pods to be assigned only to the
+			// nodes we observe initially.
+			nodeLabels = make(map[string]string)
+			nodeLabels["kubelet_cleanup"] = "true"
+			nodes := framework.GetReadySchedulableNodesOrDie(c)
+			numNodes = len(nodes.Items)
+			Expect(numNodes).NotTo(BeZero())
+			nodeNames = sets.NewString()
+			// If there are a lot of nodes, we don't want to use all of them
+			// (if there are 1000 nodes in the cluster, starting 10 pods/node
+			// will take ~10 minutes today). And there is also deletion phase.
+			// Instead, we choose at most 10 nodes.
+			if numNodes > maxNodesToCheck {
+				numNodes = maxNodesToCheck
+			}
+			for i := 0; i < numNodes; i++ {
+				nodeNames.Insert(nodes.Items[i].Name)
+			}
+			updateNodeLabels(c, nodeNames, nodeLabels, nil)
+
+			// Start resourceMonitor only in small clusters.
+			if len(nodes.Items) <= maxNodesToCheck {
+				resourceMonitor = framework.NewResourceMonitor(f.ClientSet, framework.TargetContainers(), containerStatsPollingInterval)
+				resourceMonitor.Start()
+			}
+		})
+
+		AfterEach(func() {
+			if resourceMonitor != nil {
+				resourceMonitor.Stop()
+			}
+			// If we added labels to nodes in this test, remove them now.
+			updateNodeLabels(c, nodeNames, nil, nodeLabels)
+		})
+
+		for _, itArg := range deleteTests {
+			name := fmt.Sprintf(
+				"kubelet should be able to delete %d pods per node in %v.", itArg.podsPerNode, itArg.timeout)
+			It(name, func() {
+				totalPods := itArg.podsPerNode * numNodes
+				By(fmt.Sprintf("Creating a RC of %d pods and wait until all pods of this RC are running", totalPods))
+				rcName := fmt.Sprintf("cleanup%d-%s", totalPods, string(uuid.NewUUID()))
+
+				Expect(framework.RunRC(testutils.RCConfig{
+					Client:         f.ClientSet,
+					InternalClient: f.InternalClientset,
+					Name:           rcName,
+					Namespace:      f.Namespace.Name,
+					Image:          framework.GetPauseImageName(f.ClientSet),
+					Replicas:       totalPods,
+					NodeSelector:   nodeLabels,
+				})).NotTo(HaveOccurred())
+				// Perform a sanity check so that we know all desired pods are
+				// running on the nodes according to kubelet. The timeout is set to
+				// only 30 seconds here because framework.RunRC already waited for all pods to
+				// transition to the running status.
+				Expect(waitTillNPodsRunningOnNodes(f.ClientSet, nodeNames, rcName, ns, totalPods,
+					time.Second*30)).NotTo(HaveOccurred())
+				if resourceMonitor != nil {
+					resourceMonitor.LogLatest()
+				}
+
+				By("Deleting the RC")
+				framework.DeleteRCAndPods(f.ClientSet, f.InternalClientset, f.Namespace.Name, rcName)
+				// Check that the pods really are gone by querying /runningpods on the
+				// node. The /runningpods handler checks the container runtime (or its
+				// cache) and  returns a list of running pods. Some possible causes of
+				// failures are:
+				//   - kubelet deadlock
+				//   - a bug in graceful termination (if it is enabled)
+				//   - docker slow to delete pods (or resource problems causing slowness)
+				start := time.Now()
+				Expect(waitTillNPodsRunningOnNodes(f.ClientSet, nodeNames, rcName, ns, 0,
+					itArg.timeout)).NotTo(HaveOccurred())
+				framework.Logf("Deleting %d pods on %d nodes completed in %v after the RC was deleted", totalPods, len(nodeNames),
+					time.Since(start))
+				if resourceMonitor != nil {
+					resourceMonitor.LogCPUSummary()
+				}
+			})
+		}
+	})
+
+	// Test host cleanup when disrupting the volume environment.
+	SIGDescribe("host cleanup with volume mounts [sig-storage][HostCleanup][Flaky]", func() {
+
+		type hostCleanupTest struct {
+			itDescr string
+			podCmd  string
+		}
+
+		// Disrupt the nfs-server pod after a client pod accesses the nfs volume.
+		// Note: the nfs-server is stopped NOT deleted. This is done to preserve its ip addr.
+		//       If the nfs-server pod is deleted the client pod's mount can not be unmounted.
+		//       If the nfs-server pod is deleted and re-created, due to having a different ip
+		//       addr, the client pod's mount still cannot be unmounted.
+		Context("Host cleanup after disrupting NFS volume [NFS]", func() {
+			// issue #31272
+			var (
+				nfsServerPod *v1.Pod
+				nfsIP        string
+				NFSconfig    framework.VolumeTestConfig
+				pod          *v1.Pod // client pod
+			)
+
+			// fill in test slice for this context
+			testTbl := []hostCleanupTest{
+				{
+					itDescr: "after stopping the nfs-server and deleting the (sleeping) client pod, the NFS mount and the pod's UID directory should be removed.",
+					podCmd:  "sleep 6000", // keep pod running
+				},
+				{
+					itDescr: "after stopping the nfs-server and deleting the (active) client pod, the NFS mount and the pod's UID directory should be removed.",
+					podCmd:  "while true; do echo FeFieFoFum >>/mnt/SUCCESS; sleep 1; cat /mnt/SUCCESS; done",
+				},
+			}
+
+			BeforeEach(func() {
+				framework.SkipUnlessProviderIs(framework.ProvidersWithSSH...)
+				NFSconfig, nfsServerPod, nfsIP = framework.NewNFSServer(c, ns, []string{"-G", "777", "/exports"})
+			})
+
+			AfterEach(func() {
+				framework.ExpectNoError(framework.DeletePodWithWait(f, c, pod), "AfterEach: Failed to delete pod ", pod.Name)
+				framework.ExpectNoError(framework.DeletePodWithWait(f, c, nfsServerPod), "AfterEach: Failed to delete pod ", nfsServerPod.Name)
+			})
+
+			// execute It blocks from above table of tests
+			for _, t := range testTbl {
+				It(t.itDescr, func() {
+					pod = createPodUsingNfs(f, c, ns, nfsIP, t.podCmd)
+
+					By("Stop the NFS server")
+					stopNfsServer(nfsServerPod)
+
+					By("Delete the pod mounted to the NFS volume")
+					framework.ExpectNoError(framework.DeletePodWithWait(f, c, pod), "Failed to delete pod ", pod.Name)
+					// pod object is now stale, but is intentionally not nil
+
+					By("Check if pod's host has been cleaned up -- expect not")
+					checkPodCleanup(c, pod, false)
+
+					By("Restart the nfs server")
+					restartNfsServer(nfsServerPod)
+
+					By("Verify host running the deleted pod is now cleaned up")
+					checkPodCleanup(c, pod, true)
+				})
+			}
+		})
+	})
+})

vendor/k8s.io/kubernetes/test/e2e/node/kubelet_perf.go

@@ -0,0 +1,285 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package node
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	clientset "k8s.io/client-go/kubernetes"
+	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
+	"k8s.io/kubernetes/test/e2e/framework"
+	testutils "k8s.io/kubernetes/test/utils"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	// Interval to poll /stats/container on a node
+	containerStatsPollingPeriod = 10 * time.Second
+	// The monitoring time for one test.
+	monitoringTime = 20 * time.Minute
+	// The periodic reporting period.
+	reportingPeriod = 5 * time.Minute
+	// Timeout for waiting for the image prepulling to complete.
+	imagePrePullingLongTimeout = time.Minute * 8
+)
+
+type resourceTest struct {
+	podsPerNode int
+	cpuLimits   framework.ContainersCPUSummary
+	memLimits   framework.ResourceUsagePerContainer
+}
+
+func logPodsOnNodes(c clientset.Interface, nodeNames []string) {
+	for _, n := range nodeNames {
+		podList, err := framework.GetKubeletRunningPods(c, n)
+		if err != nil {
+			framework.Logf("Unable to retrieve kubelet pods for node %v", n)
+			continue
+		}
+		framework.Logf("%d pods are running on node %v", len(podList.Items), n)
+	}
+}
+
+func runResourceTrackingTest(f *framework.Framework, podsPerNode int, nodeNames sets.String, rm *framework.ResourceMonitor,
+	expectedCPU map[string]map[float64]float64, expectedMemory framework.ResourceUsagePerContainer) {
+	numNodes := nodeNames.Len()
+	totalPods := podsPerNode * numNodes
+	By(fmt.Sprintf("Creating a RC of %d pods and wait until all pods of this RC are running", totalPods))
+	rcName := fmt.Sprintf("resource%d-%s", totalPods, string(uuid.NewUUID()))
+
+	// TODO: Use a more realistic workload
+	Expect(framework.RunRC(testutils.RCConfig{
+		Client:         f.ClientSet,
+		InternalClient: f.InternalClientset,
+		Name:           rcName,
+		Namespace:      f.Namespace.Name,
+		Image:          framework.GetPauseImageName(f.ClientSet),
+		Replicas:       totalPods,
+	})).NotTo(HaveOccurred())
+
+	// Log once and flush the stats.
+	rm.LogLatest()
+	rm.Reset()
+
+	By("Start monitoring resource usage")
+	// Periodically dump the cpu summary until the deadline is met.
+	// Note that without calling framework.ResourceMonitor.Reset(), the stats
+	// would occupy increasingly more memory. This should be fine
+	// for the current test duration, but we should reclaim the
+	// entries if we plan to monitor longer (e.g., 8 hours).
+	deadline := time.Now().Add(monitoringTime)
+	for time.Now().Before(deadline) {
+		timeLeft := deadline.Sub(time.Now())
+		framework.Logf("Still running...%v left", timeLeft)
+		if timeLeft < reportingPeriod {
+			time.Sleep(timeLeft)
+		} else {
+			time.Sleep(reportingPeriod)
+		}
+		logPodsOnNodes(f.ClientSet, nodeNames.List())
+	}
+
+	By("Reporting overall resource usage")
+	logPodsOnNodes(f.ClientSet, nodeNames.List())
+	usageSummary, err := rm.GetLatest()
+	Expect(err).NotTo(HaveOccurred())
+	// TODO(random-liu): Remove the original log when we migrate to new perfdash
+	framework.Logf("%s", rm.FormatResourceUsage(usageSummary))
+	// Log perf result
+	framework.PrintPerfData(framework.ResourceUsageToPerfData(rm.GetMasterNodeLatest(usageSummary)))
+	verifyMemoryLimits(f.ClientSet, expectedMemory, usageSummary)
+
+	cpuSummary := rm.GetCPUSummary()
+	framework.Logf("%s", rm.FormatCPUSummary(cpuSummary))
+	// Log perf result
+	framework.PrintPerfData(framework.CPUUsageToPerfData(rm.GetMasterNodeCPUSummary(cpuSummary)))
+	verifyCPULimits(expectedCPU, cpuSummary)
+
+	By("Deleting the RC")
+	framework.DeleteRCAndPods(f.ClientSet, f.InternalClientset, f.Namespace.Name, rcName)
+}
+
+func verifyMemoryLimits(c clientset.Interface, expected framework.ResourceUsagePerContainer, actual framework.ResourceUsagePerNode) {
+	if expected == nil {
+		return
+	}
+	var errList []string
+	for nodeName, nodeSummary := range actual {
+		var nodeErrs []string
+		for cName, expectedResult := range expected {
+			container, ok := nodeSummary[cName]
+			if !ok {
+				nodeErrs = append(nodeErrs, fmt.Sprintf("container %q: missing", cName))
+				continue
+			}
+
+			expectedValue := expectedResult.MemoryRSSInBytes
+			actualValue := container.MemoryRSSInBytes
+			if expectedValue != 0 && actualValue > expectedValue {
+				nodeErrs = append(nodeErrs, fmt.Sprintf("container %q: expected RSS memory (MB) < %d; got %d",
+					cName, expectedValue, actualValue))
+			}
+		}
+		if len(nodeErrs) > 0 {
+			errList = append(errList, fmt.Sprintf("node %v:\n %s", nodeName, strings.Join(nodeErrs, ", ")))
+			heapStats, err := framework.GetKubeletHeapStats(c, nodeName)
+			if err != nil {
+				framework.Logf("Unable to get heap stats from %q", nodeName)
+			} else {
+				framework.Logf("Heap stats on %q\n:%v", nodeName, heapStats)
+			}
+		}
+	}
+	if len(errList) > 0 {
+		framework.Failf("Memory usage exceeding limits:\n %s", strings.Join(errList, "\n"))
+	}
+}
+
+func verifyCPULimits(expected framework.ContainersCPUSummary, actual framework.NodesCPUSummary) {
+	if expected == nil {
+		return
+	}
+	var errList []string
+	for nodeName, perNodeSummary := range actual {
+		var nodeErrs []string
+		for cName, expectedResult := range expected {
+			perContainerSummary, ok := perNodeSummary[cName]
+			if !ok {
+				nodeErrs = append(nodeErrs, fmt.Sprintf("container %q: missing", cName))
+				continue
+			}
+			for p, expectedValue := range expectedResult {
+				actualValue, ok := perContainerSummary[p]
+				if !ok {
+					nodeErrs = append(nodeErrs, fmt.Sprintf("container %q: missing percentile %v", cName, p))
+					continue
+				}
+				if actualValue > expectedValue {
+					nodeErrs = append(nodeErrs, fmt.Sprintf("container %q: expected %.0fth%% usage < %.3f; got %.3f",
+						cName, p*100, expectedValue, actualValue))
+				}
+			}
+		}
+		if len(nodeErrs) > 0 {
+			errList = append(errList, fmt.Sprintf("node %v:\n %s", nodeName, strings.Join(nodeErrs, ", ")))
+		}
+	}
+	if len(errList) > 0 {
+		framework.Failf("CPU usage exceeding limits:\n %s", strings.Join(errList, "\n"))
+	}
+}
+
+// Slow by design (1 hour)
+var _ = SIGDescribe("Kubelet [Serial] [Slow]", func() {
+	var nodeNames sets.String
+	f := framework.NewDefaultFramework("kubelet-perf")
+	var om *framework.RuntimeOperationMonitor
+	var rm *framework.ResourceMonitor
+
+	BeforeEach(func() {
+		// Wait until image prepull pod has completed so that they wouldn't
+		// affect the runtime cpu usage. Fail the test if prepulling cannot
+		// finish in time.
+		if err := framework.WaitForPodsSuccess(f.ClientSet, metav1.NamespaceSystem, framework.ImagePullerLabels, imagePrePullingLongTimeout); err != nil {
+			framework.Failf("Image puller didn't complete in %v, not running resource usage test since the metrics might be adultrated", imagePrePullingLongTimeout)
+		}
+		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+		nodeNames = sets.NewString()
+		for _, node := range nodes.Items {
+			nodeNames.Insert(node.Name)
+		}
+		om = framework.NewRuntimeOperationMonitor(f.ClientSet)
+		rm = framework.NewResourceMonitor(f.ClientSet, framework.TargetContainers(), containerStatsPollingPeriod)
+		rm.Start()
+	})
+
+	AfterEach(func() {
+		rm.Stop()
+		result := om.GetLatestRuntimeOperationErrorRate()
+		framework.Logf("runtime operation error metrics:\n%s", framework.FormatRuntimeOperationErrorRate(result))
+	})
+	SIGDescribe("regular resource usage tracking", func() {
+		// We assume that the scheduler will make reasonable scheduling choices
+		// and assign ~N pods on the node.
+		// Although we want to track N pods per node, there are N + add-on pods
+		// in the cluster. The cluster add-on pods can be distributed unevenly
+		// among the nodes because they are created during the cluster
+		// initialization. This *noise* is obvious when N is small. We
+		// deliberately set higher resource usage limits to account for the
+		// noise.
+		//
+		// We set all resource limits generously because this test is mainly
+		// used to catch resource leaks in the soak cluster. For tracking
+		// kubelet/runtime resource usage, please see the node e2e benchmark
+		// dashboard. http://node-perf-dash.k8s.io/
+		//
+		// TODO(#36621): Deprecate this test once we have a node e2e soak
+		// cluster.
+		rTests := []resourceTest{
+			{
+				podsPerNode: 0,
+				cpuLimits: framework.ContainersCPUSummary{
+					stats.SystemContainerKubelet: {0.50: 0.10, 0.95: 0.20},
+					stats.SystemContainerRuntime: {0.50: 0.10, 0.95: 0.20},
+				},
+				memLimits: framework.ResourceUsagePerContainer{
+					stats.SystemContainerKubelet: &framework.ContainerResourceUsage{MemoryRSSInBytes: 200 * 1024 * 1024},
+					// The detail can be found at https://github.com/kubernetes/kubernetes/issues/28384#issuecomment-244158892
+					stats.SystemContainerRuntime: &framework.ContainerResourceUsage{MemoryRSSInBytes: 125 * 1024 * 1024},
+				},
+			},
+			{
+				cpuLimits: framework.ContainersCPUSummary{
+					stats.SystemContainerKubelet: {0.50: 0.35, 0.95: 0.50},
+					stats.SystemContainerRuntime: {0.50: 0.10, 0.95: 0.50},
+				},
+				podsPerNode: 100,
+				memLimits: framework.ResourceUsagePerContainer{
+					stats.SystemContainerKubelet: &framework.ContainerResourceUsage{MemoryRSSInBytes: 300 * 1024 * 1024},
+					stats.SystemContainerRuntime: &framework.ContainerResourceUsage{MemoryRSSInBytes: 300 * 1024 * 1024},
+				},
+			},
+		}
+		for _, testArg := range rTests {
+			itArg := testArg
+			podsPerNode := itArg.podsPerNode
+			name := fmt.Sprintf(
+				"resource tracking for %d pods per node", podsPerNode)
+			It(name, func() {
+				runResourceTrackingTest(f, podsPerNode, nodeNames, rm, itArg.cpuLimits, itArg.memLimits)
+			})
+		}
+	})
+	SIGDescribe("experimental resource usage tracking [Feature:ExperimentalResourceUsageTracking]", func() {
+		density := []int{100}
+		for i := range density {
+			podsPerNode := density[i]
+			name := fmt.Sprintf(
+				"resource tracking for %d pods per node", podsPerNode)
+			It(name, func() {
+				runResourceTrackingTest(f, podsPerNode, nodeNames, rm, nil, nil)
+			})
+		}
+	})
+})

vendor/k8s.io/kubernetes/test/e2e/node/security_context.go

@@ -0,0 +1,235 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/* This test check that SecurityContext parameters specified at the
+ * pod or the container level work as intended. These tests cannot be
+ * run when the 'SecurityContextDeny' admission controller is not used
+ * so they are skipped by default.
+ */
+
+package node
+
+import (
+	"fmt"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+)
+
+func scTestPod(hostIPC bool, hostPID bool) *v1.Pod {
+	podName := "security-context-" + string(uuid.NewUUID())
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        podName,
+			Labels:      map[string]string{"name": podName},
+			Annotations: map[string]string{},
+		},
+		Spec: v1.PodSpec{
+			HostIPC:         hostIPC,
+			HostPID:         hostPID,
+			SecurityContext: &v1.PodSecurityContext{},
+			Containers: []v1.Container{
+				{
+					Name:  "test-container",
+					Image: imageutils.GetBusyBoxImage(),
+				},
+			},
+			RestartPolicy: v1.RestartPolicyNever,
+		},
+	}
+
+	return pod
+}
+
+var _ = SIGDescribe("Security Context [Feature:SecurityContext]", func() {
+	f := framework.NewDefaultFramework("security-context")
+
+	It("should support pod.Spec.SecurityContext.SupplementalGroups", func() {
+		pod := scTestPod(false, false)
+		pod.Spec.Containers[0].Command = []string{"id", "-G"}
+		pod.Spec.SecurityContext.SupplementalGroups = []int64{1234, 5678}
+		groups := []string{"1234", "5678"}
+		f.TestContainerOutput("pod.Spec.SecurityContext.SupplementalGroups", pod, 0, groups)
+	})
+
+	It("should support pod.Spec.SecurityContext.RunAsUser", func() {
+		pod := scTestPod(false, false)
+		userID := int64(1001)
+		pod.Spec.SecurityContext.RunAsUser = &userID
+		pod.Spec.Containers[0].Command = []string{"sh", "-c", "id -u"}
+
+		f.TestContainerOutput("pod.Spec.SecurityContext.RunAsUser", pod, 0, []string{
+			fmt.Sprintf("%v", userID),
+		})
+	})
+
+	It("should support container.SecurityContext.RunAsUser", func() {
+		pod := scTestPod(false, false)
+		userID := int64(1001)
+		overrideUserID := int64(1002)
+		pod.Spec.SecurityContext.RunAsUser = &userID
+		pod.Spec.Containers[0].SecurityContext = new(v1.SecurityContext)
+		pod.Spec.Containers[0].SecurityContext.RunAsUser = &overrideUserID
+		pod.Spec.Containers[0].Command = []string{"sh", "-c", "id -u"}
+
+		f.TestContainerOutput("pod.Spec.SecurityContext.RunAsUser", pod, 0, []string{
+			fmt.Sprintf("%v", overrideUserID),
+		})
+	})
+
+	It("should support volume SELinux relabeling", func() {
+		testPodSELinuxLabeling(f, false, false)
+	})
+
+	It("should support volume SELinux relabeling when using hostIPC", func() {
+		testPodSELinuxLabeling(f, true, false)
+	})
+
+	It("should support volume SELinux relabeling when using hostPID", func() {
+		testPodSELinuxLabeling(f, false, true)
+	})
+
+	It("should support seccomp alpha unconfined annotation on the container [Feature:Seccomp]", func() {
+		// TODO: port to SecurityContext as soon as seccomp is out of alpha
+		pod := scTestPod(false, false)
+		pod.Annotations[v1.SeccompContainerAnnotationKeyPrefix+"test-container"] = "unconfined"
+		pod.Annotations[v1.SeccompPodAnnotationKey] = "docker/default"
+		pod.Spec.Containers[0].Command = []string{"grep", "ecc", "/proc/self/status"}
+		f.TestContainerOutput(v1.SeccompPodAnnotationKey, pod, 0, []string{"0"}) // seccomp disabled
+	})
+
+	It("should support seccomp alpha unconfined annotation on the pod [Feature:Seccomp]", func() {
+		// TODO: port to SecurityContext as soon as seccomp is out of alpha
+		pod := scTestPod(false, false)
+		pod.Annotations[v1.SeccompPodAnnotationKey] = "unconfined"
+		pod.Spec.Containers[0].Command = []string{"grep", "ecc", "/proc/self/status"}
+		f.TestContainerOutput(v1.SeccompPodAnnotationKey, pod, 0, []string{"0"}) // seccomp disabled
+	})
+
+	It("should support seccomp alpha docker/default annotation [Feature:Seccomp]", func() {
+		// TODO: port to SecurityContext as soon as seccomp is out of alpha
+		pod := scTestPod(false, false)
+		pod.Annotations[v1.SeccompContainerAnnotationKeyPrefix+"test-container"] = "docker/default"
+		pod.Spec.Containers[0].Command = []string{"grep", "ecc", "/proc/self/status"}
+		f.TestContainerOutput(v1.SeccompPodAnnotationKey, pod, 0, []string{"2"}) // seccomp filtered
+	})
+
+	It("should support seccomp default which is unconfined [Feature:Seccomp]", func() {
+		// TODO: port to SecurityContext as soon as seccomp is out of alpha
+		pod := scTestPod(false, false)
+		pod.Spec.Containers[0].Command = []string{"grep", "ecc", "/proc/self/status"}
+		f.TestContainerOutput(v1.SeccompPodAnnotationKey, pod, 0, []string{"0"}) // seccomp disabled
+	})
+})
+
+func testPodSELinuxLabeling(f *framework.Framework, hostIPC bool, hostPID bool) {
+	// Write and read a file with an empty_dir volume
+	// with a pod with the MCS label s0:c0,c1
+	pod := scTestPod(hostIPC, hostPID)
+	volumeName := "test-volume"
+	mountPath := "/mounted_volume"
+	pod.Spec.Containers[0].VolumeMounts = []v1.VolumeMount{
+		{
+			Name:      volumeName,
+			MountPath: mountPath,
+		},
+	}
+	pod.Spec.Volumes = []v1.Volume{
+		{
+			Name: volumeName,
+			VolumeSource: v1.VolumeSource{
+				EmptyDir: &v1.EmptyDirVolumeSource{
+					Medium: v1.StorageMediumDefault,
+				},
+			},
+		},
+	}
+	pod.Spec.SecurityContext.SELinuxOptions = &v1.SELinuxOptions{
+		Level: "s0:c0,c1",
+	}
+	pod.Spec.Containers[0].Command = []string{"sleep", "6000"}
+
+	client := f.ClientSet.Core().Pods(f.Namespace.Name)
+	pod, err := client.Create(pod)
+
+	framework.ExpectNoError(err, "Error creating pod %v", pod)
+	framework.ExpectNoError(framework.WaitForPodRunningInNamespace(f.ClientSet, pod))
+
+	testContent := "hello"
+	testFilePath := mountPath + "/TEST"
+	err = f.WriteFileViaContainer(pod.Name, pod.Spec.Containers[0].Name, testFilePath, testContent)
+	Expect(err).To(BeNil())
+	content, err := f.ReadFileViaContainer(pod.Name, pod.Spec.Containers[0].Name, testFilePath)
+	Expect(err).To(BeNil())
+	Expect(content).To(ContainSubstring(testContent))
+
+	foundPod, err := f.ClientSet.Core().Pods(f.Namespace.Name).Get(pod.Name, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+
+	// Confirm that the file can be accessed from a second
+	// pod using host_path with the same MCS label
+	volumeHostPath := fmt.Sprintf("%s/pods/%s/volumes/kubernetes.io~empty-dir/%s", framework.TestContext.KubeVolumeDir, foundPod.UID, volumeName)
+	By(fmt.Sprintf("confirming a container with the same label can read the file under --volume-dir=%s", framework.TestContext.KubeVolumeDir))
+	pod = scTestPod(hostIPC, hostPID)
+	pod.Spec.NodeName = foundPod.Spec.NodeName
+	volumeMounts := []v1.VolumeMount{
+		{
+			Name:      volumeName,
+			MountPath: mountPath,
+		},
+	}
+	volumes := []v1.Volume{
+		{
+			Name: volumeName,
+			VolumeSource: v1.VolumeSource{
+				HostPath: &v1.HostPathVolumeSource{
+					Path: volumeHostPath,
+				},
+			},
+		},
+	}
+	pod.Spec.Containers[0].VolumeMounts = volumeMounts
+	pod.Spec.Volumes = volumes
+	pod.Spec.Containers[0].Command = []string{"cat", testFilePath}
+	pod.Spec.SecurityContext.SELinuxOptions = &v1.SELinuxOptions{
+		Level: "s0:c0,c1",
+	}
+
+	f.TestContainerOutput("Pod with same MCS label reading test file", pod, 0, []string{testContent})
+	// Confirm that the same pod with a different MCS
+	// label cannot access the volume
+	pod = scTestPod(hostIPC, hostPID)
+	pod.Spec.Volumes = volumes
+	pod.Spec.Containers[0].VolumeMounts = volumeMounts
+	pod.Spec.Containers[0].Command = []string{"sleep", "6000"}
+	pod.Spec.SecurityContext.SELinuxOptions = &v1.SELinuxOptions{
+		Level: "s0:c2,c3",
+	}
+	_, err = client.Create(pod)
+	framework.ExpectNoError(err, "Error creating pod %v", pod)
+
+	err = f.WaitForPodRunning(pod.Name)
+	framework.ExpectNoError(err, "Error waiting for pod to run %v", pod)
+
+	content, err = f.ReadFileViaContainer(pod.Name, "test-container", testFilePath)
+	Expect(content).NotTo(ContainSubstring(testContent))
+}