Replace godep with dep

vendor/k8s.io/kubernetes/test/e2e/storage/BUILD

@@ -0,0 +1,88 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "empty_dir_wrapper.go",
+        "flexvolume.go",
+        "framework.go",
+        "pd.go",
+        "persistent_volumes.go",
+        "persistent_volumes-disruptive.go",
+        "persistent_volumes-gce.go",
+        "persistent_volumes-local.go",
+        "persistent_volumes-vsphere.go",
+        "pv_reclaimpolicy.go",
+        "pvc_label_selector.go",
+        "volume_io.go",
+        "volume_metrics.go",
+        "volume_provisioning.go",
+        "volumes.go",
+        "vsphere_utils.go",
+        "vsphere_volume_diskformat.go",
+        "vsphere_volume_fstype.go",
+        "vsphere_volume_ops_storm.go",
+        "vsphere_volume_placement.go",
+        "vsphere_volume_vsan_policy.go",
+    ],
+    deps = [
+        "//pkg/api/testapi:go_default_library",
+        "//pkg/api/v1/helper:go_default_library",
+        "//pkg/apis/storage/v1/util:go_default_library",
+        "//pkg/cloudprovider/providers/vsphere:go_default_library",
+        "//pkg/cloudprovider/providers/vsphere/vclib:go_default_library",
+        "//pkg/kubelet/apis:go_default_library",
+        "//pkg/kubelet/metrics:go_default_library",
+        "//pkg/volume/util/volumehelper:go_default_library",
+        "//test/e2e/framework:go_default_library",
+        "//test/e2e/framework/metrics:go_default_library",
+        "//test/e2e/generated:go_default_library",
+        "//test/utils/image:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
+        "//vendor/github.com/onsi/ginkgo:go_default_library",
+        "//vendor/github.com/onsi/gomega:go_default_library",
+        "//vendor/github.com/vmware/govmomi/find:go_default_library",
+        "//vendor/github.com/vmware/govmomi/vim25/types:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/google.golang.org/api/googleapi:go_default_library",
+        "//vendor/k8s.io/api/batch/v1:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/api/rbac/v1beta1:go_default_library",
+        "//vendor/k8s.io/api/storage/v1:go_default_library",
+        "//vendor/k8s.io/api/storage/v1beta1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/test/e2e/storage/OWNERS

@@ -0,0 +1,15 @@
+approvers:
+- saad-ali
+- rootfs
+- gnufied 
+- jingxu97
+- jsafrane
+reviewers:
+- saad-ali
+- rootfs
+- gnufied 
+- jingxu97
+- jsafrane
+- msau42
+- jeffvance
+- copejon

vendor/k8s.io/kubernetes/test/e2e/storage/empty_dir_wrapper.go

@@ -0,0 +1,392 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/kubernetes/test/e2e/framework"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+
+	"fmt"
+	"strconv"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	// These numbers are obtained empirically.
+	// If you make them too low, you'll get flaky
+	// tests instead of failing ones if the race bug reappears.
+	// If you make volume counts or pod counts too high,
+	// the tests may fail because mounting configmap/git_repo
+	// volumes is not very fast and the tests may time out
+	// waiting for pods to become Running.
+	// And of course the higher are the numbers, the
+	// slower are the tests.
+	wrappedVolumeRaceConfigMapVolumeCount    = 50
+	wrappedVolumeRaceConfigMapPodCount       = 5
+	wrappedVolumeRaceConfigMapIterationCount = 3
+	wrappedVolumeRaceGitRepoVolumeCount      = 50
+	wrappedVolumeRaceGitRepoPodCount         = 5
+	wrappedVolumeRaceGitRepoIterationCount   = 3
+	wrappedVolumeRaceRCNamePrefix            = "wrapped-volume-race-"
+)
+
+var _ = SIGDescribe("EmptyDir wrapper volumes", func() {
+	f := framework.NewDefaultFramework("emptydir-wrapper")
+
+	It("should not conflict", func() {
+		name := "emptydir-wrapper-test-" + string(uuid.NewUUID())
+		volumeName := "secret-volume"
+		volumeMountPath := "/etc/secret-volume"
+
+		secret := &v1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: f.Namespace.Name,
+				Name:      name,
+			},
+			Data: map[string][]byte{
+				"data-1": []byte("value-1\n"),
+			},
+		}
+
+		var err error
+		if secret, err = f.ClientSet.Core().Secrets(f.Namespace.Name).Create(secret); err != nil {
+			framework.Failf("unable to create test secret %s: %v", secret.Name, err)
+		}
+
+		gitVolumeName := "git-volume"
+		gitVolumeMountPath := "/etc/git-volume"
+		gitURL, gitRepo, gitCleanup := createGitServer(f)
+		defer gitCleanup()
+
+		pod := &v1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "pod-secrets-" + string(uuid.NewUUID()),
+			},
+			Spec: v1.PodSpec{
+				Volumes: []v1.Volume{
+					{
+						Name: volumeName,
+						VolumeSource: v1.VolumeSource{
+							Secret: &v1.SecretVolumeSource{
+								SecretName: name,
+							},
+						},
+					},
+					{
+						Name: gitVolumeName,
+						VolumeSource: v1.VolumeSource{
+							GitRepo: &v1.GitRepoVolumeSource{
+								Repository: gitURL,
+								Directory:  gitRepo,
+							},
+						},
+					},
+				},
+				Containers: []v1.Container{
+					{
+						Name:  "secret-test",
+						Image: imageutils.GetE2EImage(imageutils.TestWebserver),
+						VolumeMounts: []v1.VolumeMount{
+							{
+								Name:      volumeName,
+								MountPath: volumeMountPath,
+								ReadOnly:  true,
+							},
+							{
+								Name:      gitVolumeName,
+								MountPath: gitVolumeMountPath,
+							},
+						},
+					},
+				},
+			},
+		}
+		pod = f.PodClient().CreateSync(pod)
+
+		defer func() {
+			By("Cleaning up the secret")
+			if err := f.ClientSet.Core().Secrets(f.Namespace.Name).Delete(secret.Name, nil); err != nil {
+				framework.Failf("unable to delete secret %v: %v", secret.Name, err)
+			}
+			By("Cleaning up the git vol pod")
+			if err = f.ClientSet.Core().Pods(f.Namespace.Name).Delete(pod.Name, metav1.NewDeleteOptions(0)); err != nil {
+				framework.Failf("unable to delete git vol pod %v: %v", pod.Name, err)
+			}
+		}()
+	})
+
+	// The following two tests check for the problem fixed in #29641.
+	// In order to reproduce it you need to revert the fix, e.g. via
+	// git revert -n df1e925143daf34199b55ffb91d0598244888cce
+	// or
+	// curl -sL https://github.com/kubernetes/kubernetes/pull/29641.patch | patch -p1 -R
+	//
+	// After that these tests will fail because some of the pods
+	// they create never enter Running state.
+	//
+	// They need to be [Serial] and [Slow] because they try to induce
+	// the race by creating pods with many volumes and container volume mounts,
+	// which takes considerable time and may interfere with other tests.
+	//
+	// Probably should also try making tests for secrets and downwardapi,
+	// but these cases are harder because tmpfs-based emptyDir
+	// appears to be less prone to the race problem.
+
+	It("should not cause race condition when used for configmaps [Serial] [Slow]", func() {
+		configMapNames := createConfigmapsForRace(f)
+		defer deleteConfigMaps(f, configMapNames)
+		volumes, volumeMounts := makeConfigMapVolumes(configMapNames)
+		for i := 0; i < wrappedVolumeRaceConfigMapIterationCount; i++ {
+			testNoWrappedVolumeRace(f, volumes, volumeMounts, wrappedVolumeRaceConfigMapPodCount)
+		}
+	})
+
+	It("should not cause race condition when used for git_repo [Serial] [Slow]", func() {
+		gitURL, gitRepo, cleanup := createGitServer(f)
+		defer cleanup()
+		volumes, volumeMounts := makeGitRepoVolumes(gitURL, gitRepo)
+		for i := 0; i < wrappedVolumeRaceGitRepoIterationCount; i++ {
+			testNoWrappedVolumeRace(f, volumes, volumeMounts, wrappedVolumeRaceGitRepoPodCount)
+		}
+	})
+})
+
+func createGitServer(f *framework.Framework) (gitURL string, gitRepo string, cleanup func()) {
+	var err error
+	gitServerPodName := "git-server-" + string(uuid.NewUUID())
+	containerPort := 8000
+
+	labels := map[string]string{"name": gitServerPodName}
+
+	gitServerPod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   gitServerPodName,
+			Labels: labels,
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:            "git-repo",
+					Image:           imageutils.GetE2EImage(imageutils.Fakegitserver),
+					ImagePullPolicy: "IfNotPresent",
+					Ports: []v1.ContainerPort{
+						{ContainerPort: int32(containerPort)},
+					},
+				},
+			},
+		},
+	}
+	f.PodClient().CreateSync(gitServerPod)
+
+	// Portal IP and port
+	httpPort := 2345
+
+	gitServerSvc := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "git-server-svc",
+		},
+		Spec: v1.ServiceSpec{
+			Selector: labels,
+			Ports: []v1.ServicePort{
+				{
+					Name:       "http-portal",
+					Port:       int32(httpPort),
+					TargetPort: intstr.FromInt(containerPort),
+				},
+			},
+		},
+	}
+
+	if gitServerSvc, err = f.ClientSet.Core().Services(f.Namespace.Name).Create(gitServerSvc); err != nil {
+		framework.Failf("unable to create test git server service %s: %v", gitServerSvc.Name, err)
+	}
+
+	return "http://" + gitServerSvc.Spec.ClusterIP + ":" + strconv.Itoa(httpPort), "test", func() {
+		By("Cleaning up the git server pod")
+		if err := f.ClientSet.Core().Pods(f.Namespace.Name).Delete(gitServerPod.Name, metav1.NewDeleteOptions(0)); err != nil {
+			framework.Failf("unable to delete git server pod %v: %v", gitServerPod.Name, err)
+		}
+		By("Cleaning up the git server svc")
+		if err := f.ClientSet.Core().Services(f.Namespace.Name).Delete(gitServerSvc.Name, nil); err != nil {
+			framework.Failf("unable to delete git server svc %v: %v", gitServerSvc.Name, err)
+		}
+	}
+}
+
+func makeGitRepoVolumes(gitURL, gitRepo string) (volumes []v1.Volume, volumeMounts []v1.VolumeMount) {
+	for i := 0; i < wrappedVolumeRaceGitRepoVolumeCount; i++ {
+		volumeName := fmt.Sprintf("racey-git-repo-%d", i)
+		volumes = append(volumes, v1.Volume{
+			Name: volumeName,
+			VolumeSource: v1.VolumeSource{
+				GitRepo: &v1.GitRepoVolumeSource{
+					Repository: gitURL,
+					Directory:  gitRepo,
+				},
+			},
+		})
+		volumeMounts = append(volumeMounts, v1.VolumeMount{
+			Name:      volumeName,
+			MountPath: fmt.Sprintf("/etc/git-volume-%d", i),
+		})
+	}
+	return
+}
+
+func createConfigmapsForRace(f *framework.Framework) (configMapNames []string) {
+	By(fmt.Sprintf("Creating %d configmaps", wrappedVolumeRaceConfigMapVolumeCount))
+	for i := 0; i < wrappedVolumeRaceConfigMapVolumeCount; i++ {
+		configMapName := fmt.Sprintf("racey-configmap-%d", i)
+		configMapNames = append(configMapNames, configMapName)
+		configMap := &v1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: f.Namespace.Name,
+				Name:      configMapName,
+			},
+			Data: map[string]string{
+				"data-1": "value-1",
+			},
+		}
+		_, err := f.ClientSet.Core().ConfigMaps(f.Namespace.Name).Create(configMap)
+		framework.ExpectNoError(err)
+	}
+	return
+}
+
+func deleteConfigMaps(f *framework.Framework, configMapNames []string) {
+	By("Cleaning up the configMaps")
+	for _, configMapName := range configMapNames {
+		err := f.ClientSet.Core().ConfigMaps(f.Namespace.Name).Delete(configMapName, nil)
+		Expect(err).NotTo(HaveOccurred(), "unable to delete configMap %v", configMapName)
+	}
+}
+
+func makeConfigMapVolumes(configMapNames []string) (volumes []v1.Volume, volumeMounts []v1.VolumeMount) {
+	for i, configMapName := range configMapNames {
+		volumeName := fmt.Sprintf("racey-configmap-%d", i)
+		volumes = append(volumes, v1.Volume{
+			Name: volumeName,
+			VolumeSource: v1.VolumeSource{
+				ConfigMap: &v1.ConfigMapVolumeSource{
+					LocalObjectReference: v1.LocalObjectReference{
+						Name: configMapName,
+					},
+					Items: []v1.KeyToPath{
+						{
+							Key:  "data-1",
+							Path: "data-1",
+						},
+					},
+				},
+			},
+		})
+		volumeMounts = append(volumeMounts, v1.VolumeMount{
+			Name:      volumeName,
+			MountPath: fmt.Sprintf("/etc/config-%d", i),
+		})
+	}
+	return
+}
+
+func testNoWrappedVolumeRace(f *framework.Framework, volumes []v1.Volume, volumeMounts []v1.VolumeMount, podCount int32) {
+	rcName := wrappedVolumeRaceRCNamePrefix + string(uuid.NewUUID())
+	nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+	Expect(len(nodeList.Items)).To(BeNumerically(">", 0))
+	targetNode := nodeList.Items[0]
+
+	By("Creating RC which spawns configmap-volume pods")
+	affinity := &v1.Affinity{
+		NodeAffinity: &v1.NodeAffinity{
+			RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
+				NodeSelectorTerms: []v1.NodeSelectorTerm{
+					{
+						MatchExpressions: []v1.NodeSelectorRequirement{
+							{
+								Key:      "kubernetes.io/hostname",
+								Operator: v1.NodeSelectorOpIn,
+								Values:   []string{targetNode.Name},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	rc := &v1.ReplicationController{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: rcName,
+		},
+		Spec: v1.ReplicationControllerSpec{
+			Replicas: &podCount,
+			Selector: map[string]string{
+				"name": rcName,
+			},
+			Template: &v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{"name": rcName},
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name:    "test-container",
+							Image:   imageutils.GetBusyBoxImage(),
+							Command: []string{"sleep", "10000"},
+							Resources: v1.ResourceRequirements{
+								Requests: v1.ResourceList{
+									v1.ResourceCPU: resource.MustParse("10m"),
+								},
+							},
+							VolumeMounts: volumeMounts,
+						},
+					},
+					Affinity:  affinity,
+					DNSPolicy: v1.DNSDefault,
+					Volumes:   volumes,
+				},
+			},
+		},
+	}
+	_, err := f.ClientSet.Core().ReplicationControllers(f.Namespace.Name).Create(rc)
+	Expect(err).NotTo(HaveOccurred(), "error creating replication controller")
+
+	defer func() {
+		err := framework.DeleteRCAndPods(f.ClientSet, f.InternalClientset, f.Namespace.Name, rcName)
+		framework.ExpectNoError(err)
+	}()
+
+	pods, err := framework.PodsCreated(f.ClientSet, f.Namespace.Name, rcName, podCount)
+
+	By("Ensuring each pod is running")
+
+	// Wait for the pods to enter the running state. Waiting loops until the pods
+	// are running so non-running pods cause a timeout for this test.
+	for _, pod := range pods.Items {
+		if pod.DeletionTimestamp != nil {
+			continue
+		}
+		err = f.WaitForPodRunning(pod.Name)
+		Expect(err).NotTo(HaveOccurred(), "Failed waiting for pod %s to enter running state", pod.Name)
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/storage/flexvolume.go

@@ -0,0 +1,229 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"fmt"
+	"math/rand"
+	"net"
+	"path"
+
+	. "github.com/onsi/ginkgo"
+	"k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/generated"
+)
+
+const (
+	sshPort                = "22"
+	driverDir              = "test/e2e/testing-manifests/flexvolume/"
+	defaultVolumePluginDir = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec"
+	// TODO: change this and config-test.sh when default flex volume install path is changed for GCI
+	// On gci, root is read-only and controller-manager containerized. Assume
+	// controller-manager has started with --flex-volume-plugin-dir equal to this
+	// (see cluster/gce/config-test.sh)
+	gciVolumePluginDir = "/etc/srv/kubernetes/kubelet-plugins/volume/exec"
+)
+
+// testFlexVolume tests that a client pod using a given flexvolume driver
+// successfully mounts it and runs
+func testFlexVolume(driver string, cs clientset.Interface, config framework.VolumeTestConfig, f *framework.Framework, clean bool) {
+	tests := []framework.VolumeTest{
+		{
+			Volume: v1.VolumeSource{
+				FlexVolume: &v1.FlexVolumeSource{
+					Driver: "k8s/" + driver,
+				},
+			},
+			File: "index.html",
+			// Must match content of examples/volumes/flexvolume/dummy(-attachable) domount
+			ExpectedContent: "Hello from flexvolume!",
+		},
+	}
+	framework.TestVolumeClient(cs, config, nil, tests)
+
+	if clean {
+		framework.VolumeTestCleanup(f, config)
+	}
+}
+
+// installFlex installs the driver found at filePath on the node, and restarts
+// kubelet if 'restart' is true. If node is nil, installs on the master, and restarts
+// controller-manager if 'restart' is true.
+func installFlex(node *v1.Node, vendor, driver, filePath string, restart bool) {
+	flexDir := getFlexDir(node == nil, vendor, driver)
+	flexFile := path.Join(flexDir, driver)
+
+	host := ""
+	if node != nil {
+		host = framework.GetNodeExternalIP(node)
+	} else {
+		host = net.JoinHostPort(framework.GetMasterHost(), sshPort)
+	}
+
+	cmd := fmt.Sprintf("sudo mkdir -p %s", flexDir)
+	sshAndLog(cmd, host)
+
+	data := generated.ReadOrDie(filePath)
+	cmd = fmt.Sprintf("sudo tee <<'EOF' %s\n%s\nEOF", flexFile, string(data))
+	sshAndLog(cmd, host)
+
+	cmd = fmt.Sprintf("sudo chmod +x %s", flexFile)
+	sshAndLog(cmd, host)
+
+	if !restart {
+		return
+	}
+
+	if node != nil {
+		err := framework.RestartKubelet(host)
+		framework.ExpectNoError(err)
+		err = framework.WaitForKubeletUp(host)
+		framework.ExpectNoError(err)
+	} else {
+		err := framework.RestartControllerManager()
+		framework.ExpectNoError(err)
+		err = framework.WaitForControllerManagerUp()
+		framework.ExpectNoError(err)
+	}
+}
+
+func uninstallFlex(node *v1.Node, vendor, driver string) {
+	flexDir := getFlexDir(node == nil, vendor, driver)
+
+	host := ""
+	if node != nil {
+		host = framework.GetNodeExternalIP(node)
+	} else {
+		host = net.JoinHostPort(framework.GetMasterHost(), sshPort)
+	}
+
+	cmd := fmt.Sprintf("sudo rm -r %s", flexDir)
+	sshAndLog(cmd, host)
+}
+
+func getFlexDir(master bool, vendor, driver string) string {
+	volumePluginDir := defaultVolumePluginDir
+	if framework.ProviderIs("gce") {
+		if (master && framework.MasterOSDistroIs("gci")) || (!master && framework.NodeOSDistroIs("gci")) {
+			volumePluginDir = gciVolumePluginDir
+		}
+	}
+	flexDir := path.Join(volumePluginDir, fmt.Sprintf("/%s~%s/", vendor, driver))
+	return flexDir
+}
+
+func sshAndLog(cmd, host string) {
+	result, err := framework.SSH(cmd, host, framework.TestContext.Provider)
+	framework.LogSSHResult(result)
+	framework.ExpectNoError(err)
+	if result.Code != 0 {
+		framework.Failf("%s returned non-zero, stderr: %s", cmd, result.Stderr)
+	}
+}
+
+var _ = SIGDescribe("Flexvolumes [Disruptive] [Feature:FlexVolume]", func() {
+	f := framework.NewDefaultFramework("flexvolume")
+
+	// If 'false', the test won't clear its volumes upon completion. Useful for debugging,
+	// note that namespace deletion is handled by delete-namespace flag
+	clean := true
+
+	var cs clientset.Interface
+	var ns *v1.Namespace
+	var node v1.Node
+	var config framework.VolumeTestConfig
+	var suffix string
+
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("gce")
+		framework.SkipUnlessMasterOSDistroIs("gci")
+		framework.SkipUnlessNodeOSDistroIs("debian", "gci")
+		framework.SkipUnlessSSHKeyPresent()
+
+		cs = f.ClientSet
+		ns = f.Namespace
+		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+		node = nodes.Items[rand.Intn(len(nodes.Items))]
+		config = framework.VolumeTestConfig{
+			Namespace:      ns.Name,
+			Prefix:         "flex",
+			ClientNodeName: node.Name,
+		}
+		suffix = ns.Name
+	})
+
+	It("should be mountable when non-attachable", func() {
+		driver := "dummy"
+		driverInstallAs := driver + "-" + suffix
+
+		By(fmt.Sprintf("installing flexvolume %s on node %s as %s", path.Join(driverDir, driver), node.Name, driverInstallAs))
+		installFlex(&node, "k8s", driverInstallAs, path.Join(driverDir, driver), true /* restart */)
+
+		testFlexVolume(driverInstallAs, cs, config, f, clean)
+
+		By("waiting for flex client pod to terminate")
+		if err := f.WaitForPodTerminated(config.Prefix+"-client", ""); !apierrs.IsNotFound(err) {
+			framework.ExpectNoError(err, "Failed to wait client pod terminated: %v", err)
+		}
+
+		By(fmt.Sprintf("uninstalling flexvolume %s from node %s", driverInstallAs, node.Name))
+		uninstallFlex(&node, "k8s", driverInstallAs)
+	})
+
+	It("should be mountable when attachable", func() {
+		driver := "dummy-attachable"
+		driverInstallAs := driver + "-" + suffix
+
+		By(fmt.Sprintf("installing flexvolume %s on node %s as %s", path.Join(driverDir, driver), node.Name, driverInstallAs))
+		installFlex(&node, "k8s", driverInstallAs, path.Join(driverDir, driver), true /* restart */)
+		By(fmt.Sprintf("installing flexvolume %s on master as %s", path.Join(driverDir, driver), driverInstallAs))
+		installFlex(nil, "k8s", driverInstallAs, path.Join(driverDir, driver), true /* restart */)
+
+		testFlexVolume(driverInstallAs, cs, config, f, clean)
+
+		By("waiting for flex client pod to terminate")
+		if err := f.WaitForPodTerminated(config.Prefix+"-client", ""); !apierrs.IsNotFound(err) {
+			framework.ExpectNoError(err, "Failed to wait client pod terminated: %v", err)
+		}
+
+		By(fmt.Sprintf("uninstalling flexvolume %s from node %s", driverInstallAs, node.Name))
+		uninstallFlex(&node, "k8s", driverInstallAs)
+		By(fmt.Sprintf("uninstalling flexvolume %s from master", driverInstallAs))
+		uninstallFlex(nil, "k8s", driverInstallAs)
+	})
+
+	It("should install plugin without kubelet restart", func() {
+		driver := "dummy"
+		driverInstallAs := driver + "-" + suffix
+
+		By(fmt.Sprintf("installing flexvolume %s on node %s as %s", path.Join(driverDir, driver), node.Name, driverInstallAs))
+		installFlex(&node, "k8s", driverInstallAs, path.Join(driverDir, driver), false /* restart */)
+
+		testFlexVolume(driverInstallAs, cs, config, f, clean)
+
+		By("waiting for flex client pod to terminate")
+		if err := f.WaitForPodTerminated(config.Prefix+"-client", ""); !apierrs.IsNotFound(err) {
+			framework.ExpectNoError(err, "Failed to wait client pod terminated: %v", err)
+		}
+
+		By(fmt.Sprintf("uninstalling flexvolume %s from node %s", driverInstallAs, node.Name))
+		uninstallFlex(&node, "k8s", driverInstallAs)
+	})
+})

vendor/k8s.io/kubernetes/test/e2e/storage/framework.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import "github.com/onsi/ginkgo"
+
+func SIGDescribe(text string, body func()) bool {
+	return ginkgo.Describe("[sig-storage] "+text, body)
+}

vendor/k8s.io/kubernetes/test/e2e/storage/pd.go

@@ -0,0 +1,755 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"fmt"
+	mathrand "math/rand"
+	"os/exec"
+	"strings"
+	"time"
+
+	"google.golang.org/api/googleapi"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/kubernetes/pkg/api/testapi"
+	"k8s.io/kubernetes/test/e2e/framework"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+)
+
+const (
+	gcePDDetachTimeout  = 10 * time.Minute
+	gcePDDetachPollTime = 10 * time.Second
+	nodeStatusTimeout   = 10 * time.Minute
+	nodeStatusPollTime  = 1 * time.Second
+	maxReadRetry        = 3
+)
+
+var _ = SIGDescribe("Pod Disks", func() {
+	var (
+		podClient  v1core.PodInterface
+		nodeClient v1core.NodeInterface
+		host0Name  types.NodeName
+		host1Name  types.NodeName
+		nodes      *v1.NodeList
+	)
+	f := framework.NewDefaultFramework("pod-disks")
+
+	BeforeEach(func() {
+		framework.SkipUnlessNodeCountIsAtLeast(2)
+
+		podClient = f.ClientSet.Core().Pods(f.Namespace.Name)
+		nodeClient = f.ClientSet.Core().Nodes()
+		nodes = framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+
+		Expect(len(nodes.Items)).To(BeNumerically(">=", 2), "Requires at least 2 nodes")
+
+		host0Name = types.NodeName(nodes.Items[0].ObjectMeta.Name)
+		host1Name = types.NodeName(nodes.Items[1].ObjectMeta.Name)
+
+		mathrand.Seed(time.Now().UTC().UnixNano())
+	})
+
+	It("should schedule a pod w/ a RW PD, ungracefully remove it, then schedule it on another host [Slow]", func() {
+		framework.SkipUnlessProviderIs("gce", "gke", "aws")
+
+		By("creating PD")
+		diskName, err := framework.CreatePDWithRetry()
+		framework.ExpectNoError(err, "Error creating PD")
+
+		host0Pod := testPDPod([]string{diskName}, host0Name, false /* readOnly */, 1 /* numContainers */)
+		host1Pod := testPDPod([]string{diskName}, host1Name, false /* readOnly */, 1 /* numContainers */)
+		containerName := "mycontainer"
+
+		defer func() {
+			// Teardown pods, PD. Ignore errors.
+			// Teardown should do nothing unless test failed.
+			By("cleaning up PD-RW test environment")
+			podClient.Delete(host0Pod.Name, metav1.NewDeleteOptions(0))
+			podClient.Delete(host1Pod.Name, metav1.NewDeleteOptions(0))
+			detachAndDeletePDs(diskName, []types.NodeName{host0Name, host1Name})
+		}()
+
+		By("submitting host0Pod to kubernetes")
+		_, err = podClient.Create(host0Pod)
+		framework.ExpectNoError(err, fmt.Sprintf("Failed to create host0Pod: %v", err))
+
+		framework.ExpectNoError(f.WaitForPodRunningSlow(host0Pod.Name))
+
+		testFile := "/testpd1/tracker"
+		testFileContents := fmt.Sprintf("%v", mathrand.Int())
+
+		framework.ExpectNoError(f.WriteFileViaContainer(host0Pod.Name, containerName, testFile, testFileContents))
+		framework.Logf("Wrote value: %v", testFileContents)
+
+		// Verify that disk shows up for in node 1's VolumeInUse list
+		framework.ExpectNoError(waitForPDInVolumesInUse(nodeClient, diskName, host0Name, nodeStatusTimeout, true /* shouldExist */))
+
+		By("deleting host0Pod")
+		// Delete pod with 0 grace period
+		framework.ExpectNoError(podClient.Delete(host0Pod.Name, metav1.NewDeleteOptions(0)), "Failed to delete host0Pod")
+
+		By("submitting host1Pod to kubernetes")
+		_, err = podClient.Create(host1Pod)
+		framework.ExpectNoError(err, "Failed to create host1Pod")
+
+		framework.ExpectNoError(f.WaitForPodRunningSlow(host1Pod.Name))
+
+		verifyPDContentsViaContainer(f, host1Pod.Name, containerName, map[string]string{testFile: testFileContents})
+
+		// Verify that disk is removed from node 1's VolumeInUse list
+		framework.ExpectNoError(waitForPDInVolumesInUse(nodeClient, diskName, host0Name, nodeStatusTimeout, false /* shouldExist */))
+
+		By("deleting host1Pod")
+		framework.ExpectNoError(podClient.Delete(host1Pod.Name, metav1.NewDeleteOptions(0)), "Failed to delete host1Pod")
+
+		By("Test completed successfully, waiting for PD to safely detach")
+		waitForPDDetach(diskName, host0Name)
+		waitForPDDetach(diskName, host1Name)
+
+		return
+	})
+
+	It("Should schedule a pod w/ a RW PD, gracefully remove it, then schedule it on another host [Slow]", func() {
+		framework.SkipUnlessProviderIs("gce", "gke", "aws")
+
+		By("creating PD")
+		diskName, err := framework.CreatePDWithRetry()
+		framework.ExpectNoError(err, "Error creating PD")
+
+		host0Pod := testPDPod([]string{diskName}, host0Name, false /* readOnly */, 1 /* numContainers */)
+		host1Pod := testPDPod([]string{diskName}, host1Name, false /* readOnly */, 1 /* numContainers */)
+		containerName := "mycontainer"
+
+		defer func() {
+			// Teardown pods, PD. Ignore errors.
+			// Teardown should do nothing unless test failed.
+			By("cleaning up PD-RW test environment")
+			podClient.Delete(host0Pod.Name, &metav1.DeleteOptions{})
+			podClient.Delete(host1Pod.Name, &metav1.DeleteOptions{})
+			detachAndDeletePDs(diskName, []types.NodeName{host0Name, host1Name})
+		}()
+
+		By("submitting host0Pod to kubernetes")
+		_, err = podClient.Create(host0Pod)
+		framework.ExpectNoError(err, fmt.Sprintf("Failed to create host0Pod: %v", err))
+
+		framework.ExpectNoError(f.WaitForPodRunningSlow(host0Pod.Name))
+
+		testFile := "/testpd1/tracker"
+		testFileContents := fmt.Sprintf("%v", mathrand.Int())
+
+		framework.ExpectNoError(f.WriteFileViaContainer(host0Pod.Name, containerName, testFile, testFileContents))
+		framework.Logf("Wrote value: %v", testFileContents)
+
+		// Verify that disk shows up for in node 1's VolumeInUse list
+		framework.ExpectNoError(waitForPDInVolumesInUse(nodeClient, diskName, host0Name, nodeStatusTimeout, true /* shouldExist */))
+
+		By("deleting host0Pod")
+		// Delete pod with default grace period 30s
+		framework.ExpectNoError(podClient.Delete(host0Pod.Name, &metav1.DeleteOptions{}), "Failed to delete host0Pod")
+
+		By("submitting host1Pod to kubernetes")
+		_, err = podClient.Create(host1Pod)
+		framework.ExpectNoError(err, "Failed to create host1Pod")
+
+		framework.ExpectNoError(f.WaitForPodRunningSlow(host1Pod.Name))
+
+		verifyPDContentsViaContainer(f, host1Pod.Name, containerName, map[string]string{testFile: testFileContents})
+
+		// Verify that disk is removed from node 1's VolumeInUse list
+		framework.ExpectNoError(waitForPDInVolumesInUse(nodeClient, diskName, host0Name, nodeStatusTimeout, false /* shouldExist */))
+
+		By("deleting host1Pod")
+		framework.ExpectNoError(podClient.Delete(host1Pod.Name, &metav1.DeleteOptions{}), "Failed to delete host1Pod")
+
+		By("Test completed successfully, waiting for PD to safely detach")
+		waitForPDDetach(diskName, host0Name)
+		waitForPDDetach(diskName, host1Name)
+
+		return
+	})
+
+	It("should schedule a pod w/ a readonly PD on two hosts, then remove both ungracefully. [Slow]", func() {
+		framework.SkipUnlessProviderIs("gce", "gke")
+
+		By("creating PD")
+		diskName, err := framework.CreatePDWithRetry()
+		framework.ExpectNoError(err, "Error creating PD")
+
+		rwPod := testPDPod([]string{diskName}, host0Name, false /* readOnly */, 1 /* numContainers */)
+		host0ROPod := testPDPod([]string{diskName}, host0Name, true /* readOnly */, 1 /* numContainers */)
+		host1ROPod := testPDPod([]string{diskName}, host1Name, true /* readOnly */, 1 /* numContainers */)
+
+		defer func() {
+			By("cleaning up PD-RO test environment")
+			// Teardown pods, PD. Ignore errors.
+			// Teardown should do nothing unless test failed.
+			podClient.Delete(rwPod.Name, metav1.NewDeleteOptions(0))
+			podClient.Delete(host0ROPod.Name, metav1.NewDeleteOptions(0))
+			podClient.Delete(host1ROPod.Name, metav1.NewDeleteOptions(0))
+			detachAndDeletePDs(diskName, []types.NodeName{host0Name, host1Name})
+		}()
+
+		By("submitting rwPod to ensure PD is formatted")
+		_, err = podClient.Create(rwPod)
+		framework.ExpectNoError(err, "Failed to create rwPod")
+		framework.ExpectNoError(f.WaitForPodRunningSlow(rwPod.Name))
+		// Delete pod with 0 grace period
+		framework.ExpectNoError(podClient.Delete(rwPod.Name, metav1.NewDeleteOptions(0)), "Failed to delete host0Pod")
+		framework.ExpectNoError(waitForPDDetach(diskName, host0Name))
+
+		By("submitting host0ROPod to kubernetes")
+		_, err = podClient.Create(host0ROPod)
+		framework.ExpectNoError(err, "Failed to create host0ROPod")
+
+		By("submitting host1ROPod to kubernetes")
+		_, err = podClient.Create(host1ROPod)
+		framework.ExpectNoError(err, "Failed to create host1ROPod")
+
+		framework.ExpectNoError(f.WaitForPodRunningSlow(host0ROPod.Name))
+
+		framework.ExpectNoError(f.WaitForPodRunningSlow(host1ROPod.Name))
+
+		By("deleting host0ROPod")
+		framework.ExpectNoError(podClient.Delete(host0ROPod.Name, metav1.NewDeleteOptions(0)), "Failed to delete host0ROPod")
+
+		By("deleting host1ROPod")
+		framework.ExpectNoError(podClient.Delete(host1ROPod.Name, metav1.NewDeleteOptions(0)), "Failed to delete host1ROPod")
+
+		By("Test completed successfully, waiting for PD to safely detach")
+		waitForPDDetach(diskName, host0Name)
+		waitForPDDetach(diskName, host1Name)
+	})
+
+	It("Should schedule a pod w/ a readonly PD on two hosts, then remove both gracefully. [Slow]", func() {
+		framework.SkipUnlessProviderIs("gce", "gke")
+
+		By("creating PD")
+		diskName, err := framework.CreatePDWithRetry()
+		framework.ExpectNoError(err, "Error creating PD")
+
+		rwPod := testPDPod([]string{diskName}, host0Name, false /* readOnly */, 1 /* numContainers */)
+		host0ROPod := testPDPod([]string{diskName}, host0Name, true /* readOnly */, 1 /* numContainers */)
+		host1ROPod := testPDPod([]string{diskName}, host1Name, true /* readOnly */, 1 /* numContainers */)
+
+		defer func() {
+			By("cleaning up PD-RO test environment")
+			// Teardown pods, PD. Ignore errors.
+			// Teardown should do nothing unless test failed.
+			podClient.Delete(rwPod.Name, &metav1.DeleteOptions{})
+			podClient.Delete(host0ROPod.Name, &metav1.DeleteOptions{})
+			podClient.Delete(host1ROPod.Name, &metav1.DeleteOptions{})
+			detachAndDeletePDs(diskName, []types.NodeName{host0Name, host1Name})
+		}()
+
+		By("submitting rwPod to ensure PD is formatted")
+		_, err = podClient.Create(rwPod)
+		framework.ExpectNoError(err, "Failed to create rwPod")
+		framework.ExpectNoError(f.WaitForPodRunningSlow(rwPod.Name))
+		// Delete pod with default grace period 30s
+		framework.ExpectNoError(podClient.Delete(rwPod.Name, &metav1.DeleteOptions{}), "Failed to delete host0Pod")
+		framework.ExpectNoError(waitForPDDetach(diskName, host0Name))
+
+		By("submitting host0ROPod to kubernetes")
+		_, err = podClient.Create(host0ROPod)
+		framework.ExpectNoError(err, "Failed to create host0ROPod")
+
+		By("submitting host1ROPod to kubernetes")
+		_, err = podClient.Create(host1ROPod)
+		framework.ExpectNoError(err, "Failed to create host1ROPod")
+
+		framework.ExpectNoError(f.WaitForPodRunningSlow(host0ROPod.Name))
+
+		framework.ExpectNoError(f.WaitForPodRunningSlow(host1ROPod.Name))
+
+		By("deleting host0ROPod")
+		framework.ExpectNoError(podClient.Delete(host0ROPod.Name, &metav1.DeleteOptions{}), "Failed to delete host0ROPod")
+
+		By("deleting host1ROPod")
+		framework.ExpectNoError(podClient.Delete(host1ROPod.Name, &metav1.DeleteOptions{}), "Failed to delete host1ROPod")
+
+		By("Test completed successfully, waiting for PD to safely detach")
+		waitForPDDetach(diskName, host0Name)
+		waitForPDDetach(diskName, host1Name)
+	})
+
+	It("should schedule a pod w/ a RW PD shared between multiple containers, write to PD, delete pod, verify contents, and repeat in rapid succession [Slow]", func() {
+		framework.SkipUnlessProviderIs("gce", "gke", "aws")
+
+		By("creating PD")
+		diskName, err := framework.CreatePDWithRetry()
+		framework.ExpectNoError(err, "Error creating PD")
+		numContainers := 4
+		var host0Pod *v1.Pod
+
+		defer func() {
+			By("cleaning up PD-RW test environment")
+			// Teardown pods, PD. Ignore errors.
+			// Teardown should do nothing unless test failed.
+			if host0Pod != nil {
+				podClient.Delete(host0Pod.Name, metav1.NewDeleteOptions(0))
+			}
+			detachAndDeletePDs(diskName, []types.NodeName{host0Name})
+		}()
+
+		fileAndContentToVerify := make(map[string]string)
+		for i := 0; i < 3; i++ {
+			framework.Logf("PD Read/Writer Iteration #%v", i)
+			By("submitting host0Pod to kubernetes")
+			host0Pod = testPDPod([]string{diskName}, host0Name, false /* readOnly */, numContainers)
+			_, err = podClient.Create(host0Pod)
+			framework.ExpectNoError(err, fmt.Sprintf("Failed to create host0Pod: %v", err))
+
+			framework.ExpectNoError(f.WaitForPodRunningSlow(host0Pod.Name))
+
+			// randomly select a container and read/verify pd contents from it
+			containerName := fmt.Sprintf("mycontainer%v", mathrand.Intn(numContainers)+1)
+			verifyPDContentsViaContainer(f, host0Pod.Name, containerName, fileAndContentToVerify)
+
+			// Randomly select a container to write a file to PD from
+			containerName = fmt.Sprintf("mycontainer%v", mathrand.Intn(numContainers)+1)
+			testFile := fmt.Sprintf("/testpd1/tracker%v", i)
+			testFileContents := fmt.Sprintf("%v", mathrand.Int())
+			fileAndContentToVerify[testFile] = testFileContents
+			framework.ExpectNoError(f.WriteFileViaContainer(host0Pod.Name, containerName, testFile, testFileContents))
+			framework.Logf("Wrote value: \"%v\" to PD %q from pod %q container %q", testFileContents, diskName, host0Pod.Name, containerName)
+
+			// Randomly select a container and read/verify pd contents from it
+			containerName = fmt.Sprintf("mycontainer%v", mathrand.Intn(numContainers)+1)
+			verifyPDContentsViaContainer(f, host0Pod.Name, containerName, fileAndContentToVerify)
+
+			By("deleting host0Pod")
+			framework.ExpectNoError(podClient.Delete(host0Pod.Name, metav1.NewDeleteOptions(0)), "Failed to delete host0Pod")
+		}
+
+		By("Test completed successfully, waiting for PD to safely detach")
+		waitForPDDetach(diskName, host0Name)
+	})
+
+	It("should schedule a pod w/two RW PDs both mounted to one container, write to PD, verify contents, delete pod, recreate pod, verify contents, and repeat in rapid succession [Slow]", func() {
+		framework.SkipUnlessProviderIs("gce", "gke", "aws")
+
+		By("creating PD1")
+		disk1Name, err := framework.CreatePDWithRetry()
+		framework.ExpectNoError(err, "Error creating PD1")
+		By("creating PD2")
+		disk2Name, err := framework.CreatePDWithRetry()
+		framework.ExpectNoError(err, "Error creating PD2")
+		var host0Pod *v1.Pod
+
+		defer func() {
+			By("cleaning up PD-RW test environment")
+			// Teardown pods, PD. Ignore errors.
+			// Teardown should do nothing unless test failed.
+			if host0Pod != nil {
+				podClient.Delete(host0Pod.Name, metav1.NewDeleteOptions(0))
+			}
+			detachAndDeletePDs(disk1Name, []types.NodeName{host0Name})
+			detachAndDeletePDs(disk2Name, []types.NodeName{host0Name})
+		}()
+
+		containerName := "mycontainer"
+		fileAndContentToVerify := make(map[string]string)
+		for i := 0; i < 3; i++ {
+			framework.Logf("PD Read/Writer Iteration #%v", i)
+			By("submitting host0Pod to kubernetes")
+			host0Pod = testPDPod([]string{disk1Name, disk2Name}, host0Name, false /* readOnly */, 1 /* numContainers */)
+			_, err = podClient.Create(host0Pod)
+			framework.ExpectNoError(err, fmt.Sprintf("Failed to create host0Pod: %v", err))
+
+			framework.ExpectNoError(f.WaitForPodRunningSlow(host0Pod.Name))
+
+			// Read/verify pd contents for both disks from container
+			verifyPDContentsViaContainer(f, host0Pod.Name, containerName, fileAndContentToVerify)
+
+			// Write a file to both PDs from container
+			testFilePD1 := fmt.Sprintf("/testpd1/tracker%v", i)
+			testFilePD2 := fmt.Sprintf("/testpd2/tracker%v", i)
+			testFilePD1Contents := fmt.Sprintf("%v", mathrand.Int())
+			testFilePD2Contents := fmt.Sprintf("%v", mathrand.Int())
+			fileAndContentToVerify[testFilePD1] = testFilePD1Contents
+			fileAndContentToVerify[testFilePD2] = testFilePD2Contents
+			framework.ExpectNoError(f.WriteFileViaContainer(host0Pod.Name, containerName, testFilePD1, testFilePD1Contents))
+			framework.Logf("Wrote value: \"%v\" to PD1 (%q) from pod %q container %q", testFilePD1Contents, disk1Name, host0Pod.Name, containerName)
+			framework.ExpectNoError(f.WriteFileViaContainer(host0Pod.Name, containerName, testFilePD2, testFilePD2Contents))
+			framework.Logf("Wrote value: \"%v\" to PD2 (%q) from pod %q container %q", testFilePD2Contents, disk2Name, host0Pod.Name, containerName)
+
+			// Read/verify pd contents for both disks from container
+			verifyPDContentsViaContainer(f, host0Pod.Name, containerName, fileAndContentToVerify)
+
+			By("deleting host0Pod")
+			framework.ExpectNoError(podClient.Delete(host0Pod.Name, metav1.NewDeleteOptions(0)), "Failed to delete host0Pod")
+		}
+
+		By("Test completed successfully, waiting for PD to safely detach")
+		waitForPDDetach(disk1Name, host0Name)
+		waitForPDDetach(disk2Name, host0Name)
+	})
+
+	It("should be able to detach from a node which was deleted [Slow] [Disruptive]", func() {
+		framework.SkipUnlessProviderIs("gce")
+
+		initialGroupSize, err := framework.GroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup)
+		framework.ExpectNoError(err, "Error getting group size")
+
+		By("Creating a pd")
+		diskName, err := framework.CreatePDWithRetry()
+		framework.ExpectNoError(err, "Error creating a pd")
+
+		host0Pod := testPDPod([]string{diskName}, host0Name, false, 1)
+
+		containerName := "mycontainer"
+
+		defer func() {
+			By("Cleaning up PD-RW test env")
+			podClient.Delete(host0Pod.Name, metav1.NewDeleteOptions(0))
+			detachAndDeletePDs(diskName, []types.NodeName{host0Name})
+			framework.WaitForNodeToBeReady(f.ClientSet, string(host0Name), nodeStatusTimeout)
+			framework.WaitForAllNodesSchedulable(f.ClientSet, nodeStatusTimeout)
+			nodes = framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+			Expect(len(nodes.Items)).To(Equal(initialGroupSize), "Requires node count to return to initial group size.")
+		}()
+
+		By("submitting host0Pod to kubernetes")
+		_, err = podClient.Create(host0Pod)
+		framework.ExpectNoError(err, fmt.Sprintf("Failed to create host0pod: %v", err))
+
+		framework.ExpectNoError(f.WaitForPodRunningSlow(host0Pod.Name))
+
+		testFile := "/testpd1/tracker"
+		testFileContents := fmt.Sprintf("%v", mathrand.Int())
+
+		framework.ExpectNoError(f.WriteFileViaContainer(host0Pod.Name, containerName, testFile, testFileContents))
+		framework.Logf("Wrote value: %v", testFileContents)
+
+		// Verify that disk shows up in node 0's volumeInUse list
+		framework.ExpectNoError(waitForPDInVolumesInUse(nodeClient, diskName, host0Name, nodeStatusTimeout, true /* should exist*/))
+
+		output, err := exec.Command("gcloud", "compute", "instances", "list", "--project="+framework.TestContext.CloudConfig.ProjectID).CombinedOutput()
+		framework.ExpectNoError(err, fmt.Sprintf("Unable to get list of node instances err=%v output=%s", err, output))
+		Expect(true, strings.Contains(string(output), string(host0Name)))
+
+		By("deleting host0")
+
+		output, err = exec.Command("gcloud", "compute", "instances", "delete", string(host0Name), "--project="+framework.TestContext.CloudConfig.ProjectID, "--zone="+framework.TestContext.CloudConfig.Zone).CombinedOutput()
+		framework.ExpectNoError(err, fmt.Sprintf("Failed to delete host0pod: err=%v output=%s", err, output))
+
+		output, err = exec.Command("gcloud", "compute", "instances", "list", "--project="+framework.TestContext.CloudConfig.ProjectID).CombinedOutput()
+		framework.ExpectNoError(err, fmt.Sprintf("Unable to get list of node instances err=%v output=%s", err, output))
+		Expect(false, strings.Contains(string(output), string(host0Name)))
+
+		// The disk should be detached from host0 on it's deletion
+		By("Waiting for pd to detach from host0")
+		waitForPDDetach(diskName, host0Name)
+		framework.ExpectNoError(framework.WaitForGroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup, int32(initialGroupSize)), "Unable to get back the cluster to inital size")
+		return
+	})
+
+	It("should be able to detach from a node whose api object was deleted [Slow] [Disruptive]", func() {
+		framework.SkipUnlessProviderIs("gce")
+		initialGroupSize, err := framework.GroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup)
+		framework.ExpectNoError(err, "Error getting group size")
+		By("Creating a pd")
+		diskName, err := framework.CreatePDWithRetry()
+		framework.ExpectNoError(err, "Error creating a pd")
+
+		host0Pod := testPDPod([]string{diskName}, host0Name, false, 1)
+		originalCount := len(nodes.Items)
+		containerName := "mycontainer"
+		nodeToDelete := &nodes.Items[0]
+		defer func() {
+			By("Cleaning up PD-RW test env")
+			detachAndDeletePDs(diskName, []types.NodeName{host0Name})
+			nodeToDelete.ObjectMeta.SetResourceVersion("0")
+			// need to set the resource version or else the Create() fails
+			_, err := nodeClient.Create(nodeToDelete)
+			framework.ExpectNoError(err, "Unable to re-create the deleted node")
+			framework.ExpectNoError(framework.WaitForGroupSize(framework.TestContext.CloudConfig.NodeInstanceGroup, int32(initialGroupSize)), "Unable to get the node group back to the original size")
+			framework.WaitForNodeToBeReady(f.ClientSet, nodeToDelete.Name, nodeStatusTimeout)
+			framework.WaitForAllNodesSchedulable(f.ClientSet, nodeStatusTimeout)
+			nodes = framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+			Expect(len(nodes.Items)).To(Equal(originalCount), "Requires node count to return to original node count.")
+		}()
+
+		By("submitting host0Pod to kubernetes")
+		_, err = podClient.Create(host0Pod)
+		framework.ExpectNoError(err, fmt.Sprintf("Failed to create host0pod: %v", err))
+
+		framework.ExpectNoError(f.WaitForPodRunningSlow(host0Pod.Name))
+
+		testFile := "/testpd1/tracker"
+		testFileContents := fmt.Sprintf("%v", mathrand.Int())
+
+		framework.ExpectNoError(f.WriteFileViaContainer(host0Pod.Name, containerName, testFile, testFileContents))
+		framework.Logf("Wrote value: %v", testFileContents)
+
+		// Verify that disk shows up in node 0's volumeInUse list
+		framework.ExpectNoError(waitForPDInVolumesInUse(nodeClient, diskName, host0Name, nodeStatusTimeout, true /* should exist*/))
+
+		By("deleting api object of host0")
+		framework.ExpectNoError(nodeClient.Delete(string(host0Name), metav1.NewDeleteOptions(0)), "Unable to delete host0")
+
+		By("deleting host0pod")
+		framework.ExpectNoError(podClient.Delete(host0Pod.Name, metav1.NewDeleteOptions(0)), "Unable to delete host0Pod")
+		// The disk should be detached from host0 on its deletion
+		By("Waiting for pd to detach from host0")
+		framework.ExpectNoError(waitForPDDetach(diskName, host0Name), "Timed out waiting for detach pd")
+	})
+
+	It("should be able to delete a non-existent PD without error", func() {
+		framework.SkipUnlessProviderIs("gce")
+
+		By("delete a PD")
+		framework.ExpectNoError(framework.DeletePDWithRetry("non-exist"))
+	})
+})
+
+func verifyPDContentsViaContainer(f *framework.Framework, podName, containerName string, fileAndContentToVerify map[string]string) {
+	for filePath, expectedContents := range fileAndContentToVerify {
+		var value string
+		// Add a retry to avoid temporal failure in reading the content
+		for i := 0; i < maxReadRetry; i++ {
+			v, err := f.ReadFileViaContainer(podName, containerName, filePath)
+			value = v
+			if err != nil {
+				framework.Logf("Error reading file: %v", err)
+			}
+			framework.ExpectNoError(err)
+			framework.Logf("Read file %q with content: %v (iteration %d)", filePath, v, i)
+			if strings.TrimSpace(v) != strings.TrimSpace(expectedContents) {
+				framework.Logf("Warning: read content <%q> does not match execpted content <%q>.", v, expectedContents)
+				size, err := f.CheckFileSizeViaContainer(podName, containerName, filePath)
+				if err != nil {
+					framework.Logf("Error checking file size: %v", err)
+				}
+				framework.Logf("Check file %q size: %q", filePath, size)
+			} else {
+				break
+			}
+		}
+		Expect(strings.TrimSpace(value)).To(Equal(strings.TrimSpace(expectedContents)))
+	}
+}
+
+func detachPD(nodeName types.NodeName, pdName string) error {
+	if framework.TestContext.Provider == "gce" || framework.TestContext.Provider == "gke" {
+		gceCloud, err := framework.GetGCECloud()
+		if err != nil {
+			return err
+		}
+
+		err = gceCloud.DetachDisk(pdName, nodeName)
+		if err != nil {
+			if gerr, ok := err.(*googleapi.Error); ok && strings.Contains(gerr.Message, "Invalid value for field 'disk'") {
+				// PD already detached, ignore error.
+				return nil
+			}
+
+			framework.Logf("Error detaching PD %q: %v", pdName, err)
+		}
+
+		return err
+	} else if framework.TestContext.Provider == "aws" {
+		client := ec2.New(session.New())
+
+		tokens := strings.Split(pdName, "/")
+		awsVolumeID := tokens[len(tokens)-1]
+
+		request := ec2.DetachVolumeInput{
+			VolumeId: aws.String(awsVolumeID),
+		}
+
+		_, err := client.DetachVolume(&request)
+		if err != nil {
+			return fmt.Errorf("error detaching EBS volume: %v", err)
+		}
+		return nil
+	} else {
+		return fmt.Errorf("Provider does not support volume detaching")
+	}
+}
+
+func testPDPod(diskNames []string, targetNode types.NodeName, readOnly bool, numContainers int) *v1.Pod {
+	containers := make([]v1.Container, numContainers)
+	for i := range containers {
+		containers[i].Name = "mycontainer"
+		if numContainers > 1 {
+			containers[i].Name = fmt.Sprintf("mycontainer%v", i+1)
+		}
+
+		containers[i].Image = imageutils.GetBusyBoxImage()
+
+		containers[i].Command = []string{"sleep", "6000"}
+
+		containers[i].VolumeMounts = make([]v1.VolumeMount, len(diskNames))
+		for k := range diskNames {
+			containers[i].VolumeMounts[k].Name = fmt.Sprintf("testpd%v", k+1)
+			containers[i].VolumeMounts[k].MountPath = fmt.Sprintf("/testpd%v", k+1)
+		}
+
+		containers[i].Resources.Limits = v1.ResourceList{}
+		containers[i].Resources.Limits[v1.ResourceCPU] = *resource.NewQuantity(int64(0), resource.DecimalSI)
+
+	}
+
+	pod := &v1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: testapi.Groups[v1.GroupName].GroupVersion().String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "pd-test-" + string(uuid.NewUUID()),
+		},
+		Spec: v1.PodSpec{
+			Containers: containers,
+			NodeName:   string(targetNode),
+		},
+	}
+
+	if framework.TestContext.Provider == "gce" || framework.TestContext.Provider == "gke" {
+		pod.Spec.Volumes = make([]v1.Volume, len(diskNames))
+		for k, diskName := range diskNames {
+			pod.Spec.Volumes[k].Name = fmt.Sprintf("testpd%v", k+1)
+			pod.Spec.Volumes[k].VolumeSource = v1.VolumeSource{
+				GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
+					PDName:   diskName,
+					FSType:   "ext4",
+					ReadOnly: readOnly,
+				},
+			}
+		}
+	} else if framework.TestContext.Provider == "aws" {
+		pod.Spec.Volumes = make([]v1.Volume, len(diskNames))
+		for k, diskName := range diskNames {
+			pod.Spec.Volumes[k].Name = fmt.Sprintf("testpd%v", k+1)
+			pod.Spec.Volumes[k].VolumeSource = v1.VolumeSource{
+				AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
+					VolumeID: diskName,
+					FSType:   "ext4",
+					ReadOnly: readOnly,
+				},
+			}
+		}
+	} else {
+		panic("Unknown provider: " + framework.TestContext.Provider)
+	}
+
+	return pod
+}
+
+// Waits for specified PD to to detach from specified hostName
+func waitForPDDetach(diskName string, nodeName types.NodeName) error {
+	if framework.TestContext.Provider == "gce" || framework.TestContext.Provider == "gke" {
+		framework.Logf("Waiting for GCE PD %q to detach from node %q.", diskName, nodeName)
+		gceCloud, err := framework.GetGCECloud()
+		if err != nil {
+			return err
+		}
+
+		for start := time.Now(); time.Since(start) < gcePDDetachTimeout; time.Sleep(gcePDDetachPollTime) {
+			diskAttached, err := gceCloud.DiskIsAttached(diskName, nodeName)
+			if err != nil {
+				framework.Logf("Error waiting for PD %q to detach from node %q. 'DiskIsAttached(...)' failed with %v", diskName, nodeName, err)
+				return err
+			}
+
+			if !diskAttached {
+				// Specified disk does not appear to be attached to specified node
+				framework.Logf("GCE PD %q appears to have successfully detached from %q.", diskName, nodeName)
+				return nil
+			}
+
+			framework.Logf("Waiting for GCE PD %q to detach from %q.", diskName, nodeName)
+		}
+
+		return fmt.Errorf("Gave up waiting for GCE PD %q to detach from %q after %v", diskName, nodeName, gcePDDetachTimeout)
+	}
+
+	return nil
+}
+
+func detachAndDeletePDs(diskName string, hosts []types.NodeName) {
+	for _, host := range hosts {
+		framework.Logf("Detaching GCE PD %q from node %q.", diskName, host)
+		detachPD(host, diskName)
+		By(fmt.Sprintf("Waiting for PD %q to detach from %q", diskName, host))
+		waitForPDDetach(diskName, host)
+	}
+	By(fmt.Sprintf("Deleting PD %q", diskName))
+	framework.ExpectNoError(framework.DeletePDWithRetry(diskName))
+}
+
+func waitForPDInVolumesInUse(
+	nodeClient v1core.NodeInterface,
+	diskName string,
+	nodeName types.NodeName,
+	timeout time.Duration,
+	shouldExist bool) error {
+	logStr := "to contain"
+	if !shouldExist {
+		logStr = "to NOT contain"
+	}
+	framework.Logf(
+		"Waiting for node %s's VolumesInUse Status %s PD %q",
+		nodeName, logStr, diskName)
+	for start := time.Now(); time.Since(start) < timeout; time.Sleep(nodeStatusPollTime) {
+		nodeObj, err := nodeClient.Get(string(nodeName), metav1.GetOptions{})
+		if err != nil || nodeObj == nil {
+			framework.Logf(
+				"Failed to fetch node object %q from API server. err=%v",
+				nodeName, err)
+			continue
+		}
+
+		exists := false
+		for _, volumeInUse := range nodeObj.Status.VolumesInUse {
+			volumeInUseStr := string(volumeInUse)
+			if strings.Contains(volumeInUseStr, diskName) {
+				if shouldExist {
+					framework.Logf(
+						"Found PD %q in node %q's VolumesInUse Status: %q",
+						diskName, nodeName, volumeInUseStr)
+					return nil
+				}
+
+				exists = true
+			}
+		}
+
+		if !shouldExist && !exists {
+			framework.Logf(
+				"Verified PD %q does not exist in node %q's VolumesInUse Status.",
+				diskName, nodeName)
+			return nil
+		}
+	}
+
+	return fmt.Errorf(
+		"Timed out waiting for node %s VolumesInUse Status %s diskName %q",
+		nodeName, logStr, diskName)
+}

vendor/k8s.io/kubernetes/test/e2e/storage/persistent_volumes-disruptive.go

@@ -0,0 +1,292 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+type testBody func(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume)
+type disruptiveTest struct {
+	testItStmt string
+	runTest    testBody
+}
+type kubeletOpt string
+
+const (
+	MinNodes                    = 2
+	NodeStateTimeout            = 1 * time.Minute
+	kStart           kubeletOpt = "start"
+	kStop            kubeletOpt = "stop"
+	kRestart         kubeletOpt = "restart"
+)
+
+var _ = SIGDescribe("PersistentVolumes[Disruptive][Flaky]", func() {
+
+	f := framework.NewDefaultFramework("disruptive-pv")
+	var (
+		c                         clientset.Interface
+		ns                        string
+		nfsServerPod              *v1.Pod
+		nfsPVconfig               framework.PersistentVolumeConfig
+		pvcConfig                 framework.PersistentVolumeClaimConfig
+		nfsServerIP, clientNodeIP string
+		clientNode                *v1.Node
+		volLabel                  labels.Set
+		selector                  *metav1.LabelSelector
+	)
+
+	BeforeEach(func() {
+		// To protect the NFS volume pod from the kubelet restart, we isolate it on its own node.
+		framework.SkipUnlessNodeCountIsAtLeast(MinNodes)
+		framework.SkipIfProviderIs("local")
+
+		c = f.ClientSet
+		ns = f.Namespace.Name
+		volLabel = labels.Set{framework.VolumeSelectorKey: ns}
+		selector = metav1.SetAsLabelSelector(volLabel)
+		// Start the NFS server pod.
+		_, nfsServerPod, nfsServerIP = framework.NewNFSServer(c, ns, []string{"-G", "777", "/exports"})
+		nfsPVconfig = framework.PersistentVolumeConfig{
+			NamePrefix: "nfs-",
+			Labels:     volLabel,
+			PVSource: v1.PersistentVolumeSource{
+				NFS: &v1.NFSVolumeSource{
+					Server:   nfsServerIP,
+					Path:     "/exports",
+					ReadOnly: false,
+				},
+			},
+		}
+		pvcConfig = framework.PersistentVolumeClaimConfig{
+			Annotations: map[string]string{
+				v1.BetaStorageClassAnnotation: "",
+			},
+			Selector: selector,
+		}
+		// Get the first ready node IP that is not hosting the NFS pod.
+		if clientNodeIP == "" {
+			framework.Logf("Designating test node")
+			nodes := framework.GetReadySchedulableNodesOrDie(c)
+			for _, node := range nodes.Items {
+				if node.Name != nfsServerPod.Spec.NodeName {
+					clientNode = &node
+					clientNodeIP = framework.GetNodeExternalIP(clientNode)
+					break
+				}
+			}
+			Expect(clientNodeIP).NotTo(BeEmpty())
+		}
+	})
+
+	AfterEach(func() {
+		framework.DeletePodWithWait(f, c, nfsServerPod)
+	})
+
+	Context("when kubelet restarts", func() {
+		var (
+			clientPod *v1.Pod
+			pv        *v1.PersistentVolume
+			pvc       *v1.PersistentVolumeClaim
+		)
+
+		BeforeEach(func() {
+			framework.Logf("Initializing test spec")
+			clientPod, pv, pvc = initTestCase(f, c, nfsPVconfig, pvcConfig, ns, clientNode.Name)
+		})
+
+		AfterEach(func() {
+			framework.Logf("Tearing down test spec")
+			tearDownTestCase(c, f, ns, clientPod, pvc, pv)
+			pv, pvc, clientPod = nil, nil, nil
+		})
+
+		// Test table housing the It() title string and test spec.  runTest is type testBody, defined at
+		// the start of this file.  To add tests, define a function mirroring the testBody signature and assign
+		// to runTest.
+		disruptiveTestTable := []disruptiveTest{
+			{
+				testItStmt: "Should test that a file written to the mount before kubelet restart is readable after restart.",
+				runTest:    testKubeletRestartsAndRestoresMount,
+			},
+			{
+				testItStmt: "Should test that a volume mounted to a pod that is deleted while the kubelet is down unmounts when the kubelet returns.",
+				runTest:    testVolumeUnmountsFromDeletedPod,
+			},
+		}
+
+		// Test loop executes each disruptiveTest iteratively.
+		for _, test := range disruptiveTestTable {
+			func(t disruptiveTest) {
+				It(t.testItStmt, func() {
+					By("Executing Spec")
+					t.runTest(c, f, clientPod, pvc, pv)
+				})
+			}(test)
+		}
+	})
+})
+
+// testKubeletRestartsAndRestoresMount tests that a volume mounted to a pod remains mounted after a kubelet restarts
+func testKubeletRestartsAndRestoresMount(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) {
+	By("Writing to the volume.")
+	file := "/mnt/_SUCCESS"
+	out, err := podExec(clientPod, fmt.Sprintf("touch %s", file))
+	framework.Logf(out)
+	Expect(err).NotTo(HaveOccurred())
+
+	By("Restarting kubelet")
+	kubeletCommand(kRestart, c, clientPod)
+
+	By("Testing that written file is accessible.")
+	out, err = podExec(clientPod, fmt.Sprintf("cat %s", file))
+	framework.Logf(out)
+	Expect(err).NotTo(HaveOccurred())
+	framework.Logf("Volume mount detected on pod %s and written file %s is readable post-restart.", clientPod.Name, file)
+}
+
+// testVolumeUnmountsFromDeletedPod tests that a volume unmounts if the client pod was deleted while the kubelet was down.
+func testVolumeUnmountsFromDeletedPod(c clientset.Interface, f *framework.Framework, clientPod *v1.Pod, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) {
+	nodeIP, err := framework.GetHostExternalAddress(c, clientPod)
+	Expect(err).NotTo(HaveOccurred())
+	nodeIP = nodeIP + ":22"
+
+	By("Expecting the volume mount to be found.")
+	result, err := framework.SSH(fmt.Sprintf("mount | grep %s", clientPod.UID), nodeIP, framework.TestContext.Provider)
+	framework.LogSSHResult(result)
+	Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.")
+	Expect(result.Code).To(BeZero(), fmt.Sprintf("Expected grep exit code of 0, got %d", result.Code))
+
+	By("Stopping the kubelet.")
+	kubeletCommand(kStop, c, clientPod)
+	defer func() {
+		if err != nil {
+			kubeletCommand(kStart, c, clientPod)
+		}
+	}()
+	By(fmt.Sprintf("Deleting Pod %q", clientPod.Name))
+	err = c.Core().Pods(clientPod.Namespace).Delete(clientPod.Name, &metav1.DeleteOptions{})
+	Expect(err).NotTo(HaveOccurred())
+	By("Starting the kubelet and waiting for pod to delete.")
+	kubeletCommand(kStart, c, clientPod)
+	err = f.WaitForPodTerminated(clientPod.Name, "")
+	if !apierrs.IsNotFound(err) && err != nil {
+		Expect(err).NotTo(HaveOccurred(), "Expected pod to terminate.")
+	}
+
+	By("Expecting the volume mount not to be found.")
+	result, err = framework.SSH(fmt.Sprintf("mount | grep %s", clientPod.UID), nodeIP, framework.TestContext.Provider)
+	framework.LogSSHResult(result)
+	Expect(err).NotTo(HaveOccurred(), "Encountered SSH error.")
+	Expect(result.Stdout).To(BeEmpty(), "Expected grep stdout to be empty (i.e. no mount found).")
+	framework.Logf("Volume unmounted on node %s", clientPod.Spec.NodeName)
+}
+
+// initTestCase initializes spec resources (pv, pvc, and pod) and returns pointers to be consumed
+// by the test.
+func initTestCase(f *framework.Framework, c clientset.Interface, pvConfig framework.PersistentVolumeConfig, pvcConfig framework.PersistentVolumeClaimConfig, ns, nodeName string) (*v1.Pod, *v1.PersistentVolume, *v1.PersistentVolumeClaim) {
+	pv, pvc, err := framework.CreatePVPVC(c, pvConfig, pvcConfig, ns, false)
+	defer func() {
+		if err != nil {
+			framework.DeletePersistentVolumeClaim(c, pvc.Name, ns)
+			framework.DeletePersistentVolume(c, pv.Name)
+		}
+	}()
+	Expect(err).NotTo(HaveOccurred())
+	pod := framework.MakePod(ns, []*v1.PersistentVolumeClaim{pvc}, true, "")
+	pod.Spec.NodeName = nodeName
+	framework.Logf("Creating NFS client pod.")
+	pod, err = c.CoreV1().Pods(ns).Create(pod)
+	framework.Logf("NFS client Pod %q created on Node %q", pod.Name, nodeName)
+	Expect(err).NotTo(HaveOccurred())
+	defer func() {
+		if err != nil {
+			framework.DeletePodWithWait(f, c, pod)
+		}
+	}()
+	err = framework.WaitForPodRunningInNamespace(c, pod)
+	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Pod %q timed out waiting for phase: Running", pod.Name))
+	// Return created api objects
+	pod, err = c.CoreV1().Pods(ns).Get(pod.Name, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+	pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Get(pvc.Name, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+	pv, err = c.CoreV1().PersistentVolumes().Get(pv.Name, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+	return pod, pv, pvc
+}
+
+// tearDownTestCase destroy resources created by initTestCase.
+func tearDownTestCase(c clientset.Interface, f *framework.Framework, ns string, client *v1.Pod, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) {
+	// Ignore deletion errors.  Failing on them will interrupt test cleanup.
+	framework.DeletePodWithWait(f, c, client)
+	framework.DeletePersistentVolumeClaim(c, pvc.Name, ns)
+	framework.DeletePersistentVolume(c, pv.Name)
+}
+
+// kubeletCommand performs `start`, `restart`, or `stop` on the kubelet running on the node of the target pod and waits
+// for the desired statues..
+// - First issues the command via `systemctl`
+// - If `systemctl` returns stderr "command not found, issues the command via `service`
+// - If `service` also returns stderr "command not found", the test is aborted.
+// Allowed kubeletOps are `kStart`, `kStop`, and `kRestart`
+func kubeletCommand(kOp kubeletOpt, c clientset.Interface, pod *v1.Pod) {
+	nodeIP, err := framework.GetHostExternalAddress(c, pod)
+	Expect(err).NotTo(HaveOccurred())
+	nodeIP = nodeIP + ":22"
+	systemctlCmd := fmt.Sprintf("sudo systemctl %s kubelet", string(kOp))
+	framework.Logf("Attempting `%s`", systemctlCmd)
+	sshResult, err := framework.SSH(systemctlCmd, nodeIP, framework.TestContext.Provider)
+	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName))
+	framework.LogSSHResult(sshResult)
+	if strings.Contains(sshResult.Stderr, "command not found") {
+		serviceCmd := fmt.Sprintf("sudo service kubelet %s", string(kOp))
+		framework.Logf("Attempting `%s`", serviceCmd)
+		sshResult, err = framework.SSH(serviceCmd, nodeIP, framework.TestContext.Provider)
+		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("SSH to Node %q errored.", pod.Spec.NodeName))
+		framework.LogSSHResult(sshResult)
+	}
+	Expect(sshResult.Code).To(BeZero(), "Failed to [%s] kubelet:\n%#v", string(kOp), sshResult)
+	// On restart, waiting for node NotReady prevents a race condition where the node takes a few moments to leave the
+	// Ready state which in turn short circuits WaitForNodeToBeReady()
+	if kOp == kStop || kOp == kRestart {
+		if ok := framework.WaitForNodeToBeNotReady(c, pod.Spec.NodeName, NodeStateTimeout); !ok {
+			framework.Failf("Node %s failed to enter NotReady state", pod.Spec.NodeName)
+		}
+	}
+	if kOp == kStart || kOp == kRestart {
+		if ok := framework.WaitForNodeToBeReady(c, pod.Spec.NodeName, NodeStateTimeout); !ok {
+			framework.Failf("Node %s failed to enter Ready state", pod.Spec.NodeName)
+		}
+	}
+}
+
+// podExec wraps RunKubectl to execute a bash cmd in target pod
+func podExec(pod *v1.Pod, bashExec string) (string, error) {
+	return framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", pod.Namespace), pod.Name, "--", "/bin/sh", "-c", bashExec)
+}

vendor/k8s.io/kubernetes/test/e2e/storage/persistent_volumes-gce.go

@@ -0,0 +1,162 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+// verifyGCEDiskAttached performs a sanity check to verify the PD attached to the node
+func verifyGCEDiskAttached(diskName string, nodeName types.NodeName) bool {
+	gceCloud, err := framework.GetGCECloud()
+	Expect(err).NotTo(HaveOccurred())
+	isAttached, err := gceCloud.DiskIsAttached(diskName, nodeName)
+	Expect(err).NotTo(HaveOccurred())
+	return isAttached
+}
+
+// initializeGCETestSpec creates a PV, PVC, and ClientPod that will run until killed by test or clean up.
+func initializeGCETestSpec(c clientset.Interface, ns string, pvConfig framework.PersistentVolumeConfig, pvcConfig framework.PersistentVolumeClaimConfig, isPrebound bool) (*v1.Pod, *v1.PersistentVolume, *v1.PersistentVolumeClaim) {
+	By("Creating the PV and PVC")
+	pv, pvc, err := framework.CreatePVPVC(c, pvConfig, pvcConfig, ns, isPrebound)
+	Expect(err).NotTo(HaveOccurred())
+	framework.ExpectNoError(framework.WaitOnPVandPVC(c, ns, pv, pvc))
+
+	By("Creating the Client Pod")
+	clientPod, err := framework.CreateClientPod(c, ns, pvc)
+	Expect(err).NotTo(HaveOccurred())
+	return clientPod, pv, pvc
+}
+
+// Testing configurations of single a PV/PVC pair attached to a GCE PD
+var _ = SIGDescribe("PersistentVolumes GCEPD", func() {
+	var (
+		c         clientset.Interface
+		diskName  string
+		ns        string
+		err       error
+		pv        *v1.PersistentVolume
+		pvc       *v1.PersistentVolumeClaim
+		clientPod *v1.Pod
+		pvConfig  framework.PersistentVolumeConfig
+		pvcConfig framework.PersistentVolumeClaimConfig
+		volLabel  labels.Set
+		selector  *metav1.LabelSelector
+		node      types.NodeName
+	)
+
+	f := framework.NewDefaultFramework("pv")
+	BeforeEach(func() {
+		c = f.ClientSet
+		ns = f.Namespace.Name
+
+		// Enforce binding only within test space via selector labels
+		volLabel = labels.Set{framework.VolumeSelectorKey: ns}
+		selector = metav1.SetAsLabelSelector(volLabel)
+
+		framework.SkipUnlessProviderIs("gce", "gke")
+		By("Initializing Test Spec")
+		diskName, err = framework.CreatePDWithRetry()
+		Expect(err).NotTo(HaveOccurred())
+		pvConfig = framework.PersistentVolumeConfig{
+			NamePrefix: "gce-",
+			Labels:     volLabel,
+			PVSource: v1.PersistentVolumeSource{
+				GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
+					PDName:   diskName,
+					FSType:   "ext3",
+					ReadOnly: false,
+				},
+			},
+			Prebind: nil,
+		}
+		pvcConfig = framework.PersistentVolumeClaimConfig{
+			Annotations: map[string]string{
+				v1.BetaStorageClassAnnotation: "",
+			},
+			Selector: selector,
+		}
+		clientPod, pv, pvc = initializeGCETestSpec(c, ns, pvConfig, pvcConfig, false)
+		node = types.NodeName(clientPod.Spec.NodeName)
+	})
+
+	AfterEach(func() {
+		framework.Logf("AfterEach: Cleaning up test resources")
+		if c != nil {
+			framework.ExpectNoError(framework.DeletePodWithWait(f, c, clientPod))
+			if errs := framework.PVPVCCleanup(c, ns, pv, pvc); len(errs) > 0 {
+				framework.Failf("AfterEach: Failed to delete PVC and/or PV. Errors: %v", utilerrors.NewAggregate(errs))
+			}
+			clientPod, pv, pvc, node = nil, nil, nil, ""
+			if diskName != "" {
+				framework.ExpectNoError(framework.DeletePDWithRetry(diskName))
+			}
+		}
+	})
+
+	// Attach a persistent disk to a pod using a PVC.
+	// Delete the PVC and then the pod.  Expect the pod to succeed in unmounting and detaching PD on delete.
+	It("should test that deleting a PVC before the pod does not cause pod deletion to fail on PD detach", func() {
+
+		By("Deleting the Claim")
+		framework.ExpectNoError(framework.DeletePersistentVolumeClaim(c, pvc.Name, ns), "Unable to delete PVC ", pvc.Name)
+		Expect(verifyGCEDiskAttached(diskName, node)).To(BeTrue())
+
+		By("Deleting the Pod")
+		framework.ExpectNoError(framework.DeletePodWithWait(f, c, clientPod), "Failed to delete pod ", clientPod.Name)
+
+		By("Verifying Persistent Disk detach")
+		framework.ExpectNoError(waitForPDDetach(diskName, node), "PD ", diskName, " did not detach")
+	})
+
+	// Attach a persistent disk to a pod using a PVC.
+	// Delete the PV and then the pod.  Expect the pod to succeed in unmounting and detaching PD on delete.
+	It("should test that deleting the PV before the pod does not cause pod deletion to fail on PD detach", func() {
+
+		By("Deleting the Persistent Volume")
+		framework.ExpectNoError(framework.DeletePersistentVolume(c, pv.Name), "Failed to delete PV ", pv.Name)
+		Expect(verifyGCEDiskAttached(diskName, node)).To(BeTrue())
+
+		By("Deleting the client pod")
+		framework.ExpectNoError(framework.DeletePodWithWait(f, c, clientPod), "Failed to delete pod ", clientPod.Name)
+
+		By("Verifying Persistent Disk detaches")
+		framework.ExpectNoError(waitForPDDetach(diskName, node), "PD ", diskName, " did not detach")
+	})
+
+	// Test that a Pod and PVC attached to a GCEPD successfully unmounts and detaches when the encompassing Namespace is deleted.
+	It("should test that deleting the Namespace of a PVC and Pod causes the successful detach of Persistent Disk", func() {
+
+		By("Deleting the Namespace")
+		err := c.CoreV1().Namespaces().Delete(ns, nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = framework.WaitForNamespacesDeleted(c, []string{ns}, framework.DefaultNamespaceDeletionTimeout)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Verifying Persistent Disk detaches")
+		framework.ExpectNoError(waitForPDDetach(diskName, node), "PD ", diskName, " did not detach")
+	})
+})

vendor/k8s.io/kubernetes/test/e2e/storage/persistent_volumes-local.go

@@ -0,0 +1,793 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"encoding/json"
+	"fmt"
+	"path"
+	"path/filepath"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	batchv1 "k8s.io/api/batch/v1"
+	"k8s.io/api/core/v1"
+	rbacv1beta1 "k8s.io/api/rbac/v1beta1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+type localTestConfig struct {
+	ns     string
+	nodes  *v1.NodeList
+	node0  *v1.Node
+	client clientset.Interface
+	scName string
+}
+
+type LocalVolumeType string
+
+const (
+	// default local volume type, aka a directory
+	DirectoryLocalVolumeType LocalVolumeType = "dir"
+	// creates a tmpfs and mounts it
+	TmpfsLocalVolumeType LocalVolumeType = "tmpfs"
+)
+
+type localTestVolume struct {
+	// Node that the volume is on
+	node *v1.Node
+	// Path to the volume on the host node
+	hostDir string
+	// PVC for this volume
+	pvc *v1.PersistentVolumeClaim
+	// PV for this volume
+	pv *v1.PersistentVolume
+	// Type of local volume
+	localVolumeType LocalVolumeType
+}
+
+const (
+	// TODO: This may not be available/writable on all images.
+	hostBase      = "/tmp"
+	containerBase = "/myvol"
+	// 'hostBase + discoveryDir' is the path for volume discovery.
+	discoveryDir = "disks"
+	// Path to the first volume in the test containers
+	// created via createLocalPod or makeLocalPod
+	// leveraging pv_util.MakePod
+	volumeDir = "/mnt/volume1"
+	// testFile created in setupLocalVolume
+	testFile = "test-file"
+	// testFileContent written into testFile
+	testFileContent = "test-file-content"
+	testSCPrefix    = "local-volume-test-storageclass"
+
+	// Following are constants used for provisioner e2e tests.
+	//
+	// testServiceAccount is the service account for bootstrapper
+	testServiceAccount = "local-storage-bootstrapper"
+	// testRoleBinding is the cluster-admin rolebinding for bootstrapper
+	testRoleBinding = "local-storage:bootstrapper"
+	// volumeConfigName is the configmap passed to bootstrapper and provisioner
+	volumeConfigName = "local-volume-config"
+	// bootstrapper and provisioner images used for e2e tests
+	bootstrapperImageName = "quay.io/external_storage/local-volume-provisioner-bootstrap:v1.0.1"
+	provisionerImageName  = "quay.io/external_storage/local-volume-provisioner:v1.0.1"
+	// provisioner daemonSetName name, must match the one defined in bootstrapper
+	daemonSetName = "local-volume-provisioner"
+	// provisioner node/pv cluster role binding, must match the one defined in bootstrapper
+	nodeBindingName = "local-storage:provisioner-node-binding"
+	pvBindingName   = "local-storage:provisioner-pv-binding"
+	// A sample request size
+	testRequestSize = "10Mi"
+)
+
+// Common selinux labels
+var selinuxLabel = &v1.SELinuxOptions{
+	Level: "s0:c0,c1"}
+
+var _ = SIGDescribe("PersistentVolumes-local [Feature:LocalPersistentVolumes] [Serial]", func() {
+	f := framework.NewDefaultFramework("persistent-local-volumes-test")
+
+	var (
+		config *localTestConfig
+		scName string
+	)
+
+	BeforeEach(func() {
+		// Get all the schedulable nodes
+		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+		Expect(len(nodes.Items)).NotTo(BeZero(), "No available nodes for scheduling")
+		scName = fmt.Sprintf("%v-%v", testSCPrefix, f.Namespace.Name)
+		// Choose the first node
+		node0 := &nodes.Items[0]
+
+		config = &localTestConfig{
+			ns:     f.Namespace.Name,
+			client: f.ClientSet,
+			nodes:  nodes,
+			node0:  node0,
+			scName: scName,
+		}
+	})
+
+	Context("when one pod requests one prebound PVC", func() {
+
+		var testVol *localTestVolume
+
+		BeforeEach(func() {
+			testVol = setupLocalVolumePVCPV(config, DirectoryLocalVolumeType)
+		})
+
+		AfterEach(func() {
+			cleanupLocalVolume(config, testVol)
+		})
+
+		It("should be able to mount and read from the volume using one-command containers", func() {
+			By("Creating a pod to read from the PV")
+			readCmd := createReadCmd(volumeDir, testFile)
+			podSpec := makeLocalPod(config, testVol, readCmd)
+			//testFileContent was written during setupLocalVolume
+			f.TestContainerOutput("pod reads PV", podSpec, 0, []string{testFileContent})
+		})
+
+		It("should be able to mount and write to the volume using one-command containers", func() {
+			By("Creating a pod to write to the PV")
+			writeCmd, readCmd := createWriteAndReadCmds(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/)
+			writeThenReadCmd := fmt.Sprintf("%s;%s", writeCmd, readCmd)
+			podSpec := makeLocalPod(config, testVol, writeThenReadCmd)
+			f.TestContainerOutput("pod writes to PV", podSpec, 0, []string{testVol.hostDir})
+		})
+
+		It("should be able to mount volume and read from pod1", func() {
+			By("Creating pod1")
+			pod1, pod1Err := createLocalPod(config, testVol)
+			Expect(pod1Err).NotTo(HaveOccurred())
+
+			pod1NodeName, pod1NodeNameErr := podNodeName(config, pod1)
+			Expect(pod1NodeNameErr).NotTo(HaveOccurred())
+			framework.Logf("pod1 %q created on Node %q", pod1.Name, pod1NodeName)
+			Expect(pod1NodeName).To(Equal(config.node0.Name))
+
+			By("Reading in pod1")
+			// testFileContent was written during setupLocalVolume
+			testReadFileContent(volumeDir, testFile, testFileContent, pod1)
+
+			By("Deleting pod1")
+			framework.DeletePodOrFail(config.client, config.ns, pod1.Name)
+		})
+
+		It("should be able to mount volume and write from pod1", func() {
+			By("Creating pod1")
+			pod1, pod1Err := createLocalPod(config, testVol)
+			Expect(pod1Err).NotTo(HaveOccurred())
+
+			pod1NodeName, pod1NodeNameErr := podNodeName(config, pod1)
+			Expect(pod1NodeNameErr).NotTo(HaveOccurred())
+			framework.Logf("pod1 %q created on Node %q", pod1.Name, pod1NodeName)
+			Expect(pod1NodeName).To(Equal(config.node0.Name))
+
+			// testFileContent was written during setupLocalVolume
+			testReadFileContent(volumeDir, testFile, testFileContent, pod1)
+
+			By("Writing in pod1")
+			writeCmd, _ := createWriteAndReadCmds(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/)
+			podRWCmdExec(pod1, writeCmd)
+
+			By("Deleting pod1")
+			framework.DeletePodOrFail(config.client, config.ns, pod1.Name)
+		})
+	})
+
+	Context("when two pods request one prebound PVC one after other", func() {
+
+		var testVol *localTestVolume
+
+		BeforeEach(func() {
+			testVol = setupLocalVolumePVCPV(config, DirectoryLocalVolumeType)
+		})
+
+		AfterEach(func() {
+			cleanupLocalVolume(config, testVol)
+		})
+
+		It("should be able to mount volume, write from pod1, and read from pod2 using one-command containers", func() {
+			By("Creating pod1 to write to the PV")
+			writeCmd, readCmd := createWriteAndReadCmds(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/)
+			writeThenReadCmd := fmt.Sprintf("%s;%s", writeCmd, readCmd)
+			podSpec1 := makeLocalPod(config, testVol, writeThenReadCmd)
+			f.TestContainerOutput("pod writes to PV", podSpec1, 0, []string{testVol.hostDir})
+
+			By("Creating pod2 to read from the PV")
+			podSpec2 := makeLocalPod(config, testVol, readCmd)
+			f.TestContainerOutput("pod reads PV", podSpec2, 0, []string{testVol.hostDir})
+		})
+	})
+
+	LocalVolumeTypes := []LocalVolumeType{DirectoryLocalVolumeType, TmpfsLocalVolumeType}
+
+	Context("when two pods mount a local volume at the same time", func() {
+		It("should be able to write from pod1 and read from pod2", func() {
+			for _, testVolType := range LocalVolumeTypes {
+				var testVol *localTestVolume
+				By(fmt.Sprintf("local-volume-type: %s", testVolType))
+				testVol = setupLocalVolumePVCPV(config, testVolType)
+				twoPodsReadWriteTest(config, testVol)
+				cleanupLocalVolume(config, testVol)
+			}
+		})
+	})
+
+	Context("when two pods mount a local volume one after the other", func() {
+		It("should be able to write from pod1 and read from pod2", func() {
+			for _, testVolType := range LocalVolumeTypes {
+				var testVol *localTestVolume
+				By(fmt.Sprintf("local-volume-type: %s", testVolType))
+				testVol = setupLocalVolumePVCPV(config, testVolType)
+				twoPodsReadWriteSerialTest(config, testVol)
+				cleanupLocalVolume(config, testVol)
+			}
+		})
+	})
+
+	Context("when using local volume provisioner", func() {
+		var (
+			volumePath string
+		)
+
+		BeforeEach(func() {
+			setupLocalVolumeProvisioner(config)
+			volumePath = path.Join(
+				hostBase, discoveryDir, fmt.Sprintf("vol-%v", string(uuid.NewUUID())))
+		})
+
+		AfterEach(func() {
+			cleanupLocalVolumeProvisioner(config, volumePath)
+		})
+
+		It("should create and recreate local persistent volume", func() {
+			By("Creating bootstrapper pod to start provisioner daemonset")
+			createBootstrapperJob(config)
+			kind := schema.GroupKind{Group: "extensions", Kind: "DaemonSet"}
+			framework.WaitForControlledPodsRunning(config.client, config.ns, daemonSetName, kind)
+
+			By("Creating a directory under discovery path")
+			framework.Logf("creating local volume under path %q", volumePath)
+			mkdirCmd := fmt.Sprintf("mkdir %v -m 777", volumePath)
+			err := framework.IssueSSHCommand(mkdirCmd, framework.TestContext.Provider, config.node0)
+			Expect(err).NotTo(HaveOccurred())
+			By("Waiting for a PersitentVolume to be created")
+			oldPV, err := waitForLocalPersistentVolume(config.client, volumePath)
+			Expect(err).NotTo(HaveOccurred())
+
+			// Create a persistent volume claim for local volume: the above volume will be bound.
+			By("Creating a persistent volume claim")
+			claim, err := config.client.Core().PersistentVolumeClaims(config.ns).Create(newLocalClaim(config))
+			Expect(err).NotTo(HaveOccurred())
+			err = framework.WaitForPersistentVolumeClaimPhase(
+				v1.ClaimBound, config.client, claim.Namespace, claim.Name, framework.Poll, 1*time.Minute)
+			Expect(err).NotTo(HaveOccurred())
+
+			claim, err = config.client.Core().PersistentVolumeClaims(config.ns).Get(claim.Name, metav1.GetOptions{})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(claim.Spec.VolumeName).To(Equal(oldPV.Name))
+
+			// Delete the persistent volume claim: file will be cleaned up and volume be re-created.
+			By("Deleting the persistent volume claim to clean up persistent volume and re-create one")
+			writeCmd, _ := createWriteAndReadCmds(volumePath, testFile, testFileContent)
+			err = framework.IssueSSHCommand(writeCmd, framework.TestContext.Provider, config.node0)
+			Expect(err).NotTo(HaveOccurred())
+			err = config.client.Core().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, &metav1.DeleteOptions{})
+			Expect(err).NotTo(HaveOccurred())
+
+			By("Waiting for a new PersistentVolume to be re-created")
+			newPV, err := waitForLocalPersistentVolume(config.client, volumePath)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(newPV.UID).NotTo(Equal(oldPV.UID))
+			fileDoesntExistCmd := createFileDoesntExistCmd(volumePath, testFile)
+			err = framework.IssueSSHCommand(fileDoesntExistCmd, framework.TestContext.Provider, config.node0)
+			Expect(err).NotTo(HaveOccurred())
+		})
+	})
+})
+
+// The tests below are run against multiple mount point types
+
+// Test two pods at the same time, write from pod1, and read from pod2
+func twoPodsReadWriteTest(config *localTestConfig, testVol *localTestVolume) {
+	By("Creating pod1 to write to the PV")
+	pod1, pod1Err := createLocalPod(config, testVol)
+	Expect(pod1Err).NotTo(HaveOccurred())
+
+	framework.ExpectNoError(framework.WaitForPodRunningInNamespace(config.client, pod1))
+	pod1NodeName, pod1NodeNameErr := podNodeName(config, pod1)
+	Expect(pod1NodeNameErr).NotTo(HaveOccurred())
+	framework.Logf("Pod1 %q created on Node %q", pod1.Name, pod1NodeName)
+	Expect(pod1NodeName).To(Equal(config.node0.Name))
+
+	// testFileContent was written during setupLocalVolume
+	testReadFileContent(volumeDir, testFile, testFileContent, pod1)
+
+	By("Creating pod2 to read from the PV")
+	pod2, pod2Err := createLocalPod(config, testVol)
+	Expect(pod2Err).NotTo(HaveOccurred())
+
+	framework.ExpectNoError(framework.WaitForPodRunningInNamespace(config.client, pod2))
+	pod2NodeName, pod2NodeNameErr := podNodeName(config, pod2)
+	Expect(pod2NodeNameErr).NotTo(HaveOccurred())
+	framework.Logf("Pod2 %q created on Node %q", pod2.Name, pod2NodeName)
+	Expect(pod2NodeName).To(Equal(config.node0.Name))
+
+	// testFileContent was written during setupLocalVolume
+	testReadFileContent(volumeDir, testFile, testFileContent, pod2)
+
+	writeCmd := createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/)
+
+	By("Writing in pod1")
+	podRWCmdExec(pod1, writeCmd)
+
+	By("Reading in pod2")
+	testReadFileContent(volumeDir, testFile, testVol.hostDir, pod2)
+
+	By("Deleting pod1")
+	framework.DeletePodOrFail(config.client, config.ns, pod1.Name)
+	By("Deleting pod2")
+	framework.DeletePodOrFail(config.client, config.ns, pod2.Name)
+}
+
+// Test two pods one after other, write from pod1, and read from pod2
+func twoPodsReadWriteSerialTest(config *localTestConfig, testVol *localTestVolume) {
+	By("Creating pod1")
+	pod1, pod1Err := createLocalPod(config, testVol)
+	Expect(pod1Err).NotTo(HaveOccurred())
+
+	framework.ExpectNoError(framework.WaitForPodRunningInNamespace(config.client, pod1))
+	pod1NodeName, pod1NodeNameErr := podNodeName(config, pod1)
+	Expect(pod1NodeNameErr).NotTo(HaveOccurred())
+	framework.Logf("Pod1 %q created on Node %q", pod1.Name, pod1NodeName)
+	Expect(pod1NodeName).To(Equal(config.node0.Name))
+
+	// testFileContent was written during setupLocalVolume
+	testReadFileContent(volumeDir, testFile, testFileContent, pod1)
+
+	writeCmd := createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/)
+
+	By("Writing in pod1")
+	podRWCmdExec(pod1, writeCmd)
+
+	By("Deleting pod1")
+	framework.DeletePodOrFail(config.client, config.ns, pod1.Name)
+
+	By("Creating pod2")
+	pod2, pod2Err := createLocalPod(config, testVol)
+	Expect(pod2Err).NotTo(HaveOccurred())
+
+	framework.ExpectNoError(framework.WaitForPodRunningInNamespace(config.client, pod2))
+	pod2NodeName, pod2NodeNameErr := podNodeName(config, pod2)
+	Expect(pod2NodeNameErr).NotTo(HaveOccurred())
+	framework.Logf("Pod2 %q created on Node %q", pod2.Name, pod2NodeName)
+	Expect(pod2NodeName).To(Equal(config.node0.Name))
+
+	By("Reading in pod2")
+	testReadFileContent(volumeDir, testFile, testVol.hostDir, pod2)
+
+	By("Deleting pod2")
+	framework.DeletePodOrFail(config.client, config.ns, pod2.Name)
+}
+
+// podNode wraps RunKubectl to get node where pod is running
+func podNodeName(config *localTestConfig, pod *v1.Pod) (string, error) {
+	runtimePod, runtimePodErr := config.client.Core().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{})
+	return runtimePod.Spec.NodeName, runtimePodErr
+}
+
+// setupLocalVolume setups a directory to user for local PV
+func setupLocalVolume(config *localTestConfig, localVolumeType LocalVolumeType) *localTestVolume {
+	testDirName := "local-volume-test-" + string(uuid.NewUUID())
+	hostDir := filepath.Join(hostBase, testDirName)
+
+	if localVolumeType == TmpfsLocalVolumeType {
+		createAndMountTmpfsLocalVolume(config, hostDir)
+	}
+
+	// populate volume with testFile containing testFileContent
+	writeCmd, _ := createWriteAndReadCmds(hostDir, testFile, testFileContent)
+	By(fmt.Sprintf("Creating local volume on node %q at path %q", config.node0.Name, hostDir))
+	err := framework.IssueSSHCommand(writeCmd, framework.TestContext.Provider, config.node0)
+	Expect(err).NotTo(HaveOccurred())
+	return &localTestVolume{
+		node:            config.node0,
+		hostDir:         hostDir,
+		localVolumeType: localVolumeType,
+	}
+}
+
+// Deletes the PVC/PV, and launches a pod with hostpath volume to remove the test directory
+func cleanupLocalVolume(config *localTestConfig, volume *localTestVolume) {
+	if volume == nil {
+		return
+	}
+
+	By("Cleaning up PVC and PV")
+	errs := framework.PVPVCCleanup(config.client, config.ns, volume.pv, volume.pvc)
+	if len(errs) > 0 {
+		framework.Failf("Failed to delete PV and/or PVC: %v", utilerrors.NewAggregate(errs))
+	}
+
+	if volume.localVolumeType == TmpfsLocalVolumeType {
+		unmountTmpfsLocalVolume(config, volume.hostDir)
+	}
+
+	By("Removing the test directory")
+	removeCmd := fmt.Sprintf("rm -r %s", volume.hostDir)
+	err := framework.IssueSSHCommand(removeCmd, framework.TestContext.Provider, config.node0)
+	Expect(err).NotTo(HaveOccurred())
+}
+
+func makeLocalPVCConfig(config *localTestConfig) framework.PersistentVolumeClaimConfig {
+	return framework.PersistentVolumeClaimConfig{
+		AccessModes:      []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
+		StorageClassName: &config.scName,
+	}
+}
+
+func makeLocalPVConfig(config *localTestConfig, volume *localTestVolume) framework.PersistentVolumeConfig {
+	// TODO: hostname may not be the best option
+	nodeKey := "kubernetes.io/hostname"
+	if volume.node.Labels == nil {
+		framework.Failf("Node does not have labels")
+	}
+	nodeValue, found := volume.node.Labels[nodeKey]
+	if !found {
+		framework.Failf("Node does not have required label %q", nodeKey)
+	}
+
+	return framework.PersistentVolumeConfig{
+		PVSource: v1.PersistentVolumeSource{
+			Local: &v1.LocalVolumeSource{
+				Path: volume.hostDir,
+			},
+		},
+		NamePrefix:       "local-pv",
+		StorageClassName: config.scName,
+		NodeAffinity: &v1.NodeAffinity{
+			RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
+				NodeSelectorTerms: []v1.NodeSelectorTerm{
+					{
+						MatchExpressions: []v1.NodeSelectorRequirement{
+							{
+								Key:      nodeKey,
+								Operator: v1.NodeSelectorOpIn,
+								Values:   []string{nodeValue},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+// Creates a PVC and PV with prebinding
+func createLocalPVCPV(config *localTestConfig, volume *localTestVolume) {
+	pvcConfig := makeLocalPVCConfig(config)
+	pvConfig := makeLocalPVConfig(config, volume)
+	var err error
+	volume.pv, volume.pvc, err = framework.CreatePVPVC(config.client, pvConfig, pvcConfig, config.ns, true)
+	framework.ExpectNoError(err)
+	framework.ExpectNoError(framework.WaitOnPVandPVC(config.client, config.ns, volume.pv, volume.pvc))
+}
+
+func makeLocalPod(config *localTestConfig, volume *localTestVolume, cmd string) *v1.Pod {
+	return framework.MakeSecPod(config.ns, []*v1.PersistentVolumeClaim{volume.pvc}, false, cmd, false, false, selinuxLabel)
+}
+
+// createSecPod should be used when Pod requires non default SELinux labels
+func createSecPod(config *localTestConfig, volume *localTestVolume, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions) (*v1.Pod, error) {
+	pod, err := framework.CreateSecPod(config.client, config.ns, []*v1.PersistentVolumeClaim{volume.pvc}, false, "", hostIPC, hostPID, seLinuxLabel)
+	podNodeName, podNodeNameErr := podNodeName(config, pod)
+	Expect(podNodeNameErr).NotTo(HaveOccurred())
+	framework.Logf("Security Context POD %q created on Node %q", pod.Name, podNodeName)
+	Expect(podNodeName).To(Equal(config.node0.Name))
+	return pod, err
+}
+
+func createLocalPod(config *localTestConfig, volume *localTestVolume) (*v1.Pod, error) {
+	return framework.CreateSecPod(config.client, config.ns, []*v1.PersistentVolumeClaim{volume.pvc}, false, "", false, false, selinuxLabel)
+}
+
+func createAndMountTmpfsLocalVolume(config *localTestConfig, dir string) {
+	By(fmt.Sprintf("Creating tmpfs mount point on node %q at path %q", config.node0.Name, dir))
+	err := framework.IssueSSHCommand(fmt.Sprintf("mkdir -p %q && sudo mount -t tmpfs -o size=1m tmpfs-%q %q", dir, dir, dir), framework.TestContext.Provider, config.node0)
+	Expect(err).NotTo(HaveOccurred())
+}
+
+func unmountTmpfsLocalVolume(config *localTestConfig, dir string) {
+	By(fmt.Sprintf("Unmount tmpfs mount point on node %q at path %q", config.node0.Name, dir))
+	err := framework.IssueSSHCommand(fmt.Sprintf("sudo umount %q", dir), framework.TestContext.Provider, config.node0)
+	Expect(err).NotTo(HaveOccurred())
+}
+
+// Create corresponding write and read commands
+// to be executed via SSH on the node with the local PV
+func createWriteAndReadCmds(testFileDir string, testFile string, writeTestFileContent string) (writeCmd string, readCmd string) {
+	writeCmd = createWriteCmd(testFileDir, testFile, writeTestFileContent)
+	readCmd = createReadCmd(testFileDir, testFile)
+	return writeCmd, readCmd
+}
+
+func createWriteCmd(testFileDir string, testFile string, writeTestFileContent string) string {
+	testFilePath := filepath.Join(testFileDir, testFile)
+	return fmt.Sprintf("mkdir -p %s; echo %s > %s", testFileDir, writeTestFileContent, testFilePath)
+}
+func createReadCmd(testFileDir string, testFile string) string {
+	testFilePath := filepath.Join(testFileDir, testFile)
+	return fmt.Sprintf("cat %s", testFilePath)
+}
+
+// Read testFile and evaluate whether it contains the testFileContent
+func testReadFileContent(testFileDir string, testFile string, testFileContent string, pod *v1.Pod) {
+	readCmd := createReadCmd(volumeDir, testFile)
+	readOut := podRWCmdExec(pod, readCmd)
+	Expect(readOut).To(ContainSubstring(testFileContent))
+}
+
+// Create command to verify that the file doesn't exist
+// to be executed via SSH on the node with the local PV
+func createFileDoesntExistCmd(testFileDir string, testFile string) string {
+	testFilePath := filepath.Join(testFileDir, testFile)
+	return fmt.Sprintf("[ ! -e %s ]", testFilePath)
+}
+
+// Execute a read or write command in a pod.
+// Fail on error
+func podRWCmdExec(pod *v1.Pod, cmd string) string {
+	out, err := podExec(pod, cmd)
+	Expect(err).NotTo(HaveOccurred())
+	return out
+}
+
+// Initialize test volume on node
+// and create local PVC and PV
+func setupLocalVolumePVCPV(config *localTestConfig, localVolumeType LocalVolumeType) *localTestVolume {
+	By("Initializing test volume")
+	testVol := setupLocalVolume(config, localVolumeType)
+
+	By("Creating local PVC and PV")
+	createLocalPVCPV(config, testVol)
+
+	return testVol
+}
+
+func setupLocalVolumeProvisioner(config *localTestConfig) {
+	By("Bootstrapping local volume provisioner")
+	createServiceAccount(config)
+	createClusterRoleBinding(config)
+	createVolumeConfigMap(config)
+
+	By("Initializing local volume discovery base path")
+	mkdirCmd := fmt.Sprintf("mkdir %v -m 777", path.Join(hostBase, discoveryDir))
+	err := framework.IssueSSHCommand(mkdirCmd, framework.TestContext.Provider, config.node0)
+	Expect(err).NotTo(HaveOccurred())
+}
+
+func cleanupLocalVolumeProvisioner(config *localTestConfig, volumePath string) {
+	By("Cleaning up cluster role binding")
+	deleteClusterRoleBinding(config)
+
+	By("Removing the test directory")
+	removeCmd := fmt.Sprintf("rm -r %s", path.Join(hostBase, discoveryDir))
+	err := framework.IssueSSHCommand(removeCmd, framework.TestContext.Provider, config.node0)
+	Expect(err).NotTo(HaveOccurred())
+
+	By("Cleaning up persistent volume")
+	pv, err := findLocalPersistentVolume(config.client, volumePath)
+	Expect(err).NotTo(HaveOccurred())
+	err = config.client.Core().PersistentVolumes().Delete(pv.Name, &metav1.DeleteOptions{})
+	Expect(err).NotTo(HaveOccurred())
+}
+
+func createServiceAccount(config *localTestConfig) {
+	serviceAccount := v1.ServiceAccount{
+		TypeMeta:   metav1.TypeMeta{APIVersion: "v1", Kind: "ServiceAccount"},
+		ObjectMeta: metav1.ObjectMeta{Name: testServiceAccount, Namespace: config.ns},
+	}
+	_, err := config.client.CoreV1().ServiceAccounts(config.ns).Create(&serviceAccount)
+	Expect(err).NotTo(HaveOccurred())
+}
+
+func createClusterRoleBinding(config *localTestConfig) {
+	subjects := []rbacv1beta1.Subject{
+		{
+			Kind:      rbacv1beta1.ServiceAccountKind,
+			Name:      testServiceAccount,
+			Namespace: config.ns,
+		},
+	}
+
+	binding := rbacv1beta1.ClusterRoleBinding{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "rbac.authorization.k8s.io/v1beta1",
+			Kind:       "ClusterRoleBinding",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: testRoleBinding,
+		},
+		RoleRef: rbacv1beta1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "ClusterRole",
+			Name:     "cluster-admin",
+		},
+		Subjects: subjects,
+	}
+
+	_, err := config.client.RbacV1beta1().ClusterRoleBindings().Create(&binding)
+	Expect(err).NotTo(HaveOccurred())
+}
+
+func deleteClusterRoleBinding(config *localTestConfig) {
+	err := config.client.RbacV1beta1().ClusterRoleBindings().Delete(testRoleBinding, metav1.NewDeleteOptions(0))
+	Expect(err).NotTo(HaveOccurred())
+	// These role bindings are created in provisioner; we just ensure it's
+	// deleted and do not panic on error.
+	config.client.RbacV1beta1().ClusterRoleBindings().Delete(nodeBindingName, metav1.NewDeleteOptions(0))
+	config.client.RbacV1beta1().ClusterRoleBindings().Delete(pvBindingName, metav1.NewDeleteOptions(0))
+}
+
+func createVolumeConfigMap(config *localTestConfig) {
+	mountConfig := struct {
+		HostDir string `json:"hostDir"`
+	}{
+		HostDir: path.Join(hostBase, discoveryDir),
+	}
+	data, err := json.Marshal(&mountConfig)
+	Expect(err).NotTo(HaveOccurred())
+
+	configMap := v1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "ConfigMap",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      volumeConfigName,
+			Namespace: config.ns,
+		},
+		Data: map[string]string{
+			config.scName: string(data),
+		},
+	}
+	_, err = config.client.CoreV1().ConfigMaps(config.ns).Create(&configMap)
+	Expect(err).NotTo(HaveOccurred())
+}
+
+func createBootstrapperJob(config *localTestConfig) {
+	bootJob := &batchv1.Job{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Job",
+			APIVersion: "batch/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "local-volume-tester-",
+		},
+		Spec: batchv1.JobSpec{
+			Template: v1.PodTemplateSpec{
+				Spec: v1.PodSpec{
+					RestartPolicy:      v1.RestartPolicyNever,
+					ServiceAccountName: testServiceAccount,
+					Containers: []v1.Container{
+						{
+							Name:  "volume-tester",
+							Image: bootstrapperImageName,
+							Env: []v1.EnvVar{
+								{
+									Name: "MY_NAMESPACE",
+									ValueFrom: &v1.EnvVarSource{
+										FieldRef: &v1.ObjectFieldSelector{
+											FieldPath: "metadata.namespace",
+										},
+									},
+								},
+							},
+							Args: []string{
+								fmt.Sprintf("--image=%v", provisionerImageName),
+								fmt.Sprintf("--volume-config=%v", volumeConfigName),
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	job, err := config.client.Batch().Jobs(config.ns).Create(bootJob)
+	Expect(err).NotTo(HaveOccurred())
+	err = framework.WaitForJobFinish(config.client, config.ns, job.Name, 1)
+	Expect(err).NotTo(HaveOccurred())
+}
+
+// newLocalClaim creates a new persistent volume claim.
+func newLocalClaim(config *localTestConfig) *v1.PersistentVolumeClaim {
+	claim := v1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "local-pvc-",
+			Namespace:    config.ns,
+		},
+		Spec: v1.PersistentVolumeClaimSpec{
+			StorageClassName: &config.scName,
+			AccessModes: []v1.PersistentVolumeAccessMode{
+				v1.ReadWriteOnce,
+			},
+			Resources: v1.ResourceRequirements{
+				Requests: v1.ResourceList{
+					v1.ResourceName(v1.ResourceStorage): resource.MustParse(testRequestSize),
+				},
+			},
+		},
+	}
+
+	return &claim
+}
+
+// waitForLocalPersistentVolume waits a local persistent volume with 'volumePath' to be available.
+func waitForLocalPersistentVolume(c clientset.Interface, volumePath string) (*v1.PersistentVolume, error) {
+	var pv *v1.PersistentVolume
+
+	for start := time.Now(); time.Since(start) < 10*time.Minute && pv == nil; time.Sleep(5 * time.Second) {
+		pvs, err := c.Core().PersistentVolumes().List(metav1.ListOptions{})
+		if err != nil {
+			return nil, err
+		}
+		if len(pvs.Items) == 0 {
+			continue
+		}
+		for _, p := range pvs.Items {
+			if p.Spec.PersistentVolumeSource.Local == nil || p.Spec.PersistentVolumeSource.Local.Path != volumePath {
+				continue
+			}
+			if p.Status.Phase != v1.VolumeAvailable {
+				continue
+			}
+			pv = &p
+			break
+		}
+	}
+	if pv == nil {
+		return nil, fmt.Errorf("Timeout while waiting for local persistent volume with path %v to be available", volumePath)
+	}
+	return pv, nil
+}
+
+// findLocalPersistentVolume finds persistent volume with 'spec.local.path' equals 'volumePath'.
+func findLocalPersistentVolume(c clientset.Interface, volumePath string) (*v1.PersistentVolume, error) {
+	pvs, err := c.Core().PersistentVolumes().List(metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+	for _, p := range pvs.Items {
+		if p.Spec.PersistentVolumeSource.Local != nil && p.Spec.PersistentVolumeSource.Local.Path == volumePath {
+			return &p, nil
+		}
+	}
+	return nil, fmt.Errorf("Unable to find local persistent volume with path %v", volumePath)
+}

vendor/k8s.io/kubernetes/test/e2e/storage/persistent_volumes-vsphere.go

@@ -0,0 +1,217 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
+	clientset "k8s.io/client-go/kubernetes"
+	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+// Testing configurations of single a PV/PVC pair attached to a vSphere Disk
+var _ = SIGDescribe("PersistentVolumes:vsphere", func() {
+	var (
+		c          clientset.Interface
+		ns         string
+		volumePath string
+		pv         *v1.PersistentVolume
+		pvc        *v1.PersistentVolumeClaim
+		clientPod  *v1.Pod
+		pvConfig   framework.PersistentVolumeConfig
+		pvcConfig  framework.PersistentVolumeClaimConfig
+		vsp        *vsphere.VSphere
+		err        error
+		node       types.NodeName
+		volLabel   labels.Set
+		selector   *metav1.LabelSelector
+	)
+
+	f := framework.NewDefaultFramework("pv")
+	/*
+		Test Setup
+
+		1. Create volume (vmdk)
+		2. Create PV with volume path for the vmdk.
+		3. Create PVC to bind with PV.
+		4. Create a POD using the PVC.
+		5. Verify Disk and Attached to the node.
+	*/
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("vsphere")
+		c = f.ClientSet
+		ns = f.Namespace.Name
+		clientPod = nil
+		pvc = nil
+		pv = nil
+
+		volLabel = labels.Set{framework.VolumeSelectorKey: ns}
+		selector = metav1.SetAsLabelSelector(volLabel)
+
+		if vsp == nil {
+			vsp, err = vsphere.GetVSphere()
+			Expect(err).NotTo(HaveOccurred())
+		}
+		if volumePath == "" {
+			volumePath, err = createVSphereVolume(vsp, nil)
+			Expect(err).NotTo(HaveOccurred())
+			pvConfig = framework.PersistentVolumeConfig{
+				NamePrefix: "vspherepv-",
+				Labels:     volLabel,
+				PVSource: v1.PersistentVolumeSource{
+					VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
+						VolumePath: volumePath,
+						FSType:     "ext4",
+					},
+				},
+				Prebind: nil,
+			}
+			pvcConfig = framework.PersistentVolumeClaimConfig{
+				Annotations: map[string]string{
+					v1.BetaStorageClassAnnotation: "",
+				},
+				Selector: selector,
+			}
+		}
+		By("Creating the PV and PVC")
+		pv, pvc, err = framework.CreatePVPVC(c, pvConfig, pvcConfig, ns, false)
+		Expect(err).NotTo(HaveOccurred())
+		framework.ExpectNoError(framework.WaitOnPVandPVC(c, ns, pv, pvc))
+
+		By("Creating the Client Pod")
+		clientPod, err = framework.CreateClientPod(c, ns, pvc)
+		Expect(err).NotTo(HaveOccurred())
+		node = types.NodeName(clientPod.Spec.NodeName)
+
+		By("Verify disk should be attached to the node")
+		isAttached, err := verifyVSphereDiskAttached(vsp, volumePath, node)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(isAttached).To(BeTrue(), "disk is not attached with the node")
+	})
+
+	AfterEach(func() {
+		framework.Logf("AfterEach: Cleaning up test resources")
+		if c != nil {
+			framework.ExpectNoError(framework.DeletePodWithWait(f, c, clientPod), "AfterEach: failed to delete pod ", clientPod.Name)
+
+			if pv != nil {
+				framework.ExpectNoError(framework.DeletePersistentVolume(c, pv.Name), "AfterEach: failed to delete PV ", pv.Name)
+			}
+			if pvc != nil {
+				framework.ExpectNoError(framework.DeletePersistentVolumeClaim(c, pvc.Name, ns), "AfterEach: failed to delete PVC ", pvc.Name)
+			}
+		}
+	})
+	/*
+		Clean up
+
+		1. Wait and verify volume is detached from the node
+		2. Delete PV
+		3. Delete Volume (vmdk)
+	*/
+	framework.AddCleanupAction(func() {
+		if len(volumePath) > 0 {
+			framework.ExpectNoError(waitForVSphereDiskToDetach(vsp, volumePath, node))
+			vsp.DeleteVolume(volumePath)
+		}
+	})
+
+	/*
+		Delete the PVC and then the pod.  Expect the pod to succeed in unmounting and detaching PD on delete.
+
+		Test Steps:
+		1. Delete PVC.
+		2. Delete POD, POD deletion should succeed.
+	*/
+
+	It("should test that deleting a PVC before the pod does not cause pod deletion to fail on vsphere volume detach", func() {
+		By("Deleting the Claim")
+		framework.ExpectNoError(framework.DeletePersistentVolumeClaim(c, pvc.Name, ns), "Failed to delete PVC ", pvc.Name)
+		pvc = nil
+
+		By("Deleting the Pod")
+		framework.ExpectNoError(framework.DeletePodWithWait(f, c, clientPod), "Failed to delete pod ", clientPod.Name)
+	})
+
+	/*
+		Delete the PV and then the pod.  Expect the pod to succeed in unmounting and detaching PD on delete.
+
+		Test Steps:
+		1. Delete PV.
+		2. Delete POD, POD deletion should succeed.
+	*/
+	It("should test that deleting the PV before the pod does not cause pod deletion to fail on vspehre volume detach", func() {
+		By("Deleting the Persistent Volume")
+		framework.ExpectNoError(framework.DeletePersistentVolume(c, pv.Name), "Failed to delete PV ", pv.Name)
+		pv = nil
+
+		By("Deleting the pod")
+		framework.ExpectNoError(framework.DeletePodWithWait(f, c, clientPod), "Failed to delete pod ", clientPod.Name)
+	})
+	/*
+		This test verifies that a volume mounted to a pod remains mounted after a kubelet restarts.
+		Steps:
+		1. Write to the volume
+		2. Restart kubelet
+		3. Verify that written file is accessible after kubelet restart
+	*/
+	It("should test that a file written to the vspehre volume mount before kubelet restart can be read after restart [Disruptive]", func() {
+		testKubeletRestartsAndRestoresMount(c, f, clientPod, pvc, pv)
+	})
+
+	/*
+		This test verifies that a volume mounted to a pod that is deleted while the kubelet is down
+		unmounts volume when the kubelet returns.
+
+		Steps:
+		1. Verify volume is mounted on the node.
+		2. Stop kubelet.
+		3. Delete pod.
+		4. Start kubelet.
+		5. Verify that volume mount not to be found.
+	*/
+	It("should test that a vspehre volume mounted to a pod that is deleted while the kubelet is down unmounts when the kubelet returns [Disruptive]", func() {
+		testVolumeUnmountsFromDeletedPod(c, f, clientPod, pvc, pv)
+	})
+
+	/*
+		This test verifies that deleting the Namespace of a PVC and Pod causes the successful detach of Persistent Disk
+
+		Steps:
+		1. Delete Namespace.
+		2. Wait for namespace to get deleted. (Namespace deletion should trigger deletion of belonging pods)
+		3. Verify volume should be detached from the node.
+	*/
+	It("should test that deleting the Namespace of a PVC and Pod causes the successful detach of vsphere volume", func() {
+		By("Deleting the Namespace")
+		err := c.CoreV1().Namespaces().Delete(ns, nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = framework.WaitForNamespacesDeleted(c, []string{ns}, 3*time.Minute)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Verifying Persistent Disk detaches")
+		waitForVSphereDiskToDetach(vsp, volumePath, node)
+	})
+})

vendor/k8s.io/kubernetes/test/e2e/storage/persistent_volumes.go

@@ -0,0 +1,306 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+// Validate PV/PVC, create and verify writer pod, delete the PVC, and validate the PV's
+// phase. Note: the PV is deleted in the AfterEach, not here.
+func completeTest(f *framework.Framework, c clientset.Interface, ns string, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) {
+	// 1. verify that the PV and PVC have bound correctly
+	By("Validating the PV-PVC binding")
+	framework.ExpectNoError(framework.WaitOnPVandPVC(c, ns, pv, pvc))
+
+	// 2. create the nfs writer pod, test if the write was successful,
+	//    then delete the pod and verify that it was deleted
+	By("Checking pod has write access to PersistentVolume")
+	framework.ExpectNoError(framework.CreateWaitAndDeletePod(f, c, ns, pvc))
+
+	// 3. delete the PVC, wait for PV to become "Released"
+	By("Deleting the PVC to invoke the reclaim policy.")
+	framework.ExpectNoError(framework.DeletePVCandValidatePV(c, ns, pvc, pv, v1.VolumeReleased))
+}
+
+// Validate pairs of PVs and PVCs, create and verify writer pod, delete PVC and validate
+// PV. Ensure each step succeeds.
+// Note: the PV is deleted in the AfterEach, not here.
+// Note: this func is serialized, we wait for each pod to be deleted before creating the
+//   next pod. Adding concurrency is a TODO item.
+func completeMultiTest(f *framework.Framework, c clientset.Interface, ns string, pvols framework.PVMap, claims framework.PVCMap, expectPhase v1.PersistentVolumePhase) error {
+	var err error
+
+	// 1. verify each PV permits write access to a client pod
+	By("Checking pod has write access to PersistentVolumes")
+	for pvcKey := range claims {
+		pvc, err := c.CoreV1().PersistentVolumeClaims(pvcKey.Namespace).Get(pvcKey.Name, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("error getting pvc %q: %v", pvcKey.Name, err)
+		}
+		if len(pvc.Spec.VolumeName) == 0 {
+			continue // claim is not bound
+		}
+		// sanity test to ensure our maps are in sync
+		_, found := pvols[pvc.Spec.VolumeName]
+		if !found {
+			return fmt.Errorf("internal: pvols map is missing volume %q", pvc.Spec.VolumeName)
+		}
+		// TODO: currently a serialized test of each PV
+		if err = framework.CreateWaitAndDeletePod(f, c, pvcKey.Namespace, pvc); err != nil {
+			return err
+		}
+	}
+
+	// 2. delete each PVC, wait for its bound PV to reach `expectedPhase`
+	By("Deleting PVCs to invoke reclaim policy")
+	if err = framework.DeletePVCandValidatePVGroup(c, ns, pvols, claims, expectPhase); err != nil {
+		return err
+	}
+	return nil
+}
+
+var _ = SIGDescribe("PersistentVolumes", func() {
+
+	// global vars for the Context()s and It()'s below
+	f := framework.NewDefaultFramework("pv")
+	var (
+		c         clientset.Interface
+		ns        string
+		pvConfig  framework.PersistentVolumeConfig
+		pvcConfig framework.PersistentVolumeClaimConfig
+		volLabel  labels.Set
+		selector  *metav1.LabelSelector
+		pv        *v1.PersistentVolume
+		pvc       *v1.PersistentVolumeClaim
+		err       error
+	)
+
+	BeforeEach(func() {
+		c = f.ClientSet
+		ns = f.Namespace.Name
+		// Enforce binding only within test space via selector labels
+		volLabel = labels.Set{framework.VolumeSelectorKey: ns}
+		selector = metav1.SetAsLabelSelector(volLabel)
+	})
+
+	// Testing configurations of a single a PV/PVC pair, multiple evenly paired PVs/PVCs,
+	// and multiple unevenly paired PV/PVCs
+	Describe("NFS", func() {
+
+		var (
+			nfsServerPod *v1.Pod
+			serverIP     string
+		)
+
+		BeforeEach(func() {
+			_, nfsServerPod, serverIP = framework.NewNFSServer(c, ns, []string{"-G", "777", "/exports"})
+			pvConfig = framework.PersistentVolumeConfig{
+				NamePrefix: "nfs-",
+				Labels:     volLabel,
+				PVSource: v1.PersistentVolumeSource{
+					NFS: &v1.NFSVolumeSource{
+						Server:   serverIP,
+						Path:     "/exports",
+						ReadOnly: false,
+					},
+				},
+			}
+			pvcConfig = framework.PersistentVolumeClaimConfig{
+				Annotations: map[string]string{
+					v1.BetaStorageClassAnnotation: "",
+				},
+				Selector: selector,
+			}
+		})
+
+		AfterEach(func() {
+			framework.ExpectNoError(framework.DeletePodWithWait(f, c, nfsServerPod), "AfterEach: Failed to delete pod ", nfsServerPod.Name)
+			pv, pvc = nil, nil
+			pvConfig, pvcConfig = framework.PersistentVolumeConfig{}, framework.PersistentVolumeClaimConfig{}
+		})
+
+		Context("with Single PV - PVC pairs", func() {
+			// Note: this is the only code where the pv is deleted.
+			AfterEach(func() {
+				framework.Logf("AfterEach: Cleaning up test resources.")
+				if errs := framework.PVPVCCleanup(c, ns, pv, pvc); len(errs) > 0 {
+					framework.Failf("AfterEach: Failed to delete PVC and/or PV. Errors: %v", utilerrors.NewAggregate(errs))
+				}
+			})
+
+			// Individual tests follow:
+			//
+			// Create an nfs PV, then a claim that matches the PV, and a pod that
+			// contains the claim. Verify that the PV and PVC bind correctly, and
+			// that the pod can write to the nfs volume.
+			It("should create a non-pre-bound PV and PVC: test write access ", func() {
+				pv, pvc, err = framework.CreatePVPVC(c, pvConfig, pvcConfig, ns, false)
+				Expect(err).NotTo(HaveOccurred())
+				completeTest(f, c, ns, pv, pvc)
+			})
+
+			// Create a claim first, then a nfs PV that matches the claim, and a
+			// pod that contains the claim. Verify that the PV and PVC bind
+			// correctly, and that the pod can write to the nfs volume.
+			It("create a PVC and non-pre-bound PV: test write access", func() {
+				pv, pvc, err = framework.CreatePVCPV(c, pvConfig, pvcConfig, ns, false)
+				Expect(err).NotTo(HaveOccurred())
+				completeTest(f, c, ns, pv, pvc)
+			})
+
+			// Create a claim first, then a pre-bound nfs PV that matches the claim,
+			// and a pod that contains the claim. Verify that the PV and PVC bind
+			// correctly, and that the pod can write to the nfs volume.
+			It("create a PVC and a pre-bound PV: test write access", func() {
+				pv, pvc, err = framework.CreatePVCPV(c, pvConfig, pvcConfig, ns, true)
+				Expect(err).NotTo(HaveOccurred())
+				completeTest(f, c, ns, pv, pvc)
+			})
+
+			// Create a nfs PV first, then a pre-bound PVC that matches the PV,
+			// and a pod that contains the claim. Verify that the PV and PVC bind
+			// correctly, and that the pod can write to the nfs volume.
+			It("create a PV and a pre-bound PVC: test write access", func() {
+				pv, pvc, err = framework.CreatePVPVC(c, pvConfig, pvcConfig, ns, true)
+				Expect(err).NotTo(HaveOccurred())
+				completeTest(f, c, ns, pv, pvc)
+			})
+		})
+
+		// Create multiple pvs and pvcs, all in the same namespace. The PVs-PVCs are
+		// verified to bind, though it's not known in advanced which PV will bind to
+		// which claim. For each pv-pvc pair create a pod that writes to the nfs mount.
+		// Note: when the number of PVs exceeds the number of PVCs the max binding wait
+		//   time will occur for each PV in excess. This is expected but the delta
+		//   should be kept small so that the tests aren't unnecessarily slow.
+		// Note: future tests may wish to incorporate the following:
+		//   a) pre-binding, b) create pvcs before pvs, c) create pvcs and pods
+		//   in different namespaces.
+		Context("with multiple PVs and PVCs all in same ns", func() {
+
+			// define the maximum number of PVs and PVCs supported by these tests
+			const maxNumPVs = 10
+			const maxNumPVCs = 10
+			// scope the pv and pvc maps to be available in the AfterEach
+			// note: these maps are created fresh in CreatePVsPVCs()
+			var pvols framework.PVMap
+			var claims framework.PVCMap
+
+			AfterEach(func() {
+				framework.Logf("AfterEach: deleting %v PVCs and %v PVs...", len(claims), len(pvols))
+				errs := framework.PVPVCMapCleanup(c, ns, pvols, claims)
+				if len(errs) > 0 {
+					errmsg := []string{}
+					for _, e := range errs {
+						errmsg = append(errmsg, e.Error())
+					}
+					framework.Failf("AfterEach: Failed to delete 1 or more PVs/PVCs. Errors: %v", strings.Join(errmsg, "; "))
+				}
+			})
+
+			// Create 2 PVs and 4 PVCs.
+			// Note: PVs are created before claims and no pre-binding
+			It("should create 2 PVs and 4 PVCs: test write access", func() {
+				numPVs, numPVCs := 2, 4
+				pvols, claims, err = framework.CreatePVsPVCs(numPVs, numPVCs, c, ns, pvConfig, pvcConfig)
+				Expect(err).NotTo(HaveOccurred())
+				framework.ExpectNoError(framework.WaitAndVerifyBinds(c, ns, pvols, claims, true))
+				framework.ExpectNoError(completeMultiTest(f, c, ns, pvols, claims, v1.VolumeReleased))
+			})
+
+			// Create 3 PVs and 3 PVCs.
+			// Note: PVs are created before claims and no pre-binding
+			It("should create 3 PVs and 3 PVCs: test write access", func() {
+				numPVs, numPVCs := 3, 3
+				pvols, claims, err = framework.CreatePVsPVCs(numPVs, numPVCs, c, ns, pvConfig, pvcConfig)
+				Expect(err).NotTo(HaveOccurred())
+				framework.ExpectNoError(framework.WaitAndVerifyBinds(c, ns, pvols, claims, true))
+				framework.ExpectNoError(completeMultiTest(f, c, ns, pvols, claims, v1.VolumeReleased))
+			})
+
+			// Create 4 PVs and 2 PVCs.
+			// Note: PVs are created before claims and no pre-binding.
+			It("should create 4 PVs and 2 PVCs: test write access [Slow]", func() {
+				numPVs, numPVCs := 4, 2
+				pvols, claims, err = framework.CreatePVsPVCs(numPVs, numPVCs, c, ns, pvConfig, pvcConfig)
+				Expect(err).NotTo(HaveOccurred())
+				framework.ExpectNoError(framework.WaitAndVerifyBinds(c, ns, pvols, claims, true))
+				framework.ExpectNoError(completeMultiTest(f, c, ns, pvols, claims, v1.VolumeReleased))
+			})
+		})
+
+		// This Context isolates and tests the "Recycle" reclaim behavior.  On deprecation of the
+		// Recycler, this entire context can be removed without affecting the test suite or leaving behind
+		// dead code.
+		Context("when invoking the Recycle reclaim policy", func() {
+			BeforeEach(func() {
+				pvConfig.ReclaimPolicy = v1.PersistentVolumeReclaimRecycle
+				pv, pvc, err = framework.CreatePVPVC(c, pvConfig, pvcConfig, ns, false)
+				Expect(err).NotTo(HaveOccurred(), "BeforeEach: Failed to create PV/PVC")
+				framework.ExpectNoError(framework.WaitOnPVandPVC(c, ns, pv, pvc), "BeforeEach: WaitOnPVandPVC failed")
+			})
+
+			AfterEach(func() {
+				framework.Logf("AfterEach: Cleaning up test resources.")
+				if errs := framework.PVPVCCleanup(c, ns, pv, pvc); len(errs) > 0 {
+					framework.Failf("AfterEach: Failed to delete PVC and/or PV. Errors: %v", utilerrors.NewAggregate(errs))
+				}
+			})
+
+			// This It() tests a scenario where a PV is written to by a Pod, recycled, then the volume checked
+			// for files. If files are found, the checking Pod fails, failing the test.  Otherwise, the pod
+			// (and test) succeed.
+			It("should test that a PV becomes Available and is clean after the PVC is deleted.", func() {
+				By("Writing to the volume.")
+				pod := framework.MakeWritePod(ns, pvc)
+				pod, err = c.CoreV1().Pods(ns).Create(pod)
+				Expect(err).NotTo(HaveOccurred())
+				framework.ExpectNoError(framework.WaitForPodSuccessInNamespace(c, pod.Name, ns))
+
+				By("Deleting the claim")
+				framework.ExpectNoError(framework.DeletePVCandValidatePV(c, ns, pvc, pv, v1.VolumeAvailable))
+
+				By("Re-mounting the volume.")
+				pvc = framework.MakePersistentVolumeClaim(pvcConfig, ns)
+				pvc, err = framework.CreatePVC(c, ns, pvc)
+				Expect(err).NotTo(HaveOccurred())
+				framework.ExpectNoError(framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, ns, pvc.Name, 2*time.Second, 60*time.Second), "Failed to reach 'Bound' for PVC ", pvc.Name)
+
+				// If a file is detected in /mnt, fail the pod and do not restart it.
+				By("Verifying the mount has been cleaned.")
+				mount := pod.Spec.Containers[0].VolumeMounts[0].MountPath
+				pod = framework.MakePod(ns, []*v1.PersistentVolumeClaim{pvc}, true, fmt.Sprintf("[ $(ls -A %s | wc -l) -eq 0 ] && exit 0 || exit 1", mount))
+				pod, err = c.CoreV1().Pods(ns).Create(pod)
+				Expect(err).NotTo(HaveOccurred())
+				framework.ExpectNoError(framework.WaitForPodSuccessInNamespace(c, pod.Name, ns))
+				framework.Logf("Pod exited without failure; the volume has been recycled.")
+			})
+		})
+	})
+})

vendor/k8s.io/kubernetes/test/e2e/storage/pv_reclaimpolicy.go

@@ -0,0 +1,196 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"strconv"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+var _ = SIGDescribe("PersistentVolumes [Feature:ReclaimPolicy]", func() {
+	f := framework.NewDefaultFramework("persistentvolumereclaim")
+	var (
+		c          clientset.Interface
+		ns         string
+		volumePath string
+		pv         *v1.PersistentVolume
+		pvc        *v1.PersistentVolumeClaim
+	)
+
+	BeforeEach(func() {
+		c = f.ClientSet
+		ns = f.Namespace.Name
+		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout))
+	})
+
+	SIGDescribe("persistentvolumereclaim:vsphere", func() {
+		BeforeEach(func() {
+			framework.SkipUnlessProviderIs("vsphere")
+			pv = nil
+			pvc = nil
+			volumePath = ""
+		})
+
+		AfterEach(func() {
+			vsp, err := vsphere.GetVSphere()
+			Expect(err).NotTo(HaveOccurred())
+			testCleanupVSpherePersistentVolumeReclaim(vsp, c, ns, volumePath, pv, pvc)
+		})
+
+		/*
+			This test verifies persistent volume should be deleted when reclaimPolicy on the PV is set to delete and
+			associated claim is deleted
+
+			Test Steps:
+			1. Create vmdk
+			2. Create PV Spec with volume path set to VMDK file created in Step-1, and PersistentVolumeReclaimPolicy is set to Delete
+			3. Create PVC with the storage request set to PV's storage capacity.
+			4. Wait for PV and PVC to bound.
+			5. Delete PVC
+			6. Verify PV is deleted automatically.
+		*/
+		It("should delete persistent volume when reclaimPolicy set to delete and associated claim is deleted", func() {
+			vsp, err := vsphere.GetVSphere()
+			Expect(err).NotTo(HaveOccurred())
+
+			volumePath, pv, pvc, err = testSetupVSpherePersistentVolumeReclaim(vsp, c, ns, v1.PersistentVolumeReclaimDelete)
+			Expect(err).NotTo(HaveOccurred())
+
+			deletePVCAfterBind(c, ns, pvc, pv)
+			pvc = nil
+
+			By("verify pv is deleted")
+			err = framework.WaitForPersistentVolumeDeleted(c, pv.Name, 3*time.Second, 300*time.Second)
+			Expect(err).NotTo(HaveOccurred())
+
+			pv = nil
+			volumePath = ""
+		})
+
+		/*
+			This test Verify persistent volume should be retained when reclaimPolicy on the PV is set to retain
+			and associated claim is deleted
+
+			Test Steps:
+			1. Create vmdk
+			2. Create PV Spec with volume path set to VMDK file created in Step-1, and PersistentVolumeReclaimPolicy is set to Retain
+			3. Create PVC with the storage request set to PV's storage capacity.
+			4. Wait for PV and PVC to bound.
+			5. Write some content in the volume.
+			6. Delete PVC
+			7. Verify PV is retained.
+			8. Delete retained PV.
+			9. Create PV Spec with the same volume path used in step 2.
+			10. Create PVC with the storage request set to PV's storage capacity.
+			11. Created POD using PVC created in Step 10 and verify volume content is matching.
+		*/
+
+		It("should retain persistent volume when reclaimPolicy set to retain when associated claim is deleted", func() {
+			var volumeFileContent = "hello from vsphere cloud provider, Random Content is :" + strconv.FormatInt(time.Now().UnixNano(), 10)
+			vsp, err := vsphere.GetVSphere()
+			Expect(err).NotTo(HaveOccurred())
+
+			volumePath, pv, pvc, err = testSetupVSpherePersistentVolumeReclaim(vsp, c, ns, v1.PersistentVolumeReclaimRetain)
+			Expect(err).NotTo(HaveOccurred())
+
+			writeContentToVSpherePV(c, pvc, volumeFileContent)
+
+			By("Delete PVC")
+			framework.ExpectNoError(framework.DeletePersistentVolumeClaim(c, pvc.Name, ns), "Failed to delete PVC ", pvc.Name)
+			pvc = nil
+
+			By("Verify PV is retained")
+			framework.Logf("Waiting for PV %v to become Released", pv.Name)
+			err = framework.WaitForPersistentVolumePhase(v1.VolumeReleased, c, pv.Name, 3*time.Second, 300*time.Second)
+			Expect(err).NotTo(HaveOccurred())
+			framework.ExpectNoError(framework.DeletePersistentVolume(c, pv.Name), "Failed to delete PV ", pv.Name)
+
+			By("Creating the PV for same volume path")
+			pv = getVSpherePersistentVolumeSpec(volumePath, v1.PersistentVolumeReclaimRetain, nil)
+			pv, err = c.CoreV1().PersistentVolumes().Create(pv)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("creating the pvc")
+			pvc = getVSpherePersistentVolumeClaimSpec(ns, nil)
+			pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Create(pvc)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("wait for the pv and pvc to bind")
+			framework.ExpectNoError(framework.WaitOnPVandPVC(c, ns, pv, pvc))
+			verifyContentOfVSpherePV(c, pvc, volumeFileContent)
+
+		})
+	})
+})
+
+// Test Setup for persistentvolumereclaim tests for vSphere Provider
+func testSetupVSpherePersistentVolumeReclaim(vsp *vsphere.VSphere, c clientset.Interface, ns string, persistentVolumeReclaimPolicy v1.PersistentVolumeReclaimPolicy) (volumePath string, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim, err error) {
+	By("running testSetupVSpherePersistentVolumeReclaim")
+	By("creating vmdk")
+	volumePath, err = createVSphereVolume(vsp, nil)
+	if err != nil {
+		return
+	}
+	By("creating the pv")
+	pv = getVSpherePersistentVolumeSpec(volumePath, persistentVolumeReclaimPolicy, nil)
+	pv, err = c.CoreV1().PersistentVolumes().Create(pv)
+	if err != nil {
+		return
+	}
+	By("creating the pvc")
+	pvc = getVSpherePersistentVolumeClaimSpec(ns, nil)
+	pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Create(pvc)
+	return
+}
+
+// Test Cleanup for persistentvolumereclaim tests for vSphere Provider
+func testCleanupVSpherePersistentVolumeReclaim(vsp *vsphere.VSphere, c clientset.Interface, ns string, volumePath string, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) {
+	By("running testCleanupVSpherePersistentVolumeReclaim")
+	if len(volumePath) > 0 {
+		vsp.DeleteVolume(volumePath)
+	}
+	if pv != nil {
+		framework.ExpectNoError(framework.DeletePersistentVolume(c, pv.Name), "Failed to delete PV ", pv.Name)
+	}
+	if pvc != nil {
+		framework.ExpectNoError(framework.DeletePersistentVolumeClaim(c, pvc.Name, ns), "Failed to delete PVC ", pvc.Name)
+	}
+}
+
+// func to wait until PV and PVC bind and once bind completes, delete the PVC
+func deletePVCAfterBind(c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) {
+	var err error
+
+	By("wait for the pv and pvc to bind")
+	framework.ExpectNoError(framework.WaitOnPVandPVC(c, ns, pv, pvc))
+
+	By("delete pvc")
+	framework.ExpectNoError(framework.DeletePersistentVolumeClaim(c, pvc.Name, ns), "Failed to delete PVC ", pvc.Name)
+	pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Get(pvc.Name, metav1.GetOptions{})
+	if !apierrs.IsNotFound(err) {
+		Expect(err).NotTo(HaveOccurred())
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/storage/pvc_label_selector.go

@@ -0,0 +1,150 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+/*
+   This is a function test for Selector-Label Volume Binding Feature
+   Test verifies volume with the matching label is bounded with the PVC.
+
+   Test Steps
+   ----------
+   1. Create VMDK.
+   2. Create pv with lable volume-type:ssd, volume path set to vmdk created in previous step, and PersistentVolumeReclaimPolicy is set to Delete.
+   3. Create PVC (pvc_vvol) with label selector to match with volume-type:vvol
+   4. Create PVC (pvc_ssd) with label selector to match with volume-type:ssd
+   5. Wait and verify pvc_ssd is bound with PV.
+   6. Verify Status of pvc_vvol is still pending.
+   7. Delete pvc_ssd.
+   8. verify associated pv is also deleted.
+   9. delete pvc_vvol
+
+*/
+var _ = SIGDescribe("PersistentVolumes [Feature:LabelSelector]", func() {
+	f := framework.NewDefaultFramework("pvclabelselector")
+	var (
+		c          clientset.Interface
+		ns         string
+		pv_ssd     *v1.PersistentVolume
+		pvc_ssd    *v1.PersistentVolumeClaim
+		pvc_vvol   *v1.PersistentVolumeClaim
+		volumePath string
+		ssdlabels  map[string]string
+		vvollabels map[string]string
+		err        error
+	)
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("vsphere")
+		c = f.ClientSet
+		ns = f.Namespace.Name
+		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout))
+		ssdlabels = make(map[string]string)
+		ssdlabels["volume-type"] = "ssd"
+		vvollabels = make(map[string]string)
+		vvollabels["volume-type"] = "vvol"
+
+	})
+
+	SIGDescribe("Selector-Label Volume Binding:vsphere", func() {
+		AfterEach(func() {
+			By("Running clean up actions")
+			if framework.ProviderIs("vsphere") {
+				testCleanupVSpherePVClabelselector(c, ns, volumePath, pv_ssd, pvc_ssd, pvc_vvol)
+			}
+		})
+		It("should bind volume with claim for given label", func() {
+			volumePath, pv_ssd, pvc_ssd, pvc_vvol, err = testSetupVSpherePVClabelselector(c, ns, ssdlabels, vvollabels)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("wait for the pvc_ssd to bind with pv_ssd")
+			framework.ExpectNoError(framework.WaitOnPVandPVC(c, ns, pv_ssd, pvc_ssd))
+
+			By("Verify status of pvc_vvol is pending")
+			err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimPending, c, ns, pvc_vvol.Name, 3*time.Second, 300*time.Second)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("delete pvc_ssd")
+			framework.ExpectNoError(framework.DeletePersistentVolumeClaim(c, pvc_ssd.Name, ns), "Failed to delete PVC ", pvc_ssd.Name)
+
+			By("verify pv_ssd is deleted")
+			err = framework.WaitForPersistentVolumeDeleted(c, pv_ssd.Name, 3*time.Second, 300*time.Second)
+			Expect(err).NotTo(HaveOccurred())
+			volumePath = ""
+
+			By("delete pvc_vvol")
+			framework.ExpectNoError(framework.DeletePersistentVolumeClaim(c, pvc_vvol.Name, ns), "Failed to delete PVC ", pvc_vvol.Name)
+		})
+	})
+})
+
+func testSetupVSpherePVClabelselector(c clientset.Interface, ns string, ssdlabels map[string]string, vvollabels map[string]string) (volumePath string, pv_ssd *v1.PersistentVolume, pvc_ssd *v1.PersistentVolumeClaim, pvc_vvol *v1.PersistentVolumeClaim, err error) {
+	volumePath = ""
+	By("creating vmdk")
+	vsp, err := vsphere.GetVSphere()
+	Expect(err).NotTo(HaveOccurred())
+	volumePath, err = createVSphereVolume(vsp, nil)
+	if err != nil {
+		return
+	}
+
+	By("creating the pv with lable volume-type:ssd")
+	pv_ssd = getVSpherePersistentVolumeSpec(volumePath, v1.PersistentVolumeReclaimDelete, ssdlabels)
+	pv_ssd, err = c.CoreV1().PersistentVolumes().Create(pv_ssd)
+	if err != nil {
+		return
+	}
+
+	By("creating pvc with label selector to match with volume-type:vvol")
+	pvc_vvol = getVSpherePersistentVolumeClaimSpec(ns, vvollabels)
+	pvc_vvol, err = c.CoreV1().PersistentVolumeClaims(ns).Create(pvc_vvol)
+	if err != nil {
+		return
+	}
+
+	By("creating pvc with label selector to match with volume-type:ssd")
+	pvc_ssd = getVSpherePersistentVolumeClaimSpec(ns, ssdlabels)
+	pvc_ssd, err = c.CoreV1().PersistentVolumeClaims(ns).Create(pvc_ssd)
+	return
+}
+
+func testCleanupVSpherePVClabelselector(c clientset.Interface, ns string, volumePath string, pv_ssd *v1.PersistentVolume, pvc_ssd *v1.PersistentVolumeClaim, pvc_vvol *v1.PersistentVolumeClaim) {
+	By("running testCleanupVSpherePVClabelselector")
+	if len(volumePath) > 0 {
+		vsp, err := vsphere.GetVSphere()
+		Expect(err).NotTo(HaveOccurred())
+		vsp.DeleteVolume(volumePath)
+	}
+	if pvc_ssd != nil {
+		framework.ExpectNoError(framework.DeletePersistentVolumeClaim(c, pvc_ssd.Name, ns), "Failed to delete PVC ", pvc_ssd.Name)
+	}
+	if pvc_vvol != nil {
+		framework.ExpectNoError(framework.DeletePersistentVolumeClaim(c, pvc_vvol.Name, ns), "Failed to delete PVC ", pvc_vvol.Name)
+	}
+	if pv_ssd != nil {
+		framework.ExpectNoError(framework.DeletePersistentVolume(c, pv_ssd.Name), "Faled to delete PV ", pv_ssd.Name)
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/storage/volume_io.go

@@ -0,0 +1,449 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+ * This test checks that the plugin VolumeSources are working when pseudo-streaming
+ * various write sizes to mounted files. Note that the plugin is defined inline in
+ * the pod spec, not via a persistent volume and claim.
+ *
+ * These tests work only when privileged containers are allowed, exporting various
+ * filesystems (NFS, GlusterFS, ...) usually needs some mounting or other privileged
+ * magic in the server pod. Note that the server containers are for testing purposes
+ * only and should not be used in production.
+ */
+
+package storage
+
+import (
+	"fmt"
+	"math"
+	"path"
+	"strconv"
+	"strings"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+const (
+	minFileSize = 1 * framework.MiB
+
+	fileSizeSmall  = 1 * framework.MiB
+	fileSizeMedium = 100 * framework.MiB
+	fileSizeLarge  = 1 * framework.GiB
+)
+
+// MD5 hashes of the test file corresponding to each file size.
+// Test files are generated in testVolumeIO()
+// If test file generation algorithm changes, these must be recomputed.
+var md5hashes = map[int64]string{
+	fileSizeSmall:  "5c34c2813223a7ca05a3c2f38c0d1710",
+	fileSizeMedium: "f2fa202b1ffeedda5f3a58bd1ae81104",
+	fileSizeLarge:  "8d763edc71bd16217664793b5a15e403",
+}
+
+// Return the plugin's client pod spec. Use an InitContainer to setup the file i/o test env.
+func makePodSpec(config framework.VolumeTestConfig, dir, initCmd string, volsrc v1.VolumeSource, podSecContext *v1.PodSecurityContext) *v1.Pod {
+	volName := fmt.Sprintf("%s-%s", config.Prefix, "io-volume")
+
+	return &v1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: config.Prefix + "-io-client",
+			Labels: map[string]string{
+				"role": config.Prefix + "-io-client",
+			},
+		},
+		Spec: v1.PodSpec{
+			InitContainers: []v1.Container{
+				{
+					Name:  config.Prefix + "-io-init",
+					Image: framework.BusyBoxImage,
+					Command: []string{
+						"/bin/sh",
+						"-c",
+						initCmd,
+					},
+					VolumeMounts: []v1.VolumeMount{
+						{
+							Name:      volName,
+							MountPath: dir,
+						},
+					},
+				},
+			},
+			Containers: []v1.Container{
+				{
+					Name:  config.Prefix + "-io-client",
+					Image: framework.BusyBoxImage,
+					Command: []string{
+						"/bin/sh",
+						"-c",
+						"sleep 3600", // keep pod alive until explicitly deleted
+					},
+					VolumeMounts: []v1.VolumeMount{
+						{
+							Name:      volName,
+							MountPath: dir,
+						},
+					},
+				},
+			},
+			SecurityContext: podSecContext,
+			Volumes: []v1.Volume{
+				{
+					Name:         volName,
+					VolumeSource: volsrc,
+				},
+			},
+			RestartPolicy: v1.RestartPolicyNever, // want pod to fail if init container fails
+		},
+	}
+}
+
+// Write `fsize` bytes to `fpath` in the pod, using dd and the `dd_input` file.
+func writeToFile(pod *v1.Pod, fpath, dd_input string, fsize int64) error {
+	By(fmt.Sprintf("writing %d bytes to test file %s", fsize, fpath))
+	loopCnt := fsize / minFileSize
+	writeCmd := fmt.Sprintf("i=0; while [ $i -lt %d ]; do dd if=%s bs=%d >>%s 2>/dev/null; let i+=1; done", loopCnt, dd_input, minFileSize, fpath)
+	_, err := podExec(pod, writeCmd)
+
+	return err
+}
+
+// Verify that the test file is the expected size and contains the expected content.
+func verifyFile(pod *v1.Pod, fpath string, expectSize int64, dd_input string) error {
+	By("verifying file size")
+	rtnstr, err := podExec(pod, fmt.Sprintf("stat -c %%s %s", fpath))
+	if err != nil || rtnstr == "" {
+		return fmt.Errorf("unable to get file size via `stat %s`: %v", fpath, err)
+	}
+	size, err := strconv.Atoi(strings.TrimSuffix(rtnstr, "\n"))
+	if err != nil {
+		return fmt.Errorf("unable to convert string %q to int: %v", rtnstr, err)
+	}
+	if int64(size) != expectSize {
+		return fmt.Errorf("size of file %s is %d, expected %d", fpath, size, expectSize)
+	}
+
+	By("verifying file hash")
+	rtnstr, err = podExec(pod, fmt.Sprintf("md5sum %s | cut -d' ' -f1", fpath))
+	if err != nil {
+		return fmt.Errorf("unable to test file hash via `md5sum %s`: %v", fpath, err)
+	}
+	actualHash := strings.TrimSuffix(rtnstr, "\n")
+	expectedHash, ok := md5hashes[expectSize]
+	if !ok {
+		return fmt.Errorf("File hash is unknown for file size %d. Was a new file size added to the test suite?",
+			expectSize)
+	}
+	if actualHash != expectedHash {
+		return fmt.Errorf("MD5 hash is incorrect for file %s with size %d. Expected: `%s`; Actual: `%s`",
+			fpath, expectSize, expectedHash, actualHash)
+	}
+
+	return nil
+}
+
+// Delete `fpath` to save some disk space on host. Delete errors are logged but ignored.
+func deleteFile(pod *v1.Pod, fpath string) {
+	By(fmt.Sprintf("deleting test file %s...", fpath))
+	_, err := podExec(pod, fmt.Sprintf("rm -f %s", fpath))
+	if err != nil {
+		// keep going, the test dir will be deleted when the volume is unmounted
+		framework.Logf("unable to delete test file %s: %v\nerror ignored, continuing test", fpath, err)
+	}
+}
+
+// Create the client pod and create files of the sizes passed in by the `fsizes` parameter. Delete the
+// client pod and the new files when done.
+// Note: the file name is appended to "/opt/<Prefix>/<namespace>", eg. "/opt/nfs/e2e-.../<file>".
+// Note: nil can be passed for the podSecContext parm, in which case it is ignored.
+// Note: `fsizes` values are enforced to each be at least `minFileSize` and a multiple of `minFileSize`
+//   bytes.
+func testVolumeIO(f *framework.Framework, cs clientset.Interface, config framework.VolumeTestConfig, volsrc v1.VolumeSource, podSecContext *v1.PodSecurityContext, file string, fsizes []int64) (err error) {
+	dir := path.Join("/opt", config.Prefix, config.Namespace)
+	dd_input := path.Join(dir, "dd_if")
+	writeBlk := strings.Repeat("abcdefghijklmnopqrstuvwxyz123456", 32) // 1KiB value
+	loopCnt := minFileSize / int64(len(writeBlk))
+	// initContainer cmd to create and fill dd's input file. The initContainer is used to create
+	// the `dd` input file which is currently 1MiB. Rather than store a 1MiB go value, a loop is
+	// used to create a 1MiB file in the target directory.
+	initCmd := fmt.Sprintf("i=0; while [ $i -lt %d ]; do echo -n %s >>%s; let i+=1; done", loopCnt, writeBlk, dd_input)
+
+	clientPod := makePodSpec(config, dir, initCmd, volsrc, podSecContext)
+
+	By(fmt.Sprintf("starting %s", clientPod.Name))
+	podsNamespacer := cs.CoreV1().Pods(config.Namespace)
+	clientPod, err = podsNamespacer.Create(clientPod)
+	if err != nil {
+		return fmt.Errorf("failed to create client pod %q: %v", clientPod.Name, err)
+	}
+	defer func() {
+		// note the test dir will be removed when the kubelet unmounts it
+		By(fmt.Sprintf("deleting client pod %q...", clientPod.Name))
+		e := framework.DeletePodWithWait(f, cs, clientPod)
+		if e != nil {
+			framework.Logf("client pod failed to delete: %v", e)
+			if err == nil { // delete err is returned if err is not set
+				err = e
+			}
+		}
+	}()
+
+	err = framework.WaitForPodRunningInNamespace(cs, clientPod)
+	if err != nil {
+		return fmt.Errorf("client pod %q not running: %v", clientPod.Name, err)
+	}
+
+	// create files of the passed-in file sizes and verify test file size and content
+	for _, fsize := range fsizes {
+		// file sizes must be a multiple of `minFileSize`
+		if math.Mod(float64(fsize), float64(minFileSize)) != 0 {
+			fsize = fsize/minFileSize + minFileSize
+		}
+		fpath := path.Join(dir, fmt.Sprintf("%s-%d", file, fsize))
+		if err = writeToFile(clientPod, fpath, dd_input, fsize); err != nil {
+			return err
+		}
+		if err = verifyFile(clientPod, fpath, fsize, dd_input); err != nil {
+			return err
+		}
+		deleteFile(clientPod, fpath)
+	}
+
+	return
+}
+
+// These tests need privileged containers which are disabled by default.
+// TODO: support all of the plugins tested in storage/volumes.go
+var _ = SIGDescribe("Volume plugin streaming [Slow]", func() {
+	f := framework.NewDefaultFramework("volume-io")
+	var (
+		config    framework.VolumeTestConfig
+		cs        clientset.Interface
+		ns        string
+		serverIP  string
+		serverPod *v1.Pod
+		volSource v1.VolumeSource
+	)
+
+	BeforeEach(func() {
+		cs = f.ClientSet
+		ns = f.Namespace.Name
+	})
+
+	////////////////////////////////////////////////////////////////////////
+	// NFS
+	////////////////////////////////////////////////////////////////////////
+	Describe("NFS", func() {
+		testFile := "nfs_io_test"
+		// client pod uses selinux
+		podSec := v1.PodSecurityContext{
+			SELinuxOptions: &v1.SELinuxOptions{
+				Level: "s0:c0,c1",
+			},
+		}
+
+		BeforeEach(func() {
+			config, serverPod, serverIP = framework.NewNFSServer(cs, ns, []string{})
+			volSource = v1.VolumeSource{
+				NFS: &v1.NFSVolumeSource{
+					Server:   serverIP,
+					Path:     "/",
+					ReadOnly: false,
+				},
+			}
+		})
+
+		AfterEach(func() {
+			framework.Logf("AfterEach: deleting NFS server pod %q...", serverPod.Name)
+			err := framework.DeletePodWithWait(f, cs, serverPod)
+			Expect(err).NotTo(HaveOccurred(), "AfterEach: NFS server pod failed to delete")
+		})
+
+		It("should write files of various sizes, verify size, validate content", func() {
+			fileSizes := []int64{fileSizeSmall, fileSizeMedium, fileSizeLarge}
+			err := testVolumeIO(f, cs, config, volSource, &podSec, testFile, fileSizes)
+			Expect(err).NotTo(HaveOccurred())
+		})
+	})
+
+	////////////////////////////////////////////////////////////////////////
+	// Gluster
+	////////////////////////////////////////////////////////////////////////
+	Describe("GlusterFS", func() {
+		var name string
+		testFile := "gluster_io_test"
+
+		BeforeEach(func() {
+			framework.SkipUnlessNodeOSDistroIs("gci")
+			// create gluster server and endpoints
+			config, serverPod, serverIP = framework.NewGlusterfsServer(cs, ns)
+			name = config.Prefix + "-server"
+			volSource = v1.VolumeSource{
+				Glusterfs: &v1.GlusterfsVolumeSource{
+					EndpointsName: name,
+					// 'test_vol' comes from test/images/volumes-tester/gluster/run_gluster.sh
+					Path:     "test_vol",
+					ReadOnly: false,
+				},
+			}
+		})
+
+		AfterEach(func() {
+			framework.Logf("AfterEach: deleting Gluster endpoints %q...", name)
+			epErr := cs.CoreV1().Endpoints(ns).Delete(name, nil)
+			framework.Logf("AfterEach: deleting Gluster server pod %q...", serverPod.Name)
+			err := framework.DeletePodWithWait(f, cs, serverPod)
+			if epErr != nil || err != nil {
+				if epErr != nil {
+					framework.Logf("AfterEach: Gluster delete endpoints failed: %v", err)
+				}
+				if err != nil {
+					framework.Logf("AfterEach: Gluster server pod delete failed: %v", err)
+				}
+				framework.Failf("AfterEach: cleanup failed")
+			}
+		})
+
+		It("should write files of various sizes, verify size, validate content", func() {
+			fileSizes := []int64{fileSizeSmall, fileSizeMedium}
+			err := testVolumeIO(f, cs, config, volSource, nil /*no secContext*/, testFile, fileSizes)
+			Expect(err).NotTo(HaveOccurred())
+		})
+	})
+
+	////////////////////////////////////////////////////////////////////////
+	// iSCSI
+	// The iscsiadm utility and iscsi target kernel modules must be installed on all nodes.
+	////////////////////////////////////////////////////////////////////////
+	Describe("iSCSI [Feature:Volumes]", func() {
+		testFile := "iscsi_io_test"
+
+		BeforeEach(func() {
+			config, serverPod, serverIP = framework.NewISCSIServer(cs, ns)
+			volSource = v1.VolumeSource{
+				ISCSI: &v1.ISCSIVolumeSource{
+					TargetPortal: serverIP + ":3260",
+					// from test/images/volumes-tester/iscsi/initiatorname.iscsi
+					IQN:      "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c",
+					Lun:      0,
+					FSType:   "ext2",
+					ReadOnly: false,
+				},
+			}
+		})
+
+		AfterEach(func() {
+			framework.Logf("AfterEach: deleting iSCSI server pod %q...", serverPod.Name)
+			err := framework.DeletePodWithWait(f, cs, serverPod)
+			Expect(err).NotTo(HaveOccurred(), "AfterEach: iSCSI server pod failed to delete")
+		})
+
+		It("should write files of various sizes, verify size, validate content", func() {
+			fileSizes := []int64{fileSizeSmall, fileSizeMedium}
+			fsGroup := int64(1234)
+			podSec := v1.PodSecurityContext{
+				FSGroup: &fsGroup,
+			}
+			err := testVolumeIO(f, cs, config, volSource, &podSec, testFile, fileSizes)
+			Expect(err).NotTo(HaveOccurred())
+		})
+	})
+
+	////////////////////////////////////////////////////////////////////////
+	// Ceph RBD
+	////////////////////////////////////////////////////////////////////////
+	Describe("Ceph-RBD [Feature:Volumes]", func() {
+		var (
+			secret *v1.Secret
+			name   string
+		)
+		testFile := "ceph-rbd_io_test"
+
+		BeforeEach(func() {
+			config, serverPod, serverIP = framework.NewRBDServer(cs, ns)
+			name = config.Prefix + "-server"
+
+			// create server secret
+			secret = &v1.Secret{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "Secret",
+					APIVersion: "v1",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: name,
+				},
+				Data: map[string][]byte{
+					// from test/images/volumes-tester/rbd/keyring
+					"key": []byte("AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA=="),
+				},
+				Type: "kubernetes.io/rbd",
+			}
+			var err error
+			secret, err = cs.CoreV1().Secrets(ns).Create(secret)
+			Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("BeforeEach: failed to create secret %q for Ceph-RBD: %v", name, err))
+
+			volSource = v1.VolumeSource{
+				RBD: &v1.RBDVolumeSource{
+					CephMonitors: []string{serverIP},
+					RBDPool:      "rbd",
+					RBDImage:     "foo",
+					RadosUser:    "admin",
+					SecretRef: &v1.LocalObjectReference{
+						Name: name,
+					},
+					FSType:   "ext2",
+					ReadOnly: true,
+				},
+			}
+		})
+
+		AfterEach(func() {
+			framework.Logf("AfterEach: deleting Ceph-RDB server secret %q...", name)
+			secErr := cs.CoreV1().Secrets(ns).Delete(name, &metav1.DeleteOptions{})
+			framework.Logf("AfterEach: deleting Ceph-RDB server pod %q...", serverPod.Name)
+			err := framework.DeletePodWithWait(f, cs, serverPod)
+			if secErr != nil || err != nil {
+				if secErr != nil {
+					framework.Logf("AfterEach: Ceph-RDB delete secret failed: %v", err)
+				}
+				if err != nil {
+					framework.Logf("AfterEach: Ceph-RDB server pod delete failed: %v", err)
+				}
+				framework.Failf("AfterEach: cleanup failed")
+			}
+		})
+
+		It("should write files of various sizes, verify size, validate content", func() {
+			fileSizes := []int64{fileSizeSmall, fileSizeMedium}
+			fsGroup := int64(1234)
+			podSec := v1.PodSecurityContext{
+				FSGroup: &fsGroup,
+			}
+			err := testVolumeIO(f, cs, config, volSource, &podSec, testFile, fileSizes)
+			Expect(err).NotTo(HaveOccurred())
+		})
+	})
+})

vendor/k8s.io/kubernetes/test/e2e/storage/volume_metrics.go

@@ -0,0 +1,230 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"fmt"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/metrics"
+)
+
+// This test needs to run in serial because other tests could interfere
+// with metrics being tested here.
+var _ = SIGDescribe("[Serial] Volume metrics", func() {
+	var (
+		c              clientset.Interface
+		ns             string
+		pvc            *v1.PersistentVolumeClaim
+		metricsGrabber *metrics.MetricsGrabber
+	)
+	f := framework.NewDefaultFramework("pv")
+
+	BeforeEach(func() {
+		c = f.ClientSet
+		ns = f.Namespace.Name
+		framework.SkipUnlessProviderIs("gce", "gke", "aws")
+		defaultScName := getDefaultStorageClassName(c)
+		verifyDefaultStorageClass(c, defaultScName, true)
+
+		test := storageClassTest{
+			name:      "default",
+			claimSize: "2Gi",
+		}
+
+		pvc = newClaim(test, ns, "default")
+		var err error
+		metricsGrabber, err = metrics.NewMetricsGrabber(c, nil, true, false, true, false, false)
+
+		if err != nil {
+			framework.Failf("Error creating metrics grabber : %v", err)
+		}
+
+		if !metricsGrabber.HasRegisteredMaster() {
+			framework.Skipf("Environment does not support getting controller-manager metrics - skipping")
+		}
+	})
+
+	AfterEach(func() {
+		framework.DeletePersistentVolumeClaim(c, pvc.Name, pvc.Namespace)
+	})
+
+	It("should create prometheus metrics for volume provisioning and attach/detach", func() {
+		var err error
+
+		controllerMetrics, err := metricsGrabber.GrabFromControllerManager()
+
+		Expect(err).NotTo(HaveOccurred(), "Error getting c-m metrics : %v", err)
+
+		storageOpMetrics := getControllerStorageMetrics(controllerMetrics)
+
+		pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(pvc)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(pvc).ToNot(Equal(nil))
+
+		claims := []*v1.PersistentVolumeClaim{pvc}
+
+		pod := framework.MakePod(ns, claims, false, "")
+		pod, err = c.CoreV1().Pods(ns).Create(pod)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = framework.WaitForPodRunningInNamespace(c, pod)
+		framework.ExpectNoError(framework.WaitForPodRunningInNamespace(c, pod), "Error starting pod ", pod.Name)
+
+		framework.Logf("Deleting pod %q/%q", pod.Namespace, pod.Name)
+		framework.ExpectNoError(framework.DeletePodWithWait(f, c, pod))
+
+		backoff := wait.Backoff{
+			Duration: 10 * time.Second,
+			Factor:   1.2,
+			Steps:    3,
+		}
+
+		updatedStorageMetrics := make(map[string]int64)
+
+		waitErr := wait.ExponentialBackoff(backoff, func() (bool, error) {
+			updatedMetrics, err := metricsGrabber.GrabFromControllerManager()
+
+			if err != nil {
+				framework.Logf("Error fetching controller-manager metrics")
+				return false, err
+			}
+			updatedStorageMetrics = getControllerStorageMetrics(updatedMetrics)
+			if len(updatedStorageMetrics) == 0 {
+				framework.Logf("Volume metrics not collected yet, going to retry")
+				return false, nil
+			}
+			return true, nil
+		})
+		Expect(waitErr).NotTo(HaveOccurred(), "Error fetching storage c-m metrics : %v", waitErr)
+
+		volumeOperations := []string{"volume_provision", "volume_detach", "volume_attach"}
+
+		for _, volumeOp := range volumeOperations {
+			verifyMetricCount(storageOpMetrics, updatedStorageMetrics, volumeOp)
+		}
+	})
+
+	It("should create volume metrics with the correct PVC ref", func() {
+		var err error
+		pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(pvc)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(pvc).ToNot(Equal(nil))
+
+		claims := []*v1.PersistentVolumeClaim{pvc}
+		pod := framework.MakePod(ns, claims, false, "")
+		pod, err = c.CoreV1().Pods(ns).Create(pod)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = framework.WaitForPodRunningInNamespace(c, pod)
+		framework.ExpectNoError(framework.WaitForPodRunningInNamespace(c, pod), "Error starting pod ", pod.Name)
+
+		pod, err = c.CoreV1().Pods(ns).Get(pod.Name, metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred())
+
+		// Wait for `VolumeStatsAggPeriod' to grab metrics
+		time.Sleep(1 * time.Minute)
+
+		// Grab kubelet metrics from the node the pod was scheduled on
+		kubeMetrics, err := metricsGrabber.GrabFromKubelet(pod.Spec.NodeName)
+		Expect(err).NotTo(HaveOccurred(), "Error getting kubelet metrics : %v", err)
+
+		framework.Logf("Deleting pod %q/%q", pod.Namespace, pod.Name)
+		framework.ExpectNoError(framework.DeletePodWithWait(f, c, pod))
+
+		// Verify volume stat metrics were collected for the referenced PVC
+		volumeStatKeys := []string{
+			kubeletmetrics.VolumeStatsUsedBytesKey,
+			kubeletmetrics.VolumeStatsCapacityBytesKey,
+			kubeletmetrics.VolumeStatsAvailableBytesKey,
+			kubeletmetrics.VolumeStatsUsedBytesKey,
+			kubeletmetrics.VolumeStatsInodesFreeKey,
+			kubeletmetrics.VolumeStatsInodesUsedKey,
+		}
+
+		for _, key := range volumeStatKeys {
+			kubeletKeyName := fmt.Sprintf("%s_%s", kubeletmetrics.KubeletSubsystem, key)
+			verifyVolumeStatMetric(kubeletKeyName, pvc.Namespace, pvc.Name, kubeMetrics)
+		}
+	})
+})
+
+func verifyMetricCount(oldMetrics map[string]int64, newMetrics map[string]int64, metricName string) {
+	oldCount, ok := oldMetrics[metricName]
+	// if metric does not exist in oldMap, it probably hasn't been emitted yet.
+	if !ok {
+		oldCount = 0
+	}
+
+	newCount, ok := newMetrics[metricName]
+	Expect(ok).To(BeTrue(), "Error getting updated metrics for %s", metricName)
+
+	Expect(oldCount + 1).To(Equal(newCount))
+}
+
+func getControllerStorageMetrics(ms metrics.ControllerManagerMetrics) map[string]int64 {
+	result := make(map[string]int64)
+
+	for method, samples := range ms {
+		if method != "storage_operation_duration_seconds_count" {
+			continue
+		}
+
+		for _, sample := range samples {
+			count := int64(sample.Value)
+			operation := string(sample.Metric["operation_name"])
+			result[operation] = count
+		}
+	}
+	return result
+}
+
+// Verifies the specified metrics are in `kubeletMetrics`
+func verifyVolumeStatMetric(metricKeyName string, namespace string, pvcName string, kubeletMetrics metrics.KubeletMetrics) {
+	found := false
+	errCount := 0
+	if samples, ok := kubeletMetrics[metricKeyName]; ok {
+		for _, sample := range samples {
+			samplePVC, ok := sample.Metric["persistentvolumeclaim"]
+			if !ok {
+				framework.Logf("Error getting pvc for metric %s, sample %s", metricKeyName, sample.String())
+				errCount++
+			}
+			sampleNS, ok := sample.Metric["namespace"]
+			if !ok {
+				framework.Logf("Error getting namespace for metric %s, sample %s", metricKeyName, sample.String())
+				errCount++
+			}
+
+			if string(samplePVC) == pvcName && string(sampleNS) == namespace {
+				found = true
+				break
+			}
+		}
+	}
+	Expect(errCount).To(Equal(0), "Found invalid samples")
+	Expect(found).To(BeTrue(), "PVC %s, Namespace %s not found for %s", pvcName, namespace, metricKeyName)
+}

vendor/k8s.io/kubernetes/test/e2e/storage/volumes.go

@@ -0,0 +1,676 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+ * This test checks that various VolumeSources are working.
+ *
+ * There are two ways, how to test the volumes:
+ * 1) With containerized server (NFS, Ceph, Gluster, iSCSI, ...)
+ * The test creates a server pod, exporting simple 'index.html' file.
+ * Then it uses appropriate VolumeSource to import this file into a client pod
+ * and checks that the pod can see the file. It does so by importing the file
+ * into web server root and loadind the index.html from it.
+ *
+ * These tests work only when privileged containers are allowed, exporting
+ * various filesystems (NFS, GlusterFS, ...) usually needs some mounting or
+ * other privileged magic in the server pod.
+ *
+ * Note that the server containers are for testing purposes only and should not
+ * be used in production.
+ *
+ * 2) With server outside of Kubernetes (Cinder, ...)
+ * Appropriate server (e.g. OpenStack Cinder) must exist somewhere outside
+ * the tested Kubernetes cluster. The test itself creates a new volume,
+ * and checks, that Kubernetes can use it as a volume.
+ */
+
+// test/e2e/common/volumes.go duplicates the GlusterFS test from this file.  Any changes made to this
+// test should be made there as well.
+
+package storage
+
+import (
+	"os/exec"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
+	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+func DeleteCinderVolume(name string) error {
+	// Try to delete the volume for several seconds - it takes
+	// a while for the plugin to detach it.
+	var output []byte
+	var err error
+	timeout := time.Second * 120
+
+	framework.Logf("Waiting up to %v for removal of cinder volume %s", timeout, name)
+	for start := time.Now(); time.Since(start) < timeout; time.Sleep(5 * time.Second) {
+		output, err = exec.Command("cinder", "delete", name).CombinedOutput()
+		if err == nil {
+			framework.Logf("Cinder volume %s deleted", name)
+			return nil
+		} else {
+			framework.Logf("Failed to delete volume %s: %v", name, err)
+		}
+	}
+	framework.Logf("Giving up deleting volume %s: %v\n%s", name, err, string(output[:]))
+	return err
+}
+
+// These tests need privileged containers, which are disabled by default.
+var _ = SIGDescribe("Volumes", func() {
+	f := framework.NewDefaultFramework("volume")
+
+	// If 'false', the test won't clear its volumes upon completion. Useful for debugging,
+	// note that namespace deletion is handled by delete-namespace flag
+	clean := true
+	// filled inside BeforeEach
+	var cs clientset.Interface
+	var namespace *v1.Namespace
+
+	BeforeEach(func() {
+		cs = f.ClientSet
+		namespace = f.Namespace
+	})
+
+	////////////////////////////////////////////////////////////////////////
+	// NFS
+	////////////////////////////////////////////////////////////////////////
+
+	Describe("NFS", func() {
+		It("should be mountable", func() {
+			config, _, serverIP := framework.NewNFSServer(cs, namespace.Name, []string{})
+			defer func() {
+				if clean {
+					framework.VolumeTestCleanup(f, config)
+				}
+			}()
+
+			tests := []framework.VolumeTest{
+				{
+					Volume: v1.VolumeSource{
+						NFS: &v1.NFSVolumeSource{
+							Server:   serverIP,
+							Path:     "/",
+							ReadOnly: true,
+						},
+					},
+					File: "index.html",
+					// Must match content of test/images/volumes-tester/nfs/index.html
+					ExpectedContent: "Hello from NFS!",
+				},
+			}
+			framework.TestVolumeClient(cs, config, nil, tests)
+		})
+	})
+
+	////////////////////////////////////////////////////////////////////////
+	// Gluster
+	////////////////////////////////////////////////////////////////////////
+
+	Describe("GlusterFS", func() {
+		It("should be mountable", func() {
+			//TODO (copejon) GFS is not supported on debian image.
+			framework.SkipUnlessNodeOSDistroIs("gci", "ubuntu")
+
+			// create gluster server and endpoints
+			config, _, _ := framework.NewGlusterfsServer(cs, namespace.Name)
+			name := config.Prefix + "-server"
+			defer func() {
+				if clean {
+					framework.VolumeTestCleanup(f, config)
+					err := cs.CoreV1().Endpoints(namespace.Name).Delete(name, nil)
+					Expect(err).NotTo(HaveOccurred(), "defer: Gluster delete endpoints failed")
+				}
+			}()
+
+			tests := []framework.VolumeTest{
+				{
+					Volume: v1.VolumeSource{
+						Glusterfs: &v1.GlusterfsVolumeSource{
+							EndpointsName: name,
+							// 'test_vol' comes from test/images/volumes-tester/gluster/run_gluster.sh
+							Path:     "test_vol",
+							ReadOnly: true,
+						},
+					},
+					File: "index.html",
+					// Must match content of test/images/volumes-tester/gluster/index.html
+					ExpectedContent: "Hello from GlusterFS!",
+				},
+			}
+			framework.TestVolumeClient(cs, config, nil, tests)
+		})
+	})
+
+	////////////////////////////////////////////////////////////////////////
+	// iSCSI
+	////////////////////////////////////////////////////////////////////////
+
+	// The test needs privileged containers, which are disabled by default.
+	// Also, make sure that iscsiadm utility and iscsi target kernel modules
+	// are installed on all nodes!
+	// Run the test with "go run hack/e2e.go ... --ginkgo.focus=iSCSI"
+
+	Describe("iSCSI [Feature:Volumes]", func() {
+		It("should be mountable", func() {
+			config, _, serverIP := framework.NewISCSIServer(cs, namespace.Name)
+			defer func() {
+				if clean {
+					framework.VolumeTestCleanup(f, config)
+				}
+			}()
+
+			tests := []framework.VolumeTest{
+				{
+					Volume: v1.VolumeSource{
+						ISCSI: &v1.ISCSIVolumeSource{
+							TargetPortal: serverIP + ":3260",
+							// from test/images/volumes-tester/iscsi/initiatorname.iscsi
+							IQN:    "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c",
+							Lun:    0,
+							FSType: "ext2",
+						},
+					},
+					File: "index.html",
+					// Must match content of test/images/volumes-tester/iscsi/block.tar.gz
+					ExpectedContent: "Hello from iSCSI",
+				},
+			}
+			fsGroup := int64(1234)
+			framework.TestVolumeClient(cs, config, &fsGroup, tests)
+		})
+	})
+
+	////////////////////////////////////////////////////////////////////////
+	// Ceph RBD
+	////////////////////////////////////////////////////////////////////////
+
+	Describe("Ceph RBD [Feature:Volumes]", func() {
+		It("should be mountable", func() {
+			config, _, serverIP := framework.NewRBDServer(cs, namespace.Name)
+			defer func() {
+				if clean {
+					framework.VolumeTestCleanup(f, config)
+				}
+			}()
+
+			// create secrets for the server
+			secret := v1.Secret{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "Secret",
+					APIVersion: "v1",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: config.Prefix + "-secret",
+				},
+				Data: map[string][]byte{
+					// from test/images/volumes-tester/rbd/keyring
+					"key": []byte("AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA=="),
+				},
+				Type: "kubernetes.io/rbd",
+			}
+
+			secClient := cs.CoreV1().Secrets(config.Namespace)
+
+			defer func() {
+				if clean {
+					secClient.Delete(config.Prefix+"-secret", nil)
+				}
+			}()
+
+			if _, err := secClient.Create(&secret); err != nil {
+				framework.Failf("Failed to create secrets for Ceph RBD: %v", err)
+			}
+
+			tests := []framework.VolumeTest{
+				{
+					Volume: v1.VolumeSource{
+						RBD: &v1.RBDVolumeSource{
+							CephMonitors: []string{serverIP},
+							RBDPool:      "rbd",
+							RBDImage:     "foo",
+							RadosUser:    "admin",
+							SecretRef: &v1.LocalObjectReference{
+								Name: config.Prefix + "-secret",
+							},
+							FSType: "ext2",
+						},
+					},
+					File: "index.html",
+					// Must match content of test/images/volumes-tester/rbd/create_block.sh
+					ExpectedContent: "Hello from RBD",
+				},
+			}
+			fsGroup := int64(1234)
+			framework.TestVolumeClient(cs, config, &fsGroup, tests)
+		})
+	})
+
+	////////////////////////////////////////////////////////////////////////
+	// Ceph
+	////////////////////////////////////////////////////////////////////////
+	Describe("CephFS [Feature:Volumes]", func() {
+		It("should be mountable", func() {
+			config := framework.VolumeTestConfig{
+				Namespace:   namespace.Name,
+				Prefix:      "cephfs",
+				ServerImage: framework.CephServerImage,
+				ServerPorts: []int{6789},
+			}
+
+			defer func() {
+				if clean {
+					framework.VolumeTestCleanup(f, config)
+				}
+			}()
+			_, serverIP := framework.CreateStorageServer(cs, config)
+			By("sleeping a bit to give ceph server time to initialize")
+			time.Sleep(20 * time.Second)
+
+			// create ceph secret
+			secret := &v1.Secret{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "Secret",
+					APIVersion: "v1",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: config.Prefix + "-secret",
+				},
+				// Must use the ceph keyring at contrib/for-tests/volumes-ceph/ceph/init.sh
+				// and encode in base64
+				Data: map[string][]byte{
+					"key": []byte("AQAMgXhVwBCeDhAA9nlPaFyfUSatGD4drFWDvQ=="),
+				},
+				Type: "kubernetes.io/cephfs",
+			}
+
+			defer func() {
+				if clean {
+					if err := cs.CoreV1().Secrets(namespace.Name).Delete(secret.Name, nil); err != nil {
+						framework.Failf("unable to delete secret %v: %v", secret.Name, err)
+					}
+				}
+			}()
+
+			var err error
+			if secret, err = cs.CoreV1().Secrets(namespace.Name).Create(secret); err != nil {
+				framework.Failf("unable to create test secret %s: %v", secret.Name, err)
+			}
+
+			tests := []framework.VolumeTest{
+				{
+					Volume: v1.VolumeSource{
+						CephFS: &v1.CephFSVolumeSource{
+							Monitors:  []string{serverIP + ":6789"},
+							User:      "kube",
+							SecretRef: &v1.LocalObjectReference{Name: config.Prefix + "-secret"},
+							ReadOnly:  true,
+						},
+					},
+					File: "index.html",
+					// Must match content of test/images/volumes-tester/ceph/index.html
+					ExpectedContent: "Hello Ceph!",
+				},
+			}
+			framework.TestVolumeClient(cs, config, nil, tests)
+		})
+	})
+
+	////////////////////////////////////////////////////////////////////////
+	// OpenStack Cinder
+	////////////////////////////////////////////////////////////////////////
+
+	// This test assumes that OpenStack client tools are installed
+	// (/usr/bin/nova, /usr/bin/cinder and /usr/bin/keystone)
+	// and that the usual OpenStack authentication env. variables are set
+	// (OS_USERNAME, OS_PASSWORD, OS_TENANT_NAME at least).
+	Describe("Cinder [Feature:Volumes]", func() {
+		It("should be mountable", func() {
+			framework.SkipUnlessProviderIs("openstack")
+			config := framework.VolumeTestConfig{
+				Namespace: namespace.Name,
+				Prefix:    "cinder",
+			}
+
+			// We assume that namespace.Name is a random string
+			volumeName := namespace.Name
+			By("creating a test Cinder volume")
+			output, err := exec.Command("cinder", "create", "--display-name="+volumeName, "1").CombinedOutput()
+			outputString := string(output[:])
+			framework.Logf("cinder output:\n%s", outputString)
+			Expect(err).NotTo(HaveOccurred())
+
+			defer func() {
+				// Ignore any cleanup errors, there is not much we can do about
+				// them. They were already logged.
+				DeleteCinderVolume(volumeName)
+			}()
+
+			// Parse 'id'' from stdout. Expected format:
+			// |     attachments     |                  []                  |
+			// |  availability_zone  |                 nova                 |
+			// ...
+			// |          id         | 1d6ff08f-5d1c-41a4-ad72-4ef872cae685 |
+			volumeID := ""
+			for _, line := range strings.Split(outputString, "\n") {
+				fields := strings.Fields(line)
+				if len(fields) != 5 {
+					continue
+				}
+				if fields[1] != "id" {
+					continue
+				}
+				volumeID = fields[3]
+				break
+			}
+			framework.Logf("Volume ID: %s", volumeID)
+			Expect(volumeID).NotTo(Equal(""))
+
+			defer func() {
+				if clean {
+					framework.Logf("Running volumeTestCleanup")
+					framework.VolumeTestCleanup(f, config)
+				}
+			}()
+
+			tests := []framework.VolumeTest{
+				{
+					Volume: v1.VolumeSource{
+						Cinder: &v1.CinderVolumeSource{
+							VolumeID: volumeID,
+							FSType:   "ext3",
+							ReadOnly: false,
+						},
+					},
+					File: "index.html",
+					// Randomize index.html to make sure we don't see the
+					// content from previous test runs.
+					ExpectedContent: "Hello from Cinder from namespace " + volumeName,
+				},
+			}
+
+			framework.InjectHtml(cs, config, tests[0].Volume, tests[0].ExpectedContent)
+
+			fsGroup := int64(1234)
+			framework.TestVolumeClient(cs, config, &fsGroup, tests)
+		})
+	})
+
+	////////////////////////////////////////////////////////////////////////
+	// GCE PD
+	////////////////////////////////////////////////////////////////////////
+	Describe("PD", func() {
+		var config framework.VolumeTestConfig
+
+		BeforeEach(func() {
+			framework.SkipUnlessProviderIs("gce", "gke")
+			config = framework.VolumeTestConfig{
+				Namespace: namespace.Name,
+				Prefix:    "pd",
+				// PD will be created in framework.TestContext.CloudConfig.Zone zone,
+				// so pods should be also scheduled there.
+				NodeSelector: map[string]string{
+					kubeletapis.LabelZoneFailureDomain: framework.TestContext.CloudConfig.Zone,
+				},
+			}
+		})
+
+		It("should be mountable with ext3", func() {
+			testGCEPD(f, config, cs, clean, "ext3")
+		})
+		It("should be mountable with ext4", func() {
+			testGCEPD(f, config, cs, clean, "ext4")
+		})
+		It("should be mountable with xfs", func() {
+			// xfs is not supported on gci
+			// and not installed by default on debian
+			framework.SkipUnlessNodeOSDistroIs("ubuntu")
+			testGCEPD(f, config, cs, clean, "xfs")
+		})
+	})
+
+	////////////////////////////////////////////////////////////////////////
+	// ConfigMap
+	////////////////////////////////////////////////////////////////////////
+	Describe("ConfigMap", func() {
+		It("should be mountable", func() {
+			config := framework.VolumeTestConfig{
+				Namespace: namespace.Name,
+				Prefix:    "configmap",
+			}
+
+			defer func() {
+				if clean {
+					framework.VolumeTestCleanup(f, config)
+				}
+			}()
+			configMap := &v1.ConfigMap{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "ConfigMap",
+					APIVersion: "v1",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name: config.Prefix + "-map",
+				},
+				Data: map[string]string{
+					"first":  "this is the first file",
+					"second": "this is the second file",
+					"third":  "this is the third file",
+				},
+			}
+			if _, err := cs.CoreV1().ConfigMaps(namespace.Name).Create(configMap); err != nil {
+				framework.Failf("unable to create test configmap: %v", err)
+			}
+			defer func() {
+				_ = cs.CoreV1().ConfigMaps(namespace.Name).Delete(configMap.Name, nil)
+			}()
+
+			// Test one ConfigMap mounted several times to test #28502
+			tests := []framework.VolumeTest{
+				{
+					Volume: v1.VolumeSource{
+						ConfigMap: &v1.ConfigMapVolumeSource{
+							LocalObjectReference: v1.LocalObjectReference{
+								Name: config.Prefix + "-map",
+							},
+							Items: []v1.KeyToPath{
+								{
+									Key:  "first",
+									Path: "firstfile",
+								},
+							},
+						},
+					},
+					File:            "firstfile",
+					ExpectedContent: "this is the first file",
+				},
+				{
+					Volume: v1.VolumeSource{
+						ConfigMap: &v1.ConfigMapVolumeSource{
+							LocalObjectReference: v1.LocalObjectReference{
+								Name: config.Prefix + "-map",
+							},
+							Items: []v1.KeyToPath{
+								{
+									Key:  "second",
+									Path: "secondfile",
+								},
+							},
+						},
+					},
+					File:            "secondfile",
+					ExpectedContent: "this is the second file",
+				},
+			}
+			framework.TestVolumeClient(cs, config, nil, tests)
+		})
+	})
+
+	////////////////////////////////////////////////////////////////////////
+	// vSphere
+	////////////////////////////////////////////////////////////////////////
+	Describe("vsphere [Feature:Volumes]", func() {
+		It("should be mountable", func() {
+			framework.SkipUnlessProviderIs("vsphere")
+			var (
+				volumePath string
+			)
+			config := framework.VolumeTestConfig{
+				Namespace: namespace.Name,
+				Prefix:    "vsphere",
+			}
+			By("creating a test vsphere volume")
+			vsp, err := vsphere.GetVSphere()
+			Expect(err).NotTo(HaveOccurred())
+
+			volumePath, err = createVSphereVolume(vsp, nil)
+			Expect(err).NotTo(HaveOccurred())
+
+			defer func() {
+				vsp.DeleteVolume(volumePath)
+			}()
+
+			defer func() {
+				if clean {
+					framework.Logf("Running volumeTestCleanup")
+					framework.VolumeTestCleanup(f, config)
+				}
+			}()
+
+			tests := []framework.VolumeTest{
+				{
+					Volume: v1.VolumeSource{
+						VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
+							VolumePath: volumePath,
+							FSType:     "ext4",
+						},
+					},
+					File: "index.html",
+					// Randomize index.html to make sure we don't see the
+					// content from previous test runs.
+					ExpectedContent: "Hello from vSphere from namespace " + namespace.Name,
+				},
+			}
+
+			framework.InjectHtml(cs, config, tests[0].Volume, tests[0].ExpectedContent)
+
+			fsGroup := int64(1234)
+			framework.TestVolumeClient(cs, config, &fsGroup, tests)
+		})
+	})
+
+	////////////////////////////////////////////////////////////////////////
+	// Azure Disk
+	////////////////////////////////////////////////////////////////////////
+	Describe("Azure Disk [Feature:Volumes]", func() {
+		It("should be mountable [Slow]", func() {
+			framework.SkipUnlessProviderIs("azure")
+			config := framework.VolumeTestConfig{
+				Namespace: namespace.Name,
+				Prefix:    "azure",
+			}
+
+			By("creating a test azure disk volume")
+			volumeName, err := framework.CreatePDWithRetry()
+			Expect(err).NotTo(HaveOccurred())
+			defer func() {
+				framework.DeletePDWithRetry(volumeName)
+			}()
+
+			defer func() {
+				if clean {
+					framework.Logf("Running volumeTestCleanup")
+					framework.VolumeTestCleanup(f, config)
+				}
+			}()
+			fsType := "ext4"
+			readOnly := false
+			diskName := volumeName[(strings.LastIndex(volumeName, "/") + 1):]
+			tests := []framework.VolumeTest{
+				{
+					Volume: v1.VolumeSource{
+						AzureDisk: &v1.AzureDiskVolumeSource{
+							DiskName:    diskName,
+							DataDiskURI: volumeName,
+							FSType:      &fsType,
+							ReadOnly:    &readOnly,
+						},
+					},
+					File: "index.html",
+					// Randomize index.html to make sure we don't see the
+					// content from previous test runs.
+					ExpectedContent: "Hello from Azure from namespace " + volumeName,
+				},
+			}
+
+			framework.InjectHtml(cs, config, tests[0].Volume, tests[0].ExpectedContent)
+
+			fsGroup := int64(1234)
+			framework.TestVolumeClient(cs, config, &fsGroup, tests)
+		})
+	})
+})
+
+func testGCEPD(f *framework.Framework, config framework.VolumeTestConfig, cs clientset.Interface, clean bool, fs string) {
+	By("creating a test gce pd volume")
+	volumeName, err := framework.CreatePDWithRetry()
+	Expect(err).NotTo(HaveOccurred())
+	defer func() {
+		// - Get NodeName from the pod spec to which the volume is mounted.
+		// - Force detach and delete.
+		pod, err := f.PodClient().Get(config.Prefix+"-client", metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred(), "Failed getting pod %q.", config.Prefix+"-client")
+		detachAndDeletePDs(volumeName, []types.NodeName{types.NodeName(pod.Spec.NodeName)})
+	}()
+
+	defer func() {
+		if clean {
+			framework.Logf("Running volumeTestCleanup")
+			framework.VolumeTestCleanup(f, config)
+		}
+	}()
+
+	tests := []framework.VolumeTest{
+		{
+			Volume: v1.VolumeSource{
+				GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
+					PDName:   volumeName,
+					FSType:   fs,
+					ReadOnly: false,
+				},
+			},
+			File: "index.html",
+			// Randomize index.html to make sure we don't see the
+			// content from previous test runs.
+			ExpectedContent: "Hello from GCE from namespace " + volumeName,
+		},
+	}
+
+	framework.InjectHtml(cs, config, tests[0].Volume, tests[0].ExpectedContent)
+
+	fsGroup := int64(1234)
+	framework.TestVolumeClient(cs, config, &fsGroup, tests)
+}

vendor/k8s.io/kubernetes/test/e2e/storage/vsphere_utils.go

@@ -0,0 +1,357 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"fmt"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	storage "k8s.io/api/storage/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	k8stype "k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib"
+	"k8s.io/kubernetes/pkg/volume/util/volumehelper"
+	"k8s.io/kubernetes/test/e2e/framework"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+)
+
+// Sanity check for vSphere testing.  Verify the persistent disk attached to the node.
+func verifyVSphereDiskAttached(vsp *vsphere.VSphere, volumePath string, nodeName types.NodeName) (bool, error) {
+	var (
+		isAttached bool
+		err        error
+	)
+	if vsp == nil {
+		vsp, err = vsphere.GetVSphere()
+		Expect(err).NotTo(HaveOccurred())
+	}
+	isAttached, err = vsp.DiskIsAttached(volumePath, nodeName)
+	Expect(err).NotTo(HaveOccurred())
+	return isAttached, err
+}
+
+// Wait until vsphere vmdk is deteched from the given node or time out after 5 minutes
+func waitForVSphereDiskToDetach(vsp *vsphere.VSphere, volumePath string, nodeName types.NodeName) error {
+	var (
+		err            error
+		diskAttached   = true
+		detachTimeout  = 5 * time.Minute
+		detachPollTime = 10 * time.Second
+	)
+	if vsp == nil {
+		vsp, err = vsphere.GetVSphere()
+		if err != nil {
+			return err
+		}
+	}
+	err = wait.Poll(detachPollTime, detachTimeout, func() (bool, error) {
+		diskAttached, err = verifyVSphereDiskAttached(vsp, volumePath, nodeName)
+		if err != nil {
+			return true, err
+		}
+		if !diskAttached {
+			framework.Logf("Volume %q appears to have successfully detached from %q.",
+				volumePath, nodeName)
+			return true, nil
+		}
+		framework.Logf("Waiting for Volume %q to detach from %q.", volumePath, nodeName)
+		return false, nil
+	})
+	if err != nil {
+		return err
+	}
+	if diskAttached {
+		return fmt.Errorf("Gave up waiting for Volume %q to detach from %q after %v", volumePath, nodeName, detachTimeout)
+	}
+	return nil
+}
+
+// function to create vsphere volume spec with given VMDK volume path, Reclaim Policy and labels
+func getVSpherePersistentVolumeSpec(volumePath string, persistentVolumeReclaimPolicy v1.PersistentVolumeReclaimPolicy, labels map[string]string) *v1.PersistentVolume {
+	var (
+		pvConfig framework.PersistentVolumeConfig
+		pv       *v1.PersistentVolume
+		claimRef *v1.ObjectReference
+	)
+	pvConfig = framework.PersistentVolumeConfig{
+		NamePrefix: "vspherepv-",
+		PVSource: v1.PersistentVolumeSource{
+			VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
+				VolumePath: volumePath,
+				FSType:     "ext4",
+			},
+		},
+		Prebind: nil,
+	}
+
+	pv = &v1.PersistentVolume{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: pvConfig.NamePrefix,
+			Annotations: map[string]string{
+				volumehelper.VolumeGidAnnotationKey: "777",
+			},
+		},
+		Spec: v1.PersistentVolumeSpec{
+			PersistentVolumeReclaimPolicy: persistentVolumeReclaimPolicy,
+			Capacity: v1.ResourceList{
+				v1.ResourceName(v1.ResourceStorage): resource.MustParse("2Gi"),
+			},
+			PersistentVolumeSource: pvConfig.PVSource,
+			AccessModes: []v1.PersistentVolumeAccessMode{
+				v1.ReadWriteOnce,
+			},
+			ClaimRef: claimRef,
+		},
+	}
+	if labels != nil {
+		pv.Labels = labels
+	}
+	return pv
+}
+
+// function to get vsphere persistent volume spec with given selector labels.
+func getVSpherePersistentVolumeClaimSpec(namespace string, labels map[string]string) *v1.PersistentVolumeClaim {
+	var (
+		pvc *v1.PersistentVolumeClaim
+	)
+	pvc = &v1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "pvc-",
+			Namespace:    namespace,
+		},
+		Spec: v1.PersistentVolumeClaimSpec{
+			AccessModes: []v1.PersistentVolumeAccessMode{
+				v1.ReadWriteOnce,
+			},
+			Resources: v1.ResourceRequirements{
+				Requests: v1.ResourceList{
+					v1.ResourceName(v1.ResourceStorage): resource.MustParse("2Gi"),
+				},
+			},
+		},
+	}
+	if labels != nil {
+		pvc.Spec.Selector = &metav1.LabelSelector{MatchLabels: labels}
+	}
+
+	return pvc
+}
+
+// function to create vmdk volume
+func createVSphereVolume(vsp *vsphere.VSphere, volumeOptions *vclib.VolumeOptions) (string, error) {
+	var (
+		volumePath string
+		err        error
+	)
+	if volumeOptions == nil {
+		volumeOptions = new(vclib.VolumeOptions)
+		volumeOptions.CapacityKB = 2097152
+		volumeOptions.Name = "e2e-vmdk-" + strconv.FormatInt(time.Now().UnixNano(), 10)
+	}
+	volumePath, err = vsp.CreateVolume(volumeOptions)
+	Expect(err).NotTo(HaveOccurred())
+	return volumePath, nil
+}
+
+// function to write content to the volume backed by given PVC
+func writeContentToVSpherePV(client clientset.Interface, pvc *v1.PersistentVolumeClaim, expectedContent string) {
+	runInPodWithVolume(client, pvc.Namespace, pvc.Name, "echo "+expectedContent+" > /mnt/test/data")
+	framework.Logf("Done with writing content to volume")
+}
+
+// function to verify content is matching on the volume backed for given PVC
+func verifyContentOfVSpherePV(client clientset.Interface, pvc *v1.PersistentVolumeClaim, expectedContent string) {
+	runInPodWithVolume(client, pvc.Namespace, pvc.Name, "grep '"+expectedContent+"' /mnt/test/data")
+	framework.Logf("Successfully verified content of the volume")
+}
+
+func getVSphereStorageClassSpec(name string, scParameters map[string]string) *storage.StorageClass {
+	var sc *storage.StorageClass
+
+	sc = &storage.StorageClass{
+		TypeMeta: metav1.TypeMeta{
+			Kind: "StorageClass",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Provisioner: "kubernetes.io/vsphere-volume",
+	}
+	if scParameters != nil {
+		sc.Parameters = scParameters
+	}
+	return sc
+}
+
+func getVSphereClaimSpecWithStorageClassAnnotation(ns string, storageclass *storage.StorageClass) *v1.PersistentVolumeClaim {
+	scAnnotation := make(map[string]string)
+	scAnnotation[v1.BetaStorageClassAnnotation] = storageclass.Name
+
+	claim := &v1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "pvc-",
+			Namespace:    ns,
+			Annotations:  scAnnotation,
+		},
+		Spec: v1.PersistentVolumeClaimSpec{
+			AccessModes: []v1.PersistentVolumeAccessMode{
+				v1.ReadWriteOnce,
+			},
+			Resources: v1.ResourceRequirements{
+				Requests: v1.ResourceList{
+					v1.ResourceName(v1.ResourceStorage): resource.MustParse("2Gi"),
+				},
+			},
+		},
+	}
+	return claim
+}
+
+// func to get pod spec with given volume claim, node selector labels and command
+func getVSpherePodSpecWithClaim(claimName string, nodeSelectorKV map[string]string, command string) *v1.Pod {
+	pod := &v1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "pod-pvc-",
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:    "volume-tester",
+					Image:   imageutils.GetBusyBoxImage(),
+					Command: []string{"/bin/sh"},
+					Args:    []string{"-c", command},
+					VolumeMounts: []v1.VolumeMount{
+						{
+							Name:      "my-volume",
+							MountPath: "/mnt/test",
+						},
+					},
+				},
+			},
+			RestartPolicy: v1.RestartPolicyNever,
+			Volumes: []v1.Volume{
+				{
+					Name: "my-volume",
+					VolumeSource: v1.VolumeSource{
+						PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
+							ClaimName: claimName,
+							ReadOnly:  false,
+						},
+					},
+				},
+			},
+		},
+	}
+	if nodeSelectorKV != nil {
+		pod.Spec.NodeSelector = nodeSelectorKV
+	}
+	return pod
+}
+
+// func to get pod spec with given volume paths, node selector lables and container commands
+func getVSpherePodSpecWithVolumePaths(volumePaths []string, keyValuelabel map[string]string, commands []string) *v1.Pod {
+	var volumeMounts []v1.VolumeMount
+	var volumes []v1.Volume
+
+	for index, volumePath := range volumePaths {
+		name := fmt.Sprintf("volume%v", index+1)
+		volumeMounts = append(volumeMounts, v1.VolumeMount{Name: name, MountPath: "/mnt/" + name})
+		vsphereVolume := new(v1.VsphereVirtualDiskVolumeSource)
+		vsphereVolume.VolumePath = volumePath
+		vsphereVolume.FSType = "ext4"
+		volumes = append(volumes, v1.Volume{Name: name})
+		volumes[index].VolumeSource.VsphereVolume = vsphereVolume
+	}
+
+	if commands == nil || len(commands) == 0 {
+		commands = []string{
+			"/bin/sh",
+			"-c",
+			"while true; do sleep 2; done",
+		}
+	}
+	pod := &v1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "vsphere-e2e-",
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:         "vsphere-e2e-container-" + string(uuid.NewUUID()),
+					Image:        imageutils.GetBusyBoxImage(),
+					Command:      commands,
+					VolumeMounts: volumeMounts,
+				},
+			},
+			RestartPolicy: v1.RestartPolicyNever,
+			Volumes:       volumes,
+		},
+	}
+
+	if keyValuelabel != nil {
+		pod.Spec.NodeSelector = keyValuelabel
+	}
+	return pod
+}
+
+func verifyFilesExistOnVSphereVolume(namespace string, podName string, filePaths []string) {
+	for _, filePath := range filePaths {
+		_, err := framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", namespace), podName, "--", "/bin/ls", filePath)
+		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to verify file: %q on the pod: %q", filePath, podName))
+	}
+}
+
+func createEmptyFilesOnVSphereVolume(namespace string, podName string, filePaths []string) {
+	for _, filePath := range filePaths {
+		err := framework.CreateEmptyFileOnPod(namespace, podName, filePath)
+		Expect(err).NotTo(HaveOccurred())
+	}
+}
+
+// verify volumes are attached to the node and are accessible in pod
+func verifyVSphereVolumesAccessible(pod *v1.Pod, persistentvolumes []*v1.PersistentVolume, vsp *vsphere.VSphere) {
+	nodeName := pod.Spec.NodeName
+	namespace := pod.Namespace
+	for index, pv := range persistentvolumes {
+		// Verify disks are attached to the node
+		isAttached, err := verifyVSphereDiskAttached(vsp, pv.Spec.VsphereVolume.VolumePath, k8stype.NodeName(nodeName))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(isAttached).To(BeTrue(), fmt.Sprintf("disk %v is not attached with the node", pv.Spec.VsphereVolume.VolumePath))
+		// Verify Volumes are accessible
+		filepath := filepath.Join("/mnt/", fmt.Sprintf("volume%v", index+1), "/emptyFile.txt")
+		_, err = framework.LookForStringInPodExec(namespace, pod.Name, []string{"/bin/touch", filepath}, "", time.Minute)
+		Expect(err).NotTo(HaveOccurred())
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/storage/vsphere_volume_diskformat.go

@@ -0,0 +1,214 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vmware/govmomi/find"
+	"github.com/vmware/govmomi/vim25/types"
+	"golang.org/x/net/context"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8stype "k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	clientset "k8s.io/client-go/kubernetes"
+	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+/*
+	Test to verify diskformat specified in storage-class is being honored while volume creation.
+	Valid and supported options are eagerzeroedthick, zeroedthick and thin
+
+	Steps
+	1. Create StorageClass with diskformat set to valid type
+	2. Create PVC which uses the StorageClass created in step 1.
+	3. Wait for PV to be provisioned.
+	4. Wait for PVC's status to become Bound
+	5. Create pod using PVC on specific node.
+	6. Wait for Disk to be attached to the node.
+	7. Get node VM's devices and find PV's Volume Disk.
+	8. Get Backing Info of the Volume Disk and obtain EagerlyScrub and ThinProvisioned
+	9. Based on the value of EagerlyScrub and ThinProvisioned, verify diskformat is correct.
+	10. Delete pod and Wait for Volume Disk to be detached from the Node.
+	11. Delete PVC, PV and Storage Class
+*/
+
+var _ = SIGDescribe("Volume Disk Format", func() {
+	f := framework.NewDefaultFramework("volume-disk-format")
+	var (
+		client            clientset.Interface
+		namespace         string
+		nodeName          string
+		isNodeLabeled     bool
+		nodeKeyValueLabel map[string]string
+		nodeLabelValue    string
+	)
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("vsphere")
+		client = f.ClientSet
+		namespace = f.Namespace.Name
+		nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+		if len(nodeList.Items) != 0 {
+			nodeName = nodeList.Items[0].Name
+		} else {
+			framework.Failf("Unable to find ready and schedulable Node")
+		}
+		if !isNodeLabeled {
+			nodeLabelValue := "vsphere_e2e_" + string(uuid.NewUUID())
+			nodeKeyValueLabel = make(map[string]string)
+			nodeKeyValueLabel["vsphere_e2e_label"] = nodeLabelValue
+			framework.AddOrUpdateLabelOnNode(client, nodeName, "vsphere_e2e_label", nodeLabelValue)
+			isNodeLabeled = true
+		}
+	})
+	framework.AddCleanupAction(func() {
+		if len(nodeLabelValue) > 0 {
+			framework.RemoveLabelOffNode(client, nodeName, "vsphere_e2e_label")
+		}
+	})
+
+	It("verify disk format type - eagerzeroedthick is honored for dynamically provisioned pv using storageclass", func() {
+		By("Invoking Test for diskformat: eagerzeroedthick")
+		invokeTest(f, client, namespace, nodeName, nodeKeyValueLabel, "eagerzeroedthick")
+	})
+	It("verify disk format type - zeroedthick is honored for dynamically provisioned pv using storageclass", func() {
+		By("Invoking Test for diskformat: zeroedthick")
+		invokeTest(f, client, namespace, nodeName, nodeKeyValueLabel, "zeroedthick")
+	})
+	It("verify disk format type - thin is honored for dynamically provisioned pv using storageclass", func() {
+		By("Invoking Test for diskformat: thin")
+		invokeTest(f, client, namespace, nodeName, nodeKeyValueLabel, "thin")
+	})
+})
+
+func invokeTest(f *framework.Framework, client clientset.Interface, namespace string, nodeName string, nodeKeyValueLabel map[string]string, diskFormat string) {
+
+	framework.Logf("Invoking Test for DiskFomat: %s", diskFormat)
+	scParameters := make(map[string]string)
+	scParameters["diskformat"] = diskFormat
+
+	By("Creating Storage Class With DiskFormat")
+	storageClassSpec := getVSphereStorageClassSpec("thinsc", scParameters)
+	storageclass, err := client.StorageV1().StorageClasses().Create(storageClassSpec)
+	Expect(err).NotTo(HaveOccurred())
+
+	defer client.StorageV1().StorageClasses().Delete(storageclass.Name, nil)
+
+	By("Creating PVC using the Storage Class")
+	pvclaimSpec := getVSphereClaimSpecWithStorageClassAnnotation(namespace, storageclass)
+	pvclaim, err := client.CoreV1().PersistentVolumeClaims(namespace).Create(pvclaimSpec)
+	Expect(err).NotTo(HaveOccurred())
+
+	defer func() {
+		client.CoreV1().PersistentVolumeClaims(namespace).Delete(pvclaimSpec.Name, nil)
+	}()
+
+	By("Waiting for claim to be in bound phase")
+	err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, pvclaim.Namespace, pvclaim.Name, framework.Poll, framework.ClaimProvisionTimeout)
+	Expect(err).NotTo(HaveOccurred())
+
+	// Get new copy of the claim
+	pvclaim, err = client.CoreV1().PersistentVolumeClaims(pvclaim.Namespace).Get(pvclaim.Name, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+
+	// Get the bound PV
+	pv, err := client.CoreV1().PersistentVolumes().Get(pvclaim.Spec.VolumeName, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+
+	/*
+		PV is required to be attached to the Node. so that using govmomi API we can grab Disk's Backing Info
+		to check EagerlyScrub and ThinProvisioned property
+	*/
+	By("Creating pod to attach PV to the node")
+	// Create pod to attach Volume to Node
+	podSpec := getVSpherePodSpecWithClaim(pvclaim.Name, nodeKeyValueLabel, "while true ; do sleep 2 ; done")
+	pod, err := client.CoreV1().Pods(namespace).Create(podSpec)
+	Expect(err).NotTo(HaveOccurred())
+
+	vsp, err := vsphere.GetVSphere()
+	Expect(err).NotTo(HaveOccurred())
+	verifyVSphereDiskAttached(vsp, pv.Spec.VsphereVolume.VolumePath, k8stype.NodeName(nodeName))
+
+	By("Waiting for pod to be running")
+	Expect(framework.WaitForPodNameRunningInNamespace(client, pod.Name, namespace)).To(Succeed())
+	Expect(verifyDiskFormat(nodeName, pv.Spec.VsphereVolume.VolumePath, diskFormat)).To(BeTrue(), "DiskFormat Verification Failed")
+
+	var volumePaths []string
+	volumePaths = append(volumePaths, pv.Spec.VsphereVolume.VolumePath)
+
+	By("Delete pod and wait for volume to be detached from node")
+	deletePodAndWaitForVolumeToDetach(f, client, pod, vsp, nodeName, volumePaths)
+
+}
+
+func verifyDiskFormat(nodeName string, pvVolumePath string, diskFormat string) bool {
+	By("Verifing disk format")
+	eagerlyScrub := false
+	thinProvisioned := false
+	diskFound := false
+	pvvmdkfileName := filepath.Base(pvVolumePath) + filepath.Ext(pvVolumePath)
+
+	govMoMiClient, err := vsphere.GetgovmomiClient(nil)
+	Expect(err).NotTo(HaveOccurred())
+
+	f := find.NewFinder(govMoMiClient.Client, true)
+	ctx, _ := context.WithCancel(context.Background())
+	vm, err := f.VirtualMachine(ctx, os.Getenv("VSPHERE_WORKING_DIR")+nodeName)
+	Expect(err).NotTo(HaveOccurred())
+
+	vmDevices, err := vm.Device(ctx)
+	Expect(err).NotTo(HaveOccurred())
+
+	disks := vmDevices.SelectByType((*types.VirtualDisk)(nil))
+
+	for _, disk := range disks {
+		backing := disk.GetVirtualDevice().Backing.(*types.VirtualDiskFlatVer2BackingInfo)
+		backingFileName := filepath.Base(backing.FileName) + filepath.Ext(backing.FileName)
+		if backingFileName == pvvmdkfileName {
+			diskFound = true
+			if backing.EagerlyScrub != nil {
+				eagerlyScrub = *backing.EagerlyScrub
+			}
+			if backing.ThinProvisioned != nil {
+				thinProvisioned = *backing.ThinProvisioned
+			}
+			break
+		}
+	}
+
+	Expect(diskFound).To(BeTrue(), "Failed to find disk")
+	isDiskFormatCorrect := false
+	if diskFormat == "eagerzeroedthick" {
+		if eagerlyScrub == true && thinProvisioned == false {
+			isDiskFormatCorrect = true
+		}
+	} else if diskFormat == "zeroedthick" {
+		if eagerlyScrub == false && thinProvisioned == false {
+			isDiskFormatCorrect = true
+		}
+	} else if diskFormat == "thin" {
+		if eagerlyScrub == false && thinProvisioned == true {
+			isDiskFormatCorrect = true
+		}
+	}
+	return isDiskFormatCorrect
+}

vendor/k8s.io/kubernetes/test/e2e/storage/vsphere_volume_fstype.go

@@ -0,0 +1,111 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	k8stype "k8s.io/apimachinery/pkg/types"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+/*
+	Test to verify fstype specified in storage-class is being honored after volume creation.
+
+	Steps
+	1. Create StorageClass with fstype set to valid type (default case included).
+	2. Create PVC which uses the StorageClass created in step 1.
+	3. Wait for PV to be provisioned.
+	4. Wait for PVC's status to become Bound.
+	5. Create pod using PVC on specific node.
+	6. Wait for Disk to be attached to the node.
+	7. Execute command in the pod to get fstype.
+	8. Delete pod and Wait for Volume Disk to be detached from the Node.
+	9. Delete PVC, PV and Storage Class.
+*/
+
+var _ = SIGDescribe("vsphere Volume fstype", func() {
+	f := framework.NewDefaultFramework("volume-fstype")
+	var (
+		client    clientset.Interface
+		namespace string
+	)
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("vsphere")
+		client = f.ClientSet
+		namespace = f.Namespace.Name
+		nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+		Expect(len(nodeList.Items)).NotTo(BeZero(), "Unable to find ready and schedulable Node")
+	})
+
+	It("verify fstype - ext3 formatted volume", func() {
+		By("Invoking Test for fstype: ext3")
+		invokeTestForFstype(f, client, namespace, "ext3", "ext3")
+	})
+
+	It("verify disk format type - default value should be ext4", func() {
+		By("Invoking Test for fstype: Default Value")
+		invokeTestForFstype(f, client, namespace, "", "ext4")
+	})
+})
+
+func invokeTestForFstype(f *framework.Framework, client clientset.Interface, namespace string, fstype string, expectedContent string) {
+	framework.Logf("Invoking Test for fstype: %s", fstype)
+	scParameters := make(map[string]string)
+	scParameters["fstype"] = fstype
+
+	By("Creating Storage Class With Fstype")
+	storageclass, err := client.StorageV1().StorageClasses().Create(getVSphereStorageClassSpec("fstype", scParameters))
+	Expect(err).NotTo(HaveOccurred())
+	defer client.StorageV1().StorageClasses().Delete(storageclass.Name, nil)
+
+	By("Creating PVC using the Storage Class")
+	pvclaim, err := client.CoreV1().PersistentVolumeClaims(namespace).Create(getVSphereClaimSpecWithStorageClassAnnotation(namespace, storageclass))
+	Expect(err).NotTo(HaveOccurred())
+	defer framework.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace)
+
+	var pvclaims []*v1.PersistentVolumeClaim
+	pvclaims = append(pvclaims, pvclaim)
+	By("Waiting for claim to be in bound phase")
+	persistentvolumes, err := framework.WaitForPVClaimBoundPhase(client, pvclaims)
+	Expect(err).NotTo(HaveOccurred())
+
+	By("Creating pod to attach PV to the node")
+	// Create pod to attach Volume to Node
+	pod, err := framework.CreatePod(client, namespace, pvclaims, false, "")
+	Expect(err).NotTo(HaveOccurred())
+
+	// Asserts: Right disk is attached to the pod
+	vsp, err := vsphere.GetVSphere()
+	Expect(err).NotTo(HaveOccurred())
+	By("Verify the volume is accessible and available in the pod")
+	verifyVSphereVolumesAccessible(pod, persistentvolumes, vsp)
+
+	_, err = framework.LookForStringInPodExec(namespace, pod.Name, []string{"/bin/cat", "/mnt/test/fstype"}, expectedContent, time.Minute)
+	Expect(err).NotTo(HaveOccurred())
+
+	By("Deleting pod")
+	framework.DeletePodWithWait(f, client, pod)
+
+	By("Waiting for volumes to be detached from the node")
+	waitForVSphereDiskToDetach(vsp, persistentvolumes[0].Spec.VsphereVolume.VolumePath, k8stype.NodeName(pod.Spec.NodeName))
+}

vendor/k8s.io/kubernetes/test/e2e/storage/vsphere_volume_ops_storm.go

@@ -0,0 +1,126 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	storage "k8s.io/api/storage/v1"
+	k8stype "k8s.io/apimachinery/pkg/types"
+	clientset "k8s.io/client-go/kubernetes"
+	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+/*
+	Test to perform Disk Ops storm.
+
+	Steps
+    	1. Create storage class for thin Provisioning.
+    	2. Create 30 PVCs using above storage class in annotation, requesting 2 GB files.
+    	3. Wait until all disks are ready and all PVs and PVCs get bind. (CreateVolume storm)
+    	4. Create pod to mount volumes using PVCs created in step 2. (AttachDisk storm)
+    	5. Wait for pod status to be running.
+    	6. Verify all volumes accessible and available in the pod.
+    	7. Delete pod.
+    	8. wait until volumes gets detached. (DetachDisk storm)
+    	9. Delete all PVCs. This should delete all Disks. (DeleteVolume storm)
+		10. Delete storage class.
+*/
+
+var _ = SIGDescribe("vsphere volume operations storm", func() {
+	f := framework.NewDefaultFramework("volume-ops-storm")
+	const DEFAULT_VOLUME_OPS_SCALE = 30
+	var (
+		client            clientset.Interface
+		namespace         string
+		storageclass      *storage.StorageClass
+		pvclaims          []*v1.PersistentVolumeClaim
+		persistentvolumes []*v1.PersistentVolume
+		err               error
+		volume_ops_scale  int
+		vsp               *vsphere.VSphere
+	)
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("vsphere")
+		client = f.ClientSet
+		namespace = f.Namespace.Name
+		nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+		if len(nodeList.Items) == 0 {
+			framework.Failf("Unable to find ready and schedulable Node")
+		}
+		if os.Getenv("VOLUME_OPS_SCALE") != "" {
+			volume_ops_scale, err = strconv.Atoi(os.Getenv("VOLUME_OPS_SCALE"))
+			Expect(err).NotTo(HaveOccurred())
+		} else {
+			volume_ops_scale = DEFAULT_VOLUME_OPS_SCALE
+		}
+		pvclaims = make([]*v1.PersistentVolumeClaim, volume_ops_scale)
+		vsp, err = vsphere.GetVSphere()
+		Expect(err).NotTo(HaveOccurred())
+	})
+	AfterEach(func() {
+		By("Deleting PVCs")
+		for _, claim := range pvclaims {
+			framework.DeletePersistentVolumeClaim(client, claim.Name, namespace)
+		}
+		By("Deleting StorageClass")
+		err = client.StorageV1().StorageClasses().Delete(storageclass.Name, nil)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("should create pod with many volumes and verify no attach call fails", func() {
+		By(fmt.Sprintf("Running test with VOLUME_OPS_SCALE: %v", volume_ops_scale))
+		By("Creating Storage Class")
+		scParameters := make(map[string]string)
+		scParameters["diskformat"] = "thin"
+		storageclass, err = client.StorageV1().StorageClasses().Create(getVSphereStorageClassSpec("thinsc", scParameters))
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Creating PVCs using the Storage Class")
+		count := 0
+		for count < volume_ops_scale {
+			pvclaims[count], err = framework.CreatePVC(client, namespace, getVSphereClaimSpecWithStorageClassAnnotation(namespace, storageclass))
+			Expect(err).NotTo(HaveOccurred())
+			count++
+		}
+
+		By("Waiting for all claims to be in bound phase")
+		persistentvolumes, err = framework.WaitForPVClaimBoundPhase(client, pvclaims)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Creating pod to attach PVs to the node")
+		pod, err := framework.CreatePod(client, namespace, pvclaims, false, "")
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Verify all volumes are accessible and available in the pod")
+		verifyVSphereVolumesAccessible(pod, persistentvolumes, vsp)
+
+		By("Deleting pod")
+		framework.ExpectNoError(framework.DeletePodWithWait(f, client, pod))
+
+		By("Waiting for volumes to be detached from the node")
+		for _, pv := range persistentvolumes {
+			waitForVSphereDiskToDetach(vsp, pv.Spec.VsphereVolume.VolumePath, k8stype.NodeName(pod.Spec.NodeName))
+		}
+	})
+})

vendor/k8s.io/kubernetes/test/e2e/storage/vsphere_volume_placement.go

@@ -0,0 +1,387 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	clientset "k8s.io/client-go/kubernetes"
+	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+var _ = SIGDescribe("Volume Placement", func() {
+	f := framework.NewDefaultFramework("volume-placement")
+	var (
+		c                  clientset.Interface
+		ns                 string
+		vsp                *vsphere.VSphere
+		volumePaths        []string
+		node1Name          string
+		node1KeyValueLabel map[string]string
+		node2Name          string
+		node2KeyValueLabel map[string]string
+		isNodeLabeled      bool
+		err                error
+	)
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("vsphere")
+		c = f.ClientSet
+		ns = f.Namespace.Name
+		framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout))
+		if !isNodeLabeled {
+			node1Name, node1KeyValueLabel, node2Name, node2KeyValueLabel = testSetupVolumePlacement(c, ns)
+			isNodeLabeled = true
+		}
+		By("creating vmdk")
+		vsp, err = vsphere.GetVSphere()
+		Expect(err).NotTo(HaveOccurred())
+		volumePath, err := createVSphereVolume(vsp, nil)
+		Expect(err).NotTo(HaveOccurred())
+		volumePaths = append(volumePaths, volumePath)
+	})
+
+	AfterEach(func() {
+		for _, volumePath := range volumePaths {
+			vsp.DeleteVolume(volumePath)
+		}
+		volumePaths = nil
+	})
+
+	/*
+		Steps
+		1. Remove labels assigned to node 1 and node 2
+		2. Delete VMDK volume
+	*/
+	framework.AddCleanupAction(func() {
+		if len(node1KeyValueLabel) > 0 {
+			framework.RemoveLabelOffNode(c, node1Name, "vsphere_e2e_label")
+		}
+		if len(node2KeyValueLabel) > 0 {
+			framework.RemoveLabelOffNode(c, node2Name, "vsphere_e2e_label")
+		}
+	})
+	/*
+		Steps
+
+		1. Create pod Spec with volume path of the vmdk and NodeSelector set to label assigned to node1.
+		2. Create pod and wait for pod to become ready.
+		3. Verify volume is attached to the node1.
+		4. Create empty file on the volume to verify volume is writable.
+		5. Verify newly created file and previously created files exist on the volume.
+		6. Delete pod.
+		7. Wait for volume to be detached from the node1.
+		8. Repeat Step 1 to 7 and make sure back to back pod creation on same worker node with the same volume is working as expected.
+
+	*/
+
+	It("should create and delete pod with the same volume source on the same worker node", func() {
+		var volumeFiles []string
+		pod := createPodWithVolumeAndNodeSelector(c, ns, vsp, node1Name, node1KeyValueLabel, volumePaths)
+
+		// Create empty files on the mounted volumes on the pod to verify volume is writable
+		// Verify newly and previously created files present on the volume mounted on the pod
+		newEmptyFileName := fmt.Sprintf("/mnt/volume1/%v_1.txt", ns)
+		volumeFiles = append(volumeFiles, newEmptyFileName)
+		createAndVerifyFilesOnVolume(ns, pod.Name, []string{newEmptyFileName}, volumeFiles)
+		deletePodAndWaitForVolumeToDetach(f, c, pod, vsp, node1Name, volumePaths)
+
+		By(fmt.Sprintf("Creating pod on the same node: %v", node1Name))
+		pod = createPodWithVolumeAndNodeSelector(c, ns, vsp, node1Name, node1KeyValueLabel, volumePaths)
+
+		// Create empty files on the mounted volumes on the pod to verify volume is writable
+		// Verify newly and previously created files present on the volume mounted on the pod
+		newEmptyFileName = fmt.Sprintf("/mnt/volume1/%v_2.txt", ns)
+		volumeFiles = append(volumeFiles, newEmptyFileName)
+		createAndVerifyFilesOnVolume(ns, pod.Name, []string{newEmptyFileName}, volumeFiles)
+		deletePodAndWaitForVolumeToDetach(f, c, pod, vsp, node1Name, volumePaths)
+	})
+
+	/*
+		Steps
+
+		1. Create pod Spec with volume path of the vmdk1 and NodeSelector set to node1's label.
+		2. Create pod and wait for POD to become ready.
+		3. Verify volume is attached to the node1.
+		4. Create empty file on the volume to verify volume is writable.
+		5. Verify newly created file and previously created files exist on the volume.
+		6. Delete pod.
+		7. Wait for volume to be detached from the node1.
+		8. Create pod Spec with volume path of the vmdk1 and NodeSelector set to node2's label.
+		9. Create pod and wait for pod to become ready.
+		10. Verify volume is attached to the node2.
+		11. Create empty file on the volume to verify volume is writable.
+		12. Verify newly created file and previously created files exist on the volume.
+		13. Delete pod.
+	*/
+
+	It("should create and delete pod with the same volume source attach/detach to different worker nodes", func() {
+		var volumeFiles []string
+		pod := createPodWithVolumeAndNodeSelector(c, ns, vsp, node1Name, node1KeyValueLabel, volumePaths)
+		// Create empty files on the mounted volumes on the pod to verify volume is writable
+		// Verify newly and previously created files present on the volume mounted on the pod
+		newEmptyFileName := fmt.Sprintf("/mnt/volume1/%v_1.txt", ns)
+		volumeFiles = append(volumeFiles, newEmptyFileName)
+		createAndVerifyFilesOnVolume(ns, pod.Name, []string{newEmptyFileName}, volumeFiles)
+		deletePodAndWaitForVolumeToDetach(f, c, pod, vsp, node1Name, volumePaths)
+
+		By(fmt.Sprintf("Creating pod on the another node: %v", node2Name))
+		pod = createPodWithVolumeAndNodeSelector(c, ns, vsp, node2Name, node2KeyValueLabel, volumePaths)
+
+		newEmptyFileName = fmt.Sprintf("/mnt/volume1/%v_2.txt", ns)
+		volumeFiles = append(volumeFiles, newEmptyFileName)
+		// Create empty files on the mounted volumes on the pod to verify volume is writable
+		// Verify newly and previously created files present on the volume mounted on the pod
+		createAndVerifyFilesOnVolume(ns, pod.Name, []string{newEmptyFileName}, volumeFiles)
+		deletePodAndWaitForVolumeToDetach(f, c, pod, vsp, node2Name, volumePaths)
+	})
+
+	/*
+		Test multiple volumes from same datastore within the same pod
+		1. Create volumes - vmdk2
+		2. Create pod Spec with volume path of vmdk1 (vmdk1 is created in test setup) and vmdk2.
+		3. Create pod using spec created in step-2 and wait for pod to become ready.
+		4. Verify both volumes are attached to the node on which pod are created. Write some data to make sure volume are accessible.
+		5. Delete pod.
+		6. Wait for vmdk1 and vmdk2 to be detached from node.
+		7. Create pod using spec created in step-2 and wait for pod to become ready.
+		8. Verify both volumes are attached to the node on which PODs are created. Verify volume contents are matching with the content written in step 4.
+		9. Delete POD.
+		10. Wait for vmdk1 and vmdk2 to be detached from node.
+	*/
+
+	It("should create and delete pod with multiple volumes from same datastore", func() {
+		By("creating another vmdk")
+		volumePath, err := createVSphereVolume(vsp, nil)
+		Expect(err).NotTo(HaveOccurred())
+		volumePaths = append(volumePaths, volumePath)
+
+		By(fmt.Sprintf("Creating pod on the node: %v with volume: %v and volume: %v", node1Name, volumePaths[0], volumePaths[1]))
+		pod := createPodWithVolumeAndNodeSelector(c, ns, vsp, node1Name, node1KeyValueLabel, volumePaths)
+		// Create empty files on the mounted volumes on the pod to verify volume is writable
+		// Verify newly and previously created files present on the volume mounted on the pod
+		volumeFiles := []string{
+			fmt.Sprintf("/mnt/volume1/%v_1.txt", ns),
+			fmt.Sprintf("/mnt/volume2/%v_1.txt", ns),
+		}
+		createAndVerifyFilesOnVolume(ns, pod.Name, volumeFiles, volumeFiles)
+		deletePodAndWaitForVolumeToDetach(f, c, pod, vsp, node1Name, volumePaths)
+		By(fmt.Sprintf("Creating pod on the node: %v with volume :%v and volume: %v", node1Name, volumePaths[0], volumePaths[1]))
+		pod = createPodWithVolumeAndNodeSelector(c, ns, vsp, node1Name, node1KeyValueLabel, volumePaths)
+		// Create empty files on the mounted volumes on the pod to verify volume is writable
+		// Verify newly and previously created files present on the volume mounted on the pod
+		newEmptyFilesNames := []string{
+			fmt.Sprintf("/mnt/volume1/%v_2.txt", ns),
+			fmt.Sprintf("/mnt/volume2/%v_2.txt", ns),
+		}
+		volumeFiles = append(volumeFiles, newEmptyFilesNames[0])
+		volumeFiles = append(volumeFiles, newEmptyFilesNames[1])
+		createAndVerifyFilesOnVolume(ns, pod.Name, newEmptyFilesNames, volumeFiles)
+	})
+
+	/*
+		Test multiple volumes from different datastore within the same pod
+		1. Create volumes - vmdk2 on non default shared datastore.
+		2. Create pod Spec with volume path of vmdk1 (vmdk1 is created in test setup on default datastore) and vmdk2.
+		3. Create pod using spec created in step-2 and wait for pod to become ready.
+		4. Verify both volumes are attached to the node on which pod are created. Write some data to make sure volume are accessible.
+		5. Delete pod.
+		6. Wait for vmdk1 and vmdk2 to be detached from node.
+		7. Create pod using spec created in step-2 and wait for pod to become ready.
+		8. Verify both volumes are attached to the node on which PODs are created. Verify volume contents are matching with the content written in step 4.
+		9. Delete POD.
+		10. Wait for vmdk1 and vmdk2 to be detached from node.
+	*/
+	It("should create and delete pod with multiple volumes from different datastore", func() {
+		By("creating another vmdk on non default shared datastore")
+		var volumeOptions *vclib.VolumeOptions
+		volumeOptions = new(vclib.VolumeOptions)
+		volumeOptions.CapacityKB = 2097152
+		volumeOptions.Name = "e2e-vmdk-" + strconv.FormatInt(time.Now().UnixNano(), 10)
+		volumeOptions.Datastore = os.Getenv("VSPHERE_SECOND_SHARED_DATASTORE")
+		volumePath, err := createVSphereVolume(vsp, volumeOptions)
+		Expect(err).NotTo(HaveOccurred())
+		volumePaths = append(volumePaths, volumePath)
+
+		By(fmt.Sprintf("Creating pod on the node: %v with volume :%v  and volume: %v", node1Name, volumePaths[0], volumePaths[1]))
+		pod := createPodWithVolumeAndNodeSelector(c, ns, vsp, node1Name, node1KeyValueLabel, volumePaths)
+
+		// Create empty files on the mounted volumes on the pod to verify volume is writable
+		// Verify newly and previously created files present on the volume mounted on the pod
+		volumeFiles := []string{
+			fmt.Sprintf("/mnt/volume1/%v_1.txt", ns),
+			fmt.Sprintf("/mnt/volume2/%v_1.txt", ns),
+		}
+		createAndVerifyFilesOnVolume(ns, pod.Name, volumeFiles, volumeFiles)
+		deletePodAndWaitForVolumeToDetach(f, c, pod, vsp, node1Name, volumePaths)
+
+		By(fmt.Sprintf("Creating pod on the node: %v with volume :%v  and volume: %v", node1Name, volumePaths[0], volumePaths[1]))
+		pod = createPodWithVolumeAndNodeSelector(c, ns, vsp, node1Name, node1KeyValueLabel, volumePaths)
+		// Create empty files on the mounted volumes on the pod to verify volume is writable
+		// Verify newly and previously created files present on the volume mounted on the pod
+		newEmptyFileNames := []string{
+			fmt.Sprintf("/mnt/volume1/%v_2.txt", ns),
+			fmt.Sprintf("/mnt/volume2/%v_2.txt", ns),
+		}
+		volumeFiles = append(volumeFiles, newEmptyFileNames[0])
+		volumeFiles = append(volumeFiles, newEmptyFileNames[1])
+		createAndVerifyFilesOnVolume(ns, pod.Name, newEmptyFileNames, volumeFiles)
+		deletePodAndWaitForVolumeToDetach(f, c, pod, vsp, node1Name, volumePaths)
+	})
+
+	/*
+		Test Back-to-back pod creation/deletion with different volume sources on the same worker node
+		    1. Create volumes - vmdk2
+		    2. Create pod Spec - pod-SpecA with volume path of vmdk1 and NodeSelector set to label assigned to node1.
+		    3. Create pod Spec - pod-SpecB with volume path of vmdk2 and NodeSelector set to label assigned to node1.
+		    4. Create pod-A using pod-SpecA and wait for pod to become ready.
+		    5. Create pod-B using pod-SpecB and wait for POD to become ready.
+		    6. Verify volumes are attached to the node.
+		    7. Create empty file on the volume to make sure volume is accessible. (Perform this step on pod-A and pod-B)
+		    8. Verify file created in step 5 is present on the volume. (perform this step on pod-A and pod-B)
+		    9. Delete pod-A and pod-B
+		    10. Repeatedly (5 times) perform step 4 to 9 and verify associated volume's content is matching.
+		    11. Wait for vmdk1 and vmdk2 to be detached from node.
+	*/
+	It("test back to back pod creation and deletion with different volume sources on the same worker node", func() {
+		var (
+			podA                *v1.Pod
+			podB                *v1.Pod
+			testvolumePathsPodA []string
+			testvolumePathsPodB []string
+			podAFiles           []string
+			podBFiles           []string
+		)
+
+		defer func() {
+			By("clean up undeleted pods")
+			framework.ExpectNoError(framework.DeletePodWithWait(f, c, podA), "defer: Failed to delete pod ", podA.Name)
+			framework.ExpectNoError(framework.DeletePodWithWait(f, c, podB), "defer: Failed to delete pod ", podB.Name)
+			By(fmt.Sprintf("wait for volumes to be detached from the node: %v", node1Name))
+			for _, volumePath := range volumePaths {
+				framework.ExpectNoError(waitForVSphereDiskToDetach(vsp, volumePath, types.NodeName(node1Name)))
+			}
+		}()
+
+		testvolumePathsPodA = append(testvolumePathsPodA, volumePaths[0])
+		// Create another VMDK Volume
+		By("creating another vmdk")
+		volumePath, err := createVSphereVolume(vsp, nil)
+		Expect(err).NotTo(HaveOccurred())
+		volumePaths = append(volumePaths, volumePath)
+		testvolumePathsPodB = append(testvolumePathsPodA, volumePath)
+
+		for index := 0; index < 5; index++ {
+			By(fmt.Sprintf("Creating pod-A on the node: %v with volume: %v", node1Name, testvolumePathsPodA[0]))
+			podA = createPodWithVolumeAndNodeSelector(c, ns, vsp, node1Name, node1KeyValueLabel, testvolumePathsPodA)
+
+			By(fmt.Sprintf("Creating pod-B on the node: %v with volume: %v", node1Name, testvolumePathsPodB[0]))
+			podB = createPodWithVolumeAndNodeSelector(c, ns, vsp, node1Name, node1KeyValueLabel, testvolumePathsPodB)
+
+			podAFileName := fmt.Sprintf("/mnt/volume1/podA_%v_%v.txt", ns, index+1)
+			podBFileName := fmt.Sprintf("/mnt/volume1/podB_%v_%v.txt", ns, index+1)
+			podAFiles = append(podAFiles, podAFileName)
+			podBFiles = append(podBFiles, podBFileName)
+
+			// Create empty files on the mounted volumes on the pod to verify volume is writable
+			By("Creating empty file on volume mounted on pod-A")
+			framework.CreateEmptyFileOnPod(ns, podA.Name, podAFileName)
+
+			By("Creating empty file volume mounted on pod-B")
+			framework.CreateEmptyFileOnPod(ns, podB.Name, podBFileName)
+
+			// Verify newly and previously created files present on the volume mounted on the pod
+			By("Verify newly Created file and previously created files present on volume mounted on pod-A")
+			verifyFilesExistOnVSphereVolume(ns, podA.Name, podAFiles)
+			By("Verify newly Created file and previously created files present on volume mounted on pod-B")
+			verifyFilesExistOnVSphereVolume(ns, podB.Name, podBFiles)
+
+			By("Deleting pod-A")
+			framework.ExpectNoError(framework.DeletePodWithWait(f, c, podA), "Failed to delete pod ", podA.Name)
+			By("Deleting pod-B")
+			framework.ExpectNoError(framework.DeletePodWithWait(f, c, podB), "Failed to delete pod ", podB.Name)
+		}
+	})
+})
+
+func testSetupVolumePlacement(client clientset.Interface, namespace string) (node1Name string, node1KeyValueLabel map[string]string, node2Name string, node2KeyValueLabel map[string]string) {
+	nodes := framework.GetReadySchedulableNodesOrDie(client)
+	if len(nodes.Items) < 2 {
+		framework.Skipf("Requires at least %d nodes (not %d)", 2, len(nodes.Items))
+	}
+	node1Name = nodes.Items[0].Name
+	node2Name = nodes.Items[1].Name
+	node1LabelValue := "vsphere_e2e_" + string(uuid.NewUUID())
+	node1KeyValueLabel = make(map[string]string)
+	node1KeyValueLabel["vsphere_e2e_label"] = node1LabelValue
+	framework.AddOrUpdateLabelOnNode(client, node1Name, "vsphere_e2e_label", node1LabelValue)
+
+	node2LabelValue := "vsphere_e2e_" + string(uuid.NewUUID())
+	node2KeyValueLabel = make(map[string]string)
+	node2KeyValueLabel["vsphere_e2e_label"] = node2LabelValue
+	framework.AddOrUpdateLabelOnNode(client, node2Name, "vsphere_e2e_label", node2LabelValue)
+	return node1Name, node1KeyValueLabel, node2Name, node2KeyValueLabel
+}
+
+func createPodWithVolumeAndNodeSelector(client clientset.Interface, namespace string, vsp *vsphere.VSphere, nodeName string, nodeKeyValueLabel map[string]string, volumePaths []string) *v1.Pod {
+	var pod *v1.Pod
+	var err error
+	By(fmt.Sprintf("Creating pod on the node: %v", nodeName))
+	podspec := getVSpherePodSpecWithVolumePaths(volumePaths, nodeKeyValueLabel, nil)
+
+	pod, err = client.CoreV1().Pods(namespace).Create(podspec)
+	Expect(err).NotTo(HaveOccurred())
+	By("Waiting for pod to be ready")
+	Expect(framework.WaitForPodNameRunningInNamespace(client, pod.Name, namespace)).To(Succeed())
+
+	By(fmt.Sprintf("Verify volume is attached to the node:%v", nodeName))
+	for _, volumePath := range volumePaths {
+		isAttached, err := verifyVSphereDiskAttached(vsp, volumePath, types.NodeName(nodeName))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(isAttached).To(BeTrue(), "disk:"+volumePath+" is not attached with the node")
+	}
+	return pod
+}
+
+func createAndVerifyFilesOnVolume(namespace string, podname string, newEmptyfilesToCreate []string, filesToCheck []string) {
+	// Create empty files on the mounted volumes on the pod to verify volume is writable
+	By(fmt.Sprintf("Creating empty file on volume mounted on: %v", podname))
+	createEmptyFilesOnVSphereVolume(namespace, podname, newEmptyfilesToCreate)
+
+	// Verify newly and previously created files present on the volume mounted on the pod
+	By(fmt.Sprintf("Verify newly Created file and previously created files present on volume mounted on: %v", podname))
+	verifyFilesExistOnVSphereVolume(namespace, podname, filesToCheck)
+}
+
+func deletePodAndWaitForVolumeToDetach(f *framework.Framework, c clientset.Interface, pod *v1.Pod, vsp *vsphere.VSphere, nodeName string, volumePaths []string) {
+	By("Deleting pod")
+	framework.ExpectNoError(framework.DeletePodWithWait(f, c, pod), "Failed to delete pod ", pod.Name)
+
+	By("Waiting for volume to be detached from the node")
+	for _, volumePath := range volumePaths {
+		framework.ExpectNoError(waitForVSphereDiskToDetach(vsp, volumePath, types.NodeName(nodeName)))
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/storage/vsphere_volume_vsan_policy.go

@@ -0,0 +1,309 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"strings"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8stype "k8s.io/apimachinery/pkg/types"
+	clientset "k8s.io/client-go/kubernetes"
+	vsphere "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+const (
+	VmfsDatastore                              = "sharedVmfs-0"
+	VsanDatastore                              = "vsanDatastore"
+	Datastore                                  = "datastore"
+	Policy_DiskStripes                         = "diskStripes"
+	Policy_HostFailuresToTolerate              = "hostFailuresToTolerate"
+	Policy_CacheReservation                    = "cacheReservation"
+	Policy_ObjectSpaceReservation              = "objectSpaceReservation"
+	Policy_IopsLimit                           = "iopsLimit"
+	DiskFormat                                 = "diskformat"
+	ThinDisk                                   = "thin"
+	SpbmStoragePolicy                          = "storagepolicyname"
+	BronzeStoragePolicy                        = "bronze"
+	HostFailuresToTolerateCapabilityVal        = "0"
+	CacheReservationCapabilityVal              = "20"
+	DiskStripesCapabilityVal                   = "1"
+	ObjectSpaceReservationCapabilityVal        = "30"
+	IopsLimitCapabilityVal                     = "100"
+	StripeWidthCapabilityVal                   = "2"
+	DiskStripesCapabilityInvalidVal            = "14"
+	HostFailuresToTolerateCapabilityInvalidVal = "4"
+)
+
+/*
+   Test to verify the storage policy based management for dynamic volume provisioning inside kubernetes.
+   There are 2 ways to achive it:
+   1. Specify VSAN storage capabilities in the storage-class.
+   2. Use existing vCenter SPBM storage policies.
+
+   Valid VSAN storage capabilities are mentioned below:
+   1. hostFailuresToTolerate
+   2. forceProvisioning
+   3. cacheReservation
+   4. diskStripes
+   5. objectSpaceReservation
+   6. iopsLimit
+
+   Steps
+   1. Create StorageClass with.
+   		a. VSAN storage capabilities set to valid/invalid values (or)
+		b. Use existing vCenter SPBM storage policies.
+   2. Create PVC which uses the StorageClass created in step 1.
+   3. Wait for PV to be provisioned.
+   4. Wait for PVC's status to become Bound
+   5. Create pod using PVC on specific node.
+   6. Wait for Disk to be attached to the node.
+   7. Delete pod and Wait for Volume Disk to be detached from the Node.
+   8. Delete PVC, PV and Storage Class
+*/
+
+var _ = SIGDescribe("vSphere Storage policy support for dynamic provisioning", func() {
+	f := framework.NewDefaultFramework("volume-vsan-policy")
+	var (
+		client       clientset.Interface
+		namespace    string
+		scParameters map[string]string
+	)
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("vsphere")
+		client = f.ClientSet
+		namespace = f.Namespace.Name
+		scParameters = make(map[string]string)
+		nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+		if !(len(nodeList.Items) > 0) {
+			framework.Failf("Unable to find ready and schedulable Node")
+		}
+	})
+
+	// Valid policy.
+	It("verify VSAN storage capability with valid hostFailuresToTolerate and cacheReservation values is honored for dynamically provisioned pvc using storageclass", func() {
+		By(fmt.Sprintf("Invoking Test for VSAN policy hostFailuresToTolerate: %s, cacheReservation: %s", HostFailuresToTolerateCapabilityVal, CacheReservationCapabilityVal))
+		scParameters[Policy_HostFailuresToTolerate] = HostFailuresToTolerateCapabilityVal
+		scParameters[Policy_CacheReservation] = CacheReservationCapabilityVal
+		framework.Logf("Invoking Test for VSAN storage capabilities: %+v", scParameters)
+		invokeValidPolicyTest(f, client, namespace, scParameters)
+	})
+
+	// Valid policy.
+	It("verify VSAN storage capability with valid diskStripes and objectSpaceReservation values is honored for dynamically provisioned pvc using storageclass", func() {
+		By(fmt.Sprintf("Invoking Test for VSAN policy diskStripes: %s, objectSpaceReservation: %s", DiskStripesCapabilityVal, ObjectSpaceReservationCapabilityVal))
+		scParameters[Policy_DiskStripes] = "1"
+		scParameters[Policy_ObjectSpaceReservation] = "30"
+		framework.Logf("Invoking Test for VSAN storage capabilities: %+v", scParameters)
+		invokeValidPolicyTest(f, client, namespace, scParameters)
+	})
+
+	// Valid policy.
+	It("verify VSAN storage capability with valid diskStripes and objectSpaceReservation values and a VSAN datastore is honored for dynamically provisioned pvc using storageclass", func() {
+		By(fmt.Sprintf("Invoking Test for VSAN policy diskStripes: %s, objectSpaceReservation: %s", DiskStripesCapabilityVal, ObjectSpaceReservationCapabilityVal))
+		scParameters[Policy_DiskStripes] = DiskStripesCapabilityVal
+		scParameters[Policy_ObjectSpaceReservation] = ObjectSpaceReservationCapabilityVal
+		scParameters[Datastore] = VsanDatastore
+		framework.Logf("Invoking Test for VSAN storage capabilities: %+v", scParameters)
+		invokeValidPolicyTest(f, client, namespace, scParameters)
+	})
+
+	// Valid policy.
+	It("verify VSAN storage capability with valid objectSpaceReservation and iopsLimit values is honored for dynamically provisioned pvc using storageclass", func() {
+		By(fmt.Sprintf("Invoking Test for VSAN policy objectSpaceReservation: %s, iopsLimit: %s", ObjectSpaceReservationCapabilityVal, IopsLimitCapabilityVal))
+		scParameters[Policy_ObjectSpaceReservation] = ObjectSpaceReservationCapabilityVal
+		scParameters[Policy_IopsLimit] = IopsLimitCapabilityVal
+		framework.Logf("Invoking Test for VSAN storage capabilities: %+v", scParameters)
+		invokeValidPolicyTest(f, client, namespace, scParameters)
+	})
+
+	// Invalid VSAN storage capabilties parameters.
+	It("verify VSAN storage capability with invalid capability name objectSpaceReserve is not honored for dynamically provisioned pvc using storageclass", func() {
+		By(fmt.Sprintf("Invoking Test for VSAN policy objectSpaceReserve: %s, stripeWidth: %s", ObjectSpaceReservationCapabilityVal, StripeWidthCapabilityVal))
+		scParameters["objectSpaceReserve"] = ObjectSpaceReservationCapabilityVal
+		scParameters[Policy_DiskStripes] = StripeWidthCapabilityVal
+		framework.Logf("Invoking Test for VSAN storage capabilities: %+v", scParameters)
+		err := invokeInvalidPolicyTestNeg(client, namespace, scParameters)
+		Expect(err).To(HaveOccurred())
+		errorMsg := "invalid option \\\"objectSpaceReserve\\\" for volume plugin kubernetes.io/vsphere-volume"
+		if !strings.Contains(err.Error(), errorMsg) {
+			Expect(err).NotTo(HaveOccurred(), errorMsg)
+		}
+	})
+
+	// Invalid policy on a VSAN test bed.
+	// diskStripes value has to be between 1 and 12.
+	It("verify VSAN storage capability with invalid diskStripes value is not honored for dynamically provisioned pvc using storageclass", func() {
+		By(fmt.Sprintf("Invoking Test for VSAN policy diskStripes: %s, cacheReservation: %s", DiskStripesCapabilityInvalidVal, CacheReservationCapabilityVal))
+		scParameters[Policy_DiskStripes] = DiskStripesCapabilityInvalidVal
+		scParameters[Policy_CacheReservation] = CacheReservationCapabilityVal
+		framework.Logf("Invoking Test for VSAN storage capabilities: %+v", scParameters)
+		err := invokeInvalidPolicyTestNeg(client, namespace, scParameters)
+		Expect(err).To(HaveOccurred())
+		errorMsg := "Invalid value for " + Policy_DiskStripes + "."
+		if !strings.Contains(err.Error(), errorMsg) {
+			Expect(err).NotTo(HaveOccurred(), errorMsg)
+		}
+	})
+
+	// Invalid policy on a VSAN test bed.
+	// hostFailuresToTolerate value has to be between 0 and 3 including.
+	It("verify VSAN storage capability with invalid hostFailuresToTolerate value is not honored for dynamically provisioned pvc using storageclass", func() {
+		By(fmt.Sprintf("Invoking Test for VSAN policy hostFailuresToTolerate: %s", HostFailuresToTolerateCapabilityInvalidVal))
+		scParameters[Policy_HostFailuresToTolerate] = HostFailuresToTolerateCapabilityInvalidVal
+		framework.Logf("Invoking Test for VSAN storage capabilities: %+v", scParameters)
+		err := invokeInvalidPolicyTestNeg(client, namespace, scParameters)
+		Expect(err).To(HaveOccurred())
+		errorMsg := "Invalid value for " + Policy_HostFailuresToTolerate + "."
+		if !strings.Contains(err.Error(), errorMsg) {
+			Expect(err).NotTo(HaveOccurred(), errorMsg)
+		}
+	})
+
+	// Specify a valid VSAN policy on a non-VSAN test bed.
+	// The test should fail.
+	It("verify VSAN storage capability with non-vsan datastore is not honored for dynamically provisioned pvc using storageclass", func() {
+		By(fmt.Sprintf("Invoking Test for VSAN policy diskStripes: %s, objectSpaceReservation: %s and a non-VSAN datastore: %s", DiskStripesCapabilityVal, ObjectSpaceReservationCapabilityVal, VmfsDatastore))
+		scParameters[Policy_DiskStripes] = DiskStripesCapabilityVal
+		scParameters[Policy_ObjectSpaceReservation] = ObjectSpaceReservationCapabilityVal
+		scParameters[Datastore] = VmfsDatastore
+		framework.Logf("Invoking Test for VSAN storage capabilities: %+v", scParameters)
+		err := invokeInvalidPolicyTestNeg(client, namespace, scParameters)
+		Expect(err).To(HaveOccurred())
+		errorMsg := "The specified datastore: \\\"" + VmfsDatastore + "\\\" is not a VSAN datastore. " +
+			"The policy parameters will work only with VSAN Datastore."
+		if !strings.Contains(err.Error(), errorMsg) {
+			Expect(err).NotTo(HaveOccurred(), errorMsg)
+		}
+	})
+
+	It("verify an existing and compatible SPBM policy is honored for dynamically provisioned pvc using storageclass", func() {
+		By(fmt.Sprintf("Invoking Test for SPBM policy: %s", os.Getenv("VSPHERE_SPBM_GOLD_POLICY")))
+		goldPolicy := os.Getenv("VSPHERE_SPBM_GOLD_POLICY")
+		Expect(goldPolicy).NotTo(BeEmpty())
+		scParameters[SpbmStoragePolicy] = goldPolicy
+		scParameters[DiskFormat] = ThinDisk
+		framework.Logf("Invoking Test for SPBM storage policy: %+v", scParameters)
+		invokeValidPolicyTest(f, client, namespace, scParameters)
+	})
+
+	It("verify if a SPBM policy is not honored on a non-compatible datastore for dynamically provisioned pvc using storageclass", func() {
+		By(fmt.Sprintf("Invoking Test for SPBM policy: %s and datastore: %s", os.Getenv("VSPHERE_SPBM_TAG_POLICY"), VsanDatastore))
+		tagPolicy := os.Getenv("VSPHERE_SPBM_TAG_POLICY")
+		Expect(tagPolicy).NotTo(BeEmpty())
+		scParameters[SpbmStoragePolicy] = tagPolicy
+		scParameters[Datastore] = VsanDatastore
+		scParameters[DiskFormat] = ThinDisk
+		framework.Logf("Invoking Test for SPBM storage policy on a non-compatible datastore: %+v", scParameters)
+		err := invokeInvalidPolicyTestNeg(client, namespace, scParameters)
+		Expect(err).To(HaveOccurred())
+		errorMsg := "User specified datastore is not compatible with the storagePolicy: \\\"" + os.Getenv("VSPHERE_SPBM_TAG_POLICY") + "\\\""
+		if !strings.Contains(err.Error(), errorMsg) {
+			Expect(err).NotTo(HaveOccurred(), errorMsg)
+		}
+	})
+
+	It("verify if a non-existing SPBM policy is not honored for dynamically provisioned pvc using storageclass", func() {
+		By(fmt.Sprintf("Invoking Test for SPBM policy: %s", BronzeStoragePolicy))
+		scParameters[SpbmStoragePolicy] = BronzeStoragePolicy
+		scParameters[DiskFormat] = ThinDisk
+		framework.Logf("Invoking Test for non-existing SPBM storage policy: %+v", scParameters)
+		err := invokeInvalidPolicyTestNeg(client, namespace, scParameters)
+		Expect(err).To(HaveOccurred())
+		errorMsg := "no pbm profile found with name: \\\"" + BronzeStoragePolicy + "\\"
+		if !strings.Contains(err.Error(), errorMsg) {
+			Expect(err).NotTo(HaveOccurred(), errorMsg)
+		}
+	})
+
+	It("verify an if a SPBM policy and VSAN capabilities cannot be honored for dynamically provisioned pvc using storageclass", func() {
+		By(fmt.Sprintf("Invoking Test for SPBM policy: %s with VSAN storage capabilities", os.Getenv("VSPHERE_SPBM_GOLD_POLICY")))
+		goldPolicy := os.Getenv("VSPHERE_SPBM_GOLD_POLICY")
+		Expect(goldPolicy).NotTo(BeEmpty())
+		scParameters[SpbmStoragePolicy] = goldPolicy
+		Expect(scParameters[SpbmStoragePolicy]).NotTo(BeEmpty())
+		scParameters[Policy_DiskStripes] = DiskStripesCapabilityVal
+		scParameters[DiskFormat] = ThinDisk
+		framework.Logf("Invoking Test for SPBM storage policy and VSAN capabilities together: %+v", scParameters)
+		err := invokeInvalidPolicyTestNeg(client, namespace, scParameters)
+		Expect(err).To(HaveOccurred())
+		errorMsg := "Cannot specify storage policy capabilities along with storage policy name. Please specify only one"
+		if !strings.Contains(err.Error(), errorMsg) {
+			Expect(err).NotTo(HaveOccurred(), errorMsg)
+		}
+	})
+})
+
+func invokeValidPolicyTest(f *framework.Framework, client clientset.Interface, namespace string, scParameters map[string]string) {
+	By("Creating Storage Class With storage policy params")
+	storageclass, err := client.StorageV1().StorageClasses().Create(getVSphereStorageClassSpec("storagepolicysc", scParameters))
+	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to create storage class with err: %v", err))
+	defer client.StorageV1().StorageClasses().Delete(storageclass.Name, nil)
+
+	By("Creating PVC using the Storage Class")
+	pvclaim, err := framework.CreatePVC(client, namespace, getVSphereClaimSpecWithStorageClassAnnotation(namespace, storageclass))
+	Expect(err).NotTo(HaveOccurred())
+	defer framework.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace)
+
+	var pvclaims []*v1.PersistentVolumeClaim
+	pvclaims = append(pvclaims, pvclaim)
+	By("Waiting for claim to be in bound phase")
+	persistentvolumes, err := framework.WaitForPVClaimBoundPhase(client, pvclaims)
+	Expect(err).NotTo(HaveOccurred())
+
+	By("Creating pod to attach PV to the node")
+	// Create pod to attach Volume to Node
+	pod, err := framework.CreatePod(client, namespace, pvclaims, false, "")
+	Expect(err).NotTo(HaveOccurred())
+
+	vsp, err := vsphere.GetVSphere()
+	Expect(err).NotTo(HaveOccurred())
+	By("Verify the volume is accessible and available in the pod")
+	verifyVSphereVolumesAccessible(pod, persistentvolumes, vsp)
+
+	By("Deleting pod")
+	framework.DeletePodWithWait(f, client, pod)
+
+	By("Waiting for volumes to be detached from the node")
+	waitForVSphereDiskToDetach(vsp, persistentvolumes[0].Spec.VsphereVolume.VolumePath, k8stype.NodeName(pod.Spec.NodeName))
+}
+
+func invokeInvalidPolicyTestNeg(client clientset.Interface, namespace string, scParameters map[string]string) error {
+	By("Creating Storage Class With storage policy params")
+	storageclass, err := client.StorageV1().StorageClasses().Create(getVSphereStorageClassSpec("storagepolicysc", scParameters))
+	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to create storage class with err: %v", err))
+	defer client.StorageV1().StorageClasses().Delete(storageclass.Name, nil)
+
+	By("Creating PVC using the Storage Class")
+	pvclaim, err := framework.CreatePVC(client, namespace, getVSphereClaimSpecWithStorageClassAnnotation(namespace, storageclass))
+	Expect(err).NotTo(HaveOccurred())
+	defer framework.DeletePersistentVolumeClaim(client, pvclaim.Name, namespace)
+
+	By("Waiting for claim to be in bound phase")
+	err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, pvclaim.Namespace, pvclaim.Name, framework.Poll, 2*time.Minute)
+	Expect(err).To(HaveOccurred())
+
+	eventList, err := client.CoreV1().Events(pvclaim.Namespace).List(metav1.ListOptions{})
+	return fmt.Errorf("Failure message: %+q", eventList.Items[0].Message)
+}