Replace godep with dep

vendor/k8s.io/kubernetes/pkg/volume/rbd/BUILD

@@ -0,0 +1,63 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "disk_manager.go",
+        "doc.go",
+        "rbd.go",
+        "rbd_util.go",
+    ],
+    deps = [
+        "//pkg/util/mount:go_default_library",
+        "//pkg/util/node:go_default_library",
+        "//pkg/util/strings:go_default_library",
+        "//pkg/volume:go_default_library",
+        "//pkg/volume/util:go_default_library",
+        "//pkg/volume/util/volumehelper:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["rbd_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//pkg/util/mount:go_default_library",
+        "//pkg/volume:go_default_library",
+        "//pkg/volume/testing:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
+        "//vendor/k8s.io/client-go/util/testing:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/volume/rbd/OWNERS

@@ -0,0 +1,15 @@
+approvers:
+- rootfs
+reviewers:
+- sjenning
+- thockin
+- smarterclayton
+- brendandburns
+- derekwaynecarr
+- pmorie
+- saad-ali
+- justinsb
+- jsafrane
+- rootfs
+- jingxu97
+- msau42

vendor/k8s.io/kubernetes/pkg/volume/rbd/disk_manager.go

@@ -0,0 +1,131 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+//
+// diskManager interface and diskSetup/TearDown functions abstract commonly used procedures to setup a block volume
+// rbd volume implements diskManager, calls diskSetup when creating a volume, and calls diskTearDown inside volume unmounter.
+// TODO: consolidate, refactor, and share diskManager among iSCSI, GCE PD, and RBD
+//
+
+package rbd
+
+import (
+	"os"
+
+	"github.com/golang/glog"
+	"k8s.io/api/core/v1"
+	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/volume"
+)
+
+// Abstract interface to disk operations.
+type diskManager interface {
+	MakeGlobalPDName(disk rbd) string
+	// Attaches the disk to the kubelet's host machine.
+	AttachDisk(disk rbdMounter) error
+	// Detaches the disk from the kubelet's host machine.
+	DetachDisk(disk rbdUnmounter, mntPath string) error
+	// Creates a rbd image
+	CreateImage(provisioner *rbdVolumeProvisioner) (r *v1.RBDVolumeSource, volumeSizeGB int, err error)
+	// Deletes a rbd image
+	DeleteImage(deleter *rbdVolumeDeleter) error
+}
+
+// utility to mount a disk based filesystem
+func diskSetUp(manager diskManager, b rbdMounter, volPath string, mounter mount.Interface, fsGroup *int64) error {
+	globalPDPath := manager.MakeGlobalPDName(*b.rbd)
+	// TODO: handle failed mounts here.
+	notMnt, err := mounter.IsLikelyNotMountPoint(volPath)
+
+	if err != nil && !os.IsNotExist(err) {
+		glog.Errorf("cannot validate mountpoint: %s", volPath)
+		return err
+	}
+	if !notMnt {
+		return nil
+	}
+	if err := manager.AttachDisk(b); err != nil {
+		glog.Errorf("failed to attach disk")
+		return err
+	}
+
+	if err := os.MkdirAll(volPath, 0750); err != nil {
+		glog.Errorf("failed to mkdir:%s", volPath)
+		return err
+	}
+	// Perform a bind mount to the full path to allow duplicate mounts of the same disk.
+	options := []string{"bind"}
+	if (&b).GetAttributes().ReadOnly {
+		options = append(options, "ro")
+	}
+	mountOptions := volume.JoinMountOptions(b.mountOptions, options)
+	err = mounter.Mount(globalPDPath, volPath, "", mountOptions)
+	if err != nil {
+		glog.Errorf("failed to bind mount:%s", globalPDPath)
+		return err
+	}
+	glog.V(3).Infof("rbd: successfully bind mount %s to %s with options %v", globalPDPath, volPath, mountOptions)
+
+	if !b.ReadOnly {
+		volume.SetVolumeOwnership(&b, fsGroup)
+	}
+
+	return nil
+}
+
+// utility to tear down a disk based filesystem
+func diskTearDown(manager diskManager, c rbdUnmounter, volPath string, mounter mount.Interface) error {
+	notMnt, err := mounter.IsLikelyNotMountPoint(volPath)
+	if err != nil {
+		glog.Errorf("cannot validate mountpoint %s", volPath)
+		return err
+	}
+	if notMnt {
+		return os.Remove(volPath)
+	}
+
+	refs, err := mount.GetMountRefs(mounter, volPath)
+	if err != nil {
+		glog.Errorf("failed to get reference count %s", volPath)
+		return err
+	}
+	if err := mounter.Unmount(volPath); err != nil {
+		glog.Errorf("failed to umount %s", volPath)
+		return err
+	}
+	// If len(refs) is 1, then all bind mounts have been removed, and the
+	// remaining reference is the global mount. It is safe to detach.
+	if len(refs) == 1 {
+		mntPath := refs[0]
+		if err := manager.DetachDisk(c, mntPath); err != nil {
+			glog.Errorf("failed to detach disk from %s", mntPath)
+			return err
+		}
+	}
+
+	notMnt, mntErr := mounter.IsLikelyNotMountPoint(volPath)
+	if mntErr != nil {
+		glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
+		return err
+	}
+	if notMnt {
+		if err := os.Remove(volPath); err != nil {
+			return err
+		}
+	}
+	return nil
+
+}

vendor/k8s.io/kubernetes/pkg/volume/rbd/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package rbd contains the internal representation of Rados Block Store (Ceph)
+// volumes.
+package rbd // import "k8s.io/kubernetes/pkg/volume/rbd"

vendor/k8s.io/kubernetes/pkg/volume/rbd/rbd.go

@@ -0,0 +1,533 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"fmt"
+	dstrings "strings"
+
+	"github.com/golang/glog"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/util/strings"
+	"k8s.io/kubernetes/pkg/volume"
+	volutil "k8s.io/kubernetes/pkg/volume/util"
+	"k8s.io/kubernetes/pkg/volume/util/volumehelper"
+)
+
+var (
+	supportedFeatures = sets.NewString("layering")
+)
+
+// This is the primary entrypoint for volume plugins.
+func ProbeVolumePlugins() []volume.VolumePlugin {
+	return []volume.VolumePlugin{&rbdPlugin{nil}}
+}
+
+type rbdPlugin struct {
+	host volume.VolumeHost
+}
+
+var _ volume.VolumePlugin = &rbdPlugin{}
+var _ volume.PersistentVolumePlugin = &rbdPlugin{}
+var _ volume.DeletableVolumePlugin = &rbdPlugin{}
+var _ volume.ProvisionableVolumePlugin = &rbdPlugin{}
+
+const (
+	rbdPluginName                  = "kubernetes.io/rbd"
+	secretKeyName                  = "key" // key name used in secret
+	rbdImageFormat1                = "1"
+	rbdImageFormat2                = "2"
+	rbdDefaultAdminId              = "admin"
+	rbdDefaultAdminSecretNamespace = "default"
+	rbdDefaultPool                 = "rbd"
+	rbdDefaultUserId               = rbdDefaultAdminId
+)
+
+func getPath(uid types.UID, volName string, host volume.VolumeHost) string {
+	return host.GetPodVolumeDir(uid, strings.EscapeQualifiedNameForDisk(rbdPluginName), volName)
+}
+
+func (plugin *rbdPlugin) Init(host volume.VolumeHost) error {
+	plugin.host = host
+	return nil
+}
+
+func (plugin *rbdPlugin) GetPluginName() string {
+	return rbdPluginName
+}
+
+func (plugin *rbdPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
+	volumeSource, _, err := getVolumeSource(spec)
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf(
+		"%v:%v",
+		volumeSource.CephMonitors,
+		volumeSource.RBDImage), nil
+}
+
+func (plugin *rbdPlugin) CanSupport(spec *volume.Spec) bool {
+	if (spec.Volume != nil && spec.Volume.RBD == nil) || (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD == nil) {
+		return false
+	}
+
+	return true
+}
+
+func (plugin *rbdPlugin) RequiresRemount() bool {
+	return false
+}
+
+func (plugin *rbdPlugin) SupportsMountOption() bool {
+	return true
+}
+
+func (plugin *rbdPlugin) SupportsBulkVolumeVerification() bool {
+	return false
+}
+
+func (plugin *rbdPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
+	return []v1.PersistentVolumeAccessMode{
+		v1.ReadWriteOnce,
+		v1.ReadOnlyMany,
+	}
+}
+
+func (plugin *rbdPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
+	var secret string
+	var err error
+	source, _ := plugin.getRBDVolumeSource(spec)
+
+	if source.SecretRef != nil {
+		if secret, err = parsePodSecret(pod, source.SecretRef.Name, plugin.host.GetKubeClient()); err != nil {
+			glog.Errorf("Couldn't get secret from %v/%v", pod.Namespace, source.SecretRef)
+			return nil, err
+		}
+	}
+
+	// Inject real implementations here, test through the internal function.
+	return plugin.newMounterInternal(spec, pod.UID, &RBDUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()), secret)
+}
+
+func (plugin *rbdPlugin) getRBDVolumeSource(spec *volume.Spec) (*v1.RBDVolumeSource, bool) {
+	// rbd volumes used directly in a pod have a ReadOnly flag set by the pod author.
+	// rbd volumes used as a PersistentVolume gets the ReadOnly flag indirectly through the persistent-claim volume used to mount the PV
+	if spec.Volume != nil && spec.Volume.RBD != nil {
+		return spec.Volume.RBD, spec.Volume.RBD.ReadOnly
+	} else {
+		return spec.PersistentVolume.Spec.RBD, spec.ReadOnly
+	}
+}
+
+func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, exec mount.Exec, secret string) (volume.Mounter, error) {
+	source, readOnly := plugin.getRBDVolumeSource(spec)
+	pool := source.RBDPool
+	id := source.RadosUser
+	keyring := source.Keyring
+
+	return &rbdMounter{
+		rbd: &rbd{
+			podUID:          podUID,
+			volName:         spec.Name(),
+			Image:           source.RBDImage,
+			Pool:            pool,
+			ReadOnly:        readOnly,
+			manager:         manager,
+			mounter:         &mount.SafeFormatAndMount{Interface: mounter, Exec: exec},
+			exec:            exec,
+			plugin:          plugin,
+			MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)),
+		},
+		Mon:          source.CephMonitors,
+		Id:           id,
+		Keyring:      keyring,
+		Secret:       secret,
+		fsType:       source.FSType,
+		mountOptions: volume.MountOptionFromSpec(spec),
+	}, nil
+}
+
+func (plugin *rbdPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
+	// Inject real implementations here, test through the internal function.
+	return plugin.newUnmounterInternal(volName, podUID, &RBDUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()))
+}
+
+func (plugin *rbdPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager, mounter mount.Interface, exec mount.Exec) (volume.Unmounter, error) {
+	return &rbdUnmounter{
+		rbdMounter: &rbdMounter{
+			rbd: &rbd{
+				podUID:          podUID,
+				volName:         volName,
+				manager:         manager,
+				mounter:         &mount.SafeFormatAndMount{Interface: mounter, Exec: exec},
+				exec:            exec,
+				plugin:          plugin,
+				MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host)),
+			},
+			Mon: make([]string, 0),
+		},
+	}, nil
+}
+
+func (plugin *rbdPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+	rbdVolume := &v1.Volume{
+		Name: volumeName,
+		VolumeSource: v1.VolumeSource{
+			RBD: &v1.RBDVolumeSource{
+				CephMonitors: []string{},
+			},
+		},
+	}
+	return volume.NewSpecFromVolume(rbdVolume), nil
+}
+
+func (plugin *rbdPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) {
+	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD == nil {
+		return nil, fmt.Errorf("spec.PersistentVolumeSource.Spec.RBD is nil")
+	}
+	class, err := volutil.GetClassForVolume(plugin.host.GetKubeClient(), spec.PersistentVolume)
+	if err != nil {
+		return nil, err
+	}
+	adminSecretName := ""
+	adminSecretNamespace := rbdDefaultAdminSecretNamespace
+	admin := ""
+
+	for k, v := range class.Parameters {
+		switch dstrings.ToLower(k) {
+		case "adminid":
+			admin = v
+		case "adminsecretname":
+			adminSecretName = v
+		case "adminsecretnamespace":
+			adminSecretNamespace = v
+		}
+	}
+
+	if admin == "" {
+		admin = rbdDefaultAdminId
+	}
+	secret, err := parsePVSecret(adminSecretNamespace, adminSecretName, plugin.host.GetKubeClient())
+	if err != nil {
+		return nil, fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
+	}
+	return plugin.newDeleterInternal(spec, admin, secret, &RBDUtil{})
+}
+
+func (plugin *rbdPlugin) newDeleterInternal(spec *volume.Spec, admin, secret string, manager diskManager) (volume.Deleter, error) {
+	return &rbdVolumeDeleter{
+		rbdMounter: &rbdMounter{
+			rbd: &rbd{
+				volName: spec.Name(),
+				Image:   spec.PersistentVolume.Spec.RBD.RBDImage,
+				Pool:    spec.PersistentVolume.Spec.RBD.RBDPool,
+				manager: manager,
+				plugin:  plugin,
+				mounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(plugin.GetPluginName())},
+				exec:    plugin.host.GetExec(plugin.GetPluginName()),
+			},
+			Mon:         spec.PersistentVolume.Spec.RBD.CephMonitors,
+			adminId:     admin,
+			adminSecret: secret,
+		}}, nil
+}
+
+func (plugin *rbdPlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) {
+	return plugin.newProvisionerInternal(options, &RBDUtil{})
+}
+
+func (plugin *rbdPlugin) newProvisionerInternal(options volume.VolumeOptions, manager diskManager) (volume.Provisioner, error) {
+	return &rbdVolumeProvisioner{
+		rbdMounter: &rbdMounter{
+			rbd: &rbd{
+				manager: manager,
+				plugin:  plugin,
+				mounter: &mount.SafeFormatAndMount{Interface: plugin.host.GetMounter(plugin.GetPluginName())},
+				exec:    plugin.host.GetExec(plugin.GetPluginName()),
+			},
+		},
+		options: options,
+	}, nil
+}
+
+type rbdVolumeProvisioner struct {
+	*rbdMounter
+	options volume.VolumeOptions
+}
+
+func (r *rbdVolumeProvisioner) Provision() (*v1.PersistentVolume, error) {
+	if !volume.AccessModesContainedInAll(r.plugin.GetAccessModes(), r.options.PVC.Spec.AccessModes) {
+		return nil, fmt.Errorf("invalid AccessModes %v: only AccessModes %v are supported", r.options.PVC.Spec.AccessModes, r.plugin.GetAccessModes())
+	}
+
+	if r.options.PVC.Spec.Selector != nil {
+		return nil, fmt.Errorf("claim Selector is not supported")
+	}
+	var err error
+	adminSecretName := ""
+	adminSecretNamespace := rbdDefaultAdminSecretNamespace
+	secretName := ""
+	secret := ""
+	imageFormat := rbdImageFormat2
+	fstype := ""
+
+	for k, v := range r.options.Parameters {
+		switch dstrings.ToLower(k) {
+		case "monitors":
+			arr := dstrings.Split(v, ",")
+			for _, m := range arr {
+				r.Mon = append(r.Mon, m)
+			}
+		case "adminid":
+			r.adminId = v
+		case "adminsecretname":
+			adminSecretName = v
+		case "adminsecretnamespace":
+			adminSecretNamespace = v
+		case "userid":
+			r.Id = v
+		case "pool":
+			r.Pool = v
+		case "usersecretname":
+			secretName = v
+		case "imageformat":
+			imageFormat = v
+		case "imagefeatures":
+			arr := dstrings.Split(v, ",")
+			for _, f := range arr {
+				if !supportedFeatures.Has(f) {
+					return nil, fmt.Errorf("invalid feature %q for volume plugin %s, supported features are: %v", f, r.plugin.GetPluginName(), supportedFeatures)
+				} else {
+					r.imageFeatures = append(r.imageFeatures, f)
+				}
+			}
+		case volume.VolumeParameterFSType:
+			fstype = v
+		default:
+			return nil, fmt.Errorf("invalid option %q for volume plugin %s", k, r.plugin.GetPluginName())
+		}
+	}
+	// sanity check
+	if imageFormat != rbdImageFormat1 && imageFormat != rbdImageFormat2 {
+		return nil, fmt.Errorf("invalid ceph imageformat %s, expecting %s or %s",
+			imageFormat, rbdImageFormat1, rbdImageFormat2)
+	}
+	r.imageFormat = imageFormat
+	if adminSecretName == "" {
+		return nil, fmt.Errorf("missing Ceph admin secret name")
+	}
+	if secret, err = parsePVSecret(adminSecretNamespace, adminSecretName, r.plugin.host.GetKubeClient()); err != nil {
+		return nil, fmt.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
+	}
+	r.adminSecret = secret
+	if len(r.Mon) < 1 {
+		return nil, fmt.Errorf("missing Ceph monitors")
+	}
+	if secretName == "" {
+		return nil, fmt.Errorf("missing user secret name")
+	}
+	if r.adminId == "" {
+		r.adminId = rbdDefaultAdminId
+	}
+	if r.Pool == "" {
+		r.Pool = rbdDefaultPool
+	}
+	if r.Id == "" {
+		r.Id = r.adminId
+	}
+
+	// create random image name
+	image := fmt.Sprintf("kubernetes-dynamic-pvc-%s", uuid.NewUUID())
+	r.rbdMounter.Image = image
+	rbd, sizeMB, err := r.manager.CreateImage(r)
+	if err != nil {
+		glog.Errorf("rbd: create volume failed, err: %v", err)
+		return nil, err
+	}
+	glog.Infof("successfully created rbd image %q", image)
+	pv := new(v1.PersistentVolume)
+	metav1.SetMetaDataAnnotation(&pv.ObjectMeta, volumehelper.VolumeDynamicallyCreatedByKey, "rbd-dynamic-provisioner")
+	rbd.SecretRef = new(v1.LocalObjectReference)
+	rbd.SecretRef.Name = secretName
+	rbd.RadosUser = r.Id
+	rbd.FSType = fstype
+	pv.Spec.PersistentVolumeSource.RBD = rbd
+	pv.Spec.PersistentVolumeReclaimPolicy = r.options.PersistentVolumeReclaimPolicy
+	pv.Spec.AccessModes = r.options.PVC.Spec.AccessModes
+	if len(pv.Spec.AccessModes) == 0 {
+		pv.Spec.AccessModes = r.plugin.GetAccessModes()
+	}
+	pv.Spec.Capacity = v1.ResourceList{
+		v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dMi", sizeMB)),
+	}
+	pv.Spec.MountOptions = r.options.MountOptions
+	return pv, nil
+}
+
+type rbdVolumeDeleter struct {
+	*rbdMounter
+}
+
+func (r *rbdVolumeDeleter) GetPath() string {
+	return getPath(r.podUID, r.volName, r.plugin.host)
+}
+
+func (r *rbdVolumeDeleter) Delete() error {
+	return r.manager.DeleteImage(r)
+}
+
+type rbd struct {
+	volName  string
+	podUID   types.UID
+	Pool     string
+	Image    string
+	ReadOnly bool
+	plugin   *rbdPlugin
+	mounter  *mount.SafeFormatAndMount
+	exec     mount.Exec
+	// Utility interface that provides API calls to the provider to attach/detach disks.
+	manager                diskManager
+	volume.MetricsProvider `json:"-"`
+}
+
+func (rbd *rbd) GetPath() string {
+	// safe to use PodVolumeDir now: volume teardown occurs before pod is cleaned up
+	return getPath(rbd.podUID, rbd.volName, rbd.plugin.host)
+}
+
+type rbdMounter struct {
+	*rbd
+	// capitalized so they can be exported in persistRBD()
+	Mon           []string
+	Id            string
+	Keyring       string
+	Secret        string
+	fsType        string
+	adminSecret   string
+	adminId       string
+	mountOptions  []string
+	imageFormat   string
+	imageFeatures []string
+}
+
+var _ volume.Mounter = &rbdMounter{}
+
+func (b *rbd) GetAttributes() volume.Attributes {
+	return volume.Attributes{
+		ReadOnly:        b.ReadOnly,
+		Managed:         !b.ReadOnly,
+		SupportsSELinux: true,
+	}
+}
+
+// Checks prior to mount operations to verify that the required components (binaries, etc.)
+// to mount the volume are available on the underlying node.
+// If not, it returns an error
+func (b *rbdMounter) CanMount() error {
+	return nil
+}
+
+func (b *rbdMounter) SetUp(fsGroup *int64) error {
+	return b.SetUpAt(b.GetPath(), fsGroup)
+}
+
+func (b *rbdMounter) SetUpAt(dir string, fsGroup *int64) error {
+	// diskSetUp checks mountpoints and prevent repeated calls
+	glog.V(4).Infof("rbd: attempting to SetUp and mount %s", dir)
+	err := diskSetUp(b.manager, *b, dir, b.mounter, fsGroup)
+	if err != nil {
+		glog.Errorf("rbd: failed to setup mount %s %v", dir, err)
+	}
+	return err
+}
+
+type rbdUnmounter struct {
+	*rbdMounter
+}
+
+var _ volume.Unmounter = &rbdUnmounter{}
+
+// Unmounts the bind mount, and detaches the disk only if the disk
+// resource was the last reference to that disk on the kubelet.
+func (c *rbdUnmounter) TearDown() error {
+	return c.TearDownAt(c.GetPath())
+}
+
+func (c *rbdUnmounter) TearDownAt(dir string) error {
+	if pathExists, pathErr := volutil.PathExists(dir); pathErr != nil {
+		return fmt.Errorf("Error checking if path exists: %v", pathErr)
+	} else if !pathExists {
+		glog.Warningf("Warning: Unmount skipped because path does not exist: %v", dir)
+		return nil
+	}
+	return diskTearDown(c.manager, *c, dir, c.mounter)
+}
+
+func getVolumeSource(
+	spec *volume.Spec) (*v1.RBDVolumeSource, bool, error) {
+	if spec.Volume != nil && spec.Volume.RBD != nil {
+		return spec.Volume.RBD, spec.Volume.RBD.ReadOnly, nil
+	} else if spec.PersistentVolume != nil &&
+		spec.PersistentVolume.Spec.RBD != nil {
+		return spec.PersistentVolume.Spec.RBD, spec.ReadOnly, nil
+	}
+
+	return nil, false, fmt.Errorf("Spec does not reference a RBD volume type")
+}
+
+func parsePodSecret(pod *v1.Pod, secretName string, kubeClient clientset.Interface) (string, error) {
+	secret, err := volutil.GetSecretForPod(pod, secretName, kubeClient)
+	if err != nil {
+		glog.Errorf("failed to get secret from [%q/%q]", pod.Namespace, secretName)
+		return "", fmt.Errorf("failed to get secret from [%q/%q]", pod.Namespace, secretName)
+	}
+	return parseSecretMap(secret)
+}
+
+func parsePVSecret(namespace, secretName string, kubeClient clientset.Interface) (string, error) {
+	secret, err := volutil.GetSecretForPV(namespace, secretName, rbdPluginName, kubeClient)
+	if err != nil {
+		glog.Errorf("failed to get secret from [%q/%q]", namespace, secretName)
+		return "", fmt.Errorf("failed to get secret from [%q/%q]", namespace, secretName)
+	}
+	return parseSecretMap(secret)
+}
+
+// parseSecretMap locates the secret by key name.
+func parseSecretMap(secretMap map[string]string) (string, error) {
+	if len(secretMap) == 0 {
+		return "", fmt.Errorf("empty secret map")
+	}
+	secret := ""
+	for k, v := range secretMap {
+		if k == secretKeyName {
+			return v, nil
+		}
+		secret = v
+	}
+	// If not found, the last secret in the map wins as done before
+	return secret, nil
+}

vendor/k8s.io/kubernetes/pkg/volume/rbd/rbd_test.go

@@ -0,0 +1,331 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/fake"
+	utiltesting "k8s.io/client-go/util/testing"
+	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/volume"
+	volumetest "k8s.io/kubernetes/pkg/volume/testing"
+)
+
+func TestCanSupport(t *testing.T) {
+	tmpDir, err := utiltesting.MkTmpdir("rbd_test")
+	if err != nil {
+		t.Fatalf("error creating temp dir: %v", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	plugMgr := volume.VolumePluginMgr{}
+	plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil))
+
+	plug, err := plugMgr.FindPluginByName("kubernetes.io/rbd")
+	if err != nil {
+		t.Errorf("Can't find the plugin by name")
+	}
+	if plug.GetPluginName() != "kubernetes.io/rbd" {
+		t.Errorf("Wrong name: %s", plug.GetPluginName())
+	}
+	if plug.CanSupport(&volume.Spec{Volume: &v1.Volume{VolumeSource: v1.VolumeSource{}}}) {
+		t.Errorf("Expected false")
+	}
+}
+
+type fakeDiskManager struct {
+	tmpDir string
+}
+
+func NewFakeDiskManager() *fakeDiskManager {
+	return &fakeDiskManager{
+		tmpDir: utiltesting.MkTmpdirOrDie("rbd_test"),
+	}
+}
+
+func (fake *fakeDiskManager) Cleanup() {
+	os.RemoveAll(fake.tmpDir)
+}
+
+func (fake *fakeDiskManager) MakeGlobalPDName(disk rbd) string {
+	return fake.tmpDir
+}
+func (fake *fakeDiskManager) AttachDisk(b rbdMounter) error {
+	globalPath := b.manager.MakeGlobalPDName(*b.rbd)
+	err := os.MkdirAll(globalPath, 0750)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (fake *fakeDiskManager) DetachDisk(c rbdUnmounter, mntPath string) error {
+	globalPath := c.manager.MakeGlobalPDName(*c.rbd)
+	err := os.RemoveAll(globalPath)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (fake *fakeDiskManager) CreateImage(provisioner *rbdVolumeProvisioner) (r *v1.RBDVolumeSource, volumeSizeGB int, err error) {
+	return nil, 0, fmt.Errorf("not implemented")
+}
+
+func (fake *fakeDiskManager) DeleteImage(deleter *rbdVolumeDeleter) error {
+	return fmt.Errorf("not implemented")
+}
+
+func doTestPlugin(t *testing.T, spec *volume.Spec) {
+	tmpDir, err := utiltesting.MkTmpdir("rbd_test")
+	if err != nil {
+		t.Fatalf("error creating temp dir: %v", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	plugMgr := volume.VolumePluginMgr{}
+	plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, nil, nil))
+
+	plug, err := plugMgr.FindPluginByName("kubernetes.io/rbd")
+	if err != nil {
+		t.Errorf("Can't find the plugin by name")
+	}
+	fdm := NewFakeDiskManager()
+	defer fdm.Cleanup()
+	exec := mount.NewFakeExec(nil)
+	mounter, err := plug.(*rbdPlugin).newMounterInternal(spec, types.UID("poduid"), fdm, &mount.FakeMounter{}, exec, "secrets")
+	if err != nil {
+		t.Errorf("Failed to make a new Mounter: %v", err)
+	}
+	if mounter == nil {
+		t.Error("Got a nil Mounter")
+	}
+
+	path := mounter.GetPath()
+	expectedPath := fmt.Sprintf("%s/pods/poduid/volumes/kubernetes.io~rbd/vol1", tmpDir)
+	if path != expectedPath {
+		t.Errorf("Unexpected path, expected %q, got: %q", expectedPath, path)
+	}
+
+	if err := mounter.SetUp(nil); err != nil {
+		t.Errorf("Expected success, got: %v", err)
+	}
+	if _, err := os.Stat(path); err != nil {
+		if os.IsNotExist(err) {
+			t.Errorf("SetUp() failed, volume path not created: %s", path)
+		} else {
+			t.Errorf("SetUp() failed: %v", err)
+		}
+	}
+
+	unmounter, err := plug.(*rbdPlugin).newUnmounterInternal("vol1", types.UID("poduid"), fdm, &mount.FakeMounter{}, exec)
+	if err != nil {
+		t.Errorf("Failed to make a new Unmounter: %v", err)
+	}
+	if unmounter == nil {
+		t.Error("Got a nil Unmounter")
+	}
+
+	if err := unmounter.TearDown(); err != nil {
+		t.Errorf("Expected success, got: %v", err)
+	}
+	if _, err := os.Stat(path); err == nil {
+		t.Errorf("TearDown() failed, volume path still exists: %s", path)
+	} else if !os.IsNotExist(err) {
+		t.Errorf("SetUp() failed: %v", err)
+	}
+}
+
+func TestPluginVolume(t *testing.T) {
+	vol := &v1.Volume{
+		Name: "vol1",
+		VolumeSource: v1.VolumeSource{
+			RBD: &v1.RBDVolumeSource{
+				CephMonitors: []string{"a", "b"},
+				RBDImage:     "bar",
+				FSType:       "ext4",
+			},
+		},
+	}
+	doTestPlugin(t, volume.NewSpecFromVolume(vol))
+}
+func TestPluginPersistentVolume(t *testing.T) {
+	vol := &v1.PersistentVolume{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "vol1",
+		},
+		Spec: v1.PersistentVolumeSpec{
+			PersistentVolumeSource: v1.PersistentVolumeSource{
+				RBD: &v1.RBDVolumeSource{
+					CephMonitors: []string{"a", "b"},
+					RBDImage:     "bar",
+					FSType:       "ext4",
+				},
+			},
+		},
+	}
+
+	doTestPlugin(t, volume.NewSpecFromPersistentVolume(vol, false))
+}
+
+func TestPersistentClaimReadOnlyFlag(t *testing.T) {
+	tmpDir, err := utiltesting.MkTmpdir("rbd_test")
+	if err != nil {
+		t.Fatalf("error creating temp dir: %v", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	pv := &v1.PersistentVolume{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "pvA",
+		},
+		Spec: v1.PersistentVolumeSpec{
+			PersistentVolumeSource: v1.PersistentVolumeSource{
+				RBD: &v1.RBDVolumeSource{
+					CephMonitors: []string{"a", "b"},
+					RBDImage:     "bar",
+					FSType:       "ext4",
+				},
+			},
+			ClaimRef: &v1.ObjectReference{
+				Name: "claimA",
+			},
+		},
+	}
+
+	claim := &v1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "claimA",
+			Namespace: "nsA",
+		},
+		Spec: v1.PersistentVolumeClaimSpec{
+			VolumeName: "pvA",
+		},
+		Status: v1.PersistentVolumeClaimStatus{
+			Phase: v1.ClaimBound,
+		},
+	}
+
+	client := fake.NewSimpleClientset(pv, claim)
+
+	plugMgr := volume.VolumePluginMgr{}
+	plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpDir, client, nil))
+	plug, _ := plugMgr.FindPluginByName(rbdPluginName)
+
+	// readOnly bool is supplied by persistent-claim volume source when its mounter creates other volumes
+	spec := volume.NewSpecFromPersistentVolume(pv, true)
+	pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}}
+	mounter, _ := plug.NewMounter(spec, pod, volume.VolumeOptions{})
+	if mounter == nil {
+		t.Fatalf("Got a nil Mounter")
+	}
+
+	if !mounter.GetAttributes().ReadOnly {
+		t.Errorf("Expected true for mounter.IsReadOnly")
+	}
+}
+
+func TestPersistAndLoadRBD(t *testing.T) {
+	tmpDir, err := utiltesting.MkTmpdir("rbd_test")
+	if err != nil {
+		t.Fatalf("error creating temp dir: %v", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	testcases := []struct {
+		rbdMounter               rbdMounter
+		expectedJSONStr          string
+		expectedLoadedRBDMounter rbdMounter
+	}{
+		{
+			rbdMounter{},
+			`{"Mon":null,"Id":"","Keyring":"","Secret":""}`,
+			rbdMounter{},
+		},
+		{
+			rbdMounter{
+				rbd: &rbd{
+					podUID:          "poduid",
+					Pool:            "kube",
+					Image:           "some-test-image",
+					ReadOnly:        false,
+					MetricsProvider: volume.NewMetricsStatFS("/tmp"),
+				},
+				Mon:     []string{"127.0.0.1"},
+				Id:      "kube",
+				Keyring: "",
+				Secret:  "QVFEcTdKdFp4SmhtTFJBQUNwNDI3UnhGRzBvQ1Y0SUJwLy9pRUE9PQ==",
+			},
+			`
+{
+	"Pool": "kube",
+	"Image": "some-test-image",
+	"ReadOnly": false,
+	"Mon": ["127.0.0.1"],
+	"Id": "kube",
+	"Keyring": "",
+	"Secret": "QVFEcTdKdFp4SmhtTFJBQUNwNDI3UnhGRzBvQ1Y0SUJwLy9pRUE9PQ=="
+}
+			`,
+			rbdMounter{
+				rbd: &rbd{
+					Pool:     "kube",
+					Image:    "some-test-image",
+					ReadOnly: false,
+				},
+				Mon:     []string{"127.0.0.1"},
+				Id:      "kube",
+				Keyring: "",
+				Secret:  "QVFEcTdKdFp4SmhtTFJBQUNwNDI3UnhGRzBvQ1Y0SUJwLy9pRUE9PQ==",
+			},
+		},
+	}
+
+	util := &RBDUtil{}
+	for _, c := range testcases {
+		err = util.persistRBD(c.rbdMounter, tmpDir)
+		if err != nil {
+			t.Errorf("failed to persist rbd: %v, err: %v", c.rbdMounter, err)
+		}
+		jsonFile := filepath.Join(tmpDir, "rbd.json")
+		jsonData, err := ioutil.ReadFile(jsonFile)
+		if err != nil {
+			t.Errorf("failed to read json file %s: %v", jsonFile, err)
+		}
+		if !assert.JSONEq(t, c.expectedJSONStr, string(jsonData)) {
+			t.Errorf("json file does not match expected one: %s, should be %s", string(jsonData), c.expectedJSONStr)
+		}
+		tmpRBDMounter := rbdMounter{}
+		err = util.loadRBD(&tmpRBDMounter, tmpDir)
+		if err != nil {
+			t.Errorf("faild to load rbd: %v", err)
+		}
+		if !reflect.DeepEqual(tmpRBDMounter, c.expectedLoadedRBDMounter) {
+			t.Errorf("loaded rbd does not equal to expected one: %v, should be %v", tmpRBDMounter, c.rbdMounter)
+		}
+	}
+}

vendor/k8s.io/kubernetes/pkg/volume/rbd/rbd_util.go

@@ -0,0 +1,476 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+//
+// utility functions to setup rbd volume
+// mainly implement diskManager interface
+//
+
+package rbd
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+	"k8s.io/api/core/v1"
+	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/util/node"
+	"k8s.io/kubernetes/pkg/volume"
+)
+
+const (
+	imageWatcherStr = "watcher="
+	kubeLockMagic   = "kubelet_lock_magic_"
+	rbdCmdErr       = "executable file not found in $PATH"
+)
+
+// search /sys/bus for rbd device that matches given pool and image
+func getDevFromImageAndPool(pool, image string) (string, bool) {
+	// /sys/bus/rbd/devices/X/name and /sys/bus/rbd/devices/X/pool
+	sys_path := "/sys/bus/rbd/devices"
+	if dirs, err := ioutil.ReadDir(sys_path); err == nil {
+		for _, f := range dirs {
+			// pool and name format:
+			// see rbd_pool_show() and rbd_name_show() at
+			// https://github.com/torvalds/linux/blob/master/drivers/block/rbd.c
+			name := f.Name()
+			// first match pool, then match name
+			poolFile := path.Join(sys_path, name, "pool")
+			poolBytes, err := ioutil.ReadFile(poolFile)
+			if err != nil {
+				glog.V(4).Infof("Error reading %s: %v", poolFile, err)
+				continue
+			}
+			if strings.TrimSpace(string(poolBytes)) != pool {
+				glog.V(4).Infof("Device %s is not %q: %q", name, pool, string(poolBytes))
+				continue
+			}
+			imgFile := path.Join(sys_path, name, "name")
+			imgBytes, err := ioutil.ReadFile(imgFile)
+			if err != nil {
+				glog.V(4).Infof("Error reading %s: %v", imgFile, err)
+				continue
+			}
+			if strings.TrimSpace(string(imgBytes)) != image {
+				glog.V(4).Infof("Device %s is not %q: %q", name, image, string(imgBytes))
+				continue
+			}
+			// found a match, check if device exists
+			devicePath := "/dev/rbd" + name
+			if _, err := os.Lstat(devicePath); err == nil {
+				return devicePath, true
+			}
+		}
+	}
+	return "", false
+}
+
+// stat a path, if not exists, retry maxRetries times
+func waitForPath(pool, image string, maxRetries int) (string, bool) {
+	for i := 0; i < maxRetries; i++ {
+		devicePath, found := getDevFromImageAndPool(pool, image)
+		if found {
+			return devicePath, true
+		}
+		if i == maxRetries-1 {
+			break
+		}
+		time.Sleep(time.Second)
+	}
+	return "", false
+}
+
+// make a directory like /var/lib/kubelet/plugins/kubernetes.io/pod/rbd/pool-image-image
+func makePDNameInternal(host volume.VolumeHost, pool string, image string) string {
+	return path.Join(host.GetPluginDir(rbdPluginName), "rbd", pool+"-image-"+image)
+}
+
+type RBDUtil struct{}
+
+func (util *RBDUtil) MakeGlobalPDName(rbd rbd) string {
+	return makePDNameInternal(rbd.plugin.host, rbd.Pool, rbd.Image)
+}
+func rbdErrors(runErr, resultErr error) error {
+	if runErr.Error() == rbdCmdErr {
+		return fmt.Errorf("rbd: rbd cmd not found")
+	}
+	return resultErr
+}
+
+func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error {
+	var err error
+	var output, locker string
+	var cmd []byte
+	var secret_opt []string
+
+	if b.Secret != "" {
+		secret_opt = []string{"--key=" + b.Secret}
+	} else {
+		secret_opt = []string{"-k", b.Keyring}
+	}
+	if len(b.adminId) == 0 {
+		b.adminId = b.Id
+	}
+	if len(b.adminSecret) == 0 {
+		b.adminSecret = b.Secret
+	}
+
+	// construct lock id using host name and a magic prefix
+	lock_id := kubeLockMagic + node.GetHostname("")
+
+	l := len(b.Mon)
+	// avoid mount storm, pick a host randomly
+	start := rand.Int() % l
+	// iterate all hosts until mount succeeds.
+	for i := start; i < start+l; i++ {
+		mon := b.Mon[i%l]
+		// cmd "rbd lock list" serves two purposes:
+		// for fencing, check if lock already held for this host
+		// this edge case happens if host crashes in the middle of acquiring lock and mounting rbd
+		// for defencing, get the locker name, something like "client.1234"
+		args := []string{"lock", "list", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon}
+		args = append(args, secret_opt...)
+		cmd, err = b.exec.Run("rbd", args...)
+		output = string(cmd)
+		glog.Infof("lock list output %q", output)
+		if err != nil {
+			continue
+		}
+
+		if lock {
+			// check if lock is already held for this host by matching lock_id and rbd lock id
+			if strings.Contains(output, lock_id) {
+				// this host already holds the lock, exit
+				glog.V(1).Infof("rbd: lock already held for %s", lock_id)
+				return nil
+			}
+			// clean up orphaned lock if no watcher on the image
+			used, statusErr := util.rbdStatus(&b)
+			if statusErr == nil && !used {
+				re := regexp.MustCompile("client.* " + kubeLockMagic + ".*")
+				locks := re.FindAllStringSubmatch(output, -1)
+				for _, v := range locks {
+					if len(v) > 0 {
+						lockInfo := strings.Split(v[0], " ")
+						if len(lockInfo) > 2 {
+							args := []string{"lock", "remove", b.Image, lockInfo[1], lockInfo[0], "--pool", b.Pool, "--id", b.Id, "-m", mon}
+							args = append(args, secret_opt...)
+							cmd, err = b.exec.Run("rbd", args...)
+							glog.Infof("remove orphaned locker %s from client %s: err %v, output: %s", lockInfo[1], lockInfo[0], err, string(cmd))
+						}
+					}
+				}
+			}
+
+			// hold a lock: rbd lock add
+			args := []string{"lock", "add", b.Image, lock_id, "--pool", b.Pool, "--id", b.Id, "-m", mon}
+			args = append(args, secret_opt...)
+			cmd, err = b.exec.Run("rbd", args...)
+		} else {
+			// defencing, find locker name
+			ind := strings.LastIndex(output, lock_id) - 1
+			for i := ind; i >= 0; i-- {
+				if output[i] == '\n' {
+					locker = output[(i + 1):ind]
+					break
+				}
+			}
+			// remove a lock: rbd lock remove
+			args := []string{"lock", "remove", b.Image, lock_id, locker, "--pool", b.Pool, "--id", b.Id, "-m", mon}
+			args = append(args, secret_opt...)
+			cmd, err = b.exec.Run("rbd", args...)
+		}
+
+		if err == nil {
+			//lock is acquired
+			break
+		}
+	}
+	return err
+}
+
+func (util *RBDUtil) persistRBD(rbd rbdMounter, mnt string) error {
+	file := path.Join(mnt, "rbd.json")
+	fp, err := os.Create(file)
+	if err != nil {
+		return fmt.Errorf("rbd: create err %s/%s", file, err)
+	}
+	defer fp.Close()
+
+	encoder := json.NewEncoder(fp)
+	if err = encoder.Encode(rbd); err != nil {
+		return fmt.Errorf("rbd: encode err: %v.", err)
+	}
+
+	return nil
+}
+
+func (util *RBDUtil) loadRBD(mounter *rbdMounter, mnt string) error {
+	file := path.Join(mnt, "rbd.json")
+	fp, err := os.Open(file)
+	if err != nil {
+		return fmt.Errorf("rbd: open err %s/%s", file, err)
+	}
+	defer fp.Close()
+
+	decoder := json.NewDecoder(fp)
+	if err = decoder.Decode(mounter); err != nil {
+		return fmt.Errorf("rbd: decode err: %v.", err)
+	}
+
+	return nil
+}
+
+func (util *RBDUtil) fencing(b rbdMounter) error {
+	// no need to fence readOnly
+	if (&b).GetAttributes().ReadOnly {
+		return nil
+	}
+	return util.rbdLock(b, true)
+}
+
+func (util *RBDUtil) defencing(c rbdUnmounter) error {
+	// no need to fence readOnly
+	if c.ReadOnly {
+		return nil
+	}
+
+	return util.rbdLock(*c.rbdMounter, false)
+}
+
+func (util *RBDUtil) AttachDisk(b rbdMounter) error {
+	var err error
+	var output []byte
+
+	// create mount point
+	globalPDPath := b.manager.MakeGlobalPDName(*b.rbd)
+	notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
+	// in the first time, the path shouldn't exist and IsLikelyNotMountPoint is expected to get NotExist
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("rbd: %s failed to check mountpoint", globalPDPath)
+	}
+	if !notMnt {
+		return nil
+	}
+	if err = os.MkdirAll(globalPDPath, 0750); err != nil {
+		return fmt.Errorf("rbd: failed to mkdir %s, error", globalPDPath)
+	}
+
+	devicePath, found := waitForPath(b.Pool, b.Image, 1)
+	if !found {
+		_, err = b.exec.Run("modprobe", "rbd")
+		if err != nil {
+			glog.Warningf("rbd: failed to load rbd kernel module:%v", err)
+		}
+
+		// fence off other mappers
+		if err = util.fencing(b); err != nil {
+			return rbdErrors(err, fmt.Errorf("rbd: failed to lock image %s (maybe locked by other nodes), error %v", b.Image, err))
+		}
+		// rbd lock remove needs ceph and image config
+		// but kubelet doesn't get them from apiserver during teardown
+		// so persit rbd config so upon disk detach, rbd lock can be removed
+		// since rbd json is persisted in the same local directory that is used as rbd mountpoint later,
+		// the json file remains invisible during rbd mount and thus won't be removed accidentally.
+		util.persistRBD(b, globalPDPath)
+
+		// rbd map
+		l := len(b.Mon)
+		// avoid mount storm, pick a host randomly
+		start := rand.Int() % l
+		// iterate all hosts until mount succeeds.
+		for i := start; i < start+l; i++ {
+			mon := b.Mon[i%l]
+			glog.V(1).Infof("rbd: map mon %s", mon)
+			if b.Secret != "" {
+				output, err = b.exec.Run("rbd",
+					"map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "--key="+b.Secret)
+			} else {
+				output, err = b.exec.Run("rbd",
+					"map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "-k", b.Keyring)
+			}
+			if err == nil {
+				break
+			}
+			glog.V(1).Infof("rbd: map error %v %s", err, string(output))
+		}
+		if err != nil {
+			return fmt.Errorf("rbd: map failed %v %s", err, string(output))
+		}
+		devicePath, found = waitForPath(b.Pool, b.Image, 10)
+		if !found {
+			return errors.New("Could not map image: Timeout after 10s")
+		}
+		glog.V(3).Infof("rbd: successfully map image %s/%s to %s", b.Pool, b.Image, devicePath)
+	}
+
+	// mount it
+	if err = b.mounter.FormatAndMount(devicePath, globalPDPath, b.fsType, nil); err != nil {
+		err = fmt.Errorf("rbd: failed to mount rbd volume %s [%s] to %s, error %v", devicePath, b.fsType, globalPDPath, err)
+	}
+	glog.V(3).Infof("rbd: successfully mount image %s/%s at %s", b.Pool, b.Image, globalPDPath)
+	return err
+}
+
+func (util *RBDUtil) DetachDisk(c rbdUnmounter, mntPath string) error {
+	device, cnt, err := mount.GetDeviceNameFromMount(c.mounter, mntPath)
+	if err != nil {
+		return fmt.Errorf("rbd detach disk: failed to get device from mnt: %s\nError: %v", mntPath, err)
+	}
+	if err = c.mounter.Unmount(mntPath); err != nil {
+		return fmt.Errorf("rbd detach disk: failed to umount: %s\nError: %v", mntPath, err)
+	}
+	glog.V(3).Infof("rbd: successfully umount mountpoint %s", mntPath)
+	// if device is no longer used, see if can unmap
+	if cnt <= 1 {
+		// rbd unmap
+		_, err = c.exec.Run("rbd", "unmap", device)
+		if err != nil {
+			return rbdErrors(err, fmt.Errorf("rbd: failed to unmap device %s:Error: %v", device, err))
+		}
+
+		// load ceph and image/pool info to remove fencing
+		if err := util.loadRBD(c.rbdMounter, mntPath); err == nil {
+			// remove rbd lock
+			util.defencing(c)
+		}
+
+		glog.V(3).Infof("rbd: successfully unmap device %s", device)
+	}
+	return nil
+}
+
+func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDVolumeSource, size int, err error) {
+	var output []byte
+	capacity := p.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
+	volSizeBytes := capacity.Value()
+	// convert to MB that rbd defaults on
+	sz := int(volume.RoundUpSize(volSizeBytes, 1024*1024))
+	volSz := fmt.Sprintf("%d", sz)
+	// rbd create
+	l := len(p.rbdMounter.Mon)
+	// pick a mon randomly
+	start := rand.Int() % l
+	// iterate all monitors until create succeeds.
+	for i := start; i < start+l; i++ {
+		mon := p.Mon[i%l]
+		if p.rbdMounter.imageFormat == rbdImageFormat2 {
+			glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", p.rbdMounter.Image, volSz, p.rbdMounter.imageFormat, p.rbdMounter.imageFeatures, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret)
+		} else {
+			glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, volSz, p.rbdMounter.imageFormat, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret)
+		}
+		args := []string{"create", p.rbdMounter.Image, "--size", volSz, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key=" + p.rbdMounter.adminSecret, "--image-format", p.rbdMounter.imageFormat}
+		if p.rbdMounter.imageFormat == rbdImageFormat2 {
+			// if no image features is provided, it results in empty string
+			// which disable all RBD image format 2 features as we expected
+			features := strings.Join(p.rbdMounter.imageFeatures, ",")
+			args = append(args, "--image-feature", features)
+		}
+		output, err = p.exec.Run("rbd", args...)
+		if err == nil {
+			break
+		} else {
+			glog.Warningf("failed to create rbd image, output %v", string(output))
+		}
+	}
+
+	if err != nil {
+		return nil, 0, fmt.Errorf("failed to create rbd image: %v, command output: %s", err, string(output))
+	}
+
+	return &v1.RBDVolumeSource{
+		CephMonitors: p.rbdMounter.Mon,
+		RBDImage:     p.rbdMounter.Image,
+		RBDPool:      p.rbdMounter.Pool,
+	}, sz, nil
+}
+
+func (util *RBDUtil) DeleteImage(p *rbdVolumeDeleter) error {
+	var output []byte
+	found, err := util.rbdStatus(p.rbdMounter)
+	if err != nil {
+		return err
+	}
+	if found {
+		glog.Info("rbd is still being used ", p.rbdMounter.Image)
+		return fmt.Errorf("rbd %s is still being used", p.rbdMounter.Image)
+	}
+	// rbd rm
+	l := len(p.rbdMounter.Mon)
+	// pick a mon randomly
+	start := rand.Int() % l
+	// iterate all monitors until rm succeeds.
+	for i := start; i < start+l; i++ {
+		mon := p.rbdMounter.Mon[i%l]
+		glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret)
+		output, err = p.exec.Run("rbd",
+			"rm", p.rbdMounter.Image, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key="+p.rbdMounter.adminSecret)
+		if err == nil {
+			return nil
+		} else {
+			glog.Errorf("failed to delete rbd image: %v, command output: %s", err, string(output))
+		}
+	}
+	return err
+}
+
+// run rbd status command to check if there is watcher on the image
+func (util *RBDUtil) rbdStatus(b *rbdMounter) (bool, error) {
+	var err error
+	var output string
+	var cmd []byte
+
+	l := len(b.Mon)
+	start := rand.Int() % l
+	// iterate all hosts until mount succeeds.
+	for i := start; i < start+l; i++ {
+		mon := b.Mon[i%l]
+		// cmd "rbd status" list the rbd client watch with the following output:
+		// Watchers:
+		//   watcher=10.16.153.105:0/710245699 client.14163 cookie=1
+		glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", b.Image, mon, b.Pool, b.adminId, b.adminSecret)
+		cmd, err = b.exec.Run("rbd",
+			"status", b.Image, "--pool", b.Pool, "-m", mon, "--id", b.adminId, "--key="+b.adminSecret)
+		output = string(cmd)
+
+		if err != nil {
+			if err.Error() == rbdCmdErr {
+				glog.Errorf("rbd cmd not found")
+			} else {
+				// ignore error code, just checkout output for watcher string
+				glog.Warningf("failed to execute rbd status on mon %s", mon)
+			}
+		}
+
+		if strings.Contains(output, imageWatcherStr) {
+			glog.V(4).Infof("rbd: watchers on %s: %s", b.Image, output)
+			return true, nil
+		} else {
+			glog.Warningf("rbd: no watchers on %s", b.Image)
+			return false, nil
+		}
+	}
+	return false, nil
+}