Replace godep with dep

vendor/k8s.io/kubernetes/pkg/cloudprovider/doc.go

@@ -15,4 +15,4 @@ limitations under the License.
 */
 
 // Package cloudprovider supplies interfaces and implementations for cloud service providers.
-package cloudprovider
+package cloudprovider // import "k8s.io/kubernetes/pkg/cloudprovider"

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/BUILD

@@ -0,0 +1,47 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["providers.go"],
+    deps = [
+        "//pkg/cloudprovider/providers/aws:go_default_library",
+        "//pkg/cloudprovider/providers/azure:go_default_library",
+        "//pkg/cloudprovider/providers/cloudstack:go_default_library",
+        "//pkg/cloudprovider/providers/gce:go_default_library",
+        "//pkg/cloudprovider/providers/openstack:go_default_library",
+        "//pkg/cloudprovider/providers/ovirt:go_default_library",
+        "//pkg/cloudprovider/providers/photon:go_default_library",
+        "//pkg/cloudprovider/providers/rackspace:go_default_library",
+        "//pkg/cloudprovider/providers/vsphere:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/cloudprovider/providers/aws:all-srcs",
+        "//pkg/cloudprovider/providers/azure:all-srcs",
+        "//pkg/cloudprovider/providers/cloudstack:all-srcs",
+        "//pkg/cloudprovider/providers/fake:all-srcs",
+        "//pkg/cloudprovider/providers/gce:all-srcs",
+        "//pkg/cloudprovider/providers/openstack:all-srcs",
+        "//pkg/cloudprovider/providers/ovirt:all-srcs",
+        "//pkg/cloudprovider/providers/photon:all-srcs",
+        "//pkg/cloudprovider/providers/rackspace:all-srcs",
+        "//pkg/cloudprovider/providers/vsphere:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/BUILD

@@ -0,0 +1,96 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "aws.go",
+        "aws_instancegroups.go",
+        "aws_loadbalancer.go",
+        "aws_metrics.go",
+        "aws_routes.go",
+        "aws_utils.go",
+        "device_allocator.go",
+        "instances.go",
+        "log_handler.go",
+        "regions.go",
+        "retry_handler.go",
+        "sets_ippermissions.go",
+        "tags.go",
+        "volumes.go",
+    ],
+    deps = [
+        "//pkg/api/v1/service:go_default_library",
+        "//pkg/cloudprovider:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//pkg/credentialprovider/aws:go_default_library",
+        "//pkg/kubelet/apis:go_default_library",
+        "//pkg/volume:go_default_library",
+        "//pkg/volume/util:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/awserr:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/credentials:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/ec2metadata:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/request:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/autoscaling:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/elb:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/kms:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//vendor/gopkg.in/gcfg.v1:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "aws_loadbalancer_test.go",
+        "aws_test.go",
+        "device_allocator_test.go",
+        "instances_test.go",
+        "regions_test.go",
+        "retry_handler_test.go",
+        "tags_test.go",
+    ],
+    library = ":go_default_library",
+    deps = [
+        "//pkg/kubelet/apis:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/autoscaling:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/elb:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/kms:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/github.com/stretchr/testify/mock:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/OWNERS

@@ -0,0 +1,8 @@
+approvers:
+- justinsb
+- zmerlynn
+reviewers:
+- gnufied
+- jsafrane
+- justinsb
+- zmerlynn

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/aws_instancegroups.go

@@ -0,0 +1,90 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/autoscaling"
+	"github.com/golang/glog"
+)
+
+// AWSCloud implements InstanceGroups
+var _ InstanceGroups = &Cloud{}
+
+// ResizeInstanceGroup sets the size of the specificed instancegroup Exported
+// so it can be used by the e2e tests, which don't want to instantiate a full
+// cloudprovider.
+func ResizeInstanceGroup(asg ASG, instanceGroupName string, size int) error {
+	request := &autoscaling.UpdateAutoScalingGroupInput{
+		AutoScalingGroupName: aws.String(instanceGroupName),
+		MinSize:              aws.Int64(int64(size)),
+		MaxSize:              aws.Int64(int64(size)),
+	}
+	if _, err := asg.UpdateAutoScalingGroup(request); err != nil {
+		return fmt.Errorf("error resizing AWS autoscaling group: %q", err)
+	}
+	return nil
+}
+
+// Implement InstanceGroups.ResizeInstanceGroup
+// Set the size to the fixed size
+func (c *Cloud) ResizeInstanceGroup(instanceGroupName string, size int) error {
+	return ResizeInstanceGroup(c.asg, instanceGroupName, size)
+}
+
+// DescribeInstanceGroup gets info about the specified instancegroup
+// Exported so it can be used by the e2e tests,
+// which don't want to instantiate a full cloudprovider.
+func DescribeInstanceGroup(asg ASG, instanceGroupName string) (InstanceGroupInfo, error) {
+	request := &autoscaling.DescribeAutoScalingGroupsInput{
+		AutoScalingGroupNames: []*string{aws.String(instanceGroupName)},
+	}
+	response, err := asg.DescribeAutoScalingGroups(request)
+	if err != nil {
+		return nil, fmt.Errorf("error listing AWS autoscaling group (%s): %q", instanceGroupName, err)
+	}
+
+	if len(response.AutoScalingGroups) == 0 {
+		return nil, nil
+	}
+	if len(response.AutoScalingGroups) > 1 {
+		glog.Warning("AWS returned multiple autoscaling groups with name ", instanceGroupName)
+	}
+	group := response.AutoScalingGroups[0]
+	return &awsInstanceGroup{group: group}, nil
+}
+
+// Implement InstanceGroups.DescribeInstanceGroup
+// Queries the cloud provider for information about the specified instance group
+func (c *Cloud) DescribeInstanceGroup(instanceGroupName string) (InstanceGroupInfo, error) {
+	return DescribeInstanceGroup(c.asg, instanceGroupName)
+}
+
+// awsInstanceGroup implements InstanceGroupInfo
+var _ InstanceGroupInfo = &awsInstanceGroup{}
+
+type awsInstanceGroup struct {
+	group *autoscaling.Group
+}
+
+// Implement InstanceGroupInfo.CurrentSize
+// The number of instances currently running under control of this group
+func (g *awsInstanceGroup) CurrentSize() (int, error) {
+	return len(g.group.Instances), nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/aws_loadbalancer.go

@@ -0,0 +1,544 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/elb"
+	"github.com/golang/glog"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+const ProxyProtocolPolicyName = "k8s-proxyprotocol-enabled"
+
+// getLoadBalancerAdditionalTags converts the comma separated list of key-value
+// pairs in the ServiceAnnotationLoadBalancerAdditionalTags annotation and returns
+// it as a map.
+func getLoadBalancerAdditionalTags(annotations map[string]string) map[string]string {
+	additionalTags := make(map[string]string)
+	if additionalTagsList, ok := annotations[ServiceAnnotationLoadBalancerAdditionalTags]; ok {
+		additionalTagsList = strings.TrimSpace(additionalTagsList)
+
+		// Break up list of "Key1=Val,Key2=Val2"
+		tagList := strings.Split(additionalTagsList, ",")
+
+		// Break up "Key=Val"
+		for _, tagSet := range tagList {
+			tag := strings.Split(strings.TrimSpace(tagSet), "=")
+
+			// Accept "Key=val" or "Key=" or just "Key"
+			if len(tag) >= 2 && len(tag[0]) != 0 {
+				// There is a key and a value, so save it
+				additionalTags[tag[0]] = tag[1]
+			} else if len(tag) == 1 && len(tag[0]) != 0 {
+				// Just "Key"
+				additionalTags[tag[0]] = ""
+			}
+		}
+	}
+
+	return additionalTags
+}
+
+func (c *Cloud) ensureLoadBalancer(namespacedName types.NamespacedName, loadBalancerName string, listeners []*elb.Listener, subnetIDs []string, securityGroupIDs []string, internalELB, proxyProtocol bool, loadBalancerAttributes *elb.LoadBalancerAttributes, annotations map[string]string) (*elb.LoadBalancerDescription, error) {
+	loadBalancer, err := c.describeLoadBalancer(loadBalancerName)
+	if err != nil {
+		return nil, err
+	}
+
+	dirty := false
+
+	if loadBalancer == nil {
+		createRequest := &elb.CreateLoadBalancerInput{}
+		createRequest.LoadBalancerName = aws.String(loadBalancerName)
+
+		createRequest.Listeners = listeners
+
+		if internalELB {
+			createRequest.Scheme = aws.String("internal")
+		}
+
+		// We are supposed to specify one subnet per AZ.
+		// TODO: What happens if we have more than one subnet per AZ?
+		createRequest.Subnets = stringPointerArray(subnetIDs)
+
+		createRequest.SecurityGroups = stringPointerArray(securityGroupIDs)
+
+		// Get additional tags set by the user
+		tags := getLoadBalancerAdditionalTags(annotations)
+
+		// Add default tags
+		tags[TagNameKubernetesService] = namespacedName.String()
+		tags = c.tagging.buildTags(ResourceLifecycleOwned, tags)
+
+		for k, v := range tags {
+			createRequest.Tags = append(createRequest.Tags, &elb.Tag{
+				Key: aws.String(k), Value: aws.String(v),
+			})
+		}
+
+		glog.Infof("Creating load balancer for %v with name: %s", namespacedName, loadBalancerName)
+		_, err := c.elb.CreateLoadBalancer(createRequest)
+		if err != nil {
+			return nil, err
+		}
+
+		if proxyProtocol {
+			err = c.createProxyProtocolPolicy(loadBalancerName)
+			if err != nil {
+				return nil, err
+			}
+
+			for _, listener := range listeners {
+				glog.V(2).Infof("Adjusting AWS loadbalancer proxy protocol on node port %d. Setting to true", *listener.InstancePort)
+				err := c.setBackendPolicies(loadBalancerName, *listener.InstancePort, []*string{aws.String(ProxyProtocolPolicyName)})
+				if err != nil {
+					return nil, err
+				}
+			}
+		}
+
+		dirty = true
+	} else {
+		// TODO: Sync internal vs non-internal
+
+		{
+			// Sync subnets
+			expected := sets.NewString(subnetIDs...)
+			actual := stringSetFromPointers(loadBalancer.Subnets)
+
+			additions := expected.Difference(actual)
+			removals := actual.Difference(expected)
+
+			if removals.Len() != 0 {
+				request := &elb.DetachLoadBalancerFromSubnetsInput{}
+				request.LoadBalancerName = aws.String(loadBalancerName)
+				request.Subnets = stringSetToPointers(removals)
+				glog.V(2).Info("Detaching load balancer from removed subnets")
+				_, err := c.elb.DetachLoadBalancerFromSubnets(request)
+				if err != nil {
+					return nil, fmt.Errorf("error detaching AWS loadbalancer from subnets: %q", err)
+				}
+				dirty = true
+			}
+
+			if additions.Len() != 0 {
+				request := &elb.AttachLoadBalancerToSubnetsInput{}
+				request.LoadBalancerName = aws.String(loadBalancerName)
+				request.Subnets = stringSetToPointers(additions)
+				glog.V(2).Info("Attaching load balancer to added subnets")
+				_, err := c.elb.AttachLoadBalancerToSubnets(request)
+				if err != nil {
+					return nil, fmt.Errorf("error attaching AWS loadbalancer to subnets: %q", err)
+				}
+				dirty = true
+			}
+		}
+
+		{
+			// Sync security groups
+			expected := sets.NewString(securityGroupIDs...)
+			actual := stringSetFromPointers(loadBalancer.SecurityGroups)
+
+			if !expected.Equal(actual) {
+				// This call just replaces the security groups, unlike e.g. subnets (!)
+				request := &elb.ApplySecurityGroupsToLoadBalancerInput{}
+				request.LoadBalancerName = aws.String(loadBalancerName)
+				request.SecurityGroups = stringPointerArray(securityGroupIDs)
+				glog.V(2).Info("Applying updated security groups to load balancer")
+				_, err := c.elb.ApplySecurityGroupsToLoadBalancer(request)
+				if err != nil {
+					return nil, fmt.Errorf("error applying AWS loadbalancer security groups: %q", err)
+				}
+				dirty = true
+			}
+		}
+
+		{
+			// Sync listeners
+			listenerDescriptions := loadBalancer.ListenerDescriptions
+
+			foundSet := make(map[int]bool)
+			removals := []*int64{}
+			for _, listenerDescription := range listenerDescriptions {
+				actual := listenerDescription.Listener
+				if actual == nil {
+					glog.Warning("Ignoring empty listener in AWS loadbalancer: ", loadBalancerName)
+					continue
+				}
+
+				found := -1
+				for i, expected := range listeners {
+					if elbProtocolsAreEqual(actual.Protocol, expected.Protocol) {
+						continue
+					}
+					if elbProtocolsAreEqual(actual.InstanceProtocol, expected.InstanceProtocol) {
+						continue
+					}
+					if orZero(actual.InstancePort) != orZero(expected.InstancePort) {
+						continue
+					}
+					if orZero(actual.LoadBalancerPort) != orZero(expected.LoadBalancerPort) {
+						continue
+					}
+					if awsArnEquals(actual.SSLCertificateId, expected.SSLCertificateId) {
+						continue
+					}
+					found = i
+				}
+				if found != -1 {
+					foundSet[found] = true
+				} else {
+					removals = append(removals, actual.LoadBalancerPort)
+				}
+			}
+
+			additions := []*elb.Listener{}
+			for i := range listeners {
+				if foundSet[i] {
+					continue
+				}
+				additions = append(additions, listeners[i])
+			}
+
+			if len(removals) != 0 {
+				request := &elb.DeleteLoadBalancerListenersInput{}
+				request.LoadBalancerName = aws.String(loadBalancerName)
+				request.LoadBalancerPorts = removals
+				glog.V(2).Info("Deleting removed load balancer listeners")
+				_, err := c.elb.DeleteLoadBalancerListeners(request)
+				if err != nil {
+					return nil, fmt.Errorf("error deleting AWS loadbalancer listeners: %q", err)
+				}
+				dirty = true
+			}
+
+			if len(additions) != 0 {
+				request := &elb.CreateLoadBalancerListenersInput{}
+				request.LoadBalancerName = aws.String(loadBalancerName)
+				request.Listeners = additions
+				glog.V(2).Info("Creating added load balancer listeners")
+				_, err := c.elb.CreateLoadBalancerListeners(request)
+				if err != nil {
+					return nil, fmt.Errorf("error creating AWS loadbalancer listeners: %q", err)
+				}
+				dirty = true
+			}
+		}
+
+		{
+			// Sync proxy protocol state for new and existing listeners
+
+			proxyPolicies := make([]*string, 0)
+			if proxyProtocol {
+				// Ensure the backend policy exists
+
+				// NOTE The documentation for the AWS API indicates we could get an HTTP 400
+				// back if a policy of the same name already exists. However, the aws-sdk does not
+				// seem to return an error to us in these cases. Therefore, this will issue an API
+				// request every time.
+				err := c.createProxyProtocolPolicy(loadBalancerName)
+				if err != nil {
+					return nil, err
+				}
+
+				proxyPolicies = append(proxyPolicies, aws.String(ProxyProtocolPolicyName))
+			}
+
+			foundBackends := make(map[int64]bool)
+			proxyProtocolBackends := make(map[int64]bool)
+			for _, backendListener := range loadBalancer.BackendServerDescriptions {
+				foundBackends[*backendListener.InstancePort] = false
+				proxyProtocolBackends[*backendListener.InstancePort] = proxyProtocolEnabled(backendListener)
+			}
+
+			for _, listener := range listeners {
+				setPolicy := false
+				instancePort := *listener.InstancePort
+
+				if currentState, ok := proxyProtocolBackends[instancePort]; !ok {
+					// This is a new ELB backend so we only need to worry about
+					// potentially adding a policy and not removing an
+					// existing one
+					setPolicy = proxyProtocol
+				} else {
+					foundBackends[instancePort] = true
+					// This is an existing ELB backend so we need to determine
+					// if the state changed
+					setPolicy = (currentState != proxyProtocol)
+				}
+
+				if setPolicy {
+					glog.V(2).Infof("Adjusting AWS loadbalancer proxy protocol on node port %d. Setting to %t", instancePort, proxyProtocol)
+					err := c.setBackendPolicies(loadBalancerName, instancePort, proxyPolicies)
+					if err != nil {
+						return nil, err
+					}
+					dirty = true
+				}
+			}
+
+			// We now need to figure out if any backend policies need removed
+			// because these old policies will stick around even if there is no
+			// corresponding listener anymore
+			for instancePort, found := range foundBackends {
+				if !found {
+					glog.V(2).Infof("Adjusting AWS loadbalancer proxy protocol on node port %d. Setting to false", instancePort)
+					err := c.setBackendPolicies(loadBalancerName, instancePort, []*string{})
+					if err != nil {
+						return nil, err
+					}
+					dirty = true
+				}
+			}
+		}
+	}
+
+	// Whether the ELB was new or existing, sync attributes regardless. This accounts for things
+	// that cannot be specified at the time of creation and can only be modified after the fact,
+	// e.g. idle connection timeout.
+	{
+		describeAttributesRequest := &elb.DescribeLoadBalancerAttributesInput{}
+		describeAttributesRequest.LoadBalancerName = aws.String(loadBalancerName)
+		describeAttributesOutput, err := c.elb.DescribeLoadBalancerAttributes(describeAttributesRequest)
+		if err != nil {
+			glog.Warning("Unable to retrieve load balancer attributes during attribute sync")
+			return nil, err
+		}
+
+		foundAttributes := &describeAttributesOutput.LoadBalancerAttributes
+
+		// Update attributes if they're dirty
+		if !reflect.DeepEqual(loadBalancerAttributes, foundAttributes) {
+			glog.V(2).Infof("Updating load-balancer attributes for %q", loadBalancerName)
+
+			modifyAttributesRequest := &elb.ModifyLoadBalancerAttributesInput{}
+			modifyAttributesRequest.LoadBalancerName = aws.String(loadBalancerName)
+			modifyAttributesRequest.LoadBalancerAttributes = loadBalancerAttributes
+			_, err = c.elb.ModifyLoadBalancerAttributes(modifyAttributesRequest)
+			if err != nil {
+				return nil, fmt.Errorf("Unable to update load balancer attributes during attribute sync: %q", err)
+			}
+			dirty = true
+		}
+	}
+
+	if dirty {
+		loadBalancer, err = c.describeLoadBalancer(loadBalancerName)
+		if err != nil {
+			glog.Warning("Unable to retrieve load balancer after creation/update")
+			return nil, err
+		}
+	}
+
+	return loadBalancer, nil
+}
+
+// elbProtocolsAreEqual checks if two ELB protocol strings are considered the same
+// Comparison is case insensitive
+func elbProtocolsAreEqual(l, r *string) bool {
+	if l == nil || r == nil {
+		return l == r
+	}
+	return strings.EqualFold(aws.StringValue(l), aws.StringValue(r))
+}
+
+// awsArnEquals checks if two ARN strings are considered the same
+// Comparison is case insensitive
+func awsArnEquals(l, r *string) bool {
+	if l == nil || r == nil {
+		return l == r
+	}
+	return strings.EqualFold(aws.StringValue(l), aws.StringValue(r))
+}
+
+// Makes sure that the health check for an ELB matches the configured health check node port
+func (c *Cloud) ensureLoadBalancerHealthCheck(loadBalancer *elb.LoadBalancerDescription, protocol string, port int32, path string) error {
+	name := aws.StringValue(loadBalancer.LoadBalancerName)
+
+	actual := loadBalancer.HealthCheck
+
+	// Default AWS settings
+	expectedHealthyThreshold := int64(2)
+	expectedUnhealthyThreshold := int64(6)
+	expectedTimeout := int64(5)
+	expectedInterval := int64(10)
+
+	expectedTarget := protocol + ":" + strconv.FormatInt(int64(port), 10) + path
+
+	if expectedTarget == aws.StringValue(actual.Target) &&
+		expectedHealthyThreshold == orZero(actual.HealthyThreshold) &&
+		expectedUnhealthyThreshold == orZero(actual.UnhealthyThreshold) &&
+		expectedTimeout == orZero(actual.Timeout) &&
+		expectedInterval == orZero(actual.Interval) {
+		return nil
+	}
+
+	glog.V(2).Infof("Updating load-balancer health-check for %q", name)
+
+	healthCheck := &elb.HealthCheck{}
+	healthCheck.HealthyThreshold = &expectedHealthyThreshold
+	healthCheck.UnhealthyThreshold = &expectedUnhealthyThreshold
+	healthCheck.Timeout = &expectedTimeout
+	healthCheck.Interval = &expectedInterval
+	healthCheck.Target = &expectedTarget
+
+	request := &elb.ConfigureHealthCheckInput{}
+	request.HealthCheck = healthCheck
+	request.LoadBalancerName = loadBalancer.LoadBalancerName
+
+	_, err := c.elb.ConfigureHealthCheck(request)
+	if err != nil {
+		return fmt.Errorf("error configuring load-balancer health-check for %q: %q", name, err)
+	}
+
+	return nil
+}
+
+// Makes sure that exactly the specified hosts are registered as instances with the load balancer
+func (c *Cloud) ensureLoadBalancerInstances(loadBalancerName string, lbInstances []*elb.Instance, instanceIDs map[awsInstanceID]*ec2.Instance) error {
+	expected := sets.NewString()
+	for id := range instanceIDs {
+		expected.Insert(string(id))
+	}
+
+	actual := sets.NewString()
+	for _, lbInstance := range lbInstances {
+		actual.Insert(aws.StringValue(lbInstance.InstanceId))
+	}
+
+	additions := expected.Difference(actual)
+	removals := actual.Difference(expected)
+
+	addInstances := []*elb.Instance{}
+	for _, instanceId := range additions.List() {
+		addInstance := &elb.Instance{}
+		addInstance.InstanceId = aws.String(instanceId)
+		addInstances = append(addInstances, addInstance)
+	}
+
+	removeInstances := []*elb.Instance{}
+	for _, instanceId := range removals.List() {
+		removeInstance := &elb.Instance{}
+		removeInstance.InstanceId = aws.String(instanceId)
+		removeInstances = append(removeInstances, removeInstance)
+	}
+
+	if len(addInstances) > 0 {
+		registerRequest := &elb.RegisterInstancesWithLoadBalancerInput{}
+		registerRequest.Instances = addInstances
+		registerRequest.LoadBalancerName = aws.String(loadBalancerName)
+		_, err := c.elb.RegisterInstancesWithLoadBalancer(registerRequest)
+		if err != nil {
+			return err
+		}
+		glog.V(1).Infof("Instances added to load-balancer %s", loadBalancerName)
+	}
+
+	if len(removeInstances) > 0 {
+		deregisterRequest := &elb.DeregisterInstancesFromLoadBalancerInput{}
+		deregisterRequest.Instances = removeInstances
+		deregisterRequest.LoadBalancerName = aws.String(loadBalancerName)
+		_, err := c.elb.DeregisterInstancesFromLoadBalancer(deregisterRequest)
+		if err != nil {
+			return err
+		}
+		glog.V(1).Infof("Instances removed from load-balancer %s", loadBalancerName)
+	}
+
+	return nil
+}
+
+func (c *Cloud) createProxyProtocolPolicy(loadBalancerName string) error {
+	request := &elb.CreateLoadBalancerPolicyInput{
+		LoadBalancerName: aws.String(loadBalancerName),
+		PolicyName:       aws.String(ProxyProtocolPolicyName),
+		PolicyTypeName:   aws.String("ProxyProtocolPolicyType"),
+		PolicyAttributes: []*elb.PolicyAttribute{
+			{
+				AttributeName:  aws.String("ProxyProtocol"),
+				AttributeValue: aws.String("true"),
+			},
+		},
+	}
+	glog.V(2).Info("Creating proxy protocol policy on load balancer")
+	_, err := c.elb.CreateLoadBalancerPolicy(request)
+	if err != nil {
+		return fmt.Errorf("error creating proxy protocol policy on load balancer: %q", err)
+	}
+
+	return nil
+}
+
+func (c *Cloud) setBackendPolicies(loadBalancerName string, instancePort int64, policies []*string) error {
+	request := &elb.SetLoadBalancerPoliciesForBackendServerInput{
+		InstancePort:     aws.Int64(instancePort),
+		LoadBalancerName: aws.String(loadBalancerName),
+		PolicyNames:      policies,
+	}
+	if len(policies) > 0 {
+		glog.V(2).Infof("Adding AWS loadbalancer backend policies on node port %d", instancePort)
+	} else {
+		glog.V(2).Infof("Removing AWS loadbalancer backend policies on node port %d", instancePort)
+	}
+	_, err := c.elb.SetLoadBalancerPoliciesForBackendServer(request)
+	if err != nil {
+		return fmt.Errorf("error adjusting AWS loadbalancer backend policies: %q", err)
+	}
+
+	return nil
+}
+
+func proxyProtocolEnabled(backend *elb.BackendServerDescription) bool {
+	for _, policy := range backend.PolicyNames {
+		if aws.StringValue(policy) == ProxyProtocolPolicyName {
+			return true
+		}
+	}
+
+	return false
+}
+
+// findInstancesForELB gets the EC2 instances corresponding to the Nodes, for setting up an ELB
+// We ignore Nodes (with a log message) where the instanceid cannot be determined from the provider,
+// and we ignore instances which are not found
+func (c *Cloud) findInstancesForELB(nodes []*v1.Node) (map[awsInstanceID]*ec2.Instance, error) {
+	// Map to instance ids ignoring Nodes where we cannot find the id (but logging)
+	instanceIDs := mapToAWSInstanceIDsTolerant(nodes)
+
+	cacheCriteria := cacheCriteria{
+		// MaxAge not required, because we only care about security groups, which should not change
+		HasInstances: instanceIDs, // Refresh if any of the instance ids are missing
+	}
+	snapshot, err := c.instanceCache.describeAllInstancesCached(cacheCriteria)
+	if err != nil {
+		return nil, err
+	}
+
+	instances := snapshot.FindInstances(instanceIDs)
+	// We ignore instances that cannot be found
+
+	return instances, nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/aws_loadbalancer_test.go

@@ -0,0 +1,127 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"testing"
+)
+
+func TestElbProtocolsAreEqual(t *testing.T) {
+	grid := []struct {
+		L        *string
+		R        *string
+		Expected bool
+	}{
+		{
+			L:        aws.String("http"),
+			R:        aws.String("http"),
+			Expected: true,
+		},
+		{
+			L:        aws.String("HTTP"),
+			R:        aws.String("http"),
+			Expected: true,
+		},
+		{
+			L:        aws.String("HTTP"),
+			R:        aws.String("TCP"),
+			Expected: false,
+		},
+		{
+			L:        aws.String(""),
+			R:        aws.String("TCP"),
+			Expected: false,
+		},
+		{
+			L:        aws.String(""),
+			R:        aws.String(""),
+			Expected: true,
+		},
+		{
+			L:        nil,
+			R:        aws.String(""),
+			Expected: false,
+		},
+		{
+			L:        aws.String(""),
+			R:        nil,
+			Expected: false,
+		},
+		{
+			L:        nil,
+			R:        nil,
+			Expected: true,
+		},
+	}
+	for _, g := range grid {
+		actual := elbProtocolsAreEqual(g.L, g.R)
+		if actual != g.Expected {
+			t.Errorf("unexpected result from protocolsEquals(%v, %v)", g.L, g.R)
+		}
+	}
+}
+
+func TestAWSARNEquals(t *testing.T) {
+	grid := []struct {
+		L        *string
+		R        *string
+		Expected bool
+	}{
+		{
+			L:        aws.String("arn:aws:acm:us-east-1:123456789012:certificate/12345678-1234-1234-1234-123456789012"),
+			R:        aws.String("arn:aws:acm:us-east-1:123456789012:certificate/12345678-1234-1234-1234-123456789012"),
+			Expected: true,
+		},
+		{
+			L:        aws.String("ARN:AWS:ACM:US-EAST-1:123456789012:CERTIFICATE/12345678-1234-1234-1234-123456789012"),
+			R:        aws.String("arn:aws:acm:us-east-1:123456789012:certificate/12345678-1234-1234-1234-123456789012"),
+			Expected: true,
+		},
+		{
+			L:        aws.String("arn:aws:acm:us-east-1:123456789012:certificate/12345678-1234-1234-1234-123456789012"),
+			R:        aws.String(""),
+			Expected: false,
+		},
+		{
+			L:        aws.String(""),
+			R:        aws.String(""),
+			Expected: true,
+		},
+		{
+			L:        nil,
+			R:        aws.String(""),
+			Expected: false,
+		},
+		{
+			L:        aws.String(""),
+			R:        nil,
+			Expected: false,
+		},
+		{
+			L:        nil,
+			R:        nil,
+			Expected: true,
+		},
+	}
+	for _, g := range grid {
+		actual := awsArnEquals(g.L, g.R)
+		if actual != g.Expected {
+			t.Errorf("unexpected result from awsArnEquals(%v, %v)", g.L, g.R)
+		}
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/aws_metrics.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var awsApiMetric = prometheus.NewHistogramVec(
+	prometheus.HistogramOpts{
+		Name: "cloudprovider_aws_api_request_duration_seconds",
+		Help: "Latency of aws api call",
+	},
+	[]string{"request"},
+)
+
+var awsApiErrorMetric = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "cloudprovider_aws_api_request_errors",
+		Help: "AWS Api errors",
+	},
+	[]string{"request"},
+)
+
+func registerMetrics() {
+	prometheus.MustRegister(awsApiMetric)
+	prometheus.MustRegister(awsApiErrorMetric)
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/aws_routes.go

@@ -0,0 +1,217 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+)
+
+func (c *Cloud) findRouteTable(clusterName string) (*ec2.RouteTable, error) {
+	// This should be unnecessary (we already filter on TagNameKubernetesCluster,
+	// and something is broken if cluster name doesn't match, but anyway...
+	// TODO: All clouds should be cluster-aware by default
+	var tables []*ec2.RouteTable
+
+	if c.cfg.Global.RouteTableID != "" {
+		request := &ec2.DescribeRouteTablesInput{Filters: []*ec2.Filter{newEc2Filter("route-table-id", c.cfg.Global.RouteTableID)}}
+		response, err := c.ec2.DescribeRouteTables(request)
+		if err != nil {
+			return nil, err
+		}
+
+		tables = response
+	} else {
+		request := &ec2.DescribeRouteTablesInput{Filters: c.tagging.addFilters(nil)}
+		response, err := c.ec2.DescribeRouteTables(request)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, table := range response {
+			if c.tagging.hasClusterTag(table.Tags) {
+				tables = append(tables, table)
+			}
+		}
+	}
+
+	if len(tables) == 0 {
+		return nil, fmt.Errorf("unable to find route table for AWS cluster: %s", clusterName)
+	}
+
+	if len(tables) != 1 {
+		return nil, fmt.Errorf("found multiple matching AWS route tables for AWS cluster: %s", clusterName)
+	}
+	return tables[0], nil
+}
+
+// ListRoutes implements Routes.ListRoutes
+// List all routes that match the filter
+func (c *Cloud) ListRoutes(clusterName string) ([]*cloudprovider.Route, error) {
+	table, err := c.findRouteTable(clusterName)
+	if err != nil {
+		return nil, err
+	}
+
+	var routes []*cloudprovider.Route
+	var instanceIDs []*string
+
+	for _, r := range table.Routes {
+		instanceID := aws.StringValue(r.InstanceId)
+
+		if instanceID == "" {
+			continue
+		}
+
+		instanceIDs = append(instanceIDs, &instanceID)
+	}
+
+	instances, err := c.getInstancesByIDs(instanceIDs)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, r := range table.Routes {
+		destinationCIDR := aws.StringValue(r.DestinationCidrBlock)
+		if destinationCIDR == "" {
+			continue
+		}
+
+		route := &cloudprovider.Route{
+			Name:            clusterName + "-" + destinationCIDR,
+			DestinationCIDR: destinationCIDR,
+		}
+
+		// Capture blackhole routes
+		if aws.StringValue(r.State) == ec2.RouteStateBlackhole {
+			route.Blackhole = true
+			routes = append(routes, route)
+			continue
+		}
+
+		// Capture instance routes
+		instanceID := aws.StringValue(r.InstanceId)
+		if instanceID != "" {
+			instance, found := instances[instanceID]
+			if found {
+				route.TargetNode = mapInstanceToNodeName(instance)
+				routes = append(routes, route)
+			} else {
+				glog.Warningf("unable to find instance ID %s in the list of instances being routed to", instanceID)
+			}
+		}
+	}
+
+	return routes, nil
+}
+
+// Sets the instance attribute "source-dest-check" to the specified value
+func (c *Cloud) configureInstanceSourceDestCheck(instanceID string, sourceDestCheck bool) error {
+	request := &ec2.ModifyInstanceAttributeInput{}
+	request.InstanceId = aws.String(instanceID)
+	request.SourceDestCheck = &ec2.AttributeBooleanValue{Value: aws.Bool(sourceDestCheck)}
+
+	_, err := c.ec2.ModifyInstanceAttribute(request)
+	if err != nil {
+		return fmt.Errorf("error configuring source-dest-check on instance %s: %q", instanceID, err)
+	}
+	return nil
+}
+
+// CreateRoute implements Routes.CreateRoute
+// Create the described route
+func (c *Cloud) CreateRoute(clusterName string, nameHint string, route *cloudprovider.Route) error {
+	instance, err := c.getInstanceByNodeName(route.TargetNode)
+	if err != nil {
+		return err
+	}
+
+	// In addition to configuring the route itself, we also need to configure the instance to accept that traffic
+	// On AWS, this requires turning source-dest checks off
+	err = c.configureInstanceSourceDestCheck(aws.StringValue(instance.InstanceId), false)
+	if err != nil {
+		return err
+	}
+
+	table, err := c.findRouteTable(clusterName)
+	if err != nil {
+		return err
+	}
+
+	var deleteRoute *ec2.Route
+	for _, r := range table.Routes {
+		destinationCIDR := aws.StringValue(r.DestinationCidrBlock)
+
+		if destinationCIDR != route.DestinationCIDR {
+			continue
+		}
+
+		if aws.StringValue(r.State) == ec2.RouteStateBlackhole {
+			deleteRoute = r
+		}
+	}
+
+	if deleteRoute != nil {
+		glog.Infof("deleting blackholed route: %s", aws.StringValue(deleteRoute.DestinationCidrBlock))
+
+		request := &ec2.DeleteRouteInput{}
+		request.DestinationCidrBlock = deleteRoute.DestinationCidrBlock
+		request.RouteTableId = table.RouteTableId
+
+		_, err = c.ec2.DeleteRoute(request)
+		if err != nil {
+			return fmt.Errorf("error deleting blackholed AWS route (%s): %q", aws.StringValue(deleteRoute.DestinationCidrBlock), err)
+		}
+	}
+
+	request := &ec2.CreateRouteInput{}
+	// TODO: use ClientToken for idempotency?
+	request.DestinationCidrBlock = aws.String(route.DestinationCIDR)
+	request.InstanceId = instance.InstanceId
+	request.RouteTableId = table.RouteTableId
+
+	_, err = c.ec2.CreateRoute(request)
+	if err != nil {
+		return fmt.Errorf("error creating AWS route (%s): %q", route.DestinationCIDR, err)
+	}
+
+	return nil
+}
+
+// DeleteRoute implements Routes.DeleteRoute
+// Delete the specified route
+func (c *Cloud) DeleteRoute(clusterName string, route *cloudprovider.Route) error {
+	table, err := c.findRouteTable(clusterName)
+	if err != nil {
+		return err
+	}
+
+	request := &ec2.DeleteRouteInput{}
+	request.DestinationCidrBlock = aws.String(route.DestinationCIDR)
+	request.RouteTableId = table.RouteTableId
+
+	_, err = c.ec2.DeleteRoute(request)
+	if err != nil {
+		return fmt.Errorf("error deleting AWS route (%s): %q", route.DestinationCIDR, err)
+	}
+
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/aws_utils.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+func stringSetToPointers(in sets.String) []*string {
+	if in == nil {
+		return nil
+	}
+	out := make([]*string, 0, len(in))
+	for k := range in {
+		out = append(out, aws.String(k))
+	}
+	return out
+}
+
+func stringSetFromPointers(in []*string) sets.String {
+	if in == nil {
+		return nil
+	}
+	out := sets.NewString()
+	for i := range in {
+		out.Insert(aws.StringValue(in[i]))
+	}
+	return out
+}
+
+// orZero returns the value, or 0 if the pointer is nil
+// Deprecated: prefer aws.Int64Value
+func orZero(v *int64) int64 {
+	return aws.Int64Value(v)
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/device_allocator.go

@@ -0,0 +1,130 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"fmt"
+	"sort"
+	"sync"
+)
+
+// ExistingDevices is a map of assigned devices. Presence of a key with a device
+// name in the map means that the device is allocated. Value is irrelevant and
+// can be used for anything that DeviceAllocator user wants.
+// Only the relevant part of device name should be in the map, e.g. "ba" for
+// "/dev/xvdba".
+type ExistingDevices map[mountDevice]awsVolumeID
+
+// On AWS, we should assign new (not yet used) device names to attached volumes.
+// If we reuse a previously used name, we may get the volume "attaching" forever,
+// see https://aws.amazon.com/premiumsupport/knowledge-center/ebs-stuck-attaching/.
+// DeviceAllocator finds available device name, taking into account already
+// assigned device names from ExistingDevices map. It tries to find the next
+// device name to the previously assigned one (from previous DeviceAllocator
+// call), so all available device names are used eventually and it minimizes
+// device name reuse.
+// All these allocations are in-memory, nothing is written to / read from
+// /dev directory.
+type DeviceAllocator interface {
+	// GetNext returns a free device name or error when there is no free device
+	// name. Only the device suffix is returned, e.g. "ba" for "/dev/xvdba".
+	// It's up to the called to add appropriate "/dev/sd" or "/dev/xvd" prefix.
+	GetNext(existingDevices ExistingDevices) (mountDevice, error)
+
+	// Deprioritize the device so as it can't be used immediately again
+	Deprioritize(mountDevice)
+
+	// Lock the deviceAllocator
+	Lock()
+
+	// Unlock the deviceAllocator
+	Unlock()
+}
+
+type deviceAllocator struct {
+	possibleDevices map[mountDevice]int
+	counter         int
+	deviceLock      sync.Mutex
+}
+
+var _ DeviceAllocator = &deviceAllocator{}
+
+type devicePair struct {
+	deviceName  mountDevice
+	deviceIndex int
+}
+
+type devicePairList []devicePair
+
+func (p devicePairList) Len() int           { return len(p) }
+func (p devicePairList) Less(i, j int) bool { return p[i].deviceIndex < p[j].deviceIndex }
+func (p devicePairList) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+// Allocates device names according to scheme ba..bz, ca..cz
+// it moves along the ring and always picks next device until
+// device list is exhausted.
+func NewDeviceAllocator() DeviceAllocator {
+	possibleDevices := make(map[mountDevice]int)
+	for _, firstChar := range []rune{'b', 'c'} {
+		for i := 'a'; i <= 'z'; i++ {
+			dev := mountDevice([]rune{firstChar, i})
+			possibleDevices[dev] = 0
+		}
+	}
+	return &deviceAllocator{
+		possibleDevices: possibleDevices,
+		counter:         0,
+	}
+}
+
+// GetNext gets next available device from the pool, this function assumes that caller
+// holds the necessary lock on deviceAllocator
+func (d *deviceAllocator) GetNext(existingDevices ExistingDevices) (mountDevice, error) {
+	for _, devicePair := range d.sortByCount() {
+		if _, found := existingDevices[devicePair.deviceName]; !found {
+			return devicePair.deviceName, nil
+		}
+	}
+	return "", fmt.Errorf("no devices are available")
+}
+
+func (d *deviceAllocator) sortByCount() devicePairList {
+	dpl := make(devicePairList, 0)
+	for deviceName, deviceIndex := range d.possibleDevices {
+		dpl = append(dpl, devicePair{deviceName, deviceIndex})
+	}
+	sort.Sort(dpl)
+	return dpl
+}
+
+func (d *deviceAllocator) Lock() {
+	d.deviceLock.Lock()
+}
+
+func (d *deviceAllocator) Unlock() {
+	d.deviceLock.Unlock()
+}
+
+// Deprioritize the device so as it can't be used immediately again
+func (d *deviceAllocator) Deprioritize(chosen mountDevice) {
+	d.deviceLock.Lock()
+	defer d.deviceLock.Unlock()
+	if _, ok := d.possibleDevices[chosen]; ok {
+		d.counter++
+		d.possibleDevices[chosen] = d.counter
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/device_allocator_test.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import "testing"
+
+func TestDeviceAllocator(t *testing.T) {
+	tests := []struct {
+		name            string
+		existingDevices ExistingDevices
+		deviceMap       map[mountDevice]int
+		expectedOutput  mountDevice
+	}{
+		{
+			"empty device list with wrap",
+			ExistingDevices{},
+			generateUnsortedDeviceList(),
+			"bd", // next to 'zz' is the first one, 'ba'
+		},
+	}
+
+	for _, test := range tests {
+		allocator := NewDeviceAllocator().(*deviceAllocator)
+		for k, v := range test.deviceMap {
+			allocator.possibleDevices[k] = v
+		}
+
+		got, err := allocator.GetNext(test.existingDevices)
+		if err != nil {
+			t.Errorf("text %q: unexpected error: %v", test.name, err)
+		}
+		if got != test.expectedOutput {
+			t.Errorf("text %q: expected %q, got %q", test.name, test.expectedOutput, got)
+		}
+	}
+}
+
+func generateUnsortedDeviceList() map[mountDevice]int {
+	possibleDevices := make(map[mountDevice]int)
+	for _, firstChar := range []rune{'b', 'c'} {
+		for i := 'a'; i <= 'z'; i++ {
+			dev := mountDevice([]rune{firstChar, i})
+			possibleDevices[dev] = 3
+		}
+	}
+	possibleDevices["bd"] = 0
+	return possibleDevices
+}
+
+func TestDeviceAllocatorError(t *testing.T) {
+	allocator := NewDeviceAllocator().(*deviceAllocator)
+	existingDevices := ExistingDevices{}
+
+	// make all devices used
+	var first, second byte
+	for first = 'b'; first <= 'c'; first++ {
+		for second = 'a'; second <= 'z'; second++ {
+			device := [2]byte{first, second}
+			existingDevices[mountDevice(device[:])] = "used"
+		}
+	}
+
+	device, err := allocator.GetNext(existingDevices)
+	if err == nil {
+		t.Errorf("expected error, got device  %q", device)
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/instances.go

@@ -0,0 +1,269 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/golang/glog"
+	"k8s.io/api/core/v1"
+	"sync"
+	"time"
+)
+
+// awsInstanceID represents the ID of the instance in the AWS API, e.g. i-12345678
+// The "traditional" format is "i-12345678"
+// A new longer format is also being introduced: "i-12345678abcdef01"
+// We should not assume anything about the length or format, though it seems
+// reasonable to assume that instances will continue to start with "i-".
+type awsInstanceID string
+
+func (i awsInstanceID) awsString() *string {
+	return aws.String(string(i))
+}
+
+// kubernetesInstanceID represents the id for an instance in the kubernetes API;
+// the following form
+//  * aws:///<zone>/<awsInstanceId>
+//  * aws:////<awsInstanceId>
+//  * <awsInstanceId>
+type kubernetesInstanceID string
+
+// mapToAWSInstanceID extracts the awsInstanceID from the kubernetesInstanceID
+func (name kubernetesInstanceID) mapToAWSInstanceID() (awsInstanceID, error) {
+	s := string(name)
+
+	if !strings.HasPrefix(s, "aws://") {
+		// Assume a bare aws volume id (vol-1234...)
+		// Build a URL with an empty host (AZ)
+		s = "aws://" + "/" + "/" + s
+	}
+	url, err := url.Parse(s)
+	if err != nil {
+		return "", fmt.Errorf("Invalid instance name (%s): %v", name, err)
+	}
+	if url.Scheme != "aws" {
+		return "", fmt.Errorf("Invalid scheme for AWS instance (%s)", name)
+	}
+
+	awsID := ""
+	tokens := strings.Split(strings.Trim(url.Path, "/"), "/")
+	if len(tokens) == 1 {
+		// instanceId
+		awsID = tokens[0]
+	} else if len(tokens) == 2 {
+		// az/instanceId
+		awsID = tokens[1]
+	}
+
+	// We sanity check the resulting volume; the two known formats are
+	// i-12345678 and i-12345678abcdef01
+	// TODO: Regex match?
+	if awsID == "" || strings.Contains(awsID, "/") || !strings.HasPrefix(awsID, "i-") {
+		return "", fmt.Errorf("Invalid format for AWS instance (%s)", name)
+	}
+
+	return awsInstanceID(awsID), nil
+}
+
+// mapToAWSInstanceID extracts the awsInstanceIDs from the Nodes, returning an error if a Node cannot be mapped
+func mapToAWSInstanceIDs(nodes []*v1.Node) ([]awsInstanceID, error) {
+	var instanceIDs []awsInstanceID
+	for _, node := range nodes {
+		if node.Spec.ProviderID == "" {
+			return nil, fmt.Errorf("node %q did not have ProviderID set", node.Name)
+		}
+		instanceID, err := kubernetesInstanceID(node.Spec.ProviderID).mapToAWSInstanceID()
+		if err != nil {
+			return nil, fmt.Errorf("unable to parse ProviderID %q for node %q", node.Spec.ProviderID, node.Name)
+		}
+		instanceIDs = append(instanceIDs, instanceID)
+	}
+
+	return instanceIDs, nil
+}
+
+// mapToAWSInstanceIDsTolerant extracts the awsInstanceIDs from the Nodes, skipping Nodes that cannot be mapped
+func mapToAWSInstanceIDsTolerant(nodes []*v1.Node) []awsInstanceID {
+	var instanceIDs []awsInstanceID
+	for _, node := range nodes {
+		if node.Spec.ProviderID == "" {
+			glog.Warningf("node %q did not have ProviderID set", node.Name)
+			continue
+		}
+		instanceID, err := kubernetesInstanceID(node.Spec.ProviderID).mapToAWSInstanceID()
+		if err != nil {
+			glog.Warningf("unable to parse ProviderID %q for node %q", node.Spec.ProviderID, node.Name)
+			continue
+		}
+		instanceIDs = append(instanceIDs, instanceID)
+	}
+
+	return instanceIDs
+}
+
+// Gets the full information about this instance from the EC2 API
+func describeInstance(ec2Client EC2, instanceID awsInstanceID) (*ec2.Instance, error) {
+	request := &ec2.DescribeInstancesInput{
+		InstanceIds: []*string{instanceID.awsString()},
+	}
+
+	instances, err := ec2Client.DescribeInstances(request)
+	if err != nil {
+		return nil, err
+	}
+	if len(instances) == 0 {
+		return nil, fmt.Errorf("no instances found for instance: %s", instanceID)
+	}
+	if len(instances) > 1 {
+		return nil, fmt.Errorf("multiple instances found for instance: %s", instanceID)
+	}
+	return instances[0], nil
+}
+
+// instanceCache manages the cache of DescribeInstances
+type instanceCache struct {
+	// TODO: Get rid of this field, send all calls through the instanceCache
+	cloud *Cloud
+
+	mutex    sync.Mutex
+	snapshot *allInstancesSnapshot
+}
+
+// Gets the full information about these instance from the EC2 API
+func (c *instanceCache) describeAllInstancesUncached() (*allInstancesSnapshot, error) {
+	now := time.Now()
+
+	glog.V(4).Infof("EC2 DescribeInstances - fetching all instances")
+
+	filters := []*ec2.Filter{}
+	instances, err := c.cloud.describeInstances(filters)
+	if err != nil {
+		return nil, err
+	}
+
+	m := make(map[awsInstanceID]*ec2.Instance)
+	for _, i := range instances {
+		id := awsInstanceID(aws.StringValue(i.InstanceId))
+		m[id] = i
+	}
+
+	snapshot := &allInstancesSnapshot{now, m}
+
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+
+	if c.snapshot != nil && snapshot.olderThan(c.snapshot) {
+		// If this happens a lot, we could run this function in a mutex and only return one result
+		glog.Infof("Not caching concurrent AWS DescribeInstances results")
+	} else {
+		c.snapshot = snapshot
+	}
+
+	return snapshot, nil
+}
+
+// cacheCriteria holds criteria that must hold to use a cached snapshot
+type cacheCriteria struct {
+	// MaxAge indicates the maximum age of a cached snapshot we can accept.
+	// If set to 0 (i.e. unset), cached values will not time out because of age.
+	MaxAge time.Duration
+
+	// HasInstances is a list of awsInstanceIDs that must be in a cached snapshot for it to be considered valid.
+	// If an instance is not found in the cached snapshot, the snapshot be ignored and we will re-fetch.
+	HasInstances []awsInstanceID
+}
+
+// describeAllInstancesCached returns all instances, using cached results if applicable
+func (c *instanceCache) describeAllInstancesCached(criteria cacheCriteria) (*allInstancesSnapshot, error) {
+	var err error
+	snapshot := c.getSnapshot()
+	if snapshot != nil && !snapshot.MeetsCriteria(criteria) {
+		snapshot = nil
+	}
+
+	if snapshot == nil {
+		snapshot, err = c.describeAllInstancesUncached()
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		glog.V(6).Infof("EC2 DescribeInstances - using cached results")
+	}
+
+	return snapshot, nil
+}
+
+// getSnapshot returns a snapshot if one exists
+func (c *instanceCache) getSnapshot() *allInstancesSnapshot {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+
+	return c.snapshot
+}
+
+// olderThan is a simple helper to encapsulate timestamp comparison
+func (s *allInstancesSnapshot) olderThan(other *allInstancesSnapshot) bool {
+	// After() is technically broken by time changes until we have monotonic time
+	return other.timestamp.After(s.timestamp)
+}
+
+// MeetsCriteria returns true if the snapshot meets the criteria in cacheCriteria
+func (s *allInstancesSnapshot) MeetsCriteria(criteria cacheCriteria) bool {
+	if criteria.MaxAge > 0 {
+		// Sub() is technically broken by time changes until we have monotonic time
+		now := time.Now()
+		if now.Sub(s.timestamp) > criteria.MaxAge {
+			glog.V(6).Infof("instanceCache snapshot cannot be used as is older than MaxAge=%s", criteria.MaxAge)
+			return false
+		}
+	}
+
+	if len(criteria.HasInstances) != 0 {
+		for _, id := range criteria.HasInstances {
+			if nil == s.instances[id] {
+				glog.V(6).Infof("instanceCache snapshot cannot be used as does not contain instance %s", id)
+				return false
+			}
+		}
+	}
+
+	return true
+}
+
+// allInstancesSnapshot holds the results from querying for all instances,
+// along with the timestamp for cache-invalidation purposes
+type allInstancesSnapshot struct {
+	timestamp time.Time
+	instances map[awsInstanceID]*ec2.Instance
+}
+
+// FindInstances returns the instances corresponding to the specified ids.  If an id is not found, it is ignored.
+func (s *allInstancesSnapshot) FindInstances(ids []awsInstanceID) map[awsInstanceID]*ec2.Instance {
+	m := make(map[awsInstanceID]*ec2.Instance)
+	for _, id := range ids {
+		instance := s.instances[id]
+		if instance != nil {
+			m[id] = instance
+		}
+	}
+	return m
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/instances_test.go

@@ -0,0 +1,199 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/stretchr/testify/assert"
+	"k8s.io/api/core/v1"
+	"testing"
+	"time"
+)
+
+func TestParseInstance(t *testing.T) {
+	tests := []struct {
+		Kubernetes  kubernetesInstanceID
+		Aws         awsInstanceID
+		ExpectError bool
+	}{
+		{
+			Kubernetes: "aws:///us-east-1a/i-12345678",
+			Aws:        "i-12345678",
+		},
+		{
+			Kubernetes: "aws:////i-12345678",
+			Aws:        "i-12345678",
+		},
+		{
+			Kubernetes: "i-12345678",
+			Aws:        "i-12345678",
+		},
+		{
+			Kubernetes: "aws:///us-east-1a/i-12345678abcdef01",
+			Aws:        "i-12345678abcdef01",
+		},
+		{
+			Kubernetes: "aws:////i-12345678abcdef01",
+			Aws:        "i-12345678abcdef01",
+		},
+		{
+			Kubernetes: "i-12345678abcdef01",
+			Aws:        "i-12345678abcdef01",
+		},
+		{
+			Kubernetes:  "vol-123456789",
+			ExpectError: true,
+		},
+		{
+			Kubernetes:  "aws:///us-east-1a/vol-12345678abcdef01",
+			ExpectError: true,
+		},
+		{
+			Kubernetes:  "aws://accountid/us-east-1a/vol-12345678abcdef01",
+			ExpectError: true,
+		},
+		{
+			Kubernetes:  "aws:///us-east-1a/vol-12345678abcdef01/suffix",
+			ExpectError: true,
+		},
+		{
+			Kubernetes:  "",
+			ExpectError: true,
+		},
+	}
+
+	for _, test := range tests {
+		awsID, err := test.Kubernetes.mapToAWSInstanceID()
+		if err != nil {
+			if !test.ExpectError {
+				t.Errorf("unexpected error parsing %s: %v", test.Kubernetes, err)
+			}
+		} else {
+			if test.ExpectError {
+				t.Errorf("expected error parsing %s", test.Kubernetes)
+			} else if test.Aws != awsID {
+				t.Errorf("unexpected value parsing %s, got %s", test.Kubernetes, awsID)
+			}
+		}
+	}
+
+	for _, test := range tests {
+		node := &v1.Node{}
+		node.Spec.ProviderID = string(test.Kubernetes)
+
+		awsInstanceIds, err := mapToAWSInstanceIDs([]*v1.Node{node})
+		if err != nil {
+			if !test.ExpectError {
+				t.Errorf("unexpected error parsing %s: %v", test.Kubernetes, err)
+			}
+		} else {
+			if test.ExpectError {
+				t.Errorf("expected error parsing %s", test.Kubernetes)
+			} else if len(awsInstanceIds) != 1 {
+				t.Errorf("unexpected value parsing %s, got %s", test.Kubernetes, awsInstanceIds)
+			} else if awsInstanceIds[0] != test.Aws {
+				t.Errorf("unexpected value parsing %s, got %s", test.Kubernetes, awsInstanceIds)
+			}
+		}
+
+		awsInstanceIds = mapToAWSInstanceIDsTolerant([]*v1.Node{node})
+		if test.ExpectError {
+			if len(awsInstanceIds) != 0 {
+				t.Errorf("unexpected results parsing %s: %s", test.Kubernetes, awsInstanceIds)
+			}
+		} else {
+			if len(awsInstanceIds) != 1 {
+				t.Errorf("unexpected value parsing %s, got %s", test.Kubernetes, awsInstanceIds)
+			} else if awsInstanceIds[0] != test.Aws {
+				t.Errorf("unexpected value parsing %s, got %s", test.Kubernetes, awsInstanceIds)
+			}
+		}
+	}
+}
+
+func TestSnapshotMeetsCriteria(t *testing.T) {
+	snapshot := &allInstancesSnapshot{timestamp: time.Now().Add(-3601 * time.Second)}
+
+	if !snapshot.MeetsCriteria(cacheCriteria{}) {
+		t.Errorf("Snapshot should always meet empty criteria")
+	}
+
+	if snapshot.MeetsCriteria(cacheCriteria{MaxAge: time.Hour}) {
+		t.Errorf("Snapshot did not honor MaxAge")
+	}
+
+	if snapshot.MeetsCriteria(cacheCriteria{HasInstances: []awsInstanceID{awsInstanceID("i-12345678")}}) {
+		t.Errorf("Snapshot did not honor HasInstances with missing instances")
+	}
+
+	snapshot.instances = make(map[awsInstanceID]*ec2.Instance)
+	snapshot.instances[awsInstanceID("i-12345678")] = &ec2.Instance{}
+
+	if !snapshot.MeetsCriteria(cacheCriteria{HasInstances: []awsInstanceID{awsInstanceID("i-12345678")}}) {
+		t.Errorf("Snapshot did not honor HasInstances with matching instances")
+	}
+
+	if snapshot.MeetsCriteria(cacheCriteria{HasInstances: []awsInstanceID{awsInstanceID("i-12345678"), awsInstanceID("i-00000000")}}) {
+		t.Errorf("Snapshot did not honor HasInstances with partially matching instances")
+	}
+}
+
+func TestOlderThan(t *testing.T) {
+	t1 := time.Now()
+	t2 := t1.Add(time.Second)
+
+	s1 := &allInstancesSnapshot{timestamp: t1}
+	s2 := &allInstancesSnapshot{timestamp: t2}
+
+	assert.True(t, s1.olderThan(s2), "s1 should be olderThan s2")
+	assert.False(t, s2.olderThan(s1), "s2 not should be olderThan s1")
+	assert.False(t, s1.olderThan(s1), "s1 not should be olderThan itself")
+}
+
+func TestSnapshotFindInstances(t *testing.T) {
+	snapshot := &allInstancesSnapshot{}
+
+	snapshot.instances = make(map[awsInstanceID]*ec2.Instance)
+	{
+		id := awsInstanceID("i-12345678")
+		snapshot.instances[id] = &ec2.Instance{InstanceId: id.awsString()}
+	}
+	{
+		id := awsInstanceID("i-23456789")
+		snapshot.instances[id] = &ec2.Instance{InstanceId: id.awsString()}
+	}
+
+	instances := snapshot.FindInstances([]awsInstanceID{awsInstanceID("i-12345678"), awsInstanceID("i-23456789"), awsInstanceID("i-00000000")})
+	if len(instances) != 2 {
+		t.Errorf("findInstances returned %d results, expected 2", len(instances))
+	}
+
+	for _, id := range []awsInstanceID{awsInstanceID("i-12345678"), awsInstanceID("i-23456789")} {
+		i := instances[id]
+		if i == nil {
+			t.Errorf("findInstances did not return %s", id)
+			continue
+		}
+		if aws.StringValue(i.InstanceId) != string(id) {
+			t.Errorf("findInstances did not return expected instanceId for %s", id)
+		}
+		if i != snapshot.instances[id] {
+			t.Errorf("findInstances did not return expected instance (reference equality) for %s", id)
+		}
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/log_handler.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/golang/glog"
+)
+
+// Handler for aws-sdk-go that logs all requests
+func awsHandlerLogger(req *request.Request) {
+	service, name := awsServiceAndName(req)
+	glog.V(4).Infof("AWS request: %s %s", service, name)
+}
+
+func awsSendHandlerLogger(req *request.Request) {
+	service, name := awsServiceAndName(req)
+	glog.V(4).Infof("AWS API Send: %s %s %v %v", service, name, req.Operation, req.Params)
+}
+
+func awsValidateResponseHandlerLogger(req *request.Request) {
+	service, name := awsServiceAndName(req)
+	glog.V(4).Infof("AWS API ValidateResponse: %s %s %v %v %s", service, name, req.Operation, req.Params, req.HTTPResponse.Status)
+}
+
+func awsServiceAndName(req *request.Request) (string, string) {
+	service := req.ClientInfo.ServiceName
+
+	name := "?"
+	if req.Operation != nil {
+		name = req.Operation.Name
+	}
+	return service, name
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/regions.go

@@ -0,0 +1,94 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"github.com/golang/glog"
+	"k8s.io/apimachinery/pkg/util/sets"
+	awscredentialprovider "k8s.io/kubernetes/pkg/credentialprovider/aws"
+	"sync"
+)
+
+// WellKnownRegions is the complete list of regions known to the AWS cloudprovider
+// and credentialprovider.
+var WellKnownRegions = [...]string{
+	// from `aws ec2 describe-regions --region us-east-1 --query Regions[].RegionName | sort`
+	"ap-northeast-1",
+	"ap-northeast-2",
+	"ap-south-1",
+	"ap-southeast-1",
+	"ap-southeast-2",
+	"ca-central-1",
+	"eu-central-1",
+	"eu-west-1",
+	"eu-west-2",
+	"sa-east-1",
+	"us-east-1",
+	"us-east-2",
+	"us-west-1",
+	"us-west-2",
+
+	// these are not registered in many / most accounts
+	"cn-north-1",
+	"us-gov-west-1",
+}
+
+// awsRegionsMutex protects awsRegions
+var awsRegionsMutex sync.Mutex
+
+// awsRegions is a set of recognized regions
+var awsRegions sets.String
+
+// RecognizeRegion is called for each AWS region we know about.
+// It currently registers a credential provider for that region.
+// There are two paths to discovering a region:
+//  * we hard-code some well-known regions
+//  * if a region is discovered from instance metadata, we add that
+func RecognizeRegion(region string) {
+	awsRegionsMutex.Lock()
+	defer awsRegionsMutex.Unlock()
+
+	if awsRegions == nil {
+		awsRegions = sets.NewString()
+	}
+
+	if awsRegions.Has(region) {
+		glog.V(6).Infof("found AWS region %q again - ignoring", region)
+		return
+	}
+
+	glog.V(4).Infof("found AWS region %q", region)
+
+	awscredentialprovider.RegisterCredentialsProvider(region)
+
+	awsRegions.Insert(region)
+}
+
+// RecognizeWellKnownRegions calls RecognizeRegion on each WellKnownRegion
+func RecognizeWellKnownRegions() {
+	for _, region := range WellKnownRegions {
+		RecognizeRegion(region)
+	}
+}
+
+// isRegionValid checks if the region is in the set of known regions
+func isRegionValid(region string) bool {
+	awsRegionsMutex.Lock()
+	defer awsRegionsMutex.Unlock()
+
+	return awsRegions.Has(region)
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/regions_test.go

@@ -0,0 +1,85 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"testing"
+)
+
+// TestRegions does basic checking of region verification / addition
+func TestRegions(t *testing.T) {
+	RecognizeWellKnownRegions()
+
+	tests := []struct {
+		Add            string
+		Lookup         string
+		ExpectIsRegion bool
+	}{
+		{
+			Lookup:         "us-east-1",
+			ExpectIsRegion: true,
+		},
+		{
+			Lookup:         "us-east-1a",
+			ExpectIsRegion: false,
+		},
+		{
+			Add:            "us-test-1",
+			Lookup:         "us-east-1",
+			ExpectIsRegion: true,
+		},
+		{
+			Lookup:         "us-test-1",
+			ExpectIsRegion: true,
+		},
+		{
+			Add:            "us-test-1",
+			Lookup:         "us-test-1",
+			ExpectIsRegion: true,
+		},
+	}
+
+	for _, test := range tests {
+		if test.Add != "" {
+			RecognizeRegion(test.Add)
+		}
+
+		if test.Lookup != "" {
+			if isRegionValid(test.Lookup) != test.ExpectIsRegion {
+				t.Fatalf("region valid mismatch: %q", test.Lookup)
+			}
+		}
+	}
+}
+
+// TestRecognizesNewRegion verifies that we see a region from metadata, we recognize it as valid
+func TestRecognizesNewRegion(t *testing.T) {
+	region := "us-testrecognizesnewregion-1"
+	if isRegionValid(region) {
+		t.Fatalf("region already valid: %q", region)
+	}
+
+	awsServices := NewFakeAWSServices().withAz(region + "a")
+	_, err := newAWSCloud(nil, awsServices)
+	if err != nil {
+		t.Errorf("error building AWS cloud: %v", err)
+	}
+
+	if !isRegionValid(region) {
+		t.Fatalf("newly discovered region not valid: %q", region)
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/retry_handler.go

@@ -0,0 +1,161 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"math"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/golang/glog"
+)
+
+const (
+	decayIntervalSeconds = 20
+	decayFraction        = 0.8
+	maxDelay             = 60 * time.Second
+)
+
+// CrossRequestRetryDelay inserts delays before AWS calls, when we are observing RequestLimitExceeded errors
+// Note that we share a CrossRequestRetryDelay across multiple AWS requests; this is a process-wide back-off,
+// whereas the aws-sdk-go implements a per-request exponential backoff/retry
+type CrossRequestRetryDelay struct {
+	backoff Backoff
+}
+
+// Create a new CrossRequestRetryDelay
+func NewCrossRequestRetryDelay() *CrossRequestRetryDelay {
+	c := &CrossRequestRetryDelay{}
+	c.backoff.init(decayIntervalSeconds, decayFraction, maxDelay)
+	return c
+}
+
+// Added to the Sign chain; called before each request
+func (c *CrossRequestRetryDelay) BeforeSign(r *request.Request) {
+	now := time.Now()
+	delay := c.backoff.ComputeDelayForRequest(now)
+	if delay > 0 {
+		glog.Warningf("Inserting delay before AWS request (%s) to avoid RequestLimitExceeded: %s",
+			describeRequest(r), delay.String())
+		r.Config.SleepDelay(delay)
+
+		// Avoid clock skew problems
+		r.Time = now
+	}
+}
+
+// Return a user-friendly string describing the request, for use in log messages
+func describeRequest(r *request.Request) string {
+	service := r.ClientInfo.ServiceName
+
+	name := "?"
+	if r.Operation != nil {
+		name = r.Operation.Name
+	}
+
+	return service + "::" + name
+}
+
+// Added to the AfterRetry chain; called after any error
+func (c *CrossRequestRetryDelay) AfterRetry(r *request.Request) {
+	if r.Error == nil {
+		return
+	}
+	awsError, ok := r.Error.(awserr.Error)
+	if !ok {
+		return
+	}
+	if awsError.Code() == "RequestLimitExceeded" {
+		c.backoff.ReportError()
+		glog.Warningf("Got RequestLimitExceeded error on AWS request (%s)",
+			describeRequest(r))
+	}
+}
+
+// Backoff manages a backoff that varies based on the recently observed failures
+type Backoff struct {
+	decayIntervalSeconds int64
+	decayFraction        float64
+	maxDelay             time.Duration
+
+	mutex sync.Mutex
+
+	// We count all requests & the number of requests which hit a
+	// RequestLimit.  We only really care about 'recent' requests, so we
+	// decay the counts exponentially to bias towards recent values.
+	countErrorsRequestLimit float32
+	countRequests           float32
+	lastDecay               int64
+}
+
+func (b *Backoff) init(decayIntervalSeconds int, decayFraction float64, maxDelay time.Duration) {
+	b.lastDecay = time.Now().Unix()
+	// Bias so that if the first request hits the limit we don't immediately apply the full delay
+	b.countRequests = 4
+	b.decayIntervalSeconds = int64(decayIntervalSeconds)
+	b.decayFraction = decayFraction
+	b.maxDelay = maxDelay
+}
+
+// Computes the delay required for a request, also updating internal state to count this request
+func (b *Backoff) ComputeDelayForRequest(now time.Time) time.Duration {
+	b.mutex.Lock()
+	defer b.mutex.Unlock()
+
+	// Apply exponential decay to the counters
+	timeDeltaSeconds := now.Unix() - b.lastDecay
+	if timeDeltaSeconds > b.decayIntervalSeconds {
+		intervals := float64(timeDeltaSeconds) / float64(b.decayIntervalSeconds)
+		decay := float32(math.Pow(b.decayFraction, intervals))
+		b.countErrorsRequestLimit *= decay
+		b.countRequests *= decay
+		b.lastDecay = now.Unix()
+	}
+
+	// Count this request
+	b.countRequests += 1.0
+
+	// Compute the failure rate
+	errorFraction := float32(0.0)
+	if b.countRequests > 0.5 {
+		// Avoid tiny residuals & rounding errors
+		errorFraction = b.countErrorsRequestLimit / b.countRequests
+	}
+
+	// Ignore a low fraction of errors
+	// This also allows them to time-out
+	if errorFraction < 0.1 {
+		return time.Duration(0)
+	}
+
+	// Delay by the max delay multiplied by the recent error rate
+	// (i.e. we apply a linear delay function)
+	// TODO: This is pretty arbitrary
+	delay := time.Nanosecond * time.Duration(float32(b.maxDelay.Nanoseconds())*errorFraction)
+	// Round down to the nearest second for sanity
+	return time.Second * time.Duration(int(delay.Seconds()))
+}
+
+// Called when we observe a throttling error
+func (b *Backoff) ReportError() {
+	b.mutex.Lock()
+	defer b.mutex.Unlock()
+
+	b.countErrorsRequestLimit += 1.0
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/retry_handler_test.go

@@ -0,0 +1,135 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"testing"
+	"time"
+)
+
+// There follows a group of tests for the backoff logic.  There's nothing
+// particularly special about the values chosen: if we tweak the values in the
+// backoff logic then we might well have to update the tests.  However the key
+// behavioural elements should remain (e.g. no errors => no backoff), and these
+// are each tested by one of the tests below.
+
+// Test that we don't apply any delays when there are no errors
+func TestBackoffNoErrors(t *testing.T) {
+	b := &Backoff{}
+	b.init(decayIntervalSeconds, decayFraction, maxDelay)
+
+	now := time.Now()
+	for i := 0; i < 100; i++ {
+		d := b.ComputeDelayForRequest(now)
+		if d.Nanoseconds() != 0 {
+			t.Fatalf("unexpected delay during no-error case")
+		}
+		now = now.Add(time.Second)
+	}
+}
+
+// Test that we always apply a delay when there are errors, and also that we
+// don't "flap" - that our own delay doesn't cause us to oscillate between
+// delay and no-delay.
+func TestBackoffAllErrors(t *testing.T) {
+	b := &Backoff{}
+	b.init(decayIntervalSeconds, decayFraction, maxDelay)
+
+	now := time.Now()
+	// Warm up
+	for i := 0; i < 10; i++ {
+		_ = b.ComputeDelayForRequest(now)
+		b.ReportError()
+		now = now.Add(time.Second)
+	}
+
+	for i := 0; i < 100; i++ {
+		d := b.ComputeDelayForRequest(now)
+		b.ReportError()
+		if d.Seconds() < 5 {
+			t.Fatalf("unexpected short-delay during all-error case: %v", d)
+		}
+		t.Logf("delay @%d %v", i, d)
+		now = now.Add(d)
+	}
+}
+
+// Test that we do come close to our max delay, when we see all errors at 1
+// second intervals (this simulates multiple concurrent requests, because we
+// don't wait for delay in between requests)
+func TestBackoffHitsMax(t *testing.T) {
+	b := &Backoff{}
+	b.init(decayIntervalSeconds, decayFraction, maxDelay)
+
+	now := time.Now()
+	for i := 0; i < 100; i++ {
+		_ = b.ComputeDelayForRequest(now)
+		b.ReportError()
+		now = now.Add(time.Second)
+	}
+
+	for i := 0; i < 10; i++ {
+		d := b.ComputeDelayForRequest(now)
+		b.ReportError()
+		if float32(d.Nanoseconds()) < (float32(maxDelay.Nanoseconds()) * 0.95) {
+			t.Fatalf("expected delay to be >= 95 percent of max delay, was %v", d)
+		}
+		t.Logf("delay @%d %v", i, d)
+		now = now.Add(time.Second)
+	}
+}
+
+// Test that after a phase of errors, we eventually stop applying a delay once there are
+// no more errors.
+func TestBackoffRecovers(t *testing.T) {
+	b := &Backoff{}
+	b.init(decayIntervalSeconds, decayFraction, maxDelay)
+
+	now := time.Now()
+
+	// Phase of all-errors
+	for i := 0; i < 100; i++ {
+		_ = b.ComputeDelayForRequest(now)
+		b.ReportError()
+		now = now.Add(time.Second)
+	}
+
+	for i := 0; i < 10; i++ {
+		d := b.ComputeDelayForRequest(now)
+		b.ReportError()
+		if d.Seconds() < 5 {
+			t.Fatalf("unexpected short-delay during all-error phase: %v", d)
+		}
+		t.Logf("error phase delay @%d %v", i, d)
+		now = now.Add(time.Second)
+	}
+
+	// Phase of no errors
+	for i := 0; i < 100; i++ {
+		_ = b.ComputeDelayForRequest(now)
+		now = now.Add(3 * time.Second)
+	}
+
+	for i := 0; i < 10; i++ {
+		d := b.ComputeDelayForRequest(now)
+		if d.Seconds() != 0 {
+			t.Fatalf("unexpected delay during error recovery phase: %v", d)
+		}
+		t.Logf("no-error phase delay @%d %v", i, d)
+		now = now.Add(time.Second)
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/sets_ippermissions.go

@@ -0,0 +1,146 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+)
+
+type IPPermissionSet map[string]*ec2.IpPermission
+
+func NewIPPermissionSet(items ...*ec2.IpPermission) IPPermissionSet {
+	s := make(IPPermissionSet)
+	s.Insert(items...)
+	return s
+}
+
+// Ungroup splits permissions out into individual permissions
+// EC2 will combine permissions with the same port but different SourceRanges together, for example
+// We ungroup them so we can process them
+func (s IPPermissionSet) Ungroup() IPPermissionSet {
+	l := []*ec2.IpPermission{}
+	for _, p := range s.List() {
+		if len(p.IpRanges) <= 1 {
+			l = append(l, p)
+			continue
+		}
+		for _, ipRange := range p.IpRanges {
+			c := &ec2.IpPermission{}
+			*c = *p
+			c.IpRanges = []*ec2.IpRange{ipRange}
+			l = append(l, c)
+		}
+	}
+
+	l2 := []*ec2.IpPermission{}
+	for _, p := range l {
+		if len(p.UserIdGroupPairs) <= 1 {
+			l2 = append(l2, p)
+			continue
+		}
+		for _, u := range p.UserIdGroupPairs {
+			c := &ec2.IpPermission{}
+			*c = *p
+			c.UserIdGroupPairs = []*ec2.UserIdGroupPair{u}
+			l2 = append(l, c)
+		}
+	}
+
+	l3 := []*ec2.IpPermission{}
+	for _, p := range l2 {
+		if len(p.PrefixListIds) <= 1 {
+			l3 = append(l3, p)
+			continue
+		}
+		for _, v := range p.PrefixListIds {
+			c := &ec2.IpPermission{}
+			*c = *p
+			c.PrefixListIds = []*ec2.PrefixListId{v}
+			l3 = append(l3, c)
+		}
+	}
+
+	return NewIPPermissionSet(l3...)
+}
+
+// Insert adds items to the set.
+func (s IPPermissionSet) Insert(items ...*ec2.IpPermission) {
+	for _, p := range items {
+		k := keyForIPPermission(p)
+		s[k] = p
+	}
+}
+
+// List returns the contents as a slice.  Order is not defined.
+func (s IPPermissionSet) List() []*ec2.IpPermission {
+	res := make([]*ec2.IpPermission, 0, len(s))
+	for _, v := range s {
+		res = append(res, v)
+	}
+	return res
+}
+
+// IsSuperset returns true if and only if s1 is a superset of s2.
+func (s1 IPPermissionSet) IsSuperset(s2 IPPermissionSet) bool {
+	for k := range s2 {
+		_, found := s1[k]
+		if !found {
+			return false
+		}
+	}
+	return true
+}
+
+// Equal returns true if and only if s1 is equal (as a set) to s2.
+// Two sets are equal if their membership is identical.
+// (In practice, this means same elements, order doesn't matter)
+func (s1 IPPermissionSet) Equal(s2 IPPermissionSet) bool {
+	return len(s1) == len(s2) && s1.IsSuperset(s2)
+}
+
+// Difference returns a set of objects that are not in s2
+// For example:
+// s1 = {a1, a2, a3}
+// s2 = {a1, a2, a4, a5}
+// s1.Difference(s2) = {a3}
+// s2.Difference(s1) = {a4, a5}
+func (s IPPermissionSet) Difference(s2 IPPermissionSet) IPPermissionSet {
+	result := NewIPPermissionSet()
+	for k, v := range s {
+		_, found := s2[k]
+		if !found {
+			result[k] = v
+		}
+	}
+	return result
+}
+
+// Len returns the size of the set.
+func (s IPPermissionSet) Len() int {
+	return len(s)
+}
+
+func keyForIPPermission(p *ec2.IpPermission) string {
+	v, err := json.Marshal(p)
+	if err != nil {
+		panic(fmt.Sprintf("error building JSON representation of ec2.IpPermission: %v", err))
+	}
+	return string(v)
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/tags.go

@@ -0,0 +1,282 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"fmt"
+
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/golang/glog"
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+// TagNameKubernetesClusterPrefix is the tag name we use to differentiate multiple
+// logically independent clusters running in the same AZ.
+// The tag key = TagNameKubernetesClusterPrefix + clusterID
+// The tag value is an ownership value
+const TagNameKubernetesClusterPrefix = "kubernetes.io/cluster/"
+
+// TagNameKubernetesClusterLegacy is the legacy tag name we use to differentiate multiple
+// logically independent clusters running in the same AZ.  The problem with it was that it
+// did not allow shared resources.
+const TagNameKubernetesClusterLegacy = "KubernetesCluster"
+
+type ResourceLifecycle string
+
+const (
+	// ResourceLifecycleOwned is the value we use when tagging resources to indicate
+	// that the resource is considered owned and managed by the cluster,
+	// and in particular that the lifecycle is tied to the lifecycle of the cluster.
+	ResourceLifecycleOwned = "owned"
+	// ResourceLifecycleShared is the value we use when tagging resources to indicate
+	// that the resource is shared between multiple clusters, and should not be destroyed
+	// if the cluster is destroyed.
+	ResourceLifecycleShared = "shared"
+)
+
+type awsTagging struct {
+	// ClusterID is our cluster identifier: we tag AWS resources with this value,
+	// and thus we can run two independent clusters in the same VPC or subnets.
+	// This gives us similar functionality to GCE projects.
+	ClusterID string
+
+	// usesLegacyTags is true if we are using the legacy TagNameKubernetesClusterLegacy tags
+	usesLegacyTags bool
+}
+
+func (t *awsTagging) init(legacyClusterID string, clusterID string) error {
+	if legacyClusterID != "" {
+		if clusterID != "" && legacyClusterID != clusterID {
+			return fmt.Errorf("ClusterID tags did not match: %q vs %q", clusterID, legacyClusterID)
+		}
+		t.usesLegacyTags = true
+		clusterID = legacyClusterID
+	}
+
+	t.ClusterID = clusterID
+
+	if clusterID != "" {
+		glog.Infof("AWS cloud filtering on ClusterID: %v", clusterID)
+	} else {
+		glog.Warning("AWS cloud - no clusterID filtering applied for shared resources; do not run multiple clusters in this AZ.")
+	}
+
+	return nil
+}
+
+// Extracts a clusterID from the given tags, if one is present
+// If no clusterID is found, returns "", nil
+// If multiple (different) clusterIDs are found, returns an error
+func (t *awsTagging) initFromTags(tags []*ec2.Tag) error {
+	legacyClusterID, newClusterID, err := findClusterIDs(tags)
+	if err != nil {
+		return err
+	}
+
+	if legacyClusterID == "" && newClusterID == "" {
+		glog.Errorf("Tag %q nor %q not found; Kubernetes may behave unexpectedly.", TagNameKubernetesClusterLegacy, TagNameKubernetesClusterPrefix+"...")
+	}
+
+	return t.init(legacyClusterID, newClusterID)
+}
+
+// Extracts the legacy & new cluster ids from the given tags, if they are present
+// If duplicate tags are found, returns an error
+func findClusterIDs(tags []*ec2.Tag) (string, string, error) {
+	legacyClusterID := ""
+	newClusterID := ""
+
+	for _, tag := range tags {
+		tagKey := aws.StringValue(tag.Key)
+		if strings.HasPrefix(tagKey, TagNameKubernetesClusterPrefix) {
+			id := strings.TrimPrefix(tagKey, TagNameKubernetesClusterPrefix)
+			if newClusterID != "" {
+				return "", "", fmt.Errorf("Found multiple cluster tags with prefix %s (%q and %q)", TagNameKubernetesClusterPrefix, newClusterID, id)
+			}
+			newClusterID = id
+		}
+
+		if tagKey == TagNameKubernetesClusterLegacy {
+			id := aws.StringValue(tag.Value)
+			if legacyClusterID != "" {
+				return "", "", fmt.Errorf("Found multiple %s tags (%q and %q)", TagNameKubernetesClusterLegacy, legacyClusterID, id)
+			}
+			legacyClusterID = id
+		}
+	}
+
+	return legacyClusterID, newClusterID, nil
+}
+
+func (t *awsTagging) clusterTagKey() string {
+	return TagNameKubernetesClusterPrefix + t.ClusterID
+}
+
+func (t *awsTagging) hasClusterTag(tags []*ec2.Tag) bool {
+	// if the clusterID is not configured -- we consider all instances.
+	if len(t.ClusterID) == 0 {
+		return true
+	}
+	clusterTagKey := t.clusterTagKey()
+	for _, tag := range tags {
+		tagKey := aws.StringValue(tag.Key)
+		// For 1.6, we continue to recognize the legacy tags, for the 1.5 -> 1.6 upgrade
+		if tagKey == TagNameKubernetesClusterLegacy {
+			return aws.StringValue(tag.Value) == t.ClusterID
+		}
+
+		if tagKey == clusterTagKey {
+			return true
+		}
+	}
+	return false
+}
+
+// Ensure that a resource has the correct tags
+// If it has no tags, we assume that this was a problem caused by an error in between creation and tagging,
+// and we add the tags.  If it has a different cluster's tags, that is an error.
+func (c *awsTagging) readRepairClusterTags(client EC2, resourceID string, lifecycle ResourceLifecycle, additionalTags map[string]string, observedTags []*ec2.Tag) error {
+	actualTagMap := make(map[string]string)
+	for _, tag := range observedTags {
+		actualTagMap[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value)
+	}
+
+	expectedTags := c.buildTags(lifecycle, additionalTags)
+
+	addTags := make(map[string]string)
+	for k, expected := range expectedTags {
+		actual := actualTagMap[k]
+		if actual == expected {
+			continue
+		}
+		if actual == "" {
+			glog.Warningf("Resource %q was missing expected cluster tag %q.  Will add (with value %q)", resourceID, k, expected)
+			addTags[k] = expected
+		} else {
+			return fmt.Errorf("resource %q has tag belonging to another cluster: %q=%q (expected %q)", resourceID, k, actual, expected)
+		}
+	}
+
+	if len(addTags) == 0 {
+		return nil
+	}
+
+	if err := c.createTags(client, resourceID, lifecycle, addTags); err != nil {
+		return fmt.Errorf("error adding missing tags to resource %q: %q", resourceID, err)
+	}
+
+	return nil
+}
+
+// createTags calls EC2 CreateTags, but adds retry-on-failure logic
+// We retry mainly because if we create an object, we cannot tag it until it is "fully created" (eventual consistency)
+// The error code varies though (depending on what we are tagging), so we simply retry on all errors
+func (t *awsTagging) createTags(client EC2, resourceID string, lifecycle ResourceLifecycle, additionalTags map[string]string) error {
+	tags := t.buildTags(lifecycle, additionalTags)
+
+	if tags == nil || len(tags) == 0 {
+		return nil
+	}
+
+	var awsTags []*ec2.Tag
+	for k, v := range tags {
+		tag := &ec2.Tag{
+			Key:   aws.String(k),
+			Value: aws.String(v),
+		}
+		awsTags = append(awsTags, tag)
+	}
+
+	backoff := wait.Backoff{
+		Duration: createTagInitialDelay,
+		Factor:   createTagFactor,
+		Steps:    createTagSteps,
+	}
+	request := &ec2.CreateTagsInput{}
+	request.Resources = []*string{&resourceID}
+	request.Tags = awsTags
+
+	var lastErr error
+	err := wait.ExponentialBackoff(backoff, func() (bool, error) {
+		_, err := client.CreateTags(request)
+		if err == nil {
+			return true, nil
+		}
+
+		// We could check that the error is retryable, but the error code changes based on what we are tagging
+		// SecurityGroup: InvalidGroup.NotFound
+		glog.V(2).Infof("Failed to create tags; will retry.  Error was %q", err)
+		lastErr = err
+		return false, nil
+	})
+	if err == wait.ErrWaitTimeout {
+		// return real CreateTags error instead of timeout
+		err = lastErr
+	}
+	return err
+}
+
+// Add additional filters, to match on our tags
+// This lets us run multiple k8s clusters in a single EC2 AZ
+func (t *awsTagging) addFilters(filters []*ec2.Filter) []*ec2.Filter {
+	// if there are no clusterID configured - no filtering by special tag names
+	// should be applied to revert to legacy behaviour.
+	if len(t.ClusterID) == 0 {
+		if len(filters) == 0 {
+			// We can't pass a zero-length Filters to AWS (it's an error)
+			// So if we end up with no filters; just return nil
+			return nil
+		}
+		return filters
+	}
+	// For 1.6, we always recognize the legacy tag, for the 1.5 -> 1.6 upgrade
+	// There are no "or" filters by key, so we look for both the legacy and new key, and then we have to post-filter
+	f := newEc2Filter("tag-key", TagNameKubernetesClusterLegacy, t.clusterTagKey())
+
+	// We can't pass a zero-length Filters to AWS (it's an error)
+	// So if we end up with no filters; we need to return nil
+	filters = append(filters, f)
+	return filters
+}
+
+func (t *awsTagging) buildTags(lifecycle ResourceLifecycle, additionalTags map[string]string) map[string]string {
+	tags := make(map[string]string)
+	for k, v := range additionalTags {
+		tags[k] = v
+	}
+
+	// no clusterID is a sign of misconfigured cluster, but we can't be tagging the resources with empty
+	// strings
+	if len(t.ClusterID) == 0 {
+		return tags
+	}
+
+	// We only create legacy tags if we are using legacy tags, i.e. if we have seen a legacy tag on our instance
+	if t.usesLegacyTags {
+		tags[TagNameKubernetesClusterLegacy] = t.ClusterID
+	}
+	tags[t.clusterTagKey()] = string(lifecycle)
+
+	return tags
+}
+
+func (t *awsTagging) clusterID() string {
+	return t.ClusterID
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/tags_test.go

@@ -0,0 +1,111 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"strings"
+	"testing"
+)
+
+func TestFilterTags(t *testing.T) {
+	awsServices := NewFakeAWSServices()
+	c, err := newAWSCloud(strings.NewReader("[global]"), awsServices)
+	if err != nil {
+		t.Errorf("Error building aws cloud: %v", err)
+		return
+	}
+
+	if c.tagging.ClusterID != TestClusterId {
+		t.Errorf("unexpected ClusterID: %v", c.tagging.ClusterID)
+	}
+}
+
+func TestFindClusterID(t *testing.T) {
+	grid := []struct {
+		Tags           map[string]string
+		ExpectedNew    string
+		ExpectedLegacy string
+		ExpectError    bool
+	}{
+		{
+			Tags: map[string]string{},
+		},
+		{
+			Tags: map[string]string{
+				TagNameKubernetesClusterLegacy: "a",
+			},
+			ExpectedLegacy: "a",
+		},
+		{
+			Tags: map[string]string{
+				TagNameKubernetesClusterPrefix + "a": "owned",
+			},
+			ExpectedNew: "a",
+		},
+		{
+			Tags: map[string]string{
+				TagNameKubernetesClusterPrefix + "a": "",
+			},
+			ExpectedNew: "a",
+		},
+		{
+			Tags: map[string]string{
+				TagNameKubernetesClusterLegacy:       "a",
+				TagNameKubernetesClusterPrefix + "a": "",
+			},
+			ExpectedLegacy: "a",
+			ExpectedNew:    "a",
+		},
+		{
+			Tags: map[string]string{
+				TagNameKubernetesClusterPrefix + "a": "",
+				TagNameKubernetesClusterPrefix + "b": "",
+			},
+			ExpectError: true,
+		},
+	}
+	for _, g := range grid {
+		var ec2Tags []*ec2.Tag
+		for k, v := range g.Tags {
+			ec2Tags = append(ec2Tags, &ec2.Tag{Key: aws.String(k), Value: aws.String(v)})
+		}
+		actualLegacy, actualNew, err := findClusterIDs(ec2Tags)
+		if g.ExpectError {
+			if err == nil {
+				t.Errorf("expected error for tags %v", g.Tags)
+				continue
+			}
+		} else {
+			if err != nil {
+				t.Errorf("unexpected error for tags %v: %v", g.Tags, err)
+				continue
+			}
+
+			if g.ExpectedNew != actualNew {
+				t.Errorf("unexpected new clusterid for tags %v: %s vs %s", g.Tags, g.ExpectedNew, actualNew)
+				continue
+			}
+
+			if g.ExpectedLegacy != actualLegacy {
+				t.Errorf("unexpected new clusterid for tags %v: %s vs %s", g.Tags, g.ExpectedLegacy, actualLegacy)
+				continue
+			}
+		}
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/aws/volumes.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+)
+
+// awsVolumeID represents the ID of the volume in the AWS API, e.g. vol-12345678
+// The "traditional" format is "vol-12345678"
+// A new longer format is also being introduced: "vol-12345678abcdef01"
+// We should not assume anything about the length or format, though it seems
+// reasonable to assume that volumes will continue to start with "vol-".
+type awsVolumeID string
+
+func (i awsVolumeID) awsString() *string {
+	return aws.String(string(i))
+}
+
+// KubernetesVolumeID represents the id for a volume in the kubernetes API;
+// a few forms are recognized:
+//  * aws://<zone>/<awsVolumeId>
+//  * aws:///<awsVolumeId>
+//  * <awsVolumeId>
+type KubernetesVolumeID string
+
+// mapToAWSVolumeID extracts the awsVolumeID from the KubernetesVolumeID
+func (name KubernetesVolumeID) mapToAWSVolumeID() (awsVolumeID, error) {
+	// name looks like aws://availability-zone/awsVolumeId
+
+	// The original idea of the URL-style name was to put the AZ into the
+	// host, so we could find the AZ immediately from the name without
+	// querying the API.  But it turns out we don't actually need it for
+	// multi-AZ clusters, as we put the AZ into the labels on the PV instead.
+	// However, if in future we want to support multi-AZ cluster
+	// volume-awareness without using PersistentVolumes, we likely will
+	// want the AZ in the host.
+
+	s := string(name)
+
+	if !strings.HasPrefix(s, "aws://") {
+		// Assume a bare aws volume id (vol-1234...)
+		// Build a URL with an empty host (AZ)
+		s = "aws://" + "" + "/" + s
+	}
+	url, err := url.Parse(s)
+	if err != nil {
+		// TODO: Maybe we should pass a URL into the Volume functions
+		return "", fmt.Errorf("Invalid disk name (%s): %v", name, err)
+	}
+	if url.Scheme != "aws" {
+		return "", fmt.Errorf("Invalid scheme for AWS volume (%s)", name)
+	}
+
+	awsID := url.Path
+	awsID = strings.Trim(awsID, "/")
+
+	// We sanity check the resulting volume; the two known formats are
+	// vol-12345678 and vol-12345678abcdef01
+	// TODO: Regex match?
+	if strings.Contains(awsID, "/") || !strings.HasPrefix(awsID, "vol-") {
+		return "", fmt.Errorf("Invalid format for AWS volume (%s)", name)
+	}
+
+	return awsVolumeID(awsID), nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/BUILD

@@ -0,0 +1,79 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "azure.go",
+        "azure_backoff.go",
+        "azure_blobDiskController.go",
+        "azure_controllerCommon.go",
+        "azure_file.go",
+        "azure_instance_metadata.go",
+        "azure_instances.go",
+        "azure_loadbalancer.go",
+        "azure_managedDiskController.go",
+        "azure_routes.go",
+        "azure_storage.go",
+        "azure_storageaccount.go",
+        "azure_util.go",
+        "azure_wrap.go",
+        "azure_zones.go",
+    ],
+    deps = [
+        "//pkg/api/v1/service:go_default_library",
+        "//pkg/cloudprovider:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//pkg/version:go_default_library",
+        "//pkg/volume:go_default_library",
+        "//vendor/github.com/Azure/azure-sdk-for-go/arm/compute:go_default_library",
+        "//vendor/github.com/Azure/azure-sdk-for-go/arm/disk:go_default_library",
+        "//vendor/github.com/Azure/azure-sdk-for-go/arm/network:go_default_library",
+        "//vendor/github.com/Azure/azure-sdk-for-go/arm/storage:go_default_library",
+        "//vendor/github.com/Azure/azure-sdk-for-go/storage:go_default_library",
+        "//vendor/github.com/Azure/go-autorest/autorest:go_default_library",
+        "//vendor/github.com/Azure/go-autorest/autorest/adal:go_default_library",
+        "//vendor/github.com/Azure/go-autorest/autorest/azure:go_default_library",
+        "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library",
+        "//vendor/github.com/ghodss/yaml:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/rubiojr/go-vhd/vhd:go_default_library",
+        "//vendor/golang.org/x/crypto/pkcs12:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["azure_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//pkg/api/v1/service:go_default_library",
+        "//vendor/github.com/Azure/azure-sdk-for-go/arm/network:go_default_library",
+        "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/OWNERS

@@ -0,0 +1,4 @@
+approvers:
+- brendandburns
+- colemickens
+- jdumars

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/azure.go

@@ -0,0 +1,450 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"crypto/rsa"
+	"crypto/x509"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"time"
+
+	"k8s.io/client-go/util/flowcontrol"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+	"k8s.io/kubernetes/pkg/controller"
+	"k8s.io/kubernetes/pkg/version"
+
+	"github.com/Azure/azure-sdk-for-go/arm/compute"
+	"github.com/Azure/azure-sdk-for-go/arm/disk"
+	"github.com/Azure/azure-sdk-for-go/arm/network"
+	"github.com/Azure/azure-sdk-for-go/arm/storage"
+	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/adal"
+	"github.com/Azure/go-autorest/autorest/azure"
+	"github.com/ghodss/yaml"
+	"github.com/golang/glog"
+	"golang.org/x/crypto/pkcs12"
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+const (
+	// CloudProviderName is the value used for the --cloud-provider flag
+	CloudProviderName      = "azure"
+	rateLimitQPSDefault    = 1.0
+	rateLimitBucketDefault = 5
+	backoffRetriesDefault  = 6
+	backoffExponentDefault = 1.5
+	backoffDurationDefault = 5 // in seconds
+	backoffJitterDefault   = 1.0
+)
+
+// Config holds the configuration parsed from the --cloud-config flag
+// All fields are required unless otherwise specified
+type Config struct {
+	// The cloud environment identifier. Takes values from https://github.com/Azure/go-autorest/blob/ec5f4903f77ed9927ac95b19ab8e44ada64c1356/autorest/azure/environments.go#L13
+	Cloud string `json:"cloud" yaml:"cloud"`
+	// The AAD Tenant ID for the Subscription that the cluster is deployed in
+	TenantID string `json:"tenantId" yaml:"tenantId"`
+	// The ID of the Azure Subscription that the cluster is deployed in
+	SubscriptionID string `json:"subscriptionId" yaml:"subscriptionId"`
+	// The name of the resource group that the cluster is deployed in
+	ResourceGroup string `json:"resourceGroup" yaml:"resourceGroup"`
+	// The location of the resource group that the cluster is deployed in
+	Location string `json:"location" yaml:"location"`
+	// The name of the VNet that the cluster is deployed in
+	VnetName string `json:"vnetName" yaml:"vnetName"`
+	// The name of the resource group that the Vnet is deployed in
+	VnetResourceGroup string `json:"vnetResourceGroup" yaml:"vnetResourceGroup"`
+	// The name of the subnet that the cluster is deployed in
+	SubnetName string `json:"subnetName" yaml:"subnetName"`
+	// The name of the security group attached to the cluster's subnet
+	SecurityGroupName string `json:"securityGroupName" yaml:"securityGroupName"`
+	// (Optional in 1.6) The name of the route table attached to the subnet that the cluster is deployed in
+	RouteTableName string `json:"routeTableName" yaml:"routeTableName"`
+	// (Optional) The name of the availability set that should be used as the load balancer backend
+	// If this is set, the Azure cloudprovider will only add nodes from that availability set to the load
+	// balancer backend pool. If this is not set, and multiple agent pools (availability sets) are used, then
+	// the cloudprovider will try to add all nodes to a single backend pool which is forbidden.
+	// In other words, if you use multiple agent pools (availability sets), you MUST set this field.
+	PrimaryAvailabilitySetName string `json:"primaryAvailabilitySetName" yaml:"primaryAvailabilitySetName"`
+
+	// The ClientID for an AAD application with RBAC access to talk to Azure RM APIs
+	AADClientID string `json:"aadClientId" yaml:"aadClientId"`
+	// The ClientSecret for an AAD application with RBAC access to talk to Azure RM APIs
+	AADClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"`
+	// The path of a client certificate for an AAD application with RBAC access to talk to Azure RM APIs
+	AADClientCertPath string `json:"aadClientCertPath" yaml:"aadClientCertPath"`
+	// The password of the client certificate for an AAD application with RBAC access to talk to Azure RM APIs
+	AADClientCertPassword string `json:"aadClientCertPassword" yaml:"aadClientCertPassword"`
+	// Enable exponential backoff to manage resource request retries
+	CloudProviderBackoff bool `json:"cloudProviderBackoff" yaml:"cloudProviderBackoff"`
+	// Backoff retry limit
+	CloudProviderBackoffRetries int `json:"cloudProviderBackoffRetries" yaml:"cloudProviderBackoffRetries"`
+	// Backoff exponent
+	CloudProviderBackoffExponent float64 `json:"cloudProviderBackoffExponent" yaml:"cloudProviderBackoffExponent"`
+	// Backoff duration
+	CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration" yaml:"cloudProviderBackoffDuration"`
+	// Backoff jitter
+	CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter" yaml:"cloudProviderBackoffJitter"`
+	// Enable rate limiting
+	CloudProviderRateLimit bool `json:"cloudProviderRateLimit" yaml:"cloudProviderRateLimit"`
+	// Rate limit QPS
+	CloudProviderRateLimitQPS float32 `json:"cloudProviderRateLimitQPS" yaml:"cloudProviderRateLimitQPS"`
+	// Rate limit Bucket Size
+	CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket" yaml:"cloudProviderRateLimitBucket"`
+
+	// Use instance metadata service where possible
+	UseInstanceMetadata bool `json:"useInstanceMetadata" yaml:"useInstanceMetadata"`
+
+	// Use managed service identity for the virtual machine to access Azure ARM APIs
+	UseManagedIdentityExtension bool `json:"useManagedIdentityExtension"`
+}
+
+// Cloud holds the config and clients
+type Cloud struct {
+	Config
+	Environment              azure.Environment
+	RoutesClient             network.RoutesClient
+	SubnetsClient            network.SubnetsClient
+	InterfacesClient         network.InterfacesClient
+	RouteTablesClient        network.RouteTablesClient
+	LoadBalancerClient       network.LoadBalancersClient
+	PublicIPAddressesClient  network.PublicIPAddressesClient
+	SecurityGroupsClient     network.SecurityGroupsClient
+	VirtualMachinesClient    compute.VirtualMachinesClient
+	StorageAccountClient     storage.AccountsClient
+	DisksClient              disk.DisksClient
+	operationPollRateLimiter flowcontrol.RateLimiter
+	resourceRequestBackoff   wait.Backoff
+	metadata                 *InstanceMetadata
+
+	*BlobDiskController
+	*ManagedDiskController
+	*controllerCommon
+}
+
+func init() {
+	cloudprovider.RegisterCloudProvider(CloudProviderName, NewCloud)
+}
+
+// decodePkcs12 decodes a PKCS#12 client certificate by extracting the public certificate and
+// the private RSA key
+func decodePkcs12(pkcs []byte, password string) (*x509.Certificate, *rsa.PrivateKey, error) {
+	privateKey, certificate, err := pkcs12.Decode(pkcs, password)
+	if err != nil {
+		return nil, nil, fmt.Errorf("decoding the PKCS#12 client certificate: %v", err)
+	}
+	rsaPrivateKey, isRsaKey := privateKey.(*rsa.PrivateKey)
+	if !isRsaKey {
+		return nil, nil, fmt.Errorf("PKCS#12 certificate must contain a RSA private key")
+	}
+
+	return certificate, rsaPrivateKey, nil
+}
+
+// GetServicePrincipalToken creates a new service principal token based on the configuration
+func GetServicePrincipalToken(config *Config, env *azure.Environment) (*adal.ServicePrincipalToken, error) {
+	oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID)
+	if err != nil {
+		return nil, fmt.Errorf("creating the OAuth config: %v", err)
+	}
+
+	if config.UseManagedIdentityExtension {
+		glog.V(2).Infoln("azure: using managed identity extension to retrieve access token")
+		return adal.NewServicePrincipalTokenFromMSI(
+			*oauthConfig,
+			env.ServiceManagementEndpoint)
+	}
+
+	if len(config.AADClientSecret) > 0 {
+		glog.V(2).Infoln("azure: using client_id+client_secret to retrieve access token")
+		return adal.NewServicePrincipalToken(
+			*oauthConfig,
+			config.AADClientID,
+			config.AADClientSecret,
+			env.ServiceManagementEndpoint)
+	}
+
+	if len(config.AADClientCertPath) > 0 && len(config.AADClientCertPassword) > 0 {
+		glog.V(2).Infoln("azure: using jwt client_assertion (client_cert+client_private_key) to retrieve access token")
+		certData, err := ioutil.ReadFile(config.AADClientCertPath)
+		if err != nil {
+			return nil, fmt.Errorf("reading the client certificate from file %s: %v", config.AADClientCertPath, err)
+		}
+		certificate, privateKey, err := decodePkcs12(certData, config.AADClientCertPassword)
+		if err != nil {
+			return nil, fmt.Errorf("decoding the client certificate: %v", err)
+		}
+		return adal.NewServicePrincipalTokenFromCertificate(
+			*oauthConfig,
+			config.AADClientID,
+			certificate,
+			privateKey,
+			env.ServiceManagementEndpoint)
+	}
+
+	return nil, fmt.Errorf("No credentials provided for AAD application %s", config.AADClientID)
+}
+
+// NewCloud returns a Cloud with initialized clients
+func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) {
+	config, env, err := ParseConfig(configReader)
+	if err != nil {
+		return nil, err
+	}
+	az := Cloud{
+		Config:      *config,
+		Environment: *env,
+	}
+
+	servicePrincipalToken, err := GetServicePrincipalToken(config, env)
+	if err != nil {
+		return nil, err
+	}
+
+	az.SubnetsClient = network.NewSubnetsClient(az.SubscriptionID)
+	az.SubnetsClient.BaseURI = az.Environment.ResourceManagerEndpoint
+	az.SubnetsClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
+	az.SubnetsClient.PollingDelay = 5 * time.Second
+	configureUserAgent(&az.SubnetsClient.Client)
+
+	az.RouteTablesClient = network.NewRouteTablesClient(az.SubscriptionID)
+	az.RouteTablesClient.BaseURI = az.Environment.ResourceManagerEndpoint
+	az.RouteTablesClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
+	az.RouteTablesClient.PollingDelay = 5 * time.Second
+	configureUserAgent(&az.RouteTablesClient.Client)
+
+	az.RoutesClient = network.NewRoutesClient(az.SubscriptionID)
+	az.RoutesClient.BaseURI = az.Environment.ResourceManagerEndpoint
+	az.RoutesClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
+	az.RoutesClient.PollingDelay = 5 * time.Second
+	configureUserAgent(&az.RoutesClient.Client)
+
+	az.InterfacesClient = network.NewInterfacesClient(az.SubscriptionID)
+	az.InterfacesClient.BaseURI = az.Environment.ResourceManagerEndpoint
+	az.InterfacesClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
+	az.InterfacesClient.PollingDelay = 5 * time.Second
+	configureUserAgent(&az.InterfacesClient.Client)
+
+	az.LoadBalancerClient = network.NewLoadBalancersClient(az.SubscriptionID)
+	az.LoadBalancerClient.BaseURI = az.Environment.ResourceManagerEndpoint
+	az.LoadBalancerClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
+	az.LoadBalancerClient.PollingDelay = 5 * time.Second
+	configureUserAgent(&az.LoadBalancerClient.Client)
+
+	az.VirtualMachinesClient = compute.NewVirtualMachinesClient(az.SubscriptionID)
+	az.VirtualMachinesClient.BaseURI = az.Environment.ResourceManagerEndpoint
+	az.VirtualMachinesClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
+	az.VirtualMachinesClient.PollingDelay = 5 * time.Second
+	configureUserAgent(&az.VirtualMachinesClient.Client)
+
+	az.PublicIPAddressesClient = network.NewPublicIPAddressesClient(az.SubscriptionID)
+	az.PublicIPAddressesClient.BaseURI = az.Environment.ResourceManagerEndpoint
+	az.PublicIPAddressesClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
+	az.PublicIPAddressesClient.PollingDelay = 5 * time.Second
+	configureUserAgent(&az.PublicIPAddressesClient.Client)
+
+	az.SecurityGroupsClient = network.NewSecurityGroupsClient(az.SubscriptionID)
+	az.SecurityGroupsClient.BaseURI = az.Environment.ResourceManagerEndpoint
+	az.SecurityGroupsClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
+	az.SecurityGroupsClient.PollingDelay = 5 * time.Second
+	configureUserAgent(&az.SecurityGroupsClient.Client)
+
+	az.StorageAccountClient = storage.NewAccountsClientWithBaseURI(az.Environment.ResourceManagerEndpoint, az.SubscriptionID)
+	az.StorageAccountClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
+	configureUserAgent(&az.StorageAccountClient.Client)
+
+	az.DisksClient = disk.NewDisksClientWithBaseURI(az.Environment.ResourceManagerEndpoint, az.SubscriptionID)
+	az.DisksClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
+	configureUserAgent(&az.DisksClient.Client)
+
+	// Conditionally configure rate limits
+	if az.CloudProviderRateLimit {
+		// Assign rate limit defaults if no configuration was passed in
+		if az.CloudProviderRateLimitQPS == 0 {
+			az.CloudProviderRateLimitQPS = rateLimitQPSDefault
+		}
+		if az.CloudProviderRateLimitBucket == 0 {
+			az.CloudProviderRateLimitBucket = rateLimitBucketDefault
+		}
+		az.operationPollRateLimiter = flowcontrol.NewTokenBucketRateLimiter(
+			az.CloudProviderRateLimitQPS,
+			az.CloudProviderRateLimitBucket)
+		glog.V(2).Infof("Azure cloudprovider using rate limit config: QPS=%d, bucket=%d",
+			az.CloudProviderRateLimitQPS,
+			az.CloudProviderRateLimitBucket)
+	} else {
+		// if rate limits are configured off, az.operationPollRateLimiter.Accept() is a no-op
+		az.operationPollRateLimiter = flowcontrol.NewFakeAlwaysRateLimiter()
+	}
+
+	// Conditionally configure resource request backoff
+	if az.CloudProviderBackoff {
+		// Assign backoff defaults if no configuration was passed in
+		if az.CloudProviderBackoffRetries == 0 {
+			az.CloudProviderBackoffRetries = backoffRetriesDefault
+		}
+		if az.CloudProviderBackoffExponent == 0 {
+			az.CloudProviderBackoffExponent = backoffExponentDefault
+		}
+		if az.CloudProviderBackoffDuration == 0 {
+			az.CloudProviderBackoffDuration = backoffDurationDefault
+		}
+		if az.CloudProviderBackoffJitter == 0 {
+			az.CloudProviderBackoffJitter = backoffJitterDefault
+		}
+		az.resourceRequestBackoff = wait.Backoff{
+			Steps:    az.CloudProviderBackoffRetries,
+			Factor:   az.CloudProviderBackoffExponent,
+			Duration: time.Duration(az.CloudProviderBackoffDuration) * time.Second,
+			Jitter:   az.CloudProviderBackoffJitter,
+		}
+		glog.V(2).Infof("Azure cloudprovider using retry backoff: retries=%d, exponent=%f, duration=%d, jitter=%f",
+			az.CloudProviderBackoffRetries,
+			az.CloudProviderBackoffExponent,
+			az.CloudProviderBackoffDuration,
+			az.CloudProviderBackoffJitter)
+	}
+
+	az.metadata = NewInstanceMetadata()
+
+	if err := initDiskControllers(&az); err != nil {
+		return nil, err
+	}
+	return &az, nil
+}
+
+// ParseConfig returns a parsed configuration and azure.Environment for an Azure cloudprovider config file
+func ParseConfig(configReader io.Reader) (*Config, *azure.Environment, error) {
+	var config Config
+	var env azure.Environment
+
+	if configReader == nil {
+		return &config, &env, nil
+	}
+
+	configContents, err := ioutil.ReadAll(configReader)
+	if err != nil {
+		return nil, nil, err
+	}
+	err = yaml.Unmarshal(configContents, &config)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if config.Cloud == "" {
+		env = azure.PublicCloud
+	} else {
+		env, err = azure.EnvironmentFromName(config.Cloud)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+	return &config, &env, nil
+}
+
+// Initialize passes a Kubernetes clientBuilder interface to the cloud provider
+func (az *Cloud) Initialize(clientBuilder controller.ControllerClientBuilder) {}
+
+// LoadBalancer returns a balancer interface. Also returns true if the interface is supported, false otherwise.
+func (az *Cloud) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
+	return az, true
+}
+
+// Instances returns an instances interface. Also returns true if the interface is supported, false otherwise.
+func (az *Cloud) Instances() (cloudprovider.Instances, bool) {
+	return az, true
+}
+
+// Zones returns a zones interface. Also returns true if the interface is supported, false otherwise.
+func (az *Cloud) Zones() (cloudprovider.Zones, bool) {
+	return az, true
+}
+
+// Clusters returns a clusters interface.  Also returns true if the interface is supported, false otherwise.
+func (az *Cloud) Clusters() (cloudprovider.Clusters, bool) {
+	return nil, false
+}
+
+// Routes returns a routes interface along with whether the interface is supported.
+func (az *Cloud) Routes() (cloudprovider.Routes, bool) {
+	return az, true
+}
+
+// ScrubDNS provides an opportunity for cloud-provider-specific code to process DNS settings for pods.
+func (az *Cloud) ScrubDNS(nameservers, searches []string) (nsOut, srchOut []string) {
+	return nameservers, searches
+}
+
+// HasClusterID returns true if the cluster has a clusterID
+func (az *Cloud) HasClusterID() bool {
+	return true
+}
+
+// ProviderName returns the cloud provider ID.
+func (az *Cloud) ProviderName() string {
+	return CloudProviderName
+}
+
+// configureUserAgent configures the autorest client with a user agent that
+// includes "kubernetes" and the full kubernetes git version string
+// example:
+// Azure-SDK-for-Go/7.0.1-beta arm-network/2016-09-01; kubernetes-cloudprovider/v1.7.0-alpha.2.711+a2fadef8170bb0-dirty;
+func configureUserAgent(client *autorest.Client) {
+	k8sVersion := version.Get().GitVersion
+	client.UserAgent = fmt.Sprintf("%s; kubernetes-cloudprovider/%s", client.UserAgent, k8sVersion)
+}
+
+func initDiskControllers(az *Cloud) error {
+	// Common controller contains the function
+	// needed by both blob disk and managed disk controllers
+
+	common := &controllerCommon{
+		aadResourceEndPoint:   az.Environment.ServiceManagementEndpoint,
+		clientID:              az.AADClientID,
+		clientSecret:          az.AADClientSecret,
+		location:              az.Location,
+		storageEndpointSuffix: az.Environment.StorageEndpointSuffix,
+		managementEndpoint:    az.Environment.ResourceManagerEndpoint,
+		resourceGroup:         az.ResourceGroup,
+		tenantID:              az.TenantID,
+		tokenEndPoint:         az.Environment.ActiveDirectoryEndpoint,
+		subscriptionID:        az.SubscriptionID,
+		cloud:                 az,
+	}
+
+	// BlobDiskController: contains the function needed to
+	// create/attach/detach/delete blob based (unmanaged disks)
+	blobController, err := newBlobDiskController(common)
+	if err != nil {
+		return fmt.Errorf("AzureDisk -  failed to init Blob Disk Controller with error (%s)", err.Error())
+	}
+
+	// ManagedDiskController: contains the functions needed to
+	// create/attach/detach/delete managed disks
+	managedController, err := newManagedDiskController(common)
+	if err != nil {
+		return fmt.Errorf("AzureDisk -  failed to init Managed  Disk Controller with error (%s)", err.Error())
+	}
+
+	az.BlobDiskController = blobController
+	az.ManagedDiskController = managedController
+	az.controllerCommon = common
+
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/azure_backoff.go

@@ -0,0 +1,242 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/Azure/azure-sdk-for-go/arm/compute"
+	"github.com/Azure/azure-sdk-for-go/arm/network"
+	"github.com/Azure/go-autorest/autorest"
+	"github.com/golang/glog"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// GetVirtualMachineWithRetry invokes az.getVirtualMachine with exponential backoff retry
+func (az *Cloud) GetVirtualMachineWithRetry(name types.NodeName) (compute.VirtualMachine, bool, error) {
+	var machine compute.VirtualMachine
+	var exists bool
+	err := wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {
+		var retryErr error
+		machine, exists, retryErr = az.getVirtualMachine(name)
+		if retryErr != nil {
+			glog.Errorf("backoff: failure, will retry,err=%v", retryErr)
+			return false, nil
+		}
+		glog.V(2).Infof("backoff: success")
+		return true, nil
+	})
+	return machine, exists, err
+}
+
+// VirtualMachineClientGetWithRetry invokes az.VirtualMachinesClient.Get with exponential backoff retry
+func (az *Cloud) VirtualMachineClientGetWithRetry(resourceGroup, vmName string, types compute.InstanceViewTypes) (compute.VirtualMachine, error) {
+	var machine compute.VirtualMachine
+	err := wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {
+		var retryErr error
+		machine, retryErr = az.VirtualMachinesClient.Get(resourceGroup, vmName, types)
+		if retryErr != nil {
+			glog.Errorf("backoff: failure, will retry,err=%v", retryErr)
+			return false, nil
+		}
+		glog.V(2).Infof("backoff: success")
+		return true, nil
+	})
+	return machine, err
+}
+
+// GetIPForMachineWithRetry invokes az.getIPForMachine with exponential backoff retry
+func (az *Cloud) GetIPForMachineWithRetry(name types.NodeName) (string, error) {
+	var ip string
+	err := wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {
+		var retryErr error
+		ip, retryErr = az.getIPForMachine(name)
+		if retryErr != nil {
+			glog.Errorf("backoff: failure, will retry,err=%v", retryErr)
+			return false, nil
+		}
+		glog.V(2).Infof("backoff: success")
+		return true, nil
+	})
+	return ip, err
+}
+
+// CreateOrUpdateSGWithRetry invokes az.SecurityGroupsClient.CreateOrUpdate with exponential backoff retry
+func (az *Cloud) CreateOrUpdateSGWithRetry(sg network.SecurityGroup) error {
+	return wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {
+		az.operationPollRateLimiter.Accept()
+		glog.V(10).Infof("SecurityGroupsClient.CreateOrUpdate(%s): start", *sg.Name)
+		respChan, errChan := az.SecurityGroupsClient.CreateOrUpdate(az.ResourceGroup, *sg.Name, sg, nil)
+		resp := <-respChan
+		err := <-errChan
+		glog.V(10).Infof("SecurityGroupsClient.CreateOrUpdate(%s): end", *sg.Name)
+		return processRetryResponse(resp.Response, err)
+	})
+}
+
+// CreateOrUpdateLBWithRetry invokes az.LoadBalancerClient.CreateOrUpdate with exponential backoff retry
+func (az *Cloud) CreateOrUpdateLBWithRetry(lb network.LoadBalancer) error {
+	return wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {
+		az.operationPollRateLimiter.Accept()
+		glog.V(10).Infof("LoadBalancerClient.CreateOrUpdate(%s): start", *lb.Name)
+		respChan, errChan := az.LoadBalancerClient.CreateOrUpdate(az.ResourceGroup, *lb.Name, lb, nil)
+		resp := <-respChan
+		err := <-errChan
+		glog.V(10).Infof("LoadBalancerClient.CreateOrUpdate(%s): end", *lb.Name)
+		return processRetryResponse(resp.Response, err)
+	})
+}
+
+// CreateOrUpdatePIPWithRetry invokes az.PublicIPAddressesClient.CreateOrUpdate with exponential backoff retry
+func (az *Cloud) CreateOrUpdatePIPWithRetry(pip network.PublicIPAddress) error {
+	return wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {
+		az.operationPollRateLimiter.Accept()
+		glog.V(10).Infof("PublicIPAddressesClient.CreateOrUpdate(%s): start", *pip.Name)
+		respChan, errChan := az.PublicIPAddressesClient.CreateOrUpdate(az.ResourceGroup, *pip.Name, pip, nil)
+		resp := <-respChan
+		err := <-errChan
+		glog.V(10).Infof("PublicIPAddressesClient.CreateOrUpdate(%s): end", *pip.Name)
+		return processRetryResponse(resp.Response, err)
+	})
+}
+
+// CreateOrUpdateInterfaceWithRetry invokes az.PublicIPAddressesClient.CreateOrUpdate with exponential backoff retry
+func (az *Cloud) CreateOrUpdateInterfaceWithRetry(nic network.Interface) error {
+	return wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {
+		az.operationPollRateLimiter.Accept()
+		glog.V(10).Infof("InterfacesClient.CreateOrUpdate(%s): start", *nic.Name)
+		respChan, errChan := az.InterfacesClient.CreateOrUpdate(az.ResourceGroup, *nic.Name, nic, nil)
+		resp := <-respChan
+		err := <-errChan
+		glog.V(10).Infof("InterfacesClient.CreateOrUpdate(%s): end", *nic.Name)
+		return processRetryResponse(resp.Response, err)
+	})
+}
+
+// DeletePublicIPWithRetry invokes az.PublicIPAddressesClient.Delete with exponential backoff retry
+func (az *Cloud) DeletePublicIPWithRetry(pipName string) error {
+	return wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {
+		az.operationPollRateLimiter.Accept()
+		glog.V(10).Infof("PublicIPAddressesClient.Delete(%s): start", pipName)
+		respChan, errChan := az.PublicIPAddressesClient.Delete(az.ResourceGroup, pipName, nil)
+		resp := <-respChan
+		err := <-errChan
+		glog.V(10).Infof("PublicIPAddressesClient.Delete(%s): end", pipName)
+		return processRetryResponse(resp, err)
+	})
+}
+
+// DeleteLBWithRetry invokes az.LoadBalancerClient.Delete with exponential backoff retry
+func (az *Cloud) DeleteLBWithRetry(lbName string) error {
+	return wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {
+		az.operationPollRateLimiter.Accept()
+		glog.V(10).Infof("LoadBalancerClient.Delete(%s): start", lbName)
+		respChan, errChan := az.LoadBalancerClient.Delete(az.ResourceGroup, lbName, nil)
+		resp := <-respChan
+		err := <-errChan
+		glog.V(10).Infof("LoadBalancerClient.Delete(%s): end", lbName)
+		return processRetryResponse(resp, err)
+	})
+}
+
+// CreateOrUpdateRouteTableWithRetry invokes az.RouteTablesClient.CreateOrUpdate with exponential backoff retry
+func (az *Cloud) CreateOrUpdateRouteTableWithRetry(routeTable network.RouteTable) error {
+	return wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {
+		az.operationPollRateLimiter.Accept()
+		glog.V(10).Infof("RouteTablesClient.CreateOrUpdate(%s): start", routeTable)
+		respChan, errChan := az.RouteTablesClient.CreateOrUpdate(az.ResourceGroup, az.RouteTableName, routeTable, nil)
+		resp := <-respChan
+		err := <-errChan
+		glog.V(10).Infof("RouteTablesClient.CreateOrUpdate(%s): end", routeTable)
+		return processRetryResponse(resp.Response, err)
+	})
+}
+
+// CreateOrUpdateRouteWithRetry invokes az.RoutesClient.CreateOrUpdate with exponential backoff retry
+func (az *Cloud) CreateOrUpdateRouteWithRetry(route network.Route) error {
+	return wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {
+		az.operationPollRateLimiter.Accept()
+		glog.V(10).Infof("RoutesClient.CreateOrUpdate(%s): start", *route.Name)
+		respChan, errChan := az.RoutesClient.CreateOrUpdate(az.ResourceGroup, az.RouteTableName, *route.Name, route, nil)
+		resp := <-respChan
+		err := <-errChan
+		glog.V(10).Infof("RoutesClient.CreateOrUpdate(%s): end", *route.Name)
+		return processRetryResponse(resp.Response, err)
+	})
+}
+
+// DeleteRouteWithRetry invokes az.RoutesClient.Delete with exponential backoff retry
+func (az *Cloud) DeleteRouteWithRetry(routeName string) error {
+	return wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {
+		az.operationPollRateLimiter.Accept()
+		glog.V(10).Infof("RoutesClient.Delete(%s): start", az.RouteTableName)
+		respChan, errChan := az.RoutesClient.Delete(az.ResourceGroup, az.RouteTableName, routeName, nil)
+		resp := <-respChan
+		err := <-errChan
+		glog.V(10).Infof("RoutesClient.Delete(%s): end", az.RouteTableName)
+		return processRetryResponse(resp, err)
+	})
+}
+
+// CreateOrUpdateVMWithRetry invokes az.VirtualMachinesClient.CreateOrUpdate with exponential backoff retry
+func (az *Cloud) CreateOrUpdateVMWithRetry(vmName string, newVM compute.VirtualMachine) error {
+	return wait.ExponentialBackoff(az.resourceRequestBackoff, func() (bool, error) {
+		az.operationPollRateLimiter.Accept()
+		glog.V(10).Infof("VirtualMachinesClient.CreateOrUpdate(%s): start", vmName)
+		respChan, errChan := az.VirtualMachinesClient.CreateOrUpdate(az.ResourceGroup, vmName, newVM, nil)
+		resp := <-respChan
+		err := <-errChan
+		glog.V(10).Infof("VirtualMachinesClient.CreateOrUpdate(%s): end", vmName)
+		return processRetryResponse(resp.Response, err)
+	})
+}
+
+// A wait.ConditionFunc function to deal with common HTTP backoff response conditions
+func processRetryResponse(resp autorest.Response, err error) (bool, error) {
+	if isSuccessHTTPResponse(resp) {
+		glog.V(2).Infof("backoff: success, HTTP response=%d", resp.StatusCode)
+		return true, nil
+	}
+	if shouldRetryAPIRequest(resp, err) {
+		glog.Errorf("backoff: failure, will retry, HTTP response=%d, err=%v", resp.StatusCode, err)
+		// suppress the error object so that backoff process continues
+		return false, nil
+	}
+	// Fall-through: stop periodic backoff, return error object from most recent request
+	return true, err
+}
+
+// shouldRetryAPIRequest determines if the response from an HTTP request suggests periodic retry behavior
+func shouldRetryAPIRequest(resp autorest.Response, err error) bool {
+	if err != nil {
+		return true
+	}
+	// HTTP 4xx or 5xx suggests we should retry
+	if 399 < resp.StatusCode && resp.StatusCode < 600 {
+		return true
+	}
+	return false
+}
+
+// isSuccessHTTPResponse determines if the response from an HTTP request suggests success
+func isSuccessHTTPResponse(resp autorest.Response) bool {
+	// HTTP 2xx suggests a successful response
+	if 199 < resp.StatusCode && resp.StatusCode < 300 {
+		return true
+	}
+	return false
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/azure_blobDiskController.go

@@ -0,0 +1,808 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"math"
+	"net/url"
+	"os"
+	"regexp"
+	"sync"
+
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	storage "github.com/Azure/azure-sdk-for-go/arm/storage"
+	azstorage "github.com/Azure/azure-sdk-for-go/storage"
+	"github.com/Azure/go-autorest/autorest/to"
+	"github.com/golang/glog"
+	"github.com/rubiojr/go-vhd/vhd"
+	kwait "k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/kubernetes/pkg/volume"
+)
+
+const (
+	vhdContainerName         = "vhds"
+	useHTTPSForBlobBasedDisk = true
+	blobServiceName          = "blob"
+)
+
+type storageAccountState struct {
+	name                    string
+	saType                  storage.SkuName
+	key                     string
+	diskCount               int32
+	isValidating            int32
+	defaultContainerCreated bool
+}
+
+//BlobDiskController : blob disk controller struct
+type BlobDiskController struct {
+	common   *controllerCommon
+	accounts map[string]*storageAccountState
+}
+
+var defaultContainerName = ""
+var storageAccountNamePrefix = ""
+var storageAccountNameMatch = ""
+var initFlag int64
+
+var accountsLock = &sync.Mutex{}
+
+func newBlobDiskController(common *controllerCommon) (*BlobDiskController, error) {
+	c := BlobDiskController{common: common}
+	err := c.init()
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &c, nil
+}
+
+// CreateVolume creates a VHD blob in a given storage account, will create the given storage account if it does not exist in current resource group
+func (c *BlobDiskController) CreateVolume(name, storageAccount string, storageAccountType storage.SkuName, location string, requestGB int) (string, string, int, error) {
+	key, err := c.common.cloud.getStorageAccesskey(storageAccount)
+	if err != nil {
+		glog.V(2).Infof("azureDisk - no key found for storage account %s in resource group %s, begin to create a new storage account", storageAccount, c.common.resourceGroup)
+
+		cp := storage.AccountCreateParameters{
+			Sku:      &storage.Sku{Name: storageAccountType},
+			Tags:     &map[string]*string{"created-by": to.StringPtr("azure-dd")},
+			Location: &location}
+		cancel := make(chan struct{})
+
+		_, errchan := c.common.cloud.StorageAccountClient.Create(c.common.resourceGroup, storageAccount, cp, cancel)
+		err = <-errchan
+		if err != nil {
+			return "", "", 0, fmt.Errorf(fmt.Sprintf("Create Storage Account %s, error: %s", storageAccount, err))
+		}
+
+		key, err = c.common.cloud.getStorageAccesskey(storageAccount)
+		if err != nil {
+			return "", "", 0, fmt.Errorf("no key found for storage account %s even after creating a new storage account", storageAccount)
+		}
+
+		glog.Errorf("no key found for storage account %s in resource group %s", storageAccount, c.common.resourceGroup)
+		return "", "", 0, err
+	}
+
+	client, err := azstorage.NewBasicClient(storageAccount, key)
+	if err != nil {
+		return "", "", 0, err
+	}
+	blobClient := client.GetBlobService()
+
+	container := blobClient.GetContainerReference(vhdContainerName)
+	_, err = container.CreateIfNotExists(&azstorage.CreateContainerOptions{Access: azstorage.ContainerAccessTypePrivate})
+	if err != nil {
+		return "", "", 0, err
+	}
+
+	diskName, diskURI, err := c.createVHDBlobDisk(blobClient, storageAccount, name, vhdContainerName, int64(requestGB))
+	if err != nil {
+		return "", "", 0, err
+	}
+
+	glog.V(4).Infof("azureDisk - created vhd blob uri: %s", diskURI)
+	return diskName, diskURI, requestGB, err
+}
+
+// DeleteVolume deletes a VHD blob
+func (c *BlobDiskController) DeleteVolume(diskURI string) error {
+	glog.V(4).Infof("azureDisk - begin to delete volume %s", diskURI)
+	accountName, blob, err := c.common.cloud.getBlobNameAndAccountFromURI(diskURI)
+	if err != nil {
+		return fmt.Errorf("failed to parse vhd URI %v", err)
+	}
+	key, err := c.common.cloud.getStorageAccesskey(accountName)
+	if err != nil {
+		return fmt.Errorf("no key for storage account %s, err %v", accountName, err)
+	}
+	err = c.common.cloud.deleteVhdBlob(accountName, key, blob)
+	if err != nil {
+		glog.Warningf("azureDisk - failed to delete blob %s err: %v", diskURI, err)
+		detail := err.Error()
+		if strings.Contains(detail, errLeaseIDMissing) {
+			// disk is still being used
+			// see https://msdn.microsoft.com/en-us/library/microsoft.windowsazure.storage.blob.protocol.bloberrorcodestrings.leaseidmissing.aspx
+			return volume.NewDeletedVolumeInUseError(fmt.Sprintf("disk %q is still in use while being deleted", diskURI))
+		}
+		return fmt.Errorf("failed to delete vhd %v, account %s, blob %s, err: %v", diskURI, accountName, blob, err)
+	}
+	glog.V(4).Infof("azureDisk - blob %s deleted", diskURI)
+	return nil
+
+}
+
+// get diskURI https://foo.blob.core.windows.net/vhds/bar.vhd and return foo (account) and bar.vhd (blob name)
+func (c *BlobDiskController) getBlobNameAndAccountFromURI(diskURI string) (string, string, error) {
+	scheme := "http"
+	if useHTTPSForBlobBasedDisk {
+		scheme = "https"
+	}
+	host := fmt.Sprintf("%s://(.*).%s.%s", scheme, blobServiceName, c.common.storageEndpointSuffix)
+	reStr := fmt.Sprintf("%s/%s/(.*)", host, vhdContainerName)
+	re := regexp.MustCompile(reStr)
+	res := re.FindSubmatch([]byte(diskURI))
+	if len(res) < 3 {
+		return "", "", fmt.Errorf("invalid vhd URI for regex %s: %s", reStr, diskURI)
+	}
+	return string(res[1]), string(res[2]), nil
+}
+
+func (c *BlobDiskController) createVHDBlobDisk(blobClient azstorage.BlobStorageClient, accountName, vhdName, containerName string, sizeGB int64) (string, string, error) {
+	container := blobClient.GetContainerReference(containerName)
+	_, err := container.CreateIfNotExists(&azstorage.CreateContainerOptions{Access: azstorage.ContainerAccessTypePrivate})
+	if err != nil {
+		return "", "", err
+	}
+
+	size := 1024 * 1024 * 1024 * sizeGB
+	vhdSize := size + vhd.VHD_HEADER_SIZE /* header size */
+	// Blob name in URL must end with '.vhd' extension.
+	vhdName = vhdName + ".vhd"
+
+	tags := make(map[string]string)
+	tags["createdby"] = "k8sAzureDataDisk"
+	glog.V(4).Infof("azureDisk - creating page blob %name in container %s account %s", vhdName, containerName, accountName)
+
+	blob := container.GetBlobReference(vhdName)
+	blob.Properties.ContentLength = vhdSize
+	blob.Metadata = tags
+	err = blob.PutPageBlob(nil)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to put page blob %s in container %s: %v", vhdName, containerName, err)
+	}
+
+	// add VHD signature to the blob
+	h, err := createVHDHeader(uint64(size))
+	if err != nil {
+		blob.DeleteIfExists(nil)
+		return "", "", fmt.Errorf("failed to create vhd header, err: %v", err)
+	}
+
+	blobRange := azstorage.BlobRange{
+		Start: uint64(size),
+		End:   uint64(vhdSize - 1),
+	}
+	if err = blob.WriteRange(blobRange, bytes.NewBuffer(h[:vhd.VHD_HEADER_SIZE]), nil); err != nil {
+		glog.Infof("azureDisk - failed to put header page for data disk %s in container %s account %s, error was %s\n",
+			vhdName, containerName, accountName, err.Error())
+		return "", "", err
+	}
+
+	scheme := "http"
+	if useHTTPSForBlobBasedDisk {
+		scheme = "https"
+	}
+
+	host := fmt.Sprintf("%s://%s.%s.%s", scheme, accountName, blobServiceName, c.common.storageEndpointSuffix)
+	uri := fmt.Sprintf("%s/%s/%s", host, containerName, vhdName)
+	return vhdName, uri, nil
+}
+
+// delete a vhd blob
+func (c *BlobDiskController) deleteVhdBlob(accountName, accountKey, blobName string) error {
+	client, err := azstorage.NewBasicClient(accountName, accountKey)
+	if err != nil {
+		return err
+	}
+	blobSvc := client.GetBlobService()
+
+	container := blobSvc.GetContainerReference(vhdContainerName)
+	blob := container.GetBlobReference(blobName)
+	return blob.Delete(nil)
+}
+
+//CreateBlobDisk : create a blob disk in a node
+func (c *BlobDiskController) CreateBlobDisk(dataDiskName string, storageAccountType storage.SkuName, sizeGB int, forceStandAlone bool) (string, error) {
+	glog.V(4).Infof("azureDisk - creating blob data disk named:%s on StorageAccountType:%s StandAlone:%v", dataDiskName, storageAccountType, forceStandAlone)
+
+	var storageAccountName = ""
+	var err error
+
+	if forceStandAlone {
+		// we have to wait until the storage account is is created
+		storageAccountName = "p" + MakeCRC32(c.common.subscriptionID+c.common.resourceGroup+dataDiskName)
+		err = c.createStorageAccount(storageAccountName, storageAccountType, c.common.location, false)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		storageAccountName, err = c.findSANameForDisk(storageAccountType)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	blobClient, err := c.getBlobSvcClient(storageAccountName)
+	if err != nil {
+		return "", err
+	}
+
+	_, diskURI, err := c.createVHDBlobDisk(blobClient, storageAccountName, dataDiskName, defaultContainerName, int64(sizeGB))
+	if err != nil {
+		return "", err
+	}
+
+	if !forceStandAlone {
+		atomic.AddInt32(&c.accounts[storageAccountName].diskCount, 1)
+	}
+
+	return diskURI, nil
+}
+
+//DeleteBlobDisk : delete a blob disk from a node
+func (c *BlobDiskController) DeleteBlobDisk(diskURI string, wasForced bool) error {
+	storageAccountName, vhdName, err := diskNameandSANameFromURI(diskURI)
+	if err != nil {
+		return err
+	}
+
+	_, ok := c.accounts[storageAccountName]
+	if !ok {
+		// the storage account is specified by user
+		glog.V(4).Infof("azureDisk - deleting volume %s", diskURI)
+		return c.DeleteVolume(diskURI)
+	}
+	// if forced (as in one disk = one storage account)
+	// delete the account completely
+	if wasForced {
+		return c.deleteStorageAccount(storageAccountName)
+	}
+
+	blobSvc, err := c.getBlobSvcClient(storageAccountName)
+	if err != nil {
+		return err
+	}
+
+	glog.V(4).Infof("azureDisk - About to delete vhd file %s on storage account %s container %s", vhdName, storageAccountName, defaultContainerName)
+
+	container := blobSvc.GetContainerReference(defaultContainerName)
+	blob := container.GetBlobReference(vhdName)
+	_, err = blob.DeleteIfExists(nil)
+
+	if c.accounts[storageAccountName].diskCount == -1 {
+		if diskCount, err := c.getDiskCount(storageAccountName); err != nil {
+			c.accounts[storageAccountName].diskCount = int32(diskCount)
+		} else {
+			glog.Warningf("azureDisk - failed to get disk count for %s however the delete disk operation was ok", storageAccountName)
+			return nil // we have failed to aquire a new count. not an error condition
+		}
+	}
+	atomic.AddInt32(&c.accounts[storageAccountName].diskCount, -1)
+	return err
+}
+
+// Init tries best effort to ensure that 2 accounts standard/premium were created
+// to be used by shared blob disks. This to increase the speed pvc provisioning (in most of cases)
+func (c *BlobDiskController) init() error {
+	if !c.shouldInit() {
+		return nil
+	}
+
+	c.setUniqueStrings()
+
+	// get accounts
+	accounts, err := c.getAllStorageAccounts()
+	if err != nil {
+		return err
+	}
+	c.accounts = accounts
+
+	if len(c.accounts) == 0 {
+		counter := 1
+		for counter <= storageAccountsCountInit {
+
+			accountType := storage.PremiumLRS
+			if n := math.Mod(float64(counter), 2); n == 0 {
+				accountType = storage.StandardLRS
+			}
+
+			// We don't really care if these calls failed
+			// at this stage, we are trying to ensure 2 accounts (Standard/Premium)
+			// are there ready for PVC creation
+
+			// if we failed here, the accounts will be created in the process
+			// of creating PVC
+
+			// nor do we care if they were partially created, as the entire
+			// account creation process is idempotent
+			go func(thisNext int) {
+				newAccountName := getAccountNameForNum(thisNext)
+
+				glog.Infof("azureDisk - BlobDiskController init process  will create new storageAccount:%s type:%s", newAccountName, accountType)
+				err := c.createStorageAccount(newAccountName, accountType, c.common.location, true)
+				// TODO return created and error from
+				if err != nil {
+					glog.Infof("azureDisk - BlobDiskController init: create account %s with error:%s", newAccountName, err.Error())
+
+				} else {
+					glog.Infof("azureDisk - BlobDiskController init: created account %s", newAccountName)
+				}
+			}(counter)
+			counter = counter + 1
+		}
+	}
+
+	return nil
+}
+
+//Sets unique strings to be used as accountnames && || blob containers names
+func (c *BlobDiskController) setUniqueStrings() {
+	uniqueString := c.common.resourceGroup + c.common.location + c.common.subscriptionID
+	hash := MakeCRC32(uniqueString)
+	//used to generate a unqie container name used by this cluster PVC
+	defaultContainerName = hash
+
+	storageAccountNamePrefix = fmt.Sprintf(storageAccountNameTemplate, hash)
+	// Used to filter relevant accounts (accounts used by shared PVC)
+	storageAccountNameMatch = storageAccountNamePrefix
+	// Used as a template to create new names for relevant accounts
+	storageAccountNamePrefix = storageAccountNamePrefix + "%s"
+}
+func (c *BlobDiskController) getStorageAccountKey(SAName string) (string, error) {
+	if account, exists := c.accounts[SAName]; exists && account.key != "" {
+		return c.accounts[SAName].key, nil
+	}
+	listKeysResult, err := c.common.cloud.StorageAccountClient.ListKeys(c.common.resourceGroup, SAName)
+	if err != nil {
+		return "", err
+	}
+	if listKeysResult.Keys == nil {
+		return "", fmt.Errorf("azureDisk - empty listKeysResult in storage account:%s keys", SAName)
+	}
+	for _, v := range *listKeysResult.Keys {
+		if v.Value != nil && *v.Value == "key1" {
+			if _, ok := c.accounts[SAName]; !ok {
+				glog.Warningf("azureDisk - account %s was not cached while getting keys", SAName)
+				return *v.Value, nil
+			}
+		}
+
+		c.accounts[SAName].key = *v.Value
+		return c.accounts[SAName].key, nil
+	}
+
+	return "", fmt.Errorf("couldn't find key named key1 in storage account:%s keys", SAName)
+}
+
+func (c *BlobDiskController) getBlobSvcClient(SAName string) (azstorage.BlobStorageClient, error) {
+	key := ""
+	var client azstorage.Client
+	var blobSvc azstorage.BlobStorageClient
+	var err error
+	if key, err = c.getStorageAccountKey(SAName); err != nil {
+		return blobSvc, err
+	}
+
+	if client, err = azstorage.NewBasicClient(SAName, key); err != nil {
+		return blobSvc, err
+	}
+
+	blobSvc = client.GetBlobService()
+	return blobSvc, nil
+}
+
+func (c *BlobDiskController) ensureDefaultContainer(storageAccountName string) error {
+	var err error
+	var blobSvc azstorage.BlobStorageClient
+
+	// short circut the check via local cache
+	// we are forgiving the fact that account may not be in cache yet
+	if v, ok := c.accounts[storageAccountName]; ok && v.defaultContainerCreated {
+		return nil
+	}
+
+	// not cached, check existance and readiness
+	bExist, provisionState, _ := c.getStorageAccountState(storageAccountName)
+
+	// account does not exist
+	if !bExist {
+		return fmt.Errorf("azureDisk - account %s does not exist while trying to create/ensure default container", storageAccountName)
+	}
+
+	// account exists but not ready yet
+	if provisionState != storage.Succeeded {
+		// we don't want many attempts to validate the account readiness
+		// here hence we are locking
+		counter := 1
+		for swapped := atomic.CompareAndSwapInt32(&c.accounts[storageAccountName].isValidating, 0, 1); swapped != true; {
+			time.Sleep(3 * time.Second)
+			counter = counter + 1
+			// check if we passed the max sleep
+			if counter >= 20 {
+				return fmt.Errorf("azureDisk - timeout waiting to aquire lock to validate account:%s readiness", storageAccountName)
+			}
+		}
+
+		// swapped
+		defer func() {
+			c.accounts[storageAccountName].isValidating = 0
+		}()
+
+		// short circut the check again.
+		if v, ok := c.accounts[storageAccountName]; ok && v.defaultContainerCreated {
+			return nil
+		}
+
+		err = kwait.ExponentialBackoff(defaultBackOff, func() (bool, error) {
+			_, provisionState, err := c.getStorageAccountState(storageAccountName)
+
+			if err != nil {
+				glog.V(4).Infof("azureDisk - GetStorageAccount:%s err %s", storageAccountName, err.Error())
+				return false, err
+			}
+
+			if provisionState == storage.Succeeded {
+				return true, nil
+			}
+
+			glog.V(4).Infof("azureDisk - GetStorageAccount:%s not ready yet", storageAccountName)
+			// leave it for next loop/sync loop
+			return false, fmt.Errorf("azureDisk - Account %s has not been flagged Succeeded by ARM", storageAccountName)
+		})
+		// we have failed to ensure that account is ready for us to create
+		// the default vhd container
+		if err != nil {
+			return err
+		}
+	}
+
+	if blobSvc, err = c.getBlobSvcClient(storageAccountName); err != nil {
+		return err
+	}
+
+	container := blobSvc.GetContainerReference(defaultContainerName)
+	bCreated, err := container.CreateIfNotExists(&azstorage.CreateContainerOptions{Access: azstorage.ContainerAccessTypePrivate})
+	if err != nil {
+		return err
+	}
+	if bCreated {
+		glog.V(2).Infof("azureDisk - storage account:%s had no default container(%s) and it was created \n", storageAccountName, defaultContainerName)
+	}
+
+	// flag so we no longer have to check on ARM
+	c.accounts[storageAccountName].defaultContainerCreated = true
+	return nil
+}
+
+// Gets Disk counts per storage account
+func (c *BlobDiskController) getDiskCount(SAName string) (int, error) {
+	// if we have it in cache
+	if c.accounts[SAName].diskCount != -1 {
+		return int(c.accounts[SAName].diskCount), nil
+	}
+
+	var err error
+	var blobSvc azstorage.BlobStorageClient
+
+	if err = c.ensureDefaultContainer(SAName); err != nil {
+		return 0, err
+	}
+
+	if blobSvc, err = c.getBlobSvcClient(SAName); err != nil {
+		return 0, err
+	}
+	params := azstorage.ListBlobsParameters{}
+
+	container := blobSvc.GetContainerReference(defaultContainerName)
+	response, err := container.ListBlobs(params)
+	if err != nil {
+		return 0, err
+	}
+	glog.V(4).Infof("azure-Disk -  refreshed data count for account %s and found %v", SAName, len(response.Blobs))
+	c.accounts[SAName].diskCount = int32(len(response.Blobs))
+
+	return int(c.accounts[SAName].diskCount), nil
+}
+
+// shouldInit ensures that we only init the plugin once
+// and we only do that in the controller
+
+func (c *BlobDiskController) shouldInit() bool {
+	if os.Args[0] == "kube-controller-manager" || (os.Args[0] == "/hyperkube" && os.Args[1] == "controller-manager") {
+		swapped := atomic.CompareAndSwapInt64(&initFlag, 0, 1)
+		if swapped {
+			return true
+		}
+	}
+	return false
+}
+
+func (c *BlobDiskController) getAllStorageAccounts() (map[string]*storageAccountState, error) {
+	accountListResult, err := c.common.cloud.StorageAccountClient.List()
+	if err != nil {
+		return nil, err
+	}
+	if accountListResult.Value == nil {
+		return nil, fmt.Errorf("azureDisk - empty accountListResult")
+	}
+
+	accounts := make(map[string]*storageAccountState)
+	for _, v := range *accountListResult.Value {
+		if strings.Index(*v.Name, storageAccountNameMatch) != 0 {
+			continue
+		}
+		if v.Name == nil || v.Sku == nil {
+			glog.Infof("azureDisk - accountListResult Name or Sku is nil")
+			continue
+		}
+		glog.Infof("azureDisk - identified account %s as part of shared PVC accounts", *v.Name)
+
+		sastate := &storageAccountState{
+			name:      *v.Name,
+			saType:    (*v.Sku).Name,
+			diskCount: -1,
+		}
+
+		accounts[*v.Name] = sastate
+	}
+
+	return accounts, nil
+}
+
+func (c *BlobDiskController) createStorageAccount(storageAccountName string, storageAccountType storage.SkuName, location string, checkMaxAccounts bool) error {
+	bExist, _, _ := c.getStorageAccountState(storageAccountName)
+	if bExist {
+		newAccountState := &storageAccountState{
+			diskCount: -1,
+			saType:    storageAccountType,
+			name:      storageAccountName,
+		}
+
+		c.addAccountState(storageAccountName, newAccountState)
+	}
+	// Account Does not exist
+	if !bExist {
+		if len(c.accounts) == maxStorageAccounts && checkMaxAccounts {
+			return fmt.Errorf("azureDisk - can not create new storage account, current storage accounts count:%v Max is:%v", len(c.accounts), maxStorageAccounts)
+		}
+
+		glog.V(2).Infof("azureDisk - Creating storage account %s type %s \n", storageAccountName, string(storageAccountType))
+
+		cp := storage.AccountCreateParameters{
+			Sku:      &storage.Sku{Name: storageAccountType},
+			Tags:     &map[string]*string{"created-by": to.StringPtr("azure-dd")},
+			Location: &location}
+		cancel := make(chan struct{})
+
+		_, errChan := c.common.cloud.StorageAccountClient.Create(c.common.resourceGroup, storageAccountName, cp, cancel)
+		err := <-errChan
+		if err != nil {
+			return fmt.Errorf(fmt.Sprintf("Create Storage Account: %s, error: %s", storageAccountName, err))
+		}
+
+		newAccountState := &storageAccountState{
+			diskCount: -1,
+			saType:    storageAccountType,
+			name:      storageAccountName,
+		}
+
+		c.addAccountState(storageAccountName, newAccountState)
+	}
+
+	if !bExist {
+		// SA Accounts takes time to be provisioned
+		// so if this account was just created allow it sometime
+		// before polling
+		glog.V(2).Infof("azureDisk - storage account %s was just created, allowing time before polling status")
+		time.Sleep(25 * time.Second) // as observed 25 is the average time for SA to be provisioned
+	}
+
+	// finally, make sure that we default container is created
+	// before handing it back over
+	return c.ensureDefaultContainer(storageAccountName)
+}
+
+// finds a new suitable storageAccount for this disk
+func (c *BlobDiskController) findSANameForDisk(storageAccountType storage.SkuName) (string, error) {
+	maxDiskCount := maxDisksPerStorageAccounts
+	SAName := ""
+	totalDiskCounts := 0
+	countAccounts := 0 // account of this type.
+	for _, v := range c.accounts {
+		// filter out any stand-alone disks/accounts
+		if strings.Index(v.name, storageAccountNameMatch) != 0 {
+			continue
+		}
+
+		// note: we compute avge stratified by type.
+		// this to enable user to grow per SA type to avoid low
+		//avg utilization on one account type skewing all data.
+
+		if v.saType == storageAccountType {
+			// compute average
+			dCount, err := c.getDiskCount(v.name)
+			if err != nil {
+				return "", err
+			}
+			totalDiskCounts = totalDiskCounts + dCount
+			countAccounts = countAccounts + 1
+			// empty account
+			if dCount == 0 {
+				glog.V(2).Infof("azureDisk - account %s identified for a new disk  is because it has 0 allocated disks", v.name)
+				return v.name, nil // shortcircut, avg is good and no need to adjust
+			}
+			// if this account is less allocated
+			if dCount < maxDiskCount {
+				maxDiskCount = dCount
+				SAName = v.name
+			}
+		}
+	}
+
+	// if we failed to find storageaccount
+	if SAName == "" {
+		glog.V(2).Infof("azureDisk - failed to identify a suitable account for new disk and will attempt to create new account")
+		SAName = getAccountNameForNum(c.getNextAccountNum())
+		err := c.createStorageAccount(SAName, storageAccountType, c.common.location, true)
+		if err != nil {
+			return "", err
+		}
+		return SAName, nil
+	}
+
+	disksAfter := totalDiskCounts + 1 // with the new one!
+
+	avgUtilization := float64(disksAfter) / float64(countAccounts*maxDisksPerStorageAccounts)
+	aboveAvg := (avgUtilization > storageAccountUtilizationBeforeGrowing)
+
+	// avg are not create and we should craete more accounts if we can
+	if aboveAvg && countAccounts < maxStorageAccounts {
+		glog.V(2).Infof("azureDisk - shared storageAccounts utilzation(%v) >  grow-at-avg-utilization (%v). New storage account will be created", avgUtilization, storageAccountUtilizationBeforeGrowing)
+		SAName = getAccountNameForNum(c.getNextAccountNum())
+		err := c.createStorageAccount(SAName, storageAccountType, c.common.location, true)
+		if err != nil {
+			return "", err
+		}
+		return SAName, nil
+	}
+
+	// avergates are not ok and we are at capacity(max storage accounts allowed)
+	if aboveAvg && countAccounts == maxStorageAccounts {
+		glog.Infof("azureDisk - shared storageAccounts utilzation(%v) > grow-at-avg-utilization (%v). But k8s maxed on SAs for PVC(%v). k8s will now exceed grow-at-avg-utilization without adding accounts",
+			avgUtilization, storageAccountUtilizationBeforeGrowing, maxStorageAccounts)
+	}
+
+	// we found a  storage accounts && [ avg are ok || we reached max sa count ]
+	return SAName, nil
+}
+func (c *BlobDiskController) getNextAccountNum() int {
+	max := 0
+
+	for k := range c.accounts {
+		// filter out accounts that are for standalone
+		if strings.Index(k, storageAccountNameMatch) != 0 {
+			continue
+		}
+		num := getAccountNumFromName(k)
+		if num > max {
+			max = num
+		}
+	}
+
+	return max + 1
+}
+
+func (c *BlobDiskController) deleteStorageAccount(storageAccountName string) error {
+	resp, err := c.common.cloud.StorageAccountClient.Delete(c.common.resourceGroup, storageAccountName)
+	if err != nil {
+		return fmt.Errorf("azureDisk - Delete of storage account '%s' failed with status %s...%v", storageAccountName, resp.Status, err)
+	}
+
+	c.removeAccountState(storageAccountName)
+
+	glog.Infof("azureDisk - Storage Account %s was deleted", storageAccountName)
+	return nil
+}
+
+//Gets storage account exist, provisionStatus, Error if any
+func (c *BlobDiskController) getStorageAccountState(storageAccountName string) (bool, storage.ProvisioningState, error) {
+	account, err := c.common.cloud.StorageAccountClient.GetProperties(c.common.resourceGroup, storageAccountName)
+	if err != nil {
+		return false, "", err
+	}
+	return true, account.AccountProperties.ProvisioningState, nil
+}
+
+func (c *BlobDiskController) addAccountState(key string, state *storageAccountState) {
+	accountsLock.Lock()
+	defer accountsLock.Unlock()
+
+	if _, ok := c.accounts[key]; !ok {
+		c.accounts[key] = state
+	}
+}
+
+func (c *BlobDiskController) removeAccountState(key string) {
+	accountsLock.Lock()
+	defer accountsLock.Unlock()
+	delete(c.accounts, key)
+}
+
+// pads account num with zeros as needed
+func getAccountNameForNum(num int) string {
+	sNum := strconv.Itoa(num)
+	missingZeros := 3 - len(sNum)
+	strZero := ""
+	for missingZeros > 0 {
+		strZero = strZero + "0"
+		missingZeros = missingZeros - 1
+	}
+
+	sNum = strZero + sNum
+	return fmt.Sprintf(storageAccountNamePrefix, sNum)
+}
+
+func getAccountNumFromName(accountName string) int {
+	nameLen := len(accountName)
+	num, _ := strconv.Atoi(accountName[nameLen-3:])
+
+	return num
+}
+
+func createVHDHeader(size uint64) ([]byte, error) {
+	h := vhd.CreateFixedHeader(size, &vhd.VHDOptions{})
+	b := new(bytes.Buffer)
+	err := binary.Write(b, binary.BigEndian, h)
+	if err != nil {
+		return nil, err
+	}
+	return b.Bytes(), nil
+}
+
+func diskNameandSANameFromURI(diskURI string) (string, string, error) {
+	uri, err := url.Parse(diskURI)
+	if err != nil {
+		return "", "", err
+	}
+
+	hostName := uri.Host
+	storageAccountName := strings.Split(hostName, ".")[0]
+
+	segments := strings.Split(uri.Path, "/")
+	diskNameVhd := segments[len(segments)-1]
+
+	return storageAccountName, diskNameVhd, nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/azure_controllerCommon.go

@@ -0,0 +1,270 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"k8s.io/apimachinery/pkg/types"
+	kwait "k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+
+	"github.com/Azure/azure-sdk-for-go/arm/compute"
+	"github.com/golang/glog"
+)
+
+const (
+	defaultDataDiskCount       int = 16 // which will allow you to work with most medium size VMs (if not found in map)
+	storageAccountNameTemplate     = "pvc%s"
+
+	// for limits check https://docs.microsoft.com/en-us/azure/azure-subscription-service-limits#storage-limits
+	maxStorageAccounts                     = 100 // max # is 200 (250 with special request). this allows 100 for everything else including stand alone disks
+	maxDisksPerStorageAccounts             = 60
+	storageAccountUtilizationBeforeGrowing = 0.5
+	storageAccountsCountInit               = 2 // When the plug-in is init-ed, 2 storage accounts will be created to allow fast pvc create/attach/mount
+
+	maxLUN               = 64 // max number of LUNs per VM
+	errLeaseFailed       = "AcquireDiskLeaseFailed"
+	errLeaseIDMissing    = "LeaseIdMissing"
+	errContainerNotFound = "ContainerNotFound"
+)
+
+var defaultBackOff = kwait.Backoff{
+	Steps:    20,
+	Duration: 2 * time.Second,
+	Factor:   1.5,
+	Jitter:   0.0,
+}
+
+type controllerCommon struct {
+	tenantID              string
+	subscriptionID        string
+	location              string
+	storageEndpointSuffix string
+	resourceGroup         string
+	clientID              string
+	clientSecret          string
+	managementEndpoint    string
+	tokenEndPoint         string
+	aadResourceEndPoint   string
+	aadToken              string
+	expiresOn             time.Time
+	cloud                 *Cloud
+}
+
+// AttachDisk attaches a vhd to vm
+// the vhd must exist, can be identified by diskName, diskURI, and lun.
+func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI string, nodeName types.NodeName, lun int32, cachingMode compute.CachingTypes) error {
+	vm, exists, err := c.cloud.getVirtualMachine(nodeName)
+	if err != nil {
+		return err
+	} else if !exists {
+		return cloudprovider.InstanceNotFound
+	}
+	disks := *vm.StorageProfile.DataDisks
+	if isManagedDisk {
+		disks = append(disks,
+			compute.DataDisk{
+				Name:         &diskName,
+				Lun:          &lun,
+				Caching:      cachingMode,
+				CreateOption: "attach",
+				ManagedDisk: &compute.ManagedDiskParameters{
+					ID: &diskURI,
+				},
+			})
+	} else {
+		disks = append(disks,
+			compute.DataDisk{
+				Name: &diskName,
+				Vhd: &compute.VirtualHardDisk{
+					URI: &diskURI,
+				},
+				Lun:          &lun,
+				Caching:      cachingMode,
+				CreateOption: "attach",
+			})
+	}
+
+	newVM := compute.VirtualMachine{
+		Location: vm.Location,
+		VirtualMachineProperties: &compute.VirtualMachineProperties{
+			StorageProfile: &compute.StorageProfile{
+				DataDisks: &disks,
+			},
+		},
+	}
+	vmName := mapNodeNameToVMName(nodeName)
+	glog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk", c.resourceGroup, vmName)
+	c.cloud.operationPollRateLimiter.Accept()
+	respChan, errChan := c.cloud.VirtualMachinesClient.CreateOrUpdate(c.resourceGroup, vmName, newVM, nil)
+	resp := <-respChan
+	err = <-errChan
+	if c.cloud.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
+		glog.V(2).Infof("azureDisk - update(%s) backing off: vm(%s)", c.resourceGroup, vmName)
+		retryErr := c.cloud.CreateOrUpdateVMWithRetry(vmName, newVM)
+		if retryErr != nil {
+			err = retryErr
+			glog.V(2).Infof("azureDisk - update(%s) abort backoff: vm(%s)", c.resourceGroup, vmName)
+		}
+	}
+	if err != nil {
+		glog.Errorf("azureDisk - azure attach failed, err: %v", err)
+		detail := err.Error()
+		if strings.Contains(detail, errLeaseFailed) {
+			// if lease cannot be acquired, immediately detach the disk and return the original error
+			glog.Infof("azureDisk - failed to acquire disk lease, try detach")
+			c.cloud.DetachDiskByName(diskName, diskURI, nodeName)
+		}
+	} else {
+		glog.V(4).Infof("azureDisk - azure attach succeeded")
+	}
+	return err
+}
+
+// DetachDiskByName detaches a vhd from host
+// the vhd can be identified by diskName or diskURI
+func (c *controllerCommon) DetachDiskByName(diskName, diskURI string, nodeName types.NodeName) error {
+	vm, exists, err := c.cloud.getVirtualMachine(nodeName)
+	if err != nil || !exists {
+		// if host doesn't exist, no need to detach
+		glog.Warningf("azureDisk - cannot find node %s, skip detaching disk %s", nodeName, diskName)
+		return nil
+	}
+
+	disks := *vm.StorageProfile.DataDisks
+	bFoundDisk := false
+	for i, disk := range disks {
+		if disk.Lun != nil && (disk.Name != nil && diskName != "" && *disk.Name == diskName) ||
+			(disk.Vhd != nil && disk.Vhd.URI != nil && diskURI != "" && *disk.Vhd.URI == diskURI) ||
+			(disk.ManagedDisk != nil && diskURI != "" && *disk.ManagedDisk.ID == diskURI) {
+			// found the disk
+			glog.V(4).Infof("azureDisk - detach disk: name %q uri %q", diskName, diskURI)
+			disks = append(disks[:i], disks[i+1:]...)
+			bFoundDisk = true
+			break
+		}
+	}
+
+	if !bFoundDisk {
+		return fmt.Errorf("detach azure disk failure, disk %s not found, diskURI: %s", diskName, diskURI)
+	}
+
+	newVM := compute.VirtualMachine{
+		Location: vm.Location,
+		VirtualMachineProperties: &compute.VirtualMachineProperties{
+			StorageProfile: &compute.StorageProfile{
+				DataDisks: &disks,
+			},
+		},
+	}
+	vmName := mapNodeNameToVMName(nodeName)
+	glog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk", c.resourceGroup, vmName)
+	c.cloud.operationPollRateLimiter.Accept()
+	respChan, errChan := c.cloud.VirtualMachinesClient.CreateOrUpdate(c.resourceGroup, vmName, newVM, nil)
+	resp := <-respChan
+	err = <-errChan
+	if c.cloud.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
+		glog.V(2).Infof("azureDisk - update(%s) backing off: vm(%s)", c.resourceGroup, vmName)
+		retryErr := c.cloud.CreateOrUpdateVMWithRetry(vmName, newVM)
+		if retryErr != nil {
+			err = retryErr
+			glog.V(2).Infof("azureDisk - update(%s) abort backoff: vm(%s)", c.cloud.ResourceGroup, vmName)
+		}
+	}
+	if err != nil {
+		glog.Errorf("azureDisk - azure disk detach failed, err: %v", err)
+	} else {
+		glog.V(4).Infof("azureDisk - azure disk detach succeeded")
+	}
+	return err
+}
+
+// GetDiskLun finds the lun on the host that the vhd is attached to, given a vhd's diskName and diskURI
+func (c *controllerCommon) GetDiskLun(diskName, diskURI string, nodeName types.NodeName) (int32, error) {
+	vm, exists, err := c.cloud.getVirtualMachine(nodeName)
+	if err != nil {
+		return -1, err
+	} else if !exists {
+		return -1, cloudprovider.InstanceNotFound
+	}
+	disks := *vm.StorageProfile.DataDisks
+	for _, disk := range disks {
+		if disk.Lun != nil && (disk.Name != nil && diskName != "" && *disk.Name == diskName) ||
+			(disk.Vhd != nil && disk.Vhd.URI != nil && diskURI != "" && *disk.Vhd.URI == diskURI) ||
+			(disk.ManagedDisk != nil && *disk.ManagedDisk.ID == diskURI) {
+			// found the disk
+			glog.V(4).Infof("azureDisk - find disk: lun %d name %q uri %q", *disk.Lun, diskName, diskURI)
+			return *disk.Lun, nil
+		}
+	}
+	return -1, fmt.Errorf("Cannot find Lun for disk %s", diskName)
+}
+
+// GetNextDiskLun searches all vhd attachment on the host and find unused lun
+// return -1 if all luns are used
+func (c *controllerCommon) GetNextDiskLun(nodeName types.NodeName) (int32, error) {
+	vm, exists, err := c.cloud.getVirtualMachine(nodeName)
+	if err != nil {
+		return -1, err
+	} else if !exists {
+		return -1, cloudprovider.InstanceNotFound
+	}
+	used := make([]bool, maxLUN)
+	disks := *vm.StorageProfile.DataDisks
+	for _, disk := range disks {
+		if disk.Lun != nil {
+			used[*disk.Lun] = true
+		}
+	}
+	for k, v := range used {
+		if !v {
+			return int32(k), nil
+		}
+	}
+	return -1, fmt.Errorf("All Luns are used")
+}
+
+// DisksAreAttached checks if a list of volumes are attached to the node with the specified NodeName
+func (c *controllerCommon) DisksAreAttached(diskNames []string, nodeName types.NodeName) (map[string]bool, error) {
+	attached := make(map[string]bool)
+	for _, diskName := range diskNames {
+		attached[diskName] = false
+	}
+	vm, exists, err := c.cloud.getVirtualMachine(nodeName)
+	if !exists {
+		// if host doesn't exist, no need to detach
+		glog.Warningf("azureDisk - Cannot find node %q, DisksAreAttached will assume disks %v are not attached to it.",
+			nodeName, diskNames)
+		return attached, nil
+	} else if err != nil {
+		return attached, err
+	}
+
+	disks := *vm.StorageProfile.DataDisks
+	for _, disk := range disks {
+		for _, diskName := range diskNames {
+			if disk.Name != nil && diskName != "" && *disk.Name == diskName {
+				attached[diskName] = true
+			}
+		}
+	}
+
+	return attached, nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/azure_file.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"fmt"
+
+	azs "github.com/Azure/azure-sdk-for-go/storage"
+	"github.com/golang/glog"
+)
+
+const (
+	useHTTPS = true
+)
+
+// create file share
+func (az *Cloud) createFileShare(accountName, accountKey, name string, sizeGB int) error {
+	fileClient, err := az.getFileSvcClient(accountName, accountKey)
+	if err != nil {
+		return err
+	}
+	// create a file share and set quota
+	// Note. Per https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Create-Share,
+	// setting x-ms-share-quota can set quota on the new share, but in reality, setting quota in CreateShare
+	// receives error "The metadata specified is invalid. It has characters that are not permitted."
+	// As a result,breaking into two API calls: create share and set quota
+	share := fileClient.GetShareReference(name)
+	if err = share.Create(nil); err != nil {
+		return fmt.Errorf("failed to create file share, err: %v", err)
+	}
+	share.Properties.Quota = sizeGB
+	if err = share.SetProperties(nil); err != nil {
+		if err := share.Delete(nil); err != nil {
+			glog.Errorf("Error deleting share: %v", err)
+		}
+		return fmt.Errorf("failed to set quota on file share %s, err: %v", name, err)
+	}
+	return nil
+}
+
+// delete a file share
+func (az *Cloud) deleteFileShare(accountName, accountKey, name string) error {
+	fileClient, err := az.getFileSvcClient(accountName, accountKey)
+	if err == nil {
+		share := fileClient.GetShareReference(name)
+		return share.Delete(nil)
+	}
+	return nil
+}
+
+func (az *Cloud) getFileSvcClient(accountName, accountKey string) (*azs.FileServiceClient, error) {
+	client, err := azs.NewClient(accountName, accountKey, az.Environment.StorageEndpointSuffix, azs.DefaultAPIVersion, useHTTPS)
+	if err != nil {
+		return nil, fmt.Errorf("error creating azure client: %v", err)
+	}
+	f := client.GetFileService()
+	return &f, nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/azure_instance_metadata.go

@@ -0,0 +1,113 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+)
+
+const metadataURL = "http://169.254.169.254/metadata/"
+
+// NetworkMetadata contains metadata about an instance's network
+type NetworkMetadata struct {
+	Interface []NetworkInterface `json:"interface"`
+}
+
+// NetworkInterface represents an instances network interface.
+type NetworkInterface struct {
+	IPV4 NetworkData `json:"ipv4"`
+	IPV6 NetworkData `json:"ipv6"`
+	MAC  string      `json:"macAddress"`
+}
+
+// NetworkData contains IP information for a network.
+type NetworkData struct {
+	IPAddress []IPAddress `json:"ipAddress"`
+	Subnet    []Subnet    `json:"subnet"`
+}
+
+// IPAddress represents IP address information.
+type IPAddress struct {
+	PrivateIP string `json:"privateIPAddress"`
+	PublicIP  string `json:"publicIPAddress"`
+}
+
+// Subnet represents subnet information.
+type Subnet struct {
+	Address string `json:"address"`
+	Prefix  string `json:"prefix"`
+}
+
+// InstanceMetadata knows how to query the Azure instance metadata server.
+type InstanceMetadata struct {
+	baseURL string
+}
+
+// NewInstanceMetadata creates an instance of the InstanceMetadata accessor object.
+func NewInstanceMetadata() *InstanceMetadata {
+	return &InstanceMetadata{
+		baseURL: metadataURL,
+	}
+}
+
+// makeMetadataURL makes a complete metadata URL from the given path.
+func (i *InstanceMetadata) makeMetadataURL(path string) string {
+	return i.baseURL + path
+}
+
+// Object queries the metadata server and populates the passed in object
+func (i *InstanceMetadata) Object(path string, obj interface{}) error {
+	data, err := i.queryMetadataBytes(path, "json")
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(data, obj)
+}
+
+// Text queries the metadata server and returns the corresponding text
+func (i *InstanceMetadata) Text(path string) (string, error) {
+	data, err := i.queryMetadataBytes(path, "text")
+	if err != nil {
+		return "", err
+	}
+	return string(data), err
+}
+
+func (i *InstanceMetadata) queryMetadataBytes(path, format string) ([]byte, error) {
+	client := &http.Client{}
+
+	req, err := http.NewRequest("GET", i.makeMetadataURL(path), nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Metadata", "True")
+
+	q := req.URL.Query()
+	q.Add("format", format)
+	q.Add("api-version", "2017-04-02")
+	req.URL.RawQuery = q.Encode()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	return ioutil.ReadAll(resp.Body)
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/azure_instances.go

@@ -0,0 +1,233 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"errors"
+	"fmt"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+
+	"github.com/Azure/azure-sdk-for-go/arm/compute"
+	"github.com/golang/glog"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// NodeAddresses returns the addresses of the specified instance.
+func (az *Cloud) NodeAddresses(name types.NodeName) ([]v1.NodeAddress, error) {
+	if az.UseInstanceMetadata {
+		ipAddress := IPAddress{}
+		err := az.metadata.Object("instance/network/interface/0/ipv4/ipAddress/0", &ipAddress)
+		if err != nil {
+			return nil, err
+		}
+		addresses := []v1.NodeAddress{
+			{Type: v1.NodeInternalIP, Address: ipAddress.PrivateIP},
+			{Type: v1.NodeHostName, Address: string(name)},
+		}
+		if len(ipAddress.PublicIP) > 0 {
+			addr := v1.NodeAddress{
+				Type:    v1.NodeExternalIP,
+				Address: ipAddress.PublicIP,
+			}
+			addresses = append(addresses, addr)
+		}
+		return addresses, nil
+	}
+	ip, err := az.getIPForMachine(name)
+	if err != nil {
+		if az.CloudProviderBackoff {
+			glog.V(2).Infof("NodeAddresses(%s) backing off", name)
+			ip, err = az.GetIPForMachineWithRetry(name)
+			if err != nil {
+				glog.V(2).Infof("NodeAddresses(%s) abort backoff", name)
+				return nil, err
+			}
+		} else {
+			glog.Errorf("error: az.NodeAddresses, az.getIPForMachine(%s), err=%v", name, err)
+			return nil, err
+		}
+	}
+
+	return []v1.NodeAddress{
+		{Type: v1.NodeInternalIP, Address: ip},
+		{Type: v1.NodeHostName, Address: string(name)},
+	}, nil
+}
+
+// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID
+// This method will not be called from the node that is requesting this ID. i.e. metadata service
+// and other local methods cannot be used here
+func (az *Cloud) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) {
+	name, err := splitProviderID(providerID)
+	if err != nil {
+		return nil, err
+	}
+
+	return az.NodeAddresses(name)
+}
+
+// ExternalID returns the cloud provider ID of the specified instance (deprecated).
+func (az *Cloud) ExternalID(name types.NodeName) (string, error) {
+	return az.InstanceID(name)
+}
+
+// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running.
+// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager.
+func (az *Cloud) InstanceExistsByProviderID(providerID string) (bool, error) {
+	return false, errors.New("unimplemented")
+}
+
+func (az *Cloud) isCurrentInstance(name types.NodeName) (bool, error) {
+	nodeName := mapNodeNameToVMName(name)
+	metadataName, err := az.metadata.Text("instance/compute/name")
+	return (metadataName == nodeName), err
+}
+
+// InstanceID returns the cloud provider ID of the specified instance.
+// Note that if the instance does not exist or is no longer running, we must return ("", cloudprovider.InstanceNotFound)
+func (az *Cloud) InstanceID(name types.NodeName) (string, error) {
+	if az.UseInstanceMetadata {
+		isLocalInstance, err := az.isCurrentInstance(name)
+		if err != nil {
+			return "", err
+		}
+		if isLocalInstance {
+			externalInstanceID, err := az.metadata.Text("instance/compute/vmId")
+			if err == nil {
+				return externalInstanceID, nil
+			}
+		}
+	}
+	var machine compute.VirtualMachine
+	var exists bool
+	var err error
+	az.operationPollRateLimiter.Accept()
+	machine, exists, err = az.getVirtualMachine(name)
+	if err != nil {
+		if az.CloudProviderBackoff {
+			glog.V(2).Infof("InstanceID(%s) backing off", name)
+			machine, exists, err = az.GetVirtualMachineWithRetry(name)
+			if err != nil {
+				glog.V(2).Infof("InstanceID(%s) abort backoff", name)
+				return "", err
+			}
+		} else {
+			return "", err
+		}
+	} else if !exists {
+		return "", cloudprovider.InstanceNotFound
+	}
+	return *machine.ID, nil
+}
+
+// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID
+// This method will not be called from the node that is requesting this ID. i.e. metadata service
+// and other local methods cannot be used here
+func (az *Cloud) InstanceTypeByProviderID(providerID string) (string, error) {
+	name, err := splitProviderID(providerID)
+	if err != nil {
+		return "", err
+	}
+
+	return az.InstanceType(name)
+}
+
+// InstanceType returns the type of the specified instance.
+// Note that if the instance does not exist or is no longer running, we must return ("", cloudprovider.InstanceNotFound)
+// (Implementer Note): This is used by kubelet. Kubelet will label the node. Real log from kubelet:
+//       Adding node label from cloud provider: beta.kubernetes.io/instance-type=[value]
+func (az *Cloud) InstanceType(name types.NodeName) (string, error) {
+	if az.UseInstanceMetadata {
+		isLocalInstance, err := az.isCurrentInstance(name)
+		if err != nil {
+			return "", err
+		}
+		if isLocalInstance {
+			machineType, err := az.metadata.Text("instance/compute/vmSize")
+			if err == nil {
+				return machineType, nil
+			}
+		}
+	}
+	machine, exists, err := az.getVirtualMachine(name)
+	if err != nil {
+		glog.Errorf("error: az.InstanceType(%s), az.getVirtualMachine(%s) err=%v", name, name, err)
+		return "", err
+	} else if !exists {
+		return "", cloudprovider.InstanceNotFound
+	}
+	return string(machine.HardwareProfile.VMSize), nil
+}
+
+// AddSSHKeyToAllInstances adds an SSH public key as a legal identity for all instances
+// expected format for the key is standard ssh-keygen format: <protocol> <blob>
+func (az *Cloud) AddSSHKeyToAllInstances(user string, keyData []byte) error {
+	return fmt.Errorf("not supported")
+}
+
+// CurrentNodeName returns the name of the node we are currently running on
+// On most clouds (e.g. GCE) this is the hostname, so we provide the hostname
+func (az *Cloud) CurrentNodeName(hostname string) (types.NodeName, error) {
+	return types.NodeName(hostname), nil
+}
+
+func (az *Cloud) listAllNodesInResourceGroup() ([]compute.VirtualMachine, error) {
+	allNodes := []compute.VirtualMachine{}
+
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("VirtualMachinesClient.List(%s): start", az.ResourceGroup)
+	result, err := az.VirtualMachinesClient.List(az.ResourceGroup)
+	glog.V(10).Infof("VirtualMachinesClient.List(%s): end", az.ResourceGroup)
+	if err != nil {
+		glog.Errorf("error: az.listAllNodesInResourceGroup(), az.VirtualMachinesClient.List(%s), err=%v", az.ResourceGroup, err)
+		return nil, err
+	}
+
+	morePages := (result.Value != nil && len(*result.Value) > 1)
+
+	for morePages {
+		allNodes = append(allNodes, *result.Value...)
+
+		az.operationPollRateLimiter.Accept()
+		glog.V(10).Infof("VirtualMachinesClient.ListAllNextResults(%v): start", az.ResourceGroup)
+		result, err = az.VirtualMachinesClient.ListAllNextResults(result)
+		glog.V(10).Infof("VirtualMachinesClient.ListAllNextResults(%v): end", az.ResourceGroup)
+		if err != nil {
+			glog.Errorf("error: az.listAllNodesInResourceGroup(), az.VirtualMachinesClient.ListAllNextResults(%v), err=%v", result, err)
+			return nil, err
+		}
+
+		morePages = (result.Value != nil && len(*result.Value) > 1)
+	}
+
+	return allNodes, nil
+
+}
+
+// mapNodeNameToVMName maps a k8s NodeName to an Azure VM Name
+// This is a simple string cast.
+func mapNodeNameToVMName(nodeName types.NodeName) string {
+	return string(nodeName)
+}
+
+// mapVMNameToNodeName maps an Azure VM Name to a k8s NodeName
+// This is a simple string cast.
+func mapVMNameToNodeName(vmName string) types.NodeName {
+	return types.NodeName(vmName)
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/azure_loadbalancer.go

@@ -0,0 +1,996 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"k8s.io/api/core/v1"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	serviceapi "k8s.io/kubernetes/pkg/api/v1/service"
+
+	"github.com/Azure/azure-sdk-for-go/arm/compute"
+	"github.com/Azure/azure-sdk-for-go/arm/network"
+	"github.com/Azure/go-autorest/autorest/to"
+	"github.com/golang/glog"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// ServiceAnnotationLoadBalancerInternal is the annotation used on the service
+const ServiceAnnotationLoadBalancerInternal = "service.beta.kubernetes.io/azure-load-balancer-internal"
+
+// GetLoadBalancer returns whether the specified load balancer exists, and
+// if so, what its status is.
+func (az *Cloud) GetLoadBalancer(clusterName string, service *v1.Service) (status *v1.LoadBalancerStatus, exists bool, err error) {
+	isInternal := requiresInternalLoadBalancer(service)
+	lbName := getLoadBalancerName(clusterName, isInternal)
+	serviceName := getServiceName(service)
+
+	lb, existsLb, err := az.getAzureLoadBalancer(lbName)
+	if err != nil {
+		return nil, false, err
+	}
+	if !existsLb {
+		glog.V(5).Infof("get(%s): lb(%s) - doesn't exist", serviceName, lbName)
+		return nil, false, nil
+	}
+
+	var lbIP *string
+
+	if isInternal {
+		lbFrontendIPConfigName := getFrontendIPConfigName(service)
+		for _, ipConfiguration := range *lb.FrontendIPConfigurations {
+			if lbFrontendIPConfigName == *ipConfiguration.Name {
+				lbIP = ipConfiguration.PrivateIPAddress
+				break
+			}
+		}
+	} else {
+		// TODO: Consider also read address from lb's FrontendIPConfigurations
+		pipName, err := az.determinePublicIPName(clusterName, service)
+		if err != nil {
+			return nil, false, err
+		}
+		pip, existsPip, err := az.getPublicIPAddress(pipName)
+		if err != nil {
+			return nil, false, err
+		}
+		if existsPip {
+			lbIP = pip.IPAddress
+		}
+	}
+
+	if lbIP == nil {
+		glog.V(5).Infof("get(%s): lb(%s) - IP doesn't exist", serviceName, lbName)
+		return nil, false, nil
+	}
+
+	return &v1.LoadBalancerStatus{
+		Ingress: []v1.LoadBalancerIngress{{IP: *lbIP}},
+	}, true, nil
+}
+
+func (az *Cloud) determinePublicIPName(clusterName string, service *v1.Service) (string, error) {
+	loadBalancerIP := service.Spec.LoadBalancerIP
+	if len(loadBalancerIP) == 0 {
+		return getPublicIPName(clusterName, service), nil
+	}
+
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("PublicIPAddressesClient.List(%v): start", az.ResourceGroup)
+	list, err := az.PublicIPAddressesClient.List(az.ResourceGroup)
+	glog.V(10).Infof("PublicIPAddressesClient.List(%v): end", az.ResourceGroup)
+	if err != nil {
+		return "", err
+	}
+
+	if list.Value != nil {
+		for ix := range *list.Value {
+			ip := &(*list.Value)[ix]
+			if ip.PublicIPAddressPropertiesFormat.IPAddress != nil &&
+				*ip.PublicIPAddressPropertiesFormat.IPAddress == loadBalancerIP {
+				return *ip.Name, nil
+			}
+		}
+	}
+	// TODO: follow next link here? Will there really ever be that many public IPs?
+
+	return "", fmt.Errorf("user supplied IP Address %s was not found", loadBalancerIP)
+}
+
+// EnsureLoadBalancer creates a new load balancer 'name', or updates the existing one. Returns the status of the balancer
+func (az *Cloud) EnsureLoadBalancer(clusterName string, service *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) {
+	isInternal := requiresInternalLoadBalancer(service)
+	lbName := getLoadBalancerName(clusterName, isInternal)
+
+	// When a client updates the internal load balancer annotation,
+	// the service may be switched from an internal LB to a public one, or vise versa.
+	// Here we'll firstly ensure service do not lie in the opposite LB.
+	err := az.cleanupLoadBalancer(clusterName, service, !isInternal)
+	if err != nil {
+		return nil, err
+	}
+
+	serviceName := getServiceName(service)
+	glog.V(5).Infof("ensure(%s): START clusterName=%q lbName=%q", serviceName, clusterName, lbName)
+
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("SecurityGroupsClient.Get(%q): start", az.SecurityGroupName)
+	sg, err := az.SecurityGroupsClient.Get(az.ResourceGroup, az.SecurityGroupName, "")
+	glog.V(10).Infof("SecurityGroupsClient.Get(%q): end", az.SecurityGroupName)
+	if err != nil {
+		return nil, err
+	}
+	sg, sgNeedsUpdate, err := az.reconcileSecurityGroup(sg, clusterName, service, true /* wantLb */)
+	if err != nil {
+		return nil, err
+	}
+	if sgNeedsUpdate {
+		glog.V(3).Infof("ensure(%s): sg(%s) - updating", serviceName, *sg.Name)
+		// azure-sdk-for-go introduced contraint validation which breaks the updating here if we don't set these
+		// to nil. This is a workaround until https://github.com/Azure/go-autorest/issues/112 is fixed
+		sg.SecurityGroupPropertiesFormat.NetworkInterfaces = nil
+		sg.SecurityGroupPropertiesFormat.Subnets = nil
+		az.operationPollRateLimiter.Accept()
+		glog.V(10).Infof("SecurityGroupsClient.CreateOrUpdate(%q): start", *sg.Name)
+		respChan, errChan := az.SecurityGroupsClient.CreateOrUpdate(az.ResourceGroup, *sg.Name, sg, nil)
+		resp := <-respChan
+		err := <-errChan
+		glog.V(10).Infof("SecurityGroupsClient.CreateOrUpdate(%q): end", *sg.Name)
+		if az.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
+			glog.V(2).Infof("ensure(%s) backing off: sg(%s) - updating", serviceName, *sg.Name)
+			retryErr := az.CreateOrUpdateSGWithRetry(sg)
+			if retryErr != nil {
+				glog.V(2).Infof("ensure(%s) abort backoff: sg(%s) - updating", serviceName, *sg.Name)
+				return nil, retryErr
+			}
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	lb, existsLb, err := az.getAzureLoadBalancer(lbName)
+	if err != nil {
+		return nil, err
+	}
+	if !existsLb {
+		lb = network.LoadBalancer{
+			Name:                         &lbName,
+			Location:                     &az.Location,
+			LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{},
+		}
+	}
+
+	var lbIP *string
+	var fipConfigurationProperties *network.FrontendIPConfigurationPropertiesFormat
+
+	if isInternal {
+		subnet, existsSubnet, err := az.getSubnet(az.VnetName, az.SubnetName)
+		if err != nil {
+			return nil, err
+		}
+
+		if !existsSubnet {
+			return nil, fmt.Errorf("ensure(%s): lb(%s) - failed to get subnet: %s/%s", serviceName, lbName, az.VnetName, az.SubnetName)
+		}
+
+		configProperties := network.FrontendIPConfigurationPropertiesFormat{
+			Subnet: &network.Subnet{
+				ID: subnet.ID,
+			},
+		}
+
+		loadBalancerIP := service.Spec.LoadBalancerIP
+		if loadBalancerIP != "" {
+			configProperties.PrivateIPAllocationMethod = network.Static
+			configProperties.PrivateIPAddress = &loadBalancerIP
+			lbIP = &loadBalancerIP
+		} else {
+			// We'll need to call GetLoadBalancer later to retrieve allocated IP.
+			configProperties.PrivateIPAllocationMethod = network.Dynamic
+		}
+
+		fipConfigurationProperties = &configProperties
+	} else {
+		pipName, err := az.determinePublicIPName(clusterName, service)
+		if err != nil {
+			return nil, err
+		}
+		pip, err := az.ensurePublicIPExists(serviceName, pipName)
+		if err != nil {
+			return nil, err
+		}
+
+		lbIP = pip.IPAddress
+		fipConfigurationProperties = &network.FrontendIPConfigurationPropertiesFormat{
+			PublicIPAddress: &network.PublicIPAddress{ID: pip.ID},
+		}
+	}
+
+	lb, lbNeedsUpdate, err := az.reconcileLoadBalancer(lb, fipConfigurationProperties, clusterName, service, nodes)
+	if err != nil {
+		return nil, err
+	}
+	if !existsLb || lbNeedsUpdate {
+		glog.V(3).Infof("ensure(%s): lb(%s) - updating", serviceName, lbName)
+		az.operationPollRateLimiter.Accept()
+		glog.V(10).Infof("LoadBalancerClient.CreateOrUpdate(%q): start", *lb.Name)
+		respChan, errChan := az.LoadBalancerClient.CreateOrUpdate(az.ResourceGroup, *lb.Name, lb, nil)
+		resp := <-respChan
+		err := <-errChan
+		glog.V(10).Infof("LoadBalancerClient.CreateOrUpdate(%q): end", *lb.Name)
+		if az.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
+			glog.V(2).Infof("ensure(%s) backing off: lb(%s) - updating", serviceName, lbName)
+			retryErr := az.CreateOrUpdateLBWithRetry(lb)
+			if retryErr != nil {
+				glog.V(2).Infof("ensure(%s) abort backoff: lb(%s) - updating", serviceName, lbName)
+				return nil, retryErr
+			}
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Add the machines to the backend pool if they're not already
+	lbBackendName := getBackendPoolName(clusterName)
+	lbBackendPoolID := az.getBackendPoolID(lbName, lbBackendName)
+	hostUpdates := make([]func() error, len(nodes))
+	for i, node := range nodes {
+		localNodeName := node.Name
+		f := func() error {
+			err := az.ensureHostInPool(serviceName, types.NodeName(localNodeName), lbBackendPoolID)
+			if err != nil {
+				return fmt.Errorf("ensure(%s): lb(%s) - failed to ensure host in pool: %q", serviceName, lbName, err)
+			}
+			return nil
+		}
+		hostUpdates[i] = f
+	}
+
+	errs := utilerrors.AggregateGoroutines(hostUpdates...)
+	if errs != nil {
+		return nil, utilerrors.Flatten(errs)
+	}
+
+	glog.V(2).Infof("ensure(%s): lb(%s) finished", serviceName, lbName)
+
+	if lbIP == nil {
+		lbStatus, exists, err := az.GetLoadBalancer(clusterName, service)
+		if err != nil {
+			return nil, err
+		}
+		if !exists {
+			return nil, fmt.Errorf("ensure(%s): lb(%s) - failed to get back load balancer", serviceName, lbName)
+		}
+		return lbStatus, nil
+	}
+
+	return &v1.LoadBalancerStatus{
+		Ingress: []v1.LoadBalancerIngress{{IP: *lbIP}},
+	}, nil
+}
+
+// UpdateLoadBalancer updates hosts under the specified load balancer.
+func (az *Cloud) UpdateLoadBalancer(clusterName string, service *v1.Service, nodes []*v1.Node) error {
+	_, err := az.EnsureLoadBalancer(clusterName, service, nodes)
+	return err
+}
+
+// EnsureLoadBalancerDeleted deletes the specified load balancer if it
+// exists, returning nil if the load balancer specified either didn't exist or
+// was successfully deleted.
+// This construction is useful because many cloud providers' load balancers
+// have multiple underlying components, meaning a Get could say that the LB
+// doesn't exist even if some part of it is still laying around.
+func (az *Cloud) EnsureLoadBalancerDeleted(clusterName string, service *v1.Service) error {
+	isInternal := requiresInternalLoadBalancer(service)
+	lbName := getLoadBalancerName(clusterName, isInternal)
+	serviceName := getServiceName(service)
+
+	glog.V(5).Infof("delete(%s): START clusterName=%q lbName=%q", serviceName, clusterName, lbName)
+
+	err := az.cleanupLoadBalancer(clusterName, service, isInternal)
+	if err != nil {
+		return err
+	}
+
+	sg, existsSg, err := az.getSecurityGroup()
+	if err != nil {
+		return err
+	}
+	if existsSg {
+		reconciledSg, sgNeedsUpdate, reconcileErr := az.reconcileSecurityGroup(sg, clusterName, service, false /* wantLb */)
+		if reconcileErr != nil {
+			return reconcileErr
+		}
+		if sgNeedsUpdate {
+			glog.V(3).Infof("delete(%s): sg(%s) - updating", serviceName, az.SecurityGroupName)
+			// azure-sdk-for-go introduced contraint validation which breaks the updating here if we don't set these
+			// to nil. This is a workaround until https://github.com/Azure/go-autorest/issues/112 is fixed
+			sg.SecurityGroupPropertiesFormat.NetworkInterfaces = nil
+			sg.SecurityGroupPropertiesFormat.Subnets = nil
+			az.operationPollRateLimiter.Accept()
+			glog.V(10).Infof("SecurityGroupsClient.CreateOrUpdate(%q): start", *reconciledSg.Name)
+			respChan, errChan := az.SecurityGroupsClient.CreateOrUpdate(az.ResourceGroup, *reconciledSg.Name, reconciledSg, nil)
+			resp := <-respChan
+			err := <-errChan
+			glog.V(10).Infof("SecurityGroupsClient.CreateOrUpdate(%q): end", *reconciledSg.Name)
+			if az.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
+				glog.V(2).Infof("delete(%s) backing off: sg(%s) - updating", serviceName, az.SecurityGroupName)
+				retryErr := az.CreateOrUpdateSGWithRetry(reconciledSg)
+				if retryErr != nil {
+					err = retryErr
+					glog.V(2).Infof("delete(%s) abort backoff: sg(%s) - updating", serviceName, az.SecurityGroupName)
+				}
+			}
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	glog.V(2).Infof("delete(%s): FINISH", serviceName)
+	return nil
+}
+
+func (az *Cloud) cleanupLoadBalancer(clusterName string, service *v1.Service, isInternalLb bool) error {
+	lbName := getLoadBalancerName(clusterName, isInternalLb)
+	serviceName := getServiceName(service)
+
+	glog.V(10).Infof("ensure lb deleted: clusterName=%q, serviceName=%s, lbName=%q", clusterName, serviceName, lbName)
+
+	lb, existsLb, err := az.getAzureLoadBalancer(lbName)
+	if err != nil {
+		return err
+	}
+	if existsLb {
+		var publicIPToCleanup *string
+
+		if !isInternalLb {
+			// Find public ip resource to clean up from IP configuration
+			lbFrontendIPConfigName := getFrontendIPConfigName(service)
+			for _, config := range *lb.FrontendIPConfigurations {
+				if strings.EqualFold(*config.Name, lbFrontendIPConfigName) {
+					if config.PublicIPAddress != nil {
+						// Only ID property is available
+						publicIPToCleanup = config.PublicIPAddress.ID
+					}
+					break
+				}
+			}
+		}
+
+		lb, lbNeedsUpdate, reconcileErr := az.reconcileLoadBalancer(lb, nil, clusterName, service, []*v1.Node{})
+		if reconcileErr != nil {
+			return reconcileErr
+		}
+		if lbNeedsUpdate {
+			if len(*lb.FrontendIPConfigurations) > 0 {
+				glog.V(3).Infof("delete(%s): lb(%s) - updating", serviceName, lbName)
+				az.operationPollRateLimiter.Accept()
+				glog.V(10).Infof("LoadBalancerClient.CreateOrUpdate(%q): start", *lb.Name)
+				respChan, errChan := az.LoadBalancerClient.CreateOrUpdate(az.ResourceGroup, *lb.Name, lb, nil)
+				resp := <-respChan
+				err := <-errChan
+				glog.V(10).Infof("LoadBalancerClient.CreateOrUpdate(%q): end", *lb.Name)
+				if az.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
+					glog.V(2).Infof("delete(%s) backing off: sg(%s) - updating", serviceName, az.SecurityGroupName)
+					retryErr := az.CreateOrUpdateLBWithRetry(lb)
+					if retryErr != nil {
+						err = retryErr
+						glog.V(2).Infof("delete(%s) abort backoff: sg(%s) - updating", serviceName, az.SecurityGroupName)
+					}
+				}
+				if err != nil {
+					return err
+				}
+			} else {
+				glog.V(3).Infof("delete(%s): lb(%s) - deleting; no remaining frontendipconfigs", serviceName, lbName)
+
+				az.operationPollRateLimiter.Accept()
+				glog.V(10).Infof("LoadBalancerClient.Delete(%q): start", lbName)
+				respChan, errChan := az.LoadBalancerClient.Delete(az.ResourceGroup, lbName, nil)
+				resp := <-respChan
+				err := <-errChan
+				glog.V(10).Infof("LoadBalancerClient.Delete(%q): end", lbName)
+				if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) {
+					glog.V(2).Infof("delete(%s) backing off: lb(%s) - deleting; no remaining frontendipconfigs", serviceName, lbName)
+					retryErr := az.DeleteLBWithRetry(lbName)
+					if retryErr != nil {
+						err = retryErr
+						glog.V(2).Infof("delete(%s) abort backoff: lb(%s) - deleting; no remaining frontendipconfigs", serviceName, lbName)
+					}
+				}
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		// Public IP can be deleted after frontend ip configuration rule deleted.
+		if publicIPToCleanup != nil {
+			// Only delete an IP address if we created it, deducing by name.
+			if index := strings.LastIndex(*publicIPToCleanup, "/"); index != -1 {
+				managedPipName := getPublicIPName(clusterName, service)
+				pipName := (*publicIPToCleanup)[index+1:]
+				if strings.EqualFold(managedPipName, pipName) {
+					glog.V(5).Infof("Deleting public IP resource %q.", pipName)
+					err = az.ensurePublicIPDeleted(serviceName, pipName)
+					if err != nil {
+						return err
+					}
+				} else {
+					glog.V(5).Infof("Public IP resource %q found, but it does not match managed name %q, skip deleting.", pipName, managedPipName)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (az *Cloud) ensurePublicIPExists(serviceName, pipName string) (*network.PublicIPAddress, error) {
+	pip, existsPip, err := az.getPublicIPAddress(pipName)
+	if err != nil {
+		return nil, err
+	}
+	if existsPip {
+		return &pip, nil
+	}
+
+	pip.Name = to.StringPtr(pipName)
+	pip.Location = to.StringPtr(az.Location)
+	pip.PublicIPAddressPropertiesFormat = &network.PublicIPAddressPropertiesFormat{
+		PublicIPAllocationMethod: network.Static,
+	}
+	pip.Tags = &map[string]*string{"service": &serviceName}
+
+	glog.V(3).Infof("ensure(%s): pip(%s) - creating", serviceName, *pip.Name)
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("PublicIPAddressesClient.CreateOrUpdate(%q): start", *pip.Name)
+	respChan, errChan := az.PublicIPAddressesClient.CreateOrUpdate(az.ResourceGroup, *pip.Name, pip, nil)
+	resp := <-respChan
+	err = <-errChan
+	glog.V(10).Infof("PublicIPAddressesClient.CreateOrUpdate(%q): end", *pip.Name)
+	if az.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
+		glog.V(2).Infof("ensure(%s) backing off: pip(%s) - creating", serviceName, *pip.Name)
+		retryErr := az.CreateOrUpdatePIPWithRetry(pip)
+		if retryErr != nil {
+			glog.V(2).Infof("ensure(%s) abort backoff: pip(%s) - creating", serviceName, *pip.Name)
+			err = retryErr
+		}
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("PublicIPAddressesClient.Get(%q): start", *pip.Name)
+	pip, err = az.PublicIPAddressesClient.Get(az.ResourceGroup, *pip.Name, "")
+	glog.V(10).Infof("PublicIPAddressesClient.Get(%q): end", *pip.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	return &pip, nil
+
+}
+
+func (az *Cloud) ensurePublicIPDeleted(serviceName, pipName string) error {
+	glog.V(2).Infof("ensure(%s): pip(%s) - deleting", serviceName, pipName)
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("PublicIPAddressesClient.Delete(%q): start", pipName)
+	resp, deleteErrChan := az.PublicIPAddressesClient.Delete(az.ResourceGroup, pipName, nil)
+	deleteErr := <-deleteErrChan
+	glog.V(10).Infof("PublicIPAddressesClient.Delete(%q): end", pipName) // response not read yet...
+	if az.CloudProviderBackoff && shouldRetryAPIRequest(<-resp, deleteErr) {
+		glog.V(2).Infof("ensure(%s) backing off: pip(%s) - deleting", serviceName, pipName)
+		retryErr := az.DeletePublicIPWithRetry(pipName)
+		if retryErr != nil {
+			glog.V(2).Infof("ensure(%s) abort backoff: pip(%s) - deleting", serviceName, pipName)
+			return retryErr
+		}
+	}
+	_, realErr := checkResourceExistsFromError(deleteErr)
+	if realErr != nil {
+		return nil
+	}
+	return nil
+}
+
+// This ensures load balancer exists and the frontend ip config is setup.
+// This also reconciles the Service's Ports  with the LoadBalancer config.
+// This entails adding rules/probes for expected Ports and removing stale rules/ports.
+func (az *Cloud) reconcileLoadBalancer(lb network.LoadBalancer, fipConfigurationProperties *network.FrontendIPConfigurationPropertiesFormat, clusterName string, service *v1.Service, nodes []*v1.Node) (network.LoadBalancer, bool, error) {
+	isInternal := requiresInternalLoadBalancer(service)
+	lbName := getLoadBalancerName(clusterName, isInternal)
+	serviceName := getServiceName(service)
+	lbFrontendIPConfigName := getFrontendIPConfigName(service)
+	lbFrontendIPConfigID := az.getFrontendIPConfigID(lbName, lbFrontendIPConfigName)
+	lbBackendPoolName := getBackendPoolName(clusterName)
+	lbBackendPoolID := az.getBackendPoolID(lbName, lbBackendPoolName)
+
+	wantLb := fipConfigurationProperties != nil
+	dirtyLb := false
+
+	// Ensure LoadBalancer's Backend Pool Configuration
+	if wantLb {
+		newBackendPools := []network.BackendAddressPool{}
+		if lb.BackendAddressPools != nil {
+			newBackendPools = *lb.BackendAddressPools
+		}
+
+		foundBackendPool := false
+		for _, bp := range newBackendPools {
+			if strings.EqualFold(*bp.Name, lbBackendPoolName) {
+				glog.V(10).Infof("reconcile(%s)(%t): lb backendpool - found wanted backendpool. not adding anything", serviceName, wantLb)
+				foundBackendPool = true
+				break
+			} else {
+				glog.V(10).Infof("reconcile(%s)(%t): lb backendpool - found other backendpool %s", serviceName, wantLb, *bp.Name)
+			}
+		}
+		if !foundBackendPool {
+			newBackendPools = append(newBackendPools, network.BackendAddressPool{
+				Name: to.StringPtr(lbBackendPoolName),
+			})
+			glog.V(10).Infof("reconcile(%s)(%t): lb backendpool - adding backendpool", serviceName, wantLb)
+
+			dirtyLb = true
+			lb.BackendAddressPools = &newBackendPools
+		}
+	}
+
+	// Ensure LoadBalancer's Frontend IP Configurations
+	dirtyConfigs := false
+	newConfigs := []network.FrontendIPConfiguration{}
+	if lb.FrontendIPConfigurations != nil {
+		newConfigs = *lb.FrontendIPConfigurations
+	}
+	if !wantLb {
+		for i := len(newConfigs) - 1; i >= 0; i-- {
+			config := newConfigs[i]
+			if strings.EqualFold(*config.Name, lbFrontendIPConfigName) {
+				glog.V(3).Infof("reconcile(%s)(%t): lb frontendconfig(%s) - dropping", serviceName, wantLb, lbFrontendIPConfigName)
+				newConfigs = append(newConfigs[:i], newConfigs[i+1:]...)
+				dirtyConfigs = true
+			}
+		}
+	} else {
+		foundConfig := false
+		for _, config := range newConfigs {
+			if strings.EqualFold(*config.Name, lbFrontendIPConfigName) {
+				foundConfig = true
+				break
+			}
+		}
+		if !foundConfig {
+			newConfigs = append(newConfigs,
+				network.FrontendIPConfiguration{
+					Name: to.StringPtr(lbFrontendIPConfigName),
+					FrontendIPConfigurationPropertiesFormat: fipConfigurationProperties,
+				})
+			glog.V(10).Infof("reconcile(%s)(%t): lb frontendconfig(%s) - adding", serviceName, wantLb, lbFrontendIPConfigName)
+			dirtyConfigs = true
+		}
+	}
+	if dirtyConfigs {
+		dirtyLb = true
+		lb.FrontendIPConfigurations = &newConfigs
+	}
+
+	// update probes/rules
+	var ports []v1.ServicePort
+	if wantLb {
+		ports = service.Spec.Ports
+	} else {
+		ports = []v1.ServicePort{}
+	}
+
+	var expectedProbes []network.Probe
+	var expectedRules []network.LoadBalancingRule
+	for _, port := range ports {
+		lbRuleName := getLoadBalancerRuleName(service, port)
+
+		transportProto, _, probeProto, err := getProtocolsFromKubernetesProtocol(port.Protocol)
+		if err != nil {
+			return lb, false, err
+		}
+
+		if serviceapi.NeedsHealthCheck(service) {
+			if port.Protocol == v1.ProtocolUDP {
+				// ERROR: this isn't supported
+				// health check (aka source ip preservation) is not
+				// compatible with UDP (it uses an HTTP check)
+				return lb, false, fmt.Errorf("services requiring health checks are incompatible with UDP ports")
+			}
+
+			podPresencePath, podPresencePort := serviceapi.GetServiceHealthCheckPathPort(service)
+
+			expectedProbes = append(expectedProbes, network.Probe{
+				Name: &lbRuleName,
+				ProbePropertiesFormat: &network.ProbePropertiesFormat{
+					RequestPath:       to.StringPtr(podPresencePath),
+					Protocol:          network.ProbeProtocolHTTP,
+					Port:              to.Int32Ptr(podPresencePort),
+					IntervalInSeconds: to.Int32Ptr(5),
+					NumberOfProbes:    to.Int32Ptr(2),
+				},
+			})
+		} else if port.Protocol != v1.ProtocolUDP {
+			// we only add the expected probe if we're doing TCP
+			expectedProbes = append(expectedProbes, network.Probe{
+				Name: &lbRuleName,
+				ProbePropertiesFormat: &network.ProbePropertiesFormat{
+					Protocol:          *probeProto,
+					Port:              to.Int32Ptr(port.NodePort),
+					IntervalInSeconds: to.Int32Ptr(5),
+					NumberOfProbes:    to.Int32Ptr(2),
+				},
+			})
+		}
+
+		loadDistribution := network.Default
+		if service.Spec.SessionAffinity == v1.ServiceAffinityClientIP {
+			loadDistribution = network.SourceIP
+		}
+		expectedRule := network.LoadBalancingRule{
+			Name: &lbRuleName,
+			LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
+				Protocol: *transportProto,
+				FrontendIPConfiguration: &network.SubResource{
+					ID: to.StringPtr(lbFrontendIPConfigID),
+				},
+				BackendAddressPool: &network.SubResource{
+					ID: to.StringPtr(lbBackendPoolID),
+				},
+				LoadDistribution: loadDistribution,
+				FrontendPort:     to.Int32Ptr(port.Port),
+				BackendPort:      to.Int32Ptr(port.Port),
+				EnableFloatingIP: to.BoolPtr(true),
+			},
+		}
+
+		// we didn't construct the probe objects for UDP because they're not used/needed/allowed
+		if port.Protocol != v1.ProtocolUDP {
+			expectedRule.Probe = &network.SubResource{
+				ID: to.StringPtr(az.getLoadBalancerProbeID(lbName, lbRuleName)),
+			}
+		}
+
+		expectedRules = append(expectedRules, expectedRule)
+	}
+
+	// remove unwanted probes
+	dirtyProbes := false
+	var updatedProbes []network.Probe
+	if lb.Probes != nil {
+		updatedProbes = *lb.Probes
+	}
+	for i := len(updatedProbes) - 1; i >= 0; i-- {
+		existingProbe := updatedProbes[i]
+		if serviceOwnsRule(service, *existingProbe.Name) {
+			glog.V(10).Infof("reconcile(%s)(%t): lb probe(%s) - considering evicting", serviceName, wantLb, *existingProbe.Name)
+			keepProbe := false
+			if findProbe(expectedProbes, existingProbe) {
+				glog.V(10).Infof("reconcile(%s)(%t): lb probe(%s) - keeping", serviceName, wantLb, *existingProbe.Name)
+				keepProbe = true
+			}
+			if !keepProbe {
+				updatedProbes = append(updatedProbes[:i], updatedProbes[i+1:]...)
+				glog.V(10).Infof("reconcile(%s)(%t): lb probe(%s) - dropping", serviceName, wantLb, *existingProbe.Name)
+				dirtyProbes = true
+			}
+		}
+	}
+	// add missing, wanted probes
+	for _, expectedProbe := range expectedProbes {
+		foundProbe := false
+		if findProbe(updatedProbes, expectedProbe) {
+			glog.V(10).Infof("reconcile(%s)(%t): lb probe(%s) - already exists", serviceName, wantLb, *expectedProbe.Name)
+			foundProbe = true
+		}
+		if !foundProbe {
+			glog.V(10).Infof("reconcile(%s)(%t): lb probe(%s) - adding", serviceName, wantLb, *expectedProbe.Name)
+			updatedProbes = append(updatedProbes, expectedProbe)
+			dirtyProbes = true
+		}
+	}
+	if dirtyProbes {
+		dirtyLb = true
+		lb.Probes = &updatedProbes
+	}
+
+	// update rules
+	dirtyRules := false
+	var updatedRules []network.LoadBalancingRule
+	if lb.LoadBalancingRules != nil {
+		updatedRules = *lb.LoadBalancingRules
+	}
+	// update rules: remove unwanted
+	for i := len(updatedRules) - 1; i >= 0; i-- {
+		existingRule := updatedRules[i]
+		if serviceOwnsRule(service, *existingRule.Name) {
+			keepRule := false
+			glog.V(10).Infof("reconcile(%s)(%t): lb rule(%s) - considering evicting", serviceName, wantLb, *existingRule.Name)
+			if findRule(expectedRules, existingRule) {
+				glog.V(10).Infof("reconcile(%s)(%t): lb rule(%s) - keeping", serviceName, wantLb, *existingRule.Name)
+				keepRule = true
+			}
+			if !keepRule {
+				glog.V(3).Infof("reconcile(%s)(%t): lb rule(%s) - dropping", serviceName, wantLb, *existingRule.Name)
+				updatedRules = append(updatedRules[:i], updatedRules[i+1:]...)
+				dirtyRules = true
+			}
+		}
+	}
+	// update rules: add needed
+	for _, expectedRule := range expectedRules {
+		foundRule := false
+		if findRule(updatedRules, expectedRule) {
+			glog.V(10).Infof("reconcile(%s)(%t): lb rule(%s) - already exists", serviceName, wantLb, *expectedRule.Name)
+			foundRule = true
+		}
+		if !foundRule {
+			glog.V(10).Infof("reconcile(%s)(%t): lb rule(%s) adding", serviceName, wantLb, *expectedRule.Name)
+			updatedRules = append(updatedRules, expectedRule)
+			dirtyRules = true
+		}
+	}
+	if dirtyRules {
+		dirtyLb = true
+		lb.LoadBalancingRules = &updatedRules
+	}
+
+	return lb, dirtyLb, nil
+}
+
+// This reconciles the Network Security Group similar to how the LB is reconciled.
+// This entails adding required, missing SecurityRules and removing stale rules.
+func (az *Cloud) reconcileSecurityGroup(sg network.SecurityGroup, clusterName string, service *v1.Service, wantLb bool) (network.SecurityGroup, bool, error) {
+	serviceName := getServiceName(service)
+	var ports []v1.ServicePort
+	if wantLb {
+		ports = service.Spec.Ports
+	} else {
+		ports = []v1.ServicePort{}
+	}
+
+	sourceRanges, err := serviceapi.GetLoadBalancerSourceRanges(service)
+	if err != nil {
+		return sg, false, err
+	}
+	var sourceAddressPrefixes []string
+	if sourceRanges == nil || serviceapi.IsAllowAll(sourceRanges) {
+		if !requiresInternalLoadBalancer(service) {
+			sourceAddressPrefixes = []string{"Internet"}
+		}
+	} else {
+		for _, ip := range sourceRanges {
+			sourceAddressPrefixes = append(sourceAddressPrefixes, ip.String())
+		}
+	}
+	expectedSecurityRules := make([]network.SecurityRule, len(ports)*len(sourceAddressPrefixes))
+
+	for i, port := range ports {
+		_, securityProto, _, err := getProtocolsFromKubernetesProtocol(port.Protocol)
+		if err != nil {
+			return sg, false, err
+		}
+		for j := range sourceAddressPrefixes {
+			ix := i*len(sourceAddressPrefixes) + j
+			securityRuleName := getSecurityRuleName(service, port, sourceAddressPrefixes[j])
+			expectedSecurityRules[ix] = network.SecurityRule{
+				Name: to.StringPtr(securityRuleName),
+				SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
+					Protocol:                 *securityProto,
+					SourcePortRange:          to.StringPtr("*"),
+					DestinationPortRange:     to.StringPtr(strconv.Itoa(int(port.Port))),
+					SourceAddressPrefix:      to.StringPtr(sourceAddressPrefixes[j]),
+					DestinationAddressPrefix: to.StringPtr("*"),
+					Access:    network.SecurityRuleAccessAllow,
+					Direction: network.SecurityRuleDirectionInbound,
+				},
+			}
+		}
+	}
+
+	// update security rules
+	dirtySg := false
+	var updatedRules []network.SecurityRule
+	if sg.SecurityRules != nil {
+		updatedRules = *sg.SecurityRules
+	}
+	// update security rules: remove unwanted
+	for i := len(updatedRules) - 1; i >= 0; i-- {
+		existingRule := updatedRules[i]
+		if serviceOwnsRule(service, *existingRule.Name) {
+			glog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - considering evicting", serviceName, wantLb, *existingRule.Name)
+			keepRule := false
+			if findSecurityRule(expectedSecurityRules, existingRule) {
+				glog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - keeping", serviceName, wantLb, *existingRule.Name)
+				keepRule = true
+			}
+			if !keepRule {
+				glog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - dropping", serviceName, wantLb, *existingRule.Name)
+				updatedRules = append(updatedRules[:i], updatedRules[i+1:]...)
+				dirtySg = true
+			}
+		}
+	}
+	// update security rules: add needed
+	for _, expectedRule := range expectedSecurityRules {
+		foundRule := false
+		if findSecurityRule(updatedRules, expectedRule) {
+			glog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - already exists", serviceName, wantLb, *expectedRule.Name)
+			foundRule = true
+		}
+		if !foundRule {
+			glog.V(10).Infof("reconcile(%s)(%t): sg rule(%s) - adding", serviceName, wantLb, *expectedRule.Name)
+
+			nextAvailablePriority, err := getNextAvailablePriority(updatedRules)
+			if err != nil {
+				return sg, false, err
+			}
+
+			expectedRule.Priority = to.Int32Ptr(nextAvailablePriority)
+			updatedRules = append(updatedRules, expectedRule)
+			dirtySg = true
+		}
+	}
+	if dirtySg {
+		sg.SecurityRules = &updatedRules
+	}
+	return sg, dirtySg, nil
+}
+
+func findProbe(probes []network.Probe, probe network.Probe) bool {
+	for _, existingProbe := range probes {
+		if strings.EqualFold(*existingProbe.Name, *probe.Name) {
+			return true
+		}
+	}
+	return false
+}
+
+func findRule(rules []network.LoadBalancingRule, rule network.LoadBalancingRule) bool {
+	for _, existingRule := range rules {
+		if strings.EqualFold(*existingRule.Name, *rule.Name) {
+			return true
+		}
+	}
+	return false
+}
+
+func findSecurityRule(rules []network.SecurityRule, rule network.SecurityRule) bool {
+	for _, existingRule := range rules {
+		if strings.EqualFold(*existingRule.Name, *rule.Name) {
+			return true
+		}
+	}
+	return false
+}
+
+// This ensures the given VM's Primary NIC's Primary IP Configuration is
+// participating in the specified LoadBalancer Backend Pool.
+func (az *Cloud) ensureHostInPool(serviceName string, nodeName types.NodeName, backendPoolID string) error {
+	var machine compute.VirtualMachine
+	vmName := mapNodeNameToVMName(nodeName)
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("VirtualMachinesClient.Get(%q): start", vmName)
+	machine, err := az.VirtualMachinesClient.Get(az.ResourceGroup, vmName, "")
+	glog.V(10).Infof("VirtualMachinesClient.Get(%q): end", vmName)
+	if err != nil {
+		if az.CloudProviderBackoff {
+			glog.V(2).Infof("ensureHostInPool(%s, %s, %s) backing off", serviceName, nodeName, backendPoolID)
+			machine, err = az.VirtualMachineClientGetWithRetry(az.ResourceGroup, vmName, "")
+			if err != nil {
+				glog.V(2).Infof("ensureHostInPool(%s, %s, %s) abort backoff", serviceName, nodeName, backendPoolID)
+				return err
+			}
+		} else {
+			return err
+		}
+	}
+
+	primaryNicID, err := getPrimaryInterfaceID(machine)
+	if err != nil {
+		return err
+	}
+	nicName, err := getLastSegment(primaryNicID)
+	if err != nil {
+		return err
+	}
+
+	// Check availability set
+	if az.PrimaryAvailabilitySetName != "" {
+		expectedAvailabilitySetName := az.getAvailabilitySetID(az.PrimaryAvailabilitySetName)
+		if !strings.EqualFold(*machine.AvailabilitySet.ID, expectedAvailabilitySetName) {
+			glog.V(3).Infof(
+				"nicupdate(%s): skipping nic (%s) since it is not in the primaryAvailabilitSet(%s)",
+				serviceName, nicName, az.PrimaryAvailabilitySetName)
+			return nil
+		}
+	}
+
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("InterfacesClient.Get(%q): start", nicName)
+	nic, err := az.InterfacesClient.Get(az.ResourceGroup, nicName, "")
+	glog.V(10).Infof("InterfacesClient.Get(%q): end", nicName)
+	if err != nil {
+		return err
+	}
+
+	var primaryIPConfig *network.InterfaceIPConfiguration
+	primaryIPConfig, err = getPrimaryIPConfig(nic)
+	if err != nil {
+		return err
+	}
+
+	foundPool := false
+	newBackendPools := []network.BackendAddressPool{}
+	if primaryIPConfig.LoadBalancerBackendAddressPools != nil {
+		newBackendPools = *primaryIPConfig.LoadBalancerBackendAddressPools
+	}
+	for _, existingPool := range newBackendPools {
+		if strings.EqualFold(backendPoolID, *existingPool.ID) {
+			foundPool = true
+			break
+		}
+	}
+	if !foundPool {
+		newBackendPools = append(newBackendPools,
+			network.BackendAddressPool{
+				ID: to.StringPtr(backendPoolID),
+			})
+
+		primaryIPConfig.LoadBalancerBackendAddressPools = &newBackendPools
+
+		glog.V(3).Infof("nicupdate(%s): nic(%s) - updating", serviceName, nicName)
+		az.operationPollRateLimiter.Accept()
+		glog.V(10).Infof("InterfacesClient.CreateOrUpdate(%q): start", *nic.Name)
+		respChan, errChan := az.InterfacesClient.CreateOrUpdate(az.ResourceGroup, *nic.Name, nic, nil)
+		resp := <-respChan
+		err := <-errChan
+		glog.V(10).Infof("InterfacesClient.CreateOrUpdate(%q): end", *nic.Name)
+		if az.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
+			glog.V(2).Infof("nicupdate(%s) backing off: nic(%s) - updating, err=%v", serviceName, nicName, err)
+			retryErr := az.CreateOrUpdateInterfaceWithRetry(nic)
+			if retryErr != nil {
+				err = retryErr
+				glog.V(2).Infof("nicupdate(%s) abort backoff: nic(%s) - updating", serviceName, nicName)
+			}
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Check if service requires an internal load balancer.
+func requiresInternalLoadBalancer(service *v1.Service) bool {
+	if l, ok := service.Annotations[ServiceAnnotationLoadBalancerInternal]; ok {
+		return l == "true"
+	}
+
+	return false
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/azure_managedDiskController.go

@@ -0,0 +1,129 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"path"
+	"strings"
+
+	"github.com/Azure/azure-sdk-for-go/arm/disk"
+	storage "github.com/Azure/azure-sdk-for-go/arm/storage"
+	"github.com/golang/glog"
+	kwait "k8s.io/apimachinery/pkg/util/wait"
+)
+
+//ManagedDiskController : managed disk controller struct
+type ManagedDiskController struct {
+	common *controllerCommon
+}
+
+func newManagedDiskController(common *controllerCommon) (*ManagedDiskController, error) {
+	return &ManagedDiskController{common: common}, nil
+}
+
+//CreateManagedDisk : create managed disk
+func (c *ManagedDiskController) CreateManagedDisk(diskName string, storageAccountType storage.SkuName, sizeGB int, tags map[string]string) (string, error) {
+	glog.V(4).Infof("azureDisk - creating new managed Name:%s StorageAccountType:%s Size:%v", diskName, storageAccountType, sizeGB)
+
+	newTags := make(map[string]*string)
+	azureDDTag := "kubernetes-azure-dd"
+	newTags["created-by"] = &azureDDTag
+
+	// insert original tags to newTags
+	if tags != nil {
+		for k, v := range tags {
+			// Azure won't allow / (forward slash) in tags
+			newKey := strings.Replace(k, "/", "-", -1)
+			newValue := strings.Replace(v, "/", "-", -1)
+			newTags[newKey] = &newValue
+		}
+	}
+
+	diskSizeGB := int32(sizeGB)
+	model := disk.Model{
+		Location: &c.common.location,
+		Tags:     &newTags,
+		Properties: &disk.Properties{
+			AccountType:  disk.StorageAccountTypes(storageAccountType),
+			DiskSizeGB:   &diskSizeGB,
+			CreationData: &disk.CreationData{CreateOption: disk.Empty},
+		}}
+	cancel := make(chan struct{})
+	respChan, errChan := c.common.cloud.DisksClient.CreateOrUpdate(c.common.resourceGroup, diskName, model, cancel)
+	<-respChan
+	err := <-errChan
+	if err != nil {
+		return "", err
+	}
+
+	diskID := ""
+
+	err = kwait.ExponentialBackoff(defaultBackOff, func() (bool, error) {
+		provisonState, id, err := c.getDisk(diskName)
+		diskID = id
+		// We are waiting for provisioningState==Succeeded
+		// We don't want to hand-off managed disks to k8s while they are
+		//still being provisioned, this is to avoid some race conditions
+		if err != nil {
+			return false, err
+		}
+		if strings.ToLower(provisonState) == "succeeded" {
+			return true, nil
+		}
+		return false, nil
+	})
+
+	if err != nil {
+		glog.V(2).Infof("azureDisk - created new MD Name:%s StorageAccountType:%s Size:%v but was unable to confirm provisioningState in poll process", diskName, storageAccountType, sizeGB)
+	} else {
+		glog.V(2).Infof("azureDisk - created new MD Name:%s StorageAccountType:%s Size:%v", diskName, storageAccountType, sizeGB)
+	}
+
+	return diskID, nil
+}
+
+//DeleteManagedDisk : delete managed disk
+func (c *ManagedDiskController) DeleteManagedDisk(diskURI string) error {
+	diskName := path.Base(diskURI)
+	cancel := make(chan struct{})
+	respChan, errChan := c.common.cloud.DisksClient.Delete(c.common.resourceGroup, diskName, cancel)
+	<-respChan
+	err := <-errChan
+	if err != nil {
+		return err
+	}
+	// We don't need poll here, k8s will immediatly stop referencing the disk
+	// the disk will be evantually deleted - cleanly - by ARM
+
+	glog.V(2).Infof("azureDisk - deleted a managed disk: %s", diskURI)
+
+	return nil
+}
+
+// return: disk provisionState, diskID, error
+func (c *ManagedDiskController) getDisk(diskName string) (string, string, error) {
+	result, err := c.common.cloud.DisksClient.Get(c.common.resourceGroup, diskName)
+	if err != nil {
+		return "", "", err
+	}
+
+	if result.Properties != nil && (*result.Properties).ProvisioningState != nil {
+		return *(*result.Properties).ProvisioningState, *result.ID, nil
+	}
+
+	return "", "", err
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/azure_routes.go

@@ -0,0 +1,184 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"fmt"
+
+	"k8s.io/kubernetes/pkg/cloudprovider"
+
+	"github.com/Azure/azure-sdk-for-go/arm/network"
+	"github.com/Azure/go-autorest/autorest/to"
+	"github.com/golang/glog"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// ListRoutes lists all managed routes that belong to the specified clusterName
+func (az *Cloud) ListRoutes(clusterName string) (routes []*cloudprovider.Route, err error) {
+	glog.V(10).Infof("list: START clusterName=%q", clusterName)
+	routeTable, existsRouteTable, err := az.getRouteTable()
+	if err != nil {
+		return nil, err
+	}
+	if !existsRouteTable {
+		return []*cloudprovider.Route{}, nil
+	}
+
+	var kubeRoutes []*cloudprovider.Route
+	if routeTable.Routes != nil {
+		kubeRoutes = make([]*cloudprovider.Route, len(*routeTable.Routes))
+		for i, route := range *routeTable.Routes {
+			instance := mapRouteNameToNodeName(*route.Name)
+			cidr := *route.AddressPrefix
+			glog.V(10).Infof("list: * instance=%q, cidr=%q", instance, cidr)
+
+			kubeRoutes[i] = &cloudprovider.Route{
+				Name:            *route.Name,
+				TargetNode:      instance,
+				DestinationCIDR: cidr,
+			}
+		}
+	}
+
+	glog.V(10).Info("list: FINISH")
+	return kubeRoutes, nil
+}
+
+// CreateRoute creates the described managed route
+// route.Name will be ignored, although the cloud-provider may use nameHint
+// to create a more user-meaningful name.
+func (az *Cloud) CreateRoute(clusterName string, nameHint string, kubeRoute *cloudprovider.Route) error {
+	glog.V(2).Infof("create: creating route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR)
+
+	routeTable, existsRouteTable, err := az.getRouteTable()
+	if err != nil {
+		glog.V(2).Infof("create error: couldn't get routetable. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR)
+		return err
+	}
+	if !existsRouteTable {
+		routeTable = network.RouteTable{
+			Name:                       to.StringPtr(az.RouteTableName),
+			Location:                   to.StringPtr(az.Location),
+			RouteTablePropertiesFormat: &network.RouteTablePropertiesFormat{},
+		}
+
+		glog.V(3).Infof("create: creating routetable. routeTableName=%q", az.RouteTableName)
+		az.operationPollRateLimiter.Accept()
+		glog.V(10).Infof("RouteTablesClient.CreateOrUpdate(%q): start", az.RouteTableName)
+		respChan, errChan := az.RouteTablesClient.CreateOrUpdate(az.ResourceGroup, az.RouteTableName, routeTable, nil)
+		resp := <-respChan
+		err := <-errChan
+		glog.V(10).Infof("RouteTablesClient.CreateOrUpdate(%q): end", az.RouteTableName)
+		if az.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
+			glog.V(2).Infof("create backing off: creating routetable. routeTableName=%q", az.RouteTableName)
+			retryErr := az.CreateOrUpdateRouteTableWithRetry(routeTable)
+			if retryErr != nil {
+				err = retryErr
+				glog.V(2).Infof("create abort backoff: creating routetable. routeTableName=%q", az.RouteTableName)
+			}
+		}
+		if err != nil {
+			return err
+		}
+
+		glog.V(10).Infof("RouteTablesClient.Get(%q): start", az.RouteTableName)
+		routeTable, err = az.RouteTablesClient.Get(az.ResourceGroup, az.RouteTableName, "")
+		glog.V(10).Infof("RouteTablesClient.Get(%q): end", az.RouteTableName)
+		if err != nil {
+			return err
+		}
+	}
+
+	targetIP, err := az.getIPForMachine(kubeRoute.TargetNode)
+	if err != nil {
+		return err
+	}
+
+	routeName := mapNodeNameToRouteName(kubeRoute.TargetNode)
+	route := network.Route{
+		Name: to.StringPtr(routeName),
+		RoutePropertiesFormat: &network.RoutePropertiesFormat{
+			AddressPrefix:    to.StringPtr(kubeRoute.DestinationCIDR),
+			NextHopType:      network.RouteNextHopTypeVirtualAppliance,
+			NextHopIPAddress: to.StringPtr(targetIP),
+		},
+	}
+
+	glog.V(3).Infof("create: creating route: instance=%q cidr=%q", kubeRoute.TargetNode, kubeRoute.DestinationCIDR)
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("RoutesClient.CreateOrUpdate(%q): start", az.RouteTableName)
+	respChan, errChan := az.RoutesClient.CreateOrUpdate(az.ResourceGroup, az.RouteTableName, *route.Name, route, nil)
+	resp := <-respChan
+	err = <-errChan
+	glog.V(10).Infof("RoutesClient.CreateOrUpdate(%q): end", az.RouteTableName)
+	if az.CloudProviderBackoff && shouldRetryAPIRequest(resp.Response, err) {
+		glog.V(2).Infof("create backing off: creating route: instance=%q cidr=%q", kubeRoute.TargetNode, kubeRoute.DestinationCIDR)
+		retryErr := az.CreateOrUpdateRouteWithRetry(route)
+		if retryErr != nil {
+			err = retryErr
+			glog.V(2).Infof("create abort backoff: creating route: instance=%q cidr=%q", kubeRoute.TargetNode, kubeRoute.DestinationCIDR)
+		}
+	}
+	if err != nil {
+		return err
+	}
+
+	glog.V(2).Infof("create: route created. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR)
+	return nil
+}
+
+// DeleteRoute deletes the specified managed route
+// Route should be as returned by ListRoutes
+func (az *Cloud) DeleteRoute(clusterName string, kubeRoute *cloudprovider.Route) error {
+	glog.V(2).Infof("delete: deleting route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR)
+
+	routeName := mapNodeNameToRouteName(kubeRoute.TargetNode)
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("RoutesClient.Delete(%q): start", az.RouteTableName)
+	respChan, errChan := az.RoutesClient.Delete(az.ResourceGroup, az.RouteTableName, routeName, nil)
+	resp := <-respChan
+	err := <-errChan
+	glog.V(10).Infof("RoutesClient.Delete(%q): end", az.RouteTableName)
+
+	if az.CloudProviderBackoff && shouldRetryAPIRequest(resp, err) {
+		glog.V(2).Infof("delete backing off: deleting route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR)
+		retryErr := az.DeleteRouteWithRetry(routeName)
+		if retryErr != nil {
+			err = retryErr
+			glog.V(2).Infof("delete abort backoff: deleting route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR)
+		}
+	}
+	if err != nil {
+		return err
+	}
+
+	glog.V(2).Infof("delete: route deleted. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR)
+	return nil
+}
+
+// This must be kept in sync with mapRouteNameToNodeName.
+// These two functions enable stashing the instance name in the route
+// and then retrieving it later when listing. This is needed because
+// Azure does not let you put tags/descriptions on the Route itself.
+func mapNodeNameToRouteName(nodeName types.NodeName) string {
+	return fmt.Sprintf("%s", nodeName)
+}
+
+// Used with mapNodeNameToRouteName. See comment on mapNodeNameToRouteName.
+func mapRouteNameToNodeName(routeName string) types.NodeName {
+	return types.NodeName(fmt.Sprintf("%s", routeName))
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/azure_storage.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"fmt"
+
+	"github.com/golang/glog"
+)
+
+// CreateFileShare creates a file share, using a matching storage account
+func (az *Cloud) CreateFileShare(name, storageAccount, storageType, location string, requestGB int) (string, string, error) {
+	var err error
+	accounts := []accountWithLocation{}
+	if len(storageAccount) > 0 {
+		accounts = append(accounts, accountWithLocation{Name: storageAccount})
+	} else {
+		// find a storage account
+		accounts, err = az.getStorageAccounts()
+		if err != nil {
+			// TODO: create a storage account and container
+			return "", "", err
+		}
+	}
+	for _, account := range accounts {
+		glog.V(4).Infof("account %s type %s location %s", account.Name, account.StorageType, account.Location)
+		if ((storageType == "" || account.StorageType == storageType) && (location == "" || account.Location == location)) || len(storageAccount) > 0 {
+			// find the access key with this account
+			key, err := az.getStorageAccesskey(account.Name)
+			if err != nil {
+				glog.V(2).Infof("no key found for storage account %s", account.Name)
+				continue
+			}
+
+			err = az.createFileShare(account.Name, key, name, requestGB)
+			if err != nil {
+				glog.V(2).Infof("failed to create share %s in account %s: %v", name, account.Name, err)
+				continue
+			}
+			glog.V(4).Infof("created share %s in account %s", name, account.Name)
+			return account.Name, key, err
+		}
+	}
+	return "", "", fmt.Errorf("failed to find a matching storage account")
+}
+
+// DeleteFileShare deletes a file share using storage account name and key
+func (az *Cloud) DeleteFileShare(accountName, key, name string) error {
+	err := az.deleteFileShare(accountName, key, name)
+	if err != nil {
+		return err
+	}
+	glog.V(4).Infof("share %s deleted", name)
+	return nil
+
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/azure_storageaccount.go

@@ -0,0 +1,85 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/golang/glog"
+)
+
+type accountWithLocation struct {
+	Name, StorageType, Location string
+}
+
+// getStorageAccounts gets the storage accounts' name, type, location in a resource group
+func (az *Cloud) getStorageAccounts() ([]accountWithLocation, error) {
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("StorageAccountClient.ListByResourceGroup(%v): start", az.ResourceGroup)
+	result, err := az.StorageAccountClient.ListByResourceGroup(az.ResourceGroup)
+	glog.V(10).Infof("StorageAccountClient.ListByResourceGroup(%v): end", az.ResourceGroup)
+	if err != nil {
+		return nil, err
+	}
+	if result.Value == nil {
+		return nil, fmt.Errorf("no storage accounts from resource group %s", az.ResourceGroup)
+	}
+
+	accounts := []accountWithLocation{}
+	for _, acct := range *result.Value {
+		if acct.Name != nil {
+			name := *acct.Name
+			loc := ""
+			if acct.Location != nil {
+				loc = *acct.Location
+			}
+			storageType := ""
+			if acct.Sku != nil {
+				storageType = string((*acct.Sku).Name)
+			}
+			accounts = append(accounts, accountWithLocation{Name: name, StorageType: storageType, Location: loc})
+		}
+	}
+
+	return accounts, nil
+}
+
+// getStorageAccesskey gets the storage account access key
+func (az *Cloud) getStorageAccesskey(account string) (string, error) {
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("StorageAccountClient.ListKeys(%q): start", account)
+	result, err := az.StorageAccountClient.ListKeys(az.ResourceGroup, account)
+	glog.V(10).Infof("StorageAccountClient.ListKeys(%q): end", account)
+	if err != nil {
+		return "", err
+	}
+	if result.Keys == nil {
+		return "", fmt.Errorf("empty keys")
+	}
+
+	for _, k := range *result.Keys {
+		if k.Value != nil && *k.Value != "" {
+			v := *k.Value
+			if ind := strings.LastIndex(v, " "); ind >= 0 {
+				v = v[(ind + 1):]
+			}
+			return v, nil
+		}
+	}
+	return "", fmt.Errorf("no valid keys")
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/azure_test.go

@@ -0,0 +1,889 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"strings"
+	"testing"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	serviceapi "k8s.io/kubernetes/pkg/api/v1/service"
+
+	"github.com/Azure/azure-sdk-for-go/arm/network"
+	"github.com/Azure/go-autorest/autorest/to"
+)
+
+var testClusterName = "testCluster"
+
+// Test additional of a new service/port.
+func TestReconcileLoadBalancerAddPort(t *testing.T) {
+	az := getTestCloud()
+	svc := getTestService("servicea", v1.ProtocolTCP, 80)
+	configProperties := getTestPublicFipConfigurationProperties()
+	lb := getTestLoadBalancer()
+	nodes := []*v1.Node{}
+
+	svc.Spec.Ports = append(svc.Spec.Ports, v1.ServicePort{
+		Name:     fmt.Sprintf("port-udp-%d", 1234),
+		Protocol: v1.ProtocolUDP,
+		Port:     1234,
+		NodePort: getBackendPort(1234),
+	})
+
+	lb, updated, err := az.reconcileLoadBalancer(lb, &configProperties, testClusterName, &svc, nodes)
+	if err != nil {
+		t.Errorf("Unexpected error: %q", err)
+	}
+
+	if !updated {
+		t.Error("Expected the loadbalancer to need an update")
+	}
+
+	// ensure we got a frontend ip configuration
+	if len(*lb.FrontendIPConfigurations) != 1 {
+		t.Error("Expected the loadbalancer to have a frontend ip configuration")
+	}
+
+	validateLoadBalancer(t, lb, svc)
+}
+
+func TestReconcileLoadBalancerNodeHealth(t *testing.T) {
+	az := getTestCloud()
+	svc := getTestService("servicea", v1.ProtocolTCP, 80)
+	svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal
+	svc.Spec.HealthCheckNodePort = int32(32456)
+	configProperties := getTestPublicFipConfigurationProperties()
+	lb := getTestLoadBalancer()
+
+	nodes := []*v1.Node{}
+
+	lb, updated, err := az.reconcileLoadBalancer(lb, &configProperties, testClusterName, &svc, nodes)
+	if err != nil {
+		t.Errorf("Unexpected error: %q", err)
+	}
+
+	if !updated {
+		t.Error("Expected the loadbalancer to need an update")
+	}
+
+	// ensure we got a frontend ip configuration
+	if len(*lb.FrontendIPConfigurations) != 1 {
+		t.Error("Expected the loadbalancer to have a frontend ip configuration")
+	}
+
+	validateLoadBalancer(t, lb, svc)
+}
+
+// Test removing all services results in removing the frontend ip configuration
+func TestReconcileLoadBalancerRemoveService(t *testing.T) {
+	az := getTestCloud()
+	svc := getTestService("servicea", v1.ProtocolTCP, 80, 443)
+	lb := getTestLoadBalancer()
+	configProperties := getTestPublicFipConfigurationProperties()
+	nodes := []*v1.Node{}
+
+	lb, updated, err := az.reconcileLoadBalancer(lb, &configProperties, testClusterName, &svc, nodes)
+	if err != nil {
+		t.Errorf("Unexpected error: %q", err)
+	}
+	validateLoadBalancer(t, lb, svc)
+
+	lb, updated, err = az.reconcileLoadBalancer(lb, nil, testClusterName, &svc, nodes)
+	if err != nil {
+		t.Errorf("Unexpected error: %q", err)
+	}
+
+	if !updated {
+		t.Error("Expected the loadbalancer to need an update")
+	}
+
+	// ensure we abandoned the frontend ip configuration
+	if len(*lb.FrontendIPConfigurations) != 0 {
+		t.Error("Expected the loadbalancer to have no frontend ip configuration")
+	}
+
+	validateLoadBalancer(t, lb)
+}
+
+// Test removing all service ports results in removing the frontend ip configuration
+func TestReconcileLoadBalancerRemoveAllPortsRemovesFrontendConfig(t *testing.T) {
+	az := getTestCloud()
+	svc := getTestService("servicea", v1.ProtocolTCP, 80)
+	lb := getTestLoadBalancer()
+	configProperties := getTestPublicFipConfigurationProperties()
+	nodes := []*v1.Node{}
+
+	lb, updated, err := az.reconcileLoadBalancer(lb, &configProperties, testClusterName, &svc, nodes)
+	if err != nil {
+		t.Errorf("Unexpected error: %q", err)
+	}
+	validateLoadBalancer(t, lb, svc)
+
+	svcUpdated := getTestService("servicea", v1.ProtocolTCP)
+	lb, updated, err = az.reconcileLoadBalancer(lb, nil, testClusterName, &svcUpdated, nodes)
+	if err != nil {
+		t.Errorf("Unexpected error: %q", err)
+	}
+
+	if !updated {
+		t.Error("Expected the loadbalancer to need an update")
+	}
+
+	// ensure we abandoned the frontend ip configuration
+	if len(*lb.FrontendIPConfigurations) != 0 {
+		t.Error("Expected the loadbalancer to have no frontend ip configuration")
+	}
+
+	validateLoadBalancer(t, lb, svcUpdated)
+}
+
+// Test removal of a port from an existing service.
+func TestReconcileLoadBalancerRemovesPort(t *testing.T) {
+	az := getTestCloud()
+	svc := getTestService("servicea", v1.ProtocolTCP, 80, 443)
+	configProperties := getTestPublicFipConfigurationProperties()
+	nodes := []*v1.Node{}
+
+	existingLoadBalancer := getTestLoadBalancer(svc)
+
+	svcUpdated := getTestService("servicea", v1.ProtocolTCP, 80)
+	updatedLoadBalancer, _, err := az.reconcileLoadBalancer(existingLoadBalancer, &configProperties, testClusterName, &svcUpdated, nodes)
+	if err != nil {
+		t.Errorf("Unexpected error: %q", err)
+	}
+
+	validateLoadBalancer(t, updatedLoadBalancer, svcUpdated)
+}
+
+// Test reconciliation of multiple services on same port
+func TestReconcileLoadBalancerMultipleServices(t *testing.T) {
+	az := getTestCloud()
+	svc1 := getTestService("servicea", v1.ProtocolTCP, 80, 443)
+	svc2 := getTestService("serviceb", v1.ProtocolTCP, 80)
+	configProperties := getTestPublicFipConfigurationProperties()
+	nodes := []*v1.Node{}
+
+	existingLoadBalancer := getTestLoadBalancer()
+
+	updatedLoadBalancer, _, err := az.reconcileLoadBalancer(existingLoadBalancer, &configProperties, testClusterName, &svc1, nodes)
+	if err != nil {
+		t.Errorf("Unexpected error: %q", err)
+	}
+
+	updatedLoadBalancer, _, err = az.reconcileLoadBalancer(updatedLoadBalancer, &configProperties, testClusterName, &svc2, nodes)
+	if err != nil {
+		t.Errorf("Unexpected error: %q", err)
+	}
+
+	validateLoadBalancer(t, updatedLoadBalancer, svc1, svc2)
+}
+
+func TestReconcileSecurityGroupNewServiceAddsPort(t *testing.T) {
+	az := getTestCloud()
+	svc1 := getTestService("serviceea", v1.ProtocolTCP, 80)
+
+	sg := getTestSecurityGroup()
+
+	sg, _, err := az.reconcileSecurityGroup(sg, testClusterName, &svc1, true)
+	if err != nil {
+		t.Errorf("Unexpected error: %q", err)
+	}
+
+	validateSecurityGroup(t, sg, svc1)
+}
+
+func TestReconcileSecurityGroupNewInternalServiceAddsPort(t *testing.T) {
+	az := getTestCloud()
+	svc1 := getInternalTestService("serviceea", 80)
+
+	sg := getTestSecurityGroup()
+
+	sg, _, err := az.reconcileSecurityGroup(sg, testClusterName, &svc1, true)
+	if err != nil {
+		t.Errorf("Unexpected error: %q", err)
+	}
+
+	validateSecurityGroup(t, sg, svc1)
+}
+
+func TestReconcileSecurityGroupRemoveService(t *testing.T) {
+	service1 := getTestService("servicea", v1.ProtocolTCP, 81)
+	service2 := getTestService("serviceb", v1.ProtocolTCP, 82)
+
+	sg := getTestSecurityGroup(service1, service2)
+
+	validateSecurityGroup(t, sg, service1, service2)
+	az := getTestCloud()
+	sg, _, err := az.reconcileSecurityGroup(sg, testClusterName, &service1, false)
+	if err != nil {
+		t.Errorf("Unexpected error: %q", err)
+	}
+
+	validateSecurityGroup(t, sg, service2)
+}
+
+func TestReconcileSecurityGroupRemoveServiceRemovesPort(t *testing.T) {
+	az := getTestCloud()
+	svc := getTestService("servicea", v1.ProtocolTCP, 80, 443)
+
+	sg := getTestSecurityGroup(svc)
+
+	svcUpdated := getTestService("servicea", v1.ProtocolTCP, 80)
+	sg, _, err := az.reconcileSecurityGroup(sg, testClusterName, &svcUpdated, true)
+	if err != nil {
+		t.Errorf("Unexpected error: %q", err)
+	}
+
+	validateSecurityGroup(t, sg, svcUpdated)
+}
+
+func TestReconcileSecurityWithSourceRanges(t *testing.T) {
+	az := getTestCloud()
+	svc := getTestService("servicea", v1.ProtocolTCP, 80, 443)
+	svc.Spec.LoadBalancerSourceRanges = []string{
+		"192.168.0.0/24",
+		"10.0.0.0/32",
+	}
+
+	sg := getTestSecurityGroup(svc)
+	sg, _, err := az.reconcileSecurityGroup(sg, testClusterName, &svc, true)
+	if err != nil {
+		t.Errorf("Unexpected error: %q", err)
+	}
+
+	validateSecurityGroup(t, sg, svc)
+}
+
+func getTestCloud() *Cloud {
+	return &Cloud{
+		Config: Config{
+			TenantID:          "tenant",
+			SubscriptionID:    "subscription",
+			ResourceGroup:     "rg",
+			Location:          "westus",
+			VnetName:          "vnet",
+			SubnetName:        "subnet",
+			SecurityGroupName: "nsg",
+			RouteTableName:    "rt",
+		},
+	}
+}
+
+func getBackendPort(port int32) int32 {
+	return port + 10000
+}
+
+func getTestPublicFipConfigurationProperties() network.FrontendIPConfigurationPropertiesFormat {
+	return network.FrontendIPConfigurationPropertiesFormat{
+		PublicIPAddress: &network.PublicIPAddress{ID: to.StringPtr("/this/is/a/public/ip/address/id")},
+	}
+}
+
+func getTestService(identifier string, proto v1.Protocol, requestedPorts ...int32) v1.Service {
+	ports := []v1.ServicePort{}
+	for _, port := range requestedPorts {
+		ports = append(ports, v1.ServicePort{
+			Name:     fmt.Sprintf("port-tcp-%d", port),
+			Protocol: proto,
+			Port:     port,
+			NodePort: getBackendPort(port),
+		})
+	}
+
+	svc := v1.Service{
+		Spec: v1.ServiceSpec{
+			Type:  v1.ServiceTypeLoadBalancer,
+			Ports: ports,
+		},
+	}
+	svc.Name = identifier
+	svc.Namespace = "default"
+	svc.UID = types.UID(identifier)
+	svc.Annotations = make(map[string]string)
+
+	return svc
+}
+
+func getInternalTestService(identifier string, requestedPorts ...int32) v1.Service {
+	svc := getTestService(identifier, v1.ProtocolTCP, requestedPorts...)
+	svc.Annotations[ServiceAnnotationLoadBalancerInternal] = "true"
+
+	return svc
+}
+
+func getTestLoadBalancer(services ...v1.Service) network.LoadBalancer {
+	rules := []network.LoadBalancingRule{}
+	probes := []network.Probe{}
+
+	for _, service := range services {
+		for _, port := range service.Spec.Ports {
+			ruleName := getLoadBalancerRuleName(&service, port)
+			rules = append(rules, network.LoadBalancingRule{
+				Name: to.StringPtr(ruleName),
+				LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{
+					FrontendPort: to.Int32Ptr(port.Port),
+					BackendPort:  to.Int32Ptr(port.Port),
+				},
+			})
+			probes = append(probes, network.Probe{
+				Name: to.StringPtr(ruleName),
+				ProbePropertiesFormat: &network.ProbePropertiesFormat{
+					Port: to.Int32Ptr(port.NodePort),
+				},
+			})
+		}
+	}
+
+	lb := network.LoadBalancer{
+		LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{
+			LoadBalancingRules: &rules,
+			Probes:             &probes,
+		},
+	}
+
+	return lb
+}
+
+func getServiceSourceRanges(service *v1.Service) []string {
+	if len(service.Spec.LoadBalancerSourceRanges) == 0 {
+		if !requiresInternalLoadBalancer(service) {
+			return []string{"Internet"}
+		}
+	}
+
+	return service.Spec.LoadBalancerSourceRanges
+}
+
+func getTestSecurityGroup(services ...v1.Service) network.SecurityGroup {
+	rules := []network.SecurityRule{}
+
+	for _, service := range services {
+		for _, port := range service.Spec.Ports {
+			sources := getServiceSourceRanges(&service)
+			for _, src := range sources {
+				ruleName := getSecurityRuleName(&service, port, src)
+				rules = append(rules, network.SecurityRule{
+					Name: to.StringPtr(ruleName),
+					SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
+						SourceAddressPrefix:  to.StringPtr(src),
+						DestinationPortRange: to.StringPtr(fmt.Sprintf("%d", port.Port)),
+					},
+				})
+			}
+		}
+	}
+
+	sg := network.SecurityGroup{
+		SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{
+			SecurityRules: &rules,
+		},
+	}
+
+	return sg
+}
+
+func validateLoadBalancer(t *testing.T, loadBalancer network.LoadBalancer, services ...v1.Service) {
+	expectedRuleCount := 0
+	expectedFrontendIPCount := 0
+	expectedProbeCount := 0
+	for _, svc := range services {
+		if len(svc.Spec.Ports) > 0 {
+			expectedFrontendIPCount++
+		}
+		for _, wantedRule := range svc.Spec.Ports {
+			expectedRuleCount++
+			wantedRuleName := getLoadBalancerRuleName(&svc, wantedRule)
+			foundRule := false
+			for _, actualRule := range *loadBalancer.LoadBalancingRules {
+				if strings.EqualFold(*actualRule.Name, wantedRuleName) &&
+					*actualRule.FrontendPort == wantedRule.Port &&
+					*actualRule.BackendPort == wantedRule.Port {
+					foundRule = true
+					break
+				}
+			}
+			if !foundRule {
+				t.Errorf("Expected load balancer rule but didn't find it: %q", wantedRuleName)
+			}
+
+			// if UDP rule, there is no probe
+			if wantedRule.Protocol == v1.ProtocolUDP {
+				continue
+			}
+
+			expectedProbeCount++
+			foundProbe := false
+			if serviceapi.NeedsHealthCheck(&svc) {
+				path, port := serviceapi.GetServiceHealthCheckPathPort(&svc)
+				for _, actualProbe := range *loadBalancer.Probes {
+					if strings.EqualFold(*actualProbe.Name, wantedRuleName) &&
+						*actualProbe.Port == port &&
+						*actualProbe.RequestPath == path &&
+						actualProbe.Protocol == network.ProbeProtocolHTTP {
+						foundProbe = true
+						break
+					}
+				}
+			} else {
+				for _, actualProbe := range *loadBalancer.Probes {
+					if strings.EqualFold(*actualProbe.Name, wantedRuleName) &&
+						*actualProbe.Port == wantedRule.NodePort {
+						foundProbe = true
+						break
+					}
+				}
+			}
+			if !foundProbe {
+				for _, actualProbe := range *loadBalancer.Probes {
+					t.Logf("Probe: %s %d", *actualProbe.Name, *actualProbe.Port)
+				}
+				t.Errorf("Expected loadbalancer probe but didn't find it: %q", wantedRuleName)
+			}
+		}
+	}
+
+	frontendIPCount := len(*loadBalancer.FrontendIPConfigurations)
+	if frontendIPCount != expectedFrontendIPCount {
+		t.Errorf("Expected the loadbalancer to have %d frontend IPs. Found %d.\n%v", expectedFrontendIPCount, frontendIPCount, loadBalancer.FrontendIPConfigurations)
+	}
+
+	lenRules := len(*loadBalancer.LoadBalancingRules)
+	if lenRules != expectedRuleCount {
+		t.Errorf("Expected the loadbalancer to have %d rules. Found %d.\n%v", expectedRuleCount, lenRules, loadBalancer.LoadBalancingRules)
+	}
+
+	lenProbes := len(*loadBalancer.Probes)
+	if lenProbes != expectedProbeCount {
+		t.Errorf("Expected the loadbalancer to have %d probes. Found %d.", expectedRuleCount, lenProbes)
+	}
+}
+
+func validateSecurityGroup(t *testing.T, securityGroup network.SecurityGroup, services ...v1.Service) {
+	expectedRuleCount := 0
+	for _, svc := range services {
+		for _, wantedRule := range svc.Spec.Ports {
+			sources := getServiceSourceRanges(&svc)
+			for _, source := range sources {
+				wantedRuleName := getSecurityRuleName(&svc, wantedRule, source)
+				expectedRuleCount++
+				foundRule := false
+				for _, actualRule := range *securityGroup.SecurityRules {
+					if strings.EqualFold(*actualRule.Name, wantedRuleName) &&
+						*actualRule.SourceAddressPrefix == source &&
+						*actualRule.DestinationPortRange == fmt.Sprintf("%d", wantedRule.Port) {
+						foundRule = true
+						break
+					}
+				}
+				if !foundRule {
+					t.Errorf("Expected security group rule but didn't find it: %q", wantedRuleName)
+				}
+			}
+		}
+	}
+
+	lenRules := len(*securityGroup.SecurityRules)
+	if lenRules != expectedRuleCount {
+		t.Errorf("Expected the loadbalancer to have %d rules. Found %d.\n", expectedRuleCount, lenRules)
+	}
+}
+
+func TestSecurityRulePriorityPicksNextAvailablePriority(t *testing.T) {
+	rules := []network.SecurityRule{}
+
+	var expectedPriority int32 = loadBalancerMinimumPriority + 50
+
+	var i int32
+	for i = loadBalancerMinimumPriority; i < expectedPriority; i++ {
+		rules = append(rules, network.SecurityRule{
+			SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
+				Priority: to.Int32Ptr(i),
+			},
+		})
+	}
+
+	priority, err := getNextAvailablePriority(rules)
+	if err != nil {
+		t.Errorf("Unexpectected error: %q", err)
+	}
+
+	if priority != expectedPriority {
+		t.Errorf("Expected priority %d. Got priority %d.", expectedPriority, priority)
+	}
+}
+
+func TestSecurityRulePriorityFailsIfExhausted(t *testing.T) {
+	rules := []network.SecurityRule{}
+
+	var i int32
+	for i = loadBalancerMinimumPriority; i < loadBalancerMaximumPriority; i++ {
+		rules = append(rules, network.SecurityRule{
+			SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{
+				Priority: to.Int32Ptr(i),
+			},
+		})
+	}
+
+	_, err := getNextAvailablePriority(rules)
+	if err == nil {
+		t.Error("Expectected an error. There are no priority levels left.")
+	}
+}
+
+func TestProtocolTranslationTCP(t *testing.T) {
+	proto := v1.ProtocolTCP
+	transportProto, securityGroupProto, probeProto, err := getProtocolsFromKubernetesProtocol(proto)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if *transportProto != network.TransportProtocolTCP {
+		t.Errorf("Expected TCP LoadBalancer Rule Protocol. Got %v", transportProto)
+	}
+	if *securityGroupProto != network.SecurityRuleProtocolTCP {
+		t.Errorf("Expected TCP SecurityGroup Protocol. Got %v", transportProto)
+	}
+	if *probeProto != network.ProbeProtocolTCP {
+		t.Errorf("Expected TCP LoadBalancer Probe Protocol. Got %v", transportProto)
+	}
+}
+
+func TestProtocolTranslationUDP(t *testing.T) {
+	proto := v1.ProtocolUDP
+	transportProto, securityGroupProto, probeProto, _ := getProtocolsFromKubernetesProtocol(proto)
+	if *transportProto != network.TransportProtocolUDP {
+		t.Errorf("Expected UDP LoadBalancer Rule Protocol. Got %v", transportProto)
+	}
+	if *securityGroupProto != network.SecurityRuleProtocolUDP {
+		t.Errorf("Expected UDP SecurityGroup Protocol. Got %v", transportProto)
+	}
+	if probeProto != nil {
+		t.Errorf("Expected UDP LoadBalancer Probe Protocol. Got %v", transportProto)
+	}
+}
+
+// Test Configuration deserialization (json)
+func TestNewCloudFromJSON(t *testing.T) {
+	config := `{
+		"tenantId": "--tenant-id--",
+		"subscriptionId": "--subscription-id--",
+		"aadClientId": "--aad-client-id--",
+		"aadClientSecret": "--aad-client-secret--",
+		"aadClientCertPath": "--aad-client-cert-path--",
+		"aadClientCertPassword": "--aad-client-cert-password--",
+		"resourceGroup": "--resource-group--",
+		"location": "--location--",
+		"subnetName": "--subnet-name--",
+		"securityGroupName": "--security-group-name--",
+		"vnetName": "--vnet-name--",
+		"routeTableName": "--route-table-name--",
+		"primaryAvailabilitySetName": "--primary-availability-set-name--",
+		"cloudProviderBackoff": true,
+		"cloudProviderBackoffRetries": 6,
+		"cloudProviderBackoffExponent": 1.5,
+		"cloudProviderBackoffDuration": 5,
+		"cloudProviderBackoffJitter": 1.0,
+		"cloudProviderRatelimit": true,
+		"cloudProviderRateLimitQPS": 0.5,
+		"cloudProviderRateLimitBucket": 5
+	}`
+	validateConfig(t, config)
+}
+
+// Test Backoff and Rate Limit defaults (json)
+func TestCloudDefaultConfigFromJSON(t *testing.T) {
+	config := `{
+                "aadClientId": "--aad-client-id--",
+                "aadClientSecret": "--aad-client-secret--"
+        }`
+
+	validateEmptyConfig(t, config)
+}
+
+// Test Backoff and Rate Limit defaults (yaml)
+func TestCloudDefaultConfigFromYAML(t *testing.T) {
+	config := `
+aadClientId: --aad-client-id--
+aadClientSecret: --aad-client-secret--
+`
+	validateEmptyConfig(t, config)
+}
+
+// Test Configuration deserialization (yaml)
+func TestNewCloudFromYAML(t *testing.T) {
+	config := `
+tenantId: --tenant-id--
+subscriptionId: --subscription-id--
+aadClientId: --aad-client-id--
+aadClientSecret: --aad-client-secret--
+aadClientCertPath: --aad-client-cert-path--
+aadClientCertPassword: --aad-client-cert-password--
+resourceGroup: --resource-group--
+location: --location--
+subnetName: --subnet-name--
+securityGroupName: --security-group-name--
+vnetName: --vnet-name--
+routeTableName: --route-table-name--
+primaryAvailabilitySetName: --primary-availability-set-name--
+cloudProviderBackoff: true
+cloudProviderBackoffRetries: 6
+cloudProviderBackoffExponent: 1.5
+cloudProviderBackoffDuration: 5
+cloudProviderBackoffJitter: 1.0
+cloudProviderRatelimit: true
+cloudProviderRateLimitQPS: 0.5
+cloudProviderRateLimitBucket: 5
+`
+	validateConfig(t, config)
+}
+
+func validateConfig(t *testing.T, config string) {
+	azureCloud := getCloudFromConfig(t, config)
+
+	if azureCloud.TenantID != "--tenant-id--" {
+		t.Errorf("got incorrect value for TenantID")
+	}
+	if azureCloud.SubscriptionID != "--subscription-id--" {
+		t.Errorf("got incorrect value for SubscriptionID")
+	}
+	if azureCloud.AADClientID != "--aad-client-id--" {
+		t.Errorf("got incorrect value for AADClientID")
+	}
+	if azureCloud.AADClientSecret != "--aad-client-secret--" {
+		t.Errorf("got incorrect value for AADClientSecret")
+	}
+	if azureCloud.AADClientCertPath != "--aad-client-cert-path--" {
+		t.Errorf("got incorrect value for AADClientCertPath")
+	}
+	if azureCloud.AADClientCertPassword != "--aad-client-cert-password--" {
+		t.Errorf("got incorrect value for AADClientCertPassword")
+	}
+	if azureCloud.ResourceGroup != "--resource-group--" {
+		t.Errorf("got incorrect value for ResourceGroup")
+	}
+	if azureCloud.Location != "--location--" {
+		t.Errorf("got incorrect value for Location")
+	}
+	if azureCloud.SubnetName != "--subnet-name--" {
+		t.Errorf("got incorrect value for SubnetName")
+	}
+	if azureCloud.SecurityGroupName != "--security-group-name--" {
+		t.Errorf("got incorrect value for SecurityGroupName")
+	}
+	if azureCloud.VnetName != "--vnet-name--" {
+		t.Errorf("got incorrect value for VnetName")
+	}
+	if azureCloud.RouteTableName != "--route-table-name--" {
+		t.Errorf("got incorrect value for RouteTableName")
+	}
+	if azureCloud.PrimaryAvailabilitySetName != "--primary-availability-set-name--" {
+		t.Errorf("got incorrect value for PrimaryAvailabilitySetName")
+	}
+	if azureCloud.CloudProviderBackoff != true {
+		t.Errorf("got incorrect value for CloudProviderBackoff")
+	}
+	if azureCloud.CloudProviderBackoffRetries != 6 {
+		t.Errorf("got incorrect value for CloudProviderBackoffRetries")
+	}
+	if azureCloud.CloudProviderBackoffExponent != 1.5 {
+		t.Errorf("got incorrect value for CloudProviderBackoffExponent")
+	}
+	if azureCloud.CloudProviderBackoffDuration != 5 {
+		t.Errorf("got incorrect value for CloudProviderBackoffDuration")
+	}
+	if azureCloud.CloudProviderBackoffJitter != 1.0 {
+		t.Errorf("got incorrect value for CloudProviderBackoffJitter")
+	}
+	if azureCloud.CloudProviderRateLimit != true {
+		t.Errorf("got incorrect value for CloudProviderRateLimit")
+	}
+	if azureCloud.CloudProviderRateLimitQPS != 0.5 {
+		t.Errorf("got incorrect value for CloudProviderRateLimitQPS")
+	}
+	if azureCloud.CloudProviderRateLimitBucket != 5 {
+		t.Errorf("got incorrect value for CloudProviderRateLimitBucket")
+	}
+}
+
+func getCloudFromConfig(t *testing.T, config string) *Cloud {
+	configReader := strings.NewReader(config)
+	cloud, err := NewCloud(configReader)
+	if err != nil {
+		t.Error(err)
+	}
+	azureCloud, ok := cloud.(*Cloud)
+	if !ok {
+		t.Error("NewCloud returned incorrect type")
+	}
+	return azureCloud
+}
+
+// TODO include checks for other appropriate default config parameters
+func validateEmptyConfig(t *testing.T, config string) {
+	azureCloud := getCloudFromConfig(t, config)
+
+	// backoff should be disabled by default if not explicitly enabled in config
+	if azureCloud.CloudProviderBackoff != false {
+		t.Errorf("got incorrect value for CloudProviderBackoff")
+	}
+
+	// rate limits should be disabled by default if not explicitly enabled in config
+	if azureCloud.CloudProviderRateLimit != false {
+		t.Errorf("got incorrect value for CloudProviderRateLimit")
+	}
+}
+
+func TestDecodeInstanceInfo(t *testing.T) {
+	response := `{"ID":"_azdev","UD":"0","FD":"99"}`
+
+	faultDomain, err := readFaultDomain(strings.NewReader(response))
+	if err != nil {
+		t.Error("Unexpected error in ReadFaultDomain")
+	}
+
+	if faultDomain == nil {
+		t.Error("Fault domain was unexpectedly nil")
+	}
+
+	if *faultDomain != "99" {
+		t.Error("got incorrect fault domain")
+	}
+}
+
+func TestSplitProviderID(t *testing.T) {
+	providers := []struct {
+		providerID string
+		name       types.NodeName
+
+		fail bool
+	}{
+		{
+			providerID: CloudProviderName + ":///subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroupName/providers/Microsoft.Compute/virtualMachines/k8s-agent-AAAAAAAA-0",
+			name:       "k8s-agent-AAAAAAAA-0",
+			fail:       false,
+		},
+		{
+			providerID: CloudProviderName + ":/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroupName/providers/Microsoft.Compute/virtualMachines/k8s-agent-AAAAAAAA-0",
+			name:       "",
+			fail:       true,
+		},
+		{
+			providerID: CloudProviderName + "://",
+			name:       "",
+			fail:       true,
+		},
+		{
+			providerID: ":///subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroupName/providers/Microsoft.Compute/virtualMachines/k8s-agent-AAAAAAAA-0",
+			name:       "",
+			fail:       true,
+		},
+		{
+			providerID: "aws:///subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/myResourceGroupName/providers/Microsoft.Compute/virtualMachines/k8s-agent-AAAAAAAA-0",
+			name:       "",
+			fail:       true,
+		},
+	}
+
+	for _, test := range providers {
+		name, err := splitProviderID(test.providerID)
+		if (err != nil) != test.fail {
+			t.Errorf("Expected to failt=%t, with pattern %v", test.fail, test)
+		}
+
+		if test.fail {
+			continue
+		}
+
+		if name != test.name {
+			t.Errorf("Expected %v, but got %v", test.name, name)
+		}
+
+	}
+}
+
+func TestMetadataURLGeneration(t *testing.T) {
+	metadata := NewInstanceMetadata()
+	fullPath := metadata.makeMetadataURL("some/path")
+	if fullPath != "http://169.254.169.254/metadata/some/path" {
+		t.Errorf("Expected http://169.254.169.254/metadata/some/path saw %s", fullPath)
+	}
+}
+
+func TestMetadataParsing(t *testing.T) {
+	data := `
+{
+    "interface": [
+      {
+        "ipv4": {
+          "ipAddress": [
+            {
+              "privateIpAddress": "10.0.1.4",
+              "publicIpAddress": "X.X.X.X"
+            }
+          ],
+          "subnet": [
+            {
+              "address": "10.0.1.0",
+              "prefix": "24"
+            }
+          ]
+        },
+        "ipv6": {
+          "ipAddress": [
+
+          ]
+        },
+        "macAddress": "002248020E1E"
+      }
+    ]
+}	
+`
+
+	network := NetworkMetadata{}
+	if err := json.Unmarshal([]byte(data), &network); err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	ip := network.Interface[0].IPV4.IPAddress[0].PrivateIP
+	if ip != "10.0.1.4" {
+		t.Errorf("Unexpected value: %s, expected 10.0.1.4", ip)
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, data)
+	}))
+	defer server.Close()
+
+	metadata := &InstanceMetadata{
+		baseURL: server.URL,
+	}
+
+	networkJSON := NetworkMetadata{}
+	if err := metadata.Object("/some/path", &networkJSON); err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	if !reflect.DeepEqual(network, networkJSON) {
+		t.Errorf("Unexpected inequality:\n%#v\nvs\n%#v", network, networkJSON)
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/azure_util.go

@@ -0,0 +1,355 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"errors"
+	"fmt"
+	"hash/crc32"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+
+	"github.com/Azure/azure-sdk-for-go/arm/compute"
+	"github.com/Azure/azure-sdk-for-go/arm/network"
+	"github.com/golang/glog"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+const (
+	loadBalancerMinimumPriority = 500
+	loadBalancerMaximumPriority = 4096
+
+	machineIDTemplate           = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachines/%s"
+	availabilitySetIDTemplate   = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/availabilitySets/%s"
+	frontendIPConfigIDTemplate  = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s"
+	backendPoolIDTemplate       = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/backendAddressPools/%s"
+	loadBalancerRuleIDTemplate  = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/loadBalancingRules/%s"
+	loadBalancerProbeIDTemplate = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/probes/%s"
+	securityRuleIDTemplate      = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/networkSecurityGroups/%s/securityRules/%s"
+)
+
+var providerIDRE = regexp.MustCompile(`^` + CloudProviderName + `://(?:.*)/Microsoft.Compute/virtualMachines/(.+)$`)
+
+// returns the full identifier of a machine
+func (az *Cloud) getMachineID(machineName string) string {
+	return fmt.Sprintf(
+		machineIDTemplate,
+		az.SubscriptionID,
+		az.ResourceGroup,
+		machineName)
+}
+
+// returns the full identifier of an availabilitySet
+func (az *Cloud) getAvailabilitySetID(availabilitySetName string) string {
+	return fmt.Sprintf(
+		availabilitySetIDTemplate,
+		az.SubscriptionID,
+		az.ResourceGroup,
+		availabilitySetName)
+}
+
+// returns the full identifier of a loadbalancer frontendipconfiguration.
+func (az *Cloud) getFrontendIPConfigID(lbName, backendPoolName string) string {
+	return fmt.Sprintf(
+		frontendIPConfigIDTemplate,
+		az.SubscriptionID,
+		az.ResourceGroup,
+		lbName,
+		backendPoolName)
+}
+
+// returns the full identifier of a loadbalancer backendpool.
+func (az *Cloud) getBackendPoolID(lbName, backendPoolName string) string {
+	return fmt.Sprintf(
+		backendPoolIDTemplate,
+		az.SubscriptionID,
+		az.ResourceGroup,
+		lbName,
+		backendPoolName)
+}
+
+// returns the full identifier of a loadbalancer rule.
+func (az *Cloud) getLoadBalancerRuleID(lbName, lbRuleName string) string {
+	return fmt.Sprintf(
+		loadBalancerRuleIDTemplate,
+		az.SubscriptionID,
+		az.ResourceGroup,
+		lbName,
+		lbRuleName)
+}
+
+// returns the full identifier of a loadbalancer probe.
+func (az *Cloud) getLoadBalancerProbeID(lbName, lbRuleName string) string {
+	return fmt.Sprintf(
+		loadBalancerProbeIDTemplate,
+		az.SubscriptionID,
+		az.ResourceGroup,
+		lbName,
+		lbRuleName)
+}
+
+// returns the full identifier of a network security group security rule.
+func (az *Cloud) getSecurityRuleID(securityRuleName string) string {
+	return fmt.Sprintf(
+		securityRuleIDTemplate,
+		az.SubscriptionID,
+		az.ResourceGroup,
+		az.SecurityGroupName,
+		securityRuleName)
+}
+
+// returns the deepest child's identifier from a full identifier string.
+func getLastSegment(ID string) (string, error) {
+	parts := strings.Split(ID, "/")
+	name := parts[len(parts)-1]
+	if len(name) == 0 {
+		return "", fmt.Errorf("resource name was missing from identifier")
+	}
+
+	return name, nil
+}
+
+// returns the equivalent LoadBalancerRule, SecurityRule and LoadBalancerProbe
+// protocol types for the given Kubernetes protocol type.
+func getProtocolsFromKubernetesProtocol(protocol v1.Protocol) (*network.TransportProtocol, *network.SecurityRuleProtocol, *network.ProbeProtocol, error) {
+	var transportProto network.TransportProtocol
+	var securityProto network.SecurityRuleProtocol
+	var probeProto network.ProbeProtocol
+
+	switch protocol {
+	case v1.ProtocolTCP:
+		transportProto = network.TransportProtocolTCP
+		securityProto = network.SecurityRuleProtocolTCP
+		probeProto = network.ProbeProtocolTCP
+		return &transportProto, &securityProto, &probeProto, nil
+	case v1.ProtocolUDP:
+		transportProto = network.TransportProtocolUDP
+		securityProto = network.SecurityRuleProtocolUDP
+		return &transportProto, &securityProto, nil, nil
+	default:
+		return &transportProto, &securityProto, &probeProto, fmt.Errorf("Only TCP and UDP are supported for Azure LoadBalancers")
+	}
+
+}
+
+// This returns the full identifier of the primary NIC for the given VM.
+func getPrimaryInterfaceID(machine compute.VirtualMachine) (string, error) {
+	if len(*machine.NetworkProfile.NetworkInterfaces) == 1 {
+		return *(*machine.NetworkProfile.NetworkInterfaces)[0].ID, nil
+	}
+
+	for _, ref := range *machine.NetworkProfile.NetworkInterfaces {
+		if *ref.Primary {
+			return *ref.ID, nil
+		}
+	}
+
+	return "", fmt.Errorf("failed to find a primary nic for the vm. vmname=%q", *machine.Name)
+}
+
+func getPrimaryIPConfig(nic network.Interface) (*network.InterfaceIPConfiguration, error) {
+	if len(*nic.IPConfigurations) == 1 {
+		return &((*nic.IPConfigurations)[0]), nil
+	}
+
+	for _, ref := range *nic.IPConfigurations {
+		if *ref.Primary {
+			return &ref, nil
+		}
+	}
+
+	return nil, fmt.Errorf("failed to determine the determine primary ipconfig. nicname=%q", *nic.Name)
+}
+
+// For a load balancer, all frontend ip should reference either a subnet or publicIpAddress.
+// Thus Azure do not allow mixed type (public and internal) load balancer.
+// So we'd have a separate name for internal load balancer.
+// This would be the name for Azure LoadBalancer resource.
+func getLoadBalancerName(clusterName string, isInternal bool) string {
+	if isInternal {
+		return fmt.Sprintf("%s-internal", clusterName)
+	}
+
+	return clusterName
+}
+
+func getBackendPoolName(clusterName string) string {
+	return clusterName
+}
+
+func getLoadBalancerRuleName(service *v1.Service, port v1.ServicePort) string {
+	return fmt.Sprintf("%s-%s-%d", getRulePrefix(service), port.Protocol, port.Port)
+}
+
+func getSecurityRuleName(service *v1.Service, port v1.ServicePort, sourceAddrPrefix string) string {
+	safePrefix := strings.Replace(sourceAddrPrefix, "/", "_", -1)
+	return fmt.Sprintf("%s-%s-%d-%s", getRulePrefix(service), port.Protocol, port.Port, safePrefix)
+}
+
+// This returns a human-readable version of the Service used to tag some resources.
+// This is only used for human-readable convenience, and not to filter.
+func getServiceName(service *v1.Service) string {
+	return fmt.Sprintf("%s/%s", service.Namespace, service.Name)
+}
+
+// This returns a prefix for loadbalancer/security rules.
+func getRulePrefix(service *v1.Service) string {
+	return cloudprovider.GetLoadBalancerName(service)
+}
+
+func getPublicIPName(clusterName string, service *v1.Service) string {
+	return fmt.Sprintf("%s-%s", clusterName, cloudprovider.GetLoadBalancerName(service))
+}
+
+func serviceOwnsRule(service *v1.Service, rule string) bool {
+	prefix := getRulePrefix(service)
+	return strings.HasPrefix(strings.ToUpper(rule), strings.ToUpper(prefix))
+}
+
+func getFrontendIPConfigName(service *v1.Service) string {
+	return cloudprovider.GetLoadBalancerName(service)
+}
+
+// This returns the next available rule priority level for a given set of security rules.
+func getNextAvailablePriority(rules []network.SecurityRule) (int32, error) {
+	var smallest int32 = loadBalancerMinimumPriority
+	var spread int32 = 1
+
+outer:
+	for smallest < loadBalancerMaximumPriority {
+		for _, rule := range rules {
+			if *rule.Priority == smallest {
+				smallest += spread
+				continue outer
+			}
+		}
+		// no one else had it
+		return smallest, nil
+	}
+
+	return -1, fmt.Errorf("SecurityGroup priorities are exhausted")
+}
+
+func (az *Cloud) getIPForMachine(nodeName types.NodeName) (string, error) {
+	az.operationPollRateLimiter.Accept()
+	machine, exists, err := az.getVirtualMachine(nodeName)
+	if !exists {
+		return "", cloudprovider.InstanceNotFound
+	}
+	if err != nil {
+		glog.Errorf("error: az.getIPForMachine(%s), az.getVirtualMachine(%s), err=%v", nodeName, nodeName, err)
+		return "", err
+	}
+
+	nicID, err := getPrimaryInterfaceID(machine)
+	if err != nil {
+		glog.Errorf("error: az.getIPForMachine(%s), getPrimaryInterfaceID(%v), err=%v", nodeName, machine, err)
+		return "", err
+	}
+
+	nicName, err := getLastSegment(nicID)
+	if err != nil {
+		glog.Errorf("error: az.getIPForMachine(%s), getLastSegment(%s), err=%v", nodeName, nicID, err)
+		return "", err
+	}
+
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("InterfacesClient.Get(%q): start", nicName)
+	nic, err := az.InterfacesClient.Get(az.ResourceGroup, nicName, "")
+	glog.V(10).Infof("InterfacesClient.Get(%q): end", nicName)
+	if err != nil {
+		glog.Errorf("error: az.getIPForMachine(%s), az.InterfacesClient.Get(%s, %s, %s), err=%v", nodeName, az.ResourceGroup, nicName, "", err)
+		return "", err
+	}
+
+	ipConfig, err := getPrimaryIPConfig(nic)
+	if err != nil {
+		glog.Errorf("error: az.getIPForMachine(%s), getPrimaryIPConfig(%v), err=%v", nodeName, nic, err)
+		return "", err
+	}
+
+	targetIP := *ipConfig.PrivateIPAddress
+	return targetIP, nil
+}
+
+// splitProviderID converts a providerID to a NodeName.
+func splitProviderID(providerID string) (types.NodeName, error) {
+	matches := providerIDRE.FindStringSubmatch(providerID)
+	if len(matches) != 2 {
+		return "", errors.New("error splitting providerID")
+	}
+	return types.NodeName(matches[1]), nil
+}
+
+var polyTable = crc32.MakeTable(crc32.Koopman)
+
+//MakeCRC32 : convert string to CRC32 format
+func MakeCRC32(str string) string {
+	crc := crc32.New(polyTable)
+	crc.Write([]byte(str))
+	hash := crc.Sum32()
+	return strconv.FormatUint(uint64(hash), 10)
+}
+
+//ExtractVMData : extract dataDisks, storageProfile from a map struct
+func ExtractVMData(vmData map[string]interface{}) (dataDisks []interface{},
+	storageProfile map[string]interface{},
+	hardwareProfile map[string]interface{}, err error) {
+	props, ok := vmData["properties"].(map[string]interface{})
+	if !ok {
+		return nil, nil, nil, fmt.Errorf("convert vmData(properties) to map error")
+	}
+
+	storageProfile, ok = props["storageProfile"].(map[string]interface{})
+	if !ok {
+		return nil, nil, nil, fmt.Errorf("convert vmData(storageProfile) to map error")
+	}
+
+	hardwareProfile, ok = props["hardwareProfile"].(map[string]interface{})
+	if !ok {
+		return nil, nil, nil, fmt.Errorf("convert vmData(hardwareProfile) to map error")
+	}
+
+	dataDisks, ok = storageProfile["dataDisks"].([]interface{})
+	if !ok {
+		return nil, nil, nil, fmt.Errorf("convert vmData(dataDisks) to map error")
+	}
+	return dataDisks, storageProfile, hardwareProfile, nil
+}
+
+//ExtractDiskData : extract provisioningState, diskState from a map struct
+func ExtractDiskData(diskData interface{}) (provisioningState string, diskState string, err error) {
+	fragment, ok := diskData.(map[string]interface{})
+	if !ok {
+		return "", "", fmt.Errorf("convert diskData to map error")
+	}
+
+	properties, ok := fragment["properties"].(map[string]interface{})
+	if !ok {
+		return "", "", fmt.Errorf("convert diskData(properties) to map error")
+	}
+
+	provisioningState, ok = properties["provisioningState"].(string) // if there is a disk, provisioningState property will be there
+	if ref, ok := properties["diskState"]; ok {
+		diskState = ref.(string)
+	}
+	return provisioningState, diskState, nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/azure_wrap.go

@@ -0,0 +1,169 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"net/http"
+
+	"github.com/Azure/azure-sdk-for-go/arm/compute"
+	"github.com/Azure/azure-sdk-for-go/arm/network"
+	"github.com/Azure/go-autorest/autorest"
+	"github.com/golang/glog"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// checkExistsFromError inspects an error and returns a true if err is nil,
+// false if error is an autorest.Error with StatusCode=404 and will return the
+// error back if error is another status code or another type of error.
+func checkResourceExistsFromError(err error) (bool, error) {
+	if err == nil {
+		return true, nil
+	}
+	v, ok := err.(autorest.DetailedError)
+	if ok && v.StatusCode == http.StatusNotFound {
+		return false, nil
+	}
+	return false, v
+}
+
+func (az *Cloud) getVirtualMachine(nodeName types.NodeName) (vm compute.VirtualMachine, exists bool, err error) {
+	var realErr error
+
+	vmName := string(nodeName)
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("VirtualMachinesClient.Get(%s): start", vmName)
+	vm, err = az.VirtualMachinesClient.Get(az.ResourceGroup, vmName, "")
+	glog.V(10).Infof("VirtualMachinesClient.Get(%s): end", vmName)
+
+	exists, realErr = checkResourceExistsFromError(err)
+	if realErr != nil {
+		return vm, false, realErr
+	}
+
+	if !exists {
+		return vm, false, nil
+	}
+
+	return vm, exists, err
+}
+
+func (az *Cloud) getRouteTable() (routeTable network.RouteTable, exists bool, err error) {
+	var realErr error
+
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("RouteTablesClient.Get(%s): start", az.RouteTableName)
+	routeTable, err = az.RouteTablesClient.Get(az.ResourceGroup, az.RouteTableName, "")
+	glog.V(10).Infof("RouteTablesClient.Get(%s): end", az.RouteTableName)
+
+	exists, realErr = checkResourceExistsFromError(err)
+	if realErr != nil {
+		return routeTable, false, realErr
+	}
+
+	if !exists {
+		return routeTable, false, nil
+	}
+
+	return routeTable, exists, err
+}
+
+func (az *Cloud) getSecurityGroup() (sg network.SecurityGroup, exists bool, err error) {
+	var realErr error
+
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("SecurityGroupsClient.Get(%s): start", az.SecurityGroupName)
+	sg, err = az.SecurityGroupsClient.Get(az.ResourceGroup, az.SecurityGroupName, "")
+	glog.V(10).Infof("SecurityGroupsClient.Get(%s): end", az.SecurityGroupName)
+
+	exists, realErr = checkResourceExistsFromError(err)
+	if realErr != nil {
+		return sg, false, realErr
+	}
+
+	if !exists {
+		return sg, false, nil
+	}
+
+	return sg, exists, err
+}
+
+func (az *Cloud) getAzureLoadBalancer(name string) (lb network.LoadBalancer, exists bool, err error) {
+	var realErr error
+
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("LoadBalancerClient.Get(%s): start", name)
+	lb, err = az.LoadBalancerClient.Get(az.ResourceGroup, name, "")
+	glog.V(10).Infof("LoadBalancerClient.Get(%s): end", name)
+
+	exists, realErr = checkResourceExistsFromError(err)
+	if realErr != nil {
+		return lb, false, realErr
+	}
+
+	if !exists {
+		return lb, false, nil
+	}
+
+	return lb, exists, err
+}
+
+func (az *Cloud) getPublicIPAddress(name string) (pip network.PublicIPAddress, exists bool, err error) {
+	var realErr error
+
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("PublicIPAddressesClient.Get(%s): start", name)
+	pip, err = az.PublicIPAddressesClient.Get(az.ResourceGroup, name, "")
+	glog.V(10).Infof("PublicIPAddressesClient.Get(%s): end", name)
+
+	exists, realErr = checkResourceExistsFromError(err)
+	if realErr != nil {
+		return pip, false, realErr
+	}
+
+	if !exists {
+		return pip, false, nil
+	}
+
+	return pip, exists, err
+}
+
+func (az *Cloud) getSubnet(virtualNetworkName string, subnetName string) (subnet network.Subnet, exists bool, err error) {
+	var realErr error
+	var rg string
+
+	if len(az.VnetResourceGroup) > 0 {
+		rg = az.VnetResourceGroup
+	} else {
+		rg = az.ResourceGroup
+	}
+
+	az.operationPollRateLimiter.Accept()
+	glog.V(10).Infof("SubnetsClient.Get(%s): start", subnetName)
+	subnet, err = az.SubnetsClient.Get(rg, virtualNetworkName, subnetName, "")
+	glog.V(10).Infof("SubnetsClient.Get(%s): end", subnetName)
+
+	exists, realErr = checkResourceExistsFromError(err)
+	if realErr != nil {
+		return subnet, false, realErr
+	}
+
+	if !exists {
+		return subnet, false, nil
+	}
+
+	return subnet, exists, err
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/azure/azure_zones.go

@@ -0,0 +1,113 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"sync"
+
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+
+	"github.com/Azure/azure-sdk-for-go/arm/compute"
+)
+
+const instanceInfoURL = "http://169.254.169.254/metadata/v1/InstanceInfo"
+
+var faultMutex = &sync.Mutex{}
+var faultDomain *string
+
+type instanceInfo struct {
+	ID           string `json:"ID"`
+	UpdateDomain string `json:"UD"`
+	FaultDomain  string `json:"FD"`
+}
+
+// GetZone returns the Zone containing the current failure zone and locality region that the program is running in
+func (az *Cloud) GetZone() (cloudprovider.Zone, error) {
+	faultMutex.Lock()
+	if faultDomain == nil {
+		var err error
+		faultDomain, err = fetchFaultDomain()
+		if err != nil {
+			return cloudprovider.Zone{}, err
+		}
+	}
+	zone := cloudprovider.Zone{
+		FailureDomain: *faultDomain,
+		Region:        az.Location,
+	}
+	faultMutex.Unlock()
+	return zone, nil
+}
+
+// GetZoneByProviderID implements Zones.GetZoneByProviderID
+// This is particularly useful in external cloud providers where the kubelet
+// does not initialize node data.
+func (az *Cloud) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) {
+	nodeName, err := splitProviderID(providerID)
+	if err != nil {
+		return cloudprovider.Zone{}, err
+	}
+	return az.GetZoneByNodeName(nodeName)
+}
+
+// GetZoneByNodeName implements Zones.GetZoneByNodeName
+// This is particularly useful in external cloud providers where the kubelet
+// does not initialize node data.
+func (az *Cloud) GetZoneByNodeName(nodeName types.NodeName) (cloudprovider.Zone, error) {
+
+	vm, err := az.VirtualMachinesClient.Get(az.ResourceGroup, string(nodeName), compute.InstanceView)
+
+	if err != nil {
+		return cloudprovider.Zone{}, err
+	}
+
+	failureDomain := strconv.Itoa(int(*vm.VirtualMachineProperties.InstanceView.PlatformFaultDomain))
+
+	zone := cloudprovider.Zone{
+		FailureDomain: failureDomain,
+		Region:        *(vm.Location),
+	}
+	return zone, nil
+}
+
+func fetchFaultDomain() (*string, error) {
+	resp, err := http.Get(instanceInfoURL)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	return readFaultDomain(resp.Body)
+}
+
+func readFaultDomain(reader io.Reader) (*string, error) {
+	var instanceInfo instanceInfo
+	body, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(body, &instanceInfo)
+	if err != nil {
+		return nil, err
+	}
+	return &instanceInfo.FaultDomain, nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/cloudstack/BUILD

@@ -0,0 +1,58 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cloudstack.go",
+        "cloudstack_instances.go",
+        "cloudstack_loadbalancer.go",
+        "metadata.go",
+        "metadata_other.go",
+    ] + select({
+        "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "metadata_linux.go",
+        ],
+        "//conditions:default": [],
+    }),
+    deps = [
+        "//pkg/cloudprovider:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//vendor/github.com/d2g/dhcp4:go_default_library",
+        "//vendor/github.com/d2g/dhcp4client:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/kardianos/osext:go_default_library",
+        "//vendor/github.com/xanzy/go-cloudstack/cloudstack:go_default_library",
+        "//vendor/gopkg.in/gcfg.v1:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["cloudstack_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/cloudstack/OWNERS

@@ -0,0 +1,4 @@
+approvers:
+- ngtuna
+- sebgoa
+- svanharmelen

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/cloudstack/cloudstack.go

@@ -0,0 +1,264 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudstack
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/golang/glog"
+	"github.com/kardianos/osext"
+	"github.com/xanzy/go-cloudstack/cloudstack"
+	"gopkg.in/gcfg.v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+	"k8s.io/kubernetes/pkg/controller"
+)
+
+// ProviderName is the name of this cloud provider.
+const ProviderName = "cloudstack"
+
+// CSConfig wraps the config for the CloudStack cloud provider.
+type CSConfig struct {
+	Global struct {
+		APIURL      string `gcfg:"api-url"`
+		APIKey      string `gcfg:"api-key"`
+		SecretKey   string `gcfg:"secret-key"`
+		SSLNoVerify bool   `gcfg:"ssl-no-verify"`
+		ProjectID   string `gcfg:"project-id"`
+		Zone        string `gcfg:"zone"`
+	}
+}
+
+// CSCloud is an implementation of Interface for CloudStack.
+type CSCloud struct {
+	client    *cloudstack.CloudStackClient
+	metadata  *metadata
+	projectID string // If non-"", all resources will be created within this project
+	zone      string
+}
+
+func init() {
+	cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) {
+		cfg, err := readConfig(config)
+		if err != nil {
+			return nil, err
+		}
+
+		return newCSCloud(cfg)
+	})
+}
+
+func readConfig(config io.Reader) (*CSConfig, error) {
+	cfg := &CSConfig{}
+
+	if config == nil {
+		return cfg, nil
+	}
+
+	if err := gcfg.ReadInto(cfg, config); err != nil {
+		return nil, fmt.Errorf("could not parse cloud provider config: %v", err)
+	}
+
+	return cfg, nil
+}
+
+// newCSCloud creates a new instance of CSCloud.
+func newCSCloud(cfg *CSConfig) (*CSCloud, error) {
+	cs := &CSCloud{
+		projectID: cfg.Global.ProjectID,
+		zone:      cfg.Global.Zone,
+	}
+
+	exe, err := osext.Executable()
+	if err != nil {
+		return nil, fmt.Errorf("cloud not find the service executable: %v", err)
+	}
+
+	// When running the kubelet service it's fine to not specify a config file (or only a
+	// partial config file) as all needed info can be retrieved anonymously using metadata.
+	if filepath.Base(exe) == "kubelet" || filepath.Base(exe) == "kubelet.exe" {
+		// In CloudStack your metadata is always served by the DHCP server.
+		dhcpServer, err := findDHCPServer()
+		if err == nil {
+			glog.V(4).Infof("Found metadata server: %v", dhcpServer)
+			cs.metadata = &metadata{dhcpServer: dhcpServer, zone: cs.zone}
+		} else {
+			glog.Errorf("Error searching metadata server: %v", err)
+		}
+	}
+
+	if cfg.Global.APIURL != "" && cfg.Global.APIKey != "" && cfg.Global.SecretKey != "" {
+		cs.client = cloudstack.NewAsyncClient(cfg.Global.APIURL, cfg.Global.APIKey, cfg.Global.SecretKey, !cfg.Global.SSLNoVerify)
+	}
+
+	if cs.client == nil {
+		if cs.metadata != nil {
+			glog.V(2).Infof("No API URL, key and secret are provided, so only using metadata!")
+		} else {
+			return nil, errors.New("no cloud provider config given")
+		}
+	}
+
+	return cs, nil
+}
+
+// Initialize passes a Kubernetes clientBuilder interface to the cloud provider
+func (cs *CSCloud) Initialize(clientBuilder controller.ControllerClientBuilder) {}
+
+// LoadBalancer returns an implementation of LoadBalancer for CloudStack.
+func (cs *CSCloud) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
+	if cs.client == nil {
+		return nil, false
+	}
+
+	return cs, true
+}
+
+// Instances returns an implementation of Instances for CloudStack.
+func (cs *CSCloud) Instances() (cloudprovider.Instances, bool) {
+	if cs.metadata != nil {
+		return cs.metadata, true
+	}
+
+	if cs.client == nil {
+		return nil, false
+	}
+
+	return cs, true
+}
+
+// Zones returns an implementation of Zones for CloudStack.
+func (cs *CSCloud) Zones() (cloudprovider.Zones, bool) {
+	if cs.metadata != nil {
+		return cs.metadata, true
+	}
+
+	if cs.client == nil {
+		return nil, false
+	}
+
+	return cs, true
+}
+
+// Clusters returns an implementation of Clusters for CloudStack.
+func (cs *CSCloud) Clusters() (cloudprovider.Clusters, bool) {
+	if cs.client == nil {
+		return nil, false
+	}
+
+	return nil, false
+}
+
+// Routes returns an implementation of Routes for CloudStack.
+func (cs *CSCloud) Routes() (cloudprovider.Routes, bool) {
+	if cs.client == nil {
+		return nil, false
+	}
+
+	return nil, false
+}
+
+// ProviderName returns the cloud provider ID.
+func (cs *CSCloud) ProviderName() string {
+	return ProviderName
+}
+
+// ScrubDNS filters DNS settings for pods.
+func (cs *CSCloud) ScrubDNS(nameservers, searches []string) (nsOut, srchOut []string) {
+	return nameservers, searches
+}
+
+// HasClusterID returns true if the cluster has a clusterID
+func (cs *CSCloud) HasClusterID() bool {
+	return true
+}
+
+// GetZone returns the Zone containing the region that the program is running in.
+func (cs *CSCloud) GetZone() (cloudprovider.Zone, error) {
+	zone := cloudprovider.Zone{}
+
+	if cs.zone == "" {
+		hostname, err := os.Hostname()
+		if err != nil {
+			return zone, fmt.Errorf("failed to get hostname for retrieving the zone: %v", err)
+		}
+
+		instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByName(hostname)
+		if err != nil {
+			if count == 0 {
+				return zone, fmt.Errorf("could not find instance for retrieving the zone: %v", err)
+			}
+			return zone, fmt.Errorf("error getting instance for retrieving the zone: %v", err)
+		}
+
+		cs.zone = instance.Zonename
+	}
+
+	glog.V(2).Infof("Current zone is %v", cs.zone)
+	zone.FailureDomain = cs.zone
+	zone.Region = cs.zone
+
+	return zone, nil
+}
+
+// GetZoneByProviderID returns the Zone, found by using the provider ID.
+func (cs *CSCloud) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) {
+	zone := cloudprovider.Zone{}
+
+	instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByID(
+		providerID,
+		cloudstack.WithProject(cs.projectID),
+	)
+	if err != nil {
+		if count == 0 {
+			return zone, fmt.Errorf("could not find node by ID: %v", providerID)
+		}
+		return zone, fmt.Errorf("error retrieving zone: %v", err)
+	}
+
+	glog.V(2).Infof("Current zone is %v", cs.zone)
+	zone.FailureDomain = instance.Zonename
+	zone.Region = instance.Zonename
+
+	return zone, nil
+}
+
+// GetZoneByNodeName returns the Zone, found by using the node name.
+func (cs *CSCloud) GetZoneByNodeName(nodeName types.NodeName) (cloudprovider.Zone, error) {
+	zone := cloudprovider.Zone{}
+
+	instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByName(
+		string(nodeName),
+		cloudstack.WithProject(cs.projectID),
+	)
+	if err != nil {
+		if count == 0 {
+			return zone, fmt.Errorf("could not find node: %v", nodeName)
+		}
+		return zone, fmt.Errorf("error retrieving zone: %v", err)
+	}
+
+	glog.V(2).Infof("Current zone is %v", cs.zone)
+	zone.FailureDomain = instance.Zonename
+	zone.Region = instance.Zonename
+
+	return zone, nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/cloudstack/cloudstack_instances.go

@@ -0,0 +1,159 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudstack
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/golang/glog"
+	"github.com/xanzy/go-cloudstack/cloudstack"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+)
+
+// NodeAddresses returns the addresses of the specified instance.
+func (cs *CSCloud) NodeAddresses(name types.NodeName) ([]v1.NodeAddress, error) {
+	instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByName(
+		string(name),
+		cloudstack.WithProject(cs.projectID),
+	)
+	if err != nil {
+		if count == 0 {
+			return nil, cloudprovider.InstanceNotFound
+		}
+		return nil, fmt.Errorf("error retrieving node addresses: %v", err)
+	}
+
+	return cs.nodeAddresses(instance)
+}
+
+// NodeAddressesByProviderID returns the addresses of the specified instance.
+func (cs *CSCloud) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) {
+	instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByID(
+		providerID,
+		cloudstack.WithProject(cs.projectID),
+	)
+	if err != nil {
+		if count == 0 {
+			return nil, cloudprovider.InstanceNotFound
+		}
+		return nil, fmt.Errorf("error retrieving node addresses: %v", err)
+	}
+
+	return cs.nodeAddresses(instance)
+}
+
+func (cs *CSCloud) nodeAddresses(instance *cloudstack.VirtualMachine) ([]v1.NodeAddress, error) {
+	if len(instance.Nic) == 0 {
+		return nil, errors.New("instance does not have an internal IP")
+	}
+
+	addresses := []v1.NodeAddress{
+		{Type: v1.NodeInternalIP, Address: instance.Nic[0].Ipaddress},
+	}
+
+	if instance.Publicip != "" {
+		addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: instance.Publicip})
+	} else {
+		// Since there is no sane way to determine the external IP if the host isn't
+		// using static NAT, we will just fire a log message and omit the external IP.
+		glog.V(4).Infof("Could not determine the public IP of host %v (%v)", instance.Name, instance.Id)
+	}
+
+	return addresses, nil
+}
+
+// ExternalID returns the cloud provider ID of the specified instance (deprecated).
+func (cs *CSCloud) ExternalID(name types.NodeName) (string, error) {
+	return cs.InstanceID(name)
+}
+
+// InstanceID returns the cloud provider ID of the specified instance.
+func (cs *CSCloud) InstanceID(name types.NodeName) (string, error) {
+	instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByName(
+		string(name),
+		cloudstack.WithProject(cs.projectID),
+	)
+	if err != nil {
+		if count == 0 {
+			return "", cloudprovider.InstanceNotFound
+		}
+		return "", fmt.Errorf("error retrieving instance ID: %v", err)
+	}
+
+	return instance.Id, nil
+}
+
+// InstanceType returns the type of the specified instance.
+func (cs *CSCloud) InstanceType(name types.NodeName) (string, error) {
+	instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByName(
+		string(name),
+		cloudstack.WithProject(cs.projectID),
+	)
+	if err != nil {
+		if count == 0 {
+			return "", cloudprovider.InstanceNotFound
+		}
+		return "", fmt.Errorf("error retrieving instance type: %v", err)
+	}
+
+	return instance.Serviceofferingname, nil
+}
+
+// InstanceTypeByProviderID returns the type of the specified instance.
+func (cs *CSCloud) InstanceTypeByProviderID(providerID string) (string, error) {
+	instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByID(
+		providerID,
+		cloudstack.WithProject(cs.projectID),
+	)
+	if err != nil {
+		if count == 0 {
+			return "", cloudprovider.InstanceNotFound
+		}
+		return "", fmt.Errorf("error retrieving instance type: %v", err)
+	}
+
+	return instance.Serviceofferingname, nil
+}
+
+// AddSSHKeyToAllInstances is currently not implemented.
+func (cs *CSCloud) AddSSHKeyToAllInstances(user string, keyData []byte) error {
+	return errors.New("AddSSHKeyToAllInstances not implemented")
+}
+
+// CurrentNodeName returns the name of the node we are currently running on.
+func (cs *CSCloud) CurrentNodeName(hostname string) (types.NodeName, error) {
+	return types.NodeName(hostname), nil
+}
+
+// InstanceExistsByProviderID returns if the instance still exists.
+func (cs *CSCloud) InstanceExistsByProviderID(providerID string) (bool, error) {
+	_, count, err := cs.client.VirtualMachine.GetVirtualMachineByID(
+		providerID,
+		cloudstack.WithProject(cs.projectID),
+	)
+	if err != nil {
+		if count == 0 {
+			return false, nil
+		}
+		return false, fmt.Errorf("error retrieving instance: %v", err)
+	}
+
+	return true, nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/cloudstack/cloudstack_loadbalancer.go

@@ -0,0 +1,542 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudstack
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/golang/glog"
+	"github.com/xanzy/go-cloudstack/cloudstack"
+	"k8s.io/api/core/v1"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+)
+
+type loadBalancer struct {
+	*cloudstack.CloudStackClient
+
+	name      string
+	algorithm string
+	hostIDs   []string
+	ipAddr    string
+	ipAddrID  string
+	networkID string
+	projectID string
+	rules     map[string]*cloudstack.LoadBalancerRule
+}
+
+// GetLoadBalancer returns whether the specified load balancer exists, and if so, what its status is.
+func (cs *CSCloud) GetLoadBalancer(clusterName string, service *v1.Service) (*v1.LoadBalancerStatus, bool, error) {
+	glog.V(4).Infof("GetLoadBalancer(%v, %v, %v)", clusterName, service.Namespace, service.Name)
+
+	// Get the load balancer details and existing rules.
+	lb, err := cs.getLoadBalancer(service)
+	if err != nil {
+		return nil, false, err
+	}
+
+	// If we don't have any rules, the load balancer does not exist.
+	if len(lb.rules) == 0 {
+		return nil, false, nil
+	}
+
+	glog.V(4).Infof("Found a load balancer associated with IP %v", lb.ipAddr)
+
+	status := &v1.LoadBalancerStatus{}
+	status.Ingress = append(status.Ingress, v1.LoadBalancerIngress{IP: lb.ipAddr})
+
+	return status, true, nil
+}
+
+// EnsureLoadBalancer creates a new load balancer, or updates the existing one. Returns the status of the balancer.
+func (cs *CSCloud) EnsureLoadBalancer(clusterName string, service *v1.Service, nodes []*v1.Node) (status *v1.LoadBalancerStatus, err error) {
+	glog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v)", clusterName, service.Namespace, service.Name, service.Spec.LoadBalancerIP, service.Spec.Ports, nodes)
+
+	if len(service.Spec.Ports) == 0 {
+		return nil, fmt.Errorf("requested load balancer with no ports")
+	}
+
+	// Get the load balancer details and existing rules.
+	lb, err := cs.getLoadBalancer(service)
+	if err != nil {
+		return nil, err
+	}
+
+	// Set the load balancer algorithm.
+	switch service.Spec.SessionAffinity {
+	case v1.ServiceAffinityNone:
+		lb.algorithm = "roundrobin"
+	case v1.ServiceAffinityClientIP:
+		lb.algorithm = "source"
+	default:
+		return nil, fmt.Errorf("unsupported load balancer affinity: %v", service.Spec.SessionAffinity)
+	}
+
+	// Verify that all the hosts belong to the same network, and retrieve their ID's.
+	lb.hostIDs, lb.networkID, err = cs.verifyHosts(nodes)
+	if err != nil {
+		return nil, err
+	}
+
+	if !lb.hasLoadBalancerIP() {
+		// Create or retrieve the load balancer IP.
+		if err := lb.getLoadBalancerIP(service.Spec.LoadBalancerIP); err != nil {
+			return nil, err
+		}
+
+		if lb.ipAddr != "" && lb.ipAddr != service.Spec.LoadBalancerIP {
+			defer func(lb *loadBalancer) {
+				if err != nil {
+					if err := lb.releaseLoadBalancerIP(); err != nil {
+						glog.Errorf(err.Error())
+					}
+				}
+			}(lb)
+		}
+	}
+
+	glog.V(4).Infof("Load balancer %v is associated with IP %v", lb.name, lb.ipAddr)
+
+	for _, port := range service.Spec.Ports {
+		// All ports have their own load balancer rule, so add the port to lbName to keep the names unique.
+		lbRuleName := fmt.Sprintf("%s-%d", lb.name, port.Port)
+
+		// If the load balancer rule exists and is up-to-date, we move on to the next rule.
+		exists, needsUpdate, err := lb.checkLoadBalancerRule(lbRuleName, port)
+		if err != nil {
+			return nil, err
+		}
+		if exists && !needsUpdate {
+			glog.V(4).Infof("Load balancer rule %v is up-to-date", lbRuleName)
+			// Delete the rule from the map, to prevent it being deleted.
+			delete(lb.rules, lbRuleName)
+			continue
+		}
+
+		if needsUpdate {
+			glog.V(4).Infof("Updating load balancer rule: %v", lbRuleName)
+			if err := lb.updateLoadBalancerRule(lbRuleName); err != nil {
+				return nil, err
+			}
+			// Delete the rule from the map, to prevent it being deleted.
+			delete(lb.rules, lbRuleName)
+			continue
+		}
+
+		glog.V(4).Infof("Creating load balancer rule: %v", lbRuleName)
+		lbRule, err := lb.createLoadBalancerRule(lbRuleName, port)
+		if err != nil {
+			return nil, err
+		}
+
+		glog.V(4).Infof("Assigning hosts (%v) to load balancer rule: %v", lb.hostIDs, lbRuleName)
+		if err = lb.assignHostsToRule(lbRule, lb.hostIDs); err != nil {
+			return nil, err
+		}
+
+	}
+
+	// Cleanup any rules that are now still in the rules map, as they are no longer needed.
+	for _, lbRule := range lb.rules {
+		glog.V(4).Infof("Deleting obsolete load balancer rule: %v", lbRule.Name)
+		if err := lb.deleteLoadBalancerRule(lbRule); err != nil {
+			return nil, err
+		}
+	}
+
+	status = &v1.LoadBalancerStatus{}
+	status.Ingress = []v1.LoadBalancerIngress{{IP: lb.ipAddr}}
+
+	return status, nil
+}
+
+// UpdateLoadBalancer updates hosts under the specified load balancer.
+func (cs *CSCloud) UpdateLoadBalancer(clusterName string, service *v1.Service, nodes []*v1.Node) error {
+	glog.V(4).Infof("UpdateLoadBalancer(%v, %v, %v, %v)", clusterName, service.Namespace, service.Name, nodes)
+
+	// Get the load balancer details and existing rules.
+	lb, err := cs.getLoadBalancer(service)
+	if err != nil {
+		return err
+	}
+
+	// Verify that all the hosts belong to the same network, and retrieve their ID's.
+	lb.hostIDs, _, err = cs.verifyHosts(nodes)
+	if err != nil {
+		return err
+	}
+
+	for _, lbRule := range lb.rules {
+		p := lb.LoadBalancer.NewListLoadBalancerRuleInstancesParams(lbRule.Id)
+
+		// Retrieve all VMs currently associated to this load balancer rule.
+		l, err := lb.LoadBalancer.ListLoadBalancerRuleInstances(p)
+		if err != nil {
+			return fmt.Errorf("error retrieving associated instances: %v", err)
+		}
+
+		assign, remove := symmetricDifference(lb.hostIDs, l.LoadBalancerRuleInstances)
+
+		if len(assign) > 0 {
+			glog.V(4).Infof("Assigning new hosts (%v) to load balancer rule: %v", assign, lbRule.Name)
+			if err := lb.assignHostsToRule(lbRule, assign); err != nil {
+				return err
+			}
+		}
+
+		if len(remove) > 0 {
+			glog.V(4).Infof("Removing old hosts (%v) from load balancer rule: %v", assign, lbRule.Name)
+			if err := lb.removeHostsFromRule(lbRule, remove); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// EnsureLoadBalancerDeleted deletes the specified load balancer if it exists, returning
+// nil if the load balancer specified either didn't exist or was successfully deleted.
+func (cs *CSCloud) EnsureLoadBalancerDeleted(clusterName string, service *v1.Service) error {
+	glog.V(4).Infof("EnsureLoadBalancerDeleted(%v, %v, %v)", clusterName, service.Namespace, service.Name)
+
+	// Get the load balancer details and existing rules.
+	lb, err := cs.getLoadBalancer(service)
+	if err != nil {
+		return err
+	}
+
+	for _, lbRule := range lb.rules {
+		glog.V(4).Infof("Deleting load balancer rule: %v", lbRule.Name)
+		if err := lb.deleteLoadBalancerRule(lbRule); err != nil {
+			return err
+		}
+	}
+
+	if lb.ipAddr != "" && lb.ipAddr != service.Spec.LoadBalancerIP {
+		glog.V(4).Infof("Releasing load balancer IP: %v", lb.ipAddr)
+		if err := lb.releaseLoadBalancerIP(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// getLoadBalancer retrieves the IP address and ID and all the existing rules it can find.
+func (cs *CSCloud) getLoadBalancer(service *v1.Service) (*loadBalancer, error) {
+	lb := &loadBalancer{
+		CloudStackClient: cs.client,
+		name:             cloudprovider.GetLoadBalancerName(service),
+		projectID:        cs.projectID,
+		rules:            make(map[string]*cloudstack.LoadBalancerRule),
+	}
+
+	p := cs.client.LoadBalancer.NewListLoadBalancerRulesParams()
+	p.SetKeyword(lb.name)
+	p.SetListall(true)
+
+	if cs.projectID != "" {
+		p.SetProjectid(cs.projectID)
+	}
+
+	l, err := cs.client.LoadBalancer.ListLoadBalancerRules(p)
+	if err != nil {
+		return nil, fmt.Errorf("error retrieving load balancer rules: %v", err)
+	}
+
+	for _, lbRule := range l.LoadBalancerRules {
+		lb.rules[lbRule.Name] = lbRule
+
+		if lb.ipAddr != "" && lb.ipAddr != lbRule.Publicip {
+			glog.Warningf("Load balancer for service %v/%v has rules associated with different IP's: %v, %v", service.Namespace, service.Name, lb.ipAddr, lbRule.Publicip)
+		}
+
+		lb.ipAddr = lbRule.Publicip
+		lb.ipAddrID = lbRule.Publicipid
+	}
+
+	glog.V(4).Infof("Load balancer %v contains %d rule(s)", lb.name, len(lb.rules))
+
+	return lb, nil
+}
+
+// verifyHosts verifies if all hosts belong to the same network, and returns the host ID's and network ID.
+func (cs *CSCloud) verifyHosts(nodes []*v1.Node) ([]string, string, error) {
+	hostNames := map[string]bool{}
+	for _, node := range nodes {
+		hostNames[node.Name] = true
+	}
+
+	p := cs.client.VirtualMachine.NewListVirtualMachinesParams()
+	p.SetListall(true)
+
+	if cs.projectID != "" {
+		p.SetProjectid(cs.projectID)
+	}
+
+	l, err := cs.client.VirtualMachine.ListVirtualMachines(p)
+	if err != nil {
+		return nil, "", fmt.Errorf("error retrieving list of hosts: %v", err)
+	}
+
+	var hostIDs []string
+	var networkID string
+
+	// Check if the virtual machine is in the hosts slice, then add the corresponding ID.
+	for _, vm := range l.VirtualMachines {
+		if hostNames[vm.Name] {
+			if networkID != "" && networkID != vm.Nic[0].Networkid {
+				return nil, "", fmt.Errorf("found hosts that belong to different networks")
+			}
+
+			networkID = vm.Nic[0].Networkid
+			hostIDs = append(hostIDs, vm.Id)
+		}
+	}
+
+	return hostIDs, networkID, nil
+}
+
+// hasLoadBalancerIP returns true if we have a load balancer address and ID.
+func (lb *loadBalancer) hasLoadBalancerIP() bool {
+	return lb.ipAddr != "" && lb.ipAddrID != ""
+}
+
+// getLoadBalancerIP retieves an existing IP or associates a new IP.
+func (lb *loadBalancer) getLoadBalancerIP(loadBalancerIP string) error {
+	if loadBalancerIP != "" {
+		return lb.getPublicIPAddress(loadBalancerIP)
+	}
+
+	return lb.associatePublicIPAddress()
+}
+
+// getPublicIPAddressID retrieves the ID of the given IP, and sets the address and it's ID.
+func (lb *loadBalancer) getPublicIPAddress(loadBalancerIP string) error {
+	glog.V(4).Infof("Retrieve load balancer IP details: %v", loadBalancerIP)
+
+	p := lb.Address.NewListPublicIpAddressesParams()
+	p.SetIpaddress(loadBalancerIP)
+	p.SetListall(true)
+
+	if lb.projectID != "" {
+		p.SetProjectid(lb.projectID)
+	}
+
+	l, err := lb.Address.ListPublicIpAddresses(p)
+	if err != nil {
+		return fmt.Errorf("error retrieving IP address: %v", err)
+	}
+
+	if l.Count != 1 {
+		return fmt.Errorf("could not find IP address %v", loadBalancerIP)
+	}
+
+	lb.ipAddr = l.PublicIpAddresses[0].Ipaddress
+	lb.ipAddrID = l.PublicIpAddresses[0].Id
+
+	return nil
+}
+
+// associatePublicIPAddress associates a new IP and sets the address and it's ID.
+func (lb *loadBalancer) associatePublicIPAddress() error {
+	glog.V(4).Infof("Allocate new IP for load balancer: %v", lb.name)
+	// If a network belongs to a VPC, the IP address needs to be associated with
+	// the VPC instead of with the network.
+	network, count, err := lb.Network.GetNetworkByID(lb.networkID, cloudstack.WithProject(lb.projectID))
+	if err != nil {
+		if count == 0 {
+			return fmt.Errorf("could not find network %v", lb.networkID)
+		}
+		return fmt.Errorf("error retrieving network: %v", err)
+	}
+
+	p := lb.Address.NewAssociateIpAddressParams()
+
+	if network.Vpcid != "" {
+		p.SetVpcid(network.Vpcid)
+	} else {
+		p.SetNetworkid(lb.networkID)
+	}
+
+	if lb.projectID != "" {
+		p.SetProjectid(lb.projectID)
+	}
+
+	// Associate a new IP address
+	r, err := lb.Address.AssociateIpAddress(p)
+	if err != nil {
+		return fmt.Errorf("error associating new IP address: %v", err)
+	}
+
+	lb.ipAddr = r.Ipaddress
+	lb.ipAddrID = r.Id
+
+	return nil
+}
+
+// releasePublicIPAddress releases an associated IP.
+func (lb *loadBalancer) releaseLoadBalancerIP() error {
+	p := lb.Address.NewDisassociateIpAddressParams(lb.ipAddrID)
+
+	if _, err := lb.Address.DisassociateIpAddress(p); err != nil {
+		return fmt.Errorf("error releasing load balancer IP %v: %v", lb.ipAddr, err)
+	}
+
+	return nil
+}
+
+// checkLoadBalancerRule checks if the rule already exists and if it does, if it can be updated. If
+// it does exist but cannot be updated, it will delete the existing rule so it can be created again.
+func (lb *loadBalancer) checkLoadBalancerRule(lbRuleName string, port v1.ServicePort) (bool, bool, error) {
+	lbRule, ok := lb.rules[lbRuleName]
+	if !ok {
+		return false, false, nil
+	}
+
+	// Check if any of the values we cannot update (those that require a new load balancer rule) are changed.
+	if lbRule.Publicip == lb.ipAddr && lbRule.Privateport == strconv.Itoa(int(port.NodePort)) && lbRule.Publicport == strconv.Itoa(int(port.Port)) {
+		return true, lbRule.Algorithm != lb.algorithm, nil
+	}
+
+	// Delete the load balancer rule so we can create a new one using the new values.
+	if err := lb.deleteLoadBalancerRule(lbRule); err != nil {
+		return false, false, err
+	}
+
+	return false, false, nil
+}
+
+// updateLoadBalancerRule updates a load balancer rule.
+func (lb *loadBalancer) updateLoadBalancerRule(lbRuleName string) error {
+	lbRule := lb.rules[lbRuleName]
+
+	p := lb.LoadBalancer.NewUpdateLoadBalancerRuleParams(lbRule.Id)
+	p.SetAlgorithm(lb.algorithm)
+
+	_, err := lb.LoadBalancer.UpdateLoadBalancerRule(p)
+	return err
+}
+
+// createLoadBalancerRule creates a new load balancer rule and returns it's ID.
+func (lb *loadBalancer) createLoadBalancerRule(lbRuleName string, port v1.ServicePort) (*cloudstack.LoadBalancerRule, error) {
+	p := lb.LoadBalancer.NewCreateLoadBalancerRuleParams(
+		lb.algorithm,
+		lbRuleName,
+		int(port.NodePort),
+		int(port.Port),
+	)
+
+	p.SetNetworkid(lb.networkID)
+	p.SetPublicipid(lb.ipAddrID)
+
+	switch port.Protocol {
+	case v1.ProtocolTCP:
+		p.SetProtocol("TCP")
+	case v1.ProtocolUDP:
+		p.SetProtocol("UDP")
+	default:
+		return nil, fmt.Errorf("unsupported load balancer protocol: %v", port.Protocol)
+	}
+
+	// Do not create corresponding firewall rule.
+	p.SetOpenfirewall(false)
+
+	// Create a new load balancer rule.
+	r, err := lb.LoadBalancer.CreateLoadBalancerRule(p)
+	if err != nil {
+		return nil, fmt.Errorf("error creating load balancer rule %v: %v", lbRuleName, err)
+	}
+
+	lbRule := &cloudstack.LoadBalancerRule{
+		Id:          r.Id,
+		Algorithm:   r.Algorithm,
+		Cidrlist:    r.Cidrlist,
+		Name:        r.Name,
+		Networkid:   r.Networkid,
+		Privateport: r.Privateport,
+		Publicport:  r.Publicport,
+		Publicip:    r.Publicip,
+		Publicipid:  r.Publicipid,
+	}
+
+	return lbRule, nil
+}
+
+// deleteLoadBalancerRule deletes a load balancer rule.
+func (lb *loadBalancer) deleteLoadBalancerRule(lbRule *cloudstack.LoadBalancerRule) error {
+	p := lb.LoadBalancer.NewDeleteLoadBalancerRuleParams(lbRule.Id)
+
+	if _, err := lb.LoadBalancer.DeleteLoadBalancerRule(p); err != nil {
+		return fmt.Errorf("error deleting load balancer rule %v: %v", lbRule.Name, err)
+	}
+
+	// Delete the rule from the map as it no longer exists
+	delete(lb.rules, lbRule.Name)
+
+	return nil
+}
+
+// assignHostsToRule assigns hosts to a load balancer rule.
+func (lb *loadBalancer) assignHostsToRule(lbRule *cloudstack.LoadBalancerRule, hostIDs []string) error {
+	p := lb.LoadBalancer.NewAssignToLoadBalancerRuleParams(lbRule.Id)
+	p.SetVirtualmachineids(hostIDs)
+
+	if _, err := lb.LoadBalancer.AssignToLoadBalancerRule(p); err != nil {
+		return fmt.Errorf("error assigning hosts to load balancer rule %v: %v", lbRule.Name, err)
+	}
+
+	return nil
+}
+
+// removeHostsFromRule removes hosts from a load balancer rule.
+func (lb *loadBalancer) removeHostsFromRule(lbRule *cloudstack.LoadBalancerRule, hostIDs []string) error {
+	p := lb.LoadBalancer.NewRemoveFromLoadBalancerRuleParams(lbRule.Id)
+	p.SetVirtualmachineids(hostIDs)
+
+	if _, err := lb.LoadBalancer.RemoveFromLoadBalancerRule(p); err != nil {
+		return fmt.Errorf("error removing hosts from load balancer rule %v: %v", lbRule.Name, err)
+	}
+
+	return nil
+}
+
+// symmetricDifference returns the symmetric difference between the old (existing) and new (wanted) host ID's.
+func symmetricDifference(hostIDs []string, lbInstances []*cloudstack.VirtualMachine) ([]string, []string) {
+	new := make(map[string]bool)
+	for _, hostID := range hostIDs {
+		new[hostID] = true
+	}
+
+	var remove []string
+	for _, instance := range lbInstances {
+		if new[instance.Id] {
+			delete(new, instance.Id)
+			continue
+		}
+
+		remove = append(remove, instance.Id)
+	}
+
+	var assign []string
+	for hostID := range new {
+		assign = append(assign, hostID)
+	}
+
+	return assign, remove
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/cloudstack/cloudstack_test.go

@@ -0,0 +1,117 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudstack
+
+import (
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const testClusterName = "testCluster"
+
+func TestReadConfig(t *testing.T) {
+	_, err := readConfig(nil)
+	if err != nil {
+		t.Fatalf("Should not return an error when no config is provided: %v", err)
+	}
+
+	cfg, err := readConfig(strings.NewReader(`
+ [Global]
+ api-url				= https://cloudstack.url
+ api-key				= a-valid-api-key
+ secret-key			= a-valid-secret-key
+ ssl-no-verify	= true
+ project-id			= a-valid-project-id
+ `))
+	if err != nil {
+		t.Fatalf("Should succeed when a valid config is provided: %v", err)
+	}
+
+	if cfg.Global.APIURL != "https://cloudstack.url" {
+		t.Errorf("incorrect api-url: %s", cfg.Global.APIURL)
+	}
+	if cfg.Global.APIKey != "a-valid-api-key" {
+		t.Errorf("incorrect api-key: %s", cfg.Global.APIKey)
+	}
+	if cfg.Global.SecretKey != "a-valid-secret-key" {
+		t.Errorf("incorrect secret-key: %s", cfg.Global.SecretKey)
+	}
+	if !cfg.Global.SSLNoVerify {
+		t.Errorf("incorrect ssl-no-verify: %t", cfg.Global.SSLNoVerify)
+	}
+}
+
+// This allows acceptance testing against an existing CloudStack environment.
+func configFromEnv() (*CSConfig, bool) {
+	cfg := &CSConfig{}
+
+	cfg.Global.APIURL = os.Getenv("CS_API_URL")
+	cfg.Global.APIKey = os.Getenv("CS_API_KEY")
+	cfg.Global.SecretKey = os.Getenv("CS_SECRET_KEY")
+	cfg.Global.ProjectID = os.Getenv("CS_PROJECT_ID")
+
+	// It is save to ignore the error here. If the input cannot be parsed SSLNoVerify
+	// will still be a bool with its zero value (false) which is the expected default.
+	cfg.Global.SSLNoVerify, _ = strconv.ParseBool(os.Getenv("CS_SSL_NO_VERIFY"))
+
+	// Check if we have the minimum required info to be able to connect to CloudStack.
+	ok := cfg.Global.APIURL != "" && cfg.Global.APIKey != "" && cfg.Global.SecretKey != ""
+
+	return cfg, ok
+}
+
+func TestNewCSCloud(t *testing.T) {
+	cfg, ok := configFromEnv()
+	if !ok {
+		t.Skipf("No config found in environment")
+	}
+
+	_, err := newCSCloud(cfg)
+	if err != nil {
+		t.Fatalf("Failed to construct/authenticate CloudStack: %v", err)
+	}
+}
+
+func TestLoadBalancer(t *testing.T) {
+	cfg, ok := configFromEnv()
+	if !ok {
+		t.Skipf("No config found in environment")
+	}
+
+	cs, err := newCSCloud(cfg)
+	if err != nil {
+		t.Fatalf("Failed to construct/authenticate CloudStack: %v", err)
+	}
+
+	lb, ok := cs.LoadBalancer()
+	if !ok {
+		t.Fatalf("LoadBalancer() returned false")
+	}
+
+	_, exists, err := lb.GetLoadBalancer(testClusterName, &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "noexist"}})
+	if err != nil {
+		t.Fatalf("GetLoadBalancer(\"noexist\") returned error: %s", err)
+	}
+	if exists {
+		t.Fatalf("GetLoadBalancer(\"noexist\") returned exists")
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/cloudstack/metadata.go

@@ -0,0 +1,211 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudstack
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+
+	"github.com/d2g/dhcp4"
+	"github.com/golang/glog"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+)
+
+type metadata struct {
+	dhcpServer string
+	zone       string
+}
+
+type metadataType string
+
+const (
+	metadataTypeExternalIP   metadataType = "public-ipv4"
+	metadataTypeInternalIP   metadataType = "local-ipv4"
+	metadataTypeInstanceID   metadataType = "instance-id"
+	metadataTypeInstanceType metadataType = "service-offering"
+	metadataTypeZone         metadataType = "availability-zone"
+)
+
+// NodeAddresses returns the addresses of the specified instance.
+func (m *metadata) NodeAddresses(name types.NodeName) ([]v1.NodeAddress, error) {
+	externalIP, err := m.get(metadataTypeExternalIP)
+	if err != nil {
+		return nil, fmt.Errorf("could not get external IP: %v", err)
+	}
+
+	internalIP, err := m.get(metadataTypeInternalIP)
+	if err != nil {
+		return nil, fmt.Errorf("could not get internal IP: %v", err)
+	}
+
+	return []v1.NodeAddress{
+		{Type: v1.NodeExternalIP, Address: externalIP},
+		{Type: v1.NodeInternalIP, Address: internalIP},
+	}, nil
+}
+
+// NodeAddressesByProviderID returns the addresses of the specified instance.
+func (m *metadata) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) {
+	return nil, errors.New("NodeAddressesByProviderID not implemented")
+}
+
+// ExternalID returns the cloud provider ID of the specified instance (deprecated).
+func (m *metadata) ExternalID(name types.NodeName) (string, error) {
+	return m.InstanceID(name)
+}
+
+// InstanceID returns the cloud provider ID of the specified instance.
+func (m *metadata) InstanceID(name types.NodeName) (string, error) {
+	instanceID, err := m.get(metadataTypeInstanceID)
+	if err != nil {
+		return "", fmt.Errorf("could not get instance ID: %v", err)
+	}
+
+	zone, err := m.get(metadataTypeZone)
+	if err != nil {
+		return "", fmt.Errorf("could not get zone: %v", err)
+	}
+
+	return "/" + zone + "/" + instanceID, nil
+}
+
+// InstanceType returns the type of the specified instance.
+func (m *metadata) InstanceType(name types.NodeName) (string, error) {
+	instanceType, err := m.get(metadataTypeInstanceType)
+	if err == nil {
+		return "", fmt.Errorf("could not get instance type: %v", err)
+	}
+
+	return instanceType, nil
+}
+
+// InstanceTypeByProviderID returns the type of the specified instance.
+func (m *metadata) InstanceTypeByProviderID(providerID string) (string, error) {
+	return "", errors.New("InstanceTypeByProviderID not implemented")
+}
+
+// AddSSHKeyToAllInstances is currently not implemented.
+func (m *metadata) AddSSHKeyToAllInstances(user string, keyData []byte) error {
+	return errors.New("AddSSHKeyToAllInstances not implemented")
+}
+
+// CurrentNodeName returns the name of the node we are currently running on.
+func (m *metadata) CurrentNodeName(hostname string) (types.NodeName, error) {
+	return types.NodeName(hostname), nil
+}
+
+// InstanceExistsByProviderID returns if the instance still exists.
+func (m *metadata) InstanceExistsByProviderID(providerID string) (bool, error) {
+	return false, errors.New("InstanceExistsByProviderID not implemented")
+}
+
+// GetZone returns the Zone containing the region that the program is running in.
+func (m *metadata) GetZone() (cloudprovider.Zone, error) {
+	zone := cloudprovider.Zone{}
+
+	if m.zone == "" {
+		zoneName, err := m.get(metadataTypeZone)
+		if err != nil {
+			return zone, fmt.Errorf("could not get zone: %v", err)
+		}
+
+		m.zone = zoneName
+	}
+
+	glog.V(2).Infof("Current zone is %v", zone)
+	zone.FailureDomain = m.zone
+	zone.Region = m.zone
+
+	return zone, nil
+}
+
+// GetZoneByProviderID returns the Zone, found by using the provider ID.
+func (m *metadata) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) {
+	return cloudprovider.Zone{}, errors.New("GetZoneByProviderID not implemented")
+}
+
+// GetZoneByNodeName returns the Zone, found by using the node name.
+func (m *metadata) GetZoneByNodeName(nodeName types.NodeName) (cloudprovider.Zone, error) {
+	return cloudprovider.Zone{}, errors.New("GetZoneByNodeName not implemented")
+}
+
+func (m *metadata) get(mdType metadataType) (string, error) {
+	url := fmt.Sprintf("http://%s/latest/meta-data/%s", m.dhcpServer, mdType)
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", fmt.Errorf("error reading metadata: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected HTTP status: %d", resp.StatusCode)
+	}
+
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("error reading response body: %d", resp.StatusCode)
+	}
+
+	return string(data), nil
+}
+
+func findDHCPServer() (string, error) {
+	nics, err := net.Interfaces()
+	if err != nil {
+		return "", fmt.Errorf("could not get interfaces: %v", err)
+	}
+
+	for _, nic := range nics {
+		if nic.Flags&net.FlagUp == 1 && nic.Flags&net.FlagLoopback == 0 && nic.Flags&net.FlagPointToPoint == 0 {
+			addrs, err := nic.Addrs()
+			if err != nil {
+				return "", fmt.Errorf("error reading IP addresses from interface %v: %v", nic.Name, err)
+			}
+
+			if addrs != nil {
+				client, err := newDHCPClient(&nic)
+				if err != nil {
+					return "", fmt.Errorf("error creating new DHCP client: %v", err)
+				}
+
+				discoverPacket, err := client.SendDiscoverPacket()
+				if err != nil {
+					return "", fmt.Errorf("error sending DHCP discover package: %v", err)
+				}
+
+				offerPacket, err := client.GetOffer(&discoverPacket)
+				if err != nil {
+					return "", fmt.Errorf("error recieving DHCP offer package: %v", err)
+				}
+
+				offerPacketOptions := offerPacket.ParseOptions()
+
+				if ipaddr, ok := offerPacketOptions[dhcp4.OptionServerIdentifier]; ok {
+					return net.IP(ipaddr).String(), nil
+				}
+			}
+		}
+	}
+
+	return "", errors.New("no server found")
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/cloudstack/metadata_linux.go

@@ -0,0 +1,40 @@
+// +build linux
+
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudstack
+
+import (
+	"net"
+	"time"
+
+	"github.com/d2g/dhcp4client"
+)
+
+func newDHCPClient(nic *net.Interface) (*dhcp4client.Client, error) {
+	pktsock, err := dhcp4client.NewPacketSock(nic.Index)
+	if err != nil {
+		return nil, err
+	}
+
+	return dhcp4client.New(
+		dhcp4client.HardwareAddr(nic.HardwareAddr),
+		dhcp4client.Timeout(2*time.Second),
+		dhcp4client.Broadcast(false),
+		dhcp4client.Connection(pktsock),
+	)
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/cloudstack/metadata_other.go

@@ -0,0 +1,40 @@
+// +build !linux
+
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudstack
+
+import (
+	"net"
+	"time"
+
+	"github.com/d2g/dhcp4client"
+)
+
+func newDHCPClient(nic *net.Interface) (*dhcp4client.Client, error) {
+	inetsock, err := dhcp4client.NewInetSock()
+	if err != nil {
+		return nil, err
+	}
+
+	return dhcp4client.New(
+		dhcp4client.HardwareAddr(nic.HardwareAddr),
+		dhcp4client.Timeout(2*time.Second),
+		dhcp4client.Broadcast(false),
+		dhcp4client.Connection(inetsock),
+	)
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/fake/BUILD

@@ -0,0 +1,33 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "fake.go",
+    ],
+    deps = [
+        "//pkg/cloudprovider:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/fake/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package fake is a test-double implementation of cloudprovider
+// Interface, LoadBalancer and Instances. It is useful for testing.
+package fake // import "k8s.io/kubernetes/pkg/cloudprovider/providers/fake"

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/fake/fake.go

@@ -0,0 +1,332 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fake
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"regexp"
+	"sync"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+	"k8s.io/kubernetes/pkg/controller"
+)
+
+const defaultProviderName = "fake"
+
+// FakeBalancer is a fake storage of balancer information
+type FakeBalancer struct {
+	Name           string
+	Region         string
+	LoadBalancerIP string
+	Ports          []v1.ServicePort
+	Hosts          []*v1.Node
+}
+
+type FakeUpdateBalancerCall struct {
+	Service *v1.Service
+	Hosts   []*v1.Node
+}
+
+// FakeCloud is a test-double implementation of Interface, LoadBalancer, Instances, and Routes. It is useful for testing.
+type FakeCloud struct {
+	Exists bool
+	Err    error
+
+	ExistsByProviderID bool
+	ErrByProviderID    error
+
+	Calls         []string
+	Addresses     []v1.NodeAddress
+	ExtID         map[types.NodeName]string
+	InstanceTypes map[types.NodeName]string
+	Machines      []types.NodeName
+	NodeResources *v1.NodeResources
+	ClusterList   []string
+	MasterName    string
+	ExternalIP    net.IP
+	Balancers     map[string]FakeBalancer
+	UpdateCalls   []FakeUpdateBalancerCall
+	RouteMap      map[string]*FakeRoute
+	Lock          sync.Mutex
+	Provider      string
+	addCallLock   sync.Mutex
+	cloudprovider.Zone
+	VolumeLabelMap map[string]map[string]string
+}
+
+type FakeRoute struct {
+	ClusterName string
+	Route       cloudprovider.Route
+}
+
+func (f *FakeCloud) addCall(desc string) {
+	f.addCallLock.Lock()
+	defer f.addCallLock.Unlock()
+	f.Calls = append(f.Calls, desc)
+}
+
+// ClearCalls clears internal record of method calls to this FakeCloud.
+func (f *FakeCloud) ClearCalls() {
+	f.Calls = []string{}
+}
+
+// Initialize passes a Kubernetes clientBuilder interface to the cloud provider
+func (f *FakeCloud) Initialize(clientBuilder controller.ControllerClientBuilder) {}
+
+func (f *FakeCloud) ListClusters() ([]string, error) {
+	return f.ClusterList, f.Err
+}
+
+func (f *FakeCloud) Master(name string) (string, error) {
+	return f.MasterName, f.Err
+}
+
+func (f *FakeCloud) Clusters() (cloudprovider.Clusters, bool) {
+	return f, true
+}
+
+// ProviderName returns the cloud provider ID.
+func (f *FakeCloud) ProviderName() string {
+	if f.Provider == "" {
+		return defaultProviderName
+	}
+	return f.Provider
+}
+
+// ScrubDNS filters DNS settings for pods.
+func (f *FakeCloud) ScrubDNS(nameservers, searches []string) (nsOut, srchOut []string) {
+	return nameservers, searches
+}
+
+// HasClusterID returns true if the cluster has a clusterID
+func (f *FakeCloud) HasClusterID() bool {
+	return true
+}
+
+// LoadBalancer returns a fake implementation of LoadBalancer.
+// Actually it just returns f itself.
+func (f *FakeCloud) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
+	return f, true
+}
+
+// Instances returns a fake implementation of Instances.
+//
+// Actually it just returns f itself.
+func (f *FakeCloud) Instances() (cloudprovider.Instances, bool) {
+	return f, true
+}
+
+func (f *FakeCloud) Zones() (cloudprovider.Zones, bool) {
+	return f, true
+}
+
+func (f *FakeCloud) Routes() (cloudprovider.Routes, bool) {
+	return f, true
+}
+
+// GetLoadBalancer is a stub implementation of LoadBalancer.GetLoadBalancer.
+func (f *FakeCloud) GetLoadBalancer(clusterName string, service *v1.Service) (*v1.LoadBalancerStatus, bool, error) {
+	status := &v1.LoadBalancerStatus{}
+	status.Ingress = []v1.LoadBalancerIngress{{IP: f.ExternalIP.String()}}
+
+	return status, f.Exists, f.Err
+}
+
+// EnsureLoadBalancer is a test-spy implementation of LoadBalancer.EnsureLoadBalancer.
+// It adds an entry "create" into the internal method call record.
+func (f *FakeCloud) EnsureLoadBalancer(clusterName string, service *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) {
+	f.addCall("create")
+	if f.Balancers == nil {
+		f.Balancers = make(map[string]FakeBalancer)
+	}
+
+	name := cloudprovider.GetLoadBalancerName(service)
+	spec := service.Spec
+
+	zone, err := f.GetZone()
+	if err != nil {
+		return nil, err
+	}
+	region := zone.Region
+
+	f.Balancers[name] = FakeBalancer{name, region, spec.LoadBalancerIP, spec.Ports, nodes}
+
+	status := &v1.LoadBalancerStatus{}
+	status.Ingress = []v1.LoadBalancerIngress{{IP: f.ExternalIP.String()}}
+
+	return status, f.Err
+}
+
+// UpdateLoadBalancer is a test-spy implementation of LoadBalancer.UpdateLoadBalancer.
+// It adds an entry "update" into the internal method call record.
+func (f *FakeCloud) UpdateLoadBalancer(clusterName string, service *v1.Service, nodes []*v1.Node) error {
+	f.addCall("update")
+	f.UpdateCalls = append(f.UpdateCalls, FakeUpdateBalancerCall{service, nodes})
+	return f.Err
+}
+
+// EnsureLoadBalancerDeleted is a test-spy implementation of LoadBalancer.EnsureLoadBalancerDeleted.
+// It adds an entry "delete" into the internal method call record.
+func (f *FakeCloud) EnsureLoadBalancerDeleted(clusterName string, service *v1.Service) error {
+	f.addCall("delete")
+	return f.Err
+}
+
+func (f *FakeCloud) AddSSHKeyToAllInstances(user string, keyData []byte) error {
+	return errors.New("unimplemented")
+}
+
+// Implementation of Instances.CurrentNodeName
+func (f *FakeCloud) CurrentNodeName(hostname string) (types.NodeName, error) {
+	return types.NodeName(hostname), nil
+}
+
+// NodeAddresses is a test-spy implementation of Instances.NodeAddresses.
+// It adds an entry "node-addresses" into the internal method call record.
+func (f *FakeCloud) NodeAddresses(instance types.NodeName) ([]v1.NodeAddress, error) {
+	f.addCall("node-addresses")
+	return f.Addresses, f.Err
+}
+
+// NodeAddressesByProviderID is a test-spy implementation of Instances.NodeAddressesByProviderID.
+// It adds an entry "node-addresses-by-provider-id" into the internal method call record.
+func (f *FakeCloud) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) {
+	f.addCall("node-addresses-by-provider-id")
+	return f.Addresses, f.Err
+}
+
+// ExternalID is a test-spy implementation of Instances.ExternalID.
+// It adds an entry "external-id" into the internal method call record.
+// It returns an external id to the mapped instance name, if not found, it will return "ext-{instance}"
+func (f *FakeCloud) ExternalID(nodeName types.NodeName) (string, error) {
+	f.addCall("external-id")
+	return f.ExtID[nodeName], f.Err
+}
+
+// InstanceID returns the cloud provider ID of the node with the specified Name.
+func (f *FakeCloud) InstanceID(nodeName types.NodeName) (string, error) {
+	f.addCall("instance-id")
+	return f.ExtID[nodeName], nil
+}
+
+// InstanceType returns the type of the specified instance.
+func (f *FakeCloud) InstanceType(instance types.NodeName) (string, error) {
+	f.addCall("instance-type")
+	return f.InstanceTypes[instance], nil
+}
+
+// InstanceTypeByProviderID returns the type of the specified instance.
+func (f *FakeCloud) InstanceTypeByProviderID(providerID string) (string, error) {
+	f.addCall("instance-type-by-provider-id")
+	return f.InstanceTypes[types.NodeName(providerID)], nil
+}
+
+// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running.
+// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager.
+func (f *FakeCloud) InstanceExistsByProviderID(providerID string) (bool, error) {
+	f.addCall("instance-exists-by-provider-id")
+	return f.ExistsByProviderID, f.ErrByProviderID
+}
+
+// List is a test-spy implementation of Instances.List.
+// It adds an entry "list" into the internal method call record.
+func (f *FakeCloud) List(filter string) ([]types.NodeName, error) {
+	f.addCall("list")
+	result := []types.NodeName{}
+	for _, machine := range f.Machines {
+		if match, _ := regexp.MatchString(filter, string(machine)); match {
+			result = append(result, machine)
+		}
+	}
+	return result, f.Err
+}
+
+func (f *FakeCloud) GetZone() (cloudprovider.Zone, error) {
+	f.addCall("get-zone")
+	return f.Zone, f.Err
+}
+
+// GetZoneByProviderID implements Zones.GetZoneByProviderID
+// This is particularly useful in external cloud providers where the kubelet
+// does not initialize node data.
+func (f *FakeCloud) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) {
+	f.addCall("get-zone-by-provider-id")
+	return f.Zone, f.Err
+}
+
+// GetZoneByNodeName implements Zones.GetZoneByNodeName
+// This is particularly useful in external cloud providers where the kubelet
+// does not initialize node data.
+func (f *FakeCloud) GetZoneByNodeName(nodeName types.NodeName) (cloudprovider.Zone, error) {
+	f.addCall("get-zone-by-node-name")
+	return f.Zone, f.Err
+}
+
+func (f *FakeCloud) ListRoutes(clusterName string) ([]*cloudprovider.Route, error) {
+	f.Lock.Lock()
+	defer f.Lock.Unlock()
+	f.addCall("list-routes")
+	var routes []*cloudprovider.Route
+	for _, fakeRoute := range f.RouteMap {
+		if clusterName == fakeRoute.ClusterName {
+			routeCopy := fakeRoute.Route
+			routes = append(routes, &routeCopy)
+		}
+	}
+	return routes, f.Err
+}
+
+func (f *FakeCloud) CreateRoute(clusterName string, nameHint string, route *cloudprovider.Route) error {
+	f.Lock.Lock()
+	defer f.Lock.Unlock()
+	f.addCall("create-route")
+	name := clusterName + "-" + nameHint
+	if _, exists := f.RouteMap[name]; exists {
+		f.Err = fmt.Errorf("route %q already exists", name)
+		return f.Err
+	}
+	fakeRoute := FakeRoute{}
+	fakeRoute.Route = *route
+	fakeRoute.Route.Name = name
+	fakeRoute.ClusterName = clusterName
+	f.RouteMap[name] = &fakeRoute
+	return nil
+}
+
+func (f *FakeCloud) DeleteRoute(clusterName string, route *cloudprovider.Route) error {
+	f.Lock.Lock()
+	defer f.Lock.Unlock()
+	f.addCall("delete-route")
+	name := route.Name
+	if _, exists := f.RouteMap[name]; !exists {
+		f.Err = fmt.Errorf("no route found with name %q", name)
+		return f.Err
+	}
+	delete(f.RouteMap, name)
+	return nil
+}
+
+func (c *FakeCloud) GetLabelsForVolume(pv *v1.PersistentVolume) (map[string]string, error) {
+	if val, ok := c.VolumeLabelMap[pv.Name]; ok {
+		return val, nil
+	}
+	return nil, fmt.Errorf("label not found for volume")
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/doc.go

@@ -16,4 +16,4 @@ limitations under the License.
 
 // Package gce is an implementation of Interface, LoadBalancer
 // and Instances for Google Compute Engine.
-package gce
+package gce // import "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_address_manager_test.go

@@ -0,0 +1,137 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	computebeta "google.golang.org/api/compute/v0.beta"
+)
+
+const testSvcName = "my-service"
+const testRegion = "us-central1"
+const testSubnet = "/projects/x/testRegions/us-central1/testSubnetworks/customsub"
+const testLBName = "a111111111111111"
+
+// TestAddressManagerNoRequestedIP tests the typical case of passing in no requested IP
+func TestAddressManagerNoRequestedIP(t *testing.T) {
+	svc := NewFakeCloudAddressService()
+	targetIP := ""
+
+	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
+	testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal))
+	testReleaseAddress(t, mgr, svc, testLBName, testRegion)
+}
+
+// TestAddressManagerBasic tests the typical case of reserving and unreserving an address.
+func TestAddressManagerBasic(t *testing.T) {
+	svc := NewFakeCloudAddressService()
+	targetIP := "1.1.1.1"
+
+	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
+	testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal))
+	testReleaseAddress(t, mgr, svc, testLBName, testRegion)
+}
+
+// TestAddressManagerOrphaned tests the case where the address exists with the IP being equal
+// to the requested address (forwarding rule or loadbalancer IP).
+func TestAddressManagerOrphaned(t *testing.T) {
+	svc := NewFakeCloudAddressService()
+	targetIP := "1.1.1.1"
+
+	addr := &computebeta.Address{Name: testLBName, Address: targetIP, AddressType: string(schemeInternal)}
+	err := svc.ReserveBetaRegionAddress(addr, testRegion)
+	require.NoError(t, err)
+
+	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
+	testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal))
+	testReleaseAddress(t, mgr, svc, testLBName, testRegion)
+}
+
+// TestAddressManagerOutdatedOrphan tests the case where an address exists but points to
+// an IP other than the forwarding rule or loadbalancer IP.
+func TestAddressManagerOutdatedOrphan(t *testing.T) {
+	svc := NewFakeCloudAddressService()
+	previousAddress := "1.1.0.0"
+	targetIP := "1.1.1.1"
+
+	addr := &computebeta.Address{Name: testLBName, Address: previousAddress, AddressType: string(schemeExternal)}
+	err := svc.ReserveBetaRegionAddress(addr, testRegion)
+	require.NoError(t, err)
+
+	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
+	testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal))
+	testReleaseAddress(t, mgr, svc, testLBName, testRegion)
+}
+
+// TestAddressManagerExternallyOwned tests the case where the address exists but isn't
+// owned by the controller.
+func TestAddressManagerExternallyOwned(t *testing.T) {
+	svc := NewFakeCloudAddressService()
+	targetIP := "1.1.1.1"
+
+	addr := &computebeta.Address{Name: "my-important-address", Address: targetIP, AddressType: string(schemeInternal)}
+	err := svc.ReserveBetaRegionAddress(addr, testRegion)
+	require.NoError(t, err)
+
+	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
+	ipToUse, err := mgr.HoldAddress()
+	require.NoError(t, err)
+	assert.NotEmpty(t, ipToUse)
+
+	_, err = svc.GetRegionAddress(testLBName, testRegion)
+	assert.True(t, isNotFound(err))
+
+	testReleaseAddress(t, mgr, svc, testLBName, testRegion)
+}
+
+// TestAddressManagerExternallyOwned tests the case where the address exists but isn't
+// owned by the controller. However, this address has the wrong type.
+func TestAddressManagerBadExternallyOwned(t *testing.T) {
+	svc := NewFakeCloudAddressService()
+	targetIP := "1.1.1.1"
+
+	addr := &computebeta.Address{Name: "my-important-address", Address: targetIP, AddressType: string(schemeExternal)}
+	err := svc.ReserveBetaRegionAddress(addr, testRegion)
+	require.NoError(t, err)
+
+	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
+	_, err = mgr.HoldAddress()
+	assert.NotNil(t, err)
+}
+
+func testHoldAddress(t *testing.T, mgr *addressManager, svc CloudAddressService, name, region, targetIP, scheme string) {
+	ipToUse, err := mgr.HoldAddress()
+	require.NoError(t, err)
+	assert.NotEmpty(t, ipToUse)
+
+	addr, err := svc.GetBetaRegionAddress(name, region)
+	require.NoError(t, err)
+	if targetIP != "" {
+		assert.EqualValues(t, targetIP, addr.Address)
+	}
+	assert.EqualValues(t, scheme, addr.AddressType)
+}
+
+func testReleaseAddress(t *testing.T, mgr *addressManager, svc CloudAddressService, name, region string) {
+	err := mgr.ReleaseAddress()
+	require.NoError(t, err)
+	_, err = svc.GetBetaRegionAddress(name, region)
+	assert.True(t, isNotFound(err))
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_annotations_test.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"testing"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestServiceNetworkTierAnnotationKey(t *testing.T) {
+	createTestService := func() *v1.Service {
+		return &v1.Service{
+			ObjectMeta: metav1.ObjectMeta{
+				UID:       "randome-uid",
+				Name:      "test-svc",
+				Namespace: "test-ns",
+			},
+		}
+	}
+
+	for testName, testCase := range map[string]struct {
+		annotations  map[string]string
+		expectedTier NetworkTier
+		expectErr    bool
+	}{
+		"Use the default when the annotation does not exist": {
+			annotations:  nil,
+			expectedTier: NetworkTierDefault,
+		},
+		"Standard tier": {
+			annotations:  map[string]string{NetworkTierAnnotationKey: "Standard"},
+			expectedTier: NetworkTierStandard,
+		},
+		"Premium tier": {
+			annotations:  map[string]string{NetworkTierAnnotationKey: "Premium"},
+			expectedTier: NetworkTierPremium,
+		},
+		"Report an error on invalid network tier value": {
+			annotations:  map[string]string{NetworkTierAnnotationKey: "Unknown-tier"},
+			expectedTier: NetworkTierPremium,
+			expectErr:    true,
+		},
+	} {
+		t.Run(testName, func(t *testing.T) {
+			svc := createTestService()
+			svc.Annotations = testCase.annotations
+			actualTier, err := GetServiceNetworkTier(svc)
+			assert.Equal(t, testCase.expectedTier, actualTier)
+			assert.Equal(t, testCase.expectErr, err != nil)
+		})
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_disks_test.go

@@ -0,0 +1,927 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"testing"
+
+	"fmt"
+
+	computealpha "google.golang.org/api/compute/v0.alpha"
+	computebeta "google.golang.org/api/compute/v0.beta"
+	compute "google.golang.org/api/compute/v1"
+	"google.golang.org/api/googleapi"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
+)
+
+// TODO TODO write a test for GetDiskByNameUnknownZone and make sure casting logic works
+// TODO TODO verify that RegionDisks.Get does not return non-replica disks
+
+func TestCreateDisk_Basic(t *testing.T) {
+	/* Arrange */
+	gceProjectId := "test-project"
+	gceRegion := "fake-region"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	alphaFeatureGate, featureGateErr := NewAlphaFeatureGate([]string{})
+	if featureGateErr != nil {
+		t.Error(featureGateErr)
+	}
+	gce := GCECloud{
+		manager:          fakeManager,
+		managedZones:     []string{"zone1"},
+		projectID:        gceProjectId,
+		AlphaFeatureGate: alphaFeatureGate,
+	}
+
+	diskName := "disk"
+	diskType := DiskTypeSSD
+	zone := "zone1"
+	const sizeGb int64 = 128
+	tags := make(map[string]string)
+	tags["test-tag"] = "test-value"
+
+	expectedDiskTypeURI := gceComputeAPIEndpoint + "projects/" + fmt.Sprintf(
+		diskTypeURITemplateSingleZone, gceProjectId, zone, diskType)
+	expectedDescription := "{\"test-tag\":\"test-value\"}"
+
+	/* Act */
+	err := gce.CreateDisk(diskName, diskType, zone, sizeGb, tags)
+
+	/* Assert */
+	if err != nil {
+		t.Error(err)
+	}
+	if !fakeManager.createDiskCalled {
+		t.Error("Never called GCE disk create.")
+	}
+	if !fakeManager.doesOpMatch {
+		t.Error("Ops used in WaitForZoneOp does not match what's returned by CreateDisk.")
+	}
+
+	// Partial check of equality between disk description sent to GCE and parameters of method.
+	diskToCreate := fakeManager.diskToCreateStable
+	if diskToCreate.Name != diskName {
+		t.Errorf("Expected disk name: %s; Actual: %s", diskName, diskToCreate.Name)
+	}
+
+	if diskToCreate.Type != expectedDiskTypeURI {
+		t.Errorf("Expected disk type: %s; Actual: %s", expectedDiskTypeURI, diskToCreate.Type)
+	}
+	if diskToCreate.SizeGb != sizeGb {
+		t.Errorf("Expected disk size: %d; Actual: %d", sizeGb, diskToCreate.SizeGb)
+	}
+	if diskToCreate.Description != expectedDescription {
+		t.Errorf("Expected tag string: %s; Actual: %s", expectedDescription, diskToCreate.Description)
+	}
+}
+
+func TestCreateRegionalDisk_Basic(t *testing.T) {
+	/* Arrange */
+	gceProjectId := "test-project"
+	gceRegion := "fake-region"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	alphaFeatureGate, featureGateErr := NewAlphaFeatureGate([]string{GCEDiskAlphaFeatureGate})
+	if featureGateErr != nil {
+		t.Error(featureGateErr)
+	}
+	gce := GCECloud{
+		manager:          fakeManager,
+		managedZones:     []string{"zone1", "zone3", "zone2"},
+		projectID:        gceProjectId,
+		AlphaFeatureGate: alphaFeatureGate,
+	}
+
+	diskName := "disk"
+	diskType := DiskTypeSSD
+	replicaZones := sets.NewString("zone1", "zone2")
+	const sizeGb int64 = 128
+	tags := make(map[string]string)
+	tags["test-tag"] = "test-value"
+
+	expectedDiskTypeURI := gceComputeAPIEndpointAlpha + "projects/" + fmt.Sprintf(
+		diskTypeURITemplateRegional, gceProjectId, gceRegion, diskType)
+	expectedDescription := "{\"test-tag\":\"test-value\"}"
+
+	/* Act */
+	err := gce.CreateRegionalDisk(diskName, diskType, replicaZones, sizeGb, tags)
+
+	/* Assert */
+	if err != nil {
+		t.Error(err)
+	}
+	if !fakeManager.createDiskCalled {
+		t.Error("Never called GCE disk create.")
+	}
+	if !fakeManager.doesOpMatch {
+		t.Error("Ops used in WaitForZoneOp does not match what's returned by CreateDisk.")
+	}
+
+	// Partial check of equality between disk description sent to GCE and parameters of method.
+	diskToCreate := fakeManager.diskToCreateStable
+	if diskToCreate.Name != diskName {
+		t.Errorf("Expected disk name: %s; Actual: %s", diskName, diskToCreate.Name)
+	}
+
+	if diskToCreate.Type != expectedDiskTypeURI {
+		t.Errorf("Expected disk type: %s; Actual: %s", expectedDiskTypeURI, diskToCreate.Type)
+	}
+	if diskToCreate.SizeGb != sizeGb {
+		t.Errorf("Expected disk size: %d; Actual: %d", sizeGb, diskToCreate.SizeGb)
+	}
+	if diskToCreate.Description != expectedDescription {
+		t.Errorf("Expected tag string: %s; Actual: %s", expectedDescription, diskToCreate.Description)
+	}
+}
+
+func TestCreateDisk_DiskAlreadyExists(t *testing.T) {
+	/* Arrange */
+	gceProjectId := "test-project"
+	gceRegion := "fake-region"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	alphaFeatureGate, featureGateErr := NewAlphaFeatureGate([]string{})
+	if featureGateErr != nil {
+		t.Error(featureGateErr)
+	}
+	gce := GCECloud{
+		manager:          fakeManager,
+		managedZones:     []string{"zone1"},
+		AlphaFeatureGate: alphaFeatureGate,
+	}
+
+	// Inject disk AlreadyExists error.
+	alreadyExistsError := googleapi.ErrorItem{Reason: "alreadyExists"}
+	fakeManager.waitForOpError = &googleapi.Error{
+		Errors: []googleapi.ErrorItem{alreadyExistsError},
+	}
+
+	/* Act */
+	err := gce.CreateDisk("disk", DiskTypeSSD, "zone1", 128, nil)
+
+	/* Assert */
+	if err != nil {
+		t.Error(
+			"Expected success when a disk with the given name already exists, but an error is returned.")
+	}
+}
+
+func TestCreateDisk_WrongZone(t *testing.T) {
+	/* Arrange */
+	gceProjectId := "test-project"
+	gceRegion := "fake-region"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	gce := GCECloud{manager: fakeManager, managedZones: []string{"zone1"}}
+
+	diskName := "disk"
+	diskType := DiskTypeSSD
+	const sizeGb int64 = 128
+
+	/* Act */
+	err := gce.CreateDisk(diskName, diskType, "zone2", sizeGb, nil)
+
+	/* Assert */
+	if err == nil {
+		t.Error("Expected error when zone is not managed, but none returned.")
+	}
+}
+
+func TestCreateDisk_NoManagedZone(t *testing.T) {
+	/* Arrange */
+	gceProjectId := "test-project"
+	gceRegion := "fake-region"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	gce := GCECloud{manager: fakeManager, managedZones: []string{}}
+
+	diskName := "disk"
+	diskType := DiskTypeSSD
+	const sizeGb int64 = 128
+
+	/* Act */
+	err := gce.CreateDisk(diskName, diskType, "zone1", sizeGb, nil)
+
+	/* Assert */
+	if err == nil {
+		t.Error("Expected error when managedZones is empty, but none returned.")
+	}
+}
+
+func TestCreateDisk_BadDiskType(t *testing.T) {
+	/* Arrange */
+	gceProjectId := "test-project"
+	gceRegion := "fake-region"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	gce := GCECloud{manager: fakeManager, managedZones: []string{"zone1"}}
+
+	diskName := "disk"
+	diskType := "arbitrary-disk"
+	zone := "zone1"
+	const sizeGb int64 = 128
+
+	/* Act */
+	err := gce.CreateDisk(diskName, diskType, zone, sizeGb, nil)
+
+	/* Assert */
+	if err == nil {
+		t.Error("Expected error when disk type is not supported, but none returned.")
+	}
+}
+
+func TestCreateDisk_MultiZone(t *testing.T) {
+	/* Arrange */
+	gceProjectId := "test-project"
+	gceRegion := "fake-region"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	alphaFeatureGate, featureGateErr := NewAlphaFeatureGate([]string{})
+	if featureGateErr != nil {
+		t.Error(featureGateErr)
+	}
+	gce := GCECloud{
+		manager:          fakeManager,
+		managedZones:     []string{"zone1", "zone2", "zone3"},
+		AlphaFeatureGate: alphaFeatureGate,
+	}
+
+	diskName := "disk"
+	diskType := DiskTypeStandard
+	const sizeGb int64 = 128
+
+	/* Act & Assert */
+	for _, zone := range gce.managedZones {
+		diskName = zone + "disk"
+		err := gce.CreateDisk(diskName, diskType, zone, sizeGb, nil)
+		if err != nil {
+			t.Errorf("Error creating disk in zone '%v'; error: \"%v\"", zone, err)
+		}
+	}
+}
+
+func TestDeleteDisk_Basic(t *testing.T) {
+	/* Arrange */
+	gceProjectId := "test-project"
+	gceRegion := "fake-region"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	alphaFeatureGate, featureGateErr := NewAlphaFeatureGate([]string{})
+	if featureGateErr != nil {
+		t.Error(featureGateErr)
+	}
+	gce := GCECloud{
+		manager:          fakeManager,
+		managedZones:     []string{"zone1"},
+		AlphaFeatureGate: alphaFeatureGate,
+	}
+	diskName := "disk"
+	diskType := DiskTypeSSD
+	zone := "zone1"
+	const sizeGb int64 = 128
+
+	gce.CreateDisk(diskName, diskType, zone, sizeGb, nil)
+
+	/* Act */
+	err := gce.DeleteDisk(diskName)
+
+	/* Assert */
+	if err != nil {
+		t.Error(err)
+	}
+	if !fakeManager.deleteDiskCalled {
+		t.Error("Never called GCE disk delete.")
+	}
+	if !fakeManager.doesOpMatch {
+		t.Error("Ops used in WaitForZoneOp does not match what's returned by DeleteDisk.")
+	}
+
+}
+
+func TestDeleteDisk_NotFound(t *testing.T) {
+	/* Arrange */
+	gceProjectId := "test-project"
+	gceRegion := "fake-region"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	alphaFeatureGate, featureGateErr := NewAlphaFeatureGate([]string{})
+	if featureGateErr != nil {
+		t.Error(featureGateErr)
+	}
+	gce := GCECloud{
+		manager:          fakeManager,
+		managedZones:     []string{"zone1"},
+		AlphaFeatureGate: alphaFeatureGate,
+	}
+	diskName := "disk"
+
+	/* Act */
+	err := gce.DeleteDisk(diskName)
+
+	/* Assert */
+	if err != nil {
+		t.Error("Expected successful operation when disk is not found, but an error is returned.")
+	}
+}
+
+func TestDeleteDisk_ResourceBeingUsed(t *testing.T) {
+	/* Arrange */
+	gceProjectId := "test-project"
+	gceRegion := "fake-region"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	alphaFeatureGate, featureGateErr := NewAlphaFeatureGate([]string{})
+	if featureGateErr != nil {
+		t.Error(featureGateErr)
+	}
+	gce := GCECloud{
+		manager:          fakeManager,
+		managedZones:     []string{"zone1"},
+		AlphaFeatureGate: alphaFeatureGate,
+	}
+	diskName := "disk"
+	diskType := DiskTypeSSD
+	zone := "zone1"
+	const sizeGb int64 = 128
+
+	gce.CreateDisk(diskName, diskType, zone, sizeGb, nil)
+	fakeManager.resourceInUse = true
+
+	/* Act */
+	err := gce.DeleteDisk(diskName)
+
+	/* Assert */
+	if err == nil {
+		t.Error("Expected error when disk is in use, but none returned.")
+	}
+}
+
+func TestDeleteDisk_SameDiskMultiZone(t *testing.T) {
+	/* Assert */
+	gceProjectId := "test-project"
+	gceRegion := "fake-region"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	alphaFeatureGate, featureGateErr := NewAlphaFeatureGate([]string{})
+	if featureGateErr != nil {
+		t.Error(featureGateErr)
+	}
+	gce := GCECloud{
+		manager:          fakeManager,
+		managedZones:     []string{"zone1", "zone2", "zone3"},
+		AlphaFeatureGate: alphaFeatureGate,
+	}
+	diskName := "disk"
+	diskType := DiskTypeSSD
+	const sizeGb int64 = 128
+
+	for _, zone := range gce.managedZones {
+		gce.CreateDisk(diskName, diskType, zone, sizeGb, nil)
+	}
+
+	/* Act */
+	// DeleteDisk will call FakeServiceManager.GetDisk() with all zones,
+	// and FakeServiceManager.GetDisk() always returns a disk,
+	// so DeleteDisk thinks a disk with diskName exists in all zones.
+	err := gce.DeleteDisk(diskName)
+
+	/* Assert */
+	if err == nil {
+		t.Error("Expected error when disk is found in multiple zones, but none returned.")
+	}
+}
+
+func TestDeleteDisk_DiffDiskMultiZone(t *testing.T) {
+	/* Arrange */
+	gceProjectId := "test-project"
+	gceRegion := "fake-region"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	alphaFeatureGate, featureGateErr := NewAlphaFeatureGate([]string{})
+	if featureGateErr != nil {
+		t.Error(featureGateErr)
+	}
+	gce := GCECloud{
+		manager:          fakeManager,
+		managedZones:     []string{"zone1"},
+		AlphaFeatureGate: alphaFeatureGate,
+	}
+	diskName := "disk"
+	diskType := DiskTypeSSD
+	const sizeGb int64 = 128
+
+	for _, zone := range gce.managedZones {
+		diskName = zone + "disk"
+		gce.CreateDisk(diskName, diskType, zone, sizeGb, nil)
+	}
+
+	/* Act & Assert */
+	var err error
+	for _, zone := range gce.managedZones {
+		diskName = zone + "disk"
+		err = gce.DeleteDisk(diskName)
+		if err != nil {
+			t.Errorf("Error deleting disk in zone '%v'; error: \"%v\"", zone, err)
+		}
+	}
+}
+
+func TestGetAutoLabelsForPD_Basic(t *testing.T) {
+	/* Arrange */
+	gceProjectId := "test-project"
+	gceRegion := "us-central1"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	diskName := "disk"
+	diskType := DiskTypeSSD
+	zone := "us-central1-c"
+	const sizeGb int64 = 128
+	alphaFeatureGate, featureGateErr := NewAlphaFeatureGate([]string{})
+	if featureGateErr != nil {
+		t.Error(featureGateErr)
+	}
+	gce := GCECloud{
+		manager:          fakeManager,
+		managedZones:     []string{zone},
+		AlphaFeatureGate: alphaFeatureGate,
+	}
+
+	gce.CreateDisk(diskName, diskType, zone, sizeGb, nil)
+
+	/* Act */
+	labels, err := gce.GetAutoLabelsForPD(diskName, zone)
+
+	/* Assert */
+	if err != nil {
+		t.Error(err)
+	}
+	if labels[kubeletapis.LabelZoneFailureDomain] != zone {
+		t.Errorf("Failure domain is '%v', but zone is '%v'",
+			labels[kubeletapis.LabelZoneFailureDomain], zone)
+	}
+	if labels[kubeletapis.LabelZoneRegion] != gceRegion {
+		t.Errorf("Region is '%v', but region is 'us-central1'", labels[kubeletapis.LabelZoneRegion])
+	}
+}
+
+func TestGetAutoLabelsForPD_NoZone(t *testing.T) {
+	/* Arrange */
+	gceProjectId := "test-project"
+	gceRegion := "europe-west1"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	diskName := "disk"
+	diskType := DiskTypeStandard
+	zone := "europe-west1-d"
+	const sizeGb int64 = 128
+	alphaFeatureGate, featureGateErr := NewAlphaFeatureGate([]string{})
+	if featureGateErr != nil {
+		t.Error(featureGateErr)
+	}
+	gce := GCECloud{
+		manager:          fakeManager,
+		managedZones:     []string{zone},
+		AlphaFeatureGate: alphaFeatureGate,
+	}
+	gce.CreateDisk(diskName, diskType, zone, sizeGb, nil)
+
+	/* Act */
+	labels, err := gce.GetAutoLabelsForPD(diskName, "")
+
+	/* Assert */
+	if err != nil {
+		t.Error(err)
+	}
+	if labels[kubeletapis.LabelZoneFailureDomain] != zone {
+		t.Errorf("Failure domain is '%v', but zone is '%v'",
+			labels[kubeletapis.LabelZoneFailureDomain], zone)
+	}
+	if labels[kubeletapis.LabelZoneRegion] != gceRegion {
+		t.Errorf("Region is '%v', but region is 'europe-west1'", labels[kubeletapis.LabelZoneRegion])
+	}
+}
+
+func TestGetAutoLabelsForPD_DiskNotFound(t *testing.T) {
+	/* Arrange */
+	gceProjectId := "test-project"
+	gceRegion := "fake-region"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	diskName := "disk"
+	zone := "asia-northeast1-a"
+	gce := GCECloud{manager: fakeManager, managedZones: []string{zone}}
+
+	/* Act */
+	_, err := gce.GetAutoLabelsForPD(diskName, zone)
+
+	/* Assert */
+	if err == nil {
+		t.Error("Expected error when the specified disk does not exist, but none returned.")
+	}
+}
+
+func TestGetAutoLabelsForPD_DiskNotFoundAndNoZone(t *testing.T) {
+	/* Arrange */
+	gceProjectId := "test-project"
+	gceRegion := "fake-region"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	diskName := "disk"
+	alphaFeatureGate, featureGateErr := NewAlphaFeatureGate([]string{})
+	if featureGateErr != nil {
+		t.Error(featureGateErr)
+	}
+	gce := GCECloud{
+		manager:          fakeManager,
+		managedZones:     []string{},
+		AlphaFeatureGate: alphaFeatureGate,
+	}
+
+	/* Act */
+	_, err := gce.GetAutoLabelsForPD(diskName, "")
+
+	/* Assert */
+	if err == nil {
+		t.Error("Expected error when the specified disk does not exist, but none returned.")
+	}
+}
+
+func TestGetAutoLabelsForPD_DupDisk(t *testing.T) {
+	/* Arrange */
+	gceProjectId := "test-project"
+	gceRegion := "us-west1"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	diskName := "disk"
+	diskType := DiskTypeStandard
+	zone := "us-west1-b"
+	const sizeGb int64 = 128
+
+	alphaFeatureGate, featureGateErr := NewAlphaFeatureGate([]string{})
+	if featureGateErr != nil {
+		t.Error(featureGateErr)
+	}
+	gce := GCECloud{
+		manager:          fakeManager,
+		managedZones:     []string{"us-west1-b", "asia-southeast1-a"},
+		AlphaFeatureGate: alphaFeatureGate,
+	}
+	for _, zone := range gce.managedZones {
+		gce.CreateDisk(diskName, diskType, zone, sizeGb, nil)
+	}
+
+	/* Act */
+	labels, err := gce.GetAutoLabelsForPD(diskName, zone)
+
+	/* Assert */
+	if err != nil {
+		t.Error("Disk name and zone uniquely identifies a disk, yet an error is returned.")
+	}
+	if labels[kubeletapis.LabelZoneFailureDomain] != zone {
+		t.Errorf("Failure domain is '%v', but zone is '%v'",
+			labels[kubeletapis.LabelZoneFailureDomain], zone)
+	}
+	if labels[kubeletapis.LabelZoneRegion] != gceRegion {
+		t.Errorf("Region is '%v', but region is 'us-west1'", labels[kubeletapis.LabelZoneRegion])
+	}
+}
+
+func TestGetAutoLabelsForPD_DupDiskNoZone(t *testing.T) {
+	/* Arrange */
+	gceProjectId := "test-project"
+	gceRegion := "fake-region"
+	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	diskName := "disk"
+	diskType := DiskTypeStandard
+	const sizeGb int64 = 128
+
+	alphaFeatureGate, featureGateErr := NewAlphaFeatureGate([]string{})
+	if featureGateErr != nil {
+		t.Error(featureGateErr)
+	}
+	gce := GCECloud{
+		manager:          fakeManager,
+		managedZones:     []string{"us-west1-b", "asia-southeast1-a"},
+		AlphaFeatureGate: alphaFeatureGate,
+	}
+	for _, zone := range gce.managedZones {
+		gce.CreateDisk(diskName, diskType, zone, sizeGb, nil)
+	}
+
+	/* Act */
+	_, err := gce.GetAutoLabelsForPD(diskName, "")
+
+	/* Assert */
+	if err == nil {
+		t.Error("Expected error when the disk is duplicated and zone is not specified, but none returned.")
+	}
+}
+
+type targetClientAPI int
+
+const (
+	targetStable targetClientAPI = iota
+	targetBeta
+	targetAlpha
+)
+
+type FakeServiceManager struct {
+	// Common fields shared among tests
+	targetAPI      targetClientAPI
+	gceProjectID   string
+	gceRegion      string
+	opAlpha        *computealpha.Operation // Mocks an operation returned by GCE API calls
+	opBeta         *computebeta.Operation  // Mocks an operation returned by GCE API calls
+	opStable       *compute.Operation      // Mocks an operation returned by GCE API calls
+	doesOpMatch    bool
+	zonalDisks     map[string]string      // zone: diskName
+	regionalDisks  map[string]sets.String // diskName: zones
+	waitForOpError error                  // Error to be returned by WaitForZoneOp or WaitForRegionalOp
+
+	// Fields for TestCreateDisk
+	createDiskCalled   bool
+	diskToCreateAlpha  *computealpha.Disk
+	diskToCreateBeta   *computebeta.Disk
+	diskToCreateStable *compute.Disk
+
+	// Fields for TestDeleteDisk
+	deleteDiskCalled bool
+	resourceInUse    bool // Marks the disk as in-use
+}
+
+func newFakeManager(gceProjectID string, gceRegion string) *FakeServiceManager {
+	return &FakeServiceManager{
+		zonalDisks:    make(map[string]string),
+		regionalDisks: make(map[string]sets.String),
+		gceProjectID:  gceProjectID,
+		gceRegion:     gceRegion,
+	}
+}
+
+/**
+ * Upon disk creation, disk info is stored in FakeServiceManager
+ * to be used by other tested methods.
+ */
+func (manager *FakeServiceManager) CreateDisk(
+	name string,
+	sizeGb int64,
+	tagsStr string,
+	diskType string,
+	zone string) (gceObject, error) {
+	manager.createDiskCalled = true
+
+	switch t := manager.targetAPI; t {
+	case targetStable:
+		manager.opStable = &compute.Operation{}
+		diskTypeURI := gceComputeAPIEndpoint + "projects/" + fmt.Sprintf(diskTypeURITemplateSingleZone, manager.gceProjectID, zone, diskType)
+		diskToCreateV1 := &compute.Disk{
+			Name:        name,
+			SizeGb:      sizeGb,
+			Description: tagsStr,
+			Type:        diskTypeURI,
+		}
+		manager.diskToCreateStable = diskToCreateV1
+		manager.zonalDisks[zone] = diskToCreateV1.Name
+		return manager.opStable, nil
+	case targetBeta:
+		manager.opBeta = &computebeta.Operation{}
+		diskTypeURI := gceComputeAPIEndpoint + "projects/" + fmt.Sprintf(diskTypeURITemplateSingleZone, manager.gceProjectID, zone, diskType)
+		diskToCreateBeta := &computebeta.Disk{
+			Name:        name,
+			SizeGb:      sizeGb,
+			Description: tagsStr,
+			Type:        diskTypeURI,
+		}
+		manager.diskToCreateBeta = diskToCreateBeta
+		manager.zonalDisks[zone] = diskToCreateBeta.Name
+		return manager.opBeta, nil
+	case targetAlpha:
+		manager.opAlpha = &computealpha.Operation{}
+		diskTypeURI := gceComputeAPIEndpointAlpha + "projects/" + fmt.Sprintf(diskTypeURITemplateSingleZone, manager.gceProjectID, zone, diskType)
+		diskToCreateAlpha := &computealpha.Disk{
+			Name:        name,
+			SizeGb:      sizeGb,
+			Description: tagsStr,
+			Type:        diskTypeURI,
+		}
+		manager.diskToCreateAlpha = diskToCreateAlpha
+		manager.zonalDisks[zone] = diskToCreateAlpha.Name
+		return manager.opAlpha, nil
+	default:
+		return nil, fmt.Errorf("unexpected type: %T", t)
+	}
+}
+
+/**
+ * Upon disk creation, disk info is stored in FakeServiceManager
+ * to be used by other tested methods.
+ */
+func (manager *FakeServiceManager) CreateRegionalDisk(
+	name string,
+	sizeGb int64,
+	tagsStr string,
+	diskType string,
+	zones sets.String) (gceObject, error) {
+	manager.createDiskCalled = true
+	diskTypeURI := gceComputeAPIEndpointAlpha + "projects/" + fmt.Sprintf(diskTypeURITemplateRegional, manager.gceProjectID, manager.gceRegion, diskType)
+
+	switch t := manager.targetAPI; t {
+	case targetStable:
+		manager.opStable = &compute.Operation{}
+		diskToCreateV1 := &compute.Disk{
+			Name:        name,
+			SizeGb:      sizeGb,
+			Description: tagsStr,
+			Type:        diskTypeURI,
+		}
+		manager.diskToCreateStable = diskToCreateV1
+		manager.regionalDisks[diskToCreateV1.Name] = zones
+		return manager.opStable, nil
+	case targetBeta:
+		return nil, fmt.Errorf("RegionalDisk CreateDisk op not supported in beta.")
+	case targetAlpha:
+		return nil, fmt.Errorf("RegionalDisk CreateDisk op not supported in alpha.")
+	default:
+		return nil, fmt.Errorf("unexpected type: %T", t)
+	}
+}
+
+func (manager *FakeServiceManager) AttachDisk(
+	disk *GCEDisk,
+	readWrite string,
+	instanceZone string,
+	instanceName string) (gceObject, error) {
+
+	switch t := manager.targetAPI; t {
+	case targetStable:
+		manager.opStable = &compute.Operation{}
+		return manager.opStable, nil
+	case targetBeta:
+		manager.opBeta = &computebeta.Operation{}
+		return manager.opBeta, nil
+	case targetAlpha:
+		manager.opAlpha = &computealpha.Operation{}
+		return manager.opAlpha, nil
+	default:
+		return nil, fmt.Errorf("unexpected type: %T", t)
+	}
+}
+
+func (manager *FakeServiceManager) DetachDisk(
+	instanceZone string,
+	instanceName string,
+	devicePath string) (gceObject, error) {
+	switch t := manager.targetAPI; t {
+	case targetStable:
+		manager.opStable = &compute.Operation{}
+		return manager.opStable, nil
+	case targetBeta:
+		manager.opBeta = &computebeta.Operation{}
+		return manager.opBeta, nil
+	case targetAlpha:
+		manager.opAlpha = &computealpha.Operation{}
+		return manager.opAlpha, nil
+	default:
+		return nil, fmt.Errorf("unexpected type: %T", t)
+	}
+}
+
+/**
+ * Gets disk info stored in the FakeServiceManager.
+ */
+func (manager *FakeServiceManager) GetDisk(
+	zone string, diskName string) (*GCEDisk, error) {
+
+	if manager.zonalDisks[zone] == "" {
+		return nil, cloudprovider.DiskNotFound
+	}
+
+	if manager.resourceInUse {
+		errorItem := googleapi.ErrorItem{Reason: "resourceInUseByAnotherResource"}
+		err := &googleapi.Error{Errors: []googleapi.ErrorItem{errorItem}}
+		return nil, err
+	}
+
+	return &GCEDisk{
+		Region:   manager.gceRegion,
+		ZoneInfo: singleZone{lastComponent(zone)},
+		Name:     diskName,
+		Kind:     "compute#disk",
+		Type:     "type",
+	}, nil
+}
+
+/**
+ * Gets disk info stored in the FakeServiceManager.
+ */
+func (manager *FakeServiceManager) GetRegionalDisk(
+	diskName string) (*GCEDisk, error) {
+
+	if _, ok := manager.regionalDisks[diskName]; !ok {
+		return nil, cloudprovider.DiskNotFound
+	}
+
+	if manager.resourceInUse {
+		errorItem := googleapi.ErrorItem{Reason: "resourceInUseByAnotherResource"}
+		err := &googleapi.Error{Errors: []googleapi.ErrorItem{errorItem}}
+		return nil, err
+	}
+
+	return &GCEDisk{
+		Region:   manager.gceRegion,
+		ZoneInfo: multiZone{manager.regionalDisks[diskName]},
+		Name:     diskName,
+		Kind:     "compute#disk",
+		Type:     "type",
+	}, nil
+}
+
+/**
+ * Disk info is removed from the FakeServiceManager.
+ */
+func (manager *FakeServiceManager) DeleteDisk(
+	zone string,
+	disk string) (gceObject, error) {
+
+	manager.deleteDiskCalled = true
+	delete(manager.zonalDisks, zone)
+
+	switch t := manager.targetAPI; t {
+	case targetStable:
+		manager.opStable = &compute.Operation{}
+		return manager.opStable, nil
+	case targetBeta:
+		manager.opBeta = &computebeta.Operation{}
+		return manager.opBeta, nil
+	case targetAlpha:
+		manager.opAlpha = &computealpha.Operation{}
+		return manager.opAlpha, nil
+	default:
+		return nil, fmt.Errorf("unexpected type: %T", t)
+	}
+}
+
+func (manager *FakeServiceManager) DeleteRegionalDisk(
+	disk string) (gceObject, error) {
+
+	manager.deleteDiskCalled = true
+	delete(manager.regionalDisks, disk)
+
+	switch t := manager.targetAPI; t {
+	case targetStable:
+		manager.opStable = &compute.Operation{}
+		return manager.opStable, nil
+	case targetBeta:
+		manager.opBeta = &computebeta.Operation{}
+		return manager.opBeta, nil
+	case targetAlpha:
+		manager.opAlpha = &computealpha.Operation{}
+		return manager.opAlpha, nil
+	default:
+		return nil, fmt.Errorf("unexpected type: %T", t)
+	}
+}
+
+func (manager *FakeServiceManager) WaitForZoneOp(
+	op gceObject,
+	zone string,
+	mc *metricContext) error {
+	switch v := op.(type) {
+	case *computealpha.Operation:
+		if op.(*computealpha.Operation) == manager.opAlpha {
+			manager.doesOpMatch = true
+		}
+	case *computebeta.Operation:
+		if op.(*computebeta.Operation) == manager.opBeta {
+			manager.doesOpMatch = true
+		}
+	case *compute.Operation:
+		if op.(*compute.Operation) == manager.opStable {
+			manager.doesOpMatch = true
+		}
+	default:
+		return fmt.Errorf("unexpected type: %T", v)
+	}
+	return manager.waitForOpError
+}
+
+func (manager *FakeServiceManager) WaitForRegionalOp(
+	op gceObject, mc *metricContext) error {
+	switch v := op.(type) {
+	case *computealpha.Operation:
+		if op.(*computealpha.Operation) == manager.opAlpha {
+			manager.doesOpMatch = true
+		}
+	case *computebeta.Operation:
+		if op.(*computebeta.Operation) == manager.opBeta {
+			manager.doesOpMatch = true
+		}
+	case *compute.Operation:
+		if op.(*compute.Operation) == manager.opStable {
+			manager.doesOpMatch = true
+		}
+	default:
+		return fmt.Errorf("unexpected type: %T", v)
+	}
+	return manager.waitForOpError
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_healthchecks_test.go

@@ -0,0 +1,124 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"testing"
+
+	"k8s.io/api/core/v1"
+)
+
+func TestIsAtLeastMinNodesHealthCheckVersion(t *testing.T) {
+	testCases := []struct {
+		version string
+		expect  bool
+	}{
+		{"v1.7.3", true},
+		{"v1.7.2", true},
+		{"v1.7.2-alpha.2.597+276d289b90d322", true},
+		{"v1.6.0-beta.3.472+831q821c907t31a", false},
+		{"v1.5.2", false},
+	}
+
+	for _, tc := range testCases {
+		if res := isAtLeastMinNodesHealthCheckVersion(tc.version); res != tc.expect {
+			t.Errorf("%v: want %v, got %v", tc.version, tc.expect, res)
+		}
+	}
+}
+
+func TestSupportsNodesHealthCheck(t *testing.T) {
+	testCases := []struct {
+		desc   string
+		nodes  []*v1.Node
+		expect bool
+	}{
+		{
+			"All nodes support nodes health check",
+			[]*v1.Node{
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.7.2",
+						},
+					},
+				},
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.7.2-alpha.2.597+276d289b90d322",
+						},
+					},
+				},
+			},
+			true,
+		},
+		{
+			"All nodes don't support nodes health check",
+			[]*v1.Node{
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.6.0-beta.3.472+831q821c907t31a",
+						},
+					},
+				},
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.5.2",
+						},
+					},
+				},
+			},
+			false,
+		},
+		{
+			"One node doesn't support nodes health check",
+			[]*v1.Node{
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.7.3",
+						},
+					},
+				},
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.7.2-alpha.2.597+276d289b90d322",
+						},
+					},
+				},
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.5.2",
+						},
+					},
+				},
+			},
+			false,
+		},
+	}
+
+	for _, tc := range testCases {
+		if res := supportsNodesHealthCheck(tc.nodes); res != tc.expect {
+			t.Errorf("%v: want %v, got %v", tc.desc, tc.expect, res)
+		}
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_loadbalancer_external_test.go

@@ -0,0 +1,239 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	computealpha "google.golang.org/api/compute/v0.alpha"
+
+	"k8s.io/api/core/v1"
+)
+
+func TestEnsureStaticIP(t *testing.T) {
+	fcas := NewFakeCloudAddressService()
+	ipName := "some-static-ip"
+	serviceName := ""
+	region := "us-central1"
+
+	// First ensure call
+	ip, existed, err := ensureStaticIP(fcas, ipName, serviceName, region, "", NetworkTierDefault)
+	if err != nil || existed || ip == "" {
+		t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, "") = %v, %v, %v; want valid ip, false, nil`, fcas, ipName, serviceName, region, ip, existed, err)
+	}
+
+	// Second ensure call
+	var ipPrime string
+	ipPrime, existed, err = ensureStaticIP(fcas, ipName, serviceName, region, ip, NetworkTierDefault)
+	if err != nil || !existed || ip != ipPrime {
+		t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, %v) = %v, %v, %v; want %v, true, nil`, fcas, ipName, serviceName, region, ip, ipPrime, existed, err, ip)
+	}
+}
+
+func TestEnsureStaticIPWithTier(t *testing.T) {
+	s := NewFakeCloudAddressService()
+	serviceName := ""
+	region := "us-east1"
+
+	for desc, tc := range map[string]struct {
+		name     string
+		netTier  NetworkTier
+		expected string
+	}{
+		"Premium (default)": {
+			name:     "foo-1",
+			netTier:  NetworkTierPremium,
+			expected: "PREMIUM",
+		},
+		"Standard": {
+			name:     "foo-2",
+			netTier:  NetworkTierStandard,
+			expected: "STANDARD",
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			ip, existed, err := ensureStaticIP(s, tc.name, serviceName, region, "", tc.netTier)
+			assert.NoError(t, err)
+			assert.False(t, existed)
+			assert.NotEqual(t, "", ip)
+			// Get the Address from the fake address service and verify that the tier
+			// is set correctly.
+			alphaAddr, err := s.GetAlphaRegionAddress(tc.name, region)
+			require.NoError(t, err)
+			assert.Equal(t, tc.expected, alphaAddr.NetworkTier)
+		})
+	}
+}
+
+func TestVerifyRequestedIP(t *testing.T) {
+	region := "test-region"
+	lbRef := "test-lb"
+	s := NewFakeCloudAddressService()
+
+	for desc, tc := range map[string]struct {
+		requestedIP     string
+		fwdRuleIP       string
+		netTier         NetworkTier
+		addrList        []*computealpha.Address
+		expectErr       bool
+		expectUserOwned bool
+	}{
+		"requested IP exists": {
+			requestedIP:     "1.1.1.1",
+			netTier:         NetworkTierPremium,
+			addrList:        []*computealpha.Address{{Name: "foo", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
+			expectErr:       false,
+			expectUserOwned: true,
+		},
+		"requested IP is not static, but is in use by the fwd rule": {
+			requestedIP: "1.1.1.1",
+			fwdRuleIP:   "1.1.1.1",
+			netTier:     NetworkTierPremium,
+			expectErr:   false,
+		},
+		"requested IP is not static and is not used by the fwd rule": {
+			requestedIP: "1.1.1.1",
+			fwdRuleIP:   "2.2.2.2",
+			netTier:     NetworkTierPremium,
+			expectErr:   true,
+		},
+		"no requested IP": {
+			netTier:   NetworkTierPremium,
+			expectErr: false,
+		},
+		"requested IP exists, but network tier does not match": {
+			requestedIP: "1.1.1.1",
+			netTier:     NetworkTierStandard,
+			addrList:    []*computealpha.Address{{Name: "foo", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
+			expectErr:   true,
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			s.SetRegionalAddresses(region, tc.addrList)
+			isUserOwnedIP, err := verifyUserRequestedIP(s, region, tc.requestedIP, tc.fwdRuleIP, lbRef, tc.netTier)
+			assert.Equal(t, tc.expectErr, err != nil, fmt.Sprintf("err: %v", err))
+			assert.Equal(t, tc.expectUserOwned, isUserOwnedIP)
+		})
+	}
+}
+
+func TestCreateForwardingRuleWithTier(t *testing.T) {
+	s := NewFakeCloudForwardingRuleService()
+	// Common variables among the tests.
+	ports := []v1.ServicePort{{Name: "foo", Protocol: v1.ProtocolTCP, Port: int32(123)}}
+	region := "test-region"
+	target := "test-target-pool"
+	svcName := "foo-svc"
+
+	for desc, tc := range map[string]struct {
+		netTier      NetworkTier
+		expectedRule *computealpha.ForwardingRule
+	}{
+		"Premium tier": {
+			netTier: NetworkTierPremium,
+			expectedRule: &computealpha.ForwardingRule{
+				Name:        "lb-1",
+				Description: `{"kubernetes.io/service-name":"foo-svc"}`,
+				IPAddress:   "1.1.1.1",
+				IPProtocol:  "TCP",
+				PortRange:   "123-123",
+				Target:      target,
+				NetworkTier: "PREMIUM",
+			},
+		},
+		"Standard tier": {
+			netTier: NetworkTierStandard,
+			expectedRule: &computealpha.ForwardingRule{
+				Name:        "lb-2",
+				Description: `{"kubernetes.io/service-name":"foo-svc"}`,
+				IPAddress:   "2.2.2.2",
+				IPProtocol:  "TCP",
+				PortRange:   "123-123",
+				Target:      target,
+				NetworkTier: "STANDARD",
+			},
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			lbName := tc.expectedRule.Name
+			ipAddr := tc.expectedRule.IPAddress
+
+			err := createForwardingRule(s, lbName, svcName, region, ipAddr, target, ports, tc.netTier)
+			assert.NoError(t, err)
+
+			alphaRule, err := s.GetAlphaRegionForwardingRule(lbName, region)
+			assert.NoError(t, err)
+			assert.Equal(t, tc.expectedRule, alphaRule)
+		})
+	}
+}
+
+func TestDeleteAddressWithWrongTier(t *testing.T) {
+	region := "test-region"
+	lbRef := "test-lb"
+	s := NewFakeCloudAddressService()
+
+	for desc, tc := range map[string]struct {
+		addrName     string
+		netTier      NetworkTier
+		addrList     []*computealpha.Address
+		expectDelete bool
+	}{
+		"Network tiers (premium) match; do nothing": {
+			addrName: "foo1",
+			netTier:  NetworkTierPremium,
+			addrList: []*computealpha.Address{{Name: "foo1", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
+		},
+		"Network tiers (standard) match; do nothing": {
+			addrName: "foo2",
+			netTier:  NetworkTierStandard,
+			addrList: []*computealpha.Address{{Name: "foo2", Address: "1.1.1.2", NetworkTier: "STANDARD"}},
+		},
+		"Wrong network tier (standard); delete address": {
+			addrName:     "foo3",
+			netTier:      NetworkTierPremium,
+			addrList:     []*computealpha.Address{{Name: "foo3", Address: "1.1.1.3", NetworkTier: "STANDARD"}},
+			expectDelete: true,
+		},
+		"Wrong network tier (preimium); delete address": {
+			addrName:     "foo4",
+			netTier:      NetworkTierStandard,
+			addrList:     []*computealpha.Address{{Name: "foo4", Address: "1.1.1.4", NetworkTier: "PREMIUM"}},
+			expectDelete: true,
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			s.SetRegionalAddresses(region, tc.addrList)
+			// Sanity check to ensure we inject the right address.
+			_, err := s.GetRegionAddress(tc.addrName, region)
+			require.NoError(t, err)
+
+			err = deleteAddressWithWrongTier(s, region, tc.addrName, lbRef, tc.netTier)
+			assert.NoError(t, err)
+			// Check whether the address still exists.
+			_, err = s.GetRegionAddress(tc.addrName, region)
+			if tc.expectDelete {
+				assert.True(t, isNotFound(err))
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_test.go

@@ -0,0 +1,628 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"encoding/json"
+	"reflect"
+	"strings"
+	"testing"
+
+	"golang.org/x/oauth2/google"
+
+	computealpha "google.golang.org/api/compute/v0.alpha"
+	computebeta "google.golang.org/api/compute/v0.beta"
+	computev1 "google.golang.org/api/compute/v1"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+)
+
+func TestReadConfigFile(t *testing.T) {
+	const s = `[Global]
+token-url = my-token-url
+token-body = my-token-body
+project-id = my-project
+network-project-id = my-network-project
+network-name = my-network
+subnetwork-name = my-subnetwork
+secondary-range-name = my-secondary-range
+node-tags = my-node-tag1
+node-instance-prefix = my-prefix
+multizone = true
+   `
+	reader := strings.NewReader(s)
+	config, err := readConfig(reader)
+	if err != nil {
+		t.Fatalf("Unexpected config parsing error %v", err)
+	}
+
+	expected := &ConfigFile{Global: ConfigGlobal{
+		TokenURL:           "my-token-url",
+		TokenBody:          "my-token-body",
+		ProjectID:          "my-project",
+		NetworkProjectID:   "my-network-project",
+		NetworkName:        "my-network",
+		SubnetworkName:     "my-subnetwork",
+		SecondaryRangeName: "my-secondary-range",
+		NodeTags:           []string{"my-node-tag1"},
+		NodeInstancePrefix: "my-prefix",
+		Multizone:          true,
+	}}
+
+	if !reflect.DeepEqual(expected, config) {
+		t.Fatalf("Expected config file values to be read into ConfigFile struct.  \nExpected:\n%+v\nActual:\n%+v", expected, config)
+	}
+}
+
+func TestExtraKeyInConfig(t *testing.T) {
+	const s = `[Global]
+project-id = my-project
+unknown-key = abc
+network-name = my-network
+   `
+	reader := strings.NewReader(s)
+	config, err := readConfig(reader)
+	if err != nil {
+		t.Fatalf("Unexpected config parsing error %v", err)
+	}
+	if config.Global.ProjectID != "my-project" || config.Global.NetworkName != "my-network" {
+		t.Fatalf("Expected config values to continue to be read despite extra key-value pair.")
+	}
+}
+
+func TestGetRegion(t *testing.T) {
+	zoneName := "us-central1-b"
+	regionName, err := GetGCERegion(zoneName)
+	if err != nil {
+		t.Fatalf("unexpected error from GetGCERegion: %v", err)
+	}
+	if regionName != "us-central1" {
+		t.Errorf("Unexpected region from GetGCERegion: %s", regionName)
+	}
+	gce := &GCECloud{
+		localZone: zoneName,
+		region:    regionName,
+	}
+	zones, ok := gce.Zones()
+	if !ok {
+		t.Fatalf("Unexpected missing zones impl")
+	}
+	zone, err := zones.GetZone()
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	if zone.Region != "us-central1" {
+		t.Errorf("Unexpected region: %s", zone.Region)
+	}
+}
+
+func TestComparingHostURLs(t *testing.T) {
+	tests := []struct {
+		host1       string
+		zone        string
+		name        string
+		expectEqual bool
+	}{
+		{
+			host1:       "https://www.googleapis.com/compute/v1/projects/1234567/zones/us-central1-f/instances/kubernetes-node-fhx1",
+			zone:        "us-central1-f",
+			name:        "kubernetes-node-fhx1",
+			expectEqual: true,
+		},
+		{
+			host1:       "https://www.googleapis.com/compute/v1/projects/cool-project/zones/us-central1-f/instances/kubernetes-node-fhx1",
+			zone:        "us-central1-f",
+			name:        "kubernetes-node-fhx1",
+			expectEqual: true,
+		},
+		{
+			host1:       "https://www.googleapis.com/compute/v23/projects/1234567/zones/us-central1-f/instances/kubernetes-node-fhx1",
+			zone:        "us-central1-f",
+			name:        "kubernetes-node-fhx1",
+			expectEqual: true,
+		},
+		{
+			host1:       "https://www.googleapis.com/compute/v24/projects/1234567/regions/us-central1/zones/us-central1-f/instances/kubernetes-node-fhx1",
+			zone:        "us-central1-f",
+			name:        "kubernetes-node-fhx1",
+			expectEqual: true,
+		},
+		{
+			host1:       "https://www.googleapis.com/compute/v1/projects/1234567/zones/us-central1-f/instances/kubernetes-node-fhx1",
+			zone:        "us-central1-c",
+			name:        "kubernetes-node-fhx1",
+			expectEqual: false,
+		},
+		{
+			host1:       "https://www.googleapis.com/compute/v1/projects/1234567/zones/us-central1-f/instances/kubernetes-node-fhx",
+			zone:        "us-central1-f",
+			name:        "kubernetes-node-fhx1",
+			expectEqual: false,
+		},
+		{
+			host1:       "https://www.googleapis.com/compute/v1/projects/1234567/zones/us-central1-f/instances/kubernetes-node-fhx1",
+			zone:        "us-central1-f",
+			name:        "kubernetes-node-fhx",
+			expectEqual: false,
+		},
+	}
+
+	for _, test := range tests {
+		link1 := hostURLToComparablePath(test.host1)
+		testInstance := &gceInstance{
+			Name: canonicalizeInstanceName(test.name),
+			Zone: test.zone,
+		}
+		link2 := testInstance.makeComparableHostPath()
+		if test.expectEqual && link1 != link2 {
+			t.Errorf("expected link1 and link2 to be equal, got %s and %s", link1, link2)
+		} else if !test.expectEqual && link1 == link2 {
+			t.Errorf("expected link1 and link2 not to be equal, got %s and %s", link1, link2)
+		}
+	}
+}
+
+func TestScrubDNS(t *testing.T) {
+	tcs := []struct {
+		nameserversIn  []string
+		searchesIn     []string
+		nameserversOut []string
+		searchesOut    []string
+	}{
+		{
+			nameserversIn:  []string{"1.2.3.4", "5.6.7.8"},
+			nameserversOut: []string{"1.2.3.4", "5.6.7.8"},
+		},
+		{
+			searchesIn:  []string{"c.prj.internal.", "12345678910.google.internal.", "google.internal."},
+			searchesOut: []string{"c.prj.internal.", "google.internal."},
+		},
+		{
+			searchesIn:  []string{"c.prj.internal.", "12345678910.google.internal.", "zone.c.prj.internal.", "google.internal."},
+			searchesOut: []string{"c.prj.internal.", "zone.c.prj.internal.", "google.internal."},
+		},
+		{
+			searchesIn:  []string{"c.prj.internal.", "12345678910.google.internal.", "zone.c.prj.internal.", "google.internal.", "unexpected"},
+			searchesOut: []string{"c.prj.internal.", "zone.c.prj.internal.", "google.internal.", "unexpected"},
+		},
+	}
+	gce := &GCECloud{}
+	for i := range tcs {
+		n, s := gce.ScrubDNS(tcs[i].nameserversIn, tcs[i].searchesIn)
+		if !reflect.DeepEqual(n, tcs[i].nameserversOut) {
+			t.Errorf("Expected %v, got %v", tcs[i].nameserversOut, n)
+		}
+		if !reflect.DeepEqual(s, tcs[i].searchesOut) {
+			t.Errorf("Expected %v, got %v", tcs[i].searchesOut, s)
+		}
+	}
+}
+
+func TestSplitProviderID(t *testing.T) {
+	providers := []struct {
+		providerID string
+
+		project  string
+		zone     string
+		instance string
+
+		fail bool
+	}{
+		{
+			providerID: ProviderName + "://project-example-164317/us-central1-f/kubernetes-node-fhx1",
+			project:    "project-example-164317",
+			zone:       "us-central1-f",
+			instance:   "kubernetes-node-fhx1",
+			fail:       false,
+		},
+		{
+			providerID: ProviderName + "://project-example.164317/us-central1-f/kubernetes-node-fhx1",
+			project:    "project-example.164317",
+			zone:       "us-central1-f",
+			instance:   "kubernetes-node-fhx1",
+			fail:       false,
+		},
+		{
+			providerID: ProviderName + "://project-example-164317/us-central1-fkubernetes-node-fhx1",
+			project:    "",
+			zone:       "",
+			instance:   "",
+			fail:       true,
+		},
+		{
+			providerID: ProviderName + ":/project-example-164317/us-central1-f/kubernetes-node-fhx1",
+			project:    "",
+			zone:       "",
+			instance:   "",
+			fail:       true,
+		},
+		{
+			providerID: "aws://project-example-164317/us-central1-f/kubernetes-node-fhx1",
+			project:    "",
+			zone:       "",
+			instance:   "",
+			fail:       true,
+		},
+		{
+			providerID: ProviderName + "://project-example-164317/us-central1-f/kubernetes-node-fhx1/",
+			project:    "",
+			zone:       "",
+			instance:   "",
+			fail:       true,
+		},
+		{
+			providerID: ProviderName + "://project-example.164317//kubernetes-node-fhx1",
+			project:    "",
+			zone:       "",
+			instance:   "",
+			fail:       true,
+		},
+		{
+			providerID: ProviderName + "://project-example.164317/kubernetes-node-fhx1",
+			project:    "",
+			zone:       "",
+			instance:   "",
+			fail:       true,
+		},
+	}
+
+	for _, test := range providers {
+		project, zone, instance, err := splitProviderID(test.providerID)
+		if (err != nil) != test.fail {
+			t.Errorf("Expected to fail=%t, with pattern %v", test.fail, test)
+		}
+
+		if test.fail {
+			continue
+		}
+
+		if project != test.project {
+			t.Errorf("Expected %v, but got %v", test.project, project)
+		}
+		if zone != test.zone {
+			t.Errorf("Expected %v, but got %v", test.zone, zone)
+		}
+		if instance != test.instance {
+			t.Errorf("Expected %v, but got %v", test.instance, instance)
+		}
+	}
+}
+
+func TestGetZoneByProviderID(t *testing.T) {
+	tests := []struct {
+		providerID string
+
+		expectedZone cloudprovider.Zone
+
+		fail        bool
+		description string
+	}{
+		{
+			providerID:   ProviderName + "://project-example-164317/us-central1-f/kubernetes-node-fhx1",
+			expectedZone: cloudprovider.Zone{FailureDomain: "us-central1-f", Region: "us-central1"},
+			fail:         false,
+			description:  "standard gce providerID",
+		},
+		{
+			providerID:   ProviderName + "://project-example-164317/us-central1-f/kubernetes-node-fhx1/",
+			expectedZone: cloudprovider.Zone{},
+			fail:         true,
+			description:  "too many slashes('/') trailing",
+		},
+		{
+			providerID:   ProviderName + "://project-example.164317//kubernetes-node-fhx1",
+			expectedZone: cloudprovider.Zone{},
+			fail:         true,
+			description:  "too many slashes('/') embedded",
+		},
+		{
+			providerID:   ProviderName + "://project-example-164317/uscentral1f/kubernetes-node-fhx1",
+			expectedZone: cloudprovider.Zone{},
+			fail:         true,
+			description:  "invalid name of the GCE zone",
+		},
+	}
+
+	gce := &GCECloud{
+		localZone: "us-central1-f",
+		region:    "us-central1",
+	}
+	for _, test := range tests {
+		zone, err := gce.GetZoneByProviderID(test.providerID)
+		if (err != nil) != test.fail {
+			t.Errorf("Expected to fail=%t, provider ID %v, tests %s", test.fail, test, test.description)
+		}
+
+		if test.fail {
+			continue
+		}
+
+		if zone != test.expectedZone {
+			t.Errorf("Expected %v, but got %v", test.expectedZone, zone)
+		}
+	}
+}
+
+func TestGenerateCloudConfigs(t *testing.T) {
+	configBoilerplate := ConfigGlobal{
+		TokenURL:           "",
+		TokenBody:          "",
+		ProjectID:          "project-id",
+		NetworkName:        "network-name",
+		SubnetworkName:     "",
+		SecondaryRangeName: "",
+		NodeTags:           []string{"node-tag"},
+		NodeInstancePrefix: "node-prefix",
+		Multizone:          false,
+		ApiEndpoint:        "",
+		LocalZone:          "us-central1-a",
+		AlphaFeatures:      []string{},
+	}
+
+	cloudBoilerplate := CloudConfig{
+		ApiEndpoint:        "",
+		ProjectID:          "project-id",
+		NetworkProjectID:   "",
+		Region:             "us-central1",
+		Zone:               "us-central1-a",
+		ManagedZones:       []string{"us-central1-a"},
+		NetworkName:        "network-name",
+		SubnetworkName:     "",
+		NetworkURL:         "",
+		SubnetworkURL:      "",
+		SecondaryRangeName: "",
+		NodeTags:           []string{"node-tag"},
+		TokenSource:        google.ComputeTokenSource(""),
+		NodeInstancePrefix: "node-prefix",
+		UseMetadataServer:  true,
+		AlphaFeatureGate:   &AlphaFeatureGate{map[string]bool{}},
+	}
+
+	testCases := []struct {
+		name   string
+		config func() ConfigGlobal
+		cloud  func() CloudConfig
+	}{
+		{
+			name:   "Empty Config",
+			config: func() ConfigGlobal { return configBoilerplate },
+			cloud:  func() CloudConfig { return cloudBoilerplate },
+		},
+		{
+			name: "Nil token URL",
+			config: func() ConfigGlobal {
+				v := configBoilerplate
+				v.TokenURL = "nil"
+				return v
+			},
+			cloud: func() CloudConfig {
+				v := cloudBoilerplate
+				v.TokenSource = nil
+				return v
+			},
+		},
+		{
+			name: "Network Project ID",
+			config: func() ConfigGlobal {
+				v := configBoilerplate
+				v.NetworkProjectID = "my-awesome-project"
+				return v
+			},
+			cloud: func() CloudConfig {
+				v := cloudBoilerplate
+				v.NetworkProjectID = "my-awesome-project"
+				return v
+			},
+		},
+		{
+			name: "Specified API Endpint",
+			config: func() ConfigGlobal {
+				v := configBoilerplate
+				v.ApiEndpoint = "https://www.googleapis.com/compute/staging_v1/"
+				return v
+			},
+			cloud: func() CloudConfig {
+				v := cloudBoilerplate
+				v.ApiEndpoint = "https://www.googleapis.com/compute/staging_v1/"
+				return v
+			},
+		},
+		{
+			name: "Network & Subnetwork names",
+			config: func() ConfigGlobal {
+				v := configBoilerplate
+				v.NetworkName = "my-network"
+				v.SubnetworkName = "my-subnetwork"
+				return v
+			},
+			cloud: func() CloudConfig {
+				v := cloudBoilerplate
+				v.NetworkName = "my-network"
+				v.SubnetworkName = "my-subnetwork"
+				return v
+			},
+		},
+		{
+			name: "Network & Subnetwork URLs",
+			config: func() ConfigGlobal {
+				v := configBoilerplate
+				v.NetworkName = "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/my-network"
+				v.SubnetworkName = "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/my-subnetwork"
+				return v
+			},
+			cloud: func() CloudConfig {
+				v := cloudBoilerplate
+				v.NetworkName = ""
+				v.SubnetworkName = ""
+				v.NetworkURL = "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/my-network"
+				v.SubnetworkURL = "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/my-subnetwork"
+				return v
+			},
+		},
+		{
+			name: "Multizone",
+			config: func() ConfigGlobal {
+				v := configBoilerplate
+				v.Multizone = true
+				return v
+			},
+			cloud: func() CloudConfig {
+				v := cloudBoilerplate
+				v.ManagedZones = nil
+				return v
+			},
+		},
+		{
+			name: "Secondary Range Name",
+			config: func() ConfigGlobal {
+				v := configBoilerplate
+				v.SecondaryRangeName = "my-secondary"
+				return v
+			},
+			cloud: func() CloudConfig {
+				v := cloudBoilerplate
+				v.SecondaryRangeName = "my-secondary"
+				return v
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			resultCloud, err := generateCloudConfig(&ConfigFile{Global: tc.config()})
+			if err != nil {
+				t.Fatalf("Unexpect error: %v", err)
+			}
+
+			v := tc.cloud()
+			if !reflect.DeepEqual(*resultCloud, v) {
+				t.Errorf("Got: \n%v\nWant\n%v\n", v, *resultCloud)
+			}
+		})
+	}
+}
+
+func TestConvertToV1Operation(t *testing.T) {
+	v1Op := getTestOperation()
+	enc, _ := v1Op.MarshalJSON()
+	var op interface{}
+	var alphaOp computealpha.Operation
+	var betaOp computebeta.Operation
+
+	if err := json.Unmarshal(enc, &alphaOp); err != nil {
+		t.Errorf("Failed to unmarshal operation: %v", err)
+	}
+
+	if err := json.Unmarshal(enc, &betaOp); err != nil {
+		t.Errorf("Failed to unmarshal operation: %v", err)
+	}
+
+	op = convertToV1Operation(&alphaOp)
+	if _, ok := op.(*computev1.Operation); ok {
+		if !reflect.DeepEqual(op, v1Op) {
+			t.Errorf("Failed to maintain consistency across conversion")
+		}
+	} else {
+		t.Errorf("Expect output to be type v1 operation, but got %v", op)
+	}
+
+	op = convertToV1Operation(&betaOp)
+	if _, ok := op.(*computev1.Operation); ok {
+		if !reflect.DeepEqual(op, v1Op) {
+			t.Errorf("Failed to maintain consistency across conversion")
+		}
+	} else {
+		t.Errorf("Expect output to be type v1 operation, but got %v", op)
+	}
+}
+
+func getTestOperation() *computev1.Operation {
+	return &computev1.Operation{
+		Name:        "test",
+		Description: "test",
+		Id:          uint64(12345),
+		Error: &computev1.OperationError{
+			Errors: []*computev1.OperationErrorErrors{
+				{
+					Code:    "555",
+					Message: "error",
+				},
+			},
+		},
+	}
+}
+
+func TestNewAlphaFeatureGate(t *testing.T) {
+	knownAlphaFeatures["foo"] = true
+	knownAlphaFeatures["bar"] = true
+
+	testCases := []struct {
+		alphaFeatures  []string
+		expectEnabled  []string
+		expectDisabled []string
+		expectError    bool
+	}{
+		// enable foo bar
+		{
+			alphaFeatures:  []string{"foo", "bar"},
+			expectEnabled:  []string{"foo", "bar"},
+			expectDisabled: []string{"aaa"},
+			expectError:    false,
+		},
+		// no alpha feature
+		{
+			alphaFeatures:  []string{},
+			expectEnabled:  []string{},
+			expectDisabled: []string{"foo", "bar"},
+			expectError:    false,
+		},
+		// unsupported alpha feature
+		{
+			alphaFeatures:  []string{"aaa", "foo"},
+			expectError:    true,
+			expectEnabled:  []string{"foo"},
+			expectDisabled: []string{"aaa"},
+		},
+		// enable foo
+		{
+			alphaFeatures:  []string{"foo"},
+			expectEnabled:  []string{"foo"},
+			expectDisabled: []string{"bar"},
+			expectError:    false,
+		},
+	}
+
+	for _, tc := range testCases {
+		featureGate, err := NewAlphaFeatureGate(tc.alphaFeatures)
+
+		if (tc.expectError && err == nil) || (!tc.expectError && err != nil) {
+			t.Errorf("Expect error to be %v, but got error %v", tc.expectError, err)
+		}
+
+		for _, key := range tc.expectEnabled {
+			if !featureGate.Enabled(key) {
+				t.Errorf("Expect %q to be enabled.", key)
+			}
+		}
+		for _, key := range tc.expectDisabled {
+			if featureGate.Enabled(key) {
+				t.Errorf("Expect %q to be disabled.", key)
+			}
+		}
+	}
+	delete(knownAlphaFeatures, "foo")
+	delete(knownAlphaFeatures, "bar")
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/metrics_test.go

@@ -0,0 +1,28 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestVerifyMetricLabelCardinality(t *testing.T) {
+	mc := newGenericMetricContext("foo", "get", "us-central1", "<n/a>", "alpha")
+	assert.Len(t, mc.attributes, len(metricLabels), "cardinalities of labels and values must match")
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/openstack/BUILD

@@ -0,0 +1,98 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "metadata.go",
+        "openstack.go",
+        "openstack_client.go",
+        "openstack_instances.go",
+        "openstack_loadbalancer.go",
+        "openstack_metrics.go",
+        "openstack_routes.go",
+        "openstack_volumes.go",
+    ],
+    deps = [
+        "//pkg/api/v1/helper:go_default_library",
+        "//pkg/api/v1/service:go_default_library",
+        "//pkg/cloudprovider:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//pkg/util/mount:go_default_library",
+        "//pkg/volume:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v1/apiversions:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v1/volumes:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/attachinterfaces:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/servers:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/trusts:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/identity/v3/tokens:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas/members:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas/monitors:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas/pools:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas/vips:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/listeners:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/loadbalancers:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/monitors:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/lbaas_v2/pools:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/rules:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/ports:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/pagination:go_default_library",
+        "//vendor/github.com/mitchellh/mapstructure:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//vendor/gopkg.in/gcfg.v1:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/client-go/util/cert:go_default_library",
+        "//vendor/k8s.io/utils/exec:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "metadata_test.go",
+        "openstack_routes_test.go",
+        "openstack_test.go",
+    ],
+    library = ":go_default_library",
+    deps = [
+        "//pkg/cloudprovider:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v1/apiversions:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/servers:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/rand:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/openstack/MAINTAINERS.md

@@ -0,0 +1,6 @@
+# Maintainers
+
+* [Angus Lees](https://github.com/anguslees)
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/pkg/cloudprovider/providers/openstack/MAINTAINERS.md?pixel)]()

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/openstack/OWNERS

@@ -0,0 +1,8 @@
+approvers:
+- anguslees
+- NickrenREN
+- dims
+reviewers:
+- anguslees
+- NickrenREN
+- dims

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/openstack/metadata.go

@@ -0,0 +1,156 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openstack
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/utils/exec"
+)
+
+// metadataUrl is URL to OpenStack metadata server. It's hardcoded IPv4
+// link-local address as documented in "OpenStack Cloud Administrator Guide",
+// chapter Compute - Networking with nova-network.
+// https://docs.openstack.org/admin-guide/compute-networking-nova.html#metadata-service
+const metadataUrl = "http://169.254.169.254/openstack/2012-08-10/meta_data.json"
+
+// Config drive is defined as an iso9660 or vfat (deprecated) drive
+// with the "config-2" label.
+// http://docs.openstack.org/user-guide/cli-config-drive.html
+const configDriveLabel = "config-2"
+const configDrivePath = "openstack/2012-08-10/meta_data.json"
+
+var ErrBadMetadata = errors.New("Invalid OpenStack metadata, got empty uuid")
+
+// Assumes the "2012-08-10" meta_data.json format.
+// See http://docs.openstack.org/user-guide/cli_config_drive.html
+type Metadata struct {
+	Uuid             string `json:"uuid"`
+	Name             string `json:"name"`
+	AvailabilityZone string `json:"availability_zone"`
+	// .. and other fields we don't care about.  Expand as necessary.
+}
+
+// parseMetadata reads JSON from OpenStack metadata server and parses
+// instance ID out of it.
+func parseMetadata(r io.Reader) (*Metadata, error) {
+	var metadata Metadata
+	json := json.NewDecoder(r)
+	if err := json.Decode(&metadata); err != nil {
+		return nil, err
+	}
+
+	if metadata.Uuid == "" {
+		return nil, ErrBadMetadata
+	}
+
+	return &metadata, nil
+}
+
+func getMetadataFromConfigDrive() (*Metadata, error) {
+	// Try to read instance UUID from config drive.
+	dev := "/dev/disk/by-label/" + configDriveLabel
+	if _, err := os.Stat(dev); os.IsNotExist(err) {
+		out, err := exec.New().Command(
+			"blkid", "-l",
+			"-t", "LABEL="+configDriveLabel,
+			"-o", "device",
+		).CombinedOutput()
+		if err != nil {
+			glog.V(2).Infof("Unable to run blkid: %v", err)
+			return nil, err
+		}
+		dev = strings.TrimSpace(string(out))
+	}
+
+	mntdir, err := ioutil.TempDir("", "configdrive")
+	if err != nil {
+		return nil, err
+	}
+	defer os.Remove(mntdir)
+
+	glog.V(4).Infof("Attempting to mount configdrive %s on %s", dev, mntdir)
+
+	mounter := mount.New("" /* default mount path */)
+	err = mounter.Mount(dev, mntdir, "iso9660", []string{"ro"})
+	if err != nil {
+		err = mounter.Mount(dev, mntdir, "vfat", []string{"ro"})
+	}
+	if err != nil {
+		glog.Errorf("Error mounting configdrive %s: %v", dev, err)
+		return nil, err
+	}
+	defer mounter.Unmount(mntdir)
+
+	glog.V(4).Infof("Configdrive mounted on %s", mntdir)
+
+	f, err := os.Open(
+		filepath.Join(mntdir, configDrivePath))
+	if err != nil {
+		glog.Errorf("Error reading %s on config drive: %v", configDrivePath, err)
+		return nil, err
+	}
+	defer f.Close()
+
+	return parseMetadata(f)
+}
+
+func getMetadataFromMetadataService() (*Metadata, error) {
+	// Try to get JSON from metdata server.
+	glog.V(4).Infof("Attempting to fetch metadata from %s", metadataUrl)
+	resp, err := http.Get(metadataUrl)
+	if err != nil {
+		glog.V(3).Infof("Cannot read %s: %v", metadataUrl, err)
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		err = fmt.Errorf("Unexpected status code when reading metadata from %s: %s", metadataUrl, resp.Status)
+		glog.V(3).Infof("%v", err)
+		return nil, err
+	}
+
+	return parseMetadata(resp.Body)
+}
+
+// Metadata is fixed for the current host, so cache the value process-wide
+var metadataCache *Metadata
+
+func getMetadata() (*Metadata, error) {
+	if metadataCache == nil {
+		md, err := getMetadataFromConfigDrive()
+		if err != nil {
+			md, err = getMetadataFromMetadataService()
+		}
+		if err != nil {
+			return nil, err
+		}
+		metadataCache = md
+	}
+	return metadataCache, nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/openstack/metadata_test.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openstack
+
+import (
+	"strings"
+	"testing"
+)
+
+var FakeMetadata = Metadata{
+	Uuid:             "83679162-1378-4288-a2d4-70e13ec132aa",
+	Name:             "test",
+	AvailabilityZone: "nova",
+}
+
+func SetMetadataFixture(value *Metadata) {
+	metadataCache = value
+}
+
+func ClearMetadata() {
+	metadataCache = nil
+}
+
+func TestParseMetadata(t *testing.T) {
+	_, err := parseMetadata(strings.NewReader("bogus"))
+	if err == nil {
+		t.Errorf("Should fail when bad data is provided: %s", err)
+	}
+
+	data := strings.NewReader(`
+{
+    "availability_zone": "nova",
+    "files": [
+        {
+            "content_path": "/content/0000",
+            "path": "/etc/network/interfaces"
+        },
+        {
+            "content_path": "/content/0001",
+            "path": "known_hosts"
+        }
+    ],
+    "hostname": "test.novalocal",
+    "launch_index": 0,
+    "name": "test",
+    "meta": {
+        "role": "webservers",
+        "essential": "false"
+    },
+    "public_keys": {
+        "mykey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQDBqUfVvCSez0/Wfpd8dLLgZXV9GtXQ7hnMN+Z0OWQUyebVEHey1CXuin0uY1cAJMhUq8j98SiW+cU0sU4J3x5l2+xi1bodDm1BtFWVeLIOQINpfV1n8fKjHB+ynPpe1F6tMDvrFGUlJs44t30BrujMXBe8Rq44cCk6wqyjATA3rQ== Generated by Nova\n"
+    },
+    "uuid": "83679162-1378-4288-a2d4-70e13ec132aa"
+}
+`)
+	md, err := parseMetadata(data)
+	if err != nil {
+		t.Fatalf("Should succeed when provided with valid data: %s", err)
+	}
+
+	if md.Name != "test" {
+		t.Errorf("incorrect name: %s", md.Name)
+	}
+
+	if md.Uuid != "83679162-1378-4288-a2d4-70e13ec132aa" {
+		t.Errorf("incorrect uuid: %s", md.Uuid)
+	}
+
+	if md.AvailabilityZone != "nova" {
+		t.Errorf("incorrect az: %s", md.AvailabilityZone)
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/openstack/openstack.go

@@ -0,0 +1,747 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openstack
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/gophercloud/gophercloud"
+	"github.com/gophercloud/gophercloud/openstack"
+	apiversions_v1 "github.com/gophercloud/gophercloud/openstack/blockstorage/v1/apiversions"
+	"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/attachinterfaces"
+	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
+	"github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/trusts"
+	tokens3 "github.com/gophercloud/gophercloud/openstack/identity/v3/tokens"
+	"github.com/gophercloud/gophercloud/pagination"
+	"github.com/mitchellh/mapstructure"
+	"gopkg.in/gcfg.v1"
+
+	"github.com/golang/glog"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	netutil "k8s.io/apimachinery/pkg/util/net"
+	certutil "k8s.io/client-go/util/cert"
+	v1helper "k8s.io/kubernetes/pkg/api/v1/helper"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+	"k8s.io/kubernetes/pkg/controller"
+)
+
+const (
+	ProviderName     = "openstack"
+	AvailabilityZone = "availability_zone"
+)
+
+var ErrNotFound = errors.New("Failed to find object")
+var ErrMultipleResults = errors.New("Multiple results where only one expected")
+var ErrNoAddressFound = errors.New("No address found for host")
+
+// encoding.TextUnmarshaler interface for time.Duration
+type MyDuration struct {
+	time.Duration
+}
+
+func (d *MyDuration) UnmarshalText(text []byte) error {
+	res, err := time.ParseDuration(string(text))
+	if err != nil {
+		return err
+	}
+	d.Duration = res
+	return nil
+}
+
+type LoadBalancer struct {
+	network *gophercloud.ServiceClient
+	compute *gophercloud.ServiceClient
+	opts    LoadBalancerOpts
+}
+
+type LoadBalancerOpts struct {
+	LBVersion            string     `gcfg:"lb-version"`          // overrides autodetection. v1 or v2
+	SubnetId             string     `gcfg:"subnet-id"`           // overrides autodetection.
+	FloatingNetworkId    string     `gcfg:"floating-network-id"` // If specified, will create floating ip for loadbalancer, or do not create floating ip.
+	LBMethod             string     `gcfg:"lb-method"`           // default to ROUND_ROBIN.
+	CreateMonitor        bool       `gcfg:"create-monitor"`
+	MonitorDelay         MyDuration `gcfg:"monitor-delay"`
+	MonitorTimeout       MyDuration `gcfg:"monitor-timeout"`
+	MonitorMaxRetries    uint       `gcfg:"monitor-max-retries"`
+	ManageSecurityGroups bool       `gcfg:"manage-security-groups"`
+	NodeSecurityGroupID  string     `gcfg:"node-security-group"`
+}
+
+type BlockStorageOpts struct {
+	BSVersion       string `gcfg:"bs-version"`        // overrides autodetection. v1 or v2. Defaults to auto
+	TrustDevicePath bool   `gcfg:"trust-device-path"` // See Issue #33128
+}
+
+type RouterOpts struct {
+	RouterId string `gcfg:"router-id"` // required
+}
+
+// OpenStack is an implementation of cloud provider Interface for OpenStack.
+type OpenStack struct {
+	provider  *gophercloud.ProviderClient
+	region    string
+	lbOpts    LoadBalancerOpts
+	bsOpts    BlockStorageOpts
+	routeOpts RouterOpts
+	// InstanceID of the server where this OpenStack object is instantiated.
+	localInstanceID string
+}
+
+type Config struct {
+	Global struct {
+		AuthUrl    string `gcfg:"auth-url"`
+		Username   string
+		UserId     string `gcfg:"user-id"`
+		Password   string
+		TenantId   string `gcfg:"tenant-id"`
+		TenantName string `gcfg:"tenant-name"`
+		TrustId    string `gcfg:"trust-id"`
+		DomainId   string `gcfg:"domain-id"`
+		DomainName string `gcfg:"domain-name"`
+		Region     string
+		CAFile     string `gcfg:"ca-file"`
+	}
+	LoadBalancer LoadBalancerOpts
+	BlockStorage BlockStorageOpts
+	Route        RouterOpts
+}
+
+func init() {
+	RegisterMetrics()
+
+	cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) {
+		cfg, err := readConfig(config)
+		if err != nil {
+			return nil, err
+		}
+		return newOpenStack(cfg)
+	})
+}
+
+func (cfg Config) toAuthOptions() gophercloud.AuthOptions {
+	return gophercloud.AuthOptions{
+		IdentityEndpoint: cfg.Global.AuthUrl,
+		Username:         cfg.Global.Username,
+		UserID:           cfg.Global.UserId,
+		Password:         cfg.Global.Password,
+		TenantID:         cfg.Global.TenantId,
+		TenantName:       cfg.Global.TenantName,
+		DomainID:         cfg.Global.DomainId,
+		DomainName:       cfg.Global.DomainName,
+
+		// Persistent service, so we need to be able to renew tokens.
+		AllowReauth: true,
+	}
+}
+
+func (cfg Config) toAuth3Options() tokens3.AuthOptions {
+	return tokens3.AuthOptions{
+		IdentityEndpoint: cfg.Global.AuthUrl,
+		Username:         cfg.Global.Username,
+		UserID:           cfg.Global.UserId,
+		Password:         cfg.Global.Password,
+		DomainID:         cfg.Global.DomainId,
+		DomainName:       cfg.Global.DomainName,
+		AllowReauth:      true,
+	}
+}
+
+func readConfig(config io.Reader) (Config, error) {
+	if config == nil {
+		err := fmt.Errorf("no OpenStack cloud provider config file given")
+		return Config{}, err
+	}
+
+	var cfg Config
+
+	// Set default values for config params
+	cfg.BlockStorage.BSVersion = "auto"
+	cfg.BlockStorage.TrustDevicePath = false
+
+	err := gcfg.ReadInto(&cfg, config)
+	return cfg, err
+}
+
+// Tiny helper for conditional unwind logic
+type Caller bool
+
+func NewCaller() Caller   { return Caller(true) }
+func (c *Caller) Disarm() { *c = false }
+
+func (c *Caller) Call(f func()) {
+	if *c {
+		f()
+	}
+}
+
+func readInstanceID() (string, error) {
+	// Try to find instance ID on the local filesystem (created by cloud-init)
+	const instanceIDFile = "/var/lib/cloud/data/instance-id"
+	idBytes, err := ioutil.ReadFile(instanceIDFile)
+	if err == nil {
+		instanceID := string(idBytes)
+		instanceID = strings.TrimSpace(instanceID)
+		glog.V(3).Infof("Got instance id from %s: %s", instanceIDFile, instanceID)
+		if instanceID != "" {
+			return instanceID, nil
+		}
+		// Fall through to metadata server lookup
+	}
+
+	md, err := getMetadata()
+	if err != nil {
+		return "", err
+	}
+
+	return md.Uuid, nil
+}
+
+// check opts for OpenStack
+func checkOpenStackOpts(openstackOpts *OpenStack) error {
+	lbOpts := openstackOpts.lbOpts
+
+	// if need to create health monitor for Neutron LB,
+	// monitor-delay, monitor-timeout and monitor-max-retries should be set.
+	emptyDuration := MyDuration{}
+	if lbOpts.CreateMonitor {
+		if lbOpts.MonitorDelay == emptyDuration {
+			return fmt.Errorf("monitor-delay not set in cloud provider config")
+		}
+		if lbOpts.MonitorTimeout == emptyDuration {
+			return fmt.Errorf("monitor-timeout not set in cloud provider config")
+		}
+		if lbOpts.MonitorMaxRetries == uint(0) {
+			return fmt.Errorf("monitor-max-retries not set in cloud provider config")
+		}
+	}
+
+	// if enable ManageSecurityGroups, node-security-group should be set.
+	if lbOpts.ManageSecurityGroups {
+		if len(lbOpts.NodeSecurityGroupID) == 0 {
+			return fmt.Errorf("node-security-group not set in cloud provider config")
+		}
+	}
+
+	return nil
+}
+
+func newOpenStack(cfg Config) (*OpenStack, error) {
+	provider, err := openstack.NewClient(cfg.Global.AuthUrl)
+	if err != nil {
+		return nil, err
+	}
+	if cfg.Global.CAFile != "" {
+		roots, err := certutil.NewPool(cfg.Global.CAFile)
+		if err != nil {
+			return nil, err
+		}
+		config := &tls.Config{}
+		config.RootCAs = roots
+		provider.HTTPClient.Transport = netutil.SetOldTransportDefaults(&http.Transport{TLSClientConfig: config})
+
+	}
+	if cfg.Global.TrustId != "" {
+		opts := cfg.toAuth3Options()
+		authOptsExt := trusts.AuthOptsExt{
+			TrustID:            cfg.Global.TrustId,
+			AuthOptionsBuilder: &opts,
+		}
+		err = openstack.AuthenticateV3(provider, authOptsExt, gophercloud.EndpointOpts{})
+	} else {
+		err = openstack.Authenticate(provider, cfg.toAuthOptions())
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	os := OpenStack{
+		provider:  provider,
+		region:    cfg.Global.Region,
+		lbOpts:    cfg.LoadBalancer,
+		bsOpts:    cfg.BlockStorage,
+		routeOpts: cfg.Route,
+	}
+
+	err = checkOpenStackOpts(&os)
+	if err != nil {
+		return nil, err
+	}
+
+	return &os, nil
+}
+
+// Initialize passes a Kubernetes clientBuilder interface to the cloud provider
+func (os *OpenStack) Initialize(clientBuilder controller.ControllerClientBuilder) {}
+
+// mapNodeNameToServerName maps a k8s NodeName to an OpenStack Server Name
+// This is a simple string cast.
+func mapNodeNameToServerName(nodeName types.NodeName) string {
+	return string(nodeName)
+}
+
+// mapServerToNodeName maps an OpenStack Server to a k8s NodeName
+func mapServerToNodeName(server *servers.Server) types.NodeName {
+	// Node names are always lowercase, and (at least)
+	// routecontroller does case-sensitive string comparisons
+	// assuming this
+	return types.NodeName(strings.ToLower(server.Name))
+}
+
+func foreachServer(client *gophercloud.ServiceClient, opts servers.ListOptsBuilder, handler func(*servers.Server) (bool, error)) error {
+	pager := servers.List(client, opts)
+
+	err := pager.EachPage(func(page pagination.Page) (bool, error) {
+		s, err := servers.ExtractServers(page)
+		if err != nil {
+			return false, err
+		}
+		for _, server := range s {
+			ok, err := handler(&server)
+			if !ok || err != nil {
+				return false, err
+			}
+		}
+		return true, nil
+	})
+	return err
+}
+
+func getServerByName(client *gophercloud.ServiceClient, name types.NodeName) (*servers.Server, error) {
+	opts := servers.ListOpts{
+		Name:   fmt.Sprintf("^%s$", regexp.QuoteMeta(mapNodeNameToServerName(name))),
+		Status: "ACTIVE",
+	}
+	pager := servers.List(client, opts)
+
+	serverList := make([]servers.Server, 0, 1)
+
+	err := pager.EachPage(func(page pagination.Page) (bool, error) {
+		s, err := servers.ExtractServers(page)
+		if err != nil {
+			return false, err
+		}
+		serverList = append(serverList, s...)
+		if len(serverList) > 1 {
+			return false, ErrMultipleResults
+		}
+		return true, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if len(serverList) == 0 {
+		return nil, ErrNotFound
+	}
+
+	return &serverList[0], nil
+}
+
+func nodeAddresses(srv *servers.Server) ([]v1.NodeAddress, error) {
+	addrs := []v1.NodeAddress{}
+
+	type Address struct {
+		IpType string `mapstructure:"OS-EXT-IPS:type"`
+		Addr   string
+	}
+
+	var addresses map[string][]Address
+	err := mapstructure.Decode(srv.Addresses, &addresses)
+	if err != nil {
+		return nil, err
+	}
+
+	for network, addrList := range addresses {
+		for _, props := range addrList {
+			var addressType v1.NodeAddressType
+			if props.IpType == "floating" || network == "public" {
+				addressType = v1.NodeExternalIP
+			} else {
+				addressType = v1.NodeInternalIP
+			}
+
+			v1helper.AddToNodeAddresses(&addrs,
+				v1.NodeAddress{
+					Type:    addressType,
+					Address: props.Addr,
+				},
+			)
+		}
+	}
+
+	// AccessIPs are usually duplicates of "public" addresses.
+	if srv.AccessIPv4 != "" {
+		v1helper.AddToNodeAddresses(&addrs,
+			v1.NodeAddress{
+				Type:    v1.NodeExternalIP,
+				Address: srv.AccessIPv4,
+			},
+		)
+	}
+
+	if srv.AccessIPv6 != "" {
+		v1helper.AddToNodeAddresses(&addrs,
+			v1.NodeAddress{
+				Type:    v1.NodeExternalIP,
+				Address: srv.AccessIPv6,
+			},
+		)
+	}
+
+	return addrs, nil
+}
+
+func getAddressesByName(client *gophercloud.ServiceClient, name types.NodeName) ([]v1.NodeAddress, error) {
+	srv, err := getServerByName(client, name)
+	if err != nil {
+		return nil, err
+	}
+
+	return nodeAddresses(srv)
+}
+
+func getAddressByName(client *gophercloud.ServiceClient, name types.NodeName) (string, error) {
+	addrs, err := getAddressesByName(client, name)
+	if err != nil {
+		return "", err
+	} else if len(addrs) == 0 {
+		return "", ErrNoAddressFound
+	}
+
+	for _, addr := range addrs {
+		if addr.Type == v1.NodeInternalIP {
+			return addr.Address, nil
+		}
+	}
+
+	return addrs[0].Address, nil
+}
+
+// getAttachedInterfacesByID returns the node interfaces of the specified instance.
+func getAttachedInterfacesByID(client *gophercloud.ServiceClient, serviceID string) ([]attachinterfaces.Interface, error) {
+	var interfaces []attachinterfaces.Interface
+
+	pager := attachinterfaces.List(client, serviceID)
+	err := pager.EachPage(func(page pagination.Page) (bool, error) {
+		s, err := attachinterfaces.ExtractInterfaces(page)
+		if err != nil {
+			return false, err
+		}
+		interfaces = append(interfaces, s...)
+		return true, nil
+	})
+	if err != nil {
+		return interfaces, err
+	}
+
+	return interfaces, nil
+}
+
+func (os *OpenStack) Clusters() (cloudprovider.Clusters, bool) {
+	return nil, false
+}
+
+// ProviderName returns the cloud provider ID.
+func (os *OpenStack) ProviderName() string {
+	return ProviderName
+}
+
+// ScrubDNS filters DNS settings for pods.
+func (os *OpenStack) ScrubDNS(nameServers, searches []string) ([]string, []string) {
+	return nameServers, searches
+}
+
+// HasClusterID returns true if the cluster has a clusterID
+func (os *OpenStack) HasClusterID() bool {
+	return true
+}
+
+func (os *OpenStack) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
+	glog.V(4).Info("openstack.LoadBalancer() called")
+
+	// TODO: Search for and support Rackspace loadbalancer API, and others.
+	network, err := os.NewNetworkV2()
+	if err != nil {
+		return nil, false
+	}
+
+	compute, err := os.NewComputeV2()
+	if err != nil {
+		return nil, false
+	}
+
+	lbVersion := os.lbOpts.LBVersion
+	if lbVersion == "" {
+		// No version specified, try newest supported by server
+		netExts, err := networkExtensions(network)
+		if err != nil {
+			glog.Warningf("Failed to list neutron extensions: %v", err)
+			return nil, false
+		}
+
+		if netExts["lbaasv2"] {
+			lbVersion = "v2"
+		} else if netExts["lbaas"] {
+			lbVersion = "v1"
+		} else {
+			glog.Warningf("Failed to find neutron LBaaS extension (v1 or v2)")
+			return nil, false
+		}
+		glog.V(3).Infof("Using LBaaS extension %v", lbVersion)
+	}
+
+	glog.V(1).Info("Claiming to support LoadBalancer")
+
+	if lbVersion == "v2" {
+		return &LbaasV2{LoadBalancer{network, compute, os.lbOpts}}, true
+	} else if lbVersion == "v1" {
+		// Since LBaaS v1 is deprecated in the OpenStack Liberty release, so deprecate LBaaSV1 at V1.8, then remove LBaaSV1 after V1.9.
+		// Reference OpenStack doc:	https://docs.openstack.org/mitaka/networking-guide/config-lbaas.html
+		glog.Warningf("The LBaaS v1 of OpenStack cloud provider has been deprecated, Please use LBaaS v2")
+		return &LbaasV1{LoadBalancer{network, compute, os.lbOpts}}, true
+	} else {
+		glog.Warningf("Config error: unrecognised lb-version \"%v\"", lbVersion)
+		return nil, false
+	}
+}
+
+func isNotFound(err error) bool {
+	e, ok := err.(*gophercloud.ErrUnexpectedResponseCode)
+	return ok && e.Actual == http.StatusNotFound
+}
+
+func (os *OpenStack) Zones() (cloudprovider.Zones, bool) {
+	glog.V(1).Info("Claiming to support Zones")
+
+	return os, true
+}
+
+func (os *OpenStack) GetZone() (cloudprovider.Zone, error) {
+	md, err := getMetadata()
+	if err != nil {
+		return cloudprovider.Zone{}, err
+	}
+
+	zone := cloudprovider.Zone{
+		FailureDomain: md.AvailabilityZone,
+		Region:        os.region,
+	}
+	glog.V(1).Infof("Current zone is %v", zone)
+
+	return zone, nil
+}
+
+// GetZoneByProviderID implements Zones.GetZoneByProviderID
+// This is particularly useful in external cloud providers where the kubelet
+// does not initialize node data.
+func (os *OpenStack) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) {
+	instanceID, err := instanceIDFromProviderID(providerID)
+	if err != nil {
+		return cloudprovider.Zone{}, err
+	}
+
+	compute, err := os.NewComputeV2()
+	if err != nil {
+		return cloudprovider.Zone{}, err
+	}
+
+	srv, err := servers.Get(compute, instanceID).Extract()
+	if err != nil {
+		return cloudprovider.Zone{}, err
+	}
+
+	zone := cloudprovider.Zone{
+		FailureDomain: srv.Metadata[AvailabilityZone],
+		Region:        os.region,
+	}
+	glog.V(4).Infof("The instance %s in zone %v", srv.Name, zone)
+
+	return zone, nil
+}
+
+// GetZoneByNodeName implements Zones.GetZoneByNodeName
+// This is particularly useful in external cloud providers where the kubelet
+// does not initialize node data.
+func (os *OpenStack) GetZoneByNodeName(nodeName types.NodeName) (cloudprovider.Zone, error) {
+	compute, err := os.NewComputeV2()
+	if err != nil {
+		return cloudprovider.Zone{}, err
+	}
+
+	srv, err := getServerByName(compute, nodeName)
+	if err != nil {
+		if err == ErrNotFound {
+			return cloudprovider.Zone{}, cloudprovider.InstanceNotFound
+		}
+		return cloudprovider.Zone{}, err
+	}
+
+	zone := cloudprovider.Zone{
+		FailureDomain: srv.Metadata[AvailabilityZone],
+		Region:        os.region,
+	}
+	glog.V(4).Infof("The instance %s in zone %v", srv.Name, zone)
+
+	return zone, nil
+}
+
+func (os *OpenStack) Routes() (cloudprovider.Routes, bool) {
+	glog.V(4).Info("openstack.Routes() called")
+
+	network, err := os.NewNetworkV2()
+	if err != nil {
+		return nil, false
+	}
+
+	netExts, err := networkExtensions(network)
+	if err != nil {
+		glog.Warningf("Failed to list neutron extensions: %v", err)
+		return nil, false
+	}
+
+	if !netExts["extraroute"] {
+		glog.V(3).Infof("Neutron extraroute extension not found, required for Routes support")
+		return nil, false
+	}
+
+	compute, err := os.NewComputeV2()
+	if err != nil {
+		return nil, false
+	}
+
+	r, err := NewRoutes(compute, network, os.routeOpts)
+	if err != nil {
+		glog.Warningf("Error initialising Routes support: %v", err)
+		return nil, false
+	}
+
+	glog.V(1).Info("Claiming to support Routes")
+
+	return r, true
+}
+
+// Implementation of sort interface for blockstorage version probing
+type APIVersionsByID []apiversions_v1.APIVersion
+
+func (apiVersions APIVersionsByID) Len() int {
+	return len(apiVersions)
+}
+
+func (apiVersions APIVersionsByID) Swap(i, j int) {
+	apiVersions[i], apiVersions[j] = apiVersions[j], apiVersions[i]
+}
+
+func (apiVersions APIVersionsByID) Less(i, j int) bool {
+	return apiVersions[i].ID > apiVersions[j].ID
+}
+
+func autoVersionSelector(apiVersion *apiversions_v1.APIVersion) string {
+	switch strings.ToLower(apiVersion.ID) {
+	case "v2.0":
+		return "v2"
+	case "v1.0":
+		return "v1"
+	default:
+		return ""
+	}
+}
+
+func doBsApiVersionAutodetect(availableApiVersions []apiversions_v1.APIVersion) string {
+	sort.Sort(APIVersionsByID(availableApiVersions))
+	for _, status := range []string{"CURRENT", "SUPPORTED"} {
+		for _, version := range availableApiVersions {
+			if strings.ToUpper(version.Status) == status {
+				if detectedApiVersion := autoVersionSelector(&version); detectedApiVersion != "" {
+					glog.V(3).Infof("Blockstorage API version probing has found a suitable %s api version: %s", status, detectedApiVersion)
+					return detectedApiVersion
+				}
+			}
+		}
+	}
+
+	return ""
+
+}
+
+func (os *OpenStack) volumeService(forceVersion string) (volumeService, error) {
+	bsVersion := ""
+	if forceVersion == "" {
+		bsVersion = os.bsOpts.BSVersion
+	} else {
+		bsVersion = forceVersion
+	}
+
+	switch bsVersion {
+	case "v1":
+		sClient, err := os.NewBlockStorageV1()
+		if err != nil {
+			return nil, err
+		}
+		return &VolumesV1{sClient, os.bsOpts}, nil
+	case "v2":
+		sClient, err := os.NewBlockStorageV2()
+		if err != nil {
+			return nil, err
+		}
+		return &VolumesV2{sClient, os.bsOpts}, nil
+	case "auto":
+		sClient, err := os.NewBlockStorageV1()
+		if err != nil {
+			return nil, err
+		}
+		availableApiVersions := []apiversions_v1.APIVersion{}
+		err = apiversions_v1.List(sClient).EachPage(func(page pagination.Page) (bool, error) {
+			// returning false from this handler stops page iteration, error is propagated to the upper function
+			apiversions, err := apiversions_v1.ExtractAPIVersions(page)
+			if err != nil {
+				glog.Errorf("Unable to extract api versions from page: %v", err)
+				return false, err
+			}
+			availableApiVersions = append(availableApiVersions, apiversions...)
+			return true, nil
+		})
+
+		if err != nil {
+			glog.Errorf("Error when retrieving list of supported blockstorage api versions: %v", err)
+			return nil, err
+		}
+		if autodetectedVersion := doBsApiVersionAutodetect(availableApiVersions); autodetectedVersion != "" {
+			return os.volumeService(autodetectedVersion)
+		} else {
+			// Nothing suitable found, failed autodetection, just exit with appropriate message
+			err_txt := "BlockStorage API version autodetection failed. " +
+				"Please set it explicitly in cloud.conf in section [BlockStorage] with key `bs-version`"
+			return nil, errors.New(err_txt)
+		}
+
+	default:
+		err_txt := fmt.Sprintf("Config error: unrecognised bs-version \"%v\"", os.bsOpts.BSVersion)
+		glog.Warningf(err_txt)
+		return nil, errors.New(err_txt)
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/openstack/openstack_client.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openstack
+
+import (
+	"github.com/gophercloud/gophercloud"
+	"github.com/gophercloud/gophercloud/openstack"
+
+	"github.com/golang/glog"
+)
+
+func (os *OpenStack) NewNetworkV2() (*gophercloud.ServiceClient, error) {
+	network, err := openstack.NewNetworkV2(os.provider, gophercloud.EndpointOpts{
+		Region: os.region,
+	})
+	if err != nil {
+		glog.Warningf("Failed to find network v2 endpoint for region %s: %v", os.region, err)
+		return nil, err
+	}
+	return network, nil
+}
+
+func (os *OpenStack) NewComputeV2() (*gophercloud.ServiceClient, error) {
+	compute, err := openstack.NewComputeV2(os.provider, gophercloud.EndpointOpts{
+		Region: os.region,
+	})
+	if err != nil {
+		glog.Warningf("Failed to find compute v2 endpoint for region %s: %v", os.region, err)
+		return nil, err
+	}
+	return compute, nil
+}
+
+func (os *OpenStack) NewBlockStorageV1() (*gophercloud.ServiceClient, error) {
+	storage, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{
+		Region: os.region,
+	})
+	if err != nil {
+		glog.Errorf("Unable to initialize cinder v1 client for region %s: %v", os.region, err)
+		return nil, err
+	}
+	return storage, nil
+}
+
+func (os *OpenStack) NewBlockStorageV2() (*gophercloud.ServiceClient, error) {
+	storage, err := openstack.NewBlockStorageV2(os.provider, gophercloud.EndpointOpts{
+		Region: os.region,
+	})
+	if err != nil {
+		glog.Errorf("Unable to initialize cinder v2 client for region %s: %v", os.region, err)
+		return nil, err
+	}
+	return storage, nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/openstack/openstack_instances.go

@@ -0,0 +1,201 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openstack
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+
+	"github.com/golang/glog"
+	"github.com/gophercloud/gophercloud"
+	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+)
+
+type Instances struct {
+	compute *gophercloud.ServiceClient
+}
+
+// Instances returns an implementation of Instances for OpenStack.
+func (os *OpenStack) Instances() (cloudprovider.Instances, bool) {
+	glog.V(4).Info("openstack.Instances() called")
+
+	compute, err := os.NewComputeV2()
+	if err != nil {
+		return nil, false
+	}
+
+	glog.V(1).Info("Claiming to support Instances")
+
+	return &Instances{compute}, true
+}
+
+// Implementation of Instances.CurrentNodeName
+// Note this is *not* necessarily the same as hostname.
+func (i *Instances) CurrentNodeName(hostname string) (types.NodeName, error) {
+	md, err := getMetadata()
+	if err != nil {
+		return "", err
+	}
+	return types.NodeName(md.Name), nil
+}
+
+func (i *Instances) AddSSHKeyToAllInstances(user string, keyData []byte) error {
+	return errors.New("unimplemented")
+}
+
+func (i *Instances) NodeAddresses(name types.NodeName) ([]v1.NodeAddress, error) {
+	glog.V(4).Infof("NodeAddresses(%v) called", name)
+
+	addrs, err := getAddressesByName(i.compute, name)
+	if err != nil {
+		return nil, err
+	}
+
+	glog.V(4).Infof("NodeAddresses(%v) => %v", name, addrs)
+	return addrs, nil
+}
+
+// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID
+// This method will not be called from the node that is requesting this ID. i.e. metadata service
+// and other local methods cannot be used here
+func (i *Instances) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) {
+	instanceID, err := instanceIDFromProviderID(providerID)
+
+	if err != nil {
+		return []v1.NodeAddress{}, err
+	}
+
+	server, err := servers.Get(i.compute, instanceID).Extract()
+
+	if err != nil {
+		return []v1.NodeAddress{}, err
+	}
+
+	addresses, err := nodeAddresses(server)
+	if err != nil {
+		return []v1.NodeAddress{}, err
+	}
+
+	return addresses, nil
+}
+
+// ExternalID returns the cloud provider ID of the specified instance (deprecated).
+func (i *Instances) ExternalID(name types.NodeName) (string, error) {
+	srv, err := getServerByName(i.compute, name)
+	if err != nil {
+		if err == ErrNotFound {
+			return "", cloudprovider.InstanceNotFound
+		}
+		return "", err
+	}
+	return srv.ID, nil
+}
+
+// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running.
+// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager.
+func (i *Instances) InstanceExistsByProviderID(providerID string) (bool, error) {
+	return false, errors.New("unimplemented")
+}
+
+// InstanceID returns the kubelet's cloud provider ID.
+func (os *OpenStack) InstanceID() (string, error) {
+	if len(os.localInstanceID) == 0 {
+		id, err := readInstanceID()
+		if err != nil {
+			return "", err
+		}
+		os.localInstanceID = id
+	}
+	return os.localInstanceID, nil
+}
+
+// InstanceID returns the cloud provider ID of the specified instance.
+func (i *Instances) InstanceID(name types.NodeName) (string, error) {
+	srv, err := getServerByName(i.compute, name)
+	if err != nil {
+		if err == ErrNotFound {
+			return "", cloudprovider.InstanceNotFound
+		}
+		return "", err
+	}
+	// In the future it is possible to also return an endpoint as:
+	// <endpoint>/<instanceid>
+	return "/" + srv.ID, nil
+}
+
+// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID
+// This method will not be called from the node that is requesting this ID. i.e. metadata service
+// and other local methods cannot be used here
+func (i *Instances) InstanceTypeByProviderID(providerID string) (string, error) {
+	instanceID, err := instanceIDFromProviderID(providerID)
+
+	if err != nil {
+		return "", err
+	}
+
+	server, err := servers.Get(i.compute, instanceID).Extract()
+
+	if err != nil {
+		return "", err
+	}
+
+	return srvInstanceType(server)
+}
+
+// InstanceType returns the type of the specified instance.
+func (i *Instances) InstanceType(name types.NodeName) (string, error) {
+	srv, err := getServerByName(i.compute, name)
+
+	if err != nil {
+		return "", err
+	}
+
+	return srvInstanceType(srv)
+}
+
+func srvInstanceType(srv *servers.Server) (string, error) {
+	keys := []string{"name", "id", "original_name"}
+	for _, key := range keys {
+		val, found := srv.Flavor[key]
+		if found {
+			flavor, ok := val.(string)
+			if ok {
+				return flavor, nil
+			}
+		}
+	}
+	return "", fmt.Errorf("flavor name/id not found")
+}
+
+// instanceIDFromProviderID splits a provider's id and return instanceID.
+// A providerID is build out of '${ProviderName}:///${instance-id}'which contains ':///'.
+// See cloudprovider.GetInstanceProviderID and Instances.InstanceID.
+func instanceIDFromProviderID(providerID string) (instanceID string, err error) {
+	// If Instances.InstanceID or cloudprovider.GetInstanceProviderID is changed, the regexp should be changed too.
+	var providerIdRegexp = regexp.MustCompile(`^` + ProviderName + `:///([^/]+)$`)
+
+	matches := providerIdRegexp.FindStringSubmatch(providerID)
+	if len(matches) != 2 {
+		return "", fmt.Errorf("ProviderID \"%s\" didn't match expected format \"openstack:///InstanceID\"", providerID)
+	}
+	return matches[1], nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/openstack/openstack_metrics.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openstack
+
+import "github.com/prometheus/client_golang/prometheus"
+
+const (
+	OpenstackSubsystem         = "openstack"
+	OpenstackOperationKey      = "cloudprovider_openstack_api_request_duration_seconds"
+	OpenstackOperationErrorKey = "cloudprovider_openstack_api_request_errors"
+)
+
+var (
+	OpenstackOperationsLatency = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Subsystem: OpenstackSubsystem,
+			Name:      OpenstackOperationKey,
+			Help:      "Latency of openstack api call",
+		},
+		[]string{"request"},
+	)
+
+	OpenstackApiRequestErrors = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: OpenstackSubsystem,
+			Name:      OpenstackOperationErrorKey,
+			Help:      "Cumulative number of openstack Api call errors",
+		},
+		[]string{"request"},
+	)
+)
+
+func RegisterMetrics() {
+	prometheus.MustRegister(OpenstackOperationsLatency)
+	prometheus.MustRegister(OpenstackApiRequestErrors)
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/openstack/openstack_routes.go

@@ -0,0 +1,323 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openstack
+
+import (
+	"errors"
+
+	"github.com/gophercloud/gophercloud"
+	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
+	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers"
+	neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
+
+	"github.com/golang/glog"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+)
+
+var ErrNoRouterId = errors.New("router-id not set in cloud provider config")
+
+type Routes struct {
+	compute *gophercloud.ServiceClient
+	network *gophercloud.ServiceClient
+	opts    RouterOpts
+}
+
+func NewRoutes(compute *gophercloud.ServiceClient, network *gophercloud.ServiceClient, opts RouterOpts) (cloudprovider.Routes, error) {
+	if opts.RouterId == "" {
+		return nil, ErrNoRouterId
+	}
+
+	return &Routes{
+		compute: compute,
+		network: network,
+		opts:    opts,
+	}, nil
+}
+
+func (r *Routes) ListRoutes(clusterName string) ([]*cloudprovider.Route, error) {
+	glog.V(4).Infof("ListRoutes(%v)", clusterName)
+
+	nodeNamesByAddr := make(map[string]types.NodeName)
+	err := foreachServer(r.compute, servers.ListOpts{Status: "ACTIVE"}, func(srv *servers.Server) (bool, error) {
+		addrs, err := nodeAddresses(srv)
+		if err != nil {
+			return false, err
+		}
+
+		name := mapServerToNodeName(srv)
+		for _, addr := range addrs {
+			nodeNamesByAddr[addr.Address] = name
+		}
+
+		return true, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	router, err := routers.Get(r.network, r.opts.RouterId).Extract()
+	if err != nil {
+		return nil, err
+	}
+
+	var routes []*cloudprovider.Route
+	for _, item := range router.Routes {
+		nodeName, ok := nodeNamesByAddr[item.NextHop]
+		if !ok {
+			// Not one of our routes?
+			glog.V(4).Infof("Skipping route with unknown nexthop %v", item.NextHop)
+			continue
+		}
+		route := cloudprovider.Route{
+			Name:            item.DestinationCIDR,
+			TargetNode:      nodeName,
+			DestinationCIDR: item.DestinationCIDR,
+		}
+		routes = append(routes, &route)
+	}
+
+	return routes, nil
+}
+
+func updateRoutes(network *gophercloud.ServiceClient, router *routers.Router, newRoutes []routers.Route) (func(), error) {
+	origRoutes := router.Routes // shallow copy
+
+	_, err := routers.Update(network, router.ID, routers.UpdateOpts{
+		Routes: newRoutes,
+	}).Extract()
+	if err != nil {
+		return nil, err
+	}
+
+	unwinder := func() {
+		glog.V(4).Info("Reverting routes change to router ", router.ID)
+		_, err := routers.Update(network, router.ID, routers.UpdateOpts{
+			Routes: origRoutes,
+		}).Extract()
+		if err != nil {
+			glog.Warning("Unable to reset routes during error unwind: ", err)
+		}
+	}
+
+	return unwinder, nil
+}
+
+func updateAllowedAddressPairs(network *gophercloud.ServiceClient, port *neutronports.Port, newPairs []neutronports.AddressPair) (func(), error) {
+	origPairs := port.AllowedAddressPairs // shallow copy
+
+	_, err := neutronports.Update(network, port.ID, neutronports.UpdateOpts{
+		AllowedAddressPairs: &newPairs,
+	}).Extract()
+	if err != nil {
+		return nil, err
+	}
+
+	unwinder := func() {
+		glog.V(4).Info("Reverting allowed-address-pairs change to port ", port.ID)
+		_, err := neutronports.Update(network, port.ID, neutronports.UpdateOpts{
+			AllowedAddressPairs: &origPairs,
+		}).Extract()
+		if err != nil {
+			glog.Warning("Unable to reset allowed-address-pairs during error unwind: ", err)
+		}
+	}
+
+	return unwinder, nil
+}
+
+func (r *Routes) CreateRoute(clusterName string, nameHint string, route *cloudprovider.Route) error {
+	glog.V(4).Infof("CreateRoute(%v, %v, %v)", clusterName, nameHint, route)
+
+	onFailure := NewCaller()
+
+	addr, err := getAddressByName(r.compute, route.TargetNode)
+	if err != nil {
+		return err
+	}
+
+	glog.V(4).Infof("Using nexthop %v for node %v", addr, route.TargetNode)
+
+	router, err := routers.Get(r.network, r.opts.RouterId).Extract()
+	if err != nil {
+		return err
+	}
+
+	routes := router.Routes
+
+	for _, item := range routes {
+		if item.DestinationCIDR == route.DestinationCIDR && item.NextHop == addr {
+			glog.V(4).Infof("Skipping existing route: %v", route)
+			return nil
+		}
+	}
+
+	routes = append(routes, routers.Route{
+		DestinationCIDR: route.DestinationCIDR,
+		NextHop:         addr,
+	})
+
+	unwind, err := updateRoutes(r.network, router, routes)
+	if err != nil {
+		return err
+	}
+	defer onFailure.Call(unwind)
+
+	// get the port of addr on target node.
+	portID, err := getPortIDByIP(r.compute, route.TargetNode, addr)
+	if err != nil {
+		return err
+	}
+	port, err := getPortByID(r.network, portID)
+	if err != nil {
+		return err
+	}
+
+	found := false
+	for _, item := range port.AllowedAddressPairs {
+		if item.IPAddress == route.DestinationCIDR {
+			glog.V(4).Info("Found existing allowed-address-pair: ", item)
+			found = true
+			break
+		}
+	}
+
+	if !found {
+		newPairs := append(port.AllowedAddressPairs, neutronports.AddressPair{
+			IPAddress: route.DestinationCIDR,
+		})
+		unwind, err := updateAllowedAddressPairs(r.network, port, newPairs)
+		if err != nil {
+			return err
+		}
+		defer onFailure.Call(unwind)
+	}
+
+	glog.V(4).Infof("Route created: %v", route)
+	onFailure.Disarm()
+	return nil
+}
+
+func (r *Routes) DeleteRoute(clusterName string, route *cloudprovider.Route) error {
+	glog.V(4).Infof("DeleteRoute(%v, %v)", clusterName, route)
+
+	onFailure := NewCaller()
+
+	addr, err := getAddressByName(r.compute, route.TargetNode)
+	if err != nil {
+		return err
+	}
+
+	router, err := routers.Get(r.network, r.opts.RouterId).Extract()
+	if err != nil {
+		return err
+	}
+
+	routes := router.Routes
+	index := -1
+	for i, item := range routes {
+		if item.DestinationCIDR == route.DestinationCIDR && item.NextHop == addr {
+			index = i
+			break
+		}
+	}
+
+	if index == -1 {
+		glog.V(4).Infof("Skipping non-existent route: %v", route)
+		return nil
+	}
+
+	// Delete element `index`
+	routes[index] = routes[len(routes)-1]
+	routes = routes[:len(routes)-1]
+
+	unwind, err := updateRoutes(r.network, router, routes)
+	if err != nil {
+		return err
+	}
+	defer onFailure.Call(unwind)
+
+	// get the port of addr on target node.
+	portID, err := getPortIDByIP(r.compute, route.TargetNode, addr)
+	if err != nil {
+		return err
+	}
+	port, err := getPortByID(r.network, portID)
+	if err != nil {
+		return err
+	}
+
+	addr_pairs := port.AllowedAddressPairs
+	index = -1
+	for i, item := range addr_pairs {
+		if item.IPAddress == route.DestinationCIDR {
+			index = i
+			break
+		}
+	}
+
+	if index != -1 {
+		// Delete element `index`
+		addr_pairs[index] = addr_pairs[len(addr_pairs)-1]
+		addr_pairs = addr_pairs[:len(addr_pairs)-1]
+
+		unwind, err := updateAllowedAddressPairs(r.network, port, addr_pairs)
+		if err != nil {
+			return err
+		}
+		defer onFailure.Call(unwind)
+	}
+
+	glog.V(4).Infof("Route deleted: %v", route)
+	onFailure.Disarm()
+	return nil
+}
+
+func getPortIDByIP(compute *gophercloud.ServiceClient, targetNode types.NodeName, ipAddress string) (string, error) {
+	srv, err := getServerByName(compute, targetNode)
+	if err != nil {
+		return "", err
+	}
+
+	interfaces, err := getAttachedInterfacesByID(compute, srv.ID)
+	if err != nil {
+		return "", err
+	}
+
+	for _, intf := range interfaces {
+		for _, fixedIP := range intf.FixedIPs {
+			if fixedIP.IPAddress == ipAddress {
+				return intf.PortID, nil
+			}
+		}
+	}
+
+	return "", ErrNotFound
+}
+
+func getPortByID(client *gophercloud.ServiceClient, portID string) (*neutronports.Port, error) {
+	targetPort, err := neutronports.Get(client, portID).Extract()
+	if err != nil {
+		return nil, err
+	}
+
+	if targetPort == nil {
+		return nil, ErrNotFound
+	}
+
+	return targetPort, nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/openstack/openstack_routes_test.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openstack
+
+import (
+	"net"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+)
+
+func TestRoutes(t *testing.T) {
+	const clusterName = "ignored"
+
+	cfg, ok := configFromEnv()
+	if !ok {
+		t.Skipf("No config found in environment")
+	}
+
+	os, err := newOpenStack(cfg)
+	if err != nil {
+		t.Fatalf("Failed to construct/authenticate OpenStack: %s", err)
+	}
+
+	r, ok := os.Routes()
+	if !ok {
+		t.Fatalf("Routes() returned false - perhaps your stack doens't support Neutron?")
+	}
+
+	newroute := cloudprovider.Route{
+		DestinationCIDR: "10.164.2.0/24",
+		TargetNode:      types.NodeName("testinstance"),
+	}
+	err = r.CreateRoute(clusterName, "myhint", &newroute)
+	if err != nil {
+		t.Fatalf("CreateRoute error: %v", err)
+	}
+
+	routelist, err := r.ListRoutes(clusterName)
+	if err != nil {
+		t.Fatalf("ListRoutes() error: %v", err)
+	}
+	for _, route := range routelist {
+		_, cidr, err := net.ParseCIDR(route.DestinationCIDR)
+		if err != nil {
+			t.Logf("Ignoring route %s, unparsable CIDR: %v", route.Name, err)
+			continue
+		}
+		t.Logf("%s via %s", cidr, route.TargetNode)
+	}
+
+	err = r.DeleteRoute(clusterName, &newroute)
+	if err != nil {
+		t.Fatalf("DeleteRoute error: %v", err)
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/openstack/openstack_test.go

@@ -0,0 +1,595 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openstack
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/gophercloud/gophercloud"
+	"github.com/gophercloud/gophercloud/openstack/blockstorage/v1/apiversions"
+	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
+	"k8s.io/api/core/v1"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/rand"
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+const (
+	volumeAvailableStatus = "available"
+	volumeInUseStatus     = "in-use"
+	testClusterName       = "testCluster"
+
+	volumeStatusTimeoutSeconds = 30
+	// volumeStatus* is configuration of exponential backoff for
+	// waiting for specified volume status. Starting with 1
+	// seconds, multiplying by 1.2 with each step and taking 13 steps at maximum
+	// it will time out after 32s, which roughly corresponds to 30s
+	volumeStatusInitDealy = 1 * time.Second
+	volumeStatusFactor    = 1.2
+	volumeStatusSteps     = 13
+)
+
+func WaitForVolumeStatus(t *testing.T, os *OpenStack, volumeName string, status string) {
+	backoff := wait.Backoff{
+		Duration: volumeStatusInitDealy,
+		Factor:   volumeStatusFactor,
+		Steps:    volumeStatusSteps,
+	}
+	err := wait.ExponentialBackoff(backoff, func() (bool, error) {
+		getVol, err := os.getVolume(volumeName)
+		if err != nil {
+			return false, err
+		}
+		if getVol.Status == status {
+			t.Logf("Volume (%s) status changed to %s after %v seconds\n",
+				volumeName,
+				status,
+				volumeStatusTimeoutSeconds)
+			return true, nil
+		} else {
+			return false, nil
+		}
+	})
+	if err == wait.ErrWaitTimeout {
+		t.Logf("Volume (%s) status did not change to %s after %v seconds\n",
+			volumeName,
+			status,
+			volumeStatusTimeoutSeconds)
+		return
+	}
+	if err != nil {
+		t.Fatalf("Cannot get existing Cinder volume (%s): %v", volumeName, err)
+	}
+}
+
+func TestReadConfig(t *testing.T) {
+	_, err := readConfig(nil)
+	if err == nil {
+		t.Errorf("Should fail when no config is provided: %s", err)
+	}
+
+	cfg, err := readConfig(strings.NewReader(`
+ [Global]
+ auth-url = http://auth.url
+ username = user
+ [LoadBalancer]
+ create-monitor = yes
+ monitor-delay = 1m
+ monitor-timeout = 30s
+ monitor-max-retries = 3
+ [BlockStorage]
+ bs-version = auto
+ trust-device-path = yes
+
+ `))
+	if err != nil {
+		t.Fatalf("Should succeed when a valid config is provided: %s", err)
+	}
+	if cfg.Global.AuthUrl != "http://auth.url" {
+		t.Errorf("incorrect authurl: %s", cfg.Global.AuthUrl)
+	}
+
+	if !cfg.LoadBalancer.CreateMonitor {
+		t.Errorf("incorrect lb.createmonitor: %t", cfg.LoadBalancer.CreateMonitor)
+	}
+	if cfg.LoadBalancer.MonitorDelay.Duration != 1*time.Minute {
+		t.Errorf("incorrect lb.monitordelay: %s", cfg.LoadBalancer.MonitorDelay)
+	}
+	if cfg.LoadBalancer.MonitorTimeout.Duration != 30*time.Second {
+		t.Errorf("incorrect lb.monitortimeout: %s", cfg.LoadBalancer.MonitorTimeout)
+	}
+	if cfg.LoadBalancer.MonitorMaxRetries != 3 {
+		t.Errorf("incorrect lb.monitormaxretries: %d", cfg.LoadBalancer.MonitorMaxRetries)
+	}
+	if cfg.BlockStorage.TrustDevicePath != true {
+		t.Errorf("incorrect bs.trustdevicepath: %v", cfg.BlockStorage.TrustDevicePath)
+	}
+	if cfg.BlockStorage.BSVersion != "auto" {
+		t.Errorf("incorrect bs.bs-version: %v", cfg.BlockStorage.BSVersion)
+	}
+}
+
+func TestToAuthOptions(t *testing.T) {
+	cfg := Config{}
+	cfg.Global.Username = "user"
+	// etc.
+
+	ao := cfg.toAuthOptions()
+
+	if !ao.AllowReauth {
+		t.Errorf("Will need to be able to reauthenticate")
+	}
+	if ao.Username != cfg.Global.Username {
+		t.Errorf("Username %s != %s", ao.Username, cfg.Global.Username)
+	}
+}
+
+func TestCheckOpenStackOpts(t *testing.T) {
+	delay := MyDuration{60 * time.Second}
+	timeout := MyDuration{30 * time.Second}
+	tests := []struct {
+		name          string
+		openstackOpts *OpenStack
+		expectedError error
+	}{
+		{
+			name: "test1",
+			openstackOpts: &OpenStack{
+				provider: nil,
+				lbOpts: LoadBalancerOpts{
+					LBVersion:            "v2",
+					SubnetId:             "6261548e-ffde-4bc7-bd22-59c83578c5ef",
+					FloatingNetworkId:    "38b8b5f9-64dc-4424-bf86-679595714786",
+					LBMethod:             "ROUND_ROBIN",
+					CreateMonitor:        true,
+					MonitorDelay:         delay,
+					MonitorTimeout:       timeout,
+					MonitorMaxRetries:    uint(3),
+					ManageSecurityGroups: true,
+					NodeSecurityGroupID:  "b41d28c2-d02f-4e1e-8ffb-23b8e4f5c144",
+				},
+			},
+			expectedError: nil,
+		},
+		{
+			name: "test2",
+			openstackOpts: &OpenStack{
+				provider: nil,
+				lbOpts: LoadBalancerOpts{
+					LBVersion:            "v2",
+					FloatingNetworkId:    "38b8b5f9-64dc-4424-bf86-679595714786",
+					LBMethod:             "ROUND_ROBIN",
+					CreateMonitor:        true,
+					MonitorDelay:         delay,
+					MonitorTimeout:       timeout,
+					MonitorMaxRetries:    uint(3),
+					ManageSecurityGroups: true,
+					NodeSecurityGroupID:  "b41d28c2-d02f-4e1e-8ffb-23b8e4f5c144",
+				},
+			},
+			expectedError: nil,
+		},
+		{
+			name: "test3",
+			openstackOpts: &OpenStack{
+				provider: nil,
+				lbOpts: LoadBalancerOpts{
+					LBVersion:            "v2",
+					SubnetId:             "6261548e-ffde-4bc7-bd22-59c83578c5ef",
+					FloatingNetworkId:    "38b8b5f9-64dc-4424-bf86-679595714786",
+					LBMethod:             "ROUND_ROBIN",
+					CreateMonitor:        true,
+					ManageSecurityGroups: true,
+					NodeSecurityGroupID:  "b41d28c2-d02f-4e1e-8ffb-23b8e4f5c144",
+				},
+			},
+			expectedError: fmt.Errorf("monitor-delay not set in cloud provider config"),
+		},
+		{
+			name: "test4",
+			openstackOpts: &OpenStack{
+				provider: nil,
+				lbOpts: LoadBalancerOpts{
+					LBVersion:            "v2",
+					SubnetId:             "6261548e-ffde-4bc7-bd22-59c83578c5ef",
+					FloatingNetworkId:    "38b8b5f9-64dc-4424-bf86-679595714786",
+					LBMethod:             "ROUND_ROBIN",
+					CreateMonitor:        true,
+					MonitorDelay:         delay,
+					MonitorTimeout:       timeout,
+					MonitorMaxRetries:    uint(3),
+					ManageSecurityGroups: true,
+				},
+			},
+			expectedError: fmt.Errorf("node-security-group not set in cloud provider config"),
+		},
+	}
+
+	for _, testcase := range tests {
+		err := checkOpenStackOpts(testcase.openstackOpts)
+
+		if err == nil && testcase.expectedError == nil {
+			continue
+		}
+		if (err != nil && testcase.expectedError == nil) || (err == nil && testcase.expectedError != nil) || err.Error() != testcase.expectedError.Error() {
+			t.Errorf("%s failed: expected err=%q, got %q",
+				testcase.name, testcase.expectedError, err)
+		}
+	}
+}
+
+func TestCaller(t *testing.T) {
+	called := false
+	myFunc := func() { called = true }
+
+	c := NewCaller()
+	c.Call(myFunc)
+
+	if !called {
+		t.Errorf("Caller failed to call function in default case")
+	}
+
+	c.Disarm()
+	called = false
+	c.Call(myFunc)
+
+	if called {
+		t.Error("Caller still called function when disarmed")
+	}
+
+	// Confirm the "usual" deferred Caller pattern works as expected
+
+	called = false
+	success_case := func() {
+		c := NewCaller()
+		defer c.Call(func() { called = true })
+		c.Disarm()
+	}
+	if success_case(); called {
+		t.Error("Deferred success case still invoked unwind")
+	}
+
+	called = false
+	failure_case := func() {
+		c := NewCaller()
+		defer c.Call(func() { called = true })
+	}
+	if failure_case(); !called {
+		t.Error("Deferred failure case failed to invoke unwind")
+	}
+}
+
+// An arbitrary sort.Interface, just for easier comparison
+type AddressSlice []v1.NodeAddress
+
+func (a AddressSlice) Len() int           { return len(a) }
+func (a AddressSlice) Less(i, j int) bool { return a[i].Address < a[j].Address }
+func (a AddressSlice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+
+func TestNodeAddresses(t *testing.T) {
+	srv := servers.Server{
+		Status:     "ACTIVE",
+		HostID:     "29d3c8c896a45aa4c34e52247875d7fefc3d94bbcc9f622b5d204362",
+		AccessIPv4: "50.56.176.99",
+		AccessIPv6: "2001:4800:790e:510:be76:4eff:fe04:82a8",
+		Addresses: map[string]interface{}{
+			"private": []interface{}{
+				map[string]interface{}{
+					"OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:7c:1b:2b",
+					"version":                 float64(4),
+					"addr":                    "10.0.0.32",
+					"OS-EXT-IPS:type":         "fixed",
+				},
+				map[string]interface{}{
+					"version":         float64(4),
+					"addr":            "50.56.176.36",
+					"OS-EXT-IPS:type": "floating",
+				},
+				map[string]interface{}{
+					"version": float64(4),
+					"addr":    "10.0.0.31",
+					// No OS-EXT-IPS:type
+				},
+			},
+			"public": []interface{}{
+				map[string]interface{}{
+					"version": float64(4),
+					"addr":    "50.56.176.35",
+				},
+				map[string]interface{}{
+					"version": float64(6),
+					"addr":    "2001:4800:780e:510:be76:4eff:fe04:84a8",
+				},
+			},
+		},
+	}
+
+	addrs, err := nodeAddresses(&srv)
+	if err != nil {
+		t.Fatalf("nodeAddresses returned error: %v", err)
+	}
+
+	sort.Sort(AddressSlice(addrs))
+	t.Logf("addresses is %v", addrs)
+
+	want := []v1.NodeAddress{
+		{Type: v1.NodeInternalIP, Address: "10.0.0.31"},
+		{Type: v1.NodeInternalIP, Address: "10.0.0.32"},
+		{Type: v1.NodeExternalIP, Address: "2001:4800:780e:510:be76:4eff:fe04:84a8"},
+		{Type: v1.NodeExternalIP, Address: "2001:4800:790e:510:be76:4eff:fe04:82a8"},
+		{Type: v1.NodeExternalIP, Address: "50.56.176.35"},
+		{Type: v1.NodeExternalIP, Address: "50.56.176.36"},
+		{Type: v1.NodeExternalIP, Address: "50.56.176.99"},
+	}
+
+	if !reflect.DeepEqual(want, addrs) {
+		t.Errorf("nodeAddresses returned incorrect value %v", addrs)
+	}
+}
+
+// This allows acceptance testing against an existing OpenStack
+// install, using the standard OS_* OpenStack client environment
+// variables.
+// FIXME: it would be better to hermetically test against canned JSON
+// requests/responses.
+func configFromEnv() (cfg Config, ok bool) {
+	cfg.Global.AuthUrl = os.Getenv("OS_AUTH_URL")
+
+	cfg.Global.TenantId = os.Getenv("OS_TENANT_ID")
+	// Rax/nova _insists_ that we don't specify both tenant ID and name
+	if cfg.Global.TenantId == "" {
+		cfg.Global.TenantName = os.Getenv("OS_TENANT_NAME")
+	}
+
+	cfg.Global.Username = os.Getenv("OS_USERNAME")
+	cfg.Global.Password = os.Getenv("OS_PASSWORD")
+	cfg.Global.Region = os.Getenv("OS_REGION_NAME")
+	cfg.Global.DomainId = os.Getenv("OS_DOMAIN_ID")
+	cfg.Global.DomainName = os.Getenv("OS_DOMAIN_NAME")
+
+	ok = (cfg.Global.AuthUrl != "" &&
+		cfg.Global.Username != "" &&
+		cfg.Global.Password != "" &&
+		(cfg.Global.TenantId != "" || cfg.Global.TenantName != "" ||
+			cfg.Global.DomainId != "" || cfg.Global.DomainName != ""))
+
+	return
+}
+
+func TestNewOpenStack(t *testing.T) {
+	cfg, ok := configFromEnv()
+	if !ok {
+		t.Skipf("No config found in environment")
+	}
+
+	_, err := newOpenStack(cfg)
+	if err != nil {
+		t.Fatalf("Failed to construct/authenticate OpenStack: %s", err)
+	}
+}
+
+func TestLoadBalancer(t *testing.T) {
+	cfg, ok := configFromEnv()
+	if !ok {
+		t.Skipf("No config found in environment")
+	}
+
+	versions := []string{"v1", "v2", ""}
+
+	for _, v := range versions {
+		t.Logf("Trying LBVersion = '%s'\n", v)
+		cfg.LoadBalancer.LBVersion = v
+
+		os, err := newOpenStack(cfg)
+		if err != nil {
+			t.Fatalf("Failed to construct/authenticate OpenStack: %s", err)
+		}
+
+		lb, ok := os.LoadBalancer()
+		if !ok {
+			t.Fatalf("LoadBalancer() returned false - perhaps your stack doesn't support Neutron?")
+		}
+
+		_, exists, err := lb.GetLoadBalancer(testClusterName, &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "noexist"}})
+		if err != nil {
+			t.Fatalf("GetLoadBalancer(\"noexist\") returned error: %s", err)
+		}
+		if exists {
+			t.Fatalf("GetLoadBalancer(\"noexist\") returned exists")
+		}
+	}
+}
+
+func TestZones(t *testing.T) {
+	SetMetadataFixture(&FakeMetadata)
+	defer ClearMetadata()
+
+	os := OpenStack{
+		provider: &gophercloud.ProviderClient{
+			IdentityBase: "http://auth.url/",
+		},
+		region: "myRegion",
+	}
+
+	z, ok := os.Zones()
+	if !ok {
+		t.Fatalf("Zones() returned false")
+	}
+
+	zone, err := z.GetZone()
+	if err != nil {
+		t.Fatalf("GetZone() returned error: %s", err)
+	}
+
+	if zone.Region != "myRegion" {
+		t.Fatalf("GetZone() returned wrong region (%s)", zone.Region)
+	}
+
+	if zone.FailureDomain != "nova" {
+		t.Fatalf("GetZone() returned wrong failure domain (%s)", zone.FailureDomain)
+	}
+}
+
+func TestVolumes(t *testing.T) {
+	cfg, ok := configFromEnv()
+	if !ok {
+		t.Skipf("No config found in environment")
+	}
+
+	os, err := newOpenStack(cfg)
+	if err != nil {
+		t.Fatalf("Failed to construct/authenticate OpenStack: %s", err)
+	}
+
+	tags := map[string]string{
+		"test": "value",
+	}
+	vol, _, err := os.CreateVolume("kubernetes-test-volume-"+rand.String(10), 1, "", "", &tags)
+	if err != nil {
+		t.Fatalf("Cannot create a new Cinder volume: %v", err)
+	}
+	t.Logf("Volume (%s) created\n", vol)
+
+	WaitForVolumeStatus(t, os, vol, volumeAvailableStatus)
+
+	id, err := os.InstanceID()
+	if err != nil {
+		t.Fatalf("Cannot find instance id: %v", err)
+	}
+
+	diskId, err := os.AttachDisk(id, vol)
+	if err != nil {
+		t.Fatalf("Cannot AttachDisk Cinder volume %s: %v", vol, err)
+	}
+	t.Logf("Volume (%s) attached, disk ID: %s\n", vol, diskId)
+
+	WaitForVolumeStatus(t, os, vol, volumeInUseStatus)
+
+	devicePath := os.GetDevicePath(diskId)
+	if !strings.HasPrefix(devicePath, "/dev/disk/by-id/") {
+		t.Fatalf("GetDevicePath returned and unexpected path for Cinder volume %s, returned %s", vol, devicePath)
+	}
+	t.Logf("Volume (%s) found at path: %s\n", vol, devicePath)
+
+	err = os.DetachDisk(id, vol)
+	if err != nil {
+		t.Fatalf("Cannot DetachDisk Cinder volume %s: %v", vol, err)
+	}
+	t.Logf("Volume (%s) detached\n", vol)
+
+	WaitForVolumeStatus(t, os, vol, volumeAvailableStatus)
+
+	err = os.DeleteVolume(vol)
+	if err != nil {
+		t.Fatalf("Cannot delete Cinder volume %s: %v", vol, err)
+	}
+	t.Logf("Volume (%s) deleted\n", vol)
+
+}
+
+func TestCinderAutoDetectApiVersion(t *testing.T) {
+	updated := "" // not relevant to this test, can be set to any value
+	status_current := "CURRENT"
+	status_supported := "SUPpORTED" // lowercase to test regression resitance if api returns different case
+	status_deprecated := "DEPRECATED"
+
+	var result_version, api_version [4]string
+
+	for ver := 0; ver <= 3; ver++ {
+		api_version[ver] = fmt.Sprintf("v%d.0", ver)
+		result_version[ver] = fmt.Sprintf("v%d", ver)
+	}
+	result_version[0] = ""
+	api_current_v1 := apiversions.APIVersion{ID: api_version[1], Status: status_current, Updated: updated}
+	api_current_v2 := apiversions.APIVersion{ID: api_version[2], Status: status_current, Updated: updated}
+	api_current_v3 := apiversions.APIVersion{ID: api_version[3], Status: status_current, Updated: updated}
+
+	api_supported_v1 := apiversions.APIVersion{ID: api_version[1], Status: status_supported, Updated: updated}
+	api_supported_v2 := apiversions.APIVersion{ID: api_version[2], Status: status_supported, Updated: updated}
+
+	api_deprecated_v1 := apiversions.APIVersion{ID: api_version[1], Status: status_deprecated, Updated: updated}
+	api_deprecated_v2 := apiversions.APIVersion{ID: api_version[2], Status: status_deprecated, Updated: updated}
+
+	var testCases = []struct {
+		test_case     []apiversions.APIVersion
+		wanted_result string
+	}{
+		{[]apiversions.APIVersion{api_current_v1}, result_version[1]},
+		{[]apiversions.APIVersion{api_current_v2}, result_version[2]},
+		{[]apiversions.APIVersion{api_supported_v1, api_current_v2}, result_version[2]},                     // current always selected
+		{[]apiversions.APIVersion{api_current_v1, api_supported_v2}, result_version[1]},                     // current always selected
+		{[]apiversions.APIVersion{api_current_v3, api_supported_v2, api_deprecated_v1}, result_version[2]},  // with current v3, but should fall back to v2
+		{[]apiversions.APIVersion{api_current_v3, api_deprecated_v2, api_deprecated_v1}, result_version[0]}, // v3 is not supported
+	}
+
+	for _, suite := range testCases {
+		if autodetectedVersion := doBsApiVersionAutodetect(suite.test_case); autodetectedVersion != suite.wanted_result {
+			t.Fatalf("Autodetect for suite: %s, failed with result: '%s', wanted '%s'", suite.test_case, autodetectedVersion, suite.wanted_result)
+		}
+	}
+}
+
+func TestInstanceIDFromProviderID(t *testing.T) {
+	testCases := []struct {
+		providerID string
+		instanceID string
+		fail       bool
+	}{
+		{
+			providerID: ProviderName + "://" + "/" + "7b9cf879-7146-417c-abfd-cb4272f0c935",
+			instanceID: "7b9cf879-7146-417c-abfd-cb4272f0c935",
+			fail:       false,
+		},
+		{
+			providerID: "openstack://7b9cf879-7146-417c-abfd-cb4272f0c935",
+			instanceID: "",
+			fail:       true,
+		},
+		{
+			providerID: "7b9cf879-7146-417c-abfd-cb4272f0c935",
+			instanceID: "",
+			fail:       true,
+		},
+		{
+			providerID: "other-provider:///7b9cf879-7146-417c-abfd-cb4272f0c935",
+			instanceID: "",
+			fail:       true,
+		},
+	}
+
+	for _, test := range testCases {
+		instanceID, err := instanceIDFromProviderID(test.providerID)
+		if (err != nil) != test.fail {
+			t.Errorf("%s yielded `err != nil` as %t. expected %t", test.providerID, (err != nil), test.fail)
+		}
+
+		if test.fail {
+			continue
+		}
+
+		if instanceID != test.instanceID {
+			t.Errorf("%s yielded %s. expected %s", test.providerID, instanceID, test.instanceID)
+		}
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/openstack/openstack_volumes.go

@@ -0,0 +1,449 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openstack
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"path"
+	"strings"
+	"time"
+
+	k8s_volume "k8s.io/kubernetes/pkg/volume"
+
+	"github.com/gophercloud/gophercloud"
+	volumes_v1 "github.com/gophercloud/gophercloud/openstack/blockstorage/v1/volumes"
+	volumes_v2 "github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes"
+	"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/golang/glog"
+)
+
+type volumeService interface {
+	createVolume(opts VolumeCreateOpts) (string, string, error)
+	getVolume(volumeID string) (Volume, error)
+	deleteVolume(volumeName string) error
+}
+
+// Volumes implementation for v1
+type VolumesV1 struct {
+	blockstorage *gophercloud.ServiceClient
+	opts         BlockStorageOpts
+}
+
+// Volumes implementation for v2
+type VolumesV2 struct {
+	blockstorage *gophercloud.ServiceClient
+	opts         BlockStorageOpts
+}
+
+type Volume struct {
+	// ID of the instance, to which this volume is attached. "" if not attached
+	AttachedServerId string
+	// Device file path
+	AttachedDevice string
+	// Unique identifier for the volume.
+	ID string
+	// Human-readable display name for the volume.
+	Name string
+	// Current status of the volume.
+	Status string
+}
+
+type VolumeCreateOpts struct {
+	Size         int
+	Availability string
+	Name         string
+	VolumeType   string
+	Metadata     map[string]string
+}
+
+const (
+	VolumeAvailableStatus = "available"
+	VolumeInUseStatus     = "in-use"
+	VolumeDeletedStatus   = "deleted"
+	VolumeErrorStatus     = "error"
+)
+
+func (volumes *VolumesV1) createVolume(opts VolumeCreateOpts) (string, string, error) {
+	startTime := time.Now()
+
+	create_opts := volumes_v1.CreateOpts{
+		Name:             opts.Name,
+		Size:             opts.Size,
+		VolumeType:       opts.VolumeType,
+		AvailabilityZone: opts.Availability,
+		Metadata:         opts.Metadata,
+	}
+
+	vol, err := volumes_v1.Create(volumes.blockstorage, create_opts).Extract()
+	timeTaken := time.Since(startTime).Seconds()
+	recordOpenstackOperationMetric("create_v1_volume", timeTaken, err)
+	if err != nil {
+		return "", "", err
+	}
+	return vol.ID, vol.AvailabilityZone, nil
+}
+
+func (volumes *VolumesV2) createVolume(opts VolumeCreateOpts) (string, string, error) {
+	startTime := time.Now()
+
+	create_opts := volumes_v2.CreateOpts{
+		Name:             opts.Name,
+		Size:             opts.Size,
+		VolumeType:       opts.VolumeType,
+		AvailabilityZone: opts.Availability,
+		Metadata:         opts.Metadata,
+	}
+
+	vol, err := volumes_v2.Create(volumes.blockstorage, create_opts).Extract()
+	timeTaken := time.Since(startTime).Seconds()
+	recordOpenstackOperationMetric("create_v2_volume", timeTaken, err)
+	if err != nil {
+		return "", "", err
+	}
+	return vol.ID, vol.AvailabilityZone, nil
+}
+
+func (volumes *VolumesV1) getVolume(volumeID string) (Volume, error) {
+	startTime := time.Now()
+	volumeV1, err := volumes_v1.Get(volumes.blockstorage, volumeID).Extract()
+	timeTaken := time.Since(startTime).Seconds()
+	recordOpenstackOperationMetric("get_v1_volume", timeTaken, err)
+	if err != nil {
+		glog.Errorf("Error occurred getting volume by ID: %s", volumeID)
+		return Volume{}, err
+	}
+
+	volume := Volume{
+		ID:     volumeV1.ID,
+		Name:   volumeV1.Name,
+		Status: volumeV1.Status,
+	}
+
+	if len(volumeV1.Attachments) > 0 && volumeV1.Attachments[0]["server_id"] != nil {
+		volume.AttachedServerId = volumeV1.Attachments[0]["server_id"].(string)
+		volume.AttachedDevice = volumeV1.Attachments[0]["device"].(string)
+	}
+
+	return volume, nil
+}
+
+func (volumes *VolumesV2) getVolume(volumeID string) (Volume, error) {
+	startTime := time.Now()
+	volumeV2, err := volumes_v2.Get(volumes.blockstorage, volumeID).Extract()
+	timeTaken := time.Since(startTime).Seconds()
+	recordOpenstackOperationMetric("get_v2_volume", timeTaken, err)
+	if err != nil {
+		glog.Errorf("Error occurred getting volume by ID: %s", volumeID)
+		return Volume{}, err
+	}
+
+	volume := Volume{
+		ID:     volumeV2.ID,
+		Name:   volumeV2.Name,
+		Status: volumeV2.Status,
+	}
+
+	if len(volumeV2.Attachments) > 0 {
+		volume.AttachedServerId = volumeV2.Attachments[0].ServerID
+		volume.AttachedDevice = volumeV2.Attachments[0].Device
+	}
+
+	return volume, nil
+}
+
+func (volumes *VolumesV1) deleteVolume(volumeID string) error {
+	startTime := time.Now()
+	err := volumes_v1.Delete(volumes.blockstorage, volumeID).ExtractErr()
+	timeTaken := time.Since(startTime).Seconds()
+	recordOpenstackOperationMetric("delete_v1_volume", timeTaken, err)
+	if err != nil {
+		glog.Errorf("Cannot delete volume %s: %v", volumeID, err)
+	}
+
+	return err
+}
+
+func (volumes *VolumesV2) deleteVolume(volumeID string) error {
+	startTime := time.Now()
+	err := volumes_v2.Delete(volumes.blockstorage, volumeID).ExtractErr()
+	timeTaken := time.Since(startTime).Seconds()
+	recordOpenstackOperationMetric("delete_v2_volume", timeTaken, err)
+	if err != nil {
+		glog.Errorf("Cannot delete volume %s: %v", volumeID, err)
+	}
+
+	return err
+}
+
+func (os *OpenStack) OperationPending(diskName string) (bool, string, error) {
+	volume, err := os.getVolume(diskName)
+	if err != nil {
+		return false, "", err
+	}
+	volumeStatus := volume.Status
+	if volumeStatus == VolumeErrorStatus {
+		glog.Errorf("status of volume %s is %s", diskName, volumeStatus)
+		return false, volumeStatus, nil
+	}
+	if volumeStatus == VolumeAvailableStatus || volumeStatus == VolumeInUseStatus || volumeStatus == VolumeDeletedStatus {
+		return false, volume.Status, nil
+	}
+	return true, volumeStatus, nil
+}
+
+// AttachDisk attaches given cinder volume to the compute running kubelet
+func (os *OpenStack) AttachDisk(instanceID, volumeID string) (string, error) {
+	volume, err := os.getVolume(volumeID)
+	if err != nil {
+		return "", err
+	}
+
+	cClient, err := os.NewComputeV2()
+	if err != nil {
+		return "", err
+	}
+
+	if volume.AttachedServerId != "" {
+		if instanceID == volume.AttachedServerId {
+			glog.V(4).Infof("Disk %s is already attached to instance %s", volumeID, instanceID)
+			return volume.ID, nil
+		}
+		errmsg := fmt.Sprintf("Disk %s is attached to a different instance (%s)", volumeID, volume.AttachedServerId)
+		glog.V(2).Infof(errmsg)
+		return "", errors.New(errmsg)
+	}
+
+	startTime := time.Now()
+	// add read only flag here if possible spothanis
+	_, err = volumeattach.Create(cClient, instanceID, &volumeattach.CreateOpts{
+		VolumeID: volume.ID,
+	}).Extract()
+	timeTaken := time.Since(startTime).Seconds()
+	recordOpenstackOperationMetric("attach_disk", timeTaken, err)
+	if err != nil {
+		glog.Errorf("Failed to attach %s volume to %s compute: %v", volumeID, instanceID, err)
+		return "", err
+	}
+	glog.V(2).Infof("Successfully attached %s volume to %s compute", volumeID, instanceID)
+	return volume.ID, nil
+}
+
+// DetachDisk detaches given cinder volume from the compute running kubelet
+func (os *OpenStack) DetachDisk(instanceID, volumeID string) error {
+	volume, err := os.getVolume(volumeID)
+	if err != nil {
+		return err
+	}
+	if volume.Status == VolumeAvailableStatus {
+		// "available" is fine since that means the volume is detached from instance already.
+		glog.V(2).Infof("volume: %s has been detached from compute: %s ", volume.ID, instanceID)
+		return nil
+	}
+
+	if volume.Status != VolumeInUseStatus {
+		errmsg := fmt.Sprintf("can not detach volume %s, its status is %s.", volume.Name, volume.Status)
+		glog.Errorf(errmsg)
+		return errors.New(errmsg)
+	}
+	cClient, err := os.NewComputeV2()
+	if err != nil {
+		return err
+	}
+	if volume.AttachedServerId != instanceID {
+		errMsg := fmt.Sprintf("Disk: %s has no attachments or is not attached to compute: %s", volume.Name, instanceID)
+		glog.Errorf(errMsg)
+		return errors.New(errMsg)
+	} else {
+		startTime := time.Now()
+		// This is a blocking call and effects kubelet's performance directly.
+		// We should consider kicking it out into a separate routine, if it is bad.
+		err = volumeattach.Delete(cClient, instanceID, volume.ID).ExtractErr()
+		timeTaken := time.Since(startTime).Seconds()
+		recordOpenstackOperationMetric("detach_disk", timeTaken, err)
+		if err != nil {
+			glog.Errorf("Failed to delete volume %s from compute %s attached %v", volume.ID, instanceID, err)
+			return err
+		}
+		glog.V(2).Infof("Successfully detached volume: %s from compute: %s", volume.ID, instanceID)
+	}
+
+	return nil
+}
+
+// getVolume retrieves Volume by its ID.
+func (os *OpenStack) getVolume(volumeID string) (Volume, error) {
+	volumes, err := os.volumeService("")
+	if err != nil || volumes == nil {
+		glog.Errorf("Unable to initialize cinder client for region: %s", os.region)
+		return Volume{}, err
+	}
+	return volumes.getVolume(volumeID)
+}
+
+// CreateVolume creates a volume of given size (in GiB)
+func (os *OpenStack) CreateVolume(name string, size int, vtype, availability string, tags *map[string]string) (string, string, error) {
+	volumes, err := os.volumeService("")
+	if err != nil || volumes == nil {
+		glog.Errorf("Unable to initialize cinder client for region: %s", os.region)
+		return "", "", err
+	}
+
+	opts := VolumeCreateOpts{
+		Name:         name,
+		Size:         size,
+		VolumeType:   vtype,
+		Availability: availability,
+	}
+	if tags != nil {
+		opts.Metadata = *tags
+	}
+
+	volumeID, volumeAZ, err := volumes.createVolume(opts)
+
+	if err != nil {
+		glog.Errorf("Failed to create a %d GB volume: %v", size, err)
+		return "", "", err
+	}
+
+	glog.Infof("Created volume %v in Availability Zone: %v", volumeID, volumeAZ)
+	return volumeID, volumeAZ, nil
+}
+
+// GetDevicePath returns the path of an attached block storage volume, specified by its id.
+func (os *OpenStack) GetDevicePath(volumeID string) string {
+	// Build a list of candidate device paths
+	candidateDeviceNodes := []string{
+		// KVM
+		fmt.Sprintf("virtio-%s", volumeID[:20]),
+		// KVM virtio-scsi
+		fmt.Sprintf("scsi-0QEMU_QEMU_HARDDISK_%s", volumeID[:20]),
+		// ESXi
+		fmt.Sprintf("wwn-0x%s", strings.Replace(volumeID, "-", "", -1)),
+	}
+
+	files, _ := ioutil.ReadDir("/dev/disk/by-id/")
+
+	for _, f := range files {
+		for _, c := range candidateDeviceNodes {
+			if c == f.Name() {
+				glog.V(4).Infof("Found disk attached as %q; full devicepath: %s\n", f.Name(), path.Join("/dev/disk/by-id/", f.Name()))
+				return path.Join("/dev/disk/by-id/", f.Name())
+			}
+		}
+	}
+
+	glog.Warningf("Failed to find device for the volumeID: %q\n", volumeID)
+	return ""
+}
+
+func (os *OpenStack) DeleteVolume(volumeID string) error {
+	used, err := os.diskIsUsed(volumeID)
+	if err != nil {
+		return err
+	}
+	if used {
+		msg := fmt.Sprintf("Cannot delete the volume %q, it's still attached to a node", volumeID)
+		return k8s_volume.NewDeletedVolumeInUseError(msg)
+	}
+
+	volumes, err := os.volumeService("")
+	if err != nil || volumes == nil {
+		glog.Errorf("Unable to initialize cinder client for region: %s", os.region)
+		return err
+	}
+
+	err = volumes.deleteVolume(volumeID)
+	if err != nil {
+		glog.Errorf("Cannot delete volume %s: %v", volumeID, err)
+	}
+	return nil
+
+}
+
+// GetAttachmentDiskPath gets device path of attached volume to the compute running kubelet, as known by cinder
+func (os *OpenStack) GetAttachmentDiskPath(instanceID, volumeID string) (string, error) {
+	// See issue #33128 - Cinder does not always tell you the right device path, as such
+	// we must only use this value as a last resort.
+	volume, err := os.getVolume(volumeID)
+	if err != nil {
+		return "", err
+	}
+	if volume.Status != VolumeInUseStatus {
+		errmsg := fmt.Sprintf("can not get device path of volume %s, its status is %s.", volume.Name, volume.Status)
+		glog.Errorf(errmsg)
+		return "", errors.New(errmsg)
+	}
+	if volume.AttachedServerId != "" {
+		if instanceID == volume.AttachedServerId {
+			// Attachment[0]["device"] points to the device path
+			// see http://developer.openstack.org/api-ref-blockstorage-v1.html
+			return volume.AttachedDevice, nil
+		} else {
+			errMsg := fmt.Sprintf("Disk %q is attached to a different compute: %q, should be detached before proceeding", volumeID, volume.AttachedServerId)
+			glog.Errorf(errMsg)
+			return "", errors.New(errMsg)
+		}
+	}
+	return "", fmt.Errorf("volume %s has no ServerId.", volumeID)
+}
+
+// DiskIsAttached queries if a volume is attached to a compute instance
+func (os *OpenStack) DiskIsAttached(instanceID, volumeID string) (bool, error) {
+	volume, err := os.getVolume(volumeID)
+	if err != nil {
+		return false, err
+	}
+
+	return instanceID == volume.AttachedServerId, nil
+}
+
+// DisksAreAttached queries if a list of volumes are attached to a compute instance
+func (os *OpenStack) DisksAreAttached(instanceID string, volumeIDs []string) (map[string]bool, error) {
+	attached := make(map[string]bool)
+	for _, volumeID := range volumeIDs {
+		isAttached, _ := os.DiskIsAttached(instanceID, volumeID)
+		attached[volumeID] = isAttached
+	}
+	return attached, nil
+}
+
+// diskIsUsed returns true a disk is attached to any node.
+func (os *OpenStack) diskIsUsed(volumeID string) (bool, error) {
+	volume, err := os.getVolume(volumeID)
+	if err != nil {
+		return false, err
+	}
+	return volume.AttachedServerId != "", nil
+}
+
+// ShouldTrustDevicePath queries if we should trust the cinder provide deviceName, See issue #33128
+func (os *OpenStack) ShouldTrustDevicePath() bool {
+	return os.bsOpts.TrustDevicePath
+}
+
+// recordOpenstackOperationMetric records openstack operation metrics
+func recordOpenstackOperationMetric(operation string, timeTaken float64, err error) {
+	if err != nil {
+		OpenstackApiRequestErrors.With(prometheus.Labels{"request": operation}).Inc()
+	} else {
+		OpenstackOperationsLatency.With(prometheus.Labels{"request": operation}).Observe(timeTaken)
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/ovirt/BUILD

@@ -0,0 +1,39 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["ovirt.go"],
+    deps = [
+        "//pkg/cloudprovider:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//vendor/gopkg.in/gcfg.v1:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["ovirt_test.go"],
+    library = ":go_default_library",
+    deps = ["//pkg/cloudprovider:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/ovirt/ovirt.go

@@ -0,0 +1,325 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ovirt
+
+import (
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"path"
+	"sort"
+	"strings"
+
+	"gopkg.in/gcfg.v1"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+	"k8s.io/kubernetes/pkg/controller"
+)
+
+const ProviderName = "ovirt"
+
+type OVirtInstance struct {
+	UUID      string
+	Name      string
+	IPAddress string
+}
+
+type OVirtInstanceMap map[string]OVirtInstance
+
+type OVirtCloud struct {
+	VmsRequest   *url.URL
+	HostsRequest *url.URL
+}
+
+type OVirtApiConfig struct {
+	Connection struct {
+		ApiEntry string `gcfg:"uri"`
+		Username string `gcfg:"username"`
+		Password string `gcfg:"password"`
+	}
+	Filters struct {
+		VmsQuery string `gcfg:"vms"`
+	}
+}
+
+type XmlVmAddress struct {
+	Address string `xml:"address,attr"`
+}
+
+type XmlVmInfo struct {
+	UUID      string         `xml:"id,attr"`
+	Name      string         `xml:"name"`
+	Hostname  string         `xml:"guest_info>fqdn"`
+	Addresses []XmlVmAddress `xml:"guest_info>ips>ip"`
+	State     string         `xml:"status>state"`
+}
+
+type XmlVmsList struct {
+	XMLName xml.Name    `xml:"vms"`
+	Vm      []XmlVmInfo `xml:"vm"`
+}
+
+func init() {
+	cloudprovider.RegisterCloudProvider(ProviderName,
+		func(config io.Reader) (cloudprovider.Interface, error) {
+			return newOVirtCloud(config)
+		})
+}
+
+func newOVirtCloud(config io.Reader) (*OVirtCloud, error) {
+	if config == nil {
+		return nil, fmt.Errorf("missing configuration file for ovirt cloud provider")
+	}
+
+	oVirtConfig := OVirtApiConfig{}
+
+	/* defaults */
+	oVirtConfig.Connection.Username = "admin@internal"
+
+	if err := gcfg.ReadInto(&oVirtConfig, config); err != nil {
+		return nil, err
+	}
+
+	if oVirtConfig.Connection.ApiEntry == "" {
+		return nil, fmt.Errorf("missing ovirt uri in cloud provider configuration")
+	}
+
+	request, err := url.Parse(oVirtConfig.Connection.ApiEntry)
+	if err != nil {
+		return nil, err
+	}
+
+	request.Path = path.Join(request.Path, "vms")
+	request.User = url.UserPassword(oVirtConfig.Connection.Username, oVirtConfig.Connection.Password)
+	request.RawQuery = url.Values{"search": {oVirtConfig.Filters.VmsQuery}}.Encode()
+
+	return &OVirtCloud{VmsRequest: request}, nil
+}
+
+// Initialize passes a Kubernetes clientBuilder interface to the cloud provider
+func (v *OVirtCloud) Initialize(clientBuilder controller.ControllerClientBuilder) {}
+
+func (v *OVirtCloud) Clusters() (cloudprovider.Clusters, bool) {
+	return nil, false
+}
+
+// ProviderName returns the cloud provider ID.
+func (v *OVirtCloud) ProviderName() string {
+	return ProviderName
+}
+
+// ScrubDNS filters DNS settings for pods.
+func (v *OVirtCloud) ScrubDNS(nameservers, searches []string) (nsOut, srchOut []string) {
+	return nameservers, searches
+}
+
+// HasClusterID returns true if the cluster has a clusterID
+func (v *OVirtCloud) HasClusterID() bool {
+	return true
+}
+
+// LoadBalancer returns an implementation of LoadBalancer for oVirt cloud
+func (v *OVirtCloud) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
+	return nil, false
+}
+
+// Instances returns an implementation of Instances for oVirt cloud
+func (v *OVirtCloud) Instances() (cloudprovider.Instances, bool) {
+	return v, true
+}
+
+// Zones returns an implementation of Zones for oVirt cloud
+func (v *OVirtCloud) Zones() (cloudprovider.Zones, bool) {
+	return nil, false
+}
+
+// Routes returns an implementation of Routes for oVirt cloud
+func (v *OVirtCloud) Routes() (cloudprovider.Routes, bool) {
+	return nil, false
+}
+
+// NodeAddresses returns the NodeAddresses of the instance with the specified nodeName.
+func (v *OVirtCloud) NodeAddresses(nodeName types.NodeName) ([]v1.NodeAddress, error) {
+	name := mapNodeNameToInstanceName(nodeName)
+	instance, err := v.fetchInstance(name)
+	if err != nil {
+		return nil, err
+	}
+
+	var address net.IP
+
+	if instance.IPAddress != "" {
+		address = net.ParseIP(instance.IPAddress)
+		if address == nil {
+			return nil, fmt.Errorf("couldn't parse address: %s", instance.IPAddress)
+		}
+	} else {
+		resolved, err := net.LookupIP(name)
+		if err != nil || len(resolved) < 1 {
+			return nil, fmt.Errorf("couldn't lookup address: %s", name)
+		}
+		address = resolved[0]
+	}
+
+	return []v1.NodeAddress{
+		{Type: v1.NodeInternalIP, Address: address.String()},
+		{Type: v1.NodeExternalIP, Address: address.String()},
+	}, nil
+}
+
+// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID
+// This method will not be called from the node that is requesting this ID. i.e. metadata service
+// and other local methods cannot be used here
+func (v *OVirtCloud) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) {
+	return []v1.NodeAddress{}, errors.New("unimplemented")
+}
+
+// mapNodeNameToInstanceName maps from a k8s NodeName to an ovirt instance name (the hostname)
+// This is a simple string cast
+func mapNodeNameToInstanceName(nodeName types.NodeName) string {
+	return string(nodeName)
+}
+
+// ExternalID returns the cloud provider ID of the specified node with the specified NodeName (deprecated).
+func (v *OVirtCloud) ExternalID(nodeName types.NodeName) (string, error) {
+	name := mapNodeNameToInstanceName(nodeName)
+	instance, err := v.fetchInstance(name)
+	if err != nil {
+		return "", err
+	}
+	return instance.UUID, nil
+}
+
+// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running.
+// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager.
+func (v *OVirtCloud) InstanceExistsByProviderID(providerID string) (bool, error) {
+	return false, errors.New("unimplemented")
+}
+
+// InstanceID returns the cloud provider ID of the node with the specified NodeName.
+func (v *OVirtCloud) InstanceID(nodeName types.NodeName) (string, error) {
+	name := mapNodeNameToInstanceName(nodeName)
+	instance, err := v.fetchInstance(name)
+	if err != nil {
+		return "", err
+	}
+	// TODO: define a way to identify the provider instance to complete
+	// the format <provider_instance_id>/<instance_id>.
+	return "/" + instance.UUID, err
+}
+
+// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID
+// This method will not be called from the node that is requesting this ID. i.e. metadata service
+// and other local methods cannot be used here
+func (v *OVirtCloud) InstanceTypeByProviderID(providerID string) (string, error) {
+	return "", errors.New("unimplemented")
+}
+
+// InstanceType returns the type of the specified instance.
+func (v *OVirtCloud) InstanceType(name types.NodeName) (string, error) {
+	return "", nil
+}
+
+func getInstancesFromXml(body io.Reader) (OVirtInstanceMap, error) {
+	if body == nil {
+		return nil, fmt.Errorf("ovirt rest-api response body is missing")
+	}
+
+	content, err := ioutil.ReadAll(body)
+	if err != nil {
+		return nil, err
+	}
+
+	vmlist := XmlVmsList{}
+
+	if err := xml.Unmarshal(content, &vmlist); err != nil {
+		return nil, err
+	}
+
+	instances := make(OVirtInstanceMap)
+
+	for _, vm := range vmlist.Vm {
+		// Always return only vms that are up and running
+		if vm.Hostname != "" && strings.ToLower(vm.State) == "up" {
+			address := ""
+			if len(vm.Addresses) > 0 {
+				address = vm.Addresses[0].Address
+			}
+
+			instances[vm.Hostname] = OVirtInstance{
+				UUID:      vm.UUID,
+				Name:      vm.Name,
+				IPAddress: address,
+			}
+		}
+	}
+
+	return instances, nil
+}
+
+func (v *OVirtCloud) fetchAllInstances() (OVirtInstanceMap, error) {
+	response, err := http.Get(v.VmsRequest.String())
+	if err != nil {
+		return nil, err
+	}
+
+	defer response.Body.Close()
+
+	return getInstancesFromXml(response.Body)
+}
+
+func (v *OVirtCloud) fetchInstance(name string) (*OVirtInstance, error) {
+	allInstances, err := v.fetchAllInstances()
+	if err != nil {
+		return nil, err
+	}
+
+	instance, found := allInstances[name]
+	if !found {
+		return nil, fmt.Errorf("cannot find instance: %s", name)
+	}
+
+	return &instance, nil
+}
+
+func (m *OVirtInstanceMap) ListSortedNames() []string {
+	var names []string
+
+	for k := range *m {
+		names = append(names, k)
+	}
+
+	sort.Strings(names)
+
+	return names
+}
+
+// Implementation of Instances.CurrentNodeName
+func (v *OVirtCloud) CurrentNodeName(hostname string) (types.NodeName, error) {
+	return types.NodeName(hostname), nil
+}
+
+func (v *OVirtCloud) AddSSHKeyToAllInstances(user string, keyData []byte) error {
+	return errors.New("unimplemented")
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/ovirt/ovirt_test.go

@@ -0,0 +1,126 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ovirt
+
+import (
+	"io"
+	"strings"
+	"testing"
+
+	"k8s.io/kubernetes/pkg/cloudprovider"
+)
+
+func TestOVirtCloudConfiguration(t *testing.T) {
+	config1 := (io.Reader)(nil)
+
+	_, err1 := cloudprovider.GetCloudProvider("ovirt", config1)
+	if err1 == nil {
+		t.Fatalf("An error is expected when the configuration is missing")
+	}
+
+	config2 := strings.NewReader("")
+
+	_, err2 := cloudprovider.GetCloudProvider("ovirt", config2)
+	if err2 == nil {
+		t.Fatalf("An error is expected when the configuration is empty")
+	}
+
+	config3 := strings.NewReader(`
+[connection]
+	`)
+
+	_, err3 := cloudprovider.GetCloudProvider("ovirt", config3)
+	if err3 == nil {
+		t.Fatalf("An error is expected when the uri is missing")
+	}
+
+	config4 := strings.NewReader(`
+[connection]
+uri = https://localhost:8443/ovirt-engine/api
+`)
+
+	_, err4 := cloudprovider.GetCloudProvider("ovirt", config4)
+	if err4 != nil {
+		t.Fatalf("Unexpected error creating the provider: %s", err4)
+	}
+}
+
+func TestOVirtCloudXmlParsing(t *testing.T) {
+	body1 := (io.Reader)(nil)
+
+	_, err1 := getInstancesFromXml(body1)
+	if err1 == nil {
+		t.Fatalf("An error is expected when body is missing")
+	}
+
+	body2 := strings.NewReader("")
+
+	_, err2 := getInstancesFromXml(body2)
+	if err2 == nil {
+		t.Fatalf("An error is expected when body is empty")
+	}
+
+	body3 := strings.NewReader(`
+<vms>
+  <vm></vm>
+</vms>
+`)
+
+	instances3, err3 := getInstancesFromXml(body3)
+	if err3 != nil {
+		t.Fatalf("Unexpected error listing instances: %s", err3)
+	}
+	if len(instances3) > 0 {
+		t.Fatalf("Unexpected number of instance(s): %d", len(instances3))
+	}
+
+	body4 := strings.NewReader(`
+<vms>
+  <vm>
+    <status><state>Up</state></status>
+    <guest_info><fqdn>host1</fqdn></guest_info>
+  </vm>
+  <vm>
+    <!-- empty -->
+  </vm>
+  <vm>
+    <status><state>Up</state></status>
+  </vm>
+  <vm>
+    <status><state>Down</state></status>
+    <guest_info><fqdn>host2</fqdn></guest_info>
+  </vm>
+  <vm>
+    <status><state>Up</state></status>
+    <guest_info><fqdn>host3</fqdn></guest_info>
+  </vm>
+</vms>
+`)
+
+	instances4, err4 := getInstancesFromXml(body4)
+	if err4 != nil {
+		t.Fatalf("Unexpected error listing instances: %s", err4)
+	}
+	if len(instances4) != 2 {
+		t.Fatalf("Unexpected number of instance(s): %d", len(instances4))
+	}
+
+	names := instances4.ListSortedNames()
+	if names[0] != "host1" || names[1] != "host3" {
+		t.Fatalf("Unexpected instance(s): %s", instances4)
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/photon/BUILD

@@ -0,0 +1,46 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["photon.go"],
+    deps = [
+        "//pkg/api/v1/helper:go_default_library",
+        "//pkg/cloudprovider:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/vmware/photon-controller-go-sdk/photon:go_default_library",
+        "//vendor/gopkg.in/gcfg.v1:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["photon_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//pkg/cloudprovider:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/rand:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/photon/OWNERS

@@ -0,0 +1,4 @@
+maintainers:
+- luomiao
+- kerneltime
+- abrarshivani

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/photon/photon.go

@@ -0,0 +1,739 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// This version of Photon cloud provider supports the disk interface
+// for Photon persistent disk volume plugin. LoadBalancer, Routes, and
+// Zones are currently not supported.
+// The use of Photon cloud provider requires to start kubelet, kube-apiserver,
+// and kube-controller-manager with config flag: '--cloud-provider=photon
+// --cloud-config=[path_to_config_file]'. When running multi-node kubernetes
+// using docker, the config file should be located inside /etc/kubernetes.
+package photon
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"os"
+	"strings"
+
+	"github.com/golang/glog"
+	"github.com/vmware/photon-controller-go-sdk/photon"
+	"gopkg.in/gcfg.v1"
+	"k8s.io/api/core/v1"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+	v1helper "k8s.io/kubernetes/pkg/api/v1/helper"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+	"k8s.io/kubernetes/pkg/controller"
+)
+
+const (
+	ProviderName = "photon"
+	DiskSpecKind = "persistent-disk"
+	MAC_OUI_VC   = "00:50:56"
+	MAC_OUI_ESX  = "00:0c:29"
+)
+
+// overrideIP indicates if the hostname is overriden by IP address, such as when
+// running multi-node kubernetes using docker. In this case the user should set
+// overrideIP = true in cloud config file. Default value is false.
+var overrideIP bool = false
+
+// Photon is an implementation of the cloud provider interface for Photon Controller.
+type PCCloud struct {
+	cfg *PCConfig
+	// InstanceID of the server where this PCCloud object is instantiated.
+	localInstanceID string
+	// local $HOSTNAME
+	localHostname string
+	// hostname from K8S, could be overridden
+	localK8sHostname string
+	// Photon project ID. We assume that there is only one Photon Controller project
+	// in the environment per current Photon Controller deployment methodology.
+	projID string
+	cloudprovider.Zone
+	photonClient *photon.Client
+	logger       *log.Logger
+}
+
+type PCConfig struct {
+	Global struct {
+		// the Photon Controller endpoint IP address
+		CloudTarget string `gcfg:"target"`
+		// Photon Controller project name
+		Project string `gcfg:"project"`
+		// when kubelet is started with '--hostname-override=${IP_ADDRESS}', set to true;
+		// otherwise, set to false.
+		OverrideIP bool `gcfg:"overrideIP"`
+		// VM ID for this node
+		VMID string `gcfg:"vmID"`
+		// Authentication enabled or not
+		AuthEnabled bool `gcfg:"authentication"`
+	}
+}
+
+// Disks is interface for manipulation with PhotonController Persistent Disks.
+type Disks interface {
+	// AttachDisk attaches given disk to given node. Current node
+	// is used when nodeName is empty string.
+	AttachDisk(pdID string, nodeName k8stypes.NodeName) error
+
+	// DetachDisk detaches given disk to given node. Current node
+	// is used when nodeName is empty string.
+	DetachDisk(pdID string, nodeName k8stypes.NodeName) error
+
+	// DiskIsAttached checks if a disk is attached to the given node.
+	DiskIsAttached(pdID string, nodeName k8stypes.NodeName) (bool, error)
+
+	// DisksAreAttached is a batch function to check if a list of disks are attached
+	// to the node with the specified NodeName.
+	DisksAreAttached(pdIDs []string, nodeName k8stypes.NodeName) (map[string]bool, error)
+
+	// CreateDisk creates a new PD with given properties.
+	CreateDisk(volumeOptions *VolumeOptions) (pdID string, err error)
+
+	// DeleteDisk deletes PD.
+	DeleteDisk(pdID string) error
+}
+
+// VolumeOptions specifies capacity, tags, name and flavorID for a volume.
+type VolumeOptions struct {
+	CapacityGB int
+	Tags       map[string]string
+	Name       string
+	Flavor     string
+}
+
+func readConfig(config io.Reader) (PCConfig, error) {
+	if config == nil {
+		err := fmt.Errorf("cloud provider config file is missing. Please restart kubelet with --cloud-provider=photon --cloud-config=[path_to_config_file]")
+		return PCConfig{}, err
+	}
+
+	var cfg PCConfig
+	err := gcfg.ReadInto(&cfg, config)
+	return cfg, err
+}
+
+func init() {
+	cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) {
+		cfg, err := readConfig(config)
+		if err != nil {
+			glog.Errorf("Photon Cloud Provider: failed to read in cloud provider config file. Error[%v]", err)
+			return nil, err
+		}
+		return newPCCloud(cfg)
+	})
+}
+
+// Retrieve the Photon VM ID from the Photon Controller endpoint based on the node name
+func getVMIDbyNodename(pc *PCCloud, nodeName string) (string, error) {
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for getVMIDbyNodename, error: [%v]", err)
+		return "", err
+	}
+
+	vmList, err := photonClient.Projects.GetVMs(pc.projID, nil)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to GetVMs from project %s with nodeName %s, error: [%v]", pc.projID, nodeName, err)
+		return "", err
+	}
+
+	for _, vm := range vmList.Items {
+		if vm.Name == nodeName {
+			return vm.ID, nil
+		}
+	}
+
+	return "", fmt.Errorf("No matching started VM is found with name %s", nodeName)
+}
+
+// Retrieve the Photon VM ID from the Photon Controller endpoint based on the IP address
+func getVMIDbyIP(pc *PCCloud, IPAddress string) (string, error) {
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for getVMIDbyNodename, error: [%v]", err)
+		return "", err
+	}
+
+	vmList, err := photonClient.Projects.GetVMs(pc.projID, nil)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to GetVMs for project %s. error: [%v]", pc.projID, err)
+		return "", err
+	}
+
+	for _, vm := range vmList.Items {
+		task, err := photonClient.VMs.GetNetworks(vm.ID)
+		if err != nil {
+			glog.Warningf("Photon Cloud Provider: GetNetworks failed for vm.ID %s, error [%v]", vm.ID, err)
+		} else {
+			task, err = photonClient.Tasks.Wait(task.ID)
+			if err != nil {
+				glog.Warningf("Photon Cloud Provider: Wait task for GetNetworks failed for vm.ID %s, error [%v]", vm.ID, err)
+			} else {
+				networkConnections := task.ResourceProperties.(map[string]interface{})
+				networks := networkConnections["networkConnections"].([]interface{})
+				for _, nt := range networks {
+					network := nt.(map[string]interface{})
+					if val, ok := network["ipAddress"]; ok && val != nil {
+						ipAddr := val.(string)
+						if ipAddr == IPAddress {
+							return vm.ID, nil
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return "", fmt.Errorf("No matching VM is found with IP %s", IPAddress)
+}
+
+func getPhotonClient(pc *PCCloud) (*photon.Client, error) {
+	var err error
+	if len(pc.cfg.Global.CloudTarget) == 0 {
+		return nil, fmt.Errorf("Photon Controller endpoint was not specified.")
+	}
+
+	options := &photon.ClientOptions{
+		IgnoreCertificate: true,
+	}
+
+	pc.photonClient = photon.NewClient(pc.cfg.Global.CloudTarget, options, pc.logger)
+	if pc.cfg.Global.AuthEnabled == true {
+		// work around before metadata is available
+		file, err := os.Open("/etc/kubernetes/pc_login_info")
+		if err != nil {
+			glog.Errorf("Photon Cloud Provider: Authentication is enabled but found no username/password at /etc/kubernetes/pc_login_info. Error[%v]", err)
+			return nil, err
+		}
+		defer file.Close()
+		scanner := bufio.NewScanner(file)
+		if !scanner.Scan() {
+			glog.Errorf("Photon Cloud Provider: Empty username inside /etc/kubernetes/pc_login_info.")
+			return nil, fmt.Errorf("Failed to create authentication enabled client with invalid username")
+		}
+		username := scanner.Text()
+		if !scanner.Scan() {
+			glog.Errorf("Photon Cloud Provider: Empty password set inside /etc/kubernetes/pc_login_info.")
+			return nil, fmt.Errorf("Failed to create authentication enabled client with invalid password")
+		}
+		password := scanner.Text()
+
+		token_options, err := pc.photonClient.Auth.GetTokensByPassword(username, password)
+		if err != nil {
+			glog.Errorf("Photon Cloud Provider: failed to get tokens by password")
+			return nil, err
+		}
+
+		options = &photon.ClientOptions{
+			IgnoreCertificate: true,
+			TokenOptions: &photon.TokenOptions{
+				AccessToken: token_options.AccessToken,
+			},
+		}
+		pc.photonClient = photon.NewClient(pc.cfg.Global.CloudTarget, options, pc.logger)
+	}
+
+	status, err := pc.photonClient.Status.Get()
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: new client creation failed. Error[%v]", err)
+		return nil, err
+	}
+	glog.V(2).Infof("Photon Cloud Provider: Status of the new photon controller client: %v", status)
+
+	return pc.photonClient, nil
+}
+
+func newPCCloud(cfg PCConfig) (*PCCloud, error) {
+	projID := cfg.Global.Project
+	vmID := cfg.Global.VMID
+
+	// Get local hostname
+	hostname, err := os.Hostname()
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: get hostname failed. Error[%v]", err)
+		return nil, err
+	}
+	pc := PCCloud{
+		cfg:              &cfg,
+		localInstanceID:  vmID,
+		localHostname:    hostname,
+		localK8sHostname: "",
+		projID:           projID,
+	}
+
+	overrideIP = cfg.Global.OverrideIP
+
+	return &pc, nil
+}
+
+// Initialize passes a Kubernetes clientBuilder interface to the cloud provider
+func (pc *PCCloud) Initialize(clientBuilder controller.ControllerClientBuilder) {}
+
+// Instances returns an implementation of Instances for Photon Controller.
+func (pc *PCCloud) Instances() (cloudprovider.Instances, bool) {
+	return pc, true
+}
+
+// List is an implementation of Instances.List.
+func (pc *PCCloud) List(filter string) ([]k8stypes.NodeName, error) {
+	return nil, nil
+}
+
+// NodeAddresses is an implementation of Instances.NodeAddresses.
+func (pc *PCCloud) NodeAddresses(nodeName k8stypes.NodeName) ([]v1.NodeAddress, error) {
+	nodeAddrs := []v1.NodeAddress{}
+	name := string(nodeName)
+
+	if name == pc.localK8sHostname {
+		ifaces, err := net.Interfaces()
+		if err != nil {
+			glog.Errorf("Photon Cloud Provider: net.Interfaces() failed for NodeAddresses. Error[%v]", err)
+			return nodeAddrs, err
+		}
+
+		for _, i := range ifaces {
+			addrs, err := i.Addrs()
+			if err != nil {
+				glog.Warningf("Photon Cloud Provider: Failed to extract addresses for NodeAddresses. Error[%v]", err)
+			} else {
+				for _, addr := range addrs {
+					if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
+						if ipnet.IP.To4() != nil {
+							// Filter external IP by MAC address OUIs from vCenter and from ESX
+							if strings.HasPrefix(i.HardwareAddr.String(), MAC_OUI_VC) ||
+								strings.HasPrefix(i.HardwareAddr.String(), MAC_OUI_ESX) {
+								v1helper.AddToNodeAddresses(&nodeAddrs,
+									v1.NodeAddress{
+										Type:    v1.NodeExternalIP,
+										Address: ipnet.IP.String(),
+									},
+								)
+							} else {
+								v1helper.AddToNodeAddresses(&nodeAddrs,
+									v1.NodeAddress{
+										Type:    v1.NodeInternalIP,
+										Address: ipnet.IP.String(),
+									},
+								)
+							}
+						}
+					}
+				}
+			}
+		}
+		return nodeAddrs, nil
+	}
+
+	// Inquiring IP addresses from photon controller endpoint only for a node other than this node.
+	// This is assumed to be done by master only.
+	vmID, err := getInstanceID(pc, name)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: getInstanceID failed for NodeAddresses. Error[%v]", err)
+		return nodeAddrs, err
+	}
+
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for NodeAddresses, error: [%v]", err)
+		return nodeAddrs, err
+	}
+
+	// Retrieve the Photon VM's IP addresses from the Photon Controller endpoint based on the VM ID
+	vmList, err := photonClient.Projects.GetVMs(pc.projID, nil)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to GetVMs for project %s. Error[%v]", pc.projID, err)
+		return nodeAddrs, err
+	}
+
+	for _, vm := range vmList.Items {
+		if vm.ID == vmID {
+			task, err := photonClient.VMs.GetNetworks(vm.ID)
+			if err != nil {
+				glog.Errorf("Photon Cloud Provider: GetNetworks failed for node %s with vm.ID %s. Error[%v]", name, vm.ID, err)
+				return nodeAddrs, err
+			} else {
+				task, err = photonClient.Tasks.Wait(task.ID)
+				if err != nil {
+					glog.Errorf("Photon Cloud Provider: Wait task for GetNetworks failed for node %s with vm.ID %s. Error[%v]", name, vm.ID, err)
+					return nodeAddrs, err
+				} else {
+					networkConnections := task.ResourceProperties.(map[string]interface{})
+					networks := networkConnections["networkConnections"].([]interface{})
+					for _, nt := range networks {
+						ipAddr := "-"
+						macAddr := "-"
+						network := nt.(map[string]interface{})
+						if val, ok := network["ipAddress"]; ok && val != nil {
+							ipAddr = val.(string)
+						}
+						if val, ok := network["macAddress"]; ok && val != nil {
+							macAddr = val.(string)
+						}
+						if ipAddr != "-" {
+							if strings.HasPrefix(macAddr, MAC_OUI_VC) ||
+								strings.HasPrefix(macAddr, MAC_OUI_ESX) {
+								v1helper.AddToNodeAddresses(&nodeAddrs,
+									v1.NodeAddress{
+										Type:    v1.NodeExternalIP,
+										Address: ipAddr,
+									},
+								)
+							} else {
+								v1helper.AddToNodeAddresses(&nodeAddrs,
+									v1.NodeAddress{
+										Type:    v1.NodeInternalIP,
+										Address: ipAddr,
+									},
+								)
+							}
+						}
+					}
+					return nodeAddrs, nil
+				}
+			}
+		}
+	}
+
+	glog.Errorf("Failed to find the node %s from Photon Controller endpoint", name)
+	return nodeAddrs, fmt.Errorf("Failed to find the node %s from Photon Controller endpoint", name)
+}
+
+// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID
+// This method will not be called from the node that is requesting this ID. i.e. metadata service
+// and other local methods cannot be used here
+func (pc *PCCloud) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) {
+	return []v1.NodeAddress{}, errors.New("unimplemented")
+}
+
+func (pc *PCCloud) AddSSHKeyToAllInstances(user string, keyData []byte) error {
+	return errors.New("unimplemented")
+}
+
+func (pc *PCCloud) CurrentNodeName(hostname string) (k8stypes.NodeName, error) {
+	pc.localK8sHostname = hostname
+	return k8stypes.NodeName(hostname), nil
+}
+
+func getInstanceID(pc *PCCloud, name string) (string, error) {
+	var vmID string
+	var err error
+
+	if overrideIP == true {
+		vmID, err = getVMIDbyIP(pc, name)
+	} else {
+		vmID, err = getVMIDbyNodename(pc, name)
+	}
+	if err != nil {
+		return "", err
+	}
+
+	if vmID == "" {
+		err = cloudprovider.InstanceNotFound
+	}
+
+	return vmID, err
+}
+
+// ExternalID returns the cloud provider ID of the specified instance (deprecated).
+func (pc *PCCloud) ExternalID(nodeName k8stypes.NodeName) (string, error) {
+	name := string(nodeName)
+	if name == pc.localK8sHostname {
+		return pc.localInstanceID, nil
+	} else {
+		// We assume only master need to get InstanceID of a node other than itself
+		ID, err := getInstanceID(pc, name)
+		if err != nil {
+			glog.Errorf("Photon Cloud Provider: getInstanceID failed for ExternalID. Error[%v]", err)
+			return ID, err
+		} else {
+			return ID, nil
+		}
+	}
+}
+
+// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running.
+// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager.
+func (pc *PCCloud) InstanceExistsByProviderID(providerID string) (bool, error) {
+	return false, errors.New("unimplemented")
+}
+
+// InstanceID returns the cloud provider ID of the specified instance.
+func (pc *PCCloud) InstanceID(nodeName k8stypes.NodeName) (string, error) {
+	name := string(nodeName)
+	if name == pc.localK8sHostname {
+		return pc.localInstanceID, nil
+	} else {
+		// We assume only master need to get InstanceID of a node other than itself
+		ID, err := getInstanceID(pc, name)
+		if err != nil {
+			glog.Errorf("Photon Cloud Provider: getInstanceID failed for InstanceID. Error[%v]", err)
+			return ID, err
+		} else {
+			return ID, nil
+		}
+	}
+}
+
+// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID
+// This method will not be called from the node that is requesting this ID. i.e. metadata service
+// and other local methods cannot be used here
+func (pc *PCCloud) InstanceTypeByProviderID(providerID string) (string, error) {
+	return "", errors.New("unimplemented")
+}
+
+func (pc *PCCloud) InstanceType(nodeName k8stypes.NodeName) (string, error) {
+	return "", nil
+}
+
+func (pc *PCCloud) Clusters() (cloudprovider.Clusters, bool) {
+	return nil, true
+}
+
+// ProviderName returns the cloud provider ID.
+func (pc *PCCloud) ProviderName() string {
+	return ProviderName
+}
+
+// LoadBalancer returns an implementation of LoadBalancer for Photon Controller.
+func (pc *PCCloud) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
+	return nil, false
+}
+
+// Zones returns an implementation of Zones for Photon Controller.
+func (pc *PCCloud) Zones() (cloudprovider.Zones, bool) {
+	return pc, true
+}
+
+func (pc *PCCloud) GetZone() (cloudprovider.Zone, error) {
+	return pc.Zone, nil
+}
+
+// GetZoneByProviderID implements Zones.GetZoneByProviderID
+// This is particularly useful in external cloud providers where the kubelet
+// does not initialize node data.
+func (pc *PCCloud) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) {
+	return cloudprovider.Zone{}, errors.New("GetZoneByProviderID not implemented")
+}
+
+// GetZoneByNodeName implements Zones.GetZoneByNodeName
+// This is particularly useful in external cloud providers where the kubelet
+// does not initialize node data.
+func (pc *PCCloud) GetZoneByNodeName(nodeName k8stypes.NodeName) (cloudprovider.Zone, error) {
+	return cloudprovider.Zone{}, errors.New("GetZoneByNodeName not imeplemented")
+}
+
+// Routes returns a false since the interface is not supported for photon controller.
+func (pc *PCCloud) Routes() (cloudprovider.Routes, bool) {
+	return nil, false
+}
+
+// ScrubDNS filters DNS settings for pods.
+func (pc *PCCloud) ScrubDNS(nameservers, searches []string) (nsOut, srchOut []string) {
+	return nameservers, searches
+}
+
+// HasClusterID returns true if the cluster has a clusterID
+func (pc *PCCloud) HasClusterID() bool {
+	return true
+}
+
+// Attaches given virtual disk volume to the compute running kubelet.
+func (pc *PCCloud) AttachDisk(pdID string, nodeName k8stypes.NodeName) error {
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for AttachDisk, error: [%v]", err)
+		return err
+	}
+
+	operation := &photon.VmDiskOperation{
+		DiskID: pdID,
+	}
+
+	vmID, err := pc.InstanceID(nodeName)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: pc.InstanceID failed for AttachDisk. Error[%v]", err)
+		return err
+	}
+
+	task, err := photonClient.VMs.AttachDisk(vmID, operation)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to attach disk with pdID %s. Error[%v]", pdID, err)
+		return err
+	}
+
+	_, err = photonClient.Tasks.Wait(task.ID)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to wait for task to attach disk with pdID %s. Error[%v]", pdID, err)
+		return err
+	}
+
+	return nil
+}
+
+// Detaches given virtual disk volume from the compute running kubelet.
+func (pc *PCCloud) DetachDisk(pdID string, nodeName k8stypes.NodeName) error {
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for DetachDisk, error: [%v]", err)
+		return err
+	}
+
+	operation := &photon.VmDiskOperation{
+		DiskID: pdID,
+	}
+
+	vmID, err := pc.InstanceID(nodeName)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: pc.InstanceID failed for DetachDisk. Error[%v]", err)
+		return err
+	}
+
+	task, err := photonClient.VMs.DetachDisk(vmID, operation)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to detach disk with pdID %s. Error[%v]", pdID, err)
+		return err
+	}
+
+	_, err = photonClient.Tasks.Wait(task.ID)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to wait for task to detach disk with pdID %s. Error[%v]", pdID, err)
+		return err
+	}
+
+	return nil
+}
+
+// DiskIsAttached returns if disk is attached to the VM using controllers supported by the plugin.
+func (pc *PCCloud) DiskIsAttached(pdID string, nodeName k8stypes.NodeName) (bool, error) {
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for DiskIsAttached, error: [%v]", err)
+		return false, err
+	}
+
+	disk, err := photonClient.Disks.Get(pdID)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to Get disk with pdID %s. Error[%v]", pdID, err)
+		return false, err
+	}
+
+	vmID, err := pc.InstanceID(nodeName)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: pc.InstanceID failed for DiskIsAttached. Error[%v]", err)
+		return false, err
+	}
+
+	for _, vm := range disk.VMs {
+		if vm == vmID {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+// DisksAreAttached returns if disks are attached to the VM using controllers supported by the plugin.
+func (pc *PCCloud) DisksAreAttached(pdIDs []string, nodeName k8stypes.NodeName) (map[string]bool, error) {
+	attached := make(map[string]bool)
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for DisksAreAttached, error: [%v]", err)
+		return attached, err
+	}
+
+	for _, pdID := range pdIDs {
+		attached[pdID] = false
+	}
+
+	vmID, err := pc.InstanceID(nodeName)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: pc.InstanceID failed for DiskIsAttached. Error[%v]", err)
+		return attached, err
+	}
+
+	for _, pdID := range pdIDs {
+		disk, err := photonClient.Disks.Get(pdID)
+		if err != nil {
+			glog.Warningf("Photon Cloud Provider: failed to get VMs for persistent disk %s, err [%v]", pdID, err)
+		} else {
+			for _, vm := range disk.VMs {
+				if vm == vmID {
+					attached[pdID] = true
+				}
+			}
+		}
+	}
+
+	return attached, nil
+}
+
+// Create a volume of given size (in GB).
+func (pc *PCCloud) CreateDisk(volumeOptions *VolumeOptions) (pdID string, err error) {
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for CreateDisk, error: [%v]", err)
+		return "", err
+	}
+
+	diskSpec := photon.DiskCreateSpec{}
+	diskSpec.Name = volumeOptions.Name
+	diskSpec.Flavor = volumeOptions.Flavor
+	diskSpec.CapacityGB = volumeOptions.CapacityGB
+	diskSpec.Kind = DiskSpecKind
+
+	task, err := photonClient.Projects.CreateDisk(pc.projID, &diskSpec)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to CreateDisk. Error[%v]", err)
+		return "", err
+	}
+
+	waitTask, err := photonClient.Tasks.Wait(task.ID)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to wait for task to CreateDisk. Error[%v]", err)
+		return "", err
+	}
+
+	return waitTask.Entity.ID, nil
+}
+
+// Deletes a volume given volume name.
+func (pc *PCCloud) DeleteDisk(pdID string) error {
+	photonClient, err := getPhotonClient(pc)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to get photon client for DeleteDisk, error: [%v]", err)
+		return err
+	}
+
+	task, err := photonClient.Disks.Delete(pdID)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to DeleteDisk. Error[%v]", err)
+		return err
+	}
+
+	_, err = photonClient.Tasks.Wait(task.ID)
+	if err != nil {
+		glog.Errorf("Photon Cloud Provider: Failed to wait for task to DeleteDisk. Error[%v]", err)
+		return err
+	}
+
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/photon/photon_test.go

@@ -0,0 +1,216 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package photon
+
+import (
+	"log"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/rand"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+)
+
+func configFromEnv() (TestVM string, TestFlavor string, cfg PCConfig, ok bool) {
+	var AuthEnabled bool
+	var OverrideIP bool
+	var err error
+	cfg.Global.CloudTarget = os.Getenv("PHOTON_TARGET")
+	cfg.Global.Project = os.Getenv("PHOTON_PROJECT")
+	cfg.Global.VMID = os.Getenv("PHOTON_VMID")
+	if os.Getenv("PHOTON_AUTH_ENABLED") != "" {
+		AuthEnabled, err = strconv.ParseBool(os.Getenv("PHOTON_AUTH_ENABLED"))
+	} else {
+		AuthEnabled = false
+	}
+	if err != nil {
+		log.Fatal(err)
+	}
+	cfg.Global.AuthEnabled = AuthEnabled
+	if os.Getenv("PHOTON_OVERRIDE_IP") != "" {
+		OverrideIP, err = strconv.ParseBool(os.Getenv("PHOTON_OVERRIDE_IP"))
+	} else {
+		OverrideIP = false
+	}
+	if err != nil {
+		log.Fatal(err)
+	}
+	cfg.Global.OverrideIP = OverrideIP
+
+	TestVM = os.Getenv("PHOTON_TEST_VM")
+	if os.Getenv("PHOTON_TEST_FLAVOR") != "" {
+		TestFlavor = os.Getenv("PHOTON_TEST_FLAVOR")
+	} else {
+		TestFlavor = ""
+	}
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	ok = (cfg.Global.CloudTarget != "" &&
+		cfg.Global.Project != "" &&
+		cfg.Global.VMID != "" &&
+		TestVM != "")
+
+	return
+}
+
+func TestReadConfig(t *testing.T) {
+	_, err := readConfig(nil)
+	if err == nil {
+		t.Errorf("Should fail when no config is provided: %s", err)
+	}
+
+	cfg, err := readConfig(strings.NewReader(`
+[Global]
+target = 0.0.0.0
+project = project
+overrideIP = true
+vmID = vmid
+authentication = false
+`))
+	if err != nil {
+		t.Fatalf("Should succeed when a valid config is provided: %s", err)
+	}
+
+	if cfg.Global.CloudTarget != "0.0.0.0" {
+		t.Errorf("incorrect photon target ip: %s", cfg.Global.CloudTarget)
+	}
+
+	if cfg.Global.Project != "project" {
+		t.Errorf("incorrect project: %s", cfg.Global.Project)
+	}
+
+	if cfg.Global.VMID != "vmid" {
+		t.Errorf("incorrect vmid: %s", cfg.Global.VMID)
+	}
+}
+
+func TestNewPCCloud(t *testing.T) {
+	_, _, cfg, ok := configFromEnv()
+	if !ok {
+		t.Skipf("No config found in environment")
+	}
+
+	_, err := newPCCloud(cfg)
+	if err != nil {
+		t.Fatalf("Failed to create new Photon client: %s", err)
+	}
+}
+
+func TestInstances(t *testing.T) {
+	testVM, _, cfg, ok := configFromEnv()
+	if !ok {
+		t.Skipf("No config found in environment")
+	}
+	NodeName := types.NodeName(testVM)
+
+	pc, err := newPCCloud(cfg)
+	if err != nil {
+		t.Fatalf("Failed to create new Photon client: %s", err)
+	}
+
+	i, ok := pc.Instances()
+	if !ok {
+		t.Fatalf("Instances() returned false")
+	}
+
+	externalId, err := i.ExternalID(NodeName)
+	if err != nil {
+		t.Fatalf("Instances.ExternalID(%s) failed: %s", testVM, err)
+	}
+	t.Logf("Found ExternalID(%s) = %s\n", testVM, externalId)
+
+	nonExistingVM := types.NodeName(rand.String(15))
+	externalId, err = i.ExternalID(nonExistingVM)
+	if err == cloudprovider.InstanceNotFound {
+		t.Logf("VM %s was not found as expected\n", nonExistingVM)
+	} else if err == nil {
+		t.Fatalf("Instances.ExternalID did not fail as expected, VM %s was found", nonExistingVM)
+	} else {
+		t.Fatalf("Instances.ExternalID did not fail as expected, err: %v", err)
+	}
+
+	instanceId, err := i.InstanceID(NodeName)
+	if err != nil {
+		t.Fatalf("Instances.InstanceID(%s) failed: %s", testVM, err)
+	}
+	t.Logf("Found InstanceID(%s) = %s\n", testVM, instanceId)
+
+	instanceId, err = i.InstanceID(nonExistingVM)
+	if err == cloudprovider.InstanceNotFound {
+		t.Logf("VM %s was not found as expected\n", nonExistingVM)
+	} else if err == nil {
+		t.Fatalf("Instances.InstanceID did not fail as expected, VM %s was found", nonExistingVM)
+	} else {
+		t.Fatalf("Instances.InstanceID did not fail as expected, err: %v", err)
+	}
+
+	addrs, err := i.NodeAddresses(NodeName)
+	if err != nil {
+		t.Fatalf("Instances.NodeAddresses(%s) failed: %s", testVM, err)
+	}
+	t.Logf("Found NodeAddresses(%s) = %s\n", testVM, addrs)
+}
+
+func TestVolumes(t *testing.T) {
+	testVM, testFlavor, cfg, ok := configFromEnv()
+	if !ok {
+		t.Skipf("No config found in environment")
+	}
+
+	pc, err := newPCCloud(cfg)
+	if err != nil {
+		t.Fatalf("Failed to create new Photon client: %s", err)
+	}
+
+	NodeName := types.NodeName(testVM)
+
+	volumeOptions := &VolumeOptions{
+		CapacityGB: 2,
+		Tags:       nil,
+		Name:       "kubernetes-test-volume-" + rand.String(10),
+		Flavor:     testFlavor}
+
+	pdID, err := pc.CreateDisk(volumeOptions)
+	if err != nil {
+		t.Fatalf("Cannot create a Photon persistent disk: %v", err)
+	}
+
+	err = pc.AttachDisk(pdID, NodeName)
+	if err != nil {
+		t.Fatalf("Cannot attach persistent disk(%s) to VM(%s): %v", pdID, testVM, err)
+	}
+
+	_, err = pc.DiskIsAttached(pdID, NodeName)
+	if err != nil {
+		t.Fatalf("Cannot attach persistent disk(%s) to VM(%s): %v", pdID, testVM, err)
+	}
+
+	err = pc.DetachDisk(pdID, NodeName)
+	if err != nil {
+		t.Fatalf("Cannot detach persisten disk(%s) from VM(%s): %v", pdID, testVM, err)
+	}
+
+	err = pc.DeleteDisk(pdID)
+	if err != nil {
+		t.Fatalf("Cannot delete persisten disk(%s): %v", pdID, err)
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/providers.go

@@ -0,0 +1,30 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudprovider
+
+import (
+	// Cloud providers
+	_ "k8s.io/kubernetes/pkg/cloudprovider/providers/aws"
+	_ "k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
+	_ "k8s.io/kubernetes/pkg/cloudprovider/providers/cloudstack"
+	_ "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
+	_ "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack"
+	_ "k8s.io/kubernetes/pkg/cloudprovider/providers/ovirt"
+	_ "k8s.io/kubernetes/pkg/cloudprovider/providers/photon"
+	_ "k8s.io/kubernetes/pkg/cloudprovider/providers/rackspace"
+	_ "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
+)

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/rackspace/BUILD

@@ -0,0 +1,48 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["rackspace.go"],
+    deps = [
+        "//pkg/cloudprovider:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/rackspace/gophercloud:go_default_library",
+        "//vendor/github.com/rackspace/gophercloud/openstack/compute/v2/extensions/volumeattach:go_default_library",
+        "//vendor/github.com/rackspace/gophercloud/openstack/compute/v2/servers:go_default_library",
+        "//vendor/github.com/rackspace/gophercloud/pagination:go_default_library",
+        "//vendor/github.com/rackspace/gophercloud/rackspace:go_default_library",
+        "//vendor/github.com/rackspace/gophercloud/rackspace/blockstorage/v1/volumes:go_default_library",
+        "//vendor/github.com/rackspace/gophercloud/rackspace/compute/v2/servers:go_default_library",
+        "//vendor/github.com/rackspace/gophercloud/rackspace/compute/v2/volumeattach:go_default_library",
+        "//vendor/gopkg.in/gcfg.v1:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["rackspace_test.go"],
+    library = ":go_default_library",
+    deps = ["//vendor/github.com/rackspace/gophercloud:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/rackspace/rackspace.go

@@ -0,0 +1,786 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rackspace
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"os"
+	"regexp"
+	"time"
+
+	"gopkg.in/gcfg.v1"
+
+	"github.com/golang/glog"
+	"github.com/rackspace/gophercloud"
+	osvolumeattach "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/volumeattach"
+	osservers "github.com/rackspace/gophercloud/openstack/compute/v2/servers"
+	"github.com/rackspace/gophercloud/pagination"
+	"github.com/rackspace/gophercloud/rackspace"
+	"github.com/rackspace/gophercloud/rackspace/blockstorage/v1/volumes"
+	"github.com/rackspace/gophercloud/rackspace/compute/v2/servers"
+	"github.com/rackspace/gophercloud/rackspace/compute/v2/volumeattach"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+	"k8s.io/kubernetes/pkg/controller"
+)
+
+const (
+	ProviderName          = "rackspace"
+	MetaDataPath          = "/media/configdrive/openstack/latest/meta_data.json"
+	VolumeAvailableStatus = "available"
+	VolumeInUseStatus     = "in-use"
+	VolumeErrorStatus     = "error"
+)
+
+var ErrNotFound = errors.New("Failed to find object")
+var ErrMultipleResults = errors.New("Multiple results where only one expected")
+var ErrNoAddressFound = errors.New("No address found for host")
+var ErrAttrNotFound = errors.New("Expected attribute not found")
+
+// encoding.TextUnmarshaler interface for time.Duration
+type MyDuration struct {
+	time.Duration
+}
+
+func (d *MyDuration) UnmarshalText(text []byte) error {
+	res, err := time.ParseDuration(string(text))
+	if err != nil {
+		return err
+	}
+	d.Duration = res
+	return nil
+}
+
+type MetaData struct {
+	UUID string `json:"uuid"`
+	Name string `json:"name"`
+}
+
+type LoadBalancerOpts struct {
+	SubnetId          string     `gcfg:"subnet-id"` // required
+	CreateMonitor     bool       `gcfg:"create-monitor"`
+	MonitorDelay      MyDuration `gcfg:"monitor-delay"`
+	MonitorTimeout    MyDuration `gcfg:"monitor-timeout"`
+	MonitorMaxRetries uint       `gcfg:"monitor-max-retries"`
+}
+
+// Rackspace is an implementation of cloud provider Interface for Rackspace.
+type Rackspace struct {
+	provider *gophercloud.ProviderClient
+	region   string
+	lbOpts   LoadBalancerOpts
+}
+
+type Config struct {
+	Global struct {
+		AuthUrl    string `gcfg:"auth-url"`
+		Username   string
+		UserId     string `gcfg:"user-id"`
+		Password   string
+		ApiKey     string `gcfg:"api-key"`
+		TenantId   string `gcfg:"tenant-id"`
+		TenantName string `gcfg:"tenant-name"`
+		DomainId   string `gcfg:"domain-id"`
+		DomainName string `gcfg:"domain-name"`
+		Region     string
+	}
+	LoadBalancer LoadBalancerOpts
+}
+
+func probeNodeAddress(compute *gophercloud.ServiceClient, name string) (string, error) {
+	id, err := readInstanceID()
+	if err == nil {
+		srv, err := servers.Get(compute, id).Extract()
+		if err != nil {
+			return "", err
+		}
+		return getAddressByServer(srv)
+	}
+
+	ip, err := getAddressByName(compute, name)
+	if err != nil {
+		return "", err
+	}
+
+	return ip, nil
+}
+
+func probeInstanceID(client *gophercloud.ServiceClient, name string) (string, error) {
+	// Attempt to read id from config drive.
+	id, err := readInstanceID()
+	if err == nil {
+		return id, nil
+	}
+
+	// Attempt to get the server by the name from the API
+	server, err := getServerByName(client, name)
+	if err != nil {
+		return "", err
+	}
+
+	return server.ID, nil
+}
+
+func parseMetaData(file io.Reader) (string, error) {
+	metaDataBytes, err := ioutil.ReadAll(file)
+	if err != nil {
+		return "", fmt.Errorf("Cannot read %s: %v", file, err)
+	}
+
+	metaData := MetaData{}
+	err = json.Unmarshal(metaDataBytes, &metaData)
+	if err != nil {
+		return "", fmt.Errorf("Cannot parse %s: %v", MetaDataPath, err)
+	}
+
+	return metaData.UUID, nil
+}
+
+func readInstanceID() (string, error) {
+	file, err := os.Open(MetaDataPath)
+	if err != nil {
+		return "", fmt.Errorf("Cannot open %s: %v", MetaDataPath, err)
+	}
+	defer file.Close()
+
+	return parseMetaData(file)
+}
+
+func init() {
+	cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) {
+		cfg, err := readConfig(config)
+		if err != nil {
+			return nil, err
+		}
+		return newRackspace(cfg)
+	})
+}
+
+func (cfg Config) toAuthOptions() gophercloud.AuthOptions {
+	return gophercloud.AuthOptions{
+		IdentityEndpoint: cfg.Global.AuthUrl,
+		Username:         cfg.Global.Username,
+		UserID:           cfg.Global.UserId,
+		Password:         cfg.Global.Password,
+		APIKey:           cfg.Global.ApiKey,
+		TenantID:         cfg.Global.TenantId,
+		TenantName:       cfg.Global.TenantName,
+
+		// Persistent service, so we need to be able to renew tokens
+		AllowReauth: true,
+	}
+}
+
+func readConfig(config io.Reader) (Config, error) {
+	if config == nil {
+		err := fmt.Errorf("no Rackspace cloud provider config file given")
+		return Config{}, err
+	}
+
+	var cfg Config
+	err := gcfg.ReadInto(&cfg, config)
+	return cfg, err
+}
+
+func newRackspace(cfg Config) (*Rackspace, error) {
+	provider, err := rackspace.AuthenticatedClient(cfg.toAuthOptions())
+	if err != nil {
+		return nil, err
+	}
+
+	os := Rackspace{
+		provider: provider,
+		region:   cfg.Global.Region,
+		lbOpts:   cfg.LoadBalancer,
+	}
+
+	return &os, nil
+}
+
+// Initialize passes a Kubernetes clientBuilder interface to the cloud provider
+func (os *Rackspace) Initialize(clientBuilder controller.ControllerClientBuilder) {}
+
+type Instances struct {
+	compute *gophercloud.ServiceClient
+}
+
+// Instances returns an implementation of Instances for Rackspace.
+func (os *Rackspace) Instances() (cloudprovider.Instances, bool) {
+	glog.V(2).Info("rackspace.Instances() called")
+
+	compute, err := os.getComputeClient()
+	if err != nil {
+		glog.Warningf("Failed to find compute endpoint: %v", err)
+		return nil, false
+	}
+	glog.V(1).Info("Claiming to support Instances")
+
+	return &Instances{compute}, true
+}
+
+func serverHasAddress(srv osservers.Server, ip string) bool {
+	if ip == firstAddr(srv.Addresses["private"]) {
+		return true
+	}
+	if ip == firstAddr(srv.Addresses["public"]) {
+		return true
+	}
+	if ip == srv.AccessIPv4 {
+		return true
+	}
+	if ip == srv.AccessIPv6 {
+		return true
+	}
+	return false
+}
+
+func getServerByAddress(client *gophercloud.ServiceClient, name string) (*osservers.Server, error) {
+	pager := servers.List(client, nil)
+
+	serverList := make([]osservers.Server, 0, 1)
+
+	err := pager.EachPage(func(page pagination.Page) (bool, error) {
+		s, err := servers.ExtractServers(page)
+		if err != nil {
+			return false, err
+		}
+		for _, v := range s {
+			if serverHasAddress(v, name) {
+				serverList = append(serverList, v)
+			}
+		}
+		if len(serverList) > 1 {
+			return false, ErrMultipleResults
+		}
+		return true, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if len(serverList) == 0 {
+		return nil, ErrNotFound
+	} else if len(serverList) > 1 {
+		return nil, ErrMultipleResults
+	}
+
+	return &serverList[0], nil
+}
+
+func getServerByName(client *gophercloud.ServiceClient, name string) (*osservers.Server, error) {
+	if net.ParseIP(name) != nil {
+		// we're an IP, so we'll have to walk the full list of servers to
+		// figure out which one we are.
+		return getServerByAddress(client, name)
+	}
+	opts := osservers.ListOpts{
+		Name:   fmt.Sprintf("^%s$", regexp.QuoteMeta(name)),
+		Status: "ACTIVE",
+	}
+	pager := servers.List(client, opts)
+
+	serverList := make([]osservers.Server, 0, 1)
+
+	err := pager.EachPage(func(page pagination.Page) (bool, error) {
+		s, err := servers.ExtractServers(page)
+		if err != nil {
+			return false, err
+		}
+		serverList = append(serverList, s...)
+		if len(serverList) > 1 {
+			return false, ErrMultipleResults
+		}
+		return true, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if len(serverList) == 0 {
+		return nil, ErrNotFound
+	} else if len(serverList) > 1 {
+		return nil, ErrMultipleResults
+	}
+
+	return &serverList[0], nil
+}
+
+func firstAddr(netblob interface{}) string {
+	// Run-time types for the win :(
+	list, ok := netblob.([]interface{})
+	if !ok || len(list) < 1 {
+		return ""
+	}
+	props, ok := list[0].(map[string]interface{})
+	if !ok {
+		return ""
+	}
+	tmp, ok := props["addr"]
+	if !ok {
+		return ""
+	}
+	addr, ok := tmp.(string)
+	if !ok {
+		return ""
+	}
+	return addr
+}
+
+func getAddressByServer(srv *osservers.Server) (string, error) {
+	var s string
+	if s == "" {
+		s = firstAddr(srv.Addresses["private"])
+	}
+	if s == "" {
+		s = firstAddr(srv.Addresses["public"])
+	}
+	if s == "" {
+		s = srv.AccessIPv4
+	}
+	if s == "" {
+		s = srv.AccessIPv6
+	}
+	if s == "" {
+		return "", ErrNoAddressFound
+	}
+	return s, nil
+}
+
+func getAddressByName(api *gophercloud.ServiceClient, name string) (string, error) {
+	srv, err := getServerByName(api, name)
+	if err != nil {
+		return "", err
+	}
+
+	return getAddressByServer(srv)
+}
+
+func (i *Instances) NodeAddresses(nodeName types.NodeName) ([]v1.NodeAddress, error) {
+	glog.V(2).Infof("NodeAddresses(%v) called", nodeName)
+	serverName := mapNodeNameToServerName(nodeName)
+	ip, err := probeNodeAddress(i.compute, serverName)
+	if err != nil {
+		return nil, err
+	}
+
+	glog.V(2).Infof("NodeAddresses(%v) => %v", serverName, ip)
+
+	// net.ParseIP().String() is to maintain compatibility with the old code
+	parsedIP := net.ParseIP(ip).String()
+	return []v1.NodeAddress{
+		{Type: v1.NodeInternalIP, Address: parsedIP},
+		{Type: v1.NodeExternalIP, Address: parsedIP},
+	}, nil
+}
+
+// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID
+// This method will not be called from the node that is requesting this ID. i.e. metadata service
+// and other local methods cannot be used here
+func (i *Instances) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) {
+	instanceID, err := instanceIDFromProviderID(providerID)
+
+	if err != nil {
+		return []v1.NodeAddress{}, err
+	}
+
+	server, err := servers.Get(i.compute, instanceID).Extract()
+
+	if err != nil {
+		return []v1.NodeAddress{}, err
+	}
+
+	addresses, err := i.NodeAddresses(mapServerToNodeName(server))
+
+	if err != nil {
+		return []v1.NodeAddress{}, err
+	}
+
+	return addresses, nil
+}
+
+// mapNodeNameToServerName maps from a k8s NodeName to a rackspace Server Name
+// This is a simple string cast.
+func mapNodeNameToServerName(nodeName types.NodeName) string {
+	return string(nodeName)
+}
+
+// mapServerToNodeName maps a rackspace Server to an k8s NodeName
+func mapServerToNodeName(s *osservers.Server) types.NodeName {
+	return types.NodeName(s.Name)
+}
+
+// ExternalID returns the cloud provider ID of the node with the specified Name (deprecated).
+func (i *Instances) ExternalID(nodeName types.NodeName) (string, error) {
+	serverName := mapNodeNameToServerName(nodeName)
+	return probeInstanceID(i.compute, serverName)
+}
+
+// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running.
+// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager.
+func (i *Instances) InstanceExistsByProviderID(providerID string) (bool, error) {
+	return false, errors.New("unimplemented")
+}
+
+// InstanceID returns the cloud provider ID of the kubelet's instance.
+func (rs *Rackspace) InstanceID() (string, error) {
+	return readInstanceID()
+}
+
+// InstanceID returns the cloud provider ID of the node with the specified Name.
+func (i *Instances) InstanceID(nodeName types.NodeName) (string, error) {
+	serverName := mapNodeNameToServerName(nodeName)
+	return probeInstanceID(i.compute, serverName)
+}
+
+// InstanceType returns the type of the specified instance.
+func (i *Instances) InstanceType(name types.NodeName) (string, error) {
+	serverName := mapNodeNameToServerName(name)
+
+	srv, err := getServerByName(i.compute, serverName)
+	if err != nil {
+		return "", err
+	}
+
+	return srvInstanceType(srv)
+}
+
+func srvInstanceType(srv *osservers.Server) (string, error) {
+	val, ok := srv.Flavor["name"]
+
+	if !ok {
+		return "", fmt.Errorf("flavor name not present in server info")
+	}
+
+	flavor, ok := val.(string)
+
+	if !ok {
+		return "", fmt.Errorf("flavor name is not a string")
+	}
+
+	return flavor, nil
+}
+
+func instanceIDFromProviderID(providerID string) (instanceID string, err error) {
+	var providerIDRegexp = regexp.MustCompile(`^rackspace://([^/]+)$`)
+	matches := providerIDRegexp.FindStringSubmatch(providerID)
+	if len(matches) != 2 {
+		return "", fmt.Errorf("ProviderID \"%s\" didn't match expected format \"rackspace://InstanceID\"", providerID)
+	}
+
+	return matches[1], nil
+}
+
+// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID
+// This method will not be called from the node that is requesting this ID. i.e. metadata service
+// and other local methods cannot be used here
+func (i *Instances) InstanceTypeByProviderID(providerID string) (string, error) {
+	instanceID, err := instanceIDFromProviderID(providerID)
+
+	if err != nil {
+		return "", err
+	}
+
+	server, err := servers.Get(i.compute, instanceID).Extract()
+
+	if err != nil {
+		return "", err
+	}
+
+	return srvInstanceType(server)
+}
+
+func (i *Instances) AddSSHKeyToAllInstances(user string, keyData []byte) error {
+	return errors.New("unimplemented")
+}
+
+// Implementation of Instances.CurrentNodeName
+func (i *Instances) CurrentNodeName(hostname string) (types.NodeName, error) {
+	// Beware when changing this, nodename == hostname assumption is crucial to
+	// apiserver => kubelet communication.
+	return types.NodeName(hostname), nil
+}
+
+func (os *Rackspace) Clusters() (cloudprovider.Clusters, bool) {
+	return nil, false
+}
+
+// ProviderName returns the cloud provider ID.
+func (os *Rackspace) ProviderName() string {
+	return ProviderName
+}
+
+// ScrubDNS filters DNS settings for pods.
+func (os *Rackspace) ScrubDNS(nameservers, searches []string) (nsOut, srchOut []string) {
+	return nameservers, searches
+}
+
+// HasClusterID returns true if the cluster has a clusterID
+func (os *Rackspace) HasClusterID() bool {
+	return true
+}
+
+func (os *Rackspace) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
+	return nil, false
+}
+
+func (os *Rackspace) Zones() (cloudprovider.Zones, bool) {
+	glog.V(1).Info("Claiming to support Zones")
+
+	return os, true
+}
+
+func (os *Rackspace) Routes() (cloudprovider.Routes, bool) {
+	return nil, false
+}
+
+func (os *Rackspace) GetZone() (cloudprovider.Zone, error) {
+	glog.V(1).Infof("Current zone is %v", os.region)
+
+	return cloudprovider.Zone{Region: os.region}, nil
+}
+
+// GetZoneByProviderID implements Zones.GetZoneByProviderID
+// This is particularly useful in external cloud providers where the kubelet
+// does not initialize node data.
+func (os *Rackspace) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) {
+	return cloudprovider.Zone{}, errors.New("GetZoneByProviderID not implemented")
+}
+
+// GetZoneByNodeName implements Zones.GetZoneByNodeName
+// This is particularly useful in external cloud providers where the kubelet
+// does not initialize node data.
+func (os *Rackspace) GetZoneByNodeName(nodeName types.NodeName) (cloudprovider.Zone, error) {
+	return cloudprovider.Zone{}, errors.New("GetZoneByNodeName not imeplemented")
+}
+
+// Create a volume of given size (in GiB)
+func (rs *Rackspace) CreateVolume(name string, size int, vtype, availability string, tags *map[string]string) (string, string, error) {
+	return "", "", errors.New("unimplemented")
+}
+
+func (rs *Rackspace) DeleteVolume(volumeID string) error {
+	return errors.New("unimplemented")
+}
+
+func (rs *Rackspace) OperationPending(diskName string) (bool, string, error) {
+	disk, err := rs.getVolume(diskName)
+	if err != nil {
+		return false, "", err
+	}
+	volumeStatus := disk.Status
+	if volumeStatus == VolumeErrorStatus {
+		glog.Errorf("status of volume %s is %s", diskName, volumeStatus)
+		return false, volumeStatus, nil
+	}
+	if volumeStatus == VolumeAvailableStatus || volumeStatus == VolumeInUseStatus {
+		return false, disk.Status, nil
+	}
+	return true, volumeStatus, nil
+}
+
+// Attaches given cinder volume to the compute running kubelet
+func (rs *Rackspace) AttachDisk(instanceID, volumeID string) (string, error) {
+	volume, err := rs.getVolume(volumeID)
+	if err != nil {
+		return "", err
+	}
+
+	if volume.Status != VolumeAvailableStatus {
+		errmsg := fmt.Sprintf("volume %s status is %s, not %s, can not be attached to instance %s.", volume.Name, volume.Status, VolumeAvailableStatus, instanceID)
+		glog.Errorf(errmsg)
+		return "", errors.New(errmsg)
+	}
+
+	compute, err := rs.getComputeClient()
+	if err != nil {
+		return "", err
+	}
+
+	if len(volume.Attachments) > 0 {
+		if instanceID == volume.Attachments[0]["server_id"] {
+			glog.V(4).Infof("Volume: %q is already attached to compute: %q", volumeID, instanceID)
+			return volume.ID, nil
+		}
+
+		errMsg := fmt.Sprintf("Volume %q is attached to a different compute: %q, should be detached before proceeding", volumeID, volume.Attachments[0]["server_id"])
+		glog.Errorf(errMsg)
+		return "", errors.New(errMsg)
+	}
+
+	_, err = volumeattach.Create(compute, instanceID, &osvolumeattach.CreateOpts{
+		VolumeID: volume.ID,
+	}).Extract()
+	if err != nil {
+		glog.Errorf("Failed to attach %s volume to %s compute", volumeID, instanceID)
+		return "", err
+	}
+	glog.V(2).Infof("Successfully attached %s volume to %s compute", volumeID, instanceID)
+	return volume.ID, nil
+}
+
+// GetDevicePath returns the path of an attached block storage volume, specified by its id.
+func (rs *Rackspace) GetDevicePath(volumeID string) string {
+	volume, err := rs.getVolume(volumeID)
+	if err != nil {
+		return ""
+	}
+	attachments := volume.Attachments
+	if len(attachments) != 1 {
+		glog.Warningf("Unexpected number of volume attachments on %s: %d", volumeID, len(attachments))
+		return ""
+	}
+	return attachments[0]["device"].(string)
+}
+
+// Takes a partial/full disk id or volumeName
+func (rs *Rackspace) getVolume(volumeID string) (*volumes.Volume, error) {
+	client, err := rackspace.NewBlockStorageV1(rs.provider, gophercloud.EndpointOpts{
+		Region: rs.region,
+	})
+
+	volume, err := volumes.Get(client, volumeID).Extract()
+	if err != nil {
+		glog.Errorf("Error occurred getting volume by ID: %s", volumeID)
+		return &volumes.Volume{}, err
+	}
+	return volume, nil
+}
+
+func (rs *Rackspace) getComputeClient() (*gophercloud.ServiceClient, error) {
+	client, err := rackspace.NewComputeV2(rs.provider, gophercloud.EndpointOpts{
+		Region: rs.region,
+	})
+	if err != nil || client == nil {
+		glog.Errorf("Unable to initialize nova client for region: %s", rs.region)
+	}
+	return client, nil
+}
+
+// Detaches given cinder volume from the compute running kubelet
+func (rs *Rackspace) DetachDisk(instanceID, volumeID string) error {
+	volume, err := rs.getVolume(volumeID)
+	if err != nil {
+		return err
+	}
+
+	if volume.Status != VolumeInUseStatus {
+		errmsg := fmt.Sprintf("can not detach volume %s, its status is %s.", volume.Name, volume.Status)
+		glog.Errorf(errmsg)
+		return errors.New(errmsg)
+	}
+
+	compute, err := rs.getComputeClient()
+	if err != nil {
+		return err
+	}
+
+	if len(volume.Attachments) > 1 {
+		// Rackspace does not support "multiattach", this is a sanity check.
+		errmsg := fmt.Sprintf("Volume %s is attached to multiple instances, which is not supported by this provider.", volume.ID)
+		return errors.New(errmsg)
+	}
+
+	if len(volume.Attachments) > 0 && instanceID == volume.Attachments[0]["server_id"] {
+		// This is a blocking call and effects kubelet's performance directly.
+		// We should consider kicking it out into a separate routine, if it is bad.
+		err = volumeattach.Delete(compute, instanceID, volume.ID).ExtractErr()
+		if err != nil {
+			glog.Errorf("Failed to delete volume %s from compute %s attached %v", volume.ID, instanceID, err)
+			return err
+		}
+		glog.V(2).Infof("Successfully detached volume: %s from compute: %s", volume.ID, instanceID)
+	} else {
+		errMsg := fmt.Sprintf("Disk: %s has no attachments or is not attached to compute: %s", volume.Name, instanceID)
+		glog.Errorf(errMsg)
+		return errors.New(errMsg)
+	}
+
+	return nil
+}
+
+// Get device path of attached volume to the compute running kubelet, as known by cinder
+func (rs *Rackspace) GetAttachmentDiskPath(instanceID, volumeID string) (string, error) {
+	// See issue #33128 - Cinder does not always tell you the right device path, as such
+	// we must only use this value as a last resort.
+	volume, err := rs.getVolume(volumeID)
+	if err != nil {
+		return "", err
+	}
+
+	if volume.Status != VolumeInUseStatus {
+		errmsg := fmt.Sprintf("can not get device path of volume %s, its status is %s.", volume.Name, volume.Status)
+		glog.Errorf(errmsg)
+		return "", errors.New(errmsg)
+	}
+	if len(volume.Attachments) > 0 && volume.Attachments[0]["server_id"] != nil {
+		if instanceID == volume.Attachments[0]["server_id"] {
+			// Attachment[0]["device"] points to the device path
+			// see http://developer.openstack.org/api-ref-blockstorage-v1.html
+			return volume.Attachments[0]["device"].(string), nil
+		} else {
+			errMsg := fmt.Sprintf("Disk %q is attached to a different compute: %q, should be detached before proceeding", volumeID, volume.Attachments[0]["server_id"])
+			glog.Errorf(errMsg)
+			return "", errors.New(errMsg)
+		}
+	}
+	return "", fmt.Errorf("volume %s is not attached to %s", volumeID, instanceID)
+}
+
+// query if a volume is attached to a compute instance
+func (rs *Rackspace) DiskIsAttached(instanceID, volumeID string) (bool, error) {
+	volume, err := rs.getVolume(volumeID)
+	if err != nil {
+		return false, err
+	}
+	if len(volume.Attachments) > 0 && volume.Attachments[0]["server_id"] != nil && instanceID == volume.Attachments[0]["server_id"] {
+		return true, nil
+	}
+	return false, nil
+}
+
+// query if a list volumes are attached to a compute instance
+func (rs *Rackspace) DisksAreAttached(instanceID string, volumeIDs []string) (map[string]bool, error) {
+	attached := make(map[string]bool)
+	for _, volumeID := range volumeIDs {
+		attached[volumeID] = false
+	}
+	var returnedErr error
+	for _, volumeID := range volumeIDs {
+		result, err := rs.DiskIsAttached(instanceID, volumeID)
+		if err != nil {
+			returnedErr = fmt.Errorf("Error in checking disk %q attached: %v \n %v", volumeID, err, returnedErr)
+			continue
+		}
+		if result {
+			attached[volumeID] = true
+		}
+
+	}
+	return attached, returnedErr
+}
+
+// query if we should trust the cinder provide deviceName, See issue #33128
+func (rs *Rackspace) ShouldTrustDevicePath() bool {
+	return true
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/rackspace/rackspace_test.go

@@ -0,0 +1,207 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rackspace
+
+import (
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/rackspace/gophercloud"
+)
+
+func TestReadConfig(t *testing.T) {
+	_, err := readConfig(nil)
+	if err == nil {
+		t.Errorf("Should fail when no config is provided: %s", err)
+	}
+
+	cfg, err := readConfig(strings.NewReader(`
+[Global]
+auth-url = http://auth.url
+username = user
+[LoadBalancer]
+create-monitor = yes
+monitor-delay = 1m
+monitor-timeout = 30s
+monitor-max-retries = 3
+`))
+	if err != nil {
+		t.Fatalf("Should succeed when a valid config is provided: %s", err)
+	}
+	if cfg.Global.AuthUrl != "http://auth.url" {
+		t.Errorf("incorrect authurl: %s", cfg.Global.AuthUrl)
+	}
+
+	if !cfg.LoadBalancer.CreateMonitor {
+		t.Errorf("incorrect lb.createmonitor: %t", cfg.LoadBalancer.CreateMonitor)
+	}
+	if cfg.LoadBalancer.MonitorDelay.Duration != 1*time.Minute {
+		t.Errorf("incorrect lb.monitordelay: %s", cfg.LoadBalancer.MonitorDelay)
+	}
+	if cfg.LoadBalancer.MonitorTimeout.Duration != 30*time.Second {
+		t.Errorf("incorrect lb.monitortimeout: %s", cfg.LoadBalancer.MonitorTimeout)
+	}
+	if cfg.LoadBalancer.MonitorMaxRetries != 3 {
+		t.Errorf("incorrect lb.monitormaxretries: %d", cfg.LoadBalancer.MonitorMaxRetries)
+	}
+}
+
+func TestToAuthOptions(t *testing.T) {
+	cfg := Config{}
+	cfg.Global.Username = "user"
+	// etc.
+
+	ao := cfg.toAuthOptions()
+
+	if !ao.AllowReauth {
+		t.Errorf("Will need to be able to reauthenticate")
+	}
+	if ao.Username != cfg.Global.Username {
+		t.Errorf("Username %s != %s", ao.Username, cfg.Global.Username)
+	}
+}
+
+// This allows acceptance testing against an existing Rackspace
+// install, using the standard OS_* Rackspace client environment
+// variables.
+// FIXME: it would be better to hermetically test against canned JSON
+// requests/responses.
+func configFromEnv() (cfg Config, ok bool) {
+	cfg.Global.AuthUrl = os.Getenv("OS_AUTH_URL")
+
+	cfg.Global.TenantId = os.Getenv("OS_TENANT_ID")
+	// Rax/nova _insists_ that we don't specify both tenant ID and name
+	if cfg.Global.TenantId == "" {
+		cfg.Global.TenantName = os.Getenv("OS_TENANT_NAME")
+	}
+
+	cfg.Global.Username = os.Getenv("OS_USERNAME")
+	cfg.Global.Password = os.Getenv("OS_PASSWORD")
+	cfg.Global.ApiKey = os.Getenv("OS_API_KEY")
+	cfg.Global.Region = os.Getenv("OS_REGION_NAME")
+	cfg.Global.DomainId = os.Getenv("OS_DOMAIN_ID")
+	cfg.Global.DomainName = os.Getenv("OS_DOMAIN_NAME")
+
+	ok = (cfg.Global.AuthUrl != "" &&
+		cfg.Global.Username != "" &&
+		(cfg.Global.Password != "" || cfg.Global.ApiKey != "") &&
+		(cfg.Global.TenantId != "" || cfg.Global.TenantName != "" ||
+			cfg.Global.DomainId != "" || cfg.Global.DomainName != ""))
+
+	return
+}
+
+func TestParseMetaData(t *testing.T) {
+	_, err := parseMetaData(strings.NewReader(""))
+	if err == nil {
+		t.Errorf("Should fail when invalid meta data is provided: %s", err)
+	}
+
+	id, err := parseMetaData(strings.NewReader(`
+	{
+		"UUID":"someuuid",
+		"name":"somename",
+		"project_id":"someprojectid"
+	}
+	`))
+	if err != nil {
+		t.Fatalf("Should succeed when valid meta data is provided: %s", err)
+	}
+	if id != "someuuid" {
+		t.Errorf("incorrect uuid: %s", id)
+	}
+}
+
+func TestNewRackspace(t *testing.T) {
+	cfg, ok := configFromEnv()
+	if !ok {
+		t.Skipf("No config found in environment")
+	}
+
+	_, err := newRackspace(cfg)
+	if err != nil {
+		t.Fatalf("Failed to construct/authenticate Rackspace: %s", err)
+	}
+}
+
+func TestZones(t *testing.T) {
+	os := Rackspace{
+		provider: &gophercloud.ProviderClient{
+			IdentityBase: "http://auth.url/",
+		},
+		region: "myRegion",
+	}
+
+	z, ok := os.Zones()
+	if !ok {
+		t.Fatalf("Zones() returned false")
+	}
+
+	zone, err := z.GetZone()
+	if err != nil {
+		t.Fatalf("GetZone() returned error: %s", err)
+	}
+
+	if zone.Region != "myRegion" {
+		t.Fatalf("GetZone() returned wrong region (%s)", zone.Region)
+	}
+}
+
+func TestInstanceIDFromProviderID(t *testing.T) {
+	testCases := []struct {
+		providerID string
+		instanceID string
+		fail       bool
+	}{
+		{
+			providerID: ProviderName + "://7b9cf879-7146-417c-abfd-cb4272f0c935",
+			instanceID: "7b9cf879-7146-417c-abfd-cb4272f0c935",
+			fail:       false,
+		},
+		{
+			providerID: "7b9cf879-7146-417c-abfd-cb4272f0c935",
+			instanceID: "",
+			fail:       true,
+		},
+		{
+			providerID: "other-provider://7b9cf879-7146-417c-abfd-cb4272f0c935",
+			instanceID: "",
+			fail:       true,
+		},
+		{
+			providerID: "",
+			instanceID: "",
+			fail:       true,
+		},
+	}
+
+	for _, test := range testCases {
+		instanceID, err := instanceIDFromProviderID(test.providerID)
+		if (err != nil) != test.fail {
+			t.Errorf("%s yielded `err != nil` as %t. expected %t", test.providerID, (err != nil), test.fail)
+		}
+
+		if test.fail {
+			continue
+		}
+		if instanceID != test.instanceID {
+			t.Errorf("%s yielded %s. expected %s", test.providerID, instanceID, test.instanceID)
+		}
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/BUILD

@@ -0,0 +1,60 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "vsphere.go",
+        "vsphere_util.go",
+    ],
+    deps = [
+        "//pkg/api/v1/helper:go_default_library",
+        "//pkg/cloudprovider:go_default_library",
+        "//pkg/cloudprovider/providers/vsphere/vclib:go_default_library",
+        "//pkg/cloudprovider/providers/vsphere/vclib/diskmanagers:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/vmware/govmomi:go_default_library",
+        "//vendor/github.com/vmware/govmomi/object:go_default_library",
+        "//vendor/github.com/vmware/govmomi/vim25:go_default_library",
+        "//vendor/github.com/vmware/govmomi/vim25/mo:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/gopkg.in/gcfg.v1:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["vsphere_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//pkg/cloudprovider:go_default_library",
+        "//pkg/cloudprovider/providers/vsphere/vclib:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/rand:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/cloudprovider/providers/vsphere/vclib:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/OWNERS

@@ -0,0 +1,7 @@
+approvers:
+- abrarshivani
+- baludontu
+- divyenpatel
+- imkin
+- kerneltime
+- luomiao

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib/BUILD

@@ -0,0 +1,56 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "connection.go",
+        "constants.go",
+        "custom_errors.go",
+        "datacenter.go",
+        "datastore.go",
+        "folder.go",
+        "pbm.go",
+        "utils.go",
+        "virtualmachine.go",
+        "vmoptions.go",
+        "volumeoptions.go",
+        "vsphere_metrics.go",
+    ],
+    deps = [
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//vendor/github.com/vmware/govmomi:go_default_library",
+        "//vendor/github.com/vmware/govmomi/find:go_default_library",
+        "//vendor/github.com/vmware/govmomi/object:go_default_library",
+        "//vendor/github.com/vmware/govmomi/pbm:go_default_library",
+        "//vendor/github.com/vmware/govmomi/pbm/types:go_default_library",
+        "//vendor/github.com/vmware/govmomi/property:go_default_library",
+        "//vendor/github.com/vmware/govmomi/session:go_default_library",
+        "//vendor/github.com/vmware/govmomi/vim25:go_default_library",
+        "//vendor/github.com/vmware/govmomi/vim25/mo:go_default_library",
+        "//vendor/github.com/vmware/govmomi/vim25/soap:go_default_library",
+        "//vendor/github.com/vmware/govmomi/vim25/types:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/cloudprovider/providers/vsphere/vclib/diskmanagers:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib/connection.go

@@ -0,0 +1,99 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vclib
+
+import (
+	"fmt"
+	neturl "net/url"
+	"sync"
+
+	"github.com/golang/glog"
+	"github.com/vmware/govmomi"
+	"github.com/vmware/govmomi/session"
+	"github.com/vmware/govmomi/vim25"
+	"golang.org/x/net/context"
+)
+
+// VSphereConnection contains information for connecting to vCenter
+type VSphereConnection struct {
+	GoVmomiClient     *govmomi.Client
+	Username          string
+	Password          string
+	Hostname          string
+	Port              string
+	Insecure          bool
+	RoundTripperCount uint
+}
+
+var (
+	clientLock sync.Mutex
+)
+
+// Connect makes connection to vCenter and sets VSphereConnection.GoVmomiClient.
+// If connection.GoVmomiClient is already set, it obtains the existing user session.
+// if user session is not valid, connection.GoVmomiClient will be set to the new client.
+func (connection *VSphereConnection) Connect(ctx context.Context) error {
+	var err error
+	clientLock.Lock()
+	defer clientLock.Unlock()
+
+	if connection.GoVmomiClient == nil {
+		connection.GoVmomiClient, err = connection.NewClient(ctx)
+		if err != nil {
+			glog.Errorf("Failed to create govmomi client. err: %+v", err)
+			return err
+		}
+		return nil
+	}
+	m := session.NewManager(connection.GoVmomiClient.Client)
+	userSession, err := m.UserSession(ctx)
+	if err != nil {
+		glog.Errorf("Error while obtaining user session. err: %+v", err)
+		return err
+	}
+	if userSession != nil {
+		return nil
+	}
+	glog.Warningf("Creating new client session since the existing session is not valid or not authenticated")
+	connection.GoVmomiClient.Logout(ctx)
+	connection.GoVmomiClient, err = connection.NewClient(ctx)
+	if err != nil {
+		glog.Errorf("Failed to create govmomi client. err: %+v", err)
+		return err
+	}
+	return nil
+}
+
+// NewClient creates a new govmomi client for the VSphereConnection obj
+func (connection *VSphereConnection) NewClient(ctx context.Context) (*govmomi.Client, error) {
+	url, err := neturl.Parse(fmt.Sprintf("https://%s:%s/sdk", connection.Hostname, connection.Port))
+	if err != nil {
+		glog.Errorf("Failed to parse URL: %s. err: %+v", url, err)
+		return nil, err
+	}
+	url.User = neturl.UserPassword(connection.Username, connection.Password)
+	client, err := govmomi.NewClient(ctx, url, connection.Insecure)
+	if err != nil {
+		glog.Errorf("Failed to create new client. err: %+v", err)
+		return nil, err
+	}
+	if connection.RoundTripperCount == 0 {
+		connection.RoundTripperCount = RoundTripperDefaultCount
+	}
+	client.RoundTripper = vim25.Retry(client.RoundTripper, vim25.TemporaryNetworkError(int(connection.RoundTripperCount)))
+	return client, nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib/constants.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vclib
+
+// Volume Constnts
+const (
+	ThinDiskType             = "thin"
+	PreallocatedDiskType     = "preallocated"
+	EagerZeroedThickDiskType = "eagerZeroedThick"
+	ZeroedThickDiskType      = "zeroedThick"
+)
+
+// Controller Constants
+const (
+	SCSIControllerLimit       = 4
+	SCSIControllerDeviceLimit = 15
+	SCSIDeviceSlots           = 16
+	SCSIReservedSlot          = 7
+
+	SCSIControllerType        = "scsi"
+	LSILogicControllerType    = "lsiLogic"
+	BusLogicControllerType    = "busLogic"
+	LSILogicSASControllerType = "lsiLogic-sas"
+	PVSCSIControllerType      = "pvscsi"
+)
+
+// Other Constants
+const (
+	LogLevel                 = 4
+	DatastoreProperty        = "datastore"
+	ResourcePoolProperty     = "resourcePool"
+	DatastoreInfoProperty    = "info"
+	VirtualMachineType       = "VirtualMachine"
+	RoundTripperDefaultCount = 3
+	VSANDatastoreType        = "vsan"
+	DummyVMPrefixName        = "vsphere-k8s"
+	ActivePowerState         = "poweredOn"
+)

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib/custom_errors.go

@@ -0,0 +1,37 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vclib
+
+import "errors"
+
+// Error Messages
+const (
+	FileAlreadyExistErrMsg     = "File requested already exist"
+	NoDiskUUIDFoundErrMsg      = "No disk UUID found"
+	NoDevicesFoundErrMsg       = "No devices found"
+	DiskNotFoundErrMsg         = "No vSphere disk ID found"
+	InvalidVolumeOptionsErrMsg = "VolumeOptions verification failed"
+)
+
+// Error constants
+var (
+	ErrFileAlreadyExist     = errors.New(FileAlreadyExistErrMsg)
+	ErrNoDiskUUIDFound      = errors.New(NoDiskUUIDFoundErrMsg)
+	ErrNoDevicesFound       = errors.New(NoDevicesFoundErrMsg)
+	ErrNoDiskIDFound        = errors.New(DiskNotFoundErrMsg)
+	ErrInvalidVolumeOptions = errors.New(InvalidVolumeOptionsErrMsg)
+)

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib/datacenter.go

@@ -0,0 +1,164 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vclib
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/golang/glog"
+	"github.com/vmware/govmomi/find"
+	"github.com/vmware/govmomi/object"
+	"github.com/vmware/govmomi/property"
+	"github.com/vmware/govmomi/vim25/mo"
+	"github.com/vmware/govmomi/vim25/types"
+	"golang.org/x/net/context"
+)
+
+// Datacenter extends the govmomi Datacenter object
+type Datacenter struct {
+	*object.Datacenter
+}
+
+// GetDatacenter returns the DataCenter Object for the given datacenterPath
+// If datacenter is located in a folder, include full path to datacenter else just provide the datacenter name
+func GetDatacenter(ctx context.Context, connection *VSphereConnection, datacenterPath string) (*Datacenter, error) {
+	finder := find.NewFinder(connection.GoVmomiClient.Client, true)
+	datacenter, err := finder.Datacenter(ctx, datacenterPath)
+	if err != nil {
+		glog.Errorf("Failed to find the datacenter: %s. err: %+v", datacenterPath, err)
+		return nil, err
+	}
+	dc := Datacenter{datacenter}
+	return &dc, nil
+}
+
+// GetVMByUUID gets the VM object from the given vmUUID
+func (dc *Datacenter) GetVMByUUID(ctx context.Context, vmUUID string) (*VirtualMachine, error) {
+	s := object.NewSearchIndex(dc.Client())
+	vmUUID = strings.ToLower(strings.TrimSpace(vmUUID))
+	svm, err := s.FindByUuid(ctx, dc.Datacenter, vmUUID, true, nil)
+	if err != nil {
+		glog.Errorf("Failed to find VM by UUID. VM UUID: %s, err: %+v", vmUUID, err)
+		return nil, err
+	}
+	if svm == nil {
+		glog.Errorf("Unable to find VM by UUID. VM UUID: %s", vmUUID)
+		return nil, fmt.Errorf("Failed to find VM by UUID: %s", vmUUID)
+	}
+	virtualMachine := VirtualMachine{object.NewVirtualMachine(dc.Client(), svm.Reference()), dc}
+	return &virtualMachine, nil
+}
+
+// GetVMByPath gets the VM object from the given vmPath
+// vmPath should be the full path to VM and not just the name
+func (dc *Datacenter) GetVMByPath(ctx context.Context, vmPath string) (*VirtualMachine, error) {
+	finder := getFinder(dc)
+	vm, err := finder.VirtualMachine(ctx, vmPath)
+	if err != nil {
+		glog.Errorf("Failed to find VM by Path. VM Path: %s, err: %+v", vmPath, err)
+		return nil, err
+	}
+	virtualMachine := VirtualMachine{vm, dc}
+	return &virtualMachine, nil
+}
+
+// GetDatastoreByPath gets the Datastore object from the given vmDiskPath
+func (dc *Datacenter) GetDatastoreByPath(ctx context.Context, vmDiskPath string) (*Datastore, error) {
+	datastorePathObj := new(object.DatastorePath)
+	isSuccess := datastorePathObj.FromString(vmDiskPath)
+	if !isSuccess {
+		glog.Errorf("Failed to parse vmDiskPath: %s", vmDiskPath)
+		return nil, errors.New("Failed to parse vmDiskPath")
+	}
+	finder := getFinder(dc)
+	ds, err := finder.Datastore(ctx, datastorePathObj.Datastore)
+	if err != nil {
+		glog.Errorf("Failed while searching for datastore: %s. err: %+v", datastorePathObj.Datastore, err)
+		return nil, err
+	}
+	datastore := Datastore{ds, dc}
+	return &datastore, nil
+}
+
+// GetDatastoreByName gets the Datastore object for the given datastore name
+func (dc *Datacenter) GetDatastoreByName(ctx context.Context, name string) (*Datastore, error) {
+	finder := getFinder(dc)
+	ds, err := finder.Datastore(ctx, name)
+	if err != nil {
+		glog.Errorf("Failed while searching for datastore: %s. err: %+v", name, err)
+		return nil, err
+	}
+	datastore := Datastore{ds, dc}
+	return &datastore, nil
+}
+
+// GetFolderByPath gets the Folder Object from the given folder path
+// folderPath should be the full path to folder
+func (dc *Datacenter) GetFolderByPath(ctx context.Context, folderPath string) (*Folder, error) {
+	finder := getFinder(dc)
+	vmFolder, err := finder.Folder(ctx, folderPath)
+	if err != nil {
+		glog.Errorf("Failed to get the folder reference for %s. err: %+v", folderPath, err)
+		return nil, err
+	}
+	folder := Folder{vmFolder, dc}
+	return &folder, nil
+}
+
+// GetVMMoList gets the VM Managed Objects with the given properties from the VM object
+func (dc *Datacenter) GetVMMoList(ctx context.Context, vmObjList []*VirtualMachine, properties []string) ([]mo.VirtualMachine, error) {
+	var vmMoList []mo.VirtualMachine
+	var vmRefs []types.ManagedObjectReference
+	if len(vmObjList) < 1 {
+		glog.Errorf("VirtualMachine Object list is empty")
+		return nil, fmt.Errorf("VirtualMachine Object list is empty")
+	}
+
+	for _, vmObj := range vmObjList {
+		vmRefs = append(vmRefs, vmObj.Reference())
+	}
+	pc := property.DefaultCollector(dc.Client())
+	err := pc.Retrieve(ctx, vmRefs, properties, &vmMoList)
+	if err != nil {
+		glog.Errorf("Failed to get VM managed objects from VM objects. vmObjList: %+v, properties: %+v, err: %v", vmObjList, properties, err)
+		return nil, err
+	}
+	return vmMoList, nil
+}
+
+// GetDatastoreMoList gets the Datastore Managed Objects with the given properties from the datastore objects
+func (dc *Datacenter) GetDatastoreMoList(ctx context.Context, dsObjList []*Datastore, properties []string) ([]mo.Datastore, error) {
+	var dsMoList []mo.Datastore
+	var dsRefs []types.ManagedObjectReference
+	if len(dsObjList) < 1 {
+		glog.Errorf("Datastore Object list is empty")
+		return nil, fmt.Errorf("Datastore Object list is empty")
+	}
+
+	for _, dsObj := range dsObjList {
+		dsRefs = append(dsRefs, dsObj.Reference())
+	}
+	pc := property.DefaultCollector(dc.Client())
+	err := pc.Retrieve(ctx, dsRefs, properties, &dsMoList)
+	if err != nil {
+		glog.Errorf("Failed to get Datastore managed objects from datastore objects. dsObjList: %+v, properties: %+v, err: %v", dsObjList, properties, err)
+		return nil, err
+	}
+	return dsMoList, nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib/datastore.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vclib
+
+import (
+	"github.com/golang/glog"
+	"github.com/vmware/govmomi/object"
+	"github.com/vmware/govmomi/property"
+	"github.com/vmware/govmomi/vim25/mo"
+	"github.com/vmware/govmomi/vim25/soap"
+	"github.com/vmware/govmomi/vim25/types"
+	"golang.org/x/net/context"
+)
+
+// Datastore extends the govmomi Datastore object
+type Datastore struct {
+	*object.Datastore
+	Datacenter *Datacenter
+}
+
+// CreateDirectory creates the directory at location specified by directoryPath.
+// If the intermediate level folders do not exist, and the parameter createParents is true, all the non-existent folders are created.
+// directoryPath must be in the format "[vsanDatastore] kubevols"
+func (ds *Datastore) CreateDirectory(ctx context.Context, directoryPath string, createParents bool) error {
+	fileManager := object.NewFileManager(ds.Client())
+	err := fileManager.MakeDirectory(ctx, directoryPath, ds.Datacenter.Datacenter, createParents)
+	if err != nil {
+		if soap.IsSoapFault(err) {
+			soapFault := soap.ToSoapFault(err)
+			if _, ok := soapFault.VimFault().(types.FileAlreadyExists); ok {
+				return ErrFileAlreadyExist
+			}
+		}
+		return err
+	}
+	glog.V(LogLevel).Infof("Created dir with path as %+q", directoryPath)
+	return nil
+}
+
+// GetType returns the type of datastore
+func (ds *Datastore) GetType(ctx context.Context) (string, error) {
+	var dsMo mo.Datastore
+	pc := property.DefaultCollector(ds.Client())
+	err := pc.RetrieveOne(ctx, ds.Datastore.Reference(), []string{"summary"}, &dsMo)
+	if err != nil {
+		glog.Errorf("Failed to retrieve datastore summary property. err: %v", err)
+		return "", err
+	}
+	return dsMo.Summary.Type, nil
+}
+
+// IsCompatibleWithStoragePolicy returns true if datastore is compatible with given storage policy else return false
+// for not compatible datastore, fault message is also returned
+func (ds *Datastore) IsCompatibleWithStoragePolicy(ctx context.Context, storagePolicyID string) (bool, string, error) {
+	pbmClient, err := NewPbmClient(ctx, ds.Client())
+	if err != nil {
+		glog.Errorf("Failed to get new PbmClient Object. err: %v", err)
+		return false, "", err
+	}
+	return pbmClient.IsDatastoreCompatible(ctx, storagePolicyID, ds)
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/BUILD

@@ -0,0 +1,35 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "vdm.go",
+        "virtualdisk.go",
+        "vmdm.go",
+    ],
+    deps = [
+        "//pkg/cloudprovider/providers/vsphere/vclib:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/vmware/govmomi/object:go_default_library",
+        "//vendor/github.com/vmware/govmomi/vim25/types:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vdm.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package diskmanagers
+
+import (
+	"time"
+
+	"golang.org/x/net/context"
+
+	"github.com/golang/glog"
+	"github.com/vmware/govmomi/object"
+	"github.com/vmware/govmomi/vim25/types"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib"
+)
+
+// virtualDiskManager implements VirtualDiskProvider Interface for creating and deleting volume using VirtualDiskManager
+type virtualDiskManager struct {
+	diskPath      string
+	volumeOptions *vclib.VolumeOptions
+}
+
+// Create implements Disk's Create interface
+// Contains implementation of virtualDiskManager based Provisioning
+func (diskManager virtualDiskManager) Create(ctx context.Context, datastore *vclib.Datastore) (err error) {
+	if diskManager.volumeOptions.SCSIControllerType == "" {
+		diskManager.volumeOptions.SCSIControllerType = vclib.LSILogicControllerType
+	}
+	// Create virtual disk
+	diskFormat := vclib.DiskFormatValidType[diskManager.volumeOptions.DiskFormat]
+	// Create a virtual disk manager
+	vdm := object.NewVirtualDiskManager(datastore.Client())
+	// Create specification for new virtual disk
+	vmDiskSpec := &types.FileBackedVirtualDiskSpec{
+		VirtualDiskSpec: types.VirtualDiskSpec{
+			AdapterType: diskManager.volumeOptions.SCSIControllerType,
+			DiskType:    diskFormat,
+		},
+		CapacityKb: int64(diskManager.volumeOptions.CapacityKB),
+	}
+	requestTime := time.Now()
+	// Create virtual disk
+	task, err := vdm.CreateVirtualDisk(ctx, diskManager.diskPath, datastore.Datacenter.Datacenter, vmDiskSpec)
+	if err != nil {
+		vclib.RecordvSphereMetric(vclib.APICreateVolume, requestTime, err)
+		glog.Errorf("Failed to create virtual disk: %s. err: %+v", diskManager.diskPath, err)
+		return err
+	}
+	err = task.Wait(ctx)
+	vclib.RecordvSphereMetric(vclib.APICreateVolume, requestTime, err)
+	if err != nil {
+		glog.Errorf("Failed to create virtual disk: %s. err: %+v", diskManager.diskPath, err)
+		return err
+	}
+	return nil
+}
+
+// Delete implements Disk's Delete interface
+func (diskManager virtualDiskManager) Delete(ctx context.Context, datastore *vclib.Datastore) error {
+	// Create a virtual disk manager
+	virtualDiskManager := object.NewVirtualDiskManager(datastore.Client())
+	diskPath := vclib.RemoveClusterFromVDiskPath(diskManager.diskPath)
+	requestTime := time.Now()
+	// Delete virtual disk
+	task, err := virtualDiskManager.DeleteVirtualDisk(ctx, diskPath, datastore.Datacenter.Datacenter)
+	if err != nil {
+		glog.Errorf("Failed to delete virtual disk. err: %v", err)
+		vclib.RecordvSphereMetric(vclib.APIDeleteVolume, requestTime, err)
+		return err
+	}
+	err = task.Wait(ctx)
+	vclib.RecordvSphereMetric(vclib.APIDeleteVolume, requestTime, err)
+	if err != nil {
+		glog.Errorf("Failed to delete virtual disk. err: %v", err)
+		return err
+	}
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/virtualdisk.go

@@ -0,0 +1,80 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package diskmanagers
+
+import (
+	"fmt"
+
+	"github.com/golang/glog"
+	"golang.org/x/net/context"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib"
+)
+
+// VirtualDisk is for the Disk Management
+type VirtualDisk struct {
+	DiskPath      string
+	VolumeOptions *vclib.VolumeOptions
+	VMOptions     *vclib.VMOptions
+}
+
+// VirtualDisk Operations Const
+const (
+	VirtualDiskCreateOperation = "Create"
+	VirtualDiskDeleteOperation = "Delete"
+)
+
+// VirtualDiskProvider defines interfaces for creating disk
+type VirtualDiskProvider interface {
+	Create(ctx context.Context, datastore *vclib.Datastore) error
+	Delete(ctx context.Context, datastore *vclib.Datastore) error
+}
+
+// getDiskManager returns vmDiskManager or vdmDiskManager based on given volumeoptions
+func getDiskManager(disk *VirtualDisk, diskOperation string) VirtualDiskProvider {
+	var diskProvider VirtualDiskProvider
+	switch diskOperation {
+	case VirtualDiskDeleteOperation:
+		diskProvider = virtualDiskManager{disk.DiskPath, disk.VolumeOptions}
+	case VirtualDiskCreateOperation:
+		if disk.VolumeOptions.StoragePolicyName != "" || disk.VolumeOptions.VSANStorageProfileData != "" || disk.VolumeOptions.StoragePolicyID != "" {
+			diskProvider = vmDiskManager{disk.DiskPath, disk.VolumeOptions, disk.VMOptions}
+		} else {
+			diskProvider = virtualDiskManager{disk.DiskPath, disk.VolumeOptions}
+		}
+	}
+	return diskProvider
+}
+
+// Create gets appropriate disk manager and calls respective create method
+func (virtualDisk *VirtualDisk) Create(ctx context.Context, datastore *vclib.Datastore) error {
+	if virtualDisk.VolumeOptions.DiskFormat == "" {
+		virtualDisk.VolumeOptions.DiskFormat = vclib.ThinDiskType
+	}
+	if !virtualDisk.VolumeOptions.VerifyVolumeOptions() {
+		glog.Error("VolumeOptions verification failed. volumeOptions: ", virtualDisk.VolumeOptions)
+		return vclib.ErrInvalidVolumeOptions
+	}
+	if virtualDisk.VolumeOptions.StoragePolicyID != "" && virtualDisk.VolumeOptions.StoragePolicyName != "" {
+		return fmt.Errorf("Storage Policy ID and Storage Policy Name both set, Please set only one parameter")
+	}
+	return getDiskManager(virtualDisk, VirtualDiskCreateOperation).Create(ctx, datastore)
+}
+
+// Delete gets appropriate disk manager and calls respective delete method
+func (virtualDisk *VirtualDisk) Delete(ctx context.Context, datastore *vclib.Datastore) error {
+	return getDiskManager(virtualDisk, VirtualDiskDeleteOperation).Delete(ctx, datastore)
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vmdm.go

@@ -0,0 +1,249 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package diskmanagers
+
+import (
+	"fmt"
+	"hash/fnv"
+	"strings"
+
+	"github.com/golang/glog"
+	"github.com/vmware/govmomi/object"
+	"github.com/vmware/govmomi/vim25/types"
+	"golang.org/x/net/context"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib"
+)
+
+// vmDiskManager implements VirtualDiskProvider interface for creating volume using Virtual Machine Reconfigure approach
+type vmDiskManager struct {
+	diskPath      string
+	volumeOptions *vclib.VolumeOptions
+	vmOptions     *vclib.VMOptions
+}
+
+// Create implements Disk's Create interface
+// Contains implementation of VM based Provisioning to provision disk with SPBM Policy or VSANStorageProfileData
+func (vmdisk vmDiskManager) Create(ctx context.Context, datastore *vclib.Datastore) (err error) {
+	if vmdisk.volumeOptions.SCSIControllerType == "" {
+		vmdisk.volumeOptions.SCSIControllerType = vclib.PVSCSIControllerType
+	}
+	pbmClient, err := vclib.NewPbmClient(ctx, datastore.Client())
+	if err != nil {
+		glog.Errorf("Error occurred while creating new pbmClient, err: %+v", err)
+		return err
+	}
+
+	if vmdisk.volumeOptions.StoragePolicyID == "" && vmdisk.volumeOptions.StoragePolicyName != "" {
+		vmdisk.volumeOptions.StoragePolicyID, err = pbmClient.ProfileIDByName(ctx, vmdisk.volumeOptions.StoragePolicyName)
+		if err != nil {
+			glog.Errorf("Error occurred while getting Profile Id from Profile Name: %s, err: %+v", vmdisk.volumeOptions.StoragePolicyName, err)
+			return err
+		}
+	}
+	if vmdisk.volumeOptions.StoragePolicyID != "" {
+		compatible, faultMessage, err := datastore.IsCompatibleWithStoragePolicy(ctx, vmdisk.volumeOptions.StoragePolicyID)
+		if err != nil {
+			glog.Errorf("Error occurred while checking datastore compatibility with storage policy id: %s, err: %+v", vmdisk.volumeOptions.StoragePolicyID, err)
+			return err
+		}
+
+		if !compatible {
+			glog.Errorf("Datastore: %s is not compatible with Policy: %s", datastore.Name(), vmdisk.volumeOptions.StoragePolicyName)
+			return fmt.Errorf("User specified datastore is not compatible with the storagePolicy: %q. Failed with faults: %+q", vmdisk.volumeOptions.StoragePolicyName, faultMessage)
+		}
+	}
+
+	storageProfileSpec := &types.VirtualMachineDefinedProfileSpec{}
+	// Is PBM storage policy ID is present, set the storage spec profile ID,
+	// else, set raw the VSAN policy string.
+	if vmdisk.volumeOptions.StoragePolicyID != "" {
+		storageProfileSpec.ProfileId = vmdisk.volumeOptions.StoragePolicyID
+	} else if vmdisk.volumeOptions.VSANStorageProfileData != "" {
+		// Check Datastore type - VSANStorageProfileData is only applicable to vSAN Datastore
+		dsType, err := datastore.GetType(ctx)
+		if err != nil {
+			return err
+		}
+		if dsType != vclib.VSANDatastoreType {
+			glog.Errorf("The specified datastore: %q is not a VSAN datastore", datastore.Name())
+			return fmt.Errorf("The specified datastore: %q is not a VSAN datastore."+
+				" The policy parameters will work only with VSAN Datastore."+
+				" So, please specify a valid VSAN datastore in Storage class definition.", datastore.Name())
+		}
+		storageProfileSpec.ProfileId = ""
+		storageProfileSpec.ProfileData = &types.VirtualMachineProfileRawData{
+			ExtensionKey: "com.vmware.vim.sps",
+			ObjectData:   vmdisk.volumeOptions.VSANStorageProfileData,
+		}
+	} else {
+		glog.Errorf("Both volumeOptions.StoragePolicyID and volumeOptions.VSANStorageProfileData are not set. One of them should be set")
+		return fmt.Errorf("Both volumeOptions.StoragePolicyID and volumeOptions.VSANStorageProfileData are not set. One of them should be set")
+	}
+	var dummyVM *vclib.VirtualMachine
+	// Check if VM already exist in the folder.
+	// If VM is already present, use it, else create a new dummy VM.
+	fnvHash := fnv.New32a()
+	fnvHash.Write([]byte(vmdisk.volumeOptions.Name))
+	dummyVMFullName := vclib.DummyVMPrefixName + "-" + fmt.Sprint(fnvHash.Sum32())
+	dummyVM, err = datastore.Datacenter.GetVMByPath(ctx, vmdisk.vmOptions.VMFolder.InventoryPath+"/"+dummyVMFullName)
+	if err != nil {
+		// Create a dummy VM
+		glog.V(1).Info("Creating Dummy VM: %q", dummyVMFullName)
+		dummyVM, err = vmdisk.createDummyVM(ctx, datastore.Datacenter, dummyVMFullName)
+		if err != nil {
+			glog.Errorf("Failed to create Dummy VM. err: %v", err)
+			return err
+		}
+	}
+
+	// Reconfigure the VM to attach the disk with the VSAN policy configured
+	virtualMachineConfigSpec := types.VirtualMachineConfigSpec{}
+	disk, _, err := dummyVM.CreateDiskSpec(ctx, vmdisk.diskPath, datastore, vmdisk.volumeOptions)
+	if err != nil {
+		glog.Errorf("Failed to create Disk Spec. err: %v", err)
+		return err
+	}
+	deviceConfigSpec := &types.VirtualDeviceConfigSpec{
+		Device:        disk,
+		Operation:     types.VirtualDeviceConfigSpecOperationAdd,
+		FileOperation: types.VirtualDeviceConfigSpecFileOperationCreate,
+	}
+
+	deviceConfigSpec.Profile = append(deviceConfigSpec.Profile, storageProfileSpec)
+	virtualMachineConfigSpec.DeviceChange = append(virtualMachineConfigSpec.DeviceChange, deviceConfigSpec)
+	fileAlreadyExist := false
+	task, err := dummyVM.Reconfigure(ctx, virtualMachineConfigSpec)
+	err = task.Wait(ctx)
+	if err != nil {
+		fileAlreadyExist = isAlreadyExists(vmdisk.diskPath, err)
+		if fileAlreadyExist {
+			//Skip error and continue to detach the disk as the disk was already created on the datastore.
+			glog.V(vclib.LogLevel).Info("File: %v already exists", vmdisk.diskPath)
+		} else {
+			glog.Errorf("Failed to attach the disk to VM: %q with err: %+v", dummyVMFullName, err)
+			return err
+		}
+	}
+	// Detach the disk from the dummy VM.
+	err = dummyVM.DetachDisk(ctx, vmdisk.diskPath)
+	if err != nil {
+		if vclib.DiskNotFoundErrMsg == err.Error() && fileAlreadyExist {
+			// Skip error if disk was already detached from the dummy VM but still present on the datastore.
+			glog.V(vclib.LogLevel).Info("File: %v is already detached", vmdisk.diskPath)
+		} else {
+			glog.Errorf("Failed to detach the disk: %q from VM: %q with err: %+v", vmdisk.diskPath, dummyVMFullName, err)
+			return err
+		}
+	}
+	//  Delete the dummy VM
+	err = dummyVM.DeleteVM(ctx)
+	if err != nil {
+		glog.Errorf("Failed to destroy the vm: %q with err: %+v", dummyVMFullName, err)
+	}
+	return nil
+}
+
+func (vmdisk vmDiskManager) Delete(ctx context.Context, datastore *vclib.Datastore) error {
+	return fmt.Errorf("vmDiskManager.Delete is not supported")
+}
+
+// CreateDummyVM create a Dummy VM at specified location with given name.
+func (vmdisk vmDiskManager) createDummyVM(ctx context.Context, datacenter *vclib.Datacenter, vmName string) (*vclib.VirtualMachine, error) {
+	// Create a virtual machine config spec with 1 SCSI adapter.
+	virtualMachineConfigSpec := types.VirtualMachineConfigSpec{
+		Name: vmName,
+		Files: &types.VirtualMachineFileInfo{
+			VmPathName: "[" + vmdisk.volumeOptions.Datastore + "]",
+		},
+		NumCPUs:  1,
+		MemoryMB: 4,
+		DeviceChange: []types.BaseVirtualDeviceConfigSpec{
+			&types.VirtualDeviceConfigSpec{
+				Operation: types.VirtualDeviceConfigSpecOperationAdd,
+				Device: &types.ParaVirtualSCSIController{
+					VirtualSCSIController: types.VirtualSCSIController{
+						SharedBus: types.VirtualSCSISharingNoSharing,
+						VirtualController: types.VirtualController{
+							BusNumber: 0,
+							VirtualDevice: types.VirtualDevice{
+								Key: 1000,
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	task, err := vmdisk.vmOptions.VMFolder.CreateVM(ctx, virtualMachineConfigSpec, vmdisk.vmOptions.VMResourcePool, nil)
+	if err != nil {
+		glog.Errorf("Failed to create VM. err: %+v", err)
+		return nil, err
+	}
+
+	dummyVMTaskInfo, err := task.WaitForResult(ctx, nil)
+	if err != nil {
+		glog.Errorf("Error occurred while waiting for create VM task result. err: %+v", err)
+		return nil, err
+	}
+
+	vmRef := dummyVMTaskInfo.Result.(object.Reference)
+	dummyVM := object.NewVirtualMachine(datacenter.Client(), vmRef.Reference())
+	return &vclib.VirtualMachine{VirtualMachine: dummyVM, Datacenter: datacenter}, nil
+}
+
+// CleanUpDummyVMs deletes stale dummyVM's
+func CleanUpDummyVMs(ctx context.Context, folder *vclib.Folder, dc *vclib.Datacenter) error {
+	vmList, err := folder.GetVirtualMachines(ctx)
+	if err != nil {
+		glog.V(4).Infof("Failed to get virtual machines in the kubernetes cluster: %s, err: %+v", folder.InventoryPath, err)
+		return err
+	}
+	if vmList == nil || len(vmList) == 0 {
+		glog.Errorf("No virtual machines found in the kubernetes cluster: %s", folder.InventoryPath)
+		return fmt.Errorf("No virtual machines found in the kubernetes cluster: %s", folder.InventoryPath)
+	}
+	var dummyVMList []*vclib.VirtualMachine
+	// Loop through VM's in the Kubernetes cluster to find dummy VM's
+	for _, vm := range vmList {
+		vmName, err := vm.ObjectName(ctx)
+		if err != nil {
+			glog.V(4).Infof("Unable to get name from VM with err: %+v", err)
+			continue
+		}
+		if strings.HasPrefix(vmName, vclib.DummyVMPrefixName) {
+			vmObj := vclib.VirtualMachine{VirtualMachine: object.NewVirtualMachine(dc.Client(), vm.Reference()), Datacenter: dc}
+			dummyVMList = append(dummyVMList, &vmObj)
+		}
+	}
+	for _, vm := range dummyVMList {
+		err = vm.DeleteVM(ctx)
+		if err != nil {
+			glog.V(4).Infof("Unable to delete dummy VM with err: %+v", err)
+			continue
+		}
+	}
+	return nil
+}
+
+func isAlreadyExists(path string, err error) bool {
+	errorMessage := fmt.Sprintf("Cannot complete the operation because the file or folder %s already exists", path)
+	if errorMessage == err.Error() {
+		return true
+	}
+	return false
+}