Update dependencies to K8s 1.8

vendor/k8s.io/kubernetes/pkg/cloudprovider/BUILD

@@ -1,7 +1,5 @@
 package(default_visibility = ["//visibility:public"])
 
-licenses(["notice"])
-
 load(
     "@io_bazel_rules_go//go:def.bzl",
     "go_library",
@@ -14,7 +12,6 @@ go_library(
         "doc.go",
         "plugins.go",
     ],
-    tags = ["automanaged"],
     deps = [
         "//pkg/controller:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",

vendor/k8s.io/kubernetes/pkg/cloudprovider/OWNERS

@@ -38,3 +38,4 @@ reviewers:
 - jdef
 - freehan
 - jingxu97
+- wlan0

vendor/k8s.io/kubernetes/pkg/cloudprovider/cloud.go

@@ -45,6 +45,8 @@ type Interface interface {
 	ProviderName() string
 	// ScrubDNS provides an opportunity for cloud-provider-specific code to process DNS settings for pods.
 	ScrubDNS(nameservers, searches []string) (nsOut, srchOut []string)
+	// HasClusterID returns true if a ClusterID is required and set
+	HasClusterID() bool
 }
 
 // Clusters is an abstract, pluggable interface for clusters of containers.
@@ -122,7 +124,7 @@ type Instances interface {
 	// ProviderID is a unique identifier of the node. This will not be called
 	// from the node whose nodeaddresses are being queried. i.e. local metadata
 	// services cannot be used in this method to obtain nodeaddresses
-	NodeAddressesByProviderID(providerId string) ([]v1.NodeAddress, error)
+	NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error)
 	// ExternalID returns the cloud provider ID of the node with the specified NodeName.
 	// Note that if the instance does not exist or is no longer running, we must return ("", cloudprovider.InstanceNotFound)
 	ExternalID(nodeName types.NodeName) (string, error)
@@ -138,6 +140,9 @@ type Instances interface {
 	// CurrentNodeName returns the name of the node we are currently running on
 	// On most clouds (e.g. GCE) this is the hostname, so we provide the hostname
 	CurrentNodeName(hostname string) (types.NodeName, error)
+	// InstanceExistsByProviderID returns true if the instance for the given provider id still is running.
+	// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager.
+	InstanceExistsByProviderID(providerID string) (bool, error)
 }
 
 // Route is a representation of an advanced routing rule.
@@ -182,5 +187,23 @@ type Zone struct {
 // Zones is an abstract, pluggable interface for zone enumeration.
 type Zones interface {
 	// GetZone returns the Zone containing the current failure zone and locality region that the program is running in
+	// In most cases, this method is called from the kubelet querying a local metadata service to aquire its zone.
+	// For the case of external cloud providers, use GetZoneByProviderID or GetZoneByNodeName since GetZone
+	// can no longer be called from the kubelets.
 	GetZone() (Zone, error)
+
+	// GetZoneByProviderID returns the Zone containing the current zone and locality region of the node specified by providerId
+	// This method is particularly used in the context of external cloud providers where node initialization must be down
+	// outside the kubelets.
+	GetZoneByProviderID(providerID string) (Zone, error)
+
+	// GetZoneByNodeName returns the Zone containing the current zone and locality region of the node specified by node name
+	// This method is particularly used in the context of external cloud providers where node initialization must be down
+	// outside the kubelets.
+	GetZoneByNodeName(nodeName types.NodeName) (Zone, error)
+}
+
+// PVLabeler is an abstract, pluggable interface for fetching labels for volumes
+type PVLabeler interface {
+	GetLabelsForVolume(pv *v1.PersistentVolume) (map[string]string, error)
 }

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/BUILD

@@ -1,7 +1,5 @@
 package(default_visibility = ["//visibility:public"])
 
-licenses(["notice"])
-
 load(
     "@io_bazel_rules_go//go:def.bzl",
     "go_library",
@@ -13,8 +11,10 @@ go_library(
     srcs = [
         "doc.go",
         "gce.go",
+        "gce_address_manager.go",
         "gce_addresses.go",
         "gce_addresses_fakes.go",
+        "gce_alpha.go",
         "gce_annotations.go",
         "gce_backendservice.go",
         "gce_cert.go",
@@ -23,6 +23,7 @@ go_library(
         "gce_disks.go",
         "gce_firewall.go",
         "gce_forwardingrule.go",
+        "gce_forwardingrule_fakes.go",
         "gce_healthchecks.go",
         "gce_instancegroup.go",
         "gce_instances.go",
@@ -38,10 +39,10 @@ go_library(
         "gce_urlmap.go",
         "gce_util.go",
         "gce_zones.go",
+        "kms.go",
         "metrics.go",
         "token_source.go",
     ],
-    tags = ["automanaged"],
     deps = [
         "//pkg/api/v1/service:go_default_library",
         "//pkg/cloudprovider:go_default_library",
@@ -51,6 +52,7 @@ go_library(
         "//pkg/util/net/sets:go_default_library",
         "//pkg/util/version:go_default_library",
         "//pkg/volume:go_default_library",
+        "//pkg/volume/util:go_default_library",
         "//vendor/cloud.google.com/go/compute/metadata:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
@@ -72,8 +74,13 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
         "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+        "//vendor/k8s.io/client-go/tools/record:go_default_library",
         "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
     ],
 )
@@ -81,19 +88,28 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
+        "gce_address_manager_test.go",
+        "gce_annotations_test.go",
         "gce_disks_test.go",
         "gce_healthchecks_test.go",
         "gce_loadbalancer_external_test.go",
         "gce_test.go",
+        "metrics_test.go",
     ],
     library = ":go_default_library",
-    tags = ["automanaged"],
     deps = [
         "//pkg/cloudprovider:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/github.com/stretchr/testify/require:go_default_library",
+        "//vendor/golang.org/x/oauth2/google:go_default_library",
+        "//vendor/google.golang.org/api/compute/v0.alpha:go_default_library",
+        "//vendor/google.golang.org/api/compute/v0.beta:go_default_library",
         "//vendor/google.golang.org/api/compute/v1:go_default_library",
         "//vendor/google.golang.org/api/googleapi:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_address_manager.go

@@ -0,0 +1,198 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"fmt"
+	"net/http"
+
+	computebeta "google.golang.org/api/compute/v0.beta"
+
+	"github.com/golang/glog"
+)
+
+type addressManager struct {
+	logPrefix   string
+	svc         CloudAddressService
+	name        string
+	serviceName string
+	targetIP    string
+	addressType lbScheme
+	region      string
+	subnetURL   string
+	tryRelease  bool
+}
+
+func newAddressManager(svc CloudAddressService, serviceName, region, subnetURL, name, targetIP string, addressType lbScheme) *addressManager {
+	return &addressManager{
+		svc:         svc,
+		logPrefix:   fmt.Sprintf("AddressManager(%q)", name),
+		region:      region,
+		serviceName: serviceName,
+		name:        name,
+		targetIP:    targetIP,
+		addressType: addressType,
+		tryRelease:  true,
+		subnetURL:   subnetURL,
+	}
+}
+
+// HoldAddress will ensure that the IP is reserved with an address - either owned by the controller
+// or by a user. If the address is not the addressManager.name, then it's assumed to be a user's address.
+// The string returned is the reserved IP address.
+func (am *addressManager) HoldAddress() (string, error) {
+	// HoldAddress starts with retrieving the address that we use for this load balancer (by name).
+	// Retrieving an address by IP will indicate if the IP is reserved and if reserved by the user
+	// or the controller, but won't tell us the current state of the controller's IP. The address
+	// could be reserving another address; therefore, it would need to be deleted. In the normal
+	// case of using a controller address, retrieving the address by name results in the fewest API
+	// calls since it indicates whether a Delete is necessary before Reserve.
+	glog.V(4).Infof("%v: attempting hold of IP %q Type %q", am.logPrefix, am.targetIP, am.addressType)
+	// Get the address in case it was orphaned earlier
+	addr, err := am.svc.GetBetaRegionAddress(am.name, am.region)
+	if err != nil && !isNotFound(err) {
+		return "", err
+	}
+
+	if addr != nil {
+		// If address exists, check if the address had the expected attributes.
+		validationError := am.validateAddress(addr)
+		if validationError == nil {
+			glog.V(4).Infof("%v: address %q already reserves IP %q Type %q. No further action required.", am.logPrefix, addr.Name, addr.Address, addr.AddressType)
+			return addr.Address, nil
+		}
+
+		glog.V(2).Infof("%v: deleting existing address because %v", am.logPrefix, validationError)
+		err := am.svc.DeleteRegionAddress(addr.Name, am.region)
+		if err != nil {
+			if isNotFound(err) {
+				glog.V(4).Infof("%v: address %q was not found. Ignoring.", am.logPrefix, addr.Name)
+			} else {
+				return "", err
+			}
+		} else {
+			glog.V(4).Infof("%v: successfully deleted previous address %q", am.logPrefix, addr.Name)
+		}
+	}
+
+	return am.ensureAddressReservation()
+}
+
+// ReleaseAddress will release the address if it's owned by the controller.
+func (am *addressManager) ReleaseAddress() error {
+	if !am.tryRelease {
+		glog.V(4).Infof("%v: not attempting release of address %q.", am.logPrefix, am.targetIP)
+		return nil
+	}
+
+	glog.V(4).Infof("%v: releasing address %q named %q", am.logPrefix, am.targetIP, am.name)
+	// Controller only ever tries to unreserve the address named with the load balancer's name.
+	err := am.svc.DeleteRegionAddress(am.name, am.region)
+	if err != nil {
+		if isNotFound(err) {
+			glog.Warningf("%v: address %q was not found. Ignoring.", am.logPrefix, am.name)
+			return nil
+		}
+
+		return err
+	}
+
+	glog.V(4).Infof("%v: successfully released IP %q named %q", am.logPrefix, am.targetIP, am.name)
+	return nil
+}
+
+func (am *addressManager) ensureAddressReservation() (string, error) {
+	// Try reserving the IP with controller-owned address name
+	// If am.targetIP is an empty string, a new IP will be created.
+	newAddr := &computebeta.Address{
+		Name:        am.name,
+		Description: fmt.Sprintf(`{"kubernetes.io/service-name":"%s"}`, am.serviceName),
+		Address:     am.targetIP,
+		AddressType: string(am.addressType),
+		Subnetwork:  am.subnetURL,
+	}
+
+	reserveErr := am.svc.ReserveBetaRegionAddress(newAddr, am.region)
+	if reserveErr == nil {
+		if newAddr.Address != "" {
+			glog.V(4).Infof("%v: successfully reserved IP %q with name %q", am.logPrefix, newAddr.Address, newAddr.Name)
+			return newAddr.Address, nil
+		}
+
+		addr, err := am.svc.GetRegionAddress(newAddr.Name, am.region)
+		if err != nil {
+			return "", err
+		}
+
+		glog.V(4).Infof("%v: successfully created address %q which reserved IP %q", am.logPrefix, addr.Name, addr.Address)
+		return addr.Address, nil
+	} else if !isHTTPErrorCode(reserveErr, http.StatusConflict) && !isHTTPErrorCode(reserveErr, http.StatusBadRequest) {
+		// If the IP is already reserved:
+		//    by an internal address: a StatusConflict is returned
+		//    by an external address: a BadRequest is returned
+		return "", reserveErr
+	}
+
+	// If the target IP was empty, we cannot try to find which IP caused a conflict.
+	// If the name was already used, then the next sync will attempt deletion of that address.
+	if am.targetIP == "" {
+		return "", fmt.Errorf("failed to reserve address %q with no specific IP, err: %v", am.name, reserveErr)
+	}
+
+	// Reserving the address failed due to a conflict or bad request. The address manager just checked that no address
+	// exists with the name, so it may belong to the user.
+	addr, err := am.svc.GetBetaRegionAddressByIP(am.region, am.targetIP)
+	if err != nil {
+		return "", fmt.Errorf("failed to get address by IP %q after reservation attempt, err: %q, reservation err: %q", am.targetIP, err, reserveErr)
+	}
+
+	// Check that the address attributes are as required.
+	if err := am.validateAddress(addr); err != nil {
+		return "", err
+	}
+
+	if am.isManagedAddress(addr) {
+		// The address with this name is checked at the beginning of 'HoldAddress()', but for some reason
+		// it was re-created by this point. May be possible that two controllers are running.
+		glog.Warning("%v: address %q unexpectedly existed with IP %q.", am.logPrefix, addr.Name, am.targetIP)
+	} else {
+		// If the retrieved address is not named with the loadbalancer name, then the controller does not own it, but will allow use of it.
+		glog.V(4).Infof("%v: address %q was already reserved with name: %q, description: %q", am.logPrefix, am.targetIP, addr.Name, addr.Description)
+		am.tryRelease = false
+	}
+
+	return addr.Address, nil
+}
+
+func (am *addressManager) validateAddress(addr *computebeta.Address) error {
+	if am.targetIP != "" && am.targetIP != addr.Address {
+		return fmt.Errorf("address %q does not have the expected IP %q, actual: %q", addr.Name, am.targetIP, addr.Address)
+	}
+	if addr.AddressType != string(am.addressType) {
+		return fmt.Errorf("address %q does not have the expected address type %q, actual: %q", addr.Name, am.addressType, addr.AddressType)
+	}
+
+	return nil
+}
+
+func (am *addressManager) isManagedAddress(addr *computebeta.Address) bool {
+	return addr.Name == am.name
+}
+
+func ensureAddressDeleted(svc CloudAddressService, name, region string) error {
+	return ignoreNotFound(svc.DeleteRegionAddress(name, region))
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_addresses.go

@@ -17,16 +17,21 @@ limitations under the License.
 package gce
 
 import (
-	"time"
+	"fmt"
 
+	"github.com/golang/glog"
+
+	computealpha "google.golang.org/api/compute/v0.alpha"
+	computebeta "google.golang.org/api/compute/v0.beta"
 	compute "google.golang.org/api/compute/v1"
 )
 
 func newAddressMetricContext(request, region string) *metricContext {
-	return &metricContext{
-		start:      time.Now(),
-		attributes: []string{"address_" + request, region, unusedMetricLabel},
-	}
+	return newAddressMetricContextWithVersion(request, region, computeV1Version)
+}
+
+func newAddressMetricContextWithVersion(request, region, version string) *metricContext {
+	return newGenericMetricContext("address", request, region, unusedMetricLabel, version)
 }
 
 // ReserveGlobalAddress creates a global address.
@@ -69,6 +74,26 @@ func (gce *GCECloud) ReserveRegionAddress(addr *compute.Address, region string)
 	return gce.waitForRegionOp(op, region, mc)
 }
 
+// ReserveAlphaRegionAddress creates an Alpha, regional address.
+func (gce *GCECloud) ReserveAlphaRegionAddress(addr *computealpha.Address, region string) error {
+	mc := newAddressMetricContextWithVersion("reserve", region, computeAlphaVersion)
+	op, err := gce.serviceAlpha.Addresses.Insert(gce.projectID, region, addr).Do()
+	if err != nil {
+		return mc.Observe(err)
+	}
+	return gce.waitForRegionOp(op, region, mc)
+}
+
+// ReserveBetaRegionAddress creates a beta region address
+func (gce *GCECloud) ReserveBetaRegionAddress(addr *computebeta.Address, region string) error {
+	mc := newAddressMetricContextWithVersion("reserve", region, computeBetaVersion)
+	op, err := gce.serviceBeta.Addresses.Insert(gce.projectID, region, addr).Do()
+	if err != nil {
+		return mc.Observe(err)
+	}
+	return gce.waitForRegionOp(op, region, mc)
+}
+
 // DeleteRegionAddress deletes a region address by name.
 func (gce *GCECloud) DeleteRegionAddress(name, region string) error {
 	mc := newAddressMetricContext("delete", region)
@@ -85,3 +110,81 @@ func (gce *GCECloud) GetRegionAddress(name, region string) (*compute.Address, er
 	v, err := gce.service.Addresses.Get(gce.projectID, region, name).Do()
 	return v, mc.Observe(err)
 }
+
+// GetAlphaRegionAddress returns the Alpha, regional address by name.
+func (gce *GCECloud) GetAlphaRegionAddress(name, region string) (*computealpha.Address, error) {
+	mc := newAddressMetricContextWithVersion("get", region, computeAlphaVersion)
+	v, err := gce.serviceAlpha.Addresses.Get(gce.projectID, region, name).Do()
+	return v, mc.Observe(err)
+}
+
+// GetBetaRegionAddress returns the beta region address by name
+func (gce *GCECloud) GetBetaRegionAddress(name, region string) (*computebeta.Address, error) {
+	mc := newAddressMetricContextWithVersion("get", region, computeBetaVersion)
+	v, err := gce.serviceBeta.Addresses.Get(gce.projectID, region, name).Do()
+	return v, mc.Observe(err)
+}
+
+// GetRegionAddressByIP returns the regional address matching the given IP address.
+func (gce *GCECloud) GetRegionAddressByIP(region, ipAddress string) (*compute.Address, error) {
+	mc := newAddressMetricContext("list", region)
+	addrs, err := gce.service.Addresses.List(gce.projectID, region).Filter("address eq " + ipAddress).Do()
+	// Record the metrics for the call.
+	mc.Observe(err)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(addrs.Items) > 1 {
+		// We don't expect more than one match.
+		addrsToPrint := []compute.Address{}
+		for _, addr := range addrs.Items {
+			addrsToPrint = append(addrsToPrint, *addr)
+		}
+		glog.Errorf("More than one addresses matching the IP %q: %+v", ipAddress, addrsToPrint)
+	}
+	for _, addr := range addrs.Items {
+		if addr.Address == ipAddress {
+			return addr, nil
+		}
+	}
+	return nil, makeGoogleAPINotFoundError(fmt.Sprintf("Address with IP %q was not found in region %q", ipAddress, region))
+}
+
+// GetBetaRegionAddressByIP returns the beta regional address matching the given IP address.
+func (gce *GCECloud) GetBetaRegionAddressByIP(region, ipAddress string) (*computebeta.Address, error) {
+	mc := newAddressMetricContext("list", region)
+	addrs, err := gce.serviceBeta.Addresses.List(gce.projectID, region).Filter("address eq " + ipAddress).Do()
+	// Record the metrics for the call.
+	mc.Observe(err)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(addrs.Items) > 1 {
+		// We don't expect more than one match.
+		addrsToPrint := []computebeta.Address{}
+		for _, addr := range addrs.Items {
+			addrsToPrint = append(addrsToPrint, *addr)
+		}
+		glog.Errorf("More than one addresses matching the IP %q: %+v", ipAddress, addrsToPrint)
+	}
+	for _, addr := range addrs.Items {
+		if addr.Address == ipAddress {
+			return addr, nil
+		}
+	}
+	return nil, makeGoogleAPINotFoundError(fmt.Sprintf("Address with IP %q was not found in region %q", ipAddress, region))
+}
+
+// TODO(#51665): retire this function once Network Tiers becomes Beta in GCP.
+func (gce *GCECloud) getNetworkTierFromAddress(name, region string) (string, error) {
+	if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) {
+		return NetworkTierDefault.ToGCEValue(), nil
+	}
+	addr, err := gce.GetAlphaRegionAddress(name, region)
+	if err != nil {
+		return handleAlphaNetworkTierGetError(err)
+	}
+	return addr.NetworkTier, nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_addresses_fakes.go

@@ -17,13 +17,18 @@ limitations under the License.
 package gce
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"net/http"
 
+	computealpha "google.golang.org/api/compute/v0.alpha"
+	computebeta "google.golang.org/api/compute/v0.beta"
 	compute "google.golang.org/api/compute/v1"
-	"google.golang.org/api/googleapi"
 )
 
+// test
+
 type FakeCloudAddressService struct {
 	count int
 	// reservedAddrs tracks usage of IP addresses
@@ -31,32 +36,60 @@ type FakeCloudAddressService struct {
 	reservedAddrs map[string]bool
 	// addrsByRegionAndName
 	// Outer key is for region string; inner key is for address name.
-	addrsByRegionAndName map[string]map[string]*compute.Address
+	addrsByRegionAndName map[string]map[string]*computealpha.Address
 }
 
+// FakeCloudAddressService Implements CloudAddressService
+var _ CloudAddressService = &FakeCloudAddressService{}
+
 func NewFakeCloudAddressService() *FakeCloudAddressService {
 	return &FakeCloudAddressService{
 		reservedAddrs:        make(map[string]bool),
-		addrsByRegionAndName: make(map[string]map[string]*compute.Address),
+		addrsByRegionAndName: make(map[string]map[string]*computealpha.Address),
 	}
 }
 
-func (cas *FakeCloudAddressService) ReserveRegionAddress(addr *compute.Address, region string) error {
+// SetRegionalAddresses sets the addresses of ther region. This is used for
+// setting the test environment.
+func (cas *FakeCloudAddressService) SetRegionalAddresses(region string, addrs []*computealpha.Address) {
+	// Reset addresses in the region.
+	cas.addrsByRegionAndName[region] = make(map[string]*computealpha.Address)
+
+	for _, addr := range addrs {
+		cas.reservedAddrs[addr.Address] = true
+		cas.addrsByRegionAndName[region][addr.Name] = addr
+	}
+}
+
+func (cas *FakeCloudAddressService) ReserveAlphaRegionAddress(addr *computealpha.Address, region string) error {
 	if addr.Address == "" {
 		addr.Address = fmt.Sprintf("1.2.3.%d", cas.count)
 		cas.count++
 	}
 
+	if addr.AddressType == "" {
+		addr.AddressType = string(schemeExternal)
+	}
+
 	if cas.reservedAddrs[addr.Address] {
-		return &googleapi.Error{Code: http.StatusConflict}
+		msg := "IP in use"
+		// When the IP is already in use, this call returns an error code based
+		// on the type (internal vs external) of the address. This is to be
+		// consistent with actual GCE API.
+		switch lbScheme(addr.AddressType) {
+		case schemeExternal:
+			return makeGoogleAPIError(http.StatusBadRequest, msg)
+		default:
+			return makeGoogleAPIError(http.StatusConflict, msg)
+		}
 	}
 
 	if _, exists := cas.addrsByRegionAndName[region]; !exists {
-		cas.addrsByRegionAndName[region] = make(map[string]*compute.Address)
+		cas.addrsByRegionAndName[region] = make(map[string]*computealpha.Address)
 	}
 
 	if _, exists := cas.addrsByRegionAndName[region][addr.Name]; exists {
-		return &googleapi.Error{Code: http.StatusConflict}
+		return makeGoogleAPIError(http.StatusConflict, "name in use")
 	}
 
 	cas.addrsByRegionAndName[region][addr.Name] = addr
@@ -64,14 +97,141 @@ func (cas *FakeCloudAddressService) ReserveRegionAddress(addr *compute.Address,
 	return nil
 }
 
-func (cas *FakeCloudAddressService) GetRegionAddress(name, region string) (*compute.Address, error) {
+func (cas *FakeCloudAddressService) ReserveBetaRegionAddress(addr *computebeta.Address, region string) error {
+	alphaAddr := convertToAlphaAddress(addr)
+	return cas.ReserveAlphaRegionAddress(alphaAddr, region)
+}
+
+func (cas *FakeCloudAddressService) ReserveRegionAddress(addr *compute.Address, region string) error {
+	alphaAddr := convertToAlphaAddress(addr)
+	return cas.ReserveAlphaRegionAddress(alphaAddr, region)
+}
+
+func (cas *FakeCloudAddressService) GetAlphaRegionAddress(name, region string) (*computealpha.Address, error) {
 	if _, exists := cas.addrsByRegionAndName[region]; !exists {
-		return nil, &googleapi.Error{Code: http.StatusNotFound}
+		return nil, makeGoogleAPINotFoundError("")
 	}
 
 	if addr, exists := cas.addrsByRegionAndName[region][name]; !exists {
-		return nil, &googleapi.Error{Code: http.StatusNotFound}
+		return nil, makeGoogleAPINotFoundError("")
 	} else {
 		return addr, nil
 	}
 }
+
+func (cas *FakeCloudAddressService) GetBetaRegionAddress(name, region string) (*computebeta.Address, error) {
+	addr, err := cas.GetAlphaRegionAddress(name, region)
+	if addr != nil {
+		return convertToBetaAddress(addr), err
+	}
+	return nil, err
+}
+
+func (cas *FakeCloudAddressService) GetRegionAddress(name, region string) (*compute.Address, error) {
+	addr, err := cas.GetAlphaRegionAddress(name, region)
+	if addr != nil {
+		return convertToV1Address(addr), err
+	}
+	return nil, err
+}
+
+func (cas *FakeCloudAddressService) DeleteRegionAddress(name, region string) error {
+	if _, exists := cas.addrsByRegionAndName[region]; !exists {
+		return makeGoogleAPINotFoundError("")
+	}
+
+	addr, exists := cas.addrsByRegionAndName[region][name]
+	if !exists {
+		return makeGoogleAPINotFoundError("")
+	}
+
+	delete(cas.reservedAddrs, addr.Address)
+	delete(cas.addrsByRegionAndName[region], name)
+	return nil
+}
+
+func (cas *FakeCloudAddressService) GetAlphaRegionAddressByIP(region, ipAddress string) (*computealpha.Address, error) {
+	if _, exists := cas.addrsByRegionAndName[region]; !exists {
+		return nil, makeGoogleAPINotFoundError("")
+	}
+
+	for _, addr := range cas.addrsByRegionAndName[region] {
+		if addr.Address == ipAddress {
+			return addr, nil
+		}
+	}
+	return nil, makeGoogleAPINotFoundError("")
+}
+
+func (cas *FakeCloudAddressService) GetBetaRegionAddressByIP(name, region string) (*computebeta.Address, error) {
+	addr, err := cas.GetAlphaRegionAddressByIP(name, region)
+	if addr != nil {
+		return convertToBetaAddress(addr), nil
+	}
+	return nil, err
+}
+
+func (cas *FakeCloudAddressService) GetRegionAddressByIP(name, region string) (*compute.Address, error) {
+	addr, err := cas.GetAlphaRegionAddressByIP(name, region)
+	if addr != nil {
+		return convertToV1Address(addr), nil
+	}
+	return nil, err
+}
+
+func (cas *FakeCloudAddressService) getNetworkTierFromAddress(name, region string) (string, error) {
+	addr, err := cas.GetAlphaRegionAddress(name, region)
+	if err != nil {
+		return "", err
+	}
+	return addr.NetworkTier, nil
+}
+
+func convertToV1Address(object gceObject) *compute.Address {
+	enc, err := object.MarshalJSON()
+	if err != nil {
+		panic(fmt.Sprintf("Failed to encode to json: %v", err))
+	}
+	var addr compute.Address
+	if err := json.Unmarshal(enc, &addr); err != nil {
+		panic(fmt.Sprintf("Failed to convert GCE apiObject %v to v1 address: %v", object, err))
+	}
+	return &addr
+}
+
+func convertToAlphaAddress(object gceObject) *computealpha.Address {
+	enc, err := object.MarshalJSON()
+	if err != nil {
+		panic(fmt.Sprintf("Failed to encode to json: %v", err))
+	}
+	var addr computealpha.Address
+	if err := json.Unmarshal(enc, &addr); err != nil {
+		panic(fmt.Sprintf("Failed to convert GCE apiObject %v to alpha address: %v", object, err))
+	}
+	// Set the default values for the Alpha fields.
+	addr.NetworkTier = NetworkTierDefault.ToGCEValue()
+	return &addr
+}
+
+func convertToBetaAddress(object gceObject) *computebeta.Address {
+	enc, err := object.MarshalJSON()
+	if err != nil {
+		panic(fmt.Sprintf("Failed to encode to json: %v", err))
+	}
+	var addr computebeta.Address
+	if err := json.Unmarshal(enc, &addr); err != nil {
+		panic(fmt.Sprintf("Failed to convert GCE apiObject %v to beta address: %v", object, err))
+	}
+	return &addr
+}
+
+func (cas *FakeCloudAddressService) String() string {
+	var b bytes.Buffer
+	for region, regAddresses := range cas.addrsByRegionAndName {
+		b.WriteString(fmt.Sprintf("%v:\n", region))
+		for name, addr := range regAddresses {
+			b.WriteString(fmt.Sprintf(" %v:  %v\n", name, addr.Address))
+		}
+	}
+	return b.String()
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_alpha.go

@@ -0,0 +1,60 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"fmt"
+
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+)
+
+const (
+	// alpha: v1.8 (for Services)
+	//
+	// Allows Services backed by a GCP load balancer to choose what network
+	// tier to use. Currently supports "Standard" and "Premium" (default).
+	AlphaFeatureNetworkTiers = "NetworkTiers"
+
+	GCEDiskAlphaFeatureGate = "DiskAlphaAPI"
+)
+
+// All known alpha features
+var knownAlphaFeatures = map[string]bool{
+	AlphaFeatureNetworkTiers: true,
+	GCEDiskAlphaFeatureGate:  true,
+}
+
+type AlphaFeatureGate struct {
+	features map[string]bool
+}
+
+func (af *AlphaFeatureGate) Enabled(key string) bool {
+	return af.features[key]
+}
+
+func NewAlphaFeatureGate(features []string) (*AlphaFeatureGate, error) {
+	errList := []error{}
+	featureMap := make(map[string]bool)
+	for _, name := range features {
+		if _, ok := knownAlphaFeatures[name]; !ok {
+			errList = append(errList, fmt.Errorf("alpha feature %q is not supported.", name))
+		} else {
+			featureMap[name] = true
+		}
+	}
+	return &AlphaFeatureGate{featureMap}, utilerrors.NewAggregate(errList)
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_annotations.go

@@ -16,9 +16,17 @@ limitations under the License.
 
 package gce
 
-import "k8s.io/api/core/v1"
+import (
+	"fmt"
+	"strings"
+
+	"github.com/golang/glog"
+
+	"k8s.io/api/core/v1"
+)
 
 type LoadBalancerType string
+type NetworkTier string
 
 const (
 	// ServiceAnnotationLoadBalancerType is annotated on a service with type LoadBalancer
@@ -26,12 +34,27 @@ const (
 	// Currently, only "internal" is supported.
 	ServiceAnnotationLoadBalancerType = "cloud.google.com/load-balancer-type"
 
-	LBTypeInternal LoadBalancerType = "internal"
+	LBTypeInternal LoadBalancerType = "Internal"
+	// Deprecating the lowercase spelling of Internal.
+	deprecatedTypeInternalLowerCase LoadBalancerType = "internal"
 
 	// ServiceAnnotationInternalBackendShare is annotated on a service with "true" when users
 	// want to share GCP Backend Services for a set of internal load balancers.
 	// ALPHA feature - this may be removed in a future release.
-	ServiceAnnotationILBBackendShare = "cloud.google.com/load-balancer-backend-share"
+	ServiceAnnotationILBBackendShare = "alpha.cloud.google.com/load-balancer-backend-share"
+	// This annotation did not correctly specify "alpha", so both annotations will be checked.
+	deprecatedServiceAnnotationILBBackendShare = "cloud.google.com/load-balancer-backend-share"
+
+	// NetworkTierAnnotationKey is annotated on a Service object to indicate which
+	// network tier a GCP LB should use. The valid values are "Standard" and
+	// "Premium" (default).
+	NetworkTierAnnotationKey      = "cloud.google.com/network-tier"
+	NetworkTierAnnotationStandard = "Standard"
+	NetworkTierAnnotationPremium  = "Premium"
+
+	NetworkTierStandard NetworkTier = NetworkTierAnnotationStandard
+	NetworkTierPremium  NetworkTier = NetworkTierAnnotationPremium
+	NetworkTierDefault  NetworkTier = NetworkTierPremium
 )
 
 // GetLoadBalancerAnnotationType returns the type of GCP load balancer which should be assembled.
@@ -48,8 +71,8 @@ func GetLoadBalancerAnnotationType(service *v1.Service) (LoadBalancerType, bool)
 	}
 
 	switch v {
-	case LBTypeInternal:
-		return v, true
+	case LBTypeInternal, deprecatedTypeInternalLowerCase:
+		return LBTypeInternal, true
 	default:
 		return v, false
 	}
@@ -58,10 +81,54 @@ func GetLoadBalancerAnnotationType(service *v1.Service) (LoadBalancerType, bool)
 // GetLoadBalancerAnnotationBackendShare returns whether this service's backend service should be
 // shared with other load balancers. Health checks and the healthcheck firewall will be shared regardless.
 func GetLoadBalancerAnnotationBackendShare(service *v1.Service) bool {
-	l, exists := service.Annotations[ServiceAnnotationILBBackendShare]
-	if exists && l == "true" {
+	if l, exists := service.Annotations[ServiceAnnotationILBBackendShare]; exists && l == "true" {
+		return true
+	}
+
+	// Check for deprecated annotation key
+	if l, exists := service.Annotations[deprecatedServiceAnnotationILBBackendShare]; exists && l == "true" {
+		glog.Warningf("Annotation %q is deprecated and replaced with an alpha-specific key: %q", deprecatedServiceAnnotationILBBackendShare, ServiceAnnotationILBBackendShare)
 		return true
 	}
 
 	return false
 }
+
+// GetServiceNetworkTier returns the network tier of GCP load balancer
+// which should be assembled, and an error if the specified tier is not
+// supported.
+func GetServiceNetworkTier(service *v1.Service) (NetworkTier, error) {
+	l, ok := service.Annotations[NetworkTierAnnotationKey]
+	if !ok {
+		return NetworkTierDefault, nil
+	}
+
+	v := NetworkTier(l)
+	switch v {
+	case NetworkTierStandard:
+		fallthrough
+	case NetworkTierPremium:
+		return v, nil
+	default:
+		return NetworkTierDefault, fmt.Errorf("unsupported network tier: %q", v)
+	}
+}
+
+// ToGCEValue converts NetworkTier to a string that we can populate the
+// NetworkTier field of GCE objects.
+func (n NetworkTier) ToGCEValue() string {
+	return strings.ToUpper(string(n))
+}
+
+// NetworkTierGCEValueToType converts the value of the NetworkTier field of a
+// GCE object to the NetworkTier type.
+func NetworkTierGCEValueToType(s string) NetworkTier {
+	switch s {
+	case "STANDARD":
+		return NetworkTierStandard
+	case "PREMIUM":
+		return NetworkTierPremium
+	default:
+		return NetworkTier(s)
+	}
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_backendservice.go

@@ -18,16 +18,12 @@ package gce
 
 import (
 	"net/http"
-	"time"
 
 	compute "google.golang.org/api/compute/v1"
 )
 
 func newBackendServiceMetricContext(request, region string) *metricContext {
-	return &metricContext{
-		start:      time.Now(),
-		attributes: []string{"backendservice_" + request, region, unusedMetricLabel},
-	}
+	return newGenericMetricContext("backendservice", request, region, unusedMetricLabel, computeV1Version)
 }
 
 // GetGlobalBackendService retrieves a backend by name.

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_cert.go

@@ -18,16 +18,12 @@ package gce
 
 import (
 	"net/http"
-	"time"
 
 	compute "google.golang.org/api/compute/v1"
 )
 
 func newCertMetricContext(request string) *metricContext {
-	return &metricContext{
-		start:      time.Now(),
-		attributes: []string{"cert_" + request, unusedMetricLabel, unusedMetricLabel},
-	}
+	return newGenericMetricContext("cert", request, unusedMetricLabel, unusedMetricLabel, computeV1Version)
 }
 
 // GetSslCertificate returns the SslCertificate by name.

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_clusterid.go

@@ -62,7 +62,7 @@ type ClusterID struct {
 func (gce *GCECloud) watchClusterID() {
 	gce.ClusterID = ClusterID{
 		cfgMapKey: fmt.Sprintf("%v/%v", UIDNamespace, UIDConfigMapName),
-		client:    gce.clientBuilder.ClientOrDie("cloud-provider"),
+		client:    gce.client,
 	}
 
 	mapEventHandler := cache.ResourceEventHandlerFuncs{

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_clusters.go

@@ -16,13 +16,8 @@ limitations under the License.
 
 package gce
 
-import "time"
-
 func newClustersMetricContext(request, zone string) *metricContext {
-	return &metricContext{
-		start:      time.Now(),
-		attributes: []string{"clusters_" + request, unusedMetricLabel, zone},
-	}
+	return newGenericMetricContext("clusters", request, unusedMetricLabel, zone, computeV1Version)
 }
 
 func (gce *GCECloud) ListClusters() ([]string, error) {

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_disks.go

@@ -21,15 +21,17 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
-	"time"
+
+	"k8s.io/api/core/v1"
 
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 	"k8s.io/kubernetes/pkg/volume"
+	volumeutil "k8s.io/kubernetes/pkg/volume/util"
 
 	"github.com/golang/glog"
-	compute "google.golang.org/api/compute/v1"
 	"google.golang.org/api/googleapi"
 )
 
@@ -39,8 +41,15 @@ const (
 	DiskTypeSSD      = "pd-ssd"
 	DiskTypeStandard = "pd-standard"
 
-	diskTypeDefault     = DiskTypeStandard
-	diskTypeUriTemplate = "%s/zones/%s/diskTypes/%s"
+	diskTypeDefault               = DiskTypeStandard
+	diskTypeURITemplateSingleZone = "%s/zones/%s/diskTypes/%s"   // {gce.projectID}/zones/{disk.Zone}/diskTypes/{disk.Type}"
+	diskTypeURITemplateRegional   = "%s/regions/%s/diskTypes/%s" // {gce.projectID}/regions/{disk.Region}/diskTypes/{disk.Type}"
+	diskTypePersistent            = "PERSISTENT"
+
+	diskSourceURITemplateSingleZone = "%s/zones/%s/disks/%s"   // {gce.projectID}/zones/{disk.Zone}/disks/{disk.Name}"
+	diskSourceURITemplateRegional   = "%s/regions/%s/disks/%s" //{gce.projectID}/regions/{disk.Region}/disks/repd"
+
+	replicaZoneURITemplateSingleZone = "%s/zones/%s" // {gce.projectID}/zones/{disk.Zone}
 )
 
 // Disks is interface for manipulation with GCE PDs.
@@ -64,6 +73,11 @@ type Disks interface {
 	// as JSON into Description field.
 	CreateDisk(name string, diskType string, zone string, sizeGb int64, tags map[string]string) error
 
+	// CreateRegionalDisk creates a new Regional Persistent Disk, with the
+	// specified properties, replicated to the specified zones. Tags are
+	// serialized as JSON into Description field.
+	CreateRegionalDisk(name string, diskType string, replicaZones sets.String, sizeGb int64, tags map[string]string) error
+
 	// DeleteDisk deletes PD.
 	DeleteDisk(diskToDelete string) error
 
@@ -77,18 +91,55 @@ type Disks interface {
 // GCECloud implements Disks.
 var _ Disks = (*GCECloud)(nil)
 
+// GCECloud implements PVLabeler.
+var _ cloudprovider.PVLabeler = (*GCECloud)(nil)
+
 type GCEDisk struct {
-	Zone string
-	Name string
-	Kind string
-	Type string
+	ZoneInfo zoneType
+	Region   string
+	Name     string
+	Kind     string
+	Type     string
 }
 
-func newDiskMetricContext(request, zone string) *metricContext {
-	return &metricContext{
-		start:      time.Now(),
-		attributes: []string{"disk_" + request, unusedMetricLabel, zone},
+type zoneType interface {
+	isZoneType()
+}
+
+type multiZone struct {
+	replicaZones sets.String
+}
+
+type singleZone struct {
+	zone string
+}
+
+func (m multiZone) isZoneType()  {}
+func (s singleZone) isZoneType() {}
+
+func newDiskMetricContextZonal(request, region, zone string) *metricContext {
+	return newGenericMetricContext("disk", request, region, zone, computeV1Version)
+}
+
+func newDiskMetricContextRegional(request, region string) *metricContext {
+	return newGenericMetricContext("disk", request, region, unusedMetricLabel, computeV1Version)
+}
+
+func (gce *GCECloud) GetLabelsForVolume(pv *v1.PersistentVolume) (map[string]string, error) {
+	// Ignore any volumes that are being provisioned
+	if pv.Spec.GCEPersistentDisk.PDName == volume.ProvisionedVolumeName {
+		return nil, nil
 	}
+
+	// If the zone is already labeled, honor the hint
+	zone := pv.Labels[kubeletapis.LabelZoneFailureDomain]
+
+	labels, err := gce.GetAutoLabelsForPD(pv.Spec.GCEPersistentDisk.PDName, zone)
+	if err != nil {
+		return nil, err
+	}
+
+	return labels, nil
 }
 
 func (gce *GCECloud) AttachDisk(diskName string, nodeName types.NodeName, readOnly bool) error {
@@ -97,25 +148,41 @@ func (gce *GCECloud) AttachDisk(diskName string, nodeName types.NodeName, readOn
 	if err != nil {
 		return fmt.Errorf("error getting instance %q", instanceName)
 	}
-	disk, err := gce.getDiskByName(diskName, instance.Zone)
-	if err != nil {
-		return err
+
+	// Try fetching as regional PD
+	var disk *GCEDisk
+	var mc *metricContext
+	if gce.AlphaFeatureGate.Enabled(GCEDiskAlphaFeatureGate) {
+		disk, err = gce.getRegionalDiskByName(diskName)
+		if err != nil {
+			glog.V(5).Infof("Could not find regional PD named %q to Attach. Will look for a zonal PD", diskName)
+			err = nil
+		} else {
+			mc = newDiskMetricContextRegional("attach", gce.region)
+		}
 	}
+
+	if disk == nil {
+		disk, err = gce.getDiskByName(diskName, instance.Zone)
+		if err != nil {
+			return err
+		}
+		mc = newDiskMetricContextZonal("attach", gce.region, instance.Zone)
+	}
+
 	readWrite := "READ_WRITE"
 	if readOnly {
 		readWrite = "READ_ONLY"
 	}
-	attachedDisk := gce.convertDiskToAttachedDisk(disk, readWrite)
 
-	mc := newDiskMetricContext("attach", instance.Zone)
-	attachOp, err := gce.service.Instances.AttachDisk(
-		gce.projectID, disk.Zone, instance.Name, attachedDisk).Do()
+	attachOp, err := gce.manager.AttachDisk(
+		disk, readWrite, instance.Zone, instance.Name)
 
 	if err != nil {
 		return mc.Observe(err)
 	}
 
-	return gce.waitForZoneOp(attachOp, disk.Zone, mc)
+	return gce.manager.WaitForZoneOp(attachOp, instance.Zone, mc)
 }
 
 func (gce *GCECloud) DetachDisk(devicePath string, nodeName types.NodeName) error {
@@ -134,13 +201,13 @@ func (gce *GCECloud) DetachDisk(devicePath string, nodeName types.NodeName) erro
 		return fmt.Errorf("error getting instance %q", instanceName)
 	}
 
-	mc := newDiskMetricContext("detach", inst.Zone)
-	detachOp, err := gce.service.Instances.DetachDisk(gce.projectID, inst.Zone, inst.Name, devicePath).Do()
+	mc := newDiskMetricContextZonal("detach", gce.region, inst.Zone)
+	detachOp, err := gce.manager.DetachDisk(inst.Zone, inst.Name, devicePath)
 	if err != nil {
 		return mc.Observe(err)
 	}
 
-	return gce.waitForZoneOp(detachOp, inst.Zone, mc)
+	return gce.manager.WaitForZoneOp(detachOp, inst.Zone, mc)
 }
 
 func (gce *GCECloud) DiskIsAttached(diskName string, nodeName types.NodeName) (bool, error) {
@@ -209,14 +276,7 @@ func (gce *GCECloud) CreateDisk(
 
 	// Do not allow creation of PDs in zones that are not managed. Such PDs
 	// then cannot be deleted by DeleteDisk.
-	isManaged := false
-	for _, managedZone := range gce.managedZones {
-		if zone == managedZone {
-			isManaged = true
-			break
-		}
-	}
-	if !isManaged {
+	if isManaged := gce.verifyZoneIsManaged(zone); !isManaged {
 		return fmt.Errorf("kubernetes does not manage zone %q", zone)
 	}
 
@@ -225,30 +285,16 @@ func (gce *GCECloud) CreateDisk(
 		return err
 	}
 
-	switch diskType {
-	case DiskTypeSSD, DiskTypeStandard:
-		// noop
-	case "":
-		diskType = diskTypeDefault
-	default:
-		return fmt.Errorf("invalid GCE disk type %q", diskType)
+	diskType, err = getDiskType(diskType)
+	if err != nil {
+		return err
 	}
 
-	projectsApiEndpoint := gceComputeAPIEndpoint + "projects/"
-	if gce.service != nil {
-		projectsApiEndpoint = gce.service.BasePath
-	}
-	diskTypeUri := projectsApiEndpoint + fmt.Sprintf(diskTypeUriTemplate, gce.projectID, zone, diskType)
+	mc := newDiskMetricContextZonal("create", gce.region, zone)
 
-	diskToCreate := &compute.Disk{
-		Name:        name,
-		SizeGb:      sizeGb,
-		Description: tagsStr,
-		Type:        diskTypeUri,
-	}
+	createOp, err := gce.manager.CreateDisk(
+		name, sizeGb, tagsStr, diskType, zone)
 
-	mc := newDiskMetricContext("create", zone)
-	createOp, err := gce.manager.CreateDisk(gce.projectID, zone, diskToCreate)
 	if isGCEError(err, "alreadyExists") {
 		glog.Warningf("GCE PD %q already exists, reusing", name)
 		return nil
@@ -264,6 +310,76 @@ func (gce *GCECloud) CreateDisk(
 	return err
 }
 
+// CreateRegionalDisk creates a new Regional Persistent Disk, with the specified
+// name & size, replicated to the specified zones. It stores specified tags
+// encoded in JSON in Description field.
+func (gce *GCECloud) CreateRegionalDisk(
+	name string, diskType string, replicaZones sets.String, sizeGb int64, tags map[string]string) error {
+
+	// Do not allow creation of PDs in zones that are not managed. Such PDs
+	// then cannot be deleted by DeleteDisk.
+	unmanagedZones := []string{}
+	for _, zone := range replicaZones.UnsortedList() {
+		if isManaged := gce.verifyZoneIsManaged(zone); !isManaged {
+			unmanagedZones = append(unmanagedZones, zone)
+		}
+	}
+
+	if len(unmanagedZones) > 0 {
+		return fmt.Errorf("kubernetes does not manage specified zones: %q. Managed Zones: %q", unmanagedZones, gce.managedZones)
+	}
+
+	tagsStr, err := gce.encodeDiskTags(tags)
+	if err != nil {
+		return err
+	}
+
+	diskType, err = getDiskType(diskType)
+	if err != nil {
+		return err
+	}
+
+	mc := newDiskMetricContextRegional("create", gce.region)
+
+	createOp, err := gce.manager.CreateRegionalDisk(
+		name, sizeGb, tagsStr, diskType, replicaZones)
+
+	if isGCEError(err, "alreadyExists") {
+		glog.Warningf("GCE PD %q already exists, reusing", name)
+		return nil
+	} else if err != nil {
+		return mc.Observe(err)
+	}
+
+	err = gce.manager.WaitForRegionalOp(createOp, mc)
+	if isGCEError(err, "alreadyExists") {
+		glog.Warningf("GCE PD %q already exists, reusing", name)
+		return nil
+	}
+	return err
+}
+
+func (gce *GCECloud) verifyZoneIsManaged(zone string) bool {
+	for _, managedZone := range gce.managedZones {
+		if zone == managedZone {
+			return true
+		}
+	}
+
+	return false
+}
+
+func getDiskType(diskType string) (string, error) {
+	switch diskType {
+	case DiskTypeSSD, DiskTypeStandard:
+		return diskType, nil
+	case "":
+		return diskTypeDefault, nil
+	default:
+		return "", fmt.Errorf("invalid GCE disk type %q", diskType)
+	}
+}
+
 func (gce *GCECloud) DeleteDisk(diskToDelete string) error {
 	err := gce.doDeleteDisk(diskToDelete)
 	if isGCEError(err, "resourceInUseByAnotherResource") {
@@ -285,40 +401,66 @@ func (gce *GCECloud) GetAutoLabelsForPD(name string, zone string) (map[string]st
 	var disk *GCEDisk
 	var err error
 	if zone == "" {
-		// We would like as far as possible to avoid this case,
-		// because GCE doesn't guarantee that volumes are uniquely named per region,
-		// just per zone.  However, creation of GCE PDs was originally done only
-		// by name, so we have to continue to support that.
-		// However, wherever possible the zone should be passed (and it is passed
-		// for most cases that we can control, e.g. dynamic volume provisioning)
+		// For regional PDs this is fine, but for zonal PDs we would like as far
+		// as possible to avoid this case, because GCE doesn't guarantee that
+		// volumes are uniquely named per region, just per zone. However,
+		// creation of GCE PDs was originally done only by name, so we have to
+		// continue to support that.
+		// However, wherever possible the zone should be passed (and it is
+		// passed for most cases that we can control, e.g. dynamic volume
+		// provisioning).
 		disk, err = gce.GetDiskByNameUnknownZone(name)
 		if err != nil {
 			return nil, err
 		}
-		zone = disk.Zone
 	} else {
 		// We could assume the disks exists; we have all the information we need
 		// However it is more consistent to ensure the disk exists,
 		// and in future we may gather addition information (e.g. disk type, IOPS etc)
-		disk, err = gce.getDiskByName(name, zone)
+		zoneSet, err := volumeutil.LabelZonesToSet(zone)
 		if err != nil {
-			return nil, err
+			glog.Warningf("Failed to parse zone field: %q. Will use raw field.", zone)
+		}
+
+		if len(zoneSet) > 1 {
+			// Regional PD
+			disk, err = gce.getRegionalDiskByName(name)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			// Zonal PD
+			disk, err = gce.getDiskByName(name, zone)
+			if err != nil {
+				return nil, err
+			}
 		}
 	}
 
-	region, err := GetGCERegion(zone)
-	if err != nil {
-		return nil, err
-	}
-
-	if zone == "" || region == "" {
-		// Unexpected, but sanity-check
-		return nil, fmt.Errorf("PD did not have zone/region information: %q", disk.Name)
-	}
-
 	labels := make(map[string]string)
-	labels[kubeletapis.LabelZoneFailureDomain] = zone
-	labels[kubeletapis.LabelZoneRegion] = region
+	switch zoneInfo := disk.ZoneInfo.(type) {
+	case singleZone:
+		if zoneInfo.zone == "" || disk.Region == "" {
+			// Unexpected, but sanity-check
+			return nil, fmt.Errorf("PD did not have zone/region information: %v", disk)
+		}
+		labels[kubeletapis.LabelZoneFailureDomain] = zoneInfo.zone
+		labels[kubeletapis.LabelZoneRegion] = disk.Region
+	case multiZone:
+		if zoneInfo.replicaZones == nil || zoneInfo.replicaZones.Len() <= 0 {
+			// Unexpected, but sanity-check
+			return nil, fmt.Errorf("PD is regional but does not have any replicaZones specified: %v", disk)
+		}
+		labels[kubeletapis.LabelZoneFailureDomain] =
+			volumeutil.ZonesSetToLabelValue(zoneInfo.replicaZones)
+		labels[kubeletapis.LabelZoneRegion] = disk.Region
+	case nil:
+		// Unexpected, but sanity-check
+		return nil, fmt.Errorf("PD did not have ZoneInfo: %v", disk)
+	default:
+		// Unexpected, but sanity-check
+		return nil, fmt.Errorf("disk.ZoneInfo has unexpected type %T", zoneInfo)
+	}
 
 	return labels, nil
 }
@@ -326,16 +468,10 @@ func (gce *GCECloud) GetAutoLabelsForPD(name string, zone string) (map[string]st
 // Returns a GCEDisk for the disk, if it is found in the specified zone.
 // If not found, returns (nil, nil)
 func (gce *GCECloud) findDiskByName(diskName string, zone string) (*GCEDisk, error) {
-	mc := newDiskMetricContext("get", zone)
-	disk, err := gce.manager.GetDisk(gce.projectID, zone, diskName)
+	mc := newDiskMetricContextZonal("get", gce.region, zone)
+	disk, err := gce.manager.GetDisk(zone, diskName)
 	if err == nil {
-		d := &GCEDisk{
-			Zone: lastComponent(disk.Zone),
-			Name: disk.Name,
-			Kind: disk.Kind,
-			Type: disk.Type,
-		}
-		return d, mc.Observe(nil)
+		return disk, mc.Observe(nil)
 	}
 	if !isHTTPErrorCode(err, http.StatusNotFound) {
 		return nil, mc.Observe(err)
@@ -352,10 +488,40 @@ func (gce *GCECloud) getDiskByName(diskName string, zone string) (*GCEDisk, erro
 	return disk, err
 }
 
+// Returns a GCEDisk for the regional disk, if it is found.
+// If not found, returns (nil, nil)
+func (gce *GCECloud) findRegionalDiskByName(diskName string) (*GCEDisk, error) {
+	mc := newDiskMetricContextRegional("get", gce.region)
+	disk, err := gce.manager.GetRegionalDisk(diskName)
+	if err == nil {
+		return disk, mc.Observe(nil)
+	}
+	if !isHTTPErrorCode(err, http.StatusNotFound) {
+		return nil, mc.Observe(err)
+	}
+	return nil, mc.Observe(nil)
+}
+
+// Like findRegionalDiskByName, but returns an error if the disk is not found
+func (gce *GCECloud) getRegionalDiskByName(diskName string) (*GCEDisk, error) {
+	disk, err := gce.findRegionalDiskByName(diskName)
+	if disk == nil && err == nil {
+		return nil, fmt.Errorf("GCE regional persistent disk not found: diskName=%q", diskName)
+	}
+	return disk, err
+}
+
 // Scans all managed zones to return the GCE PD
 // Prefer getDiskByName, if the zone can be established
 // Return cloudprovider.DiskNotFound if the given disk cannot be found in any zone
 func (gce *GCECloud) GetDiskByNameUnknownZone(diskName string) (*GCEDisk, error) {
+	if gce.AlphaFeatureGate.Enabled(GCEDiskAlphaFeatureGate) {
+		regionalDisk, err := gce.getRegionalDiskByName(diskName)
+		if err == nil {
+			return regionalDisk, err
+		}
+	}
+
 	// Note: this is the gotcha right now with GCE PD support:
 	// disk names are not unique per-region.
 	// (I can create two volumes with name "myvol" in e.g. us-central1-b & us-central1-f)
@@ -378,7 +544,17 @@ func (gce *GCECloud) GetDiskByNameUnknownZone(diskName string) (*GCEDisk, error)
 			continue
 		}
 		if found != nil {
-			return nil, fmt.Errorf("GCE persistent disk name was found in multiple zones: %q", diskName)
+			switch zoneInfo := disk.ZoneInfo.(type) {
+			case multiZone:
+				if zoneInfo.replicaZones.Has(zone) {
+					glog.Warningf("GCE PD name (%q) was found in multiple zones (%q), but ok because it is a RegionalDisk.",
+						diskName, zoneInfo.replicaZones)
+					continue
+				}
+				return nil, fmt.Errorf("GCE PD name was found in multiple zones: %q", diskName)
+			default:
+				return nil, fmt.Errorf("GCE PD name was found in multiple zones: %q", diskName)
+			}
 		}
 		found = disk
 	}
@@ -412,25 +588,27 @@ func (gce *GCECloud) doDeleteDisk(diskToDelete string) error {
 		return err
 	}
 
-	mc := newDiskMetricContext("delete", disk.Zone)
+	var mc *metricContext
 
-	deleteOp, err := gce.manager.DeleteDisk(gce.projectID, disk.Zone, disk.Name)
-	if err != nil {
-		return mc.Observe(err)
-	}
-
-	return gce.manager.WaitForZoneOp(deleteOp, disk.Zone, mc)
-}
-
-// Converts a Disk resource to an AttachedDisk resource.
-func (gce *GCECloud) convertDiskToAttachedDisk(disk *GCEDisk, readWrite string) *compute.AttachedDisk {
-	return &compute.AttachedDisk{
-		DeviceName: disk.Name,
-		Kind:       disk.Kind,
-		Mode:       readWrite,
-		Source: gce.service.BasePath + strings.Join([]string{
-			gce.projectID, "zones", disk.Zone, "disks", disk.Name}, "/"),
-		Type: "PERSISTENT",
+	switch zoneInfo := disk.ZoneInfo.(type) {
+	case singleZone:
+		mc = newDiskMetricContextZonal("delete", disk.Region, zoneInfo.zone)
+		deleteOp, err := gce.manager.DeleteDisk(zoneInfo.zone, disk.Name)
+		if err != nil {
+			return mc.Observe(err)
+		}
+		return gce.manager.WaitForZoneOp(deleteOp, zoneInfo.zone, mc)
+	case multiZone:
+		mc = newDiskMetricContextRegional("delete", disk.Region)
+		deleteOp, err := gce.manager.DeleteRegionalDisk(disk.Name)
+		if err != nil {
+			return mc.Observe(err)
+		}
+		return gce.manager.WaitForRegionalOp(deleteOp, mc)
+	case nil:
+		return fmt.Errorf("PD has nil ZoneInfo: %v", disk)
+	default:
+		return fmt.Errorf("disk.ZoneInfo has unexpected type %T", zoneInfo)
 	}
 }
 

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_firewall.go

@@ -17,53 +17,48 @@ limitations under the License.
 package gce
 
 import (
-	"time"
-
 	compute "google.golang.org/api/compute/v1"
 )
 
 func newFirewallMetricContext(request string) *metricContext {
-	return &metricContext{
-		start:      time.Now(),
-		attributes: []string{"firewall_" + request, unusedMetricLabel, unusedMetricLabel},
-	}
+	return newGenericMetricContext("firewall", request, unusedMetricLabel, unusedMetricLabel, computeV1Version)
 }
 
 // GetFirewall returns the Firewall by name.
 func (gce *GCECloud) GetFirewall(name string) (*compute.Firewall, error) {
 	mc := newFirewallMetricContext("get")
-	v, err := gce.service.Firewalls.Get(gce.projectID, name).Do()
+	v, err := gce.service.Firewalls.Get(gce.NetworkProjectID(), name).Do()
 	return v, mc.Observe(err)
 }
 
 // CreateFirewall creates the passed firewall
 func (gce *GCECloud) CreateFirewall(f *compute.Firewall) error {
 	mc := newFirewallMetricContext("create")
-	op, err := gce.service.Firewalls.Insert(gce.projectID, f).Do()
+	op, err := gce.service.Firewalls.Insert(gce.NetworkProjectID(), f).Do()
 	if err != nil {
 		return mc.Observe(err)
 	}
 
-	return gce.waitForGlobalOp(op, mc)
+	return gce.waitForGlobalOpInProject(op, gce.NetworkProjectID(), mc)
 }
 
 // DeleteFirewall deletes the given firewall rule.
 func (gce *GCECloud) DeleteFirewall(name string) error {
 	mc := newFirewallMetricContext("delete")
-	op, err := gce.service.Firewalls.Delete(gce.projectID, name).Do()
+	op, err := gce.service.Firewalls.Delete(gce.NetworkProjectID(), name).Do()
 	if err != nil {
 		return mc.Observe(err)
 	}
-	return gce.waitForGlobalOp(op, mc)
+	return gce.waitForGlobalOpInProject(op, gce.NetworkProjectID(), mc)
 }
 
 // UpdateFirewall applies the given firewall as an update to an existing service.
 func (gce *GCECloud) UpdateFirewall(f *compute.Firewall) error {
 	mc := newFirewallMetricContext("update")
-	op, err := gce.service.Firewalls.Update(gce.projectID, f.Name, f).Do()
+	op, err := gce.service.Firewalls.Update(gce.NetworkProjectID(), f.Name, f).Do()
 	if err != nil {
 		return mc.Observe(err)
 	}
 
-	return gce.waitForGlobalOp(op, mc)
+	return gce.waitForGlobalOpInProject(op, gce.NetworkProjectID(), mc)
 }

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_forwardingrule.go

@@ -17,16 +17,15 @@ limitations under the License.
 package gce
 
 import (
-	"time"
-
+	computealpha "google.golang.org/api/compute/v0.alpha"
 	compute "google.golang.org/api/compute/v1"
 )
 
 func newForwardingRuleMetricContext(request, region string) *metricContext {
-	return &metricContext{
-		start:      time.Now(),
-		attributes: []string{"forwardingrule_" + request, region, unusedMetricLabel},
-	}
+	return newForwardingRuleMetricContextWithVersion(request, region, computeV1Version)
+}
+func newForwardingRuleMetricContextWithVersion(request, region, version string) *metricContext {
+	return newGenericMetricContext("forwardingrule", request, region, unusedMetricLabel, version)
 }
 
 // CreateGlobalForwardingRule creates the passed GlobalForwardingRule
@@ -85,6 +84,13 @@ func (gce *GCECloud) GetRegionForwardingRule(name, region string) (*compute.Forw
 	return v, mc.Observe(err)
 }
 
+// GetAlphaRegionForwardingRule returns the Alpha forwarding rule by name & region.
+func (gce *GCECloud) GetAlphaRegionForwardingRule(name, region string) (*computealpha.ForwardingRule, error) {
+	mc := newForwardingRuleMetricContextWithVersion("get", region, computeAlphaVersion)
+	v, err := gce.serviceAlpha.ForwardingRules.Get(gce.projectID, region, name).Do()
+	return v, mc.Observe(err)
+}
+
 // ListRegionForwardingRules lists all RegionalForwardingRules in the project & region.
 func (gce *GCECloud) ListRegionForwardingRules(region string) (*compute.ForwardingRuleList, error) {
 	mc := newForwardingRuleMetricContext("list", region)
@@ -93,6 +99,14 @@ func (gce *GCECloud) ListRegionForwardingRules(region string) (*compute.Forwardi
 	return v, mc.Observe(err)
 }
 
+// ListRegionForwardingRules lists all RegionalForwardingRules in the project & region.
+func (gce *GCECloud) ListAlphaRegionForwardingRules(region string) (*computealpha.ForwardingRuleList, error) {
+	mc := newForwardingRuleMetricContextWithVersion("list", region, computeAlphaVersion)
+	// TODO: use PageToken to list all not just the first 500
+	v, err := gce.serviceAlpha.ForwardingRules.List(gce.projectID, region).Do()
+	return v, mc.Observe(err)
+}
+
 // CreateRegionForwardingRule creates and returns a
 // RegionalForwardingRule that points to the given BackendService
 func (gce *GCECloud) CreateRegionForwardingRule(rule *compute.ForwardingRule, region string) error {
@@ -105,6 +119,18 @@ func (gce *GCECloud) CreateRegionForwardingRule(rule *compute.ForwardingRule, re
 	return gce.waitForRegionOp(op, region, mc)
 }
 
+// CreateAlphaRegionForwardingRule creates and returns an Alpha
+// forwarding fule in the given region.
+func (gce *GCECloud) CreateAlphaRegionForwardingRule(rule *computealpha.ForwardingRule, region string) error {
+	mc := newForwardingRuleMetricContextWithVersion("create", region, computeAlphaVersion)
+	op, err := gce.serviceAlpha.ForwardingRules.Insert(gce.projectID, region, rule).Do()
+	if err != nil {
+		return mc.Observe(err)
+	}
+
+	return gce.waitForRegionOp(op, region, mc)
+}
+
 // DeleteRegionForwardingRule deletes the RegionalForwardingRule by name & region.
 func (gce *GCECloud) DeleteRegionForwardingRule(name, region string) error {
 	mc := newForwardingRuleMetricContext("delete", region)
@@ -115,3 +141,15 @@ func (gce *GCECloud) DeleteRegionForwardingRule(name, region string) error {
 
 	return gce.waitForRegionOp(op, region, mc)
 }
+
+// TODO(#51665): retire this function once Network Tiers becomes Beta in GCP.
+func (gce *GCECloud) getNetworkTierFromForwardingRule(name, region string) (string, error) {
+	if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) {
+		return NetworkTierDefault.ToGCEValue(), nil
+	}
+	fwdRule, err := gce.GetAlphaRegionForwardingRule(name, region)
+	if err != nil {
+		return handleAlphaNetworkTierGetError(err)
+	}
+	return fwdRule.NetworkTier, nil
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_forwardingrule_fakes.go

@@ -0,0 +1,138 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	computealpha "google.golang.org/api/compute/v0.alpha"
+	compute "google.golang.org/api/compute/v1"
+	"google.golang.org/api/googleapi"
+)
+
+type FakeCloudForwardingRuleService struct {
+	// fwdRulesByRegionAndName
+	// Outer key is for region string; inner key is for fwdRuleess name.
+	fwdRulesByRegionAndName map[string]map[string]*computealpha.ForwardingRule
+}
+
+// FakeCloudForwardingRuleService Implements CloudForwardingRuleService
+var _ CloudForwardingRuleService = &FakeCloudForwardingRuleService{}
+
+func NewFakeCloudForwardingRuleService() *FakeCloudForwardingRuleService {
+	return &FakeCloudForwardingRuleService{
+		fwdRulesByRegionAndName: make(map[string]map[string]*computealpha.ForwardingRule),
+	}
+}
+
+// SetRegionalForwardingRulees sets the fwdRuleesses of ther region. This is used for
+// setting the test environment.
+func (f *FakeCloudForwardingRuleService) SetRegionalForwardingRulees(region string, fwdRules []*computealpha.ForwardingRule) {
+	// Reset fwdRuleesses in the region.
+	f.fwdRulesByRegionAndName[region] = make(map[string]*computealpha.ForwardingRule)
+
+	for _, fwdRule := range fwdRules {
+		f.fwdRulesByRegionAndName[region][fwdRule.Name] = fwdRule
+	}
+}
+
+func (f *FakeCloudForwardingRuleService) CreateAlphaRegionForwardingRule(fwdRule *computealpha.ForwardingRule, region string) error {
+	if _, exists := f.fwdRulesByRegionAndName[region]; !exists {
+		f.fwdRulesByRegionAndName[region] = make(map[string]*computealpha.ForwardingRule)
+	}
+
+	if _, exists := f.fwdRulesByRegionAndName[region][fwdRule.Name]; exists {
+		return &googleapi.Error{Code: http.StatusConflict}
+	}
+
+	f.fwdRulesByRegionAndName[region][fwdRule.Name] = fwdRule
+	return nil
+}
+
+func (f *FakeCloudForwardingRuleService) CreateRegionForwardingRule(fwdRule *compute.ForwardingRule, region string) error {
+	alphafwdRule := convertToAlphaForwardingRule(fwdRule)
+	return f.CreateAlphaRegionForwardingRule(alphafwdRule, region)
+}
+
+func (f *FakeCloudForwardingRuleService) DeleteRegionForwardingRule(name, region string) error {
+	if _, exists := f.fwdRulesByRegionAndName[region]; !exists {
+		return makeGoogleAPINotFoundError("")
+	}
+
+	if _, exists := f.fwdRulesByRegionAndName[region][name]; !exists {
+		return makeGoogleAPINotFoundError("")
+	}
+	delete(f.fwdRulesByRegionAndName[region], name)
+	return nil
+}
+
+func (f *FakeCloudForwardingRuleService) GetAlphaRegionForwardingRule(name, region string) (*computealpha.ForwardingRule, error) {
+	if _, exists := f.fwdRulesByRegionAndName[region]; !exists {
+		return nil, makeGoogleAPINotFoundError("")
+	}
+
+	if fwdRule, exists := f.fwdRulesByRegionAndName[region][name]; !exists {
+		return nil, makeGoogleAPINotFoundError("")
+	} else {
+		return fwdRule, nil
+	}
+}
+
+func (f *FakeCloudForwardingRuleService) GetRegionForwardingRule(name, region string) (*compute.ForwardingRule, error) {
+	fwdRule, err := f.GetAlphaRegionForwardingRule(name, region)
+	if fwdRule != nil {
+		return convertToV1ForwardingRule(fwdRule), err
+	}
+	return nil, err
+}
+
+func (f *FakeCloudForwardingRuleService) getNetworkTierFromForwardingRule(name, region string) (string, error) {
+	fwdRule, err := f.GetAlphaRegionForwardingRule(name, region)
+	if err != nil {
+		return "", err
+	}
+	return fwdRule.NetworkTier, nil
+}
+
+func convertToV1ForwardingRule(object gceObject) *compute.ForwardingRule {
+	enc, err := object.MarshalJSON()
+	if err != nil {
+		panic(fmt.Sprintf("Failed to encode to json: %v", err))
+	}
+	var fwdRule compute.ForwardingRule
+	if err := json.Unmarshal(enc, &fwdRule); err != nil {
+		panic(fmt.Sprintf("Failed to convert GCE apiObject %v to v1 fwdRuleess: %v", object, err))
+	}
+	return &fwdRule
+}
+
+func convertToAlphaForwardingRule(object gceObject) *computealpha.ForwardingRule {
+	enc, err := object.MarshalJSON()
+	if err != nil {
+		panic(fmt.Sprintf("Failed to encode to json: %v", err))
+	}
+	var fwdRule computealpha.ForwardingRule
+	if err := json.Unmarshal(enc, &fwdRule); err != nil {
+		panic(fmt.Sprintf("Failed to convert GCE apiObject %v to alpha fwdRuleess: %v", object, err))
+	}
+	// Set the default values for the Alpha fields.
+	fwdRule.NetworkTier = NetworkTierDefault.ToGCEValue()
+
+	return &fwdRule
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_healthchecks.go

@@ -17,8 +17,6 @@ limitations under the License.
 package gce
 
 import (
-	"time"
-
 	"k8s.io/api/core/v1"
 	"k8s.io/kubernetes/pkg/master/ports"
 	utilversion "k8s.io/kubernetes/pkg/util/version"
@@ -45,10 +43,7 @@ func init() {
 }
 
 func newHealthcheckMetricContext(request string) *metricContext {
-	return &metricContext{
-		start:      time.Now(),
-		attributes: []string{"healthcheck_" + request, unusedMetricLabel, unusedMetricLabel},
-	}
+	return newGenericMetricContext("healthcheck", request, unusedMetricLabel, unusedMetricLabel, computeV1Version)
 }
 
 // GetHttpHealthCheck returns the given HttpHealthCheck by name.

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_instancegroup.go

@@ -16,17 +16,10 @@ limitations under the License.
 
 package gce
 
-import (
-	"time"
-
-	compute "google.golang.org/api/compute/v1"
-)
+import compute "google.golang.org/api/compute/v1"
 
 func newInstanceGroupMetricContext(request string, zone string) *metricContext {
-	return &metricContext{
-		start:      time.Now(),
-		attributes: []string{"instancegroup_" + request, unusedMetricLabel, zone},
-	}
+	return newGenericMetricContext("instancegroup", request, unusedMetricLabel, zone, computeV1Version)
 }
 
 // CreateInstanceGroup creates an instance group with the given

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_instances.go

@@ -17,7 +17,9 @@ limitations under the License.
 package gce
 
 import (
+	"errors"
 	"fmt"
+	"net"
 	"net/http"
 	"strconv"
 	"strings"
@@ -25,6 +27,7 @@ import (
 
 	"cloud.google.com/go/compute/metadata"
 	"github.com/golang/glog"
+	computealpha "google.golang.org/api/compute/v0.alpha"
 	computebeta "google.golang.org/api/compute/v0.beta"
 	compute "google.golang.org/api/compute/v1"
 
@@ -41,10 +44,7 @@ const (
 )
 
 func newInstancesMetricContext(request, zone string) *metricContext {
-	return &metricContext{
-		start:      time.Now(),
-		attributes: []string{"instances_" + request, unusedMetricLabel, zone},
-	}
+	return newGenericMetricContext("instances", request, unusedMetricLabel, zone, computeV1Version)
 }
 
 func splitNodesByZone(nodes []*v1.Node) map[string][]*v1.Node {
@@ -154,6 +154,12 @@ func (gce *GCECloud) ExternalID(nodeName types.NodeName) (string, error) {
 	return strconv.FormatUint(inst.ID, 10), nil
 }
 
+// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running.
+// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager.
+func (gce *GCECloud) InstanceExistsByProviderID(providerID string) (bool, error) {
+	return false, errors.New("unimplemented")
+}
+
 // InstanceID returns the cloud provider ID of the node with the specified NodeName.
 func (gce *GCECloud) InstanceID(nodeName types.NodeName) (string, error) {
 	instanceName := mapNodeNameToInstanceName(nodeName)
@@ -318,6 +324,43 @@ func (gce *GCECloud) AliasRanges(nodeName types.NodeName) (cidrs []string, err e
 	return
 }
 
+// AddAliasToInstance adds an alias to the given instance from the named
+// secondary range.
+func (gce *GCECloud) AddAliasToInstance(nodeName types.NodeName, alias *net.IPNet) error {
+
+	v1instance, err := gce.getInstanceByName(mapNodeNameToInstanceName(nodeName))
+	if err != nil {
+		return err
+	}
+	instance, err := gce.serviceAlpha.Instances.Get(gce.projectID, v1instance.Zone, v1instance.Name).Do()
+	if err != nil {
+		return err
+	}
+
+	switch len(instance.NetworkInterfaces) {
+	case 0:
+		return fmt.Errorf("Instance %q has no network interfaces", nodeName)
+	case 1:
+	default:
+		glog.Warningf("Instance %q has more than one network interface, using only the first (%v)",
+			nodeName, instance.NetworkInterfaces)
+	}
+
+	iface := instance.NetworkInterfaces[0]
+	iface.AliasIpRanges = append(iface.AliasIpRanges, &computealpha.AliasIpRange{
+		IpCidrRange:         alias.String(),
+		SubnetworkRangeName: gce.secondaryRangeName,
+	})
+
+	mc := newInstancesMetricContext("addalias", v1instance.Zone)
+	op, err := gce.serviceAlpha.Instances.UpdateNetworkInterface(
+		gce.projectID, instance.Zone, instance.Name, iface.Name, iface).Do()
+	if err != nil {
+		return mc.Observe(err)
+	}
+	return gce.waitForZoneOp(op, v1instance.Zone, mc)
+}
+
 // Gets the named instances, returning cloudprovider.InstanceNotFound if any instance is not found
 func (gce *GCECloud) getInstancesByNames(names []string) ([]*gceInstance, error) {
 	instances := make(map[string]*gceInstance)

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_interfaces.go

@@ -16,12 +16,46 @@ limitations under the License.
 
 package gce
 
-import compute "google.golang.org/api/compute/v1"
+import (
+	computealpha "google.golang.org/api/compute/v0.alpha"
+	computebeta "google.golang.org/api/compute/v0.beta"
+	compute "google.golang.org/api/compute/v1"
+)
+
+// These interfaces are added for testability.
 
 // CloudAddressService is an interface for managing addresses
 type CloudAddressService interface {
-	ReserveRegionAddress(*compute.Address, string) error
-	GetRegionAddress(string, string) (*compute.Address, error)
-	// TODO: Mock `DeleteRegionAddress(name, region string) endpoint
+	ReserveRegionAddress(address *compute.Address, region string) error
+	GetRegionAddress(name string, region string) (*compute.Address, error)
+	GetRegionAddressByIP(region, ipAddress string) (*compute.Address, error)
+	DeleteRegionAddress(name, region string) error
 	// TODO: Mock Global endpoints
+
+	// Alpha API.
+	GetAlphaRegionAddress(name, region string) (*computealpha.Address, error)
+	ReserveAlphaRegionAddress(addr *computealpha.Address, region string) error
+
+	// Beta API
+	ReserveBetaRegionAddress(address *computebeta.Address, region string) error
+	GetBetaRegionAddress(name string, region string) (*computebeta.Address, error)
+	GetBetaRegionAddressByIP(region, ipAddress string) (*computebeta.Address, error)
+
+	// TODO(#51665): Remove this once the Network Tiers becomes Alpha in GCP.
+	getNetworkTierFromAddress(name, region string) (string, error)
+}
+
+// CloudForwardingRuleService is an interface for managing forwarding rules.
+// TODO: Expand the interface to include more methods.
+type CloudForwardingRuleService interface {
+	GetRegionForwardingRule(name, region string) (*compute.ForwardingRule, error)
+	CreateRegionForwardingRule(rule *compute.ForwardingRule, region string) error
+	DeleteRegionForwardingRule(name, region string) error
+
+	// Alpha API.
+	GetAlphaRegionForwardingRule(name, region string) (*computealpha.ForwardingRule, error)
+	CreateAlphaRegionForwardingRule(rule *computealpha.ForwardingRule, region string) error
+
+	// Needed for the Alpha "Network Tiers" feature.
+	getNetworkTierFromForwardingRule(name, region string) (string, error)
 }

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_loadbalancer.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"net"
 	"strings"
-	"time"
 
 	"github.com/golang/glog"
 
@@ -40,10 +39,7 @@ var (
 )
 
 func newLoadBalancerMetricContext(request, region string) *metricContext {
-	return &metricContext{
-		start:      time.Now(),
-		attributes: []string{"loadbalancer_" + request, region, unusedMetricLabel},
-	}
+	return newGenericMetricContext("loadbalancer", request, region, unusedMetricLabel, computeV1Version)
 }
 
 type lbScheme string
@@ -141,6 +137,9 @@ func (gce *GCECloud) EnsureLoadBalancer(clusterName string, svc *v1.Service, nod
 			if err != nil {
 				return nil, err
 			}
+
+			// Assume the ensureDeleted function successfully deleted the forwarding rule.
+			existingFwdRule = nil
 		}
 	}
 

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go

@@ -31,6 +31,7 @@ import (
 	netsets "k8s.io/kubernetes/pkg/util/net/sets"
 
 	"github.com/golang/glog"
+	computealpha "google.golang.org/api/compute/v0.alpha"
 	compute "google.golang.org/api/compute/v1"
 )
 
@@ -55,7 +56,7 @@ func (gce *GCECloud) ensureExternalLoadBalancer(clusterName, clusterID string, a
 	}
 
 	loadBalancerName := cloudprovider.GetLoadBalancerName(apiService)
-	loadBalancerIP := apiService.Spec.LoadBalancerIP
+	requestedIP := apiService.Spec.LoadBalancerIP
 	ports := apiService.Spec.Ports
 	portStr := []string{}
 	for _, p := range apiService.Spec.Ports {
@@ -66,10 +67,23 @@ func (gce *GCECloud) ensureExternalLoadBalancer(clusterName, clusterID string, a
 
 	serviceName := types.NamespacedName{Namespace: apiService.Namespace, Name: apiService.Name}
 	glog.V(2).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)",
-		loadBalancerName, gce.region, loadBalancerIP, portStr, hostNames, serviceName, apiService.Annotations)
+		loadBalancerName, gce.region, requestedIP, portStr, hostNames, serviceName, apiService.Annotations)
+
+	lbRefStr := fmt.Sprintf("%v(%v)", loadBalancerName, serviceName)
+	// Check the current and the desired network tiers. If they do not match,
+	// tear down the existing resources with the wrong tier.
+	netTier, err := gce.getServiceNetworkTier(apiService)
+	if err != nil {
+		glog.Errorf("EnsureLoadBalancer(%s): failed to get the desired network tier: %v", lbRefStr, err)
+		return nil, err
+	}
+	glog.V(4).Infof("EnsureLoadBalancer(%s): desired network tier %q ", lbRefStr, netTier)
+	if gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) {
+		gce.deleteWrongNetworkTieredResources(loadBalancerName, lbRefStr, netTier)
+	}
 
 	// Check if the forwarding rule exists, and if so, what its IP is.
-	fwdRuleExists, fwdRuleNeedsUpdate, fwdRuleIP, err := gce.forwardingRuleNeedsUpdate(loadBalancerName, gce.region, loadBalancerIP, ports)
+	fwdRuleExists, fwdRuleNeedsUpdate, fwdRuleIP, err := gce.forwardingRuleNeedsUpdate(loadBalancerName, gce.region, requestedIP, ports)
 	if err != nil {
 		return nil, err
 	}
@@ -93,7 +107,7 @@ func (gce *GCECloud) ensureExternalLoadBalancer(clusterName, clusterID string, a
 	// forwarding rule creation as the last thing that needs to be done in this
 	// function in order to maintain the invariant that "if the forwarding rule
 	// exists, the LB has been fully created".
-	ipAddress := ""
+	ipAddressToUse := ""
 
 	// Through this process we try to keep track of whether it is safe to
 	// release the IP that was allocated.  If the user specifically asked for
@@ -110,75 +124,41 @@ func (gce *GCECloud) ensureExternalLoadBalancer(clusterName, clusterID string, a
 		}
 		if isSafeToReleaseIP {
 			if err := gce.DeleteRegionAddress(loadBalancerName, gce.region); err != nil && !isNotFound(err) {
-				glog.Errorf("failed to release static IP %s for load balancer (%v(%v), %v): %v", ipAddress, loadBalancerName, serviceName, gce.region, err)
+				glog.Errorf("Failed to release static IP %s for load balancer (%v(%v), %v): %v", ipAddressToUse, loadBalancerName, serviceName, gce.region, err)
 			} else if isNotFound(err) {
-				glog.V(2).Infof("EnsureLoadBalancer(%v(%v)): address %s is not reserved.", loadBalancerName, serviceName, ipAddress)
+				glog.V(2).Infof("EnsureLoadBalancer(%v(%v)): address %s is not reserved.", loadBalancerName, serviceName, ipAddressToUse)
 			} else {
-				glog.V(2).Infof("EnsureLoadBalancer(%v(%v)): released static IP %s", loadBalancerName, serviceName, ipAddress)
+				glog.V(2).Infof("EnsureLoadBalancer(%v(%v)): released static IP %s", loadBalancerName, serviceName, ipAddressToUse)
 			}
 		} else {
-			glog.Warningf("orphaning static IP %s during update of load balancer (%v(%v), %v): %v", ipAddress, loadBalancerName, serviceName, gce.region, err)
+			glog.Warningf("orphaning static IP %s during update of load balancer (%v(%v), %v): %v", ipAddressToUse, loadBalancerName, serviceName, gce.region, err)
 		}
 	}()
 
-	if loadBalancerIP != "" {
-		// If a specific IP address has been requested, we have to respect the
-		// user's request and use that IP. If the forwarding rule was already using
-		// a different IP, it will be harmlessly abandoned because it was only an
-		// ephemeral IP (or it was a different static IP owned by the user, in which
-		// case we shouldn't delete it anyway).
-		if isStatic, err := gce.projectOwnsStaticIP(loadBalancerName, gce.region, loadBalancerIP); err != nil {
-			return nil, fmt.Errorf("failed to test if this GCE project owns the static IP %s: %v", loadBalancerIP, err)
-		} else if isStatic {
-			// The requested IP is a static IP, owned and managed by the user.
-			isUserOwnedIP = true
-			isSafeToReleaseIP = false
-			ipAddress = loadBalancerIP
-			glog.V(4).Infof("EnsureLoadBalancer(%v(%v)): using user-provided static IP %s", loadBalancerName, serviceName, ipAddress)
-		} else if loadBalancerIP == fwdRuleIP {
-			// The requested IP is not a static IP, but is currently assigned
-			// to this forwarding rule, so we can keep it.
-			isUserOwnedIP = false
-			isSafeToReleaseIP = true
-			ipAddress, _, err = ensureStaticIP(gce, loadBalancerName, serviceName.String(), gce.region, fwdRuleIP)
-			if err != nil {
-				return nil, fmt.Errorf("failed to ensure static IP %s: %v", fwdRuleIP, err)
-			}
-			glog.V(4).Infof("EnsureLoadBalancer(%v(%v)): using user-provided non-static IP %s", loadBalancerName, serviceName, ipAddress)
-		} else {
-			// The requested IP is not static and it is not assigned to the
-			// current forwarding rule.  It might be attached to a different
-			// rule or it might not be part of this project at all.  Either
-			// way, we can't use it.
-			return nil, fmt.Errorf("requested ip %s is neither static nor assigned to LB %s(%v): %v", loadBalancerIP, loadBalancerName, serviceName, err)
-		}
-	} else {
-		// The user did not request a specific IP.
-		isUserOwnedIP = false
-
-		// This will either allocate a new static IP if the forwarding rule didn't
-		// already have an IP, or it will promote the forwarding rule's current
-		// IP from ephemeral to static, or it will just get the IP if it is
-		// already static.
-		existed := false
-		ipAddress, existed, err = ensureStaticIP(gce, loadBalancerName, serviceName.String(), gce.region, fwdRuleIP)
+	if requestedIP != "" {
+		// If user requests a specific IP address, verify first. No mutation to
+		// the GCE resources will be performed in the verification process.
+		isUserOwnedIP, err = verifyUserRequestedIP(gce, gce.region, requestedIP, fwdRuleIP, lbRefStr, netTier)
 		if err != nil {
-			return nil, fmt.Errorf("failed to ensure static IP %s: %v", fwdRuleIP, err)
+			return nil, err
 		}
-		if existed {
-			// If the IP was not specifically requested by the user, but it
-			// already existed, it seems to be a failed update cycle.  We can
-			// use this IP and try to run through the process again, but we
-			// should not release the IP unless it is explicitly flagged as OK.
-			isSafeToReleaseIP = false
-			glog.V(4).Infof("EnsureLoadBalancer(%v(%v)): adopting static IP %s", loadBalancerName, serviceName, ipAddress)
-		} else {
-			// For total clarity.  The IP did not pre-exist and the user did
-			// not ask for a particular one, so we can release the IP in case
-			// of failure or success.
-			isSafeToReleaseIP = true
-			glog.V(4).Infof("EnsureLoadBalancer(%v(%v)): allocated static IP %s", loadBalancerName, serviceName, ipAddress)
+		ipAddressToUse = requestedIP
+	}
+
+	if !isUserOwnedIP {
+		// If we are not using the user-owned IP, either promote the
+		// emphemeral IP used by the fwd rule, or create a new static IP.
+		ipAddr, existed, err := ensureStaticIP(gce, loadBalancerName, serviceName.String(), gce.region, fwdRuleIP, netTier)
+		if err != nil {
+			return nil, fmt.Errorf("failed to ensure a static IP for the LB: %v", err)
 		}
+		glog.V(4).Infof("EnsureLoadBalancer(%s): ensured IP address %s (tier: %s)", lbRefStr, ipAddr, netTier)
+		// If the IP was not owned by the user, but it already existed, it
+		// could indicate that the previous update cycle failed. We can use
+		// this IP and try to run through the process again, but we should
+		// not release the IP unless it is explicitly flagged as OK.
+		isSafeToReleaseIP = !existed
+		ipAddressToUse = ipAddr
 	}
 
 	// Deal with the firewall next. The reason we do this here rather than last
@@ -190,24 +170,24 @@ func (gce *GCECloud) ensureExternalLoadBalancer(clusterName, clusterID string, a
 		return nil, err
 	}
 
-	firewallExists, firewallNeedsUpdate, err := gce.firewallNeedsUpdate(loadBalancerName, serviceName.String(), gce.region, ipAddress, ports, sourceRanges)
+	firewallExists, firewallNeedsUpdate, err := gce.firewallNeedsUpdate(loadBalancerName, serviceName.String(), gce.region, ipAddressToUse, ports, sourceRanges)
 	if err != nil {
 		return nil, err
 	}
 
 	if firewallNeedsUpdate {
-		desc := makeFirewallDescription(serviceName.String(), ipAddress)
+		desc := makeFirewallDescription(serviceName.String(), ipAddressToUse)
 		// Unlike forwarding rules and target pools, firewalls can be updated
 		// without needing to be deleted and recreated.
 		if firewallExists {
 			glog.Infof("EnsureLoadBalancer(%v(%v)): updating firewall", loadBalancerName, serviceName)
-			if err := gce.updateFirewall(makeFirewallName(loadBalancerName), gce.region, desc, sourceRanges, ports, hosts); err != nil {
+			if err := gce.updateFirewall(apiService, makeFirewallName(loadBalancerName), gce.region, desc, sourceRanges, ports, hosts); err != nil {
 				return nil, err
 			}
 			glog.Infof("EnsureLoadBalancer(%v(%v)): updated firewall", loadBalancerName, serviceName)
 		} else {
 			glog.Infof("EnsureLoadBalancer(%v(%v)): creating firewall", loadBalancerName, serviceName)
-			if err := gce.createFirewall(makeFirewallName(loadBalancerName), gce.region, desc, sourceRanges, ports, hosts); err != nil {
+			if err := gce.createFirewall(apiService, makeFirewallName(loadBalancerName), gce.region, desc, sourceRanges, ports, hosts); err != nil {
 				return nil, err
 			}
 			glog.Infof("EnsureLoadBalancer(%v(%v)): created firewall", loadBalancerName, serviceName)
@@ -279,7 +259,7 @@ func (gce *GCECloud) ensureExternalLoadBalancer(clusterName, clusterID string, a
 		if hcToDelete != nil {
 			hcNames = append(hcNames, hcToDelete.Name)
 		}
-		if err := gce.DeleteExternalTargetPoolAndChecks(loadBalancerName, gce.region, clusterID, hcNames...); err != nil {
+		if err := gce.DeleteExternalTargetPoolAndChecks(apiService, loadBalancerName, gce.region, clusterID, hcNames...); err != nil {
 			return nil, fmt.Errorf("failed to delete existing target pool %s for load balancer update: %v", loadBalancerName, err)
 		}
 		glog.Infof("EnsureLoadBalancer(%v(%v)): deleted target pool", loadBalancerName, serviceName)
@@ -293,7 +273,7 @@ func (gce *GCECloud) ensureExternalLoadBalancer(clusterName, clusterID string, a
 			createInstances = createInstances[:maxTargetPoolCreateInstances]
 		}
 		// Pass healthchecks to createTargetPool which needs them as health check links in the target pool
-		if err := gce.createTargetPool(loadBalancerName, serviceName.String(), ipAddress, gce.region, clusterID, createInstances, affinityType, hcToCreate); err != nil {
+		if err := gce.createTargetPool(apiService, loadBalancerName, serviceName.String(), ipAddressToUse, gce.region, clusterID, createInstances, affinityType, hcToCreate); err != nil {
 			return nil, fmt.Errorf("failed to create target pool %s: %v", loadBalancerName, err)
 		}
 		if hcToCreate != nil {
@@ -315,8 +295,8 @@ func (gce *GCECloud) ensureExternalLoadBalancer(clusterName, clusterID string, a
 		}
 	}
 	if tpNeedsUpdate || fwdRuleNeedsUpdate {
-		glog.Infof("EnsureLoadBalancer(%v(%v)): creating forwarding rule, IP %s", loadBalancerName, serviceName, ipAddress)
-		if err := gce.createForwardingRule(loadBalancerName, serviceName.String(), gce.region, ipAddress, ports); err != nil {
+		glog.Infof("EnsureLoadBalancer(%v(%v)): creating forwarding rule, IP %s (tier: %s)", loadBalancerName, serviceName, ipAddressToUse, netTier)
+		if err := createForwardingRule(gce, loadBalancerName, serviceName.String(), gce.region, ipAddressToUse, gce.targetPoolURL(loadBalancerName), ports, netTier); err != nil {
 			return nil, fmt.Errorf("failed to create forwarding rule %s: %v", loadBalancerName, err)
 		}
 		// End critical section.  It is safe to release the static IP (which
@@ -324,11 +304,11 @@ func (gce *GCECloud) ensureExternalLoadBalancer(clusterName, clusterID string, a
 		// of a user-requested IP, the "is user-owned" flag will be set,
 		// preventing it from actually being released.
 		isSafeToReleaseIP = true
-		glog.Infof("EnsureLoadBalancer(%v(%v)): created forwarding rule, IP %s", loadBalancerName, serviceName, ipAddress)
+		glog.Infof("EnsureLoadBalancer(%v(%v)): created forwarding rule, IP %s", loadBalancerName, serviceName, ipAddressToUse)
 	}
 
 	status := &v1.LoadBalancerStatus{}
-	status.Ingress = []v1.LoadBalancerIngress{{IP: ipAddress}}
+	status.Ingress = []v1.LoadBalancerIngress{{IP: ipAddressToUse}}
 
 	return status, nil
 }
@@ -375,7 +355,16 @@ func (gce *GCECloud) ensureExternalLoadBalancerDeleted(clusterName, clusterID st
 	}
 
 	errs := utilerrors.AggregateGoroutines(
-		func() error { return ignoreNotFound(gce.DeleteFirewall(makeFirewallName(loadBalancerName))) },
+		func() error {
+			fwName := makeFirewallName(loadBalancerName)
+			err := ignoreNotFound(gce.DeleteFirewall(fwName))
+			if isForbidden(err) && gce.OnXPN() {
+				glog.V(4).Infof("ensureExternalLoadBalancerDeleted(%v): do not have permission to delete firewall rule (on XPN). Raising event.", loadBalancerName)
+				gce.raiseFirewallChangeNeededEvent(service, FirewallToGCloudDeleteCmd(fwName, gce.NetworkProjectID()))
+				return nil
+			}
+			return err
+		},
 		// Even though we don't hold on to static IPs for load balancers, it's
 		// possible that EnsureLoadBalancer left one around in a failed
 		// creation/update attempt, so make sure we clean it up here just in case.
@@ -386,7 +375,7 @@ func (gce *GCECloud) ensureExternalLoadBalancerDeleted(clusterName, clusterID st
 			if err := ignoreNotFound(gce.DeleteRegionForwardingRule(loadBalancerName, gce.region)); err != nil {
 				return err
 			}
-			if err := gce.DeleteExternalTargetPoolAndChecks(loadBalancerName, gce.region, clusterID, hcNames...); err != nil {
+			if err := gce.DeleteExternalTargetPoolAndChecks(service, loadBalancerName, gce.region, clusterID, hcNames...); err != nil {
 				return err
 			}
 			return nil
@@ -398,7 +387,7 @@ func (gce *GCECloud) ensureExternalLoadBalancerDeleted(clusterName, clusterID st
 	return nil
 }
 
-func (gce *GCECloud) DeleteExternalTargetPoolAndChecks(name, region, clusterID string, hcNames ...string) error {
+func (gce *GCECloud) DeleteExternalTargetPoolAndChecks(service *v1.Service, name, region, clusterID string, hcNames ...string) error {
 	if err := gce.DeleteTargetPool(name, region); err != nil && isHTTPErrorCode(err, http.StatusNotFound) {
 		glog.Infof("Target pool %s already deleted. Continuing to delete other resources.", name)
 	} else if err != nil {
@@ -440,9 +429,10 @@ func (gce *GCECloud) DeleteExternalTargetPoolAndChecks(name, region, clusterID s
 			// So we should delete the health check firewall as well.
 			fwName := MakeHealthCheckFirewallName(clusterID, hcName, isNodesHealthCheck)
 			glog.Infof("Deleting firewall %v.", fwName)
-			if err := gce.DeleteFirewall(fwName); err != nil {
-				if isHTTPErrorCode(err, http.StatusNotFound) {
-					glog.V(4).Infof("Firewall %v is already deleted.", fwName)
+			if err := ignoreNotFound(gce.DeleteFirewall(fwName)); err != nil {
+				if isForbidden(err) && gce.OnXPN() {
+					glog.V(4).Infof("DeleteExternalTargetPoolAndChecks(%v): do not have permission to delete firewall rule (on XPN). Raising event.", hcName)
+					gce.raiseFirewallChangeNeededEvent(service, FirewallToGCloudDeleteCmd(fwName, gce.NetworkProjectID()))
 					return nil
 				}
 				return err
@@ -456,7 +446,57 @@ func (gce *GCECloud) DeleteExternalTargetPoolAndChecks(name, region, clusterID s
 	return nil
 }
 
-func (gce *GCECloud) createTargetPool(name, serviceName, ipAddress, region, clusterID string, hosts []*gceInstance, affinityType v1.ServiceAffinity, hc *compute.HttpHealthCheck) error {
+// verifyUserRequestedIP checks the user-provided IP to see whether it meets
+// all the expected attributes for the load balancer, and returns an error if
+// the verification failed. It also returns a boolean to indicate whether the
+// IP address is considered owned by the user (i.e., not managed by the
+// controller.
+func verifyUserRequestedIP(s CloudAddressService, region, requestedIP, fwdRuleIP, lbRef string, desiredNetTier NetworkTier) (isUserOwnedIP bool, err error) {
+	if requestedIP == "" {
+		return false, nil
+	}
+	// If a specific IP address has been requested, we have to respect the
+	// user's request and use that IP. If the forwarding rule was already using
+	// a different IP, it will be harmlessly abandoned because it was only an
+	// ephemeral IP (or it was a different static IP owned by the user, in which
+	// case we shouldn't delete it anyway).
+	existingAddress, err := s.GetRegionAddressByIP(region, requestedIP)
+	if err != nil && !isNotFound(err) {
+		glog.Errorf("verifyUserRequestedIP: failed to check whether the requested IP %q for LB %s exists: %v", requestedIP, lbRef, err)
+		return false, err
+	}
+	if err == nil {
+		// The requested IP is a static IP, owned and managed by the user.
+
+		// Check if the network tier of the static IP matches the desired
+		// network tier.
+		netTierStr, err := s.getNetworkTierFromAddress(existingAddress.Name, region)
+		if err != nil {
+			return false, fmt.Errorf("failed to check the network tier of the IP %q: %v", requestedIP, err)
+		}
+		netTier := NetworkTierGCEValueToType(netTierStr)
+		if netTier != desiredNetTier {
+			glog.Errorf("verifyUserRequestedIP: requested static IP %q (name: %s) for LB %s has network tier %s, need %s.", requestedIP, existingAddress.Name, lbRef, netTier, desiredNetTier)
+			return false, fmt.Errorf("requrested IP %q belongs to the %s network tier; expected %s", requestedIP, netTier, desiredNetTier)
+		}
+		glog.V(4).Infof("verifyUserRequestedIP: the requested static IP %q (name: %s, tier: %s) for LB %s exists.", requestedIP, existingAddress.Name, netTier, lbRef)
+		return true, nil
+	}
+	if requestedIP == fwdRuleIP {
+		// The requested IP is not a static IP, but is currently assigned
+		// to this forwarding rule, so we can just use it.
+		glog.V(4).Infof("verifyUserRequestedIP: the requested IP %q is not static, but is currently in use by for LB %s", requestedIP, lbRef)
+		return false, nil
+	}
+	// The requested IP is not static and it is not assigned to the
+	// current forwarding rule.  It might be attached to a different
+	// rule or it might not be part of this project at all.  Either
+	// way, we can't use it.
+	glog.Errorf("verifyUserRequestedIP: requested IP %q for LB %s is neither static nor assigned to the LB", requestedIP, lbRef)
+	return false, fmt.Errorf("requested ip %q is neither static nor assigned to the LB", requestedIP)
+}
+
+func (gce *GCECloud) createTargetPool(svc *v1.Service, name, serviceName, ipAddress, region, clusterID string, hosts []*gceInstance, affinityType v1.ServiceAffinity, hc *compute.HttpHealthCheck) error {
 	// health check management is coupled with targetPools to prevent leaks. A
 	// target pool is the only thing that requires a health check, so we delete
 	// associated checks on teardown, and ensure checks on setup.
@@ -469,14 +509,14 @@ func (gce *GCECloud) createTargetPool(name, serviceName, ipAddress, region, clus
 			gce.sharedResourceLock.Lock()
 			defer gce.sharedResourceLock.Unlock()
 		}
-		if !gce.OnXPN() {
-			if err := gce.ensureHttpHealthCheckFirewall(serviceName, ipAddress, region, clusterID, hosts, hc.Name, int32(hc.Port), isNodesHealthCheck); err != nil {
-				return err
-			}
+
+		if err := gce.ensureHttpHealthCheckFirewall(svc, serviceName, ipAddress, region, clusterID, hosts, hc.Name, int32(hc.Port), isNodesHealthCheck); err != nil {
+			return err
 		}
 		var err error
+		hcRequestPath, hcPort := hc.RequestPath, hc.Port
 		if hc, err = gce.ensureHttpHealthCheck(hc.Name, hc.RequestPath, int32(hc.Port)); err != nil || hc == nil {
-			return fmt.Errorf("Failed to ensure health check for %v port %d path %v: %v", name, hc.Port, hc.RequestPath, err)
+			return fmt.Errorf("Failed to ensure health check for %v port %d path %v: %v", name, hcPort, hcRequestPath, err)
 		}
 		hcLinks = append(hcLinks, hc.SelfLink)
 	}
@@ -541,8 +581,8 @@ func (gce *GCECloud) updateTargetPool(loadBalancerName string, existing sets.Str
 	return nil
 }
 
-func (gce *GCECloud) targetPoolURL(name, region string) string {
-	return gce.service.BasePath + strings.Join([]string{gce.projectID, "regions", region, "targetPools", name}, "/")
+func (gce *GCECloud) targetPoolURL(name string) string {
+	return gce.service.BasePath + strings.Join([]string{gce.projectID, "regions", gce.region, "targetPools", name}, "/")
 }
 
 func makeHttpHealthCheck(name, path string, port int32) *compute.HttpHealthCheck {
@@ -721,12 +761,7 @@ func translateAffinityType(affinityType v1.ServiceAffinity) string {
 }
 
 func (gce *GCECloud) firewallNeedsUpdate(name, serviceName, region, ipAddress string, ports []v1.ServicePort, sourceRanges netsets.IPNet) (exists bool, needsUpdate bool, err error) {
-	if gce.OnXPN() {
-		glog.V(2).Infoln("firewallNeedsUpdate: Cluster is on XPN network - skipping firewall creation")
-		return false, false, nil
-	}
-
-	fw, err := gce.service.Firewalls.Get(gce.projectID, makeFirewallName(name)).Do()
+	fw, err := gce.service.Firewalls.Get(gce.NetworkProjectID(), makeFirewallName(name)).Do()
 	if err != nil {
 		if isHTTPErrorCode(err, http.StatusNotFound) {
 			return false, true, nil
@@ -763,7 +798,7 @@ func (gce *GCECloud) firewallNeedsUpdate(name, serviceName, region, ipAddress st
 	return true, false, nil
 }
 
-func (gce *GCECloud) ensureHttpHealthCheckFirewall(serviceName, ipAddress, region, clusterID string, hosts []*gceInstance, hcName string, hcPort int32, isNodesHealthCheck bool) error {
+func (gce *GCECloud) ensureHttpHealthCheckFirewall(svc *v1.Service, serviceName, ipAddress, region, clusterID string, hosts []*gceInstance, hcName string, hcPort int32, isNodesHealthCheck bool) error {
 	// Prepare the firewall params for creating / checking.
 	desc := fmt.Sprintf(`{"kubernetes.io/cluster-id":"%s"}`, clusterID)
 	if !isNodesHealthCheck {
@@ -773,13 +808,13 @@ func (gce *GCECloud) ensureHttpHealthCheckFirewall(serviceName, ipAddress, regio
 	ports := []v1.ServicePort{{Protocol: "tcp", Port: hcPort}}
 
 	fwName := MakeHealthCheckFirewallName(clusterID, hcName, isNodesHealthCheck)
-	fw, err := gce.service.Firewalls.Get(gce.projectID, fwName).Do()
+	fw, err := gce.service.Firewalls.Get(gce.NetworkProjectID(), fwName).Do()
 	if err != nil {
 		if !isHTTPErrorCode(err, http.StatusNotFound) {
 			return fmt.Errorf("error getting firewall for health checks: %v", err)
 		}
 		glog.Infof("Creating firewall %v for health checks.", fwName)
-		if err := gce.createFirewall(fwName, region, desc, sourceRanges, ports, hosts); err != nil {
+		if err := gce.createFirewall(svc, fwName, region, desc, sourceRanges, ports, hosts); err != nil {
 			return err
 		}
 		glog.Infof("Created firewall %v for health checks.", fwName)
@@ -789,10 +824,10 @@ func (gce *GCECloud) ensureHttpHealthCheckFirewall(serviceName, ipAddress, regio
 	if fw.Description != desc ||
 		len(fw.Allowed) != 1 ||
 		fw.Allowed[0].IPProtocol != string(ports[0].Protocol) ||
-		!equalStringSets(fw.Allowed[0].Ports, []string{string(ports[0].Port)}) ||
+		!equalStringSets(fw.Allowed[0].Ports, []string{strconv.Itoa(int(ports[0].Port))}) ||
 		!equalStringSets(fw.SourceRanges, sourceRanges.StringSlice()) {
 		glog.Warningf("Firewall %v exists but parameters have drifted - updating...", fwName)
-		if err := gce.updateFirewall(fwName, region, desc, sourceRanges, ports, hosts); err != nil {
+		if err := gce.updateFirewall(svc, fwName, region, desc, sourceRanges, ports, hosts); err != nil {
 			glog.Warningf("Failed to reconcile firewall %v parameters.", fwName)
 			return err
 		}
@@ -801,44 +836,77 @@ func (gce *GCECloud) ensureHttpHealthCheckFirewall(serviceName, ipAddress, regio
 	return nil
 }
 
-func (gce *GCECloud) createForwardingRule(name, serviceName, region, ipAddress string, ports []v1.ServicePort) error {
+func createForwardingRule(s CloudForwardingRuleService, name, serviceName, region, ipAddress, target string, ports []v1.ServicePort, netTier NetworkTier) error {
 	portRange, err := loadBalancerPortRange(ports)
 	if err != nil {
 		return err
 	}
-	req := &compute.ForwardingRule{
-		Name:        name,
-		Description: fmt.Sprintf(`{"kubernetes.io/service-name":"%s"}`, serviceName),
-		IPAddress:   ipAddress,
-		IPProtocol:  string(ports[0].Protocol),
-		PortRange:   portRange,
-		Target:      gce.targetPoolURL(name, region),
+	desc := makeServiceDescription(serviceName)
+	ipProtocol := string(ports[0].Protocol)
+
+	switch netTier {
+	case NetworkTierPremium:
+		rule := &compute.ForwardingRule{
+			Name:        name,
+			Description: desc,
+			IPAddress:   ipAddress,
+			IPProtocol:  ipProtocol,
+			PortRange:   portRange,
+			Target:      target,
+		}
+		err = s.CreateRegionForwardingRule(rule, region)
+	default:
+		rule := &computealpha.ForwardingRule{
+			Name:        name,
+			Description: desc,
+			IPAddress:   ipAddress,
+			IPProtocol:  ipProtocol,
+			PortRange:   portRange,
+			Target:      target,
+			NetworkTier: netTier.ToGCEValue(),
+		}
+		err = s.CreateAlphaRegionForwardingRule(rule, region)
 	}
 
-	if err = gce.CreateRegionForwardingRule(req, region); err != nil && !isHTTPErrorCode(err, http.StatusConflict) {
+	if err != nil && !isHTTPErrorCode(err, http.StatusConflict) {
+		return err
+	}
+
+	return nil
+}
+
+func (gce *GCECloud) createFirewall(svc *v1.Service, name, region, desc string, sourceRanges netsets.IPNet, ports []v1.ServicePort, hosts []*gceInstance) error {
+	firewall, err := gce.firewallObject(name, region, desc, sourceRanges, ports, hosts)
+	if err != nil {
+		return err
+	}
+	if err = gce.CreateFirewall(firewall); err != nil {
+		if isHTTPErrorCode(err, http.StatusConflict) {
+			return nil
+		} else if isForbidden(err) && gce.OnXPN() {
+			glog.V(4).Infof("createFirewall(%v): do not have permission to create firewall rule (on XPN). Raising event.", firewall.Name)
+			gce.raiseFirewallChangeNeededEvent(svc, FirewallToGCloudCreateCmd(firewall, gce.NetworkProjectID()))
+			return nil
+		}
 		return err
 	}
 	return nil
 }
 
-func (gce *GCECloud) createFirewall(name, region, desc string, sourceRanges netsets.IPNet, ports []v1.ServicePort, hosts []*gceInstance) error {
-	firewall, err := gce.firewallObject(name, region, desc, sourceRanges, ports, hosts)
-	if err != nil {
-		return err
-	}
-	if err = gce.CreateFirewall(firewall); err != nil && !isHTTPErrorCode(err, http.StatusConflict) {
-		return err
-	}
-	return nil
-}
-
-func (gce *GCECloud) updateFirewall(name, region, desc string, sourceRanges netsets.IPNet, ports []v1.ServicePort, hosts []*gceInstance) error {
+func (gce *GCECloud) updateFirewall(svc *v1.Service, name, region, desc string, sourceRanges netsets.IPNet, ports []v1.ServicePort, hosts []*gceInstance) error {
 	firewall, err := gce.firewallObject(name, region, desc, sourceRanges, ports, hosts)
 	if err != nil {
 		return err
 	}
 
-	if err = gce.UpdateFirewall(firewall); err != nil && !isHTTPErrorCode(err, http.StatusConflict) {
+	if err = gce.UpdateFirewall(firewall); err != nil {
+		if isHTTPErrorCode(err, http.StatusConflict) {
+			return nil
+		} else if isForbidden(err) && gce.OnXPN() {
+			glog.V(4).Infof("updateFirewall(%v): do not have permission to update firewall rule (on XPN). Raising event.", firewall.Name)
+			gce.raiseFirewallChangeNeededEvent(svc, FirewallToGCloudUpdateCmd(firewall, gce.NetworkProjectID()))
+			return nil
+		}
 		return err
 	}
 	return nil
@@ -880,52 +948,43 @@ func (gce *GCECloud) firewallObject(name, region, desc string, sourceRanges nets
 	return firewall, nil
 }
 
-func (gce *GCECloud) projectOwnsStaticIP(name, region string, ipAddress string) (bool, error) {
-	pageToken := ""
-	page := 0
-	for ; page == 0 || (pageToken != "" && page < maxPages); page++ {
-		listCall := gce.service.Addresses.List(gce.projectID, region)
-		if pageToken != "" {
-			listCall = listCall.PageToken(pageToken)
-		}
-		addresses, err := listCall.Do()
-		if err != nil {
-			return false, fmt.Errorf("failed to list gce IP addresses: %v", err)
-		}
-		pageToken = addresses.NextPageToken
-		for _, addr := range addresses.Items {
-			if addr.Address == ipAddress {
-				// This project does own the address, so return success.
-				return true, nil
-			}
-		}
-	}
-	if page >= maxPages {
-		glog.Errorf("projectOwnsStaticIP exceeded maxPages=%d for Addresses.List; truncating.", maxPages)
-	}
-	return false, nil
-}
-
-func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP string) (ipAddress string, existing bool, err error) {
+func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP string, netTier NetworkTier) (ipAddress string, existing bool, err error) {
 	// If the address doesn't exist, this will create it.
 	// If the existingIP exists but is ephemeral, this will promote it to static.
 	// If the address already exists, this will harmlessly return a StatusConflict
 	// and we'll grab the IP before returning.
 	existed := false
-	addressObj := &compute.Address{
-		Name:        name,
-		Description: fmt.Sprintf(`{"kubernetes.io/service-name":"%s"}`, serviceName),
-	}
+	desc := makeServiceDescription(serviceName)
 
-	if existingIP != "" {
-		addressObj.Address = existingIP
-	}
-
-	if err = s.ReserveRegionAddress(addressObj, region); err != nil {
-		if !isHTTPErrorCode(err, http.StatusConflict) {
-			return "", false, fmt.Errorf("error creating gce static IP address: %v", err)
+	var creationErr error
+	switch netTier {
+	case NetworkTierPremium:
+		addressObj := &compute.Address{
+			Name:        name,
+			Description: desc,
+		}
+		if existingIP != "" {
+			addressObj.Address = existingIP
+		}
+		creationErr = s.ReserveRegionAddress(addressObj, region)
+	default:
+		addressObj := &computealpha.Address{
+			Name:        name,
+			Description: desc,
+			NetworkTier: netTier.ToGCEValue(),
+		}
+		if existingIP != "" {
+			addressObj.Address = existingIP
+		}
+		creationErr = s.ReserveAlphaRegionAddress(addressObj, region)
+	}
+
+	if creationErr != nil {
+		// GCE returns StatusConflict if the name conflicts; it returns
+		// StatusBadRequest if the IP conflicts.
+		if !isHTTPErrorCode(creationErr, http.StatusConflict) && !isHTTPErrorCode(creationErr, http.StatusBadRequest) {
+			return "", false, fmt.Errorf("error creating gce static IP address: %v", creationErr)
 		}
-		// StatusConflict == the IP exists already.
 		existed = true
 	}
 
@@ -936,3 +995,73 @@ func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP
 
 	return addr.Address, existed, nil
 }
+
+func (gce *GCECloud) getServiceNetworkTier(svc *v1.Service) (NetworkTier, error) {
+	if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) {
+		return NetworkTierDefault, nil
+	}
+	tier, err := GetServiceNetworkTier(svc)
+	if err != nil {
+		// Returns an error if the annotation is invalid.
+		return NetworkTier(""), err
+	}
+	return tier, nil
+}
+
+func (gce *GCECloud) deleteWrongNetworkTieredResources(lbName, lbRef string, desiredNetTier NetworkTier) error {
+	logPrefix := fmt.Sprintf("deleteWrongNetworkTieredResources:(%s)", lbRef)
+	if err := deleteFWDRuleWithWrongTier(gce, gce.region, lbName, logPrefix, desiredNetTier); err != nil {
+		return err
+	}
+	if err := deleteAddressWithWrongTier(gce, gce.region, lbName, logPrefix, desiredNetTier); err != nil {
+		return err
+	}
+	return nil
+}
+
+// deleteFWDRuleWithWrongTier checks the network tier of existing forwarding
+// rule and delete the rule if the tier does not matched the desired tier.
+func deleteFWDRuleWithWrongTier(s CloudForwardingRuleService, region, name, logPrefix string, desiredNetTier NetworkTier) error {
+	tierStr, err := s.getNetworkTierFromForwardingRule(name, region)
+	if isNotFound(err) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+	existingTier := NetworkTierGCEValueToType(tierStr)
+	if existingTier == desiredNetTier {
+		return nil
+	}
+	glog.V(2).Infof("%s: Network tiers do not match; existing forwarding rule: %q, desired: %q. Deleting the forwarding rule",
+		logPrefix, existingTier, desiredNetTier)
+	err = s.DeleteRegionForwardingRule(name, region)
+	return ignoreNotFound(err)
+}
+
+// deleteAddressWithWrongTier checks the network tier of existing address
+// and delete the address if the tier does not matched the desired tier.
+func deleteAddressWithWrongTier(s CloudAddressService, region, name, logPrefix string, desiredNetTier NetworkTier) error {
+	// We only check the IP address matching the reserved name that the
+	// controller assigned to the LB. We make the assumption that an address of
+	// such name is owned by the controller and is safe to release. Whether an
+	// IP is owned by the user is not clearly defined in the current code, and
+	// this assumption may not match some of the existing logic in the code.
+	// However, this is okay since network tiering is still Alpha and will be
+	// properly gated.
+	// TODO(#51665): Re-evaluate the "ownership" of the IP address to ensure
+	// we don't release IP unintentionally.
+	tierStr, err := s.getNetworkTierFromAddress(name, region)
+	if isNotFound(err) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+	existingTier := NetworkTierGCEValueToType(tierStr)
+	if existingTier == desiredNetTier {
+		return nil
+	}
+	glog.V(2).Infof("%s: Network tiers do not match; existing address: %q, desired: %q. Deleting the address",
+		logPrefix, existingTier, desiredNetTier)
+	err = s.DeleteRegionAddress(name, region)
+	return ignoreNotFound(err)
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_loadbalancer_internal.go

@@ -34,8 +34,6 @@ const (
 	allInstances = "ALL"
 )
 
-type lbBalancingMode string
-
 func (gce *GCECloud) ensureInternalLoadBalancer(clusterName, clusterID string, svc *v1.Service, existingFwdRule *compute.ForwardingRule, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) {
 	nm := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}
 	ports, protocol := getPortsAndProtocol(svc.Spec.Ports)
@@ -79,19 +77,25 @@ func (gce *GCECloud) ensureInternalLoadBalancer(clusterName, clusterID string, s
 		return nil, err
 	}
 
+	// Determine IP which will be used for this LB. If no forwarding rule has been established
+	// or specified in the Service spec, then requestedIP = "".
+	requestedIP := determineRequestedIP(svc, existingFwdRule)
+	addrMgr := newAddressManager(gce, nm.String(), gce.Region(), gce.getInternalSubnetURL(), loadBalancerName, requestedIP, schemeInternal)
+	ipToUse, err := addrMgr.HoldAddress()
+	if err != nil {
+		return nil, err
+	}
+	glog.V(2).Infof("ensureInternalLoadBalancer(%v): reserved IP %q for the forwarding rule", loadBalancerName, ipToUse)
+
 	// Ensure firewall rules if necessary
-	if gce.OnXPN() {
-		glog.V(2).Infof("ensureInternalLoadBalancer: cluster is on a cross-project network (XPN) network project %v, compute project %v - skipping firewall creation", gce.networkProjectID, gce.projectID)
-	} else {
-		if err = gce.ensureInternalFirewalls(loadBalancerName, clusterID, nm, svc, strconv.Itoa(int(hcPort)), sharedHealthCheck, nodes); err != nil {
-			return nil, err
-		}
+	if err = gce.ensureInternalFirewalls(loadBalancerName, ipToUse, clusterID, nm, svc, strconv.Itoa(int(hcPort)), sharedHealthCheck, nodes); err != nil {
+		return nil, err
 	}
 
 	expectedFwdRule := &compute.ForwardingRule{
 		Name:                loadBalancerName,
 		Description:         fmt.Sprintf(`{"kubernetes.io/service-name":"%s"}`, nm.String()),
-		IPAddress:           svc.Spec.LoadBalancerIP,
+		IPAddress:           ipToUse,
 		BackendService:      backendServiceLink,
 		Ports:               ports,
 		IPProtocol:          string(protocol),
@@ -126,25 +130,25 @@ func (gce *GCECloud) ensureInternalLoadBalancer(clusterName, clusterID string, s
 		if err = gce.CreateRegionForwardingRule(expectedFwdRule, gce.region); err != nil {
 			return nil, err
 		}
+		glog.V(2).Infof("ensureInternalLoadBalancer(%v): created forwarding rule", loadBalancerName)
 	}
 
 	// Delete the previous internal load balancer resources if necessary
 	if existingBackendService != nil {
-		gce.clearPreviousInternalResources(loadBalancerName, existingBackendService, backendServiceName, hcName)
+		gce.clearPreviousInternalResources(svc, loadBalancerName, existingBackendService, backendServiceName, hcName)
 	}
 
-	// Get the most recent forwarding rule for the new address.
-	existingFwdRule, err = gce.GetRegionForwardingRule(loadBalancerName, gce.region)
-	if err != nil {
-		return nil, err
+	// Now that the controller knows the forwarding rule exists, we can release the address.
+	if err := addrMgr.ReleaseAddress(); err != nil {
+		glog.Errorf("ensureInternalLoadBalancer: failed to release address reservation, possibly causing an orphan: %v", err)
 	}
 
 	status := &v1.LoadBalancerStatus{}
-	status.Ingress = []v1.LoadBalancerIngress{{IP: existingFwdRule.IPAddress}}
+	status.Ingress = []v1.LoadBalancerIngress{{IP: ipToUse}}
 	return status, nil
 }
 
-func (gce *GCECloud) clearPreviousInternalResources(loadBalancerName string, existingBackendService *compute.BackendService, expectedBSName, expectedHCName string) {
+func (gce *GCECloud) clearPreviousInternalResources(svc *v1.Service, loadBalancerName string, existingBackendService *compute.BackendService, expectedBSName, expectedHCName string) {
 	// If a new backend service was created, delete the old one.
 	if existingBackendService.Name != expectedBSName {
 		glog.V(2).Infof("clearPreviousInternalResources(%v): expected backend service %q does not match previous %q - deleting backend service", loadBalancerName, expectedBSName, existingBackendService.Name)
@@ -158,7 +162,7 @@ func (gce *GCECloud) clearPreviousInternalResources(loadBalancerName string, exi
 		existingHCName := getNameFromLink(existingBackendService.HealthChecks[0])
 		if existingHCName != expectedHCName {
 			glog.V(2).Infof("clearPreviousInternalResources(%v): expected health check %q does not match previous %q - deleting health check", loadBalancerName, expectedHCName, existingHCName)
-			if err := gce.teardownInternalHealthCheckAndFirewall(existingHCName); err != nil {
+			if err := gce.teardownInternalHealthCheckAndFirewall(svc, existingHCName); err != nil {
 				glog.Warningf("clearPreviousInternalResources: could not delete existing healthcheck: %v, err: %v", existingHCName, err)
 			}
 		}
@@ -198,6 +202,9 @@ func (gce *GCECloud) ensureInternalLoadBalancerDeleted(clusterName, clusterID st
 	gce.sharedResourceLock.Lock()
 	defer gce.sharedResourceLock.Unlock()
 
+	glog.V(2).Infof("ensureInternalLoadBalancerDeleted(%v): attempting delete of region internal address", loadBalancerName)
+	ensureAddressDeleted(gce, loadBalancerName, gce.region)
+
 	glog.V(2).Infof("ensureInternalLoadBalancerDeleted(%v): deleting region internal forwarding rule", loadBalancerName)
 	if err := gce.DeleteRegionForwardingRule(loadBalancerName, gce.region); err != nil && !isNotFound(err) {
 		return err
@@ -211,12 +218,17 @@ func (gce *GCECloud) ensureInternalLoadBalancerDeleted(clusterName, clusterID st
 
 	glog.V(2).Infof("ensureInternalLoadBalancerDeleted(%v): deleting firewall for traffic", loadBalancerName)
 	if err := gce.DeleteFirewall(loadBalancerName); err != nil {
-		return err
+		if isForbidden(err) && gce.OnXPN() {
+			glog.V(2).Infof("ensureInternalLoadBalancerDeleted(%v): could not delete traffic firewall on XPN cluster. Raising event.", loadBalancerName)
+			gce.raiseFirewallChangeNeededEvent(svc, FirewallToGCloudDeleteCmd(loadBalancerName, gce.NetworkProjectID()))
+		} else {
+			return err
+		}
 	}
 
 	hcName := makeHealthCheckName(loadBalancerName, clusterID, sharedHealthCheck)
 	glog.V(2).Infof("ensureInternalLoadBalancerDeleted(%v): deleting health check %v and its firewall", loadBalancerName, hcName)
-	if err := gce.teardownInternalHealthCheckAndFirewall(hcName); err != nil {
+	if err := gce.teardownInternalHealthCheckAndFirewall(svc, hcName); err != nil {
 		return err
 	}
 
@@ -245,7 +257,7 @@ func (gce *GCECloud) teardownInternalBackendService(bsName string) error {
 	return nil
 }
 
-func (gce *GCECloud) teardownInternalHealthCheckAndFirewall(hcName string) error {
+func (gce *GCECloud) teardownInternalHealthCheckAndFirewall(svc *v1.Service, hcName string) error {
 	if err := gce.DeleteHealthCheck(hcName); err != nil {
 		if isNotFound(err) {
 			glog.V(2).Infof("teardownInternalHealthCheckAndFirewall(%v): health check does not exist.", hcName)
@@ -261,13 +273,19 @@ func (gce *GCECloud) teardownInternalHealthCheckAndFirewall(hcName string) error
 
 	hcFirewallName := makeHealthCheckFirewallNameFromHC(hcName)
 	if err := gce.DeleteFirewall(hcFirewallName); err != nil && !isNotFound(err) {
+		if isForbidden(err) && gce.OnXPN() {
+			glog.V(2).Infof("teardownInternalHealthCheckAndFirewall(%v): could not delete health check traffic firewall on XPN cluster. Raising Event.", hcName)
+			gce.raiseFirewallChangeNeededEvent(svc, FirewallToGCloudDeleteCmd(hcFirewallName, gce.NetworkProjectID()))
+			return nil
+		}
+
 		return fmt.Errorf("failed to delete health check firewall: %v, err: %v", hcFirewallName, err)
 	}
 	glog.V(2).Infof("teardownInternalHealthCheckAndFirewall(%v): health check firewall deleted", hcFirewallName)
 	return nil
 }
 
-func (gce *GCECloud) ensureInternalFirewall(fwName, fwDesc string, sourceRanges []string, ports []string, protocol v1.Protocol, nodes []*v1.Node) error {
+func (gce *GCECloud) ensureInternalFirewall(svc *v1.Service, fwName, fwDesc string, sourceRanges []string, ports []string, protocol v1.Protocol, nodes []*v1.Node) error {
 	glog.V(2).Infof("ensureInternalFirewall(%v): checking existing firewall", fwName)
 	targetTags, err := gce.GetNodeTags(nodeNames(nodes))
 	if err != nil {
@@ -295,7 +313,13 @@ func (gce *GCECloud) ensureInternalFirewall(fwName, fwDesc string, sourceRanges
 
 	if existingFirewall == nil {
 		glog.V(2).Infof("ensureInternalFirewall(%v): creating firewall", fwName)
-		return gce.CreateFirewall(expectedFirewall)
+		err = gce.CreateFirewall(expectedFirewall)
+		if err != nil && isForbidden(err) && gce.OnXPN() {
+			glog.V(2).Infof("ensureInternalFirewall(%v): do not have permission to create firewall rule (on XPN). Raising event.", fwName)
+			gce.raiseFirewallChangeNeededEvent(svc, FirewallToGCloudCreateCmd(expectedFirewall, gce.NetworkProjectID()))
+			return nil
+		}
+		return err
 	}
 
 	if firewallRuleEqual(expectedFirewall, existingFirewall) {
@@ -303,18 +327,24 @@ func (gce *GCECloud) ensureInternalFirewall(fwName, fwDesc string, sourceRanges
 	}
 
 	glog.V(2).Infof("ensureInternalFirewall(%v): updating firewall", fwName)
-	return gce.UpdateFirewall(expectedFirewall)
+	err = gce.UpdateFirewall(expectedFirewall)
+	if err != nil && isForbidden(err) && gce.OnXPN() {
+		glog.V(2).Infof("ensureInternalFirewall(%v): do not have permission to update firewall rule (on XPN). Raising event.", fwName)
+		gce.raiseFirewallChangeNeededEvent(svc, FirewallToGCloudUpdateCmd(expectedFirewall, gce.NetworkProjectID()))
+		return nil
+	}
+	return err
 }
 
-func (gce *GCECloud) ensureInternalFirewalls(loadBalancerName, clusterID string, nm types.NamespacedName, svc *v1.Service, healthCheckPort string, sharedHealthCheck bool, nodes []*v1.Node) error {
+func (gce *GCECloud) ensureInternalFirewalls(loadBalancerName, ipAddress, clusterID string, nm types.NamespacedName, svc *v1.Service, healthCheckPort string, sharedHealthCheck bool, nodes []*v1.Node) error {
 	// First firewall is for ingress traffic
-	fwDesc := makeFirewallDescription(nm.String(), svc.Spec.LoadBalancerIP)
+	fwDesc := makeFirewallDescription(nm.String(), ipAddress)
 	ports, protocol := getPortsAndProtocol(svc.Spec.Ports)
 	sourceRanges, err := v1_service.GetLoadBalancerSourceRanges(svc)
 	if err != nil {
 		return err
 	}
-	err = gce.ensureInternalFirewall(loadBalancerName, fwDesc, sourceRanges.StringSlice(), ports, protocol, nodes)
+	err = gce.ensureInternalFirewall(svc, loadBalancerName, fwDesc, sourceRanges.StringSlice(), ports, protocol, nodes)
 	if err != nil {
 		return err
 	}
@@ -322,7 +352,7 @@ func (gce *GCECloud) ensureInternalFirewalls(loadBalancerName, clusterID string,
 	// Second firewall is for health checking nodes / services
 	fwHCName := makeHealthCheckFirewallName(loadBalancerName, clusterID, sharedHealthCheck)
 	hcSrcRanges := LoadBalancerSrcRanges()
-	return gce.ensureInternalFirewall(fwHCName, "", hcSrcRanges, []string{healthCheckPort}, v1.ProtocolTCP, nodes)
+	return gce.ensureInternalFirewall(svc, fwHCName, "", hcSrcRanges, []string{healthCheckPort}, v1.ProtocolTCP, nodes)
 }
 
 func (gce *GCECloud) ensureInternalHealthCheck(name string, svcName types.NamespacedName, shared bool, path string, port int32) (*compute.HealthCheck, error) {
@@ -632,6 +662,20 @@ func (gce *GCECloud) getBackendServiceLink(name string) string {
 	return gce.service.BasePath + strings.Join([]string{gce.projectID, "regions", gce.region, "backendServices", name}, "/")
 }
 
+// getInternalSubnetURL first attempts to return the configured SubnetURL.
+// If subnetwork-name was not specified, then a best-effort generation is made.
+// Note subnet names might not be the network name for some auto networks.
+func (gce *GCECloud) getInternalSubnetURL() string {
+	if gce.SubnetworkURL() != "" {
+		return gce.SubnetworkURL()
+	}
+
+	networkName := getNameFromLink(gce.NetworkURL())
+	v := gceSubnetworkURL("", gce.NetworkProjectID(), gce.Region(), networkName)
+	glog.Warningf("Generating subnetwork URL based off network since subnet name/URL was not configured: %q", v)
+	return v
+}
+
 func getNameFromLink(link string) string {
 	if link == "" {
 		return ""
@@ -640,3 +684,15 @@ func getNameFromLink(link string) string {
 	fields := strings.Split(link, "/")
 	return fields[len(fields)-1]
 }
+
+func determineRequestedIP(svc *v1.Service, fwdRule *compute.ForwardingRule) string {
+	if svc.Spec.LoadBalancerIP != "" {
+		return svc.Spec.LoadBalancerIP
+	}
+
+	if fwdRule != nil {
+		return fwdRule.IPAddress
+	}
+
+	return ""
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_loadbalancer_naming.go

@@ -84,6 +84,11 @@ func makeBackendServiceDescription(nm types.NamespacedName, shared bool) string
 
 // External Load Balancer
 
+// makeServiceDescription is used to generate descriptions for forwarding rules and addresses.
+func makeServiceDescription(serviceName string) string {
+	return fmt.Sprintf(`{"kubernetes.io/service-name":"%s"}`, serviceName)
+}
+
 // makeNodesHealthCheckName returns name of the health check resource used by
 // the GCE load balancers (l4) for performing health checks on nodes.
 func makeNodesHealthCheckName(clusterID string) string {

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_op.go

@@ -17,17 +17,20 @@ limitations under the License.
 package gce
 
 import (
+	"encoding/json"
 	"fmt"
 	"time"
 
 	"k8s.io/apimachinery/pkg/util/wait"
 
 	"github.com/golang/glog"
-	compute "google.golang.org/api/compute/v1"
+	computealpha "google.golang.org/api/compute/v0.alpha"
+	computebeta "google.golang.org/api/compute/v0.beta"
+	computev1 "google.golang.org/api/compute/v1"
 	"google.golang.org/api/googleapi"
 )
 
-func (gce *GCECloud) waitForOp(op *compute.Operation, getOperation func(operationName string) (*compute.Operation, error), mc *metricContext) error {
+func (gce *GCECloud) waitForOp(op *computev1.Operation, getOperation func(operationName string) (*computev1.Operation, error), mc *metricContext) error {
 	if op == nil {
 		return mc.Observe(fmt.Errorf("operation must not be nil"))
 	}
@@ -72,11 +75,11 @@ func (gce *GCECloud) waitForOp(op *compute.Operation, getOperation func(operatio
 	})
 }
 
-func opIsDone(op *compute.Operation) bool {
+func opIsDone(op *computev1.Operation) bool {
 	return op != nil && op.Status == "DONE"
 }
 
-func getErrorFromOp(op *compute.Operation) error {
+func getErrorFromOp(op *computev1.Operation) error {
 	if op != nil && op.Error != nil && len(op.Error.Errors) > 0 {
 		err := &googleapi.Error{
 			Code:    int(op.HttpErrorStatusCode),
@@ -89,20 +92,89 @@ func getErrorFromOp(op *compute.Operation) error {
 	return nil
 }
 
-func (gce *GCECloud) waitForGlobalOp(op *compute.Operation, mc *metricContext) error {
-	return gce.waitForOp(op, func(operationName string) (*compute.Operation, error) {
-		return gce.service.GlobalOperations.Get(gce.projectID, operationName).Do()
-	}, mc)
+func (gce *GCECloud) waitForGlobalOp(op gceObject, mc *metricContext) error {
+	return gce.waitForGlobalOpInProject(op, gce.ProjectID(), mc)
 }
 
-func (gce *GCECloud) waitForRegionOp(op *compute.Operation, region string, mc *metricContext) error {
-	return gce.waitForOp(op, func(operationName string) (*compute.Operation, error) {
-		return gce.service.RegionOperations.Get(gce.projectID, region, operationName).Do()
-	}, mc)
+func (gce *GCECloud) waitForRegionOp(op gceObject, region string, mc *metricContext) error {
+	return gce.waitForRegionOpInProject(op, gce.ProjectID(), region, mc)
 }
 
-func (gce *GCECloud) waitForZoneOp(op *compute.Operation, zone string, mc *metricContext) error {
-	return gce.waitForOp(op, func(operationName string) (*compute.Operation, error) {
-		return gce.service.ZoneOperations.Get(gce.projectID, zone, operationName).Do()
-	}, mc)
+func (gce *GCECloud) waitForZoneOp(op gceObject, zone string, mc *metricContext) error {
+	return gce.waitForZoneOpInProject(op, gce.ProjectID(), zone, mc)
+}
+
+func (gce *GCECloud) waitForGlobalOpInProject(op gceObject, projectID string, mc *metricContext) error {
+	switch v := op.(type) {
+	case *computealpha.Operation:
+		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
+			op, err := gce.serviceAlpha.GlobalOperations.Get(projectID, operationName).Do()
+			return convertToV1Operation(op), err
+		}, mc)
+	case *computebeta.Operation:
+		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
+			op, err := gce.serviceBeta.GlobalOperations.Get(projectID, operationName).Do()
+			return convertToV1Operation(op), err
+		}, mc)
+	case *computev1.Operation:
+		return gce.waitForOp(op.(*computev1.Operation), func(operationName string) (*computev1.Operation, error) {
+			return gce.service.GlobalOperations.Get(projectID, operationName).Do()
+		}, mc)
+	default:
+		return fmt.Errorf("unexpected type: %T", v)
+	}
+}
+
+func (gce *GCECloud) waitForRegionOpInProject(op gceObject, projectID, region string, mc *metricContext) error {
+	switch v := op.(type) {
+	case *computealpha.Operation:
+		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
+			op, err := gce.serviceAlpha.RegionOperations.Get(projectID, region, operationName).Do()
+			return convertToV1Operation(op), err
+		}, mc)
+	case *computebeta.Operation:
+		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
+			op, err := gce.serviceBeta.RegionOperations.Get(projectID, region, operationName).Do()
+			return convertToV1Operation(op), err
+		}, mc)
+	case *computev1.Operation:
+		return gce.waitForOp(op.(*computev1.Operation), func(operationName string) (*computev1.Operation, error) {
+			return gce.service.RegionOperations.Get(projectID, region, operationName).Do()
+		}, mc)
+	default:
+		return fmt.Errorf("unexpected type: %T", v)
+	}
+}
+
+func (gce *GCECloud) waitForZoneOpInProject(op gceObject, projectID, zone string, mc *metricContext) error {
+	switch v := op.(type) {
+	case *computealpha.Operation:
+		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
+			op, err := gce.serviceAlpha.ZoneOperations.Get(projectID, zone, operationName).Do()
+			return convertToV1Operation(op), err
+		}, mc)
+	case *computebeta.Operation:
+		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
+			op, err := gce.serviceBeta.ZoneOperations.Get(projectID, zone, operationName).Do()
+			return convertToV1Operation(op), err
+		}, mc)
+	case *computev1.Operation:
+		return gce.waitForOp(op.(*computev1.Operation), func(operationName string) (*computev1.Operation, error) {
+			return gce.service.ZoneOperations.Get(projectID, zone, operationName).Do()
+		}, mc)
+	default:
+		return fmt.Errorf("unexpected type: %T", v)
+	}
+}
+
+func convertToV1Operation(object gceObject) *computev1.Operation {
+	enc, err := object.MarshalJSON()
+	if err != nil {
+		panic(fmt.Sprintf("Failed to encode to json: %v", err))
+	}
+	var op computev1.Operation
+	if err := json.Unmarshal(enc, &op); err != nil {
+		panic(fmt.Sprintf("Failed to convert GCE apiObject %v to v1 operation: %v", object, err))
+	}
+	return &op
 }

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_routes.go

@@ -20,8 +20,6 @@ import (
 	"fmt"
 	"net/http"
 	"path"
-	"strings"
-	"time"
 
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/kubernetes/pkg/cloudprovider"
@@ -31,10 +29,7 @@ import (
 )
 
 func newRoutesMetricContext(request string) *metricContext {
-	return &metricContext{
-		start:      time.Now(),
-		attributes: []string{"routes_" + request, unusedMetricLabel, unusedMetricLabel},
-	}
+	return newGenericMetricContext("routes", request, unusedMetricLabel, unusedMetricLabel, computeV1Version)
 }
 
 func (gce *GCECloud) ListRoutes(clusterName string) ([]*cloudprovider.Route, error) {
@@ -43,10 +38,15 @@ func (gce *GCECloud) ListRoutes(clusterName string) ([]*cloudprovider.Route, err
 	page := 0
 	for ; page == 0 || (pageToken != "" && page < maxPages); page++ {
 		mc := newRoutesMetricContext("list_page")
-		listCall := gce.service.Routes.List(gce.projectID)
+		listCall := gce.service.Routes.List(gce.NetworkProjectID())
 
 		prefix := truncateClusterName(clusterName)
-		listCall = listCall.Filter("name eq " + prefix + "-.*")
+		// Filter for routes starting with clustername AND belonging to the
+		// relevant gcp network AND having description = "k8s-node-route".
+		filter := "(name eq " + prefix + "-.*) "
+		filter = filter + "(network eq " + gce.NetworkURL() + ") "
+		filter = filter + "(description eq " + k8sNodeRouteTag + ")"
+		listCall = listCall.Filter(filter)
 		if pageToken != "" {
 			listCall = listCall.PageToken(pageToken)
 		}
@@ -58,18 +58,6 @@ func (gce *GCECloud) ListRoutes(clusterName string) ([]*cloudprovider.Route, err
 		}
 		pageToken = res.NextPageToken
 		for _, r := range res.Items {
-			if r.Network != gce.networkURL {
-				continue
-			}
-			// Not managed if route description != "k8s-node-route"
-			if r.Description != k8sNodeRouteTag {
-				continue
-			}
-			// Not managed if route name doesn't start with <clusterName>
-			if !strings.HasPrefix(r.Name, prefix) {
-				continue
-			}
-
 			target := path.Base(r.NextHopInstance)
 			// TODO: Should we lastComponent(target) this?
 			targetNodeName := types.NodeName(target) // NodeName == Instance Name on GCE
@@ -92,11 +80,11 @@ func (gce *GCECloud) CreateRoute(clusterName string, nameHint string, route *clo
 	}
 
 	mc := newRoutesMetricContext("create")
-	insertOp, err := gce.service.Routes.Insert(gce.projectID, &compute.Route{
+	insertOp, err := gce.service.Routes.Insert(gce.NetworkProjectID(), &compute.Route{
 		Name:            routeName,
 		DestRange:       route.DestinationCIDR,
 		NextHopInstance: fmt.Sprintf("zones/%s/instances/%s", targetInstance.Zone, targetInstance.Name),
-		Network:         gce.networkURL,
+		Network:         gce.NetworkURL(),
 		Priority:        1000,
 		Description:     k8sNodeRouteTag,
 	}).Do()
@@ -108,16 +96,16 @@ func (gce *GCECloud) CreateRoute(clusterName string, nameHint string, route *clo
 			return mc.Observe(err)
 		}
 	}
-	return gce.waitForGlobalOp(insertOp, mc)
+	return gce.waitForGlobalOpInProject(insertOp, gce.NetworkProjectID(), mc)
 }
 
 func (gce *GCECloud) DeleteRoute(clusterName string, route *cloudprovider.Route) error {
 	mc := newRoutesMetricContext("delete")
-	deleteOp, err := gce.service.Routes.Delete(gce.projectID, route.Name).Do()
+	deleteOp, err := gce.service.Routes.Delete(gce.NetworkProjectID(), route.Name).Do()
 	if err != nil {
 		return mc.Observe(err)
 	}
-	return gce.waitForGlobalOp(deleteOp, mc)
+	return gce.waitForGlobalOpInProject(deleteOp, gce.NetworkProjectID(), mc)
 }
 
 func truncateClusterName(clusterName string) string {

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_targetpool.go

@@ -16,17 +16,10 @@ limitations under the License.
 
 package gce
 
-import (
-	"time"
-
-	compute "google.golang.org/api/compute/v1"
-)
+import compute "google.golang.org/api/compute/v1"
 
 func newTargetPoolMetricContext(request, region string) *metricContext {
-	return &metricContext{
-		start:      time.Now(),
-		attributes: []string{"targetpool_" + request, region, unusedMetricLabel},
-	}
+	return newGenericMetricContext("targetpool", request, region, unusedMetricLabel, computeV1Version)
 }
 
 // GetTargetPool returns the TargetPool by name.

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_targetproxy.go

@@ -18,16 +18,12 @@ package gce
 
 import (
 	"net/http"
-	"time"
 
 	compute "google.golang.org/api/compute/v1"
 )
 
 func newTargetProxyMetricContext(request string) *metricContext {
-	return &metricContext{
-		start:      time.Now(),
-		attributes: []string{"targetproxy_" + request, unusedMetricLabel, unusedMetricLabel},
-	}
+	return newGenericMetricContext("targetproxy", request, unusedMetricLabel, unusedMetricLabel, computeV1Version)
 }
 
 // GetTargetHttpProxy returns the UrlMap by name.

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_urlmap.go

@@ -18,16 +18,12 @@ package gce
 
 import (
 	"net/http"
-	"time"
 
 	compute "google.golang.org/api/compute/v1"
 )
 
 func newUrlMapMetricContext(request string) *metricContext {
-	return &metricContext{
-		start:      time.Now(),
-		attributes: []string{"urlmap_" + request, unusedMetricLabel, unusedMetricLabel},
-	}
+	return newGenericMetricContext("urlmap", request, unusedMetricLabel, unusedMetricLabel, computeV1Version)
 }
 
 // GetUrlMap returns the UrlMap by name.

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_util.go

@@ -23,6 +23,7 @@ import (
 	"regexp"
 	"strings"
 
+	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 
@@ -58,6 +59,43 @@ func getProjectAndZone() (string, string, error) {
 	return projectID, zone, nil
 }
 
+func (gce *GCECloud) raiseFirewallChangeNeededEvent(svc *v1.Service, cmd string) {
+	msg := fmt.Sprintf("Firewall change required by network admin: `%v`", cmd)
+	if gce.eventRecorder != nil && svc != nil {
+		gce.eventRecorder.Event(svc, v1.EventTypeNormal, "LoadBalancerManualChange", msg)
+	}
+}
+
+// FirewallToGCloudCreateCmd generates a gcloud command to create a firewall with specified params
+func FirewallToGCloudCreateCmd(fw *compute.Firewall, projectID string) string {
+	args := firewallToGcloudArgs(fw, projectID)
+	return fmt.Sprintf("gcloud compute firewall-rules create %v --network %v %v", fw.Name, getNameFromLink(fw.Network), args)
+}
+
+// FirewallToGCloudCreateCmd generates a gcloud command to update a firewall to specified params
+func FirewallToGCloudUpdateCmd(fw *compute.Firewall, projectID string) string {
+	args := firewallToGcloudArgs(fw, projectID)
+	return fmt.Sprintf("gcloud compute firewall-rules update %v %v", fw.Name, args)
+}
+
+// FirewallToGCloudCreateCmd generates a gcloud command to delete a firewall to specified params
+func FirewallToGCloudDeleteCmd(fwName, projectID string) string {
+	return fmt.Sprintf("gcloud compute firewall-rules delete %v --project %v", fwName, projectID)
+}
+
+func firewallToGcloudArgs(fw *compute.Firewall, projectID string) string {
+	var allPorts []string
+	for _, a := range fw.Allowed {
+		for _, p := range a.Ports {
+			allPorts = append(allPorts, fmt.Sprintf("%v:%v", a.IPProtocol, p))
+		}
+	}
+	allow := strings.Join(allPorts, ",")
+	srcRngs := strings.Join(fw.SourceRanges, ",")
+	targets := strings.Join(fw.TargetTags, ",")
+	return fmt.Sprintf("--description %q --allow %v --source-ranges %v --target-tags %v --project %v", fw.Description, allow, srcRngs, targets, projectID)
+}
+
 // Take a GCE instance 'hostname' and break it down to something that can be fed
 // to the GCE API client library.  Basically this means reducing 'kubernetes-
 // node-2.c.my-proj.internal' to 'kubernetes-node-2' if necessary.
@@ -149,3 +187,27 @@ func ignoreNotFound(err error) error {
 func isNotFoundOrInUse(err error) bool {
 	return isNotFound(err) || isInUsedByError(err)
 }
+
+func isForbidden(err error) bool {
+	return isHTTPErrorCode(err, http.StatusForbidden)
+}
+
+func makeGoogleAPINotFoundError(message string) error {
+	return &googleapi.Error{Code: http.StatusNotFound, Message: message}
+}
+
+func makeGoogleAPIError(code int, message string) error {
+	return &googleapi.Error{Code: code, Message: message}
+}
+
+// TODO(#51665): Remove this once Network Tiers becomes Beta in GCP.
+func handleAlphaNetworkTierGetError(err error) (string, error) {
+	if isForbidden(err) {
+		// Network tier is still an Alpha feature in GCP, and not every project
+		// is whitelisted to access the API. If we cannot access the API, just
+		// assume the tier is premium.
+		return NetworkTierDefault.ToGCEValue(), nil
+	}
+	// Can't get the network tier, just return an error.
+	return "", err
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/gce_zones.go

@@ -18,19 +18,16 @@ package gce
 
 import (
 	"fmt"
-	"time"
+	"strings"
 
 	compute "google.golang.org/api/compute/v1"
 
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/kubernetes/pkg/cloudprovider"
-	"strings"
 )
 
 func newZonesMetricContext(request, region string) *metricContext {
-	return &metricContext{
-		start:      time.Now(),
-		attributes: []string{"zones_" + request, region, unusedMetricLabel},
-	}
+	return newGenericMetricContext("zones", request, region, unusedMetricLabel, computeV1Version)
 }
 
 // GetZone creates a cloudprovider.Zone of the current zone and region
@@ -41,6 +38,37 @@ func (gce *GCECloud) GetZone() (cloudprovider.Zone, error) {
 	}, nil
 }
 
+// GetZoneByProviderID implements Zones.GetZoneByProviderID
+// This is particularly useful in external cloud providers where the kubelet
+// does not initialize node data.
+func (gce *GCECloud) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) {
+	_, zone, _, err := splitProviderID(providerID)
+	if err != nil {
+		return cloudprovider.Zone{}, err
+	}
+	region, err := GetGCERegion(zone)
+	if err != nil {
+		return cloudprovider.Zone{}, err
+	}
+	return cloudprovider.Zone{FailureDomain: zone, Region: region}, nil
+}
+
+// GetZoneByNodeName implements Zones.GetZoneByNodeName
+// This is particularly useful in external cloud providers where the kubelet
+// does not initialize node data.
+func (gce *GCECloud) GetZoneByNodeName(nodeName types.NodeName) (cloudprovider.Zone, error) {
+	instanceName := mapNodeNameToInstanceName(nodeName)
+	instance, err := gce.getInstanceByName(instanceName)
+	if err != nil {
+		return cloudprovider.Zone{}, err
+	}
+	region, err := GetGCERegion(instance.Zone)
+	if err != nil {
+		return cloudprovider.Zone{}, err
+	}
+	return cloudprovider.Zone{FailureDomain: instance.Zone, Region: region}, nil
+}
+
 // ListZonesInRegion returns all zones in a GCP region
 func (gce *GCECloud) ListZonesInRegion(region string) ([]*compute.Zone, error) {
 	mc := newZonesMetricContext("list", region)

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/kms.go

@@ -0,0 +1,167 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io"
+
+	"github.com/golang/glog"
+	cloudkms "google.golang.org/api/cloudkms/v1"
+	"google.golang.org/api/googleapi"
+	gcfg "gopkg.in/gcfg.v1"
+	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
+)
+
+const (
+	// KMSServiceName is the name of the cloudkms provider registered by this cloud.
+	KMSServiceName = "gcp-cloudkms"
+
+	defaultGKMSKeyRing         = "google-container-engine"
+	defaultGKMSKeyRingLocation = "global"
+)
+
+// gkmsConfig contains the GCE specific KMS configuration for setting up a KMS connection.
+type gkmsConfig struct {
+	Global struct {
+		// location is the KMS location of the KeyRing to be used for encryption.
+		// It can be found by checking the available KeyRings in the IAM UI.
+		// This is not the same as the GCP location of the project.
+		// +optional
+		Location string `gcfg:"kms-location"`
+		// keyRing is the keyRing of the hosted key to be used. The default value is "google-kubernetes".
+		// +optional
+		KeyRing string `gcfg:"kms-keyring"`
+		// cryptoKey is the name of the key to be used for encryption of Data-Encryption-Keys.
+		CryptoKey string `gcfg:"kms-cryptokey"`
+	}
+}
+
+// readGCPCloudKMSConfig parses and returns the configuration parameters for Google Cloud KMS.
+func readGCPCloudKMSConfig(reader io.Reader) (*gkmsConfig, error) {
+	cfg := &gkmsConfig{}
+	if err := gcfg.FatalOnly(gcfg.ReadInto(cfg, reader)); err != nil {
+		glog.Errorf("Couldn't read Google Cloud KMS config: %v", err)
+		return nil, err
+	}
+	return cfg, nil
+}
+
+// gkmsService provides Encrypt and Decrypt methods which allow cryptographic operations
+// using Google Cloud KMS service.
+type gkmsService struct {
+	parentName      string
+	cloudkmsService *cloudkms.Service
+}
+
+// getGCPCloudKMSService provides a Google Cloud KMS based implementation of envelope.Service.
+func (gce *GCECloud) getGCPCloudKMSService(config io.Reader) (envelope.Service, error) {
+	kmsConfig, err := readGCPCloudKMSConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
+	// Hosting on GCE/GKE with Google KMS encryption provider
+	cloudkmsService := gce.GetKMSService()
+
+	// Set defaults for location and keyRing.
+	location := kmsConfig.Global.Location
+	if len(location) == 0 {
+		location = defaultGKMSKeyRingLocation
+	}
+	keyRing := kmsConfig.Global.KeyRing
+	if len(keyRing) == 0 {
+		keyRing = defaultGKMSKeyRing
+	}
+
+	cryptoKey := kmsConfig.Global.CryptoKey
+	if len(cryptoKey) == 0 {
+		return nil, fmt.Errorf("missing cryptoKey for cloudprovided KMS: " + KMSServiceName)
+	}
+
+	parentName := fmt.Sprintf("projects/%s/locations/%s", gce.projectID, location)
+
+	// Create the keyRing if it does not exist yet
+	_, err = cloudkmsService.Projects.Locations.KeyRings.Create(parentName,
+		&cloudkms.KeyRing{}).KeyRingId(keyRing).Do()
+	if err != nil && unrecoverableCreationError(err) {
+		return nil, err
+	}
+	parentName = parentName + "/keyRings/" + keyRing
+
+	// Create the cryptoKey if it does not exist yet
+	_, err = cloudkmsService.Projects.Locations.KeyRings.CryptoKeys.Create(parentName,
+		&cloudkms.CryptoKey{
+			Purpose: "ENCRYPT_DECRYPT",
+		}).CryptoKeyId(cryptoKey).Do()
+	if err != nil && unrecoverableCreationError(err) {
+		return nil, err
+	}
+	parentName = parentName + "/cryptoKeys/" + cryptoKey
+
+	service := &gkmsService{
+		parentName:      parentName,
+		cloudkmsService: cloudkmsService,
+	}
+
+	// Sanity check before startup. For non-GCP clusters, the user's account may not have permissions to create
+	// the key. We need to verify the existence of the key before apiserver startup.
+	_, err = service.Encrypt([]byte("test"))
+	if err != nil {
+		return nil, fmt.Errorf("failed to encrypt data using Google cloudkms, using key %s. Ensure that the keyRing and cryptoKey exist. Got error: %v", parentName, err)
+	}
+
+	return service, nil
+}
+
+// Decrypt decrypts a base64 representation of encrypted bytes.
+func (t *gkmsService) Decrypt(data string) ([]byte, error) {
+	resp, err := t.cloudkmsService.Projects.Locations.KeyRings.CryptoKeys.
+		Decrypt(t.parentName, &cloudkms.DecryptRequest{
+			Ciphertext: data,
+		}).Do()
+	if err != nil {
+		return nil, err
+	}
+	return base64.StdEncoding.DecodeString(resp.Plaintext)
+}
+
+// Encrypt encrypts bytes, and returns base64 representation of the ciphertext.
+func (t *gkmsService) Encrypt(data []byte) (string, error) {
+	resp, err := t.cloudkmsService.Projects.Locations.KeyRings.CryptoKeys.
+		Encrypt(t.parentName, &cloudkms.EncryptRequest{
+			Plaintext: base64.StdEncoding.EncodeToString(data),
+		}).Do()
+	if err != nil {
+		return "", err
+	}
+	return resp.Ciphertext, nil
+}
+
+// unrecoverableCreationError decides if Kubernetes should ignore the encountered Google KMS
+// error. Only to be used for errors seen while creating a KeyRing or CryptoKey.
+func unrecoverableCreationError(err error) bool {
+	apiError, isAPIError := err.(*googleapi.Error)
+	// 409 means the object exists.
+	// 403 means we do not have permission to create the object, the user must do it.
+	// Else, it is an unrecoverable error.
+	if !isAPIError || (apiError.Code != 409 && apiError.Code != 403) {
+		return true
+	}
+	return false
+}

vendor/k8s.io/kubernetes/pkg/cloudprovider/providers/gce/metrics.go

@@ -22,21 +22,33 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
+const (
+	// Version strings for recording metrics.
+	computeV1Version    = "v1"
+	computeAlphaVersion = "alpha"
+	computeBetaVersion  = "beta"
+)
+
 type apiCallMetrics struct {
 	latency *prometheus.HistogramVec
 	errors  *prometheus.CounterVec
 }
 
 var (
-	apiMetrics = registerAPIMetrics(
+	metricLabels = []string{
 		"request", // API function that is begin invoked.
 		"region",  // region (optional).
 		"zone",    // zone (optional).
-	)
+		"version", // API version.
+	}
+
+	apiMetrics = registerAPIMetrics(metricLabels...)
 )
 
 type metricContext struct {
-	start      time.Time
+	start time.Time
+	// The cardinalities of attributes and metricLabels (defined above) must
+	// match, or prometheus will panic.
 	attributes []string
 }
 
@@ -54,6 +66,13 @@ func (mc *metricContext) Observe(err error) error {
 	return err
 }
 
+func newGenericMetricContext(prefix, request, region, zone, version string) *metricContext {
+	return &metricContext{
+		start:      time.Now(),
+		attributes: []string{prefix + "_" + request, region, zone, version},
+	}
+}
+
 // registerApiMetrics adds metrics definitions for a category of API calls.
 func registerAPIMetrics(attributes ...string) *apiCallMetrics {
 	metrics := &apiCallMetrics{