Replace godep with dep

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/service/BUILD

@@ -0,0 +1,84 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["servicecontroller.go"],
+    deps = [
+        "//federation/apis/federation:go_default_library",
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset:go_default_library",
+        "//federation/pkg/federation-controller/service/ingress:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//federation/pkg/federation-controller/util/clusterselector:go_default_library",
+        "//federation/pkg/federation-controller/util/deletionhelper:go_default_library",
+        "//federation/pkg/federation-controller/util/eventsink:go_default_library",
+        "//pkg/api:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+        "//vendor/k8s.io/client-go/tools/record:go_default_library",
+        "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
+        "//vendor/k8s.io/client-go/util/workqueue:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["servicecontroller_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset/fake:go_default_library",
+        "//federation/pkg/federation-controller/service/ingress:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//federation/pkg/federation-controller/util/deletionhelper:go_default_library",
+        "//federation/pkg/federation-controller/util/test:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/stretchr/testify/require:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
+        "//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//federation/pkg/federation-controller/service/dns:all-srcs",
+        "//federation/pkg/federation-controller/service/ingress:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/service/dns/BUILD

@@ -0,0 +1,59 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["dns_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset/fake:go_default_library",
+        "//federation/pkg/dnsprovider/providers/google/clouddns:go_default_library",
+        "//federation/pkg/federation-controller/service/ingress:go_default_library",
+        "//federation/pkg/federation-controller/util/test:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["dns.go"],
+    deps = [
+        "//federation/client/clientset_generated/federation_clientset:go_default_library",
+        "//federation/pkg/dnsprovider:go_default_library",
+        "//federation/pkg/dnsprovider/rrstype:go_default_library",
+        "//federation/pkg/federation-controller/service/ingress:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+        "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
+        "//vendor/k8s.io/client-go/util/workqueue:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/service/dns/dns.go

@@ -0,0 +1,549 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dns
+
+import (
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pkgruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
+	corelisters "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/flowcontrol"
+	"k8s.io/client-go/util/workqueue"
+	fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	"k8s.io/kubernetes/federation/pkg/dnsprovider"
+	"k8s.io/kubernetes/federation/pkg/dnsprovider/rrstype"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/service/ingress"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
+
+	"github.com/golang/glog"
+)
+
+const (
+	ControllerName = "service-dns"
+
+	UserAgentName = "federation-service-dns-controller"
+
+	// minDNSTTL is the minimum safe DNS TTL value to use (in seconds).  We use this as the TTL for all DNS records.
+	minDNSTTL = 180
+
+	serviceSyncPeriod = 30 * time.Second
+)
+
+type ServiceDNSController struct {
+	// Client to federation api server
+	federationClient fedclientset.Interface
+	dns              dnsprovider.Interface
+	federationName   string
+	// serviceDNSSuffix is the DNS suffix we use when publishing service DNS names
+	serviceDNSSuffix string
+	// zoneName and zoneID are used to identify the zone in which to put records
+	zoneName string
+	zoneID   string
+	dnsZones dnsprovider.Zones
+	// each federation should be configured with a single zone (e.g. "mycompany.com")
+	dnsZone dnsprovider.Zone
+	// Informer Store for federated services
+	serviceStore corelisters.ServiceLister
+	// Informer controller for federated services
+	serviceController  cache.Controller
+	workQueue          workqueue.Interface
+	objectDeliverer    *util.DelayingDeliverer
+	flowcontrolBackoff *flowcontrol.Backoff
+}
+
+// NewServiceDNSController returns a new service dns controller to manage DNS records for federated services
+func NewServiceDNSController(client fedclientset.Interface, dnsProvider, dnsProviderConfig, federationName,
+	serviceDNSSuffix, zoneName, zoneID string) (*ServiceDNSController, error) {
+	dns, err := dnsprovider.InitDnsProvider(dnsProvider, dnsProviderConfig)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("DNS provider could not be initialized: %v", err))
+		return nil, err
+	}
+	d := &ServiceDNSController{
+		federationClient:   client,
+		dns:                dns,
+		federationName:     federationName,
+		serviceDNSSuffix:   serviceDNSSuffix,
+		zoneName:           zoneName,
+		zoneID:             zoneID,
+		workQueue:          workqueue.New(),
+		objectDeliverer:    util.NewDelayingDeliverer(),
+		flowcontrolBackoff: flowcontrol.NewBackOff(5*time.Second, time.Minute),
+	}
+	if err := d.validateConfig(); err != nil {
+		runtime.HandleError(fmt.Errorf("Invalid configuration passed to DNS provider: %v", err))
+		return nil, err
+	}
+	if err := d.retrieveOrCreateDNSZone(); err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to retrieve DNS zone: %v", err))
+		return nil, err
+	}
+
+	// Start informer in federated API servers on federated services
+	var serviceIndexer cache.Indexer
+	serviceIndexer, d.serviceController = cache.NewIndexerInformer(
+		&cache.ListWatch{
+			ListFunc: func(options metav1.ListOptions) (pkgruntime.Object, error) {
+				return client.Core().Services(metav1.NamespaceAll).List(options)
+			},
+			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+				return client.Core().Services(metav1.NamespaceAll).Watch(options)
+			},
+		},
+		&v1.Service{},
+		serviceSyncPeriod,
+		util.NewTriggerOnAllChanges(func(obj pkgruntime.Object) { d.workQueue.Add(obj) }),
+		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
+	)
+	d.serviceStore = corelisters.NewServiceLister(serviceIndexer)
+
+	return d, nil
+}
+
+func (s *ServiceDNSController) DNSControllerRun(workers int, stopCh <-chan struct{}) {
+	defer runtime.HandleCrash()
+	defer s.workQueue.ShutDown()
+
+	glog.Infof("Starting federation service dns controller")
+
+	s.objectDeliverer.StartWithHandler(func(item *util.DelayingDelivererItem) {
+		s.workQueue.Add(item.Value.(*v1.Service))
+	})
+	defer s.objectDeliverer.Stop()
+
+	util.StartBackoffGC(s.flowcontrolBackoff, stopCh)
+	go s.serviceController.Run(stopCh)
+
+	for i := 0; i < workers; i++ {
+		go wait.Until(s.worker, time.Second, stopCh)
+	}
+
+	<-stopCh
+	glog.Infof("Stopping federation service dns controller")
+}
+
+// Adds backoff to delay if this delivery is related to some failure. Resets backoff if there was no failure.
+func (s *ServiceDNSController) deliverService(service *v1.Service, delay time.Duration, failed bool) {
+	if failed {
+		s.flowcontrolBackoff.Next(service.String(), time.Now())
+		delay = delay + s.flowcontrolBackoff.Get(service.String())
+	} else {
+		s.flowcontrolBackoff.Reset(service.String())
+	}
+	s.objectDeliverer.DeliverAfter(service.String(), service, delay)
+}
+
+func wantsDNSRecords(service *v1.Service) bool {
+	return service.Spec.Type == v1.ServiceTypeLoadBalancer
+}
+
+func (s *ServiceDNSController) workerFunction() bool {
+	item, quit := s.workQueue.Get()
+	if quit {
+		return true
+	}
+	defer s.workQueue.Done(item)
+
+	service := item.(*v1.Service)
+
+	if !wantsDNSRecords(service) {
+		return false
+	}
+
+	ingress, err := ingress.ParseFederatedServiceIngress(service)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Error in parsing lb ingress for service %s/%s: %v", service.Namespace, service.Name, err))
+		return false
+	}
+	for _, clusterIngress := range ingress.Items {
+		err = s.ensureDNSRecords(clusterIngress.Cluster, service)
+		if err != nil {
+			runtime.HandleError(fmt.Errorf("Error when ensuring DNS records for service %s/%s: %v", service.Namespace, service.Name, err))
+			s.deliverService(service, 0, true)
+		}
+	}
+	return false
+}
+
+func (s *ServiceDNSController) worker() {
+	for {
+		if quit := s.workerFunction(); quit {
+			glog.Infof("service dns controller worker queue shutting down")
+			return
+		}
+	}
+}
+
+func (s *ServiceDNSController) validateConfig() error {
+	if s.federationName == "" {
+		return fmt.Errorf("DNSController should not be run without federationName")
+	}
+	if s.zoneName == "" && s.zoneID == "" {
+		return fmt.Errorf("DNSController must be run with either zoneName or zoneID")
+	}
+	if s.serviceDNSSuffix == "" {
+		if s.zoneName == "" {
+			return fmt.Errorf("DNSController must be run with zoneName, if serviceDnsSuffix is not set")
+		}
+		s.serviceDNSSuffix = s.zoneName
+	}
+	if s.dns == nil {
+		return fmt.Errorf("DNSController should not be run without a dnsprovider")
+	}
+	zones, ok := s.dns.Zones()
+	if !ok {
+		return fmt.Errorf("the dns provider does not support zone enumeration, which is required for creating dns records")
+	}
+	s.dnsZones = zones
+	return nil
+}
+
+func (s *ServiceDNSController) retrieveOrCreateDNSZone() error {
+	matchingZones, err := getDNSZones(s.zoneName, s.zoneID, s.dnsZones)
+	if err != nil {
+		return fmt.Errorf("error querying for DNS zones: %v", err)
+	}
+	switch len(matchingZones) {
+	case 0: // No matching zones for s.zoneName, so create one
+		if s.zoneName == "" {
+			return fmt.Errorf("DNSController must be run with zoneName to create zone automatically")
+		}
+		glog.Infof("DNS zone %q not found.  Creating DNS zone %q.", s.zoneName, s.zoneName)
+		managedZone, err := s.dnsZones.New(s.zoneName)
+		if err != nil {
+			return err
+		}
+		zone, err := s.dnsZones.Add(managedZone)
+		if err != nil {
+			return err
+		}
+		glog.Infof("DNS zone %q successfully created.  Note that DNS resolution will not work until you have registered this name with "+
+			"a DNS registrar and they have changed the authoritative name servers for your domain to point to your DNS provider", zone.Name())
+	case 1: // s.zoneName matches exactly one DNS zone
+		s.dnsZone = matchingZones[0]
+	default: // s.zoneName matches more than one DNS zone
+		return fmt.Errorf("Multiple matching DNS zones found for %q; please specify zoneID", s.zoneName)
+	}
+	return nil
+}
+
+// getHealthyEndpoints returns the hostnames and/or IP addresses of healthy endpoints for the service, at a zone, region and global level (or an error)
+func (s *ServiceDNSController) getHealthyEndpoints(clusterName string, service *v1.Service) (zoneEndpoints, regionEndpoints, globalEndpoints []string, err error) {
+	var (
+		zoneNames  []string
+		regionName string
+	)
+	if zoneNames, regionName, err = s.getClusterZoneNames(clusterName); err != nil {
+		return nil, nil, nil, err
+	}
+
+	// If federated service is deleted, return empty endpoints, so that DNS records are removed
+	if service.DeletionTimestamp != nil {
+		return zoneEndpoints, regionEndpoints, globalEndpoints, nil
+	}
+
+	serviceIngress, err := ingress.ParseFederatedServiceIngress(service)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	for _, lbClusterIngress := range serviceIngress.Items {
+		lbClusterName := lbClusterIngress.Cluster
+		lbZoneNames, lbRegionName, err := s.getClusterZoneNames(lbClusterName)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		for _, ingress := range lbClusterIngress.Items {
+			var address string
+			// We should get either an IP address or a hostname - use whichever one we get
+			if ingress.IP != "" {
+				address = ingress.IP
+			} else if ingress.Hostname != "" {
+				address = ingress.Hostname
+			}
+			if len(address) <= 0 {
+				return nil, nil, nil, fmt.Errorf("Service %s/%s in cluster %s has neither LoadBalancerStatus.ingress.ip nor LoadBalancerStatus.ingress.hostname. Cannot use it as endpoint for federated service",
+					service.Name, service.Namespace, clusterName)
+			}
+			for _, lbZoneName := range lbZoneNames {
+				for _, zoneName := range zoneNames {
+					if lbZoneName == zoneName {
+						zoneEndpoints = append(zoneEndpoints, address)
+					}
+				}
+			}
+			if lbRegionName == regionName {
+				regionEndpoints = append(regionEndpoints, address)
+			}
+			globalEndpoints = append(globalEndpoints, address)
+		}
+	}
+	return zoneEndpoints, regionEndpoints, globalEndpoints, nil
+}
+
+// getClusterZoneNames returns the name of the zones (and the region) where the specified cluster exists (e.g. zones "us-east1-c" on GCE, or "us-east-1b" on AWS)
+func (s *ServiceDNSController) getClusterZoneNames(clusterName string) ([]string, string, error) {
+	cluster, err := s.federationClient.Federation().Clusters().Get(clusterName, metav1.GetOptions{})
+	if err != nil {
+		return nil, "", err
+	}
+	return cluster.Status.Zones, cluster.Status.Region, nil
+}
+
+// getDNSZones returns the DNS zones matching dnsZoneName and dnsZoneID (if specified)
+func getDNSZones(dnsZoneName string, dnsZoneID string, dnsZonesInterface dnsprovider.Zones) ([]dnsprovider.Zone, error) {
+	// TODO: We need query-by-name and query-by-id functions
+	dnsZones, err := dnsZonesInterface.List()
+	if err != nil {
+		return nil, err
+	}
+
+	var matches []dnsprovider.Zone
+	findName := strings.TrimSuffix(dnsZoneName, ".")
+	for _, dnsZone := range dnsZones {
+		if dnsZoneID != "" {
+			if dnsZoneID != dnsZone.ID() {
+				continue
+			}
+		}
+		if findName != "" {
+			if strings.TrimSuffix(dnsZone.Name(), ".") != findName {
+				continue
+			}
+		}
+		matches = append(matches, dnsZone)
+	}
+
+	return matches, nil
+}
+
+// NOTE: that if the named resource record set does not exist, but no
+// error occurred, the returned list will be empty, and the error will
+// be nil
+func getRrset(dnsName string, rrsetsInterface dnsprovider.ResourceRecordSets) ([]dnsprovider.ResourceRecordSet, error) {
+	return rrsetsInterface.Get(dnsName)
+}
+
+func findRrset(list []dnsprovider.ResourceRecordSet, rrset dnsprovider.ResourceRecordSet) dnsprovider.ResourceRecordSet {
+	for i, elem := range list {
+		if dnsprovider.ResourceRecordSetsEquivalent(rrset, elem) {
+			return list[i]
+		}
+	}
+	return nil
+}
+
+/* getResolvedEndpoints performs DNS resolution on the provided slice of endpoints (which might be DNS names or IPv4 addresses)
+   and returns a list of IPv4 addresses.  If any of the endpoints are neither valid IPv4 addresses nor resolvable DNS names,
+   non-nil error is also returned (possibly along with a partially complete list of resolved endpoints.
+*/
+func getResolvedEndpoints(endpoints []string) ([]string, error) {
+	resolvedEndpoints := sets.String{}
+	for _, endpoint := range endpoints {
+		if net.ParseIP(endpoint) == nil {
+			// It's not a valid IP address, so assume it's a DNS name, and try to resolve it,
+			// replacing its DNS name with its IP addresses in expandedEndpoints
+			ipAddrs, err := net.LookupHost(endpoint)
+			if err != nil {
+				return resolvedEndpoints.List(), err
+			}
+			for _, ip := range ipAddrs {
+				resolvedEndpoints = resolvedEndpoints.Union(sets.NewString(ip))
+			}
+		} else {
+			resolvedEndpoints = resolvedEndpoints.Union(sets.NewString(endpoint))
+		}
+	}
+	return resolvedEndpoints.List(), nil
+}
+
+/* ensureDNSRrsets ensures (idempotently, and with minimum mutations) that all of the DNS resource record sets for dnsName are consistent with endpoints.
+   if endpoints is nil or empty, a CNAME record to uplevelCname is ensured.
+*/
+func (s *ServiceDNSController) ensureDNSRrsets(dnsZone dnsprovider.Zone, dnsName string, endpoints []string, uplevelCname string) error {
+	rrsets, supported := dnsZone.ResourceRecordSets()
+	if !supported {
+		return fmt.Errorf("Failed to ensure DNS records for %s. DNS provider does not support the ResourceRecordSets interface", dnsName)
+	}
+	rrsetList, err := getRrset(dnsName, rrsets) // TODO: rrsets.Get(dnsName)
+	if err != nil {
+		return err
+	}
+	if len(rrsetList) == 0 {
+		glog.V(4).Infof("No recordsets found for DNS name %q.  Need to add either A records (if we have healthy endpoints), or a CNAME record to %q", dnsName, uplevelCname)
+		if len(endpoints) < 1 {
+			glog.V(4).Infof("There are no healthy endpoint addresses at level %q, so CNAME to %q, if provided", dnsName, uplevelCname)
+			if uplevelCname != "" {
+				glog.V(4).Infof("Creating CNAME to %q for %q", uplevelCname, dnsName)
+				newRrset := rrsets.New(dnsName, []string{uplevelCname}, minDNSTTL, rrstype.CNAME)
+				glog.V(4).Infof("Adding recordset %v", newRrset)
+				err = rrsets.StartChangeset().Add(newRrset).Apply()
+				if err != nil {
+					return err
+				}
+				glog.V(4).Infof("Successfully created CNAME to %q for %q", uplevelCname, dnsName)
+			} else {
+				glog.V(4).Infof("We want no record for %q, and we have no record, so we're all good.", dnsName)
+			}
+		} else {
+			// We have valid endpoint addresses, so just add them as A records.
+			// But first resolve DNS names, as some cloud providers (like AWS) expose
+			// load balancers behind DNS names, not IP addresses.
+			glog.V(4).Infof("We have valid endpoint addresses %v at level %q, so add them as A records, after resolving DNS names", endpoints, dnsName)
+			resolvedEndpoints, err := getResolvedEndpoints(endpoints)
+			if err != nil {
+				return err // TODO: We could potentially add the ones we did get back, even if some of them failed to resolve.
+			}
+			newRrset := rrsets.New(dnsName, resolvedEndpoints, minDNSTTL, rrstype.A)
+			glog.V(4).Infof("Adding recordset %v", newRrset)
+			err = rrsets.StartChangeset().Add(newRrset).Apply()
+			if err != nil {
+				return err
+			}
+			glog.V(4).Infof("Successfully added recordset %v", newRrset)
+		}
+	} else {
+		// the rrsets already exists, so make it right.
+		glog.V(4).Infof("Recordset %v already exists. Ensuring that it is correct.", rrsetList)
+		if len(endpoints) < 1 {
+			// Need an appropriate CNAME record.  Check that we have it.
+			newRrset := rrsets.New(dnsName, []string{uplevelCname}, minDNSTTL, rrstype.CNAME)
+			glog.V(4).Infof("No healthy endpoints for %s. Have recordsets %v. Need recordset %v", dnsName, rrsetList, newRrset)
+			found := findRrset(rrsetList, newRrset)
+			if found != nil {
+				// The existing rrset is equivalent to the required one - our work is done here
+				glog.V(4).Infof("Existing recordset %v is equivalent to needed recordset %v, our work is done here.", rrsetList, newRrset)
+				return nil
+			} else {
+				// Need to replace the existing one with a better one (or just remove it if we have no healthy endpoints).
+				glog.V(4).Infof("Existing recordset %v not equivalent to needed recordset %v removing existing and adding needed.", rrsetList, newRrset)
+				changeSet := rrsets.StartChangeset()
+				for i := range rrsetList {
+					changeSet = changeSet.Remove(rrsetList[i])
+				}
+				if uplevelCname != "" {
+					changeSet = changeSet.Add(newRrset)
+					if err := changeSet.Apply(); err != nil {
+						return err
+					}
+					glog.V(4).Infof("Successfully replaced needed recordset %v -> %v", found, newRrset)
+				} else {
+					if err := changeSet.Apply(); err != nil {
+						return err
+					}
+					glog.V(4).Infof("Successfully removed existing recordset %v", found)
+					glog.V(4).Infof("Uplevel CNAME is empty string. Not adding recordset %v", newRrset)
+				}
+			}
+		} else {
+			// We have an rrset in DNS, possibly with some missing addresses and some unwanted addresses.
+			// And we have healthy endpoints.  Just replace what's there with the healthy endpoints, if it's not already correct.
+			glog.V(4).Infof("%s: Healthy endpoints %v exist. Recordset %v exists.  Reconciling.", dnsName, endpoints, rrsetList)
+			resolvedEndpoints, err := getResolvedEndpoints(endpoints)
+			if err != nil { // Some invalid addresses or otherwise unresolvable DNS names.
+				return err // TODO: We could potentially add the ones we did get back, even if some of them failed to resolve.
+			}
+			newRrset := rrsets.New(dnsName, resolvedEndpoints, minDNSTTL, rrstype.A)
+			glog.V(4).Infof("Have recordset %v. Need recordset %v", rrsetList, newRrset)
+			found := findRrset(rrsetList, newRrset)
+			if found != nil {
+				glog.V(4).Infof("Existing recordset %v is equivalent to needed recordset %v, our work is done here.", found, newRrset)
+				// TODO: We could be more thorough about checking for equivalence to avoid unnecessary updates, but in the
+				//       worst case we'll just replace what's there with an equivalent, if not exactly identical record set.
+				return nil
+			} else {
+				// Need to replace the existing one with a better one
+				glog.V(4).Infof("Existing recordset %v is not equivalent to needed recordset %v, removing existing and adding needed.", found, newRrset)
+				changeSet := rrsets.StartChangeset()
+				for i := range rrsetList {
+					changeSet = changeSet.Remove(rrsetList[i])
+				}
+				changeSet = changeSet.Add(newRrset)
+				if err = changeSet.Apply(); err != nil {
+					return err
+				}
+				glog.V(4).Infof("Successfully replaced recordset %v -> %v", found, newRrset)
+			}
+		}
+	}
+	return nil
+}
+
+/* ensureDNSRecords ensures (idempotently, and with minimum mutations) that all of the DNS records for a service in a given cluster are correct,
+given the current state of that service in that cluster.  This should be called every time the state of a service might have changed
+(either w.r.t. its loadbalancer address, or if the number of healthy backend endpoints for that service transitioned from zero to non-zero
+(or vice versa).  Only shards of the service which have both a loadbalancer ingress IP address or hostname AND at least one healthy backend endpoint
+are included in DNS records for that service (at all of zone, region and global levels). All other addresses are removed.  Also, if no shards exist
+in the zone or region of the cluster, a CNAME reference to the next higher level is ensured to exist. */
+func (s *ServiceDNSController) ensureDNSRecords(clusterName string, service *v1.Service) error {
+	// Quinton: Pseudocode....
+	// See https://github.com/kubernetes/kubernetes/pull/25107#issuecomment-218026648
+	// For each service we need the following DNS names:
+	// mysvc.myns.myfed.svc.z1.r1.mydomain.com  (for zone z1 in region r1)
+	//         - an A record to IP address of specific shard in that zone (if that shard exists and has healthy endpoints)
+	//         - OR a CNAME record to the next level up, i.e. mysvc.myns.myfed.svc.r1.mydomain.com  (if a healthy shard does not exist in zone z1)
+	// mysvc.myns.myfed.svc.r1.mydomain.com
+	//         - a set of A records to IP addresses of all healthy shards in region r1, if one or more of these exist
+	//         - OR a CNAME record to the next level up, i.e. mysvc.myns.myfed.svc.mydomain.com (if no healthy shards exist in region r1)
+	// mysvc.myns.myfed.svc.mydomain.com
+	//         - a set of A records to IP addresses of all healthy shards in all regions, if one or more of these exist.
+	//         - no record (NXRECORD response) if no healthy shards exist in any regions
+	//
+	// Each service has the current known state of loadbalancer ingress for the federated cluster stored in annotations.
+	// So generate the DNS records based on the current state and ensure those desired DNS records match the
+	// actual DNS records (add new records, remove deleted records, and update changed records).
+	//
+	serviceName := service.Name
+	namespaceName := service.Namespace
+	zoneNames, regionName, err := s.getClusterZoneNames(clusterName)
+	if err != nil {
+		return err
+	}
+	if zoneNames == nil {
+		return fmt.Errorf("failed to get cluster zone names")
+	}
+	commonPrefix := serviceName + "." + namespaceName + "." + s.federationName + ".svc"
+	// dnsNames is the path up the DNS search tree, starting at the leaf
+	dnsNames := []string{
+		strings.Join([]string{commonPrefix, zoneNames[0], regionName, s.serviceDNSSuffix}, "."), // zone level - TODO might need other zone names for multi-zone clusters
+		strings.Join([]string{commonPrefix, regionName, s.serviceDNSSuffix}, "."),               // region level, one up from zone level
+		strings.Join([]string{commonPrefix, s.serviceDNSSuffix}, "."),                           // global level, one up from region level
+		"", // nowhere to go up from global level
+	}
+
+	zoneEndpoints, regionEndpoints, globalEndpoints, err := s.getHealthyEndpoints(clusterName, service)
+	if err != nil {
+		return err
+	}
+	endpoints := [][]string{zoneEndpoints, regionEndpoints, globalEndpoints}
+	for i, endpoint := range endpoints {
+		if err = s.ensureDNSRrsets(s.dnsZone, dnsNames[i], endpoint, dnsNames[i+1]); err != nil {
+			return err
+		}
+	}
+	return nil
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/service/dns/dns_test.go

@@ -0,0 +1,256 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dns
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	fakefedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset/fake"
+	"k8s.io/kubernetes/federation/pkg/dnsprovider/providers/google/clouddns" // Only for unit testing purposes.
+	"k8s.io/kubernetes/federation/pkg/federation-controller/service/ingress"
+	. "k8s.io/kubernetes/federation/pkg/federation-controller/util/test"
+)
+
+// NewClusterWithRegionZone builds a new cluster object with given region and zone attributes.
+func NewClusterWithRegionZone(name string, readyStatus v1.ConditionStatus, region, zone string) *v1beta1.Cluster {
+	cluster := NewCluster(name, readyStatus)
+	cluster.Status.Zones = []string{zone}
+	cluster.Status.Region = region
+	return cluster
+}
+
+func TestServiceController_ensureDnsRecords(t *testing.T) {
+	cluster1Name := "c1"
+	cluster2Name := "c2"
+	cluster1 := NewClusterWithRegionZone(cluster1Name, v1.ConditionTrue, "fooregion", "foozone")
+	cluster2 := NewClusterWithRegionZone(cluster2Name, v1.ConditionTrue, "barregion", "barzone")
+	globalDNSName := "servicename.servicenamespace.myfederation.svc.federation.example.com"
+	fooRegionDNSName := "servicename.servicenamespace.myfederation.svc.fooregion.federation.example.com"
+	fooZoneDNSName := "servicename.servicenamespace.myfederation.svc.foozone.fooregion.federation.example.com"
+	barRegionDNSName := "servicename.servicenamespace.myfederation.svc.barregion.federation.example.com"
+	barZoneDNSName := "servicename.servicenamespace.myfederation.svc.barzone.barregion.federation.example.com"
+
+	tests := []struct {
+		name     string
+		service  v1.Service
+		expected []string
+	}{
+		{
+			name: "ServiceWithSingleLBIngress",
+			service: v1.Service{
+				ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{
+					ingress.FederatedServiceIngressAnnotation: ingress.NewFederatedServiceIngress().
+						AddEndpoints(cluster1Name, []string{"198.51.100.1"}).
+						AddEndpoints(cluster2Name, []string{}).
+						String()},
+				},
+			},
+			expected: []string{
+				"example.com:" + globalDNSName + ":A:180:[198.51.100.1]",
+				"example.com:" + fooRegionDNSName + ":A:180:[198.51.100.1]",
+				"example.com:" + fooZoneDNSName + ":A:180:[198.51.100.1]",
+				"example.com:" + barRegionDNSName + ":CNAME:180:[" + globalDNSName + "]",
+				"example.com:" + barZoneDNSName + ":CNAME:180:[" + barRegionDNSName + "]",
+			},
+		},
+		/*
+			TODO: getResolvedEndpoints preforms DNS lookup.
+			Mock and maybe look at error handling when some endpoints resolve, but also caching?
+			{
+				name: "withname",
+				service: v1.Service{
+					ObjectMeta: metav1.ObjectMeta{},
+				},
+				expected: []string{
+					"example.com:"+globalDNSName+":A:180:[198.51.100.1]",
+					"example.com:"+fooRegionDNSName+":A:180:[198.51.100.1]",
+					"example.com:"+fooZoneDNSName+":A:180:[198.51.100.1]",
+				},
+			},
+		*/
+		{
+			name: "ServiceWithNoLBIngress",
+			service: v1.Service{
+				ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{
+					ingress.FederatedServiceIngressAnnotation: ingress.NewFederatedServiceIngress().
+						AddEndpoints(cluster1Name, []string{}).
+						AddEndpoints(cluster2Name, []string{}).
+						String()},
+				},
+			},
+			expected: []string{
+				"example.com:" + fooRegionDNSName + ":CNAME:180:[" + globalDNSName + "]",
+				"example.com:" + fooZoneDNSName + ":CNAME:180:[" + fooRegionDNSName + "]",
+				"example.com:" + barRegionDNSName + ":CNAME:180:[" + globalDNSName + "]",
+				"example.com:" + barZoneDNSName + ":CNAME:180:[" + barRegionDNSName + "]",
+			},
+		},
+		{
+			name: "ServiceWithMultipleLBIngress",
+			service: v1.Service{
+				ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{
+					ingress.FederatedServiceIngressAnnotation: ingress.NewFederatedServiceIngress().
+						AddEndpoints(cluster1Name, []string{"198.51.100.1"}).
+						AddEndpoints(cluster2Name, []string{"198.51.200.1"}).
+						String()},
+				},
+			},
+			expected: []string{
+				"example.com:" + globalDNSName + ":A:180:[198.51.100.1 198.51.200.1]",
+				"example.com:" + fooRegionDNSName + ":A:180:[198.51.100.1]",
+				"example.com:" + fooZoneDNSName + ":A:180:[198.51.100.1]",
+				"example.com:" + barRegionDNSName + ":A:180:[198.51.200.1]",
+				"example.com:" + barZoneDNSName + ":A:180:[198.51.200.1]",
+			},
+		},
+		{
+			name: "ServiceWithLBIngressAndServiceDeleted",
+			service: v1.Service{
+				ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{
+					ingress.FederatedServiceIngressAnnotation: ingress.NewFederatedServiceIngress().
+						AddEndpoints(cluster1Name, []string{"198.51.100.1"}).
+						AddEndpoints(cluster2Name, []string{"198.51.200.1"}).
+						String()},
+					DeletionTimestamp: &metav1.Time{Time: time.Now()},
+				},
+			},
+			expected: []string{
+				// TODO: Ideally we should expect that there are no DNS records when federated service is deleted. Need to remove these leaks in future
+				"example.com:" + fooRegionDNSName + ":CNAME:180:[" + globalDNSName + "]",
+				"example.com:" + fooZoneDNSName + ":CNAME:180:[" + fooRegionDNSName + "]",
+				"example.com:" + barRegionDNSName + ":CNAME:180:[" + globalDNSName + "]",
+				"example.com:" + barZoneDNSName + ":CNAME:180:[" + barRegionDNSName + "]",
+			},
+		},
+		{
+			name: "ServiceWithMultipleLBIngressAndOneLBIngressGettingRemoved",
+			service: v1.Service{
+				ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{
+					ingress.FederatedServiceIngressAnnotation: ingress.NewFederatedServiceIngress().
+						AddEndpoints(cluster1Name, []string{"198.51.100.1"}).
+						AddEndpoints(cluster2Name, []string{"198.51.200.1"}).
+						RemoveEndpoint(cluster2Name, "198.51.200.1").
+						String()},
+				},
+			},
+			expected: []string{
+				"example.com:" + globalDNSName + ":A:180:[198.51.100.1]",
+				"example.com:" + fooRegionDNSName + ":A:180:[198.51.100.1]",
+				"example.com:" + fooZoneDNSName + ":A:180:[198.51.100.1]",
+				"example.com:" + barRegionDNSName + ":CNAME:180:[" + globalDNSName + "]",
+				"example.com:" + barZoneDNSName + ":CNAME:180:[" + barRegionDNSName + "]",
+			},
+		},
+		{
+			name: "ServiceWithMultipleLBIngressAndAllLBIngressGettingRemoved",
+			service: v1.Service{
+				ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{
+					ingress.FederatedServiceIngressAnnotation: ingress.NewFederatedServiceIngress().
+						AddEndpoints(cluster1Name, []string{"198.51.100.1"}).
+						AddEndpoints(cluster2Name, []string{"198.51.200.1"}).
+						RemoveEndpoint(cluster1Name, "198.51.100.1").
+						RemoveEndpoint(cluster2Name, "198.51.200.1").
+						String()},
+				},
+			},
+			expected: []string{
+				"example.com:" + fooRegionDNSName + ":CNAME:180:[" + globalDNSName + "]",
+				"example.com:" + fooZoneDNSName + ":CNAME:180:[" + fooRegionDNSName + "]",
+				"example.com:" + barRegionDNSName + ":CNAME:180:[" + globalDNSName + "]",
+				"example.com:" + barZoneDNSName + ":CNAME:180:[" + barRegionDNSName + "]",
+			},
+		},
+	}
+	for _, test := range tests {
+		fakedns, _ := clouddns.NewFakeInterface()
+		fakednsZones, ok := fakedns.Zones()
+		if !ok {
+			t.Error("Unable to fetch zones")
+		}
+		fakeClient := &fakefedclientset.Clientset{}
+		RegisterFakeClusterGet(&fakeClient.Fake, &v1beta1.ClusterList{Items: []v1beta1.Cluster{*cluster1, *cluster2}})
+		d := ServiceDNSController{
+			federationClient: fakeClient,
+			dns:              fakedns,
+			dnsZones:         fakednsZones,
+			serviceDNSSuffix: "federation.example.com",
+			zoneName:         "example.com",
+			federationName:   "myfederation",
+		}
+
+		dnsZones, err := getDNSZones(d.zoneName, d.zoneID, d.dnsZones)
+		if err != nil {
+			t.Errorf("Test failed for %s, Get DNS Zones failed: %v", test.name, err)
+		}
+		d.dnsZone = dnsZones[0]
+		test.service.Name = "servicename"
+		test.service.Namespace = "servicenamespace"
+
+		ingress, err := ingress.ParseFederatedServiceIngress(&test.service)
+		if err != nil {
+			t.Errorf("Error in parsing lb ingress for service %s/%s: %v", test.service.Namespace, test.service.Name, err)
+			return
+		}
+		for _, clusterIngress := range ingress.Items {
+			d.ensureDNSRecords(clusterIngress.Cluster, &test.service)
+		}
+
+		zones, err := fakednsZones.List()
+		if err != nil {
+			t.Errorf("error querying zones: %v", err)
+		}
+
+		// Dump every record to a testable-by-string-comparison form
+		records := []string{}
+		for _, z := range zones {
+			zoneName := z.Name()
+
+			rrs, ok := z.ResourceRecordSets()
+			if !ok {
+				t.Errorf("cannot get rrs for zone %q", zoneName)
+			}
+
+			rrList, err := rrs.List()
+			if err != nil {
+				t.Errorf("error querying rr for zone %q: %v", zoneName, err)
+			}
+			for _, rr := range rrList {
+				rrdatas := rr.Rrdatas()
+
+				// Put in consistent (testable-by-string-comparison) order
+				sort.Strings(rrdatas)
+				records = append(records, fmt.Sprintf("%s:%s:%s:%d:%s", zoneName, rr.Name(), rr.Type(), rr.Ttl(), rrdatas))
+			}
+		}
+
+		// Ignore order of records
+		sort.Strings(records)
+		sort.Strings(test.expected)
+
+		if !reflect.DeepEqual(records, test.expected) {
+			t.Errorf("Test %q failed.  Actual=%v, Expected=%v", test.name, records, test.expected)
+		}
+
+	}
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/service/ingress/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["ingress.go"],
+    deps = [
+        "//federation/apis/federation:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/service/ingress/ingress.go

@@ -0,0 +1,136 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ingress
+
+import (
+	"encoding/json"
+	"sort"
+	"strings"
+
+	"k8s.io/api/core/v1"
+	fedapi "k8s.io/kubernetes/federation/apis/federation"
+)
+
+// Compile time check for interface adherence
+var _ sort.Interface = &FederatedServiceIngress{}
+
+const (
+	FederatedServiceIngressAnnotation = "federation.kubernetes.io/service-ingresses"
+)
+
+// FederatedServiceIngress implements sort.Interface.
+type FederatedServiceIngress struct {
+	fedapi.FederatedServiceIngress
+}
+
+func NewFederatedServiceIngress() *FederatedServiceIngress {
+	return &FederatedServiceIngress{}
+}
+
+func (ingress *FederatedServiceIngress) String() string {
+	annotationBytes, _ := json.Marshal(ingress)
+	return string(annotationBytes[:])
+}
+
+// Len is to satisfy of sort.Interface.
+func (ingress *FederatedServiceIngress) Len() int {
+	return len(ingress.Items)
+}
+
+// Less is to satisfy of sort.Interface.
+func (ingress *FederatedServiceIngress) Less(i, j int) bool {
+	return (strings.Compare(ingress.Items[i].Cluster, ingress.Items[j].Cluster) < 0)
+}
+
+// Swap is to satisfy of sort.Interface.
+func (ingress *FederatedServiceIngress) Swap(i, j int) {
+	ingress.Items[i].Cluster, ingress.Items[j].Cluster = ingress.Items[j].Cluster, ingress.Items[i].Cluster
+	ingress.Items[i].Items, ingress.Items[j].Items = ingress.Items[j].Items, ingress.Items[i].Items
+}
+
+// GetClusterLoadBalancerIngresses returns loadbalancer ingresses for given cluster if exist otherwise returns an empty slice
+func (ingress *FederatedServiceIngress) GetClusterLoadBalancerIngresses(cluster string) []v1.LoadBalancerIngress {
+	for _, clusterIngress := range ingress.Items {
+		if cluster == clusterIngress.Cluster {
+			return clusterIngress.Items
+		}
+	}
+	return []v1.LoadBalancerIngress{}
+}
+
+// AddClusterLoadBalancerIngresses adds the ladbalancer ingresses for a given cluster to federated service ingress
+func (ingress *FederatedServiceIngress) AddClusterLoadBalancerIngresses(cluster string, loadbalancerIngresses []v1.LoadBalancerIngress) {
+	for i, clusterIngress := range ingress.Items {
+		if cluster == clusterIngress.Cluster {
+			ingress.Items[i].Items = append(ingress.Items[i].Items, loadbalancerIngresses...)
+			return
+		}
+	}
+	clusterNewIngress := fedapi.ClusterServiceIngress{Cluster: cluster, Items: loadbalancerIngresses}
+	ingress.Items = append(ingress.Items, clusterNewIngress)
+	sort.Sort(ingress)
+}
+
+// AddEndpoints add one or more endpoints to federated service ingress.
+// endpoints are federated cluster's loadbalancer ip/hostname for the service
+func (ingress *FederatedServiceIngress) AddEndpoints(cluster string, endpoints []string) *FederatedServiceIngress {
+	lbIngress := []v1.LoadBalancerIngress{}
+	for _, endpoint := range endpoints {
+		lbIngress = append(lbIngress, v1.LoadBalancerIngress{IP: endpoint})
+	}
+	ingress.AddClusterLoadBalancerIngresses(cluster, lbIngress)
+	return ingress
+}
+
+// RemoveEndpoint removes a single endpoint (ip/hostname) from the federated service ingress
+func (ingress *FederatedServiceIngress) RemoveEndpoint(cluster string, endpoint string) *FederatedServiceIngress {
+	for i, clusterIngress := range ingress.Items {
+		if cluster == clusterIngress.Cluster {
+			for j, lbIngress := range clusterIngress.Items {
+				if lbIngress.IP == endpoint {
+					ingress.Items[i].Items = append(ingress.Items[i].Items[:j], ingress.Items[i].Items[j+1:]...)
+				}
+			}
+		}
+	}
+	return ingress
+}
+
+// ParseFederatedServiceIngress extracts federated service ingresses from a federated service
+func ParseFederatedServiceIngress(service *v1.Service) (*FederatedServiceIngress, error) {
+	ingress := FederatedServiceIngress{}
+	if service.Annotations == nil {
+		return &ingress, nil
+	}
+	federatedServiceIngressString, found := service.Annotations[FederatedServiceIngressAnnotation]
+	if !found {
+		return &ingress, nil
+	}
+	if err := json.Unmarshal([]byte(federatedServiceIngressString), &ingress); err != nil {
+		return &ingress, err
+	}
+	return &ingress, nil
+}
+
+// UpdateIngressAnnotation updates the federated service with service ingress annotation
+func UpdateIngressAnnotation(service *v1.Service, ingress *FederatedServiceIngress) *v1.Service {
+	if service.Annotations == nil {
+		service.Annotations = make(map[string]string)
+	}
+	service.Annotations[FederatedServiceIngressAnnotation] = ingress.String()
+	return service
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/service/servicecontroller.go

@@ -0,0 +1,735 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package service
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	pkgruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	corelisters "k8s.io/client-go/listers/core/v1"
+	cache "k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/flowcontrol"
+	"k8s.io/client-go/util/workqueue"
+	fedapi "k8s.io/kubernetes/federation/apis/federation"
+	v1beta1 "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/service/ingress"
+	fedutil "k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/clusterselector"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/deletionhelper"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/eventsink"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/controller"
+)
+
+const (
+	serviceSyncPeriod = 30 * time.Second
+
+	UserAgentName = "federation-service-controller"
+
+	reviewDelay           = 10 * time.Second
+	updateTimeout         = 30 * time.Second
+	allClustersKey        = "ALL_CLUSTERS"
+	clusterAvailableDelay = time.Second * 20
+	ControllerName        = "services"
+)
+
+var (
+	RequiredResources = []schema.GroupVersionResource{v1.SchemeGroupVersion.WithResource("services")}
+)
+
+type ServiceController struct {
+	federationClient fedclientset.Interface
+	// A store of services, populated by the serviceController
+	serviceStore corelisters.ServiceLister
+	// Watches changes to all services
+	serviceController cache.Controller
+	federatedInformer fedutil.FederatedInformer
+	eventBroadcaster  record.EventBroadcaster
+	eventRecorder     record.EventRecorder
+	// services that need to be synced
+	queue *workqueue.Type
+
+	// For triggering all services reconciliation. This is used when
+	// a new cluster becomes available.
+	clusterDeliverer *fedutil.DelayingDeliverer
+
+	deletionHelper *deletionhelper.DeletionHelper
+
+	reviewDelay           time.Duration
+	clusterAvailableDelay time.Duration
+	updateTimeout         time.Duration
+
+	endpointFederatedInformer fedutil.FederatedInformer
+	federatedUpdater          fedutil.FederatedUpdater
+	objectDeliverer           *fedutil.DelayingDeliverer
+	flowcontrolBackoff        *flowcontrol.Backoff
+}
+
+// New returns a new service controller to keep service objects between
+// the federation and member clusters in sync.
+func New(federationClient fedclientset.Interface) *ServiceController {
+	broadcaster := record.NewBroadcaster()
+	broadcaster.StartRecordingToSink(eventsink.NewFederatedEventSink(federationClient))
+	recorder := broadcaster.NewRecorder(api.Scheme, v1.EventSource{Component: UserAgentName})
+
+	s := &ServiceController{
+		federationClient:      federationClient,
+		eventBroadcaster:      broadcaster,
+		eventRecorder:         recorder,
+		queue:                 workqueue.New(),
+		reviewDelay:           reviewDelay,
+		clusterAvailableDelay: clusterAvailableDelay,
+		updateTimeout:         updateTimeout,
+		flowcontrolBackoff:    flowcontrol.NewBackOff(5*time.Second, time.Minute),
+	}
+	s.objectDeliverer = fedutil.NewDelayingDeliverer()
+	s.clusterDeliverer = fedutil.NewDelayingDeliverer()
+	var serviceIndexer cache.Indexer
+	serviceIndexer, s.serviceController = cache.NewIndexerInformer(
+		&cache.ListWatch{
+			ListFunc: func(options metav1.ListOptions) (pkgruntime.Object, error) {
+				return s.federationClient.Core().Services(metav1.NamespaceAll).List(options)
+			},
+			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+				return s.federationClient.Core().Services(metav1.NamespaceAll).Watch(options)
+			},
+		},
+		&v1.Service{},
+		serviceSyncPeriod,
+		fedutil.NewTriggerOnAllChanges(func(obj pkgruntime.Object) {
+			glog.V(5).Infof("Delivering notification from federation: %v", obj)
+			s.deliverObject(obj, 0, false)
+		}),
+		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
+	)
+	s.serviceStore = corelisters.NewServiceLister(serviceIndexer)
+
+	clusterLifecycle := fedutil.ClusterLifecycleHandlerFuncs{
+		ClusterAvailable: func(cluster *v1beta1.Cluster) {
+			s.clusterDeliverer.DeliverAfter(allClustersKey, nil, clusterAvailableDelay)
+		},
+	}
+	fedInformerFactory := func(cluster *v1beta1.Cluster, targetClient kubeclientset.Interface) (cache.Store, cache.Controller) {
+		return cache.NewInformer(
+			&cache.ListWatch{
+				ListFunc: func(options metav1.ListOptions) (pkgruntime.Object, error) {
+					return targetClient.Core().Services(metav1.NamespaceAll).List(options)
+				},
+				WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+					return targetClient.Core().Services(metav1.NamespaceAll).Watch(options)
+				},
+			},
+			&v1.Service{},
+			controller.NoResyncPeriodFunc(),
+			// Trigger reconciliation whenever something in federated cluster is changed. In most cases it
+			// would be just confirmation that some service operation succeeded.
+			fedutil.NewTriggerOnAllChanges(
+				func(obj pkgruntime.Object) {
+					glog.V(5).Infof("Delivering service notification from federated cluster %s: %v", cluster.Name, obj)
+					s.deliverObject(obj, s.reviewDelay, false)
+				},
+			))
+	}
+
+	s.federatedInformer = fedutil.NewFederatedInformer(federationClient, fedInformerFactory, &clusterLifecycle)
+
+	s.federatedUpdater = fedutil.NewFederatedUpdater(s.federatedInformer, "service", updateTimeout, s.eventRecorder,
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			svc := obj.(*v1.Service)
+			_, err := client.Core().Services(svc.Namespace).Create(svc)
+			return err
+		},
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			svc := obj.(*v1.Service)
+			_, err := client.Core().Services(svc.Namespace).Update(svc)
+			return err
+		},
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			svc := obj.(*v1.Service)
+			orphanDependents := false
+			err := client.Core().Services(svc.Namespace).Delete(svc.Name, &metav1.DeleteOptions{OrphanDependents: &orphanDependents})
+			return err
+		})
+
+	// Federated informers on endpoints in federated clusters.
+	// This will enable to check if service ingress endpoints in federated clusters are reachable
+	s.endpointFederatedInformer = fedutil.NewFederatedInformer(
+		federationClient,
+		func(cluster *v1beta1.Cluster, targetClient kubeclientset.Interface) (
+			cache.Store, cache.Controller) {
+			return cache.NewInformer(
+				&cache.ListWatch{
+					ListFunc: func(options metav1.ListOptions) (pkgruntime.Object, error) {
+						return targetClient.Core().Endpoints(metav1.NamespaceAll).List(options)
+					},
+					WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+						return targetClient.Core().Endpoints(metav1.NamespaceAll).Watch(options)
+					},
+				},
+				&v1.Endpoints{},
+				controller.NoResyncPeriodFunc(),
+				fedutil.NewTriggerOnMetaAndFieldChanges(
+					"Subsets",
+					func(obj pkgruntime.Object) {
+						glog.V(5).Infof("Delivering endpoint notification from federated cluster %s :%v", cluster.Name, obj)
+						s.deliverObject(obj, s.reviewDelay, false)
+					},
+				))
+		},
+		&fedutil.ClusterLifecycleHandlerFuncs{},
+	)
+
+	s.deletionHelper = deletionhelper.NewDeletionHelper(
+		s.updateService,
+		// objNameFunc
+		func(obj pkgruntime.Object) string {
+			service := obj.(*v1.Service)
+			return fmt.Sprintf("%s/%s", service.Namespace, service.Name)
+		},
+		s.federatedInformer,
+		s.federatedUpdater,
+	)
+
+	return s
+}
+
+// Sends the given updated object to apiserver.
+// Assumes that the given object is a service.
+func (s *ServiceController) updateService(obj pkgruntime.Object) (pkgruntime.Object, error) {
+	service := obj.(*v1.Service)
+	return s.federationClient.Core().Services(service.Namespace).Update(service)
+}
+
+// Run starts informers, delay deliverers and workers. Workers continuously watch for events which could
+// be from federation or federated clusters and tries to reconcile the service objects from federation to
+// federated clusters.
+func (s *ServiceController) Run(workers int, stopCh <-chan struct{}) {
+	glog.Infof("Starting federation service controller")
+
+	defer runtime.HandleCrash()
+	defer s.queue.ShutDown()
+
+	s.federatedInformer.Start()
+	defer s.federatedInformer.Stop()
+
+	s.endpointFederatedInformer.Start()
+	defer s.endpointFederatedInformer.Stop()
+
+	s.objectDeliverer.StartWithHandler(func(item *fedutil.DelayingDelivererItem) {
+		s.queue.Add(item.Value.(string))
+	})
+	defer s.objectDeliverer.Stop()
+
+	s.clusterDeliverer.StartWithHandler(func(_ *fedutil.DelayingDelivererItem) {
+		s.deliverServicesOnClusterChange()
+	})
+	defer s.clusterDeliverer.Stop()
+
+	fedutil.StartBackoffGC(s.flowcontrolBackoff, stopCh)
+	go s.serviceController.Run(stopCh)
+
+	for i := 0; i < workers; i++ {
+		go wait.Until(s.fedServiceWorker, time.Second, stopCh)
+	}
+
+	<-stopCh
+	glog.Infof("Shutting down federation service controller")
+}
+
+type reconciliationStatus string
+
+const (
+	statusAllOk               = reconciliationStatus("ALL_OK")
+	statusRecoverableError    = reconciliationStatus("RECOVERABLE_ERROR")
+	statusNonRecoverableError = reconciliationStatus("NON_RECOVERABLE_ERROR")
+	statusNotSynced           = reconciliationStatus("NOSYNC")
+)
+
+func (s *ServiceController) workerFunction() bool {
+	key, quit := s.queue.Get()
+	if quit {
+		return true
+	}
+	defer s.queue.Done(key)
+
+	service := key.(string)
+	status := s.reconcileService(service)
+	switch status {
+	case statusAllOk:
+	// do nothing, reconcile is successful.
+	case statusNotSynced:
+		glog.V(5).Infof("Delivering notification for %q after clusterAvailableDelay", service)
+		s.deliverService(service, s.clusterAvailableDelay, false)
+	case statusRecoverableError:
+		s.deliverService(service, 0, true)
+	case statusNonRecoverableError:
+		// do nothing, error is already logged.
+	}
+	return false
+}
+
+// fedServiceWorker runs a worker thread that just dequeues items, processes them, and marks them done.
+func (s *ServiceController) fedServiceWorker() {
+	for {
+		if quit := s.workerFunction(); quit {
+			glog.Infof("service controller worker queue shutting down")
+			return
+		}
+	}
+}
+
+// delete deletes the given service or returns error if the deletion was not complete.
+func (s *ServiceController) delete(service *v1.Service) error {
+	glog.V(3).Infof("Handling deletion of service: %v", *service)
+	_, err := s.deletionHelper.HandleObjectInUnderlyingClusters(service)
+	if err != nil {
+		return err
+	}
+
+	err = s.federationClient.Core().Services(service.Namespace).Delete(service.Name, nil)
+	if err != nil {
+		// Its all good if the error is not found error. That means it is deleted already and we do not have to do anything.
+		// This is expected when we are processing an update as a result of service finalizer deletion.
+		// The process that deleted the last finalizer is also going to delete the service and we do not have to do anything.
+		if !errors.IsNotFound(err) {
+			return fmt.Errorf("failed to delete service: %v", err)
+		}
+	}
+	return nil
+}
+
+func (s *ServiceController) deliverServicesOnClusterChange() {
+	if !s.isSynced() {
+		s.clusterDeliverer.DeliverAfter(allClustersKey, nil, s.clusterAvailableDelay)
+	}
+	glog.V(5).Infof("Delivering all service as cluster status changed")
+	serviceList, err := s.serviceStore.List(labels.Everything())
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("error listing federated services: %v", err))
+		s.clusterDeliverer.DeliverAfter(allClustersKey, nil, 0)
+	}
+	for _, service := range serviceList {
+		s.deliverObject(service, 0, false)
+	}
+}
+
+func (s *ServiceController) deliverObject(object interface{}, delay time.Duration, failed bool) {
+	switch value := object.(type) {
+	case *v1.Service:
+		s.deliverService(types.NamespacedName{Namespace: value.Namespace, Name: value.Name}.String(), delay, failed)
+	case *v1.Endpoints:
+		s.deliverService(types.NamespacedName{Namespace: value.Namespace, Name: value.Name}.String(), delay, failed)
+	default:
+		glog.Warningf("Unknown object received: %v", object)
+	}
+}
+
+// Adds backoff to delay if this delivery is related to some failure. Resets backoff if there was no failure.
+func (s *ServiceController) deliverService(key string, delay time.Duration, failed bool) {
+	if failed {
+		s.flowcontrolBackoff.Next(key, time.Now())
+		delay = delay + s.flowcontrolBackoff.Get(key)
+	} else {
+		s.flowcontrolBackoff.Reset(key)
+	}
+	s.objectDeliverer.DeliverAfter(key, key, delay)
+}
+
+// Check whether all data stores are in sync. False is returned if any of the informer/stores is not yet synced with
+// the corresponding api server.
+func (s *ServiceController) isSynced() bool {
+	if !s.federatedInformer.ClustersSynced() {
+		glog.V(2).Infof("Cluster list not synced")
+		return false
+	}
+	serviceClusters, err := s.federatedInformer.GetReadyClusters()
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get ready clusters: %v", err))
+		return false
+	}
+	if !s.federatedInformer.GetTargetStore().ClustersSynced(serviceClusters) {
+		return false
+	}
+
+	if !s.endpointFederatedInformer.ClustersSynced() {
+		glog.V(2).Infof("Cluster list not synced")
+		return false
+	}
+	endpointClusters, err := s.endpointFederatedInformer.GetReadyClusters()
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get ready clusters: %v", err))
+		return false
+	}
+	if !s.endpointFederatedInformer.GetTargetStore().ClustersSynced(endpointClusters) {
+		return false
+	}
+
+	return true
+}
+
+// reconcileService triggers reconciliation of a federated service with corresponding services in federated clusters.
+// This function is called on service Addition/Deletion/Update either in federated cluster or in federation.
+func (s *ServiceController) reconcileService(key string) reconciliationStatus {
+	if !s.isSynced() {
+		glog.V(4).Infof("Data store not synced, delaying reconcilation: %v", key)
+		return statusNotSynced
+	}
+
+	namespace, name, err := cache.SplitMetaNamespaceKey(key)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Invalid key %q received, unable to split key to namespace and name, err: %v", key, err))
+		return statusNonRecoverableError
+	}
+
+	service, err := s.serviceStore.Services(namespace).Get(name)
+	if errors.IsNotFound(err) {
+		// Not a federated service, ignoring.
+		return statusAllOk
+	} else if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to retrieve federated service %q from store: %v", key, err))
+		return statusRecoverableError
+	}
+
+	glog.V(3).Infof("Reconciling federated service: %s", key)
+
+	// Create a copy before modifying the service to prevent race condition with other readers of service from store
+	fedServiceObj, err := api.Scheme.DeepCopy(service)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Error in copying obj: %s, %v", key, err))
+		return statusNonRecoverableError
+	}
+	fedService, ok := fedServiceObj.(*v1.Service)
+	if err != nil || !ok {
+		runtime.HandleError(fmt.Errorf("Unknown obj received from store: %#v, %v", fedServiceObj, err))
+		return statusNonRecoverableError
+	}
+
+	// Handle deletion of federated service
+	if fedService.DeletionTimestamp != nil {
+		if err := s.delete(fedService); err != nil {
+			runtime.HandleError(fmt.Errorf("Failed to delete %s: %v", key, err))
+			s.eventRecorder.Eventf(fedService, api.EventTypeWarning, "DeleteFailed", "Deleting service failed: %v", err)
+			return statusRecoverableError
+		}
+		glog.V(3).Infof("Deleting federated service succeeded: %s", key)
+		s.eventRecorder.Eventf(fedService, api.EventTypeNormal, "DeleteSucceed", "Deleting service succeeded")
+		return statusAllOk
+	}
+
+	// Add the required finalizers before creating a service in underlying clusters. This ensures that the
+	// dependent services in underlying clusters are deleted when the federated service is deleted.
+	updatedServiceObj, err := s.deletionHelper.EnsureFinalizers(fedService)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to ensure setting finalizer for service %s: %v", key, err))
+		return statusRecoverableError
+	}
+	fedService = updatedServiceObj.(*v1.Service)
+
+	// Synchronize the federated service in all underlying ready clusters.
+	clusters, err := s.federatedInformer.GetReadyClusters()
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get ready cluster list: %v", err))
+		return statusRecoverableError
+	}
+
+	newLBStatus := newLoadbalancerStatus()
+	newServiceIngress := ingress.NewFederatedServiceIngress()
+	operations := make([]fedutil.FederatedOperation, 0)
+	for _, cluster := range clusters {
+		// Aggregate all operations to perform on all federated clusters
+		operation, err := getOperationsToPerformOnCluster(s.federatedInformer, cluster, fedService, clusterselector.SendToCluster)
+		if err != nil {
+			return statusRecoverableError
+		}
+		if operation != nil {
+			operations = append(operations, *operation)
+		}
+
+		// Aggregate LoadBalancerStatus from all services in federated clusters to update status in federated service
+		lbStatus, err := s.getServiceStatusInCluster(cluster, key)
+		if err != nil {
+			return statusRecoverableError
+		}
+		if len(lbStatus.Ingress) > 0 {
+			newLBStatus.Ingress = append(newLBStatus.Ingress, lbStatus.Ingress...)
+
+			// Add/Update federated service ingress only if there are reachable endpoints backing the lb service
+			endpoints, err := s.getServiceEndpointsInCluster(cluster, key)
+			if err != nil {
+				return statusRecoverableError
+			}
+			// if there are no endpoints created for the service then the loadbalancer ingress
+			// is not reachable, so do not consider such loadbalancer ingresses for federated
+			// service ingresses
+			if len(endpoints) > 0 {
+				clusterIngress := fedapi.ClusterServiceIngress{
+					Cluster: cluster.Name,
+					Items:   lbStatus.Ingress,
+				}
+				newServiceIngress.Items = append(newServiceIngress.Items, clusterIngress)
+			}
+		}
+	}
+
+	if len(operations) != 0 {
+		err = s.federatedUpdater.Update(operations)
+		if err != nil {
+			if !errors.IsAlreadyExists(err) {
+				runtime.HandleError(fmt.Errorf("Failed to execute updates for %s: %v", key, err))
+				return statusRecoverableError
+			}
+		}
+	}
+
+	// Update the federated service if there are any updates in clustered service (status/endpoints)
+	err = s.updateFederatedService(fedService, newLBStatus, newServiceIngress)
+	if err != nil {
+		return statusRecoverableError
+	}
+
+	glog.V(5).Infof("Everything is in order in federated clusters for service %s", key)
+	return statusAllOk
+}
+
+type clusterSelectorFunc func(map[string]string, map[string]string) (bool, error)
+
+// getOperationsToPerformOnCluster returns the operations to be performed so that clustered service is in sync with federated service
+func getOperationsToPerformOnCluster(informer fedutil.FederatedInformer, cluster *v1beta1.Cluster, fedService *v1.Service, selector clusterSelectorFunc) (*fedutil.FederatedOperation, error) {
+	var operation *fedutil.FederatedOperation
+	var operationType fedutil.FederatedOperationType = ""
+
+	key := types.NamespacedName{Namespace: fedService.Namespace, Name: fedService.Name}.String()
+	clusterServiceObj, found, err := informer.GetTargetStore().GetByKey(cluster.Name, key)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get %s service from %s: %v", key, cluster.Name, err))
+		return nil, err
+	}
+
+	send, err := selector(cluster.Labels, fedService.ObjectMeta.Annotations)
+	if err != nil {
+		glog.Errorf("Error processing ClusterSelector cluster: %s for service map: %s error: %s", cluster.Name, key, err.Error())
+		return nil, err
+	} else if !send {
+		glog.V(5).Infof("Skipping cluster: %s for service: %s reason: cluster selectors do not match: %-v %-v", cluster.Name, key, cluster.ObjectMeta.Labels, fedService.ObjectMeta.Annotations[v1beta1.FederationClusterSelectorAnnotation])
+	}
+
+	desiredService := &v1.Service{
+		ObjectMeta: fedutil.DeepCopyRelevantObjectMeta(fedService.ObjectMeta),
+		Spec:       *(fedutil.DeepCopyApiTypeOrPanic(&fedService.Spec).(*v1.ServiceSpec)),
+	}
+	switch {
+	case found && send:
+		clusterService, ok := clusterServiceObj.(*v1.Service)
+		if !ok {
+			runtime.HandleError(fmt.Errorf("Unexpected error for %q: %v", key, err))
+			return nil, err
+		}
+
+		// ClusterIP and NodePort are allocated to Service by cluster, so retain the same if any while updating
+		desiredService.Spec.ClusterIP = clusterService.Spec.ClusterIP
+		for _, cPort := range clusterService.Spec.Ports {
+			for i, fPort := range clusterService.Spec.Ports {
+				if fPort.Name == cPort.Name && fPort.Protocol == cPort.Protocol && fPort.Port == cPort.Port {
+					desiredService.Spec.Ports[i].NodePort = cPort.NodePort
+				}
+			}
+		}
+
+		// Update existing service, if needed.
+		if !Equivalent(desiredService, clusterService) {
+			operationType = fedutil.OperationTypeUpdate
+
+			glog.V(4).Infof("Service in underlying cluster %s does not match, Desired: %+v, Existing: %+v", cluster.Name, desiredService, clusterService)
+
+			// ResourceVersion of cluster service can be different from federated service,
+			// so do not update ResourceVersion while updating cluster service
+			desiredService.ResourceVersion = clusterService.ResourceVersion
+		} else {
+			glog.V(5).Infof("Service in underlying cluster %s is up to date: %+v", cluster.Name, desiredService)
+		}
+	case found && !send:
+		operationType = fedutil.OperationTypeDelete
+	case !found && send:
+		operationType = fedutil.OperationTypeAdd
+		desiredService.ResourceVersion = ""
+
+		glog.V(4).Infof("Creating service in underlying cluster %s: %+v", cluster.Name, desiredService)
+	}
+
+	if len(operationType) > 0 {
+		operation = &fedutil.FederatedOperation{
+			Type:        operationType,
+			Obj:         desiredService,
+			ClusterName: cluster.Name,
+			Key:         key,
+		}
+	}
+	return operation, nil
+}
+
+// getServiceStatusInCluster returns service status in federated cluster
+func (s *ServiceController) getServiceStatusInCluster(cluster *v1beta1.Cluster, key string) (*v1.LoadBalancerStatus, error) {
+	lbStatus := &v1.LoadBalancerStatus{}
+
+	clusterServiceObj, serviceFound, err := s.federatedInformer.GetTargetStore().GetByKey(cluster.Name, key)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get %s service from %s: %v", key, cluster.Name, err))
+		return lbStatus, err
+	}
+	if serviceFound {
+		clusterService, ok := clusterServiceObj.(*v1.Service)
+		if !ok {
+			err = fmt.Errorf("Unknown object received: %v", clusterServiceObj)
+			runtime.HandleError(err)
+			return lbStatus, err
+		}
+		lbStatus = &clusterService.Status.LoadBalancer
+		newLbStatus := &loadbalancerStatus{*lbStatus}
+		sort.Sort(newLbStatus)
+	}
+	return lbStatus, nil
+}
+
+// getServiceEndpointsInCluster returns ready endpoints corresponding to service in federated cluster
+func (s *ServiceController) getServiceEndpointsInCluster(cluster *v1beta1.Cluster, key string) ([]v1.EndpointAddress, error) {
+	addresses := []v1.EndpointAddress{}
+
+	clusterEndpointsObj, endpointsFound, err := s.endpointFederatedInformer.GetTargetStore().GetByKey(cluster.Name, key)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get %s endpoint from %s: %v", key, cluster.Name, err))
+		return addresses, err
+	}
+	if endpointsFound {
+		clusterEndpoints, ok := clusterEndpointsObj.(*v1.Endpoints)
+		if !ok {
+			glog.Warningf("Unknown object received: %v", clusterEndpointsObj)
+			return addresses, fmt.Errorf("Unknown object received: %v", clusterEndpointsObj)
+		}
+		for _, subset := range clusterEndpoints.Subsets {
+			if len(subset.Addresses) > 0 {
+				addresses = append(addresses, subset.Addresses...)
+			}
+		}
+	}
+	return addresses, nil
+}
+
+// updateFederatedService updates the federated service with aggregated lbStatus and serviceIngresses
+// and also updates the dns records as needed
+func (s *ServiceController) updateFederatedService(fedService *v1.Service, newLBStatus *loadbalancerStatus, newServiceIngress *ingress.FederatedServiceIngress) error {
+	key := types.NamespacedName{Namespace: fedService.Namespace, Name: fedService.Name}.String()
+	needUpdate := false
+
+	// Sort the endpoints so that we can compare
+	sort.Sort(newLBStatus)
+	if !reflect.DeepEqual(fedService.Status.LoadBalancer.Ingress, newLBStatus.Ingress) {
+		fedService.Status.LoadBalancer.Ingress = newLBStatus.Ingress
+		glog.V(3).Infof("Federated service loadbalancer status updated for %s: %v", key, newLBStatus.Ingress)
+		needUpdate = true
+	}
+
+	existingServiceIngress, err := ingress.ParseFederatedServiceIngress(fedService)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to parse endpoint annotations for service %s: %v", key, err))
+		return err
+	}
+
+	// TODO: We should have a reliable cluster health check(should consider quorum) to detect cluster is not
+	// reachable and remove dns records for them. Until a reliable cluster health check is available, below code is
+	// a workaround to not remove the existing dns records which were created before the cluster went offline.
+	unreadyClusters, err := s.federatedInformer.GetUnreadyClusters()
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get unready cluster list: %v", err))
+		return err
+	}
+	for _, cluster := range unreadyClusters {
+		lbIngress := existingServiceIngress.GetClusterLoadBalancerIngresses(cluster.Name)
+		newServiceIngress.AddClusterLoadBalancerIngresses(cluster.Name, lbIngress)
+		glog.V(5).Infof("Cluster %s is Offline, Preserving previously available status for Service %s", cluster.Name, key)
+	}
+
+	// Update federated service status and/or ingress annotations if changed
+	sort.Sort(newServiceIngress)
+	if !reflect.DeepEqual(existingServiceIngress.Items, newServiceIngress.Items) {
+		fedService = ingress.UpdateIngressAnnotation(fedService, newServiceIngress)
+		glog.V(3).Infof("Federated service loadbalancer ingress updated for %s: existing: %#v, desired: %#v", key, existingServiceIngress, newServiceIngress)
+		needUpdate = true
+	}
+
+	if needUpdate {
+		var err error
+		fedService, err = s.federationClient.Core().Services(fedService.Namespace).UpdateStatus(fedService)
+		if err != nil {
+			runtime.HandleError(fmt.Errorf("Error updating the federation service object %s: %v", key, err))
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Equivalent Checks if cluster-independent, user provided data in two given services are equal. If in the future the
+// services structure is expanded then any field that is not populated by the api server should be included here.
+func Equivalent(s1, s2 *v1.Service) bool {
+	// TODO: should also check for all annotations except FederationServiceIngressAnnotation
+	return s1.Name == s2.Name && s1.Namespace == s2.Namespace &&
+		(reflect.DeepEqual(s1.Labels, s2.Labels) || (len(s1.Labels) == 0 && len(s2.Labels) == 0)) &&
+		reflect.DeepEqual(s1.Spec, s2.Spec)
+}
+
+type loadbalancerStatus struct {
+	v1.LoadBalancerStatus
+}
+
+func newLoadbalancerStatus() *loadbalancerStatus {
+	return &loadbalancerStatus{}
+}
+
+func (lbs loadbalancerStatus) Len() int {
+	return len(lbs.Ingress)
+}
+
+func (lbs loadbalancerStatus) Less(i, j int) bool {
+	ipComparison := strings.Compare(lbs.Ingress[i].IP, lbs.Ingress[j].IP)
+	hostnameComparison := strings.Compare(lbs.Ingress[i].Hostname, lbs.Ingress[j].Hostname)
+	if ipComparison < 0 || (ipComparison == 0 && hostnameComparison < 0) {
+		return true
+	}
+	return false
+}
+
+func (lbs loadbalancerStatus) Swap(i, j int) {
+	lbs.Ingress[i].IP, lbs.Ingress[j].IP = lbs.Ingress[j].IP, lbs.Ingress[i].IP
+	lbs.Ingress[i].Hostname, lbs.Ingress[j].Hostname = lbs.Ingress[j].Hostname, lbs.Ingress[i].Hostname
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/service/servicecontroller_test.go

@@ -0,0 +1,383 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package service
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/stretchr/testify/require"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	fakekubeclientset "k8s.io/client-go/kubernetes/fake"
+	corelisters "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	fakefedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset/fake"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/service/ingress"
+	fedutil "k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/deletionhelper"
+	. "k8s.io/kubernetes/federation/pkg/federation-controller/util/test"
+)
+
+const (
+	retryInterval = 100 * time.Millisecond
+
+	clusters  string = "clusters"
+	services  string = "services"
+	endpoints string = "endpoints"
+
+	lbIngress1       = "10.20.30.40"
+	lbIngress2       = "10.20.30.50"
+	serviceEndpoint1 = "192.168.0.1"
+	serviceEndpoint2 = "192.168.1.1"
+)
+
+var awfulError error = errors.NewGone("Something bad happened")
+
+func TestServiceController(t *testing.T) {
+	glog.Infof("Creating fake infrastructure")
+	fedClient := &fakefedclientset.Clientset{}
+	cluster1 := NewCluster("cluster1", v1.ConditionTrue)
+	cluster2 := NewCluster("cluster2", v1.ConditionTrue)
+
+	RegisterFakeClusterGet(&fedClient.Fake, &v1beta1.ClusterList{Items: []v1beta1.Cluster{*cluster1, *cluster2}})
+	RegisterFakeList(clusters, &fedClient.Fake, &v1beta1.ClusterList{Items: []v1beta1.Cluster{*cluster1, *cluster2}})
+	fedclusterWatch := RegisterFakeWatch(clusters, &fedClient.Fake)
+	RegisterFakeList(services, &fedClient.Fake, &v1.ServiceList{Items: []v1.Service{}})
+	fedServiceWatch := RegisterFakeWatch(services, &fedClient.Fake)
+	RegisterFakeOnCreate(clusters, &fedClient.Fake, fedclusterWatch)
+	RegisterFakeOnUpdate(clusters, &fedClient.Fake, fedclusterWatch)
+	RegisterFakeOnCreate(services, &fedClient.Fake, fedServiceWatch)
+	RegisterFakeOnUpdate(services, &fedClient.Fake, fedServiceWatch)
+	RegisterFakeOnDelete(services, &fedClient.Fake, fedServiceWatch, serviceObjectGetter)
+
+	cluster1Client := &fakekubeclientset.Clientset{}
+	RegisterFakeList(services, &cluster1Client.Fake, &v1.ServiceList{Items: []v1.Service{}})
+	c1ServiceWatch := RegisterFakeWatch(services, &cluster1Client.Fake)
+	RegisterFakeList(endpoints, &cluster1Client.Fake, &v1.EndpointsList{Items: []v1.Endpoints{}})
+	c1EndpointWatch := RegisterFakeWatch(endpoints, &cluster1Client.Fake)
+	RegisterFakeOnCreate(services, &cluster1Client.Fake, c1ServiceWatch)
+	RegisterFakeOnUpdate(services, &cluster1Client.Fake, c1ServiceWatch)
+	RegisterFakeOnDelete(services, &cluster1Client.Fake, c1ServiceWatch, serviceObjectGetter)
+	RegisterFakeOnCreate(endpoints, &cluster1Client.Fake, c1EndpointWatch)
+	RegisterFakeOnUpdate(endpoints, &cluster1Client.Fake, c1EndpointWatch)
+
+	cluster2Client := &fakekubeclientset.Clientset{}
+	RegisterFakeList(services, &cluster2Client.Fake, &v1.ServiceList{Items: []v1.Service{}})
+	c2ServiceWatch := RegisterFakeWatch(services, &cluster2Client.Fake)
+	RegisterFakeList(endpoints, &cluster2Client.Fake, &v1.EndpointsList{Items: []v1.Endpoints{}})
+	c2EndpointWatch := RegisterFakeWatch(endpoints, &cluster2Client.Fake)
+	RegisterFakeOnCreate(services, &cluster2Client.Fake, c2ServiceWatch)
+	RegisterFakeOnUpdate(services, &cluster2Client.Fake, c2ServiceWatch)
+	RegisterFakeOnDelete(services, &cluster2Client.Fake, c2ServiceWatch, serviceObjectGetter)
+	RegisterFakeOnCreate(endpoints, &cluster2Client.Fake, c2EndpointWatch)
+	RegisterFakeOnUpdate(endpoints, &cluster2Client.Fake, c2EndpointWatch)
+
+	fedInformerClientFactory := func(cluster *v1beta1.Cluster) (kubeclientset.Interface, error) {
+		switch cluster.Name {
+		case cluster1.Name:
+			return cluster1Client, nil
+		case cluster2.Name:
+			return cluster2Client, nil
+		default:
+			return nil, fmt.Errorf("Unknown cluster: %v", cluster.Name)
+		}
+	}
+
+	sc := New(fedClient)
+	ToFederatedInformerForTestOnly(sc.federatedInformer).SetClientFactory(fedInformerClientFactory)
+	ToFederatedInformerForTestOnly(sc.endpointFederatedInformer).SetClientFactory(fedInformerClientFactory)
+	sc.clusterAvailableDelay = 100 * time.Millisecond
+	sc.reviewDelay = 50 * time.Millisecond
+	sc.updateTimeout = 5 * time.Second
+
+	stop := make(chan struct{})
+	glog.Infof("Running Service Controller")
+	go sc.Run(5, stop)
+
+	glog.Infof("Adding cluster 1")
+	fedclusterWatch.Add(cluster1)
+
+	service := NewService("test-service-1", 80)
+
+	glog.Infof("Adding federated service")
+	fedServiceWatch.Add(service)
+	key := types.NamespacedName{Namespace: service.Namespace, Name: service.Name}.String()
+
+	glog.Infof("Test service was correctly created in cluster 1")
+	require.NoError(t, WaitForClusterService(t, sc.federatedInformer.GetTargetStore(), cluster1.Name,
+		key, service, wait.ForeverTestTimeout))
+
+	glog.Infof("Adding cluster 2")
+	fedclusterWatch.Add(cluster2)
+
+	glog.Infof("Test service was correctly created in cluster 2")
+	require.NoError(t, WaitForClusterService(t, sc.federatedInformer.GetTargetStore(), cluster2.Name,
+		key, service, wait.ForeverTestTimeout))
+
+	glog.Infof("Test federation service is updated when cluster1 service status is updated")
+	service.Status = v1.ServiceStatus{
+		LoadBalancer: v1.LoadBalancerStatus{
+			Ingress: []v1.LoadBalancerIngress{
+				{IP: lbIngress1},
+			}}}
+
+	desiredStatus := service.Status
+	desiredService := &v1.Service{Status: desiredStatus}
+
+	c1ServiceWatch.Modify(service)
+	require.NoError(t, WaitForClusterService(t, sc.federatedInformer.GetTargetStore(), cluster1.Name,
+		key, service, wait.ForeverTestTimeout))
+	require.NoError(t, WaitForFederatedServiceUpdate(t, sc.serviceStore,
+		key, desiredService, serviceStatusCompare, wait.ForeverTestTimeout))
+
+	glog.Infof("Test federation service is updated when cluster1 endpoint for the service is created")
+	desiredIngressAnnotation := ingress.NewFederatedServiceIngress().
+		AddEndpoints("cluster1", []string{lbIngress1}).
+		String()
+	desiredService = &v1.Service{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ingress.FederatedServiceIngressAnnotation: desiredIngressAnnotation}}}
+	c1EndpointWatch.Add(NewEndpoint("test-service-1", serviceEndpoint1))
+	require.NoError(t, WaitForFederatedServiceUpdate(t, sc.serviceStore,
+		key, desiredService, serviceIngressCompare, wait.ForeverTestTimeout))
+
+	glog.Infof("Test federation service is updated when cluster2 service status is updated")
+	service.Status = v1.ServiceStatus{
+		LoadBalancer: v1.LoadBalancerStatus{
+			Ingress: []v1.LoadBalancerIngress{
+				{IP: lbIngress2},
+			}}}
+	desiredStatus.LoadBalancer.Ingress = append(desiredStatus.LoadBalancer.Ingress, v1.LoadBalancerIngress{IP: lbIngress2})
+	desiredService = &v1.Service{Status: desiredStatus}
+
+	c2ServiceWatch.Modify(service)
+	require.NoError(t, WaitForClusterService(t, sc.federatedInformer.GetTargetStore(), cluster2.Name,
+		key, service, wait.ForeverTestTimeout))
+	require.NoError(t, WaitForFederatedServiceUpdate(t, sc.serviceStore,
+		key, desiredService, serviceStatusCompare, wait.ForeverTestTimeout))
+
+	glog.Infof("Test federation service is updated when cluster2 endpoint for the service is created")
+	desiredIngressAnnotation = ingress.NewFederatedServiceIngress().
+		AddEndpoints("cluster1", []string{lbIngress1}).
+		AddEndpoints("cluster2", []string{lbIngress2}).
+		String()
+	desiredService = &v1.Service{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ingress.FederatedServiceIngressAnnotation: desiredIngressAnnotation}}}
+	c2EndpointWatch.Add(NewEndpoint("test-service-1", serviceEndpoint2))
+	require.NoError(t, WaitForFederatedServiceUpdate(t, sc.serviceStore,
+		key, desiredService, serviceIngressCompare, wait.ForeverTestTimeout))
+
+	glog.Infof("Test federation service is updated when cluster1 endpoint for the service is deleted")
+	desiredIngressAnnotation = ingress.NewFederatedServiceIngress().
+		AddEndpoints("cluster2", []string{lbIngress2}).
+		String()
+	desiredService = &v1.Service{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ingress.FederatedServiceIngressAnnotation: desiredIngressAnnotation}}}
+	c1EndpointWatch.Delete(NewEndpoint("test-service-1", serviceEndpoint1))
+	require.NoError(t, WaitForFederatedServiceUpdate(t, sc.serviceStore,
+		key, desiredService, serviceIngressCompare, wait.ForeverTestTimeout))
+
+	glog.Infof("Test modifying federated service by changing the port")
+	service.Spec.Ports[0].Port = 9090
+	fedServiceWatch.Modify(service)
+	require.NoError(t, WaitForClusterService(t, sc.federatedInformer.GetTargetStore(), cluster1.Name,
+		key, service, wait.ForeverTestTimeout))
+
+	glog.Infof("Test cluster service is recreated when deleted")
+	c1Service := NewService("test-service-1", 80)
+	c1Service.DeletionTimestamp = &metav1.Time{Time: time.Now()}
+	c1ServiceWatch.Delete(c1Service)
+	require.NoError(t, WaitForClusterService(t, sc.federatedInformer.GetTargetStore(), cluster1.Name,
+		key, service, wait.ForeverTestTimeout))
+
+	glog.Infof("Test cluster services are deleted when federated service is deleted")
+	service.ObjectMeta.Finalizers = append(service.ObjectMeta.Finalizers, deletionhelper.FinalizerDeleteFromUnderlyingClusters)
+	service.DeletionTimestamp = &metav1.Time{Time: time.Now()}
+	fedServiceWatch.Modify(service)
+	require.NoError(t, WaitForClusterServiceDelete(t, sc.federatedInformer.GetTargetStore(), cluster1.Name,
+		key, wait.ForeverTestTimeout))
+	require.NoError(t, WaitForClusterServiceDelete(t, sc.federatedInformer.GetTargetStore(), cluster2.Name,
+		key, wait.ForeverTestTimeout))
+
+	close(stop)
+}
+
+func TestGetOperationsToPerformOnCluster(t *testing.T) {
+	obj := NewService("test-service-1", 80)
+	cluster1 := NewCluster("cluster1", v1.ConditionTrue)
+	fedClient := &fakefedclientset.Clientset{}
+	sc := New(fedClient)
+
+	testCases := map[string]struct {
+		expectedSendErr bool
+		sendToCluster   bool
+		operationType   fedutil.FederatedOperationType
+	}{
+		"sendToCluster error returned": {
+			expectedSendErr: true,
+		},
+		"Missing object and not matching ClusterSelector should result in no operations": {
+			sendToCluster: false,
+		},
+		"Missing object and matching ClusterSelector should result in add operation": {
+			operationType: fedutil.OperationTypeAdd,
+			sendToCluster: true,
+		},
+		// Update and Delete scenarios are tested in TestServiceController
+	}
+	for testName, testCase := range testCases {
+		t.Run(testName, func(t *testing.T) {
+
+			operations, err := getOperationsToPerformOnCluster(sc.federatedInformer, cluster1, obj, func(map[string]string, map[string]string) (bool, error) {
+				if testCase.expectedSendErr {
+					return false, awfulError
+				}
+				return testCase.sendToCluster, nil
+			})
+			if testCase.expectedSendErr {
+				require.Error(t, err, "An error was expected")
+			} else {
+				require.NoError(t, err, "An error was not expected")
+			}
+			if len(testCase.operationType) == 0 {
+				require.Nil(t, operations, "An operation was not expected")
+			} else {
+				require.NotNil(t, operations, "A single operation was expected")
+				require.Equal(t, testCase.operationType, operations.Type, "Unexpected operation returned")
+			}
+		})
+	}
+}
+
+// serviceObjectGetter gives dummy service objects to use with RegisterFakeOnDelete
+// This is just so that federated informer can be tested for delete scenarios.
+func serviceObjectGetter(name, namespace string) runtime.Object {
+	service := new(v1.Service)
+	service.Namespace = namespace
+	service.Name = name
+	return service
+}
+
+func NewService(name string, port int32) *v1.Service {
+	return &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: v1.NamespaceDefault,
+			SelfLink:  "/api/v1/namespaces/default/services/" + name,
+			Labels:    map[string]string{"app": name},
+		},
+		Spec: v1.ServiceSpec{
+			Ports: []v1.ServicePort{{Port: port}},
+			Type:  v1.ServiceTypeLoadBalancer,
+		},
+	}
+}
+
+func NewEndpoint(name, ip string) *v1.Endpoints {
+	return &v1.Endpoints{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: v1.NamespaceDefault,
+			SelfLink:  "/api/v1/namespaces/default/endpoints/" + name,
+			Labels:    map[string]string{"app": name},
+		},
+		Subsets: []v1.EndpointSubset{{
+			Addresses: []v1.EndpointAddress{{
+				IP: ip,
+			}}},
+		},
+	}
+}
+
+// WaitForClusterService waits for the cluster service to be created matching the desiredService.
+func WaitForClusterService(t *testing.T, store fedutil.FederatedReadOnlyStore, clusterName, key string, desiredService *v1.Service, timeout time.Duration) error {
+	err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) {
+		obj, found, err := store.GetByKey(clusterName, key)
+		if !found || err != nil {
+			return false, err
+		}
+		service := obj.(*v1.Service)
+		if !Equivalent(service, desiredService) {
+			glog.V(5).Infof("Waiting for clustered service, Desired: %v, Current: %v", desiredService, service)
+			return false, nil
+		}
+		glog.V(5).Infof("Clustered service is up to date: %v", service)
+		return true, nil
+	})
+	return err
+}
+
+// WaitForClusterServiceDelete waits for the cluster service to be deleted.
+func WaitForClusterServiceDelete(t *testing.T, store fedutil.FederatedReadOnlyStore, clusterName, key string, timeout time.Duration) error {
+	err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) {
+		_, found, _ := store.GetByKey(clusterName, key)
+		if !found {
+			return true, nil
+		}
+		return false, nil
+	})
+	return err
+}
+
+type serviceCompare func(current, desired *v1.Service) (match bool)
+
+func serviceStatusCompare(current, desired *v1.Service) bool {
+	if !reflect.DeepEqual(current.Status.LoadBalancer, desired.Status.LoadBalancer) {
+		glog.V(5).Infof("Waiting for loadbalancer status, Current: %v, Desired: %v", current.Status.LoadBalancer, desired.Status.LoadBalancer)
+		return false
+	}
+	glog.V(5).Infof("Loadbalancer status match: %v", current.Status.LoadBalancer)
+	return true
+}
+
+func serviceIngressCompare(current, desired *v1.Service) bool {
+	if strings.Compare(current.Annotations[ingress.FederatedServiceIngressAnnotation], desired.Annotations[ingress.FederatedServiceIngressAnnotation]) != 0 {
+		glog.V(5).Infof("Waiting for loadbalancer ingress, Current: %v, Desired: %v", current.Annotations[ingress.FederatedServiceIngressAnnotation], desired.Annotations[ingress.FederatedServiceIngressAnnotation])
+		return false
+	}
+	glog.V(5).Infof("Loadbalancer ingress match: %v", current.Annotations[ingress.FederatedServiceIngressAnnotation])
+	return true
+}
+
+// WaitForFederatedServiceUpdate waits for federated service updates to match the desiredService.
+func WaitForFederatedServiceUpdate(t *testing.T, store corelisters.ServiceLister, key string, desiredService *v1.Service, match serviceCompare, timeout time.Duration) error {
+	err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) {
+		namespace, name, err := cache.SplitMetaNamespaceKey(key)
+		if err != nil {
+			return false, err
+		}
+		service, err := store.Services(namespace).Get(name)
+		switch {
+		case errors.IsNotFound(err):
+			return false, nil
+		case err != nil:
+			return false, err
+		case !match(service, desiredService):
+			return false, nil
+		default:
+			return true, nil
+		}
+	})
+	return err
+}