Replace godep with dep

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/BUILD

@@ -0,0 +1,32 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["doc.go"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//federation/pkg/federation-controller/cluster:all-srcs",
+        "//federation/pkg/federation-controller/ingress:all-srcs",
+        "//federation/pkg/federation-controller/job:all-srcs",
+        "//federation/pkg/federation-controller/service:all-srcs",
+        "//federation/pkg/federation-controller/sync:all-srcs",
+        "//federation/pkg/federation-controller/util:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/OWNERS

@@ -0,0 +1,8 @@
+approvers:
+- quinton-hoole
+- nikhiljindal
+- madhusudancs
+reviewers:
+- quinton-hoole
+- nikhiljindal
+- madhusudancs

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/cluster/BUILD

@@ -0,0 +1,65 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["clustercontroller_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset:go_default_library",
+        "//pkg/api/testapi:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
+        "//vendor/k8s.io/client-go/rest:go_default_library",
+        "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cluster_client.go",
+        "clustercontroller.go",
+        "doc.go",
+    ],
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/cache:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//pkg/api:go_default_library",
+        "//pkg/client/clientset_generated/internalclientset:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//pkg/kubelet/apis:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/client-go/rest:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/cluster/cluster_client.go

@@ -0,0 +1,170 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/golang/glog"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	restclient "k8s.io/client-go/rest"
+	federation_v1beta1 "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	"k8s.io/kubernetes/pkg/api"
+	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
+	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
+)
+
+const (
+	UserAgentName = "Cluster-Controller"
+)
+
+type ClusterClient struct {
+	kubeClient *clientset.Clientset
+}
+
+func NewClusterClientSet(c *federation_v1beta1.Cluster) (*ClusterClient, error) {
+	clusterConfig, err := util.BuildClusterConfig(c)
+	if err != nil {
+		return nil, err
+	}
+	var clusterClientSet = ClusterClient{}
+	if clusterConfig != nil {
+		clusterClientSet.kubeClient = clientset.NewForConfigOrDie((restclient.AddUserAgent(clusterConfig, UserAgentName)))
+		if clusterClientSet.kubeClient == nil {
+			return nil, nil
+		}
+	}
+	return &clusterClientSet, nil
+}
+
+// GetClusterHealthStatus gets the kubernetes cluster health status by requesting "/healthz"
+func (self *ClusterClient) GetClusterHealthStatus() *federation_v1beta1.ClusterStatus {
+	clusterStatus := federation_v1beta1.ClusterStatus{}
+	currentTime := metav1.Now()
+	newClusterReadyCondition := federation_v1beta1.ClusterCondition{
+		Type:               federation_v1beta1.ClusterReady,
+		Status:             v1.ConditionTrue,
+		Reason:             "ClusterReady",
+		Message:            "/healthz responded with ok",
+		LastProbeTime:      currentTime,
+		LastTransitionTime: currentTime,
+	}
+	newClusterNotReadyCondition := federation_v1beta1.ClusterCondition{
+		Type:               federation_v1beta1.ClusterReady,
+		Status:             v1.ConditionFalse,
+		Reason:             "ClusterNotReady",
+		Message:            "/healthz responded without ok",
+		LastProbeTime:      currentTime,
+		LastTransitionTime: currentTime,
+	}
+	newNodeOfflineCondition := federation_v1beta1.ClusterCondition{
+		Type:               federation_v1beta1.ClusterOffline,
+		Status:             v1.ConditionTrue,
+		Reason:             "ClusterNotReachable",
+		Message:            "cluster is not reachable",
+		LastProbeTime:      currentTime,
+		LastTransitionTime: currentTime,
+	}
+	newNodeNotOfflineCondition := federation_v1beta1.ClusterCondition{
+		Type:               federation_v1beta1.ClusterOffline,
+		Status:             v1.ConditionFalse,
+		Reason:             "ClusterReachable",
+		Message:            "cluster is reachable",
+		LastProbeTime:      currentTime,
+		LastTransitionTime: currentTime,
+	}
+	body, err := self.kubeClient.DiscoveryClient.RESTClient().Get().AbsPath("/healthz").Do().Raw()
+	if err != nil {
+		clusterStatus.Conditions = append(clusterStatus.Conditions, newNodeOfflineCondition)
+	} else {
+		if !strings.EqualFold(string(body), "ok") {
+			clusterStatus.Conditions = append(clusterStatus.Conditions, newClusterNotReadyCondition, newNodeNotOfflineCondition)
+		} else {
+			clusterStatus.Conditions = append(clusterStatus.Conditions, newClusterReadyCondition)
+		}
+	}
+
+	zones, region, err := self.GetClusterZones()
+	if err != nil {
+		glog.Warningf("Failed to get zones and region for cluster with client %v: %v", self, err)
+	} else {
+		clusterStatus.Zones = zones
+		clusterStatus.Region = region
+	}
+
+	return &clusterStatus
+}
+
+// GetClusterZones gets the kubernetes cluster zones and region by inspecting labels on nodes in the cluster.
+func (self *ClusterClient) GetClusterZones() (zones []string, region string, err error) {
+	return getZoneNames(self.kubeClient)
+}
+
+// Find the name of the zone in which a Node is running
+func getZoneNameForNode(node api.Node) (string, error) {
+	for key, value := range node.Labels {
+		if key == kubeletapis.LabelZoneFailureDomain {
+			return value, nil
+		}
+	}
+	return "", fmt.Errorf("Zone name for node %s not found. No label with key %s",
+		node.Name, kubeletapis.LabelZoneFailureDomain)
+}
+
+// Find the name of the region in which a Node is running
+func getRegionNameForNode(node api.Node) (string, error) {
+	for key, value := range node.Labels {
+		if key == kubeletapis.LabelZoneRegion {
+			return value, nil
+		}
+	}
+	return "", fmt.Errorf("Region name for node %s not found. No label with key %s",
+		node.Name, kubeletapis.LabelZoneRegion)
+}
+
+// Find the names of all zones and the region in which we have nodes in this cluster.
+func getZoneNames(client *clientset.Clientset) (zones []string, region string, err error) {
+	zoneNames := sets.NewString()
+	nodes, err := client.Core().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		glog.Errorf("Failed to list nodes while getting zone names: %v", err)
+		return nil, "", err
+	}
+	for i, node := range nodes.Items {
+		// TODO: quinton-hoole make this more efficient.
+		//       For non-multi-zone clusters the zone will
+		//       be identical for all nodes, so we only need to look at one node
+		//       For multi-zone clusters we know at build time
+		//       which zones are included.  Rather get this info from there, because it's cheaper.
+		zoneName, err := getZoneNameForNode(node)
+		if err != nil {
+			return nil, "", err
+		}
+		zoneNames.Insert(zoneName)
+		if i == 0 {
+			region, err = getRegionNameForNode(node)
+			if err != nil {
+				return nil, "", err
+			}
+		}
+	}
+	return zoneNames.List(), region, nil
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/cluster/clustercontroller.go

@@ -0,0 +1,209 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/golang/glog"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+	federationv1beta1 "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	clustercache "k8s.io/kubernetes/federation/client/cache"
+	federationclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	"k8s.io/kubernetes/pkg/controller"
+)
+
+type ClusterController struct {
+	// federationClient used to operate cluster
+	federationClient federationclientset.Interface
+
+	// clusterMonitorPeriod is the period for updating status of cluster
+	clusterMonitorPeriod time.Duration
+
+	mu              sync.RWMutex
+	knownClusterSet sets.String
+	// clusterClusterStatusMap is a mapping of clusterName and cluster status of last sampling
+	clusterClusterStatusMap map[string]federationv1beta1.ClusterStatus
+	// clusterKubeClientMap is a mapping of clusterName and restclient
+	clusterKubeClientMap map[string]ClusterClient
+
+	// cluster framework and store
+	clusterController cache.Controller
+	clusterStore      clustercache.StoreToClusterLister
+}
+
+// StartClusterController starts a new cluster controller
+func StartClusterController(config *restclient.Config, stopChan <-chan struct{}, clusterMonitorPeriod time.Duration) {
+	restclient.AddUserAgent(config, "cluster-controller")
+	client := federationclientset.NewForConfigOrDie(config)
+	controller := newClusterController(client, clusterMonitorPeriod)
+	glog.Infof("Starting cluster controller")
+	controller.Run(stopChan)
+}
+
+// newClusterController returns a new cluster controller
+func newClusterController(federationClient federationclientset.Interface, clusterMonitorPeriod time.Duration) *ClusterController {
+	cc := &ClusterController{
+		knownClusterSet:         make(sets.String),
+		federationClient:        federationClient,
+		clusterMonitorPeriod:    clusterMonitorPeriod,
+		clusterClusterStatusMap: make(map[string]federationv1beta1.ClusterStatus),
+		clusterKubeClientMap:    make(map[string]ClusterClient),
+	}
+	cc.clusterStore.Store, cc.clusterController = cache.NewInformer(
+		&cache.ListWatch{
+			ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+				return cc.federationClient.Federation().Clusters().List(options)
+			},
+			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+				return cc.federationClient.Federation().Clusters().Watch(options)
+			},
+		},
+		&federationv1beta1.Cluster{},
+		controller.NoResyncPeriodFunc(),
+		cache.ResourceEventHandlerFuncs{
+			DeleteFunc: cc.delFromClusterSet,
+			AddFunc:    cc.addToClusterSet,
+		},
+	)
+	return cc
+}
+
+// delFromClusterSet delete a cluster from clusterSet and
+// delete the corresponding restclient from the map clusterKubeClientMap
+func (cc *ClusterController) delFromClusterSet(obj interface{}) {
+	cc.mu.Lock()
+	defer cc.mu.Unlock()
+	cluster := obj.(*federationv1beta1.Cluster)
+	cc.delFromClusterSetByName(cluster.Name)
+}
+
+// delFromClusterSetByName delete a cluster from clusterSet by name and
+// delete the corresponding restclient from the map clusterKubeClientMap.
+// Caller must make sure that they hold the mutex
+func (cc *ClusterController) delFromClusterSetByName(clusterName string) {
+	glog.V(1).Infof("ClusterController observed a cluster deletion: %v", clusterName)
+	cc.knownClusterSet.Delete(clusterName)
+	delete(cc.clusterKubeClientMap, clusterName)
+	delete(cc.clusterClusterStatusMap, clusterName)
+}
+
+func (cc *ClusterController) addToClusterSet(obj interface{}) {
+	cc.mu.Lock()
+	defer cc.mu.Unlock()
+	cluster := obj.(*federationv1beta1.Cluster)
+	cc.addToClusterSetWithoutLock(cluster)
+}
+
+// addToClusterSetWithoutLock inserts the new cluster to clusterSet and create
+// a corresponding restclient to map clusterKubeClientMap if the cluster is not
+// known. Caller must make sure that they hold the mutex.
+func (cc *ClusterController) addToClusterSetWithoutLock(cluster *federationv1beta1.Cluster) {
+	if cc.knownClusterSet.Has(cluster.Name) {
+		return
+	}
+	glog.V(1).Infof("ClusterController observed a new cluster: %v", cluster.Name)
+	cc.knownClusterSet.Insert(cluster.Name)
+	// create the restclient of cluster
+	restClient, err := NewClusterClientSet(cluster)
+	if err != nil || restClient == nil {
+		glog.Errorf("Failed to create corresponding restclient of kubernetes cluster: %v", err)
+		return
+	}
+	cc.clusterKubeClientMap[cluster.Name] = *restClient
+}
+
+// Run begins watching and syncing.
+func (cc *ClusterController) Run(stopChan <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	go cc.clusterController.Run(stopChan)
+	// monitor cluster status periodically, in phase 1 we just get the health state from "/healthz"
+	go wait.Until(func() {
+		if err := cc.updateClusterStatus(); err != nil {
+			glog.Errorf("Error monitoring cluster status: %v", err)
+		}
+	}, cc.clusterMonitorPeriod, stopChan)
+}
+
+// updateClusterStatus checks cluster status and get the metrics from cluster's restapi
+func (cc *ClusterController) updateClusterStatus() error {
+	clusters, err := cc.federationClient.Federation().Clusters().List(metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	for _, cluster := range clusters.Items {
+		cc.mu.RLock()
+		// skip updating status of the cluster which is not yet added to knownClusterSet.
+		if !cc.knownClusterSet.Has(cluster.Name) {
+			cc.mu.RUnlock()
+			continue
+		}
+		clusterClient, clientFound := cc.clusterKubeClientMap[cluster.Name]
+		clusterStatusOld, statusFound := cc.clusterClusterStatusMap[cluster.Name]
+		cc.mu.RUnlock()
+
+		if !clientFound {
+			glog.Warningf("Failed to get client for cluster %s", cluster.Name)
+			continue
+		}
+		clusterStatusNew := clusterClient.GetClusterHealthStatus()
+		if !statusFound {
+			glog.Infof("There is no status stored for cluster: %v before", cluster.Name)
+		} else {
+			hasTransition := false
+			if len(clusterStatusNew.Conditions) != len(clusterStatusOld.Conditions) {
+				hasTransition = true
+			} else {
+				for i := 0; i < len(clusterStatusNew.Conditions); i++ {
+					if !(strings.EqualFold(string(clusterStatusNew.Conditions[i].Type), string(clusterStatusOld.Conditions[i].Type)) &&
+						strings.EqualFold(string(clusterStatusNew.Conditions[i].Status), string(clusterStatusOld.Conditions[i].Status))) {
+						hasTransition = true
+						break
+					}
+				}
+			}
+
+			if !hasTransition {
+				for j := 0; j < len(clusterStatusNew.Conditions); j++ {
+					clusterStatusNew.Conditions[j].LastTransitionTime = clusterStatusOld.Conditions[j].LastTransitionTime
+				}
+			}
+		}
+
+		cc.mu.Lock()
+		cc.clusterClusterStatusMap[cluster.Name] = *clusterStatusNew
+		cc.mu.Unlock()
+		cluster.Status = *clusterStatusNew
+		cluster, err := cc.federationClient.Federation().Clusters().UpdateStatus(&cluster)
+		if err != nil {
+			glog.Warningf("Failed to update the status of cluster: %v ,error is : %v", cluster.Name, err)
+			// Don't return err here, as we want to continue processing remaining clusters.
+			continue
+		}
+	}
+	return nil
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/cluster/clustercontroller_test.go

@@ -0,0 +1,173 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	federationv1beta1 "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	federationclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	"k8s.io/kubernetes/pkg/api/testapi"
+)
+
+func newCluster(clusterName string, serverUrl string) *federationv1beta1.Cluster {
+	cluster := federationv1beta1.Cluster{
+		TypeMeta: metav1.TypeMeta{APIVersion: testapi.Federation.GroupVersion().String()},
+		ObjectMeta: metav1.ObjectMeta{
+			UID:  uuid.NewUUID(),
+			Name: clusterName,
+		},
+		Spec: federationv1beta1.ClusterSpec{
+			ServerAddressByClientCIDRs: []federationv1beta1.ServerAddressByClientCIDR{
+				{
+					ClientCIDR:    "0.0.0.0/0",
+					ServerAddress: serverUrl,
+				},
+			},
+		},
+	}
+	return &cluster
+}
+
+func newClusterList(cluster *federationv1beta1.Cluster) *federationv1beta1.ClusterList {
+	clusterList := federationv1beta1.ClusterList{
+		TypeMeta: metav1.TypeMeta{APIVersion: testapi.Federation.GroupVersion().String()},
+		ListMeta: metav1.ListMeta{
+			SelfLink: "foobar",
+		},
+		Items: []federationv1beta1.Cluster{},
+	}
+	clusterList.Items = append(clusterList.Items, *cluster)
+	return &clusterList
+}
+
+// init a fake http handler, simulate a federation apiserver, response the "DELETE" "PUT" "GET" "UPDATE"
+// when "canBeGotten" is false, means that user can not get the cluster cluster from apiserver
+func createHttptestFakeHandlerForFederation(clusterList *federationv1beta1.ClusterList, canBeGotten bool) *http.HandlerFunc {
+	fakeHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		clusterListString, _ := json.Marshal(*clusterList)
+		w.Header().Set("Content-Type", "application/json")
+		switch r.Method {
+		case "PUT":
+			fmt.Fprintln(w, string(clusterListString))
+		case "GET":
+			if canBeGotten {
+				fmt.Fprintln(w, string(clusterListString))
+			} else {
+				fmt.Fprintln(w, "")
+			}
+		default:
+			fmt.Fprintln(w, "")
+		}
+	})
+	return &fakeHandler
+}
+
+// init a fake http handler, simulate a cluster apiserver, response the "/healthz"
+// when "canBeGotten" is false, means that user can not get response from apiserver
+func createHttptestFakeHandlerForCluster(canBeGotten bool) *http.HandlerFunc {
+	fakeHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		switch r.Method {
+		case "GET":
+			if canBeGotten {
+				fmt.Fprintln(w, "ok")
+			} else {
+				w.WriteHeader(http.StatusNotFound)
+			}
+		default:
+			fmt.Fprintln(w, "")
+		}
+	})
+	return &fakeHandler
+}
+
+func TestUpdateClusterStatusOK(t *testing.T) {
+	clusterName := "foobarCluster"
+	// create dummy httpserver
+	testClusterServer := httptest.NewServer(createHttptestFakeHandlerForCluster(true))
+	defer testClusterServer.Close()
+	federationCluster := newCluster(clusterName, testClusterServer.URL)
+	federationClusterList := newClusterList(federationCluster)
+
+	testFederationServer := httptest.NewServer(createHttptestFakeHandlerForFederation(federationClusterList, true))
+	defer testFederationServer.Close()
+
+	restClientCfg, err := clientcmd.BuildConfigFromFlags(testFederationServer.URL, "")
+	if err != nil {
+		t.Errorf("Failed to build client config")
+	}
+	federationClientSet := federationclientset.NewForConfigOrDie(restclient.AddUserAgent(restClientCfg, "cluster-controller"))
+
+	manager := newClusterController(federationClientSet, 5)
+	manager.addToClusterSet(federationCluster)
+	err = manager.updateClusterStatus()
+	if err != nil {
+		t.Errorf("Failed to Update Cluster Status: %v", err)
+	}
+	clusterStatus, found := manager.clusterClusterStatusMap[clusterName]
+	if !found {
+		t.Errorf("Failed to Update Cluster Status")
+	} else {
+		if (clusterStatus.Conditions[1].Status != v1.ConditionFalse) || (clusterStatus.Conditions[1].Type != federationv1beta1.ClusterOffline) {
+			t.Errorf("Failed to Update Cluster Status")
+		}
+	}
+}
+
+// Test races between informer's updates and routine updates of cluster status
+// Issue https://github.com/kubernetes/kubernetes/issues/49958
+func TestUpdateClusterRace(t *testing.T) {
+	clusterName := "foobarCluster"
+	// create dummy httpserver
+	testClusterServer := httptest.NewServer(createHttptestFakeHandlerForCluster(true))
+	defer testClusterServer.Close()
+	federationCluster := newCluster(clusterName, testClusterServer.URL)
+	federationClusterList := newClusterList(federationCluster)
+
+	testFederationServer := httptest.NewServer(createHttptestFakeHandlerForFederation(federationClusterList, true))
+	defer testFederationServer.Close()
+
+	restClientCfg, err := clientcmd.BuildConfigFromFlags(testFederationServer.URL, "")
+	if err != nil {
+		t.Errorf("Failed to build client config")
+	}
+	federationClientSet := federationclientset.NewForConfigOrDie(restclient.AddUserAgent(restClientCfg, "cluster-controller"))
+
+	manager := newClusterController(federationClientSet, 1*time.Millisecond)
+
+	stop := make(chan struct{})
+	manager.Run(stop)
+
+	// try to trigger the race in UpdateClusterStatus
+	for i := 0; i < 10; i++ {
+		manager.addToClusterSet(federationCluster)
+		manager.delFromClusterSet(federationCluster)
+	}
+
+	close(stop)
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/cluster/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package cluster contains code for syncing cluster
+package cluster // import "k8s.io/kubernetes/federation/pkg/federation-controller/cluster"

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package federation_controller contains code for controllers (like the cluster
+// controller).
+package federation_controller // import "k8s.io/kubernetes/federation/pkg/federation-controller"

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/ingress/BUILD

@@ -0,0 +1,74 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["ingress_controller.go"],
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//federation/pkg/federation-controller/util/clusterselector:go_default_library",
+        "//federation/pkg/federation-controller/util/deletionhelper:go_default_library",
+        "//federation/pkg/federation-controller/util/eventsink:go_default_library",
+        "//pkg/api:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+        "//vendor/k8s.io/client-go/tools/record:go_default_library",
+        "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["ingress_controller_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset/fake:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//federation/pkg/federation-controller/util/deletionhelper:go_default_library",
+        "//federation/pkg/federation-controller/util/finalizers:go_default_library",
+        "//federation/pkg/federation-controller/util/test:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/ingress/ingress_controller.go

@@ -0,0 +1,932 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ingress
+
+import (
+	"crypto/md5"
+	"fmt"
+	"sync"
+	"time"
+
+	"k8s.io/api/core/v1"
+	extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pkgruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/watch"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/flowcontrol"
+	federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	federationclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/clusterselector"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/deletionhelper"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/eventsink"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/controller"
+
+	"github.com/golang/glog"
+)
+
+const (
+	// Special cluster name which denotes all clusters - only used internally.  It's not a valid cluster name, so effectively reserved.
+	allClustersKey = ".ALL_CLUSTERS"
+	// TODO: Get the constants below directly from the Kubernetes Ingress Controller constants - but thats in a separate repo
+	staticIPNameKeyWritable = "kubernetes.io/ingress.global-static-ip-name" // The writable annotation on Ingress to tell the controller to use a specific, named, static IP
+	staticIPNameKeyReadonly = "ingress.kubernetes.io/static-ip"             // The readonly key via which the cluster's Ingress Controller communicates which static IP it used.  If staticIPNameKeyWritable above is specified, it is used.
+	uidAnnotationKey        = "kubernetes.io/ingress.uid"                   // The annotation on federation clusters, where we store the ingress UID
+	uidConfigMapName        = "ingress-uid"                                 // Name of the config-map and key the ingress controller stores its uid in.
+	uidConfigMapNamespace   = "kube-system"
+	uidKey                  = "uid"
+	providerUidKey          = "provider-uid"
+	// Annotation on the ingress in federation control plane that is used to keep
+	// track of the first cluster in which we create ingress.
+	// We wait for ingress to be created in this cluster before creating it any
+	// other cluster.
+	firstClusterAnnotation = "ingress.federation.kubernetes.io/first-cluster"
+	ControllerName         = "ingresses"
+	UserAgentName          = "federation-ingresses-controller"
+)
+
+var (
+	RequiredResources = []schema.GroupVersionResource{extensionsv1beta1.SchemeGroupVersion.WithResource("ingresses")}
+)
+
+type IngressController struct {
+	sync.Mutex // Lock used for leader election
+	// For triggering single ingress reconciliation. This is used when there is an
+	// add/update/delete operation on an ingress in either federated API server or
+	// in some member of the federation.
+	ingressDeliverer *util.DelayingDeliverer
+
+	// For triggering reconciliation of cluster ingress controller configmap and
+	// all ingresses. This is used when a new cluster becomes available.
+	clusterDeliverer *util.DelayingDeliverer
+
+	// For triggering reconciliation of cluster ingress controller configmap.
+	// This is used when a configmap is updated in the cluster.
+	configMapDeliverer *util.DelayingDeliverer
+
+	// Contains ingresses present in members of federation.
+	ingressFederatedInformer util.FederatedInformer
+	// Contains ingress controller configmaps present in members of federation.
+	configMapFederatedInformer util.FederatedInformer
+	// For updating ingresses in members of federation.
+	federatedIngressUpdater util.FederatedUpdater
+	// For updating configmaps in members of federation.
+	federatedConfigMapUpdater util.FederatedUpdater
+	// Definitions of ingresses that should be federated.
+	ingressInformerStore cache.Store
+	// Informer controller for ingresses that should be federated.
+	ingressInformerController cache.Controller
+
+	// Client to federated api server.
+	federatedApiClient federationclientset.Interface
+
+	// Backoff manager for ingresses
+	ingressBackoff *flowcontrol.Backoff
+	// Backoff manager for configmaps
+	configMapBackoff *flowcontrol.Backoff
+
+	// For events
+	eventRecorder record.EventRecorder
+
+	deletionHelper *deletionhelper.DeletionHelper
+
+	ingressReviewDelay    time.Duration
+	configMapReviewDelay  time.Duration
+	clusterAvailableDelay time.Duration
+	smallDelay            time.Duration
+	updateTimeout         time.Duration
+}
+
+// NewIngressController returns a new ingress controller
+func NewIngressController(client federationclientset.Interface) *IngressController {
+	glog.V(4).Infof("->NewIngressController V(4)")
+	broadcaster := record.NewBroadcaster()
+	broadcaster.StartRecordingToSink(eventsink.NewFederatedEventSink(client))
+	recorder := broadcaster.NewRecorder(api.Scheme, v1.EventSource{Component: UserAgentName})
+	ic := &IngressController{
+		federatedApiClient:    client,
+		ingressReviewDelay:    time.Second * 10,
+		configMapReviewDelay:  time.Second * 10,
+		clusterAvailableDelay: time.Second * 20,
+		smallDelay:            time.Second * 3,
+		updateTimeout:         time.Second * 30,
+		ingressBackoff:        flowcontrol.NewBackOff(5*time.Second, time.Minute),
+		eventRecorder:         recorder,
+		configMapBackoff:      flowcontrol.NewBackOff(5*time.Second, time.Minute),
+	}
+
+	// Build deliverers for triggering reconciliations.
+	ic.ingressDeliverer = util.NewDelayingDeliverer()
+	ic.clusterDeliverer = util.NewDelayingDeliverer()
+	ic.configMapDeliverer = util.NewDelayingDeliverer()
+
+	// Start informer in federated API servers on ingresses that should be federated.
+	ic.ingressInformerStore, ic.ingressInformerController = cache.NewInformer(
+		&cache.ListWatch{
+			ListFunc: func(options metav1.ListOptions) (pkgruntime.Object, error) {
+				return client.Extensions().Ingresses(metav1.NamespaceAll).List(options)
+			},
+			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+				return client.Extensions().Ingresses(metav1.NamespaceAll).Watch(options)
+			},
+		},
+		&extensionsv1beta1.Ingress{},
+		controller.NoResyncPeriodFunc(),
+		util.NewTriggerOnAllChanges(
+			func(obj pkgruntime.Object) {
+				ic.deliverIngressObj(obj, 0, false)
+			},
+		))
+
+	// Federated informer on ingresses in members of federation.
+	ic.ingressFederatedInformer = util.NewFederatedInformer(
+		client,
+		func(cluster *federationapi.Cluster, targetClient kubeclientset.Interface) (cache.Store, cache.Controller) {
+			return cache.NewInformer(
+				&cache.ListWatch{
+					ListFunc: func(options metav1.ListOptions) (pkgruntime.Object, error) {
+						return targetClient.Extensions().Ingresses(metav1.NamespaceAll).List(options)
+					},
+					WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+						return targetClient.Extensions().Ingresses(metav1.NamespaceAll).Watch(options)
+					},
+				},
+				&extensionsv1beta1.Ingress{},
+				controller.NoResyncPeriodFunc(),
+				// Trigger reconciliation whenever something in federated cluster is changed. In most cases it
+				// would be just confirmation that some ingress operation succeeded.
+				util.NewTriggerOnAllChanges(
+					func(obj pkgruntime.Object) {
+						ic.deliverIngressObj(obj, ic.ingressReviewDelay, false)
+					},
+				))
+		},
+
+		&util.ClusterLifecycleHandlerFuncs{
+			ClusterAvailable: func(cluster *federationapi.Cluster) {
+				// When new cluster becomes available process all the ingresses again, and configure it's ingress controller's configmap with the correct UID
+				ic.clusterDeliverer.DeliverAfter(cluster.Name, cluster, ic.clusterAvailableDelay)
+			},
+		},
+	)
+
+	// Federated informer on configmaps for ingress controllers in members of the federation.
+	ic.configMapFederatedInformer = util.NewFederatedInformer(
+		client,
+		func(cluster *federationapi.Cluster, targetClient kubeclientset.Interface) (cache.Store, cache.Controller) {
+			glog.V(4).Infof("Returning new informer for cluster %q", cluster.Name)
+			return cache.NewInformer(
+				&cache.ListWatch{
+					ListFunc: func(options metav1.ListOptions) (pkgruntime.Object, error) {
+						if targetClient == nil {
+							glog.Errorf("Internal error: targetClient is nil")
+						}
+						return targetClient.Core().ConfigMaps(uidConfigMapNamespace).List(options) // we only want to list one by name - unfortunately Kubernetes don't have a selector for that.
+					},
+					WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+						if targetClient == nil {
+							glog.Errorf("Internal error: targetClient is nil")
+						}
+						return targetClient.Core().ConfigMaps(uidConfigMapNamespace).Watch(options) // as above
+					},
+				},
+				&v1.ConfigMap{},
+				controller.NoResyncPeriodFunc(),
+				// Trigger reconciliation whenever the ingress controller's configmap in a federated cluster is changed. In most cases it
+				// would be just confirmation that the configmap for the ingress controller is correct.
+				util.NewTriggerOnAllChanges(
+					func(obj pkgruntime.Object) {
+						ic.deliverConfigMapObj(cluster.Name, obj, ic.configMapReviewDelay, false)
+					},
+				))
+		},
+
+		&util.ClusterLifecycleHandlerFuncs{
+			ClusterAvailable: func(cluster *federationapi.Cluster) {
+				ic.clusterDeliverer.DeliverAfter(cluster.Name, cluster, ic.clusterAvailableDelay)
+			},
+		},
+	)
+
+	// Federated ingress updater along with Create/Update/Delete operations.
+	ic.federatedIngressUpdater = util.NewFederatedUpdater(ic.ingressFederatedInformer, "ingress", ic.updateTimeout, ic.eventRecorder,
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			ingress := obj.(*extensionsv1beta1.Ingress)
+			glog.V(4).Infof("Attempting to create Ingress: %v", ingress)
+			_, err := client.Extensions().Ingresses(ingress.Namespace).Create(ingress)
+			if err != nil {
+				glog.Errorf("Error creating ingress %q: %v", types.NamespacedName{Name: ingress.Name, Namespace: ingress.Namespace}, err)
+			} else {
+				glog.V(4).Infof("Successfully created ingress %q", types.NamespacedName{Name: ingress.Name, Namespace: ingress.Namespace})
+			}
+			return err
+		},
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			ingress := obj.(*extensionsv1beta1.Ingress)
+			glog.V(4).Infof("Attempting to update Ingress: %v", ingress)
+			_, err := client.Extensions().Ingresses(ingress.Namespace).Update(ingress)
+			if err != nil {
+				glog.V(4).Infof("Failed to update Ingress: %v", err)
+			} else {
+				glog.V(4).Infof("Successfully updated Ingress: %q", types.NamespacedName{Name: ingress.Name, Namespace: ingress.Namespace})
+			}
+			return err
+		},
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			ingress := obj.(*extensionsv1beta1.Ingress)
+			glog.V(4).Infof("Attempting to delete Ingress: %v", ingress)
+			orphanDependents := false
+			err := client.Extensions().Ingresses(ingress.Namespace).Delete(ingress.Name, &metav1.DeleteOptions{OrphanDependents: &orphanDependents})
+			return err
+		})
+
+	// Federated configmap updater along with Create/Update/Delete operations.  Only Update should ever be called.
+	ic.federatedConfigMapUpdater = util.NewFederatedUpdater(ic.configMapFederatedInformer, "configmap", ic.updateTimeout, ic.eventRecorder,
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			configMap := obj.(*v1.ConfigMap)
+			configMapName := types.NamespacedName{Name: configMap.Name, Namespace: configMap.Namespace}
+			glog.Errorf("Internal error: Incorrectly attempting to create ConfigMap: %q", configMapName)
+			_, err := client.Core().ConfigMaps(configMap.Namespace).Create(configMap)
+			return err
+		},
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			configMap := obj.(*v1.ConfigMap)
+			configMapName := types.NamespacedName{Name: configMap.Name, Namespace: configMap.Namespace}
+			glog.V(4).Infof("Attempting to update ConfigMap: %v", configMap)
+			_, err := client.Core().ConfigMaps(configMap.Namespace).Update(configMap)
+			if err == nil {
+				glog.V(4).Infof("Successfully updated ConfigMap %q %v", configMapName, configMap)
+			} else {
+				glog.V(4).Infof("Failed to update ConfigMap %q: %v", configMapName, err)
+			}
+			return err
+		},
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			configMap := obj.(*v1.ConfigMap)
+			configMapName := types.NamespacedName{Name: configMap.Name, Namespace: configMap.Namespace}
+			glog.Errorf("Internal error: Incorrectly attempting to delete ConfigMap: %q", configMapName)
+			err := client.Core().ConfigMaps(configMap.Namespace).Delete(configMap.Name, &metav1.DeleteOptions{})
+			return err
+		})
+
+	ic.deletionHelper = deletionhelper.NewDeletionHelper(
+		ic.updateIngress,
+		// objNameFunc
+		func(obj pkgruntime.Object) string {
+			ingress := obj.(*extensionsv1beta1.Ingress)
+			return fmt.Sprintf("%s/%s", ingress.Namespace, ingress.Name)
+		},
+		ic.ingressFederatedInformer,
+		ic.federatedIngressUpdater,
+	)
+	return ic
+}
+
+// Sends the given updated object to apiserver.
+// Assumes that the given object is an ingress.
+func (ic *IngressController) updateIngress(obj pkgruntime.Object) (pkgruntime.Object, error) {
+	ingress := obj.(*extensionsv1beta1.Ingress)
+	return ic.federatedApiClient.Extensions().Ingresses(ingress.Namespace).Update(ingress)
+}
+
+func (ic *IngressController) Run(stopChan <-chan struct{}) {
+	glog.Infof("Starting Ingress Controller")
+	go ic.ingressInformerController.Run(stopChan)
+	glog.Infof("... Starting Ingress Federated Informer")
+	ic.ingressFederatedInformer.Start()
+	glog.Infof("... Starting ConfigMap Federated Informer")
+	ic.configMapFederatedInformer.Start()
+	go func() {
+		<-stopChan
+		glog.Infof("Stopping Ingress Federated Informer")
+		ic.ingressFederatedInformer.Stop()
+		glog.Infof("Stopping ConfigMap Federated Informer")
+		ic.configMapFederatedInformer.Stop()
+		glog.Infof("Stopping ingress deliverer")
+		ic.ingressDeliverer.Stop()
+		glog.Infof("Stopping configmap deliverer")
+		ic.configMapDeliverer.Stop()
+		glog.Infof("Stopping cluster deliverer")
+		ic.clusterDeliverer.Stop()
+	}()
+	ic.ingressDeliverer.StartWithHandler(func(item *util.DelayingDelivererItem) {
+		ingress := item.Value.(types.NamespacedName)
+		glog.V(4).Infof("Ingress change delivered, reconciling: %v", ingress)
+		ic.reconcileIngress(ingress)
+	})
+	ic.clusterDeliverer.StartWithHandler(func(item *util.DelayingDelivererItem) {
+		clusterName := item.Key
+		if clusterName != allClustersKey {
+			glog.V(4).Infof("Cluster change delivered for cluster %q, reconciling configmap and ingress for that cluster", clusterName)
+		} else {
+			glog.V(4).Infof("Cluster change delivered for all clusters, reconciling configmaps and ingresses for all clusters")
+		}
+		ic.reconcileIngressesOnClusterChange(clusterName)
+		ic.reconcileConfigMapForCluster(clusterName)
+	})
+	ic.configMapDeliverer.StartWithHandler(func(item *util.DelayingDelivererItem) {
+		clusterName := item.Key
+		if clusterName != allClustersKey {
+			glog.V(4).Infof("ConfigMap change delivered for cluster %q, reconciling configmap for that cluster", clusterName)
+		} else {
+			glog.V(4).Infof("ConfigMap change delivered for all clusters, reconciling configmaps for all clusters")
+		}
+		ic.reconcileConfigMapForCluster(clusterName)
+	})
+
+	util.StartBackoffGC(ic.ingressBackoff, stopChan)
+	util.StartBackoffGC(ic.configMapBackoff, stopChan)
+}
+
+func (ic *IngressController) deliverIngressObj(obj interface{}, delay time.Duration, failed bool) {
+	ingress := obj.(*extensionsv1beta1.Ingress)
+	ic.deliverIngress(types.NamespacedName{Namespace: ingress.Namespace, Name: ingress.Name}, delay, failed)
+}
+
+func (ic *IngressController) deliverIngress(ingress types.NamespacedName, delay time.Duration, failed bool) {
+	glog.V(4).Infof("Delivering ingress: %s with delay: %v error: %v", ingress, delay, failed)
+	key := ingress.String()
+	if failed {
+		ic.ingressBackoff.Next(key, time.Now())
+		delay = delay + ic.ingressBackoff.Get(key)
+	} else {
+		ic.ingressBackoff.Reset(key)
+	}
+	ic.ingressDeliverer.DeliverAfter(key, ingress, delay)
+}
+
+func (ic *IngressController) deliverConfigMapObj(clusterName string, obj interface{}, delay time.Duration, failed bool) {
+	configMap := obj.(*v1.ConfigMap)
+	ic.deliverConfigMap(clusterName, types.NamespacedName{Namespace: configMap.Namespace, Name: configMap.Name}, delay, failed)
+}
+
+func (ic *IngressController) deliverConfigMap(cluster string, configMap types.NamespacedName, delay time.Duration, failed bool) {
+	key := cluster
+	if failed {
+		ic.configMapBackoff.Next(key, time.Now())
+		delay = delay + ic.configMapBackoff.Get(key)
+	} else {
+		ic.configMapBackoff.Reset(key)
+	}
+	glog.V(4).Infof("Delivering ConfigMap for cluster %q (delay %q): %s", cluster, delay, configMap)
+	ic.configMapDeliverer.DeliverAfter(key, configMap, delay)
+}
+
+// Check whether all data stores are in sync. False is returned if any of the informer/stores is not yet
+// synced with the corresponding api server.
+func (ic *IngressController) isSynced() bool {
+	if !ic.ingressFederatedInformer.ClustersSynced() {
+		glog.V(2).Infof("Cluster list not synced for ingress federated informer")
+		return false
+	}
+	clusters, err := ic.ingressFederatedInformer.GetReadyClusters()
+	if err != nil {
+		glog.Errorf("Failed to get ready clusters for ingress federated informer: %v", err)
+		return false
+	}
+	if !ic.ingressFederatedInformer.GetTargetStore().ClustersSynced(clusters) {
+		glog.V(2).Infof("Target store not synced for ingress federated informer")
+		return false
+	}
+	if !ic.configMapFederatedInformer.ClustersSynced() {
+		glog.V(2).Infof("Cluster list not synced for config map federated informer")
+		return false
+	}
+	clusters, err = ic.configMapFederatedInformer.GetReadyClusters()
+	if err != nil {
+		glog.Errorf("Failed to get ready clusters for configmap federated informer: %v", err)
+		return false
+	}
+	if !ic.configMapFederatedInformer.GetTargetStore().ClustersSynced(clusters) {
+		glog.V(2).Infof("Target store not synced for configmap federated informer")
+		return false
+	}
+	glog.V(4).Infof("Cluster list is synced")
+	return true
+}
+
+// The function triggers reconciliation of all federated ingresses.  clusterName is the name of the cluster that changed
+// but all ingresses in all clusters are reconciled
+func (ic *IngressController) reconcileIngressesOnClusterChange(clusterName string) {
+	glog.V(4).Infof("Reconciling ingresses on cluster change for cluster %q", clusterName)
+	if !ic.isSynced() {
+		glog.V(4).Infof("Not synced, will try again later to reconcile ingresses.")
+		ic.clusterDeliverer.DeliverAfter(clusterName, nil, ic.clusterAvailableDelay)
+	}
+	ingressList := ic.ingressInformerStore.List()
+	if len(ingressList) <= 0 {
+		glog.V(4).Infof("No federated ingresses to reconcile.")
+	}
+
+	for _, obj := range ingressList {
+		ingress := obj.(*extensionsv1beta1.Ingress)
+		nsName := types.NamespacedName{Name: ingress.Name, Namespace: ingress.Namespace}
+		glog.V(4).Infof("Delivering federated ingress %q for cluster %q", nsName, clusterName)
+		ic.deliverIngress(nsName, ic.smallDelay, false)
+	}
+}
+
+/*
+  reconcileConfigMapForCluster ensures that the configmap for the ingress controller in the cluster has objectmeta.data.UID
+  consistent with all the other clusters in the federation. If clusterName == allClustersKey, then all available clusters
+  configmaps are reconciled.
+*/
+func (ic *IngressController) reconcileConfigMapForCluster(clusterName string) {
+	glog.V(4).Infof("Reconciling ConfigMap for cluster(s) %q", clusterName)
+
+	if !ic.isSynced() {
+		ic.configMapDeliverer.DeliverAfter(clusterName, nil, ic.clusterAvailableDelay)
+		return
+	}
+
+	if clusterName == allClustersKey {
+		clusters, err := ic.configMapFederatedInformer.GetReadyClusters()
+		if err != nil {
+			glog.Errorf("Failed to get ready clusters.  redelivering %q: %v", clusterName, err)
+			ic.configMapDeliverer.DeliverAfter(clusterName, nil, ic.clusterAvailableDelay)
+			return
+		}
+		for _, cluster := range clusters {
+			glog.V(4).Infof("Delivering ConfigMap for cluster(s) %q", clusterName)
+			ic.configMapDeliverer.DeliverAt(cluster.Name, nil, time.Now())
+		}
+		return
+	} else {
+		cluster, found, err := ic.configMapFederatedInformer.GetReadyCluster(clusterName)
+		if err != nil || !found {
+			glog.Errorf("Internal error: Cluster %q queued for configmap reconciliation, but not found.  Will try again later: error = %v", clusterName, err)
+			ic.configMapDeliverer.DeliverAfter(clusterName, nil, ic.clusterAvailableDelay)
+			return
+		}
+		uidConfigMapNamespacedName := types.NamespacedName{Name: uidConfigMapName, Namespace: uidConfigMapNamespace}
+		configMapObj, found, err := ic.configMapFederatedInformer.GetTargetStore().GetByKey(cluster.Name, uidConfigMapNamespacedName.String())
+		if !found || err != nil {
+			logmsg := fmt.Sprintf("Failed to get ConfigMap %q for cluster %q.  Will try again later", uidConfigMapNamespacedName, cluster.Name)
+			if err != nil {
+				logmsg = fmt.Sprintf("%v: %v", logmsg, err)
+			}
+			if len(ic.ingressInformerStore.List()) > 0 { // Error-level if ingresses are active, Info-level otherwise.
+				glog.Errorf(logmsg)
+			} else {
+				glog.V(4).Infof(logmsg)
+			}
+			ic.configMapDeliverer.DeliverAfter(clusterName, nil, ic.configMapReviewDelay)
+			return
+		}
+		glog.V(4).Infof("Successfully got ConfigMap %q for cluster %q.", uidConfigMapNamespacedName, clusterName)
+		configMap, ok := configMapObj.(*v1.ConfigMap)
+		if !ok {
+			glog.Errorf("Internal error: The object in the ConfigMap cache for cluster %q configmap %q is not a *ConfigMap", cluster.Name, uidConfigMapNamespacedName)
+			return
+		}
+		ic.reconcileConfigMap(cluster, configMap)
+		return
+	}
+}
+
+// getProviderUid returns a provider ID based on the provided clusterName.
+func getProviderUid(clusterName string) string {
+	hashedName := md5.Sum([]byte(clusterName))
+	return fmt.Sprintf("%x", hashedName[:8])
+}
+
+/*
+  reconcileConfigMap ensures that the configmap in the cluster has a UID
+  consistent with the federation cluster's associated annotation.
+
+  1. If the UID in the configmap differs from the UID stored in the cluster's annotation, the configmap is updated.
+  2. If the UID annotation is missing from the cluster, the cluster's UID annotation is updated to be consistent
+  with the master cluster.
+  3. If there is no elected master cluster, this cluster attempts to elect itself as the master cluster.
+
+  In cases 2 and 3, the configmaps will be updated in the next cycle, triggered by the federation cluster update(s)
+
+*/
+func (ic *IngressController) reconcileConfigMap(cluster *federationapi.Cluster, configMap *v1.ConfigMap) {
+	ic.Lock() // TODO: Reduce the scope of this master election lock.
+	defer ic.Unlock()
+
+	configMapNsName := types.NamespacedName{Name: configMap.Name, Namespace: configMap.Namespace}
+	glog.V(4).Infof("Reconciling ConfigMap %q in cluster %q", configMapNsName, cluster.Name)
+
+	clusterIngressUID, clusterIngressUIDExists := cluster.ObjectMeta.Annotations[uidAnnotationKey]
+	configMapUID, ok := configMap.Data[uidKey]
+
+	if !ok {
+		glog.Errorf("Warning: ConfigMap %q in cluster %q does not contain data key %q.  Therefore it cannot become the master.", configMapNsName, cluster.Name, uidKey)
+	}
+
+	if !clusterIngressUIDExists || clusterIngressUID == "" {
+		glog.V(4).Infof("Cluster %q is the only master", cluster.Name)
+		// Second argument is the fallback, in case this is the only cluster, in which case it becomes the master
+		var err error
+		if clusterIngressUID, err = ic.updateClusterIngressUIDToMasters(cluster, configMapUID); err != nil {
+			return
+		}
+		// If we successfully update the Cluster Object, fallthrough and update the configMap.
+	}
+
+	// Figure out providerUid.
+	providerUid := getProviderUid(cluster.Name)
+	configMapProviderUid := configMap.Data[providerUidKey]
+
+	if configMapUID == clusterIngressUID && configMapProviderUid == providerUid {
+		glog.V(4).Infof("Ingress configMap update is not required: UID %q and ProviderUid %q are equal", configMapUID, providerUid)
+	} else {
+		if configMapUID != clusterIngressUID {
+			glog.V(4).Infof("Ingress configMap update is required for UID: configMapUID %q not equal to clusterIngressUID %q", configMapUID, clusterIngressUID)
+		} else if configMapProviderUid != providerUid {
+			glog.V(4).Infof("Ingress configMap update is required: configMapProviderUid %q not equal to providerUid %q", configMapProviderUid, providerUid)
+		}
+		configMap.Data[uidKey] = clusterIngressUID
+		configMap.Data[providerUidKey] = providerUid
+		operations := []util.FederatedOperation{{
+			Type:        util.OperationTypeUpdate,
+			Obj:         configMap,
+			ClusterName: cluster.Name,
+			Key:         configMapNsName.String(),
+		}}
+		glog.V(4).Infof("Calling federatedConfigMapUpdater.Update() - operations: %v", operations)
+		err := ic.federatedConfigMapUpdater.Update(operations)
+		if err != nil {
+			glog.Errorf("Failed to execute update of ConfigMap %q on cluster %q: %v", configMapNsName, cluster.Name, err)
+			ic.configMapDeliverer.DeliverAfter(cluster.Name, nil, ic.configMapReviewDelay)
+		}
+	}
+}
+
+/*
+  getMasterCluster returns the cluster which is the elected master w.r.t. ingress UID, and it's ingress UID.
+  If there is no elected master cluster, an error is returned.
+  All other clusters must use the ingress UID of the elected master.
+*/
+func (ic *IngressController) getMasterCluster() (master *federationapi.Cluster, ingressUID string, err error) {
+	clusters, err := ic.configMapFederatedInformer.GetReadyClusters()
+	if err != nil {
+		glog.Errorf("Failed to get cluster list: %v", err)
+		return nil, "", err
+	}
+
+	for _, c := range clusters {
+		UID, exists := c.ObjectMeta.Annotations[uidAnnotationKey]
+		if exists && UID != "" { // Found the master cluster
+			glog.V(4).Infof("Found master cluster %q with annotation %q=%q", c.Name, uidAnnotationKey, UID)
+			return c, UID, nil
+		}
+	}
+	return nil, "", fmt.Errorf("Failed to find master cluster with annotation %q", uidAnnotationKey)
+}
+
+/*
+  updateClusterIngressUIDToMasters takes the ingress UID annotation on the master cluster and applies it to cluster.
+  If there is no master cluster, then fallbackUID is used (and hence this cluster becomes the master).
+*/
+func (ic *IngressController) updateClusterIngressUIDToMasters(cluster *federationapi.Cluster, fallbackUID string) (string, error) {
+	masterCluster, masterUID, err := ic.getMasterCluster()
+	clusterObj, clusterErr := api.Scheme.DeepCopy(cluster) // Make a clone so that we don't clobber our input param
+	cluster, ok := clusterObj.(*federationapi.Cluster)
+	if clusterErr != nil || !ok {
+		glog.Errorf("Internal error: Failed clone cluster resource while attempting to add master ingress UID annotation (%q = %q) from master cluster %q to cluster %q, will try again later: %v", uidAnnotationKey, masterUID, masterCluster.Name, cluster.Name, clusterErr)
+		return "", clusterErr
+	}
+
+	if err == nil {
+		if masterCluster.Name != cluster.Name { // We're not the master, need to get in sync
+			if cluster.ObjectMeta.Annotations == nil {
+				cluster.ObjectMeta.Annotations = map[string]string{}
+			}
+			cluster.ObjectMeta.Annotations[uidAnnotationKey] = masterUID
+			if _, err = ic.federatedApiClient.Federation().Clusters().Update(cluster); err != nil {
+				glog.Errorf("Failed to add master ingress UID annotation (%q = %q) from master cluster %q to cluster %q, will try again later: %v", uidAnnotationKey, masterUID, masterCluster.Name, cluster.Name, err)
+				return "", err
+			} else {
+				glog.V(4).Infof("Successfully added master ingress UID annotation (%q = %q) from master cluster %q to cluster %q.", uidAnnotationKey, masterUID, masterCluster.Name, cluster.Name)
+				return masterUID, nil
+			}
+		} else {
+			glog.V(4).Infof("Cluster %q with ingress UID is already the master with annotation (%q = %q), no need to update.", cluster.Name, uidAnnotationKey, cluster.ObjectMeta.Annotations[uidAnnotationKey])
+			return cluster.ObjectMeta.Annotations[uidAnnotationKey], nil
+		}
+	} else {
+		glog.V(2).Infof("No master cluster found to source an ingress UID from for cluster %q.", cluster.Name)
+		if fallbackUID != "" {
+			glog.V(2).Infof("Attempting to elect new master cluster %q with ingress UID %q = %q", cluster.Name, uidAnnotationKey, fallbackUID)
+			if cluster.ObjectMeta.Annotations == nil {
+				cluster.ObjectMeta.Annotations = map[string]string{}
+			}
+			cluster.ObjectMeta.Annotations[uidAnnotationKey] = fallbackUID
+			if _, err = ic.federatedApiClient.Federation().Clusters().Update(cluster); err != nil {
+				glog.Errorf("Failed to add ingress UID annotation (%q = %q) to cluster %q. No master elected. Will try again later: %v", uidAnnotationKey, fallbackUID, cluster.Name, err)
+				return "", err
+			} else {
+				glog.V(4).Infof("Successfully added ingress UID annotation (%q = %q) to cluster %q.", uidAnnotationKey, fallbackUID, cluster.Name)
+				return fallbackUID, nil
+			}
+		} else {
+			glog.Errorf("No master cluster exists, and fallbackUID for cluster %q is nil.  This probably means that no clusters have an ingress controller configmap with key %q.  Federated Ingress currently supports clusters running Google Loadbalancer Controller (\"GLBC\")", cluster.Name, uidKey)
+			return "", err
+		}
+	}
+}
+
+func (ic *IngressController) isClusterReady(clusterName string) bool {
+	cluster, isReady, err := ic.ingressFederatedInformer.GetReadyCluster(clusterName)
+	return isReady && err == nil && cluster != nil
+}
+
+// updateAnnotationOnIngress updates the annotation with the given key on the given federated ingress.
+// Queues the ingress for resync when done.
+func (ic *IngressController) updateAnnotationOnIngress(ingress *extensionsv1beta1.Ingress, key, value string) {
+	if ingress.ObjectMeta.Annotations == nil {
+		ingress.ObjectMeta.Annotations = make(map[string]string)
+	}
+	ingress.ObjectMeta.Annotations[key] = value
+	ingressName := types.NamespacedName{Name: ingress.Name, Namespace: ingress.Namespace}
+	glog.V(4).Infof("Attempting to update annotation %s:%s on base federated ingress: %v", key, value, ingressName)
+	if updatedFedIngress, err := ic.federatedApiClient.Extensions().Ingresses(ingress.Namespace).Update(ingress); err != nil {
+		glog.Errorf("Failed to update annotation %s:%s on federated ingress %q, will try again later: %v", key, value, ingressName, err)
+		ic.deliverIngress(ingressName, ic.ingressReviewDelay, true)
+		return
+	} else {
+		glog.V(4).Infof("Successfully updated annotation %s:%s on federated ingress %q, after update: %q", key, value, ingress, updatedFedIngress)
+		ic.deliverIngress(ingressName, ic.smallDelay, false)
+		return
+	}
+}
+
+func (ic *IngressController) reconcileIngress(ingress types.NamespacedName) {
+	glog.V(4).Infof("Reconciling ingress %q for all clusters", ingress)
+	if !ic.isSynced() {
+		ic.deliverIngress(ingress, ic.clusterAvailableDelay, false)
+		return
+	}
+
+	key := ingress.String()
+	baseIngressObjFromStore, exist, err := ic.ingressInformerStore.GetByKey(key)
+	if err != nil {
+		glog.Errorf("Failed to query main ingress store for %v: %v", ingress, err)
+		ic.deliverIngress(ingress, 0, true)
+		return
+	}
+	if !exist {
+		// Not federated ingress, ignoring.
+		glog.V(4).Infof("Ingress %q is not federated.  Ignoring.", ingress)
+		return
+	}
+	baseIngressObj, err := api.Scheme.DeepCopy(baseIngressObjFromStore)
+	baseIngress, ok := baseIngressObj.(*extensionsv1beta1.Ingress)
+	if err != nil || !ok {
+		glog.Errorf("Internal Error %v : Object retrieved from ingressInformerStore with key %q is not of correct type *extensionsv1beta1.Ingress: %v", err, key, baseIngressObj)
+	} else {
+		glog.V(4).Infof("Base (federated) ingress: %v", baseIngress)
+	}
+
+	if baseIngress.DeletionTimestamp != nil {
+		if err := ic.delete(baseIngress); err != nil {
+			glog.Errorf("Failed to delete %s: %v", ingress, err)
+			ic.eventRecorder.Eventf(baseIngress, api.EventTypeWarning, "DeleteFailed",
+				"Ingress delete failed: %v", err)
+			ic.deliverIngress(ingress, 0, true)
+		}
+		return
+	}
+
+	glog.V(3).Infof("Ensuring delete object from underlying clusters finalizer for ingress: %s",
+		baseIngress.Name)
+	// Add the required finalizers before creating a ingress in underlying clusters.
+	updatedIngressObj, err := ic.deletionHelper.EnsureFinalizers(baseIngress)
+	if err != nil {
+		glog.Errorf("Failed to ensure delete object from underlying clusters finalizer in ingress %s: %v",
+			baseIngress.Name, err)
+		ic.deliverIngress(ingress, 0, true)
+		return
+	}
+	baseIngress = updatedIngressObj.(*extensionsv1beta1.Ingress)
+
+	glog.V(3).Infof("Syncing ingress %s in underlying clusters", baseIngress.Name)
+
+	clusters, err := ic.ingressFederatedInformer.GetReadyClusters()
+	if err != nil {
+		glog.Errorf("Failed to get cluster list: %v", err)
+		ic.deliverIngress(ingress, ic.clusterAvailableDelay, false)
+		return
+	} else {
+		glog.V(4).Infof("Found %d ready clusters across which to reconcile ingress %q", len(clusters), ingress)
+	}
+
+	operations := make([]util.FederatedOperation, 0)
+
+	for _, cluster := range clusters {
+		baseIPName, baseIPAnnotationExists := baseIngress.ObjectMeta.Annotations[staticIPNameKeyWritable]
+		firstClusterName, firstClusterExists := baseIngress.ObjectMeta.Annotations[firstClusterAnnotation]
+		clusterIngressObj, clusterIngressFound, err := ic.ingressFederatedInformer.GetTargetStore().GetByKey(cluster.Name, key)
+		if err != nil {
+			glog.Errorf("Failed to get cached ingress %s for cluster %s, will retry: %v", ingress, cluster.Name, err)
+			ic.deliverIngress(ingress, 0, true)
+			return
+		}
+		desiredIngress := &extensionsv1beta1.Ingress{}
+		objMeta, err := api.Scheme.DeepCopy(&baseIngress.ObjectMeta)
+		if err != nil {
+			glog.Errorf("Error deep copying ObjectMeta: %v", err)
+		}
+		objSpec, err := api.Scheme.DeepCopy(&baseIngress.Spec)
+		if err != nil {
+			glog.Errorf("Error deep copying Spec: %v", err)
+		}
+		objMetaCopy, ok := objMeta.(*metav1.ObjectMeta)
+		if !ok {
+			glog.Errorf("Internal error: Failed to cast to *metav1.ObjectMeta: %v", objMeta)
+		}
+		desiredIngress.ObjectMeta = *objMetaCopy
+		objSpecCopy, ok := objSpec.(*extensionsv1beta1.IngressSpec)
+		if !ok {
+			glog.Errorf("Internal error: Failed to cast to extensionsv1beta1.Ingressespec: %v", objSpec)
+		}
+		desiredIngress.Spec = *objSpecCopy
+		glog.V(4).Infof("Desired Ingress: %v", desiredIngress)
+
+		send, err := clusterselector.SendToCluster(cluster.Labels, desiredIngress.ObjectMeta.Annotations)
+		if err != nil {
+			glog.Errorf("Error processing ClusterSelector cluster: %s for Ingress map: %s error: %s", cluster.Name, key, err.Error())
+			return
+		}
+
+		switch {
+		case !clusterIngressFound && send:
+			glog.V(4).Infof("No existing Ingress %s in cluster %s - checking if appropriate to queue a create operation", ingress, cluster.Name)
+			// We can't supply server-created fields when creating a new object.
+			desiredIngress.ObjectMeta = util.DeepCopyRelevantObjectMeta(baseIngress.ObjectMeta)
+
+			// We always first create an ingress in the first available cluster. Once that ingress
+			// has been created and allocated a global IP (visible via an annotation),
+			// we record that annotation on the federated ingress, and create all other cluster
+			// ingresses with that same global IP.
+			// Note: If the first cluster becomes (e.g. temporarily) unavailable, the
+			// second cluster will become the first cluster, but eventually all ingresses
+			// will share the single global IP recorded in the annotation of the
+			// federated ingress.
+			haveFirstCluster := firstClusterExists && firstClusterName != "" && ic.isClusterReady(firstClusterName)
+			if !haveFirstCluster {
+				glog.V(4).Infof("No cluster has been chosen as the first cluster. Electing cluster %s as the first cluster to create ingress in", cluster.Name)
+				ic.updateAnnotationOnIngress(baseIngress, firstClusterAnnotation, cluster.Name)
+				return
+			}
+			if baseIPAnnotationExists || firstClusterName == cluster.Name {
+				if baseIPAnnotationExists {
+					glog.V(4).Infof("No existing Ingress %s in cluster %s and static IP annotation (%q) exists on base ingress - queuing a create operation", ingress, cluster.Name, staticIPNameKeyWritable)
+				} else {
+					glog.V(4).Infof("No existing Ingress %s in cluster %s and no static IP annotation (%q) on base ingress - queuing a create operation in first cluster", ingress, cluster.Name, staticIPNameKeyWritable)
+				}
+				operations = append(operations, util.FederatedOperation{
+					Type:        util.OperationTypeAdd,
+					Obj:         desiredIngress,
+					ClusterName: cluster.Name,
+					Key:         key,
+				})
+			} else {
+				glog.V(4).Infof("No annotation %q exists on ingress %q in federation and waiting for ingress in cluster %s. Not queueing create operation for ingress until annotation exists", staticIPNameKeyWritable, ingress, firstClusterName)
+			}
+		case clusterIngressFound && !send:
+			glog.V(5).Infof("Removing Ingress: %s from cluster: %s reason: cluster selectors do not match: %-v %-v", key, cluster.Name, cluster.ObjectMeta.Labels, desiredIngress.ObjectMeta.Annotations[federationapi.FederationClusterSelectorAnnotation])
+			operations = append(operations, util.FederatedOperation{
+				Type:        util.OperationTypeDelete,
+				Obj:         desiredIngress,
+				ClusterName: cluster.Name,
+				Key:         key,
+			})
+		case clusterIngressFound && send:
+			clusterIngress := clusterIngressObj.(*extensionsv1beta1.Ingress)
+			glog.V(4).Infof("Found existing Ingress %s in cluster %s - checking if update is required (in either direction)", ingress, cluster.Name)
+			clusterIPName, clusterIPNameExists := clusterIngress.ObjectMeta.Annotations[staticIPNameKeyReadonly]
+			baseLBStatusExists := len(baseIngress.Status.LoadBalancer.Ingress) > 0
+			clusterLBStatusExists := len(clusterIngress.Status.LoadBalancer.Ingress) > 0
+			logStr := fmt.Sprintf("Cluster ingress %q has annotation %q=%q, loadbalancer status exists? [%v], federated ingress has annotation %q=%q, loadbalancer status exists? [%v].  %%s annotation and/or loadbalancer status from cluster ingress to federated ingress.", ingress, staticIPNameKeyReadonly, clusterIPName, clusterLBStatusExists, staticIPNameKeyWritable, baseIPName, baseLBStatusExists)
+			if (!baseIPAnnotationExists && clusterIPNameExists) || (!baseLBStatusExists && clusterLBStatusExists) { // copy the IP name from the readonly annotation on the cluster ingress, to the writable annotation on the federated ingress
+				glog.V(4).Infof(logStr, "Transferring")
+				if !baseIPAnnotationExists && clusterIPNameExists {
+					ic.updateAnnotationOnIngress(baseIngress, staticIPNameKeyWritable, clusterIPName)
+					return
+				}
+				if !baseLBStatusExists && clusterLBStatusExists {
+					lbstatusObj, lbErr := api.Scheme.DeepCopy(&clusterIngress.Status.LoadBalancer)
+					lbstatus, ok := lbstatusObj.(*v1.LoadBalancerStatus)
+					if lbErr != nil || !ok {
+						glog.Errorf("Internal error: Failed to clone LoadBalancerStatus of %q in cluster %q while attempting to update master loadbalancer ingress status, will try again later. error: %v, Object to be cloned: %v", ingress, cluster.Name, lbErr, lbstatusObj)
+						ic.deliverIngress(ingress, ic.ingressReviewDelay, true)
+						return
+					}
+					baseIngress.Status.LoadBalancer = *lbstatus
+					glog.V(4).Infof("Attempting to update base federated ingress status: %v", baseIngress)
+					if updatedFedIngress, err := ic.federatedApiClient.Extensions().Ingresses(baseIngress.Namespace).UpdateStatus(baseIngress); err != nil {
+						glog.Errorf("Failed to update federated ingress status of %q (loadbalancer status), will try again later: %v", ingress, err)
+						ic.deliverIngress(ingress, ic.ingressReviewDelay, true)
+						return
+					} else {
+						glog.V(4).Infof("Successfully updated federated ingress status of %q (added loadbalancer status), after update: %q", ingress, updatedFedIngress)
+						ic.deliverIngress(ingress, ic.smallDelay, false)
+						return
+					}
+				}
+			} else {
+				glog.V(4).Infof(logStr, "Not transferring")
+			}
+			// Update existing cluster ingress, if needed.
+			if util.ObjectMetaAndSpecEquivalent(baseIngress, clusterIngress) {
+				glog.V(4).Infof("Ingress %q in cluster %q does not need an update: cluster ingress is equivalent to federated ingress", ingress, cluster.Name)
+			} else {
+				glog.V(4).Infof("Ingress %s in cluster %s needs an update: cluster ingress %v is not equivalent to federated ingress %v", ingress, cluster.Name, clusterIngress, desiredIngress)
+				objMeta, err := api.Scheme.DeepCopy(&clusterIngress.ObjectMeta)
+				if err != nil {
+					glog.Errorf("Error deep copying ObjectMeta: %v", err)
+					ic.deliverIngress(ingress, ic.ingressReviewDelay, true)
+				}
+				objMetaCopy, ok := objMeta.(*metav1.ObjectMeta)
+				if !ok {
+					glog.Errorf("Internal error: Failed to cast to metav1.ObjectMeta: %v", objMeta)
+					ic.deliverIngress(ingress, ic.ingressReviewDelay, true)
+				}
+				desiredIngress.ObjectMeta = *objMetaCopy
+				// Merge any annotations and labels on the federated ingress onto the underlying cluster ingress,
+				// overwriting duplicates.
+				if desiredIngress.ObjectMeta.Annotations == nil {
+					desiredIngress.ObjectMeta.Annotations = make(map[string]string)
+				}
+				for key, val := range baseIngress.ObjectMeta.Annotations {
+					desiredIngress.ObjectMeta.Annotations[key] = val
+				}
+				if desiredIngress.ObjectMeta.Labels == nil {
+					desiredIngress.ObjectMeta.Labels = make(map[string]string)
+				}
+				for key, val := range baseIngress.ObjectMeta.Labels {
+					desiredIngress.ObjectMeta.Labels[key] = val
+				}
+
+				operations = append(operations, util.FederatedOperation{
+					Type:        util.OperationTypeUpdate,
+					Obj:         desiredIngress,
+					ClusterName: cluster.Name,
+					Key:         key,
+				})
+				// TODO: Transfer any readonly (target-proxy, url-map etc) annotations from the master cluster to the federation, if this is the master cluster.
+				// This is only for consistency, so that the federation ingress metadata matches the underlying clusters.  It's not actually required				}
+			}
+		}
+	}
+
+	if len(operations) == 0 {
+		// Everything is in order
+		glog.V(4).Infof("Ingress %q is up-to-date in all clusters - no propagation to clusters required.", ingress)
+		return
+	}
+	glog.V(4).Infof("Calling federatedUpdater.Update() - operations: %v", operations)
+	err = ic.federatedIngressUpdater.Update(operations)
+	if err != nil {
+		glog.Errorf("Failed to execute updates for %s: %v", ingress, err)
+		ic.deliverIngress(ingress, ic.ingressReviewDelay, true)
+		return
+	}
+	// Schedule another periodic reconciliation, only to account for possible bugs in watch processing.
+	ic.deliverIngress(ingress, ic.ingressReviewDelay, false)
+}
+
+// delete deletes the given ingress or returns error if the deletion was not complete.
+func (ic *IngressController) delete(ingress *extensionsv1beta1.Ingress) error {
+	glog.V(3).Infof("Handling deletion of ingress: %v", *ingress)
+	_, err := ic.deletionHelper.HandleObjectInUnderlyingClusters(ingress)
+	if err != nil {
+		return err
+	}
+
+	err = ic.federatedApiClient.Extensions().Ingresses(ingress.Namespace).Delete(ingress.Name, nil)
+	if err != nil {
+		// Its all good if the error is not found error. That means it is deleted already and we do not have to do anything.
+		// This is expected when we are processing an update as a result of ingress finalizer deletion.
+		// The process that deleted the last finalizer is also going to delete the ingress and we do not have to do anything.
+		if !errors.IsNotFound(err) {
+			return fmt.Errorf("failed to delete ingress: %v", err)
+		}
+	}
+	return nil
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/ingress/ingress_controller_test.go

@@ -0,0 +1,397 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ingress
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+	"time"
+
+	apiv1 "k8s.io/api/core/v1"
+	extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	fakekubeclientset "k8s.io/client-go/kubernetes/fake"
+	"k8s.io/client-go/tools/cache"
+	federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	fakefedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset/fake"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/deletionhelper"
+	finalizersutil "k8s.io/kubernetes/federation/pkg/federation-controller/util/finalizers"
+	. "k8s.io/kubernetes/federation/pkg/federation-controller/util/test"
+
+	"github.com/golang/glog"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	maxTrials         = 20
+	clusters   string = "clusters"
+	ingresses  string = "ingresses"
+	configmaps string = "configmaps"
+)
+
+func TestIngressController(t *testing.T) {
+	fakeClusterList := federationapi.ClusterList{Items: []federationapi.Cluster{}}
+	fakeConfigMapList1 := apiv1.ConfigMapList{Items: []apiv1.ConfigMap{}}
+	fakeConfigMapList2 := apiv1.ConfigMapList{Items: []apiv1.ConfigMap{}}
+	cluster1 := NewCluster("cluster1", apiv1.ConditionTrue)
+	cluster2 := NewCluster("cluster2", apiv1.ConditionTrue)
+	cfg1 := NewConfigMap("foo")
+	cfg2 := NewConfigMap("bar") // Different UID from cfg1, so that we can check that they get reconciled.
+	assert.NotEqual(t, cfg1.Data[uidKey], cfg2.Data[uidKey], fmt.Sprintf("ConfigMap in cluster 2 must initially not equal that in cluster 1 for this test - please fix test"))
+
+	t.Log("Creating fake infrastructure")
+	fedClient := &fakefedclientset.Clientset{}
+	RegisterFakeList(clusters, &fedClient.Fake, &fakeClusterList)
+	RegisterFakeList(ingresses, &fedClient.Fake, &extensionsv1beta1.IngressList{Items: []extensionsv1beta1.Ingress{}})
+	fedIngressWatch := RegisterFakeWatch(ingresses, &fedClient.Fake)
+	clusterWatch := RegisterFakeWatch(clusters, &fedClient.Fake)
+	fedClusterUpdateChan := RegisterFakeCopyOnUpdate(clusters, &fedClient.Fake, clusterWatch)
+	fedIngressUpdateChan := RegisterFakeCopyOnUpdate(ingresses, &fedClient.Fake, fedIngressWatch)
+
+	cluster1Client := &fakekubeclientset.Clientset{}
+	RegisterFakeList(ingresses, &cluster1Client.Fake, &extensionsv1beta1.IngressList{Items: []extensionsv1beta1.Ingress{}})
+	RegisterFakeList(configmaps, &cluster1Client.Fake, &fakeConfigMapList1)
+	cluster1IngressWatch := RegisterFakeWatch(ingresses, &cluster1Client.Fake)
+	cluster1ConfigMapWatch := RegisterFakeWatch(configmaps, &cluster1Client.Fake)
+	cluster1IngressCreateChan := RegisterFakeCopyOnCreate(ingresses, &cluster1Client.Fake, cluster1IngressWatch)
+	cluster1IngressUpdateChan := RegisterFakeCopyOnUpdate(ingresses, &cluster1Client.Fake, cluster1IngressWatch)
+	cluster1ConfigMapUpdateChan := RegisterFakeCopyOnUpdate(configmaps, &cluster1Client.Fake, cluster1ConfigMapWatch)
+
+	cluster2Client := &fakekubeclientset.Clientset{}
+	RegisterFakeList(ingresses, &cluster2Client.Fake, &extensionsv1beta1.IngressList{Items: []extensionsv1beta1.Ingress{}})
+	RegisterFakeList(configmaps, &cluster2Client.Fake, &fakeConfigMapList2)
+	cluster2IngressWatch := RegisterFakeWatch(ingresses, &cluster2Client.Fake)
+	cluster2ConfigMapWatch := RegisterFakeWatch(configmaps, &cluster2Client.Fake)
+	cluster2IngressCreateChan := RegisterFakeCopyOnCreate(ingresses, &cluster2Client.Fake, cluster2IngressWatch)
+	cluster2ConfigMapUpdateChan := RegisterFakeCopyOnUpdate(configmaps, &cluster2Client.Fake, cluster2ConfigMapWatch)
+
+	clientFactoryFunc := func(cluster *federationapi.Cluster) (kubeclientset.Interface, error) {
+		switch cluster.Name {
+		case cluster1.Name:
+			return cluster1Client, nil
+		case cluster2.Name:
+			return cluster2Client, nil
+		default:
+			return nil, fmt.Errorf("Unknown cluster")
+		}
+	}
+	ingressController := NewIngressController(fedClient)
+	ingressInformer := ToFederatedInformerForTestOnly(ingressController.ingressFederatedInformer)
+	ingressInformer.SetClientFactory(clientFactoryFunc)
+	configMapInformer := ToFederatedInformerForTestOnly(ingressController.configMapFederatedInformer)
+	configMapInformer.SetClientFactory(clientFactoryFunc)
+	ingressController.clusterAvailableDelay = time.Second
+	ingressController.ingressReviewDelay = 100 * time.Millisecond
+	ingressController.configMapReviewDelay = 100 * time.Millisecond
+	ingressController.smallDelay = 100 * time.Millisecond
+	ingressController.updateTimeout = 5 * time.Second
+
+	stop := make(chan struct{})
+	t.Log("Running Ingress Controller")
+	ingressController.Run(stop)
+
+	// TODO: Here we are creating the ingress with first cluster annotation.
+	// Add another test without that annotation when
+	// https://github.com/kubernetes/kubernetes/issues/36540 is fixed.
+	fedIngress := extensionsv1beta1.Ingress{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-ingress",
+			Namespace: "mynamespace",
+			SelfLink:  "/api/v1/namespaces/mynamespace/ingress/test-ingress",
+			Annotations: map[string]string{
+				firstClusterAnnotation: cluster1.Name,
+			},
+		},
+		Status: extensionsv1beta1.IngressStatus{
+			LoadBalancer: apiv1.LoadBalancerStatus{
+				Ingress: make([]apiv1.LoadBalancerIngress, 0, 0),
+			},
+		},
+	}
+
+	t.Log("Adding cluster 1")
+	clusterWatch.Add(cluster1)
+
+	t.Log("Adding Ingress UID ConfigMap to cluster 1")
+	cluster1ConfigMapWatch.Add(cfg1)
+
+	t.Log("Checking that UID annotation on Cluster 1 annotation was correctly updated prior to adding Federated Ingress")
+	cluster := GetClusterFromChan(fedClusterUpdateChan)
+	assert.NotNil(t, cluster)
+	assert.Equal(t, cluster.ObjectMeta.Annotations[uidAnnotationKey], cfg1.Data[uidKey])
+
+	t.Log("Adding cluster 2")
+	clusterWatch.Add(cluster2)
+	cluster2ConfigMapWatch.Add(cfg2)
+
+	t.Log("Checking that a configmap updates are propagated prior to Federated Ingress")
+	referenceUid := cfg1.Data[uidKey]
+	uid1, providerId1 := GetConfigMapUidAndProviderId(t, cluster1ConfigMapUpdateChan)
+	uid2, providerId2 := GetConfigMapUidAndProviderId(t, cluster2ConfigMapUpdateChan)
+	t.Logf("uid2 = %v and ref = %v", uid2, referenceUid)
+	assert.True(t, referenceUid == uid1, "Expected cluster1 configmap uid %q to be equal to referenceUid %q", uid1, referenceUid)
+	assert.True(t, referenceUid == uid2, "Expected cluster2 configmap uid %q to be equal to referenceUid %q", uid2, referenceUid)
+	assert.True(t, providerId1 != providerId2, "Expected cluster1 providerUid %q to be unique and different from cluster2 providerUid %q", providerId1, providerId2)
+
+	// Test add federated ingress.
+	t.Log("Adding Federated Ingress")
+	fedIngressWatch.Add(&fedIngress)
+
+	t.Log("Checking that appropriate finalizers are added")
+	// There should be an update to add both the finalizers.
+	updatedIngress := GetIngressFromChan(t, fedIngressUpdateChan)
+	AssertHasFinalizer(t, updatedIngress, deletionhelper.FinalizerDeleteFromUnderlyingClusters)
+	AssertHasFinalizer(t, updatedIngress, metav1.FinalizerOrphanDependents)
+	fedIngress = *updatedIngress
+
+	t.Log("Checking that Ingress was correctly created in cluster 1")
+	createdIngress := GetIngressFromChan(t, cluster1IngressCreateChan)
+	assert.NotNil(t, createdIngress)
+	cluster1Ingress := *createdIngress
+	assert.True(t, reflect.DeepEqual(fedIngress.Spec, cluster1Ingress.Spec), "Spec of created ingress is not equal")
+	assert.True(t, util.ObjectMetaEquivalent(fedIngress.ObjectMeta, cluster1Ingress.ObjectMeta),
+		"Metadata of created object is not equivalent")
+
+	// Wait for finalizers to appear in federation store.
+	assert.NoError(t, WaitForFinalizersInFederationStore(ingressController, ingressController.ingressInformerStore,
+		types.NamespacedName{Namespace: fedIngress.Namespace, Name: fedIngress.Name}.String()), "finalizers not found in federated ingress")
+
+	// Wait for the cluster ingress to appear in cluster store.
+	assert.NoError(t, WaitForIngressInClusterStore(ingressController.ingressFederatedInformer.GetTargetStore(), cluster1.Name,
+		types.NamespacedName{Namespace: createdIngress.Namespace, Name: createdIngress.Name}.String()),
+		"Created ingress not found in underlying cluster store")
+
+	// Test that IP address gets transferred from cluster ingress to federated ingress.
+	t.Log("Checking that IP address gets transferred from cluster ingress to federated ingress")
+	cluster1Ingress.Status.LoadBalancer.Ingress = append(cluster1Ingress.Status.LoadBalancer.Ingress,
+		apiv1.LoadBalancerIngress{IP: "1.2.3.4"})
+	glog.Infof("Setting artificial IP address for cluster1 ingress")
+
+	for trial := 0; trial < maxTrials; trial++ {
+		cluster1IngressWatch.Modify(&cluster1Ingress)
+		// Wait for store to see the updated cluster ingress.
+		key := types.NamespacedName{Namespace: createdIngress.Namespace, Name: createdIngress.Name}.String()
+		if err := WaitForStatusUpdate(t, ingressController.ingressFederatedInformer.GetTargetStore(),
+			cluster1.Name, key, cluster1Ingress.Status.LoadBalancer, time.Second); err != nil {
+			continue
+		}
+		if err := WaitForFedStatusUpdate(t, ingressController.ingressInformerStore,
+			key, cluster1Ingress.Status.LoadBalancer, time.Second); err != nil {
+			continue
+		}
+	}
+
+	for trial := 0; trial < maxTrials; trial++ {
+		updatedIngress = GetIngressFromChan(t, fedIngressUpdateChan)
+		assert.NotNil(t, updatedIngress, "Cluster's ingress load balancer status was not correctly transferred to the federated ingress")
+		if updatedIngress == nil {
+			return
+		}
+		if reflect.DeepEqual(cluster1Ingress.Status.LoadBalancer.Ingress, updatedIngress.Status.LoadBalancer.Ingress) {
+			fedIngress.Status.LoadBalancer = updatedIngress.Status.LoadBalancer
+			break
+		} else {
+			glog.Infof("Status check failed: expected: %v actual: %v", cluster1Ingress.Status, updatedIngress.Status)
+		}
+	}
+	glog.Infof("Status check: expected: %v actual: %v", cluster1Ingress.Status, updatedIngress.Status)
+	assert.True(t, reflect.DeepEqual(cluster1Ingress.Status.LoadBalancer.Ingress, updatedIngress.Status.LoadBalancer.Ingress),
+		fmt.Sprintf("Ingress IP was not transferred from cluster ingress to federated ingress.  %v is not equal to %v",
+			cluster1Ingress.Status.LoadBalancer.Ingress, updatedIngress.Status.LoadBalancer.Ingress))
+
+	assert.NoError(t, WaitForStatusUpdate(t, ingressController.ingressFederatedInformer.GetTargetStore(),
+		cluster1.Name, types.NamespacedName{Namespace: createdIngress.Namespace, Name: createdIngress.Name}.String(),
+		cluster1Ingress.Status.LoadBalancer, time.Second))
+	assert.NoError(t, WaitForFedStatusUpdate(t, ingressController.ingressInformerStore,
+		types.NamespacedName{Namespace: createdIngress.Namespace, Name: createdIngress.Name}.String(),
+		cluster1Ingress.Status.LoadBalancer, time.Second))
+	t.Logf("expected: %v, actual: %v", createdIngress, updatedIngress)
+
+	// Test update federated ingress.
+	if fedIngress.ObjectMeta.Annotations == nil {
+		fedIngress.ObjectMeta.Annotations = make(map[string]string)
+	}
+	fedIngress.ObjectMeta.Annotations["A"] = "B"
+	fedIngress.ObjectMeta.Annotations[federationapi.FederationClusterSelectorAnnotation] = `[{"key": "cluster", "operator": "in", "values": ["cluster1","cluster2"]}]`
+	t.Log("Modifying Federated Ingress")
+	fedIngressWatch.Modify(&fedIngress)
+	t.Log("Checking that Ingress was correctly updated in cluster 1")
+	var updatedIngress2 *extensionsv1beta1.Ingress
+
+	for trial := 0; trial < maxTrials; trial++ {
+		updatedIngress2 = GetIngressFromChan(t, cluster1IngressUpdateChan)
+		assert.NotNil(t, updatedIngress2)
+		if updatedIngress2 == nil {
+			return
+		}
+		if reflect.DeepEqual(fedIngress.Spec, updatedIngress.Spec) &&
+			updatedIngress2.ObjectMeta.Annotations["A"] == fedIngress.ObjectMeta.Annotations["A"] {
+			break
+		}
+	}
+
+	assert.True(t, reflect.DeepEqual(updatedIngress2.Spec, fedIngress.Spec), "Spec of updated ingress is not equal")
+	assert.Equal(t, updatedIngress2.ObjectMeta.Annotations["A"], fedIngress.ObjectMeta.Annotations["A"], "Updated annotation not transferred from federated to cluster ingress.")
+
+	fedIngress.Annotations[staticIPNameKeyWritable] = "foo" // Make sure that the base object has a static IP name first.
+	fedIngressWatch.Modify(&fedIngress)
+
+	t.Log("Checking that the ingress got created in cluster 2 after a global ip was assigned")
+	createdIngress2 := GetIngressFromChan(t, cluster2IngressCreateChan)
+	assert.NotNil(t, createdIngress2)
+	assert.True(t, reflect.DeepEqual(fedIngress.Spec, createdIngress2.Spec), "Spec of created ingress is not equal")
+	t.Logf("created meta: %v fed meta: %v", createdIngress2.ObjectMeta, fedIngress.ObjectMeta)
+	assert.True(t, util.ObjectMetaEquivalent(fedIngress.ObjectMeta, createdIngress2.ObjectMeta), "Metadata of created object is not equivalent")
+
+	close(stop)
+}
+
+func GetConfigMapUidAndProviderId(t *testing.T, c chan runtime.Object) (string, string) {
+	updatedConfigMap := GetConfigMapFromChan(c)
+	assert.NotNil(t, updatedConfigMap, "ConfigMap should have received an update")
+	assert.NotNil(t, updatedConfigMap.Data, "ConfigMap data is empty")
+
+	for _, key := range []string{uidKey, providerUidKey} {
+		val, ok := updatedConfigMap.Data[key]
+		assert.True(t, ok, fmt.Sprintf("Didn't receive an update for key %v: %v", key, updatedConfigMap.Data))
+		assert.True(t, len(val) > 0, fmt.Sprintf("Received an empty update for key %v", key))
+	}
+	return updatedConfigMap.Data[uidKey], updatedConfigMap.Data[providerUidKey]
+}
+
+func GetIngressFromChan(t *testing.T, c chan runtime.Object) *extensionsv1beta1.Ingress {
+	obj := GetObjectFromChan(c)
+
+	if obj == nil {
+		return nil
+	}
+
+	ingress, ok := obj.(*extensionsv1beta1.Ingress)
+	if !ok {
+		t.Logf("Object on channel was not of type *extensionsv1beta1.Ingress: %v", obj)
+	}
+	return ingress
+}
+
+func GetConfigMapFromChan(c chan runtime.Object) *apiv1.ConfigMap {
+	if configMap := GetObjectFromChan(c); configMap == nil {
+		return nil
+	} else {
+		return configMap.(*apiv1.ConfigMap)
+	}
+
+}
+
+func GetClusterFromChan(c chan runtime.Object) *federationapi.Cluster {
+	if cluster := GetObjectFromChan(c); cluster == nil {
+		return nil
+	} else {
+		return cluster.(*federationapi.Cluster)
+	}
+}
+
+func NewConfigMap(uid string) *apiv1.ConfigMap {
+	return &apiv1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      uidConfigMapName,
+			Namespace: uidConfigMapNamespace,
+			SelfLink:  "/api/v1/namespaces/" + uidConfigMapNamespace + "/configmap/" + uidConfigMapName,
+			// TODO: Remove: Annotations: map[string]string{},
+		},
+		Data: map[string]string{
+			uidKey: uid,
+		},
+	}
+}
+
+// Wait for finalizers to appear in federation store.
+func WaitForFinalizersInFederationStore(ingressController *IngressController, store cache.Store, key string) error {
+	retryInterval := 100 * time.Millisecond
+	timeout := wait.ForeverTestTimeout
+	err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) {
+		obj, found, err := store.GetByKey(key)
+		if !found || err != nil {
+			return false, err
+		}
+		ingress := obj.(*extensionsv1beta1.Ingress)
+		hasOrphanFinalizer, err := finalizersutil.HasFinalizer(ingress, metav1.FinalizerOrphanDependents)
+		if err != nil {
+			return false, err
+		}
+		hasDeleteFinalizer, err := finalizersutil.HasFinalizer(ingress, deletionhelper.FinalizerDeleteFromUnderlyingClusters)
+		if err != nil {
+			return false, err
+		}
+		if hasOrphanFinalizer && hasDeleteFinalizer {
+			return true, nil
+		}
+		return false, nil
+	})
+	return err
+}
+
+// Wait for the cluster ingress to appear in cluster store.
+func WaitForIngressInClusterStore(store util.FederatedReadOnlyStore, clusterName, key string) error {
+	retryInterval := 100 * time.Millisecond
+	timeout := wait.ForeverTestTimeout
+	err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) {
+		_, found, err := store.GetByKey(clusterName, key)
+		if found && err == nil {
+			return true, nil
+		}
+		if errors.IsNotFound(err) {
+			return false, nil
+		}
+		return false, err
+	})
+	return err
+}
+
+// Wait for ingress status to be updated to match the desiredStatus.
+func WaitForStatusUpdate(t *testing.T, store util.FederatedReadOnlyStore, clusterName, key string, desiredStatus apiv1.LoadBalancerStatus, timeout time.Duration) error {
+	retryInterval := 100 * time.Millisecond
+	err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) {
+		obj, found, err := store.GetByKey(clusterName, key)
+		if !found || err != nil {
+			return false, err
+		}
+		ingress := obj.(*extensionsv1beta1.Ingress)
+		return reflect.DeepEqual(ingress.Status.LoadBalancer, desiredStatus), nil
+	})
+	return err
+}
+
+// Wait for ingress status to be updated to match the desiredStatus.
+func WaitForFedStatusUpdate(t *testing.T, store cache.Store, key string, desiredStatus apiv1.LoadBalancerStatus, timeout time.Duration) error {
+	retryInterval := 100 * time.Millisecond
+	err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) {
+		obj, found, err := store.GetByKey(key)
+		if !found || err != nil {
+			return false, err
+		}
+		ingress := obj.(*extensionsv1beta1.Ingress)
+		return reflect.DeepEqual(ingress.Status.LoadBalancer, desiredStatus), nil
+	})
+	return err
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/job/BUILD

@@ -0,0 +1,76 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["jobcontroller.go"],
+    deps = [
+        "//federation/apis/federation:go_default_library",
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//federation/pkg/federation-controller/util/deletionhelper:go_default_library",
+        "//federation/pkg/federation-controller/util/eventsink:go_default_library",
+        "//federation/pkg/federation-controller/util/planner:go_default_library",
+        "//federation/pkg/federation-controller/util/replicapreferences:go_default_library",
+        "//pkg/api:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//vendor/github.com/davecgh/go-spew/spew:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/batch/v1:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+        "//vendor/k8s.io/client-go/tools/record:go_default_library",
+        "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
+        "//vendor/k8s.io/client-go/util/workqueue:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["jobcontroller_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset/fake:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//federation/pkg/federation-controller/util/finalizers:go_default_library",
+        "//federation/pkg/federation-controller/util/test:go_default_library",
+        "//pkg/apis/batch/v1:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/k8s.io/api/batch/v1:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
+        "//vendor/k8s.io/client-go/testing:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/job/jobcontroller.go

@@ -0,0 +1,561 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package job
+
+import (
+	"fmt"
+	"reflect"
+	"time"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/golang/glog"
+
+	batchv1 "k8s.io/api/batch/v1"
+	clientv1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/flowcontrol"
+	"k8s.io/client-go/util/workqueue"
+	fed "k8s.io/kubernetes/federation/apis/federation"
+	fedv1 "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	fedutil "k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/deletionhelper"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/eventsink"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/planner"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/replicapreferences"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/controller"
+)
+
+const (
+	fedJobPreferencesAnnotation = "federation.kubernetes.io/job-preferences"
+	allClustersKey              = "THE_ALL_CLUSTER_KEY"
+	// UserAgentName is the user agent used in the federation client
+	UserAgentName = "Federation-Job-Controller"
+	// ControllerName is name of this controller
+	ControllerName = "jobs"
+)
+
+var (
+	// RequiredResources is the resource group version of the type this controller manages
+	RequiredResources       = []schema.GroupVersionResource{batchv1.SchemeGroupVersion.WithResource("jobs")}
+	jobReviewDelay          = 10 * time.Second
+	clusterAvailableDelay   = 20 * time.Second
+	clusterUnavailableDelay = 60 * time.Second
+	updateTimeout           = 30 * time.Second
+	backoffInitial          = 5 * time.Second
+	backoffMax              = 1 * time.Minute
+)
+
+// FederationJobController synchronizes the state of a federated job object
+// to clusters that are members of the federation.
+type FederationJobController struct {
+	fedClient fedclientset.Interface
+
+	jobController cache.Controller
+	jobStore      cache.Store
+
+	fedJobInformer fedutil.FederatedInformer
+
+	jobDeliverer     *fedutil.DelayingDeliverer
+	clusterDeliverer *fedutil.DelayingDeliverer
+	jobWorkQueue     workqueue.Interface
+	// For updating members of federation.
+	fedUpdater fedutil.FederatedUpdater
+
+	jobBackoff *flowcontrol.Backoff
+	// For events
+	eventRecorder record.EventRecorder
+
+	defaultPlanner *planner.Planner
+	deletionHelper *deletionhelper.DeletionHelper
+}
+
+// NewJobController creates a new federation job controller
+func NewJobController(fedClient fedclientset.Interface) *FederationJobController {
+	broadcaster := record.NewBroadcaster()
+	broadcaster.StartRecordingToSink(eventsink.NewFederatedEventSink(fedClient))
+	recorder := broadcaster.NewRecorder(api.Scheme, clientv1.EventSource{Component: "federated-job-controller"})
+	fjc := &FederationJobController{
+		fedClient:        fedClient,
+		jobDeliverer:     fedutil.NewDelayingDeliverer(),
+		clusterDeliverer: fedutil.NewDelayingDeliverer(),
+		jobWorkQueue:     workqueue.New(),
+		jobBackoff:       flowcontrol.NewBackOff(backoffInitial, backoffMax),
+		defaultPlanner: planner.NewPlanner(&fed.ReplicaAllocationPreferences{
+			Clusters: map[string]fed.ClusterPreferences{
+				"*": {Weight: 1},
+			},
+		}),
+		eventRecorder: recorder,
+	}
+
+	jobFedInformerFactory := func(cluster *fedv1.Cluster, clientset kubeclientset.Interface) (cache.Store, cache.Controller) {
+		return cache.NewInformer(
+			&cache.ListWatch{
+				ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+					return clientset.BatchV1().Jobs(metav1.NamespaceAll).List(options)
+				},
+				WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+					return clientset.BatchV1().Jobs(metav1.NamespaceAll).Watch(options)
+				},
+			},
+			&batchv1.Job{},
+			controller.NoResyncPeriodFunc(),
+			fedutil.NewTriggerOnAllChanges(
+				func(obj runtime.Object) { fjc.deliverLocalJob(obj, jobReviewDelay) },
+			),
+		)
+	}
+	clusterLifecycle := fedutil.ClusterLifecycleHandlerFuncs{
+		ClusterAvailable: func(cluster *fedv1.Cluster) {
+			fjc.clusterDeliverer.DeliverAfter(allClustersKey, nil, clusterAvailableDelay)
+		},
+		ClusterUnavailable: func(cluster *fedv1.Cluster, _ []interface{}) {
+			fjc.clusterDeliverer.DeliverAfter(allClustersKey, nil, clusterUnavailableDelay)
+		},
+	}
+	fjc.fedJobInformer = fedutil.NewFederatedInformer(fedClient, jobFedInformerFactory, &clusterLifecycle)
+
+	fjc.jobStore, fjc.jobController = cache.NewInformer(
+		&cache.ListWatch{
+			ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+				return fjc.fedClient.BatchV1().Jobs(metav1.NamespaceAll).List(options)
+			},
+			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+				return fjc.fedClient.BatchV1().Jobs(metav1.NamespaceAll).Watch(options)
+			},
+		},
+		&batchv1.Job{},
+		controller.NoResyncPeriodFunc(),
+		fedutil.NewTriggerOnMetaAndSpecChanges(
+			func(obj runtime.Object) { fjc.deliverFedJobObj(obj, 0) },
+		),
+	)
+
+	fjc.fedUpdater = fedutil.NewFederatedUpdater(fjc.fedJobInformer, "job", updateTimeout, fjc.eventRecorder,
+		func(client kubeclientset.Interface, obj runtime.Object) error {
+			rs := obj.(*batchv1.Job)
+			_, err := client.BatchV1().Jobs(rs.Namespace).Create(rs)
+			return err
+		},
+		func(client kubeclientset.Interface, obj runtime.Object) error {
+			rs := obj.(*batchv1.Job)
+			_, err := client.BatchV1().Jobs(rs.Namespace).Update(rs)
+			return err
+		},
+		func(client kubeclientset.Interface, obj runtime.Object) error {
+			rs := obj.(*batchv1.Job)
+			err := client.BatchV1().Jobs(rs.Namespace).Delete(rs.Name, &metav1.DeleteOptions{})
+			return err
+		})
+
+	fjc.deletionHelper = deletionhelper.NewDeletionHelper(
+		fjc.updateJob,
+		// objNameFunc
+		func(obj runtime.Object) string {
+			job := obj.(*batchv1.Job)
+			return job.Name
+		},
+		fjc.fedJobInformer,
+		fjc.fedUpdater,
+	)
+
+	return fjc
+}
+
+// Sends the given updated object to apiserver.
+// Assumes that the given object is a job.
+func (fjc *FederationJobController) updateJob(obj runtime.Object) (runtime.Object, error) {
+	job := obj.(*batchv1.Job)
+	return fjc.fedClient.BatchV1().Jobs(job.Namespace).Update(job)
+}
+
+// Run starts the syncing of federation jobs to the clusters.
+func (fjc *FederationJobController) Run(workers int, stopCh <-chan struct{}) {
+	go fjc.jobController.Run(stopCh)
+	fjc.fedJobInformer.Start()
+
+	fjc.jobDeliverer.StartWithHandler(func(item *fedutil.DelayingDelivererItem) {
+		fjc.jobWorkQueue.Add(item.Key)
+	})
+	fjc.clusterDeliverer.StartWithHandler(func(_ *fedutil.DelayingDelivererItem) {
+		fjc.reconcileJobsOnClusterChange()
+	})
+
+	for !fjc.isSynced() {
+		time.Sleep(5 * time.Millisecond)
+	}
+
+	for i := 0; i < workers; i++ {
+		go wait.Until(fjc.worker, time.Second, stopCh)
+	}
+
+	fedutil.StartBackoffGC(fjc.jobBackoff, stopCh)
+
+	<-stopCh
+	glog.Infof("Shutting down FederationJobController")
+	fjc.jobDeliverer.Stop()
+	fjc.clusterDeliverer.Stop()
+	fjc.jobWorkQueue.ShutDown()
+	fjc.fedJobInformer.Stop()
+}
+
+func (fjc *FederationJobController) isSynced() bool {
+	if !fjc.fedJobInformer.ClustersSynced() {
+		glog.V(3).Infof("Cluster list not synced")
+		return false
+	}
+	clusters, err := fjc.fedJobInformer.GetReadyClusters()
+	if err != nil {
+		glog.Errorf("Failed to get ready clusters: %v", err)
+		return false
+	}
+	if !fjc.fedJobInformer.GetTargetStore().ClustersSynced(clusters) {
+		glog.V(2).Infof("cluster job list not synced")
+		return false
+	}
+
+	if !fjc.jobController.HasSynced() {
+		glog.V(2).Infof("federation job list not synced")
+		return false
+	}
+	return true
+}
+
+func (fjc *FederationJobController) deliverLocalJob(obj interface{}, duration time.Duration) {
+	key, err := controller.KeyFunc(obj)
+	if err != nil {
+		glog.Errorf("Couldn't get key for object %v: %v", obj, err)
+		return
+	}
+	_, exists, err := fjc.jobStore.GetByKey(key)
+	if err != nil {
+		glog.Errorf("Couldn't get federated job %v: %v", key, err)
+		return
+	}
+	if exists { // ignore jobs exists only in local k8s
+		fjc.deliverJobByKey(key, duration, false)
+	}
+}
+
+func (fjc *FederationJobController) deliverFedJobObj(obj interface{}, delay time.Duration) {
+	key, err := controller.KeyFunc(obj)
+	if err != nil {
+		glog.Errorf("Couldn't get key for object %+v: %v", obj, err)
+		return
+	}
+	fjc.deliverJobByKey(key, delay, false)
+}
+
+func (fjc *FederationJobController) deliverJobByKey(key string, delay time.Duration, failed bool) {
+	if failed {
+		fjc.jobBackoff.Next(key, time.Now())
+		delay = delay + fjc.jobBackoff.Get(key)
+	} else {
+		fjc.jobBackoff.Reset(key)
+	}
+	fjc.jobDeliverer.DeliverAfter(key, nil, delay)
+}
+
+type reconciliationStatus string
+
+const (
+	statusAllOk       = reconciliationStatus("ALL_OK")
+	statusNeedRecheck = reconciliationStatus("RECHECK")
+	statusError       = reconciliationStatus("ERROR")
+	statusNotSynced   = reconciliationStatus("NOSYNC")
+)
+
+func (fjc *FederationJobController) worker() {
+	for {
+		item, quit := fjc.jobWorkQueue.Get()
+		if quit {
+			return
+		}
+		key := item.(string)
+		status, err := fjc.reconcileJob(key)
+		fjc.jobWorkQueue.Done(item)
+		if err != nil {
+			glog.Errorf("Error syncing job controller: %v", err)
+			fjc.deliverJobByKey(key, 0, true)
+		} else {
+			switch status {
+			case statusAllOk:
+				break
+			case statusError:
+				fjc.deliverJobByKey(key, 0, true)
+			case statusNeedRecheck:
+				fjc.deliverJobByKey(key, jobReviewDelay, false)
+			case statusNotSynced:
+				fjc.deliverJobByKey(key, clusterAvailableDelay, false)
+			default:
+				glog.Errorf("Unhandled reconciliation status: %s", status)
+				fjc.deliverJobByKey(key, jobReviewDelay, false)
+			}
+		}
+	}
+}
+
+type scheduleResult struct {
+	Parallelism *int32
+	Completions *int32
+}
+
+func (fjc *FederationJobController) schedule(fjob *batchv1.Job, clusters []*fedv1.Cluster) map[string]scheduleResult {
+	plnr := fjc.defaultPlanner
+	frsPref, err := replicapreferences.GetAllocationPreferences(fjob, fedJobPreferencesAnnotation)
+	if err != nil {
+		glog.Warningf("Invalid job specific preference, use default. rs: %v, err: %v", fjob, err)
+	}
+	if frsPref != nil { // create a new planner if user specified a preference
+		plnr = planner.NewPlanner(frsPref)
+	}
+
+	parallelism := int64(*fjob.Spec.Parallelism)
+	var clusterNames []string
+	for _, cluster := range clusters {
+		clusterNames = append(clusterNames, cluster.Name)
+	}
+	parallelismResult, _ := plnr.Plan(parallelism, clusterNames, nil, nil, fjob.Namespace+"/"+fjob.Name)
+
+	if frsPref != nil {
+		for _, clusterPref := range frsPref.Clusters {
+			clusterPref.MinReplicas = 0
+			clusterPref.MaxReplicas = nil
+		}
+		plnr = planner.NewPlanner(frsPref)
+	}
+	clusterNames = nil
+	for clusterName := range parallelismResult {
+		clusterNames = append(clusterNames, clusterName)
+	}
+	completionsResult := make(map[string]int64)
+	if fjob.Spec.Completions != nil {
+		completionsResult, _ = plnr.Plan(int64(*fjob.Spec.Completions), clusterNames, nil, nil, fjob.Namespace+"/"+fjob.Name)
+	}
+
+	results := make(map[string]scheduleResult)
+	for _, clusterName := range clusterNames {
+		paralle := int32(parallelismResult[clusterName])
+		complet := int32(completionsResult[clusterName])
+		result := scheduleResult{
+			Parallelism: &paralle,
+		}
+		if fjob.Spec.Completions != nil {
+			result.Completions = &complet
+		}
+		results[clusterName] = result
+	}
+
+	return results
+}
+
+func (fjc *FederationJobController) reconcileJob(key string) (reconciliationStatus, error) {
+	if !fjc.isSynced() {
+		return statusNotSynced, nil
+	}
+
+	glog.V(4).Infof("Start reconcile job %q", key)
+	startTime := time.Now()
+	defer glog.V(4).Infof("Finished reconcile job %q (%v)", key, time.Now().Sub(startTime))
+
+	objFromStore, exists, err := fjc.jobStore.GetByKey(key)
+	if err != nil {
+		return statusError, err
+	}
+	if !exists {
+		// deleted federated job, nothing need to do
+		return statusAllOk, nil
+	}
+
+	// Create a copy before modifying the obj to prevent race condition with other readers of obj from store.
+	obj, err := api.Scheme.DeepCopy(objFromStore)
+	fjob, ok := obj.(*batchv1.Job)
+	if err != nil || !ok {
+		return statusError, err
+	}
+
+	// delete job
+	if fjob.DeletionTimestamp != nil {
+		if err := fjc.delete(fjob); err != nil {
+			fjc.eventRecorder.Eventf(fjob, api.EventTypeNormal, "DeleteFailed", "Job delete failed: %v", err)
+			return statusError, err
+		}
+		return statusAllOk, nil
+	}
+
+	glog.V(3).Infof("Ensuring delete object from underlying clusters finalizer for job: %s\n", key)
+	// Add the required finalizers before creating a job in underlying clusters.
+	updatedJobObj, err := fjc.deletionHelper.EnsureFinalizers(fjob)
+	if err != nil {
+		return statusError, err
+	}
+	fjob = updatedJobObj.(*batchv1.Job)
+
+	clusters, err := fjc.fedJobInformer.GetReadyClusters()
+	if err != nil {
+		return statusError, err
+	}
+
+	scheduleResult := fjc.schedule(fjob, clusters)
+	glog.V(3).Infof("Start syncing local job %s: %s\n", key, spew.Sprintf("%v", scheduleResult))
+
+	fedStatus := batchv1.JobStatus{}
+	var fedStatusFailedCondition *batchv1.JobCondition
+	var fedStatusCompleteCondition *batchv1.JobCondition
+	var operations []fedutil.FederatedOperation
+	for clusterName, result := range scheduleResult {
+		ljobObj, exists, err := fjc.fedJobInformer.GetTargetStore().GetByKey(clusterName, key)
+		if err != nil {
+			return statusError, err
+		}
+		ljob := &batchv1.Job{
+			ObjectMeta: fedutil.DeepCopyRelevantObjectMeta(fjob.ObjectMeta),
+			Spec:       *fedutil.DeepCopyApiTypeOrPanic(&fjob.Spec).(*batchv1.JobSpec),
+		}
+		// use selector generated at federation level, or user specified value
+		manualSelector := true
+		ljob.Spec.ManualSelector = &manualSelector
+		ljob.Spec.Parallelism = result.Parallelism
+		ljob.Spec.Completions = result.Completions
+
+		if !exists {
+			if *ljob.Spec.Parallelism > 0 {
+				fjc.eventRecorder.Eventf(fjob, api.EventTypeNormal, "CreateInCluster", "Creating job in cluster %s", clusterName)
+				operations = append(operations, fedutil.FederatedOperation{
+					Type:        fedutil.OperationTypeAdd,
+					Obj:         ljob,
+					ClusterName: clusterName,
+				})
+			}
+		} else {
+			currentLjob := ljobObj.(*batchv1.Job)
+
+			// Update existing job, if needed.
+			if !fedutil.ObjectMetaAndSpecEquivalent(ljob, currentLjob) {
+				fjc.eventRecorder.Eventf(fjob, api.EventTypeNormal, "UpdateInCluster", "Updating job in cluster %s", clusterName)
+				operations = append(operations, fedutil.FederatedOperation{
+					Type:        fedutil.OperationTypeUpdate,
+					Obj:         ljob,
+					ClusterName: clusterName,
+				})
+			}
+
+			// collect local job status
+			for _, condition := range currentLjob.Status.Conditions {
+				if condition.Type == batchv1.JobComplete {
+					if fedStatusCompleteCondition == nil ||
+						fedStatusCompleteCondition.LastTransitionTime.Before(&condition.LastTransitionTime) {
+						fedStatusCompleteCondition = &condition
+					}
+				} else if condition.Type == batchv1.JobFailed {
+					if fedStatusFailedCondition == nil ||
+						fedStatusFailedCondition.LastTransitionTime.Before(&condition.LastTransitionTime) {
+						fedStatusFailedCondition = &condition
+					}
+				}
+			}
+			if currentLjob.Status.StartTime != nil {
+				if fedStatus.StartTime == nil || fedStatus.StartTime.After(currentLjob.Status.StartTime.Time) {
+					fedStatus.StartTime = currentLjob.Status.StartTime
+				}
+			}
+			if currentLjob.Status.CompletionTime != nil {
+				if fedStatus.CompletionTime == nil || fedStatus.CompletionTime.Before(currentLjob.Status.CompletionTime) {
+					fedStatus.CompletionTime = currentLjob.Status.CompletionTime
+				}
+			}
+			fedStatus.Active += currentLjob.Status.Active
+			fedStatus.Succeeded += currentLjob.Status.Succeeded
+			fedStatus.Failed += currentLjob.Status.Failed
+		}
+	}
+
+	// federated job fails if any local job failes
+	if fedStatusFailedCondition != nil {
+		fedStatus.Conditions = append(fedStatus.Conditions, *fedStatusFailedCondition)
+	} else if fedStatusCompleteCondition != nil {
+		fedStatus.Conditions = append(fedStatus.Conditions, *fedStatusCompleteCondition)
+	}
+	if !reflect.DeepEqual(fedStatus, fjob.Status) {
+		fjob.Status = fedStatus
+		_, err = fjc.fedClient.BatchV1().Jobs(fjob.Namespace).UpdateStatus(fjob)
+		if err != nil {
+			return statusError, err
+		}
+	}
+
+	if len(operations) == 0 {
+		// Everything is in order
+		return statusAllOk, nil
+	}
+
+	if glog.V(4) {
+		for i, op := range operations {
+			job := op.Obj.(*batchv1.Job)
+			glog.V(4).Infof("operation[%d]: %s, %s/%s/%s, %d", i, op.Type, op.ClusterName, job.Namespace, job.Name, *job.Spec.Parallelism)
+		}
+	}
+	err = fjc.fedUpdater.Update(operations)
+	if err != nil {
+		return statusError, err
+	}
+
+	// Some operations were made, reconcile after a while.
+	return statusNeedRecheck, nil
+
+}
+
+func (fjc *FederationJobController) reconcileJobsOnClusterChange() {
+	if !fjc.isSynced() {
+		fjc.clusterDeliverer.DeliverAfter(allClustersKey, nil, clusterAvailableDelay)
+	}
+	jobs := fjc.jobStore.List()
+	for _, job := range jobs {
+		key, _ := controller.KeyFunc(job)
+		fjc.deliverJobByKey(key, 0, false)
+	}
+}
+
+// delete deletes the given job or returns error if the deletion was not complete.
+func (fjc *FederationJobController) delete(job *batchv1.Job) error {
+	glog.V(3).Infof("Handling deletion of job: %s/%s\n", job.Namespace, job.Name)
+	_, err := fjc.deletionHelper.HandleObjectInUnderlyingClusters(job)
+	if err != nil {
+		return err
+	}
+
+	err = fjc.fedClient.BatchV1().Jobs(job.Namespace).Delete(job.Name, nil)
+	if err != nil {
+		// Its all good if the error is not found error. That means it is deleted already and we do not have to do anything.
+		// This is expected when we are processing an update as a result of job finalizer deletion.
+		// The process that deleted the last finalizer is also going to delete the job and we do not have to do anything.
+		if !errors.IsNotFound(err) {
+			return fmt.Errorf("failed to delete job: %s/%s, %v", job.Namespace, job.Name, err)
+		}
+	}
+	return nil
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/job/jobcontroller_test.go

@@ -0,0 +1,282 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package job
+
+import (
+	"flag"
+	"fmt"
+	"testing"
+	"time"
+
+	batchv1 "k8s.io/api/batch/v1"
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	kubeclientfake "k8s.io/client-go/kubernetes/fake"
+	core "k8s.io/client-go/testing"
+	fedv1 "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	fedclientfake "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset/fake"
+	fedutil "k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	finalizersutil "k8s.io/kubernetes/federation/pkg/federation-controller/util/finalizers"
+	testutil "k8s.io/kubernetes/federation/pkg/federation-controller/util/test"
+	batchv1internal "k8s.io/kubernetes/pkg/apis/batch/v1"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"reflect"
+	"strings"
+)
+
+func installWatchReactor(fakeClien *core.Fake, resource string) chan runtime.Object {
+	objChan := make(chan runtime.Object, 100)
+
+	fakeWatch := watch.NewRaceFreeFake()
+	fakeClien.PrependWatchReactor(resource, core.DefaultWatchReactor(fakeWatch, nil))
+	fakeClien.PrependReactor("create", resource, func(action core.Action) (handled bool, ret runtime.Object, err error) {
+		obj := action.(core.CreateAction).GetObject()
+		batchv1internal.SetDefaults_Job(obj.(*batchv1.Job))
+		fakeWatch.Add(obj)
+		objChan <- obj
+		return false, nil, nil
+	})
+	fakeClien.PrependReactor("update", resource, func(action core.Action) (handled bool, ret runtime.Object, err error) {
+		obj := action.(core.UpdateAction).GetObject()
+		fakeWatch.Modify(obj)
+		objChan <- obj
+		return false, nil, nil
+	})
+	fakeClien.PrependReactor("delete", resource, func(action core.Action) (handled bool, ret runtime.Object, err error) {
+		obj := &batchv1.Job{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      action.(core.DeleteAction).GetName(),
+				Namespace: action.GetNamespace(),
+			},
+		}
+		fakeWatch.Delete(obj)
+		objChan <- obj
+		return false, nil, nil
+	})
+
+	return objChan
+}
+
+func TestJobController(t *testing.T) {
+	flag.Set("logtostderr", "true")
+	flag.Set("v", "5")
+	flag.Parse()
+
+	jobReviewDelay = 50 * time.Millisecond
+	clusterAvailableDelay = 200 * time.Millisecond
+	clusterUnavailableDelay = 200 * time.Millisecond
+
+	fedclientset := fedclientfake.NewSimpleClientset()
+	fedChan := installWatchReactor(&fedclientset.Fake, "jobs")
+
+	fedclientset.Federation().Clusters().Create(testutil.NewCluster("k8s-1", apiv1.ConditionTrue))
+	fedclientset.Federation().Clusters().Create(testutil.NewCluster("k8s-2", apiv1.ConditionTrue))
+
+	kube1clientset := kubeclientfake.NewSimpleClientset()
+	kube1Chan := installWatchReactor(&kube1clientset.Fake, "jobs")
+	kube2clientset := kubeclientfake.NewSimpleClientset()
+	kube2Chan := installWatchReactor(&kube2clientset.Fake, "jobs")
+
+	fedInformerClientFactory := func(cluster *fedv1.Cluster) (kubeclientset.Interface, error) {
+		switch cluster.Name {
+		case "k8s-1":
+			return kube1clientset, nil
+		case "k8s-2":
+			return kube2clientset, nil
+		default:
+			return nil, fmt.Errorf("Unknown cluster: %v", cluster.Name)
+		}
+	}
+	jobController := NewJobController(fedclientset)
+	fedjobinformer := testutil.ToFederatedInformerForTestOnly(jobController.fedJobInformer)
+	fedjobinformer.SetClientFactory(fedInformerClientFactory)
+
+	stopChan := make(chan struct{})
+	defer close(stopChan)
+	go jobController.Run(5, stopChan)
+
+	test := func(job *batchv1.Job, parallelism1, parallelism2, completions1, completions2 int32) {
+		job, _ = fedclientset.Batch().Jobs(metav1.NamespaceDefault).Create(job)
+
+		joinErrors := func(errors []error) error {
+			if len(errors) == 0 {
+				return nil
+			}
+			errorStrings := []string{}
+			for _, err := range errors {
+				errorStrings = append(errorStrings, err.Error())
+			}
+			return fmt.Errorf("%s", strings.Join(errorStrings, "\n"))
+		}
+
+		// check local jobs are created with correct spec
+		checkLocalJob := func(parallelism, completions int32) testutil.CheckingFunction {
+			return func(obj runtime.Object) error {
+				errors := []error{}
+				ljob := obj.(*batchv1.Job)
+				if !fedutil.ObjectMetaEquivalent(job.ObjectMeta, ljob.ObjectMeta) {
+					errors = append(errors, fmt.Errorf("Job meta un-equivalent: %#v (expected) != %#v (actual)", job.ObjectMeta, ljob.ObjectMeta))
+				}
+				if err := checkEqual(t, *ljob.Spec.Parallelism, parallelism, "Spec.Parallelism"); err != nil {
+					errors = append(errors, err)
+				}
+				if ljob.Spec.Completions != nil {
+					if err := checkEqual(t, *ljob.Spec.Completions, completions, "Spec.Completions"); err != nil {
+						errors = append(errors, err)
+					}
+				}
+				return joinErrors(errors)
+			}
+		}
+		checkFedJob := func(obj runtime.Object) error {
+			errors := []error{}
+			return joinErrors(errors)
+		}
+		assert.NoError(t, testutil.CheckObjectFromChan(kube1Chan, checkLocalJob(parallelism1, completions1)))
+		assert.NoError(t, testutil.CheckObjectFromChan(kube2Chan, checkLocalJob(parallelism2, completions2)))
+		assert.NoError(t, testutil.CheckObjectFromChan(fedChan, checkFedJob))
+
+		// finish local jobs
+		job1, _ := kube1clientset.Batch().Jobs(metav1.NamespaceDefault).Get(job.Name, metav1.GetOptions{})
+		finishJob(job1, 100*time.Millisecond)
+		job1, _ = kube1clientset.Batch().Jobs(metav1.NamespaceDefault).UpdateStatus(job1)
+		job2, _ := kube2clientset.Batch().Jobs(metav1.NamespaceDefault).Get(job.Name, metav1.GetOptions{})
+		finishJob(job2, 100*time.Millisecond)
+		job2, _ = kube2clientset.Batch().Jobs(metav1.NamespaceDefault).UpdateStatus(job2)
+
+		// check fed job status updated
+		assert.NoError(t, testutil.CheckObjectFromChan(fedChan, func(obj runtime.Object) error {
+			errors := []error{}
+			job := obj.(*batchv1.Job)
+			if err := checkEqual(t, *job.Spec.Parallelism, *job1.Spec.Parallelism+*job2.Spec.Parallelism, "Spec.Parallelism"); err != nil {
+				errors = append(errors, err)
+			}
+			if job.Spec.Completions != nil {
+				if err := checkEqual(t, *job.Spec.Completions, *job1.Spec.Completions+*job2.Spec.Completions, "Spec.Completions"); err != nil {
+					errors = append(errors, err)
+				}
+			}
+			if err := checkEqual(t, job.Status.Succeeded, job1.Status.Succeeded+job2.Status.Succeeded, "Status.Succeeded"); err != nil {
+				errors = append(errors, err)
+			}
+			return joinErrors(errors)
+		}))
+
+		// delete fed job by set deletion time, and remove orphan finalizer
+		job, _ = fedclientset.Batch().Jobs(metav1.NamespaceDefault).Get(job.Name, metav1.GetOptions{})
+		deletionTimestamp := metav1.Now()
+		job.DeletionTimestamp = &deletionTimestamp
+		finalizersutil.RemoveFinalizers(job, sets.NewString(metav1.FinalizerOrphanDependents))
+		fedclientset.Batch().Jobs(metav1.NamespaceDefault).Update(job)
+
+		// check jobs are deleted
+		checkDeleted := func(obj runtime.Object) error {
+			djob := obj.(*batchv1.Job)
+			deletedJob := &batchv1.Job{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      djob.Name,
+					Namespace: djob.Namespace,
+				},
+			}
+			if !reflect.DeepEqual(djob, deletedJob) {
+				return fmt.Errorf("%s/%s should be deleted", djob.Namespace, djob.Name)
+			}
+			return nil
+		}
+		assert.NoError(t, testutil.CheckObjectFromChan(kube1Chan, checkDeleted))
+		assert.NoError(t, testutil.CheckObjectFromChan(kube2Chan, checkDeleted))
+		assert.NoError(t, testutil.CheckObjectFromChan(fedChan, checkDeleted))
+	}
+
+	test(newJob("job1", 2, 7), 1, 1, 4, 3)
+	test(newJob("job2", 2, -1), 1, 1, -1, -1)
+	test(newJob("job3", 7, 2), 4, 3, 1, 1)
+	test(newJob("job4", 7, 1), 4, 3, 1, 0)
+}
+
+func checkEqual(_ *testing.T, expected, actual interface{}, msg string) error {
+	if !assert.ObjectsAreEqual(expected, actual) {
+		return fmt.Errorf("%s not equal: %#v (expected) != %#v (actual)", msg, expected, actual)
+	}
+	return nil
+}
+
+func newJob(name string, parallelism int32, completions int32) *batchv1.Job {
+	job := batchv1.Job{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: metav1.NamespaceDefault,
+			SelfLink:  "/api/v1/namespaces/default/jobs/name",
+		},
+		Spec: batchv1.JobSpec{
+			Parallelism: &parallelism,
+			Completions: &completions,
+			Template: apiv1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"foo": name,
+					},
+				},
+				Spec: apiv1.PodSpec{
+					Containers: []apiv1.Container{
+						{Image: "foo/bar"},
+					},
+					RestartPolicy: apiv1.RestartPolicyNever,
+				},
+			},
+		},
+	}
+	if parallelism < 0 {
+		job.Spec.Parallelism = nil
+	}
+	if completions < 0 {
+		job.Spec.Completions = nil
+	}
+
+	batchv1internal.SetDefaults_Job(&job)
+	return &job
+}
+
+func newCondition(conditionType batchv1.JobConditionType, reason, message string) batchv1.JobCondition {
+	return batchv1.JobCondition{
+		Type:               conditionType,
+		Status:             apiv1.ConditionTrue,
+		LastProbeTime:      metav1.Now(),
+		LastTransitionTime: metav1.Now(),
+		Reason:             reason,
+		Message:            message,
+	}
+}
+
+func finishJob(job *batchv1.Job, duration time.Duration) {
+	job.Status.Conditions = append(job.Status.Conditions, newCondition(batchv1.JobComplete, "", ""))
+	if job.Spec.Completions == nil {
+		job.Status.Succeeded = 1
+	} else {
+		job.Status.Succeeded = *job.Spec.Completions
+	}
+	now := metav1.Now()
+	job.Status.StartTime = &now
+	time.Sleep(duration)
+	now = metav1.Now()
+	job.Status.CompletionTime = &now
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/service/BUILD

@@ -0,0 +1,84 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["servicecontroller.go"],
+    deps = [
+        "//federation/apis/federation:go_default_library",
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset:go_default_library",
+        "//federation/pkg/federation-controller/service/ingress:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//federation/pkg/federation-controller/util/clusterselector:go_default_library",
+        "//federation/pkg/federation-controller/util/deletionhelper:go_default_library",
+        "//federation/pkg/federation-controller/util/eventsink:go_default_library",
+        "//pkg/api:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+        "//vendor/k8s.io/client-go/tools/record:go_default_library",
+        "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
+        "//vendor/k8s.io/client-go/util/workqueue:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["servicecontroller_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset/fake:go_default_library",
+        "//federation/pkg/federation-controller/service/ingress:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//federation/pkg/federation-controller/util/deletionhelper:go_default_library",
+        "//federation/pkg/federation-controller/util/test:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/stretchr/testify/require:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
+        "//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//federation/pkg/federation-controller/service/dns:all-srcs",
+        "//federation/pkg/federation-controller/service/ingress:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/service/dns/BUILD

@@ -0,0 +1,59 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["dns_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset/fake:go_default_library",
+        "//federation/pkg/dnsprovider/providers/google/clouddns:go_default_library",
+        "//federation/pkg/federation-controller/service/ingress:go_default_library",
+        "//federation/pkg/federation-controller/util/test:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["dns.go"],
+    deps = [
+        "//federation/client/clientset_generated/federation_clientset:go_default_library",
+        "//federation/pkg/dnsprovider:go_default_library",
+        "//federation/pkg/dnsprovider/rrstype:go_default_library",
+        "//federation/pkg/federation-controller/service/ingress:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+        "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
+        "//vendor/k8s.io/client-go/util/workqueue:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/service/dns/dns.go

@@ -0,0 +1,549 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dns
+
+import (
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pkgruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
+	corelisters "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/flowcontrol"
+	"k8s.io/client-go/util/workqueue"
+	fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	"k8s.io/kubernetes/federation/pkg/dnsprovider"
+	"k8s.io/kubernetes/federation/pkg/dnsprovider/rrstype"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/service/ingress"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
+
+	"github.com/golang/glog"
+)
+
+const (
+	ControllerName = "service-dns"
+
+	UserAgentName = "federation-service-dns-controller"
+
+	// minDNSTTL is the minimum safe DNS TTL value to use (in seconds).  We use this as the TTL for all DNS records.
+	minDNSTTL = 180
+
+	serviceSyncPeriod = 30 * time.Second
+)
+
+type ServiceDNSController struct {
+	// Client to federation api server
+	federationClient fedclientset.Interface
+	dns              dnsprovider.Interface
+	federationName   string
+	// serviceDNSSuffix is the DNS suffix we use when publishing service DNS names
+	serviceDNSSuffix string
+	// zoneName and zoneID are used to identify the zone in which to put records
+	zoneName string
+	zoneID   string
+	dnsZones dnsprovider.Zones
+	// each federation should be configured with a single zone (e.g. "mycompany.com")
+	dnsZone dnsprovider.Zone
+	// Informer Store for federated services
+	serviceStore corelisters.ServiceLister
+	// Informer controller for federated services
+	serviceController  cache.Controller
+	workQueue          workqueue.Interface
+	objectDeliverer    *util.DelayingDeliverer
+	flowcontrolBackoff *flowcontrol.Backoff
+}
+
+// NewServiceDNSController returns a new service dns controller to manage DNS records for federated services
+func NewServiceDNSController(client fedclientset.Interface, dnsProvider, dnsProviderConfig, federationName,
+	serviceDNSSuffix, zoneName, zoneID string) (*ServiceDNSController, error) {
+	dns, err := dnsprovider.InitDnsProvider(dnsProvider, dnsProviderConfig)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("DNS provider could not be initialized: %v", err))
+		return nil, err
+	}
+	d := &ServiceDNSController{
+		federationClient:   client,
+		dns:                dns,
+		federationName:     federationName,
+		serviceDNSSuffix:   serviceDNSSuffix,
+		zoneName:           zoneName,
+		zoneID:             zoneID,
+		workQueue:          workqueue.New(),
+		objectDeliverer:    util.NewDelayingDeliverer(),
+		flowcontrolBackoff: flowcontrol.NewBackOff(5*time.Second, time.Minute),
+	}
+	if err := d.validateConfig(); err != nil {
+		runtime.HandleError(fmt.Errorf("Invalid configuration passed to DNS provider: %v", err))
+		return nil, err
+	}
+	if err := d.retrieveOrCreateDNSZone(); err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to retrieve DNS zone: %v", err))
+		return nil, err
+	}
+
+	// Start informer in federated API servers on federated services
+	var serviceIndexer cache.Indexer
+	serviceIndexer, d.serviceController = cache.NewIndexerInformer(
+		&cache.ListWatch{
+			ListFunc: func(options metav1.ListOptions) (pkgruntime.Object, error) {
+				return client.Core().Services(metav1.NamespaceAll).List(options)
+			},
+			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+				return client.Core().Services(metav1.NamespaceAll).Watch(options)
+			},
+		},
+		&v1.Service{},
+		serviceSyncPeriod,
+		util.NewTriggerOnAllChanges(func(obj pkgruntime.Object) { d.workQueue.Add(obj) }),
+		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
+	)
+	d.serviceStore = corelisters.NewServiceLister(serviceIndexer)
+
+	return d, nil
+}
+
+func (s *ServiceDNSController) DNSControllerRun(workers int, stopCh <-chan struct{}) {
+	defer runtime.HandleCrash()
+	defer s.workQueue.ShutDown()
+
+	glog.Infof("Starting federation service dns controller")
+
+	s.objectDeliverer.StartWithHandler(func(item *util.DelayingDelivererItem) {
+		s.workQueue.Add(item.Value.(*v1.Service))
+	})
+	defer s.objectDeliverer.Stop()
+
+	util.StartBackoffGC(s.flowcontrolBackoff, stopCh)
+	go s.serviceController.Run(stopCh)
+
+	for i := 0; i < workers; i++ {
+		go wait.Until(s.worker, time.Second, stopCh)
+	}
+
+	<-stopCh
+	glog.Infof("Stopping federation service dns controller")
+}
+
+// Adds backoff to delay if this delivery is related to some failure. Resets backoff if there was no failure.
+func (s *ServiceDNSController) deliverService(service *v1.Service, delay time.Duration, failed bool) {
+	if failed {
+		s.flowcontrolBackoff.Next(service.String(), time.Now())
+		delay = delay + s.flowcontrolBackoff.Get(service.String())
+	} else {
+		s.flowcontrolBackoff.Reset(service.String())
+	}
+	s.objectDeliverer.DeliverAfter(service.String(), service, delay)
+}
+
+func wantsDNSRecords(service *v1.Service) bool {
+	return service.Spec.Type == v1.ServiceTypeLoadBalancer
+}
+
+func (s *ServiceDNSController) workerFunction() bool {
+	item, quit := s.workQueue.Get()
+	if quit {
+		return true
+	}
+	defer s.workQueue.Done(item)
+
+	service := item.(*v1.Service)
+
+	if !wantsDNSRecords(service) {
+		return false
+	}
+
+	ingress, err := ingress.ParseFederatedServiceIngress(service)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Error in parsing lb ingress for service %s/%s: %v", service.Namespace, service.Name, err))
+		return false
+	}
+	for _, clusterIngress := range ingress.Items {
+		err = s.ensureDNSRecords(clusterIngress.Cluster, service)
+		if err != nil {
+			runtime.HandleError(fmt.Errorf("Error when ensuring DNS records for service %s/%s: %v", service.Namespace, service.Name, err))
+			s.deliverService(service, 0, true)
+		}
+	}
+	return false
+}
+
+func (s *ServiceDNSController) worker() {
+	for {
+		if quit := s.workerFunction(); quit {
+			glog.Infof("service dns controller worker queue shutting down")
+			return
+		}
+	}
+}
+
+func (s *ServiceDNSController) validateConfig() error {
+	if s.federationName == "" {
+		return fmt.Errorf("DNSController should not be run without federationName")
+	}
+	if s.zoneName == "" && s.zoneID == "" {
+		return fmt.Errorf("DNSController must be run with either zoneName or zoneID")
+	}
+	if s.serviceDNSSuffix == "" {
+		if s.zoneName == "" {
+			return fmt.Errorf("DNSController must be run with zoneName, if serviceDnsSuffix is not set")
+		}
+		s.serviceDNSSuffix = s.zoneName
+	}
+	if s.dns == nil {
+		return fmt.Errorf("DNSController should not be run without a dnsprovider")
+	}
+	zones, ok := s.dns.Zones()
+	if !ok {
+		return fmt.Errorf("the dns provider does not support zone enumeration, which is required for creating dns records")
+	}
+	s.dnsZones = zones
+	return nil
+}
+
+func (s *ServiceDNSController) retrieveOrCreateDNSZone() error {
+	matchingZones, err := getDNSZones(s.zoneName, s.zoneID, s.dnsZones)
+	if err != nil {
+		return fmt.Errorf("error querying for DNS zones: %v", err)
+	}
+	switch len(matchingZones) {
+	case 0: // No matching zones for s.zoneName, so create one
+		if s.zoneName == "" {
+			return fmt.Errorf("DNSController must be run with zoneName to create zone automatically")
+		}
+		glog.Infof("DNS zone %q not found.  Creating DNS zone %q.", s.zoneName, s.zoneName)
+		managedZone, err := s.dnsZones.New(s.zoneName)
+		if err != nil {
+			return err
+		}
+		zone, err := s.dnsZones.Add(managedZone)
+		if err != nil {
+			return err
+		}
+		glog.Infof("DNS zone %q successfully created.  Note that DNS resolution will not work until you have registered this name with "+
+			"a DNS registrar and they have changed the authoritative name servers for your domain to point to your DNS provider", zone.Name())
+	case 1: // s.zoneName matches exactly one DNS zone
+		s.dnsZone = matchingZones[0]
+	default: // s.zoneName matches more than one DNS zone
+		return fmt.Errorf("Multiple matching DNS zones found for %q; please specify zoneID", s.zoneName)
+	}
+	return nil
+}
+
+// getHealthyEndpoints returns the hostnames and/or IP addresses of healthy endpoints for the service, at a zone, region and global level (or an error)
+func (s *ServiceDNSController) getHealthyEndpoints(clusterName string, service *v1.Service) (zoneEndpoints, regionEndpoints, globalEndpoints []string, err error) {
+	var (
+		zoneNames  []string
+		regionName string
+	)
+	if zoneNames, regionName, err = s.getClusterZoneNames(clusterName); err != nil {
+		return nil, nil, nil, err
+	}
+
+	// If federated service is deleted, return empty endpoints, so that DNS records are removed
+	if service.DeletionTimestamp != nil {
+		return zoneEndpoints, regionEndpoints, globalEndpoints, nil
+	}
+
+	serviceIngress, err := ingress.ParseFederatedServiceIngress(service)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	for _, lbClusterIngress := range serviceIngress.Items {
+		lbClusterName := lbClusterIngress.Cluster
+		lbZoneNames, lbRegionName, err := s.getClusterZoneNames(lbClusterName)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		for _, ingress := range lbClusterIngress.Items {
+			var address string
+			// We should get either an IP address or a hostname - use whichever one we get
+			if ingress.IP != "" {
+				address = ingress.IP
+			} else if ingress.Hostname != "" {
+				address = ingress.Hostname
+			}
+			if len(address) <= 0 {
+				return nil, nil, nil, fmt.Errorf("Service %s/%s in cluster %s has neither LoadBalancerStatus.ingress.ip nor LoadBalancerStatus.ingress.hostname. Cannot use it as endpoint for federated service",
+					service.Name, service.Namespace, clusterName)
+			}
+			for _, lbZoneName := range lbZoneNames {
+				for _, zoneName := range zoneNames {
+					if lbZoneName == zoneName {
+						zoneEndpoints = append(zoneEndpoints, address)
+					}
+				}
+			}
+			if lbRegionName == regionName {
+				regionEndpoints = append(regionEndpoints, address)
+			}
+			globalEndpoints = append(globalEndpoints, address)
+		}
+	}
+	return zoneEndpoints, regionEndpoints, globalEndpoints, nil
+}
+
+// getClusterZoneNames returns the name of the zones (and the region) where the specified cluster exists (e.g. zones "us-east1-c" on GCE, or "us-east-1b" on AWS)
+func (s *ServiceDNSController) getClusterZoneNames(clusterName string) ([]string, string, error) {
+	cluster, err := s.federationClient.Federation().Clusters().Get(clusterName, metav1.GetOptions{})
+	if err != nil {
+		return nil, "", err
+	}
+	return cluster.Status.Zones, cluster.Status.Region, nil
+}
+
+// getDNSZones returns the DNS zones matching dnsZoneName and dnsZoneID (if specified)
+func getDNSZones(dnsZoneName string, dnsZoneID string, dnsZonesInterface dnsprovider.Zones) ([]dnsprovider.Zone, error) {
+	// TODO: We need query-by-name and query-by-id functions
+	dnsZones, err := dnsZonesInterface.List()
+	if err != nil {
+		return nil, err
+	}
+
+	var matches []dnsprovider.Zone
+	findName := strings.TrimSuffix(dnsZoneName, ".")
+	for _, dnsZone := range dnsZones {
+		if dnsZoneID != "" {
+			if dnsZoneID != dnsZone.ID() {
+				continue
+			}
+		}
+		if findName != "" {
+			if strings.TrimSuffix(dnsZone.Name(), ".") != findName {
+				continue
+			}
+		}
+		matches = append(matches, dnsZone)
+	}
+
+	return matches, nil
+}
+
+// NOTE: that if the named resource record set does not exist, but no
+// error occurred, the returned list will be empty, and the error will
+// be nil
+func getRrset(dnsName string, rrsetsInterface dnsprovider.ResourceRecordSets) ([]dnsprovider.ResourceRecordSet, error) {
+	return rrsetsInterface.Get(dnsName)
+}
+
+func findRrset(list []dnsprovider.ResourceRecordSet, rrset dnsprovider.ResourceRecordSet) dnsprovider.ResourceRecordSet {
+	for i, elem := range list {
+		if dnsprovider.ResourceRecordSetsEquivalent(rrset, elem) {
+			return list[i]
+		}
+	}
+	return nil
+}
+
+/* getResolvedEndpoints performs DNS resolution on the provided slice of endpoints (which might be DNS names or IPv4 addresses)
+   and returns a list of IPv4 addresses.  If any of the endpoints are neither valid IPv4 addresses nor resolvable DNS names,
+   non-nil error is also returned (possibly along with a partially complete list of resolved endpoints.
+*/
+func getResolvedEndpoints(endpoints []string) ([]string, error) {
+	resolvedEndpoints := sets.String{}
+	for _, endpoint := range endpoints {
+		if net.ParseIP(endpoint) == nil {
+			// It's not a valid IP address, so assume it's a DNS name, and try to resolve it,
+			// replacing its DNS name with its IP addresses in expandedEndpoints
+			ipAddrs, err := net.LookupHost(endpoint)
+			if err != nil {
+				return resolvedEndpoints.List(), err
+			}
+			for _, ip := range ipAddrs {
+				resolvedEndpoints = resolvedEndpoints.Union(sets.NewString(ip))
+			}
+		} else {
+			resolvedEndpoints = resolvedEndpoints.Union(sets.NewString(endpoint))
+		}
+	}
+	return resolvedEndpoints.List(), nil
+}
+
+/* ensureDNSRrsets ensures (idempotently, and with minimum mutations) that all of the DNS resource record sets for dnsName are consistent with endpoints.
+   if endpoints is nil or empty, a CNAME record to uplevelCname is ensured.
+*/
+func (s *ServiceDNSController) ensureDNSRrsets(dnsZone dnsprovider.Zone, dnsName string, endpoints []string, uplevelCname string) error {
+	rrsets, supported := dnsZone.ResourceRecordSets()
+	if !supported {
+		return fmt.Errorf("Failed to ensure DNS records for %s. DNS provider does not support the ResourceRecordSets interface", dnsName)
+	}
+	rrsetList, err := getRrset(dnsName, rrsets) // TODO: rrsets.Get(dnsName)
+	if err != nil {
+		return err
+	}
+	if len(rrsetList) == 0 {
+		glog.V(4).Infof("No recordsets found for DNS name %q.  Need to add either A records (if we have healthy endpoints), or a CNAME record to %q", dnsName, uplevelCname)
+		if len(endpoints) < 1 {
+			glog.V(4).Infof("There are no healthy endpoint addresses at level %q, so CNAME to %q, if provided", dnsName, uplevelCname)
+			if uplevelCname != "" {
+				glog.V(4).Infof("Creating CNAME to %q for %q", uplevelCname, dnsName)
+				newRrset := rrsets.New(dnsName, []string{uplevelCname}, minDNSTTL, rrstype.CNAME)
+				glog.V(4).Infof("Adding recordset %v", newRrset)
+				err = rrsets.StartChangeset().Add(newRrset).Apply()
+				if err != nil {
+					return err
+				}
+				glog.V(4).Infof("Successfully created CNAME to %q for %q", uplevelCname, dnsName)
+			} else {
+				glog.V(4).Infof("We want no record for %q, and we have no record, so we're all good.", dnsName)
+			}
+		} else {
+			// We have valid endpoint addresses, so just add them as A records.
+			// But first resolve DNS names, as some cloud providers (like AWS) expose
+			// load balancers behind DNS names, not IP addresses.
+			glog.V(4).Infof("We have valid endpoint addresses %v at level %q, so add them as A records, after resolving DNS names", endpoints, dnsName)
+			resolvedEndpoints, err := getResolvedEndpoints(endpoints)
+			if err != nil {
+				return err // TODO: We could potentially add the ones we did get back, even if some of them failed to resolve.
+			}
+			newRrset := rrsets.New(dnsName, resolvedEndpoints, minDNSTTL, rrstype.A)
+			glog.V(4).Infof("Adding recordset %v", newRrset)
+			err = rrsets.StartChangeset().Add(newRrset).Apply()
+			if err != nil {
+				return err
+			}
+			glog.V(4).Infof("Successfully added recordset %v", newRrset)
+		}
+	} else {
+		// the rrsets already exists, so make it right.
+		glog.V(4).Infof("Recordset %v already exists. Ensuring that it is correct.", rrsetList)
+		if len(endpoints) < 1 {
+			// Need an appropriate CNAME record.  Check that we have it.
+			newRrset := rrsets.New(dnsName, []string{uplevelCname}, minDNSTTL, rrstype.CNAME)
+			glog.V(4).Infof("No healthy endpoints for %s. Have recordsets %v. Need recordset %v", dnsName, rrsetList, newRrset)
+			found := findRrset(rrsetList, newRrset)
+			if found != nil {
+				// The existing rrset is equivalent to the required one - our work is done here
+				glog.V(4).Infof("Existing recordset %v is equivalent to needed recordset %v, our work is done here.", rrsetList, newRrset)
+				return nil
+			} else {
+				// Need to replace the existing one with a better one (or just remove it if we have no healthy endpoints).
+				glog.V(4).Infof("Existing recordset %v not equivalent to needed recordset %v removing existing and adding needed.", rrsetList, newRrset)
+				changeSet := rrsets.StartChangeset()
+				for i := range rrsetList {
+					changeSet = changeSet.Remove(rrsetList[i])
+				}
+				if uplevelCname != "" {
+					changeSet = changeSet.Add(newRrset)
+					if err := changeSet.Apply(); err != nil {
+						return err
+					}
+					glog.V(4).Infof("Successfully replaced needed recordset %v -> %v", found, newRrset)
+				} else {
+					if err := changeSet.Apply(); err != nil {
+						return err
+					}
+					glog.V(4).Infof("Successfully removed existing recordset %v", found)
+					glog.V(4).Infof("Uplevel CNAME is empty string. Not adding recordset %v", newRrset)
+				}
+			}
+		} else {
+			// We have an rrset in DNS, possibly with some missing addresses and some unwanted addresses.
+			// And we have healthy endpoints.  Just replace what's there with the healthy endpoints, if it's not already correct.
+			glog.V(4).Infof("%s: Healthy endpoints %v exist. Recordset %v exists.  Reconciling.", dnsName, endpoints, rrsetList)
+			resolvedEndpoints, err := getResolvedEndpoints(endpoints)
+			if err != nil { // Some invalid addresses or otherwise unresolvable DNS names.
+				return err // TODO: We could potentially add the ones we did get back, even if some of them failed to resolve.
+			}
+			newRrset := rrsets.New(dnsName, resolvedEndpoints, minDNSTTL, rrstype.A)
+			glog.V(4).Infof("Have recordset %v. Need recordset %v", rrsetList, newRrset)
+			found := findRrset(rrsetList, newRrset)
+			if found != nil {
+				glog.V(4).Infof("Existing recordset %v is equivalent to needed recordset %v, our work is done here.", found, newRrset)
+				// TODO: We could be more thorough about checking for equivalence to avoid unnecessary updates, but in the
+				//       worst case we'll just replace what's there with an equivalent, if not exactly identical record set.
+				return nil
+			} else {
+				// Need to replace the existing one with a better one
+				glog.V(4).Infof("Existing recordset %v is not equivalent to needed recordset %v, removing existing and adding needed.", found, newRrset)
+				changeSet := rrsets.StartChangeset()
+				for i := range rrsetList {
+					changeSet = changeSet.Remove(rrsetList[i])
+				}
+				changeSet = changeSet.Add(newRrset)
+				if err = changeSet.Apply(); err != nil {
+					return err
+				}
+				glog.V(4).Infof("Successfully replaced recordset %v -> %v", found, newRrset)
+			}
+		}
+	}
+	return nil
+}
+
+/* ensureDNSRecords ensures (idempotently, and with minimum mutations) that all of the DNS records for a service in a given cluster are correct,
+given the current state of that service in that cluster.  This should be called every time the state of a service might have changed
+(either w.r.t. its loadbalancer address, or if the number of healthy backend endpoints for that service transitioned from zero to non-zero
+(or vice versa).  Only shards of the service which have both a loadbalancer ingress IP address or hostname AND at least one healthy backend endpoint
+are included in DNS records for that service (at all of zone, region and global levels). All other addresses are removed.  Also, if no shards exist
+in the zone or region of the cluster, a CNAME reference to the next higher level is ensured to exist. */
+func (s *ServiceDNSController) ensureDNSRecords(clusterName string, service *v1.Service) error {
+	// Quinton: Pseudocode....
+	// See https://github.com/kubernetes/kubernetes/pull/25107#issuecomment-218026648
+	// For each service we need the following DNS names:
+	// mysvc.myns.myfed.svc.z1.r1.mydomain.com  (for zone z1 in region r1)
+	//         - an A record to IP address of specific shard in that zone (if that shard exists and has healthy endpoints)
+	//         - OR a CNAME record to the next level up, i.e. mysvc.myns.myfed.svc.r1.mydomain.com  (if a healthy shard does not exist in zone z1)
+	// mysvc.myns.myfed.svc.r1.mydomain.com
+	//         - a set of A records to IP addresses of all healthy shards in region r1, if one or more of these exist
+	//         - OR a CNAME record to the next level up, i.e. mysvc.myns.myfed.svc.mydomain.com (if no healthy shards exist in region r1)
+	// mysvc.myns.myfed.svc.mydomain.com
+	//         - a set of A records to IP addresses of all healthy shards in all regions, if one or more of these exist.
+	//         - no record (NXRECORD response) if no healthy shards exist in any regions
+	//
+	// Each service has the current known state of loadbalancer ingress for the federated cluster stored in annotations.
+	// So generate the DNS records based on the current state and ensure those desired DNS records match the
+	// actual DNS records (add new records, remove deleted records, and update changed records).
+	//
+	serviceName := service.Name
+	namespaceName := service.Namespace
+	zoneNames, regionName, err := s.getClusterZoneNames(clusterName)
+	if err != nil {
+		return err
+	}
+	if zoneNames == nil {
+		return fmt.Errorf("failed to get cluster zone names")
+	}
+	commonPrefix := serviceName + "." + namespaceName + "." + s.federationName + ".svc"
+	// dnsNames is the path up the DNS search tree, starting at the leaf
+	dnsNames := []string{
+		strings.Join([]string{commonPrefix, zoneNames[0], regionName, s.serviceDNSSuffix}, "."), // zone level - TODO might need other zone names for multi-zone clusters
+		strings.Join([]string{commonPrefix, regionName, s.serviceDNSSuffix}, "."),               // region level, one up from zone level
+		strings.Join([]string{commonPrefix, s.serviceDNSSuffix}, "."),                           // global level, one up from region level
+		"", // nowhere to go up from global level
+	}
+
+	zoneEndpoints, regionEndpoints, globalEndpoints, err := s.getHealthyEndpoints(clusterName, service)
+	if err != nil {
+		return err
+	}
+	endpoints := [][]string{zoneEndpoints, regionEndpoints, globalEndpoints}
+	for i, endpoint := range endpoints {
+		if err = s.ensureDNSRrsets(s.dnsZone, dnsNames[i], endpoint, dnsNames[i+1]); err != nil {
+			return err
+		}
+	}
+	return nil
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/service/dns/dns_test.go

@@ -0,0 +1,256 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dns
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	fakefedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset/fake"
+	"k8s.io/kubernetes/federation/pkg/dnsprovider/providers/google/clouddns" // Only for unit testing purposes.
+	"k8s.io/kubernetes/federation/pkg/federation-controller/service/ingress"
+	. "k8s.io/kubernetes/federation/pkg/federation-controller/util/test"
+)
+
+// NewClusterWithRegionZone builds a new cluster object with given region and zone attributes.
+func NewClusterWithRegionZone(name string, readyStatus v1.ConditionStatus, region, zone string) *v1beta1.Cluster {
+	cluster := NewCluster(name, readyStatus)
+	cluster.Status.Zones = []string{zone}
+	cluster.Status.Region = region
+	return cluster
+}
+
+func TestServiceController_ensureDnsRecords(t *testing.T) {
+	cluster1Name := "c1"
+	cluster2Name := "c2"
+	cluster1 := NewClusterWithRegionZone(cluster1Name, v1.ConditionTrue, "fooregion", "foozone")
+	cluster2 := NewClusterWithRegionZone(cluster2Name, v1.ConditionTrue, "barregion", "barzone")
+	globalDNSName := "servicename.servicenamespace.myfederation.svc.federation.example.com"
+	fooRegionDNSName := "servicename.servicenamespace.myfederation.svc.fooregion.federation.example.com"
+	fooZoneDNSName := "servicename.servicenamespace.myfederation.svc.foozone.fooregion.federation.example.com"
+	barRegionDNSName := "servicename.servicenamespace.myfederation.svc.barregion.federation.example.com"
+	barZoneDNSName := "servicename.servicenamespace.myfederation.svc.barzone.barregion.federation.example.com"
+
+	tests := []struct {
+		name     string
+		service  v1.Service
+		expected []string
+	}{
+		{
+			name: "ServiceWithSingleLBIngress",
+			service: v1.Service{
+				ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{
+					ingress.FederatedServiceIngressAnnotation: ingress.NewFederatedServiceIngress().
+						AddEndpoints(cluster1Name, []string{"198.51.100.1"}).
+						AddEndpoints(cluster2Name, []string{}).
+						String()},
+				},
+			},
+			expected: []string{
+				"example.com:" + globalDNSName + ":A:180:[198.51.100.1]",
+				"example.com:" + fooRegionDNSName + ":A:180:[198.51.100.1]",
+				"example.com:" + fooZoneDNSName + ":A:180:[198.51.100.1]",
+				"example.com:" + barRegionDNSName + ":CNAME:180:[" + globalDNSName + "]",
+				"example.com:" + barZoneDNSName + ":CNAME:180:[" + barRegionDNSName + "]",
+			},
+		},
+		/*
+			TODO: getResolvedEndpoints preforms DNS lookup.
+			Mock and maybe look at error handling when some endpoints resolve, but also caching?
+			{
+				name: "withname",
+				service: v1.Service{
+					ObjectMeta: metav1.ObjectMeta{},
+				},
+				expected: []string{
+					"example.com:"+globalDNSName+":A:180:[198.51.100.1]",
+					"example.com:"+fooRegionDNSName+":A:180:[198.51.100.1]",
+					"example.com:"+fooZoneDNSName+":A:180:[198.51.100.1]",
+				},
+			},
+		*/
+		{
+			name: "ServiceWithNoLBIngress",
+			service: v1.Service{
+				ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{
+					ingress.FederatedServiceIngressAnnotation: ingress.NewFederatedServiceIngress().
+						AddEndpoints(cluster1Name, []string{}).
+						AddEndpoints(cluster2Name, []string{}).
+						String()},
+				},
+			},
+			expected: []string{
+				"example.com:" + fooRegionDNSName + ":CNAME:180:[" + globalDNSName + "]",
+				"example.com:" + fooZoneDNSName + ":CNAME:180:[" + fooRegionDNSName + "]",
+				"example.com:" + barRegionDNSName + ":CNAME:180:[" + globalDNSName + "]",
+				"example.com:" + barZoneDNSName + ":CNAME:180:[" + barRegionDNSName + "]",
+			},
+		},
+		{
+			name: "ServiceWithMultipleLBIngress",
+			service: v1.Service{
+				ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{
+					ingress.FederatedServiceIngressAnnotation: ingress.NewFederatedServiceIngress().
+						AddEndpoints(cluster1Name, []string{"198.51.100.1"}).
+						AddEndpoints(cluster2Name, []string{"198.51.200.1"}).
+						String()},
+				},
+			},
+			expected: []string{
+				"example.com:" + globalDNSName + ":A:180:[198.51.100.1 198.51.200.1]",
+				"example.com:" + fooRegionDNSName + ":A:180:[198.51.100.1]",
+				"example.com:" + fooZoneDNSName + ":A:180:[198.51.100.1]",
+				"example.com:" + barRegionDNSName + ":A:180:[198.51.200.1]",
+				"example.com:" + barZoneDNSName + ":A:180:[198.51.200.1]",
+			},
+		},
+		{
+			name: "ServiceWithLBIngressAndServiceDeleted",
+			service: v1.Service{
+				ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{
+					ingress.FederatedServiceIngressAnnotation: ingress.NewFederatedServiceIngress().
+						AddEndpoints(cluster1Name, []string{"198.51.100.1"}).
+						AddEndpoints(cluster2Name, []string{"198.51.200.1"}).
+						String()},
+					DeletionTimestamp: &metav1.Time{Time: time.Now()},
+				},
+			},
+			expected: []string{
+				// TODO: Ideally we should expect that there are no DNS records when federated service is deleted. Need to remove these leaks in future
+				"example.com:" + fooRegionDNSName + ":CNAME:180:[" + globalDNSName + "]",
+				"example.com:" + fooZoneDNSName + ":CNAME:180:[" + fooRegionDNSName + "]",
+				"example.com:" + barRegionDNSName + ":CNAME:180:[" + globalDNSName + "]",
+				"example.com:" + barZoneDNSName + ":CNAME:180:[" + barRegionDNSName + "]",
+			},
+		},
+		{
+			name: "ServiceWithMultipleLBIngressAndOneLBIngressGettingRemoved",
+			service: v1.Service{
+				ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{
+					ingress.FederatedServiceIngressAnnotation: ingress.NewFederatedServiceIngress().
+						AddEndpoints(cluster1Name, []string{"198.51.100.1"}).
+						AddEndpoints(cluster2Name, []string{"198.51.200.1"}).
+						RemoveEndpoint(cluster2Name, "198.51.200.1").
+						String()},
+				},
+			},
+			expected: []string{
+				"example.com:" + globalDNSName + ":A:180:[198.51.100.1]",
+				"example.com:" + fooRegionDNSName + ":A:180:[198.51.100.1]",
+				"example.com:" + fooZoneDNSName + ":A:180:[198.51.100.1]",
+				"example.com:" + barRegionDNSName + ":CNAME:180:[" + globalDNSName + "]",
+				"example.com:" + barZoneDNSName + ":CNAME:180:[" + barRegionDNSName + "]",
+			},
+		},
+		{
+			name: "ServiceWithMultipleLBIngressAndAllLBIngressGettingRemoved",
+			service: v1.Service{
+				ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{
+					ingress.FederatedServiceIngressAnnotation: ingress.NewFederatedServiceIngress().
+						AddEndpoints(cluster1Name, []string{"198.51.100.1"}).
+						AddEndpoints(cluster2Name, []string{"198.51.200.1"}).
+						RemoveEndpoint(cluster1Name, "198.51.100.1").
+						RemoveEndpoint(cluster2Name, "198.51.200.1").
+						String()},
+				},
+			},
+			expected: []string{
+				"example.com:" + fooRegionDNSName + ":CNAME:180:[" + globalDNSName + "]",
+				"example.com:" + fooZoneDNSName + ":CNAME:180:[" + fooRegionDNSName + "]",
+				"example.com:" + barRegionDNSName + ":CNAME:180:[" + globalDNSName + "]",
+				"example.com:" + barZoneDNSName + ":CNAME:180:[" + barRegionDNSName + "]",
+			},
+		},
+	}
+	for _, test := range tests {
+		fakedns, _ := clouddns.NewFakeInterface()
+		fakednsZones, ok := fakedns.Zones()
+		if !ok {
+			t.Error("Unable to fetch zones")
+		}
+		fakeClient := &fakefedclientset.Clientset{}
+		RegisterFakeClusterGet(&fakeClient.Fake, &v1beta1.ClusterList{Items: []v1beta1.Cluster{*cluster1, *cluster2}})
+		d := ServiceDNSController{
+			federationClient: fakeClient,
+			dns:              fakedns,
+			dnsZones:         fakednsZones,
+			serviceDNSSuffix: "federation.example.com",
+			zoneName:         "example.com",
+			federationName:   "myfederation",
+		}
+
+		dnsZones, err := getDNSZones(d.zoneName, d.zoneID, d.dnsZones)
+		if err != nil {
+			t.Errorf("Test failed for %s, Get DNS Zones failed: %v", test.name, err)
+		}
+		d.dnsZone = dnsZones[0]
+		test.service.Name = "servicename"
+		test.service.Namespace = "servicenamespace"
+
+		ingress, err := ingress.ParseFederatedServiceIngress(&test.service)
+		if err != nil {
+			t.Errorf("Error in parsing lb ingress for service %s/%s: %v", test.service.Namespace, test.service.Name, err)
+			return
+		}
+		for _, clusterIngress := range ingress.Items {
+			d.ensureDNSRecords(clusterIngress.Cluster, &test.service)
+		}
+
+		zones, err := fakednsZones.List()
+		if err != nil {
+			t.Errorf("error querying zones: %v", err)
+		}
+
+		// Dump every record to a testable-by-string-comparison form
+		records := []string{}
+		for _, z := range zones {
+			zoneName := z.Name()
+
+			rrs, ok := z.ResourceRecordSets()
+			if !ok {
+				t.Errorf("cannot get rrs for zone %q", zoneName)
+			}
+
+			rrList, err := rrs.List()
+			if err != nil {
+				t.Errorf("error querying rr for zone %q: %v", zoneName, err)
+			}
+			for _, rr := range rrList {
+				rrdatas := rr.Rrdatas()
+
+				// Put in consistent (testable-by-string-comparison) order
+				sort.Strings(rrdatas)
+				records = append(records, fmt.Sprintf("%s:%s:%s:%d:%s", zoneName, rr.Name(), rr.Type(), rr.Ttl(), rrdatas))
+			}
+		}
+
+		// Ignore order of records
+		sort.Strings(records)
+		sort.Strings(test.expected)
+
+		if !reflect.DeepEqual(records, test.expected) {
+			t.Errorf("Test %q failed.  Actual=%v, Expected=%v", test.name, records, test.expected)
+		}
+
+	}
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/service/ingress/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["ingress.go"],
+    deps = [
+        "//federation/apis/federation:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/service/ingress/ingress.go

@@ -0,0 +1,136 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ingress
+
+import (
+	"encoding/json"
+	"sort"
+	"strings"
+
+	"k8s.io/api/core/v1"
+	fedapi "k8s.io/kubernetes/federation/apis/federation"
+)
+
+// Compile time check for interface adherence
+var _ sort.Interface = &FederatedServiceIngress{}
+
+const (
+	FederatedServiceIngressAnnotation = "federation.kubernetes.io/service-ingresses"
+)
+
+// FederatedServiceIngress implements sort.Interface.
+type FederatedServiceIngress struct {
+	fedapi.FederatedServiceIngress
+}
+
+func NewFederatedServiceIngress() *FederatedServiceIngress {
+	return &FederatedServiceIngress{}
+}
+
+func (ingress *FederatedServiceIngress) String() string {
+	annotationBytes, _ := json.Marshal(ingress)
+	return string(annotationBytes[:])
+}
+
+// Len is to satisfy of sort.Interface.
+func (ingress *FederatedServiceIngress) Len() int {
+	return len(ingress.Items)
+}
+
+// Less is to satisfy of sort.Interface.
+func (ingress *FederatedServiceIngress) Less(i, j int) bool {
+	return (strings.Compare(ingress.Items[i].Cluster, ingress.Items[j].Cluster) < 0)
+}
+
+// Swap is to satisfy of sort.Interface.
+func (ingress *FederatedServiceIngress) Swap(i, j int) {
+	ingress.Items[i].Cluster, ingress.Items[j].Cluster = ingress.Items[j].Cluster, ingress.Items[i].Cluster
+	ingress.Items[i].Items, ingress.Items[j].Items = ingress.Items[j].Items, ingress.Items[i].Items
+}
+
+// GetClusterLoadBalancerIngresses returns loadbalancer ingresses for given cluster if exist otherwise returns an empty slice
+func (ingress *FederatedServiceIngress) GetClusterLoadBalancerIngresses(cluster string) []v1.LoadBalancerIngress {
+	for _, clusterIngress := range ingress.Items {
+		if cluster == clusterIngress.Cluster {
+			return clusterIngress.Items
+		}
+	}
+	return []v1.LoadBalancerIngress{}
+}
+
+// AddClusterLoadBalancerIngresses adds the ladbalancer ingresses for a given cluster to federated service ingress
+func (ingress *FederatedServiceIngress) AddClusterLoadBalancerIngresses(cluster string, loadbalancerIngresses []v1.LoadBalancerIngress) {
+	for i, clusterIngress := range ingress.Items {
+		if cluster == clusterIngress.Cluster {
+			ingress.Items[i].Items = append(ingress.Items[i].Items, loadbalancerIngresses...)
+			return
+		}
+	}
+	clusterNewIngress := fedapi.ClusterServiceIngress{Cluster: cluster, Items: loadbalancerIngresses}
+	ingress.Items = append(ingress.Items, clusterNewIngress)
+	sort.Sort(ingress)
+}
+
+// AddEndpoints add one or more endpoints to federated service ingress.
+// endpoints are federated cluster's loadbalancer ip/hostname for the service
+func (ingress *FederatedServiceIngress) AddEndpoints(cluster string, endpoints []string) *FederatedServiceIngress {
+	lbIngress := []v1.LoadBalancerIngress{}
+	for _, endpoint := range endpoints {
+		lbIngress = append(lbIngress, v1.LoadBalancerIngress{IP: endpoint})
+	}
+	ingress.AddClusterLoadBalancerIngresses(cluster, lbIngress)
+	return ingress
+}
+
+// RemoveEndpoint removes a single endpoint (ip/hostname) from the federated service ingress
+func (ingress *FederatedServiceIngress) RemoveEndpoint(cluster string, endpoint string) *FederatedServiceIngress {
+	for i, clusterIngress := range ingress.Items {
+		if cluster == clusterIngress.Cluster {
+			for j, lbIngress := range clusterIngress.Items {
+				if lbIngress.IP == endpoint {
+					ingress.Items[i].Items = append(ingress.Items[i].Items[:j], ingress.Items[i].Items[j+1:]...)
+				}
+			}
+		}
+	}
+	return ingress
+}
+
+// ParseFederatedServiceIngress extracts federated service ingresses from a federated service
+func ParseFederatedServiceIngress(service *v1.Service) (*FederatedServiceIngress, error) {
+	ingress := FederatedServiceIngress{}
+	if service.Annotations == nil {
+		return &ingress, nil
+	}
+	federatedServiceIngressString, found := service.Annotations[FederatedServiceIngressAnnotation]
+	if !found {
+		return &ingress, nil
+	}
+	if err := json.Unmarshal([]byte(federatedServiceIngressString), &ingress); err != nil {
+		return &ingress, err
+	}
+	return &ingress, nil
+}
+
+// UpdateIngressAnnotation updates the federated service with service ingress annotation
+func UpdateIngressAnnotation(service *v1.Service, ingress *FederatedServiceIngress) *v1.Service {
+	if service.Annotations == nil {
+		service.Annotations = make(map[string]string)
+	}
+	service.Annotations[FederatedServiceIngressAnnotation] = ingress.String()
+	return service
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/service/servicecontroller.go

@@ -0,0 +1,735 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package service
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	pkgruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	corelisters "k8s.io/client-go/listers/core/v1"
+	cache "k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/flowcontrol"
+	"k8s.io/client-go/util/workqueue"
+	fedapi "k8s.io/kubernetes/federation/apis/federation"
+	v1beta1 "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/service/ingress"
+	fedutil "k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/clusterselector"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/deletionhelper"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/eventsink"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/controller"
+)
+
+const (
+	serviceSyncPeriod = 30 * time.Second
+
+	UserAgentName = "federation-service-controller"
+
+	reviewDelay           = 10 * time.Second
+	updateTimeout         = 30 * time.Second
+	allClustersKey        = "ALL_CLUSTERS"
+	clusterAvailableDelay = time.Second * 20
+	ControllerName        = "services"
+)
+
+var (
+	RequiredResources = []schema.GroupVersionResource{v1.SchemeGroupVersion.WithResource("services")}
+)
+
+type ServiceController struct {
+	federationClient fedclientset.Interface
+	// A store of services, populated by the serviceController
+	serviceStore corelisters.ServiceLister
+	// Watches changes to all services
+	serviceController cache.Controller
+	federatedInformer fedutil.FederatedInformer
+	eventBroadcaster  record.EventBroadcaster
+	eventRecorder     record.EventRecorder
+	// services that need to be synced
+	queue *workqueue.Type
+
+	// For triggering all services reconciliation. This is used when
+	// a new cluster becomes available.
+	clusterDeliverer *fedutil.DelayingDeliverer
+
+	deletionHelper *deletionhelper.DeletionHelper
+
+	reviewDelay           time.Duration
+	clusterAvailableDelay time.Duration
+	updateTimeout         time.Duration
+
+	endpointFederatedInformer fedutil.FederatedInformer
+	federatedUpdater          fedutil.FederatedUpdater
+	objectDeliverer           *fedutil.DelayingDeliverer
+	flowcontrolBackoff        *flowcontrol.Backoff
+}
+
+// New returns a new service controller to keep service objects between
+// the federation and member clusters in sync.
+func New(federationClient fedclientset.Interface) *ServiceController {
+	broadcaster := record.NewBroadcaster()
+	broadcaster.StartRecordingToSink(eventsink.NewFederatedEventSink(federationClient))
+	recorder := broadcaster.NewRecorder(api.Scheme, v1.EventSource{Component: UserAgentName})
+
+	s := &ServiceController{
+		federationClient:      federationClient,
+		eventBroadcaster:      broadcaster,
+		eventRecorder:         recorder,
+		queue:                 workqueue.New(),
+		reviewDelay:           reviewDelay,
+		clusterAvailableDelay: clusterAvailableDelay,
+		updateTimeout:         updateTimeout,
+		flowcontrolBackoff:    flowcontrol.NewBackOff(5*time.Second, time.Minute),
+	}
+	s.objectDeliverer = fedutil.NewDelayingDeliverer()
+	s.clusterDeliverer = fedutil.NewDelayingDeliverer()
+	var serviceIndexer cache.Indexer
+	serviceIndexer, s.serviceController = cache.NewIndexerInformer(
+		&cache.ListWatch{
+			ListFunc: func(options metav1.ListOptions) (pkgruntime.Object, error) {
+				return s.federationClient.Core().Services(metav1.NamespaceAll).List(options)
+			},
+			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+				return s.federationClient.Core().Services(metav1.NamespaceAll).Watch(options)
+			},
+		},
+		&v1.Service{},
+		serviceSyncPeriod,
+		fedutil.NewTriggerOnAllChanges(func(obj pkgruntime.Object) {
+			glog.V(5).Infof("Delivering notification from federation: %v", obj)
+			s.deliverObject(obj, 0, false)
+		}),
+		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
+	)
+	s.serviceStore = corelisters.NewServiceLister(serviceIndexer)
+
+	clusterLifecycle := fedutil.ClusterLifecycleHandlerFuncs{
+		ClusterAvailable: func(cluster *v1beta1.Cluster) {
+			s.clusterDeliverer.DeliverAfter(allClustersKey, nil, clusterAvailableDelay)
+		},
+	}
+	fedInformerFactory := func(cluster *v1beta1.Cluster, targetClient kubeclientset.Interface) (cache.Store, cache.Controller) {
+		return cache.NewInformer(
+			&cache.ListWatch{
+				ListFunc: func(options metav1.ListOptions) (pkgruntime.Object, error) {
+					return targetClient.Core().Services(metav1.NamespaceAll).List(options)
+				},
+				WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+					return targetClient.Core().Services(metav1.NamespaceAll).Watch(options)
+				},
+			},
+			&v1.Service{},
+			controller.NoResyncPeriodFunc(),
+			// Trigger reconciliation whenever something in federated cluster is changed. In most cases it
+			// would be just confirmation that some service operation succeeded.
+			fedutil.NewTriggerOnAllChanges(
+				func(obj pkgruntime.Object) {
+					glog.V(5).Infof("Delivering service notification from federated cluster %s: %v", cluster.Name, obj)
+					s.deliverObject(obj, s.reviewDelay, false)
+				},
+			))
+	}
+
+	s.federatedInformer = fedutil.NewFederatedInformer(federationClient, fedInformerFactory, &clusterLifecycle)
+
+	s.federatedUpdater = fedutil.NewFederatedUpdater(s.federatedInformer, "service", updateTimeout, s.eventRecorder,
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			svc := obj.(*v1.Service)
+			_, err := client.Core().Services(svc.Namespace).Create(svc)
+			return err
+		},
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			svc := obj.(*v1.Service)
+			_, err := client.Core().Services(svc.Namespace).Update(svc)
+			return err
+		},
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			svc := obj.(*v1.Service)
+			orphanDependents := false
+			err := client.Core().Services(svc.Namespace).Delete(svc.Name, &metav1.DeleteOptions{OrphanDependents: &orphanDependents})
+			return err
+		})
+
+	// Federated informers on endpoints in federated clusters.
+	// This will enable to check if service ingress endpoints in federated clusters are reachable
+	s.endpointFederatedInformer = fedutil.NewFederatedInformer(
+		federationClient,
+		func(cluster *v1beta1.Cluster, targetClient kubeclientset.Interface) (
+			cache.Store, cache.Controller) {
+			return cache.NewInformer(
+				&cache.ListWatch{
+					ListFunc: func(options metav1.ListOptions) (pkgruntime.Object, error) {
+						return targetClient.Core().Endpoints(metav1.NamespaceAll).List(options)
+					},
+					WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+						return targetClient.Core().Endpoints(metav1.NamespaceAll).Watch(options)
+					},
+				},
+				&v1.Endpoints{},
+				controller.NoResyncPeriodFunc(),
+				fedutil.NewTriggerOnMetaAndFieldChanges(
+					"Subsets",
+					func(obj pkgruntime.Object) {
+						glog.V(5).Infof("Delivering endpoint notification from federated cluster %s :%v", cluster.Name, obj)
+						s.deliverObject(obj, s.reviewDelay, false)
+					},
+				))
+		},
+		&fedutil.ClusterLifecycleHandlerFuncs{},
+	)
+
+	s.deletionHelper = deletionhelper.NewDeletionHelper(
+		s.updateService,
+		// objNameFunc
+		func(obj pkgruntime.Object) string {
+			service := obj.(*v1.Service)
+			return fmt.Sprintf("%s/%s", service.Namespace, service.Name)
+		},
+		s.federatedInformer,
+		s.federatedUpdater,
+	)
+
+	return s
+}
+
+// Sends the given updated object to apiserver.
+// Assumes that the given object is a service.
+func (s *ServiceController) updateService(obj pkgruntime.Object) (pkgruntime.Object, error) {
+	service := obj.(*v1.Service)
+	return s.federationClient.Core().Services(service.Namespace).Update(service)
+}
+
+// Run starts informers, delay deliverers and workers. Workers continuously watch for events which could
+// be from federation or federated clusters and tries to reconcile the service objects from federation to
+// federated clusters.
+func (s *ServiceController) Run(workers int, stopCh <-chan struct{}) {
+	glog.Infof("Starting federation service controller")
+
+	defer runtime.HandleCrash()
+	defer s.queue.ShutDown()
+
+	s.federatedInformer.Start()
+	defer s.federatedInformer.Stop()
+
+	s.endpointFederatedInformer.Start()
+	defer s.endpointFederatedInformer.Stop()
+
+	s.objectDeliverer.StartWithHandler(func(item *fedutil.DelayingDelivererItem) {
+		s.queue.Add(item.Value.(string))
+	})
+	defer s.objectDeliverer.Stop()
+
+	s.clusterDeliverer.StartWithHandler(func(_ *fedutil.DelayingDelivererItem) {
+		s.deliverServicesOnClusterChange()
+	})
+	defer s.clusterDeliverer.Stop()
+
+	fedutil.StartBackoffGC(s.flowcontrolBackoff, stopCh)
+	go s.serviceController.Run(stopCh)
+
+	for i := 0; i < workers; i++ {
+		go wait.Until(s.fedServiceWorker, time.Second, stopCh)
+	}
+
+	<-stopCh
+	glog.Infof("Shutting down federation service controller")
+}
+
+type reconciliationStatus string
+
+const (
+	statusAllOk               = reconciliationStatus("ALL_OK")
+	statusRecoverableError    = reconciliationStatus("RECOVERABLE_ERROR")
+	statusNonRecoverableError = reconciliationStatus("NON_RECOVERABLE_ERROR")
+	statusNotSynced           = reconciliationStatus("NOSYNC")
+)
+
+func (s *ServiceController) workerFunction() bool {
+	key, quit := s.queue.Get()
+	if quit {
+		return true
+	}
+	defer s.queue.Done(key)
+
+	service := key.(string)
+	status := s.reconcileService(service)
+	switch status {
+	case statusAllOk:
+	// do nothing, reconcile is successful.
+	case statusNotSynced:
+		glog.V(5).Infof("Delivering notification for %q after clusterAvailableDelay", service)
+		s.deliverService(service, s.clusterAvailableDelay, false)
+	case statusRecoverableError:
+		s.deliverService(service, 0, true)
+	case statusNonRecoverableError:
+		// do nothing, error is already logged.
+	}
+	return false
+}
+
+// fedServiceWorker runs a worker thread that just dequeues items, processes them, and marks them done.
+func (s *ServiceController) fedServiceWorker() {
+	for {
+		if quit := s.workerFunction(); quit {
+			glog.Infof("service controller worker queue shutting down")
+			return
+		}
+	}
+}
+
+// delete deletes the given service or returns error if the deletion was not complete.
+func (s *ServiceController) delete(service *v1.Service) error {
+	glog.V(3).Infof("Handling deletion of service: %v", *service)
+	_, err := s.deletionHelper.HandleObjectInUnderlyingClusters(service)
+	if err != nil {
+		return err
+	}
+
+	err = s.federationClient.Core().Services(service.Namespace).Delete(service.Name, nil)
+	if err != nil {
+		// Its all good if the error is not found error. That means it is deleted already and we do not have to do anything.
+		// This is expected when we are processing an update as a result of service finalizer deletion.
+		// The process that deleted the last finalizer is also going to delete the service and we do not have to do anything.
+		if !errors.IsNotFound(err) {
+			return fmt.Errorf("failed to delete service: %v", err)
+		}
+	}
+	return nil
+}
+
+func (s *ServiceController) deliverServicesOnClusterChange() {
+	if !s.isSynced() {
+		s.clusterDeliverer.DeliverAfter(allClustersKey, nil, s.clusterAvailableDelay)
+	}
+	glog.V(5).Infof("Delivering all service as cluster status changed")
+	serviceList, err := s.serviceStore.List(labels.Everything())
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("error listing federated services: %v", err))
+		s.clusterDeliverer.DeliverAfter(allClustersKey, nil, 0)
+	}
+	for _, service := range serviceList {
+		s.deliverObject(service, 0, false)
+	}
+}
+
+func (s *ServiceController) deliverObject(object interface{}, delay time.Duration, failed bool) {
+	switch value := object.(type) {
+	case *v1.Service:
+		s.deliverService(types.NamespacedName{Namespace: value.Namespace, Name: value.Name}.String(), delay, failed)
+	case *v1.Endpoints:
+		s.deliverService(types.NamespacedName{Namespace: value.Namespace, Name: value.Name}.String(), delay, failed)
+	default:
+		glog.Warningf("Unknown object received: %v", object)
+	}
+}
+
+// Adds backoff to delay if this delivery is related to some failure. Resets backoff if there was no failure.
+func (s *ServiceController) deliverService(key string, delay time.Duration, failed bool) {
+	if failed {
+		s.flowcontrolBackoff.Next(key, time.Now())
+		delay = delay + s.flowcontrolBackoff.Get(key)
+	} else {
+		s.flowcontrolBackoff.Reset(key)
+	}
+	s.objectDeliverer.DeliverAfter(key, key, delay)
+}
+
+// Check whether all data stores are in sync. False is returned if any of the informer/stores is not yet synced with
+// the corresponding api server.
+func (s *ServiceController) isSynced() bool {
+	if !s.federatedInformer.ClustersSynced() {
+		glog.V(2).Infof("Cluster list not synced")
+		return false
+	}
+	serviceClusters, err := s.federatedInformer.GetReadyClusters()
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get ready clusters: %v", err))
+		return false
+	}
+	if !s.federatedInformer.GetTargetStore().ClustersSynced(serviceClusters) {
+		return false
+	}
+
+	if !s.endpointFederatedInformer.ClustersSynced() {
+		glog.V(2).Infof("Cluster list not synced")
+		return false
+	}
+	endpointClusters, err := s.endpointFederatedInformer.GetReadyClusters()
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get ready clusters: %v", err))
+		return false
+	}
+	if !s.endpointFederatedInformer.GetTargetStore().ClustersSynced(endpointClusters) {
+		return false
+	}
+
+	return true
+}
+
+// reconcileService triggers reconciliation of a federated service with corresponding services in federated clusters.
+// This function is called on service Addition/Deletion/Update either in federated cluster or in federation.
+func (s *ServiceController) reconcileService(key string) reconciliationStatus {
+	if !s.isSynced() {
+		glog.V(4).Infof("Data store not synced, delaying reconcilation: %v", key)
+		return statusNotSynced
+	}
+
+	namespace, name, err := cache.SplitMetaNamespaceKey(key)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Invalid key %q received, unable to split key to namespace and name, err: %v", key, err))
+		return statusNonRecoverableError
+	}
+
+	service, err := s.serviceStore.Services(namespace).Get(name)
+	if errors.IsNotFound(err) {
+		// Not a federated service, ignoring.
+		return statusAllOk
+	} else if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to retrieve federated service %q from store: %v", key, err))
+		return statusRecoverableError
+	}
+
+	glog.V(3).Infof("Reconciling federated service: %s", key)
+
+	// Create a copy before modifying the service to prevent race condition with other readers of service from store
+	fedServiceObj, err := api.Scheme.DeepCopy(service)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Error in copying obj: %s, %v", key, err))
+		return statusNonRecoverableError
+	}
+	fedService, ok := fedServiceObj.(*v1.Service)
+	if err != nil || !ok {
+		runtime.HandleError(fmt.Errorf("Unknown obj received from store: %#v, %v", fedServiceObj, err))
+		return statusNonRecoverableError
+	}
+
+	// Handle deletion of federated service
+	if fedService.DeletionTimestamp != nil {
+		if err := s.delete(fedService); err != nil {
+			runtime.HandleError(fmt.Errorf("Failed to delete %s: %v", key, err))
+			s.eventRecorder.Eventf(fedService, api.EventTypeWarning, "DeleteFailed", "Deleting service failed: %v", err)
+			return statusRecoverableError
+		}
+		glog.V(3).Infof("Deleting federated service succeeded: %s", key)
+		s.eventRecorder.Eventf(fedService, api.EventTypeNormal, "DeleteSucceed", "Deleting service succeeded")
+		return statusAllOk
+	}
+
+	// Add the required finalizers before creating a service in underlying clusters. This ensures that the
+	// dependent services in underlying clusters are deleted when the federated service is deleted.
+	updatedServiceObj, err := s.deletionHelper.EnsureFinalizers(fedService)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to ensure setting finalizer for service %s: %v", key, err))
+		return statusRecoverableError
+	}
+	fedService = updatedServiceObj.(*v1.Service)
+
+	// Synchronize the federated service in all underlying ready clusters.
+	clusters, err := s.federatedInformer.GetReadyClusters()
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get ready cluster list: %v", err))
+		return statusRecoverableError
+	}
+
+	newLBStatus := newLoadbalancerStatus()
+	newServiceIngress := ingress.NewFederatedServiceIngress()
+	operations := make([]fedutil.FederatedOperation, 0)
+	for _, cluster := range clusters {
+		// Aggregate all operations to perform on all federated clusters
+		operation, err := getOperationsToPerformOnCluster(s.federatedInformer, cluster, fedService, clusterselector.SendToCluster)
+		if err != nil {
+			return statusRecoverableError
+		}
+		if operation != nil {
+			operations = append(operations, *operation)
+		}
+
+		// Aggregate LoadBalancerStatus from all services in federated clusters to update status in federated service
+		lbStatus, err := s.getServiceStatusInCluster(cluster, key)
+		if err != nil {
+			return statusRecoverableError
+		}
+		if len(lbStatus.Ingress) > 0 {
+			newLBStatus.Ingress = append(newLBStatus.Ingress, lbStatus.Ingress...)
+
+			// Add/Update federated service ingress only if there are reachable endpoints backing the lb service
+			endpoints, err := s.getServiceEndpointsInCluster(cluster, key)
+			if err != nil {
+				return statusRecoverableError
+			}
+			// if there are no endpoints created for the service then the loadbalancer ingress
+			// is not reachable, so do not consider such loadbalancer ingresses for federated
+			// service ingresses
+			if len(endpoints) > 0 {
+				clusterIngress := fedapi.ClusterServiceIngress{
+					Cluster: cluster.Name,
+					Items:   lbStatus.Ingress,
+				}
+				newServiceIngress.Items = append(newServiceIngress.Items, clusterIngress)
+			}
+		}
+	}
+
+	if len(operations) != 0 {
+		err = s.federatedUpdater.Update(operations)
+		if err != nil {
+			if !errors.IsAlreadyExists(err) {
+				runtime.HandleError(fmt.Errorf("Failed to execute updates for %s: %v", key, err))
+				return statusRecoverableError
+			}
+		}
+	}
+
+	// Update the federated service if there are any updates in clustered service (status/endpoints)
+	err = s.updateFederatedService(fedService, newLBStatus, newServiceIngress)
+	if err != nil {
+		return statusRecoverableError
+	}
+
+	glog.V(5).Infof("Everything is in order in federated clusters for service %s", key)
+	return statusAllOk
+}
+
+type clusterSelectorFunc func(map[string]string, map[string]string) (bool, error)
+
+// getOperationsToPerformOnCluster returns the operations to be performed so that clustered service is in sync with federated service
+func getOperationsToPerformOnCluster(informer fedutil.FederatedInformer, cluster *v1beta1.Cluster, fedService *v1.Service, selector clusterSelectorFunc) (*fedutil.FederatedOperation, error) {
+	var operation *fedutil.FederatedOperation
+	var operationType fedutil.FederatedOperationType = ""
+
+	key := types.NamespacedName{Namespace: fedService.Namespace, Name: fedService.Name}.String()
+	clusterServiceObj, found, err := informer.GetTargetStore().GetByKey(cluster.Name, key)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get %s service from %s: %v", key, cluster.Name, err))
+		return nil, err
+	}
+
+	send, err := selector(cluster.Labels, fedService.ObjectMeta.Annotations)
+	if err != nil {
+		glog.Errorf("Error processing ClusterSelector cluster: %s for service map: %s error: %s", cluster.Name, key, err.Error())
+		return nil, err
+	} else if !send {
+		glog.V(5).Infof("Skipping cluster: %s for service: %s reason: cluster selectors do not match: %-v %-v", cluster.Name, key, cluster.ObjectMeta.Labels, fedService.ObjectMeta.Annotations[v1beta1.FederationClusterSelectorAnnotation])
+	}
+
+	desiredService := &v1.Service{
+		ObjectMeta: fedutil.DeepCopyRelevantObjectMeta(fedService.ObjectMeta),
+		Spec:       *(fedutil.DeepCopyApiTypeOrPanic(&fedService.Spec).(*v1.ServiceSpec)),
+	}
+	switch {
+	case found && send:
+		clusterService, ok := clusterServiceObj.(*v1.Service)
+		if !ok {
+			runtime.HandleError(fmt.Errorf("Unexpected error for %q: %v", key, err))
+			return nil, err
+		}
+
+		// ClusterIP and NodePort are allocated to Service by cluster, so retain the same if any while updating
+		desiredService.Spec.ClusterIP = clusterService.Spec.ClusterIP
+		for _, cPort := range clusterService.Spec.Ports {
+			for i, fPort := range clusterService.Spec.Ports {
+				if fPort.Name == cPort.Name && fPort.Protocol == cPort.Protocol && fPort.Port == cPort.Port {
+					desiredService.Spec.Ports[i].NodePort = cPort.NodePort
+				}
+			}
+		}
+
+		// Update existing service, if needed.
+		if !Equivalent(desiredService, clusterService) {
+			operationType = fedutil.OperationTypeUpdate
+
+			glog.V(4).Infof("Service in underlying cluster %s does not match, Desired: %+v, Existing: %+v", cluster.Name, desiredService, clusterService)
+
+			// ResourceVersion of cluster service can be different from federated service,
+			// so do not update ResourceVersion while updating cluster service
+			desiredService.ResourceVersion = clusterService.ResourceVersion
+		} else {
+			glog.V(5).Infof("Service in underlying cluster %s is up to date: %+v", cluster.Name, desiredService)
+		}
+	case found && !send:
+		operationType = fedutil.OperationTypeDelete
+	case !found && send:
+		operationType = fedutil.OperationTypeAdd
+		desiredService.ResourceVersion = ""
+
+		glog.V(4).Infof("Creating service in underlying cluster %s: %+v", cluster.Name, desiredService)
+	}
+
+	if len(operationType) > 0 {
+		operation = &fedutil.FederatedOperation{
+			Type:        operationType,
+			Obj:         desiredService,
+			ClusterName: cluster.Name,
+			Key:         key,
+		}
+	}
+	return operation, nil
+}
+
+// getServiceStatusInCluster returns service status in federated cluster
+func (s *ServiceController) getServiceStatusInCluster(cluster *v1beta1.Cluster, key string) (*v1.LoadBalancerStatus, error) {
+	lbStatus := &v1.LoadBalancerStatus{}
+
+	clusterServiceObj, serviceFound, err := s.federatedInformer.GetTargetStore().GetByKey(cluster.Name, key)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get %s service from %s: %v", key, cluster.Name, err))
+		return lbStatus, err
+	}
+	if serviceFound {
+		clusterService, ok := clusterServiceObj.(*v1.Service)
+		if !ok {
+			err = fmt.Errorf("Unknown object received: %v", clusterServiceObj)
+			runtime.HandleError(err)
+			return lbStatus, err
+		}
+		lbStatus = &clusterService.Status.LoadBalancer
+		newLbStatus := &loadbalancerStatus{*lbStatus}
+		sort.Sort(newLbStatus)
+	}
+	return lbStatus, nil
+}
+
+// getServiceEndpointsInCluster returns ready endpoints corresponding to service in federated cluster
+func (s *ServiceController) getServiceEndpointsInCluster(cluster *v1beta1.Cluster, key string) ([]v1.EndpointAddress, error) {
+	addresses := []v1.EndpointAddress{}
+
+	clusterEndpointsObj, endpointsFound, err := s.endpointFederatedInformer.GetTargetStore().GetByKey(cluster.Name, key)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get %s endpoint from %s: %v", key, cluster.Name, err))
+		return addresses, err
+	}
+	if endpointsFound {
+		clusterEndpoints, ok := clusterEndpointsObj.(*v1.Endpoints)
+		if !ok {
+			glog.Warningf("Unknown object received: %v", clusterEndpointsObj)
+			return addresses, fmt.Errorf("Unknown object received: %v", clusterEndpointsObj)
+		}
+		for _, subset := range clusterEndpoints.Subsets {
+			if len(subset.Addresses) > 0 {
+				addresses = append(addresses, subset.Addresses...)
+			}
+		}
+	}
+	return addresses, nil
+}
+
+// updateFederatedService updates the federated service with aggregated lbStatus and serviceIngresses
+// and also updates the dns records as needed
+func (s *ServiceController) updateFederatedService(fedService *v1.Service, newLBStatus *loadbalancerStatus, newServiceIngress *ingress.FederatedServiceIngress) error {
+	key := types.NamespacedName{Namespace: fedService.Namespace, Name: fedService.Name}.String()
+	needUpdate := false
+
+	// Sort the endpoints so that we can compare
+	sort.Sort(newLBStatus)
+	if !reflect.DeepEqual(fedService.Status.LoadBalancer.Ingress, newLBStatus.Ingress) {
+		fedService.Status.LoadBalancer.Ingress = newLBStatus.Ingress
+		glog.V(3).Infof("Federated service loadbalancer status updated for %s: %v", key, newLBStatus.Ingress)
+		needUpdate = true
+	}
+
+	existingServiceIngress, err := ingress.ParseFederatedServiceIngress(fedService)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to parse endpoint annotations for service %s: %v", key, err))
+		return err
+	}
+
+	// TODO: We should have a reliable cluster health check(should consider quorum) to detect cluster is not
+	// reachable and remove dns records for them. Until a reliable cluster health check is available, below code is
+	// a workaround to not remove the existing dns records which were created before the cluster went offline.
+	unreadyClusters, err := s.federatedInformer.GetUnreadyClusters()
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get unready cluster list: %v", err))
+		return err
+	}
+	for _, cluster := range unreadyClusters {
+		lbIngress := existingServiceIngress.GetClusterLoadBalancerIngresses(cluster.Name)
+		newServiceIngress.AddClusterLoadBalancerIngresses(cluster.Name, lbIngress)
+		glog.V(5).Infof("Cluster %s is Offline, Preserving previously available status for Service %s", cluster.Name, key)
+	}
+
+	// Update federated service status and/or ingress annotations if changed
+	sort.Sort(newServiceIngress)
+	if !reflect.DeepEqual(existingServiceIngress.Items, newServiceIngress.Items) {
+		fedService = ingress.UpdateIngressAnnotation(fedService, newServiceIngress)
+		glog.V(3).Infof("Federated service loadbalancer ingress updated for %s: existing: %#v, desired: %#v", key, existingServiceIngress, newServiceIngress)
+		needUpdate = true
+	}
+
+	if needUpdate {
+		var err error
+		fedService, err = s.federationClient.Core().Services(fedService.Namespace).UpdateStatus(fedService)
+		if err != nil {
+			runtime.HandleError(fmt.Errorf("Error updating the federation service object %s: %v", key, err))
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Equivalent Checks if cluster-independent, user provided data in two given services are equal. If in the future the
+// services structure is expanded then any field that is not populated by the api server should be included here.
+func Equivalent(s1, s2 *v1.Service) bool {
+	// TODO: should also check for all annotations except FederationServiceIngressAnnotation
+	return s1.Name == s2.Name && s1.Namespace == s2.Namespace &&
+		(reflect.DeepEqual(s1.Labels, s2.Labels) || (len(s1.Labels) == 0 && len(s2.Labels) == 0)) &&
+		reflect.DeepEqual(s1.Spec, s2.Spec)
+}
+
+type loadbalancerStatus struct {
+	v1.LoadBalancerStatus
+}
+
+func newLoadbalancerStatus() *loadbalancerStatus {
+	return &loadbalancerStatus{}
+}
+
+func (lbs loadbalancerStatus) Len() int {
+	return len(lbs.Ingress)
+}
+
+func (lbs loadbalancerStatus) Less(i, j int) bool {
+	ipComparison := strings.Compare(lbs.Ingress[i].IP, lbs.Ingress[j].IP)
+	hostnameComparison := strings.Compare(lbs.Ingress[i].Hostname, lbs.Ingress[j].Hostname)
+	if ipComparison < 0 || (ipComparison == 0 && hostnameComparison < 0) {
+		return true
+	}
+	return false
+}
+
+func (lbs loadbalancerStatus) Swap(i, j int) {
+	lbs.Ingress[i].IP, lbs.Ingress[j].IP = lbs.Ingress[j].IP, lbs.Ingress[i].IP
+	lbs.Ingress[i].Hostname, lbs.Ingress[j].Hostname = lbs.Ingress[j].Hostname, lbs.Ingress[i].Hostname
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/service/servicecontroller_test.go

@@ -0,0 +1,383 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package service
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/stretchr/testify/require"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	fakekubeclientset "k8s.io/client-go/kubernetes/fake"
+	corelisters "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	fakefedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset/fake"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/service/ingress"
+	fedutil "k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/deletionhelper"
+	. "k8s.io/kubernetes/federation/pkg/federation-controller/util/test"
+)
+
+const (
+	retryInterval = 100 * time.Millisecond
+
+	clusters  string = "clusters"
+	services  string = "services"
+	endpoints string = "endpoints"
+
+	lbIngress1       = "10.20.30.40"
+	lbIngress2       = "10.20.30.50"
+	serviceEndpoint1 = "192.168.0.1"
+	serviceEndpoint2 = "192.168.1.1"
+)
+
+var awfulError error = errors.NewGone("Something bad happened")
+
+func TestServiceController(t *testing.T) {
+	glog.Infof("Creating fake infrastructure")
+	fedClient := &fakefedclientset.Clientset{}
+	cluster1 := NewCluster("cluster1", v1.ConditionTrue)
+	cluster2 := NewCluster("cluster2", v1.ConditionTrue)
+
+	RegisterFakeClusterGet(&fedClient.Fake, &v1beta1.ClusterList{Items: []v1beta1.Cluster{*cluster1, *cluster2}})
+	RegisterFakeList(clusters, &fedClient.Fake, &v1beta1.ClusterList{Items: []v1beta1.Cluster{*cluster1, *cluster2}})
+	fedclusterWatch := RegisterFakeWatch(clusters, &fedClient.Fake)
+	RegisterFakeList(services, &fedClient.Fake, &v1.ServiceList{Items: []v1.Service{}})
+	fedServiceWatch := RegisterFakeWatch(services, &fedClient.Fake)
+	RegisterFakeOnCreate(clusters, &fedClient.Fake, fedclusterWatch)
+	RegisterFakeOnUpdate(clusters, &fedClient.Fake, fedclusterWatch)
+	RegisterFakeOnCreate(services, &fedClient.Fake, fedServiceWatch)
+	RegisterFakeOnUpdate(services, &fedClient.Fake, fedServiceWatch)
+	RegisterFakeOnDelete(services, &fedClient.Fake, fedServiceWatch, serviceObjectGetter)
+
+	cluster1Client := &fakekubeclientset.Clientset{}
+	RegisterFakeList(services, &cluster1Client.Fake, &v1.ServiceList{Items: []v1.Service{}})
+	c1ServiceWatch := RegisterFakeWatch(services, &cluster1Client.Fake)
+	RegisterFakeList(endpoints, &cluster1Client.Fake, &v1.EndpointsList{Items: []v1.Endpoints{}})
+	c1EndpointWatch := RegisterFakeWatch(endpoints, &cluster1Client.Fake)
+	RegisterFakeOnCreate(services, &cluster1Client.Fake, c1ServiceWatch)
+	RegisterFakeOnUpdate(services, &cluster1Client.Fake, c1ServiceWatch)
+	RegisterFakeOnDelete(services, &cluster1Client.Fake, c1ServiceWatch, serviceObjectGetter)
+	RegisterFakeOnCreate(endpoints, &cluster1Client.Fake, c1EndpointWatch)
+	RegisterFakeOnUpdate(endpoints, &cluster1Client.Fake, c1EndpointWatch)
+
+	cluster2Client := &fakekubeclientset.Clientset{}
+	RegisterFakeList(services, &cluster2Client.Fake, &v1.ServiceList{Items: []v1.Service{}})
+	c2ServiceWatch := RegisterFakeWatch(services, &cluster2Client.Fake)
+	RegisterFakeList(endpoints, &cluster2Client.Fake, &v1.EndpointsList{Items: []v1.Endpoints{}})
+	c2EndpointWatch := RegisterFakeWatch(endpoints, &cluster2Client.Fake)
+	RegisterFakeOnCreate(services, &cluster2Client.Fake, c2ServiceWatch)
+	RegisterFakeOnUpdate(services, &cluster2Client.Fake, c2ServiceWatch)
+	RegisterFakeOnDelete(services, &cluster2Client.Fake, c2ServiceWatch, serviceObjectGetter)
+	RegisterFakeOnCreate(endpoints, &cluster2Client.Fake, c2EndpointWatch)
+	RegisterFakeOnUpdate(endpoints, &cluster2Client.Fake, c2EndpointWatch)
+
+	fedInformerClientFactory := func(cluster *v1beta1.Cluster) (kubeclientset.Interface, error) {
+		switch cluster.Name {
+		case cluster1.Name:
+			return cluster1Client, nil
+		case cluster2.Name:
+			return cluster2Client, nil
+		default:
+			return nil, fmt.Errorf("Unknown cluster: %v", cluster.Name)
+		}
+	}
+
+	sc := New(fedClient)
+	ToFederatedInformerForTestOnly(sc.federatedInformer).SetClientFactory(fedInformerClientFactory)
+	ToFederatedInformerForTestOnly(sc.endpointFederatedInformer).SetClientFactory(fedInformerClientFactory)
+	sc.clusterAvailableDelay = 100 * time.Millisecond
+	sc.reviewDelay = 50 * time.Millisecond
+	sc.updateTimeout = 5 * time.Second
+
+	stop := make(chan struct{})
+	glog.Infof("Running Service Controller")
+	go sc.Run(5, stop)
+
+	glog.Infof("Adding cluster 1")
+	fedclusterWatch.Add(cluster1)
+
+	service := NewService("test-service-1", 80)
+
+	glog.Infof("Adding federated service")
+	fedServiceWatch.Add(service)
+	key := types.NamespacedName{Namespace: service.Namespace, Name: service.Name}.String()
+
+	glog.Infof("Test service was correctly created in cluster 1")
+	require.NoError(t, WaitForClusterService(t, sc.federatedInformer.GetTargetStore(), cluster1.Name,
+		key, service, wait.ForeverTestTimeout))
+
+	glog.Infof("Adding cluster 2")
+	fedclusterWatch.Add(cluster2)
+
+	glog.Infof("Test service was correctly created in cluster 2")
+	require.NoError(t, WaitForClusterService(t, sc.federatedInformer.GetTargetStore(), cluster2.Name,
+		key, service, wait.ForeverTestTimeout))
+
+	glog.Infof("Test federation service is updated when cluster1 service status is updated")
+	service.Status = v1.ServiceStatus{
+		LoadBalancer: v1.LoadBalancerStatus{
+			Ingress: []v1.LoadBalancerIngress{
+				{IP: lbIngress1},
+			}}}
+
+	desiredStatus := service.Status
+	desiredService := &v1.Service{Status: desiredStatus}
+
+	c1ServiceWatch.Modify(service)
+	require.NoError(t, WaitForClusterService(t, sc.federatedInformer.GetTargetStore(), cluster1.Name,
+		key, service, wait.ForeverTestTimeout))
+	require.NoError(t, WaitForFederatedServiceUpdate(t, sc.serviceStore,
+		key, desiredService, serviceStatusCompare, wait.ForeverTestTimeout))
+
+	glog.Infof("Test federation service is updated when cluster1 endpoint for the service is created")
+	desiredIngressAnnotation := ingress.NewFederatedServiceIngress().
+		AddEndpoints("cluster1", []string{lbIngress1}).
+		String()
+	desiredService = &v1.Service{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ingress.FederatedServiceIngressAnnotation: desiredIngressAnnotation}}}
+	c1EndpointWatch.Add(NewEndpoint("test-service-1", serviceEndpoint1))
+	require.NoError(t, WaitForFederatedServiceUpdate(t, sc.serviceStore,
+		key, desiredService, serviceIngressCompare, wait.ForeverTestTimeout))
+
+	glog.Infof("Test federation service is updated when cluster2 service status is updated")
+	service.Status = v1.ServiceStatus{
+		LoadBalancer: v1.LoadBalancerStatus{
+			Ingress: []v1.LoadBalancerIngress{
+				{IP: lbIngress2},
+			}}}
+	desiredStatus.LoadBalancer.Ingress = append(desiredStatus.LoadBalancer.Ingress, v1.LoadBalancerIngress{IP: lbIngress2})
+	desiredService = &v1.Service{Status: desiredStatus}
+
+	c2ServiceWatch.Modify(service)
+	require.NoError(t, WaitForClusterService(t, sc.federatedInformer.GetTargetStore(), cluster2.Name,
+		key, service, wait.ForeverTestTimeout))
+	require.NoError(t, WaitForFederatedServiceUpdate(t, sc.serviceStore,
+		key, desiredService, serviceStatusCompare, wait.ForeverTestTimeout))
+
+	glog.Infof("Test federation service is updated when cluster2 endpoint for the service is created")
+	desiredIngressAnnotation = ingress.NewFederatedServiceIngress().
+		AddEndpoints("cluster1", []string{lbIngress1}).
+		AddEndpoints("cluster2", []string{lbIngress2}).
+		String()
+	desiredService = &v1.Service{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ingress.FederatedServiceIngressAnnotation: desiredIngressAnnotation}}}
+	c2EndpointWatch.Add(NewEndpoint("test-service-1", serviceEndpoint2))
+	require.NoError(t, WaitForFederatedServiceUpdate(t, sc.serviceStore,
+		key, desiredService, serviceIngressCompare, wait.ForeverTestTimeout))
+
+	glog.Infof("Test federation service is updated when cluster1 endpoint for the service is deleted")
+	desiredIngressAnnotation = ingress.NewFederatedServiceIngress().
+		AddEndpoints("cluster2", []string{lbIngress2}).
+		String()
+	desiredService = &v1.Service{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ingress.FederatedServiceIngressAnnotation: desiredIngressAnnotation}}}
+	c1EndpointWatch.Delete(NewEndpoint("test-service-1", serviceEndpoint1))
+	require.NoError(t, WaitForFederatedServiceUpdate(t, sc.serviceStore,
+		key, desiredService, serviceIngressCompare, wait.ForeverTestTimeout))
+
+	glog.Infof("Test modifying federated service by changing the port")
+	service.Spec.Ports[0].Port = 9090
+	fedServiceWatch.Modify(service)
+	require.NoError(t, WaitForClusterService(t, sc.federatedInformer.GetTargetStore(), cluster1.Name,
+		key, service, wait.ForeverTestTimeout))
+
+	glog.Infof("Test cluster service is recreated when deleted")
+	c1Service := NewService("test-service-1", 80)
+	c1Service.DeletionTimestamp = &metav1.Time{Time: time.Now()}
+	c1ServiceWatch.Delete(c1Service)
+	require.NoError(t, WaitForClusterService(t, sc.federatedInformer.GetTargetStore(), cluster1.Name,
+		key, service, wait.ForeverTestTimeout))
+
+	glog.Infof("Test cluster services are deleted when federated service is deleted")
+	service.ObjectMeta.Finalizers = append(service.ObjectMeta.Finalizers, deletionhelper.FinalizerDeleteFromUnderlyingClusters)
+	service.DeletionTimestamp = &metav1.Time{Time: time.Now()}
+	fedServiceWatch.Modify(service)
+	require.NoError(t, WaitForClusterServiceDelete(t, sc.federatedInformer.GetTargetStore(), cluster1.Name,
+		key, wait.ForeverTestTimeout))
+	require.NoError(t, WaitForClusterServiceDelete(t, sc.federatedInformer.GetTargetStore(), cluster2.Name,
+		key, wait.ForeverTestTimeout))
+
+	close(stop)
+}
+
+func TestGetOperationsToPerformOnCluster(t *testing.T) {
+	obj := NewService("test-service-1", 80)
+	cluster1 := NewCluster("cluster1", v1.ConditionTrue)
+	fedClient := &fakefedclientset.Clientset{}
+	sc := New(fedClient)
+
+	testCases := map[string]struct {
+		expectedSendErr bool
+		sendToCluster   bool
+		operationType   fedutil.FederatedOperationType
+	}{
+		"sendToCluster error returned": {
+			expectedSendErr: true,
+		},
+		"Missing object and not matching ClusterSelector should result in no operations": {
+			sendToCluster: false,
+		},
+		"Missing object and matching ClusterSelector should result in add operation": {
+			operationType: fedutil.OperationTypeAdd,
+			sendToCluster: true,
+		},
+		// Update and Delete scenarios are tested in TestServiceController
+	}
+	for testName, testCase := range testCases {
+		t.Run(testName, func(t *testing.T) {
+
+			operations, err := getOperationsToPerformOnCluster(sc.federatedInformer, cluster1, obj, func(map[string]string, map[string]string) (bool, error) {
+				if testCase.expectedSendErr {
+					return false, awfulError
+				}
+				return testCase.sendToCluster, nil
+			})
+			if testCase.expectedSendErr {
+				require.Error(t, err, "An error was expected")
+			} else {
+				require.NoError(t, err, "An error was not expected")
+			}
+			if len(testCase.operationType) == 0 {
+				require.Nil(t, operations, "An operation was not expected")
+			} else {
+				require.NotNil(t, operations, "A single operation was expected")
+				require.Equal(t, testCase.operationType, operations.Type, "Unexpected operation returned")
+			}
+		})
+	}
+}
+
+// serviceObjectGetter gives dummy service objects to use with RegisterFakeOnDelete
+// This is just so that federated informer can be tested for delete scenarios.
+func serviceObjectGetter(name, namespace string) runtime.Object {
+	service := new(v1.Service)
+	service.Namespace = namespace
+	service.Name = name
+	return service
+}
+
+func NewService(name string, port int32) *v1.Service {
+	return &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: v1.NamespaceDefault,
+			SelfLink:  "/api/v1/namespaces/default/services/" + name,
+			Labels:    map[string]string{"app": name},
+		},
+		Spec: v1.ServiceSpec{
+			Ports: []v1.ServicePort{{Port: port}},
+			Type:  v1.ServiceTypeLoadBalancer,
+		},
+	}
+}
+
+func NewEndpoint(name, ip string) *v1.Endpoints {
+	return &v1.Endpoints{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: v1.NamespaceDefault,
+			SelfLink:  "/api/v1/namespaces/default/endpoints/" + name,
+			Labels:    map[string]string{"app": name},
+		},
+		Subsets: []v1.EndpointSubset{{
+			Addresses: []v1.EndpointAddress{{
+				IP: ip,
+			}}},
+		},
+	}
+}
+
+// WaitForClusterService waits for the cluster service to be created matching the desiredService.
+func WaitForClusterService(t *testing.T, store fedutil.FederatedReadOnlyStore, clusterName, key string, desiredService *v1.Service, timeout time.Duration) error {
+	err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) {
+		obj, found, err := store.GetByKey(clusterName, key)
+		if !found || err != nil {
+			return false, err
+		}
+		service := obj.(*v1.Service)
+		if !Equivalent(service, desiredService) {
+			glog.V(5).Infof("Waiting for clustered service, Desired: %v, Current: %v", desiredService, service)
+			return false, nil
+		}
+		glog.V(5).Infof("Clustered service is up to date: %v", service)
+		return true, nil
+	})
+	return err
+}
+
+// WaitForClusterServiceDelete waits for the cluster service to be deleted.
+func WaitForClusterServiceDelete(t *testing.T, store fedutil.FederatedReadOnlyStore, clusterName, key string, timeout time.Duration) error {
+	err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) {
+		_, found, _ := store.GetByKey(clusterName, key)
+		if !found {
+			return true, nil
+		}
+		return false, nil
+	})
+	return err
+}
+
+type serviceCompare func(current, desired *v1.Service) (match bool)
+
+func serviceStatusCompare(current, desired *v1.Service) bool {
+	if !reflect.DeepEqual(current.Status.LoadBalancer, desired.Status.LoadBalancer) {
+		glog.V(5).Infof("Waiting for loadbalancer status, Current: %v, Desired: %v", current.Status.LoadBalancer, desired.Status.LoadBalancer)
+		return false
+	}
+	glog.V(5).Infof("Loadbalancer status match: %v", current.Status.LoadBalancer)
+	return true
+}
+
+func serviceIngressCompare(current, desired *v1.Service) bool {
+	if strings.Compare(current.Annotations[ingress.FederatedServiceIngressAnnotation], desired.Annotations[ingress.FederatedServiceIngressAnnotation]) != 0 {
+		glog.V(5).Infof("Waiting for loadbalancer ingress, Current: %v, Desired: %v", current.Annotations[ingress.FederatedServiceIngressAnnotation], desired.Annotations[ingress.FederatedServiceIngressAnnotation])
+		return false
+	}
+	glog.V(5).Infof("Loadbalancer ingress match: %v", current.Annotations[ingress.FederatedServiceIngressAnnotation])
+	return true
+}
+
+// WaitForFederatedServiceUpdate waits for federated service updates to match the desiredService.
+func WaitForFederatedServiceUpdate(t *testing.T, store corelisters.ServiceLister, key string, desiredService *v1.Service, match serviceCompare, timeout time.Duration) error {
+	err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) {
+		namespace, name, err := cache.SplitMetaNamespaceKey(key)
+		if err != nil {
+			return false, err
+		}
+		service, err := store.Services(namespace).Get(name)
+		switch {
+		case errors.IsNotFound(err):
+			return false, nil
+		case err != nil:
+			return false, err
+		case !match(service, desiredService):
+			return false, nil
+		default:
+			return true, nil
+		}
+	})
+	return err
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/sync/BUILD

@@ -0,0 +1,66 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["controller_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/pkg/federatedtypes:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//federation/pkg/federation-controller/util/test:go_default_library",
+        "//vendor/github.com/stretchr/testify/require:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["controller.go"],
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset:go_default_library",
+        "//federation/pkg/federatedtypes:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//federation/pkg/federation-controller/util/clusterselector:go_default_library",
+        "//federation/pkg/federation-controller/util/deletionhelper:go_default_library",
+        "//federation/pkg/federation-controller/util/eventsink:go_default_library",
+        "//pkg/api:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/rest:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+        "//vendor/k8s.io/client-go/tools/record:go_default_library",
+        "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
+        "//vendor/k8s.io/client-go/util/workqueue:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/sync/controller.go

@@ -0,0 +1,610 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sync
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pkgruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/flowcontrol"
+	"k8s.io/client-go/util/workqueue"
+	federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	federationclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	"k8s.io/kubernetes/federation/pkg/federatedtypes"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/clusterselector"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/deletionhelper"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/eventsink"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/controller"
+
+	"github.com/golang/glog"
+)
+
+const (
+	allClustersKey = "ALL_CLUSTERS"
+)
+
+// FederationSyncController synchronizes the state of a federated type
+// to clusters that are members of the federation.
+type FederationSyncController struct {
+	// For triggering reconciliation of a single resource. This is
+	// used when there is an add/update/delete operation on a resource
+	// in either federated API server or in some member of the
+	// federation.
+	deliverer *util.DelayingDeliverer
+
+	// For triggering reconciliation of all target resources. This is
+	// used when a new cluster becomes available.
+	clusterDeliverer *util.DelayingDeliverer
+
+	// Contains resources present in members of federation.
+	informer util.FederatedInformer
+	// For updating members of federation.
+	updater util.FederatedUpdater
+	// Definitions of resources that should be federated.
+	store cache.Store
+	// Informer controller for resources that should be federated.
+	controller cache.Controller
+
+	// Work queue allowing parallel processing of resources
+	workQueue workqueue.Interface
+
+	// Backoff manager
+	backoff *flowcontrol.Backoff
+
+	// For events
+	eventRecorder record.EventRecorder
+
+	deletionHelper *deletionhelper.DeletionHelper
+
+	reviewDelay             time.Duration
+	clusterAvailableDelay   time.Duration
+	clusterUnavailableDelay time.Duration
+	smallDelay              time.Duration
+	updateTimeout           time.Duration
+
+	adapter federatedtypes.FederatedTypeAdapter
+}
+
+// StartFederationSyncController starts a new sync controller for a type adapter
+func StartFederationSyncController(kind string, adapterFactory federatedtypes.AdapterFactory, config *restclient.Config, stopChan <-chan struct{}, minimizeLatency bool, adapterSpecificArgs map[string]interface{}) {
+	restclient.AddUserAgent(config, fmt.Sprintf("federation-%s-controller", kind))
+	client := federationclientset.NewForConfigOrDie(config)
+	adapter := adapterFactory(client, config, adapterSpecificArgs)
+	controller := newFederationSyncController(client, adapter)
+	if minimizeLatency {
+		controller.minimizeLatency()
+	}
+	glog.Infof(fmt.Sprintf("Starting federated sync controller for %s resources", kind))
+	controller.Run(stopChan)
+}
+
+// newFederationSyncController returns a new sync controller for the given client and type adapter
+func newFederationSyncController(client federationclientset.Interface, adapter federatedtypes.FederatedTypeAdapter) *FederationSyncController {
+	broadcaster := record.NewBroadcaster()
+	broadcaster.StartRecordingToSink(eventsink.NewFederatedEventSink(client))
+	recorder := broadcaster.NewRecorder(api.Scheme, v1.EventSource{Component: fmt.Sprintf("federation-%v-controller", adapter.Kind())})
+
+	s := &FederationSyncController{
+		reviewDelay:             time.Second * 10,
+		clusterAvailableDelay:   time.Second * 20,
+		clusterUnavailableDelay: time.Second * 60,
+		smallDelay:              time.Second * 3,
+		updateTimeout:           time.Second * 30,
+		workQueue:               workqueue.New(),
+		backoff:                 flowcontrol.NewBackOff(5*time.Second, time.Minute),
+		eventRecorder:           recorder,
+		adapter:                 adapter,
+	}
+
+	// Build delivereres for triggering reconciliations.
+	s.deliverer = util.NewDelayingDeliverer()
+	s.clusterDeliverer = util.NewDelayingDeliverer()
+
+	// Start informer in federated API servers on the resource type that should be federated.
+	s.store, s.controller = cache.NewInformer(
+		&cache.ListWatch{
+			ListFunc: func(options metav1.ListOptions) (pkgruntime.Object, error) {
+				return adapter.FedList(metav1.NamespaceAll, options)
+			},
+			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+				return adapter.FedWatch(metav1.NamespaceAll, options)
+			},
+		},
+		adapter.ObjectType(),
+		controller.NoResyncPeriodFunc(),
+		util.NewTriggerOnAllChanges(func(obj pkgruntime.Object) { s.deliverObj(obj, 0, false) }))
+
+	// Federated informer on the resource type in members of federation.
+	s.informer = util.NewFederatedInformer(
+		client,
+		func(cluster *federationapi.Cluster, targetClient kubeclientset.Interface) (cache.Store, cache.Controller) {
+			return cache.NewInformer(
+				&cache.ListWatch{
+					ListFunc: func(options metav1.ListOptions) (pkgruntime.Object, error) {
+						return adapter.ClusterList(targetClient, metav1.NamespaceAll, options)
+					},
+					WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+						return adapter.ClusterWatch(targetClient, metav1.NamespaceAll, options)
+					},
+				},
+				adapter.ObjectType(),
+				controller.NoResyncPeriodFunc(),
+				// Trigger reconciliation whenever something in federated cluster is changed. In most cases it
+				// would be just confirmation that some operation on the target resource type had succeeded.
+				util.NewTriggerOnAllChanges(
+					func(obj pkgruntime.Object) {
+						s.deliverObj(obj, s.reviewDelay, false)
+					},
+				))
+		},
+
+		&util.ClusterLifecycleHandlerFuncs{
+			ClusterAvailable: func(cluster *federationapi.Cluster) {
+				// When new cluster becomes available process all the target resources again.
+				s.clusterDeliverer.DeliverAt(allClustersKey, nil, time.Now().Add(s.clusterAvailableDelay))
+			},
+			// When a cluster becomes unavailable process all the target resources again.
+			ClusterUnavailable: func(cluster *federationapi.Cluster, _ []interface{}) {
+				s.clusterDeliverer.DeliverAt(allClustersKey, nil, time.Now().Add(s.clusterUnavailableDelay))
+			},
+		},
+	)
+
+	// Federated updeater along with Create/Update/Delete operations.
+	s.updater = util.NewFederatedUpdater(s.informer, adapter.Kind(), s.updateTimeout, s.eventRecorder,
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			_, err := adapter.ClusterCreate(client, obj)
+			return err
+		},
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			_, err := adapter.ClusterUpdate(client, obj)
+			return err
+		},
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			qualifiedName := adapter.QualifiedName(obj)
+			orphanDependents := false
+			err := adapter.ClusterDelete(client, qualifiedName, &metav1.DeleteOptions{OrphanDependents: &orphanDependents})
+			return err
+		})
+
+	s.deletionHelper = deletionhelper.NewDeletionHelper(
+		s.updateObject,
+		// objNameFunc
+		func(obj pkgruntime.Object) string {
+			return adapter.QualifiedName(obj).String()
+		},
+		s.informer,
+		s.updater,
+	)
+
+	return s
+}
+
+// minimizeLatency reduces delays and timeouts to make the controller more responsive (useful for testing).
+func (s *FederationSyncController) minimizeLatency() {
+	s.clusterAvailableDelay = time.Second
+	s.clusterUnavailableDelay = time.Second
+	s.reviewDelay = 50 * time.Millisecond
+	s.smallDelay = 20 * time.Millisecond
+	s.updateTimeout = 5 * time.Second
+}
+
+// Sends the given updated object to apiserver.
+func (s *FederationSyncController) updateObject(obj pkgruntime.Object) (pkgruntime.Object, error) {
+	return s.adapter.FedUpdate(obj)
+}
+
+func (s *FederationSyncController) Run(stopChan <-chan struct{}) {
+	go s.controller.Run(stopChan)
+	s.informer.Start()
+	s.deliverer.StartWithHandler(func(item *util.DelayingDelivererItem) {
+		s.workQueue.Add(item)
+	})
+	s.clusterDeliverer.StartWithHandler(func(_ *util.DelayingDelivererItem) {
+		s.reconcileOnClusterChange()
+	})
+
+	// TODO: Allow multiple workers.
+	go wait.Until(s.worker, time.Second, stopChan)
+
+	util.StartBackoffGC(s.backoff, stopChan)
+
+	// Ensure all goroutines are cleaned up when the stop channel closes
+	go func() {
+		<-stopChan
+		s.informer.Stop()
+		s.workQueue.ShutDown()
+		s.deliverer.Stop()
+		s.clusterDeliverer.Stop()
+	}()
+}
+
+type reconciliationStatus int
+
+const (
+	statusAllOK reconciliationStatus = iota
+	statusNeedsRecheck
+	statusError
+	statusNotSynced
+)
+
+func (s *FederationSyncController) worker() {
+	for {
+		obj, quit := s.workQueue.Get()
+		if quit {
+			return
+		}
+
+		item := obj.(*util.DelayingDelivererItem)
+		qualifiedName := item.Value.(*federatedtypes.QualifiedName)
+		status := s.reconcile(*qualifiedName)
+		s.workQueue.Done(item)
+
+		switch status {
+		case statusAllOK:
+			break
+		case statusError:
+			s.deliver(*qualifiedName, 0, true)
+		case statusNeedsRecheck:
+			s.deliver(*qualifiedName, s.reviewDelay, false)
+		case statusNotSynced:
+			s.deliver(*qualifiedName, s.clusterAvailableDelay, false)
+		}
+	}
+}
+
+func (s *FederationSyncController) deliverObj(obj pkgruntime.Object, delay time.Duration, failed bool) {
+	qualifiedName := s.adapter.QualifiedName(obj)
+	s.deliver(qualifiedName, delay, failed)
+}
+
+// Adds backoff to delay if this delivery is related to some failure. Resets backoff if there was no failure.
+func (s *FederationSyncController) deliver(qualifiedName federatedtypes.QualifiedName, delay time.Duration, failed bool) {
+	key := qualifiedName.String()
+	if failed {
+		s.backoff.Next(key, time.Now())
+		delay = delay + s.backoff.Get(key)
+	} else {
+		s.backoff.Reset(key)
+	}
+	s.deliverer.DeliverAfter(key, &qualifiedName, delay)
+}
+
+// Check whether all data stores are in sync. False is returned if any of the informer/stores is not yet
+// synced with the corresponding api server.
+func (s *FederationSyncController) isSynced() bool {
+	if !s.informer.ClustersSynced() {
+		glog.V(2).Infof("Cluster list not synced")
+		return false
+	}
+	clusters, err := s.informer.GetReadyClusters()
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get ready clusters: %v", err))
+		return false
+	}
+	if !s.informer.GetTargetStore().ClustersSynced(clusters) {
+		return false
+	}
+	return true
+}
+
+// The function triggers reconciliation of all target federated resources.
+func (s *FederationSyncController) reconcileOnClusterChange() {
+	if !s.isSynced() {
+		s.clusterDeliverer.DeliverAt(allClustersKey, nil, time.Now().Add(s.clusterAvailableDelay))
+	}
+	for _, obj := range s.store.List() {
+		qualifiedName := s.adapter.QualifiedName(obj.(pkgruntime.Object))
+		s.deliver(qualifiedName, s.smallDelay, false)
+	}
+}
+
+func (s *FederationSyncController) reconcile(qualifiedName federatedtypes.QualifiedName) reconciliationStatus {
+	if !s.isSynced() {
+		return statusNotSynced
+	}
+
+	kind := s.adapter.Kind()
+	key := qualifiedName.String()
+
+	glog.V(4).Infof("Starting to reconcile %v %v", kind, key)
+	startTime := time.Now()
+	defer glog.V(4).Infof("Finished reconciling %v %v (duration: %v)", kind, key, time.Now().Sub(startTime))
+
+	obj, err := s.objFromCache(kind, key)
+	if err != nil {
+		return statusError
+	}
+	if obj == nil {
+		return statusAllOK
+	}
+
+	meta := s.adapter.ObjectMeta(obj)
+	if meta.DeletionTimestamp != nil {
+		err := s.delete(obj, kind, qualifiedName)
+		if err != nil {
+			msg := "Failed to delete %s %q: %v"
+			args := []interface{}{kind, qualifiedName, err}
+			runtime.HandleError(fmt.Errorf(msg, args...))
+			s.eventRecorder.Eventf(obj, api.EventTypeWarning, "DeleteFailed", msg, args...)
+			return statusError
+		}
+		return statusAllOK
+	}
+
+	glog.V(3).Infof("Ensuring finalizers exist on %s %q", kind, key)
+	obj, err = s.deletionHelper.EnsureFinalizers(obj)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to ensure finalizers for %s %q: %v", kind, key, err))
+		return statusError
+	}
+
+	operationsAccessor := func(adapter federatedtypes.FederatedTypeAdapter, selectedClusters []*federationapi.Cluster, unselectedClusters []*federationapi.Cluster, obj pkgruntime.Object, schedulingInfo interface{}) ([]util.FederatedOperation, error) {
+		operations, err := clusterOperations(adapter, selectedClusters, unselectedClusters, obj, key, schedulingInfo, func(clusterName string) (interface{}, bool, error) {
+			return s.informer.GetTargetStore().GetByKey(clusterName, key)
+		})
+		if err != nil {
+			s.eventRecorder.Eventf(obj, api.EventTypeWarning, "FedClusterOperationsError", "Error obtaining sync operations for %s: %s error: %s", kind, key, err.Error())
+		}
+		return operations, err
+	}
+
+	return syncToClusters(
+		s.informer.GetReadyClusters,
+		operationsAccessor,
+		selectedClusters,
+		s.updater.Update,
+		s.adapter,
+		s.informer,
+		obj,
+	)
+}
+
+func (s *FederationSyncController) objFromCache(kind, key string) (pkgruntime.Object, error) {
+	cachedObj, exist, err := s.store.GetByKey(key)
+	if err != nil {
+		wrappedErr := fmt.Errorf("Failed to query %s store for %q: %v", kind, key, err)
+		runtime.HandleError(wrappedErr)
+		return nil, err
+	}
+	if !exist {
+		return nil, nil
+	}
+
+	// Create a copy before modifying the resource to prevent racing with other readers.
+	copiedObj, err := api.Scheme.DeepCopy(cachedObj)
+	if err != nil {
+		wrappedErr := fmt.Errorf("Error in retrieving %s %q from store: %v", kind, key, err)
+		runtime.HandleError(wrappedErr)
+		return nil, err
+	}
+	if !s.adapter.IsExpectedType(copiedObj) {
+		err = fmt.Errorf("Object is not the expected type: %v", copiedObj)
+		runtime.HandleError(err)
+		return nil, err
+	}
+	return copiedObj.(pkgruntime.Object), nil
+}
+
+// delete deletes the given resource or returns error if the deletion was not complete.
+func (s *FederationSyncController) delete(obj pkgruntime.Object, kind string, qualifiedName federatedtypes.QualifiedName) error {
+	glog.V(3).Infof("Handling deletion of %s %q", kind, qualifiedName)
+
+	// Perform pre-deletion cleanup for the namespace adapter
+	namespaceAdapter, ok := s.adapter.(*federatedtypes.NamespaceAdapter)
+	if ok {
+		var err error
+		obj, err = namespaceAdapter.CleanUpNamespace(obj, s.eventRecorder)
+		if err != nil {
+			return err
+		}
+	}
+
+	_, err := s.deletionHelper.HandleObjectInUnderlyingClusters(obj)
+	if err != nil {
+		return err
+	}
+
+	err = s.adapter.FedDelete(qualifiedName, nil)
+	if err != nil {
+		// Its all good if the error is not found error. That means it is deleted already and we do not have to do anything.
+		// This is expected when we are processing an update as a result of finalizer deletion.
+		// The process that deleted the last finalizer is also going to delete the resource and we do not have to do anything.
+		if !errors.IsNotFound(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+type clustersAccessorFunc func() ([]*federationapi.Cluster, error)
+type operationsFunc func(federatedtypes.FederatedTypeAdapter, []*federationapi.Cluster, []*federationapi.Cluster, pkgruntime.Object, interface{}) ([]util.FederatedOperation, error)
+type clusterSelectorFunc func(*metav1.ObjectMeta, func(map[string]string, map[string]string) (bool, error), []*federationapi.Cluster) ([]*federationapi.Cluster, []*federationapi.Cluster, error)
+type executionFunc func([]util.FederatedOperation) error
+
+// syncToClusters ensures that the state of the given object is synchronized to member clusters.
+func syncToClusters(clustersAccessor clustersAccessorFunc, operationsAccessor operationsFunc, selector clusterSelectorFunc, execute executionFunc, adapter federatedtypes.FederatedTypeAdapter, informer util.FederatedInformer, obj pkgruntime.Object) reconciliationStatus {
+	kind := adapter.Kind()
+	key := federatedtypes.ObjectKey(adapter, obj)
+
+	glog.V(3).Infof("Syncing %s %q in underlying clusters", kind, key)
+
+	clusters, err := clustersAccessor()
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get cluster list: %v", err))
+		return statusNotSynced
+	}
+
+	selectedClusters, unselectedClusters, err := selector(adapter.ObjectMeta(obj), clusterselector.SendToCluster, clusters)
+	if err != nil {
+		return statusError
+	}
+
+	var schedulingInfo interface{}
+	if adapter.IsSchedulingAdapter() {
+		schedulingAdapter, ok := adapter.(federatedtypes.SchedulingAdapter)
+		if !ok {
+			glog.Fatalf("Adapter for kind %q does not properly implement SchedulingAdapter.", kind)
+		}
+		schedulingInfo, err = schedulingAdapter.GetSchedule(obj, key, selectedClusters, informer)
+		if err != nil {
+			runtime.HandleError(fmt.Errorf("adapter.GetSchedule() failed on adapter for %s %q: %v", kind, key, err))
+			return statusError
+		}
+	}
+
+	operations, err := operationsAccessor(adapter, selectedClusters, unselectedClusters, obj, schedulingInfo)
+	if err != nil {
+		return statusError
+	}
+
+	if adapter.IsSchedulingAdapter() {
+		schedulingAdapter, ok := adapter.(federatedtypes.SchedulingAdapter)
+		if !ok {
+			glog.Fatalf("Adapter for kind %q does not properly implement SchedulingAdapter.", kind)
+		}
+		err = schedulingAdapter.UpdateFederatedStatus(obj, schedulingInfo)
+		if err != nil {
+			runtime.HandleError(fmt.Errorf("adapter.UpdateFinished() failed on adapter for %s %q: %v", kind, key, err))
+			return statusError
+		}
+	}
+
+	if len(operations) == 0 {
+		return statusAllOK
+	}
+
+	err = execute(operations)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to execute updates for %s %q: %v", kind, key, err))
+		return statusError
+	}
+
+	// Everything is in order but let's be double sure
+	return statusNeedsRecheck
+}
+
+// selectedClusters filters the provided clusters into two slices, one containing the clusters selected by selector and the other containing the rest of the provided clusters.
+func selectedClusters(objMeta *metav1.ObjectMeta, selector func(map[string]string, map[string]string) (bool, error), clusters []*federationapi.Cluster) ([]*federationapi.Cluster, []*federationapi.Cluster, error) {
+	selectedClusters := []*federationapi.Cluster{}
+	unselectedClusters := []*federationapi.Cluster{}
+
+	for _, cluster := range clusters {
+		send, err := selector(cluster.Labels, objMeta.Annotations)
+		if err != nil {
+			return nil, nil, err
+		} else if !send {
+			unselectedClusters = append(unselectedClusters, cluster)
+		} else {
+			selectedClusters = append(selectedClusters, cluster)
+		}
+	}
+	return selectedClusters, unselectedClusters, nil
+}
+
+type clusterObjectAccessorFunc func(clusterName string) (interface{}, bool, error)
+
+// clusterOperations returns the list of operations needed to synchronize the state of the given object to the provided clusters
+func clusterOperations(adapter federatedtypes.FederatedTypeAdapter, selectedClusters []*federationapi.Cluster, unselectedClusters []*federationapi.Cluster, obj pkgruntime.Object, key string, schedulingInfo interface{}, accessor clusterObjectAccessorFunc) ([]util.FederatedOperation, error) {
+	operations := make([]util.FederatedOperation, 0)
+
+	kind := adapter.Kind()
+	for _, cluster := range selectedClusters {
+		// The data should not be modified.
+		desiredObj := adapter.Copy(obj)
+
+		clusterObj, found, err := accessor(cluster.Name)
+		if err != nil {
+			wrappedErr := fmt.Errorf("Failed to get %s %q from cluster %q: %v", kind, key, cluster.Name, err)
+			runtime.HandleError(wrappedErr)
+			return nil, wrappedErr
+		}
+
+		var scheduleAction federatedtypes.ScheduleAction = federatedtypes.ActionAdd
+		if adapter.IsSchedulingAdapter() {
+			schedulingAdapter, ok := adapter.(federatedtypes.SchedulingAdapter)
+			if !ok {
+				err = fmt.Errorf("adapter for kind %s does not properly implement SchedulingAdapter.", kind)
+				glog.Fatalf("Error: %v", err)
+			}
+			var clusterTypedObj pkgruntime.Object = nil
+			if clusterObj != nil {
+				clusterTypedObj = clusterObj.(pkgruntime.Object)
+			}
+			desiredObj, scheduleAction, err = schedulingAdapter.ScheduleObject(cluster, clusterTypedObj, desiredObj, schedulingInfo)
+			if err != nil {
+				runtime.HandleError(err)
+				return nil, err
+			}
+		}
+
+		var operationType util.FederatedOperationType = ""
+		if found {
+			if scheduleAction == federatedtypes.ActionDelete {
+				operationType = util.OperationTypeDelete
+			} else {
+				clusterObj := clusterObj.(pkgruntime.Object)
+				if !adapter.Equivalent(desiredObj, clusterObj) {
+					operationType = util.OperationTypeUpdate
+				}
+			}
+		} else if scheduleAction == federatedtypes.ActionAdd {
+			operationType = util.OperationTypeAdd
+		}
+
+		if len(operationType) > 0 {
+			operations = append(operations, util.FederatedOperation{
+				Type:        operationType,
+				Obj:         desiredObj,
+				ClusterName: cluster.Name,
+				Key:         key,
+			})
+		}
+	}
+
+	for _, cluster := range unselectedClusters {
+		clusterObj, found, err := accessor(cluster.Name)
+		if err != nil {
+			wrappedErr := fmt.Errorf("Failed to get %s %q from cluster %q: %v", kind, key, cluster.Name, err)
+			runtime.HandleError(wrappedErr)
+			return nil, wrappedErr
+		}
+		if found {
+			operations = append(operations, util.FederatedOperation{
+				Type:        util.OperationTypeDelete,
+				Obj:         clusterObj.(pkgruntime.Object),
+				ClusterName: cluster.Name,
+				Key:         key,
+			})
+		}
+	}
+
+	return operations, nil
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/sync/controller_test.go

@@ -0,0 +1,231 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sync
+
+import (
+	"errors"
+	"testing"
+
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pkgruntime "k8s.io/apimachinery/pkg/runtime"
+	federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	"k8s.io/kubernetes/federation/pkg/federatedtypes"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	fedtest "k8s.io/kubernetes/federation/pkg/federation-controller/util/test"
+
+	"github.com/stretchr/testify/require"
+)
+
+var awfulError error = errors.New("Something bad happened")
+
+func TestSyncToClusters(t *testing.T) {
+	adapter := &federatedtypes.SecretAdapter{}
+	obj := adapter.NewTestObject("foo")
+
+	testCases := map[string]struct {
+		clusterError    bool
+		operationsError bool
+		executionError  bool
+		operations      []util.FederatedOperation
+		status          reconciliationStatus
+	}{
+		"Error listing clusters redelivers with cluster delay": {
+			clusterError: true,
+			status:       statusNotSynced,
+		},
+		"Error retrieving cluster operations redelivers": {
+			operationsError: true,
+			status:          statusError,
+		},
+		"No operations returns ok": {
+			status: statusAllOK,
+		},
+		"Execution error redelivers": {
+			executionError: true,
+			operations:     []util.FederatedOperation{{}},
+			status:         statusError,
+		},
+		"Successful update indicates recheck": {
+			operations: []util.FederatedOperation{{}},
+			status:     statusNeedsRecheck,
+		},
+	}
+
+	for testName, testCase := range testCases {
+		t.Run(testName, func(t *testing.T) {
+			status := syncToClusters(
+				func() ([]*federationapi.Cluster, error) {
+					if testCase.clusterError {
+						return nil, awfulError
+					}
+					return nil, nil
+				},
+				func(federatedtypes.FederatedTypeAdapter, []*federationapi.Cluster, []*federationapi.Cluster, pkgruntime.Object, interface{}) ([]util.FederatedOperation, error) {
+					if testCase.operationsError {
+						return nil, awfulError
+					}
+					return testCase.operations, nil
+				},
+				func(objMeta *metav1.ObjectMeta, selector func(map[string]string, map[string]string) (bool, error), clusters []*federationapi.Cluster) ([]*federationapi.Cluster, []*federationapi.Cluster, error) {
+					return clusters, []*federationapi.Cluster{}, nil
+				},
+				func([]util.FederatedOperation) error {
+					if testCase.executionError {
+						return awfulError
+					}
+					return nil
+				},
+				adapter,
+				nil,
+				obj,
+			)
+			require.Equal(t, testCase.status, status, "Unexpected status!")
+		})
+	}
+}
+
+func TestSelectedClusters(t *testing.T) {
+	clusterOne := fedtest.NewCluster("cluster1", apiv1.ConditionTrue)
+	clusterOne.Labels = map[string]string{"name": "cluster1"}
+	clusterTwo := fedtest.NewCluster("cluster2", apiv1.ConditionTrue)
+	clusterTwo.Labels = map[string]string{"name": "cluster2"}
+
+	clusters := []*federationapi.Cluster{clusterOne, clusterTwo}
+	testCases := map[string]struct {
+		expectedSelectorError      bool
+		clusterOneSelected         bool
+		clusterTwoSelected         bool
+		expectedSelectedClusters   []*federationapi.Cluster
+		expectedUnselectedClusters []*federationapi.Cluster
+	}{
+		"Selector returned error": {
+			expectedSelectorError: true,
+		},
+		"All clusters selected": {
+			clusterOneSelected:         true,
+			clusterTwoSelected:         true,
+			expectedSelectedClusters:   clusters,
+			expectedUnselectedClusters: []*federationapi.Cluster{},
+		},
+		"One cluster selected": {
+			clusterOneSelected:         true,
+			expectedSelectedClusters:   []*federationapi.Cluster{clusterOne},
+			expectedUnselectedClusters: []*federationapi.Cluster{clusterTwo},
+		},
+		"No clusters selected": {
+			expectedSelectedClusters:   []*federationapi.Cluster{},
+			expectedUnselectedClusters: clusters,
+		},
+	}
+
+	for testName, testCase := range testCases {
+		t.Run(testName, func(t *testing.T) {
+			selectedClusters, unselectedClusters, err := selectedClusters(&metav1.ObjectMeta{}, func(labels map[string]string, annotations map[string]string) (bool, error) {
+				if testCase.expectedSelectorError {
+					return false, awfulError
+				}
+				if labels["name"] == "cluster1" {
+					return testCase.clusterOneSelected, nil
+				}
+				if labels["name"] == "cluster2" {
+					return testCase.clusterTwoSelected, nil
+				}
+				t.Errorf("Unexpected cluster")
+				return false, nil
+			}, clusters)
+
+			if testCase.expectedSelectorError {
+				require.Error(t, err, "An error was expected")
+			} else {
+				require.NoError(t, err, "An error was not expected")
+			}
+			require.Equal(t, testCase.expectedSelectedClusters, selectedClusters, "Expected the correct clusters to be selected.")
+			require.Equal(t, testCase.expectedUnselectedClusters, unselectedClusters, "Expected the correct clusters to be unselected.")
+		})
+	}
+}
+
+func TestClusterOperations(t *testing.T) {
+	adapter := &federatedtypes.SecretAdapter{}
+	obj := adapter.NewTestObject("foo")
+	differingObj := adapter.Copy(obj)
+	federatedtypes.SetAnnotation(adapter, differingObj, "foo", "bar")
+
+	testCases := map[string]struct {
+		clusterObject pkgruntime.Object
+		expectedErr   bool
+		sendToCluster bool
+
+		operationType util.FederatedOperationType
+	}{
+		"Accessor error returned": {
+			expectedErr: true,
+		},
+		"Missing cluster object should result in add operation": {
+			operationType: util.OperationTypeAdd,
+			sendToCluster: true,
+		},
+		"Differing cluster object should result in update operation": {
+			clusterObject: differingObj,
+			operationType: util.OperationTypeUpdate,
+			sendToCluster: true,
+		},
+		"Matching object and not matching ClusterSelector should result in delete operation": {
+			clusterObject: obj,
+			operationType: util.OperationTypeDelete,
+			sendToCluster: false,
+		},
+		"Matching cluster object should not result in an operation": {
+			clusterObject: obj,
+			sendToCluster: true,
+		},
+	}
+	for testName, testCase := range testCases {
+		t.Run(testName, func(t *testing.T) {
+			clusters := []*federationapi.Cluster{fedtest.NewCluster("cluster1", apiv1.ConditionTrue)}
+			key := federatedtypes.ObjectKey(adapter, obj)
+
+			var selectedClusters, unselectedClusters []*federationapi.Cluster
+			if testCase.sendToCluster {
+				selectedClusters = clusters
+				unselectedClusters = []*federationapi.Cluster{}
+			} else {
+				selectedClusters = []*federationapi.Cluster{}
+				unselectedClusters = clusters
+			}
+			// TODO: Tests for ScheduleObject on type adapter
+			operations, err := clusterOperations(adapter, selectedClusters, unselectedClusters, obj, key, nil, func(string) (interface{}, bool, error) {
+				if testCase.expectedErr {
+					return nil, false, awfulError
+				}
+				return testCase.clusterObject, (testCase.clusterObject != nil), nil
+			})
+			if testCase.expectedErr {
+				require.Error(t, err, "An error was expected")
+			} else {
+				require.NoError(t, err, "An error was not expected")
+			}
+			if len(testCase.operationType) == 0 {
+				require.True(t, len(operations) == 0, "An operation was not expected")
+			} else {
+				require.True(t, len(operations) == 1, "A single operation was expected")
+				require.Equal(t, testCase.operationType, operations[0].Type, "Unexpected operation returned")
+			}
+		})
+	}
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/BUILD

@@ -0,0 +1,98 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "backoff.go",
+        "cluster_util.go",
+        "configmap.go",
+        "delaying_deliverer.go",
+        "deployment.go",
+        "federated_informer.go",
+        "federated_updater.go",
+        "handlers.go",
+        "meta.go",
+        "secret.go",
+    ],
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset:go_default_library",
+        "//pkg/api:go_default_library",
+        "//pkg/client/clientset_generated/internalclientset:go_default_library",
+        "//pkg/controller/deployment/util:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/rest:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+        "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
+        "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
+        "//vendor/k8s.io/client-go/tools/record:go_default_library",
+        "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "delaying_deliverer_test.go",
+        "deployment_test.go",
+        "federated_informer_test.go",
+        "federated_updater_test.go",
+        "handlers_test.go",
+        "meta_test.go",
+    ],
+    library = ":go_default_library",
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset/fake:go_default_library",
+        "//pkg/controller/deployment/util:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
+        "//vendor/k8s.io/client-go/testing:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//federation/pkg/federation-controller/util/clusterselector:all-srcs",
+        "//federation/pkg/federation-controller/util/deletionhelper:all-srcs",
+        "//federation/pkg/federation-controller/util/eventsink:all-srcs",
+        "//federation/pkg/federation-controller/util/finalizers:all-srcs",
+        "//federation/pkg/federation-controller/util/hpa:all-srcs",
+        "//federation/pkg/federation-controller/util/planner:all-srcs",
+        "//federation/pkg/federation-controller/util/podanalyzer:all-srcs",
+        "//federation/pkg/federation-controller/util/replicapreferences:all-srcs",
+        "//federation/pkg/federation-controller/util/test:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/backoff.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"time"
+
+	"k8s.io/client-go/util/flowcontrol"
+)
+
+func StartBackoffGC(backoff *flowcontrol.Backoff, stopCh <-chan struct{}) {
+	go func() {
+		for {
+			select {
+			case <-time.After(time.Minute):
+				backoff.GC()
+			case <-stopCh:
+				return
+			}
+		}
+	}()
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/cluster_util.go

@@ -0,0 +1,147 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"time"
+
+	"github.com/golang/glog"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apimachinery/pkg/util/wait"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	federation_v1beta1 "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	"k8s.io/kubernetes/pkg/api"
+	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
+)
+
+const (
+	KubeAPIQPS              = 20.0
+	KubeAPIBurst            = 30
+	KubeconfigSecretDataKey = "kubeconfig"
+	getSecretTimeout        = 1 * time.Minute
+)
+
+func BuildClusterConfig(c *federation_v1beta1.Cluster) (*restclient.Config, error) {
+	var serverAddress string
+	var clusterConfig *restclient.Config
+	hostIP, err := utilnet.ChooseHostInterface()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, item := range c.Spec.ServerAddressByClientCIDRs {
+		_, cidrnet, err := net.ParseCIDR(item.ClientCIDR)
+		if err != nil {
+			return nil, err
+		}
+		myaddr := net.ParseIP(hostIP.String())
+		if cidrnet.Contains(myaddr) == true {
+			serverAddress = item.ServerAddress
+			break
+		}
+	}
+	if serverAddress != "" {
+		if c.Spec.SecretRef == nil {
+			glog.Infof("didn't find secretRef for cluster %s. Trying insecure access", c.Name)
+			clusterConfig, err = clientcmd.BuildConfigFromFlags(serverAddress, "")
+		} else {
+			if c.Spec.SecretRef.Name == "" {
+				return nil, fmt.Errorf("found secretRef but no secret name for cluster %s", c.Name)
+			}
+			secret, err := getSecret(c.Spec.SecretRef.Name)
+			if err != nil {
+				return nil, err
+			}
+			// Pre-1.7, the secret contained a serialized kubeconfig which contained appropriate credentials.
+			// Post-1.7, the secret contains credentials for a service account.
+			// Check for the service account credentials, and use them if they exist; if not, use the
+			// serialized kubeconfig.
+			token, tokenFound := secret.Data["token"]
+			ca, caFound := secret.Data["ca.crt"]
+			if tokenFound != caFound {
+				return nil, fmt.Errorf("secret should have values for either both 'ca.crt' and 'token' in its Data, or neither: %v", secret)
+			} else if tokenFound && caFound {
+				clusterConfig, err = clientcmd.BuildConfigFromFlags(serverAddress, "")
+				clusterConfig.CAData = ca
+				clusterConfig.BearerToken = string(token)
+			} else {
+				kubeconfigGetter := KubeconfigGetterForSecret(secret)
+				clusterConfig, err = clientcmd.BuildConfigFromKubeconfigGetter(serverAddress, kubeconfigGetter)
+			}
+		}
+		if err != nil {
+			return nil, err
+		}
+		clusterConfig.QPS = KubeAPIQPS
+		clusterConfig.Burst = KubeAPIBurst
+	}
+	return clusterConfig, nil
+}
+
+// getSecret gets a secret from the cluster.
+func getSecret(secretName string) (*api.Secret, error) {
+	// Get the namespace this is running in from the env variable.
+	namespace := os.Getenv("POD_NAMESPACE")
+	if namespace == "" {
+		return nil, fmt.Errorf("unexpected: POD_NAMESPACE env var returned empty string")
+	}
+	// Get a client to talk to the k8s apiserver, to fetch secrets from it.
+	cc, err := restclient.InClusterConfig()
+	if err != nil {
+		return nil, fmt.Errorf("error in creating in-cluster config: %s", err)
+	}
+	client, err := clientset.NewForConfig(cc)
+	if err != nil {
+		return nil, fmt.Errorf("error in creating in-cluster client: %s", err)
+	}
+	var secret *api.Secret
+	err = wait.PollImmediate(1*time.Second, getSecretTimeout, func() (bool, error) {
+		secret, err = client.Core().Secrets(namespace).Get(secretName, metav1.GetOptions{})
+		if err == nil {
+			return true, nil
+		}
+		glog.Warningf("error in fetching secret: %s", err)
+		return false, nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("timed out waiting for secret: %s", err)
+	}
+	if secret == nil {
+		return nil, fmt.Errorf("unexpected: received null secret %s", secretName)
+	}
+	return secret, nil
+}
+
+// KubeconfigGetterForSecret gets the kubeconfig from the given secret.
+// This is to inject a different KubeconfigGetter in tests. We don't use
+// the standard one which calls NewInCluster in tests to avoid having to
+// set up service accounts and mount files with secret tokens.
+var KubeconfigGetterForSecret = func(secret *api.Secret) clientcmd.KubeconfigGetter {
+	return func() (*clientcmdapi.Config, error) {
+		data, ok := secret.Data[KubeconfigSecretDataKey]
+		if !ok {
+			return nil, fmt.Errorf("secret does not have data with key %s", KubeconfigSecretDataKey)
+		}
+		return clientcmd.Load(data)
+	}
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/clusterselector/BUILD

@@ -0,0 +1,40 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["clusterselector_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//vendor/github.com/stretchr/testify/require:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["clusterselector.go"],
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/selection:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/clusterselector/clusterselector.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package clusterselector
+
+import (
+	"encoding/json"
+
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/selection"
+	federation_v1beta1 "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+)
+
+// Parses the cluster selector annotation to find out if the object with that annotation should be forwarded to a cluster with the given clusterLabels.
+func SendToCluster(clusterLabels map[string]string, annotations map[string]string) (bool, error) {
+	// Check if a ClusterSelector annotation exists and send to all clusters when it does not exist
+	val, ok := annotations[federation_v1beta1.FederationClusterSelectorAnnotation]
+	if !ok {
+		return true, nil
+	}
+
+	selector, err := getSelector(val)
+	if err != nil {
+		return false, err
+	}
+	return selector.Matches(labels.Set(clusterLabels)), nil
+}
+
+func getSelector(annotation string) (labels.Selector, error) {
+	selector := labels.NewSelector()
+	requirements := make([]federation_v1beta1.ClusterSelectorRequirement, 0)
+	err := json.Unmarshal([]byte(annotation), &requirements)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, requirement := range requirements {
+		r, err := labels.NewRequirement(requirement.Key, ConvertOperator(requirement.Operator), requirement.Values)
+		if err != nil {
+			// Stop processing and assume failure since we have no way of knowing the end users intent for this or any other clusters.
+			return nil, err
+		}
+		selector = selector.Add(*r)
+	}
+
+	return selector, nil
+}
+
+// ConvertOperator converts a string operator into selection.Operator type
+func ConvertOperator(source string) selection.Operator {
+	var op selection.Operator
+	switch source {
+	case "!", "DoesNotExist":
+		op = selection.DoesNotExist
+	case "=":
+		op = selection.Equals
+	case "==":
+		op = selection.DoubleEquals
+	case "in", "In":
+		op = selection.In
+	case "!=":
+		op = selection.NotEquals
+	case "notin", "NotIn":
+		op = selection.NotIn
+	case "exists", "Exists":
+		op = selection.Exists
+	case "gt", "Gt", ">":
+		op = selection.GreaterThan
+	case "lt", "Lt", "<":
+		op = selection.LessThan
+	}
+	return op
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/clusterselector/clusterselector_test.go

@@ -0,0 +1,98 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package clusterselector
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+)
+
+func TestSendToCluster(t *testing.T) {
+
+	clusterLabels := map[string]string{
+		"location":    "europe",
+		"environment": "prod",
+		"version":     "15",
+	}
+
+	testCases := map[string]struct {
+		objectAnnotations map[string]string
+		expectedResult    bool
+		expectedErr       bool
+	}{
+		"match with single annotation": {
+			objectAnnotations: map[string]string{
+				federationapi.FederationClusterSelectorAnnotation: `[{"key": "location", "operator": "in", "values": ["europe"]}]`,
+			},
+			expectedResult: true,
+		},
+		"match on multiple annotations": {
+			objectAnnotations: map[string]string{
+				federationapi.FederationClusterSelectorAnnotation: `[{"key": "location", "operator": "in", "values": ["europe"]}, {"key": "environment", "operator": "==", "values": ["prod"]}]`,
+			},
+			expectedResult: true,
+		},
+		"mismatch on one annotation": {
+			objectAnnotations: map[string]string{
+				federationapi.FederationClusterSelectorAnnotation: `[{"key": "location", "operator": "in", "values": ["europe"]}, {"key": "environment", "operator": "==", "values": ["test"]}]`,
+			},
+			expectedResult: false,
+		},
+		"match on not equal annotation": {
+			objectAnnotations: map[string]string{
+				federationapi.FederationClusterSelectorAnnotation: `[{"key": "location", "operator": "!=", "values": ["usa"]}, {"key": "environment", "operator": "in", "values": ["prod"]}]`,
+			},
+			expectedResult: true,
+		},
+		"match on greater than annotation": {
+			objectAnnotations: map[string]string{
+				federationapi.FederationClusterSelectorAnnotation: `[{"key": "version", "operator": ">", "values": ["14"]}]`,
+			},
+			expectedResult: true,
+		},
+		"mismatch on greater than annotation": {
+			objectAnnotations: map[string]string{
+				federationapi.FederationClusterSelectorAnnotation: `[{"key": "version", "operator": ">", "values": ["15"]}]`,
+			},
+			expectedResult: false,
+		},
+		"unable to parse annotation": {
+			objectAnnotations: map[string]string{
+				federationapi.FederationClusterSelectorAnnotation: `[{"not able to parse",}]`,
+			},
+			expectedResult: false,
+			expectedErr:    true,
+		},
+	}
+
+	for testName, testCase := range testCases {
+		t.Run(testName, func(t *testing.T) {
+			result, err := SendToCluster(clusterLabels, testCase.objectAnnotations)
+
+			if testCase.expectedErr {
+				require.Error(t, err, "An error was expected")
+			} else {
+				require.NoError(t, err, "An error was not expected")
+			}
+
+			require.Equal(t, testCase.expectedResult, result, "Unexpected response from SendToCluster")
+		})
+	}
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/configmap.go

@@ -0,0 +1,31 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"reflect"
+
+	api_v1 "k8s.io/api/core/v1"
+)
+
+// Checks if cluster-independent, user provided data in two given ConfigMaps are equal. If in
+// the future the ConfigMap structure is expanded then any field that is not populated.
+// by the api server should be included here.
+func ConfigMapEquivalent(s1, s2 *api_v1.ConfigMap) bool {
+	return ObjectMetaEquivalent(s1.ObjectMeta, s2.ObjectMeta) &&
+		reflect.DeepEqual(s1.Data, s2.Data)
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/delaying_deliverer.go

@@ -0,0 +1,183 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// TODO: consider moving it to a more generic package.
+package util
+
+import (
+	"container/heap"
+	"time"
+)
+
+const (
+	// TODO: Investigate what capacity is right.
+	delayingDelivererUpdateChanCapacity = 1000
+)
+
+// DelayingDelivererItem is structure delivered by DelayingDeliverer to the
+// target channel.
+type DelayingDelivererItem struct {
+	// Key under which the value was added to deliverer.
+	Key string
+	// Value of the item.
+	Value interface{}
+	// When the item should be delivered.
+	DeliveryTime time.Time
+}
+
+type delivererHeap struct {
+	keyPosition map[string]int
+	data        []*DelayingDelivererItem
+}
+
+// Functions required by container.Heap.
+
+func (dh *delivererHeap) Len() int { return len(dh.data) }
+func (dh *delivererHeap) Less(i, j int) bool {
+	return dh.data[i].DeliveryTime.Before(dh.data[j].DeliveryTime)
+}
+func (dh *delivererHeap) Swap(i, j int) {
+	dh.keyPosition[dh.data[i].Key] = j
+	dh.keyPosition[dh.data[j].Key] = i
+	dh.data[i], dh.data[j] = dh.data[j], dh.data[i]
+}
+
+func (dh *delivererHeap) Push(x interface{}) {
+	item := x.(*DelayingDelivererItem)
+	dh.data = append(dh.data, item)
+	dh.keyPosition[item.Key] = len(dh.data) - 1
+}
+
+func (dh *delivererHeap) Pop() interface{} {
+	n := len(dh.data)
+	item := dh.data[n-1]
+	dh.data = dh.data[:n-1]
+	delete(dh.keyPosition, item.Key)
+	return item
+}
+
+// A structure that pushes the items to the target channel at a given time.
+type DelayingDeliverer struct {
+	// Channel to deliver the data when their time comes.
+	targetChannel chan *DelayingDelivererItem
+	// Store for data
+	heap *delivererHeap
+	// Channel to feed the main goroutine with updates.
+	updateChannel chan *DelayingDelivererItem
+	// To stop the main goroutine.
+	stopChannel chan struct{}
+}
+
+func NewDelayingDeliverer() *DelayingDeliverer {
+	return NewDelayingDelivererWithChannel(make(chan *DelayingDelivererItem, 100))
+}
+
+func NewDelayingDelivererWithChannel(targetChannel chan *DelayingDelivererItem) *DelayingDeliverer {
+	return &DelayingDeliverer{
+		targetChannel: targetChannel,
+		heap: &delivererHeap{
+			keyPosition: make(map[string]int),
+			data:        make([]*DelayingDelivererItem, 0),
+		},
+		updateChannel: make(chan *DelayingDelivererItem, delayingDelivererUpdateChanCapacity),
+		stopChannel:   make(chan struct{}),
+	}
+}
+
+// Deliver all items due before or equal to timestamp.
+func (d *DelayingDeliverer) deliver(timestamp time.Time) {
+	for d.heap.Len() > 0 {
+		if timestamp.Before(d.heap.data[0].DeliveryTime) {
+			return
+		}
+		item := heap.Pop(d.heap).(*DelayingDelivererItem)
+		d.targetChannel <- item
+	}
+}
+
+func (d *DelayingDeliverer) run() {
+	for {
+		now := time.Now()
+		d.deliver(now)
+
+		nextWakeUp := now.Add(time.Hour)
+		if d.heap.Len() > 0 {
+			nextWakeUp = d.heap.data[0].DeliveryTime
+		}
+		sleepTime := nextWakeUp.Sub(now)
+
+		select {
+		case <-time.After(sleepTime):
+			break // just wake up and process the data
+		case item := <-d.updateChannel:
+			if position, found := d.heap.keyPosition[item.Key]; found {
+				if item.DeliveryTime.Before(d.heap.data[position].DeliveryTime) {
+					d.heap.data[position] = item
+					heap.Fix(d.heap, position)
+				}
+				// Ignore if later.
+			} else {
+				heap.Push(d.heap, item)
+			}
+		case <-d.stopChannel:
+			return
+		}
+	}
+}
+
+// Starts the DelayingDeliverer.
+func (d *DelayingDeliverer) Start() {
+	go d.run()
+}
+
+// Stops the DelayingDeliverer. Undelivered items are discarded.
+func (d *DelayingDeliverer) Stop() {
+	close(d.stopChannel)
+}
+
+// Delivers value at the given time.
+func (d *DelayingDeliverer) DeliverAt(key string, value interface{}, deliveryTime time.Time) {
+	d.updateChannel <- &DelayingDelivererItem{
+		Key:          key,
+		Value:        value,
+		DeliveryTime: deliveryTime,
+	}
+}
+
+// Delivers value after the given delay.
+func (d *DelayingDeliverer) DeliverAfter(key string, value interface{}, delay time.Duration) {
+	d.DeliverAt(key, value, time.Now().Add(delay))
+}
+
+// Gets target channel of the deliverer.
+func (d *DelayingDeliverer) GetTargetChannel() chan *DelayingDelivererItem {
+	return d.targetChannel
+}
+
+// Starts Delaying deliverer with a handler listening on the target channel.
+func (d *DelayingDeliverer) StartWithHandler(handler func(*DelayingDelivererItem)) {
+	go func() {
+		for {
+			select {
+			case item := <-d.targetChannel:
+				handler(item)
+			case <-d.stopChannel:
+				return
+			}
+		}
+	}()
+	d.Start()
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/delaying_deliverer_test.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDelayingDeliverer(t *testing.T) {
+	targetChannel := make(chan *DelayingDelivererItem)
+	now := time.Now()
+	d := NewDelayingDelivererWithChannel(targetChannel)
+	d.Start()
+	defer d.Stop()
+	startupDelay := time.Second
+	d.DeliverAt("a", "aaa", now.Add(startupDelay+2*time.Millisecond))
+	d.DeliverAt("b", "bbb", now.Add(startupDelay+3*time.Millisecond))
+	d.DeliverAt("c", "ccc", now.Add(startupDelay+1*time.Millisecond))
+	d.DeliverAt("e", "eee", now.Add(time.Hour))
+	d.DeliverAt("e", "eee", now)
+
+	d.DeliverAt("d", "ddd", now.Add(time.Hour))
+
+	i0 := <-targetChannel
+	assert.Equal(t, "e", i0.Key)
+	assert.Equal(t, "eee", i0.Value.(string))
+	assert.Equal(t, now, i0.DeliveryTime)
+
+	i1 := <-targetChannel
+	received1 := time.Now()
+	assert.True(t, received1.Sub(now).Nanoseconds() > startupDelay.Nanoseconds())
+	assert.Equal(t, "c", i1.Key)
+
+	i2 := <-targetChannel
+	assert.Equal(t, "a", i2.Key)
+
+	i3 := <-targetChannel
+	assert.Equal(t, "b", i3.Key)
+
+	select {
+	case <-targetChannel:
+		t.Fatalf("Nothing should be received")
+	case <-time.After(time.Second):
+		// Ok. Expected
+	}
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/deletionhelper/BUILD

@@ -0,0 +1,32 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["deletion_helper.go"],
+    deps = [
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//federation/pkg/federation-controller/util/finalizers:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/deletionhelper/deletion_helper.go

@@ -0,0 +1,210 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package to help federation controllers to delete federated resources from
+// underlying clusters when the resource is deleted from federation control
+// plane.
+package deletionhelper
+
+import (
+	"fmt"
+	"strings"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	finalizersutil "k8s.io/kubernetes/federation/pkg/federation-controller/util/finalizers"
+
+	"github.com/golang/glog"
+)
+
+const (
+	// Add this finalizer to a federation resource if the resource should be
+	// deleted from all underlying clusters before being deleted from
+	// federation control plane.
+	// This is ignored if FinalizerOrphan is also present on the resource.
+	// In that case, both finalizers are removed from the resource and the
+	// resource is deleted from federation control plane without affecting
+	// the underlying clusters.
+	FinalizerDeleteFromUnderlyingClusters string = "federation.kubernetes.io/delete-from-underlying-clusters"
+)
+
+type UpdateObjFunc func(runtime.Object) (runtime.Object, error)
+type ObjNameFunc func(runtime.Object) string
+
+type DeletionHelper struct {
+	updateObjFunc UpdateObjFunc
+	objNameFunc   ObjNameFunc
+	informer      util.FederatedInformer
+	updater       util.FederatedUpdater
+}
+
+func NewDeletionHelper(
+	updateObjFunc UpdateObjFunc, objNameFunc ObjNameFunc,
+	informer util.FederatedInformer, updater util.FederatedUpdater) *DeletionHelper {
+	return &DeletionHelper{
+		updateObjFunc: updateObjFunc,
+		objNameFunc:   objNameFunc,
+		informer:      informer,
+		updater:       updater,
+	}
+}
+
+// Ensures that the given object has both FinalizerDeleteFromUnderlyingClusters
+// and FinalizerOrphan finalizers.
+// We do this so that the controller is always notified when a federation resource is deleted.
+// If user deletes the resource with nil DeleteOptions or
+// DeletionOptions.OrphanDependents = true then the apiserver removes the orphan finalizer
+// and deletion helper does a cascading deletion.
+// Otherwise, deletion helper just removes the federation resource and orphans
+// the corresponding resources in underlying clusters.
+// This method should be called before creating objects in underlying clusters.
+func (dh *DeletionHelper) EnsureFinalizers(obj runtime.Object) (
+	runtime.Object, error) {
+	finalizers := sets.String{}
+	hasFinalizer, err := finalizersutil.HasFinalizer(obj, FinalizerDeleteFromUnderlyingClusters)
+	if err != nil {
+		return obj, err
+	}
+	if !hasFinalizer {
+		finalizers.Insert(FinalizerDeleteFromUnderlyingClusters)
+	}
+	hasFinalizer, err = finalizersutil.HasFinalizer(obj, metav1.FinalizerOrphanDependents)
+	if err != nil {
+		return obj, err
+	}
+	if !hasFinalizer {
+		finalizers.Insert(metav1.FinalizerOrphanDependents)
+	}
+	if finalizers.Len() != 0 {
+		glog.V(2).Infof("Adding finalizers %v to %s", finalizers.List(), dh.objNameFunc(obj))
+		return dh.addFinalizers(obj, finalizers)
+	}
+	return obj, nil
+}
+
+// Deletes the resources corresponding to the given federated resource from
+// all underlying clusters, unless it has the FinalizerOrphan finalizer.
+// Removes FinalizerOrphan and FinalizerDeleteFromUnderlyingClusters finalizers
+// when done.
+// Callers are expected to keep calling this (with appropriate backoff) until
+// it succeeds.
+func (dh *DeletionHelper) HandleObjectInUnderlyingClusters(obj runtime.Object) (
+	runtime.Object, error) {
+	objName := dh.objNameFunc(obj)
+	glog.V(2).Infof("Handling deletion of federated dependents for object: %s", objName)
+	hasFinalizer, err := finalizersutil.HasFinalizer(obj, FinalizerDeleteFromUnderlyingClusters)
+	if err != nil {
+		return obj, err
+	}
+	if !hasFinalizer {
+		glog.V(2).Infof("obj does not have %s finalizer. Nothing to do", FinalizerDeleteFromUnderlyingClusters)
+		return obj, nil
+	}
+	hasOrphanFinalizer, err := finalizersutil.HasFinalizer(obj, metav1.FinalizerOrphanDependents)
+	if err != nil {
+		return obj, err
+	}
+	if hasOrphanFinalizer {
+		glog.V(2).Infof("Found finalizer orphan. Nothing to do, just remove the finalizer")
+		// If the obj has FinalizerOrphan finalizer, then we need to orphan the
+		// corresponding objects in underlying clusters.
+		// Just remove both the finalizers in that case.
+		finalizers := sets.NewString(FinalizerDeleteFromUnderlyingClusters, metav1.FinalizerOrphanDependents)
+		return dh.removeFinalizers(obj, finalizers)
+	}
+
+	glog.V(2).Infof("Deleting obj %s from underlying clusters", objName)
+	// Else, we need to delete the obj from all underlying clusters.
+	unreadyClusters, err := dh.informer.GetUnreadyClusters()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get a list of unready clusters: %v", err)
+	}
+	// TODO: Handle the case when cluster resource is watched after this is executed.
+	// This can happen if a namespace is deleted before its creation had been
+	// observed in all underlying clusters.
+	storeKey := dh.informer.GetTargetStore().GetKeyFor(obj)
+	clusterNsObjs, err := dh.informer.GetTargetStore().GetFromAllClusters(storeKey)
+	glog.V(3).Infof("Found %d objects in underlying clusters", len(clusterNsObjs))
+	if err != nil {
+		return nil, fmt.Errorf("failed to get object %s from underlying clusters: %v", objName, err)
+	}
+	operations := make([]util.FederatedOperation, 0)
+	for _, clusterNsObj := range clusterNsObjs {
+		operations = append(operations, util.FederatedOperation{
+			Type:        util.OperationTypeDelete,
+			ClusterName: clusterNsObj.ClusterName,
+			Obj:         clusterNsObj.Object.(runtime.Object),
+			Key:         objName,
+		})
+	}
+	err = dh.updater.Update(operations)
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute updates for obj %s: %v", objName, err)
+	}
+	if len(operations) > 0 {
+		// We have deleted a bunch of resources.
+		// Wait for the store to observe all the deletions.
+		var clusterNames []string
+		for _, op := range operations {
+			clusterNames = append(clusterNames, op.ClusterName)
+		}
+		return nil, fmt.Errorf("waiting for object %s to be deleted from clusters: %s", objName, strings.Join(clusterNames, ", "))
+	}
+
+	// We have now deleted the object from all *ready* clusters.
+	// But still need to wait for clusters that are not ready to ensure that
+	// the object has been deleted from *all* clusters.
+	if len(unreadyClusters) != 0 {
+		var clusterNames []string
+		for _, cluster := range unreadyClusters {
+			clusterNames = append(clusterNames, cluster.Name)
+		}
+		return nil, fmt.Errorf("waiting for clusters %s to become ready to verify that obj %s has been deleted", strings.Join(clusterNames, ", "), objName)
+	}
+
+	// All done. Just remove the finalizer.
+	return dh.removeFinalizers(obj, sets.NewString(FinalizerDeleteFromUnderlyingClusters))
+}
+
+// Adds the given finalizers to the given objects ObjectMeta.
+func (dh *DeletionHelper) addFinalizers(obj runtime.Object, finalizers sets.String) (runtime.Object, error) {
+	isUpdated, err := finalizersutil.AddFinalizers(obj, finalizers)
+	if err != nil || !isUpdated {
+		return obj, err
+	}
+	// Send the update to apiserver.
+	updatedObj, err := dh.updateObjFunc(obj)
+	if err != nil {
+		return nil, fmt.Errorf("failed to add finalizers %v to object %s: %v", finalizers, dh.objNameFunc(obj), err)
+	}
+	return updatedObj, nil
+}
+
+// Removes the given finalizers from the given objects ObjectMeta.
+func (dh *DeletionHelper) removeFinalizers(obj runtime.Object, finalizers sets.String) (runtime.Object, error) {
+	isUpdated, err := finalizersutil.RemoveFinalizers(obj, finalizers)
+	if err != nil || !isUpdated {
+		return obj, err
+	}
+	// Send the update to apiserver.
+	updatedObj, err := dh.updateObjFunc(obj)
+	if err != nil {
+		return nil, fmt.Errorf("failed to remove finalizers %v from object %s: %v", finalizers, dh.objNameFunc(obj), err)
+	}
+	return updatedObj, nil
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/deployment.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"reflect"
+
+	extensions_v1 "k8s.io/api/extensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	deputils "k8s.io/kubernetes/pkg/controller/deployment/util"
+)
+
+// Checks if cluster-independent, user provided data in two given Deployment are equal.
+// This function assumes that revisions are not kept in sync across the clusters.
+func DeploymentEquivalent(a, b *extensions_v1.Deployment) bool {
+	if a.Name != b.Name {
+		return false
+	}
+	if a.Namespace != b.Namespace {
+		return false
+	}
+	if !reflect.DeepEqual(a.Labels, b.Labels) && (len(a.Labels) != 0 || len(b.Labels) != 0) {
+		return false
+	}
+	hasKeysAndVals := func(x, y map[string]string) bool {
+		if x == nil {
+			x = map[string]string{}
+		}
+		if y == nil {
+			y = map[string]string{}
+		}
+		for k, v := range x {
+			if k == deputils.RevisionAnnotation {
+				continue
+			}
+			v2, found := y[k]
+			if !found || v != v2 {
+				return false
+			}
+		}
+		return true
+	}
+	return hasKeysAndVals(a.Annotations, b.Annotations) &&
+		hasKeysAndVals(b.Annotations, a.Annotations) &&
+		reflect.DeepEqual(a.Spec, b.Spec)
+}
+
+// Copies object meta for Deployment, skipping revision information.
+func DeepCopyDeploymentObjectMeta(meta metav1.ObjectMeta) metav1.ObjectMeta {
+	meta = DeepCopyRelevantObjectMeta(meta)
+	delete(meta.Annotations, deputils.RevisionAnnotation)
+	return meta
+}
+
+// Copies object meta for Deployment, skipping revision information.
+func DeepCopyDeployment(a *extensions_v1.Deployment) *extensions_v1.Deployment {
+	return &extensions_v1.Deployment{
+		ObjectMeta: DeepCopyDeploymentObjectMeta(a.ObjectMeta),
+		Spec:       *(DeepCopyApiTypeOrPanic(&a.Spec).(*extensions_v1.DeploymentSpec)),
+	}
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/deployment_test.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"testing"
+
+	extensionsv1 "k8s.io/api/extensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	deputils "k8s.io/kubernetes/pkg/controller/deployment/util"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDeploymentEquivalent(t *testing.T) {
+	d1 := newDeployment()
+	d2 := newDeployment()
+	d2.Annotations = make(map[string]string)
+
+	d3 := newDeployment()
+	d3.Annotations = map[string]string{"a": "b"}
+
+	d4 := newDeployment()
+	d4.Annotations = map[string]string{deputils.RevisionAnnotation: "9"}
+
+	assert.True(t, DeploymentEquivalent(d1, d2))
+	assert.True(t, DeploymentEquivalent(d1, d2))
+	assert.True(t, DeploymentEquivalent(d1, d4))
+	assert.True(t, DeploymentEquivalent(d4, d1))
+	assert.False(t, DeploymentEquivalent(d3, d4))
+	assert.False(t, DeploymentEquivalent(d3, d1))
+	assert.True(t, DeploymentEquivalent(d3, d3))
+}
+
+func TestDeploymentCopy(t *testing.T) {
+	d1 := newDeployment()
+	d1.Annotations = map[string]string{deputils.RevisionAnnotation: "9", "a": "b"}
+	d2 := DeepCopyDeployment(d1)
+	assert.True(t, DeploymentEquivalent(d1, d2))
+	assert.Contains(t, d2.Annotations, "a")
+	assert.NotContains(t, d2.Annotations, deputils.RevisionAnnotation)
+}
+
+func newDeployment() *extensionsv1.Deployment {
+	replicas := int32(5)
+	return &extensionsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "wrr",
+			Namespace: metav1.NamespaceDefault,
+			SelfLink:  "/api/v1/namespaces/default/deployments/name123",
+		},
+		Spec: extensionsv1.DeploymentSpec{
+			Replicas: &replicas,
+		},
+	}
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/eventsink/BUILD

@@ -0,0 +1,50 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["eventsink.go"],
+    deps = [
+        "//federation/client/clientset_generated/federation_clientset:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/client-go/tools/record:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["eventsink_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//federation/client/clientset_generated/federation_clientset/fake:go_default_library",
+        "//federation/pkg/federation-controller/util/test:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/client-go/testing:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/eventsink/eventsink.go

@@ -0,0 +1,111 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package eventsink
+
+import (
+	"reflect"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/conversion"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
+	fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+)
+
+// Implements k8s.io/client-go/tools/record.EventSink.
+type FederatedEventSink struct {
+	clientset fedclientset.Interface
+}
+
+// To check if all required functions are implemented.
+var _ record.EventSink = &FederatedEventSink{}
+
+func NewFederatedEventSink(clientset fedclientset.Interface) *FederatedEventSink {
+	return &FederatedEventSink{
+		clientset: clientset,
+	}
+}
+
+// TODO this is uses a reflection conversion path and is very expensive.  federation should update to use client-go
+
+var scheme = runtime.NewScheme()
+
+func init() {
+	// register client-go's and kube's Event type under two different GroupVersions
+	// TODO: switch to client-go client for events
+	scheme.AddKnownTypes(v1.SchemeGroupVersion, &v1.Event{})
+	scheme.AddKnownTypes(schema.GroupVersion{Group: "fake-kube-" + v1.SchemeGroupVersion.Group, Version: v1.SchemeGroupVersion.Version}, &v1.Event{})
+
+	if err := scheme.AddConversionFuncs(
+		metav1.Convert_unversioned_Time_To_unversioned_Time,
+	); err != nil {
+		panic(err)
+	}
+	if err := scheme.AddGeneratedDeepCopyFuncs(
+		conversion.GeneratedDeepCopyFunc{
+			Fn: func(in, out interface{}, c *conversion.Cloner) error {
+				in.(*metav1.Time).DeepCopyInto(out.(*metav1.Time))
+				return nil
+			},
+			InType: reflect.TypeOf(&metav1.Time{}),
+		},
+	); err != nil {
+		panic(err)
+	}
+}
+
+func (fes *FederatedEventSink) Create(event *v1.Event) (*v1.Event, error) {
+	ret, err := fes.clientset.Core().Events(event.Namespace).Create(event)
+	if err != nil {
+		return nil, err
+	}
+
+	retEvent := &v1.Event{}
+	if err := scheme.Convert(ret, retEvent, nil); err != nil {
+		return nil, err
+	}
+	return retEvent, nil
+}
+
+func (fes *FederatedEventSink) Update(event *v1.Event) (*v1.Event, error) {
+	ret, err := fes.clientset.Core().Events(event.Namespace).Update(event)
+	if err != nil {
+		return nil, err
+	}
+
+	retEvent := &v1.Event{}
+	if err := scheme.Convert(ret, retEvent, nil); err != nil {
+		return nil, err
+	}
+	return retEvent, nil
+}
+
+func (fes *FederatedEventSink) Patch(event *v1.Event, data []byte) (*v1.Event, error) {
+	ret, err := fes.clientset.Core().Events(event.Namespace).Patch(event.Name, types.StrategicMergePatchType, data)
+	if err != nil {
+		return nil, err
+	}
+
+	retEvent := &v1.Event{}
+	if err := scheme.Convert(ret, retEvent, nil); err != nil {
+		return nil, err
+	}
+	return retEvent, nil
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/eventsink/eventsink_test.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package eventsink
+
+import (
+	"testing"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	core "k8s.io/client-go/testing"
+	fakefedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset/fake"
+	. "k8s.io/kubernetes/federation/pkg/federation-controller/util/test"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEventSink(t *testing.T) {
+	fakeFederationClient := &fakefedclientset.Clientset{}
+	createdChan := make(chan runtime.Object, 100)
+	fakeFederationClient.AddReactor("create", "events", func(action core.Action) (bool, runtime.Object, error) {
+		createAction := action.(core.CreateAction)
+		obj := createAction.GetObject()
+		createdChan <- obj
+		return true, obj, nil
+	})
+	updateChan := make(chan runtime.Object, 100)
+	fakeFederationClient.AddReactor("update", "events", func(action core.Action) (bool, runtime.Object, error) {
+		updateAction := action.(core.UpdateAction)
+		obj := updateAction.GetObject()
+		updateChan <- obj
+		return true, obj, nil
+	})
+
+	event := v1.Event{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "bzium",
+			Namespace: "ns",
+		},
+	}
+	sink := NewFederatedEventSink(fakeFederationClient)
+	eventUpdated, err := sink.Create(&event)
+	assert.NoError(t, err)
+	eventV1 := GetObjectFromChan(createdChan).(*v1.Event)
+	assert.NotNil(t, eventV1)
+	// Just some simple sanity checks.
+	assert.Equal(t, event.Name, eventV1.Name)
+	assert.Equal(t, event.Name, eventUpdated.Name)
+
+	eventUpdated, err = sink.Update(&event)
+	assert.NoError(t, err)
+	eventV1 = GetObjectFromChan(updateChan).(*v1.Event)
+	assert.NotNil(t, eventV1)
+	// Just some simple sanity checks.
+	assert.Equal(t, event.Name, eventV1.Name)
+	assert.Equal(t, event.Name, eventUpdated.Name)
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/federated_informer.go

@@ -0,0 +1,524 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"reflect"
+	"sync"
+	"time"
+
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pkgruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+	federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	federationclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+
+	"github.com/golang/glog"
+)
+
+const (
+	clusterSyncPeriod = 10 * time.Minute
+	userAgentName     = "federation-controller"
+)
+
+// An object with an origin information.
+type FederatedObject struct {
+	Object      interface{}
+	ClusterName string
+}
+
+// FederatedReadOnlyStore is an overlay over multiple stores created in federated clusters.
+type FederatedReadOnlyStore interface {
+	// Returns all items in the store.
+	List() ([]FederatedObject, error)
+
+	// Returns all items from a cluster.
+	ListFromCluster(clusterName string) ([]interface{}, error)
+
+	// GetKeyFor returns the key under which the item would be put in the store.
+	GetKeyFor(item interface{}) string
+
+	// GetByKey returns the item stored under the given key in the specified cluster (if exist).
+	GetByKey(clusterName string, key string) (interface{}, bool, error)
+
+	// Returns the items stored under the given key in all clusters.
+	GetFromAllClusters(key string) ([]FederatedObject, error)
+
+	// Checks whether stores for all clusters form the lists (and only these) are there and
+	// are synced. This is only a basic check whether the data inside of the store is usable.
+	// It is not a full synchronization/locking mechanism it only tries to ensure that out-of-sync
+	// issues occur less often.	All users of the interface should assume
+	// that there may be significant delays in content updates of all kinds and write their
+	// code that it doesn't break if something is slightly out-of-sync.
+	ClustersSynced(clusters []*federationapi.Cluster) bool
+}
+
+// An interface to access federation members and clients.
+type FederationView interface {
+	// GetClientsetForCluster returns a clientset for the cluster, if present.
+	GetClientsetForCluster(clusterName string) (kubeclientset.Interface, error)
+
+	// GetUnreadyClusters returns a list of all clusters that are not ready yet.
+	GetUnreadyClusters() ([]*federationapi.Cluster, error)
+
+	// GetReadyClusters returns all clusters for which the sub-informers are run.
+	GetReadyClusters() ([]*federationapi.Cluster, error)
+
+	// GetReadyCluster returns the cluster with the given name, if found.
+	GetReadyCluster(name string) (*federationapi.Cluster, bool, error)
+
+	// ClustersSynced returns true if the view is synced (for the first time).
+	ClustersSynced() bool
+}
+
+// A structure that combines an informer running against federated api server and listening for cluster updates
+// with multiple Kubernetes API informers (called target informers) running against federation members. Whenever a new
+// cluster is added to the federation an informer is created for it using TargetInformerFactory. Informers are stopped
+// when a cluster is either put offline of deleted. It is assumed that some controller keeps an eye on the cluster list
+// and thus the clusters in ETCD are up to date.
+type FederatedInformer interface {
+	FederationView
+
+	// Returns a store created over all stores from target informers.
+	GetTargetStore() FederatedReadOnlyStore
+
+	// Starts all the processes.
+	Start()
+
+	// Stops all the processes inside the informer.
+	Stop()
+}
+
+// FederatedInformer with extra method for setting fake clients.
+type FederatedInformerForTestOnly interface {
+	FederatedInformer
+
+	SetClientFactory(func(*federationapi.Cluster) (kubeclientset.Interface, error))
+}
+
+// A function that should be used to create an informer on the target object. Store should use
+// cache.DeletionHandlingMetaNamespaceKeyFunc as a keying function.
+type TargetInformerFactory func(*federationapi.Cluster, kubeclientset.Interface) (cache.Store, cache.Controller)
+
+// A structure with cluster lifecycle handler functions. Cluster is available (and ClusterAvailable is fired)
+// when it is created in federated etcd and ready. Cluster becomes unavailable (and ClusterUnavailable is fired)
+// when it is either deleted or becomes not ready. When cluster spec (IP)is modified both ClusterAvailable
+// and ClusterUnavailable are fired.
+type ClusterLifecycleHandlerFuncs struct {
+	// Fired when the cluster becomes available.
+	ClusterAvailable func(*federationapi.Cluster)
+	// Fired when the cluster becomes unavailable. The second arg contains data that was present
+	// in the cluster before deletion.
+	ClusterUnavailable func(*federationapi.Cluster, []interface{})
+}
+
+// Builds a FederatedInformer for the given federation client and factory.
+func NewFederatedInformer(
+	federationClient federationclientset.Interface,
+	targetInformerFactory TargetInformerFactory,
+	clusterLifecycle *ClusterLifecycleHandlerFuncs) FederatedInformer {
+
+	federatedInformer := &federatedInformerImpl{
+		targetInformerFactory: targetInformerFactory,
+		clientFactory: func(cluster *federationapi.Cluster) (kubeclientset.Interface, error) {
+			clusterConfig, err := BuildClusterConfig(cluster)
+			if err == nil && clusterConfig != nil {
+				clientset := kubeclientset.NewForConfigOrDie(restclient.AddUserAgent(clusterConfig, userAgentName))
+				return clientset, nil
+			}
+			return nil, err
+		},
+		targetInformers: make(map[string]informer),
+	}
+
+	getClusterData := func(name string) []interface{} {
+		data, err := federatedInformer.GetTargetStore().ListFromCluster(name)
+		if err != nil {
+			glog.Errorf("Failed to list %s content: %v", name, err)
+			return make([]interface{}, 0)
+		}
+		return data
+	}
+
+	federatedInformer.clusterInformer.store, federatedInformer.clusterInformer.controller = cache.NewInformer(
+		&cache.ListWatch{
+			ListFunc: func(options metav1.ListOptions) (pkgruntime.Object, error) {
+				return federationClient.Federation().Clusters().List(options)
+			},
+			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+				return federationClient.Federation().Clusters().Watch(options)
+			},
+		},
+		&federationapi.Cluster{},
+		clusterSyncPeriod,
+		cache.ResourceEventHandlerFuncs{
+			DeleteFunc: func(old interface{}) {
+				oldCluster, ok := old.(*federationapi.Cluster)
+				if ok {
+					var data []interface{}
+					if clusterLifecycle.ClusterUnavailable != nil {
+						data = getClusterData(oldCluster.Name)
+					}
+					federatedInformer.deleteCluster(oldCluster)
+					if clusterLifecycle.ClusterUnavailable != nil {
+						clusterLifecycle.ClusterUnavailable(oldCluster, data)
+					}
+				}
+			},
+			AddFunc: func(cur interface{}) {
+				curCluster, ok := cur.(*federationapi.Cluster)
+				if ok && isClusterReady(curCluster) {
+					federatedInformer.addCluster(curCluster)
+					if clusterLifecycle.ClusterAvailable != nil {
+						clusterLifecycle.ClusterAvailable(curCluster)
+					}
+				} else {
+					glog.Errorf("Cluster %v not added.  Not of correct type, or cluster not ready.", cur)
+				}
+			},
+			UpdateFunc: func(old, cur interface{}) {
+				oldCluster, ok := old.(*federationapi.Cluster)
+				if !ok {
+					glog.Errorf("Internal error: Cluster %v not updated.  Old cluster not of correct type.", old)
+					return
+				}
+				curCluster, ok := cur.(*federationapi.Cluster)
+				if !ok {
+					glog.Errorf("Internal error: Cluster %v not updated.  New cluster not of correct type.", cur)
+					return
+				}
+				if isClusterReady(oldCluster) != isClusterReady(curCluster) || !reflect.DeepEqual(oldCluster.Spec, curCluster.Spec) || !reflect.DeepEqual(oldCluster.ObjectMeta.Annotations, curCluster.ObjectMeta.Annotations) {
+					var data []interface{}
+					if clusterLifecycle.ClusterUnavailable != nil {
+						data = getClusterData(oldCluster.Name)
+					}
+					federatedInformer.deleteCluster(oldCluster)
+					if clusterLifecycle.ClusterUnavailable != nil {
+						clusterLifecycle.ClusterUnavailable(oldCluster, data)
+					}
+
+					if isClusterReady(curCluster) {
+						federatedInformer.addCluster(curCluster)
+						if clusterLifecycle.ClusterAvailable != nil {
+							clusterLifecycle.ClusterAvailable(curCluster)
+						}
+					}
+				} else {
+					glog.V(4).Infof("Cluster %v not updated to %v as ready status and specs are identical", oldCluster, curCluster)
+				}
+			},
+		},
+	)
+	return federatedInformer
+}
+
+func isClusterReady(cluster *federationapi.Cluster) bool {
+	for _, condition := range cluster.Status.Conditions {
+		if condition.Type == federationapi.ClusterReady {
+			if condition.Status == apiv1.ConditionTrue {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+type informer struct {
+	controller cache.Controller
+	store      cache.Store
+	stopChan   chan struct{}
+}
+
+type federatedInformerImpl struct {
+	sync.Mutex
+
+	// Informer on federated clusters.
+	clusterInformer informer
+
+	// Target informers factory
+	targetInformerFactory TargetInformerFactory
+
+	// Structures returned by targetInformerFactory
+	targetInformers map[string]informer
+
+	// A function to build clients.
+	clientFactory func(*federationapi.Cluster) (kubeclientset.Interface, error)
+}
+
+// *federatedInformerImpl implements FederatedInformer interface.
+var _ FederatedInformer = &federatedInformerImpl{}
+
+type federatedStoreImpl struct {
+	federatedInformer *federatedInformerImpl
+}
+
+func (f *federatedInformerImpl) Stop() {
+	glog.V(4).Infof("Stopping federated informer.")
+	f.Lock()
+	defer f.Unlock()
+
+	glog.V(4).Infof("... Closing cluster informer channel.")
+	close(f.clusterInformer.stopChan)
+	for key, informer := range f.targetInformers {
+		glog.V(4).Infof("... Closing informer channel for %q.", key)
+		close(informer.stopChan)
+		// Remove each informer after it has been stopped to prevent
+		// subsequent cluster deletion from attempting to double close
+		// an informer's stop channel.
+		delete(f.targetInformers, key)
+	}
+}
+
+func (f *federatedInformerImpl) Start() {
+	f.Lock()
+	defer f.Unlock()
+
+	f.clusterInformer.stopChan = make(chan struct{})
+	go f.clusterInformer.controller.Run(f.clusterInformer.stopChan)
+}
+
+func (f *federatedInformerImpl) SetClientFactory(clientFactory func(*federationapi.Cluster) (kubeclientset.Interface, error)) {
+	f.Lock()
+	defer f.Unlock()
+
+	f.clientFactory = clientFactory
+}
+
+// GetClientsetForCluster returns a clientset for the cluster, if present.
+func (f *federatedInformerImpl) GetClientsetForCluster(clusterName string) (kubeclientset.Interface, error) {
+	f.Lock()
+	defer f.Unlock()
+	return f.getClientsetForClusterUnlocked(clusterName)
+}
+
+func (f *federatedInformerImpl) getClientsetForClusterUnlocked(clusterName string) (kubeclientset.Interface, error) {
+	// No locking needed. Will happen in f.GetCluster.
+	glog.V(4).Infof("Getting clientset for cluster %q", clusterName)
+	if cluster, found, err := f.getReadyClusterUnlocked(clusterName); found && err == nil {
+		glog.V(4).Infof("Got clientset for cluster %q", clusterName)
+		return f.clientFactory(cluster)
+	} else {
+		if err != nil {
+			return nil, err
+		}
+	}
+	return nil, fmt.Errorf("cluster %q not found", clusterName)
+}
+
+func (f *federatedInformerImpl) GetUnreadyClusters() ([]*federationapi.Cluster, error) {
+	f.Lock()
+	defer f.Unlock()
+
+	items := f.clusterInformer.store.List()
+	result := make([]*federationapi.Cluster, 0, len(items))
+	for _, item := range items {
+		if cluster, ok := item.(*federationapi.Cluster); ok {
+			if !isClusterReady(cluster) {
+				result = append(result, cluster)
+			}
+		} else {
+			return nil, fmt.Errorf("wrong data in FederatedInformerImpl cluster store: %v", item)
+		}
+	}
+	return result, nil
+}
+
+// GetReadyClusters returns all clusters for which the sub-informers are run.
+func (f *federatedInformerImpl) GetReadyClusters() ([]*federationapi.Cluster, error) {
+	f.Lock()
+	defer f.Unlock()
+
+	items := f.clusterInformer.store.List()
+	result := make([]*federationapi.Cluster, 0, len(items))
+	for _, item := range items {
+		if cluster, ok := item.(*federationapi.Cluster); ok {
+			if isClusterReady(cluster) {
+				result = append(result, cluster)
+			}
+		} else {
+			return nil, fmt.Errorf("wrong data in FederatedInformerImpl cluster store: %v", item)
+		}
+	}
+	return result, nil
+}
+
+// GetCluster returns the cluster with the given name, if found.
+func (f *federatedInformerImpl) GetReadyCluster(name string) (*federationapi.Cluster, bool, error) {
+	f.Lock()
+	defer f.Unlock()
+	return f.getReadyClusterUnlocked(name)
+}
+
+func (f *federatedInformerImpl) getReadyClusterUnlocked(name string) (*federationapi.Cluster, bool, error) {
+	if obj, exist, err := f.clusterInformer.store.GetByKey(name); exist && err == nil {
+		if cluster, ok := obj.(*federationapi.Cluster); ok {
+			if isClusterReady(cluster) {
+				return cluster, true, nil
+			}
+			return nil, false, nil
+
+		}
+		return nil, false, fmt.Errorf("wrong data in FederatedInformerImpl cluster store: %v", obj)
+
+	} else {
+		return nil, false, err
+	}
+}
+
+// Synced returns true if the view is synced (for the first time)
+func (f *federatedInformerImpl) ClustersSynced() bool {
+	return f.clusterInformer.controller.HasSynced()
+}
+
+// Adds the given cluster to federated informer.
+func (f *federatedInformerImpl) addCluster(cluster *federationapi.Cluster) {
+	f.Lock()
+	defer f.Unlock()
+	name := cluster.Name
+	if client, err := f.getClientsetForClusterUnlocked(name); err == nil {
+		store, controller := f.targetInformerFactory(cluster, client)
+		targetInformer := informer{
+			controller: controller,
+			store:      store,
+			stopChan:   make(chan struct{}),
+		}
+		f.targetInformers[name] = targetInformer
+		go targetInformer.controller.Run(targetInformer.stopChan)
+	} else {
+		// TODO: create also an event for cluster.
+		glog.Errorf("Failed to create a client for cluster: %v", err)
+	}
+}
+
+// Removes the cluster from federated informer.
+func (f *federatedInformerImpl) deleteCluster(cluster *federationapi.Cluster) {
+	f.Lock()
+	defer f.Unlock()
+	name := cluster.Name
+	if targetInformer, found := f.targetInformers[name]; found {
+		close(targetInformer.stopChan)
+	}
+	delete(f.targetInformers, name)
+}
+
+// Returns a store created over all stores from target informers.
+func (f *federatedInformerImpl) GetTargetStore() FederatedReadOnlyStore {
+	return &federatedStoreImpl{
+		federatedInformer: f,
+	}
+}
+
+// Returns all items in the store.
+func (fs *federatedStoreImpl) List() ([]FederatedObject, error) {
+	fs.federatedInformer.Lock()
+	defer fs.federatedInformer.Unlock()
+
+	result := make([]FederatedObject, 0)
+	for clusterName, targetInformer := range fs.federatedInformer.targetInformers {
+		for _, value := range targetInformer.store.List() {
+			result = append(result, FederatedObject{ClusterName: clusterName, Object: value})
+		}
+	}
+	return result, nil
+}
+
+// Returns all items in the given cluster.
+func (fs *federatedStoreImpl) ListFromCluster(clusterName string) ([]interface{}, error) {
+	fs.federatedInformer.Lock()
+	defer fs.federatedInformer.Unlock()
+
+	result := make([]interface{}, 0)
+	if targetInformer, found := fs.federatedInformer.targetInformers[clusterName]; found {
+		values := targetInformer.store.List()
+		result = append(result, values...)
+	}
+	return result, nil
+}
+
+// GetByKey returns the item stored under the given key in the specified cluster (if exist).
+func (fs *federatedStoreImpl) GetByKey(clusterName string, key string) (interface{}, bool, error) {
+	fs.federatedInformer.Lock()
+	defer fs.federatedInformer.Unlock()
+	if targetInformer, found := fs.federatedInformer.targetInformers[clusterName]; found {
+		return targetInformer.store.GetByKey(key)
+	}
+	return nil, false, nil
+}
+
+// Returns the items stored under the given key in all clusters.
+func (fs *federatedStoreImpl) GetFromAllClusters(key string) ([]FederatedObject, error) {
+	fs.federatedInformer.Lock()
+	defer fs.federatedInformer.Unlock()
+
+	result := make([]FederatedObject, 0)
+	for clusterName, targetInformer := range fs.federatedInformer.targetInformers {
+		value, exist, err := targetInformer.store.GetByKey(key)
+		if err != nil {
+			return nil, err
+		}
+		if exist {
+			result = append(result, FederatedObject{ClusterName: clusterName, Object: value})
+		}
+	}
+	return result, nil
+}
+
+// GetKeyFor returns the key under which the item would be put in the store.
+func (fs *federatedStoreImpl) GetKeyFor(item interface{}) string {
+	// TODO: support other keying functions.
+	key, _ := cache.DeletionHandlingMetaNamespaceKeyFunc(item)
+	return key
+}
+
+// Checks whether stores for all clusters form the lists (and only these) are there and
+// are synced.
+func (fs *federatedStoreImpl) ClustersSynced(clusters []*federationapi.Cluster) bool {
+
+	// Get the list of informers to check under a lock and check it outside.
+	okSoFar, informersToCheck := func() (bool, []informer) {
+		fs.federatedInformer.Lock()
+		defer fs.federatedInformer.Unlock()
+
+		if len(fs.federatedInformer.targetInformers) != len(clusters) {
+			return false, []informer{}
+		}
+		informersToCheck := make([]informer, 0, len(clusters))
+		for _, cluster := range clusters {
+			if targetInformer, found := fs.federatedInformer.targetInformers[cluster.Name]; found {
+				informersToCheck = append(informersToCheck, targetInformer)
+			} else {
+				return false, []informer{}
+			}
+		}
+		return true, informersToCheck
+	}()
+
+	if !okSoFar {
+		return false
+	}
+	for _, informerToCheck := range informersToCheck {
+		if !informerToCheck.controller.HasSynced() {
+			return false
+		}
+	}
+	return true
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/federated_informer_test.go

@@ -0,0 +1,150 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"testing"
+	"time"
+
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	fakekubeclientset "k8s.io/client-go/kubernetes/fake"
+	core "k8s.io/client-go/testing"
+	"k8s.io/client-go/tools/cache"
+	federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	fakefederationclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset/fake"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// Basic test for Federated Informer. Checks whether the subinformer are added and deleted
+// when the corresponding cluster entries appear and disappear from etcd.
+func TestFederatedInformer(t *testing.T) {
+	fakeFederationClient := &fakefederationclientset.Clientset{}
+
+	// Add a single cluster to federation and remove it when needed.
+	cluster := federationapi.Cluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "mycluster",
+		},
+		Status: federationapi.ClusterStatus{
+			Conditions: []federationapi.ClusterCondition{
+				{Type: federationapi.ClusterReady, Status: apiv1.ConditionTrue},
+			},
+		},
+	}
+	fakeFederationClient.AddReactor("list", "clusters", func(action core.Action) (bool, runtime.Object, error) {
+		return true, &federationapi.ClusterList{Items: []federationapi.Cluster{cluster}}, nil
+	})
+	deleteChan := make(chan struct{})
+	fakeFederationClient.AddWatchReactor("clusters", func(action core.Action) (bool, watch.Interface, error) {
+		fakeWatch := watch.NewFake()
+		go func() {
+			<-deleteChan
+			fakeWatch.Delete(&cluster)
+		}()
+		return true, fakeWatch, nil
+	})
+
+	fakeKubeClient := &fakekubeclientset.Clientset{}
+	// There is a single service ns1/s1 in cluster mycluster.
+	service := apiv1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "ns1",
+			Name:      "s1",
+		},
+	}
+	fakeKubeClient.AddReactor("list", "services", func(action core.Action) (bool, runtime.Object, error) {
+		return true, &apiv1.ServiceList{Items: []apiv1.Service{service}}, nil
+	})
+	fakeKubeClient.AddWatchReactor("services", func(action core.Action) (bool, watch.Interface, error) {
+		return true, watch.NewFake(), nil
+	})
+
+	targetInformerFactory := func(cluster *federationapi.Cluster, clientset kubeclientset.Interface) (cache.Store, cache.Controller) {
+		return cache.NewInformer(
+			&cache.ListWatch{
+				ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+					return clientset.Core().Services(metav1.NamespaceAll).List(options)
+				},
+				WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+					return clientset.Core().Services(metav1.NamespaceAll).Watch(options)
+				},
+			},
+			&apiv1.Service{},
+			10*time.Second,
+			cache.ResourceEventHandlerFuncs{})
+	}
+
+	addedClusters := make(chan string, 1)
+	deletedClusters := make(chan string, 1)
+	lifecycle := ClusterLifecycleHandlerFuncs{
+		ClusterAvailable: func(cluster *federationapi.Cluster) {
+			addedClusters <- cluster.Name
+			close(addedClusters)
+		},
+		ClusterUnavailable: func(cluster *federationapi.Cluster, _ []interface{}) {
+			deletedClusters <- cluster.Name
+			close(deletedClusters)
+		},
+	}
+
+	informer := NewFederatedInformer(fakeFederationClient, targetInformerFactory, &lifecycle).(*federatedInformerImpl)
+	informer.clientFactory = func(cluster *federationapi.Cluster) (kubeclientset.Interface, error) {
+		return fakeKubeClient, nil
+	}
+	assert.NotNil(t, informer)
+	informer.Start()
+
+	// Wait until mycluster is synced.
+	for !informer.GetTargetStore().ClustersSynced([]*federationapi.Cluster{&cluster}) {
+		time.Sleep(time.Millisecond * 100)
+	}
+	readyClusters, err := informer.GetReadyClusters()
+	assert.NoError(t, err)
+	assert.Contains(t, readyClusters, &cluster)
+	serviceList, err := informer.GetTargetStore().List()
+	assert.NoError(t, err)
+	federatedService := FederatedObject{ClusterName: "mycluster", Object: &service}
+	assert.Contains(t, serviceList, federatedService)
+	service1, found, err := informer.GetTargetStore().GetByKey("mycluster", "ns1/s1")
+	assert.NoError(t, err)
+	assert.True(t, found)
+	assert.EqualValues(t, &service, service1)
+	assert.Equal(t, "mycluster", <-addedClusters)
+
+	// All checked, lets delete the cluster.
+	deleteChan <- struct{}{}
+	for !informer.GetTargetStore().ClustersSynced([]*federationapi.Cluster{}) {
+		time.Sleep(time.Millisecond * 100)
+	}
+	readyClusters, err = informer.GetReadyClusters()
+	assert.NoError(t, err)
+	assert.Empty(t, readyClusters)
+
+	serviceList, err = informer.GetTargetStore().List()
+	assert.NoError(t, err)
+	assert.Empty(t, serviceList)
+
+	assert.Equal(t, "mycluster", <-deletedClusters)
+
+	// Test complete.
+	informer.Stop()
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/federated_updater.go

@@ -0,0 +1,157 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	pkgruntime "k8s.io/apimachinery/pkg/runtime"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/kubernetes/pkg/api"
+)
+
+// Type of the operation that can be executed in Federated.
+type FederatedOperationType string
+
+const (
+	OperationTypeAdd    = "add"
+	OperationTypeUpdate = "update"
+	OperationTypeDelete = "delete"
+)
+
+// FederatedOperation definition contains type (add/update/delete) and the object itself.
+type FederatedOperation struct {
+	Type        FederatedOperationType
+	ClusterName string
+	Obj         pkgruntime.Object
+	Key         string
+}
+
+// A helper that executes the given set of updates on federation, in parallel.
+type FederatedUpdater interface {
+	// Executes the given set of operations.
+	Update([]FederatedOperation) error
+}
+
+// A function that executes some operation using the passed client and object.
+type FederatedOperationHandler func(kubeclientset.Interface, pkgruntime.Object) error
+
+type federatedUpdaterImpl struct {
+	federation FederationView
+
+	kind string
+
+	timeout time.Duration
+
+	eventRecorder record.EventRecorder
+
+	addFunction    FederatedOperationHandler
+	updateFunction FederatedOperationHandler
+	deleteFunction FederatedOperationHandler
+}
+
+func NewFederatedUpdater(federation FederationView, kind string, timeout time.Duration, recorder record.EventRecorder, add, update, del FederatedOperationHandler) FederatedUpdater {
+	return &federatedUpdaterImpl{
+		federation:     federation,
+		kind:           kind,
+		timeout:        timeout,
+		eventRecorder:  recorder,
+		addFunction:    add,
+		updateFunction: update,
+		deleteFunction: del,
+	}
+}
+
+func (fu *federatedUpdaterImpl) recordEvent(obj runtime.Object, eventType, eventVerb string, args ...interface{}) {
+	messageFmt := eventVerb + " %s %q in cluster %s"
+	fu.eventRecorder.Eventf(obj, api.EventTypeNormal, eventType, messageFmt, args...)
+}
+
+// Update executes the given set of operations within the timeout specified for
+// the instance. Timeout is best-effort. There is no guarantee that the
+// underlying operations are stopped when it is reached. However the function
+// will return after the timeout with a non-nil error.
+func (fu *federatedUpdaterImpl) Update(ops []FederatedOperation) error {
+	done := make(chan error, len(ops))
+	for _, op := range ops {
+		go func(op FederatedOperation) {
+			clusterName := op.ClusterName
+
+			// TODO: Ensure that the clientset has reasonable timeout.
+			clientset, err := fu.federation.GetClientsetForCluster(clusterName)
+			if err != nil {
+				done <- err
+				return
+			}
+
+			eventArgs := []interface{}{fu.kind, op.Key, clusterName}
+			baseEventType := fmt.Sprintf("%s", op.Type)
+			eventType := fmt.Sprintf("%sInCluster", strings.Title(baseEventType))
+
+			switch op.Type {
+			case OperationTypeAdd:
+				// TODO s+OperationTypeAdd+OperationTypeCreate+
+				baseEventType = "create"
+				eventType := "CreateInCluster"
+
+				fu.recordEvent(op.Obj, eventType, "Creating", eventArgs...)
+				err = fu.addFunction(clientset, op.Obj)
+			case OperationTypeUpdate:
+				fu.recordEvent(op.Obj, eventType, "Updating", eventArgs...)
+				err = fu.updateFunction(clientset, op.Obj)
+			case OperationTypeDelete:
+				fu.recordEvent(op.Obj, eventType, "Deleting", eventArgs...)
+				err = fu.deleteFunction(clientset, op.Obj)
+				// IsNotFound error is fine since that means the object is deleted already.
+				if errors.IsNotFound(err) {
+					err = nil
+				}
+			}
+
+			if err != nil {
+				eventType := eventType + "Failed"
+				messageFmt := "Failed to " + baseEventType + " %s %q in cluster %s: %v"
+				eventArgs = append(eventArgs, err)
+				fu.eventRecorder.Eventf(op.Obj, api.EventTypeWarning, eventType, messageFmt, eventArgs...)
+			}
+
+			done <- err
+		}(op)
+	}
+	start := time.Now()
+	for i := 0; i < len(ops); i++ {
+		now := time.Now()
+		if !now.Before(start.Add(fu.timeout)) {
+			return fmt.Errorf("failed to finish all operations in %v", fu.timeout)
+		}
+		select {
+		case err := <-done:
+			if err != nil {
+				return err
+			}
+		case <-time.After(start.Add(fu.timeout).Sub(now)):
+			return fmt.Errorf("failed to finish all operations in %v", fu.timeout)
+		}
+	}
+	// All operations finished in time.
+	return nil
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/federated_updater_test.go

@@ -0,0 +1,157 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pkgruntime "k8s.io/apimachinery/pkg/runtime"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	fakekubeclientset "k8s.io/client-go/kubernetes/fake"
+	federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// Fake federation view.
+type fakeFederationView struct {
+}
+
+// Verify that fakeFederationView implements FederationView interface
+var _ FederationView = &fakeFederationView{}
+
+func (f *fakeFederationView) GetClientsetForCluster(clusterName string) (kubeclientset.Interface, error) {
+	return &fakekubeclientset.Clientset{}, nil
+}
+
+func (f *fakeFederationView) GetReadyClusters() ([]*federationapi.Cluster, error) {
+	return []*federationapi.Cluster{}, nil
+}
+
+func (f *fakeFederationView) GetUnreadyClusters() ([]*federationapi.Cluster, error) {
+	return []*federationapi.Cluster{}, nil
+}
+
+func (f *fakeFederationView) GetReadyCluster(name string) (*federationapi.Cluster, bool, error) {
+	return nil, false, nil
+}
+
+func (f *fakeFederationView) ClustersSynced() bool {
+	return true
+}
+
+type fakeEventRecorder struct{}
+
+func (f *fakeEventRecorder) Event(object pkgruntime.Object, eventtype, reason, message string) {}
+func (f *fakeEventRecorder) Eventf(object pkgruntime.Object, eventtype, reason, messageFmt string, args ...interface{}) {
+}
+func (f *fakeEventRecorder) PastEventf(object pkgruntime.Object, timestamp metav1.Time, eventtype, reason, messageFmt string, args ...interface{}) {
+}
+
+func TestFederatedUpdaterOK(t *testing.T) {
+	addChan := make(chan string, 5)
+	updateChan := make(chan string, 5)
+
+	updater := NewFederatedUpdater(&fakeFederationView{}, "foo", time.Minute, &fakeEventRecorder{},
+		func(_ kubeclientset.Interface, obj pkgruntime.Object) error {
+			service := obj.(*apiv1.Service)
+			addChan <- service.Name
+			return nil
+		},
+		func(_ kubeclientset.Interface, obj pkgruntime.Object) error {
+			service := obj.(*apiv1.Service)
+			updateChan <- service.Name
+			return nil
+		},
+		noop)
+
+	err := updater.Update([]FederatedOperation{
+		{
+			Type: OperationTypeAdd,
+			Obj:  makeService("A", "s1"),
+		},
+		{
+			Type: OperationTypeUpdate,
+			Obj:  makeService("B", "s2"),
+		},
+	})
+	assert.NoError(t, err)
+	add := <-addChan
+	update := <-updateChan
+	assert.Equal(t, "s1", add)
+	assert.Equal(t, "s2", update)
+}
+
+func TestFederatedUpdaterError(t *testing.T) {
+	updater := NewFederatedUpdater(&fakeFederationView{}, "foo", time.Minute, &fakeEventRecorder{},
+		func(_ kubeclientset.Interface, obj pkgruntime.Object) error {
+			return fmt.Errorf("boom")
+		}, noop, noop)
+
+	err := updater.Update([]FederatedOperation{
+		{
+			Type: OperationTypeAdd,
+			Obj:  makeService("A", "s1"),
+		},
+		{
+			Type: OperationTypeUpdate,
+			Obj:  makeService("B", "s1"),
+		},
+	})
+	assert.Error(t, err)
+}
+
+func TestFederatedUpdaterTimeout(t *testing.T) {
+	start := time.Now()
+	updater := NewFederatedUpdater(&fakeFederationView{}, "foo", time.Second, &fakeEventRecorder{},
+		func(_ kubeclientset.Interface, obj pkgruntime.Object) error {
+			time.Sleep(time.Minute)
+			return nil
+		},
+		noop, noop)
+
+	err := updater.Update([]FederatedOperation{
+		{
+			Type: OperationTypeAdd,
+			Obj:  makeService("A", "s1"),
+		},
+		{
+			Type: OperationTypeUpdate,
+			Obj:  makeService("B", "s1"),
+		},
+	})
+	end := time.Now()
+	assert.Error(t, err)
+	assert.True(t, start.Add(10*time.Second).After(end))
+}
+
+func makeService(cluster, name string) *apiv1.Service {
+	return &apiv1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "ns1",
+			Name:      name,
+		},
+	}
+}
+
+func noop(_ kubeclientset.Interface, _ pkgruntime.Object) error {
+	return nil
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/finalizers/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["finalizers.go"],
+    deps = [
+        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["finalizers_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+    ],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/finalizers/finalizers.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Helper functions for manipulating finalizers.
+package finalizers
+
+import (
+	meta "k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+// HasFinalizer returns true if the given object has the given finalizer in its ObjectMeta.
+func HasFinalizer(obj runtime.Object, finalizer string) (bool, error) {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return false, err
+	}
+	finalizers := sets.NewString(accessor.GetFinalizers()...)
+	return finalizers.Has(finalizer), nil
+}
+
+// AddFinalizers adds the given finalizers to the given objects ObjectMeta.
+// Returns true if the object was updated.
+func AddFinalizers(obj runtime.Object, newFinalizers sets.String) (bool, error) {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return false, err
+	}
+	oldFinalizers := sets.NewString(accessor.GetFinalizers()...)
+	if oldFinalizers.IsSuperset(newFinalizers) {
+		return false, nil
+	}
+	allFinalizers := oldFinalizers.Union(newFinalizers)
+	accessor.SetFinalizers(allFinalizers.List())
+	return true, nil
+}
+
+// RemoveFinalizers removes the given finalizers from the given objects ObjectMeta.
+// Returns true if the object was updated.
+func RemoveFinalizers(obj runtime.Object, finalizers sets.String) (bool, error) {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return false, err
+	}
+	oldFinalizers := sets.NewString(accessor.GetFinalizers()...)
+	if oldFinalizers.Intersection(finalizers).Len() == 0 {
+		return false, nil
+	}
+	newFinalizers := oldFinalizers.Difference(finalizers)
+	accessor.SetFinalizers(newFinalizers.List())
+	return true, nil
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/finalizers/finalizers_test.go

@@ -0,0 +1,171 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package finalizers
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+func newObj(finalizers []string) runtime.Object {
+	pod := v1.Pod{}
+	pod.ObjectMeta.Finalizers = finalizers
+	return &pod
+}
+
+func TestHasFinalizer(t *testing.T) {
+	testCases := []struct {
+		obj       runtime.Object
+		finalizer string
+		result    bool
+	}{
+		{
+			newObj([]string{}),
+			"",
+			false,
+		},
+		{
+			newObj([]string{}),
+			"someFinalizer",
+			false,
+		},
+		{
+			newObj([]string{"someFinalizer"}),
+			"",
+			false,
+		},
+		{
+			newObj([]string{"someFinalizer"}),
+			"anotherFinalizer",
+			false,
+		},
+		{
+			newObj([]string{"someFinalizer"}),
+			"someFinalizer",
+			true,
+		},
+		{
+			newObj([]string{"anotherFinalizer", "someFinalizer"}),
+			"someFinalizer",
+			true,
+		},
+	}
+	for index, test := range testCases {
+		hasFinalizer, _ := HasFinalizer(test.obj, test.finalizer)
+		assert.Equal(t, hasFinalizer, test.result, fmt.Sprintf("Test case %d failed. Expected: %v, actual: %v", index, test.result, hasFinalizer))
+	}
+}
+
+func TestAddFinalizers(t *testing.T) {
+	testCases := []struct {
+		obj           runtime.Object
+		finalizers    sets.String
+		isUpdated     bool
+		newFinalizers []string
+	}{
+		{
+			newObj([]string{}),
+			sets.NewString(),
+			false,
+			[]string{},
+		},
+		{
+			newObj([]string{}),
+			sets.NewString("someFinalizer"),
+			true,
+			[]string{"someFinalizer"},
+		},
+		{
+			newObj([]string{"someFinalizer"}),
+			sets.NewString(),
+			false,
+			[]string{"someFinalizer"},
+		},
+		{
+			newObj([]string{"someFinalizer"}),
+			sets.NewString("anotherFinalizer"),
+			true,
+			[]string{"anotherFinalizer", "someFinalizer"},
+		},
+		{
+			newObj([]string{"someFinalizer"}),
+			sets.NewString("someFinalizer"),
+			false,
+			[]string{"someFinalizer"},
+		},
+	}
+	for index, test := range testCases {
+		isUpdated, _ := AddFinalizers(test.obj, test.finalizers)
+		assert.Equal(t, isUpdated, test.isUpdated, fmt.Sprintf("Test case %d failed. Expected isUpdated: %v, actual: %v", index, test.isUpdated, isUpdated))
+		accessor, _ := meta.Accessor(test.obj)
+		newFinalizers := accessor.GetFinalizers()
+		assert.Equal(t, test.newFinalizers, newFinalizers, fmt.Sprintf("Test case %d failed. Expected finalizers: %v, actual: %v", index, test.newFinalizers, newFinalizers))
+	}
+}
+
+func TestRemoveFinalizers(t *testing.T) {
+	testCases := []struct {
+		obj           runtime.Object
+		finalizers    sets.String
+		isUpdated     bool
+		newFinalizers []string
+	}{
+		{
+			newObj([]string{}),
+			sets.NewString(),
+			false,
+			[]string{},
+		},
+		{
+			newObj([]string{}),
+			sets.NewString("someFinalizer"),
+			false,
+			[]string{},
+		},
+		{
+			newObj([]string{"someFinalizer"}),
+			sets.NewString(),
+			false,
+			[]string{"someFinalizer"},
+		},
+		{
+			newObj([]string{"someFinalizer"}),
+			sets.NewString("anotherFinalizer"),
+			false,
+			[]string{"someFinalizer"},
+		},
+		{
+			newObj([]string{"someFinalizer", "anotherFinalizer"}),
+			sets.NewString("someFinalizer"),
+			true,
+			[]string{"anotherFinalizer"},
+		},
+	}
+	for index, test := range testCases {
+		isUpdated, _ := RemoveFinalizers(test.obj, test.finalizers)
+		assert.Equal(t, isUpdated, test.isUpdated, fmt.Sprintf("Test case %d failed. Expected isUpdated: %v, actual: %v", index, test.isUpdated, isUpdated))
+		accessor, _ := meta.Accessor(test.obj)
+		newFinalizers := accessor.GetFinalizers()
+		assert.Equal(t, test.newFinalizers, newFinalizers, fmt.Sprintf("Test case %d failed. Expected finalizers: %v, actual: %v", index, test.newFinalizers, newFinalizers))
+	}
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/handlers.go

@@ -0,0 +1,112 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"reflect"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pkgruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/cache"
+)
+
+// Returns cache.ResourceEventHandlerFuncs that trigger the given function
+// on all object changes.
+func NewTriggerOnAllChanges(triggerFunc func(pkgruntime.Object)) *cache.ResourceEventHandlerFuncs {
+	return &cache.ResourceEventHandlerFuncs{
+		DeleteFunc: func(old interface{}) {
+			oldObj := old.(pkgruntime.Object)
+			triggerFunc(oldObj)
+		},
+		AddFunc: func(cur interface{}) {
+			curObj := cur.(pkgruntime.Object)
+			triggerFunc(curObj)
+		},
+		UpdateFunc: func(old, cur interface{}) {
+			curObj := cur.(pkgruntime.Object)
+			if !reflect.DeepEqual(old, cur) {
+				triggerFunc(curObj)
+			}
+		},
+	}
+}
+
+// Returns cache.ResourceEventHandlerFuncs that trigger the given function
+// on object add and delete as well as spec/object meta on update.
+func NewTriggerOnMetaAndSpecChanges(triggerFunc func(pkgruntime.Object)) *cache.ResourceEventHandlerFuncs {
+	getFieldOrPanic := func(obj interface{}, fieldName string) interface{} {
+		val := reflect.ValueOf(obj).Elem().FieldByName(fieldName)
+		if val.IsValid() {
+			return val.Interface()
+		} else {
+			panic(fmt.Errorf("field not found: %s", fieldName))
+		}
+	}
+	return &cache.ResourceEventHandlerFuncs{
+		DeleteFunc: func(old interface{}) {
+			oldObj := old.(pkgruntime.Object)
+			triggerFunc(oldObj)
+		},
+		AddFunc: func(cur interface{}) {
+			curObj := cur.(pkgruntime.Object)
+			triggerFunc(curObj)
+		},
+		UpdateFunc: func(old, cur interface{}) {
+			curObj := cur.(pkgruntime.Object)
+			oldMeta := getFieldOrPanic(old, "ObjectMeta").(metav1.ObjectMeta)
+			curMeta := getFieldOrPanic(cur, "ObjectMeta").(metav1.ObjectMeta)
+			if !ObjectMetaEquivalent(oldMeta, curMeta) ||
+				!reflect.DeepEqual(oldMeta.DeletionTimestamp, curMeta.DeletionTimestamp) ||
+				!reflect.DeepEqual(getFieldOrPanic(old, "Spec"), getFieldOrPanic(cur, "Spec")) {
+				triggerFunc(curObj)
+			}
+		},
+	}
+}
+
+// Returns cache.ResourceEventHandlerFuncs that trigger the given function
+// on object add/delete or ObjectMeta or given field is updated.
+func NewTriggerOnMetaAndFieldChanges(field string, triggerFunc func(pkgruntime.Object)) *cache.ResourceEventHandlerFuncs {
+	getFieldOrPanic := func(obj interface{}, fieldName string) interface{} {
+		val := reflect.ValueOf(obj).Elem().FieldByName(fieldName)
+		if val.IsValid() {
+			return val.Interface()
+		} else {
+			panic(fmt.Errorf("field not found: %s", fieldName))
+		}
+	}
+	return &cache.ResourceEventHandlerFuncs{
+		DeleteFunc: func(old interface{}) {
+			oldObj := old.(pkgruntime.Object)
+			triggerFunc(oldObj)
+		},
+		AddFunc: func(cur interface{}) {
+			curObj := cur.(pkgruntime.Object)
+			triggerFunc(curObj)
+		},
+		UpdateFunc: func(old, cur interface{}) {
+			curObj := cur.(pkgruntime.Object)
+			oldMeta := getFieldOrPanic(old, "ObjectMeta").(metav1.ObjectMeta)
+			curMeta := getFieldOrPanic(cur, "ObjectMeta").(metav1.ObjectMeta)
+			if !ObjectMetaEquivalent(oldMeta, curMeta) ||
+				!reflect.DeepEqual(getFieldOrPanic(old, field), getFieldOrPanic(cur, field)) {
+				triggerFunc(curObj)
+			}
+		},
+	}
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/handlers_test.go

@@ -0,0 +1,100 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"testing"
+
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pkgruntime "k8s.io/apimachinery/pkg/runtime"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHandlers(t *testing.T) {
+	// There is a single service ns1/s1 in cluster mycluster.
+	service := apiv1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "ns1",
+			Name:      "s1",
+		},
+	}
+	service2 := apiv1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "ns1",
+			Name:      "s1",
+			Annotations: map[string]string{
+				"A": "B",
+			},
+		},
+	}
+	triggerChan := make(chan struct{}, 1)
+	triggered := func() bool {
+		select {
+		case <-triggerChan:
+			return true
+		default:
+			return false
+		}
+	}
+
+	trigger := NewTriggerOnAllChanges(
+		func(obj pkgruntime.Object) {
+			triggerChan <- struct{}{}
+		})
+
+	trigger.OnAdd(&service)
+	assert.True(t, triggered())
+	trigger.OnDelete(&service)
+	assert.True(t, triggered())
+	trigger.OnUpdate(&service, &service)
+	assert.False(t, triggered())
+	trigger.OnUpdate(&service, &service2)
+	assert.True(t, triggered())
+
+	trigger2 := NewTriggerOnMetaAndSpecChanges(
+		func(obj pkgruntime.Object) {
+			triggerChan <- struct{}{}
+		},
+	)
+
+	trigger2.OnAdd(&service)
+	assert.True(t, triggered())
+	trigger2.OnDelete(&service)
+	assert.True(t, triggered())
+	trigger2.OnUpdate(&service, &service)
+	assert.False(t, triggered())
+	trigger2.OnUpdate(&service, &service2)
+	assert.True(t, triggered())
+
+	service3 := apiv1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "ns1",
+			Name:      "s1",
+		},
+		Status: apiv1.ServiceStatus{
+			LoadBalancer: apiv1.LoadBalancerStatus{
+				Ingress: []apiv1.LoadBalancerIngress{{
+					Hostname: "A",
+				}},
+			},
+		},
+	}
+	trigger2.OnUpdate(&service, &service3)
+	assert.False(t, triggered())
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/hpa/BUILD

@@ -0,0 +1,37 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["hpa.go"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["hpa_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//vendor/github.com/stretchr/testify/require:go_default_library",
+        "//vendor/k8s.io/api/autoscaling/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/hpa/hpa.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hpa
+
+import (
+	"encoding/json"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+const (
+	// FederatedAnnotationOnHpaTargetObj as key, is used by hpa controller to
+	// set selected cluster name list as annotation on the target object.
+	FederatedAnnotationOnHpaTargetObj = "federation.kubernetes.io/hpa-target-cluster-list"
+)
+
+// ClusterNames stores the list of clusters represented by names as appearing on federation
+// cluster objects. This is set by federation hpa and used by target objects federation
+// controller to restrict that target object to only these clusters.
+type ClusterNames struct {
+	Names []string
+}
+
+func (cn *ClusterNames) String() string {
+	annotationBytes, _ := json.Marshal(cn)
+	return string(annotationBytes[:])
+}
+
+// GetHpaTargetClusterList is used to get the list of clusters from the target object
+// annotations.
+func GetHpaTargetClusterList(obj runtime.Object) (*ClusterNames, error) {
+	accessor, _ := meta.Accessor(obj)
+	targetObjAnno := accessor.GetAnnotations()
+	if targetObjAnno == nil {
+		return nil, nil
+	}
+	targetObjAnnoString, exists := targetObjAnno[FederatedAnnotationOnHpaTargetObj]
+	if !exists {
+		return nil, nil
+	}
+
+	clusterNames := &ClusterNames{}
+	if err := json.Unmarshal([]byte(targetObjAnnoString), clusterNames); err != nil {
+		return nil, err
+	}
+	return clusterNames, nil
+}
+
+// SetHpaTargetClusterList is used to set the list of clusters on the target object
+// annotations.
+func SetHpaTargetClusterList(obj runtime.Object, clusterNames ClusterNames) runtime.Object {
+	accessor, _ := meta.Accessor(obj)
+	anno := accessor.GetAnnotations()
+	if anno == nil {
+		anno = make(map[string]string)
+		accessor.SetAnnotations(anno)
+	}
+	anno[FederatedAnnotationOnHpaTargetObj] = clusterNames.String()
+	return obj
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/hpa/hpa_test.go

@@ -0,0 +1,115 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hpa
+
+import (
+	"testing"
+
+	autoscalingv1 "k8s.io/api/autoscaling/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetHpaTargetClusterList(t *testing.T) {
+	// Any object is fine for this test.
+	obj := &autoscalingv1.HorizontalPodAutoscaler{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "myhpa",
+			Namespace: "myNamespace",
+			SelfLink:  "/api/mylink",
+		},
+	}
+
+	testCases := map[string]struct {
+		clusterNames *ClusterNames
+		expectedErr  bool
+	}{
+		"Wrong data set on annotations should return unmarshalling error when retrieving": {
+			expectedErr: true,
+		},
+		"Get clusternames on annotations with 2 clusters, should have same names, which were set": {
+			clusterNames: &ClusterNames{
+				Names: []string{
+					"c1",
+					"c2",
+				},
+			},
+		},
+	}
+
+	for testName, testCase := range testCases {
+		t.Run(testName, func(t *testing.T) {
+			accessor, _ := meta.Accessor(obj)
+			anno := accessor.GetAnnotations()
+			if anno == nil {
+				anno = make(map[string]string)
+				accessor.SetAnnotations(anno)
+			}
+			if testCase.expectedErr {
+				anno[FederatedAnnotationOnHpaTargetObj] = "{" //some random string
+			} else {
+				anno[FederatedAnnotationOnHpaTargetObj] = testCase.clusterNames.String()
+			}
+
+			readNames, err := GetHpaTargetClusterList(obj)
+
+			if testCase.expectedErr {
+				require.Error(t, err, "An error was expected")
+			} else {
+				require.Equal(t, testCase.clusterNames, readNames, "Names should have been equal")
+			}
+		})
+	}
+}
+
+func TestSetHpaTargetClusterList(t *testing.T) {
+	// Any object is fine for this test.
+	obj := &autoscalingv1.HorizontalPodAutoscaler{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "myhpa",
+			Namespace: "myNamespace",
+			SelfLink:  "/api/mylink",
+		},
+	}
+
+	testCases := map[string]struct {
+		clusterNames ClusterNames
+		expectedErr  bool
+	}{
+		"Get clusternames on annotations with 2 clusters, should have same names, which were set": {
+			clusterNames: ClusterNames{
+				Names: []string{
+					"c1",
+					"c2",
+				},
+			},
+		},
+	}
+
+	for testName, testCase := range testCases {
+		t.Run(testName, func(t *testing.T) {
+
+			SetHpaTargetClusterList(obj, testCase.clusterNames)
+			readNames, err := GetHpaTargetClusterList(obj)
+			require.NoError(t, err, "An error should not have happened")
+			require.Equal(t, &testCase.clusterNames, readNames, "Names should have been equal")
+
+		})
+	}
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/meta.go

@@ -0,0 +1,94 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"reflect"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/kubernetes/pkg/api"
+)
+
+// Copies cluster-independent, user provided data from the given ObjectMeta struct. If in
+// the future the ObjectMeta structure is expanded then any field that is not populated
+// by the api server should be included here.
+func copyObjectMeta(obj metav1.ObjectMeta) metav1.ObjectMeta {
+	return metav1.ObjectMeta{
+		Name:        obj.Name,
+		Namespace:   obj.Namespace,
+		Labels:      obj.Labels,
+		Annotations: obj.Annotations,
+	}
+}
+
+// Deep copies cluster-independent, user provided data from the given ObjectMeta struct. If in
+// the future the ObjectMeta structure is expanded then any field that is not populated
+// by the api server should be included here.
+func DeepCopyRelevantObjectMeta(obj metav1.ObjectMeta) metav1.ObjectMeta {
+	copyMeta := copyObjectMeta(obj)
+	if obj.Labels != nil {
+		copyMeta.Labels = make(map[string]string)
+		for key, val := range obj.Labels {
+			copyMeta.Labels[key] = val
+		}
+	}
+	if obj.Annotations != nil {
+		copyMeta.Annotations = make(map[string]string)
+		for key, val := range obj.Annotations {
+			copyMeta.Annotations[key] = val
+		}
+	}
+	return copyMeta
+}
+
+// Checks if cluster-independent, user provided data in two given ObjectMeta are equal. If in
+// the future the ObjectMeta structure is expanded then any field that is not populated
+// by the api server should be included here.
+func ObjectMetaEquivalent(a, b metav1.ObjectMeta) bool {
+	if a.Name != b.Name {
+		return false
+	}
+	if a.Namespace != b.Namespace {
+		return false
+	}
+	if !reflect.DeepEqual(a.Labels, b.Labels) && (len(a.Labels) != 0 || len(b.Labels) != 0) {
+		return false
+	}
+	if !reflect.DeepEqual(a.Annotations, b.Annotations) && (len(a.Annotations) != 0 || len(b.Annotations) != 0) {
+		return false
+	}
+	return true
+}
+
+// Checks if cluster-independent, user provided data in ObjectMeta and Spec in two given top
+// level api objects are equivalent.
+func ObjectMetaAndSpecEquivalent(a, b runtime.Object) bool {
+	objectMetaA := reflect.ValueOf(a).Elem().FieldByName("ObjectMeta").Interface().(metav1.ObjectMeta)
+	objectMetaB := reflect.ValueOf(b).Elem().FieldByName("ObjectMeta").Interface().(metav1.ObjectMeta)
+	specA := reflect.ValueOf(a).Elem().FieldByName("Spec").Interface()
+	specB := reflect.ValueOf(b).Elem().FieldByName("Spec").Interface()
+	return ObjectMetaEquivalent(objectMetaA, objectMetaB) && reflect.DeepEqual(specA, specB)
+}
+
+func DeepCopyApiTypeOrPanic(item interface{}) interface{} {
+	result, err := api.Scheme.DeepCopy(item)
+	if err != nil {
+		panic(err)
+	}
+	return result
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/meta_test.go

@@ -0,0 +1,117 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"testing"
+
+	api_v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestObjectMeta(t *testing.T) {
+	o1 := metav1.ObjectMeta{
+		Namespace:       "ns1",
+		Name:            "s1",
+		UID:             "1231231412",
+		ResourceVersion: "999",
+	}
+	o2 := copyObjectMeta(o1)
+	o3 := metav1.ObjectMeta{
+		Namespace:   "ns1",
+		Name:        "s1",
+		UID:         "1231231412",
+		Annotations: map[string]string{"A": "B"},
+	}
+	o4 := metav1.ObjectMeta{
+		Namespace:   "ns1",
+		Name:        "s1",
+		UID:         "1231255531412",
+		Annotations: map[string]string{"A": "B"},
+	}
+	o5 := metav1.ObjectMeta{
+		Namespace:       "ns1",
+		Name:            "s1",
+		ResourceVersion: "1231231412",
+		Annotations:     map[string]string{"A": "B"},
+	}
+	o6 := metav1.ObjectMeta{
+		Namespace:       "ns1",
+		Name:            "s1",
+		ResourceVersion: "1231255531412",
+		Annotations:     map[string]string{"A": "B"},
+	}
+	o7 := metav1.ObjectMeta{
+		Namespace:       "ns1",
+		Name:            "s1",
+		ResourceVersion: "1231255531412",
+		Annotations:     map[string]string{},
+		Labels:          map[string]string{},
+	}
+	o8 := metav1.ObjectMeta{
+		Namespace:       "ns1",
+		Name:            "s1",
+		ResourceVersion: "1231255531412",
+	}
+	assert.Equal(t, 0, len(o2.UID))
+	assert.Equal(t, 0, len(o2.ResourceVersion))
+	assert.Equal(t, o1.Name, o2.Name)
+	assert.True(t, ObjectMetaEquivalent(o1, o2))
+	assert.False(t, ObjectMetaEquivalent(o1, o3))
+	assert.True(t, ObjectMetaEquivalent(o3, o4))
+	assert.True(t, ObjectMetaEquivalent(o5, o6))
+	assert.True(t, ObjectMetaEquivalent(o3, o5))
+	assert.True(t, ObjectMetaEquivalent(o7, o8))
+	assert.True(t, ObjectMetaEquivalent(o8, o7))
+}
+
+func TestObjectMetaAndSpec(t *testing.T) {
+	s1 := api_v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "ns1",
+			Name:      "s1",
+		},
+		Spec: api_v1.ServiceSpec{
+			ExternalName: "Service1",
+		},
+	}
+	s1b := s1
+	s2 := api_v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "ns1",
+			Name:      "s2",
+		},
+		Spec: api_v1.ServiceSpec{
+			ExternalName: "Service1",
+		},
+	}
+	s3 := api_v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "ns1",
+			Name:      "s1",
+		},
+		Spec: api_v1.ServiceSpec{
+			ExternalName: "Service2",
+		},
+	}
+	assert.True(t, ObjectMetaAndSpecEquivalent(&s1, &s1b))
+	assert.False(t, ObjectMetaAndSpecEquivalent(&s1, &s2))
+	assert.False(t, ObjectMetaAndSpecEquivalent(&s1, &s3))
+	assert.False(t, ObjectMetaAndSpecEquivalent(&s2, &s3))
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/planner/BUILD

@@ -0,0 +1,36 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["planner.go"],
+    deps = ["//federation/apis/federation:go_default_library"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["planner_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//federation/apis/federation:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/planner/planner.go

@@ -0,0 +1,238 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package planner
+
+import (
+	"hash/fnv"
+	"sort"
+
+	fedapi "k8s.io/kubernetes/federation/apis/federation"
+)
+
+// Planner decides how many out of the given replicas should be placed in each of the
+// federated clusters.
+type Planner struct {
+	preferences *fedapi.ReplicaAllocationPreferences
+}
+
+type namedClusterPreferences struct {
+	clusterName string
+	hash        uint32
+	fedapi.ClusterPreferences
+}
+
+type byWeight []*namedClusterPreferences
+
+func (a byWeight) Len() int      { return len(a) }
+func (a byWeight) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+// Preferences are sorted according by decreasing weight and increasing hash (built on top of cluster name and rs name).
+// Sorting is made by a hash to avoid assigning single-replica rs to the alphabetically smallest cluster.
+func (a byWeight) Less(i, j int) bool {
+	return (a[i].Weight > a[j].Weight) || (a[i].Weight == a[j].Weight && a[i].hash < a[j].hash)
+}
+
+func NewPlanner(preferences *fedapi.ReplicaAllocationPreferences) *Planner {
+	return &Planner{
+		preferences: preferences,
+	}
+}
+
+// Distribute the desired number of replicas among the given cluster according to the planner preferences.
+// The function tries its best to assign each cluster the preferred number of replicas, however if
+// sum of MinReplicas for all cluster is bigger thant replicasToDistribute then some cluster will not
+// have all of the replicas assigned. In such case a cluster with higher weight has priority over
+// cluster with lower weight (or with lexicographically smaller name in case of draw).
+// It can also use the current replica count and estimated capacity to provide better planning and
+// adhere to rebalance policy. To avoid prioritization of clusters with smaller lexicographical names
+// a semi-random string (like replica set name) can be provided.
+// Two maps are returned:
+// * a map that contains information how many replicas will be possible to run in a cluster.
+// * a map that contains information how many extra replicas would be nice to schedule in a cluster so,
+//   if by chance, they are scheduled we will be closer to the desired replicas layout.
+func (p *Planner) Plan(replicasToDistribute int64, availableClusters []string, currentReplicaCount map[string]int64,
+	estimatedCapacity map[string]int64, replicaSetKey string) (map[string]int64, map[string]int64) {
+
+	preferences := make([]*namedClusterPreferences, 0, len(availableClusters))
+	plan := make(map[string]int64, len(preferences))
+	overflow := make(map[string]int64, len(preferences))
+
+	named := func(name string, pref fedapi.ClusterPreferences) *namedClusterPreferences {
+		// Seems to work better than addler for our case.
+		hasher := fnv.New32()
+		hasher.Write([]byte(name))
+		hasher.Write([]byte(replicaSetKey))
+
+		return &namedClusterPreferences{
+			clusterName:        name,
+			hash:               hasher.Sum32(),
+			ClusterPreferences: pref,
+		}
+	}
+
+	for _, cluster := range availableClusters {
+		if localRSP, found := p.preferences.Clusters[cluster]; found {
+			preferences = append(preferences, named(cluster, localRSP))
+		} else {
+			if localRSP, found := p.preferences.Clusters["*"]; found {
+				preferences = append(preferences, named(cluster, localRSP))
+			} else {
+				plan[cluster] = int64(0)
+			}
+		}
+	}
+	sort.Sort(byWeight(preferences))
+
+	remainingReplicas := replicasToDistribute
+
+	// Assign each cluster the minimum number of replicas it requested.
+	for _, preference := range preferences {
+		min := minInt64(preference.MinReplicas, remainingReplicas)
+		if capacity, hasCapacity := estimatedCapacity[preference.clusterName]; hasCapacity {
+			min = minInt64(min, capacity)
+		}
+		remainingReplicas -= min
+		plan[preference.clusterName] = min
+	}
+
+	// This map contains information how many replicas were assigned to
+	// the cluster based only on the current replica count and
+	// rebalance=false preference. It will be later used in remaining replica
+	// distribution code.
+	preallocated := make(map[string]int64)
+
+	if p.preferences.Rebalance == false {
+		for _, preference := range preferences {
+			planned := plan[preference.clusterName]
+			count, hasSome := currentReplicaCount[preference.clusterName]
+			if hasSome && count > planned {
+				target := count
+				if preference.MaxReplicas != nil {
+					target = minInt64(*preference.MaxReplicas, target)
+				}
+				if capacity, hasCapacity := estimatedCapacity[preference.clusterName]; hasCapacity {
+					target = minInt64(capacity, target)
+				}
+				extra := minInt64(target-planned, remainingReplicas)
+				if extra < 0 {
+					extra = 0
+				}
+				remainingReplicas -= extra
+				preallocated[preference.clusterName] = extra
+				plan[preference.clusterName] = extra + planned
+			}
+		}
+	}
+
+	modified := true
+
+	// It is possible single pass of the loop is not enough to distribute all replicas among clusters due
+	// to weight, max and rounding corner cases. In such case we iterate until either
+	// there is no replicas or no cluster gets any more replicas or the number
+	// of attempts is less than available cluster count. If there is no preallocated pods
+	// every loop either distributes all remainingReplicas or maxes out at least one cluster.
+	// If there are preallocated then the replica spreading may take longer.
+	// We reduce the number of pending preallocated replicas by at least half with each iteration so
+	// we may need log(replicasAtStart) iterations.
+	// TODO: Prove that clusterCount * log(replicas) iterations solves the problem or adjust the number.
+	// TODO: This algorithm is O(clusterCount^2 * log(replicas)) which is good for up to 100 clusters.
+	// Find something faster.
+	for trial := 0; modified && remainingReplicas > 0; trial++ {
+
+		modified = false
+		weightSum := int64(0)
+		for _, preference := range preferences {
+			weightSum += preference.Weight
+		}
+		newPreferences := make([]*namedClusterPreferences, 0, len(preferences))
+
+		distributeInThisLoop := remainingReplicas
+
+		for _, preference := range preferences {
+			if weightSum > 0 {
+				start := plan[preference.clusterName]
+				// Distribute the remaining replicas, rounding fractions always up.
+				extra := (distributeInThisLoop*preference.Weight + weightSum - 1) / weightSum
+				extra = minInt64(extra, remainingReplicas)
+
+				// Account preallocated.
+				prealloc := preallocated[preference.clusterName]
+				usedPrealloc := minInt64(extra, prealloc)
+				preallocated[preference.clusterName] = prealloc - usedPrealloc
+				extra = extra - usedPrealloc
+				if usedPrealloc > 0 {
+					modified = true
+				}
+
+				// In total there should be the amount that was there at start plus whatever is due
+				// in this iteration
+				total := start + extra
+
+				// Check if we don't overflow the cluster, and if yes don't consider this cluster
+				// in any of the following iterations.
+				full := false
+				if preference.MaxReplicas != nil && total > *preference.MaxReplicas {
+					total = *preference.MaxReplicas
+					full = true
+				}
+				if capacity, hasCapacity := estimatedCapacity[preference.clusterName]; hasCapacity && total > capacity {
+					overflow[preference.clusterName] = total - capacity
+					total = capacity
+					full = true
+				}
+
+				if !full {
+					newPreferences = append(newPreferences, preference)
+				}
+
+				// Only total-start replicas were actually taken.
+				remainingReplicas -= (total - start)
+				plan[preference.clusterName] = total
+
+				// Something extra got scheduled on this cluster.
+				if total > start {
+					modified = true
+				}
+			} else {
+				break
+			}
+		}
+		preferences = newPreferences
+	}
+
+	if p.preferences.Rebalance {
+		return plan, overflow
+	} else {
+		// If rebalance = false then overflow is trimmed at the level
+		// of replicas that it failed to place somewhere.
+		newOverflow := make(map[string]int64)
+		for key, value := range overflow {
+			value = minInt64(value, remainingReplicas)
+			if value > 0 {
+				newOverflow[key] = value
+			}
+		}
+		return plan, newOverflow
+	}
+}
+
+func minInt64(a int64, b int64) int64 {
+	if a < b {
+		return a
+	}
+	return b
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/planner/planner_test.go

@@ -0,0 +1,348 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package planner
+
+import (
+	"testing"
+
+	fedapi "k8s.io/kubernetes/federation/apis/federation"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func doCheck(t *testing.T, pref map[string]fedapi.ClusterPreferences, replicas int64, clusters []string, expected map[string]int64) {
+	planer := NewPlanner(&fedapi.ReplicaAllocationPreferences{
+		Clusters: pref,
+	})
+	plan, overflow := planer.Plan(replicas, clusters, map[string]int64{}, map[string]int64{}, "")
+	assert.EqualValues(t, expected, plan)
+	assert.Equal(t, 0, len(overflow))
+}
+
+func doCheckWithExisting(t *testing.T, pref map[string]fedapi.ClusterPreferences, replicas int64, clusters []string,
+	existing map[string]int64, expected map[string]int64) {
+	planer := NewPlanner(&fedapi.ReplicaAllocationPreferences{
+		Clusters: pref,
+	})
+	plan, overflow := planer.Plan(replicas, clusters, existing, map[string]int64{}, "")
+	assert.Equal(t, 0, len(overflow))
+	assert.EqualValues(t, expected, plan)
+}
+
+func doCheckWithExistingAndCapacity(t *testing.T, rebalance bool, pref map[string]fedapi.ClusterPreferences, replicas int64, clusters []string,
+	existing map[string]int64,
+	capacity map[string]int64,
+	expected map[string]int64,
+	expectedOverflow map[string]int64) {
+	planer := NewPlanner(&fedapi.ReplicaAllocationPreferences{
+		Rebalance: rebalance,
+		Clusters:  pref,
+	})
+	plan, overflow := planer.Plan(replicas, clusters, existing, capacity, "")
+	assert.EqualValues(t, expected, plan)
+	assert.Equal(t, expectedOverflow, overflow)
+}
+
+func pint(val int64) *int64 {
+	return &val
+}
+
+func TestEqual(t *testing.T) {
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		50, []string{"A", "B", "C"},
+		// hash dependent
+		map[string]int64{"A": 16, "B": 17, "C": 17})
+
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		50, []string{"A", "B"},
+		map[string]int64{"A": 25, "B": 25})
+
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		1, []string{"A", "B"},
+		// hash dependent
+		map[string]int64{"A": 0, "B": 1})
+
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		1, []string{"A", "B", "C", "D"},
+		// hash dependent
+		map[string]int64{"A": 0, "B": 0, "C": 0, "D": 1})
+
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		1, []string{"A"},
+		map[string]int64{"A": 1})
+
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		1, []string{},
+		map[string]int64{})
+}
+
+func TestEqualWithExisting(t *testing.T) {
+	doCheckWithExisting(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		50, []string{"A", "B", "C"},
+		map[string]int64{"C": 30},
+		map[string]int64{"A": 10, "B": 10, "C": 30})
+
+	doCheckWithExisting(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		50, []string{"A", "B"},
+		map[string]int64{"A": 30},
+		map[string]int64{"A": 30, "B": 20})
+
+	doCheckWithExisting(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		15, []string{"A", "B"},
+		map[string]int64{"A": 0, "B": 8},
+		map[string]int64{"A": 7, "B": 8})
+
+	doCheckWithExisting(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		15, []string{"A", "B"},
+		map[string]int64{"A": 1, "B": 8},
+		map[string]int64{"A": 7, "B": 8})
+
+	doCheckWithExisting(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		15, []string{"A", "B"},
+		map[string]int64{"A": 4, "B": 8},
+		map[string]int64{"A": 7, "B": 8})
+
+	doCheckWithExisting(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		15, []string{"A", "B"},
+		map[string]int64{"A": 5, "B": 8},
+		map[string]int64{"A": 7, "B": 8})
+
+	doCheckWithExisting(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		15, []string{"A", "B"},
+		map[string]int64{"A": 6, "B": 8},
+		map[string]int64{"A": 7, "B": 8})
+
+	doCheckWithExisting(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		15, []string{"A", "B"},
+		map[string]int64{"A": 7, "B": 8},
+		map[string]int64{"A": 7, "B": 8})
+
+	doCheckWithExisting(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		500000, []string{"A", "B"},
+		map[string]int64{"A": 300000},
+		map[string]int64{"A": 300000, "B": 200000})
+
+	doCheckWithExisting(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		50, []string{"A", "B"},
+		map[string]int64{"A": 10},
+		map[string]int64{"A": 25, "B": 25})
+
+	doCheckWithExisting(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		50, []string{"A", "B"},
+		map[string]int64{"A": 10, "B": 70},
+		// hash dependent
+		// TODO: Should be 10:40, update algorithm. Issue: #31816
+		map[string]int64{"A": 0, "B": 50})
+
+	doCheckWithExisting(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		1, []string{"A", "B"},
+		map[string]int64{"A": 30},
+		map[string]int64{"A": 1, "B": 0})
+
+	doCheckWithExisting(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		50, []string{"A", "B"},
+		map[string]int64{"A": 10, "B": 20},
+		map[string]int64{"A": 25, "B": 25})
+}
+
+func TestWithExistingAndCapacity(t *testing.T) {
+	// desired without capacity: map[string]int64{"A": 17, "B": 17, "C": 16})
+	doCheckWithExistingAndCapacity(t, true, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1}},
+		50, []string{"A", "B", "C"},
+		map[string]int64{},
+		map[string]int64{"C": 10},
+		map[string]int64{"A": 20, "B": 20, "C": 10},
+		map[string]int64{"C": 7})
+
+	// desired B:50 C:0
+	doCheckWithExistingAndCapacity(t, true, map[string]fedapi.ClusterPreferences{
+		"A": {Weight: 10000},
+		"B": {Weight: 1}},
+		50, []string{"B", "C"},
+		map[string]int64{},
+		map[string]int64{"B": 10},
+		map[string]int64{"B": 10, "C": 0},
+		map[string]int64{"B": 40},
+	)
+
+	// desired A:20 B:40
+	doCheckWithExistingAndCapacity(t, true, map[string]fedapi.ClusterPreferences{
+		"A": {Weight: 1},
+		"B": {Weight: 2}},
+		60, []string{"A", "B", "C"},
+		map[string]int64{},
+		map[string]int64{"B": 10},
+		map[string]int64{"A": 50, "B": 10, "C": 0},
+		map[string]int64{"B": 30})
+
+	// map[string]int64{"A": 10, "B": 30, "C": 21, "D": 10})
+	doCheckWithExistingAndCapacity(t, true, map[string]fedapi.ClusterPreferences{
+		"A": {Weight: 10000, MaxReplicas: pint(10)},
+		"B": {Weight: 1},
+		"C": {Weight: 1, MaxReplicas: pint(21)},
+		"D": {Weight: 1, MaxReplicas: pint(10)}},
+		71, []string{"A", "B", "C", "D"},
+		map[string]int64{},
+		map[string]int64{"C": 10},
+		map[string]int64{"A": 10, "B": 41, "C": 10, "D": 10},
+		map[string]int64{"C": 11},
+	)
+
+	// desired A:20 B:20
+	doCheckWithExistingAndCapacity(t, false, map[string]fedapi.ClusterPreferences{
+		"A": {Weight: 1},
+		"B": {Weight: 1}},
+		60, []string{"A", "B", "C"},
+		map[string]int64{},
+		map[string]int64{"A": 10, "B": 10},
+		map[string]int64{"A": 10, "B": 10, "C": 0},
+		map[string]int64{"A": 20, "B": 20})
+
+	// desired A:10 B:50 although A:50 B:10 is fuly acceptable because rebalance = false
+	doCheckWithExistingAndCapacity(t, false, map[string]fedapi.ClusterPreferences{
+		"A": {Weight: 1},
+		"B": {Weight: 5}},
+		60, []string{"A", "B", "C"},
+		map[string]int64{},
+		map[string]int64{"B": 10},
+		map[string]int64{"A": 50, "B": 10, "C": 0},
+		map[string]int64{})
+
+	doCheckWithExistingAndCapacity(t, false, map[string]fedapi.ClusterPreferences{
+		"*": {MinReplicas: 20, Weight: 0}},
+		50, []string{"A", "B", "C"},
+		map[string]int64{},
+		map[string]int64{"B": 10},
+		map[string]int64{"A": 20, "B": 10, "C": 20},
+		map[string]int64{})
+
+	// Actually we would like to have extra 20 in B but 15 is also good.
+	doCheckWithExistingAndCapacity(t, true, map[string]fedapi.ClusterPreferences{
+		"*": {MinReplicas: 20, Weight: 1}},
+		60, []string{"A", "B"},
+		map[string]int64{},
+		map[string]int64{"B": 10},
+		map[string]int64{"A": 50, "B": 10},
+		map[string]int64{"B": 15})
+}
+
+func TestMin(t *testing.T) {
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"*": {MinReplicas: 2, Weight: 0}},
+		50, []string{"A", "B", "C"},
+		map[string]int64{"A": 2, "B": 2, "C": 2})
+
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"*": {MinReplicas: 20, Weight: 0}},
+		50, []string{"A", "B", "C"},
+		// hash dependant.
+		map[string]int64{"A": 10, "B": 20, "C": 20})
+
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"*": {MinReplicas: 20, Weight: 0},
+		"A": {MinReplicas: 100, Weight: 1}},
+		50, []string{"A", "B", "C"},
+		map[string]int64{"A": 50, "B": 0, "C": 0})
+
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"*": {MinReplicas: 10, Weight: 1, MaxReplicas: pint(12)}},
+		50, []string{"A", "B", "C"},
+		map[string]int64{"A": 12, "B": 12, "C": 12})
+}
+
+func TestMax(t *testing.T) {
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 1, MaxReplicas: pint(2)}},
+		50, []string{"A", "B", "C"},
+		map[string]int64{"A": 2, "B": 2, "C": 2})
+
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"*": {Weight: 0, MaxReplicas: pint(2)}},
+		50, []string{"A", "B", "C"},
+		map[string]int64{"A": 0, "B": 0, "C": 0})
+}
+
+func TestWeight(t *testing.T) {
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"A": {Weight: 1},
+		"B": {Weight: 2}},
+		60, []string{"A", "B", "C"},
+		map[string]int64{"A": 20, "B": 40, "C": 0})
+
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"A": {Weight: 10000},
+		"B": {Weight: 1}},
+		50, []string{"A", "B", "C"},
+		map[string]int64{"A": 50, "B": 0, "C": 0})
+
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"A": {Weight: 10000},
+		"B": {Weight: 1}},
+		50, []string{"B", "C"},
+		map[string]int64{"B": 50, "C": 0})
+
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"A": {Weight: 10000, MaxReplicas: pint(10)},
+		"B": {Weight: 1},
+		"C": {Weight: 1}},
+		50, []string{"A", "B", "C"},
+		map[string]int64{"A": 10, "B": 20, "C": 20})
+
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"A": {Weight: 10000, MaxReplicas: pint(10)},
+		"B": {Weight: 1},
+		"C": {Weight: 1, MaxReplicas: pint(10)}},
+		50, []string{"A", "B", "C"},
+		map[string]int64{"A": 10, "B": 30, "C": 10})
+
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"A": {Weight: 10000, MaxReplicas: pint(10)},
+		"B": {Weight: 1},
+		"C": {Weight: 1, MaxReplicas: pint(21)},
+		"D": {Weight: 1, MaxReplicas: pint(10)}},
+		71, []string{"A", "B", "C", "D"},
+		map[string]int64{"A": 10, "B": 30, "C": 21, "D": 10})
+
+	doCheck(t, map[string]fedapi.ClusterPreferences{
+		"A": {Weight: 10000, MaxReplicas: pint(10)},
+		"B": {Weight: 1},
+		"C": {Weight: 1, MaxReplicas: pint(21)},
+		"D": {Weight: 1, MaxReplicas: pint(10)},
+		"E": {Weight: 1}},
+		91, []string{"A", "B", "C", "D", "E"},
+		map[string]int64{"A": 10, "B": 25, "C": 21, "D": 10, "E": 25})
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/podanalyzer/BUILD

@@ -0,0 +1,38 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["pod_helper.go"],
+    deps = ["//vendor/k8s.io/api/core/v1:go_default_library"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["pod_helper_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/podanalyzer/pod_helper.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package podanalyzer
+
+import (
+	"time"
+
+	api_v1 "k8s.io/api/core/v1"
+)
+
+type PodAnalysisResult struct {
+	// Total number of pods created.
+	Total int
+	// Number of pods that are running and ready.
+	RunningAndReady int
+	// Number of pods that have been in unschedulable state for UnshedulableThreshold seconds.
+	Unschedulable int
+
+	// TODO: Handle other scenarios like pod waiting too long for scheduler etc.
+}
+
+const (
+	// TODO: make it configurable
+	UnschedulableThreshold = 60 * time.Second
+)
+
+// AnalyzePods calculates how many pods from the list are in one of
+// the meaningful (from the replica set perspective) states. This function is
+// a temporary workaround against the current lack of ownerRef in pods.
+func AnalyzePods(pods *api_v1.PodList, currentTime time.Time) PodAnalysisResult {
+	result := PodAnalysisResult{}
+	for _, pod := range pods.Items {
+		result.Total++
+		for _, condition := range pod.Status.Conditions {
+			if pod.Status.Phase == api_v1.PodRunning {
+				if condition.Type == api_v1.PodReady {
+					result.RunningAndReady++
+				}
+			} else if condition.Type == api_v1.PodScheduled &&
+				condition.Status == api_v1.ConditionFalse &&
+				condition.Reason == api_v1.PodReasonUnschedulable &&
+				condition.LastTransitionTime.Add(UnschedulableThreshold).Before(currentTime) {
+
+				result.Unschedulable++
+			}
+		}
+	}
+	return result
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/podanalyzer/pod_helper_test.go

@@ -0,0 +1,98 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package podanalyzer
+
+import (
+	"testing"
+	"time"
+
+	api_v1 "k8s.io/api/core/v1"
+	"k8s.io/api/extensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAnalyze(t *testing.T) {
+	now := time.Now()
+	podRunning := newPod("p1",
+		api_v1.PodStatus{
+			Phase: api_v1.PodRunning,
+			Conditions: []api_v1.PodCondition{
+				{
+					Type:   api_v1.PodReady,
+					Status: api_v1.ConditionTrue,
+				},
+			},
+		})
+	podUnschedulable := newPod("pU",
+		api_v1.PodStatus{
+			Phase: api_v1.PodPending,
+			Conditions: []api_v1.PodCondition{
+				{
+					Type:               api_v1.PodScheduled,
+					Status:             api_v1.ConditionFalse,
+					Reason:             api_v1.PodReasonUnschedulable,
+					LastTransitionTime: metav1.Time{Time: now.Add(-10 * time.Minute)},
+				},
+			},
+		})
+	podOther := newPod("pO",
+		api_v1.PodStatus{
+			Phase:      api_v1.PodPending,
+			Conditions: []api_v1.PodCondition{},
+		})
+
+	result := AnalyzePods(&api_v1.PodList{Items: []api_v1.Pod{*podRunning, *podRunning, *podRunning, *podUnschedulable, *podUnschedulable}}, now)
+	assert.Equal(t, PodAnalysisResult{
+		Total:           5,
+		RunningAndReady: 3,
+		Unschedulable:   2,
+	}, result)
+
+	result = AnalyzePods(&api_v1.PodList{Items: []api_v1.Pod{*podOther}}, now)
+	assert.Equal(t, PodAnalysisResult{
+		Total:           1,
+		RunningAndReady: 0,
+		Unschedulable:   0,
+	}, result)
+}
+
+func newReplicaSet(selectorMap map[string]string) *v1beta1.ReplicaSet {
+	replicas := int32(3)
+	rs := &v1beta1.ReplicaSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "foobar",
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: v1beta1.ReplicaSetSpec{
+			Replicas: &replicas,
+			Selector: &metav1.LabelSelector{MatchLabels: selectorMap},
+		},
+	}
+	return rs
+}
+
+func newPod(name string, status api_v1.PodStatus) *api_v1.Pod {
+	return &api_v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: metav1.NamespaceDefault,
+		},
+		Status: status,
+	}
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/replicapreferences/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["preferences.go"],
+    deps = [
+        "//federation/apis/federation:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["preferences_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+    ],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/replicapreferences/preferences.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package replicapreferences
+
+import (
+	"encoding/json"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime"
+	fed "k8s.io/kubernetes/federation/apis/federation"
+)
+
+// GetAllocationPreferences reads the preferences from the annotations on the given object.
+// It takes in an object and determines the supported types.
+// Callers need to pass the string key used to store the annotations.
+// Returns nil if the annotations with the given key are not found.
+func GetAllocationPreferences(obj runtime.Object, key string) (*fed.ReplicaAllocationPreferences, error) {
+	if obj == nil {
+		return nil, nil
+	}
+
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return nil, err
+	}
+	annotations := accessor.GetAnnotations()
+	if annotations == nil {
+		return nil, nil
+	}
+
+	prefString, found := annotations[key]
+	if !found {
+		return nil, nil
+	}
+
+	var pref fed.ReplicaAllocationPreferences
+	if err := json.Unmarshal([]byte(prefString), &pref); err != nil {
+		return nil, err
+	}
+	return &pref, nil
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/replicapreferences/preferences_test.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package replicapreferences
+
+import (
+	"testing"
+
+	extensionsv1 "k8s.io/api/extensions/v1beta1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+const (
+	TestPreferencesAnnotationKey = "federation.kubernetes.io/test-preferences"
+)
+
+func TestGetAllocationPreferences(t *testing.T) {
+	testCases := []struct {
+		testname      string
+		prefs         string
+		obj           runtime.Object
+		errorExpected bool
+	}{
+		{
+			testname: "good preferences",
+			prefs: `{"rebalance": true,
+				  "clusters": {
+				    "k8s-1": {"minReplicas": 10, "maxReplicas": 20, "weight": 2},
+				    "*": {"weight": 1}
+				}}`,
+			obj: &extensionsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-obj",
+					Namespace: metav1.NamespaceDefault,
+					SelfLink:  "/api/v1/namespaces/default/obj/test-obj",
+				},
+			},
+			errorExpected: false,
+		},
+		{
+			testname: "failed preferences",
+			prefs:    `{`, // bad json
+			obj: &extensionsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-obj",
+					Namespace: metav1.NamespaceDefault,
+					SelfLink:  "/api/v1/namespaces/default/obj/test-obj",
+				},
+			},
+			errorExpected: true,
+		},
+	}
+
+	// prepare the objects
+	for _, tc := range testCases {
+		accessor, _ := meta.Accessor(tc.obj)
+		anno := accessor.GetAnnotations()
+		if anno == nil {
+			anno = make(map[string]string)
+			accessor.SetAnnotations(anno)
+		}
+		anno[TestPreferencesAnnotationKey] = tc.prefs
+	}
+
+	// test get preferences
+	for _, tc := range testCases {
+		pref, err := GetAllocationPreferences(tc.obj, TestPreferencesAnnotationKey)
+		if tc.errorExpected {
+			assert.NotNil(t, err)
+		} else {
+			assert.NotNil(t, pref)
+			assert.Nil(t, err)
+		}
+	}
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/secret.go

@@ -0,0 +1,32 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"reflect"
+
+	api_v1 "k8s.io/api/core/v1"
+)
+
+// Checks if cluster-independent, user provided data in two given Secrets are equal. If in
+// the future the Secret structure is expanded then any field that is not populated.
+// by the api server should be included here.
+func SecretEquivalent(s1, s2 api_v1.Secret) bool {
+	return ObjectMetaEquivalent(s1.ObjectMeta, s2.ObjectMeta) &&
+		reflect.DeepEqual(s1.Data, s2.Data) &&
+		reflect.DeepEqual(s1.Type, s2.Type)
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/test/BUILD

@@ -0,0 +1,39 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["test_helper.go"],
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//federation/pkg/federation-controller/util/finalizers:go_default_library",
+        "//pkg/api:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/github.com/stretchr/testify/require:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/client-go/testing:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/util/test/test_helper.go

@@ -0,0 +1,454 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testutil
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"runtime/pprof"
+	"sync"
+	"testing"
+	"time"
+
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
+	core "k8s.io/client-go/testing"
+	federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	finalizersutil "k8s.io/kubernetes/federation/pkg/federation-controller/util/finalizers"
+	"k8s.io/kubernetes/pkg/api"
+
+	"github.com/golang/glog"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	pushTimeout = 5 * time.Second
+)
+
+// A structure that distributes events to multiple watchers.
+type WatcherDispatcher struct {
+	sync.Mutex
+	watchers       []*watch.RaceFreeFakeWatcher
+	eventsSoFar    []*watch.Event
+	orderExecution chan func()
+	stopChan       chan struct{}
+}
+
+func (wd *WatcherDispatcher) register(watcher *watch.RaceFreeFakeWatcher) {
+	wd.Lock()
+	defer wd.Unlock()
+	wd.watchers = append(wd.watchers, watcher)
+	for _, event := range wd.eventsSoFar {
+		watcher.Action(event.Type, event.Object)
+	}
+}
+
+func (wd *WatcherDispatcher) Stop() {
+	wd.Lock()
+	defer wd.Unlock()
+	close(wd.stopChan)
+	glog.Infof("Stopping WatcherDispatcher")
+	for _, watcher := range wd.watchers {
+		watcher.Stop()
+	}
+}
+
+func copy(obj runtime.Object) runtime.Object {
+	objCopy, err := api.Scheme.DeepCopy(obj)
+	if err != nil {
+		panic(err)
+	}
+	return objCopy.(runtime.Object)
+}
+
+// Add sends an add event.
+func (wd *WatcherDispatcher) Add(obj runtime.Object) {
+	wd.Lock()
+	defer wd.Unlock()
+	wd.eventsSoFar = append(wd.eventsSoFar, &watch.Event{Type: watch.Added, Object: copy(obj)})
+	for _, watcher := range wd.watchers {
+		if !watcher.IsStopped() {
+			watcher.Add(copy(obj))
+		}
+	}
+}
+
+// Modify sends a modify event.
+func (wd *WatcherDispatcher) Modify(obj runtime.Object) {
+	wd.Lock()
+	defer wd.Unlock()
+	glog.V(4).Infof("->WatcherDispatcher.Modify(%v)", obj)
+	wd.eventsSoFar = append(wd.eventsSoFar, &watch.Event{Type: watch.Modified, Object: copy(obj)})
+	for i, watcher := range wd.watchers {
+		if !watcher.IsStopped() {
+			glog.V(4).Infof("->Watcher(%d).Modify(%v)", i, obj)
+			watcher.Modify(copy(obj))
+		} else {
+			glog.V(4).Infof("->Watcher(%d) is stopped.  Not calling Modify(%v)", i, obj)
+		}
+	}
+}
+
+// Delete sends a delete event.
+func (wd *WatcherDispatcher) Delete(lastValue runtime.Object) {
+	wd.Lock()
+	defer wd.Unlock()
+	wd.eventsSoFar = append(wd.eventsSoFar, &watch.Event{Type: watch.Deleted, Object: copy(lastValue)})
+	for _, watcher := range wd.watchers {
+		if !watcher.IsStopped() {
+			watcher.Delete(copy(lastValue))
+		}
+	}
+}
+
+// Error sends an Error event.
+func (wd *WatcherDispatcher) Error(errValue runtime.Object) {
+	wd.Lock()
+	defer wd.Unlock()
+	wd.eventsSoFar = append(wd.eventsSoFar, &watch.Event{Type: watch.Error, Object: copy(errValue)})
+	for _, watcher := range wd.watchers {
+		if !watcher.IsStopped() {
+			watcher.Error(copy(errValue))
+		}
+	}
+}
+
+// Action sends an event of the requested type, for table-based testing.
+func (wd *WatcherDispatcher) Action(action watch.EventType, obj runtime.Object) {
+	wd.Lock()
+	defer wd.Unlock()
+	wd.eventsSoFar = append(wd.eventsSoFar, &watch.Event{Type: action, Object: copy(obj)})
+	for _, watcher := range wd.watchers {
+		if !watcher.IsStopped() {
+			watcher.Action(action, copy(obj))
+		}
+	}
+}
+
+// RegisterFakeWatch adds a new fake watcher for the specified resource in the given fake client.
+// All subsequent requests for a watch on the client will result in returning this fake watcher.
+func RegisterFakeWatch(resource string, client *core.Fake) *WatcherDispatcher {
+	dispatcher := &WatcherDispatcher{
+		watchers:       make([]*watch.RaceFreeFakeWatcher, 0),
+		eventsSoFar:    make([]*watch.Event, 0),
+		orderExecution: make(chan func(), 100),
+		stopChan:       make(chan struct{}),
+	}
+	go func() {
+		for {
+			select {
+			case fun := <-dispatcher.orderExecution:
+				fun()
+			case <-dispatcher.stopChan:
+				return
+			}
+		}
+	}()
+
+	client.AddWatchReactor(resource, func(action core.Action) (bool, watch.Interface, error) {
+		watcher := watch.NewRaceFreeFake()
+		dispatcher.register(watcher)
+		return true, watcher, nil
+	})
+	return dispatcher
+}
+
+// RegisterFakeList registers a list response for the specified resource inside the given fake client.
+// The passed value will be returned with every list call.
+func RegisterFakeList(resource string, client *core.Fake, obj runtime.Object) {
+	client.AddReactor("list", resource, func(action core.Action) (bool, runtime.Object, error) {
+		return true, obj, nil
+	})
+}
+
+// RegisterFakeClusterGet registers a get response for the cluster resource inside the given fake client.
+func RegisterFakeClusterGet(client *core.Fake, obj runtime.Object) {
+	clusterList, ok := obj.(*federationapi.ClusterList)
+	client.AddReactor("get", "clusters", func(action core.Action) (bool, runtime.Object, error) {
+		name := action.(core.GetAction).GetName()
+		if ok {
+			for _, cluster := range clusterList.Items {
+				if cluster.Name == name {
+					return true, &cluster, nil
+				}
+			}
+		}
+		return false, nil, fmt.Errorf("could not find the requested cluster: %s", name)
+	})
+}
+
+// RegisterFakeOnCreate registers a reactor in the given fake client that passes
+// all created objects to the given watcher.
+func RegisterFakeOnCreate(resource string, client *core.Fake, watcher *WatcherDispatcher) {
+	client.AddReactor("create", resource, func(action core.Action) (bool, runtime.Object, error) {
+		createAction := action.(core.CreateAction)
+		originalObj := createAction.GetObject()
+		// Create a copy of the object here to prevent data races while reading the object in go routine.
+		obj := copy(originalObj)
+		watcher.orderExecution <- func() {
+			glog.V(4).Infof("Object created: %v", obj)
+			watcher.Add(obj)
+		}
+		return true, originalObj, nil
+	})
+}
+
+// RegisterFakeCopyOnCreate registers a reactor in the given fake client that passes
+// all created objects to the given watcher and also copies them to a channel for
+// in-test inspection.
+func RegisterFakeCopyOnCreate(resource string, client *core.Fake, watcher *WatcherDispatcher) chan runtime.Object {
+	objChan := make(chan runtime.Object, 100)
+	client.AddReactor("create", resource, func(action core.Action) (bool, runtime.Object, error) {
+		createAction := action.(core.CreateAction)
+		originalObj := createAction.GetObject()
+		// Create a copy of the object here to prevent data races while reading the object in go routine.
+		obj := copy(originalObj)
+		watcher.orderExecution <- func() {
+			glog.V(4).Infof("Object created. Writing to channel: %v", obj)
+			watcher.Add(obj)
+			objChan <- obj
+		}
+		return true, originalObj, nil
+	})
+	return objChan
+}
+
+// RegisterFakeOnUpdate registers a reactor in the given fake client that passes
+// all updated objects to the given watcher.
+func RegisterFakeOnUpdate(resource string, client *core.Fake, watcher *WatcherDispatcher) {
+	client.AddReactor("update", resource, func(action core.Action) (bool, runtime.Object, error) {
+		updateAction := action.(core.UpdateAction)
+		originalObj := updateAction.GetObject()
+		glog.V(7).Infof("Updating %s: %v", resource, updateAction.GetObject())
+
+		// Create a copy of the object here to prevent data races while reading the object in go routine.
+		obj := copy(originalObj)
+		operation := func() {
+			glog.V(4).Infof("Object updated %v", obj)
+			watcher.Modify(obj)
+		}
+		select {
+		case watcher.orderExecution <- operation:
+			break
+		case <-time.After(pushTimeout):
+			glog.Errorf("Fake client execution channel blocked")
+			glog.Errorf("Tried to push %v", updateAction)
+		}
+		return true, originalObj, nil
+	})
+	return
+}
+
+// RegisterFakeCopyOnUpdate registers a reactor in the given fake client that passes
+// all updated objects to the given watcher and also copies them to a channel for
+// in-test inspection.
+func RegisterFakeCopyOnUpdate(resource string, client *core.Fake, watcher *WatcherDispatcher) chan runtime.Object {
+	objChan := make(chan runtime.Object, 100)
+	client.AddReactor("update", resource, func(action core.Action) (bool, runtime.Object, error) {
+		updateAction := action.(core.UpdateAction)
+		originalObj := updateAction.GetObject()
+		glog.V(7).Infof("Updating %s: %v", resource, updateAction.GetObject())
+
+		// Create a copy of the object here to prevent data races while reading the object in go routine.
+		obj := copy(originalObj)
+		operation := func() {
+			glog.V(4).Infof("Object updated. Writing to channel: %v", obj)
+			watcher.Modify(obj)
+			objChan <- obj
+		}
+		select {
+		case watcher.orderExecution <- operation:
+			break
+		case <-time.After(pushTimeout):
+			glog.Errorf("Fake client execution channel blocked")
+			glog.Errorf("Tried to push %v", updateAction)
+		}
+		return true, originalObj, nil
+	})
+	return objChan
+}
+
+// RegisterFakeOnDelete registers a reactor in the given fake client that passes
+// all deleted objects to the given watcher. Since we could get only name of the
+// deleted object from DeleteAction, this register function relies on the getObject
+// function passed to get the object by name and pass it watcher.
+func RegisterFakeOnDelete(resource string, client *core.Fake, watcher *WatcherDispatcher, getObject func(name, namespace string) runtime.Object) {
+	client.AddReactor("delete", resource, func(action core.Action) (bool, runtime.Object, error) {
+		deleteAction := action.(core.DeleteAction)
+		obj := getObject(deleteAction.GetName(), deleteAction.GetNamespace())
+		glog.V(7).Infof("Deleting %s: %v", resource, obj)
+
+		operation := func() {
+			glog.V(4).Infof("Object deleted %v", obj)
+			watcher.Delete(obj)
+		}
+		select {
+		case watcher.orderExecution <- operation:
+			break
+		case <-time.After(pushTimeout):
+			glog.Errorf("Fake client execution channel blocked")
+			glog.Errorf("Tried to push %v", deleteAction)
+		}
+		return true, obj, nil
+	})
+	return
+}
+
+// Adds an update reactor to the given fake client.
+// The reactor just returns the object passed to update action.
+// This is used as a hack to workaround https://github.com/kubernetes/kubernetes/issues/40939.
+// Without this, all update actions using fake client return empty objects.
+func AddFakeUpdateReactor(resource string, client *core.Fake) {
+	client.AddReactor("update", resource, func(action core.Action) (bool, runtime.Object, error) {
+		updateAction := action.(core.UpdateAction)
+		originalObj := updateAction.GetObject()
+		return true, originalObj, nil
+	})
+}
+
+// GetObjectFromChan tries to get an api object from the given channel
+// within a reasonable time.
+func GetObjectFromChan(c chan runtime.Object) runtime.Object {
+	select {
+	case obj := <-c:
+		return obj
+	case <-time.After(wait.ForeverTestTimeout):
+		pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
+		return nil
+	}
+}
+
+type CheckingFunction func(runtime.Object) error
+
+// CheckObjectFromChan tries to get an object matching the given check function
+// within a reasonable time.
+func CheckObjectFromChan(c chan runtime.Object, checkFunction CheckingFunction) error {
+	delay := 20 * time.Second
+	var lastError error
+	for {
+		select {
+		case obj := <-c:
+			if lastError = checkFunction(obj); lastError == nil {
+				return nil
+			}
+			glog.Infof("Check function failed with %v", lastError)
+			delay = 5 * time.Second
+		case <-time.After(delay):
+			pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
+			if lastError == nil {
+				return fmt.Errorf("Failed to get an object from channel")
+			} else {
+				return lastError
+			}
+		}
+	}
+}
+
+// CompareObjectMeta returns an error when the given objects are not equivalent.
+func CompareObjectMeta(a, b metav1.ObjectMeta) error {
+	if a.Namespace != b.Namespace {
+		return fmt.Errorf("Different namespace expected:%s observed:%s", a.Namespace, b.Namespace)
+	}
+	if a.Name != b.Name {
+		return fmt.Errorf("Different name expected:%s observed:%s", a.Name, b.Name)
+	}
+	if !reflect.DeepEqual(a.Labels, b.Labels) && (len(a.Labels) != 0 || len(b.Labels) != 0) {
+		return fmt.Errorf("Labels are different expected:%v observed:%v", a.Labels, b.Labels)
+	}
+	if !reflect.DeepEqual(a.Annotations, b.Annotations) && (len(a.Annotations) != 0 || len(b.Annotations) != 0) {
+		return fmt.Errorf("Annotations are different expected:%v observed:%v", a.Annotations, b.Annotations)
+	}
+	return nil
+}
+
+func ToFederatedInformerForTestOnly(informer util.FederatedInformer) util.FederatedInformerForTestOnly {
+	inter := informer.(interface{})
+	return inter.(util.FederatedInformerForTestOnly)
+}
+
+// NewCluster builds a new cluster object.
+func NewCluster(name string, readyStatus apiv1.ConditionStatus) *federationapi.Cluster {
+	return &federationapi.Cluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        name,
+			Annotations: map[string]string{},
+			Labels:      map[string]string{"cluster": name},
+		},
+		Status: federationapi.ClusterStatus{
+			Conditions: []federationapi.ClusterCondition{
+				{Type: federationapi.ClusterReady, Status: readyStatus},
+			},
+			Zones:  []string{"foozone"},
+			Region: "fooregion",
+		},
+	}
+}
+
+// Ensure a key is in the store before returning (or timeout w/ error)
+func WaitForStoreUpdate(store util.FederatedReadOnlyStore, clusterName, key string, timeout time.Duration) error {
+	retryInterval := 100 * time.Millisecond
+	err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) {
+		_, found, err := store.GetByKey(clusterName, key)
+		return found, err
+	})
+	return err
+}
+
+// Ensure a key is in the store before returning (or timeout w/ error)
+func WaitForStoreUpdateChecking(store util.FederatedReadOnlyStore, clusterName, key string, timeout time.Duration,
+	checkFunction CheckingFunction) error {
+	retryInterval := 500 * time.Millisecond
+	var lastError error
+	err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) {
+		item, found, err := store.GetByKey(clusterName, key)
+		if err != nil || !found {
+			return found, err
+		}
+		runtimeObj := item.(runtime.Object)
+		lastError = checkFunction(runtimeObj)
+		glog.V(2).Infof("Check function failed for %s %v %v", key, runtimeObj, lastError)
+		return lastError == nil, nil
+	})
+	return err
+}
+
+func MetaAndSpecCheckingFunction(expected runtime.Object) CheckingFunction {
+	return func(obj runtime.Object) error {
+		if util.ObjectMetaAndSpecEquivalent(obj, expected) {
+			return nil
+		}
+		return fmt.Errorf("Object different expected=%#v received=%#v", expected, obj)
+	}
+}
+
+func AssertHasFinalizer(t *testing.T, obj runtime.Object, finalizer string) {
+	hasFinalizer, err := finalizersutil.HasFinalizer(obj, finalizer)
+	require.Nil(t, err)
+	assert.True(t, hasFinalizer)
+}
+
+func NewInt32(val int32) *int32 {
+	p := new(int32)
+	*p = val
+	return p
+}