Replace godep with dep

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/sync/BUILD

@@ -0,0 +1,66 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["controller_test.go"],
+    library = ":go_default_library",
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/pkg/federatedtypes:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//federation/pkg/federation-controller/util/test:go_default_library",
+        "//vendor/github.com/stretchr/testify/require:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["controller.go"],
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset:go_default_library",
+        "//federation/pkg/federatedtypes:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//federation/pkg/federation-controller/util/clusterselector:go_default_library",
+        "//federation/pkg/federation-controller/util/deletionhelper:go_default_library",
+        "//federation/pkg/federation-controller/util/eventsink:go_default_library",
+        "//pkg/api:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/rest:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+        "//vendor/k8s.io/client-go/tools/record:go_default_library",
+        "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
+        "//vendor/k8s.io/client-go/util/workqueue:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/sync/controller.go

@@ -0,0 +1,610 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sync
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pkgruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/flowcontrol"
+	"k8s.io/client-go/util/workqueue"
+	federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	federationclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	"k8s.io/kubernetes/federation/pkg/federatedtypes"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/clusterselector"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/deletionhelper"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util/eventsink"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/controller"
+
+	"github.com/golang/glog"
+)
+
+const (
+	allClustersKey = "ALL_CLUSTERS"
+)
+
+// FederationSyncController synchronizes the state of a federated type
+// to clusters that are members of the federation.
+type FederationSyncController struct {
+	// For triggering reconciliation of a single resource. This is
+	// used when there is an add/update/delete operation on a resource
+	// in either federated API server or in some member of the
+	// federation.
+	deliverer *util.DelayingDeliverer
+
+	// For triggering reconciliation of all target resources. This is
+	// used when a new cluster becomes available.
+	clusterDeliverer *util.DelayingDeliverer
+
+	// Contains resources present in members of federation.
+	informer util.FederatedInformer
+	// For updating members of federation.
+	updater util.FederatedUpdater
+	// Definitions of resources that should be federated.
+	store cache.Store
+	// Informer controller for resources that should be federated.
+	controller cache.Controller
+
+	// Work queue allowing parallel processing of resources
+	workQueue workqueue.Interface
+
+	// Backoff manager
+	backoff *flowcontrol.Backoff
+
+	// For events
+	eventRecorder record.EventRecorder
+
+	deletionHelper *deletionhelper.DeletionHelper
+
+	reviewDelay             time.Duration
+	clusterAvailableDelay   time.Duration
+	clusterUnavailableDelay time.Duration
+	smallDelay              time.Duration
+	updateTimeout           time.Duration
+
+	adapter federatedtypes.FederatedTypeAdapter
+}
+
+// StartFederationSyncController starts a new sync controller for a type adapter
+func StartFederationSyncController(kind string, adapterFactory federatedtypes.AdapterFactory, config *restclient.Config, stopChan <-chan struct{}, minimizeLatency bool, adapterSpecificArgs map[string]interface{}) {
+	restclient.AddUserAgent(config, fmt.Sprintf("federation-%s-controller", kind))
+	client := federationclientset.NewForConfigOrDie(config)
+	adapter := adapterFactory(client, config, adapterSpecificArgs)
+	controller := newFederationSyncController(client, adapter)
+	if minimizeLatency {
+		controller.minimizeLatency()
+	}
+	glog.Infof(fmt.Sprintf("Starting federated sync controller for %s resources", kind))
+	controller.Run(stopChan)
+}
+
+// newFederationSyncController returns a new sync controller for the given client and type adapter
+func newFederationSyncController(client federationclientset.Interface, adapter federatedtypes.FederatedTypeAdapter) *FederationSyncController {
+	broadcaster := record.NewBroadcaster()
+	broadcaster.StartRecordingToSink(eventsink.NewFederatedEventSink(client))
+	recorder := broadcaster.NewRecorder(api.Scheme, v1.EventSource{Component: fmt.Sprintf("federation-%v-controller", adapter.Kind())})
+
+	s := &FederationSyncController{
+		reviewDelay:             time.Second * 10,
+		clusterAvailableDelay:   time.Second * 20,
+		clusterUnavailableDelay: time.Second * 60,
+		smallDelay:              time.Second * 3,
+		updateTimeout:           time.Second * 30,
+		workQueue:               workqueue.New(),
+		backoff:                 flowcontrol.NewBackOff(5*time.Second, time.Minute),
+		eventRecorder:           recorder,
+		adapter:                 adapter,
+	}
+
+	// Build delivereres for triggering reconciliations.
+	s.deliverer = util.NewDelayingDeliverer()
+	s.clusterDeliverer = util.NewDelayingDeliverer()
+
+	// Start informer in federated API servers on the resource type that should be federated.
+	s.store, s.controller = cache.NewInformer(
+		&cache.ListWatch{
+			ListFunc: func(options metav1.ListOptions) (pkgruntime.Object, error) {
+				return adapter.FedList(metav1.NamespaceAll, options)
+			},
+			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+				return adapter.FedWatch(metav1.NamespaceAll, options)
+			},
+		},
+		adapter.ObjectType(),
+		controller.NoResyncPeriodFunc(),
+		util.NewTriggerOnAllChanges(func(obj pkgruntime.Object) { s.deliverObj(obj, 0, false) }))
+
+	// Federated informer on the resource type in members of federation.
+	s.informer = util.NewFederatedInformer(
+		client,
+		func(cluster *federationapi.Cluster, targetClient kubeclientset.Interface) (cache.Store, cache.Controller) {
+			return cache.NewInformer(
+				&cache.ListWatch{
+					ListFunc: func(options metav1.ListOptions) (pkgruntime.Object, error) {
+						return adapter.ClusterList(targetClient, metav1.NamespaceAll, options)
+					},
+					WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+						return adapter.ClusterWatch(targetClient, metav1.NamespaceAll, options)
+					},
+				},
+				adapter.ObjectType(),
+				controller.NoResyncPeriodFunc(),
+				// Trigger reconciliation whenever something in federated cluster is changed. In most cases it
+				// would be just confirmation that some operation on the target resource type had succeeded.
+				util.NewTriggerOnAllChanges(
+					func(obj pkgruntime.Object) {
+						s.deliverObj(obj, s.reviewDelay, false)
+					},
+				))
+		},
+
+		&util.ClusterLifecycleHandlerFuncs{
+			ClusterAvailable: func(cluster *federationapi.Cluster) {
+				// When new cluster becomes available process all the target resources again.
+				s.clusterDeliverer.DeliverAt(allClustersKey, nil, time.Now().Add(s.clusterAvailableDelay))
+			},
+			// When a cluster becomes unavailable process all the target resources again.
+			ClusterUnavailable: func(cluster *federationapi.Cluster, _ []interface{}) {
+				s.clusterDeliverer.DeliverAt(allClustersKey, nil, time.Now().Add(s.clusterUnavailableDelay))
+			},
+		},
+	)
+
+	// Federated updeater along with Create/Update/Delete operations.
+	s.updater = util.NewFederatedUpdater(s.informer, adapter.Kind(), s.updateTimeout, s.eventRecorder,
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			_, err := adapter.ClusterCreate(client, obj)
+			return err
+		},
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			_, err := adapter.ClusterUpdate(client, obj)
+			return err
+		},
+		func(client kubeclientset.Interface, obj pkgruntime.Object) error {
+			qualifiedName := adapter.QualifiedName(obj)
+			orphanDependents := false
+			err := adapter.ClusterDelete(client, qualifiedName, &metav1.DeleteOptions{OrphanDependents: &orphanDependents})
+			return err
+		})
+
+	s.deletionHelper = deletionhelper.NewDeletionHelper(
+		s.updateObject,
+		// objNameFunc
+		func(obj pkgruntime.Object) string {
+			return adapter.QualifiedName(obj).String()
+		},
+		s.informer,
+		s.updater,
+	)
+
+	return s
+}
+
+// minimizeLatency reduces delays and timeouts to make the controller more responsive (useful for testing).
+func (s *FederationSyncController) minimizeLatency() {
+	s.clusterAvailableDelay = time.Second
+	s.clusterUnavailableDelay = time.Second
+	s.reviewDelay = 50 * time.Millisecond
+	s.smallDelay = 20 * time.Millisecond
+	s.updateTimeout = 5 * time.Second
+}
+
+// Sends the given updated object to apiserver.
+func (s *FederationSyncController) updateObject(obj pkgruntime.Object) (pkgruntime.Object, error) {
+	return s.adapter.FedUpdate(obj)
+}
+
+func (s *FederationSyncController) Run(stopChan <-chan struct{}) {
+	go s.controller.Run(stopChan)
+	s.informer.Start()
+	s.deliverer.StartWithHandler(func(item *util.DelayingDelivererItem) {
+		s.workQueue.Add(item)
+	})
+	s.clusterDeliverer.StartWithHandler(func(_ *util.DelayingDelivererItem) {
+		s.reconcileOnClusterChange()
+	})
+
+	// TODO: Allow multiple workers.
+	go wait.Until(s.worker, time.Second, stopChan)
+
+	util.StartBackoffGC(s.backoff, stopChan)
+
+	// Ensure all goroutines are cleaned up when the stop channel closes
+	go func() {
+		<-stopChan
+		s.informer.Stop()
+		s.workQueue.ShutDown()
+		s.deliverer.Stop()
+		s.clusterDeliverer.Stop()
+	}()
+}
+
+type reconciliationStatus int
+
+const (
+	statusAllOK reconciliationStatus = iota
+	statusNeedsRecheck
+	statusError
+	statusNotSynced
+)
+
+func (s *FederationSyncController) worker() {
+	for {
+		obj, quit := s.workQueue.Get()
+		if quit {
+			return
+		}
+
+		item := obj.(*util.DelayingDelivererItem)
+		qualifiedName := item.Value.(*federatedtypes.QualifiedName)
+		status := s.reconcile(*qualifiedName)
+		s.workQueue.Done(item)
+
+		switch status {
+		case statusAllOK:
+			break
+		case statusError:
+			s.deliver(*qualifiedName, 0, true)
+		case statusNeedsRecheck:
+			s.deliver(*qualifiedName, s.reviewDelay, false)
+		case statusNotSynced:
+			s.deliver(*qualifiedName, s.clusterAvailableDelay, false)
+		}
+	}
+}
+
+func (s *FederationSyncController) deliverObj(obj pkgruntime.Object, delay time.Duration, failed bool) {
+	qualifiedName := s.adapter.QualifiedName(obj)
+	s.deliver(qualifiedName, delay, failed)
+}
+
+// Adds backoff to delay if this delivery is related to some failure. Resets backoff if there was no failure.
+func (s *FederationSyncController) deliver(qualifiedName federatedtypes.QualifiedName, delay time.Duration, failed bool) {
+	key := qualifiedName.String()
+	if failed {
+		s.backoff.Next(key, time.Now())
+		delay = delay + s.backoff.Get(key)
+	} else {
+		s.backoff.Reset(key)
+	}
+	s.deliverer.DeliverAfter(key, &qualifiedName, delay)
+}
+
+// Check whether all data stores are in sync. False is returned if any of the informer/stores is not yet
+// synced with the corresponding api server.
+func (s *FederationSyncController) isSynced() bool {
+	if !s.informer.ClustersSynced() {
+		glog.V(2).Infof("Cluster list not synced")
+		return false
+	}
+	clusters, err := s.informer.GetReadyClusters()
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get ready clusters: %v", err))
+		return false
+	}
+	if !s.informer.GetTargetStore().ClustersSynced(clusters) {
+		return false
+	}
+	return true
+}
+
+// The function triggers reconciliation of all target federated resources.
+func (s *FederationSyncController) reconcileOnClusterChange() {
+	if !s.isSynced() {
+		s.clusterDeliverer.DeliverAt(allClustersKey, nil, time.Now().Add(s.clusterAvailableDelay))
+	}
+	for _, obj := range s.store.List() {
+		qualifiedName := s.adapter.QualifiedName(obj.(pkgruntime.Object))
+		s.deliver(qualifiedName, s.smallDelay, false)
+	}
+}
+
+func (s *FederationSyncController) reconcile(qualifiedName federatedtypes.QualifiedName) reconciliationStatus {
+	if !s.isSynced() {
+		return statusNotSynced
+	}
+
+	kind := s.adapter.Kind()
+	key := qualifiedName.String()
+
+	glog.V(4).Infof("Starting to reconcile %v %v", kind, key)
+	startTime := time.Now()
+	defer glog.V(4).Infof("Finished reconciling %v %v (duration: %v)", kind, key, time.Now().Sub(startTime))
+
+	obj, err := s.objFromCache(kind, key)
+	if err != nil {
+		return statusError
+	}
+	if obj == nil {
+		return statusAllOK
+	}
+
+	meta := s.adapter.ObjectMeta(obj)
+	if meta.DeletionTimestamp != nil {
+		err := s.delete(obj, kind, qualifiedName)
+		if err != nil {
+			msg := "Failed to delete %s %q: %v"
+			args := []interface{}{kind, qualifiedName, err}
+			runtime.HandleError(fmt.Errorf(msg, args...))
+			s.eventRecorder.Eventf(obj, api.EventTypeWarning, "DeleteFailed", msg, args...)
+			return statusError
+		}
+		return statusAllOK
+	}
+
+	glog.V(3).Infof("Ensuring finalizers exist on %s %q", kind, key)
+	obj, err = s.deletionHelper.EnsureFinalizers(obj)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to ensure finalizers for %s %q: %v", kind, key, err))
+		return statusError
+	}
+
+	operationsAccessor := func(adapter federatedtypes.FederatedTypeAdapter, selectedClusters []*federationapi.Cluster, unselectedClusters []*federationapi.Cluster, obj pkgruntime.Object, schedulingInfo interface{}) ([]util.FederatedOperation, error) {
+		operations, err := clusterOperations(adapter, selectedClusters, unselectedClusters, obj, key, schedulingInfo, func(clusterName string) (interface{}, bool, error) {
+			return s.informer.GetTargetStore().GetByKey(clusterName, key)
+		})
+		if err != nil {
+			s.eventRecorder.Eventf(obj, api.EventTypeWarning, "FedClusterOperationsError", "Error obtaining sync operations for %s: %s error: %s", kind, key, err.Error())
+		}
+		return operations, err
+	}
+
+	return syncToClusters(
+		s.informer.GetReadyClusters,
+		operationsAccessor,
+		selectedClusters,
+		s.updater.Update,
+		s.adapter,
+		s.informer,
+		obj,
+	)
+}
+
+func (s *FederationSyncController) objFromCache(kind, key string) (pkgruntime.Object, error) {
+	cachedObj, exist, err := s.store.GetByKey(key)
+	if err != nil {
+		wrappedErr := fmt.Errorf("Failed to query %s store for %q: %v", kind, key, err)
+		runtime.HandleError(wrappedErr)
+		return nil, err
+	}
+	if !exist {
+		return nil, nil
+	}
+
+	// Create a copy before modifying the resource to prevent racing with other readers.
+	copiedObj, err := api.Scheme.DeepCopy(cachedObj)
+	if err != nil {
+		wrappedErr := fmt.Errorf("Error in retrieving %s %q from store: %v", kind, key, err)
+		runtime.HandleError(wrappedErr)
+		return nil, err
+	}
+	if !s.adapter.IsExpectedType(copiedObj) {
+		err = fmt.Errorf("Object is not the expected type: %v", copiedObj)
+		runtime.HandleError(err)
+		return nil, err
+	}
+	return copiedObj.(pkgruntime.Object), nil
+}
+
+// delete deletes the given resource or returns error if the deletion was not complete.
+func (s *FederationSyncController) delete(obj pkgruntime.Object, kind string, qualifiedName federatedtypes.QualifiedName) error {
+	glog.V(3).Infof("Handling deletion of %s %q", kind, qualifiedName)
+
+	// Perform pre-deletion cleanup for the namespace adapter
+	namespaceAdapter, ok := s.adapter.(*federatedtypes.NamespaceAdapter)
+	if ok {
+		var err error
+		obj, err = namespaceAdapter.CleanUpNamespace(obj, s.eventRecorder)
+		if err != nil {
+			return err
+		}
+	}
+
+	_, err := s.deletionHelper.HandleObjectInUnderlyingClusters(obj)
+	if err != nil {
+		return err
+	}
+
+	err = s.adapter.FedDelete(qualifiedName, nil)
+	if err != nil {
+		// Its all good if the error is not found error. That means it is deleted already and we do not have to do anything.
+		// This is expected when we are processing an update as a result of finalizer deletion.
+		// The process that deleted the last finalizer is also going to delete the resource and we do not have to do anything.
+		if !errors.IsNotFound(err) {
+			return err
+		}
+	}
+	return nil
+}
+
+type clustersAccessorFunc func() ([]*federationapi.Cluster, error)
+type operationsFunc func(federatedtypes.FederatedTypeAdapter, []*federationapi.Cluster, []*federationapi.Cluster, pkgruntime.Object, interface{}) ([]util.FederatedOperation, error)
+type clusterSelectorFunc func(*metav1.ObjectMeta, func(map[string]string, map[string]string) (bool, error), []*federationapi.Cluster) ([]*federationapi.Cluster, []*federationapi.Cluster, error)
+type executionFunc func([]util.FederatedOperation) error
+
+// syncToClusters ensures that the state of the given object is synchronized to member clusters.
+func syncToClusters(clustersAccessor clustersAccessorFunc, operationsAccessor operationsFunc, selector clusterSelectorFunc, execute executionFunc, adapter federatedtypes.FederatedTypeAdapter, informer util.FederatedInformer, obj pkgruntime.Object) reconciliationStatus {
+	kind := adapter.Kind()
+	key := federatedtypes.ObjectKey(adapter, obj)
+
+	glog.V(3).Infof("Syncing %s %q in underlying clusters", kind, key)
+
+	clusters, err := clustersAccessor()
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to get cluster list: %v", err))
+		return statusNotSynced
+	}
+
+	selectedClusters, unselectedClusters, err := selector(adapter.ObjectMeta(obj), clusterselector.SendToCluster, clusters)
+	if err != nil {
+		return statusError
+	}
+
+	var schedulingInfo interface{}
+	if adapter.IsSchedulingAdapter() {
+		schedulingAdapter, ok := adapter.(federatedtypes.SchedulingAdapter)
+		if !ok {
+			glog.Fatalf("Adapter for kind %q does not properly implement SchedulingAdapter.", kind)
+		}
+		schedulingInfo, err = schedulingAdapter.GetSchedule(obj, key, selectedClusters, informer)
+		if err != nil {
+			runtime.HandleError(fmt.Errorf("adapter.GetSchedule() failed on adapter for %s %q: %v", kind, key, err))
+			return statusError
+		}
+	}
+
+	operations, err := operationsAccessor(adapter, selectedClusters, unselectedClusters, obj, schedulingInfo)
+	if err != nil {
+		return statusError
+	}
+
+	if adapter.IsSchedulingAdapter() {
+		schedulingAdapter, ok := adapter.(federatedtypes.SchedulingAdapter)
+		if !ok {
+			glog.Fatalf("Adapter for kind %q does not properly implement SchedulingAdapter.", kind)
+		}
+		err = schedulingAdapter.UpdateFederatedStatus(obj, schedulingInfo)
+		if err != nil {
+			runtime.HandleError(fmt.Errorf("adapter.UpdateFinished() failed on adapter for %s %q: %v", kind, key, err))
+			return statusError
+		}
+	}
+
+	if len(operations) == 0 {
+		return statusAllOK
+	}
+
+	err = execute(operations)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("Failed to execute updates for %s %q: %v", kind, key, err))
+		return statusError
+	}
+
+	// Everything is in order but let's be double sure
+	return statusNeedsRecheck
+}
+
+// selectedClusters filters the provided clusters into two slices, one containing the clusters selected by selector and the other containing the rest of the provided clusters.
+func selectedClusters(objMeta *metav1.ObjectMeta, selector func(map[string]string, map[string]string) (bool, error), clusters []*federationapi.Cluster) ([]*federationapi.Cluster, []*federationapi.Cluster, error) {
+	selectedClusters := []*federationapi.Cluster{}
+	unselectedClusters := []*federationapi.Cluster{}
+
+	for _, cluster := range clusters {
+		send, err := selector(cluster.Labels, objMeta.Annotations)
+		if err != nil {
+			return nil, nil, err
+		} else if !send {
+			unselectedClusters = append(unselectedClusters, cluster)
+		} else {
+			selectedClusters = append(selectedClusters, cluster)
+		}
+	}
+	return selectedClusters, unselectedClusters, nil
+}
+
+type clusterObjectAccessorFunc func(clusterName string) (interface{}, bool, error)
+
+// clusterOperations returns the list of operations needed to synchronize the state of the given object to the provided clusters
+func clusterOperations(adapter federatedtypes.FederatedTypeAdapter, selectedClusters []*federationapi.Cluster, unselectedClusters []*federationapi.Cluster, obj pkgruntime.Object, key string, schedulingInfo interface{}, accessor clusterObjectAccessorFunc) ([]util.FederatedOperation, error) {
+	operations := make([]util.FederatedOperation, 0)
+
+	kind := adapter.Kind()
+	for _, cluster := range selectedClusters {
+		// The data should not be modified.
+		desiredObj := adapter.Copy(obj)
+
+		clusterObj, found, err := accessor(cluster.Name)
+		if err != nil {
+			wrappedErr := fmt.Errorf("Failed to get %s %q from cluster %q: %v", kind, key, cluster.Name, err)
+			runtime.HandleError(wrappedErr)
+			return nil, wrappedErr
+		}
+
+		var scheduleAction federatedtypes.ScheduleAction = federatedtypes.ActionAdd
+		if adapter.IsSchedulingAdapter() {
+			schedulingAdapter, ok := adapter.(federatedtypes.SchedulingAdapter)
+			if !ok {
+				err = fmt.Errorf("adapter for kind %s does not properly implement SchedulingAdapter.", kind)
+				glog.Fatalf("Error: %v", err)
+			}
+			var clusterTypedObj pkgruntime.Object = nil
+			if clusterObj != nil {
+				clusterTypedObj = clusterObj.(pkgruntime.Object)
+			}
+			desiredObj, scheduleAction, err = schedulingAdapter.ScheduleObject(cluster, clusterTypedObj, desiredObj, schedulingInfo)
+			if err != nil {
+				runtime.HandleError(err)
+				return nil, err
+			}
+		}
+
+		var operationType util.FederatedOperationType = ""
+		if found {
+			if scheduleAction == federatedtypes.ActionDelete {
+				operationType = util.OperationTypeDelete
+			} else {
+				clusterObj := clusterObj.(pkgruntime.Object)
+				if !adapter.Equivalent(desiredObj, clusterObj) {
+					operationType = util.OperationTypeUpdate
+				}
+			}
+		} else if scheduleAction == federatedtypes.ActionAdd {
+			operationType = util.OperationTypeAdd
+		}
+
+		if len(operationType) > 0 {
+			operations = append(operations, util.FederatedOperation{
+				Type:        operationType,
+				Obj:         desiredObj,
+				ClusterName: cluster.Name,
+				Key:         key,
+			})
+		}
+	}
+
+	for _, cluster := range unselectedClusters {
+		clusterObj, found, err := accessor(cluster.Name)
+		if err != nil {
+			wrappedErr := fmt.Errorf("Failed to get %s %q from cluster %q: %v", kind, key, cluster.Name, err)
+			runtime.HandleError(wrappedErr)
+			return nil, wrappedErr
+		}
+		if found {
+			operations = append(operations, util.FederatedOperation{
+				Type:        util.OperationTypeDelete,
+				Obj:         clusterObj.(pkgruntime.Object),
+				ClusterName: cluster.Name,
+				Key:         key,
+			})
+		}
+	}
+
+	return operations, nil
+}

vendor/k8s.io/kubernetes/federation/pkg/federation-controller/sync/controller_test.go

@@ -0,0 +1,231 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sync
+
+import (
+	"errors"
+	"testing"
+
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pkgruntime "k8s.io/apimachinery/pkg/runtime"
+	federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	"k8s.io/kubernetes/federation/pkg/federatedtypes"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	fedtest "k8s.io/kubernetes/federation/pkg/federation-controller/util/test"
+
+	"github.com/stretchr/testify/require"
+)
+
+var awfulError error = errors.New("Something bad happened")
+
+func TestSyncToClusters(t *testing.T) {
+	adapter := &federatedtypes.SecretAdapter{}
+	obj := adapter.NewTestObject("foo")
+
+	testCases := map[string]struct {
+		clusterError    bool
+		operationsError bool
+		executionError  bool
+		operations      []util.FederatedOperation
+		status          reconciliationStatus
+	}{
+		"Error listing clusters redelivers with cluster delay": {
+			clusterError: true,
+			status:       statusNotSynced,
+		},
+		"Error retrieving cluster operations redelivers": {
+			operationsError: true,
+			status:          statusError,
+		},
+		"No operations returns ok": {
+			status: statusAllOK,
+		},
+		"Execution error redelivers": {
+			executionError: true,
+			operations:     []util.FederatedOperation{{}},
+			status:         statusError,
+		},
+		"Successful update indicates recheck": {
+			operations: []util.FederatedOperation{{}},
+			status:     statusNeedsRecheck,
+		},
+	}
+
+	for testName, testCase := range testCases {
+		t.Run(testName, func(t *testing.T) {
+			status := syncToClusters(
+				func() ([]*federationapi.Cluster, error) {
+					if testCase.clusterError {
+						return nil, awfulError
+					}
+					return nil, nil
+				},
+				func(federatedtypes.FederatedTypeAdapter, []*federationapi.Cluster, []*federationapi.Cluster, pkgruntime.Object, interface{}) ([]util.FederatedOperation, error) {
+					if testCase.operationsError {
+						return nil, awfulError
+					}
+					return testCase.operations, nil
+				},
+				func(objMeta *metav1.ObjectMeta, selector func(map[string]string, map[string]string) (bool, error), clusters []*federationapi.Cluster) ([]*federationapi.Cluster, []*federationapi.Cluster, error) {
+					return clusters, []*federationapi.Cluster{}, nil
+				},
+				func([]util.FederatedOperation) error {
+					if testCase.executionError {
+						return awfulError
+					}
+					return nil
+				},
+				adapter,
+				nil,
+				obj,
+			)
+			require.Equal(t, testCase.status, status, "Unexpected status!")
+		})
+	}
+}
+
+func TestSelectedClusters(t *testing.T) {
+	clusterOne := fedtest.NewCluster("cluster1", apiv1.ConditionTrue)
+	clusterOne.Labels = map[string]string{"name": "cluster1"}
+	clusterTwo := fedtest.NewCluster("cluster2", apiv1.ConditionTrue)
+	clusterTwo.Labels = map[string]string{"name": "cluster2"}
+
+	clusters := []*federationapi.Cluster{clusterOne, clusterTwo}
+	testCases := map[string]struct {
+		expectedSelectorError      bool
+		clusterOneSelected         bool
+		clusterTwoSelected         bool
+		expectedSelectedClusters   []*federationapi.Cluster
+		expectedUnselectedClusters []*federationapi.Cluster
+	}{
+		"Selector returned error": {
+			expectedSelectorError: true,
+		},
+		"All clusters selected": {
+			clusterOneSelected:         true,
+			clusterTwoSelected:         true,
+			expectedSelectedClusters:   clusters,
+			expectedUnselectedClusters: []*federationapi.Cluster{},
+		},
+		"One cluster selected": {
+			clusterOneSelected:         true,
+			expectedSelectedClusters:   []*federationapi.Cluster{clusterOne},
+			expectedUnselectedClusters: []*federationapi.Cluster{clusterTwo},
+		},
+		"No clusters selected": {
+			expectedSelectedClusters:   []*federationapi.Cluster{},
+			expectedUnselectedClusters: clusters,
+		},
+	}
+
+	for testName, testCase := range testCases {
+		t.Run(testName, func(t *testing.T) {
+			selectedClusters, unselectedClusters, err := selectedClusters(&metav1.ObjectMeta{}, func(labels map[string]string, annotations map[string]string) (bool, error) {
+				if testCase.expectedSelectorError {
+					return false, awfulError
+				}
+				if labels["name"] == "cluster1" {
+					return testCase.clusterOneSelected, nil
+				}
+				if labels["name"] == "cluster2" {
+					return testCase.clusterTwoSelected, nil
+				}
+				t.Errorf("Unexpected cluster")
+				return false, nil
+			}, clusters)
+
+			if testCase.expectedSelectorError {
+				require.Error(t, err, "An error was expected")
+			} else {
+				require.NoError(t, err, "An error was not expected")
+			}
+			require.Equal(t, testCase.expectedSelectedClusters, selectedClusters, "Expected the correct clusters to be selected.")
+			require.Equal(t, testCase.expectedUnselectedClusters, unselectedClusters, "Expected the correct clusters to be unselected.")
+		})
+	}
+}
+
+func TestClusterOperations(t *testing.T) {
+	adapter := &federatedtypes.SecretAdapter{}
+	obj := adapter.NewTestObject("foo")
+	differingObj := adapter.Copy(obj)
+	federatedtypes.SetAnnotation(adapter, differingObj, "foo", "bar")
+
+	testCases := map[string]struct {
+		clusterObject pkgruntime.Object
+		expectedErr   bool
+		sendToCluster bool
+
+		operationType util.FederatedOperationType
+	}{
+		"Accessor error returned": {
+			expectedErr: true,
+		},
+		"Missing cluster object should result in add operation": {
+			operationType: util.OperationTypeAdd,
+			sendToCluster: true,
+		},
+		"Differing cluster object should result in update operation": {
+			clusterObject: differingObj,
+			operationType: util.OperationTypeUpdate,
+			sendToCluster: true,
+		},
+		"Matching object and not matching ClusterSelector should result in delete operation": {
+			clusterObject: obj,
+			operationType: util.OperationTypeDelete,
+			sendToCluster: false,
+		},
+		"Matching cluster object should not result in an operation": {
+			clusterObject: obj,
+			sendToCluster: true,
+		},
+	}
+	for testName, testCase := range testCases {
+		t.Run(testName, func(t *testing.T) {
+			clusters := []*federationapi.Cluster{fedtest.NewCluster("cluster1", apiv1.ConditionTrue)}
+			key := federatedtypes.ObjectKey(adapter, obj)
+
+			var selectedClusters, unselectedClusters []*federationapi.Cluster
+			if testCase.sendToCluster {
+				selectedClusters = clusters
+				unselectedClusters = []*federationapi.Cluster{}
+			} else {
+				selectedClusters = []*federationapi.Cluster{}
+				unselectedClusters = clusters
+			}
+			// TODO: Tests for ScheduleObject on type adapter
+			operations, err := clusterOperations(adapter, selectedClusters, unselectedClusters, obj, key, nil, func(string) (interface{}, bool, error) {
+				if testCase.expectedErr {
+					return nil, false, awfulError
+				}
+				return testCase.clusterObject, (testCase.clusterObject != nil), nil
+			})
+			if testCase.expectedErr {
+				require.Error(t, err, "An error was expected")
+			} else {
+				require.NoError(t, err, "An error was not expected")
+			}
+			if len(testCase.operationType) == 0 {
+				require.True(t, len(operations) == 0, "An operation was not expected")
+			} else {
+				require.True(t, len(operations) == 1, "A single operation was expected")
+				require.Equal(t, testCase.operationType, operations[0].Type, "Unexpected operation returned")
+			}
+		})
+	}
+}