Replace godep with dep

vendor/k8s.io/kubernetes/test/e2e_federation/BUILD

@@ -0,0 +1,72 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "apiserver.go",
+        "authn.go",
+        "crud.go",
+        "event.go",
+        "ingress.go",
+        "job.go",
+        "namespace.go",
+        "replicaset.go",
+        "service.go",
+        "upgrade.go",
+        "util.go",
+    ],
+    deps = [
+        "//federation/apis/federation:go_default_library",
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset/typed/core/v1:go_default_library",
+        "//federation/pkg/federatedtypes:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//pkg/api:go_default_library",
+        "//pkg/api/v1:go_default_library",
+        "//pkg/cloudprovider:go_default_library",
+        "//test/e2e/chaosmonkey:go_default_library",
+        "//test/e2e/common:go_default_library",
+        "//test/e2e/framework:go_default_library",
+        "//test/e2e_federation/framework:go_default_library",
+        "//test/e2e_federation/upgrades:go_default_library",
+        "//test/utils/image:go_default_library",
+        "//vendor/github.com/onsi/ginkgo:go_default_library",
+        "//vendor/github.com/onsi/gomega:go_default_library",
+        "//vendor/k8s.io/api/batch/v1:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/rand:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library",
+        "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//test/e2e_federation/framework:all-srcs",
+        "//test/e2e_federation/upgrades:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/test/e2e_federation/OWNERS

@@ -0,0 +1,16 @@
+reviewers:
+  - colhom
+  - csbell
+  - irfanurrehman
+  - madhusudancs
+  - mwielgus
+  - nikhiljindal
+  - quinton-hoole
+  - shashidharatd
+approvers:
+  - csbell
+  - madhusudancs
+  - mwielgus
+  - nikhiljindal
+  - quinton-hoole
+  - shashidharatd

vendor/k8s.io/kubernetes/test/e2e_federation/README.md

@@ -0,0 +1,3 @@
+See [e2e-tests](https://git.k8s.io/community/contributors/devel/e2e-tests.md#federation-e2e-tests)
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/test/e2e_federation//README.md?pixel)]()

vendor/k8s.io/kubernetes/test/e2e_federation/apiserver.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_federation
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+	fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
+)
+
+// Create/delete cluster api objects
+var _ = framework.KubeDescribe("Federation apiserver [Feature:Federation]", func() {
+	f := fedframework.NewDefaultFederatedFramework("federation-cluster")
+	testClusterPrefix := "test"
+
+	Describe("Cluster objects [Serial]", func() {
+		AfterEach(func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+
+			// Delete registered clusters.
+			// This is if a test failed, it should not affect other tests.
+			clusterList, err := f.FederationClientset.Federation().Clusters().List(metav1.ListOptions{LabelSelector: "prefix=" + testClusterPrefix})
+			Expect(err).NotTo(HaveOccurred())
+			for _, cluster := range clusterList.Items {
+				err := f.FederationClientset.Federation().Clusters().Delete(cluster.Name, &metav1.DeleteOptions{})
+				Expect(err).NotTo(HaveOccurred())
+			}
+		})
+
+		It("should be created and deleted successfully", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+
+			contexts := f.GetUnderlyingFederatedContexts()
+
+			By(fmt.Sprintf("Creating %d cluster objects", len(contexts)))
+			for _, context := range contexts {
+				createClusterObjectOrFail(f, &context, testClusterPrefix)
+			}
+
+			By(fmt.Sprintf("Checking that %d clusters are ready", len(contexts)))
+			for _, context := range contexts {
+				fedframework.ClusterIsReadyOrFail(f, context.Name)
+			}
+			framework.Logf("%d clusters are Ready", len(contexts))
+
+			// Verify that deletion works.
+			framework.Logf("Deleting %d clusters", len(contexts))
+			for _, context := range contexts {
+				clusterName := testClusterPrefix + context.Name
+				framework.Logf("Deleting cluster object: %s (%s, secret: %s)", clusterName, context.Cluster.Cluster.Server, context.Name)
+				err := f.FederationClientset.Federation().Clusters().Delete(clusterName, &metav1.DeleteOptions{})
+				framework.ExpectNoError(err, fmt.Sprintf("unexpected error in deleting cluster %s: %+v", clusterName, err))
+				framework.Logf("Successfully deleted cluster object: %s (%s, secret: %s)", clusterName, context.Cluster.Cluster.Server, context.Name)
+			}
+
+			// There should not be any remaining cluster.
+			framework.Logf("Verifying that zero test clusters remain")
+			clusterList, err := f.FederationClientset.Federation().Clusters().List(metav1.ListOptions{LabelSelector: "prefix=" + testClusterPrefix})
+			Expect(err).NotTo(HaveOccurred())
+			if len(clusterList.Items) != 0 {
+				framework.Failf("there should not have been any remaining clusters. Found: %+v", clusterList)
+			}
+			framework.Logf("Verified that zero clusters remain")
+		})
+	})
+
+	Describe("Admission control [NoCluster]", func() {
+		AfterEach(func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+		})
+
+		It("should not be able to create resources if namespace does not exist", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+
+			// Creating a service in a non-existing namespace should fail.
+			svcNamespace := "federation-admission-test-ns"
+			svcName := "myns"
+			clientset := f.FederationClientset
+			framework.Logf("Trying to create service %s in namespace %s, expect to get error", svcName, svcNamespace)
+			if _, err := clientset.Core().Services(svcNamespace).Create(newService(svcName, svcNamespace)); err == nil {
+				framework.Failf("Expected to get an error while creating a service in a non-existing namespace")
+			}
+
+			// Note: We have other tests that verify that we can create resources in existing namespaces, so we dont test it again here.
+		})
+	})
+})
+
+func newService(name, namespace string) *v1.Service {
+	return &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: v1.ServiceSpec{
+			Ports: []v1.ServicePort{
+				{
+					Port: 80,
+				},
+			},
+		},
+	}
+}

vendor/k8s.io/kubernetes/test/e2e_federation/authn.go

@@ -0,0 +1,207 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_federation
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	"k8s.io/kubernetes/test/e2e/framework"
+	fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// TODO: These tests should be integration tests rather than e2e tests, when the
+// integration test harness is ready.
+var _ = framework.KubeDescribe("[Feature:Federation]", func() {
+	f := fedframework.NewDefaultFederatedFramework("federation-apiserver-authn")
+
+	var _ = Describe("Federation API server authentication [NoCluster]", func() {
+		BeforeEach(func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+		})
+
+		It("should accept cluster resources when the client has certificate authentication credentials", func() {
+			fcs, err := federationClientSetWithCert()
+			framework.ExpectNoError(err)
+
+			nsName := f.FederationNamespace.Name
+			svc := createServiceOrFail(fcs, nsName, FederatedServiceName)
+			deleteServiceOrFail(f.FederationClientset, nsName, svc.Name, nil)
+		})
+
+		It("should accept cluster resources when the client has HTTP Basic authentication credentials", func() {
+			fcs, err := federationClientSetWithBasicAuth(true /* valid */)
+			framework.ExpectNoError(err)
+
+			nsName := f.FederationNamespace.Name
+			svc, err := createService(fcs, nsName, FederatedServiceName)
+			Expect(err).NotTo(HaveOccurred())
+			deleteServiceOrFail(fcs, nsName, svc.Name, nil)
+		})
+
+		It("should accept cluster resources when the client has token authentication credentials", func() {
+			fcs, err := federationClientSetWithToken(true /* valid */)
+			framework.ExpectNoError(err)
+
+			nsName := f.FederationNamespace.Name
+			svc, err := createService(fcs, nsName, FederatedServiceName)
+			Expect(err).NotTo(HaveOccurred())
+			deleteServiceOrFail(fcs, nsName, svc.Name, nil)
+		})
+
+		It("should not accept cluster resources when the client has no authentication credentials", func() {
+			fcs, err := unauthenticatedFederationClientSet()
+			framework.ExpectNoError(err)
+
+			nsName := f.FederationNamespace.Name
+			_, err = createService(fcs, nsName, FederatedServiceName)
+			Expect(errors.IsUnauthorized(err)).To(BeTrue())
+		})
+
+		// TODO: Add a test for invalid certificate credentials. The certificate is validated for
+		// correct format, so it cannot contain random noise.
+
+		It("should not accept cluster resources when the client has invalid HTTP Basic authentication credentials", func() {
+			fcs, err := federationClientSetWithBasicAuth(false /* invalid */)
+			framework.ExpectNoError(err)
+
+			nsName := f.FederationNamespace.Name
+			_, err = createService(fcs, nsName, FederatedServiceName)
+			Expect(errors.IsUnauthorized(err)).To(BeTrue())
+		})
+
+		It("should not accept cluster resources when the client has invalid token authentication credentials", func() {
+			fcs, err := federationClientSetWithToken(false /* invalid */)
+			framework.ExpectNoError(err)
+
+			nsName := f.FederationNamespace.Name
+			_, err = createService(fcs, nsName, FederatedServiceName)
+			Expect(errors.IsUnauthorized(err)).To(BeTrue())
+		})
+
+	})
+})
+
+// unauthenticatedFederationClientSet returns a Federation Clientset configured with
+// no authentication credentials.
+func unauthenticatedFederationClientSet() (*federation_clientset.Clientset, error) {
+	config, err := fedframework.LoadFederatedConfig(&clientcmd.ConfigOverrides{})
+	if err != nil {
+		return nil, err
+	}
+	config.Insecure = true
+	config.CAData = []byte{}
+	config.CertData = []byte{}
+	config.KeyData = []byte{}
+	config.BearerToken = ""
+
+	c, err := federation_clientset.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating federation clientset: %v", err)
+	}
+
+	return c, nil
+}
+
+// federationClientSetWithCert returns a Federation Clientset configured with
+// certificate authentication credentials.
+func federationClientSetWithCert() (*federation_clientset.Clientset, error) {
+	config, err := fedframework.LoadFederatedConfig(&clientcmd.ConfigOverrides{})
+	if err != nil {
+		return nil, err
+	}
+
+	config.BearerToken = ""
+
+	c, err := federation_clientset.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating federation clientset: %v", err)
+	}
+
+	return c, nil
+}
+
+// federationClientSetWithBasicAuth returns a Federation Clientset configured with
+// HTTP Basic authentication credentials.
+func federationClientSetWithBasicAuth(valid bool) (*federation_clientset.Clientset, error) {
+	config, err := fedframework.LoadFederatedConfig(&clientcmd.ConfigOverrides{})
+	if err != nil {
+		return nil, err
+	}
+
+	config.Insecure = true
+	config.CAData = []byte{}
+	config.CertData = []byte{}
+	config.KeyData = []byte{}
+	config.BearerToken = ""
+
+	if !valid {
+		config.Username = ""
+		config.Password = ""
+	} else {
+		// This is a hacky approach to getting the basic auth credentials, but since
+		// the token and the username/password cannot live in the same AuthInfo object,
+		// and because we do not want to store basic auth credentials with token and
+		// certificate credentials for security reasons, we must dig it out by hand.
+		c, err := framework.RestclientConfig(framework.TestContext.FederatedKubeContext)
+		if err != nil {
+			return nil, err
+		}
+		if authInfo, ok := c.AuthInfos[fmt.Sprintf("%s-basic-auth", framework.TestContext.FederatedKubeContext)]; ok {
+			config.Username = authInfo.Username
+			config.Password = authInfo.Password
+		}
+	}
+
+	c, err := federation_clientset.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating federation clientset: %v", err)
+	}
+
+	return c, nil
+}
+
+// federationClientSetWithToken returns a Federation Clientset configured with
+// token authentication credentials.
+func federationClientSetWithToken(valid bool) (*federation_clientset.Clientset, error) {
+	config, err := fedframework.LoadFederatedConfig(&clientcmd.ConfigOverrides{})
+	if err != nil {
+		return nil, err
+	}
+	config.Insecure = true
+	config.CAData = []byte{}
+	config.CertData = []byte{}
+	config.KeyData = []byte{}
+	config.Username = ""
+	config.Password = ""
+
+	if !valid {
+		config.BearerToken = "invalid"
+	}
+
+	c, err := federation_clientset.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating federation clientset: %v", err)
+	}
+
+	return c, nil
+}

vendor/k8s.io/kubernetes/test/e2e_federation/crud.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_federation
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo"
+
+	kubeclientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/federation/pkg/federatedtypes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
+)
+
+var _ = framework.KubeDescribe("Federated types [Feature:Federation][Experimental] ", func() {
+	var clusterClients []kubeclientset.Interface
+
+	f := fedframework.NewDefaultFederatedFramework("federated-types")
+
+	fedTypes := federatedtypes.FederatedTypes()
+	for name := range fedTypes {
+		fedType := fedTypes[name]
+		Describe(fmt.Sprintf("Federated %q resources", name), func() {
+			It("should be created, read, updated and deleted successfully", func() {
+				fedframework.SkipUnlessFederated(f.ClientSet)
+
+				// Load clients only if not skipping to avoid doing
+				// unnecessary work.  Assume clients can be shared
+				// across tests.
+				if clusterClients == nil {
+					clusterClients = f.GetClusterClients()
+				}
+				adapter := fedType.AdapterFactory(f.FederationClientset, f.FederationConfig, nil)
+				crudTester := fedframework.NewFederatedTypeCRUDTester(adapter, clusterClients)
+				obj := adapter.NewTestObject(f.FederationNamespace.Name)
+				crudTester.CheckLifecycle(obj)
+			})
+		})
+	}
+})

vendor/k8s.io/kubernetes/test/e2e_federation/event.go

@@ -0,0 +1,99 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_federation
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	"k8s.io/kubernetes/test/e2e/framework"
+	fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
+
+	. "github.com/onsi/gomega"
+)
+
+const (
+	FederationEventName = "federation-event"
+)
+
+// Create/delete event api objects.
+var _ = framework.KubeDescribe("Federation events [Feature:Federation]", func() {
+	f := fedframework.NewDefaultFederatedFramework("federation-event")
+
+	Describe("Event objects [NoCluster]", func() {
+		AfterEach(func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+
+			nsName := f.FederationNamespace.Name
+			// Delete registered events.
+			eventList, err := f.FederationClientset.Core().Events(nsName).List(metav1.ListOptions{})
+			Expect(err).NotTo(HaveOccurred())
+			for _, event := range eventList.Items {
+				err := f.FederationClientset.Core().Events(nsName).Delete(event.Name, &metav1.DeleteOptions{})
+				Expect(err).NotTo(HaveOccurred())
+			}
+		})
+
+		It("should be created and deleted successfully", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+
+			nsName := f.FederationNamespace.Name
+			event := createEventOrFail(f.FederationClientset, nsName)
+			By(fmt.Sprintf("Creation of event %q in namespace %q succeeded.  Deleting event.", event.Name, nsName))
+			// Cleanup
+			err := f.FederationClientset.Core().Events(nsName).Delete(event.Name, &metav1.DeleteOptions{})
+			framework.ExpectNoError(err, "Error deleting event %q in namespace %q", event.Name, event.Namespace)
+			By(fmt.Sprintf("Deletion of event %q in namespace %q succeeded.", event.Name, nsName))
+		})
+
+	})
+})
+
+func createEventOrFail(clientset *federation_clientset.Clientset, namespace string) *v1.Event {
+	if clientset == nil || len(namespace) == 0 {
+		Fail(fmt.Sprintf("Internal error: invalid parameters passed to createEventOrFail: clientset: %v, namespace: %v", clientset, namespace))
+	}
+	By(fmt.Sprintf("Creating federated event %q in namespace %q", FederationEventName, namespace))
+
+	event := &v1.Event{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      FederationEventName,
+			Namespace: namespace,
+		},
+		InvolvedObject: v1.ObjectReference{
+			Kind:       "Pod",
+			Name:       "pod-name",
+			Namespace:  namespace,
+			UID:        "C934D34AFB20242",
+			APIVersion: "version",
+		},
+		Source: v1.EventSource{
+			Component: "kubelet",
+			Host:      "kublet.node1",
+		},
+		Count: 1,
+		Type:  v1.EventTypeNormal,
+	}
+
+	_, err := clientset.Core().Events(namespace).Create(event)
+	framework.ExpectNoError(err, "Creating event %q in namespace %q", event.Name, namespace)
+	By(fmt.Sprintf("Successfully created federated event %q in namespace %q", FederationEventName, namespace))
+	return event
+}

vendor/k8s.io/kubernetes/test/e2e_federation/framework/BUILD

@@ -0,0 +1,50 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cluster.go",
+        "crudtester.go",
+        "framework.go",
+        "util.go",
+    ],
+    deps = [
+        "//federation/apis/federation/v1beta1:go_default_library",
+        "//federation/client/clientset_generated/federation_clientset:go_default_library",
+        "//federation/pkg/federatedtypes:go_default_library",
+        "//federation/pkg/federatedtypes/crudtester:go_default_library",
+        "//federation/pkg/federation-controller/util:go_default_library",
+        "//pkg/api:go_default_library",
+        "//pkg/api/validation:go_default_library",
+        "//test/e2e/framework:go_default_library",
+        "//vendor/github.com/onsi/ginkgo:go_default_library",
+        "//vendor/github.com/onsi/gomega:go_default_library",
+        "//vendor/gopkg.in/yaml.v2:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/rest:go_default_library",
+        "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/test/e2e_federation/framework/cluster.go

@@ -0,0 +1,240 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const federatedClustersWaitTimeout = 1 * time.Minute
+
+// ClusterSlice is a slice of clusters
+type ClusterSlice []*Cluster
+
+// Cluster keeps track of the name and client of a cluster in the federation
+type Cluster struct {
+	Name string
+	*kubeclientset.Clientset
+}
+
+// registeredClustersFromConfig configures clientsets for registered clusters from the e2e kubeconfig
+func registeredClustersFromConfig(f *Framework) ClusterSlice {
+	contexts := f.GetUnderlyingFederatedContexts()
+
+	By("Obtaining a list of all the clusters")
+	clusterList := waitForAllRegisteredClusters(f, len(contexts))
+
+	framework.Logf("Checking that %d clusters are Ready", len(contexts))
+	for _, context := range contexts {
+		ClusterIsReadyOrFail(f, context.Name)
+
+	}
+	framework.Logf("%d clusters are Ready", len(contexts))
+
+	clusters := ClusterSlice{}
+	for i, c := range clusterList.Items {
+		framework.Logf("Creating a clientset for the cluster %s", c.Name)
+		Expect(framework.TestContext.KubeConfig).ToNot(Equal(""), "KubeConfig must be specified to load clusters' client config")
+		config := restConfigFromContext(c, i)
+		clusters = append(clusters, &Cluster{
+			Name:      c.Name,
+			Clientset: clientsetFromConfig(f, config, c.Spec.ServerAddressByClientCIDRs[0].ServerAddress),
+		})
+
+	}
+	waitForNamespaceInFederatedClusters(clusters, f.FederationNamespace.Name)
+	return clusters
+}
+
+// waitForAllRegisteredClusters waits for all clusters defined in e2e context to be created
+// return ClusterList until the listed cluster items equals clusterCount
+func waitForAllRegisteredClusters(f *Framework, clusterCount int) *federationapi.ClusterList {
+	var clusterList *federationapi.ClusterList
+	if err := wait.PollImmediate(framework.Poll, federatedClustersWaitTimeout, func() (bool, error) {
+		var err error
+		clusterList, err = f.FederationClientset.Federation().Clusters().List(metav1.ListOptions{})
+		if err != nil {
+			return false, err
+		}
+		framework.Logf("%d clusters registered, waiting for %d", len(clusterList.Items), clusterCount)
+		if len(clusterList.Items) == clusterCount {
+			return true, nil
+		}
+		return false, nil
+	}); err != nil {
+		framework.Failf("Failed to list registered clusters: %+v", err)
+	}
+	return clusterList
+}
+
+func restConfigFromContext(c federationapi.Cluster, i int) *restclient.Config {
+	kubecfg, err := clientcmd.LoadFromFile(framework.TestContext.KubeConfig)
+	framework.ExpectNoError(err, "error loading KubeConfig: %v", err)
+
+	ccfg := clientcmd.NewNonInteractiveClientConfig(*kubecfg, c.Name, &clientcmd.ConfigOverrides{}, clientcmd.NewDefaultClientConfigLoadingRules())
+	cfg, err := ccfg.ClientConfig()
+	framework.ExpectNoError(err, "Error creating client config in cluster #%d (%q)", i, c.Name)
+	return cfg
+}
+
+func clientsetFromConfig(f *Framework, cfg *restclient.Config, host string) *kubeclientset.Clientset {
+	cfg.Host = host
+	cfg.QPS = f.Framework.Options.ClientQPS
+	cfg.Burst = f.Framework.Options.ClientBurst
+	return kubeclientset.NewForConfigOrDie(restclient.AddUserAgent(cfg, "federation-e2e"))
+}
+
+// waitForNamespaceInFederatedClusters waits for the federated namespace to be created in federated clusters
+func waitForNamespaceInFederatedClusters(clusters ClusterSlice, nsName string) {
+	for _, c := range clusters {
+		name := c.Name
+		By(fmt.Sprintf("Waiting for namespace %q to be created in cluster %q", nsName, name))
+		err := wait.PollImmediate(framework.Poll, FederatedDefaultTestTimeout, func() (bool, error) {
+			_, err := c.Clientset.CoreV1().Namespaces().Get(nsName, metav1.GetOptions{})
+			if errors.IsNotFound(err) {
+				return false, nil
+			} else if err != nil {
+				framework.Logf("An error occurred waiting for namespace %q to be created in cluster %q: %v", nsName, name, err)
+				return false, nil
+			}
+			By(fmt.Sprintf("Namespace %q exists in cluster %q", nsName, name))
+			return true, nil
+		})
+		framework.ExpectNoError(err, "Failed to verify federated namespace %q creation in cluster %q", nsName, name)
+	}
+}
+
+// ClusterIsReadyOrFail checks whether the named cluster is ready
+func ClusterIsReadyOrFail(f *Framework, clusterName string) {
+	By(fmt.Sprintf("Checking readiness of cluster %q", clusterName))
+	err := wait.PollImmediate(framework.Poll, FederatedDefaultTestTimeout, func() (bool, error) {
+		c, err := f.FederationClientset.Federation().Clusters().Get(clusterName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		for _, condition := range c.Status.Conditions {
+			if condition.Type == federationapi.ClusterReady && condition.Status == v1.ConditionTrue {
+				return true, nil
+			}
+		}
+		return false, nil
+	})
+	framework.ExpectNoError(err, fmt.Sprintf("Unexpected error in verifying if cluster %q is ready: %+v", clusterName, err))
+	framework.Logf("Cluster %s is Ready", clusterName)
+}
+
+// Cache the cluster config to avoid having to retrieve it for each test
+type clusterConfig struct {
+	name   string
+	host   string
+	config []byte
+}
+
+var cachedClusterConfigs []*clusterConfig
+
+// registeredClustersFromSecrets configures clientsets for cluster access from secrets in the host cluster
+func registeredClustersFromSecrets(f *Framework) ClusterSlice {
+	if cachedClusterConfigs == nil {
+		cachedClusterConfigs = clusterConfigFromSecrets(f)
+	}
+
+	clusters := ClusterSlice{}
+	for _, clusterConf := range cachedClusterConfigs {
+		restConfig := restConfigForCluster(clusterConf)
+		clientset := clientsetFromConfig(f, restConfig, clusterConf.host)
+		clusters = append(clusters, &Cluster{
+			Name:      clusterConf.name,
+			Clientset: clientset,
+		})
+	}
+
+	waitForNamespaceInFederatedClusters(clusters, f.FederationNamespace.Name)
+
+	return clusters
+}
+
+// clusterConfigFromSecrets retrieves cluster configuration from
+// secrets in the host cluster
+func clusterConfigFromSecrets(f *Framework) []*clusterConfig {
+	By("Obtaining a list of registered clusters")
+	clusterList, err := f.FederationClientset.Federation().Clusters().List(metav1.ListOptions{})
+	framework.ExpectNoError(err, fmt.Sprintf("Error retrieving list of federated clusters: %+v", err))
+	if len(clusterList.Items) == 0 {
+		framework.Failf("No registered clusters found")
+	}
+
+	clusterConfigs := []*clusterConfig{}
+	for _, c := range clusterList.Items {
+		ClusterIsReadyOrFail(f, c.Name)
+		config := clusterConfigFromSecret(f, c.Name, c.Spec.SecretRef.Name)
+		clusterConfigs = append(clusterConfigs, &clusterConfig{
+			name:   c.Name,
+			host:   c.Spec.ServerAddressByClientCIDRs[0].ServerAddress,
+			config: config,
+		})
+	}
+
+	return clusterConfigs
+}
+
+// clusterConfigFromSecret retrieves configuration for a accessing a
+// cluster from a secret in the host cluster
+func clusterConfigFromSecret(f *Framework, clusterName string, secretName string) []byte {
+	By(fmt.Sprintf("Loading configuration for cluster %q", clusterName))
+	namespace := framework.FederationSystemNamespace()
+	secret, err := f.Framework.ClientSet.Core().Secrets(namespace).Get(secretName, metav1.GetOptions{})
+	framework.ExpectNoError(err, fmt.Sprintf("Error loading config secret \"%s/%s\" for cluster %q: %+v", namespace, secretName, clusterName, err))
+
+	config, ok := secret.Data[util.KubeconfigSecretDataKey]
+	if !ok || len(config) == 0 {
+		framework.Failf("Secret \"%s/%s\" for cluster %q has no value for key %q", namespace, secretName, clusterName, util.KubeconfigSecretDataKey)
+	}
+
+	return config
+}
+
+// restConfigForCluster creates a rest client config for the given cluster config
+func restConfigForCluster(clusterConf *clusterConfig) *restclient.Config {
+	cfg, err := clientcmd.Load(clusterConf.config)
+	framework.ExpectNoError(err, fmt.Sprintf("Error loading configuration for cluster %q: %+v", clusterConf.name, err))
+
+	restConfig, err := clientcmd.NewDefaultClientConfig(*cfg, &clientcmd.ConfigOverrides{}).ClientConfig()
+	framework.ExpectNoError(err, fmt.Sprintf("Error creating client for cluster %q: %+v", clusterConf.name, err))
+	return restConfig
+}
+
+func GetZoneFromClusterName(clusterName string) string {
+	// Ref: https://github.com/kubernetes/kubernetes/blob/master/cluster/kube-util.sh#L55
+	prefix := "federation-e2e-" + framework.TestContext.Provider + "-"
+	return strings.TrimPrefix(clusterName, prefix)
+}

vendor/k8s.io/kubernetes/test/e2e_federation/framework/crudtester.go

@@ -0,0 +1,44 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	. "github.com/onsi/ginkgo"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/federation/pkg/federatedtypes"
+	"k8s.io/kubernetes/federation/pkg/federatedtypes/crudtester"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+// Adapt the methods to log/fail in e2e to the interface expected by CRUDHelper
+type e2eTestLogger struct{}
+
+func (e2eTestLogger) Fatal(msg string) {
+	Fail(msg)
+}
+
+func (e2eTestLogger) Fatalf(format string, args ...interface{}) {
+	framework.Failf(format, args...)
+}
+
+func (e2eTestLogger) Logf(format string, args ...interface{}) {
+	framework.Logf(format, args...)
+}
+
+func NewFederatedTypeCRUDTester(adapter federatedtypes.FederatedTypeAdapter, clusterClients []kubeclientset.Interface) *crudtester.FederatedTypeCRUDTester {
+	return crudtester.NewFederatedTypeCRUDTester(&e2eTestLogger{}, adapter, clusterClients, framework.Poll, FederatedDefaultTestTimeout)
+}

vendor/k8s.io/kubernetes/test/e2e_federation/framework/framework.go

@@ -0,0 +1,256 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+	"time"
+
+	"k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	yaml "gopkg.in/yaml.v2"
+)
+
+// Framework extends e2e Framework and adds federation specific fields
+type Framework struct {
+	*framework.Framework
+
+	// To make sure that this framework cleans up after itself, no matter what,
+	// we install a Cleanup action before each test and clear it after.  If we
+	// should abort, the AfterSuite hook should run all Cleanup actions.
+	cleanupHandle framework.CleanupActionHandle
+
+	FederationConfig *restclient.Config
+
+	FederationClientset *federation_clientset.Clientset
+
+	FederationNamespace *v1.Namespace
+}
+
+func NewDefaultFederatedFramework(baseName string) *Framework {
+	f := &Framework{}
+
+	// Register the federation cleanup before initializing the default
+	// e2e framework to ensure it gets called before the default
+	// framework's cleanup.
+	AfterEach(f.FederationAfterEach)
+
+	f.Framework = framework.NewDefaultFramework(baseName)
+	f.Framework.SkipNamespaceCreation = true
+
+	// Register the federation setup after initializing the default
+	// e2e framework to ensure it gets called after the default
+	// framework's setup.
+	BeforeEach(f.FederationBeforeEach)
+
+	return f
+}
+
+// FederationBeforeEach checks for federation apiserver is ready and makes a namespace.
+func (f *Framework) FederationBeforeEach() {
+	// The fact that we need this feels like a bug in ginkgo.
+	// https://github.com/onsi/ginkgo/issues/222
+	f.cleanupHandle = framework.AddCleanupAction(f.FederationAfterEach)
+
+	if f.FederationConfig == nil {
+		By("Reading the federation configuration")
+		var err error
+		f.FederationConfig, err = LoadFederatedConfig(&clientcmd.ConfigOverrides{})
+		Expect(err).NotTo(HaveOccurred())
+	}
+	if f.FederationClientset == nil {
+		By("Creating a release 1.5 federation Clientset")
+		var err error
+		f.FederationClientset, err = LoadFederationClientset(f.FederationConfig)
+		Expect(err).NotTo(HaveOccurred())
+	}
+	By("Waiting for federation-apiserver to be ready")
+	err := WaitForFederationApiserverReady(f.FederationClientset)
+	Expect(err).NotTo(HaveOccurred())
+	By("federation-apiserver is ready")
+
+	By("Creating a federation namespace")
+	ns, err := f.createFederationNamespace(f.BaseName)
+	Expect(err).NotTo(HaveOccurred())
+	f.FederationNamespace = ns
+	By(fmt.Sprintf("Created federation namespace %s", ns.Name))
+}
+
+func (f *Framework) deleteFederationNs() {
+	ns := f.FederationNamespace
+	By(fmt.Sprintf("Destroying federation namespace %q for this suite.", ns.Name))
+	timeout := 5 * time.Minute
+	if f.NamespaceDeletionTimeout != 0 {
+		timeout = f.NamespaceDeletionTimeout
+	}
+
+	clientset := f.FederationClientset
+	// First delete the namespace from federation apiserver.
+	// Also delete the corresponding namespaces from underlying clusters.
+	orphanDependents := false
+	if err := clientset.Core().Namespaces().Delete(ns.Name, &metav1.DeleteOptions{OrphanDependents: &orphanDependents}); err != nil {
+		framework.Failf("Error while deleting federation namespace %s: %s", ns.Name, err)
+	}
+	// Verify that it got deleted.
+	err := wait.PollImmediate(5*time.Second, timeout, func() (bool, error) {
+		if _, err := clientset.Core().Namespaces().Get(ns.Name, metav1.GetOptions{}); err != nil {
+			if apierrors.IsNotFound(err) {
+				return true, nil
+			}
+			framework.Logf("Error while waiting for namespace to be terminated: %v", err)
+			return false, nil
+		}
+		return false, nil
+	})
+	if err != nil {
+		if !apierrors.IsNotFound(err) {
+			framework.Failf("Couldn't delete ns %q: %s", ns.Name, err)
+		} else {
+			framework.Logf("Namespace %v was already deleted", ns.Name)
+		}
+	}
+}
+
+// FederationAfterEach deletes the namespace, after reading its events.
+func (f *Framework) FederationAfterEach() {
+	framework.RemoveCleanupAction(f.cleanupHandle)
+
+	// DeleteNamespace at the very end in defer, to avoid any
+	// expectation failures preventing deleting the namespace.
+	defer func() {
+		// Whether to delete namespace is determined by 3 factors: delete-namespace flag, delete-namespace-on-failure flag and the test result
+		// if delete-namespace set to false, namespace will always be preserved.
+		// if delete-namespace is true and delete-namespace-on-failure is false, namespace will be preserved if test failed.
+		if framework.TestContext.DeleteNamespace && (framework.TestContext.DeleteNamespaceOnFailure || !CurrentGinkgoTestDescription().Failed) {
+			// Delete the federation namespace.
+			f.deleteFederationNs()
+		}
+
+		// Paranoia-- prevent reuse!
+		f.FederationNamespace = nil
+
+		if f.FederationClientset == nil {
+			framework.Logf("Warning: framework is marked federated, but has no federation 1.5 clientset")
+			return
+		}
+	}()
+
+	// Print events if the test failed.
+	if CurrentGinkgoTestDescription().Failed && framework.TestContext.DumpLogsOnFailure {
+		// Dump federation events in federation namespace.
+		framework.DumpEventsInNamespace(func(opts metav1.ListOptions, ns string) (*v1.EventList, error) {
+			return f.FederationClientset.Core().Events(ns).List(opts)
+		}, f.FederationNamespace.Name)
+	}
+}
+
+func (f *Framework) createFederationNamespace(baseName string) (*v1.Namespace, error) {
+	clientset := f.FederationClientset
+	namespaceObj := &v1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: fmt.Sprintf("e2e-tests-%v-", baseName),
+		},
+	}
+	// Be robust about making the namespace creation call.
+	var got *v1.Namespace
+	if err := wait.PollImmediate(framework.Poll, framework.SingleCallTimeout, func() (bool, error) {
+		var err error
+		got, err = clientset.Core().Namespaces().Create(namespaceObj)
+		if err != nil {
+			framework.Logf("Unexpected error while creating namespace: %v", err)
+			return false, nil
+		}
+		return true, nil
+	}); err != nil {
+		return nil, err
+	}
+	return got, nil
+}
+
+type E2EContext struct {
+	// Raw context name,
+	RawName string `yaml:"rawName"`
+	// A valid dns subdomain which can be used as the name of kubernetes resources.
+	Name    string                 `yaml:"name"`
+	Cluster *framework.KubeCluster `yaml:"cluster"`
+	User    *framework.KubeUser    `yaml:"user"`
+}
+
+func (f *Framework) GetUnderlyingFederatedContexts() []E2EContext {
+	kubeconfig := framework.KubeConfig{}
+	configBytes, err := ioutil.ReadFile(framework.TestContext.KubeConfig)
+	framework.ExpectNoError(err)
+	err = yaml.Unmarshal(configBytes, &kubeconfig)
+	framework.ExpectNoError(err)
+
+	e2eContexts := []E2EContext{}
+	for _, context := range kubeconfig.Contexts {
+		if strings.HasPrefix(context.Name, "federation") && context.Name != framework.TestContext.FederatedKubeContext {
+			user := kubeconfig.FindUser(context.Context.User)
+			if user == nil {
+				framework.Failf("Could not find user for context %+v", context)
+			}
+
+			cluster := kubeconfig.FindCluster(context.Context.Cluster)
+			if cluster == nil {
+				framework.Failf("Could not find cluster for context %+v", context)
+			}
+
+			dnsSubdomainName, err := GetValidDNSSubdomainName(context.Name)
+			if err != nil {
+				framework.Failf("Could not convert context name %s to a valid dns subdomain name, error: %s", context.Name, err)
+			}
+			e2eContexts = append(e2eContexts, E2EContext{
+				RawName: context.Name,
+				Name:    dnsSubdomainName,
+				Cluster: cluster,
+				User:    user,
+			})
+		}
+	}
+
+	return e2eContexts
+}
+
+func (f *Framework) GetRegisteredClusters() ClusterSlice {
+	if framework.TestContext.FederationConfigFromCluster {
+		return registeredClustersFromSecrets(f)
+	} else {
+		return registeredClustersFromConfig(f)
+	}
+}
+
+func (f *Framework) GetClusterClients() []kubeclientset.Interface {
+	clusters := f.GetRegisteredClusters()
+	var clusterClients []kubeclientset.Interface
+	for _, c := range clusters {
+		clusterClients = append(clusterClients, c.Clientset)
+	}
+	return clusterClients
+}

vendor/k8s.io/kubernetes/test/e2e_federation/framework/util.go

@@ -0,0 +1,150 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+	"time"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	validationutil "k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/validation"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+const FederatedDefaultTestTimeout = 5 * time.Minute
+
+// Detects whether the federation namespace exists in the underlying cluster
+func SkipUnlessFederated(c clientset.Interface) {
+	federationNS := framework.FederationSystemNamespace()
+
+	_, err := c.Core().Namespaces().Get(federationNS, metav1.GetOptions{})
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			framework.Skipf("Could not find federation namespace %s: skipping federated test", federationNS)
+		} else {
+			framework.Failf("Unexpected error getting namespace: %v", err)
+		}
+	}
+}
+
+// WaitForFederationApiserverReady waits for the federation apiserver to be ready.
+// It tests the readiness by sending a GET request and expecting a non error response.
+func WaitForFederationApiserverReady(c *federation_clientset.Clientset) error {
+	return wait.PollImmediate(time.Second, 1*time.Minute, func() (bool, error) {
+		_, err := c.Federation().Clusters().List(metav1.ListOptions{})
+		if err != nil {
+			return false, nil
+		}
+		return true, nil
+	})
+}
+
+func LoadFederationClientset(config *restclient.Config) (*federation_clientset.Clientset, error) {
+	c, err := federation_clientset.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating federation clientset: %v", err.Error())
+	}
+	return c, nil
+}
+
+func LoadFederatedConfig(overrides *clientcmd.ConfigOverrides) (*restclient.Config, error) {
+	c, err := framework.RestclientConfig(framework.TestContext.FederatedKubeContext)
+	if err != nil {
+		return nil, fmt.Errorf("error creating federation client config: %v", err.Error())
+	}
+	cfg, err := clientcmd.NewDefaultClientConfig(*c, &clientcmd.ConfigOverrides{}).ClientConfig()
+	if cfg != nil {
+		//TODO(colhom): this is only here because https://github.com/kubernetes/kubernetes/issues/25422
+		cfg.NegotiatedSerializer = api.Codecs
+	}
+	if err != nil {
+		return cfg, fmt.Errorf("error creating federation default client config: %v", err.Error())
+	}
+	return cfg, nil
+}
+
+// GetValidDNSSubdomainName massages the given name to be a valid dns subdomain name.
+// Most resources (such as secrets, clusters) require the names to be valid dns subdomain.
+// This is a generic function (not specific to federation). Should be moved to a more generic location if others want to use it.
+func GetValidDNSSubdomainName(name string) (string, error) {
+	// "_" are not allowed. Replace them by "-".
+	name = regexp.MustCompile("_").ReplaceAllLiteralString(name, "-")
+	maxLength := validationutil.DNS1123SubdomainMaxLength
+	if len(name) > maxLength {
+		name = name[0 : maxLength-1]
+	}
+	// Verify that name now passes the validation.
+	if errors := validation.NameIsDNSSubdomain(name, false); len(errors) != 0 {
+		return "", fmt.Errorf("errors in converting name to a valid DNS subdomain %s", errors)
+	}
+	return name, nil
+}
+
+func FederationControlPlaneUpgrade(version string) error {
+	version = "v" + version
+	_, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "federation/cluster/upgrade.sh"), version)
+	return err
+}
+
+func CheckFederationVersion(c federation_clientset.Interface, want string) error {
+	framework.Logf("Checking federation version")
+	v, err := c.Discovery().ServerVersion()
+	if err != nil {
+		return fmt.Errorf("CheckFederationVersion() couldn't get the master version: %v", err)
+	}
+	// We do prefix trimming and then matching because:
+	// want looks like:  0.19.3-815-g50e67d4
+	// got  looks like: v0.19.3-815-g50e67d4034e858-dirty
+	got := strings.TrimPrefix(v.GitVersion, "v")
+	if !strings.HasPrefix(got, want) {
+		return fmt.Errorf("federation had apiserver version %s which does not start with %s",
+			got, want)
+	}
+	framework.Logf("Federation is at version %s", want)
+	return nil
+}
+
+func MasterUpgrade(context, version string) error {
+	switch framework.TestContext.Provider {
+	case "gce":
+		return masterUpgradeGCE(context, version)
+	default:
+		return fmt.Errorf("MasterUpgrade() is not implemented for provider %s", framework.TestContext.Provider)
+	}
+}
+
+func masterUpgradeGCE(context, rawVersion string) error {
+	version := "v" + rawVersion
+	// TODO: this breaks if we want to upgrade 2 clusters in same zone. use alternate methods in future to get zone of a cluster
+	zone := strings.TrimPrefix(context, "federation-e2e-"+framework.TestContext.Provider+"-")
+
+	env := append(os.Environ(), "KUBE_CONTEXT="+context, "ZONE="+zone)
+	_, _, err := framework.RunCmdEnv(env, path.Join(framework.TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-M", version)
+	return err
+}

vendor/k8s.io/kubernetes/test/e2e_federation/ingress.go

@@ -0,0 +1,638 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_federation
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net/http"
+	"os"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/api/extensions/v1beta1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apimachinery/pkg/util/wait"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
+
+	"k8s.io/kubernetes/test/e2e/framework"
+	fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	MaxRetriesOnFederatedApiserver = 3
+	FederatedIngressTimeout        = 15 * time.Minute
+	FederatedIngressDeleteTimeout  = 2 * time.Minute
+	FederatedIngressName           = "federated-ingress"
+	FederatedIngressServiceName    = "federated-ingress-service"
+	FederatedIngressTLSSecretName  = "federated-ingress-tls-secret"
+	FederatedIngressServicePodName = "federated-ingress-service-test-pod"
+	FederatedIngressHost           = "test-f8n.k8s.io."
+
+	FederatedSecretTimeout = 60 * time.Second
+
+	// TLS Certificate and Key for the ingress resource
+	// Generated using:
+	// $ openssl req -nodes -x509 -newkey rsa:2048 -keyout fedingtestkey.pem -out fedingtestcrt.pem -days 2485
+	// 2485 days is an arbitrary large number chosen below int32 seconds.
+	FederatedIngressTLSCrt = `-----BEGIN CERTIFICATE-----
+MIIDaTCCAlGgAwIBAgIJANwsCbwxm9pyMA0GCSqGSIb3DQEBCwUAMEoxCzAJBgNV
+BAYTAlVTMRMwEQYDVQQIDApTb21lLVN0YXRlMQswCQYDVQQKDAJOQTEZMBcGA1UE
+AwwQdGVzdC1mOG4uazhzLmlvLjAgFw0xNjEyMTYwNjA1NDRaGA8yMDg1MDEwMzA2
+MDU0NFowSjELMAkGA1UEBhMCVVMxEzARBgNVBAgMClNvbWUtU3RhdGUxCzAJBgNV
+BAoMAk5BMRkwFwYDVQQDDBB0ZXN0LWY4bi5rOHMuaW8uMIIBIjANBgkqhkiG9w0B
+AQEFAAOCAQ8AMIIBCgKCAQEAmsHYnLhqSeO1Q6SEjaiPiLUQV8tyGfttwNQiOT5u
+ULz6ZWYA40m/1hhla9KH9sJZ515Iq+jTtiVH0rUjryT96SjxitLCAZlxVwQ63B50
+aZF2T2OPSzvrmN+J6VGcRIq0N8fUeyp2WTIEdWlpQ7DTmDNArQqFSIvJndkLow3d
+hec7O+PErnvZQQC9zqa23rGndDzlgDJ4HJGAQNm3uYVh5WHv+wziP67T/82bEGgO
+A6EdDPWzpYxzAA1wsqz9lX5jitlbKdI56698fPR2KRelySf7OXVvZCS4/ED1lF4k
+b7fQgtBhAWe1BkuAMUl7vdRjMps7nkxmBSuxBkVQ7sb5AwIDAQABo1AwTjAdBgNV
+HQ4EFgQUjf53O/W/iE2mxuJkNjZGUfjJ9RUwHwYDVR0jBBgwFoAUjf53O/W/iE2m
+xuJkNjZGUfjJ9RUwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEABE7B
+bAiORDBA3qE5lh6JCs/lEfz93E/gOhD9oDnm9SRND4kjy7qeGxk4Wzsd/Vr+R2mi
+EZ40d4MA/mCCPnYsNQoEXMFc8IvwAbzkhh2gqTNgG0/Ks0A1mIPQNpvUcSetS4IV
+732DvB3nSnFtlzf6afw+V1Vf5ydRNuM/c9GEOOHSz+rs+9M364d+wNaFD64M72ol
+iDMAdtcrhOqkQi0lUING904jlJcyYM5oVNCCtme4F8nkIX9bxP/9Ea6VhDGPeJiX
+tVwZuudkoEbrFlEYbyLrbVeVa9oTf4Jn66iz49/+th+bUtEoTt9gk9Cul5TFgfzx
+EscdahceC7afheq6zg==
+-----END CERTIFICATE-----`
+
+	FederatedIngressTLSKey = `-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCawdicuGpJ47VD
+pISNqI+ItRBXy3IZ+23A1CI5Pm5QvPplZgDjSb/WGGVr0of2wlnnXkir6NO2JUfS
+tSOvJP3pKPGK0sIBmXFXBDrcHnRpkXZPY49LO+uY34npUZxEirQ3x9R7KnZZMgR1
+aWlDsNOYM0CtCoVIi8md2QujDd2F5zs748Sue9lBAL3Oprbesad0POWAMngckYBA
+2be5hWHlYe/7DOI/rtP/zZsQaA4DoR0M9bOljHMADXCyrP2VfmOK2Vsp0jnrr3x8
+9HYpF6XJJ/s5dW9kJLj8QPWUXiRvt9CC0GEBZ7UGS4AxSXu91GMymzueTGYFK7EG
+RVDuxvkDAgMBAAECggEAYrXGPqB6W0r88XpceibL9rzXAcjorJ3s8ZPdiHnDz4fa
+hxa69j6yOBMzjcSpqMFqquM+ozhM4d+BomqbqjmEI1ZUSuIHkRGYc5JlIMXkJvn7
+ZsPwQGKl8cqTotjFPgrizLmPVEhPWLFImsNzuxNsw6XdWQJe5VkUbrRkccqEQ8Wt
+xwq/SlRercIMnRVLOOESq8EyjOY4yDgOdIifq9K9xiI8W6nMiPs0X5AcIJoTMbCe
+cX0zUqW317awDWWP8u2GswwDDm4qPeWnXOrDkDx8Eo0dWJbmxw9su0XrM6KMvEMe
+2a/Fy/enr5Cc6/jgsh3gO5sa8dJ1Cu+wexcoEbez8QKBgQDMXlXJu/C7djke94s3
+vGxati7AGO95bBQHW+cPuN4l0rfPZ8YuUAWD4csW4BOlUPAOukreD/SKdanigR3N
+FqVPeI8rXd5kzy8/lPIOGuSkkVEpKsAJ7prFbSUVKjVPYQk2dsOEeR0r7pr2FxC9
+SBhVS/LgmPYh++iny9D0aU23hQKBgQDB2t55OE+00vgoauUc10LEY+J6tiwXuNm7
+43JtrH5ET4N+TJ2BOUl5f88TY/3QuTu6vYwlxjyn+LFuWQNhShX6lFMjt5zqPTdw
+ZPDA+9B6a45cV3YjXjRsYidpWj0D2lJgy0DbucC4f3eIhNGyFUbAQB9npKDzOeUh
+7Z+p/Grg5wKBgGUnVCLzySWgUImJUPkXZDJJ9j3SmcVpv0gdLvLTN/FUqPIZlTgb
+F3+9ZL4/zrmGpCtF/gSHtSxLLPkVm2CFkvEQ5Rw76/XNrr8zw9NDcGQcISXVKRRB
+a43IhhBBwf02NE8m3YNWRyAVi9G+fOSTKKgfXWnZjAoqG2/iK9ytum/ZAoGAYlP8
+KIxxkYy5Jvchg4GEck0f4ZJpxxaSCoWR0yN9YHTcg8Gk2pkONbyocnNzmN17+HqQ
+jdCBj8nLZedsmXqUr2dwzFskEoQ+jJoGrDyOQKoxqZELcWElQhx/VSbacAvbYRF3
+snwDzxGItgx4uNWl73oW8+FDalvhZ1Y6eGR6ad0CgYEAtlNa92Fbvd3r9O2mdyWe
+D2SXNMi45+wsNafX2sdkyb+qNN6qZXC9ylUl9h0zdky88JNgtAOgxIaRIdEZajnD
+/Zq17sTNtgpm53x16gOAgD8M+/wmBZxA+/IKfFCubuV77MbQoPfcjT5wBMRnFQnY
+Ks7c+dzaRlgDKZ6v/L/8iZU=
+-----END PRIVATE KEY-----`
+)
+
+const (
+	// timeout on a single http request.
+	reqTimeout = 10 * time.Second
+)
+
+var _ = framework.KubeDescribe("Federated ingresses [Feature:Federation]", func() {
+	f := fedframework.NewDefaultFederatedFramework("federated-ingress")
+
+	// Create/delete ingress api objects
+	// Validate federation apiserver, does not rely on underlying clusters or federation ingress controller.
+	Describe("Federated Ingresses [NoCluster]", func() {
+		AfterEach(func() {
+			nsName := f.FederationNamespace.Name
+			// Delete all ingresses.
+			deleteAllIngressesOrFail(f.FederationClientset, nsName)
+		})
+
+		It("should be created and deleted successfully", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			framework.SkipUnlessProviderIs("gce", "gke") // TODO: Federated ingress is not yet supported on non-GCP platforms.
+			nsName := f.FederationNamespace.Name
+			ingress := createIngressOrFail(f.FederationClientset, nsName, FederatedIngressServiceName, FederatedIngressTLSSecretName)
+			By(fmt.Sprintf("Creation of ingress %q in namespace %q succeeded.  Deleting ingress.", ingress.Name, nsName))
+			// Cleanup
+			err := f.FederationClientset.Extensions().Ingresses(nsName).Delete(ingress.Name, &metav1.DeleteOptions{})
+			framework.ExpectNoError(err, "Error deleting ingress %q in namespace %q", ingress.Name, ingress.Namespace)
+			By(fmt.Sprintf("Deletion of ingress %q in namespace %q succeeded.", ingress.Name, nsName))
+		})
+	})
+
+	// e2e cases for federation ingress controller
+	var _ = Describe("Federated Ingresses [Slow]", func() {
+		var (
+			clusters           fedframework.ClusterSlice
+			federationName, ns string
+			jig                *federationTestJig
+			service            *v1.Service
+			secret             *v1.Secret
+		)
+
+		// register clusters in federation apiserver
+		BeforeEach(func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			framework.SkipUnlessProviderIs("gce", "gke") // TODO: Federated ingress is not yet supported on non-GCP platforms.
+			if federationName = os.Getenv("FEDERATION_NAME"); federationName == "" {
+				federationName = DefaultFederationName
+			}
+			jig = newFederationTestJig(f.FederationClientset)
+			clusters = f.GetRegisteredClusters()
+			ns = f.FederationNamespace.Name
+			// create backend service
+			service = createLBServiceOrFail(f.FederationClientset, ns, FederatedIngressServiceName, clusters)
+			// create the TLS secret
+			secret = createTLSSecretOrFail(f.FederationClientset, ns, FederatedIngressTLSSecretName)
+			// wait for services objects sync
+			waitForServiceShardsOrFail(ns, service, clusters)
+			// wait for TLS secret sync
+			waitForSecretShardsOrFail(ns, secret, clusters)
+		})
+
+		AfterEach(func() {
+			// Delete all ingresses.
+			deleteAllIngressesOrFail(f.FederationClientset, ns)
+			if secret != nil {
+				By("Deleting secret")
+				orphanDependents := false
+				deleteSecretOrFail(f.FederationClientset, ns, secret.Name, &orphanDependents)
+				secret = nil
+			} else {
+				By("No secret to delete. Secret is nil")
+			}
+			if service != nil {
+				By("Deleting service")
+				deleteServiceOrFail(f.FederationClientset, ns, service.Name, nil)
+				By("Cleanup service shards and provider resources")
+				cleanupServiceShardsAndProviderResources(ns, service, clusters)
+				service = nil
+			} else {
+				By("No service to delete. Service is nil")
+			}
+		})
+
+		It("should create and update matching ingresses in underlying clusters", func() {
+			ingress := createIngressOrFail(f.FederationClientset, ns, FederatedIngressServiceName, FederatedIngressTLSSecretName)
+			// wait for ingress shards being created
+			waitForIngressShardsOrFail(ns, ingress, clusters)
+			ingress = updateIngressOrFail(f.FederationClientset, ns)
+			waitForIngressShardsUpdatedOrFail(ns, ingress, clusters)
+		})
+
+		It("should be deleted from underlying clusters when OrphanDependents is false", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			orphanDependents := false
+			verifyCascadingDeletionForIngress(f.FederationClientset, clusters, &orphanDependents, ns)
+			By(fmt.Sprintf("Verified that ingresses were deleted from underlying clusters"))
+		})
+
+		It("should not be deleted from underlying clusters when OrphanDependents is true", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			orphanDependents := true
+			verifyCascadingDeletionForIngress(f.FederationClientset, clusters, &orphanDependents, ns)
+			By(fmt.Sprintf("Verified that ingresses were not deleted from underlying clusters"))
+		})
+
+		It("should not be deleted from underlying clusters when OrphanDependents is nil", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			verifyCascadingDeletionForIngress(f.FederationClientset, clusters, nil, ns)
+			By(fmt.Sprintf("Verified that ingresses were not deleted from underlying clusters"))
+		})
+
+		var _ = Describe("Ingress connectivity and DNS", func() {
+
+			var backendPods BackendPodMap
+
+			BeforeEach(func() {
+				fedframework.SkipUnlessFederated(f.ClientSet)
+				// create backend pod
+				backendPods = createBackendPodsOrFail(clusters, ns, FederatedIngressServicePodName)
+				// create ingress object
+				jig.ing = createIngressOrFail(f.FederationClientset, ns, service.Name, FederatedIngressTLSSecretName)
+				// wait for ingress objects sync
+				waitForIngressShardsOrFail(ns, jig.ing, clusters)
+				By(fmt.Sprintf("Ingress created as %v", jig.ing.Name))
+			})
+
+			AfterEach(func() {
+				deleteBackendPodsOrFail(clusters, backendPods)
+				backendPods = nil
+
+				if jig.ing != nil {
+					By(fmt.Sprintf("Deleting ingress %v on all clusters", jig.ing.Name))
+					orphanDependents := false
+					deleteIngressOrFail(f.FederationClientset, ns, jig.ing.Name, &orphanDependents)
+					jig.ing = nil
+				} else {
+					By("No ingress to delete. Ingress is nil")
+				}
+			})
+
+			PIt("should be able to discover a federated ingress service via DNS", func() {
+				// we are about the ingress name
+				svcDNSNames := []string{
+					fmt.Sprintf("%s.%s", FederatedIngressServiceName, ns),
+					fmt.Sprintf("%s.%s.svc.cluster.local.", FederatedIngressServiceName, ns),
+					// TODO these two entries are not set yet
+					//fmt.Sprintf("%s.%s.%s", FederatedIngressServiceName, ns, federationName),
+					//fmt.Sprintf("%s.%s.%s.svc.cluster.local.", FederatedIngressServiceName, ns, federationName),
+				}
+				// check dns records in underlying cluster
+				for i, DNSName := range svcDNSNames {
+					discoverService(f, DNSName, true, "federated-ingress-e2e-discovery-pod-"+strconv.Itoa(i))
+				}
+				// TODO check dns record in global dns server
+			})
+
+			It("should be able to connect to a federated ingress via its load balancer", func() {
+				By(fmt.Sprintf("Waiting for Federated Ingress on %v", jig.ing.Name))
+				// check the traffic on federation ingress
+				jig.waitForFederatedIngress()
+			})
+
+		})
+	})
+})
+
+// Deletes all Ingresses in the given namespace name.
+func deleteAllIngressesOrFail(clientset *fedclientset.Clientset, nsName string) {
+	orphanDependents := false
+	err := clientset.Extensions().Ingresses(nsName).DeleteCollection(&metav1.DeleteOptions{OrphanDependents: &orphanDependents}, metav1.ListOptions{})
+	Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Error in deleting ingresses in namespace: %s", nsName))
+}
+
+// equivalent returns true if the two ingress spec are equivalent.
+func equivalentIngress(federatedIngress, clusterIngress v1beta1.Ingress) bool {
+	return reflect.DeepEqual(clusterIngress.Spec, federatedIngress.Spec)
+}
+
+// verifyCascadingDeletionForIngress verifies that ingresses are deleted from
+// underlying clusters when orphan dependents is false and they are not deleted
+// when orphan dependents is true.
+func verifyCascadingDeletionForIngress(clientset *fedclientset.Clientset, clusters fedframework.ClusterSlice, orphanDependents *bool, nsName string) {
+	ingress := createIngressOrFail(clientset, nsName, FederatedIngressServiceName, FederatedIngressTLSSecretName)
+	ingressName := ingress.Name
+	// Check subclusters if the ingress was created there.
+	By(fmt.Sprintf("Waiting for ingress %s to be created in all underlying clusters", ingressName))
+	waitForIngressShardsOrFail(nsName, ingress, clusters)
+
+	By(fmt.Sprintf("Deleting ingress %s", ingressName))
+	deleteIngressOrFail(clientset, nsName, ingressName, orphanDependents)
+
+	By(fmt.Sprintf("Verifying ingresses %s in underlying clusters", ingressName))
+	errMessages := []string{}
+	// ingress should be present in underlying clusters unless orphanDependents is false.
+	shouldExist := orphanDependents == nil || *orphanDependents == true
+	for _, cluster := range clusters {
+		clusterName := cluster.Name
+		_, err := cluster.Extensions().Ingresses(nsName).Get(ingressName, metav1.GetOptions{})
+		if shouldExist && errors.IsNotFound(err) {
+			errMessages = append(errMessages, fmt.Sprintf("unexpected NotFound error for ingress %s in cluster %s, expected ingress to exist", ingressName, clusterName))
+		} else if !shouldExist && !errors.IsNotFound(err) {
+			errMessages = append(errMessages, fmt.Sprintf("expected NotFound error for ingress %s in cluster %s, got error: %v", ingressName, clusterName, err))
+		}
+	}
+	if len(errMessages) != 0 {
+		framework.Failf("%s", strings.Join(errMessages, "; "))
+	}
+}
+
+// waitForIngressOrFail waits until a ingress is either present or absent in the cluster specified by clientset.
+// If the condition is not met within timout, it fails the calling test.
+func waitForIngressOrFail(clientset *kubeclientset.Clientset, namespace string, ingress *v1beta1.Ingress, present bool, timeout time.Duration) {
+	By(fmt.Sprintf("Fetching a federated ingress shard of ingress %q in namespace %q from cluster", ingress.Name, namespace))
+	var clusterIngress *v1beta1.Ingress
+	err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
+		var err error
+		clusterIngress, err = clientset.ExtensionsV1beta1().Ingresses(namespace).Get(ingress.Name, metav1.GetOptions{})
+		if (!present) && errors.IsNotFound(err) { // We want it gone, and it's gone.
+			By(fmt.Sprintf("Success: shard of federated ingress %q in namespace %q in cluster is absent", ingress.Name, namespace))
+			return true, nil // Success
+		}
+		if present && err == nil { // We want it present, and the Get succeeded, so we're all good.
+			By(fmt.Sprintf("Success: shard of federated ingress %q in namespace %q in cluster is present", ingress.Name, namespace))
+			return true, nil // Success
+		}
+		By(fmt.Sprintf("Ingress %q in namespace %q in cluster.  Found: %v, waiting for Found: %v, trying again in %s (err=%v)", ingress.Name, namespace, clusterIngress != nil && err == nil, present, framework.Poll, err))
+		return false, nil
+	})
+	framework.ExpectNoError(err, "Failed to verify ingress %q in namespace %q in cluster: Present=%v", ingress.Name, namespace, present)
+
+	if present && clusterIngress != nil {
+		Expect(equivalentIngress(*clusterIngress, *ingress))
+	}
+}
+
+// waitForIngressShardsOrFail waits for the ingress to appear in all clusters
+func waitForIngressShardsOrFail(namespace string, ingress *v1beta1.Ingress, clusters fedframework.ClusterSlice) {
+	framework.Logf("Waiting for ingress %q in %d clusters", ingress.Name, len(clusters))
+	for _, c := range clusters {
+		waitForIngressOrFail(c.Clientset, namespace, ingress, true, FederatedIngressTimeout)
+	}
+}
+
+// waitForIngressShardsUpdatedOrFail waits for the ingress to be updated in all clusters
+func waitForIngressShardsUpdatedOrFail(namespace string, ingress *v1beta1.Ingress, clusters fedframework.ClusterSlice) {
+	framework.Logf("Waiting for ingress %q in %d clusters", ingress.Name, len(clusters))
+	for _, c := range clusters {
+		waitForIngressUpdateOrFail(c.Clientset, namespace, ingress, FederatedIngressTimeout)
+	}
+}
+
+// waitForIngressUpdateOrFail waits until a ingress is updated in the specified cluster with same spec of federated ingress.
+// If the condition is not met within timeout, it fails the calling test.
+func waitForIngressUpdateOrFail(clientset *kubeclientset.Clientset, namespace string, ingress *v1beta1.Ingress, timeout time.Duration) {
+	By(fmt.Sprintf("Fetching a federated ingress shard of ingress %q in namespace %q from cluster", ingress.Name, namespace))
+	err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
+		clusterIngress, err := clientset.ExtensionsV1beta1().Ingresses(namespace).Get(ingress.Name, metav1.GetOptions{})
+		if err == nil { // We want it present, and the Get succeeded, so we're all good.
+			if equivalentIngress(*clusterIngress, *ingress) {
+				By(fmt.Sprintf("Success: shard of federated ingress %q in namespace %q in cluster is updated", ingress.Name, namespace))
+				return true, nil
+			}
+			By(fmt.Sprintf("Ingress %q in namespace %q in cluster, waiting for service being updated, trying again in %s (err=%v)", ingress.Name, namespace, framework.Poll, err))
+			return false, nil
+		}
+		By(fmt.Sprintf("Ingress %q in namespace %q in cluster, waiting for service being updated, trying again in %s (err=%v)", ingress.Name, namespace, framework.Poll, err))
+		return false, nil
+	})
+	framework.ExpectNoError(err, "Failed to verify ingress %q in namespace %q in cluster", ingress.Name, namespace)
+}
+
+// waitForIngressShardsGoneOrFail waits for the ingress to disappear in all clusters
+func waitForIngressShardsGoneOrFail(namespace string, ingress *v1beta1.Ingress, clusters fedframework.ClusterSlice) {
+	framework.Logf("Waiting for ingress %q in %d clusters", ingress.Name, len(clusters))
+	for _, c := range clusters {
+		waitForIngressOrFail(c.Clientset, namespace, ingress, false, FederatedIngressTimeout)
+	}
+}
+
+func deleteIngressOrFail(clientset *fedclientset.Clientset, namespace string, ingressName string, orphanDependents *bool) {
+	if clientset == nil || len(namespace) == 0 || len(ingressName) == 0 {
+		Fail(fmt.Sprintf("Internal error: invalid parameters passed to deleteIngressOrFail: clientset: %v, namespace: %v, ingress: %v", clientset, namespace, ingressName))
+	}
+	err := clientset.ExtensionsV1beta1().Ingresses(namespace).Delete(ingressName, &metav1.DeleteOptions{OrphanDependents: orphanDependents})
+	framework.ExpectNoError(err, "Error deleting ingress %q from namespace %q", ingressName, namespace)
+	// Wait for the ingress to be deleted.
+	err = wait.Poll(framework.Poll, FederatedIngressDeleteTimeout, func() (bool, error) {
+		_, err := clientset.Extensions().Ingresses(namespace).Get(ingressName, metav1.GetOptions{})
+		if err != nil && errors.IsNotFound(err) {
+			return true, nil
+		}
+		return false, err
+	})
+	if err != nil {
+		framework.Failf("Error in deleting ingress %s: %v", ingressName, err)
+	}
+}
+
+// TODO: quinton: This is largely a cut 'n paste of the above.  Yuck! Refactor as soon as we have a common interface implmented by both fedclientset.Clientset and kubeclientset.Clientset
+func deleteClusterIngressOrFail(clusterName string, clientset *kubeclientset.Clientset, namespace string, ingressName string) {
+	if clientset == nil || len(namespace) == 0 || len(ingressName) == 0 {
+		Fail(fmt.Sprintf("Internal error: invalid parameters passed to deleteClusterIngressOrFail: cluster: %q, clientset: %v, namespace: %v, ingress: %v", clusterName, clientset, namespace, ingressName))
+	}
+	err := clientset.ExtensionsV1beta1().Ingresses(namespace).Delete(ingressName, metav1.NewDeleteOptions(0))
+	framework.ExpectNoError(err, "Error deleting cluster ingress %q/%q from cluster %q", namespace, ingressName, clusterName)
+}
+
+func createIngressOrFail(clientset *fedclientset.Clientset, namespace, serviceName, secretName string) *v1beta1.Ingress {
+	if clientset == nil || len(namespace) == 0 {
+		Fail(fmt.Sprintf("Internal error: invalid parameters passed to createIngressOrFail: clientset: %v, namespace: %v", clientset, namespace))
+	}
+	By(fmt.Sprintf("Creating federated ingress %q in namespace %q", FederatedIngressName, namespace))
+
+	ingress := &v1beta1.Ingress{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: FederatedIngressName,
+		},
+		Spec: v1beta1.IngressSpec{
+			Backend: &v1beta1.IngressBackend{
+				ServiceName: serviceName,
+				ServicePort: intstr.FromInt(80),
+			},
+			TLS: []v1beta1.IngressTLS{
+				{
+					SecretName: secretName,
+				},
+			},
+		},
+	}
+
+	newIng, err := clientset.Extensions().Ingresses(namespace).Create(ingress)
+	framework.ExpectNoError(err, "Creating ingress %q in namespace %q", ingress.Name, namespace)
+	By(fmt.Sprintf("Successfully created federated ingress %q in namespace %q", FederatedIngressName, namespace))
+	return newIng
+}
+
+func updateIngressOrFail(clientset *fedclientset.Clientset, namespace string) (newIng *v1beta1.Ingress) {
+	var err error
+	if clientset == nil || len(namespace) == 0 {
+		Fail(fmt.Sprintf("Internal error: invalid parameters passed to createIngressOrFail: clientset: %v, namespace: %v", clientset, namespace))
+	}
+	ingress := &v1beta1.Ingress{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: FederatedIngressName,
+		},
+		Spec: v1beta1.IngressSpec{
+			Backend: &v1beta1.IngressBackend{
+				ServiceName: "updated-testingress-service",
+				ServicePort: intstr.FromInt(80),
+			},
+		},
+	}
+
+	err = waitForFederatedIngressExists(clientset, namespace, FederatedIngressName, FederatedIngressTimeout)
+	if err != nil {
+		framework.Failf("failed to get ingress %q: %v", FederatedIngressName, err)
+	}
+	for i := 0; i < MaxRetriesOnFederatedApiserver; i++ {
+		newIng, err = clientset.Extensions().Ingresses(namespace).Update(ingress)
+		if err == nil {
+			framework.DescribeIng(namespace)
+			return newIng
+		}
+		if !errors.IsConflict(err) && !errors.IsServerTimeout(err) {
+			framework.Failf("failed to update ingress %q: %v", FederatedIngressName, err)
+		}
+	}
+	framework.Failf("too many retries updating ingress %q", FederatedIngressName)
+	return nil
+}
+
+func (j *federationTestJig) waitForFederatedIngress() {
+	// Wait for the loadbalancer IP.
+	address, err := waitForFederatedIngressAddress(j.client, j.ing.Namespace, j.ing.Name, framework.LoadBalancerPollTimeout)
+	if err != nil {
+		framework.Failf("Ingress failed to acquire an IP address within %v", framework.LoadBalancerPollTimeout)
+	}
+	j.address = address
+	framework.Logf("Found address %v for ingress %v", j.address, j.ing.Name)
+
+	client := &http.Client{
+		// This is mostly `http.DefaultTransport` except for the
+		// `TLSClientConfig`.
+		Transport: utilnet.SetTransportDefaults(&http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+		}),
+		Timeout: reqTimeout,
+	}
+
+	// Verify that simple GET works.
+	route := fmt.Sprintf("https://%v", address)
+	framework.Logf("Testing route %v with simple GET", route)
+	framework.ExpectNoError(framework.PollURL(route, FederatedIngressHost, framework.LoadBalancerPollTimeout, framework.LoadBalancerPollInterval, client, false))
+}
+
+func createTLSSecretOrFail(clientset *fedclientset.Clientset, namespace, secretName string) *v1.Secret {
+	if clientset == nil || len(namespace) == 0 {
+		framework.Logf("Internal error: invalid parameters passed to createTLSSecretOrFail: clientset: %v, namespace: %v", clientset, namespace)
+	}
+	secret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: secretName,
+		},
+		Type: v1.SecretTypeOpaque,
+		Data: map[string][]byte{
+			"tls.crt": []byte(FederatedIngressTLSCrt),
+			"tls.key": []byte(FederatedIngressTLSKey),
+		},
+	}
+	By(fmt.Sprintf("Creating federated secret %q in namespace %q", secretName, namespace))
+	newSecret, err := clientset.Core().Secrets(namespace).Create(secret)
+	framework.ExpectNoError(err, "creating secret %q in namespace %q", secret.Name, namespace)
+	return newSecret
+}
+
+type federationTestJig struct {
+	// TODO add TLS check later
+	rootCAs map[string][]byte
+	address string
+	ing     *v1beta1.Ingress
+	client  *fedclientset.Clientset
+}
+
+func newFederationTestJig(c *fedclientset.Clientset) *federationTestJig {
+	return &federationTestJig{client: c, rootCAs: map[string][]byte{}}
+}
+
+// WaitForFederatedIngressAddress waits for the Ingress to acquire an address.
+func waitForFederatedIngressAddress(c *fedclientset.Clientset, ns, ingName string, timeout time.Duration) (string, error) {
+	var address string
+	err := wait.PollImmediate(10*time.Second, timeout, func() (bool, error) {
+		ipOrNameList, err := getFederatedIngressAddress(c, ns, ingName)
+		if err != nil || len(ipOrNameList) == 0 {
+			framework.Logf("Waiting for Ingress %v to acquire IP, error %v", ingName, err)
+			return false, nil
+		}
+		address = ipOrNameList[0]
+		return true, nil
+	})
+	return address, err
+}
+
+// waitForFederatedIngressExists waits for the Ingress object exists.
+func waitForFederatedIngressExists(c *fedclientset.Clientset, ns, ingName string, timeout time.Duration) error {
+	err := wait.PollImmediate(10*time.Second, timeout, func() (bool, error) {
+		_, err := c.Extensions().Ingresses(ns).Get(ingName, metav1.GetOptions{})
+		if err != nil {
+			framework.Logf("Waiting for Ingress %v, error %v", ingName, err)
+			return false, nil
+		}
+		return true, nil
+	})
+	return err
+}
+
+// getFederatedIngressAddress returns the ips/hostnames associated with the Ingress.
+func getFederatedIngressAddress(client *fedclientset.Clientset, ns, name string) ([]string, error) {
+	ing, err := client.Extensions().Ingresses(ns).Get(name, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	addresses := []string{}
+	for _, a := range ing.Status.LoadBalancer.Ingress {
+		if a.IP != "" {
+			addresses = append(addresses, a.IP)
+		}
+		if a.Hostname != "" {
+			addresses = append(addresses, a.Hostname)
+		}
+	}
+	return addresses, nil
+}
+
+func waitForSecretShardsOrFail(nsName string, secret *v1.Secret, clusters fedframework.ClusterSlice) {
+	framework.Logf("Waiting for secret %q in %d clusters", secret.Name, len(clusters))
+	for _, c := range clusters {
+		waitForSecretOrFail(c.Clientset, nsName, secret, true, FederatedSecretTimeout)
+	}
+}
+
+func waitForSecretOrFail(clientset *kubeclientset.Clientset, nsName string, secret *v1.Secret, present bool, timeout time.Duration) {
+	By(fmt.Sprintf("Fetching a federated secret shard of secret %q in namespace %q from cluster", secret.Name, nsName))
+	var clusterSecret *v1.Secret
+	err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
+		var err error
+		clusterSecret, err = clientset.CoreV1().Secrets(nsName).Get(secret.Name, metav1.GetOptions{})
+		if (!present) && errors.IsNotFound(err) { // We want it gone, and it's gone.
+			By(fmt.Sprintf("Success: shard of federated secret %q in namespace %q in cluster is absent", secret.Name, nsName))
+			return true, nil // Success
+		}
+		if present && err == nil { // We want it present, and the Get succeeded, so we're all good.
+			By(fmt.Sprintf("Success: shard of federated secret %q in namespace %q in cluster is present", secret.Name, nsName))
+			return true, nil // Success
+		}
+		By(fmt.Sprintf("Secret %q in namespace %q in cluster.  Found: %v, waiting for Found: %v, trying again in %s (err=%v)", secret.Name, nsName, clusterSecret != nil && err == nil, present, framework.Poll, err))
+		return false, nil
+	})
+	framework.ExpectNoError(err, "Failed to verify secret %q in namespace %q in cluster: Present=%v", secret.Name, nsName, present)
+
+	if present && clusterSecret != nil {
+		Expect(util.SecretEquivalent(*clusterSecret, *secret))
+	}
+}
+
+func deleteSecretOrFail(clientset *fedclientset.Clientset, nsName string, secretName string, orphanDependents *bool) {
+	By(fmt.Sprintf("Deleting secret %q in namespace %q", secretName, nsName))
+	err := clientset.Core().Secrets(nsName).Delete(secretName, &metav1.DeleteOptions{OrphanDependents: orphanDependents})
+	if err != nil && !errors.IsNotFound(err) {
+		framework.ExpectNoError(err, "Error deleting secret %q in namespace %q", secretName, nsName)
+	}
+
+	// Wait for the secret to be deleted.
+	err = wait.Poll(5*time.Second, wait.ForeverTestTimeout, func() (bool, error) {
+		_, err := clientset.Core().Secrets(nsName).Get(secretName, metav1.GetOptions{})
+		if err != nil && errors.IsNotFound(err) {
+			return true, nil
+		}
+		return false, err
+	})
+	if err != nil {
+		framework.Failf("Error in deleting secret %s: %v", secretName, err)
+	}
+}

vendor/k8s.io/kubernetes/test/e2e_federation/job.go

@@ -0,0 +1,293 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_federation
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	batchv1 "k8s.io/api/batch/v1"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	fedutil "k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	"k8s.io/kubernetes/test/e2e/framework"
+	fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/kubernetes/pkg/api"
+)
+
+const (
+	FederationJobName = "federation-job"
+)
+
+var _ = framework.KubeDescribe("Federation jobs [Feature:Federation]", func() {
+
+	f := fedframework.NewDefaultFederatedFramework("federation-job")
+
+	Describe("Job objects [NoCluster]", func() {
+		AfterEach(func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+
+			// Delete all jobs.
+			nsName := f.FederationNamespace.Name
+			deleteAllJobsOrFail(f.FederationClientset, nsName)
+		})
+
+		It("should be created and deleted successfully", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+
+			nsName := f.FederationNamespace.Name
+			job := createJobOrFail(f.FederationClientset, nsName)
+			By(fmt.Sprintf("Creation of job %q in namespace %q succeeded.  Deleting job.", job.Name, nsName))
+			// Cleanup
+			err := f.FederationClientset.Batch().Jobs(nsName).Delete(job.Name, &metav1.DeleteOptions{})
+			framework.ExpectNoError(err, "Error deleting job %q in namespace %q", job.Name, job.Namespace)
+			By(fmt.Sprintf("Deletion of job %q in namespace %q succeeded.", job.Name, nsName))
+		})
+
+	})
+
+	// e2e cases for federated job controller
+	Describe("Federated Job", func() {
+		var (
+			clusters fedframework.ClusterSlice
+		)
+		BeforeEach(func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			clusters = f.GetRegisteredClusters()
+		})
+
+		AfterEach(func() {
+			nsName := f.FederationNamespace.Name
+			deleteAllJobsOrFail(f.FederationClientset, nsName)
+		})
+
+		It("should create and update matching jobs in underlying clusters", func() {
+			nsName := f.FederationNamespace.Name
+			job := createJobOrFail(f.FederationClientset, nsName)
+			defer func() {
+				// cleanup. deletion of jobs is not supported for underlying clusters
+				By(fmt.Sprintf("Deleting job %q/%q", nsName, job.Name))
+				waitForJobOrFail(f.FederationClientset, nsName, job.Name, clusters)
+				f.FederationClientset.Batch().Jobs(nsName).Delete(job.Name, &metav1.DeleteOptions{})
+			}()
+
+			waitForJobOrFail(f.FederationClientset, nsName, job.Name, clusters)
+			By(fmt.Sprintf("Successfuly created and synced job %q/%q to clusters", nsName, job.Name))
+		})
+
+		It("should be deleted from underlying clusters when OrphanDependents is false", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			nsName := f.FederationNamespace.Name
+			orphanDependents := false
+			verifyCascadingDeletionForJob(f.FederationClientset, clusters, &orphanDependents, nsName)
+			By(fmt.Sprintf("Verified that jobs were deleted from underlying clusters"))
+		})
+
+		It("should not be deleted from underlying clusters when OrphanDependents is true", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			nsName := f.FederationNamespace.Name
+			orphanDependents := true
+			verifyCascadingDeletionForJob(f.FederationClientset, clusters, &orphanDependents, nsName)
+			By(fmt.Sprintf("Verified that jobs were not deleted from underlying clusters"))
+		})
+
+		It("should not be deleted from underlying clusters when OrphanDependents is nil", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			nsName := f.FederationNamespace.Name
+			verifyCascadingDeletionForJob(f.FederationClientset, clusters, nil, nsName)
+			By(fmt.Sprintf("Verified that jobs were not deleted from underlying clusters"))
+		})
+
+	})
+})
+
+// deleteAllJobsOrFail deletes all jobs in the given namespace name.
+func deleteAllJobsOrFail(clientset *fedclientset.Clientset, nsName string) {
+	jobList, err := clientset.Batch().Jobs(nsName).List(metav1.ListOptions{})
+	Expect(err).NotTo(HaveOccurred())
+	orphanDependents := false
+	for _, job := range jobList.Items {
+		deleteJobOrFail(clientset, nsName, job.Name, &orphanDependents)
+	}
+}
+
+// verifyCascadingDeletionForJob verifies that job are deleted
+// from underlying clusters when orphan dependents is false and they are not
+// deleted when orphan dependents is true.
+func verifyCascadingDeletionForJob(clientset *fedclientset.Clientset, clusters fedframework.ClusterSlice, orphanDependents *bool, nsName string) {
+	job := createJobOrFail(clientset, nsName)
+	jobName := job.Name
+	// Check subclusters if the job was created there.
+	By(fmt.Sprintf("Waiting for job %s to be created in all underlying clusters", jobName))
+	err := wait.Poll(5*time.Second, 2*time.Minute, func() (bool, error) {
+		for _, cluster := range clusters {
+			_, err := cluster.Batch().Jobs(nsName).Get(jobName, metav1.GetOptions{})
+			if err != nil && errors.IsNotFound(err) {
+				return false, nil
+			}
+			if err != nil {
+				return false, err
+			}
+		}
+		return true, nil
+	})
+	framework.ExpectNoError(err, "Not all jobs created")
+
+	By(fmt.Sprintf("Deleting job %s", jobName))
+	deleteJobOrFail(clientset, nsName, jobName, orphanDependents)
+
+	By(fmt.Sprintf("Verifying job %s in underlying clusters", jobName))
+	errMessages := []string{}
+	// job should be present in underlying clusters unless orphanDependents is false.
+	shouldExist := orphanDependents == nil || *orphanDependents == true
+	for _, cluster := range clusters {
+		clusterName := cluster.Name
+		_, err := cluster.Batch().Jobs(nsName).Get(jobName, metav1.GetOptions{})
+		if shouldExist && errors.IsNotFound(err) {
+			errMessages = append(errMessages, fmt.Sprintf("unexpected NotFound error for job %s in cluster %s, expected job to exist", jobName, clusterName))
+		} else if !shouldExist && !errors.IsNotFound(err) {
+			errMessages = append(errMessages, fmt.Sprintf("expected NotFound error for job %s in cluster %s, got error: %v", jobName, clusterName, err))
+		}
+	}
+	if len(errMessages) != 0 {
+		framework.Failf("%s", strings.Join(errMessages, "; "))
+	}
+}
+
+func waitForJobOrFail(c *fedclientset.Clientset, namespace string, jobName string, clusters fedframework.ClusterSlice) {
+	err := waitForJob(c, namespace, jobName, clusters)
+	framework.ExpectNoError(err, "Failed to verify job %q/%q, err: %v", namespace, jobName, err)
+}
+
+func waitForJob(c *fedclientset.Clientset, namespace string, jobName string, clusters fedframework.ClusterSlice) error {
+	err := wait.Poll(10*time.Second, fedframework.FederatedDefaultTestTimeout, func() (bool, error) {
+		fjob, err := c.Batch().Jobs(namespace).Get(jobName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		succeeded := int32(0)
+		for _, cluster := range clusters {
+			job, err := cluster.Batch().Jobs(namespace).Get(jobName, metav1.GetOptions{})
+			if err != nil && !errors.IsNotFound(err) {
+				By(fmt.Sprintf("Failed getting job: %q/%q/%q, err: %v", cluster.Name, namespace, jobName, err))
+				return false, err
+			}
+			if err == nil {
+				if !verifyJob(fjob, job) {
+					By(fmt.Sprintf("Job meta or spec not match for cluster %q:\n    federation: %v\n    cluster: %v", cluster.Name, fjob, job))
+					return false, nil
+				}
+				succeeded += job.Status.Succeeded
+			}
+		}
+		if succeeded == fjob.Status.Succeeded &&
+			(fjob.Spec.Completions != nil && succeeded == *fjob.Spec.Completions) {
+			return true, nil
+		}
+		By(fmt.Sprintf("Job statuses not match, federation succeeded: %v/%v, clusters succeeded: %v\n",
+			fjob.Status.Succeeded, func(p *int32) int32 {
+				if p != nil {
+					return *p
+				} else {
+					return -1
+				}
+			}(fjob.Spec.Completions), succeeded))
+		return false, nil
+	})
+
+	return err
+}
+
+func verifyJob(fedJob, localJob *batchv1.Job) bool {
+	localJobObj, _ := api.Scheme.DeepCopy(localJob)
+	localJob = localJobObj.(*batchv1.Job)
+	localJob.Spec.ManualSelector = fedJob.Spec.ManualSelector
+	localJob.Spec.Completions = fedJob.Spec.Completions
+	localJob.Spec.Parallelism = fedJob.Spec.Parallelism
+	localJob.Spec.BackoffLimit = fedJob.Spec.BackoffLimit
+	return fedutil.ObjectMetaAndSpecEquivalent(fedJob, localJob)
+}
+
+func createJobOrFail(clientset *fedclientset.Clientset, namespace string) *batchv1.Job {
+	if clientset == nil || len(namespace) == 0 {
+		Fail(fmt.Sprintf("Internal error: invalid parameters passed to createJobOrFail: clientset: %v, namespace: %v", clientset, namespace))
+	}
+	By(fmt.Sprintf("Creating federation job %q in namespace %q", FederationJobName, namespace))
+
+	job := newJobForFed(namespace, FederationJobName, 5, 5)
+
+	_, err := clientset.Batch().Jobs(namespace).Create(job)
+	framework.ExpectNoError(err, "Creating job %q in namespace %q", job.Name, namespace)
+	By(fmt.Sprintf("Successfully created federation job %q in namespace %q", FederationJobName, namespace))
+	return job
+}
+
+func deleteJobOrFail(clientset *fedclientset.Clientset, nsName string, jobName string, orphanDependents *bool) {
+	By(fmt.Sprintf("Deleting job %q in namespace %q", jobName, nsName))
+	err := clientset.Batch().Jobs(nsName).Delete(jobName, &metav1.DeleteOptions{OrphanDependents: orphanDependents})
+	if err != nil && !errors.IsNotFound(err) {
+		framework.ExpectNoError(err, "Error deleting job %q in namespace %q", jobName, nsName)
+	}
+
+	// Wait for the job to be deleted.
+	err = wait.Poll(10*time.Second, fedframework.FederatedDefaultTestTimeout, func() (bool, error) {
+		_, err := clientset.Batch().Jobs(nsName).Get(jobName, metav1.GetOptions{})
+		if err != nil && errors.IsNotFound(err) {
+			return true, nil
+		}
+		return false, err
+	})
+	if err != nil {
+		framework.Failf("Error in deleting job %s: %v", jobName, err)
+	}
+}
+
+func newJobForFed(namespace string, name string, completions int32, parallelism int32) *batchv1.Job {
+	return &batchv1.Job{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: batchv1.JobSpec{
+			Parallelism: &parallelism,
+			Completions: &completions,
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{"name": "fjob"},
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name:    "sleep",
+							Image:   imageutils.GetBusyBoxImage(),
+							Command: []string{"sleep", "1"},
+						},
+					},
+					RestartPolicy: v1.RestartPolicyNever,
+				},
+			},
+		},
+	}
+}

vendor/k8s.io/kubernetes/test/e2e_federation/namespace.go

@@ -0,0 +1,236 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_federation
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/api/extensions/v1beta1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset/typed/core/v1"
+	k8s_api_v1 "k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+	fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
+
+	. "github.com/onsi/ginkgo"
+)
+
+const (
+	eventNamePrefix      = "e2e-namespace-test-event-"
+	namespacePrefix      = "e2e-namespace-test-"
+	replicaSetNamePrefix = "e2e-namespace-test-rs-"
+)
+
+// Create/delete ingress api objects
+var _ = framework.KubeDescribe("Federation namespace [Feature:Federation]", func() {
+	f := fedframework.NewDefaultFederatedFramework("federation-namespace")
+
+	Describe("Namespace objects", func() {
+		var clusters fedframework.ClusterSlice
+
+		var nsName string
+
+		BeforeEach(func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			clusters = f.GetRegisteredClusters()
+		})
+
+		AfterEach(func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			deleteNamespace(nil, nsName,
+				f.FederationClientset.Core().Namespaces().Get,
+				f.FederationClientset.Core().Namespaces().Delete)
+			for _, cluster := range clusters {
+				deleteNamespace(nil, nsName,
+					cluster.CoreV1().Namespaces().Get,
+					cluster.CoreV1().Namespaces().Delete)
+			}
+		})
+
+		// See https://github.com/kubernetes/kubernetes/issues/38225
+		It("deletes replicasets in the namespace when the namespace is deleted", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+
+			nsName = createNamespace(f.FederationClientset.Core().Namespaces())
+			rsName := k8s_api_v1.SimpleNameGenerator.GenerateName(replicaSetNamePrefix)
+			replicaCount := int32(2)
+			rs := &v1beta1.ReplicaSet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      rsName,
+					Namespace: nsName,
+				},
+				Spec: v1beta1.ReplicaSetSpec{
+					Replicas: &replicaCount,
+					Selector: &metav1.LabelSelector{
+						MatchLabels: map[string]string{"name": "myrs"},
+					},
+					Template: v1.PodTemplateSpec{
+						ObjectMeta: metav1.ObjectMeta{
+							Labels: map[string]string{"name": "myrs"},
+						},
+						Spec: v1.PodSpec{
+							Containers: []v1.Container{
+								{
+									Name:  "nginx",
+									Image: "nginx",
+								},
+							},
+						},
+					},
+				},
+			}
+
+			By(fmt.Sprintf("Creating replicaset %s in namespace %s", rsName, nsName))
+			_, err := f.FederationClientset.Extensions().ReplicaSets(nsName).Create(rs)
+			if err != nil {
+				framework.Failf("Failed to create replicaset %v in namespace %s, err: %s", rs, nsName, err)
+			}
+
+			By(fmt.Sprintf("Deleting namespace %s", nsName))
+			deleteNamespace(nil, nsName,
+				f.FederationClientset.Core().Namespaces().Get,
+				f.FederationClientset.Core().Namespaces().Delete)
+
+			By(fmt.Sprintf("Verify that replicaset %s was deleted as well", rsName))
+
+			waitForReplicaSetToBeDeletedOrFail(f.FederationClientset, nsName, rsName)
+		})
+
+		It("all resources in the namespace should be deleted when namespace is deleted", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+
+			nsName = createNamespace(f.FederationClientset.Core().Namespaces())
+
+			// Create resources in the namespace.
+			event := v1.Event{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      k8s_api_v1.SimpleNameGenerator.GenerateName(eventNamePrefix),
+					Namespace: nsName,
+				},
+				InvolvedObject: v1.ObjectReference{
+					Kind:      "Pod",
+					Namespace: nsName,
+					Name:      "sample-pod",
+				},
+			}
+			By(fmt.Sprintf("Creating event %s in namespace %s", event.Name, nsName))
+			_, err := f.FederationClientset.Core().Events(nsName).Create(&event)
+			if err != nil {
+				framework.Failf("Failed to create event %v in namespace %s, err: %s", event, nsName, err)
+			}
+
+			By(fmt.Sprintf("Deleting namespace %s", nsName))
+			deleteNamespace(nil, nsName,
+				f.FederationClientset.Core().Namespaces().Get,
+				f.FederationClientset.Core().Namespaces().Delete)
+
+			By(fmt.Sprintf("Verify that event %s was deleted as well", event.Name))
+			latestEvent, err := f.FederationClientset.Core().Events(nsName).Get(event.Name, metav1.GetOptions{})
+			if !errors.IsNotFound(err) {
+				framework.Failf("Event %s should have been deleted. Found: %v", event.Name, latestEvent)
+			}
+			By(fmt.Sprintf("Verified that deletion succeeded"))
+		})
+	})
+})
+
+// verifyNsCascadingDeletion verifies that namespaces are deleted from
+// underlying clusters when orphan dependents is false and they are not
+// deleted when orphan dependents is true.
+func verifyNsCascadingDeletion(nsClient clientset.NamespaceInterface, clusters fedframework.ClusterSlice, orphanDependents *bool) string {
+	nsName := createNamespace(nsClient)
+	// Check subclusters if the namespace was created there.
+	By(fmt.Sprintf("Waiting for namespace %s to be created in all underlying clusters", nsName))
+	err := wait.Poll(5*time.Second, 2*time.Minute, func() (bool, error) {
+		for _, cluster := range clusters {
+			_, err := cluster.CoreV1().Namespaces().Get(nsName, metav1.GetOptions{})
+			if err != nil && !errors.IsNotFound(err) {
+				return false, err
+			}
+			if err != nil {
+				return false, nil
+			}
+		}
+		return true, nil
+	})
+	framework.ExpectNoError(err, "Not all namespaces created")
+
+	By(fmt.Sprintf("Deleting namespace %s", nsName))
+	deleteNamespace(orphanDependents, nsName, nsClient.Get, nsClient.Delete)
+
+	By(fmt.Sprintf("Verifying namespaces %s in underlying clusters", nsName))
+	errMessages := []string{}
+	// namespace should be present in underlying clusters unless orphanDependents is false.
+	shouldExist := orphanDependents == nil || *orphanDependents == true
+	for _, cluster := range clusters {
+		clusterName := cluster.Name
+		_, err := cluster.CoreV1().Namespaces().Get(nsName, metav1.GetOptions{})
+		if shouldExist && errors.IsNotFound(err) {
+			errMessages = append(errMessages, fmt.Sprintf("unexpected NotFound error for namespace %s in cluster %s, expected namespace to exist", nsName, clusterName))
+		} else if !shouldExist && !errors.IsNotFound(err) {
+			errMessages = append(errMessages, fmt.Sprintf("expected NotFound error for namespace %s in cluster %s, got error: %v", nsName, clusterName, err))
+		}
+	}
+	if len(errMessages) != 0 {
+		framework.Failf("%s", strings.Join(errMessages, "; "))
+	}
+	return nsName
+}
+
+func createNamespace(nsClient clientset.NamespaceInterface) string {
+	ns := v1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: k8s_api_v1.SimpleNameGenerator.GenerateName(namespacePrefix),
+		},
+	}
+	By(fmt.Sprintf("Creating namespace %s", ns.Name))
+	_, err := nsClient.Create(&ns)
+	framework.ExpectNoError(err, "Failed to create namespace %s", ns.Name)
+	By(fmt.Sprintf("Created namespace %s", ns.Name))
+	return ns.Name
+}
+
+func deleteNamespace(orphanDependents *bool, namespace string, getter func(name string, options metav1.GetOptions) (*v1.Namespace, error), deleter func(string, *metav1.DeleteOptions) error) {
+	By(fmt.Sprintf("Deleting namespace: %s", namespace))
+	err := deleter(namespace, &metav1.DeleteOptions{OrphanDependents: orphanDependents})
+	if errors.IsNotFound(err) {
+		return
+	} else if err != nil {
+		framework.Failf("Failed to set %s for deletion: %v", namespace, err)
+	}
+	waitForNamespaceDeletion(namespace, getter)
+}
+
+func waitForNamespaceDeletion(namespace string, getter func(name string, options metav1.GetOptions) (*v1.Namespace, error)) {
+	err := wait.Poll(5*time.Second, 2*time.Minute, func() (bool, error) {
+		_, err := getter(namespace, metav1.GetOptions{})
+		if errors.IsNotFound(err) {
+			return true, nil
+		} else if err != nil {
+			return false, err
+		}
+		return false, nil
+	})
+	if err != nil {
+		framework.Failf("Namespaces not deleted: %v", err)
+	}
+}

vendor/k8s.io/kubernetes/test/e2e_federation/replicaset.go

@@ -0,0 +1,461 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_federation
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"time"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/api/extensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	fedutil "k8s.io/kubernetes/federation/pkg/federation-controller/util"
+	"k8s.io/kubernetes/test/e2e/framework"
+	fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/kubernetes/federation/apis/federation"
+	federatedtypes "k8s.io/kubernetes/federation/pkg/federatedtypes"
+)
+
+const (
+	FederationReplicaSetPrefix = "federation-replicaset-"
+)
+
+// Create/delete replicaset api objects
+var _ = framework.KubeDescribe("Federated ReplicaSet [Feature:Federation]", func() {
+	f := fedframework.NewDefaultFederatedFramework("federation-replicaset")
+
+	Describe("ReplicaSet objects [NoCluster]", func() {
+		AfterEach(func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+
+			// Delete all replicasets.
+			nsName := f.FederationNamespace.Name
+			deleteAllReplicaSetsOrFail(f.FederationClientset, nsName)
+		})
+
+		It("should be created and deleted successfully", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+
+			nsName := f.FederationNamespace.Name
+			rs := createReplicaSetOrFail(f.FederationClientset, newReplicaSet(nsName, FederationReplicaSetPrefix, 5, nil))
+			By(fmt.Sprintf("Creation of replicaset %q in namespace %q succeeded.  Deleting replicaset.", rs.Name, nsName))
+			// Cleanup
+			err := f.FederationClientset.Extensions().ReplicaSets(nsName).Delete(rs.Name, &metav1.DeleteOptions{})
+			framework.ExpectNoError(err, "Error deleting replicaset %q in namespace %q", rs.Name, rs.Namespace)
+			By(fmt.Sprintf("Deletion of replicaset %q in namespace %q succeeded.", rs.Name, nsName))
+		})
+
+	})
+
+	// e2e cases for federated replicaset controller
+	Describe("Features", func() {
+		var (
+			clusters fedframework.ClusterSlice
+		)
+
+		BeforeEach(func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			clusters = f.GetRegisteredClusters()
+		})
+
+		// e2e cases for federated replicaset controller
+		Describe("Preferences", func() {
+			var (
+				rs *v1beta1.ReplicaSet
+			)
+
+			AfterEach(func() {
+				// Delete all replicasets.
+				nsName := f.FederationNamespace.Name
+				if rs != nil {
+					orphanDependents := false
+					By(fmt.Sprintf("Deleting replicaset \"%s/%s\"", nsName, rs.Name))
+					deleteReplicaSetOrFail(f.FederationClientset, nsName, rs.Name, &orphanDependents)
+					rs = nil
+				}
+			})
+
+			It("should create replicasets with weight preference", func() {
+				pref, replicas, expect := generateFedRSPrefsWithWeight(clusters)
+				rs = createAndUpdateFedRSWithPref(f.FederationClientset, f.FederationNamespace.Name, clusters, pref, replicas, expect)
+			})
+
+			It("should create replicasets with min replicas preference", func() {
+				pref, replicas, expect := generateFedRSPrefsWithMin(clusters)
+				rs = createAndUpdateFedRSWithPref(f.FederationClientset, f.FederationNamespace.Name, clusters, pref, replicas, expect)
+			})
+
+			It("should create replicasets with max replicas preference", func() {
+				pref, replicas, expect := generateFedRSPrefsWithMax(clusters)
+				rs = createAndUpdateFedRSWithPref(f.FederationClientset, f.FederationNamespace.Name, clusters, pref, replicas, expect)
+			})
+
+			// test for rebalancing
+			PIt("should create replicasets and rebalance them", func() {
+				nsName := f.FederationNamespace.Name
+				pref1, pref2, replicas, expect1, expect2 := generateFedRSPrefsForRebalancing(clusters)
+
+				By("Testing replicaset rebalancing")
+				framework.Logf("Replicas: %d", replicas)
+				framework.Logf("Preference 1: %#v", pref1)
+				framework.Logf("Preference 2: %#v", pref2)
+
+				rs = newReplicaSet(nsName, FederationReplicaSetPrefix, replicas, pref1)
+				rs = createReplicaSetOrFail(f.FederationClientset, rs)
+				waitForReplicaSetOrFail(f.FederationClientset, nsName, rs.Name, clusters, expect1)
+				By(fmt.Sprintf("Successfully created and synced replicaset \"%s/%s\" (%v/%v) to clusters", nsName, rs.Name, rs.Status.Replicas, *rs.Spec.Replicas))
+
+				rs = newReplicaSetWithName(nsName, rs.Name, replicas, pref2)
+				updateReplicaSetOrFail(f.FederationClientset, rs)
+				waitForReplicaSetOrFail(f.FederationClientset, nsName, rs.Name, clusters, expect1)
+				By(fmt.Sprintf("Successfully updated and synced replicaset \"%s/%s\" (%v/%v) to clusters", nsName, rs.Name, rs.Status.Replicas, *rs.Spec.Replicas))
+
+				pref2 = updateFedRSPrefsRebalance(pref2, true)
+				rs = newReplicaSetWithName(nsName, rs.Name, replicas, pref2)
+				updateReplicaSetOrFail(f.FederationClientset, rs)
+				waitForReplicaSetOrFail(f.FederationClientset, nsName, rs.Name, clusters, expect2)
+				By(fmt.Sprintf("Successfully updated and synced replicaset \"%s/%s\" (%v/%v) to clusters", nsName, rs.Name, rs.Status.Replicas, *rs.Spec.Replicas))
+			})
+		})
+	})
+})
+
+func createAndWaitForReplicasetOrFail(clientset *fedclientset.Clientset, nsName string, clusters fedframework.ClusterSlice) *v1beta1.ReplicaSet {
+	rs := createReplicaSetOrFail(clientset, newReplicaSet(nsName, FederationReplicaSetPrefix, 5, nil))
+	// Check subclusters if the replicaSet was created there.
+	By(fmt.Sprintf("Waiting for replica sets %s to be created in all underlying clusters", rs.Name))
+	err := wait.Poll(5*time.Second, 2*time.Minute, func() (bool, error) {
+		for _, cluster := range clusters {
+			_, err := cluster.Extensions().ReplicaSets(nsName).Get(rs.Name, metav1.GetOptions{})
+			if err != nil && errors.IsNotFound(err) {
+				return false, nil
+			}
+			if err != nil {
+				return false, err
+			}
+		}
+		return true, nil
+	})
+	framework.ExpectNoError(err, "Not all replica sets created")
+	return rs
+}
+
+func createAndUpdateFedRSWithPref(clientset *fedclientset.Clientset, nsName string, clusters fedframework.ClusterSlice, pref *federation.ReplicaAllocationPreferences, replicas int32, expect map[string]int32) *v1beta1.ReplicaSet {
+	framework.Logf("Replicas: %d, Preference: %#v", replicas, pref)
+	rs := newReplicaSet(nsName, FederationReplicaSetPrefix, replicas, pref)
+	rs = createReplicaSetOrFail(clientset, rs)
+
+	waitForReplicaSetOrFail(clientset, nsName, rs.Name, clusters, expect)
+	By(fmt.Sprintf("Successfully created and synced replicaset \"%s/%s\" (%v/%v) to clusters", nsName, rs.Name, rs.Status.Replicas, *rs.Spec.Replicas))
+
+	rs = newReplicaSetWithName(nsName, rs.Name, 0, pref)
+	updateReplicaSetOrFail(clientset, rs)
+	waitForReplicaSetOrFail(clientset, nsName, rs.Name, clusters, nil)
+	By(fmt.Sprintf("Successfully updated and synced replicaset \"%s/%s\" (%v/%v) to clusters", nsName, rs.Name, rs.Status.Replicas, *rs.Spec.Replicas))
+
+	rs = newReplicaSetWithName(nsName, rs.Name, replicas, pref)
+	updateReplicaSetOrFail(clientset, rs)
+	waitForReplicaSetOrFail(clientset, nsName, rs.Name, clusters, expect)
+	By(fmt.Sprintf("Successfully updated and synced replicaset \"%s/%s\" (%v/%v) to clusters", nsName, rs.Name, rs.Status.Replicas, *rs.Spec.Replicas))
+
+	return rs
+}
+
+// deleteAllReplicaSetsOrFail deletes all replicasets in the given namespace name.
+func deleteAllReplicaSetsOrFail(clientset *fedclientset.Clientset, nsName string) {
+	replicasetList, err := clientset.Extensions().ReplicaSets(nsName).List(metav1.ListOptions{})
+	Expect(err).NotTo(HaveOccurred())
+	orphanDependents := false
+	for _, replicaset := range replicasetList.Items {
+		deleteReplicaSetOrFail(clientset, nsName, replicaset.Name, &orphanDependents)
+	}
+}
+
+func generateFedRSPrefsWithWeight(clusters fedframework.ClusterSlice) (pref *federation.ReplicaAllocationPreferences, replicas int32, expect map[string]int32) {
+	By("Generating replicaset preferences with weights")
+	clusterNames := extractClusterNames(clusters)
+	pref = &federation.ReplicaAllocationPreferences{
+		Clusters: map[string]federation.ClusterPreferences{},
+	}
+	replicas = 0
+	expect = map[string]int32{}
+
+	for i, clusterName := range clusterNames {
+		if i != 0 { // do not set weight for cluster[0] thus it should have no replicas scheduled
+			pref.Clusters[clusterName] = federation.ClusterPreferences{
+				Weight: int64(i),
+			}
+			replicas += int32(i)
+			expect[clusterName] = int32(i)
+		}
+	}
+	return
+}
+
+func generateFedRSPrefsWithMin(clusters fedframework.ClusterSlice) (pref *federation.ReplicaAllocationPreferences, replicas int32, expect map[string]int32) {
+	By("Generating replicaset preferences with min replicas")
+	clusterNames := extractClusterNames(clusters)
+	pref = &federation.ReplicaAllocationPreferences{
+		Clusters: map[string]federation.ClusterPreferences{
+			clusterNames[0]: {Weight: 100},
+		},
+	}
+	replicas = 0
+	expect = map[string]int32{}
+
+	for i, clusterName := range clusterNames {
+		if i != 0 { // do not set weight and minReplicas for cluster[0] thus it should have no replicas scheduled
+			pref.Clusters[clusterName] = federation.ClusterPreferences{
+				Weight:      int64(1),
+				MinReplicas: int64(i + 2),
+			}
+			replicas += int32(i + 2)
+			expect[clusterName] = int32(i + 2)
+		}
+	}
+	// the extra replica goes to cluster[0] which has the highest weight
+	replicas += 1
+	expect[clusterNames[0]] = 1
+	return
+}
+
+func generateFedRSPrefsWithMax(clusters fedframework.ClusterSlice) (pref *federation.ReplicaAllocationPreferences, replicas int32, expect map[string]int32) {
+	By("Generating replicaset preferences with max replicas")
+	clusterNames := extractClusterNames(clusters)
+	pref = &federation.ReplicaAllocationPreferences{
+		Clusters: map[string]federation.ClusterPreferences{
+			clusterNames[0]: {Weight: 1},
+		},
+	}
+	replicas = 0
+	expect = map[string]int32{}
+
+	for i, clusterName := range clusterNames {
+		if i != 0 { // do not set maxReplicas for cluster[0] thus replicas exceeds the total maxReplicas turned to cluster[0]
+			maxReplicas := int64(i)
+			pref.Clusters[clusterName] = federation.ClusterPreferences{
+				Weight:      int64(100),
+				MaxReplicas: &maxReplicas,
+			}
+			replicas += int32(i)
+			expect[clusterName] = int32(i)
+		}
+	}
+	// extra replicas go to cluster[0] although it has the lowest weight as others hit the MaxReplicas
+	replicas += 5
+	expect[clusterNames[0]] = 5
+	return
+}
+
+func updateFedRSPrefsRebalance(pref *federation.ReplicaAllocationPreferences, rebalance bool) *federation.ReplicaAllocationPreferences {
+	pref.Rebalance = rebalance
+	return pref
+}
+
+func generateFedRSPrefsForRebalancing(clusters fedframework.ClusterSlice) (pref1, pref2 *federation.ReplicaAllocationPreferences, replicas int32, expect1, expect2 map[string]int32) {
+	By("Generating replicaset for rebalancing")
+	clusterNames := extractClusterNames(clusters)
+	replicas = 3
+
+	pref1 = &federation.ReplicaAllocationPreferences{
+		Clusters: map[string]federation.ClusterPreferences{
+			clusterNames[0]: {Weight: 1},
+			clusterNames[1]: {Weight: 2},
+		},
+	}
+	expect1 = map[string]int32{
+		clusterNames[0]: 1,
+		clusterNames[1]: 2,
+	}
+	pref2 = &federation.ReplicaAllocationPreferences{
+		Clusters: map[string]federation.ClusterPreferences{
+			clusterNames[0]: {Weight: 2},
+			clusterNames[1]: {Weight: 1},
+		},
+	}
+	expect2 = map[string]int32{
+		clusterNames[0]: 2,
+		clusterNames[1]: 1,
+	}
+	return
+}
+
+func waitForReplicaSetOrFail(c *fedclientset.Clientset, namespace string, replicaSetName string, clusters fedframework.ClusterSlice, expect map[string]int32) {
+	err := waitForReplicaSet(c, namespace, replicaSetName, clusters, expect)
+	framework.ExpectNoError(err, "Failed to verify replica set \"%s/%s\", err: %v", namespace, replicaSetName, err)
+}
+
+func waitForReplicaSet(c *fedclientset.Clientset, namespace string, replicaSetName string, clusters fedframework.ClusterSlice, expect map[string]int32) error {
+	framework.Logf("waitForReplicaSet: %s/%s; clusters: %v; expect: %v", namespace, replicaSetName, clusters, expect)
+	err := wait.Poll(10*time.Second, fedframework.FederatedDefaultTestTimeout, func() (bool, error) {
+		frs, err := c.ExtensionsV1beta1().ReplicaSets(namespace).Get(replicaSetName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		specReplicas, statusReplicas := int32(0), int32(0)
+		for _, cluster := range clusters {
+			// TODO: switch to use AppsV1beta2 ReplicaSet when apps/v1beta2 is enabled by default
+			rs, err := cluster.ExtensionsV1beta1().ReplicaSets(namespace).Get(replicaSetName, metav1.GetOptions{})
+			if err != nil && !errors.IsNotFound(err) {
+				framework.Logf("Failed getting replicaset: \"%s/%s/%s\", err: %v", cluster.Name, namespace, replicaSetName, err)
+				return false, err
+			}
+			if errors.IsNotFound(err) {
+				if expect != nil && expect[cluster.Name] > 0 {
+					framework.Logf("Replicaset \"%s/%s/%s\" with replica count %d does not exist", cluster.Name, namespace, replicaSetName, expect[cluster.Name])
+					return false, nil
+				}
+			} else {
+				if !equivalentReplicaSet(frs, rs) {
+					framework.Logf("Replicaset meta or spec does not match for cluster %q:\n    federation: %v\n    cluster: %v", cluster.Name, frs, rs)
+					return false, nil
+				}
+				if expect != nil && *rs.Spec.Replicas < expect[cluster.Name] {
+					framework.Logf("Replicas do not match for \"%s/%s/%s\": expected: >= %v, actual: %v", cluster.Name, namespace, replicaSetName, expect[cluster.Name], *rs.Spec.Replicas)
+					return false, nil
+				}
+				specReplicas += *rs.Spec.Replicas
+				statusReplicas += rs.Status.Replicas
+			}
+		}
+		if *frs.Spec.Replicas == 0 && frs.Status.Replicas != 0 {
+			framework.Logf("ReplicaSet \"%s/%s\" with zero replicas should match the status as no overflow happens: expected: 0, actual: %v", namespace, replicaSetName, frs.Status.Replicas)
+			return false, nil
+		}
+		if statusReplicas == frs.Status.Replicas && specReplicas >= *frs.Spec.Replicas {
+			return true, nil
+		}
+		framework.Logf("Replicas do not match, federation replicas: %v/%v, cluster replicas: %v/%v", frs.Status.Replicas, *frs.Spec.Replicas, statusReplicas, specReplicas)
+		return false, nil
+	})
+
+	return err
+}
+
+func equivalentReplicaSet(fedReplicaSet, localReplicaSet *v1beta1.ReplicaSet) bool {
+	localReplicaSetSpec := localReplicaSet.Spec
+	localReplicaSetSpec.Replicas = fedReplicaSet.Spec.Replicas
+	return fedutil.ObjectMetaEquivalent(fedReplicaSet.ObjectMeta, localReplicaSet.ObjectMeta) &&
+		reflect.DeepEqual(fedReplicaSet.Spec, localReplicaSetSpec)
+}
+
+func createReplicaSetOrFail(clientset *fedclientset.Clientset, replicaset *v1beta1.ReplicaSet) *v1beta1.ReplicaSet {
+	namespace := replicaset.Namespace
+	if clientset == nil || len(namespace) == 0 {
+		Fail(fmt.Sprintf("Internal error: invalid parameters passed to createReplicaSetOrFail: clientset: %v, namespace: %v", clientset, namespace))
+	}
+	By(fmt.Sprintf("Creating federation replicaset %q in namespace %q", replicaset.Name, namespace))
+
+	newRS, err := clientset.Extensions().ReplicaSets(namespace).Create(replicaset)
+	framework.ExpectNoError(err, "Creating replicaset %q in namespace %q", replicaset.Name, namespace)
+	By(fmt.Sprintf("Successfully created federation replicaset %q in namespace %q", newRS.Name, namespace))
+	return newRS
+}
+
+func deleteReplicaSetOrFail(clientset *fedclientset.Clientset, nsName string, replicaSetName string, orphanDependents *bool) {
+	By(fmt.Sprintf("Deleting replica set %q in namespace %q", replicaSetName, nsName))
+	err := clientset.Extensions().ReplicaSets(nsName).Delete(replicaSetName, &metav1.DeleteOptions{OrphanDependents: orphanDependents})
+	if err != nil && !errors.IsNotFound(err) {
+		framework.ExpectNoError(err, "Error deleting replica set %q in namespace %q", replicaSetName, nsName)
+	}
+
+	waitForReplicaSetToBeDeletedOrFail(clientset, nsName, replicaSetName)
+}
+
+func updateReplicaSetOrFail(clientset *fedclientset.Clientset, replicaset *v1beta1.ReplicaSet) *v1beta1.ReplicaSet {
+	namespace := replicaset.Namespace
+	if clientset == nil || len(namespace) == 0 {
+		Fail(fmt.Sprintf("Internal error: invalid parameters passed to updateReplicaSetOrFail: clientset: %v, namespace: %v", clientset, namespace))
+	}
+	By(fmt.Sprintf("Updating federation replicaset %q in namespace %q", replicaset.Name, namespace))
+
+	newRS, err := clientset.ExtensionsV1beta1().ReplicaSets(namespace).Update(replicaset)
+	framework.ExpectNoError(err, "Updating replicaset %q in namespace %q", replicaset.Name, namespace)
+	By(fmt.Sprintf("Successfully updated federation replicaset %q in namespace %q", replicaset.Name, namespace))
+
+	return newRS
+}
+
+func newReplicaSetObj(namespace string, replicas int32, pref *federation.ReplicaAllocationPreferences) *v1beta1.ReplicaSet {
+	// When the tests are run in parallel, replicasets from different tests can
+	// collide with each other. Prevent that by creating a unique label and
+	// label selector for each created replica set.
+	uuidString := string(uuid.NewUUID())
+	rsLabel := fmt.Sprintf("myrs-%s", uuidString)
+
+	rs := &v1beta1.ReplicaSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace:   namespace,
+			Annotations: map[string]string{},
+		},
+		Spec: v1beta1.ReplicaSetSpec{
+			Replicas: &replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"name": rsLabel},
+			},
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{"name": rsLabel},
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name:  "nginx",
+							Image: "nginx",
+						},
+					},
+				},
+			},
+		},
+	}
+	if pref != nil {
+		prefBytes, _ := json.Marshal(pref)
+		prefString := string(prefBytes)
+		rs.Annotations[federatedtypes.FedReplicaSetPreferencesAnnotation] = prefString
+	}
+	return rs
+
+}
+
+func newReplicaSet(namespace string, prefix string, replicas int32, pref *federation.ReplicaAllocationPreferences) *v1beta1.ReplicaSet {
+	rs := newReplicaSetObj(namespace, replicas, pref)
+	rs.GenerateName = prefix
+	return rs
+}
+
+func newReplicaSetWithName(namespace string, name string, replicas int32, pref *federation.ReplicaAllocationPreferences) *v1beta1.ReplicaSet {
+	rs := newReplicaSetObj(namespace, replicas, pref)
+	rs.Name = name
+	return rs
+}
+
+func extractClusterNames(clusters fedframework.ClusterSlice) []string {
+	clusterNames := make([]string, 0, len(clusters))
+	for _, cluster := range clusters {
+		clusterNames = append(clusterNames, cluster.Name)
+	}
+	return clusterNames
+}

vendor/k8s.io/kubernetes/test/e2e_federation/service.go

@@ -0,0 +1,398 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_federation
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes/scheme"
+	fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	"k8s.io/kubernetes/test/e2e/framework"
+	fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
+
+	. "github.com/onsi/ginkgo"
+)
+
+const (
+	FederatedServiceName    = "federated-service"
+	FederatedServicePodName = "federated-service-test-pod"
+)
+
+var FederatedServiceLabels = map[string]string{
+	"foo": "bar",
+}
+
+var _ = framework.KubeDescribe("Federated Services [Feature:Federation]", func() {
+	f := fedframework.NewDefaultFederatedFramework("federated-service")
+	var clusters fedframework.ClusterSlice
+	var federationName string
+
+	var _ = Describe("Without Clusters [NoCluster]", func() {
+		BeforeEach(func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			// Placeholder
+		})
+
+		AfterEach(func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+		})
+
+		It("should succeed when a service is created", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+
+			nsName := f.FederationNamespace.Name
+			service := createServiceOrFail(f.FederationClientset, nsName, FederatedServiceName)
+			By(fmt.Sprintf("Creation of service %q in namespace %q succeeded.  Deleting service.", service.Name, nsName))
+
+			// Cleanup
+			err := f.FederationClientset.CoreV1().Services(nsName).Delete(service.Name, &metav1.DeleteOptions{})
+			framework.ExpectNoError(err, "Error deleting service %q in namespace %q", service.Name, service.Namespace)
+			By(fmt.Sprintf("Deletion of service %q in namespace %q succeeded.", service.Name, nsName))
+		})
+	})
+
+	var _ = Describe("with clusters", func() {
+		BeforeEach(func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+
+			// TODO: Federation API server should be able to answer this.
+			if federationName = os.Getenv("FEDERATION_NAME"); federationName == "" {
+				federationName = DefaultFederationName
+			}
+
+			clusters = f.GetRegisteredClusters()
+		})
+
+		Describe("Federated Service", func() {
+			var (
+				service *v1.Service
+				nsName  string
+			)
+
+			BeforeEach(func() {
+				fedframework.SkipUnlessFederated(f.ClientSet)
+				nsName = f.FederationNamespace.Name
+			})
+
+			AfterEach(func() {
+				fedframework.SkipUnlessFederated(f.ClientSet)
+
+				if service != nil {
+					By(fmt.Sprintf("Deleting service shards and their provider resources in underlying clusters for service %q in namespace %q", service.Name, nsName))
+					cleanupServiceShardsAndProviderResources(nsName, service, clusters)
+					service = nil
+					nsName = ""
+				}
+			})
+
+			It("should create and update matching services in underlying clusters", func() {
+				fedframework.SkipUnlessFederated(f.ClientSet)
+				service = createServiceOrFail(f.FederationClientset, nsName, FederatedServiceName)
+				defer func() { // Cleanup
+					By(fmt.Sprintf("Deleting service %q in namespace %q", service.Name, nsName))
+					err := f.FederationClientset.CoreV1().Services(nsName).Delete(service.Name, &metav1.DeleteOptions{})
+					framework.ExpectNoError(err, "Error deleting service %q in namespace %q", service.Name, nsName)
+				}()
+				By(fmt.Sprintf("Wait for service shards to be created in all clusters for service \"%s/%s\"", nsName, service.Name))
+				waitForServiceShardsOrFail(nsName, service, clusters)
+				framework.Logf("Successfully created and synced service \"%s/%s\" to all clusters", nsName, service.Name)
+				By(fmt.Sprintf("Update federated service \"%s/%s\"", nsName, service.Name))
+				service = updateServiceOrFail(f.FederationClientset, nsName, FederatedServiceName)
+				waitForServiceShardsOrFail(nsName, service, clusters)
+				framework.Logf("Successfully updated and synced service \"%s/%s\" to clusters", nsName, service.Name)
+			})
+
+			It("should be deleted from underlying clusters when OrphanDependents is false", func() {
+				fedframework.SkipUnlessFederated(f.ClientSet)
+				orphanDependents := false
+				verifyCascadingDeletionForService(f.FederationClientset, clusters, &orphanDependents, nsName)
+				By(fmt.Sprintf("Verified that services were deleted from underlying clusters"))
+			})
+
+			It("should not be deleted from underlying clusters when OrphanDependents is true", func() {
+				fedframework.SkipUnlessFederated(f.ClientSet)
+				orphanDependents := true
+				verifyCascadingDeletionForService(f.FederationClientset, clusters, &orphanDependents, nsName)
+				By(fmt.Sprintf("Verified that services were not deleted from underlying clusters"))
+			})
+
+			It("should not be deleted from underlying clusters when OrphanDependents is nil", func() {
+				fedframework.SkipUnlessFederated(f.ClientSet)
+				verifyCascadingDeletionForService(f.FederationClientset, clusters, nil, nsName)
+				By(fmt.Sprintf("Verified that services were not deleted from underlying clusters"))
+			})
+
+			It("should recreate service shard in underlying clusters when service shard is deleted", func() {
+				fedframework.SkipUnlessFederated(f.ClientSet)
+				service = createServiceOrFail(f.FederationClientset, nsName, FederatedServiceName)
+				defer func() {
+					// Cleanup
+					By(fmt.Sprintf("Deleting service %q in namespace %q", service.Name, nsName))
+					err := f.FederationClientset.CoreV1().Services(nsName).Delete(service.Name, &metav1.DeleteOptions{})
+					framework.ExpectNoError(err, "Error deleting service %q in namespace %q", service.Name, nsName)
+				}()
+				By(fmt.Sprintf("Wait for service shards to be created in all clusters for service \"%s/%s\"", nsName, service.Name))
+				waitForServiceShardsOrFail(nsName, service, clusters)
+				framework.Logf("Successfully created and synced service \"%s/%s\" to all clusters", nsName, service.Name)
+
+				By(fmt.Sprintf("Deleting a service shard in one underlying cluster"))
+				primaryClusterName := clusters[0].Name
+				err := deleteServiceShard(clusters[0], nsName, FederatedServiceName)
+				framework.ExpectNoError(err, fmt.Sprintf("while deleting service shard %q in cluster %q", FederatedServiceName, primaryClusterName))
+
+				waitForServiceShardsOrFail(nsName, service, clusters)
+				framework.Logf("Successfully recreated service shard \"%s/%s\" in %q cluster", nsName, service.Name, primaryClusterName)
+			})
+		})
+
+		var _ = Describe("DNS", func() {
+
+			var (
+				service      *v1.Service
+				serviceShard *v1.Service
+				backendPods  BackendPodMap
+			)
+
+			BeforeEach(func() {
+				fedframework.SkipUnlessFederated(f.ClientSet)
+
+				nsName := f.FederationNamespace.Name
+
+				backendPods = createBackendPodsOrFail(clusters, nsName, FederatedServicePodName)
+
+				service = createLBServiceOrFail(f.FederationClientset, nsName, FederatedServiceName, clusters)
+				obj, err := scheme.Scheme.DeepCopy(service)
+				// Cloning shouldn't fail. On the off-chance it does, we
+				// should shallow copy service to serviceShard before
+				// failing. If we don't do this we will never really
+				// get a chance to clean up the underlying services
+				// when the cloner fails for reasons not in our
+				// control. For example, cloner bug. That will cause
+				// the resources to leak, which in turn causes the
+				// test project to run out of quota and the entire
+				// suite starts failing. So we must try as hard as
+				// possible to cleanup the underlying services. So
+				// if DeepCopy fails, we are going to try with shallow
+				// copy as a last resort.
+				if err != nil {
+					serviceCopy := *service
+					serviceShard = &serviceCopy
+					framework.ExpectNoError(err, fmt.Sprintf("Error in deep copying service %q", service.Name))
+				}
+				var ok bool
+				serviceShard, ok = obj.(*v1.Service)
+				// Same argument as above about using shallow copy
+				// as a last resort.
+				if !ok {
+					serviceCopy := *service
+					serviceShard = &serviceCopy
+					framework.ExpectNoError(err, fmt.Sprintf("Unexpected service object copied %T", obj))
+				}
+
+				waitForServiceShardsOrFail(nsName, serviceShard, clusters)
+			})
+
+			AfterEach(func() {
+				fedframework.SkipUnlessFederated(f.ClientSet)
+
+				nsName := f.FederationNamespace.Name
+				deleteBackendPodsOrFail(clusters, backendPods)
+				backendPods = nil
+
+				if service != nil {
+					deleteServiceOrFail(f.FederationClientset, nsName, service.Name, nil)
+					service = nil
+				} else {
+					By("No service to delete.  Service is nil")
+				}
+
+				if serviceShard != nil {
+					By(fmt.Sprintf("Deleting service shards and their provider resources in underlying clusters for service %q in namespace %q", serviceShard.Name, nsName))
+					cleanupServiceShardsAndProviderResources(nsName, serviceShard, clusters)
+					serviceShard = nil
+				} else {
+					By("No service shards to delete. `serviceShard` is nil")
+				}
+			})
+
+			It("should be able to discover a federated service", func() {
+				fedframework.SkipUnlessFederated(f.ClientSet)
+
+				nsName := f.FederationNamespace.Name
+				svcDNSNames := []string{
+					FederatedServiceName,
+					fmt.Sprintf("%s.%s", FederatedServiceName, nsName),
+					fmt.Sprintf("%s.%s.svc.cluster.local.", FederatedServiceName, nsName),
+					fmt.Sprintf("%s.%s.%s", FederatedServiceName, nsName, federationName),
+					fmt.Sprintf("%s.%s.%s.svc.cluster.local.", FederatedServiceName, nsName, federationName),
+				}
+				// TODO(mml): This could be much faster.  We can launch all the test
+				// pods, perhaps in the BeforeEach, and then just poll until we get
+				// successes/failures from them all.
+				for i, DNSName := range svcDNSNames {
+					discoverService(f, DNSName, true, "federated-service-e2e-discovery-pod-"+strconv.Itoa(i))
+				}
+				By("Verified that DNS rules are working as expected")
+
+				By("Deleting the service to verify that DNS rules still work")
+				err := f.FederationClientset.CoreV1().Services(nsName).Delete(FederatedServiceName, &metav1.DeleteOptions{})
+				framework.ExpectNoError(err, "Error deleting service %q in namespace %q", service.Name, service.Namespace)
+				// Service is deleted, unset the test block-global service variable.
+				service = nil
+
+				for i, DNSName := range svcDNSNames {
+					discoverService(f, DNSName, true, "federated-service-e2e-discovery-pod-"+strconv.Itoa(i))
+				}
+				By("Verified that deleting the service does not affect DNS records")
+			})
+
+			Context("non-local federated service", func() {
+				BeforeEach(func() {
+					fedframework.SkipUnlessFederated(f.ClientSet)
+
+					// Delete the backend pod from the shard which is local to the discovery pod.
+					primaryCluster := clusters[0]
+					backendPod := backendPods[primaryCluster.Name]
+					deleteOneBackendPodOrFail(primaryCluster, backendPod)
+
+				})
+
+				PIt("should be able to discover a non-local federated service", func() {
+					fedframework.SkipUnlessFederated(f.ClientSet)
+
+					nsName := f.FederationNamespace.Name
+					svcDNSNames := []string{
+						fmt.Sprintf("%s.%s.%s", FederatedServiceName, nsName, federationName),
+						fmt.Sprintf("%s.%s.%s.svc.cluster.local.", FederatedServiceName, nsName, federationName),
+					}
+					for i, name := range svcDNSNames {
+						discoverService(f, name, true, "federated-service-e2e-discovery-pod-"+strconv.Itoa(i))
+					}
+				})
+
+				// TODO(mml): This currently takes 9 minutes.  Consider reducing the
+				// TTL and/or running the pods in parallel.
+				Context("[Slow] missing local service", func() {
+					It("should never find DNS entries for a missing local service", func() {
+						fedframework.SkipUnlessFederated(f.ClientSet)
+
+						nsName := f.FederationNamespace.Name
+						localSvcDNSNames := []string{
+							FederatedServiceName,
+							fmt.Sprintf("%s.%s", FederatedServiceName, nsName),
+							fmt.Sprintf("%s.%s.svc.cluster.local.", FederatedServiceName, nsName),
+						}
+						for i, name := range localSvcDNSNames {
+							discoverService(f, name, false, "federated-service-e2e-discovery-pod-"+strconv.Itoa(i))
+						}
+					})
+				})
+			})
+		})
+	})
+})
+
+// verifyCascadingDeletionForService verifies that services are deleted from
+// underlying clusters when orphan dependents is false and they are not
+// deleted when orphan dependents is true.
+func verifyCascadingDeletionForService(clientset *fedclientset.Clientset, clusters fedframework.ClusterSlice, orphanDependents *bool, nsName string) {
+	service := createServiceOrFail(clientset, nsName, FederatedServiceName)
+	serviceName := service.Name
+	// Check subclusters if the service was created there.
+	By(fmt.Sprintf("Waiting for service %s to be created in all underlying clusters", serviceName))
+	err := wait.Poll(5*time.Second, 2*time.Minute, func() (bool, error) {
+		for _, cluster := range clusters {
+			_, err := cluster.CoreV1().Services(nsName).Get(serviceName, metav1.GetOptions{})
+			if err != nil {
+				if !errors.IsNotFound(err) {
+					return false, err
+				}
+				return false, nil
+			}
+		}
+		return true, nil
+	})
+	framework.ExpectNoError(err, "Not all services created")
+
+	By(fmt.Sprintf("Deleting service %s", serviceName))
+	deleteServiceOrFail(clientset, nsName, serviceName, orphanDependents)
+
+	By(fmt.Sprintf("Verifying services %s in underlying clusters", serviceName))
+	errMessages := []string{}
+	// service should be present in underlying clusters unless orphanDependents is false.
+	shouldExist := orphanDependents == nil || *orphanDependents == true
+	for _, cluster := range clusters {
+		clusterName := cluster.Name
+		_, err := cluster.CoreV1().Services(nsName).Get(serviceName, metav1.GetOptions{})
+		if shouldExist && errors.IsNotFound(err) {
+			errMessages = append(errMessages, fmt.Sprintf("unexpected NotFound error for service %s in cluster %s, expected service to exist", serviceName, clusterName))
+		} else if !shouldExist && !errors.IsNotFound(err) {
+			errMessages = append(errMessages, fmt.Sprintf("expected NotFound error for service %s in cluster %s, got error: %v", serviceName, clusterName, err))
+		}
+	}
+	if len(errMessages) != 0 {
+		framework.Failf("%s", strings.Join(errMessages, "; "))
+	}
+}
+
+func updateServiceOrFail(clientset *fedclientset.Clientset, namespace, name string) *v1.Service {
+	service, err := clientset.CoreV1().Services(namespace).Get(name, metav1.GetOptions{})
+	framework.ExpectNoError(err, "Getting service %q in namespace %q", name, namespace)
+	service.Spec.Selector["name"] = "update-demo"
+	newService, err := clientset.CoreV1().Services(namespace).Update(service)
+	By(fmt.Sprintf("Successfully updated federated service %q in namespace %q", name, namespace))
+	return newService
+}
+
+func deleteServiceShard(c *fedframework.Cluster, namespace, service string) error {
+	err := c.Clientset.CoreV1().Services(namespace).Delete(service, &metav1.DeleteOptions{})
+	if err != nil && !errors.IsNotFound(err) {
+		framework.Logf("Failed to delete service %q in namespace %q, in cluster %q", service, namespace, c.Name)
+		return err
+	}
+	By(fmt.Sprintf("Service %q in namespace %q in cluster %q deleted", service, namespace, c.Name))
+	return nil
+}
+
+// equivalent returns true if the two services are equivalent.  Fields which are expected to differ between
+// federated services and the underlying cluster services (e.g. ClusterIP, NodePort) are ignored.
+func equivalent(federationService, clusterService v1.Service) bool {
+	clusterService.Spec.ClusterIP = federationService.Spec.ClusterIP
+	for i := range clusterService.Spec.Ports {
+		clusterService.Spec.Ports[i].NodePort = federationService.Spec.Ports[i].NodePort
+	}
+
+	if federationService.Name != clusterService.Name || federationService.Namespace != clusterService.Namespace {
+		return false
+	}
+	if !reflect.DeepEqual(federationService.Labels, clusterService.Labels) && (len(federationService.Labels) != 0 || len(clusterService.Labels) != 0) {
+		return false
+	}
+	if !reflect.DeepEqual(federationService.Spec, clusterService.Spec) {
+		return false
+	}
+	return true
+}

vendor/k8s.io/kubernetes/test/e2e_federation/upgrade.go

@@ -0,0 +1,139 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_federation
+
+import (
+	"k8s.io/kubernetes/test/e2e/chaosmonkey"
+	"k8s.io/kubernetes/test/e2e/framework"
+	fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
+	"k8s.io/kubernetes/test/e2e_federation/upgrades"
+
+	. "github.com/onsi/ginkgo"
+)
+
+var upgradeTests = upgrades.SimpleUpgradeTests()
+
+var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
+	f := fedframework.NewDefaultFederatedFramework("federation-upgrade")
+
+	framework.KubeDescribe("Federation Control Plane upgrade", func() {
+		It("should maintain a functioning federation [Feature:FCPUpgrade]", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			cm := chaosmonkey.New(func() {
+				federationControlPlaneUpgrade(f)
+			})
+			for _, t := range upgradeTests {
+				cm.RegisterInterface(&chaosMonkeyAdapter{
+					test:        t,
+					framework:   f,
+					upgradeType: upgrades.FCPUpgrade,
+				})
+			}
+			cm.Do()
+		})
+	})
+
+	framework.KubeDescribe("Federated clusters upgrade", func() {
+		It("should maintain a functioning federation [Feature:FederatedClustersUpgrade]", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			cm := chaosmonkey.New(func() {
+				federatedClustersUpgrade(f)
+			})
+			for _, t := range upgradeTests {
+				cm.RegisterInterface(&chaosMonkeyAdapter{
+					test:        t,
+					framework:   f,
+					upgradeType: upgrades.FederatedClustersUpgrade,
+				})
+			}
+			cm.Do()
+		})
+	})
+
+	framework.KubeDescribe("FCP upgrade followed by federated clusters upgrade", func() {
+		It("should maintain a functioning federation [Feature:FCPUpgradeFollowedByFederatedClustersUpgrade]", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			cm := chaosmonkey.New(func() {
+				federationControlPlaneUpgrade(f)
+				federatedClustersUpgrade(f)
+			})
+			for _, t := range upgradeTests {
+				cm.RegisterInterface(&chaosMonkeyAdapter{
+					test:        t,
+					framework:   f,
+					upgradeType: upgrades.FCPUpgradeFollowedByFederatedClustersUpgrade,
+				})
+			}
+			cm.Do()
+		})
+	})
+
+	framework.KubeDescribe("Federated clusters upgrade followed by FCP upgrade", func() {
+		It("should maintain a functioning federation [Feature:FederatedClustersUpgradeFollowedByFCPUpgrade]", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			cm := chaosmonkey.New(func() {
+				federatedClustersUpgrade(f)
+				federationControlPlaneUpgrade(f)
+			})
+			for _, t := range upgradeTests {
+				cm.RegisterInterface(&chaosMonkeyAdapter{
+					test:        t,
+					framework:   f,
+					upgradeType: upgrades.FederatedClustersUpgradeFollowedByFCPUpgrade,
+				})
+			}
+			cm.Do()
+		})
+	})
+})
+
+type chaosMonkeyAdapter struct {
+	test        upgrades.Test
+	framework   *fedframework.Framework
+	upgradeType upgrades.FederationUpgradeType
+}
+
+func (cma *chaosMonkeyAdapter) Setup() {
+	cma.test.Setup(cma.framework)
+}
+
+func (cma *chaosMonkeyAdapter) Test(stopCh <-chan struct{}) {
+	cma.test.Test(cma.framework, stopCh, cma.upgradeType)
+}
+
+func (cma *chaosMonkeyAdapter) Teardown() {
+	cma.test.Teardown(cma.framework)
+}
+
+func federationControlPlaneUpgrade(f *fedframework.Framework) {
+	federationVersion, err := framework.RealVersion(framework.TestContext.FederationUpgradeTarget)
+	framework.ExpectNoError(err)
+	framework.ExpectNoError(fedframework.FederationControlPlaneUpgrade(federationVersion))
+	framework.ExpectNoError(fedframework.CheckFederationVersion(f.FederationClientset, federationVersion))
+}
+
+func federatedClustersUpgrade(f *fedframework.Framework) {
+	k8sVersion, err := framework.RealVersion(framework.TestContext.UpgradeTarget)
+	framework.ExpectNoError(err)
+	clusters := f.GetRegisteredClusters()
+	for _, cluster := range clusters {
+		framework.ExpectNoError(fedframework.MasterUpgrade(cluster.Name, k8sVersion))
+		framework.ExpectNoError(framework.CheckMasterVersion(cluster.Clientset, k8sVersion))
+
+		// TODO: Need to add Node upgrade. Add once this framework is stable
+	}
+}

vendor/k8s.io/kubernetes/test/e2e_federation/upgrades/BUILD

@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "simple.go",
+        "upgrade.go",
+    ],
+    deps = [
+        "//federation/pkg/federatedtypes:go_default_library",
+        "//federation/pkg/federatedtypes/crudtester:go_default_library",
+        "//test/e2e_federation/framework:go_default_library",
+        "//vendor/github.com/onsi/ginkgo:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/test/e2e_federation/upgrades/simple.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package upgrades
+
+import (
+	"fmt"
+
+	pkgruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/kubernetes/federation/pkg/federatedtypes"
+	crudtester "k8s.io/kubernetes/federation/pkg/federatedtypes/crudtester"
+	fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
+
+	. "github.com/onsi/ginkgo"
+)
+
+// SimpleUpgradeTest validates that a federated resource remains
+// propagated before and after a control plane upgrade
+type SimpleUpgradeTest struct {
+	kind           string
+	adapterFactory federatedtypes.AdapterFactory
+	crudTester     *crudtester.FederatedTypeCRUDTester
+	obj            pkgruntime.Object
+}
+
+// Setup creates a resource and validates its propagation to member clusters
+func (ut *SimpleUpgradeTest) Setup(f *fedframework.Framework) {
+	adapter := ut.adapterFactory(f.FederationClientset, f.FederationConfig, nil)
+	clients := f.GetClusterClients()
+	ut.crudTester = fedframework.NewFederatedTypeCRUDTester(adapter, clients)
+
+	By(fmt.Sprintf("Creating a resource of kind %q and validating propagation to member clusters", ut.kind))
+	obj := adapter.NewTestObject(f.Namespace.Name)
+	ut.obj = ut.crudTester.CheckCreate(obj)
+}
+
+// Test validates that a resource remains propagated post-upgrade
+func (ut *SimpleUpgradeTest) Test(f *fedframework.Framework, done <-chan struct{}, upgrade FederationUpgradeType) {
+	<-done
+	By(fmt.Sprintf("Validating that a resource of kind %q remains propagated to member clusters after upgrade", ut.kind))
+	ut.crudTester.CheckPropagation(ut.obj)
+}
+
+// Teardown cleans up remaining resources
+func (ut *SimpleUpgradeTest) Teardown(f *fedframework.Framework) {
+	// Rely on the namespace deletion to clean up everything
+}
+
+// SimpleUpgradeTests collects simple upgrade tests for registered federated types
+func SimpleUpgradeTests() []Test {
+	tests := []Test{}
+	for kind, fedType := range federatedtypes.FederatedTypes() {
+		tests = append(tests, &SimpleUpgradeTest{
+			kind:           kind,
+			adapterFactory: fedType.AdapterFactory,
+		})
+	}
+	return tests
+}

vendor/k8s.io/kubernetes/test/e2e_federation/upgrades/upgrade.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package upgrades provides a framework for testing Kubernetes federation
+// features before, during, and after different types of upgrades.
+package upgrades
+
+import fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
+
+// FederationUpgradeType represents different types of federation upgrades.
+type FederationUpgradeType int
+
+const (
+	// FCPUpgrade indicates that federation control plane is being upgraded.
+	FCPUpgrade FederationUpgradeType = iota
+
+	// FederatedClustersUpgrade indicates that federated clusters are being upgraded.
+	FederatedClustersUpgrade
+
+	// FCPUpgradeFollowedByFederatedClustersUpgrade indicates that federation control plane is upgraded
+	// followed by federated clusters upgrade.
+	FCPUpgradeFollowedByFederatedClustersUpgrade
+
+	// FederatedClustersUpgradeFollowedByFCPUpgrade indicates that federated clusters are upgraded
+	// followed by federation control plane upgrade.
+	FederatedClustersUpgradeFollowedByFCPUpgrade
+)
+
+// Test is an interface for federation upgrade tests.
+type Test interface {
+	// Setup should create and verify whatever objects need to
+	// exist before the upgrade disruption starts.
+	Setup(f *fedframework.Framework)
+
+	// Test will run during the upgrade. When the upgrade is
+	// complete, done will be closed and final validation can
+	// begin.
+	Test(f *fedframework.Framework, done <-chan struct{}, upgrade FederationUpgradeType)
+
+	// TearDown should clean up any objects that are created that
+	// aren't already cleaned up by the framework.
+	Teardown(f *fedframework.Framework)
+}

vendor/k8s.io/kubernetes/test/e2e_federation/util.go

@@ -0,0 +1,513 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_federation
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/rand"
+	"k8s.io/apimachinery/pkg/util/wait"
+	kubeclientset "k8s.io/client-go/kubernetes"
+	federationapi "k8s.io/kubernetes/federation/apis/federation/v1beta1"
+	fedclientset "k8s.io/kubernetes/federation/client/clientset_generated/federation_clientset"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+	"k8s.io/kubernetes/test/e2e/common"
+	"k8s.io/kubernetes/test/e2e/framework"
+	fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+)
+
+var (
+	DefaultFederationName = "e2e-federation"
+	// We use this to decide how long to wait for our DNS probes to succeed.
+	DNSTTL = 180 * time.Second // TODO: make k8s.io/kubernetes/federation/pkg/federation-controller/service.minDnsTtl exported, and import it here.
+)
+
+const (
+	// [30000, 32767] is the allowed default service nodeport range and our
+	// tests just use the defaults.
+	FederatedSvcNodePortFirst = 30000
+	FederatedSvcNodePortLast  = 32767
+)
+
+var FederationSuite common.Suite
+
+func createClusterObjectOrFail(f *fedframework.Framework, context *fedframework.E2EContext, clusterNamePrefix string) {
+	clusterName := clusterNamePrefix + context.Name
+	framework.Logf("Creating cluster object: %s (%s, secret: %s)", clusterName, context.Cluster.Cluster.Server, context.Name)
+	cluster := federationapi.Cluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: clusterName,
+		},
+		Spec: federationapi.ClusterSpec{
+			ServerAddressByClientCIDRs: []federationapi.ServerAddressByClientCIDR{
+				{
+					ClientCIDR:    "0.0.0.0/0",
+					ServerAddress: context.Cluster.Cluster.Server,
+				},
+			},
+			SecretRef: &v1.LocalObjectReference{
+				// Note: Name must correlate with federation build script secret name,
+				//       which currently matches the cluster name.
+				//       See federation/cluster/common.sh:132
+				Name: context.Name,
+			},
+		},
+	}
+	if clusterNamePrefix != "" {
+		cluster.Labels = map[string]string{"prefix": clusterNamePrefix}
+	}
+	_, err := f.FederationClientset.Federation().Clusters().Create(&cluster)
+	framework.ExpectNoError(err, fmt.Sprintf("creating cluster: %+v", err))
+	framework.Logf("Successfully created cluster object: %s (%s, secret: %s)", clusterName, context.Cluster.Cluster.Server, context.Name)
+}
+
+// waitForServiceOrFail waits until a service is either present or absent in the cluster specified by clientset.
+// If the condition is not met within timout, it fails the calling test.
+func waitForServiceOrFail(clientset *kubeclientset.Clientset, namespace string, service *v1.Service, present bool, timeout time.Duration) {
+	By(fmt.Sprintf("Fetching a federated service shard of service %q in namespace %q from cluster", service.Name, namespace))
+	err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
+		clusterService, err := clientset.CoreV1().Services(namespace).Get(service.Name, metav1.GetOptions{})
+		if (!present) && errors.IsNotFound(err) { // We want it gone, and it's gone.
+			By(fmt.Sprintf("Success: shard of federated service %q in namespace %q in cluster is absent", service.Name, namespace))
+			return true, nil // Success
+		}
+		if present && err == nil { // We want it present, and the Get succeeded, so we're all good.
+			if equivalent(*clusterService, *service) {
+				By(fmt.Sprintf("Success: shard of federated service %q in namespace %q in cluster is present", service.Name, namespace))
+				return true, nil // Success
+			}
+			return false, nil
+		}
+		By(fmt.Sprintf("Service %q in namespace %q in cluster.  Found: %v, waiting for Found: %v, trying again in %s (err=%v)", service.Name, namespace, clusterService != nil && err == nil, present, framework.Poll, err))
+		return false, nil
+	})
+	framework.ExpectNoError(err, "Failed to verify service %q in namespace %q in cluster: Present=%v", service.Name, namespace, present)
+}
+
+// waitForServiceShardsOrFail waits for the service to appear in all clusters
+func waitForServiceShardsOrFail(namespace string, service *v1.Service, clusters fedframework.ClusterSlice) {
+	framework.Logf("Waiting for service %q in %d clusters", service.Name, len(clusters))
+	for _, c := range clusters {
+		waitForServiceOrFail(c.Clientset, namespace, service, true, fedframework.FederatedDefaultTestTimeout)
+	}
+}
+
+func createService(clientset *fedclientset.Clientset, namespace, name string) (*v1.Service, error) {
+	if clientset == nil || len(namespace) == 0 {
+		return nil, fmt.Errorf("Internal error: invalid parameters passed to createService: clientset: %v, namespace: %v", clientset, namespace)
+	}
+	By(fmt.Sprintf("Creating federated service %q in namespace %q", name, namespace))
+
+	service := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: v1.ServiceSpec{
+			Selector: FederatedServiceLabels,
+			Type:     v1.ServiceTypeClusterIP,
+			Ports: []v1.ServicePort{
+				{
+					Name:       "http",
+					Protocol:   v1.ProtocolTCP,
+					Port:       80,
+					TargetPort: intstr.FromInt(8080),
+				},
+			},
+			SessionAffinity: v1.ServiceAffinityNone,
+		},
+	}
+
+	By(fmt.Sprintf("Trying to create service %q in namespace %q", service.Name, namespace))
+	return clientset.CoreV1().Services(namespace).Create(service)
+}
+
+func createLBService(clientset *fedclientset.Clientset, namespace, name string, clusters fedframework.ClusterSlice) (*v1.Service, error) {
+	if clientset == nil || len(namespace) == 0 {
+		return nil, fmt.Errorf("Internal error: invalid parameters passed to createService: clientset: %v, namespace: %v", clientset, namespace)
+	}
+	By(fmt.Sprintf("Creating federated service (type: load balancer) %q in namespace %q", name, namespace))
+
+	// Tests can be run in parallel, so we need a different nodePort for
+	// each test.
+	// we add in a array all the "available" ports
+	availablePorts := make([]int32, FederatedSvcNodePortLast-FederatedSvcNodePortFirst)
+	for i := range availablePorts {
+		availablePorts[i] = int32(FederatedSvcNodePortFirst + i)
+	}
+
+	var err error
+	var service *v1.Service
+	retry := 10 // the function should retry the service creation on different port only 10 time.
+
+	// until the availablePort list is not empty, lets try to create the service
+	for len(availablePorts) > 0 && retry > 0 {
+		// select the Id of an available port
+		i := rand.Intn(len(availablePorts))
+
+		By(fmt.Sprintf("try creating federated service %q in namespace %q with nodePort %d", name, namespace, availablePorts[i]))
+
+		service, err = createServiceWithNodePort(clientset, namespace, name, availablePorts[i])
+		if err == nil {
+			// check if service have been created properly in all clusters.
+			// if the service is not present in one of the clusters, we should cleanup all services
+			if err = checkServicesCreation(namespace, name, clusters); err == nil {
+				// everything was created properly so returns the federated service.
+				return service, nil
+			}
+		}
+
+		// in case of error, cleanup everything
+		if service != nil {
+			if err = deleteService(clientset, namespace, name, nil); err != nil {
+				framework.ExpectNoError(err, "Deleting service %q after a partial createService() error", service.Name)
+				return nil, err
+			}
+
+			cleanupServiceShardsAndProviderResources(namespace, service, clusters)
+		}
+
+		// creation failed, lets try with another port
+		// first remove from the availablePorts the port with which the creation failed
+		availablePorts = append(availablePorts[:i], availablePorts[i+1:]...)
+		retry--
+	}
+
+	return nil, err
+}
+
+func createServiceWithNodePort(clientset *fedclientset.Clientset, namespace, name string, nodePort int32) (*v1.Service, error) {
+	service := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: v1.ServiceSpec{
+			Selector: FederatedServiceLabels,
+			Type:     v1.ServiceTypeLoadBalancer,
+			Ports: []v1.ServicePort{
+				{
+					Name:       "http",
+					Protocol:   v1.ProtocolTCP,
+					Port:       80,
+					TargetPort: intstr.FromInt(8080),
+					NodePort:   nodePort,
+				},
+			},
+			SessionAffinity: v1.ServiceAffinityNone,
+		},
+	}
+
+	By(fmt.Sprintf("Trying to create service %q in namespace %q", service.Name, namespace))
+	return clientset.CoreV1().Services(namespace).Create(service)
+}
+
+// checkServicesCreation checks if the service have been created successfuly in all the clusters.
+// if the service is not present in at least one of the clusters, this function returns an error.
+func checkServicesCreation(namespace, serviceName string, clusters fedframework.ClusterSlice) error {
+	framework.Logf("check if service %q have been created in %d clusters", serviceName, len(clusters))
+	for _, cluster := range clusters {
+		name := cluster.Name
+		err := wait.PollImmediate(framework.Poll, fedframework.FederatedDefaultTestTimeout, func() (bool, error) {
+			var err error
+			_, err = cluster.Clientset.CoreV1().Services(namespace).Get(serviceName, metav1.GetOptions{})
+			if err != nil && !errors.IsNotFound(err) {
+				// Get failed with an error, try again.
+				framework.Logf("Failed to find service %q in namespace %q, in cluster %q: %v. Trying again in %s", serviceName, namespace, name, err, framework.Poll)
+				return false, err
+			} else if errors.IsNotFound(err) {
+				framework.Logf("Service %q in namespace %q in cluster %q not found. Trying again in %s", serviceName, namespace, name, framework.Poll)
+				return false, nil
+			}
+			By(fmt.Sprintf("Service %q in namespace %q in cluster %q found", serviceName, namespace, name))
+			return true, nil
+		})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func createServiceOrFail(clientset *fedclientset.Clientset, namespace, name string) *v1.Service {
+	service, err := createService(clientset, namespace, name)
+	framework.ExpectNoError(err, "Creating service %q in namespace %q", service.Name, namespace)
+	By(fmt.Sprintf("Successfully created federated service %q in namespace %q", name, namespace))
+	return service
+}
+
+func createLBServiceOrFail(clientset *fedclientset.Clientset, namespace, name string, clusters fedframework.ClusterSlice) *v1.Service {
+	service, err := createLBService(clientset, namespace, name, clusters)
+	framework.ExpectNoError(err, "Creating service %q in namespace %q", service.Name, namespace)
+	By(fmt.Sprintf("Successfully created federated service (type: load balancer) %q in namespace %q", name, namespace))
+	return service
+}
+
+func deleteServiceOrFail(clientset *fedclientset.Clientset, namespace string, serviceName string, orphanDependents *bool) {
+	if clientset == nil || len(namespace) == 0 || len(serviceName) == 0 {
+		Fail(fmt.Sprintf("Internal error: invalid parameters passed to deleteServiceOrFail: clientset: %v, namespace: %v, service: %v", clientset, namespace, serviceName))
+	}
+	framework.Logf("Deleting service %q in namespace %v", serviceName, namespace)
+
+	err := deleteService(clientset, namespace, serviceName, orphanDependents)
+	if err != nil {
+		framework.ExpectNoError(err, "Error deleting service %q from namespace %q", serviceName, namespace)
+	}
+}
+
+func deleteService(clientset *fedclientset.Clientset, namespace string, serviceName string, orphanDependents *bool) error {
+	err := clientset.CoreV1().Services(namespace).Delete(serviceName, &metav1.DeleteOptions{OrphanDependents: orphanDependents})
+	if err != nil {
+		return err
+	}
+	// Wait for the service to be deleted.
+	err = wait.Poll(5*time.Second, fedframework.FederatedDefaultTestTimeout, func() (bool, error) {
+		_, err := clientset.Core().Services(namespace).Get(serviceName, metav1.GetOptions{})
+		if err != nil && errors.IsNotFound(err) {
+			return true, nil
+		}
+		return false, err
+	})
+	return err
+}
+
+func cleanupServiceShardsAndProviderResources(namespace string, service *v1.Service, clusters fedframework.ClusterSlice) {
+	framework.Logf("Deleting service %q in %d clusters", service.Name, len(clusters))
+	for _, c := range clusters {
+		name := c.Name
+		var cSvc *v1.Service
+
+		err := wait.PollImmediate(framework.Poll, fedframework.FederatedDefaultTestTimeout, func() (bool, error) {
+			var err error
+			cSvc, err = c.Clientset.CoreV1().Services(namespace).Get(service.Name, metav1.GetOptions{})
+			if err != nil && !errors.IsNotFound(err) {
+				// Get failed with an error, try again.
+				framework.Logf("Failed to find service %q in namespace %q, in cluster %q: %v. Trying again in %s", service.Name, namespace, name, err, framework.Poll)
+				return false, nil
+			} else if errors.IsNotFound(err) {
+				cSvc = nil
+				By(fmt.Sprintf("Service %q in namespace %q in cluster %q not found", service.Name, namespace, name))
+				return true, err
+			}
+			By(fmt.Sprintf("Service %q in namespace %q in cluster %q found", service.Name, namespace, name))
+			return true, err
+		})
+
+		if err != nil || cSvc == nil {
+			By(fmt.Sprintf("Failed to find service %q in namespace %q, in cluster %q in %s", service.Name, namespace, name, fedframework.FederatedDefaultTestTimeout))
+			continue
+		}
+
+		if cSvc.Spec.Type == v1.ServiceTypeLoadBalancer {
+			// In federation tests, e2e zone names are used to derive federation member cluster names
+			zone := fedframework.GetZoneFromClusterName(name)
+			serviceLBName := cloudprovider.GetLoadBalancerName(cSvc)
+			framework.Logf("cleaning cloud provider resource for service %q in namespace %q, in cluster %q", service.Name, namespace, name)
+			framework.CleanupServiceResources(c.Clientset, serviceLBName, zone)
+		}
+
+		err = cleanupServiceShard(c.Clientset, name, namespace, cSvc, fedframework.FederatedDefaultTestTimeout)
+		if err != nil {
+			framework.Logf("Failed to delete service %q in namespace %q, in cluster %q: %v", service.Name, namespace, name, err)
+		}
+	}
+}
+
+func cleanupServiceShard(clientset *kubeclientset.Clientset, clusterName, namespace string, service *v1.Service, timeout time.Duration) error {
+	err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
+		err := clientset.CoreV1().Services(namespace).Delete(service.Name, &metav1.DeleteOptions{})
+		if err != nil && !errors.IsNotFound(err) {
+			// Deletion failed with an error, try again.
+			framework.Logf("Failed to delete service %q in namespace %q, in cluster %q", service.Name, namespace, clusterName)
+			return false, nil
+		}
+		By(fmt.Sprintf("Service %q in namespace %q in cluster %q deleted", service.Name, namespace, clusterName))
+		return true, nil
+	})
+	return err
+}
+
+func podExitCodeDetector(f *fedframework.Framework, name, namespace string, code int32) func() error {
+	// If we ever get any container logs, stash them here.
+	logs := ""
+
+	logerr := func(err error) error {
+		if err == nil {
+			return nil
+		}
+		if logs == "" {
+			return err
+		}
+		return fmt.Errorf("%s (%v)", logs, err)
+	}
+
+	return func() error {
+		pod, err := f.ClientSet.Core().Pods(namespace).Get(name, metav1.GetOptions{})
+		if err != nil {
+			return logerr(err)
+		}
+		if len(pod.Status.ContainerStatuses) < 1 {
+			return logerr(fmt.Errorf("no container statuses"))
+		}
+
+		// Best effort attempt to grab pod logs for debugging
+		logs, err = framework.GetPodLogs(f.ClientSet, namespace, name, pod.Spec.Containers[0].Name)
+		if err != nil {
+			framework.Logf("Cannot fetch pod logs: %v", err)
+		}
+
+		status := pod.Status.ContainerStatuses[0]
+		if status.State.Terminated == nil {
+			return logerr(fmt.Errorf("container is not in terminated state"))
+		}
+		if status.State.Terminated.ExitCode == code {
+			return nil
+		}
+
+		return logerr(fmt.Errorf("exited %d", status.State.Terminated.ExitCode))
+	}
+}
+
+func discoverService(f *fedframework.Framework, name string, exists bool, podName string) {
+	command := []string{"sh", "-c", fmt.Sprintf("until nslookup '%s'; do sleep 10; done", name)}
+	By(fmt.Sprintf("Looking up %q", name))
+
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: podName,
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:    "federated-service-discovery-container",
+					Image:   imageutils.GetBusyBoxImage(),
+					Command: command,
+				},
+			},
+			RestartPolicy: v1.RestartPolicyOnFailure,
+		},
+	}
+
+	nsName := f.FederationNamespace.Name
+	By(fmt.Sprintf("Creating pod %q in namespace %q", pod.Name, nsName))
+	_, err := f.ClientSet.Core().Pods(nsName).Create(pod)
+	framework.ExpectNoError(err, "Trying to create pod to run %q", command)
+	By(fmt.Sprintf("Successfully created pod %q in namespace %q", pod.Name, nsName))
+	defer func() {
+		By(fmt.Sprintf("Deleting pod %q from namespace %q", podName, nsName))
+		err := f.ClientSet.Core().Pods(nsName).Delete(podName, metav1.NewDeleteOptions(0))
+		framework.ExpectNoError(err, "Deleting pod %q from namespace %q", podName, nsName)
+		By(fmt.Sprintf("Deleted pod %q from namespace %q", podName, nsName))
+	}()
+
+	if exists {
+		// TODO(mml): Eventually check the IP address is correct, too.
+		Eventually(podExitCodeDetector(f, podName, nsName, 0), 3*DNSTTL, time.Second*2).
+			Should(BeNil(), "%q should exit 0, but it never did", command)
+	} else {
+		Eventually(podExitCodeDetector(f, podName, nsName, 0), 3*DNSTTL, time.Second*2).
+			ShouldNot(BeNil(), "%q should eventually not exit 0, but it always did", command)
+	}
+}
+
+// BackendPodMap maps a cluster name to a backend pod created in that cluster
+type BackendPodMap map[string]*v1.Pod
+
+// createBackendPodsOrFail creates one pod in each cluster, and returns the created pods.  If creation of any pod fails,
+// the test fails (possibly with a partially created set of pods). No retries are attempted.
+func createBackendPodsOrFail(clusters fedframework.ClusterSlice, namespace string, name string) BackendPodMap {
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+			// Namespace: namespace,
+			Labels: FederatedServiceLabels,
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:  name,
+					Image: "gcr.io/google_containers/echoserver:1.6",
+				},
+			},
+			RestartPolicy: v1.RestartPolicyAlways,
+		},
+	}
+	podMap := make(BackendPodMap)
+	for _, c := range clusters {
+		name := c.Name
+		By(fmt.Sprintf("Creating pod %q in namespace %q in cluster %q", pod.Name, namespace, name))
+		createdPod, err := c.Clientset.CoreV1().Pods(namespace).Create(pod)
+		framework.ExpectNoError(err, "Creating pod %q in namespace %q in cluster %q", name, namespace, name)
+		By(fmt.Sprintf("Successfully created pod %q in namespace %q in cluster %q: %v", pod.Name, namespace, name, *createdPod))
+		podMap[name] = createdPod
+	}
+	return podMap
+}
+
+// deleteOneBackendPodOrFail deletes exactly one backend pod which must not be nil
+// The test fails if there are any errors.
+func deleteOneBackendPodOrFail(c *fedframework.Cluster, pod *v1.Pod) {
+	Expect(pod).ToNot(BeNil())
+	err := c.Clientset.CoreV1().Pods(pod.Namespace).Delete(pod.Name, metav1.NewDeleteOptions(0))
+	msgFmt := fmt.Sprintf("Deleting Pod %q in namespace %q in cluster %q %%v", pod.Name, pod.Namespace, c.Name)
+	if errors.IsNotFound(err) {
+		framework.Logf(msgFmt, "does not exist. No need to delete it.")
+		return
+	}
+	framework.ExpectNoError(err, msgFmt, "")
+	framework.Logf(msgFmt, "was deleted")
+}
+
+// deleteBackendPodsOrFail deletes one pod from each cluster that has one.
+// If deletion of any pod fails, the test fails (possibly with a partially deleted set of pods). No retries are attempted.
+func deleteBackendPodsOrFail(clusters fedframework.ClusterSlice, backendPods BackendPodMap) {
+	if backendPods == nil {
+		return
+	}
+	for _, c := range clusters {
+		if pod, ok := backendPods[c.Name]; ok {
+			deleteOneBackendPodOrFail(c, pod)
+		} else {
+			By(fmt.Sprintf("No backend pod to delete for cluster %q", c.Name))
+		}
+	}
+}
+
+// waitForReplicatSetToBeDeletedOrFail waits for the named ReplicaSet in namespace to be deleted.
+// If the deletion fails, the enclosing test fails.
+func waitForReplicaSetToBeDeletedOrFail(clientset *fedclientset.Clientset, namespace string, replicaSet string) {
+	err := wait.Poll(5*time.Second, fedframework.FederatedDefaultTestTimeout, func() (bool, error) {
+		_, err := clientset.Extensions().ReplicaSets(namespace).Get(replicaSet, metav1.GetOptions{})
+		if err != nil && errors.IsNotFound(err) {
+			return true, nil
+		}
+		return false, err
+	})
+
+	if err != nil {
+		framework.Failf("Error in deleting replica set %s: %v", replicaSet, err)
+	}
+}