DevFW-CICD/ingress-nginx-helm
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor e2e tests to use the service ClusterIP

Browse source
This commit is contained in:
Manuel Alejandro de Brito Fontes 2019-02-22 11:03:42 -03:00
parent f04361cc06
commit 5e249d3366
No known key found for this signature in database
GPG key ID: 786136016A8BA02A
80 changed files with 777 additions and 706 deletions
Download patch file Download diff file Expand all files Collapse all files

6
test/e2e/framework/deployment.go
View file

@ -71,7 +71,7 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32
deployment := &extensions.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: f.IngressController.Namespace,
Namespace: f.Namespace,
},
Spec: extensions.DeploymentSpec{
Replicas: NewInt32(replicas),
@ -115,7 +115,7 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32
service := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: f.IngressController.Namespace,
Namespace: f.Namespace,
},
Spec: corev1.ServiceSpec{
Ports: []corev1.ServicePort{
@ -135,6 +135,6 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32
s := f.EnsureService(service)
Expect(s).NotTo(BeNil(), "expected a service but none returned")
err = WaitForEndpoints(f.KubeClientSet, DefaultTimeout, name, f.IngressController.Namespace, int(replicas))
err = WaitForEndpoints(f.KubeClientSet, DefaultTimeout, name, f.Namespace, int(replicas))
Expect(err).NotTo(HaveOccurred(), "failed to wait for endpoints to become ready")
}

2
test/e2e/framework/exec.go
View file

@ -30,7 +30,7 @@ import (
// ExecIngressPod executes a command inside the first container in ingress controller running pod
func (f *Framework) ExecIngressPod(command string) (string, error) {
pod, err := getIngressNGINXPod(f.IngressController.Namespace, f.KubeClientSet)
pod, err := getIngressNGINXPod(f.Namespace, f.KubeClientSet)
if err != nil {
return "", err
}

86
test/e2e/framework/framework.go
View file

@ -15,7 +15,6 @@ package framework
import (
"fmt"
"os"
"strings"
"time"
@ -65,13 +64,6 @@ type Framework struct {
// should abort, the AfterSuite hook should run all Cleanup actions.
cleanupHandle CleanupActionHandle
IngressController *ingressController
}
type ingressController struct {
HTTPURL string
HTTPSURL string
Namespace string
}
@ -93,7 +85,11 @@ func (f *Framework) BeforeEach() {
f.cleanupHandle = AddCleanupAction(f.AfterEach)
By("Creating a kubernetes client")
kubeConfig, err := LoadConfig(TestContext.KubeConfig, TestContext.KubeContext)
kubeConfig, err := restclient.InClusterConfig()
if err != nil {
panic(err.Error())
}
Expect(err).NotTo(HaveOccurred())
f.KubeConfig = kubeConfig
@ -104,25 +100,17 @@ func (f *Framework) BeforeEach() {
ingressNamespace, err := CreateKubeNamespace(f.BaseName, f.KubeClientSet)
Expect(err).NotTo(HaveOccurred())
f.IngressController = &ingressController{
Namespace: ingressNamespace,
}
f.Namespace = ingressNamespace
By("Starting new ingress controller")
err = f.NewIngressController(f.IngressController.Namespace)
err = f.NewIngressController(f.Namespace)
Expect(err).NotTo(HaveOccurred())
err = WaitForPodsReady(f.KubeClientSet, DefaultTimeout, 1, f.IngressController.Namespace, metav1.ListOptions{
err = WaitForPodsReady(f.KubeClientSet, DefaultTimeout, 1, f.Namespace, metav1.ListOptions{
LabelSelector: "app.kubernetes.io/name=ingress-nginx",
})
Expect(err).NotTo(HaveOccurred())
HTTPURL := f.GetNginxURL(HTTP)
f.IngressController.HTTPURL = HTTPURL
HTTPSURL := f.GetNginxURL(HTTPS)
f.IngressController.HTTPSURL = HTTPSURL
// we wait for any change in the informers and SSL certificate generation
time.Sleep(5 * time.Second)
}
@ -132,7 +120,7 @@ func (f *Framework) AfterEach() {
RemoveCleanupAction(f.cleanupHandle)
By("Waiting for test namespace to no longer exist")
err := DeleteKubeNamespace(f.KubeClientSet, f.IngressController.Namespace)
err := DeleteKubeNamespace(f.KubeClientSet, f.Namespace)
Expect(err).NotTo(HaveOccurred())
if CurrentGinkgoTestDescription().Failed {
@ -148,52 +136,34 @@ func IngressNginxDescribe(text string, body func()) bool {
return Describe("[nginx-ingress] "+text, body)
}
// GetNginxIP returns the IP address of the minikube cluster
// where the NGINX ingress controller is running
// GetNginxIP returns the number of TCP port where NGINX is running
func (f *Framework) GetNginxIP() string {
nodeIP := os.Getenv("NODE_IP")
Expect(nodeIP).NotTo(BeEmpty(), "env variable NODE_IP is empty")
return nodeIP
}
// GetNginxPort returns the number of TCP port where NGINX is running
func (f *Framework) GetNginxPort(name string) (int, error) {
s, err := f.KubeClientSet.
CoreV1().
Services(f.IngressController.Namespace).
Services(f.Namespace).
Get("ingress-nginx", metav1.GetOptions{})
if err != nil {
return -1, err
}
for _, p := range s.Spec.Ports {
if p.NodePort != 0 && p.Name == name {
return int(p.NodePort), nil
}
}
return -1, err
Expect(err).NotTo(HaveOccurred(), "unexpected error obtaning NGINX IP address")
return s.Spec.ClusterIP
}
// GetNginxURL returns the URL should be used to make a request to NGINX
func (f *Framework) GetNginxURL(scheme RequestScheme) string {
// GetURL returns the URL should be used to make a request to NGINX
func (f *Framework) GetURL(scheme RequestScheme) string {
ip := f.GetNginxIP()
port, err := f.GetNginxPort(fmt.Sprintf("%v", scheme))
Expect(err).NotTo(HaveOccurred(), "unexpected error obtaning NGINX Port")
return fmt.Sprintf("%v://%v:%v", scheme, ip, port)
return fmt.Sprintf("%v://%v", scheme, ip)
}
// WaitForNginxServer waits until the nginx configuration contains a particular server section
func (f *Framework) WaitForNginxServer(name string, matcher func(cfg string) bool) {
err := wait.Poll(Poll, time.Minute*5, f.matchNginxConditions(name, matcher))
err := wait.Poll(Poll, DefaultTimeout, f.matchNginxConditions(name, matcher))
Expect(err).NotTo(HaveOccurred(), "unexpected error waiting for nginx server condition/s")
time.Sleep(5 * time.Second)
}
// WaitForNginxConfiguration waits until the nginx configuration contains a particular configuration
func (f *Framework) WaitForNginxConfiguration(matcher func(cfg string) bool) {
err := wait.Poll(Poll, time.Minute*5, f.matchNginxConditions("", matcher))
err := wait.Poll(Poll, DefaultTimeout, f.matchNginxConditions("", matcher))
Expect(err).NotTo(HaveOccurred(), "unexpected error waiting for nginx server condition/s")
time.Sleep(5 * time.Second)
}
func nginxLogs(client kubernetes.Interface, namespace string) (string, error) {
@ -211,14 +181,14 @@ func nginxLogs(client kubernetes.Interface, namespace string) (string, error) {
// NginxLogs returns the logs of the nginx ingress controller pod running
func (f *Framework) NginxLogs() (string, error) {
return nginxLogs(f.KubeClientSet, f.IngressController.Namespace)
return nginxLogs(f.KubeClientSet, f.Namespace)
}
func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) bool) wait.ConditionFunc {
return func() (bool, error) {
pod, err := getIngressNGINXPod(f.IngressController.Namespace, f.KubeClientSet)
pod, err := getIngressNGINXPod(f.Namespace, f.KubeClientSet)
if err != nil {
return false, err
return false, nil
}
var cmd string
@ -230,7 +200,7 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b
o, err := f.ExecCommand(pod, cmd)
if err != nil {
return false, err
return false, nil
}
var match bool
@ -264,7 +234,7 @@ func (f *Framework) getNginxConfigMap() (*v1.ConfigMap, error) {
config, err := f.KubeClientSet.
CoreV1().
ConfigMaps(f.IngressController.Namespace).
ConfigMaps(f.Namespace).
Get("nginx-configuration", metav1.GetOptions{})
if err != nil {
return nil, err
@ -298,7 +268,7 @@ func (f *Framework) SetNginxConfigMapData(cmData map[string]string) {
_, err = f.KubeClientSet.
CoreV1().
ConfigMaps(f.IngressController.Namespace).
ConfigMaps(f.Namespace).
Update(config)
Expect(err).NotTo(HaveOccurred())
@ -318,14 +288,14 @@ func (f *Framework) UpdateNginxConfigMapData(key string, value string) {
// DeleteNGINXPod deletes the currently running pod. It waits for the replacement pod to be up.
// Grace period to wait for pod shutdown is in seconds.
func (f *Framework) DeleteNGINXPod(grace int64) {
ns := f.IngressController.Namespace
ns := f.Namespace
pod, err := getIngressNGINXPod(ns, f.KubeClientSet)
Expect(err).NotTo(HaveOccurred(), "expected ingress nginx pod to be running")
err = f.KubeClientSet.CoreV1().Pods(ns).Delete(pod.GetName(), metav1.NewDeleteOptions(grace))
Expect(err).NotTo(HaveOccurred(), "unexpected error deleting ingress nginx pod")
err = wait.Poll(Poll, time.Minute*5, func() (bool, error) {
err = wait.Poll(Poll, DefaultTimeout, func() (bool, error) {
pod, err := getIngressNGINXPod(ns, f.KubeClientSet)
if err != nil || pod == nil {
return false, nil

6
test/e2e/framework/grpc_fortune_teller.go
View file

@ -38,7 +38,7 @@ func (f *Framework) NewNewGRPCFortuneTellerDeploymentWithReplicas(replicas int32
deployment := &extensions.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "fortune-teller",
Namespace: f.IngressController.Namespace,
Namespace: f.Namespace,
},
Spec: extensions.DeploymentSpec{
Replicas: NewInt32(replicas),
@ -77,7 +77,7 @@ func (f *Framework) NewNewGRPCFortuneTellerDeploymentWithReplicas(replicas int32
Expect(err).NotTo(HaveOccurred())
Expect(d).NotTo(BeNil(), "expected a fortune-teller deployment")
err = WaitForPodsReady(f.KubeClientSet, DefaultTimeout, int(replicas), f.IngressController.Namespace, metav1.ListOptions{
err = WaitForPodsReady(f.KubeClientSet, DefaultTimeout, int(replicas), f.Namespace, metav1.ListOptions{
LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(),
})
Expect(err).NotTo(HaveOccurred(), "failed to wait for to become ready")
@ -85,7 +85,7 @@ func (f *Framework) NewNewGRPCFortuneTellerDeploymentWithReplicas(replicas int32
service := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "fortune-teller",
Namespace: f.IngressController.Namespace,
Namespace: f.Namespace,
},
Spec: corev1.ServiceSpec{
Ports: []corev1.ServicePort{

6
test/e2e/framework/influxdb.go
View file

@ -61,7 +61,7 @@ func (f *Framework) NewInfluxDBDeployment() {
configuration := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "influxdb-config",
Namespace: f.IngressController.Namespace,
Namespace: f.Namespace,
},
Data: map[string]string{
"influxd.conf": influxConfig,
@ -76,7 +76,7 @@ func (f *Framework) NewInfluxDBDeployment() {
deployment := &extensions.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "influxdb-svc",
Namespace: f.IngressController.Namespace,
Namespace: f.Namespace,
},
Spec: extensions.DeploymentSpec{
Replicas: NewInt32(1),
@ -140,7 +140,7 @@ func (f *Framework) NewInfluxDBDeployment() {
Expect(d).NotTo(BeNil(), "unexpected error creating deployement for influxdb")
err = WaitForPodsReady(f.KubeClientSet, DefaultTimeout, 1, f.IngressController.Namespace, metav1.ListOptions{
err = WaitForPodsReady(f.KubeClientSet, DefaultTimeout, 1, f.Namespace, metav1.ListOptions{
LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(),
})
Expect(err).NotTo(HaveOccurred(), "failed to wait for influxdb to become ready")

8
test/e2e/framework/k8s.go
View file

@ -124,7 +124,7 @@ func WaitForPodsReady(kubeClientSet kubernetes.Interface, timeout time.Duration,
return wait.Poll(2*time.Second, timeout, func() (bool, error) {
pl, err := kubeClientSet.CoreV1().Pods(namespace).List(opts)
if err != nil {
return false, err
return false, nil
}
r := 0
@ -150,11 +150,11 @@ func WaitForEndpoints(kubeClientSet kubernetes.Interface, timeout time.Duration,
return wait.Poll(2*time.Second, timeout, func() (bool, error) {
endpoint, err := kubeClientSet.CoreV1().Endpoints(ns).Get(name, metav1.GetOptions{})
if k8sErrors.IsNotFound(err) {
return false, err
return false, nil
}
Expect(err).NotTo(HaveOccurred())
if len(endpoint.Subsets) == 0 || len(endpoint.Subsets[0].Addresses) == 0 {
return false, err
return false, nil
}
r := 0
@ -191,7 +191,7 @@ func getIngressNGINXPod(ns string, kubeClientSet kubernetes.Interface) (*core.Po
LabelSelector: "app.kubernetes.io/name=ingress-nginx",
})
if err != nil {
return nil, err
return nil, nil
}
if len(l.Items) == 0 {

17
test/e2e/framework/ssl.go
View file

@ -34,7 +34,7 @@ import (
. "github.com/onsi/gomega"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
@ -338,19 +338,24 @@ func tlsConfig(serverName string, pemCA []byte) (*tls.Config, error) {
// given URL using the given TLS configuration and returns whether the TLS
// handshake completed successfully.
func matchTLSServerName(url string, tlsConfig *tls.Config) wait.ConditionFunc {
return func() (ready bool, err error) {
return func() (bool, error) {
u, err := net_url.Parse(url)
if err != nil {
return
return false, err
}
conn, err := tls.Dial("tcp", u.Host, tlsConfig)
port := u.Port()
if port == "" {
port = "443"
}
conn, err := tls.Dial("tcp", fmt.Sprintf("%v:%v", u.Host, port), tlsConfig)
if err != nil {
Logf("Unexpected TLS error: %v", err)
return false, nil
}
conn.Close()
ready = true
return
return true, nil
}
}

9
test/e2e/framework/test_context.go
View file

@ -18,17 +18,14 @@ package framework
import (
"flag"
"os"
"github.com/onsi/ginkgo/config"
"k8s.io/client-go/tools/clientcmd"
)
// TestContextType describes the client context to use in communications with the Kubernetes API.
type TestContextType struct {
KubeHost string
KubeConfig string
KubeHost string
//KubeConfig string
KubeContext string
}
@ -51,7 +48,7 @@ func RegisterCommonFlags() {
config.DefaultReporterConfig.SlowSpecThreshold = 20
flag.StringVar(&TestContext.KubeHost, "kubernetes-host", "http://127.0.0.1:8080", "The kubernetes host, or apiserver, to connect to")
flag.StringVar(&TestContext.KubeConfig, "kubernetes-config", os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to config containing embedded authinfo for kubernetes. Default value is from environment variable "+clientcmd.RecommendedConfigPathEnvVar)
//flag.StringVar(&TestContext.KubeConfig, "kubernetes-config", os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to config containing embedded authinfo for kubernetes. Default value is from environment variable "+clientcmd.RecommendedConfigPathEnvVar)
flag.StringVar(&TestContext.KubeContext, "kubernetes-context", "", "config context to use for kubernetes. If unset, will use value from 'current-context'")
}

27
test/e2e/framework/util.go
View file

@ -23,13 +23,12 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/clientcmd/api"
"k8s.io/ingress-nginx/internal/file"
@ -37,10 +36,10 @@ import (
const (
// Poll how often to poll for conditions
Poll = 2 * time.Second
Poll = 3 * time.Second
// DefaultTimeout time to wait for operations to complete
DefaultTimeout = 5 * time.Minute
DefaultTimeout = 3 * time.Minute
)
func nowStamp() string {
@ -87,15 +86,6 @@ func RestclientConfig(config, context string) (*api.Config, error) {
return c, nil
}
// LoadConfig deserializes the contents of a kubeconfig file into a REST configuration.
func LoadConfig(config, context string) (*rest.Config, error) {
c, err := RestclientConfig(config, context)
if err != nil {
return nil, err
}
return clientcmd.NewDefaultClientConfig(*c, &clientcmd.ConfigOverrides{}).ClientConfig()
}
// RunID unique identifier of the e2e run
var RunID = uuid.NewUUID()
@ -109,8 +99,9 @@ func CreateKubeNamespace(baseName string, c kubernetes.Interface) (string, error
}
// Be robust about making the namespace creation call.
var got *v1.Namespace
err := wait.PollImmediate(Poll, DefaultTimeout, func() (bool, error) {
var err error
var err error
err = wait.PollImmediate(Poll, DefaultTimeout, func() (bool, error) {
got, err = c.CoreV1().Namespaces().Create(ns)
if err != nil {
Logf("Unexpected error while creating namespace: %v", err)
@ -200,7 +191,7 @@ func secretInNamespace(c kubernetes.Interface, namespace, name string) wait.Cond
return func() (bool, error) {
s, err := c.CoreV1().Secrets(namespace).Get(name, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
return false, err
return false, nil
}
if err != nil {
return false, err
@ -268,7 +259,7 @@ func ingressInNamespace(c kubernetes.Interface, namespace, name string) wait.Con
return func() (bool, error) {
ing, err := c.ExtensionsV1beta1().Ingresses(namespace).Get(name, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
return false, err
return false, nil
}
if err != nil {
return false, err
@ -285,7 +276,7 @@ func podRunning(c kubernetes.Interface, podName, namespace string) wait.Conditio
return func() (bool, error) {
pod, err := c.CoreV1().Pods(namespace).Get(podName, metav1.GetOptions{})
if err != nil {
return false, err
return false, nil
}
switch pod.Status.Phase {
case v1.PodRunning: