DevFW-CICD/ingress-nginx-helm
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove most of the time.Sleep from the e2e tests

Browse source
This commit is contained in:
Manuel de Brito Fontes 2018-04-18 16:15:08 -03:00
parent c3ff76ae50
commit 62a80a39ad
No known key found for this signature in database
GPG key ID: 786136016A8BA02A
30 changed files with 628 additions and 465 deletions
Download patch file Download diff file Expand all files Collapse all files

6
test/e2e/framework/echo.go
View file

@ -40,7 +40,7 @@ func (f *Framework) NewEchoDeploymentWithReplicas(replicas int32) error {
deployment := &extensions.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "http-svc",
Namespace: f.Namespace.Name,
Namespace: f.IngressController.Namespace,
},
Spec: extensions.DeploymentSpec{
Replicas: NewInt32(replicas),
@ -84,7 +84,7 @@ func (f *Framework) NewEchoDeploymentWithReplicas(replicas int32) error {
return fmt.Errorf("unexpected error creating deployement for echoserver")
}
err = f.WaitForPodsReady(10*time.Second, int(replicas), metav1.ListOptions{
err = WaitForPodsReady(f.KubeClientSet, 5*time.Minute, int(replicas), f.IngressController.Namespace, metav1.ListOptions{
LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(),
})
if err != nil {
@ -94,7 +94,7 @@ func (f *Framework) NewEchoDeploymentWithReplicas(replicas int32) error {
service := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "http-svc",
Namespace: f.Namespace.Name,
Namespace: f.IngressController.Namespace,
},
Spec: corev1.ServiceSpec{
Ports: []corev1.ServicePort{

23
test/e2e/framework/exec.go
View file

@ -47,3 +47,26 @@ func (f *Framework) ExecCommand(pod *v1.Pod, command string) (string, error) {
return execOut.String(), nil
}
// NewIngressController deploys a new NGINX Ingress controller in a namespace
func (f *Framework) NewIngressController(namespace string) error {
var (
execOut bytes.Buffer
execErr bytes.Buffer
)
cmd := exec.Command("test/e2e/wait-for-nginx.sh", namespace)
cmd.Stdout = &execOut
cmd.Stderr = &execErr
err := cmd.Run()
if err != nil {
return fmt.Errorf("could not execute: %v", err)
}
if execErr.Len() > 0 {
return fmt.Errorf("stderr: %v", execErr.String())
}
return nil
}

192
test/e2e/framework/framework.go
View file

@ -21,9 +21,11 @@ import (
appsv1beta1 "k8s.io/api/apps/v1beta1"
"k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
apiextcs "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
@ -53,27 +55,26 @@ type Framework struct {
KubeConfig *restclient.Config
APIExtensionsClientSet apiextcs.Interface
// Namespace in which all test resources should reside
Namespace *v1.Namespace
// To make sure that this framework cleans up after itself, no matter what,
// we install a Cleanup action before each test and clear it after. If we
// should abort, the AfterSuite hook should run all Cleanup actions.
cleanupHandle CleanupActionHandle
// Map to hold the default nginx-configuration configmap data before a test run
DefaultNginxConfigMapData map[string]string
IngressController *ingressController
}
NginxHTTPURL string
NginxHTTPSURL string
type ingressController struct {
HTTPURL string
HTTPSURL string
Namespace string
}
// NewDefaultFramework makes a new framework and sets up a BeforeEach/AfterEach for
// you (you can write additional before/after each functions).
func NewDefaultFramework(baseName string) *Framework {
f := &Framework{
BaseName: baseName,
DefaultNginxConfigMapData: nil,
BaseName: baseName,
}
BeforeEach(f.BeforeEach)
@ -95,43 +96,48 @@ func (f *Framework) BeforeEach() {
Expect(err).NotTo(HaveOccurred())
By("Building a namespace api object")
f.Namespace, err = CreateKubeNamespace(f.BaseName, f.KubeClientSet)
ingressNamespace, err := CreateKubeNamespace(f.BaseName, f.KubeClientSet)
Expect(err).NotTo(HaveOccurred())
f.IngressController = &ingressController{
Namespace: ingressNamespace,
}
By("Starting new ingress controller")
err = f.NewIngressController(f.IngressController.Namespace)
Expect(err).NotTo(HaveOccurred())
err = WaitForPodsReady(f.KubeClientSet, 5*time.Minute, 1, f.IngressController.Namespace, metav1.ListOptions{
LabelSelector: "app=ingress-nginx",
})
Expect(err).NotTo(HaveOccurred())
By("Building NGINX HTTP URL")
f.NginxHTTPURL, err = f.GetNginxURL(HTTP)
HTTPURL, err := f.GetNginxURL(HTTP)
Expect(err).NotTo(HaveOccurred())
f.IngressController.HTTPURL = HTTPURL
By("Building NGINX HTTPS URL")
f.NginxHTTPSURL, err = f.GetNginxURL(HTTPS)
HTTPSURL, err := f.GetNginxURL(HTTPS)
Expect(err).NotTo(HaveOccurred())
By("Persist nginx-configuration configMap Data")
if f.DefaultNginxConfigMapData == nil {
f.DefaultNginxConfigMapData, err = f.GetNginxConfigMapData()
Expect(err).NotTo(HaveOccurred())
Expect(f.DefaultNginxConfigMapData).NotTo(BeNil())
}
f.IngressController.HTTPSURL = HTTPSURL
// we wait for any change in the informers and SSL certificate generation
time.Sleep(5 * time.Second)
}
// AfterEach deletes the namespace, after reading its events.
func (f *Framework) AfterEach() {
RemoveCleanupAction(f.cleanupHandle)
By("Deleting test namespace")
err := DeleteKubeNamespace(f.KubeClientSet, f.Namespace.Name)
Expect(err).NotTo(HaveOccurred())
By("Waiting for test namespace to no longer exist")
err = WaitForNoPodsInNamespace(f.KubeClientSet, f.Namespace.Name)
Expect(err).NotTo(HaveOccurred())
By("Reset nginx-configuration configMap to default state")
err = f.SetNginxConfigMapData(f.DefaultNginxConfigMapData)
err := DeleteKubeNamespace(f.KubeClientSet, f.IngressController.Namespace)
Expect(err).NotTo(HaveOccurred())
}
// IngressNginxDescribe wrapper function for ginkgo describe. Adds namespacing.
// IngressNginxDescribe wrapper function for ginkgo describe. Adds namespacing.
func IngressNginxDescribe(text string, body func()) bool {
return Describe("[nginx-ingress] "+text, body)
}
@ -148,7 +154,10 @@ func (f *Framework) GetNginxIP() (string, error) {
// GetNginxPort returns the number of TCP port where NGINX is running
func (f *Framework) GetNginxPort(name string) (int, error) {
s, err := f.KubeClientSet.CoreV1().Services("ingress-nginx").Get("ingress-nginx", metav1.GetOptions{})
s, err := f.KubeClientSet.
CoreV1().
Services(f.IngressController.Namespace).
Get("ingress-nginx", metav1.GetOptions{})
if err != nil {
return -1, err
}
@ -179,45 +188,37 @@ func (f *Framework) GetNginxURL(scheme RequestScheme) (string, error) {
// WaitForNginxServer waits until the nginx configuration contains a particular server section
func (f *Framework) WaitForNginxServer(name string, matcher func(cfg string) bool) error {
// initial wait to allow the update of the ingress controller
time.Sleep(5 * time.Second)
return wait.PollImmediate(Poll, time.Minute*2, f.matchNginxConditions(name, matcher))
return wait.Poll(Poll, time.Minute*5, f.matchNginxConditions(name, matcher))
}
// WaitForNginxConfiguration waits until the nginx configuration contains a particular configuration
func (f *Framework) WaitForNginxConfiguration(matcher func(cfg string) bool) error {
// initial wait to allow the update of the ingress controller
time.Sleep(5 * time.Second)
return wait.PollImmediate(Poll, time.Minute*2, f.matchNginxConditions("", matcher))
return wait.Poll(Poll, time.Minute*5, f.matchNginxConditions("", matcher))
}
// NginxLogs returns the logs of the nginx ingress controller pod running
func (f *Framework) NginxLogs() (string, error) {
l, err := f.KubeClientSet.CoreV1().Pods("ingress-nginx").List(metav1.ListOptions{
l, err := f.KubeClientSet.CoreV1().Pods(f.IngressController.Namespace).List(metav1.ListOptions{
LabelSelector: "app=ingress-nginx",
})
if err != nil {
return "", err
}
if len(l.Items) == 0 {
return "", fmt.Errorf("no nginx ingress controller pod is running")
}
for _, pod := range l.Items {
if strings.HasPrefix(pod.GetName(), "nginx-ingress-controller") &&
len(pod.Status.ContainerStatuses) > 0 &&
pod.Status.ContainerStatuses[0].State.Running != nil {
return f.Logs(&pod)
if strings.HasPrefix(pod.GetName(), "nginx-ingress-controller") {
if isRunning, err := podRunningReady(&pod); err == nil && isRunning {
return f.Logs(&pod)
}
}
}
return "", fmt.Errorf("no nginx ingress controller pod is running")
return "", fmt.Errorf("no nginx ingress controller pod is running (logs)")
}
func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) bool) wait.ConditionFunc {
return func() (bool, error) {
l, err := f.KubeClientSet.CoreV1().Pods("ingress-nginx").List(metav1.ListOptions{
l, err := f.KubeClientSet.CoreV1().Pods(f.IngressController.Namespace).List(metav1.ListOptions{
LabelSelector: "app=ingress-nginx",
})
if err != nil {
@ -225,7 +226,7 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b
}
if len(l.Items) == 0 {
return false, fmt.Errorf("no nginx ingress controller pod is running")
return false, nil
}
var cmd string
@ -239,17 +240,15 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b
Loop:
for _, p := range l.Items {
if strings.HasPrefix(p.GetName(), "nginx-ingress-controller") {
for _, cs := range p.Status.ContainerStatuses {
if cs.State.Running != nil && cs.Name == "nginx-ingress-controller" {
pod = &p
break Loop
}
if isRunning, err := podRunningReady(&p); err == nil && isRunning {
pod = &p
break Loop
}
}
}
if pod == nil {
return false, fmt.Errorf("no nginx ingress controller pod is running")
return false, nil
}
o, err := f.ExecCommand(pod, cmd)
@ -259,17 +258,23 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b
var match bool
errs := InterceptGomegaFailures(func() {
if glog.V(10) && len(o) > 0 {
glog.Infof("nginx.conf:\n%v", o)
}
if matcher(o) {
match = true
}
})
glog.V(2).Infof("Errors waiting for conditions: %v", errs)
if match {
return true, nil
}
if len(errs) > 0 {
glog.V(2).Infof("Errors waiting for conditions: %v", errs)
}
return false, nil
}
}
@ -279,7 +284,10 @@ func (f *Framework) getNginxConfigMap() (*v1.ConfigMap, error) {
return nil, fmt.Errorf("KubeClientSet not initialized")
}
config, err := f.KubeClientSet.CoreV1().ConfigMaps("ingress-nginx").Get("nginx-configuration", metav1.GetOptions{})
config, err := f.KubeClientSet.
CoreV1().
ConfigMaps(f.IngressController.Namespace).
Get("nginx-configuration", metav1.GetOptions{})
if err != nil {
return nil, err
}
@ -314,7 +322,10 @@ func (f *Framework) SetNginxConfigMapData(cmData map[string]string) error {
config.Data = cmData
_, err = f.KubeClientSet.CoreV1().ConfigMaps("ingress-nginx").Update(config)
_, err = f.KubeClientSet.
CoreV1().
ConfigMaps(f.IngressController.Namespace).
Update(config)
if err != nil {
return err
}
@ -343,16 +354,75 @@ func UpdateDeployment(kubeClientSet kubernetes.Interface, namespace string, name
return err
}
if err = updateFunc(deployment); err != nil {
return err
if updateFunc != nil {
if err := updateFunc(deployment); err != nil {
return err
}
}
err = WaitForPodsReady(kubeClientSet, 60*time.Second, replicas, namespace, metav1.ListOptions{
if *deployment.Spec.Replicas != int32(replicas) {
glog.Infof("updating replica count from %v to %v...", *deployment.Spec.Replicas, replicas)
deployment, err := kubeClientSet.AppsV1beta1().Deployments(namespace).Get(name, metav1.GetOptions{})
if err != nil {
return err
}
deployment.Spec.Replicas = NewInt32(int32(replicas))
_, err = kubeClientSet.AppsV1beta1().Deployments(namespace).Update(deployment)
if err != nil {
return errors.Wrapf(err, "scaling the number of replicas to %v", replicas)
}
}
err = WaitForPodsReady(kubeClientSet, 5*time.Minute, replicas, namespace, metav1.ListOptions{
LabelSelector: fields.SelectorFromSet(fields.Set(deployment.Spec.Template.ObjectMeta.Labels)).String(),
})
if err != nil {
return errors.Wrap(err, "failed to wait for nginx-ingress-controller pods to restart")
return errors.Wrapf(err, "failed to wait for nginx-ingress-controller replica count to be %v", replicas)
}
return nil
}
// NewSingleIngress creates a simple ingress rule
func NewSingleIngress(name, path, host, ns string, annotations *map[string]string) *extensions.Ingress {
if annotations == nil {
annotations = &map[string]string{}
}
ing := &extensions.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: ns,
Annotations: *annotations,
},
Spec: extensions.IngressSpec{
TLS: []extensions.IngressTLS{
{
Hosts: []string{host},
SecretName: host,
},
},
Rules: []extensions.IngressRule{
{
Host: host,
IngressRuleValue: extensions.IngressRuleValue{
HTTP: &extensions.HTTPIngressRuleValue{
Paths: []extensions.HTTPIngressPath{
{
Path: path,
Backend: extensions.IngressBackend{
ServiceName: "http-svc",
ServicePort: intstr.FromInt(80),
},
},
},
},
},
},
},
},
}
return ing
}

43
test/e2e/framework/k8s.go
View file

@ -17,6 +17,7 @@ limitations under the License.
package framework
import (
"fmt"
"time"
api "k8s.io/api/core/v1"
@ -26,6 +27,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
)
// EnsureSecret creates a Secret object or returns it if it already exists.
@ -45,10 +47,19 @@ func (f *Framework) EnsureIngress(ingress *extensions.Ingress) (*extensions.Ingr
s, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(ingress.Namespace).Update(ingress)
if err != nil {
if k8sErrors.IsNotFound(err) {
return f.KubeClientSet.ExtensionsV1beta1().Ingresses(ingress.Namespace).Create(ingress)
s, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(ingress.Namespace).Create(ingress)
if err != nil {
return nil, err
}
} else {
return nil, err
}
return nil, err
}
if s.Annotations == nil {
s.Annotations = make(map[string]string)
}
return s, nil
}
@ -76,14 +87,9 @@ func (f *Framework) EnsureDeployment(deployment *extensions.Deployment) (*extens
return d, nil
}
// WaitForPodsReady waits for a given amount of time until a group of Pods is running in the framework's namespace.
func (f *Framework) WaitForPodsReady(timeout time.Duration, expectedReplicas int, opts metav1.ListOptions) error {
return WaitForPodsReady(f.KubeClientSet, timeout, expectedReplicas, f.Namespace.Name, opts)
}
// WaitForPodsReady waits for a given amount of time until a group of Pods is running in the given namespace.
func WaitForPodsReady(kubeClientSet kubernetes.Interface, timeout time.Duration, expectedReplicas int, namespace string, opts metav1.ListOptions) error {
return wait.Poll(time.Second, timeout, func() (bool, error) {
return wait.Poll(2*time.Second, timeout, func() (bool, error) {
pl, err := kubeClientSet.CoreV1().Pods(namespace).List(opts)
if err != nil {
return false, err
@ -91,10 +97,9 @@ func WaitForPodsReady(kubeClientSet kubernetes.Interface, timeout time.Duration,
r := 0
for _, p := range pl.Items {
if p.Status.Phase != core.PodRunning {
continue
if isRunning, _ := podRunningReady(&p); isRunning {
r++
}
r++
}
if r == expectedReplicas {
@ -104,3 +109,19 @@ func WaitForPodsReady(kubeClientSet kubernetes.Interface, timeout time.Duration,
return false, nil
})
}
// podRunningReady checks whether pod p's phase is running and it has a ready
// condition of status true.
func podRunningReady(p *core.Pod) (bool, error) {
// Check the phase is running.
if p.Status.Phase != core.PodRunning {
return false, fmt.Errorf("want pod '%s' on '%s' to be '%v' but was '%v'",
p.ObjectMeta.Name, p.Spec.NodeName, core.PodRunning, p.Status.Phase)
}
// Check the ready condition is true.
if !podutil.IsPodReady(p) {
return false, fmt.Errorf("pod '%s' on '%s' didn't have condition {%v %v}; conditions: %v",
p.ObjectMeta.Name, p.Spec.NodeName, core.PodReady, core.ConditionTrue, p.Status.Conditions)
}
return true, nil
}

4
test/e2e/framework/test_context.go
View file

@ -46,6 +46,10 @@ func RegisterCommonFlags() {
// Randomize specs as well as suites
config.GinkgoConfig.RandomizeAllSpecs = true
// Default SlowSpecThreshold is 5 seconds.
// Too low for the kind of operations we need to tests
config.DefaultReporterConfig.SlowSpecThreshold = 20
flag.StringVar(&TestContext.KubeHost, "kubernetes-host", "http://127.0.0.1:8080", "The kubernetes host, or apiserver, to connect to")
flag.StringVar(&TestContext.KubeConfig, "kubernetes-config", os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to config containing embedded authinfo for kubernetes. Default value is from environment variable "+clientcmd.RecommendedConfigPathEnvVar)
flag.StringVar(&TestContext.KubeContext, "kubernetes-context", "", "config context to use for kubernetes. If unset, will use value from 'current-context'")

10
test/e2e/framework/util.go
View file

@ -100,7 +100,7 @@ func LoadConfig(config, context string) (*rest.Config, error) {
var RunID = uuid.NewUUID()
// CreateKubeNamespace creates a new namespace in the cluster
func CreateKubeNamespace(baseName string, c kubernetes.Interface) (*v1.Namespace, error) {
func CreateKubeNamespace(baseName string, c kubernetes.Interface) (string, error) {
ts := time.Now().UnixNano()
ns := &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
@ -120,9 +120,9 @@ func CreateKubeNamespace(baseName string, c kubernetes.Interface) (*v1.Namespace
return true, nil
})
if err != nil {
return nil, err
return "", err
}
return got, nil
return got.Name, nil
}
// DeleteKubeNamespace deletes a namespace and all the objects inside
@ -140,7 +140,7 @@ func ExpectNoError(err error, explain ...interface{}) {
// WaitForKubeNamespaceNotExist waits until a namespaces is not present in the cluster
func WaitForKubeNamespaceNotExist(c kubernetes.Interface, namespace string) error {
return wait.PollImmediate(Poll, time.Minute*2, namespaceNotExist(c, namespace))
return wait.PollImmediate(Poll, time.Minute*5, namespaceNotExist(c, namespace))
}
func namespaceNotExist(c kubernetes.Interface, namespace string) wait.ConditionFunc {
@ -158,7 +158,7 @@ func namespaceNotExist(c kubernetes.Interface, namespace string) wait.ConditionF
// WaitForNoPodsInNamespace waits until there are no pods running in a namespace
func WaitForNoPodsInNamespace(c kubernetes.Interface, namespace string) error {
return wait.PollImmediate(Poll, time.Minute*2, noPodsInNamespace(c, namespace))
return wait.PollImmediate(Poll, time.Minute*5, noPodsInNamespace(c, namespace))
}
func noPodsInNamespace(c kubernetes.Interface, namespace string) wait.ConditionFunc {