DevFW-CICD/ingress-nginx-helm
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix IngressClass logic for newer releases (#7341)

Browse source 
* Fix IngressClass logic for newer releases

Signed-off-by: Ricardo Pchevuzinske Katz <ricardo.katz@gmail.com>

* Change e2e tests for the new IngressClass presence

* Fix chart and admission tests

Signed-off-by: Ricardo Pchevuzinske Katz <ricardo.katz@gmail.com>

* Fix helm chart test

Signed-off-by: Ricardo Pchevuzinske Katz <ricardo.katz@gmail.com>

* Fix reviews

* Remove ingressclass code from admission
This commit is contained in:
Ricardo Katz 2021-07-28 18:58:46 -03:00 • committed by GitHub
parent 0d57e87819
commit cef147a24d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
68 changed files with 1450 additions and 637 deletions
Download patch file Download diff file Expand all files Collapse all files

64
internal/ingress/annotations/class/main.go
View file

@ -1,64 +0,0 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package class
import (
networking "k8s.io/api/networking/v1"
"k8s.io/ingress-nginx/internal/k8s"
)
const (
// IngressKey picks a specific "class" for the Ingress.
// The controller only processes Ingresses with this annotation either
// unset, or set to either the configured value or the empty string.
IngressKey = "kubernetes.io/ingress.class"
)
var (
// DefaultClass defines the default class used in the nginx ingress controller
DefaultClass = "nginx"
// IngressClass sets the runtime ingress class to use
// An empty string means accept all ingresses without
// annotation and the ones configured with class nginx
IngressClass = "nginx"
)
// IsValid returns true if the given Ingress specify the ingress.class
// annotation or IngressClassName resource for Kubernetes >= v1.18
func IsValid(ing *networking.Ingress) bool {
// 1. with annotation or IngressClass
ingress, ok := ing.GetAnnotations()[IngressKey]
if !ok && ing.Spec.IngressClassName != nil {
ingress = *ing.Spec.IngressClassName
}
// empty ingress and IngressClass equal default
if len(ingress) == 0 && IngressClass == DefaultClass {
return true
}
// k8s > v1.18.
// Processing may be redundant because k8s.IngressClass is obtained by IngressClass
// 3. without annotation and IngressClass. Check IngressClass
if k8s.IngressClass != nil {
return ingress == k8s.IngressClass.Name
}
// 4. with IngressClass
return ingress == IngressClass
}

103
internal/ingress/annotations/class/main_test.go
View file

@ -1,103 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package class
import (
"testing"
api "k8s.io/api/core/v1"
networking "k8s.io/api/networking/v1"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/ingress-nginx/internal/k8s"
)
func TestIsValidClass(t *testing.T) {
dc := DefaultClass
ic := IngressClass
k8sic := k8s.IngressClass
v1Ready := k8s.IsIngressV1Ready
// restore original values after the tests
defer func() {
DefaultClass = dc
IngressClass = ic
k8s.IngressClass = k8sic
k8s.IsIngressV1Ready = v1Ready
}()
tests := []struct {
ingress string
controller string
defClass string
annotation bool
ingressClassName bool
k8sClass *networking.IngressClass
v1Ready bool
isValid bool
}{
{"", "", "nginx", true, false, nil, false, true},
{"", "nginx", "nginx", true, false, nil, false, true},
{"nginx", "nginx", "nginx", true, false, nil, false, true},
{"custom", "custom", "nginx", true, false, nil, false, true},
{"", "killer", "nginx", true, false, nil, false, false},
{"custom", "nginx", "nginx", true, false, nil, false, false},
{"nginx", "nginx", "nginx", false, true, nil, false, true},
{"custom", "nginx", "nginx", false, true, nil, true, false},
{"nginx", "nginx", "nginx", false, true, nil, true, true},
{"", "custom", "nginx", false, false,
&networking.IngressClass{
ObjectMeta: meta_v1.ObjectMeta{
Name: "custom",
},
},
false, false},
{"", "custom", "nginx", false, false,
&networking.IngressClass{
ObjectMeta: meta_v1.ObjectMeta{
Name: "custom",
},
},
true, false},
}
for _, test := range tests {
ing := &networking.Ingress{
ObjectMeta: meta_v1.ObjectMeta{
Name: "foo",
Namespace: api.NamespaceDefault,
},
}
data := map[string]string{}
ing.SetAnnotations(data)
if test.annotation {
ing.Annotations[IngressKey] = test.ingress
}
if test.ingressClassName {
ing.Spec.IngressClassName = &[]string{test.ingress}[0]
}
IngressClass = test.controller
DefaultClass = test.defClass
k8s.IngressClass = test.k8sClass
k8s.IsIngressV1Ready = test.v1Ready
b := IsValid(ing)
if b != test.isValid {
t.Errorf("test %v - expected %v but %v was returned", test, test.isValid, b)
}
}
}

9
internal/ingress/controller/controller.go
View file

@ -33,11 +33,11 @@ import (
clientset "k8s.io/client-go/kubernetes"
"k8s.io/ingress-nginx/internal/ingress"
"k8s.io/ingress-nginx/internal/ingress/annotations"
"k8s.io/ingress-nginx/internal/ingress/annotations/class"
"k8s.io/ingress-nginx/internal/ingress/annotations/log"
"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
"k8s.io/ingress-nginx/internal/ingress/annotations/proxy"
ngx_config "k8s.io/ingress-nginx/internal/ingress/controller/config"
"k8s.io/ingress-nginx/internal/ingress/controller/ingressclass"
"k8s.io/ingress-nginx/internal/ingress/controller/store"
"k8s.io/ingress-nginx/internal/ingress/errors"
"k8s.io/ingress-nginx/internal/k8s"
@ -100,6 +100,8 @@ type Configuration struct {
DisableCatchAll bool
IngressClassConfiguration *ingressclass.IngressClassConfiguration
ValidationWebhook string
ValidationWebhookCertPath string
ValidationWebhookKeyPath string
@ -221,11 +223,6 @@ func (n *NGINXController) CheckIngress(ing *networking.Ingress) error {
return nil
}
if !class.IsValid(ing) {
klog.Warningf("ignoring ingress %v in %v based on annotation %v", ing.Name, ing.ObjectMeta.Namespace, class.IngressKey)
return nil
}
if n.cfg.Namespace != "" && ing.ObjectMeta.Namespace != n.cfg.Namespace {
klog.Warningf("ignoring ingress %v in namespace %v different from the namespace watched %s", ing.Name, ing.ObjectMeta.Namespace, n.cfg.Namespace)
return nil

26
internal/ingress/controller/controller_test.go
View file

@ -45,6 +45,7 @@ import (
"k8s.io/ingress-nginx/internal/ingress/annotations/proxyssl"
"k8s.io/ingress-nginx/internal/ingress/controller/config"
ngx_config "k8s.io/ingress-nginx/internal/ingress/controller/config"
"k8s.io/ingress-nginx/internal/ingress/controller/ingressclass"
"k8s.io/ingress-nginx/internal/ingress/controller/store"
"k8s.io/ingress-nginx/internal/ingress/defaults"
"k8s.io/ingress-nginx/internal/ingress/metric"
@ -188,18 +189,6 @@ func TestCheckIngress(t *testing.T) {
},
},
}
t.Run("When the ingress class differs from nginx", func(t *testing.T) {
ing.ObjectMeta.Annotations["kubernetes.io/ingress.class"] = "different"
nginx.command = testNginxTestCommand{
t: t,
err: fmt.Errorf("test error"),
}
if nginx.CheckIngress(ing) != nil {
t.Errorf("with a different ingress class, no error should be returned")
}
})
t.Run("when the class is the nginx one", func(t *testing.T) {
ing.ObjectMeta.Annotations["kubernetes.io/ingress.class"] = "nginx"
nginx.command = testNginxTestCommand{
@ -1899,7 +1888,12 @@ func newNGINXController(t *testing.T) *NGINXController {
10*time.Minute,
clientSet,
channels.NewRingChannel(10),
false)
false,
&ingressclass.IngressClassConfiguration{
Controller: "k8s.io/ingress-nginx",
AnnotationValue: "nginx",
},
)
sslCert := ssl.GetFakeSSLCert()
config := &Configuration{
@ -1957,7 +1951,11 @@ func newDynamicNginxController(t *testing.T, setConfigMap func(string) *v1.Confi
10*time.Minute,
clientSet,
channels.NewRingChannel(10),
false)
false,
&ingressclass.IngressClassConfiguration{
Controller: "k8s.io/ingress-nginx",
AnnotationValue: "nginx",
})
sslCert := ssl.GetFakeSSLCert()
config := &Configuration{

45
internal/ingress/controller/ingressclass/ingressclass.go Normal file
View file

@ -0,0 +1,45 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ingressclass
const (
// IngressKey picks a specific "class" for the Ingress.
// The controller only processes Ingresses with this annotation either
// unset, or set to either the configured value or the empty string.
IngressKey = "kubernetes.io/ingress.class"
// DefaultControllerName defines the default controller name for Ingress NGINX
DefaultControllerName = "k8s.io/ingress-nginx"
// DefaultAnnotationValue defines the default annotation value for the ingress-nginx controller
DefaultAnnotationValue = "nginx"
)
// IngressClassConfiguration defines the various aspects of IngressClass parsing
// and how the controller should behave in each case
type IngressClassConfiguration struct {
// Controller defines the controller value this daemon watch to.
// Defaults to "k8s.io/ingress-nginx" defined in flags
Controller string
// AnnotationValue defines the annotation value this Controller watch to, in case of the
// ingressSpecName is not found but the annotation is.
// The Annotation is deprecated and should not be used in future releases
AnnotationValue string
// WatchWithoutClass defines if Controller should watch to Ingress Objects that does
// not contain an IngressClass configuration
WatchWithoutClass bool
}

12
internal/ingress/controller/nginx.go
View file

@ -50,7 +50,6 @@ import (
adm_controller "k8s.io/ingress-nginx/internal/admission/controller"
"k8s.io/ingress-nginx/internal/file"
"k8s.io/ingress-nginx/internal/ingress"
"k8s.io/ingress-nginx/internal/ingress/annotations/class"
ngx_config "k8s.io/ingress-nginx/internal/ingress/controller/config"
"k8s.io/ingress-nginx/internal/ingress/controller/process"
"k8s.io/ingress-nginx/internal/ingress/controller/store"
@ -131,7 +130,8 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro
config.ResyncPeriod,
config.Client,
n.updateCh,
config.DisableCatchAll)
config.DisableCatchAll,
config.IngressClassConfiguration)
n.syncQueue = task.NewTaskQueue(n.syncIngress)
@ -257,10 +257,10 @@ func (n *NGINXController) Start() {
// we need to use the defined ingress class to allow multiple leaders
// in order to update information about ingress status
electionID := fmt.Sprintf("%v-%v", n.cfg.ElectionID, class.DefaultClass)
if class.IngressClass != "" {
electionID = fmt.Sprintf("%v-%v", n.cfg.ElectionID, class.IngressClass)
}
// TODO: For now, as the the IngressClass logics has changed, is up to the
// cluster admin to create different Leader Election IDs.
// Should revisit this in a future
electionID := n.cfg.ElectionID
setupLeaderElection(&leaderElectionConfig{
Client: n.cfg.Client,

39
internal/ingress/controller/store/ingressclass.go Normal file
View file

@ -0,0 +1,39 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package store
import (
networking "k8s.io/api/networking/v1"
"k8s.io/client-go/tools/cache"
)
// IngressClassLister makes a Store that lists IngressClass.
type IngressClassLister struct {
cache.Store
}
// ByKey returns the Ingress matching key in the local Ingress Store.
func (il IngressClassLister) ByKey(key string) (*networking.IngressClass, error) {
i, exists, err := il.GetByKey(key)
if err != nil {
return nil, err
}
if !exists {
return nil, NotExistsError(key)
}
return i.(*networking.IngressClass), nil
}

133
internal/ingress/controller/store/store.go
View file

@ -41,14 +41,13 @@ import (
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record"
"k8s.io/klog/v2"
"k8s.io/utils/pointer"
"k8s.io/ingress-nginx/internal/file"
"k8s.io/ingress-nginx/internal/ingress"
"k8s.io/ingress-nginx/internal/ingress/annotations"
"k8s.io/ingress-nginx/internal/ingress/annotations/class"
"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
ngx_config "k8s.io/ingress-nginx/internal/ingress/controller/config"
"k8s.io/ingress-nginx/internal/ingress/controller/ingressclass"
ngx_template "k8s.io/ingress-nginx/internal/ingress/controller/template"
"k8s.io/ingress-nginx/internal/ingress/defaults"
"k8s.io/ingress-nginx/internal/ingress/errors"
@ -121,16 +120,18 @@ type Event struct {
// Informer defines the required SharedIndexInformers that interact with the API server.
type Informer struct {
Ingress cache.SharedIndexInformer
Endpoint cache.SharedIndexInformer
Service cache.SharedIndexInformer
Secret cache.SharedIndexInformer
ConfigMap cache.SharedIndexInformer
Ingress cache.SharedIndexInformer
IngressClass cache.SharedIndexInformer
Endpoint cache.SharedIndexInformer
Service cache.SharedIndexInformer
Secret cache.SharedIndexInformer
ConfigMap cache.SharedIndexInformer
}
// Lister contains object listers (stores).
type Lister struct {
Ingress IngressLister
IngressClass IngressClassLister
Service ServiceLister
Endpoint EndpointLister
Secret SecretLister
@ -150,6 +151,7 @@ func (e NotExistsError) Error() string {
func (i *Informer) Run(stopCh chan struct{}) {
go i.Secret.Run(stopCh)
go i.Endpoint.Run(stopCh)
go i.IngressClass.Run(stopCh)
go i.Service.Run(stopCh)
go i.ConfigMap.Run(stopCh)
@ -157,6 +159,7 @@ func (i *Informer) Run(stopCh chan struct{}) {
// from the queue
if !cache.WaitForCacheSync(stopCh,
i.Endpoint.HasSynced,
i.IngressClass.HasSynced,
i.Service.HasSynced,
i.Secret.HasSynced,
i.ConfigMap.HasSynced,
@ -221,7 +224,8 @@ func New(
resyncPeriod time.Duration,
client clientset.Interface,
updateCh *channels.RingChannel,
disableCatchAll bool) Storer {
disableCatchAll bool,
icConfig *ingressclass.IngressClassConfiguration) Storer {
store := &k8sStore{
informers: &Informer{},
@ -296,6 +300,9 @@ func New(
store.informers.Ingress = infFactory.Networking().V1().Ingresses().Informer()
store.listers.Ingress.Store = store.informers.Ingress.GetStore()
store.informers.IngressClass = infFactory.Networking().V1().IngressClasses().Informer()
store.listers.IngressClass.Store = cache.NewStore(cache.MetaNamespaceKeyFunc)
store.informers.Endpoint = infFactory.Core().V1().Endpoints().Informer()
store.listers.Endpoint.Store = store.informers.Endpoint.GetStore()
@ -324,7 +331,9 @@ func New(
}
}
if !class.IsValid(ing) {
_, err := store.GetIngressClass(ing, icConfig)
if err != nil {
klog.InfoS("Ignoring ingress because of error while validating ingress class", "ingress", klog.KObj(ing), "error", err)
return
}
@ -347,12 +356,14 @@ func New(
ingEventHandler := cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
ing, _ := toIngress(obj)
if !class.IsValid(ing) {
ingressClass, _ := parser.GetStringAnnotation(class.IngressKey, ing)
klog.InfoS("Ignoring ingress", "ingress", klog.KObj(ing), "kubernetes.io/ingress.class", ingressClass, "ingressClassName", pointer.StringPtrDerefOr(ing.Spec.IngressClassName, ""))
ic, err := store.GetIngressClass(ing, icConfig)
if err != nil {
klog.InfoS("Ignoring ingress because of error while validating ingress class", "ingress", klog.KObj(ing), "error", err)
return
}
klog.InfoS("Found valid IngressClass", "ingress", klog.KObj(ing), "ingressclass", ic)
if hasCatchAllIngressRule(ing.Spec) && disableCatchAll {
klog.InfoS("Ignoring add for catch-all ingress because of --disable-catch-all", "ingress", klog.KObj(ing))
return
@ -374,21 +385,21 @@ func New(
oldIng, _ := toIngress(old)
curIng, _ := toIngress(cur)
validOld := class.IsValid(oldIng)
validCur := class.IsValid(curIng)
if !validOld && validCur {
_, errOld := store.GetIngressClass(oldIng, icConfig)
classCur, errCur := store.GetIngressClass(curIng, icConfig)
if errOld != nil && errCur == nil {
if hasCatchAllIngressRule(curIng.Spec) && disableCatchAll {
klog.InfoS("ignoring update for catch-all ingress because of --disable-catch-all", "ingress", klog.KObj(curIng))
return
}
klog.InfoS("creating ingress", "ingress", klog.KObj(curIng), "class", class.IngressKey)
klog.InfoS("creating ingress", "ingress", klog.KObj(curIng), "ingressclass", classCur)
recorder.Eventf(curIng, corev1.EventTypeNormal, "Sync", "Scheduled for sync")
} else if validOld && !validCur {
klog.InfoS("removing ingress", "ingress", klog.KObj(curIng), "class", class.IngressKey)
} else if errOld == nil && errCur != nil {
klog.InfoS("removing ingress because of unknown ingressclass", "ingress", klog.KObj(curIng))
ingDeleteHandler(old)
return
} else if validCur && !reflect.DeepEqual(old, cur) {
} else if errCur == nil && !reflect.DeepEqual(old, cur) {
if hasCatchAllIngressRule(curIng.Spec) && disableCatchAll {
klog.InfoS("ignoring update for catch-all ingress and delete old one because of --disable-catch-all", "ingress", klog.KObj(curIng))
ingDeleteHandler(old)
@ -412,6 +423,63 @@ func New(
},
}
ingressClassEventHandler := cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
ingressclass := obj.(*networkingv1.IngressClass)
if ingressclass.Spec.Controller != icConfig.Controller {
klog.InfoS("ignoring ingressclass as the spec.controller is not the same of this ingress", "ingressclass", klog.KObj(ingressclass))
return
}
err := store.listers.IngressClass.Add(ingressclass)
if err != nil {
klog.InfoS("error adding ingressclass to store", "ingressclass", klog.KObj(ingressclass), "error", err)
return
}
updateCh.In() <- Event{
Type: CreateEvent,
Obj: obj,
}
},
DeleteFunc: func(obj interface{}) {
ingressclass := obj.(*networkingv1.IngressClass)
if ingressclass.Spec.Controller != icConfig.Controller {
klog.InfoS("ignoring ingressclass as the spec.controller is not the same of this ingress", "ingressclass", klog.KObj(ingressclass))
return
}
err := store.listers.IngressClass.Delete(ingressclass)
if err != nil {
klog.InfoS("error removing ingressclass from store", "ingressclass", klog.KObj(ingressclass), "error", err)
return
}
updateCh.In() <- Event{
Type: DeleteEvent,
Obj: obj,
}
},
UpdateFunc: func(old, cur interface{}) {
oic := old.(*networkingv1.IngressClass)
cic := cur.(*networkingv1.IngressClass)
if cic.Spec.Controller != icConfig.Controller {
klog.InfoS("ignoring ingressclass as the spec.controller is not the same of this ingress", "ingressclass", klog.KObj(cic))
return
}
// TODO: In a future we might be interested in parse parameters and use as
// current IngressClass for this case, crossing with configmap
if !reflect.DeepEqual(cic.Spec.Parameters, oic.Spec.Parameters) {
err := store.listers.IngressClass.Update(cic)
if err != nil {
klog.InfoS("error updating ingressclass in store", "ingressclass", klog.KObj(cic), "error", err)
return
}
updateCh.In() <- Event{
Type: UpdateEvent,
Obj: cur,
}
}
},
}
secrEventHandler := cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
sec := obj.(*corev1.Secret)
@ -608,6 +676,7 @@ func New(
}
store.informers.Ingress.AddEventHandler(ingEventHandler)
store.informers.IngressClass.AddEventHandler(ingressClassEventHandler)
store.informers.Endpoint.AddEventHandler(epEventHandler)
store.informers.Secret.AddEventHandler(secrEventHandler)
store.informers.ConfigMap.AddEventHandler(cmEventHandler)
@ -758,6 +827,32 @@ func (s *k8sStore) GetService(key string) (*corev1.Service, error) {
return s.listers.Service.ByKey(key)
}
func (s *k8sStore) GetIngressClass(ing *networkingv1.Ingress, icConfig *ingressclass.IngressClassConfiguration) (string, error) {
// First we try ingressClassName
if ing.Spec.IngressClassName != nil {
iclass, err := s.listers.IngressClass.ByKey(*ing.Spec.IngressClassName)
if err != nil {
return "", err
}
return iclass.Name, nil
}
// Then we try annotation
if ingressclass, ok := ing.GetAnnotations()[ingressclass.IngressKey]; ok {
if ingressclass != icConfig.AnnotationValue {
return "", fmt.Errorf("ingress class annotation is not equal to the expected by Ingress Controller")
}
return ingressclass, nil
}
// Then we accept if the WithoutClass is enabled
if icConfig.WatchWithoutClass {
// Reserving "_" as a "wildcard" name
return "_", nil
}
return "", fmt.Errorf("ingress does not contain a valid IngressClass")
}
// getIngress returns the Ingress matching key.
func (s *k8sStore) getIngress(key string) (*networkingv1.Ingress, error) {
ing, err := s.listers.IngressWithAnnotation.ByKey(key)

540
internal/ingress/controller/store/store_test.go
View file

@ -37,13 +37,47 @@ import (
"sigs.k8s.io/controller-runtime/pkg/envtest"
"k8s.io/ingress-nginx/internal/ingress"
"k8s.io/ingress-nginx/internal/ingress/annotations/class"
"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
"k8s.io/ingress-nginx/internal/ingress/controller/ingressclass"
"k8s.io/ingress-nginx/test/e2e/framework"
)
var pathPrefix networking.PathType = networking.PathTypePrefix
var DefaultClassConfig = &ingressclass.IngressClassConfiguration{
Controller: ingressclass.DefaultControllerName,
AnnotationValue: ingressclass.DefaultAnnotationValue,
WatchWithoutClass: false,
}
var (
commonIngressSpec = networking.IngressSpec{
Rules: []networking.IngressRule{
{
Host: "dummy",
IngressRuleValue: networking.IngressRuleValue{
HTTP: &networking.HTTPIngressRuleValue{
Paths: []networking.HTTPIngressPath{
{
Path: "/",
PathType: &pathPrefix,
Backend: networking.IngressBackend{
Service: &networking.IngressServiceBackend{
Name: "http-svc",
Port: networking.ServiceBackendPort{
Number: 80,
},
},
},
},
},
},
},
},
},
}
)
func TestStore(t *testing.T) {
//TODO: move env definition to docker image?
os.Setenv("KUBEBUILDER_ASSETS", "/usr/local/bin")
@ -86,7 +120,8 @@ func TestStore(t *testing.T) {
10*time.Minute,
clientSet,
updateCh,
false)
false,
DefaultClassConfig)
storer.Run(stopCh)
@ -116,9 +151,10 @@ func TestStore(t *testing.T) {
}
})
t.Run("should return one event for add, update and delete of ingress", func(t *testing.T) {
t.Run("should return no event for add, update and delete of ingress as the existing ingressclass is not the expected", func(t *testing.T) {
ns := createNamespace(clientSet, t)
defer deleteNamespace(ns, clientSet, t)
createConfigMap(clientSet, ns, t)
stopCh := make(chan struct{})
@ -163,40 +199,20 @@ func TestStore(t *testing.T) {
10*time.Minute,
clientSet,
updateCh,
false)
false,
DefaultClassConfig)
storer.Run(stopCh)
ic := createIngressClass(clientSet, t, "not-k8s.io/not-ingress-nginx")
defer deleteIngressClass(ic, clientSet, t)
validSpec := commonIngressSpec
validSpec.IngressClassName = &ic
ing := ensureIngress(&networking.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "dummy",
Name: "dummy-no-class",
Namespace: ns,
},
Spec: networking.IngressSpec{
Rules: []networking.IngressRule{
{
Host: "dummy",
IngressRuleValue: networking.IngressRuleValue{
HTTP: &networking.HTTPIngressRuleValue{
Paths: []networking.HTTPIngressPath{
{
Path: "/",
PathType: &pathPrefix,
Backend: networking.IngressBackend{
Service: &networking.IngressServiceBackend{
Name: "http-svc",
Port: networking.ServiceBackendPort{
Number: 80,
},
},
},
},
},
},
},
},
},
},
Spec: validSpec,
}, clientSet, t)
err := framework.WaitForIngressInNamespace(clientSet, ns, ing.Name)
@ -205,40 +221,113 @@ func TestStore(t *testing.T) {
}
time.Sleep(1 * time.Second)
// create an invalid ingress (different class)
ni := ing.DeepCopy()
ni.Spec.Rules[0].Host = "update-dummy"
_ = ensureIngress(ni, clientSet, t)
if err != nil {
t.Errorf("error creating ingress: %v", err)
}
// Secret takes a bit to update
time.Sleep(3 * time.Second)
err = clientSet.NetworkingV1().Ingresses(ni.Namespace).Delete(context.TODO(), ni.Name, metav1.DeleteOptions{})
if err != nil {
t.Errorf("error deleting ingress: %v", err)
}
err = framework.WaitForNoIngressInNamespace(clientSet, ni.Namespace, ni.Name)
if err != nil {
t.Errorf("error waiting for secret: %v", err)
}
time.Sleep(1 * time.Second)
if atomic.LoadUint64(&add) != 0 {
t.Errorf("expected 0 event of type Create but %v occurred", add)
}
if atomic.LoadUint64(&upd) != 0 {
t.Errorf("expected 0 event of type Update but %v occurred", upd)
}
if atomic.LoadUint64(&del) != 0 {
t.Errorf("expected 0 event of type Delete but %v occurred", del)
}
})
t.Run("should return one event for add, update and delete of ingress", func(t *testing.T) {
ns := createNamespace(clientSet, t)
defer deleteNamespace(ns, clientSet, t)
ic := createIngressClass(clientSet, t, ingressclass.DefaultControllerName)
defer deleteIngressClass(ic, clientSet, t)
createConfigMap(clientSet, ns, t)
stopCh := make(chan struct{})
updateCh := channels.NewRingChannel(1024)
var add uint64
var upd uint64
var del uint64
go func(ch *channels.RingChannel) {
for {
evt, ok := <-ch.Out()
if !ok {
return
}
e := evt.(Event)
if e.Obj == nil {
continue
}
if _, ok := e.Obj.(*networking.Ingress); !ok {
continue
}
switch e.Type {
case CreateEvent:
atomic.AddUint64(&add, 1)
case UpdateEvent:
atomic.AddUint64(&upd, 1)
case DeleteEvent:
atomic.AddUint64(&del, 1)
}
}
}(updateCh)
storer := New(
ns,
fmt.Sprintf("%v/config", ns),
fmt.Sprintf("%v/tcp", ns),
fmt.Sprintf("%v/udp", ns),
"",
10*time.Minute,
clientSet,
updateCh,
false,
DefaultClassConfig)
storer.Run(stopCh)
validSpec := commonIngressSpec
validSpec.IngressClassName = &ic
ing := ensureIngress(&networking.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "dummy-class",
Namespace: ns,
},
Spec: validSpec,
}, clientSet, t)
err := framework.WaitForIngressInNamespace(clientSet, ns, ing.Name)
if err != nil {
t.Errorf("error waiting for secret: %v", err)
}
time.Sleep(1 * time.Second)
// create an invalid ingress (no ingress class and no watchWithoutClass config)
invalidIngress := ensureIngress(&networking.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "custom-class",
Name: "no-class",
Namespace: ns,
Annotations: map[string]string{
class.IngressKey: "something",
},
},
Spec: networking.IngressSpec{
Rules: []networking.IngressRule{
{
Host: "dummy",
IngressRuleValue: networking.IngressRuleValue{
HTTP: &networking.HTTPIngressRuleValue{
Paths: []networking.HTTPIngressPath{
{
Path: "/",
PathType: &pathPrefix,
Backend: networking.IngressBackend{
Service: &networking.IngressServiceBackend{
Name: "http-svc",
Port: networking.ServiceBackendPort{
Number: 80,
},
},
},
},
},
},
},
},
},
},
Spec: commonIngressSpec,
}, clientSet, t)
defer deleteIngress(invalidIngress, clientSet, t)
@ -253,7 +342,7 @@ func TestStore(t *testing.T) {
err = clientSet.NetworkingV1().Ingresses(ni.Namespace).Delete(context.TODO(), ni.Name, metav1.DeleteOptions{})
if err != nil {
t.Errorf("error creating ingress: %v", err)
t.Errorf("error deleting ingress: %v", err)
}
err = framework.WaitForNoIngressInNamespace(clientSet, ni.Namespace, ni.Name)
@ -273,7 +362,7 @@ func TestStore(t *testing.T) {
}
})
t.Run("should not receive updates for ingress with invalid class", func(t *testing.T) {
t.Run("should return two events for add and delete and one for update of ingress and watch-without-class", func(t *testing.T) {
ns := createNamespace(clientSet, t)
defer deleteNamespace(ns, clientSet, t)
createConfigMap(clientSet, ns, t)
@ -311,6 +400,224 @@ func TestStore(t *testing.T) {
}
}(updateCh)
ingressClassconfig := &ingressclass.IngressClassConfiguration{
Controller: ingressclass.DefaultControllerName,
AnnotationValue: ingressclass.DefaultAnnotationValue,
WatchWithoutClass: true,
}
storer := New(
ns,
fmt.Sprintf("%v/config", ns),
fmt.Sprintf("%v/tcp", ns),
fmt.Sprintf("%v/udp", ns),
"",
10*time.Minute,
clientSet,
updateCh,
false,
ingressClassconfig)
storer.Run(stopCh)
validIngress1 := ensureIngress(&networking.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "ing1",
Namespace: ns,
},
Spec: commonIngressSpec,
}, clientSet, t)
err := framework.WaitForIngressInNamespace(clientSet, ns, validIngress1.Name)
if err != nil {
t.Errorf("error waiting for ingress: %v", err)
}
otherIngress := commonIngressSpec
otherIngress.Rules[0].Host = "other-ingress"
validIngress2 := ensureIngress(&networking.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "ing2",
Namespace: ns,
},
Spec: otherIngress,
}, clientSet, t)
err = framework.WaitForIngressInNamespace(clientSet, ns, validIngress2.Name)
if err != nil {
t.Errorf("error waiting for ingress: %v", err)
}
time.Sleep(1 * time.Second)
validIngressUpdated := validIngress1.DeepCopy()
validIngressUpdated.Spec.Rules[0].Host = "update-dummy"
_ = ensureIngress(validIngressUpdated, clientSet, t)
if err != nil {
t.Errorf("error updating ingress: %v", err)
}
// Secret takes a bit to update
time.Sleep(3 * time.Second)
err = clientSet.NetworkingV1().Ingresses(validIngressUpdated.Namespace).Delete(context.TODO(), validIngressUpdated.Name, metav1.DeleteOptions{})
if err != nil {
t.Errorf("error deleting ingress: %v", err)
}
err = clientSet.NetworkingV1().Ingresses(validIngress2.Namespace).Delete(context.TODO(), validIngress2.Name, metav1.DeleteOptions{})
if err != nil {
t.Errorf("error deleting ingress: %v", err)
}
err = framework.WaitForNoIngressInNamespace(clientSet, validIngressUpdated.Namespace, validIngressUpdated.Name)
if err != nil {
t.Errorf("error waiting for ingress deletion: %v", err)
}
err = framework.WaitForNoIngressInNamespace(clientSet, validIngress2.Namespace, validIngress2.Name)
if err != nil {
t.Errorf("error waiting for ingress deletion: %v", err)
}
time.Sleep(1 * time.Second)
if atomic.LoadUint64(&add) != 2 {
t.Errorf("expected 0 event of type Create but %v occurred", add)
}
if atomic.LoadUint64(&upd) != 1 {
t.Errorf("expected 0 event of type Update but %v occurred", upd)
}
if atomic.LoadUint64(&del) != 2 {
t.Errorf("expected 0 event of type Delete but %v occurred", del)
}
})
t.Run("should not receive updates for ingress with invalid class annotation", func(t *testing.T) {
ns := createNamespace(clientSet, t)
defer deleteNamespace(ns, clientSet, t)
createConfigMap(clientSet, ns, t)
stopCh := make(chan struct{})
updateCh := channels.NewRingChannel(1024)
var add uint64
var upd uint64
var del uint64
// TODO: This repeats a lot, transform in a local function
go func(ch *channels.RingChannel) {
for {
evt, ok := <-ch.Out()
if !ok {
return
}
e := evt.(Event)
if e.Obj == nil {
continue
}
if _, ok := e.Obj.(*networking.Ingress); !ok {
continue
}
switch e.Type {
case CreateEvent:
atomic.AddUint64(&add, 1)
case UpdateEvent:
atomic.AddUint64(&upd, 1)
case DeleteEvent:
atomic.AddUint64(&del, 1)
}
}
}(updateCh)
storer := New(
ns,
fmt.Sprintf("%v/config", ns),
fmt.Sprintf("%v/tcp", ns),
fmt.Sprintf("%v/udp", ns),
"",
10*time.Minute,
clientSet,
updateCh,
false,
DefaultClassConfig)
storer.Run(stopCh)
// create an invalid ingress (different class)
invalidIngress := ensureIngress(&networking.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "custom-class",
Namespace: ns,
Annotations: map[string]string{
ingressclass.IngressKey: "something",
},
},
Spec: commonIngressSpec,
}, clientSet, t)
err := framework.WaitForIngressInNamespace(clientSet, ns, invalidIngress.Name)
if err != nil {
t.Errorf("error waiting for ingress: %v", err)
}
time.Sleep(1 * time.Second)
invalidIngressUpdated := invalidIngress.DeepCopy()
invalidIngressUpdated.Spec.Rules[0].Host = "update-dummy"
_ = ensureIngress(invalidIngressUpdated, clientSet, t)
if err != nil {
t.Errorf("error creating ingress: %v", err)
}
// Secret takes a bit to update
time.Sleep(3 * time.Second)
if atomic.LoadUint64(&add) != 0 {
t.Errorf("expected 0 event of type Create but %v occurred", add)
}
if atomic.LoadUint64(&upd) != 0 {
t.Errorf("expected 0 event of type Update but %v occurred", upd)
}
if atomic.LoadUint64(&del) != 0 {
t.Errorf("expected 0 event of type Delete but %v occurred", del)
}
})
t.Run("should not receive updates for ingress with invalid class specification", func(t *testing.T) {
ns := createNamespace(clientSet, t)
defer deleteNamespace(ns, clientSet, t)
ic := createIngressClass(clientSet, t, ingressclass.DefaultControllerName)
defer deleteIngressClass(ic, clientSet, t)
createConfigMap(clientSet, ns, t)
stopCh := make(chan struct{})
updateCh := channels.NewRingChannel(1024)
var add uint64
var upd uint64
var del uint64
go func(ch *channels.RingChannel) {
for {
evt, ok := <-ch.Out()
if !ok {
return
}
e := evt.(Event)
if e.Obj == nil {
continue
}
if _, ok := e.Obj.(*networking.Ingress); !ok {
continue
}
switch e.Type {
case CreateEvent:
atomic.AddUint64(&add, 1)
case UpdateEvent:
atomic.AddUint64(&upd, 1)
case DeleteEvent:
atomic.AddUint64(&del, 1)
}
}
}(updateCh)
storer := New(
ns,
fmt.Sprintf("%v/config", ns),
@ -320,44 +627,20 @@ func TestStore(t *testing.T) {
10*time.Minute,
clientSet,
updateCh,
false)
false,
DefaultClassConfig)
storer.Run(stopCh)
invalidSpec := commonIngressSpec
invalidClassName := "blo123"
invalidSpec.IngressClassName = &invalidClassName
// create an invalid ingress (different class)
invalidIngress := ensureIngress(&networking.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "custom-class",
Namespace: ns,
Annotations: map[string]string{
class.IngressKey: "something",
},
},
Spec: networking.IngressSpec{
Rules: []networking.IngressRule{
{
Host: "dummy",
IngressRuleValue: networking.IngressRuleValue{
HTTP: &networking.HTTPIngressRuleValue{
Paths: []networking.HTTPIngressPath{
{
Path: "/",
PathType: &pathPrefix,
Backend: networking.IngressBackend{
Service: &networking.IngressServiceBackend{
Name: "http-svc",
Port: networking.ServiceBackendPort{
Number: 80,
},
},
},
},
},
},
},
},
},
},
Spec: invalidSpec,
}, clientSet, t)
err := framework.WaitForIngressInNamespace(clientSet, ns, invalidIngress.Name)
if err != nil {
@ -428,7 +711,8 @@ func TestStore(t *testing.T) {
10*time.Minute,
clientSet,
updateCh,
false)
false,
DefaultClassConfig)
storer.Run(stopCh)
@ -474,6 +758,8 @@ func TestStore(t *testing.T) {
t.Run("should receive events from secret referenced from ingress", func(t *testing.T) {
ns := createNamespace(clientSet, t)
defer deleteNamespace(ns, clientSet, t)
ic := createIngressClass(clientSet, t, ingressclass.DefaultControllerName)
defer deleteIngressClass(ic, clientSet, t)
createConfigMap(clientSet, ns, t)
stopCh := make(chan struct{})
@ -494,6 +780,11 @@ func TestStore(t *testing.T) {
if e.Obj == nil {
continue
}
// We should skip IngressClass events
if _, ok := e.Obj.(*networking.IngressClass); ok {
continue
}
switch e.Type {
case CreateEvent:
atomic.AddUint64(&add, 1)
@ -514,7 +805,8 @@ func TestStore(t *testing.T) {
10*time.Minute,
clientSet,
updateCh,
false)
false,
DefaultClassConfig)
storer.Run(stopCh)
@ -527,6 +819,7 @@ func TestStore(t *testing.T) {
Namespace: ns,
},
Spec: networking.IngressSpec{
IngressClassName: &ic,
TLS: []networking.IngressTLS{
{
SecretName: secretName,
@ -586,6 +879,8 @@ func TestStore(t *testing.T) {
t.Run("should create an ingress with a secret which does not exist", func(t *testing.T) {
ns := createNamespace(clientSet, t)
defer deleteNamespace(ns, clientSet, t)
ic := createIngressClass(clientSet, t, ingressclass.DefaultControllerName)
defer deleteIngressClass(ic, clientSet, t)
createConfigMap(clientSet, ns, t)
stopCh := make(chan struct{})
@ -606,6 +901,12 @@ func TestStore(t *testing.T) {
if e.Obj == nil {
continue
}
// We should skip IngressClass objects here
if _, ok := e.Obj.(*networking.IngressClass); ok {
continue
}
switch e.Type {
case CreateEvent:
atomic.AddUint64(&add, 1)
@ -626,7 +927,8 @@ func TestStore(t *testing.T) {
10*time.Minute,
clientSet,
updateCh,
false)
false,
DefaultClassConfig)
storer.Run(stopCh)
@ -639,6 +941,7 @@ func TestStore(t *testing.T) {
Namespace: ns,
},
Spec: networking.IngressSpec{
IngressClassName: &ic,
TLS: []networking.IngressTLS{
{
Hosts: secretHosts,
@ -733,6 +1036,33 @@ func deleteNamespace(ns string, clientSet kubernetes.Interface, t *testing.T) {
}
}
func createIngressClass(clientSet kubernetes.Interface, t *testing.T, controller string) string {
t.Helper()
ingressclass := &networking.IngressClass{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("ingress-nginx-%v", time.Now().Unix()),
//Namespace: "xpto" // TODO: We don't support namespaced ingress-class yet
},
Spec: networking.IngressClassSpec{
Controller: controller,
},
}
ic, err := clientSet.NetworkingV1().IngressClasses().Create(context.TODO(), ingressclass, metav1.CreateOptions{})
if err != nil {
t.Errorf("error creating ingress class: %v", err)
}
return ic.Name
}
func deleteIngressClass(ic string, clientSet kubernetes.Interface, t *testing.T) {
t.Helper()
err := clientSet.NetworkingV1().IngressClasses().Delete(context.TODO(), ic, metav1.DeleteOptions{})
if err != nil {
t.Errorf("error deleting the ingress class: %v", err)
}
}
func createConfigMap(clientSet kubernetes.Interface, ns string, t *testing.T) string {
t.Helper()
@ -790,6 +1120,7 @@ func newStore(t *testing.T) *k8sStore {
return &k8sStore{
listers: &Lister{
// add more listers if needed
IngressClass: IngressClassLister{cache.NewStore(cache.MetaNamespaceKeyFunc)},
Ingress: IngressLister{cache.NewStore(cache.MetaNamespaceKeyFunc)},
IngressWithAnnotation: IngressWithAnnotationsLister{cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc)},
},
@ -867,18 +1198,18 @@ func TestUpdateSecretIngressMap(t *testing.T) {
func TestListIngresses(t *testing.T) {
s := newStore(t)
invalidIngressClass := "something"
validIngressClass := ingressclass.DefaultControllerName
ingressToIgnore := &ingress.Ingress{
Ingress: networking.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "test-2",
Namespace: "testns",
Annotations: map[string]string{
class.IngressKey: "something",
},
Name: "test-2",
Namespace: "testns",
CreationTimestamp: metav1.NewTime(time.Now()),
},
Spec: networking.IngressSpec{
IngressClassName: &invalidIngressClass,
DefaultBackend: &networking.IngressBackend{
Service: &networking.IngressServiceBackend{
Name: "demo",
@ -900,6 +1231,7 @@ func TestListIngresses(t *testing.T) {
CreationTimestamp: metav1.NewTime(time.Now()),
},
Spec: networking.IngressSpec{
IngressClassName: &validIngressClass,
Rules: []networking.IngressRule{
{
Host: "foo.bar",
@ -926,13 +1258,13 @@ func TestListIngresses(t *testing.T) {
}
s.listers.IngressWithAnnotation.Add(ingressWithoutPath)
ingressWithNginxClass := &ingress.Ingress{
ingressWithNginxClassAnnotation := &ingress.Ingress{
Ingress: networking.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "test-4",
Namespace: "testns",
Annotations: map[string]string{
class.IngressKey: "nginx",
ingressclass.IngressKey: ingressclass.DefaultAnnotationValue,
},
CreationTimestamp: metav1.NewTime(time.Now()),
},
@ -963,7 +1295,7 @@ func TestListIngresses(t *testing.T) {
},
},
}
s.listers.IngressWithAnnotation.Add(ingressWithNginxClass)
s.listers.IngressWithAnnotation.Add(ingressWithNginxClassAnnotation)
ingresses := s.ListIngresses()

11
internal/ingress/metric/main.go
View file

@ -26,7 +26,6 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/ingress-nginx/internal/ingress"
"k8s.io/ingress-nginx/internal/ingress/annotations/class"
"k8s.io/ingress-nginx/internal/ingress/metric/collectors"
)
@ -66,7 +65,7 @@ type collector struct {
}
// NewCollector creates a new metric collector the for ingress controller
func NewCollector(metricsPerHost bool, registry *prometheus.Registry) (Collector, error) {
func NewCollector(metricsPerHost bool, registry *prometheus.Registry, ingressclass string) (Collector, error) {
podNamespace := os.Getenv("POD_NAMESPACE")
if podNamespace == "" {
podNamespace = "default"
@ -74,22 +73,22 @@ func NewCollector(metricsPerHost bool, registry *prometheus.Registry) (Collector
podName := os.Getenv("POD_NAME")
nc, err := collectors.NewNGINXStatus(podName, podNamespace, class.IngressClass)
nc, err := collectors.NewNGINXStatus(podName, podNamespace, ingressclass)
if err != nil {
return nil, err
}
pc, err := collectors.NewNGINXProcess(podName, podNamespace, class.IngressClass)
pc, err := collectors.NewNGINXProcess(podName, podNamespace, ingressclass)
if err != nil {
return nil, err
}
s, err := collectors.NewSocketCollector(podName, podNamespace, class.IngressClass, metricsPerHost)
s, err := collectors.NewSocketCollector(podName, podNamespace, ingressclass, metricsPerHost)
if err != nil {
return nil, err
}
ic := collectors.NewController(podName, podNamespace, class.IngressClass)
ic := collectors.NewController(podName, podNamespace, ingressclass)
return Collector(&collector{
nginxStatus: nc,

4
internal/ingress/status/status_test.go
View file

@ -29,7 +29,7 @@ import (
testclient "k8s.io/client-go/kubernetes/fake"
"k8s.io/ingress-nginx/internal/ingress"
"k8s.io/ingress-nginx/internal/ingress/annotations/class"
"k8s.io/ingress-nginx/internal/ingress/controller/ingressclass"
"k8s.io/ingress-nginx/internal/k8s"
"k8s.io/ingress-nginx/internal/task"
)
@ -214,7 +214,7 @@ func buildExtensionsIngresses() []networking.Ingress {
Name: "foo_ingress_different_class",
Namespace: metav1.NamespaceDefault,
Annotations: map[string]string{
class.IngressKey: "no-nginx",
ingressclass.IngressKey: "no-nginx",
},
},
Status: networking.IngressStatus{

3
internal/k8s/main.go
View file

@ -124,9 +124,6 @@ func MetaNamespaceKey(obj interface{}) string {
// IsIngressV1Ready indicates if the running Kubernetes version is at least v1.19.0
var IsIngressV1Ready bool
// IngressClass indicates the class of the Ingress to use as filter
var IngressClass *networkingv1.IngressClass
// IngressNGINXController defines the valid value of IngressClass
// Controller field for ingress-nginx
const IngressNGINXController = "k8s.io/ingress-nginx"