DevFW-CICD/ingress-nginx-helm
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add events for NGINX reloads

Browse source
This commit is contained in:
Manuel Alejandro de Brito Fontes 2020-09-26 20:27:19 -03:00
parent c7b041fb9e
commit 29ea30a4e8
26 changed files with 319 additions and 326 deletions
Download patch file Download diff file Expand all files Collapse all files

4
internal/ingress/controller/controller.go
View file

@ -43,6 +43,7 @@ import (
"k8s.io/ingress-nginx/internal/k8s"
"k8s.io/ingress-nginx/internal/nginx"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/apis/core"
)
const (
@ -154,12 +155,15 @@ func (n *NGINXController) syncIngress(interface{}) error {
n.metricCollector.IncReloadErrorCount()
n.metricCollector.ConfigSuccess(hash, false)
klog.Errorf("Unexpected failure reloading the backend:\n%v", err)
n.recorder.Eventf(k8s.IngressNGINXPod, core.EventTypeWarning, "RELOAD", fmt.Sprintf("Error reloading NGINX: %v", err))
return err
}
klog.Infof("Backend successfully reloaded.")
n.metricCollector.ConfigSuccess(hash, true)
n.metricCollector.IncReloadCount()
n.recorder.Eventf(k8s.IngressNGINXPod, core.EventTypeNormal, "RELOAD", "NGINX reload triggered due to a change in configuration")
}
isFirstSync := n.runningConfig.Equal(&ingress.Configuration{})

36
internal/ingress/controller/controller_test.go
View file

@ -1662,13 +1662,6 @@ func testConfigMap(ns string) *v1.ConfigMap {
func newNGINXController(t *testing.T) *NGINXController {
ns := v1.NamespaceDefault
pod := &k8s.PodInfo{
Name: "testpod",
Namespace: ns,
Labels: map[string]string{
"pod-template-hash": "1234",
},
}
clientSet := fake.NewSimpleClientset()
@ -1684,6 +1677,16 @@ func newNGINXController(t *testing.T) *NGINXController {
t.Fatalf("error creating the configuration map: %v", err)
}
k8s.IngressNGINXPod = &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "testpod",
Namespace: ns,
Labels: map[string]string{
"pod-template-hash": "1234",
},
},
}
storer := store.New(
ns,
fmt.Sprintf("%v/config", ns),
@ -1693,7 +1696,6 @@ func newNGINXController(t *testing.T) *NGINXController {
10*time.Minute,
clientSet,
channels.NewRingChannel(10),
pod,
false)
sslCert := ssl.GetFakeSSLCert()
@ -1724,13 +1726,6 @@ func fakeX509Cert(dnsNames []string) *x509.Certificate {
func newDynamicNginxController(t *testing.T, setConfigMap func(string) *v1.ConfigMap) *NGINXController {
ns := v1.NamespaceDefault
pod := &k8s.PodInfo{
Name: "testpod",
Namespace: ns,
Labels: map[string]string{
"pod-template-hash": "1234",
},
}
clientSet := fake.NewSimpleClientset()
configMap := setConfigMap(ns)
@ -1740,6 +1735,16 @@ func newDynamicNginxController(t *testing.T, setConfigMap func(string) *v1.Confi
t.Fatalf("error creating the configuration map: %v", err)
}
k8s.IngressNGINXPod = &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "testpod",
Namespace: ns,
Labels: map[string]string{
"pod-template-hash": "1234",
},
},
}
storer := store.New(
ns,
fmt.Sprintf("%v/config", ns),
@ -1749,7 +1754,6 @@ func newDynamicNginxController(t *testing.T, setConfigMap func(string) *v1.Confi
10*time.Minute,
clientSet,
channels.NewRingChannel(10),
pod,
false)
sslCert := ssl.GetFakeSSLCert()

14
internal/ingress/controller/nginx.go
View file

@ -56,7 +56,6 @@ import (
ngx_template "k8s.io/ingress-nginx/internal/ingress/controller/template"
"k8s.io/ingress-nginx/internal/ingress/metric"
"k8s.io/ingress-nginx/internal/ingress/status"
"k8s.io/ingress-nginx/internal/k8s"
ing_net "k8s.io/ingress-nginx/internal/net"
"k8s.io/ingress-nginx/internal/net/dns"
"k8s.io/ingress-nginx/internal/net/ssl"
@ -118,12 +117,6 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro
}
}
pod, err := k8s.GetPodDetails(config.Client)
if err != nil {
klog.Fatalf("unexpected error obtaining pod information: %v", err)
}
n.podInfo = pod
n.store = store.New(
config.Namespace,
config.ConfigMapName,
@ -133,13 +126,12 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro
config.ResyncPeriod,
config.Client,
n.updateCh,
pod,
config.DisableCatchAll)
n.syncQueue = task.NewTaskQueue(n.syncIngress)
if config.UpdateStatus {
n.syncStatus = status.NewStatusSyncer(pod, status.Config{
n.syncStatus = status.NewStatusSyncer(status.Config{
Client: config.Client,
PublishService: config.PublishService,
PublishStatusAddress: config.PublishStatusAddress,
@ -213,8 +205,6 @@ Error loading new template: %v
// NGINXController describes a NGINX Ingress controller.
type NGINXController struct {
podInfo *k8s.PodInfo
cfg *Configuration
recorder record.EventRecorder
@ -287,8 +277,6 @@ func (n *NGINXController) Start() {
OnStoppedLeading: func() {
n.metricCollector.OnStoppedLeading(electionID)
},
PodName: n.podInfo.Name,
PodNamespace: n.podInfo.Namespace,
})
cmd := n.command.ExecCommand()

14
internal/ingress/controller/status.go
View file

@ -21,6 +21,7 @@ import (
"os"
"time"
"k8s.io/ingress-nginx/internal/k8s"
"k8s.io/klog/v2"
apiv1 "k8s.io/api/core/v1"
@ -33,9 +34,6 @@ import (
)
type leaderElectionConfig struct {
PodName string
PodNamespace string
Client clientset.Interface
ElectionID string
@ -95,17 +93,21 @@ func setupLeaderElection(config *leaderElectionConfig) {
Host: hostname,
})
ingressPod, err := k8s.GetPodDetails()
if err != nil {
klog.Fatalf("unexpected error starting leader election: %v", err)
}
lock := resourcelock.ConfigMapLock{
ConfigMapMeta: metav1.ObjectMeta{Namespace: config.PodNamespace, Name: config.ElectionID},
ConfigMapMeta: metav1.ObjectMeta{Namespace: ingressPod.Namespace, Name: config.ElectionID},
Client: config.Client.CoreV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: config.PodName,
Identity: ingressPod.Name,
EventRecorder: recorder,
},
}
ttl := 30 * time.Second
var err error
elector, err = leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
Lock: &lock,

11
internal/ingress/controller/store/store.go
View file

@ -220,8 +220,6 @@ type k8sStore struct {
backendConfigMu *sync.RWMutex
defaultSSLCertificate string
pod *k8s.PodInfo
}
// New creates a new object store to be used in the ingress controller
@ -230,7 +228,6 @@ func New(
resyncPeriod time.Duration,
client clientset.Interface,
updateCh *channels.RingChannel,
pod *k8s.PodInfo,
disableCatchAll bool) Storer {
store := &k8sStore{
@ -243,7 +240,6 @@ func New(
backendConfigMu: &sync.RWMutex{},
secretIngressMap: NewObjectRefMap(),
defaultSSLCertificate: defaultSSLCertificate,
pod: pod,
}
eventBroadcaster := record.NewBroadcaster()
@ -294,16 +290,17 @@ func New(
store.informers.Service = infFactory.Core().V1().Services().Informer()
store.listers.Service.Store = store.informers.Service.GetStore()
labelSelector := labels.SelectorFromSet(store.pod.Labels)
ingressPodInfo, _ := k8s.GetPodDetails()
labelSelector := labels.SelectorFromSet(ingressPodInfo.Labels)
store.informers.Pod = cache.NewSharedIndexInformer(
&cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (k8sruntime.Object, error) {
options.LabelSelector = labelSelector.String()
return client.CoreV1().Pods(store.pod.Namespace).List(context.TODO(), options)
return client.CoreV1().Pods(ingressPodInfo.Namespace).List(context.TODO(), options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
options.LabelSelector = labelSelector.String()
return client.CoreV1().Pods(store.pod.Namespace).Watch(context.TODO(), options)
return client.CoreV1().Pods(ingressPodInfo.Namespace).Watch(context.TODO(), options)
},
},
&corev1.Pod{},

46
internal/ingress/controller/store/store_test.go
View file

@ -47,11 +47,13 @@ import (
func TestStore(t *testing.T) {
k8s.IsNetworkingIngressAvailable = true
pod := &k8s.PodInfo{
Name: "testpod",
Namespace: v1.NamespaceDefault,
Labels: map[string]string{
"pod-template-hash": "1234",
k8s.IngressNGINXPod = &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "testpod",
Namespace: v1.NamespaceDefault,
Labels: map[string]string{
"pod-template-hash": "1234",
},
},
}
@ -94,7 +96,6 @@ func TestStore(t *testing.T) {
10*time.Minute,
clientSet,
updateCh,
pod,
false)
storer.Run(stopCh)
@ -172,7 +173,6 @@ func TestStore(t *testing.T) {
10*time.Minute,
clientSet,
updateCh,
pod,
false)
storer.Run(stopCh)
@ -320,7 +320,6 @@ func TestStore(t *testing.T) {
10*time.Minute,
clientSet,
updateCh,
pod,
false)
storer.Run(stopCh)
@ -424,7 +423,6 @@ func TestStore(t *testing.T) {
10*time.Minute,
clientSet,
updateCh,
pod,
false)
storer.Run(stopCh)
@ -511,7 +509,6 @@ func TestStore(t *testing.T) {
10*time.Minute,
clientSet,
updateCh,
pod,
false)
storer.Run(stopCh)
@ -620,7 +617,6 @@ func TestStore(t *testing.T) {
10*time.Minute,
clientSet,
updateCh,
pod,
false)
storer.Run(stopCh)
@ -777,11 +773,13 @@ func deleteIngress(ingress *networking.Ingress, clientSet kubernetes.Interface,
// newStore creates a new mock object store for tests which do not require the
// use of Informers.
func newStore(t *testing.T) *k8sStore {
pod := &k8s.PodInfo{
Name: "ingress-1",
Namespace: v1.NamespaceDefault,
Labels: map[string]string{
"pod-template-hash": "1234",
k8s.IngressNGINXPod = &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "ingress-1",
Namespace: v1.NamespaceDefault,
Labels: map[string]string{
"pod-template-hash": "1234",
},
},
}
@ -797,7 +795,6 @@ func newStore(t *testing.T) *k8sStore {
syncSecretMu: new(sync.Mutex),
backendConfigMu: new(sync.RWMutex),
secretIngressMap: NewObjectRefMap(),
pod: pod,
}
}
@ -1011,15 +1008,18 @@ func TestGetRunningControllerPodsCount(t *testing.T) {
os.Setenv("POD_NAMESPACE", "testns")
os.Setenv("POD_NAME", "ingress-1")
s := newStore(t)
s.pod = &k8s.PodInfo{
Name: "ingress-1",
Namespace: "testns",
Labels: map[string]string{
"pod-template-hash": "1234",
k8s.IngressNGINXPod = &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "ingress-1",
Namespace: "testns",
Labels: map[string]string{
"pod-template-hash": "1234",
},
},
}
s := newStore(t)
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "ingress-1",

25
internal/ingress/status/status.go
View file

@ -82,9 +82,6 @@ type Config struct {
type statusSync struct {
Config
// pod contains runtime information about this pod
pod *k8s.PodInfo
// workqueue used to keep in sync the status IP/s
// in the Ingress rules
syncQueue *task.Queue
@ -158,10 +155,8 @@ func (s statusSync) keyfunc(input interface{}) (interface{}, error) {
}
// NewStatusSyncer returns a new Syncer instance
func NewStatusSyncer(podInfo *k8s.PodInfo, config Config) Syncer {
func NewStatusSyncer(config Config) Syncer {
st := statusSync{
pod: podInfo,
Config: config,
}
st.syncQueue = task.NewCustomTaskQueue(st.sync, st.keyfunc)
@ -180,9 +175,14 @@ func (s *statusSync) runningAddresses() ([]string, error) {
return statusAddressFromService(s.PublishService, s.Client)
}
ingressPod, err := k8s.GetPodDetails()
if err != nil {
return nil, err
}
// get information about all the pods running the ingress controller
pods, err := s.Client.CoreV1().Pods(s.pod.Namespace).List(context.TODO(), metav1.ListOptions{
LabelSelector: labels.SelectorFromSet(s.pod.Labels).String(),
pods, err := s.Client.CoreV1().Pods(ingressPod.Namespace).List(context.TODO(), metav1.ListOptions{
LabelSelector: labels.SelectorFromSet(ingressPod.Labels).String(),
})
if err != nil {
return nil, err
@ -205,8 +205,13 @@ func (s *statusSync) runningAddresses() ([]string, error) {
}
func (s *statusSync) isRunningMultiplePods() bool {
pods, err := s.Client.CoreV1().Pods(s.pod.Namespace).List(context.TODO(), metav1.ListOptions{
LabelSelector: labels.SelectorFromSet(s.pod.Labels).String(),
ingressPod, err := k8s.GetPodDetails()
if err != nil {
return false
}
pods, err := s.Client.CoreV1().Pods(ingressPod.Namespace).List(context.TODO(), metav1.ListOptions{
LabelSelector: labels.SelectorFromSet(ingressPod.Labels).String(),
})
if err != nil {
return false

26
internal/ingress/status/status_test.go
View file

@ -24,6 +24,7 @@ import (
"time"
apiv1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
networking "k8s.io/api/networking/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
testclient "k8s.io/client-go/kubernetes/fake"
@ -264,13 +265,6 @@ func buildIngressLister() ingressLister {
func buildStatusSync() statusSync {
return statusSync{
pod: &k8s.PodInfo{
Name: "foo_base_pod",
Namespace: apiv1.NamespaceDefault,
Labels: map[string]string{
"lable_sig": "foo_pod",
},
},
syncQueue: task.NewTaskQueue(fakeSynFn),
Config: Config{
Client: buildSimpleClientSet(),
@ -291,14 +285,18 @@ func TestStatusActions(t *testing.T) {
UpdateStatusOnShutdown: true,
}
// create object
fkSync := NewStatusSyncer(&k8s.PodInfo{
Name: "foo_base_pod",
Namespace: apiv1.NamespaceDefault,
Labels: map[string]string{
"lable_sig": "foo_pod",
k8s.IngressNGINXPod = &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "foo_base_pod",
Namespace: apiv1.NamespaceDefault,
Labels: map[string]string{
"lable_sig": "foo_pod",
},
},
}, c)
}
// create object
fkSync := NewStatusSyncer(c)
if fkSync == nil {
t.Fatalf("expected a valid Sync")
}

32
internal/k8s/main.go
View file

@ -75,6 +75,9 @@ func GetNodeIPOrName(kubeClient clientset.Interface, name string, useInternalIP
return defaultOrInternalIP
}
// IngressNGINXPod hold information about the ingress-nginx pod
var IngressNGINXPod *apiv1.Pod
// PodInfo contains runtime information about the pod running the Ingres controller
type PodInfo struct {
Name string
@ -84,25 +87,34 @@ type PodInfo struct {
Labels map[string]string
}
// GetPodDetails returns runtime information about the pod:
// name, namespace and IP of the node where it is running
func GetPodDetails(kubeClient clientset.Interface) (*PodInfo, error) {
// GetIngressPod load the ingress-nginx pod
func GetIngressPod(kubeClient clientset.Interface) error {
podName := os.Getenv("POD_NAME")
podNs := os.Getenv("POD_NAMESPACE")
if podName == "" || podNs == "" {
return nil, fmt.Errorf("unable to get POD information (missing POD_NAME or POD_NAMESPACE environment variable")
return fmt.Errorf("unable to get POD information (missing POD_NAME or POD_NAMESPACE environment variable")
}
pod, _ := kubeClient.CoreV1().Pods(podNs).Get(context.TODO(), podName, metav1.GetOptions{})
if pod == nil {
return nil, fmt.Errorf("unable to get POD information")
IngressNGINXPod, _ = kubeClient.CoreV1().Pods(podNs).Get(context.TODO(), podName, metav1.GetOptions{})
if IngressNGINXPod == nil {
return fmt.Errorf("unable to get POD information")
}
return nil
}
// GetPodDetails returns runtime information about the pod:
// name, namespace and IP of the node where it is running
func GetPodDetails() (*PodInfo, error) {
if IngressNGINXPod == nil {
return nil, fmt.Errorf("no ingress-nginx pod details available")
}
return &PodInfo{
Name: podName,
Namespace: podNs,
Labels: pod.GetLabels(),
Name: IngressNGINXPod.Name,
Namespace: IngressNGINXPod.Namespace,
Labels: IngressNGINXPod.GetLabels(),
}, nil
}

14
internal/k8s/main_test.go
View file

@ -218,7 +218,7 @@ func TestGetPodDetails(t *testing.T) {
// POD_NAME & POD_NAMESPACE not exist
os.Setenv("POD_NAME", "")
os.Setenv("POD_NAMESPACE", "")
_, err1 := GetPodDetails(testclient.NewSimpleClientset())
err1 := GetIngressPod(testclient.NewSimpleClientset())
if err1 == nil {
t.Errorf("expected an error but returned nil")
}
@ -226,7 +226,7 @@ func TestGetPodDetails(t *testing.T) {
// POD_NAME not exist
os.Setenv("POD_NAME", "")
os.Setenv("POD_NAMESPACE", apiv1.NamespaceDefault)
_, err2 := GetPodDetails(testclient.NewSimpleClientset())
err2 := GetIngressPod(testclient.NewSimpleClientset())
if err2 == nil {
t.Errorf("expected an error but returned nil")
}
@ -234,7 +234,7 @@ func TestGetPodDetails(t *testing.T) {
// POD_NAMESPACE not exist
os.Setenv("POD_NAME", "testpod")
os.Setenv("POD_NAMESPACE", "")
_, err3 := GetPodDetails(testclient.NewSimpleClientset())
err3 := GetIngressPod(testclient.NewSimpleClientset())
if err3 == nil {
t.Errorf("expected an error but returned nil")
}
@ -242,7 +242,7 @@ func TestGetPodDetails(t *testing.T) {
// POD not exist
os.Setenv("POD_NAME", "testpod")
os.Setenv("POD_NAMESPACE", apiv1.NamespaceDefault)
_, err4 := GetPodDetails(testclient.NewSimpleClientset())
err4 := GetIngressPod(testclient.NewSimpleClientset())
if err4 == nil {
t.Errorf("expected an error but returned nil")
}
@ -273,13 +273,9 @@ func TestGetPodDetails(t *testing.T) {
},
}}})
epi, err5 := GetPodDetails(fkClient)
err5 := GetIngressPod(fkClient)
if err5 != nil {
t.Errorf("expected a PodInfo but returned error")
return
}
if epi == nil {
t.Errorf("expected a PodInfo but returned nil")
}
}