Refactor controller metrics interface
This commit is contained in:
Manuel de Brito Fontes
Manuel Alejandro de Brito Fontes
parent
bdd2c5e3be
commit
1542a12764
30 changed files with 1896 additions and 630 deletions
|
|
@ -25,6 +25,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/golang/glog"
|
||||
"github.com/mitchellh/hashstructure"
|
||||
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
extensions "k8s.io/api/extensions/v1beta1"
|
||||
|
|
@ -148,38 +149,43 @@ func (n *NGINXController) syncIngress(interface{}) error {
|
|||
}
|
||||
}
|
||||
|
||||
pcfg := ingress.Configuration{
|
||||
Backends: upstreams,
|
||||
Servers: servers,
|
||||
TCPEndpoints: n.getStreamServices(n.cfg.TCPConfigMapName, apiv1.ProtocolTCP),
|
||||
UDPEndpoints: n.getStreamServices(n.cfg.UDPConfigMapName, apiv1.ProtocolUDP),
|
||||
PassthroughBackends: passUpstreams,
|
||||
|
||||
ConfigurationChecksum: n.store.GetBackendConfiguration().Checksum,
|
||||
pcfg := &ingress.Configuration{
|
||||
Backends: upstreams,
|
||||
Servers: servers,
|
||||
TCPEndpoints: n.getStreamServices(n.cfg.TCPConfigMapName, apiv1.ProtocolTCP),
|
||||
UDPEndpoints: n.getStreamServices(n.cfg.UDPConfigMapName, apiv1.ProtocolUDP),
|
||||
PassthroughBackends: passUpstreams,
|
||||
BackendConfigChecksum: n.store.GetBackendConfiguration().Checksum,
|
||||
}
|
||||
|
||||
if n.runningConfig.Equal(&pcfg) {
|
||||
if n.runningConfig.Equal(pcfg) {
|
||||
glog.V(3).Infof("No configuration change detected, skipping backend reload.")
|
||||
return nil
|
||||
}
|
||||
|
||||
if n.cfg.DynamicConfigurationEnabled && n.IsDynamicConfigurationEnough(&pcfg) {
|
||||
if n.cfg.DynamicConfigurationEnabled && n.IsDynamicConfigurationEnough(pcfg) {
|
||||
glog.Infof("Changes handled by the dynamic configuration, skipping backend reload.")
|
||||
} else {
|
||||
glog.Infof("Configuration changes detected, backend reload required.")
|
||||
|
||||
err := n.OnUpdate(pcfg)
|
||||
hash, _ := hashstructure.Hash(pcfg, &hashstructure.HashOptions{
|
||||
TagName: "json",
|
||||
})
|
||||
|
||||
pcfg.ConfigurationChecksum = fmt.Sprintf("%v", hash)
|
||||
|
||||
err := n.OnUpdate(*pcfg)
|
||||
if err != nil {
|
||||
IncReloadErrorCount()
|
||||
ConfigSuccess(false)
|
||||
n.metricCollector.IncReloadErrorCount()
|
||||
n.metricCollector.ConfigSuccess(hash, false)
|
||||
glog.Errorf("Unexpected failure reloading the backend:\n%v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
glog.Infof("Backend successfully reloaded.")
|
||||
ConfigSuccess(true)
|
||||
IncReloadCount()
|
||||
setSSLExpireTime(servers)
|
||||
n.metricCollector.ConfigSuccess(hash, true)
|
||||
n.metricCollector.IncReloadCount()
|
||||
n.metricCollector.SetSSLExpireTime(servers)
|
||||
}
|
||||
|
||||
if n.cfg.DynamicConfigurationEnabled {
|
||||
|
|
@ -191,7 +197,7 @@ func (n *NGINXController) syncIngress(interface{}) error {
|
|||
// it takes time for NGINX to start listening on the configured ports
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
err := configureDynamically(&pcfg, n.cfg.ListenPorts.Status)
|
||||
err := configureDynamically(pcfg, n.cfg.ListenPorts.Status)
|
||||
if err == nil {
|
||||
glog.Infof("Dynamic reconfiguration succeeded.")
|
||||
} else {
|
||||
|
|
@ -200,7 +206,11 @@ func (n *NGINXController) syncIngress(interface{}) error {
|
|||
}(isFirstSync)
|
||||
}
|
||||
|
||||
n.runningConfig = &pcfg
|
||||
ri := getRemovedIngresses(n.runningConfig, pcfg)
|
||||
re := getRemovedHosts(n.runningConfig, pcfg)
|
||||
n.metricCollector.RemoveMetrics(ri, re)
|
||||
|
||||
n.runningConfig = pcfg
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -1112,3 +1122,57 @@ func extractTLSSecretName(host string, ing *extensions.Ingress,
|
|||
|
||||
return ""
|
||||
}
|
||||
|
||||
// getRemovedHosts returns a list of the hostsnames
|
||||
// that are not associated anymore to the NGINX configuration.
|
||||
func getRemovedHosts(rucfg, newcfg *ingress.Configuration) []string {
|
||||
old := sets.NewString()
|
||||
new := sets.NewString()
|
||||
|
||||
for _, s := range rucfg.Servers {
|
||||
if !old.Has(s.Hostname) {
|
||||
old.Insert(s.Hostname)
|
||||
}
|
||||
}
|
||||
|
||||
for _, s := range newcfg.Servers {
|
||||
if !new.Has(s.Hostname) {
|
||||
new.Insert(s.Hostname)
|
||||
}
|
||||
}
|
||||
|
||||
return old.Difference(new).List()
|
||||
}
|
||||
|
||||
func getRemovedIngresses(rucfg, newcfg *ingress.Configuration) []string {
|
||||
oldIngresses := sets.NewString()
|
||||
newIngresses := sets.NewString()
|
||||
|
||||
for _, server := range rucfg.Servers {
|
||||
for _, location := range server.Locations {
|
||||
if location.Ingress == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
ingKey := k8s.MetaNamespaceKey(location.Ingress)
|
||||
if !oldIngresses.Has(ingKey) {
|
||||
oldIngresses.Insert(ingKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, server := range newcfg.Servers {
|
||||
for _, location := range server.Locations {
|
||||
if location.Ingress == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
ingKey := k8s.MetaNamespaceKey(location.Ingress)
|
||||
if !newIngresses.Has(ingKey) {
|
||||
newIngresses.Insert(ingKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return oldIngresses.Difference(newIngresses).List()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,120 +0,0 @@
|
|||
/*
|
||||
Copyright 2015 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package controller
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"k8s.io/ingress-nginx/internal/ingress"
|
||||
)
|
||||
|
||||
const (
|
||||
ns = "ingress_controller"
|
||||
operation = "count"
|
||||
reloadLabel = "reloads"
|
||||
sslLabelExpire = "ssl_expire_time_seconds"
|
||||
sslLabelHost = "host"
|
||||
)
|
||||
|
||||
func init() {
|
||||
prometheus.MustRegister(reloadOperation)
|
||||
prometheus.MustRegister(reloadOperationErrors)
|
||||
prometheus.MustRegister(sslExpireTime)
|
||||
prometheus.MustRegister(configSuccess)
|
||||
prometheus.MustRegister(configSuccessTime)
|
||||
}
|
||||
|
||||
var (
|
||||
configSuccess = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Name: "config_last_reload_successfull",
|
||||
Help: `Whether the last configuration reload attemp was successful.
|
||||
Prometheus alert example:
|
||||
alert: IngressControllerFailedReload
|
||||
expr: ingress_controller_config_last_reload_successfull == 0
|
||||
for: 10m`,
|
||||
})
|
||||
configSuccessTime = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Name: "config_last_reload_successfull_timestamp_seconds",
|
||||
Help: "Timestamp of the last successful configuration reload.",
|
||||
})
|
||||
// TODO depreciate this metrics in favor of ingress_controller_config_last_reload_successfull_timestamp_seconds
|
||||
reloadOperation = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "success",
|
||||
Help: `DEPRECATED: use ingress_controller_config_last_reload_successfull_timestamp_seconds or ingress_controller_config_last_reload_successfull instead.
|
||||
Cumulative number of Ingress controller reload operations`,
|
||||
},
|
||||
[]string{operation},
|
||||
)
|
||||
// TODO depreciate this metrics in favor of ingress_controller_config_last_reload_successfull_timestamp_seconds
|
||||
reloadOperationErrors = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "errors",
|
||||
Help: `DEPRECATED: use ingress_controller_config_last_reload_successfull_timestamp_seconds or ingress_controller_config_last_reload_successfull instead.
|
||||
Cumulative number of Ingress controller errors during reload operations`,
|
||||
},
|
||||
[]string{operation},
|
||||
)
|
||||
sslExpireTime = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Name: sslLabelExpire,
|
||||
Help: "Number of seconds since 1970 to the SSL Certificate expire. An example to check if this " +
|
||||
"certificate will expire in 10 days is: \"ingress_controller_ssl_expire_time_seconds < (time() + (10 * 24 * 3600))\"",
|
||||
},
|
||||
[]string{sslLabelHost},
|
||||
)
|
||||
)
|
||||
|
||||
// IncReloadCount increment the reload counter
|
||||
func IncReloadCount() {
|
||||
reloadOperation.WithLabelValues(reloadLabel).Inc()
|
||||
}
|
||||
|
||||
// IncReloadErrorCount increment the reload error counter
|
||||
func IncReloadErrorCount() {
|
||||
reloadOperationErrors.WithLabelValues(reloadLabel).Inc()
|
||||
}
|
||||
|
||||
// ConfigSuccess set a boolean flag according to the output of the controller configuration reload
|
||||
func ConfigSuccess(success bool) {
|
||||
if success {
|
||||
ConfigSuccessTime()
|
||||
configSuccess.Set(1)
|
||||
} else {
|
||||
configSuccess.Set(0)
|
||||
}
|
||||
}
|
||||
|
||||
// ConfigSuccessTime set the current timestamp when the controller is successfully reloaded
|
||||
func ConfigSuccessTime() {
|
||||
configSuccessTime.Set(float64(time.Now().Unix()))
|
||||
}
|
||||
|
||||
func setSSLExpireTime(servers []*ingress.Server) {
|
||||
for _, s := range servers {
|
||||
if s.Hostname != defServerName {
|
||||
sslExpireTime.WithLabelValues(s.Hostname).Set(float64(s.SSLCert.ExpireTime.Unix()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -53,6 +53,7 @@ import (
|
|||
"k8s.io/ingress-nginx/internal/ingress/controller/process"
|
||||
"k8s.io/ingress-nginx/internal/ingress/controller/store"
|
||||
ngx_template "k8s.io/ingress-nginx/internal/ingress/controller/template"
|
||||
"k8s.io/ingress-nginx/internal/ingress/metric"
|
||||
"k8s.io/ingress-nginx/internal/ingress/status"
|
||||
ing_net "k8s.io/ingress-nginx/internal/net"
|
||||
"k8s.io/ingress-nginx/internal/net/dns"
|
||||
|
|
@ -70,7 +71,7 @@ var (
|
|||
)
|
||||
|
||||
// NewNGINXController creates a new NGINX Ingress controller.
|
||||
func NewNGINXController(config *Configuration, fs file.Filesystem) *NGINXController {
|
||||
func NewNGINXController(config *Configuration, mc metric.Collector, fs file.Filesystem) *NGINXController {
|
||||
eventBroadcaster := record.NewBroadcaster()
|
||||
eventBroadcaster.StartLogging(glog.Infof)
|
||||
eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{
|
||||
|
|
@ -103,6 +104,8 @@ func NewNGINXController(config *Configuration, fs file.Filesystem) *NGINXControl
|
|||
runningConfig: new(ingress.Configuration),
|
||||
|
||||
Proxy: &TCPProxy{},
|
||||
|
||||
metricCollector: mc,
|
||||
}
|
||||
|
||||
n.store = store.New(
|
||||
|
|
@ -243,6 +246,8 @@ type NGINXController struct {
|
|||
store store.Storer
|
||||
|
||||
fileSystem filesystem.Filesystem
|
||||
|
||||
metricCollector metric.Collector
|
||||
}
|
||||
|
||||
// Start starts a new NGINX master process running in the foreground.
|
||||
|
|
@ -590,6 +595,8 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
|
|||
DisableLua: n.cfg.DisableLua,
|
||||
}
|
||||
|
||||
tc.Cfg.Checksum = ingressCfg.ConfigurationChecksum
|
||||
|
||||
content, err := n.t.Write(tc)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -144,11 +144,11 @@ func TestConfigureDynamically(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
body := string(b)
|
||||
if strings.Index(body, "target") != -1 {
|
||||
if strings.Contains(body, "target") {
|
||||
t.Errorf("unexpected target reference in JSON content: %v", body)
|
||||
}
|
||||
|
||||
if strings.Index(body, "service") != -1 {
|
||||
if strings.Contains(body, "service") {
|
||||
t.Errorf("unexpected service reference in JSON content: %v", body)
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue