DevFW-CICD/ingress-nginx-helm
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor controller metrics interface

Browse source
This commit is contained in:
Manuel de Brito Fontes 2018-07-07 13:46:18 -04:00 • committed by Manuel Alejandro de Brito Fontes
parent bdd2c5e3be
commit 1542a12764
No known key found for this signature in database
GPG key ID: 786136016A8BA02A
30 changed files with 1896 additions and 630 deletions
Download patch file Download diff file Expand all files Collapse all files

218
internal/ingress/metric/collectors/controller.go Normal file
View file

@ -0,0 +1,218 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package collectors
import (
"time"
"github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/ingress-nginx/internal/ingress"
)
var (
operation = []string{"namespace", "class"}
sslLabelHost = []string{"namespace", "class", "host"}
)
// Controller defines base metrics about the ingress controller
type Controller struct {
prometheus.Collector
configHash prometheus.Gauge
configSuccess prometheus.Gauge
configSuccessTime prometheus.Gauge
reloadOperation *prometheus.CounterVec
reloadOperationErrors *prometheus.CounterVec
sslExpireTime *prometheus.GaugeVec
labels prometheus.Labels
}
// NewController creates a new prometheus collector for the
// Ingress controller operations
func NewController(pod, namespace, class string) *Controller {
constLabels := prometheus.Labels{
"controller_namespace": namespace,
"controller_class": class,
"controller_pod": pod,
}
cm := &Controller{
labels: prometheus.Labels{
"namespace": namespace,
"class": class,
},
configHash: prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: PrometheusNamespace,
Name: "config_hash",
Help: "Running configuration hash actually running",
ConstLabels: constLabels,
},
),
configSuccess: prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: PrometheusNamespace,
Name: "config_last_reload_successful",
Help: "Whether the last configuration reload attemp was successful",
ConstLabels: constLabels,
}),
configSuccessTime: prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: PrometheusNamespace,
Name: "config_last_reload_successful_timestamp_seconds",
Help: "Timestamp of the last successful configuration reload.",
ConstLabels: constLabels,
}),
reloadOperation: prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: PrometheusNamespace,
Name: "success",
Help: `Cumulative number of Ingress controller reload operations`,
},
operation,
),
reloadOperationErrors: prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: PrometheusNamespace,
Name: "errors",
Help: `Cumulative number of Ingress controller errors during reload operations`,
},
operation,
),
sslExpireTime: prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: PrometheusNamespace,
Name: "ssl_expire_time_seconds",
Help: `Number of seconds since 1970 to the SSL Certificate expire.
An example to check if this certificate will expire in 10 days is: "nginx_ingress_controller_ssl_expire_time_seconds < (time() + (10 * 24 * 3600))"`,
},
sslLabelHost,
),
}
return cm
}
// IncReloadCount increment the reload counter
func (cm *Controller) IncReloadCount() {
cm.reloadOperation.With(cm.labels).Inc()
}
// IncReloadErrorCount increment the reload error counter
func (cm *Controller) IncReloadErrorCount() {
cm.reloadOperationErrors.With(cm.labels).Inc()
}
// ConfigSuccess set a boolean flag according to the output of the controller configuration reload
func (cm *Controller) ConfigSuccess(hash uint64, success bool) {
if success {
cm.configSuccessTime.Set(float64(time.Now().Unix()))
cm.configSuccess.Set(1)
cm.configHash.Set(float64(hash))
return
}
cm.configSuccess.Set(0)
cm.configHash.Set(0)
}
// Describe implements prometheus.Collector
func (cm Controller) Describe(ch chan<- *prometheus.Desc) {
cm.configHash.Describe(ch)
cm.configSuccess.Describe(ch)
cm.configSuccessTime.Describe(ch)
cm.reloadOperation.Describe(ch)
cm.reloadOperationErrors.Describe(ch)
cm.sslExpireTime.Describe(ch)
}
// Collect implements the prometheus.Collector interface.
func (cm Controller) Collect(ch chan<- prometheus.Metric) {
cm.configHash.Collect(ch)
cm.configSuccess.Collect(ch)
cm.configSuccessTime.Collect(ch)
cm.reloadOperation.Collect(ch)
cm.reloadOperationErrors.Collect(ch)
cm.sslExpireTime.Collect(ch)
}
// SetSSLExpireTime sets the expiration time of SSL Certificates
func (cm *Controller) SetSSLExpireTime(servers []*ingress.Server) {
for _, s := range servers {
if s.Hostname != "" && s.SSLCert.ExpireTime.Unix() > 0 {
labels := make(prometheus.Labels, len(cm.labels)+1)
for k, v := range cm.labels {
labels[k] = v
}
labels["host"] = s.Hostname
cm.sslExpireTime.With(labels).Set(float64(s.SSLCert.ExpireTime.Unix()))
}
}
}
// RemoveMetrics removes metrics for hostames not available anymore
func (cm *Controller) RemoveMetrics(hosts []string, registry prometheus.Gatherer) {
mfs, err := registry.Gather()
if err != nil {
glog.Errorf("Error gathering metrics: %v", err)
return
}
glog.V(2).Infof("removing SSL certificate metrics for %v hosts", hosts)
toRemove := sets.NewString(hosts...)
for _, mf := range mfs {
metricName := mf.GetName()
if "ssl_expire_time_seconds" != metricName {
continue
}
for _, m := range mf.GetMetric() {
labels := make(map[string]string, len(m.GetLabel()))
for _, labelPair := range m.GetLabel() {
labels[*labelPair.Name] = *labelPair.Value
}
// remove labels that are constant
deleteConstants(labels)
host, ok := labels["host"]
if !ok {
continue
}
if !toRemove.Has(host) {
continue
}
glog.V(2).Infof("Removing prometheus metric from gauge %v for host %v", metricName, host)
removed := cm.sslExpireTime.Delete(labels)
if !removed {
glog.V(2).Infof("metric %v for host %v with labels not removed: %v", metricName, host, labels)
}
}
}
}

122
internal/ingress/metric/collectors/controller_test.go Normal file
View file

@ -0,0 +1,122 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package collectors
import (
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/ingress-nginx/internal/ingress"
)
func TestControllerCounters(t *testing.T) {
const metadata = `
# HELP nginx_ingress_controller_config_last_reload_successful Whether the last configuration reload attemp was successful
# TYPE nginx_ingress_controller_config_last_reload_successful gauge
# HELP nginx_ingress_controller_success Cumulative number of Ingress controller reload operations
# TYPE nginx_ingress_controller_success counter
`
cases := []struct {
name string
test func(*Controller)
metrics []string
want string
}{
{
name: "should return not increment in metrics if no operations are invoked",
test: func(cm *Controller) {
},
want: metadata + `
nginx_ingress_controller_config_last_reload_successful{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 0
`,
metrics: []string{"nginx_ingress_controller_config_last_reload_successful", "nginx_ingress_controller_success"},
},
{
name: "single increase in reload count should return 1",
test: func(cm *Controller) {
cm.IncReloadCount()
cm.ConfigSuccess(0, true)
},
want: metadata + `
nginx_ingress_controller_config_last_reload_successful{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 1
nginx_ingress_controller_success{class="nginx",namespace="default"} 1
`,
metrics: []string{"nginx_ingress_controller_config_last_reload_successful", "nginx_ingress_controller_success"},
},
{
name: "single increase in error reload count should return 1",
test: func(cm *Controller) {
cm.IncReloadErrorCount()
},
want: `
# HELP nginx_ingress_controller_errors Cumulative number of Ingress controller errors during reload operations
# TYPE nginx_ingress_controller_errors counter
nginx_ingress_controller_errors{class="nginx",namespace="default"} 1
`,
metrics: []string{"nginx_ingress_controller_errors"},
},
{
name: "should set SSL certificates metrics",
test: func(cm *Controller) {
t1, _ := time.Parse(
time.RFC3339,
"2012-11-01T22:08:41+00:00")
servers := []*ingress.Server{
{
Hostname: "demo",
SSLCert: ingress.SSLCert{
ExpireTime: t1,
},
},
{
Hostname: "invalid",
SSLCert: ingress.SSLCert{
ExpireTime: time.Unix(0, 0),
},
},
}
cm.SetSSLExpireTime(servers)
},
want: `
# HELP nginx_ingress_controller_ssl_expire_time_seconds Number of seconds since 1970 to the SSL Certificate expire.\n An example to check if this certificate will expire in 10 days is: "nginx_ingress_controller_ssl_expire_time_seconds < (time() + (10 * 24 * 3600))"
# TYPE nginx_ingress_controller_ssl_expire_time_seconds gauge
nginx_ingress_controller_ssl_expire_time_seconds{class="nginx",host="demo",namespace="default"} 1.351807721e+09
`,
metrics: []string{"nginx_ingress_controller_ssl_expire_time_seconds"},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
cm := NewController("pod", "default", "nginx")
reg := prometheus.NewPedanticRegistry()
if err := reg.Register(cm); err != nil {
t.Errorf("registering collector failed: %s", err)
}
c.test(cm)
if err := GatherAndCompare(cm, c.want, c.metrics, reg); err != nil {
t.Errorf("unexpected collecting result:\n%s", err)
}
reg.Unregister(cm)
})
}
}

20
internal/ingress/metric/collectors/main.go Normal file
View file

@ -0,0 +1,20 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package collectors
// PrometheusNamespace default metric namespace
var PrometheusNamespace = "nginx_ingress_controller"

225
internal/ingress/metric/collectors/nginx_status.go Normal file
View file

@ -0,0 +1,225 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package collectors
import (
"fmt"
"io/ioutil"
"net/http"
"regexp"
"strconv"
"github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
)
var (
ac = regexp.MustCompile(`Active connections: (\d+)`)
sahr = regexp.MustCompile(`(\d+)\s(\d+)\s(\d+)`)
reading = regexp.MustCompile(`Reading: (\d+)`)
writing = regexp.MustCompile(`Writing: (\d+)`)
waiting = regexp.MustCompile(`Waiting: (\d+)`)
)
type (
nginxStatusCollector struct {
scrapeChan chan scrapeRequest
ngxHealthPort int
ngxStatusPath string
data *nginxStatusData
}
nginxStatusData struct {
connectionsTotal *prometheus.Desc
requestsTotal *prometheus.Desc
connections *prometheus.Desc
}
basicStatus struct {
// Active total number of active connections
Active int
// Accepted total number of accepted client connections
Accepted int
// Handled total number of handled connections. Generally, the parameter value is the same as accepts unless some resource limits have been reached (for example, the worker_connections limit).
Handled int
// Requests total number of client requests.
Requests int
// Reading current number of connections where nginx is reading the request header.
Reading int
// Writing current number of connections where nginx is writing the response back to the client.
Writing int
// Waiting current number of idle client connections waiting for a request.
Waiting int
}
)
// NGINXStatusCollector defines a status collector interface
type NGINXStatusCollector interface {
prometheus.Collector
Start()
Stop()
}
// NewNGINXStatus returns a new prometheus collector the default nginx status module
func NewNGINXStatus(podName, namespace, ingressClass string, ngxHealthPort int) (NGINXStatusCollector, error) {
p := nginxStatusCollector{
scrapeChan: make(chan scrapeRequest),
ngxHealthPort: ngxHealthPort,
ngxStatusPath: "/nginx_status",
}
constLabels := prometheus.Labels{
"controller_namespace": namespace,
"controller_class": ingressClass,
"controller_pod": podName,
}
p.data = &nginxStatusData{
connectionsTotal: prometheus.NewDesc(
prometheus.BuildFQName(PrometheusNamespace, subSystem, "connections_total"),
"total number of connections with state {active, accepted, handled}",
[]string{"state"}, constLabels),
requestsTotal: prometheus.NewDesc(
prometheus.BuildFQName(PrometheusNamespace, subSystem, "requests_total"),
"total number of client requests",
nil, constLabels),
connections: prometheus.NewDesc(
prometheus.BuildFQName(PrometheusNamespace, subSystem, "connections"),
"current number of client connections with state {reading, writing, waiting}",
[]string{"state"}, constLabels),
}
return p, nil
}
// Describe implements prometheus.Collector.
func (p nginxStatusCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- p.data.connectionsTotal
ch <- p.data.requestsTotal
ch <- p.data.connections
}
// Collect implements prometheus.Collector.
func (p nginxStatusCollector) Collect(ch chan<- prometheus.Metric) {
req := scrapeRequest{results: ch, done: make(chan struct{})}
p.scrapeChan <- req
<-req.done
}
func (p nginxStatusCollector) Start() {
for req := range p.scrapeChan {
ch := req.results
p.scrape(ch)
req.done <- struct{}{}
}
}
func (p nginxStatusCollector) Stop() {
close(p.scrapeChan)
}
func httpBody(url string) ([]byte, error) {
resp, err := http.DefaultClient.Get(url)
if err != nil {
return nil, fmt.Errorf("unexpected error scraping nginx : %v", err)
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("unexpected error scraping nginx (%v)", err)
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 400 {
return nil, fmt.Errorf("unexpected error scraping nginx (status %v)", resp.StatusCode)
}
return data, nil
}
func toInt(data []string, pos int) int {
if len(data) == 0 {
return 0
}
if pos > len(data) {
return 0
}
if v, err := strconv.Atoi(data[pos]); err == nil {
return v
}
return 0
}
func parse(data string) *basicStatus {
acr := ac.FindStringSubmatch(data)
sahrr := sahr.FindStringSubmatch(data)
readingr := reading.FindStringSubmatch(data)
writingr := writing.FindStringSubmatch(data)
waitingr := waiting.FindStringSubmatch(data)
return &basicStatus{
toInt(acr, 1),
toInt(sahrr, 1),
toInt(sahrr, 2),
toInt(sahrr, 3),
toInt(readingr, 1),
toInt(writingr, 1),
toInt(waitingr, 1),
}
}
func getNginxStatus(port int, path string) (*basicStatus, error) {
url := fmt.Sprintf("http://0.0.0.0:%v%v", port, path)
glog.V(3).Infof("start scraping url: %v", url)
data, err := httpBody(url)
if err != nil {
return nil, fmt.Errorf("unexpected error scraping nginx status page: %v", err)
}
return parse(string(data)), nil
}
// nginxStatusCollector scrape the nginx status
func (p nginxStatusCollector) scrape(ch chan<- prometheus.Metric) {
s, err := getNginxStatus(p.ngxHealthPort, p.ngxStatusPath)
if err != nil {
glog.Warningf("unexpected error obtaining nginx status info: %v", err)
return
}
ch <- prometheus.MustNewConstMetric(p.data.connectionsTotal,
prometheus.CounterValue, float64(s.Active), "active")
ch <- prometheus.MustNewConstMetric(p.data.connectionsTotal,
prometheus.CounterValue, float64(s.Accepted), "accepted")
ch <- prometheus.MustNewConstMetric(p.data.connectionsTotal,
prometheus.CounterValue, float64(s.Handled), "handled")
ch <- prometheus.MustNewConstMetric(p.data.requestsTotal,
prometheus.CounterValue, float64(s.Requests))
ch <- prometheus.MustNewConstMetric(p.data.connections,
prometheus.GaugeValue, float64(s.Reading), "reading")
ch <- prometheus.MustNewConstMetric(p.data.connections,
prometheus.GaugeValue, float64(s.Writing), "writing")
ch <- prometheus.MustNewConstMetric(p.data.connections,
prometheus.GaugeValue, float64(s.Waiting), "waiting")
}

129
internal/ingress/metric/collectors/nginx_status_test.go Normal file
View file

@ -0,0 +1,129 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package collectors
import (
"fmt"
"net"
"net/http"
"net/http/httptest"
"testing"
"github.com/prometheus/client_golang/prometheus"
)
func TestStatusCollector(t *testing.T) {
cases := []struct {
name string
mock string
metrics []string
want string
}{
{
name: "should return empty metrics",
mock: `
`,
want: `
# HELP nginx_ingress_controller_nginx_process_connections_total total number of connections with state {active, accepted, handled}
# TYPE nginx_ingress_controller_nginx_process_connections_total counter
nginx_ingress_controller_nginx_process_connections_total{controller_class="nginx",controller_namespace="default",controller_pod="pod",state="accepted"} 0
nginx_ingress_controller_nginx_process_connections_total{controller_class="nginx",controller_namespace="default",controller_pod="pod",state="active"} 0
nginx_ingress_controller_nginx_process_connections_total{controller_class="nginx",controller_namespace="default",controller_pod="pod",state="handled"} 0
`,
metrics: []string{"nginx_ingress_controller_nginx_process_connections_total"},
},
{
name: "should return metrics for total connections",
mock: `
Active connections: 1
server accepts handled requests
1 2 3
Reading: 4 Writing: 5 Waiting: 6
`,
want: `
# HELP nginx_ingress_controller_nginx_process_connections_total total number of connections with state {active, accepted, handled}
# TYPE nginx_ingress_controller_nginx_process_connections_total counter
nginx_ingress_controller_nginx_process_connections_total{controller_class="nginx",controller_namespace="default",controller_pod="pod",state="accepted"} 1
nginx_ingress_controller_nginx_process_connections_total{controller_class="nginx",controller_namespace="default",controller_pod="pod",state="active"} 1
nginx_ingress_controller_nginx_process_connections_total{controller_class="nginx",controller_namespace="default",controller_pod="pod",state="handled"} 2
`,
metrics: []string{"nginx_ingress_controller_nginx_process_connections_total"},
},
{
name: "should return nginx metrics all available metrics",
mock: `
Active connections: 1
server accepts handled requests
1 2 3
Reading: 4 Writing: 5 Waiting: 6
`,
want: `
# HELP nginx_ingress_controller_nginx_process_connections current number of client connections with state {reading, writing, waiting}
# TYPE nginx_ingress_controller_nginx_process_connections gauge
nginx_ingress_controller_nginx_process_connections{controller_class="nginx",controller_namespace="default",controller_pod="pod",state="reading"} 4
nginx_ingress_controller_nginx_process_connections{controller_class="nginx",controller_namespace="default",controller_pod="pod",state="waiting"} 6
nginx_ingress_controller_nginx_process_connections{controller_class="nginx",controller_namespace="default",controller_pod="pod",state="writing"} 5
# HELP nginx_ingress_controller_nginx_process_connections_total total number of connections with state {active, accepted, handled}
# TYPE nginx_ingress_controller_nginx_process_connections_total counter
nginx_ingress_controller_nginx_process_connections_total{controller_class="nginx",controller_namespace="default",controller_pod="pod",state="accepted"} 1
nginx_ingress_controller_nginx_process_connections_total{controller_class="nginx",controller_namespace="default",controller_pod="pod",state="active"} 1
nginx_ingress_controller_nginx_process_connections_total{controller_class="nginx",controller_namespace="default",controller_pod="pod",state="handled"} 2
# HELP nginx_ingress_controller_nginx_process_requests_total total number of client requests
# TYPE nginx_ingress_controller_nginx_process_requests_total counter
nginx_ingress_controller_nginx_process_requests_total{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 3
`,
metrics: []string{
"nginx_ingress_controller_nginx_process_connections_total",
"nginx_ingress_controller_nginx_process_requests_total",
"nginx_ingress_controller_nginx_process_connections",
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
fmt.Fprintf(w, c.mock)
}))
p := server.Listener.Addr().(*net.TCPAddr).Port
cm, err := NewNGINXStatus("pod", "default", "nginx", p)
if err != nil {
t.Errorf("unexpected error creating nginx status collector: %v", err)
}
go cm.Start()
defer func() {
server.Close()
cm.Stop()
}()
reg := prometheus.NewPedanticRegistry()
if err := reg.Register(cm); err != nil {
t.Errorf("registering collector failed: %s", err)
}
if err := GatherAndCompare(cm, c.want, c.metrics, reg); err != nil {
t.Errorf("unexpected collecting result:\n%s", err)
}
reg.Unregister(cm)
})
}
}

209
internal/ingress/metric/collectors/process.go Normal file
View file

@ -0,0 +1,209 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package collectors
import (
"path/filepath"
"github.com/golang/glog"
common "github.com/ncabatoff/process-exporter"
"github.com/ncabatoff/process-exporter/proc"
"github.com/prometheus/client_golang/prometheus"
)
type scrapeRequest struct {
results chan<- prometheus.Metric
done chan struct{}
}
// Stopable defines a prometheus collector that can be stopped
type Stopable interface {
prometheus.Collector
Stop()
}
// BinaryNameMatcher ...
type BinaryNameMatcher struct {
Name string
Binary string
}
// MatchAndName returns false if the match failed, otherwise
// true and the resulting name.
func (em BinaryNameMatcher) MatchAndName(nacl common.NameAndCmdline) (bool, string) {
if len(nacl.Cmdline) == 0 {
return false, ""
}
cmd := filepath.Base(em.Binary)
return em.Name == cmd, ""
}
type namedProcessData struct {
numProcs *prometheus.Desc
cpuSecs *prometheus.Desc
readBytes *prometheus.Desc
writeBytes *prometheus.Desc
memResidentbytes *prometheus.Desc
memVirtualbytes *prometheus.Desc
startTime *prometheus.Desc
}
type namedProcess struct {
*proc.Grouper
scrapeChan chan scrapeRequest
fs *proc.FS
data namedProcessData
}
const subSystem = "nginx_process"
// NGINXProcessCollector defines a process collector interface
type NGINXProcessCollector interface {
prometheus.Collector
Start()
Stop()
}
var name = "nginx"
var binary = "/usr/bin/nginx"
// NewNGINXProcess returns a new prometheus collector for the nginx process
func NewNGINXProcess(pod, namespace, ingressClass string) (NGINXProcessCollector, error) {
fs, err := proc.NewFS("/proc")
if err != nil {
return nil, err
}
nm := BinaryNameMatcher{
Name: name,
Binary: binary,
}
p := namedProcess{
scrapeChan: make(chan scrapeRequest),
Grouper: proc.NewGrouper(true, nm),
fs: fs,
}
_, err = p.Update(p.fs.AllProcs())
if err != nil {
return nil, err
}
constLabels := prometheus.Labels{
"controller_namespace": namespace,
"controller_class": ingressClass,
"controller_pod": pod,
}
p.data = namedProcessData{
numProcs: prometheus.NewDesc(
prometheus.BuildFQName(PrometheusNamespace, subSystem, "num_procs"),
"number of processes",
nil, constLabels),
cpuSecs: prometheus.NewDesc(
prometheus.BuildFQName(PrometheusNamespace, subSystem, "cpu_seconds_total"),
"Cpu usage in seconds",
nil, constLabels),
readBytes: prometheus.NewDesc(
prometheus.BuildFQName(PrometheusNamespace, subSystem, "read_bytes_total"),
"number of bytes read",
nil, constLabels),
writeBytes: prometheus.NewDesc(
prometheus.BuildFQName(PrometheusNamespace, subSystem, "write_bytes_total"),
"number of bytes written",
nil, constLabels),
memResidentbytes: prometheus.NewDesc(
prometheus.BuildFQName(PrometheusNamespace, subSystem, "resident_memory_bytes"),
"number of bytes of memory in use",
nil, constLabels),
memVirtualbytes: prometheus.NewDesc(
prometheus.BuildFQName(PrometheusNamespace, subSystem, "virtual_memory_bytes"),
"number of bytes of memory in use",
nil, constLabels),
startTime: prometheus.NewDesc(
prometheus.BuildFQName(PrometheusNamespace, subSystem, "oldest_start_time_seconds"),
"start time in seconds since 1970/01/01",
nil, constLabels),
}
return p, nil
}
// Describe implements prometheus.Collector.
func (p namedProcess) Describe(ch chan<- *prometheus.Desc) {
ch <- p.data.cpuSecs
ch <- p.data.numProcs
ch <- p.data.readBytes
ch <- p.data.writeBytes
ch <- p.data.memResidentbytes
ch <- p.data.memVirtualbytes
ch <- p.data.startTime
}
// Collect implements prometheus.Collector.
func (p namedProcess) Collect(ch chan<- prometheus.Metric) {
req := scrapeRequest{results: ch, done: make(chan struct{})}
p.scrapeChan <- req
<-req.done
}
func (p namedProcess) Start() {
for req := range p.scrapeChan {
ch := req.results
p.scrape(ch)
req.done <- struct{}{}
}
}
func (p namedProcess) Stop() {
close(p.scrapeChan)
}
func (p namedProcess) scrape(ch chan<- prometheus.Metric) {
_, err := p.Update(p.fs.AllProcs())
if err != nil {
glog.Warningf("unexpected error obtaining nginx process info: %v", err)
return
}
for _, gcounts := range p.Groups() {
ch <- prometheus.MustNewConstMetric(p.data.numProcs,
prometheus.GaugeValue, float64(gcounts.Procs))
ch <- prometheus.MustNewConstMetric(p.data.memResidentbytes,
prometheus.GaugeValue, float64(gcounts.Memresident))
ch <- prometheus.MustNewConstMetric(p.data.memVirtualbytes,
prometheus.GaugeValue, float64(gcounts.Memvirtual))
ch <- prometheus.MustNewConstMetric(p.data.startTime,
prometheus.GaugeValue, float64(gcounts.OldestStartTime.Unix()))
ch <- prometheus.MustNewConstMetric(p.data.cpuSecs,
prometheus.CounterValue, gcounts.Cpu)
ch <- prometheus.MustNewConstMetric(p.data.readBytes,
prometheus.CounterValue, float64(gcounts.ReadBytes))
ch <- prometheus.MustNewConstMetric(p.data.writeBytes,
prometheus.CounterValue, float64(gcounts.WriteBytes))
}
}

93
internal/ingress/metric/collectors/process_test.go Normal file
View file

@ -0,0 +1,93 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package collectors
import (
"os/exec"
"syscall"
"testing"
"github.com/prometheus/client_golang/prometheus"
)
func TestProcessCollector(t *testing.T) {
cases := []struct {
name string
metrics []string
}{
{
name: "should return metrics",
metrics: []string{"nginx_ingress_controller_nginx_process_num_procs"},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
name = "sleep"
binary = "/bin/sleep"
cmd := exec.Command(binary, "1000000")
err := cmd.Start()
if err != nil {
t.Errorf("unexpected error creating dummy process: %v", err)
}
done := make(chan struct{})
go func() {
cmd.Wait()
status := cmd.ProcessState.Sys().(syscall.WaitStatus)
if status.Signaled() {
t.Logf("Signal: %v", status.Signal())
} else {
t.Logf("Status: %v", status.ExitStatus())
}
}()
cm, err := NewNGINXProcess("pod", "default", "nginx")
if err != nil {
t.Errorf("unexpected error creating nginx status collector: %v", err)
}
go cm.Start()
defer func() {
cm.Stop()
cmd.Process.Kill()
close(done)
}()
reg := prometheus.NewPedanticRegistry()
if err := reg.Register(cm); err != nil {
t.Errorf("registering collector failed: %s", err)
}
metrics, err := reg.Gather()
if err != nil {
t.Errorf("gathering metrics failed: %s", err)
}
m := filterMetrics(metrics, c.metrics)
if *m[0].GetMetric()[0].Gauge.Value < 0 {
t.Errorf("number of process should be > 0")
}
reg.Unregister(cm)
})
}
}

421
internal/ingress/metric/collectors/socket.go Normal file
View file

@ -0,0 +1,421 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package collectors
import (
"encoding/json"
"fmt"
"io"
"net"
"github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/apimachinery/pkg/util/sets"
)
type upstream struct {
Endpoint string `json:"endpoint"`
Latency float64 `json:"upstreamLatency"`
ResponseLength float64 `json:"upstreamResponseLength"`
ResponseTime float64 `json:"upstreamResponseTime"`
Status string `json:"upstreamStatus"`
}
type socketData struct {
Host string `json:"host"`
Status string `json:"status"`
ResponseLength float64 `json:"responseLength"`
Method string `json:"method"`
RequestLength float64 `json:"requestLength"`
RequestTime float64 `json:"requestTime"`
upstream
Namespace string `json:"namespace"`
Ingress string `json:"ingress"`
Service string `json:"service"`
Path string `json:"path"`
}
// SocketCollector stores prometheus metrics and ingress meta-data
type SocketCollector struct {
prometheus.Collector
requestTime *prometheus.HistogramVec
requestLength *prometheus.HistogramVec
responseTime *prometheus.HistogramVec
responseLength *prometheus.HistogramVec
upstreamLatency *prometheus.SummaryVec
bytesSent *prometheus.HistogramVec
requests *prometheus.CounterVec
listener net.Listener
metricMapping map[string]interface{}
}
var (
requestTags = []string{
"host",
"status",
"method",
"path",
// "endpoint",
"namespace",
"ingress",
"service",
}
)
// NewSocketCollector creates a new SocketCollector instance using
// the ingresss watch namespace and class used by the controller
func NewSocketCollector(pod, namespace, class string) (*SocketCollector, error) {
listener, err := net.Listen("unix", "/tmp/prometheus-nginx.socket")
if err != nil {
return nil, err
}
constLabels := prometheus.Labels{
"controller_namespace": namespace,
"controller_class": class,
"controller_pod": pod,
}
sc := &SocketCollector{
listener: listener,
responseTime: prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "response_duration_milliseconds",
Help: "The time spent on receiving the response from the upstream server",
Namespace: PrometheusNamespace,
ConstLabels: constLabels,
},
requestTags,
),
responseLength: prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "response_size",
Help: "The response length (including request line, header, and request body)",
Namespace: PrometheusNamespace,
ConstLabels: constLabels,
},
requestTags,
),
requestTime: prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "request_duration_milliseconds",
Help: "The request processing time in milliseconds",
Namespace: PrometheusNamespace,
ConstLabels: constLabels,
},
requestTags,
),
requestLength: prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "request_size",
Help: "The request length (including request line, header, and request body)",
Namespace: PrometheusNamespace,
Buckets: prometheus.LinearBuckets(10, 10, 10), // 10 buckets, each 10 bytes wide.
ConstLabels: constLabels,
},
requestTags,
),
requests: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "requests",
Help: "The total number of client requests.",
Namespace: PrometheusNamespace,
ConstLabels: constLabels,
},
[]string{"ingress", "namespace", "status"},
),
bytesSent: prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "bytes_sent",
Help: "The the number of bytes sent to a client",
Namespace: PrometheusNamespace,
Buckets: prometheus.ExponentialBuckets(10, 10, 7), // 7 buckets, exponential factor of 10.
ConstLabels: constLabels,
},
requestTags,
),
upstreamLatency: prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "ingress_upstream_latency_milliseconds",
Help: "Upstream service latency per Ingress",
Namespace: PrometheusNamespace,
ConstLabels: constLabels,
},
[]string{"ingress", "namespace", "service"},
),
}
sc.metricMapping = map[string]interface{}{
prometheus.BuildFQName(PrometheusNamespace, "", "request_duration_milliseconds"): sc.requestTime,
prometheus.BuildFQName(PrometheusNamespace, "", "request_size"): sc.requestLength,
prometheus.BuildFQName(PrometheusNamespace, "", "response_duration_milliseconds"): sc.responseTime,
prometheus.BuildFQName(PrometheusNamespace, "", "response_size"): sc.responseLength,
prometheus.BuildFQName(PrometheusNamespace, "", "bytes_sent"): sc.bytesSent,
prometheus.BuildFQName(PrometheusNamespace, "", "ingress_upstream_latency_milliseconds"): sc.upstreamLatency,
}
return sc, nil
}
func (sc *SocketCollector) handleMessage(msg []byte) {
glog.V(5).Infof("msg: %v", string(msg))
// Unmarshall bytes
var stats socketData
err := json.Unmarshal(msg, &stats)
if err != nil {
glog.Errorf("Unexpected error deserializing JSON paylod: %v", err)
return
}
requestLabels := prometheus.Labels{
"host": stats.Host,
"status": stats.Status,
"method": stats.Method,
"path": stats.Path,
//"endpoint": stats.Endpoint,
"namespace": stats.Namespace,
"ingress": stats.Ingress,
"service": stats.Service,
}
collectorLabels := prometheus.Labels{
"namespace": stats.Namespace,
"ingress": stats.Ingress,
"status": stats.Status,
}
latencyLabels := prometheus.Labels{
"namespace": stats.Namespace,
"ingress": stats.Ingress,
"service": stats.Service,
}
requestsMetric, err := sc.requests.GetMetricWith(collectorLabels)
if err != nil {
glog.Errorf("Error fetching requests metric: %v", err)
} else {
requestsMetric.Inc()
}
latencyMetric, err := sc.upstreamLatency.GetMetricWith(latencyLabels)
if err != nil {
glog.Errorf("Error fetching latency metric: %v", err)
} else {
latencyMetric.Observe(stats.Latency)
}
if stats.RequestTime != -1 {
requestTimeMetric, err := sc.requestTime.GetMetricWith(requestLabels)
if err != nil {
glog.Errorf("Error fetching request duration metric: %v", err)
} else {
requestTimeMetric.Observe(stats.RequestTime)
}
}
if stats.RequestLength != -1 {
requestLengthMetric, err := sc.requestLength.GetMetricWith(requestLabels)
if err != nil {
glog.Errorf("Error fetching request length metric: %v", err)
} else {
requestLengthMetric.Observe(stats.RequestLength)
}
}
if stats.ResponseTime != -1 {
responseTimeMetric, err := sc.responseTime.GetMetricWith(requestLabels)
if err != nil {
glog.Errorf("Error fetching upstream response time metric: %v", err)
} else {
responseTimeMetric.Observe(stats.ResponseTime)
}
}
if stats.ResponseLength != -1 {
bytesSentMetric, err := sc.bytesSent.GetMetricWith(requestLabels)
if err != nil {
glog.Errorf("Error fetching bytes sent metric: %v", err)
} else {
bytesSentMetric.Observe(stats.ResponseLength)
}
responseSizeMetric, err := sc.responseLength.GetMetricWith(requestLabels)
if err != nil {
glog.Errorf("Error fetching bytes sent metric: %v", err)
} else {
responseSizeMetric.Observe(stats.ResponseLength)
}
}
}
// Start listen for connections in the unix socket and spawns a goroutine to process the content
func (sc *SocketCollector) Start() {
for {
conn, err := sc.listener.Accept()
if err != nil {
continue
}
go handleMessages(conn, sc.handleMessage)
}
}
// Stop stops unix listener
func (sc *SocketCollector) Stop() {
sc.listener.Close()
}
// RemoveMetrics deletes prometheus metrics from prometheus for ingresses and
// host that are not available anymore.
// Ref: https://godoc.org/github.com/prometheus/client_golang/prometheus#CounterVec.Delete
func (sc *SocketCollector) RemoveMetrics(ingresses []string, registry prometheus.Gatherer) {
mfs, err := registry.Gather()
if err != nil {
glog.Errorf("Error gathering metrics: %v", err)
return
}
// 1. remove metrics of removed ingresses
glog.V(2).Infof("removing ingresses %v from metrics", ingresses)
for _, mf := range mfs {
metricName := mf.GetName()
metric, ok := sc.metricMapping[metricName]
if !ok {
continue
}
toRemove := sets.NewString(ingresses...)
for _, m := range mf.GetMetric() {
labels := make(map[string]string, len(m.GetLabel()))
for _, labelPair := range m.GetLabel() {
labels[*labelPair.Name] = *labelPair.Value
}
// remove labels that are constant
deleteConstants(labels)
ns, ok := labels["namespace"]
if !ok {
continue
}
ing, ok := labels["ingress"]
if !ok {
continue
}
ingKey := fmt.Sprintf("%v/%v", ns, ing)
if !toRemove.Has(ingKey) {
continue
}
glog.V(2).Infof("Removing prometheus metric from histogram %v for ingress %v", metricName, ingKey)
h, ok := metric.(*prometheus.HistogramVec)
if ok {
removed := h.Delete(labels)
if !removed {
glog.V(2).Infof("metric %v for ingress %v with labels not removed: %v", metricName, ingKey, labels)
}
}
s, ok := metric.(*prometheus.SummaryVec)
if ok {
removed := s.Delete(labels)
if !removed {
glog.V(2).Infof("metric %v for ingress %v with labels not removed: %v", metricName, ingKey, labels)
}
}
}
}
}
// Describe implements prometheus.Collector
func (sc SocketCollector) Describe(ch chan<- *prometheus.Desc) {
sc.requestTime.Describe(ch)
sc.requestLength.Describe(ch)
sc.requests.Describe(ch)
sc.upstreamLatency.Describe(ch)
sc.responseTime.Describe(ch)
sc.responseLength.Describe(ch)
sc.bytesSent.Describe(ch)
}
// Collect implements the prometheus.Collector interface.
func (sc SocketCollector) Collect(ch chan<- prometheus.Metric) {
sc.requestTime.Collect(ch)
sc.requestLength.Collect(ch)
sc.requests.Collect(ch)
sc.upstreamLatency.Collect(ch)
sc.responseTime.Collect(ch)
sc.responseLength.Collect(ch)
sc.bytesSent.Collect(ch)
}
const packetSize = 1024 * 65
// handleMessages process the content received in a network connection
func handleMessages(conn io.ReadCloser, fn func([]byte)) {
defer conn.Close()
msg := make([]byte, packetSize)
s, err := conn.Read(msg[0:])
if err != nil {
return
}
fn(msg[0:s])
}
func deleteConstants(labels prometheus.Labels) {
delete(labels, "controller_namespace")
delete(labels, "controller_class")
delete(labels, "controller_pod")
}

262
internal/ingress/metric/collectors/socket_test.go Normal file
View file

@ -0,0 +1,262 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package collectors
import (
"fmt"
"net"
"sync/atomic"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
)
func TestNewUDPLogListener(t *testing.T) {
var count uint64
fn := func(message []byte) {
atomic.AddUint64(&count, 1)
}
tmpFile := fmt.Sprintf("/tmp/test-socket-%v", time.Now().Nanosecond())
l, err := net.Listen("unix", tmpFile)
if err != nil {
t.Fatalf("unexpected error creating unix socket: %v", err)
}
if l == nil {
t.Fatalf("expected a listener but none returned")
}
defer l.Close()
go func() {
for {
conn, err := l.Accept()
if err != nil {
continue
}
go handleMessages(conn, fn)
}
}()
conn, _ := net.Dial("unix", tmpFile)
conn.Write([]byte("message"))
conn.Close()
time.Sleep(1 * time.Millisecond)
if atomic.LoadUint64(&count) != 1 {
t.Errorf("expected only one message from the socket listener but %v returned", atomic.LoadUint64(&count))
}
}
func TestCollector(t *testing.T) {
cases := []struct {
name string
data []string
metrics []string
wantBefore string
removeIngresses []string
wantAfter string
}{
{
name: "invalid metric object should not increase prometheus metrics",
data: []string{`#missing {
"host":"testshop.com",
"status":"200",
"bytesSent":150.0,
"method":"GET",
"path":"/admin",
"requestLength":300.0,
"requestTime":60.0,
"upstreamName":"test-upstream",
"upstreamIP":"1.1.1.1:8080",
"upstreamResponseTime":200,
"upstreamStatus":"220",
"namespace":"test-app-production",
"ingress":"web-yml",
"service":"test-app"
}`},
metrics: []string{"nginx_ingress_controller_response_duration_milliseconds"},
wantBefore: `
`,
},
{
name: "valid metric object should update prometheus metrics",
data: []string{`{
"host":"testshop.com",
"status":"200",
"bytesSent":150.0,
"method":"GET",
"path":"/admin",
"requestLength":300.0,
"requestTime":60.0,
"upstreamName":"test-upstream",
"upstreamIP":"1.1.1.1:8080",
"upstreamResponseTime":200,
"upstreamStatus":"220",
"namespace":"test-app-production",
"ingress":"web-yml",
"service":"test-app"
}`},
metrics: []string{"nginx_ingress_controller_response_duration_milliseconds"},
wantBefore: `
# HELP nginx_ingress_controller_response_duration_milliseconds The time spent on receiving the response from the upstream server
# TYPE nginx_ingress_controller_response_duration_milliseconds histogram
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.005"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.01"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.025"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.05"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.1"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.25"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.5"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="1"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="2.5"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="5"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="10"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="+Inf"} 1
nginx_ingress_controller_response_duration_milliseconds_sum{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 200
nginx_ingress_controller_response_duration_milliseconds_count{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 1
`,
removeIngresses: []string{"test-app-production/web-yml"},
wantAfter: `
`,
},
{
name: "multiple messages should increase prometheus metric by two",
data: []string{`{
"host":"testshop.com",
"status":"200",
"bytesSent":150.0,
"method":"GET",
"path":"/admin",
"requestLength":300.0,
"requestTime":60.0,
"upstreamName":"test-upstream",
"upstreamIP":"1.1.1.1:8080",
"upstreamResponseTime":200,
"upstreamStatus":"220",
"namespace":"test-app-production",
"ingress":"web-yml",
"service":"test-app"
}`, `{
"host":"testshop.com",
"status":"200",
"bytesSent":150.0,
"method":"GET",
"path":"/admin",
"requestLength":300.0,
"requestTime":60.0,
"upstreamName":"test-upstream",
"upstreamIP":"1.1.1.1:8080",
"upstreamResponseTime":200,
"upstreamStatus":"220",
"namespace":"test-app-qa",
"ingress":"web-yml-qa",
"service":"test-app-qa"
}`, `{
"host":"testshop.com",
"status":"200",
"bytesSent":150.0,
"method":"GET",
"path":"/admin",
"requestLength":300.0,
"requestTime":60.0,
"upstreamName":"test-upstream",
"upstreamIP":"1.1.1.1:8080",
"upstreamResponseTime":200,
"upstreamStatus":"220",
"namespace":"test-app-qa",
"ingress":"web-yml-qa",
"service":"test-app-qa"
}`},
metrics: []string{"nginx_ingress_controller_response_duration_milliseconds"},
wantBefore: `
# HELP nginx_ingress_controller_response_duration_milliseconds The time spent on receiving the response from the upstream server
# TYPE nginx_ingress_controller_response_duration_milliseconds histogram
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.005"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.01"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.025"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.05"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.1"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.25"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.5"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="1"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="2.5"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="5"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="10"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="+Inf"} 1
nginx_ingress_controller_response_duration_milliseconds_sum{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 200
nginx_ingress_controller_response_duration_milliseconds_count{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 1
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.005"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.01"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.025"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.05"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.1"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.25"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.5"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="1"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="2.5"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="5"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="10"} 0
nginx_ingress_controller_response_duration_milliseconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="+Inf"} 2
nginx_ingress_controller_response_duration_milliseconds_sum{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200"} 400
nginx_ingress_controller_response_duration_milliseconds_count{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200"} 2
`,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
registry := prometheus.NewPedanticRegistry()
sc, err := NewSocketCollector("pod", "default", "ingress")
if err != nil {
t.Errorf("%v: unexpected error creating new SocketCollector: %v", c.name, err)
}
if err := registry.Register(sc); err != nil {
t.Errorf("registering collector failed: %s", err)
}
for _, d := range c.data {
sc.handleMessage([]byte(d))
}
if err := GatherAndCompare(sc, c.wantBefore, c.metrics, registry); err != nil {
t.Errorf("unexpected collecting result:\n%s", err)
}
if len(c.removeIngresses) > 0 {
sc.RemoveMetrics(c.removeIngresses, registry)
time.Sleep(1 * time.Second)
if err := GatherAndCompare(sc, c.wantAfter, c.metrics, registry); err != nil {
t.Errorf("unexpected collecting result:\n%s", err)
}
}
sc.Stop()
registry.Unregister(sc)
})
}
}

183
internal/ingress/metric/collectors/testutils.go Normal file
View file

@ -0,0 +1,183 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package collectors
import (
"bytes"
"fmt"
"reflect"
"sort"
"strings"
"github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/common/expfmt"
)
// GatherAndCompare retrieves all metrics exposed by a collector and compares it
// to an expected output in the Prometheus text exposition format.
// metricNames allows only comparing the given metrics. All are compared if it's nil.
func GatherAndCompare(c prometheus.Collector, expected string, metricNames []string, reg prometheus.Gatherer) error {
expected = removeUnusedWhitespace(expected)
metrics, err := reg.Gather()
if err != nil {
return fmt.Errorf("gathering metrics failed: %s", err)
}
if metricNames != nil {
metrics = filterMetrics(metrics, metricNames)
}
var tp expfmt.TextParser
expectedMetrics, err := tp.TextToMetricFamilies(bytes.NewReader([]byte(expected)))
if err != nil {
return fmt.Errorf("parsing expected metrics failed: %s", err)
}
if !reflect.DeepEqual(metrics, normalizeMetricFamilies(expectedMetrics)) {
// Encode the gathered output to the readbale text format for comparison.
var buf1 bytes.Buffer
enc := expfmt.NewEncoder(&buf1, expfmt.FmtText)
for _, mf := range metrics {
if err := enc.Encode(mf); err != nil {
return fmt.Errorf("encoding result failed: %s", err)
}
}
// Encode normalized expected metrics again to generate them in the same ordering
// the registry does to spot differences more easily.
var buf2 bytes.Buffer
enc = expfmt.NewEncoder(&buf2, expfmt.FmtText)
for _, mf := range normalizeMetricFamilies(expectedMetrics) {
if err := enc.Encode(mf); err != nil {
return fmt.Errorf("encoding result failed: %s", err)
}
}
if buf2.String() == buf1.String() {
return nil
}
return fmt.Errorf(`
metric output does not match expectation; want:
'%s'
got:
'%s'
`, buf2.String(), buf1.String())
}
return nil
}
func filterMetrics(metrics []*dto.MetricFamily, names []string) []*dto.MetricFamily {
var filtered []*dto.MetricFamily
for _, m := range metrics {
drop := true
for _, name := range names {
if m.GetName() == name {
drop = false
break
}
}
if !drop {
filtered = append(filtered, m)
}
}
return filtered
}
func removeUnusedWhitespace(s string) string {
var (
trimmedLine string
trimmedLines []string
lines = strings.Split(s, "\n")
)
for _, l := range lines {
trimmedLine = strings.TrimSpace(l)
if len(trimmedLine) > 0 {
trimmedLines = append(trimmedLines, trimmedLine)
}
}
// The Prometheus metrics representation parser expects an empty line at the
// end otherwise fails with an unexpected EOF error.
return strings.Join(trimmedLines, "\n") + "\n"
}
// The below sorting code is copied form the Prometheus client library modulo the added
// label pair sorting.
// https://github.com/prometheus/client_golang/blob/ea6e1db4cb8127eeb0b6954f7320363e5451820f/prometheus/registry.go#L642-L684
// metricSorter is a sortable slice of *dto.Metric.
type metricSorter []*dto.Metric
func (s metricSorter) Len() int {
return len(s)
}
func (s metricSorter) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}
func (s metricSorter) Less(i, j int) bool {
sort.Sort(prometheus.LabelPairSorter(s[i].Label))
sort.Sort(prometheus.LabelPairSorter(s[j].Label))
if len(s[i].Label) != len(s[j].Label) {
return len(s[i].Label) < len(s[j].Label)
}
for n, lp := range s[i].Label {
vi := lp.GetValue()
vj := s[j].Label[n].GetValue()
if vi != vj {
return vi < vj
}
}
if s[i].TimestampMs == nil {
return false
}
if s[j].TimestampMs == nil {
return true
}
return s[i].GetTimestampMs() < s[j].GetTimestampMs()
}
// normalizeMetricFamilies returns a MetricFamily slice with empty
// MetricFamilies pruned and the remaining MetricFamilies sorted by name within
// the slice, with the contained Metrics sorted within each MetricFamily.
func normalizeMetricFamilies(metricFamiliesByName map[string]*dto.MetricFamily) []*dto.MetricFamily {
for _, mf := range metricFamiliesByName {
sort.Sort(metricSorter(mf.Metric))
}
names := make([]string, 0, len(metricFamiliesByName))
for name, mf := range metricFamiliesByName {
if len(mf.Metric) > 0 {
names = append(names, name)
}
}
sort.Strings(names)
result := make([]*dto.MetricFamily, 0, len(names))
for _, name := range names {
result = append(result, metricFamiliesByName[name])
}
return result
}