Remove GenericController and add tests

This commit is contained in:
Manuel de Brito Fontes 2017-11-04 22:18:28 -03:00
parent 1701bfc334
commit 86f39d9deb
39 changed files with 1131 additions and 1325 deletions

View file

@ -36,7 +36,7 @@ import (
// syncSecret keeps in sync Secrets used by Ingress rules with the files on
// disk to allow copy of the content of the secret to disk to be used
// by external processes.
func (ic *GenericController) syncSecret(key string) {
func (ic *NGINXController) syncSecret(key string) {
glog.V(3).Infof("starting syncing of secret %v", key)
cert, err := ic.getPemCertificate(key)
@ -70,7 +70,7 @@ func (ic *GenericController) syncSecret(key string) {
// getPemCertificate receives a secret, and creates a ingress.SSLCert as return.
// It parses the secret and verifies if it's a keypair, or a 'ca.crt' secret only.
func (ic *GenericController) getPemCertificate(secretName string) (*ingress.SSLCert, error) {
func (ic *NGINXController) getPemCertificate(secretName string) (*ingress.SSLCert, error) {
secret, err := ic.listers.Secret.GetByName(secretName)
if err != nil {
return nil, fmt.Errorf("error retrieving secret %v: %v", secretName, err)
@ -127,7 +127,7 @@ func (ic *GenericController) getPemCertificate(secretName string) (*ingress.SSLC
// checkMissingSecrets verify if one or more ingress rules contains a reference
// to a secret that is not present in the local secret store.
// In this case we call syncSecret.
func (ic *GenericController) checkMissingSecrets() {
func (ic *NGINXController) checkMissingSecrets() {
for _, obj := range ic.listers.Ingress.List() {
ing := obj.(*extensions.Ingress)

View file

@ -103,8 +103,8 @@ func buildControllerForBackendSSL() cache_client.Controller {
return cache_client.New(cfg)
}
func buildGenericControllerForBackendSSL() *GenericController {
gc := &GenericController{
func buildGenericControllerForBackendSSL() *NGINXController {
gc := &NGINXController{
syncRateLimiter: flowcontrol.NewTokenBucketRateLimiter(0.3, 1),
cfg: &Configuration{
Client: buildSimpleClientSetForBackendSSL(),

View file

@ -0,0 +1,68 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controller
import (
"fmt"
"io/ioutil"
"net/http"
"strconv"
"strings"
"github.com/golang/glog"
"github.com/ncabatoff/process-exporter/proc"
)
// Name returns the healthcheck name
func (n NGINXController) Name() string {
return "Ingress Controller"
}
// Check returns if the nginx healthz endpoint is returning ok (status code 200)
func (n NGINXController) Check(_ *http.Request) error {
res, err := http.Get(fmt.Sprintf("http://0.0.0.0:%v%v", n.cfg.ListenPorts.Status, ngxHealthPath))
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return fmt.Errorf("ingress controller is not healthy")
}
// check the nginx master process is running
fs, err := proc.NewFS("/proc")
if err != nil {
glog.Errorf("%v", err)
return err
}
f, err := ioutil.ReadFile("/run/nginx.pid")
if err != nil {
glog.Errorf("%v", err)
return err
}
pid, err := strconv.Atoi(strings.TrimRight(string(f), "\r\n"))
if err != nil {
return err
}
_, err = fs.NewProc(pid)
if err != nil {
glog.Errorf("%v", err)
return err
}
return nil
}

View file

@ -0,0 +1,563 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import (
"fmt"
"runtime"
"strconv"
"github.com/golang/glog"
apiv1 "k8s.io/api/core/v1"
"k8s.io/ingress-nginx/pkg/ingress"
"k8s.io/ingress-nginx/pkg/ingress/defaults"
)
const (
// http://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size
// Sets the maximum allowed size of the client request body
bodySize = "1m"
// http://nginx.org/en/docs/ngx_core_module.html#error_log
// Configures logging level [debug | info | notice | warn | error | crit | alert | emerg]
// Log levels above are listed in the order of increasing severity
errorLevel = "notice"
// HTTP Strict Transport Security (often abbreviated as HSTS) is a security feature (HTTP header)
// that tell browsers that it should only be communicated with using HTTPS, instead of using HTTP.
// https://developer.mozilla.org/en-US/docs/Web/Security/HTTP_strict_transport_security
// max-age is the time, in seconds, that the browser should remember that this site is only to be accessed using HTTPS.
hstsMaxAge = "15724800"
gzipTypes = "application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component"
brotliTypes = "application/xml+rss application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component"
logFormatUpstream = `%v - [$the_real_ip] - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status`
logFormatStream = `[$time_local] $protocol $status $bytes_sent $bytes_received $session_time`
// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_buffer_size
// Sets the size of the buffer used for sending data.
// 4k helps NGINX to improve TLS Time To First Byte (TTTFB)
// https://www.igvita.com/2013/12/16/optimizing-nginx-tls-time-to-first-byte/
sslBufferSize = "4k"
// Enabled ciphers list to enabled. The ciphers are specified in the format understood by the OpenSSL library
// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_ciphers
sslCiphers = "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256"
// SSL enabled protocols to use
// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_protocols
sslProtocols = "TLSv1.2"
// Time during which a client may reuse the session parameters stored in a cache.
// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_timeout
sslSessionTimeout = "10m"
// Size of the SSL shared cache between all worker processes.
// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_cache
sslSessionCacheSize = "10m"
// Default setting for load balancer algorithm
defaultLoadBalancerAlgorithm = "least_conn"
// Parameters for a shared memory zone that will keep states for various keys.
// http://nginx.org/en/docs/http/ngx_http_limit_conn_module.html#limit_conn_zone
defaultLimitConnZoneVariable = "$binary_remote_addr"
)
// Configuration represents the content of nginx.conf file
type Configuration struct {
defaults.Backend `json:",squash"`
// Sets the name of the configmap that contains the headers to pass to the client
AddHeaders string `json:"add-headers,omitempty"`
// AllowBackendServerHeader enables the return of the header Server from the backend
// instead of the generic nginx string.
// http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_hide_header
// By default this is disabled
AllowBackendServerHeader bool `json:"allow-backend-server-header"`
// AccessLogPath sets the path of the access logs if enabled
// http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log
// By default access logs go to /var/log/nginx/access.log
AccessLogPath string `json:"access-log-path,omitempty"`
// ErrorLogPath sets the path of the error logs
// http://nginx.org/en/docs/ngx_core_module.html#error_log
// By default error logs go to /var/log/nginx/error.log
ErrorLogPath string `json:"error-log-path,omitempty"`
// EnableDynamicTLSRecords enables dynamic TLS record sizes
// https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency
// By default this is enabled
EnableDynamicTLSRecords bool `json:"enable-dynamic-tls-records"`
// EnableModsecurity enables the modsecurity module for NGINX
// By default this is disabled
EnableModsecurity bool `json:"enable-modsecurity"`
// EnableModsecurity enables the OWASP ModSecurity Core Rule Set (CRS)
// By default this is disabled
EnableOWASPCoreRules bool `json:"enable-owasp-modsecurity-crs"`
// ClientHeaderBufferSize allows to configure a custom buffer
// size for reading client request header
// http://nginx.org/en/docs/http/ngx_http_core_module.html#client_header_buffer_size
ClientHeaderBufferSize string `json:"client-header-buffer-size"`
// Defines a timeout for reading client request header, in seconds
// http://nginx.org/en/docs/http/ngx_http_core_module.html#client_header_timeout
ClientHeaderTimeout int `json:"client-header-timeout,omitempty"`
// Sets buffer size for reading client request body
// http://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_buffer_size
ClientBodyBufferSize string `json:"client-body-buffer-size,omitempty"`
// Defines a timeout for reading client request body, in seconds
// http://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_timeout
ClientBodyTimeout int `json:"client-body-timeout,omitempty"`
// DisableAccessLog disables the Access Log globally from NGINX ingress controller
//http://nginx.org/en/docs/http/ngx_http_log_module.html
DisableAccessLog bool `json:"disable-access-log,omitempty"`
// DisableIpv6 disable listening on ipv6 address
DisableIpv6 bool `json:"disable-ipv6,omitempty"`
// EnableUnderscoresInHeaders enables underscores in header names
// http://nginx.org/en/docs/http/ngx_http_core_module.html#underscores_in_headers
// By default this is disabled
EnableUnderscoresInHeaders bool `json:"enable-underscores-in-headers"`
// IgnoreInvalidHeaders set if header fields with invalid names should be ignored
// http://nginx.org/en/docs/http/ngx_http_core_module.html#ignore_invalid_headers
// By default this is enabled
IgnoreInvalidHeaders bool `json:"ignore-invalid-headers"`
// EnableVtsStatus allows the replacement of the default status page with a third party module named
// nginx-module-vts - https://github.com/vozlt/nginx-module-vts
// By default this is disabled
EnableVtsStatus bool `json:"enable-vts-status,omitempty"`
// Vts config on http level
// Description: Sets parameters for a shared memory zone that will keep states for various keys. The cache is shared between all worker processe
// https://github.com/vozlt/nginx-module-vts#vhost_traffic_status_zone
// Default value is 10m
VtsStatusZoneSize string `json:"vts-status-zone-size,omitempty"`
// Vts config on http level
// Description: Enables the keys by user defined variable. The key is a key string to calculate traffic.
// The name is a group string to calculate traffic. The key and name can contain variables such as $host,
// $server_name. The name's group belongs to filterZones if specified. The key's group belongs to serverZones
// if not specified second argument name. The example with geoip module is as follows:
// https://github.com/vozlt/nginx-module-vts#vhost_traffic_status_filter_by_set_key
// Default value is $geoip_country_code country::*
VtsDefaultFilterKey string `json:"vts-default-filter-key,omitempty"`
// RetryNonIdempotent since 1.9.13 NGINX will not retry non-idempotent requests (POST, LOCK, PATCH)
// in case of an error. The previous behavior can be restored using the value true
RetryNonIdempotent bool `json:"retry-non-idempotent"`
// http://nginx.org/en/docs/ngx_core_module.html#error_log
// Configures logging level [debug | info | notice | warn | error | crit | alert | emerg]
// Log levels above are listed in the order of increasing severity
ErrorLogLevel string `json:"error-log-level,omitempty"`
// https://nginx.org/en/docs/http/ngx_http_v2_module.html#http2_max_field_size
// HTTP2MaxFieldSize Limits the maximum size of an HPACK-compressed request header field
HTTP2MaxFieldSize string `json:"http2-max-field-size,omitempty"`
// https://nginx.org/en/docs/http/ngx_http_v2_module.html#http2_max_header_size
// HTTP2MaxHeaderSize Limits the maximum size of the entire request header list after HPACK decompression
HTTP2MaxHeaderSize string `json:"http2-max-header-size,omitempty"`
// Enables or disables the header HSTS in servers running SSL
HSTS bool `json:"hsts,omitempty"`
// Enables or disables the use of HSTS in all the subdomains of the servername
// Default: true
HSTSIncludeSubdomains bool `json:"hsts-include-subdomains,omitempty"`
// HTTP Strict Transport Security (often abbreviated as HSTS) is a security feature (HTTP header)
// that tell browsers that it should only be communicated with using HTTPS, instead of using HTTP.
// https://developer.mozilla.org/en-US/docs/Web/Security/HTTP_strict_transport_security
// max-age is the time, in seconds, that the browser should remember that this site is only to be
// accessed using HTTPS.
HSTSMaxAge string `json:"hsts-max-age,omitempty"`
// Enables or disables the preload attribute in HSTS feature
HSTSPreload bool `json:"hsts-preload,omitempty"`
// Time during which a keep-alive client connection will stay open on the server side.
// The zero value disables keep-alive client connections
// http://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_timeout
KeepAlive int `json:"keep-alive,omitempty"`
// Sets the maximum number of requests that can be served through one keep-alive connection.
// http://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_requests
KeepAliveRequests int `json:"keep-alive-requests,omitempty"`
// LargeClientHeaderBuffers Sets the maximum number and size of buffers used for reading
// large client request header.
// http://nginx.org/en/docs/http/ngx_http_core_module.html#large_client_header_buffers
// Default: 4 8k
LargeClientHeaderBuffers string `json:"large-client-header-buffers"`
// Enable json escaping
// http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format
LogFormatEscapeJSON bool `json:"log-format-escape-json,omitempty"`
// Customize upstream log_format
// http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format
LogFormatUpstream string `json:"log-format-upstream,omitempty"`
// Customize stream log_format
// http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format
LogFormatStream string `json:"log-format-stream,omitempty"`
// Maximum number of simultaneous connections that can be opened by each worker process
// http://nginx.org/en/docs/ngx_core_module.html#worker_connections
MaxWorkerConnections int `json:"max-worker-connections,omitempty"`
// Sets the bucket size for the map variables hash tables.
// Default value depends on the processors cache line size.
// http://nginx.org/en/docs/http/ngx_http_map_module.html#map_hash_bucket_size
MapHashBucketSize int `json:"map-hash-bucket-size,omitempty"`
// If UseProxyProtocol is enabled ProxyRealIPCIDR defines the default the IP/network address
// of your external load balancer
ProxyRealIPCIDR []string `json:"proxy-real-ip-cidr,omitempty"`
// Sets the name of the configmap that contains the headers to pass to the backend
ProxySetHeaders string `json:"proxy-set-headers,omitempty"`
// Maximum size of the server names hash tables used in server names, map directives values,
// MIME types, names of request header strings, etcd.
// http://nginx.org/en/docs/hash.html
// http://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_max_size
ServerNameHashMaxSize int `json:"server-name-hash-max-size,omitempty"`
// Size of the bucket for the server names hash tables
// http://nginx.org/en/docs/hash.html
// http://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_bucket_size
ServerNameHashBucketSize int `json:"server-name-hash-bucket-size,omitempty"`
// Size of the bucket for the proxy headers hash tables
// http://nginx.org/en/docs/hash.html
// https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_headers_hash_max_size
ProxyHeadersHashMaxSize int `json:"proxy-headers-hash-max-size,omitempty"`
// Maximum size of the bucket for the proxy headers hash tables
// http://nginx.org/en/docs/hash.html
// https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_headers_hash_bucket_size
ProxyHeadersHashBucketSize int `json:"proxy-headers-hash-bucket-size,omitempty"`
// Enables or disables emitting nginx version in error messages and in the “Server” response header field.
// http://nginx.org/en/docs/http/ngx_http_core_module.html#server_tokens
// Default: true
ShowServerTokens bool `json:"server-tokens"`
// Enabled ciphers list to enabled. The ciphers are specified in the format understood by
// the OpenSSL library
// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_ciphers
SSLCiphers string `json:"ssl-ciphers,omitempty"`
// Specifies a curve for ECDHE ciphers.
// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_ecdh_curve
SSLECDHCurve string `json:"ssl-ecdh-curve,omitempty"`
// The secret that contains Diffie-Hellman key to help with "Perfect Forward Secrecy"
// https://www.openssl.org/docs/manmaster/apps/dhparam.html
// https://wiki.mozilla.org/Security/Server_Side_TLS#DHE_handshake_and_dhparam
// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_dhparam
SSLDHParam string `json:"ssl-dh-param,omitempty"`
// SSL enabled protocols to use
// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_protocols
SSLProtocols string `json:"ssl-protocols,omitempty"`
// Enables or disables the use of shared SSL cache among worker processes.
// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_cache
SSLSessionCache bool `json:"ssl-session-cache,omitempty"`
// Size of the SSL shared cache between all worker processes.
// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_cache
SSLSessionCacheSize string `json:"ssl-session-cache-size,omitempty"`
// Enables or disables session resumption through TLS session tickets.
// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_tickets
SSLSessionTickets bool `json:"ssl-session-tickets,omitempty"`
// Sets the secret key used to encrypt and decrypt TLS session tickets.
// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_tickets
// By default, a randomly generated key is used.
// Example: openssl rand 80 | base64 -w0
SSLSessionTicketKey string `json:"ssl-session-ticket-key,omitempty"`
// Time during which a client may reuse the session parameters stored in a cache.
// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_timeout
SSLSessionTimeout string `json:"ssl-session-timeout,omitempty"`
// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_buffer_size
// Sets the size of the buffer used for sending data.
// 4k helps NGINX to improve TLS Time To First Byte (TTTFB)
// https://www.igvita.com/2013/12/16/optimizing-nginx-tls-time-to-first-byte/
SSLBufferSize string `json:"ssl-buffer-size,omitempty"`
// Enables or disables the use of the PROXY protocol to receive client connection
// (real IP address) information passed through proxy servers and load balancers
// such as HAproxy and Amazon Elastic Load Balancer (ELB).
// https://www.nginx.com/resources/admin-guide/proxy-protocol/
UseProxyProtocol bool `json:"use-proxy-protocol,omitempty"`
// Enables or disables the use of the nginx module that compresses responses using the "gzip" method
// http://nginx.org/en/docs/http/ngx_http_gzip_module.html
UseGzip bool `json:"use-gzip,omitempty"`
// Enables or disables the use of the NGINX Brotli Module for compression
// https://github.com/google/ngx_brotli
UseBrotli bool `json:"use-brotli,omitempty"`
// Brotli Compression Level that will be used
BrotliLevel int `json:"brotli-level,omitempty"`
// MIME Types that will be compressed on-the-fly using Brotli module
BrotliTypes string `json:"brotli-types,omitempty"`
// Enables or disables the HTTP/2 support in secure connections
// http://nginx.org/en/docs/http/ngx_http_v2_module.html
// Default: true
UseHTTP2 bool `json:"use-http2,omitempty"`
// MIME types in addition to "text/html" to compress. The special value “*” matches any MIME type.
// Responses with the “text/html” type are always compressed if UseGzip is enabled
GzipTypes string `json:"gzip-types,omitempty"`
// Defines the number of worker processes. By default auto means number of available CPU cores
// http://nginx.org/en/docs/ngx_core_module.html#worker_processes
WorkerProcesses string `json:"worker-processes,omitempty"`
// Defines a timeout for a graceful shutdown of worker processes
// http://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout
WorkerShutdownTimeout string `json:"worker-shutdown-timeout,omitempty"`
// Defines the load balancing algorithm to use. The deault is round-robin
LoadBalanceAlgorithm string `json:"load-balance,omitempty"`
// Sets the bucket size for the variables hash table.
// http://nginx.org/en/docs/http/ngx_http_map_module.html#variables_hash_bucket_size
VariablesHashBucketSize int `json:"variables-hash-bucket-size,omitempty"`
// Sets the maximum size of the variables hash table.
// http://nginx.org/en/docs/http/ngx_http_map_module.html#variables_hash_max_size
VariablesHashMaxSize int `json:"variables-hash-max-size,omitempty"`
// Activates the cache for connections to upstream servers.
// The connections parameter sets the maximum number of idle keepalive connections to
// upstream servers that are preserved in the cache of each worker process. When this
// number is exceeded, the least recently used connections are closed.
// http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive
// Default: 32
UpstreamKeepaliveConnections int `json:"upstream-keepalive-connections,omitempty"`
// Sets the maximum size of the variables hash table.
// http://nginx.org/en/docs/http/ngx_http_map_module.html#variables_hash_max_size
LimitConnZoneVariable string `json:"limit-conn-zone-variable,omitempty"`
// Sets the timeout between two successive read or write operations on client or proxied server connections.
// If no data is transmitted within this time, the connection is closed.
// http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_timeout
ProxyStreamTimeout string `json:"proxy-stream-timeout,omitempty"`
// Sets the ipv4 addresses on which the server will accept requests.
BindAddressIpv4 []string `json:"bind-address-ipv4,omitempty"`
// Sets the ipv6 addresses on which the server will accept requests.
BindAddressIpv6 []string `json:"bind-address-ipv6,omitempty"`
// Sets the header field for identifying the originating IP address of a client
// Default is X-Forwarded-For
ForwardedForHeader string `json:"forwarded-for-header,omitempty"`
// Append the remote address to the X-Forwarded-For header instead of replacing it
// Default: false
ComputeFullForwardedFor bool `json:"compute-full-forwarded-for,omitempty"`
// EnableOpentracing enables the nginx Opentracing extension
// https://github.com/rnburn/nginx-opentracing
// By default this is disabled
EnableOpentracing bool `json:"enable-opentracing"`
// ZipkinCollectorHost specifies the host to use when uploading traces
ZipkinCollectorHost string `json:"zipkin-collector-host"`
// ZipkinCollectorPort specifies the port to use when uploading traces
ZipkinCollectorPort int `json:"zipkin-collector-port"`
// ZipkinServiceName specifies the service name to use for any traces created
// Default: nginx
ZipkinServiceName string `json:"zipkin-service-name"`
// HTTPSnippet adds custom configuration to the http section of the nginx configuration
HTTPSnippet string `json:"http-snippet"`
// ServerSnippet adds custom configuration to all the servers in the nginx configuration
ServerSnippet string `json:"server-snippet"`
// LocationSnippet adds custom configuration to all the locations in the nginx configuration
LocationSnippet string `json:"location-snippet"`
}
// NewDefault returns the default nginx configuration
func NewDefault() Configuration {
defIPCIDR := make([]string, 0)
defIPCIDR = append(defIPCIDR, "0.0.0.0/0")
defBindAddress := make([]string, 0)
cfg := Configuration{
AllowBackendServerHeader: false,
AccessLogPath: "/var/log/nginx/access.log",
ErrorLogPath: "/var/log/nginx/error.log",
BrotliLevel: 4,
BrotliTypes: brotliTypes,
ClientHeaderBufferSize: "1k",
ClientHeaderTimeout: 60,
ClientBodyBufferSize: "8k",
ClientBodyTimeout: 60,
EnableDynamicTLSRecords: true,
EnableUnderscoresInHeaders: false,
ErrorLogLevel: errorLevel,
ForwardedForHeader: "X-Forwarded-For",
ComputeFullForwardedFor: false,
HTTP2MaxFieldSize: "4k",
HTTP2MaxHeaderSize: "16k",
HSTS: true,
HSTSIncludeSubdomains: true,
HSTSMaxAge: hstsMaxAge,
HSTSPreload: false,
IgnoreInvalidHeaders: true,
GzipTypes: gzipTypes,
KeepAlive: 75,
KeepAliveRequests: 100,
LargeClientHeaderBuffers: "4 8k",
LogFormatEscapeJSON: false,
LogFormatStream: logFormatStream,
LogFormatUpstream: logFormatUpstream,
MaxWorkerConnections: 16384,
MapHashBucketSize: 64,
ProxyRealIPCIDR: defIPCIDR,
ServerNameHashMaxSize: 1024,
ProxyHeadersHashMaxSize: 512,
ProxyHeadersHashBucketSize: 64,
ShowServerTokens: true,
SSLBufferSize: sslBufferSize,
SSLCiphers: sslCiphers,
SSLECDHCurve: "auto",
SSLProtocols: sslProtocols,
SSLSessionCache: true,
SSLSessionCacheSize: sslSessionCacheSize,
SSLSessionTickets: true,
SSLSessionTimeout: sslSessionTimeout,
UseBrotli: true,
UseGzip: true,
WorkerProcesses: strconv.Itoa(runtime.NumCPU()),
WorkerShutdownTimeout: "10s",
LoadBalanceAlgorithm: defaultLoadBalancerAlgorithm,
VtsStatusZoneSize: "10m",
VtsDefaultFilterKey: "$geoip_country_code country::*",
VariablesHashBucketSize: 128,
VariablesHashMaxSize: 2048,
UseHTTP2: true,
ProxyStreamTimeout: "600s",
Backend: defaults.Backend{
ProxyBodySize: bodySize,
ProxyConnectTimeout: 5,
ProxyReadTimeout: 60,
ProxySendTimeout: 60,
ProxyBufferSize: "4k",
ProxyCookieDomain: "off",
ProxyCookiePath: "off",
ProxyNextUpstream: "error timeout invalid_header http_502 http_503 http_504",
ProxyRequestBuffering: "on",
SSLRedirect: true,
CustomHTTPErrors: []int{},
WhitelistSourceRange: []string{},
SkipAccessLogURLs: []string{},
LimitRate: 0,
LimitRateAfter: 0,
},
UpstreamKeepaliveConnections: 32,
LimitConnZoneVariable: defaultLimitConnZoneVariable,
BindAddressIpv4: defBindAddress,
BindAddressIpv6: defBindAddress,
ZipkinCollectorPort: 9411,
ZipkinServiceName: "nginx",
}
if glog.V(5) {
cfg.ErrorLogLevel = "debug"
}
return cfg
}
// BuildLogFormatUpstream format the log_format upstream using
// proxy_protocol_addr as remote client address if UseProxyProtocol
// is enabled.
func (cfg Configuration) BuildLogFormatUpstream() string {
if cfg.LogFormatUpstream == logFormatUpstream {
return fmt.Sprintf(cfg.LogFormatUpstream, "$the_real_ip")
}
return cfg.LogFormatUpstream
}
// TemplateConfig contains the nginx configuration to render the file nginx.conf
type TemplateConfig struct {
ProxySetHeaders map[string]string
AddHeaders map[string]string
MaxOpenFiles int
BacklogSize int
Backends []*ingress.Backend
PassthroughBackends []*ingress.SSLPassthroughBackend
Servers []*ingress.Server
TCPBackends []ingress.L4Service
UDPBackends []ingress.L4Service
HealthzURI string
CustomErrors bool
Cfg Configuration
IsIPV6Enabled bool
IsSSLPassthroughEnabled bool
RedirectServers map[string]string
ListenPorts *ListenPorts
PublishService *apiv1.Service
}
// ListenPorts describe the ports required to run the
// NGINX Ingress controller
type ListenPorts struct {
HTTP int
HTTPS int
Status int
Health int
Default int
SSLProxy int
}

View file

@ -0,0 +1,46 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import (
"fmt"
"testing"
)
func TestBuildLogFormatUpstream(t *testing.T) {
testCases := []struct {
useProxyProtocol bool // use proxy protocol
curLogFormat string
expected string
}{
{true, logFormatUpstream, fmt.Sprintf(logFormatUpstream, "$the_real_ip")},
{false, logFormatUpstream, fmt.Sprintf(logFormatUpstream, "$the_real_ip")},
{true, "my-log-format", "my-log-format"},
{false, "john-log-format", "john-log-format"},
}
for _, testCase := range testCases {
cfg := NewDefault()
cfg.UseProxyProtocol = testCase.useProxyProtocol
cfg.LogFormatUpstream = testCase.curLogFormat
result := cfg.BuildLogFormatUpstream()
if result != testCase.expected {
t.Errorf(" expected %v but return %v", testCase.expected, result)
}
}
}

View file

@ -24,7 +24,6 @@ import (
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
@ -35,12 +34,7 @@ import (
"k8s.io/apimachinery/pkg/conversion"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/flowcontrol"
"k8s.io/kubernetes/pkg/kubelet/util/sliceutils"
"k8s.io/ingress-nginx/pkg/ingress"
@ -48,11 +42,10 @@ import (
"k8s.io/ingress-nginx/pkg/ingress/annotations/healthcheck"
"k8s.io/ingress-nginx/pkg/ingress/annotations/parser"
"k8s.io/ingress-nginx/pkg/ingress/annotations/proxy"
ngx_config "k8s.io/ingress-nginx/pkg/ingress/controller/config"
"k8s.io/ingress-nginx/pkg/ingress/defaults"
"k8s.io/ingress-nginx/pkg/ingress/resolver"
"k8s.io/ingress-nginx/pkg/ingress/status"
"k8s.io/ingress-nginx/pkg/k8s"
"k8s.io/ingress-nginx/pkg/net/ssl"
"k8s.io/ingress-nginx/pkg/task"
)
@ -60,63 +53,32 @@ const (
defUpstreamName = "upstream-default-backend"
defServerName = "_"
rootLocation = "/"
fakeCertificate = "default-fake-certificate"
)
var (
// list of ports that cannot be used by TCP or UDP services
reservedPorts = []string{"80", "443", "8181", "18080"}
fakeCertificatePath = ""
fakeCertificateSHA = ""
cloner = conversion.NewCloner()
cloner *conversion.Cloner
)
// GenericController holds the boilerplate code required to build an Ingress controlller.
type GenericController struct {
cfg *Configuration
listers *ingress.StoreLister
cacheController *cacheController
annotations annotationExtractor
recorder record.EventRecorder
syncQueue *task.Queue
syncStatus status.Sync
// local store of SSL certificates
// (only certificates used in ingress)
sslCertTracker *sslCertTracker
syncRateLimiter flowcontrol.RateLimiter
// stopLock is used to enforce only a single call to Stop is active.
// Needed because we allow stopping through an http endpoint and
// allowing concurrent stoppers leads to stack traces.
stopLock *sync.Mutex
stopCh chan struct{}
// runningConfig contains the running configuration in the Backend
runningConfig *ingress.Configuration
forceReload int32
func init() {
cloner := conversion.NewCloner()
cloner.RegisterDeepCopyFunc(ingress.GetGeneratedDeepCopyFuncs)
}
// Configuration contains all the settings required by an Ingress controller
type Configuration struct {
Client clientset.Interface
APIServerHost string
KubeConfigFile string
Client clientset.Interface
ResyncPeriod time.Duration
ResyncPeriod time.Duration
ConfigMapName string
DefaultService string
IngressClass string
Namespace string
ConfigMapName string
ForceNamespaceIsolation bool
DisableNodeList bool
@ -124,15 +86,14 @@ type Configuration struct {
// optional
TCPConfigMapName string
// optional
UDPConfigMapName string
DefaultSSLCertificate string
UDPConfigMapName string
DefaultHealthzURL string
DefaultIngressClass string
DefaultSSLCertificate string
// optional
PublishService string
// Backend is the particular implementation to be used.
// (for instance NGINX)
Backend ingress.Controller
UpdateStatus bool
UseNodeInternalIP bool
@ -140,74 +101,25 @@ type Configuration struct {
UpdateStatusOnShutdown bool
SortBackends bool
}
// newIngressController creates an Ingress controller
func newIngressController(config *Configuration) *GenericController {
ListenPorts *ngx_config.ListenPorts
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartLogging(glog.Infof)
eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{
Interface: config.Client.CoreV1().Events(config.Namespace),
})
EnableSSLPassthrough bool
ic := GenericController{
cfg: config,
stopLock: &sync.Mutex{},
stopCh: make(chan struct{}),
syncRateLimiter: flowcontrol.NewTokenBucketRateLimiter(0.3, 1),
recorder: eventBroadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{
Component: "ingress-controller",
}),
sslCertTracker: newSSLCertTracker(),
}
EnableProfiling bool
ic.syncQueue = task.NewTaskQueue(ic.syncIngress)
ic.listers, ic.cacheController = ic.createListers(config.DisableNodeList)
if config.UpdateStatus {
ic.syncStatus = status.NewStatusSyncer(status.Config{
Client: config.Client,
PublishService: ic.cfg.PublishService,
IngressLister: ic.listers.Ingress,
ElectionID: config.ElectionID,
IngressClass: config.IngressClass,
DefaultIngressClass: config.DefaultIngressClass,
UpdateStatusOnShutdown: config.UpdateStatusOnShutdown,
CustomIngressStatus: ic.cfg.Backend.UpdateIngressStatus,
UseNodeInternalIP: ic.cfg.UseNodeInternalIP,
})
} else {
glog.Warning("Update of ingress status is disabled (flag --update-status=false was specified)")
}
ic.annotations = newAnnotationExtractor(ic)
ic.cfg.Backend.SetListers(ic.listers)
cloner.RegisterDeepCopyFunc(ingress.GetGeneratedDeepCopyFuncs)
return &ic
}
// Info returns information about the backend
func (ic GenericController) Info() *ingress.BackendInfo {
return ic.cfg.Backend.Info()
}
// IngressClass returns information about the backend
func (ic GenericController) IngressClass() string {
return ic.cfg.IngressClass
FakeCertificatePath string
FakeCertificateSHA string
}
// GetDefaultBackend returns the default backend
func (ic GenericController) GetDefaultBackend() defaults.Backend {
return ic.cfg.Backend.BackendDefaults()
func (n NGINXController) GetDefaultBackend() defaults.Backend {
return n.backendDefaults
}
// GetPublishService returns the configured service used to set ingress status
func (ic GenericController) GetPublishService() *apiv1.Service {
s, err := ic.listers.Service.GetByName(ic.cfg.PublishService)
func (n NGINXController) GetPublishService() *apiv1.Service {
s, err := n.listers.Service.GetByName(n.cfg.PublishService)
if err != nil {
return nil
}
@ -215,42 +127,37 @@ func (ic GenericController) GetPublishService() *apiv1.Service {
return s
}
// GetRecorder returns the event recorder
func (ic GenericController) GetRecorder() record.EventRecorder {
return ic.recorder
}
// GetSecret searches for a secret in the local secrets Store
func (ic GenericController) GetSecret(name string) (*apiv1.Secret, error) {
return ic.listers.Secret.GetByName(name)
func (n NGINXController) GetSecret(name string) (*apiv1.Secret, error) {
return n.listers.Secret.GetByName(name)
}
// GetService searches for a service in the local secrets Store
func (ic GenericController) GetService(name string) (*apiv1.Service, error) {
return ic.listers.Service.GetByName(name)
func (n NGINXController) GetService(name string) (*apiv1.Service, error) {
return n.listers.Service.GetByName(name)
}
// sync collects all the pieces required to assemble the configuration file and
// then sends the content to the backend (OnUpdate) receiving the populated
// template as response reloading the backend if is required.
func (ic *GenericController) syncIngress(item interface{}) error {
ic.syncRateLimiter.Accept()
func (n *NGINXController) syncIngress(item interface{}) error {
n.syncRateLimiter.Accept()
if ic.syncQueue.IsShuttingDown() {
if n.syncQueue.IsShuttingDown() {
return nil
}
if element, ok := item.(task.Element); ok {
if name, ok := element.Key.(string); ok {
if obj, exists, _ := ic.listers.Ingress.GetByKey(name); exists {
if obj, exists, _ := n.listers.Ingress.GetByKey(name); exists {
ing := obj.(*extensions.Ingress)
ic.readSecrets(ing)
n.readSecrets(ing)
}
}
}
// Sort ingress rules using the ResourceVersion field
ings := ic.listers.Ingress.List()
ings := n.listers.Ingress.List()
sort.SliceStable(ings, func(i, j int) bool {
ir := ings[i].(*extensions.Ingress).ResourceVersion
jr := ings[j].(*extensions.Ingress).ResourceVersion
@ -261,14 +168,14 @@ func (ic *GenericController) syncIngress(item interface{}) error {
var ingresses []*extensions.Ingress
for _, ingIf := range ings {
ing := ingIf.(*extensions.Ingress)
if !class.IsValid(ing, ic.cfg.IngressClass, ic.cfg.DefaultIngressClass) {
if !class.IsValid(ing, n.cfg.IngressClass, n.cfg.DefaultIngressClass) {
continue
}
ingresses = append(ingresses, ing)
}
upstreams, servers := ic.getBackendServers(ingresses)
upstreams, servers := n.getBackendServers(ingresses)
var passUpstreams []*ingress.SSLPassthroughBackend
for _, server := range servers {
@ -294,19 +201,19 @@ func (ic *GenericController) syncIngress(item interface{}) error {
pcfg := ingress.Configuration{
Backends: upstreams,
Servers: servers,
TCPEndpoints: ic.getStreamServices(ic.cfg.TCPConfigMapName, apiv1.ProtocolTCP),
UDPEndpoints: ic.getStreamServices(ic.cfg.UDPConfigMapName, apiv1.ProtocolUDP),
TCPEndpoints: n.getStreamServices(n.cfg.TCPConfigMapName, apiv1.ProtocolTCP),
UDPEndpoints: n.getStreamServices(n.cfg.UDPConfigMapName, apiv1.ProtocolUDP),
PassthroughBackends: passUpstreams,
}
if !ic.isForceReload() && ic.runningConfig != nil && ic.runningConfig.Equal(&pcfg) {
if !n.isForceReload() && n.runningConfig != nil && n.runningConfig.Equal(&pcfg) {
glog.V(3).Infof("skipping backend reload (no changes detected)")
return nil
}
glog.Infof("backend reload required")
err := ic.cfg.Backend.OnUpdate(pcfg)
err := n.OnUpdate(pcfg)
if err != nil {
incReloadErrorCount()
glog.Errorf("unexpected failure restarting the backend: \n%v", err)
@ -317,13 +224,13 @@ func (ic *GenericController) syncIngress(item interface{}) error {
incReloadCount()
setSSLExpireTime(servers)
ic.runningConfig = &pcfg
ic.SetForceReload(false)
n.runningConfig = &pcfg
n.SetForceReload(false)
return nil
}
func (ic *GenericController) getStreamServices(configmapName string, proto apiv1.Protocol) []ingress.L4Service {
func (n *NGINXController) getStreamServices(configmapName string, proto apiv1.Protocol) []ingress.L4Service {
glog.V(3).Infof("obtaining information about stream services of type %v located in configmap %v", proto, configmapName)
if configmapName == "" {
// no configmap configured
@ -336,7 +243,7 @@ func (ic *GenericController) getStreamServices(configmapName string, proto apiv1
return []ingress.L4Service{}
}
configmap, err := ic.listers.ConfigMap.GetByName(configmapName)
configmap, err := n.listers.ConfigMap.GetByName(configmapName)
if err != nil {
glog.Errorf("unexpected error reading configmap %v: %v", configmapName, err)
return []ingress.L4Service{}
@ -386,7 +293,7 @@ func (ic *GenericController) getStreamServices(configmapName string, proto apiv1
continue
}
svcObj, svcExists, err := ic.listers.Service.GetByKey(nsName)
svcObj, svcExists, err := n.listers.Service.GetByKey(nsName)
if err != nil {
glog.Warningf("error getting service %v: %v", nsName, err)
continue
@ -406,7 +313,7 @@ func (ic *GenericController) getStreamServices(configmapName string, proto apiv1
for _, sp := range svc.Spec.Ports {
if sp.Name == svcPort {
if sp.Protocol == proto {
endps = ic.getEndpoints(svc, &sp, proto, &healthcheck.Upstream{})
endps = n.getEndpoints(svc, &sp, proto, &healthcheck.Upstream{})
break
}
}
@ -417,7 +324,7 @@ func (ic *GenericController) getStreamServices(configmapName string, proto apiv1
for _, sp := range svc.Spec.Ports {
if sp.Port == int32(targetPort) {
if sp.Protocol == proto {
endps = ic.getEndpoints(svc, &sp, proto, &healthcheck.Upstream{})
endps = n.getEndpoints(svc, &sp, proto, &healthcheck.Upstream{})
break
}
}
@ -450,29 +357,29 @@ func (ic *GenericController) getStreamServices(configmapName string, proto apiv1
// getDefaultUpstream returns an upstream associated with the
// default backend service. In case of error retrieving information
// configure the upstream to return http code 503.
func (ic *GenericController) getDefaultUpstream() *ingress.Backend {
func (n *NGINXController) getDefaultUpstream() *ingress.Backend {
upstream := &ingress.Backend{
Name: defUpstreamName,
}
svcKey := ic.cfg.DefaultService
svcObj, svcExists, err := ic.listers.Service.GetByKey(svcKey)
svcKey := n.cfg.DefaultService
svcObj, svcExists, err := n.listers.Service.GetByKey(svcKey)
if err != nil {
glog.Warningf("unexpected error searching the default backend %v: %v", ic.cfg.DefaultService, err)
upstream.Endpoints = append(upstream.Endpoints, ic.cfg.Backend.DefaultEndpoint())
glog.Warningf("unexpected error searching the default backend %v: %v", n.cfg.DefaultService, err)
upstream.Endpoints = append(upstream.Endpoints, n.DefaultEndpoint())
return upstream
}
if !svcExists {
glog.Warningf("service %v does not exist", svcKey)
upstream.Endpoints = append(upstream.Endpoints, ic.cfg.Backend.DefaultEndpoint())
upstream.Endpoints = append(upstream.Endpoints, n.DefaultEndpoint())
return upstream
}
svc := svcObj.(*apiv1.Service)
endps := ic.getEndpoints(svc, &svc.Spec.Ports[0], apiv1.ProtocolTCP, &healthcheck.Upstream{})
endps := n.getEndpoints(svc, &svc.Spec.Ports[0], apiv1.ProtocolTCP, &healthcheck.Upstream{})
if len(endps) == 0 {
glog.Warningf("service %v does not have any active endpoints", svcKey)
endps = []ingress.Endpoint{ic.cfg.Backend.DefaultEndpoint()}
endps = []ingress.Endpoint{n.DefaultEndpoint()}
}
upstream.Service = svc
@ -482,14 +389,14 @@ func (ic *GenericController) getDefaultUpstream() *ingress.Backend {
// getBackendServers returns a list of Upstream and Server to be used by the backend
// An upstream can be used in multiple servers if the namespace, service name and port are the same
func (ic *GenericController) getBackendServers(ingresses []*extensions.Ingress) ([]*ingress.Backend, []*ingress.Server) {
du := ic.getDefaultUpstream()
upstreams := ic.createUpstreams(ingresses, du)
servers := ic.createServers(ingresses, upstreams, du)
func (n *NGINXController) getBackendServers(ingresses []*extensions.Ingress) ([]*ingress.Backend, []*ingress.Server) {
du := n.getDefaultUpstream()
upstreams := n.createUpstreams(ingresses, du)
servers := n.createServers(ingresses, upstreams, du)
for _, ing := range ingresses {
affinity := ic.annotations.SessionAffinity(ing)
anns := ic.annotations.Extract(ing)
affinity := n.annotations.SessionAffinity(ing)
anns := n.annotations.Extract(ing)
for _, rule := range ing.Spec.Rules {
host := rule.Host
@ -508,7 +415,7 @@ func (ic *GenericController) getBackendServers(ingresses []*extensions.Ingress)
}
if server.CertificateAuth.CAFileName == "" {
ca := ic.annotations.CertificateAuth(ing)
ca := n.annotations.CertificateAuth(ing)
if ca != nil {
server.CertificateAuth = *ca
// It is possible that no CAFileName is found in the secret
@ -609,7 +516,7 @@ func (ic *GenericController) getBackendServers(ingresses []*extensions.Ingress)
// check if the location contains endpoints and a custom default backend
if location.DefaultBackend != nil {
sp := location.DefaultBackend.Spec.Ports[0]
endps := ic.getEndpoints(location.DefaultBackend, &sp, apiv1.ProtocolTCP, &healthcheck.Upstream{})
endps := n.getEndpoints(location.DefaultBackend, &sp, apiv1.ProtocolTCP, &healthcheck.Upstream{})
if len(endps) > 0 {
glog.V(3).Infof("using custom default backend in server %v location %v (service %v/%v)",
server.Hostname, location.Path, location.DefaultBackend.Namespace, location.DefaultBackend.Name)
@ -656,7 +563,7 @@ func (ic *GenericController) getBackendServers(ingresses []*extensions.Ingress)
aUpstreams = append(aUpstreams, upstream)
}
if ic.cfg.SortBackends {
if n.cfg.SortBackends {
sort.SliceStable(aUpstreams, func(a, b int) bool {
return aUpstreams[a].Name < aUpstreams[b].Name
})
@ -678,17 +585,17 @@ func (ic *GenericController) getBackendServers(ingresses []*extensions.Ingress)
}
// GetAuthCertificate is used by the auth-tls annotations to get a cert from a secret
func (ic GenericController) GetAuthCertificate(name string) (*resolver.AuthSSLCert, error) {
if _, exists := ic.sslCertTracker.Get(name); !exists {
ic.syncSecret(name)
func (n NGINXController) GetAuthCertificate(name string) (*resolver.AuthSSLCert, error) {
if _, exists := n.sslCertTracker.Get(name); !exists {
n.syncSecret(name)
}
_, err := ic.listers.Secret.GetByName(name)
_, err := n.listers.Secret.GetByName(name)
if err != nil {
return &resolver.AuthSSLCert{}, fmt.Errorf("unexpected error: %v", err)
}
bc, exists := ic.sslCertTracker.Get(name)
bc, exists := n.sslCertTracker.Get(name)
if !exists {
return &resolver.AuthSSLCert{}, fmt.Errorf("secret %v does not exist", name)
}
@ -702,15 +609,15 @@ func (ic GenericController) GetAuthCertificate(name string) (*resolver.AuthSSLCe
// createUpstreams creates the NGINX upstreams for each service referenced in
// Ingress rules. The servers inside the upstream are endpoints.
func (ic *GenericController) createUpstreams(data []*extensions.Ingress, du *ingress.Backend) map[string]*ingress.Backend {
func (n *NGINXController) createUpstreams(data []*extensions.Ingress, du *ingress.Backend) map[string]*ingress.Backend {
upstreams := make(map[string]*ingress.Backend)
upstreams[defUpstreamName] = du
for _, ing := range data {
secUpstream := ic.annotations.SecureUpstream(ing)
hz := ic.annotations.HealthCheck(ing)
serviceUpstream := ic.annotations.ServiceUpstream(ing)
upstreamHashBy := ic.annotations.UpstreamHashBy(ing)
secUpstream := n.annotations.SecureUpstream(ing)
hz := n.annotations.HealthCheck(ing)
serviceUpstream := n.annotations.ServiceUpstream(ing)
upstreamHashBy := n.annotations.UpstreamHashBy(ing)
var defBackend string
if ing.Spec.Backend != nil {
@ -726,7 +633,7 @@ func (ic *GenericController) createUpstreams(data []*extensions.Ingress, du *ing
// Add the service cluster endpoint as the upstream instead of individual endpoints
// if the serviceUpstream annotation is enabled
if serviceUpstream {
endpoint, err := ic.getServiceClusterEndpoint(svcKey, ing.Spec.Backend)
endpoint, err := n.getServiceClusterEndpoint(svcKey, ing.Spec.Backend)
if err != nil {
glog.Errorf("Failed to get service cluster endpoint for service %s: %v", svcKey, err)
} else {
@ -735,7 +642,7 @@ func (ic *GenericController) createUpstreams(data []*extensions.Ingress, du *ing
}
if len(upstreams[defBackend].Endpoints) == 0 {
endps, err := ic.serviceEndpoints(svcKey, ing.Spec.Backend.ServicePort.String(), hz)
endps, err := n.serviceEndpoints(svcKey, ing.Spec.Backend.ServicePort.String(), hz)
upstreams[defBackend].Endpoints = append(upstreams[defBackend].Endpoints, endps...)
if err != nil {
glog.Warningf("error creating upstream %v: %v", defBackend, err)
@ -780,7 +687,7 @@ func (ic *GenericController) createUpstreams(data []*extensions.Ingress, du *ing
// Add the service cluster endpoint as the upstream instead of individual endpoints
// if the serviceUpstream annotation is enabled
if serviceUpstream {
endpoint, err := ic.getServiceClusterEndpoint(svcKey, &path.Backend)
endpoint, err := n.getServiceClusterEndpoint(svcKey, &path.Backend)
if err != nil {
glog.Errorf("failed to get service cluster endpoint for service %s: %v", svcKey, err)
} else {
@ -789,7 +696,7 @@ func (ic *GenericController) createUpstreams(data []*extensions.Ingress, du *ing
}
if len(upstreams[name].Endpoints) == 0 {
endp, err := ic.serviceEndpoints(svcKey, path.Backend.ServicePort.String(), hz)
endp, err := n.serviceEndpoints(svcKey, path.Backend.ServicePort.String(), hz)
if err != nil {
glog.Warningf("error obtaining service endpoints: %v", err)
continue
@ -797,7 +704,7 @@ func (ic *GenericController) createUpstreams(data []*extensions.Ingress, du *ing
upstreams[name].Endpoints = endp
}
s, err := ic.listers.Service.GetByName(svcKey)
s, err := n.listers.Service.GetByName(svcKey)
if err != nil {
glog.Warningf("error obtaining service: %v", err)
continue
@ -811,8 +718,8 @@ func (ic *GenericController) createUpstreams(data []*extensions.Ingress, du *ing
return upstreams
}
func (ic *GenericController) getServiceClusterEndpoint(svcKey string, backend *extensions.IngressBackend) (endpoint ingress.Endpoint, err error) {
svcObj, svcExists, err := ic.listers.Service.GetByKey(svcKey)
func (n *NGINXController) getServiceClusterEndpoint(svcKey string, backend *extensions.IngressBackend) (endpoint ingress.Endpoint, err error) {
svcObj, svcExists, err := n.listers.Service.GetByKey(svcKey)
if !svcExists {
return endpoint, fmt.Errorf("service %v does not exist", svcKey)
@ -848,9 +755,9 @@ func (ic *GenericController) getServiceClusterEndpoint(svcKey string, backend *e
// serviceEndpoints returns the upstream servers (endpoints) associated
// to a service.
func (ic *GenericController) serviceEndpoints(svcKey, backendPort string,
func (n *NGINXController) serviceEndpoints(svcKey, backendPort string,
hz *healthcheck.Upstream) ([]ingress.Endpoint, error) {
svc, err := ic.listers.Service.GetByName(svcKey)
svc, err := n.listers.Service.GetByName(svcKey)
var upstreams []ingress.Endpoint
if err != nil {
@ -864,12 +771,12 @@ func (ic *GenericController) serviceEndpoints(svcKey, backendPort string,
servicePort.TargetPort.String() == backendPort ||
servicePort.Name == backendPort {
endps := ic.getEndpoints(svc, &servicePort, apiv1.ProtocolTCP, hz)
endps := n.getEndpoints(svc, &servicePort, apiv1.ProtocolTCP, hz)
if len(endps) == 0 {
glog.Warningf("service %v does not have any active endpoints", svcKey)
}
if ic.cfg.SortBackends {
if n.cfg.SortBackends {
sort.SliceStable(endps, func(i, j int) bool {
iName := endps[i].Address
jName := endps[j].Address
@ -898,7 +805,7 @@ func (ic *GenericController) serviceEndpoints(svcKey, backendPort string,
Port: int32(externalPort),
TargetPort: intstr.FromString(backendPort),
}
endps := ic.getEndpoints(svc, &servicePort, apiv1.ProtocolTCP, hz)
endps := n.getEndpoints(svc, &servicePort, apiv1.ProtocolTCP, hz)
if len(endps) == 0 {
glog.Warningf("service %v does not have any active endpoints", svcKey)
return upstreams, nil
@ -908,7 +815,7 @@ func (ic *GenericController) serviceEndpoints(svcKey, backendPort string,
return upstreams, nil
}
if !ic.cfg.SortBackends {
if !n.cfg.SortBackends {
rand.Seed(time.Now().UnixNano())
for i := range upstreams {
j := rand.Intn(i + 1)
@ -923,7 +830,7 @@ func (ic *GenericController) serviceEndpoints(svcKey, backendPort string,
// FDQN referenced by ingress rules and the common name field in the referenced
// SSL certificates. Each server is configured with location / using a default
// backend specified by the user or the one inside the ingress spec.
func (ic *GenericController) createServers(data []*extensions.Ingress,
func (n *NGINXController) createServers(data []*extensions.Ingress,
upstreams map[string]*ingress.Backend,
du *ingress.Backend) map[string]*ingress.Server {
@ -932,7 +839,7 @@ func (ic *GenericController) createServers(data []*extensions.Ingress,
// remove the alias to avoid conflicts.
aliases := make(map[string]string, len(data))
bdef := ic.GetDefaultBackend()
bdef := n.GetDefaultBackend()
ngxProxy := proxy.Configuration{
BodySize: bdef.ProxyBodySize,
ConnectTimeout: bdef.ProxyConnectTimeout,
@ -946,12 +853,12 @@ func (ic *GenericController) createServers(data []*extensions.Ingress,
}
// generated on Start() with createDefaultSSLCertificate()
defaultPemFileName := fakeCertificatePath
defaultPemSHA := fakeCertificateSHA
defaultPemFileName := n.cfg.FakeCertificatePath
defaultPemSHA := n.cfg.FakeCertificateSHA
// Tries to fetch the default Certificate from nginx configuration.
// If it does not exists, use the ones generated on Start()
defaultCertificate, err := ic.getPemCertificate(ic.cfg.DefaultSSLCertificate)
defaultCertificate, err := n.getPemCertificate(n.cfg.DefaultSSLCertificate)
if err == nil {
defaultPemFileName = defaultCertificate.PemFileName
defaultPemSHA = defaultCertificate.PemSHA
@ -976,7 +883,7 @@ func (ic *GenericController) createServers(data []*extensions.Ingress,
for _, ing := range data {
// check if ssl passthrough is configured
sslpt := ic.annotations.SSLPassthrough(ing)
sslpt := n.annotations.SSLPassthrough(ing)
// default upstream server
un := du.Name
@ -1028,8 +935,8 @@ func (ic *GenericController) createServers(data []*extensions.Ingress,
// configure default location, alias, and SSL
for _, ing := range data {
// setup server-alias based on annotations
aliasAnnotation := ic.annotations.Alias(ing)
srvsnippet := ic.annotations.ServerSnippet(ing)
aliasAnnotation := n.annotations.Alias(ing)
srvsnippet := n.annotations.ServerSnippet(ing)
for _, rule := range ing.Spec.Rules {
host := rule.Host
@ -1095,7 +1002,7 @@ func (ic *GenericController) createServers(data []*extensions.Ingress,
}
key := fmt.Sprintf("%v/%v", ing.Namespace, tlsSecretName)
bc, exists := ic.sslCertTracker.Get(key)
bc, exists := n.sslCertTracker.Get(key)
if !exists {
glog.Warningf("ssl certificate \"%v\" does not exist in local store", key)
continue
@ -1130,7 +1037,7 @@ func (ic *GenericController) createServers(data []*extensions.Ingress,
}
// getEndpoints returns a list of <endpoint ip>:<port> for a given service/target port combination.
func (ic *GenericController) getEndpoints(
func (n *NGINXController) getEndpoints(
s *apiv1.Service,
servicePort *apiv1.ServicePort,
proto apiv1.Protocol,
@ -1171,7 +1078,7 @@ func (ic *GenericController) getEndpoints(
}
glog.V(3).Infof("getting endpoints for service %v/%v and port %v", s.Namespace, s.Name, servicePort.String())
ep, err := ic.listers.Endpoint.GetServiceEndpoints(s)
ep, err := n.listers.Endpoint.GetServiceEndpoints(s)
if err != nil {
glog.Warningf("unexpected error obtaining service endpoints: %v", err)
return upsServers
@ -1221,99 +1128,32 @@ func (ic *GenericController) getEndpoints(
}
// readSecrets extracts information about secrets from an Ingress rule
func (ic *GenericController) readSecrets(ing *extensions.Ingress) {
func (n *NGINXController) readSecrets(ing *extensions.Ingress) {
for _, tls := range ing.Spec.TLS {
if tls.SecretName == "" {
continue
}
key := fmt.Sprintf("%v/%v", ing.Namespace, tls.SecretName)
ic.syncSecret(key)
n.syncSecret(key)
}
key, _ := parser.GetStringAnnotation("ingress.kubernetes.io/auth-tls-secret", ing)
if key == "" {
return
}
ic.syncSecret(key)
n.syncSecret(key)
}
// Stop stops the loadbalancer controller.
func (ic GenericController) Stop() error {
ic.stopLock.Lock()
defer ic.stopLock.Unlock()
// Only try draining the workqueue if we haven't already.
if !ic.syncQueue.IsShuttingDown() {
glog.Infof("shutting down controller queues")
close(ic.stopCh)
go ic.syncQueue.Shutdown()
if ic.syncStatus != nil {
ic.syncStatus.Shutdown()
}
return nil
}
return fmt.Errorf("shutdown already in progress")
func (n *NGINXController) isForceReload() bool {
return atomic.LoadInt32(&n.forceReload) != 0
}
// Start starts the Ingress controller.
func (ic *GenericController) Start() {
glog.Infof("starting Ingress controller")
ic.cacheController.Run(ic.stopCh)
createDefaultSSLCertificate()
time.Sleep(5 * time.Second)
// initial sync of secrets to avoid unnecessary reloads
glog.Info("running initial sync of secrets")
for _, obj := range ic.listers.Ingress.List() {
ing := obj.(*extensions.Ingress)
if !class.IsValid(ing, ic.cfg.IngressClass, ic.cfg.DefaultIngressClass) {
a, _ := parser.GetStringAnnotation(class.IngressKey, ing)
glog.Infof("ignoring add for ingress %v based on annotation %v with value %v", ing.Name, class.IngressKey, a)
continue
}
ic.readSecrets(ing)
}
go ic.syncQueue.Run(time.Second, ic.stopCh)
if ic.syncStatus != nil {
go ic.syncStatus.Run(ic.stopCh)
}
go wait.Until(ic.checkMissingSecrets, 30*time.Second, ic.stopCh)
// force initial sync
ic.syncQueue.Enqueue(&extensions.Ingress{})
<-ic.stopCh
}
func (ic *GenericController) isForceReload() bool {
return atomic.LoadInt32(&ic.forceReload) != 0
}
func (ic *GenericController) SetForceReload(shouldReload bool) {
func (n *NGINXController) SetForceReload(shouldReload bool) {
if shouldReload {
atomic.StoreInt32(&ic.forceReload, 1)
ic.syncQueue.Enqueue(&extensions.Ingress{})
atomic.StoreInt32(&n.forceReload, 1)
n.syncQueue.Enqueue(&extensions.Ingress{})
} else {
atomic.StoreInt32(&ic.forceReload, 0)
atomic.StoreInt32(&n.forceReload, 0)
}
}
func createDefaultSSLCertificate() {
defCert, defKey := ssl.GetFakeSSLCert()
c, err := ssl.AddOrUpdateCertAndKey(fakeCertificate, defCert, defKey, []byte{})
if err != nil {
glog.Fatalf("Error generating self signed certificate: %v", err)
}
fakeCertificateSHA = c.PemSHA
fakeCertificatePath = c.PemFileName
}

View file

@ -1,335 +0,0 @@
package controller
import (
"encoding/json"
"flag"
"fmt"
"net/http"
"net/http/pprof"
"os"
"strings"
"syscall"
"time"
"github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/spf13/pflag"
apiv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
"k8s.io/ingress-nginx/pkg/ingress"
"k8s.io/ingress-nginx/pkg/k8s"
)
// NewIngressController returns a configured Ingress controller
func NewIngressController(backend ingress.Controller) *GenericController {
var (
flags = pflag.NewFlagSet("", pflag.ExitOnError)
apiserverHost = flags.String("apiserver-host", "", "The address of the Kubernetes Apiserver "+
"to connect to in the format of protocol://address:port, e.g., "+
"http://localhost:8080. If not specified, the assumption is that the binary runs inside a "+
"Kubernetes cluster and local discovery is attempted.")
kubeConfigFile = flags.String("kubeconfig", "", "Path to kubeconfig file with authorization and master location information.")
defaultSvc = flags.String("default-backend-service", "",
`Service used to serve a 404 page for the default backend. Takes the form
namespace/name. The controller uses the first node port of this Service for
the default backend.`)
ingressClass = flags.String("ingress-class", "",
`Name of the ingress class to route through this controller.`)
configMap = flags.String("configmap", "",
`Name of the ConfigMap that contains the custom configuration to use`)
publishSvc = flags.String("publish-service", "",
`Service fronting the ingress controllers. Takes the form
namespace/name. The controller will set the endpoint records on the
ingress objects to reflect those on the service.`)
tcpConfigMapName = flags.String("tcp-services-configmap", "",
`Name of the ConfigMap that contains the definition of the TCP services to expose.
The key in the map indicates the external port to be used. The value is the name of the
service with the format namespace/serviceName and the port of the service could be a
number of the name of the port.
The ports 80 and 443 are not allowed as external ports. This ports are reserved for the backend`)
udpConfigMapName = flags.String("udp-services-configmap", "",
`Name of the ConfigMap that contains the definition of the UDP services to expose.
The key in the map indicates the external port to be used. The value is the name of the
service with the format namespace/serviceName and the port of the service could be a
number of the name of the port.`)
resyncPeriod = flags.Duration("sync-period", 600*time.Second,
`Relist and confirm cloud resources this often. Default is 10 minutes`)
watchNamespace = flags.String("watch-namespace", apiv1.NamespaceAll,
`Namespace to watch for Ingress. Default is to watch all namespaces`)
healthzPort = flags.Int("healthz-port", 10254, "port for healthz endpoint.")
profiling = flags.Bool("profiling", true, `Enable profiling via web interface host:port/debug/pprof/`)
defSSLCertificate = flags.String("default-ssl-certificate", "", `Name of the secret
that contains a SSL certificate to be used as default for a HTTPS catch-all server`)
defHealthzURL = flags.String("health-check-path", "/healthz", `Defines
the URL to be used as health check inside in the default server in NGINX.`)
updateStatus = flags.Bool("update-status", true, `Indicates if the
ingress controller should update the Ingress status IP/hostname. Default is true`)
electionID = flags.String("election-id", "ingress-controller-leader", `Election id to use for status update.`)
forceIsolation = flags.Bool("force-namespace-isolation", false,
`Force namespace isolation. This flag is required to avoid the reference of secrets or
configmaps located in a different namespace than the specified in the flag --watch-namespace.`)
disableNodeList = flags.Bool("disable-node-list", false,
`Disable querying nodes. If --force-namespace-isolation is true, this should also be set.`)
updateStatusOnShutdown = flags.Bool("update-status-on-shutdown", true, `Indicates if the
ingress controller should update the Ingress status IP/hostname when the controller
is being stopped. Default is true`)
sortBackends = flags.Bool("sort-backends", false,
`Defines if backends and it's endpoints should be sorted`)
useNodeInternalIP = flags.Bool("report-node-internal-ip-address", false,
`Defines if the nodes IP address to be returned in the ingress status should be the internal instead of the external IP address`)
showVersion = flags.Bool("version", false,
`Shows release information about the NGINX Ingress controller`)
)
flags.AddGoFlagSet(flag.CommandLine)
backend.ConfigureFlags(flags)
flags.Parse(os.Args)
// Workaround for this issue:
// https://github.com/kubernetes/kubernetes/issues/17162
flag.CommandLine.Parse([]string{})
if *showVersion {
fmt.Println(backend.Info().String())
os.Exit(0)
}
backend.OverrideFlags(flags)
flag.Set("logtostderr", "true")
glog.Info(backend.Info())
if *ingressClass != "" {
glog.Infof("Watching for ingress class: %s", *ingressClass)
}
if *defaultSvc == "" {
glog.Fatalf("Please specify --default-backend-service")
}
kubeClient, err := createApiserverClient(*apiserverHost, *kubeConfigFile)
if err != nil {
handleFatalInitError(err)
}
ns, name, err := k8s.ParseNameNS(*defaultSvc)
if err != nil {
glog.Fatalf("invalid format for service %v: %v", *defaultSvc, err)
}
_, err = kubeClient.Core().Services(ns).Get(name, metav1.GetOptions{})
if err != nil {
if strings.Contains(err.Error(), "cannot get services in the namespace") {
glog.Fatalf("✖ It seems the cluster it is running with Authorization enabled (like RBAC) and there is no permissions for the ingress controller. Please check the configuration")
}
glog.Fatalf("no service with name %v found: %v", *defaultSvc, err)
}
glog.Infof("validated %v as the default backend", *defaultSvc)
if *publishSvc != "" {
ns, name, err := k8s.ParseNameNS(*publishSvc)
if err != nil {
glog.Fatalf("invalid service format: %v", err)
}
svc, err := kubeClient.CoreV1().Services(ns).Get(name, metav1.GetOptions{})
if err != nil {
glog.Fatalf("unexpected error getting information about service %v: %v", *publishSvc, err)
}
if len(svc.Status.LoadBalancer.Ingress) == 0 {
if len(svc.Spec.ExternalIPs) > 0 {
glog.Infof("service %v validated as assigned with externalIP", *publishSvc)
} else {
// We could poll here, but we instead just exit and rely on k8s to restart us
glog.Fatalf("service %s does not (yet) have ingress points", *publishSvc)
}
} else {
glog.Infof("service %v validated as source of Ingress status", *publishSvc)
}
}
if *watchNamespace != "" {
_, err = kubeClient.CoreV1().Namespaces().Get(*watchNamespace, metav1.GetOptions{})
if err != nil {
glog.Fatalf("no watchNamespace with name %v found: %v", *watchNamespace, err)
}
}
if resyncPeriod.Seconds() < 10 {
glog.Fatalf("resync period (%vs) is too low", resyncPeriod.Seconds())
}
err = os.MkdirAll(ingress.DefaultSSLDirectory, 0655)
if err != nil {
glog.Errorf("Failed to mkdir SSL directory: %v", err)
}
config := &Configuration{
UpdateStatus: *updateStatus,
ElectionID: *electionID,
Client: kubeClient,
ResyncPeriod: *resyncPeriod,
DefaultService: *defaultSvc,
IngressClass: *ingressClass,
DefaultIngressClass: backend.DefaultIngressClass(),
Namespace: *watchNamespace,
ConfigMapName: *configMap,
TCPConfigMapName: *tcpConfigMapName,
UDPConfigMapName: *udpConfigMapName,
DefaultSSLCertificate: *defSSLCertificate,
DefaultHealthzURL: *defHealthzURL,
PublishService: *publishSvc,
Backend: backend,
ForceNamespaceIsolation: *forceIsolation,
DisableNodeList: *disableNodeList,
UpdateStatusOnShutdown: *updateStatusOnShutdown,
SortBackends: *sortBackends,
UseNodeInternalIP: *useNodeInternalIP,
}
ic := newIngressController(config)
go registerHandlers(*profiling, *healthzPort, ic)
return ic
}
func registerHandlers(enableProfiling bool, port int, ic *GenericController) {
mux := http.NewServeMux()
// expose health check endpoint (/healthz)
healthz.InstallHandler(mux,
healthz.PingHealthz,
ic.cfg.Backend,
)
mux.Handle("/metrics", promhttp.Handler())
mux.HandleFunc("/build", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
b, _ := json.Marshal(ic.Info())
w.Write(b)
})
mux.HandleFunc("/stop", func(w http.ResponseWriter, r *http.Request) {
err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
if err != nil {
glog.Errorf("unexpected error: %v", err)
}
})
if enableProfiling {
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
}
server := &http.Server{
Addr: fmt.Sprintf(":%v", port),
Handler: mux,
}
glog.Fatal(server.ListenAndServe())
}
const (
// High enough QPS to fit all expected use cases. QPS=0 is not set here, because
// client code is overriding it.
defaultQPS = 1e6
// High enough Burst to fit all expected use cases. Burst=0 is not set here, because
// client code is overriding it.
defaultBurst = 1e6
)
// buildConfigFromFlags builds REST config based on master URL and kubeconfig path.
// If both of them are empty then in cluster config is used.
func buildConfigFromFlags(masterURL, kubeconfigPath string) (*rest.Config, error) {
if kubeconfigPath == "" && masterURL == "" {
kubeconfig, err := rest.InClusterConfig()
if err != nil {
return nil, err
}
return kubeconfig, nil
}
return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath},
&clientcmd.ConfigOverrides{
ClusterInfo: clientcmdapi.Cluster{
Server: masterURL,
},
}).ClientConfig()
}
// createApiserverClient creates new Kubernetes Apiserver client. When kubeconfig or apiserverHost param is empty
// the function assumes that it is running inside a Kubernetes cluster and attempts to
// discover the Apiserver. Otherwise, it connects to the Apiserver specified.
//
// apiserverHost param is in the format of protocol://address:port/pathPrefix, e.g.http://localhost:8001.
// kubeConfig location of kubeconfig file
func createApiserverClient(apiserverHost string, kubeConfig string) (*kubernetes.Clientset, error) {
cfg, err := buildConfigFromFlags(apiserverHost, kubeConfig)
if err != nil {
return nil, err
}
cfg.QPS = defaultQPS
cfg.Burst = defaultBurst
cfg.ContentType = "application/vnd.kubernetes.protobuf"
glog.Infof("Creating API client for %s", cfg.Host)
client, err := kubernetes.NewForConfig(cfg)
if err != nil {
return nil, err
}
v, err := client.Discovery().ServerVersion()
if err != nil {
return nil, err
}
glog.Infof("Running in Kubernetes Cluster version v%v.%v (%v) - git (%v) commit %v - platform %v",
v.Major, v.Minor, v.GitVersion, v.GitTreeState, v.GitCommit, v.Platform)
return client, nil
}
/**
* Handles fatal init error that prevents server from doing any work. Prints verbose error
* message and quits the server.
*/
func handleFatalInitError(err error) {
glog.Fatalf("Error while initializing connection to Kubernetes apiserver. "+
"This most likely means that the cluster is misconfigured (e.g., it has "+
"invalid apiserver certificates or service accounts configuration). Reason: %s\n"+
"Refer to the troubleshooting guide for more information: "+
"https://github.com/kubernetes/ingress-nginx/blob/master/docs/troubleshooting.md", err)
}

View file

@ -64,20 +64,21 @@ func (c *cacheController) Run(stopCh chan struct{}) {
}
}
func (ic *GenericController) createListers(disableNodeLister bool) (*ingress.StoreLister, *cacheController) {
func (n *NGINXController) createListers(disableNodeLister bool, stopCh chan struct{}) *ingress.StoreLister {
// from here to the end of the method all the code is just boilerplate
// required to watch Ingress, Secrets, ConfigMaps and Endoints.
// This is used to detect new content, updates or removals and act accordingly
ingEventHandler := cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
addIng := obj.(*extensions.Ingress)
if !class.IsValid(addIng, ic.cfg.IngressClass, ic.cfg.DefaultIngressClass) {
if !class.IsValid(addIng, n.cfg.IngressClass, defIngressClass) {
a, _ := parser.GetStringAnnotation(class.IngressKey, addIng)
glog.Infof("ignoring add for ingress %v based on annotation %v with value %v", addIng.Name, class.IngressKey, a)
return
}
ic.recorder.Eventf(addIng, apiv1.EventTypeNormal, "CREATE", fmt.Sprintf("Ingress %s/%s", addIng.Namespace, addIng.Name))
ic.syncQueue.Enqueue(obj)
n.recorder.Eventf(addIng, apiv1.EventTypeNormal, "CREATE", fmt.Sprintf("Ingress %s/%s", addIng.Namespace, addIng.Name))
n.syncQueue.Enqueue(obj)
},
DeleteFunc: func(obj interface{}) {
delIng, ok := obj.(*extensions.Ingress)
@ -94,29 +95,29 @@ func (ic *GenericController) createListers(disableNodeLister bool) (*ingress.Sto
return
}
}
if !class.IsValid(delIng, ic.cfg.IngressClass, ic.cfg.DefaultIngressClass) {
if !class.IsValid(delIng, n.cfg.IngressClass, defIngressClass) {
glog.Infof("ignoring delete for ingress %v based on annotation %v", delIng.Name, class.IngressKey)
return
}
ic.recorder.Eventf(delIng, apiv1.EventTypeNormal, "DELETE", fmt.Sprintf("Ingress %s/%s", delIng.Namespace, delIng.Name))
ic.syncQueue.Enqueue(obj)
n.recorder.Eventf(delIng, apiv1.EventTypeNormal, "DELETE", fmt.Sprintf("Ingress %s/%s", delIng.Namespace, delIng.Name))
n.syncQueue.Enqueue(obj)
},
UpdateFunc: func(old, cur interface{}) {
oldIng := old.(*extensions.Ingress)
curIng := cur.(*extensions.Ingress)
validOld := class.IsValid(oldIng, ic.cfg.IngressClass, ic.cfg.DefaultIngressClass)
validCur := class.IsValid(curIng, ic.cfg.IngressClass, ic.cfg.DefaultIngressClass)
validOld := class.IsValid(oldIng, n.cfg.IngressClass, defIngressClass)
validCur := class.IsValid(curIng, n.cfg.IngressClass, defIngressClass)
if !validOld && validCur {
glog.Infof("creating ingress %v based on annotation %v", curIng.Name, class.IngressKey)
ic.recorder.Eventf(curIng, apiv1.EventTypeNormal, "CREATE", fmt.Sprintf("Ingress %s/%s", curIng.Namespace, curIng.Name))
n.recorder.Eventf(curIng, apiv1.EventTypeNormal, "CREATE", fmt.Sprintf("Ingress %s/%s", curIng.Namespace, curIng.Name))
} else if validOld && !validCur {
glog.Infof("removing ingress %v based on annotation %v", curIng.Name, class.IngressKey)
ic.recorder.Eventf(curIng, apiv1.EventTypeNormal, "DELETE", fmt.Sprintf("Ingress %s/%s", curIng.Namespace, curIng.Name))
n.recorder.Eventf(curIng, apiv1.EventTypeNormal, "DELETE", fmt.Sprintf("Ingress %s/%s", curIng.Namespace, curIng.Name))
} else if validCur && !reflect.DeepEqual(old, cur) {
ic.recorder.Eventf(curIng, apiv1.EventTypeNormal, "UPDATE", fmt.Sprintf("Ingress %s/%s", curIng.Namespace, curIng.Name))
n.recorder.Eventf(curIng, apiv1.EventTypeNormal, "UPDATE", fmt.Sprintf("Ingress %s/%s", curIng.Namespace, curIng.Name))
}
ic.syncQueue.Enqueue(cur)
n.syncQueue.Enqueue(cur)
},
}
@ -125,7 +126,7 @@ func (ic *GenericController) createListers(disableNodeLister bool) (*ingress.Sto
if !reflect.DeepEqual(old, cur) {
sec := cur.(*apiv1.Secret)
key := fmt.Sprintf("%v/%v", sec.Namespace, sec.Name)
ic.syncSecret(key)
n.syncSecret(key)
}
},
DeleteFunc: func(obj interface{}) {
@ -144,23 +145,23 @@ func (ic *GenericController) createListers(disableNodeLister bool) (*ingress.Sto
}
}
key := fmt.Sprintf("%v/%v", sec.Namespace, sec.Name)
ic.sslCertTracker.DeleteAll(key)
ic.syncQueue.Enqueue(key)
n.sslCertTracker.DeleteAll(key)
n.syncQueue.Enqueue(key)
},
}
eventHandler := cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
ic.syncQueue.Enqueue(obj)
n.syncQueue.Enqueue(obj)
},
DeleteFunc: func(obj interface{}) {
ic.syncQueue.Enqueue(obj)
n.syncQueue.Enqueue(obj)
},
UpdateFunc: func(old, cur interface{}) {
oep := old.(*apiv1.Endpoints)
ocur := cur.(*apiv1.Endpoints)
if !reflect.DeepEqual(ocur.Subsets, oep.Subsets) {
ic.syncQueue.Enqueue(cur)
n.syncQueue.Enqueue(cur)
}
},
}
@ -169,33 +170,33 @@ func (ic *GenericController) createListers(disableNodeLister bool) (*ingress.Sto
AddFunc: func(obj interface{}) {
upCmap := obj.(*apiv1.ConfigMap)
mapKey := fmt.Sprintf("%s/%s", upCmap.Namespace, upCmap.Name)
if mapKey == ic.cfg.ConfigMapName {
if mapKey == n.cfg.ConfigMapName {
glog.V(2).Infof("adding configmap %v to backend", mapKey)
ic.cfg.Backend.SetConfig(upCmap)
ic.SetForceReload(true)
n.SetConfig(upCmap)
n.SetForceReload(true)
}
},
UpdateFunc: func(old, cur interface{}) {
if !reflect.DeepEqual(old, cur) {
upCmap := cur.(*apiv1.ConfigMap)
mapKey := fmt.Sprintf("%s/%s", upCmap.Namespace, upCmap.Name)
if mapKey == ic.cfg.ConfigMapName {
if mapKey == n.cfg.ConfigMapName {
glog.V(2).Infof("updating configmap backend (%v)", mapKey)
ic.cfg.Backend.SetConfig(upCmap)
ic.SetForceReload(true)
n.SetConfig(upCmap)
n.SetForceReload(true)
}
// updates to configuration configmaps can trigger an update
if mapKey == ic.cfg.ConfigMapName || mapKey == ic.cfg.TCPConfigMapName || mapKey == ic.cfg.UDPConfigMapName {
ic.recorder.Eventf(upCmap, apiv1.EventTypeNormal, "UPDATE", fmt.Sprintf("ConfigMap %v", mapKey))
ic.syncQueue.Enqueue(cur)
if mapKey == n.cfg.ConfigMapName || mapKey == n.cfg.TCPConfigMapName || mapKey == n.cfg.UDPConfigMapName {
n.recorder.Eventf(upCmap, apiv1.EventTypeNormal, "UPDATE", fmt.Sprintf("ConfigMap %v", mapKey))
n.syncQueue.Enqueue(cur)
}
}
},
}
watchNs := apiv1.NamespaceAll
if ic.cfg.ForceNamespaceIsolation && ic.cfg.Namespace != apiv1.NamespaceAll {
watchNs = ic.cfg.Namespace
if n.cfg.ForceNamespaceIsolation && n.cfg.Namespace != apiv1.NamespaceAll {
watchNs = n.cfg.Namespace
}
lister := &ingress.StoreLister{}
@ -203,34 +204,36 @@ func (ic *GenericController) createListers(disableNodeLister bool) (*ingress.Sto
controller := &cacheController{}
lister.Ingress.Store, controller.Ingress = cache.NewInformer(
cache.NewListWatchFromClient(ic.cfg.Client.ExtensionsV1beta1().RESTClient(), "ingresses", ic.cfg.Namespace, fields.Everything()),
&extensions.Ingress{}, ic.cfg.ResyncPeriod, ingEventHandler)
cache.NewListWatchFromClient(n.cfg.Client.ExtensionsV1beta1().RESTClient(), "ingresses", n.cfg.Namespace, fields.Everything()),
&extensions.Ingress{}, n.cfg.ResyncPeriod, ingEventHandler)
lister.Endpoint.Store, controller.Endpoint = cache.NewInformer(
cache.NewListWatchFromClient(ic.cfg.Client.CoreV1().RESTClient(), "endpoints", ic.cfg.Namespace, fields.Everything()),
&apiv1.Endpoints{}, ic.cfg.ResyncPeriod, eventHandler)
cache.NewListWatchFromClient(n.cfg.Client.CoreV1().RESTClient(), "endpoints", n.cfg.Namespace, fields.Everything()),
&apiv1.Endpoints{}, n.cfg.ResyncPeriod, eventHandler)
lister.Secret.Store, controller.Secret = cache.NewInformer(
cache.NewListWatchFromClient(ic.cfg.Client.CoreV1().RESTClient(), "secrets", watchNs, fields.Everything()),
&apiv1.Secret{}, ic.cfg.ResyncPeriod, secrEventHandler)
cache.NewListWatchFromClient(n.cfg.Client.CoreV1().RESTClient(), "secrets", watchNs, fields.Everything()),
&apiv1.Secret{}, n.cfg.ResyncPeriod, secrEventHandler)
lister.ConfigMap.Store, controller.Configmap = cache.NewInformer(
cache.NewListWatchFromClient(ic.cfg.Client.CoreV1().RESTClient(), "configmaps", watchNs, fields.Everything()),
&apiv1.ConfigMap{}, ic.cfg.ResyncPeriod, mapEventHandler)
cache.NewListWatchFromClient(n.cfg.Client.CoreV1().RESTClient(), "configmaps", watchNs, fields.Everything()),
&apiv1.ConfigMap{}, n.cfg.ResyncPeriod, mapEventHandler)
lister.Service.Store, controller.Service = cache.NewInformer(
cache.NewListWatchFromClient(ic.cfg.Client.CoreV1().RESTClient(), "services", ic.cfg.Namespace, fields.Everything()),
&apiv1.Service{}, ic.cfg.ResyncPeriod, cache.ResourceEventHandlerFuncs{})
cache.NewListWatchFromClient(n.cfg.Client.CoreV1().RESTClient(), "services", n.cfg.Namespace, fields.Everything()),
&apiv1.Service{}, n.cfg.ResyncPeriod, cache.ResourceEventHandlerFuncs{})
var nodeListerWatcher cache.ListerWatcher
if disableNodeLister {
nodeListerWatcher = fcache.NewFakeControllerSource()
} else {
nodeListerWatcher = cache.NewListWatchFromClient(ic.cfg.Client.CoreV1().RESTClient(), "nodes", apiv1.NamespaceAll, fields.Everything())
nodeListerWatcher = cache.NewListWatchFromClient(n.cfg.Client.CoreV1().RESTClient(), "nodes", apiv1.NamespaceAll, fields.Everything())
}
lister.Node.Store, controller.Node = cache.NewInformer(
nodeListerWatcher,
&apiv1.Node{}, ic.cfg.ResyncPeriod, cache.ResourceEventHandlerFuncs{})
&apiv1.Node{}, n.cfg.ResyncPeriod, cache.ResourceEventHandlerFuncs{})
return lister, controller
controller.Run(n.stopCh)
return lister
}

View file

@ -0,0 +1,151 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package collector
import (
"github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
)
type (
nginxStatusCollector struct {
scrapeChan chan scrapeRequest
ngxHealthPort int
ngxVtsPath string
data *nginxStatusData
watchNamespace string
ingressClass string
}
nginxStatusData struct {
active *prometheus.Desc
accepted *prometheus.Desc
handled *prometheus.Desc
requests *prometheus.Desc
reading *prometheus.Desc
writing *prometheus.Desc
waiting *prometheus.Desc
}
)
// NewNginxStatus returns a new prometheus collector the default nginx status module
func NewNginxStatus(watchNamespace, ingressClass string, ngxHealthPort int, ngxVtsPath string) Stopable {
p := nginxStatusCollector{
scrapeChan: make(chan scrapeRequest),
ngxHealthPort: ngxHealthPort,
ngxVtsPath: ngxVtsPath,
watchNamespace: watchNamespace,
ingressClass: ingressClass,
}
p.data = &nginxStatusData{
active: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "active_connections"),
"total number of active connections",
[]string{"ingress_class", "namespace"}, nil),
accepted: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "accepted_connections"),
"total number of accepted client connections",
[]string{"ingress_class", "namespace"}, nil),
handled: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "handled_connections"),
"total number of handled connections",
[]string{"ingress_class", "namespace"}, nil),
requests: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "total_requests"),
"total number of client requests",
[]string{"ingress_class", "namespace"}, nil),
reading: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "current_reading_connections"),
"current number of connections where nginx is reading the request header",
[]string{"ingress_class", "namespace"}, nil),
writing: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "current_writing_connections"),
"current number of connections where nginx is writing the response back to the client",
[]string{"ingress_class", "namespace"}, nil),
waiting: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "current_waiting_connections"),
"current number of idle client connections waiting for a request",
[]string{"ingress_class", "namespace"}, nil),
}
go p.start()
return p
}
// Describe implements prometheus.Collector.
func (p nginxStatusCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- p.data.active
ch <- p.data.accepted
ch <- p.data.handled
ch <- p.data.requests
ch <- p.data.reading
ch <- p.data.writing
ch <- p.data.waiting
}
// Collect implements prometheus.Collector.
func (p nginxStatusCollector) Collect(ch chan<- prometheus.Metric) {
req := scrapeRequest{results: ch, done: make(chan struct{})}
p.scrapeChan <- req
<-req.done
}
func (p nginxStatusCollector) start() {
for req := range p.scrapeChan {
ch := req.results
p.scrape(ch)
req.done <- struct{}{}
}
}
func (p nginxStatusCollector) Stop() {
close(p.scrapeChan)
}
// nginxStatusCollector scrape the nginx status
func (p nginxStatusCollector) scrape(ch chan<- prometheus.Metric) {
s, err := getNginxStatus(p.ngxHealthPort, p.ngxVtsPath)
if err != nil {
glog.Warningf("unexpected error obtaining nginx status info: %v", err)
return
}
ch <- prometheus.MustNewConstMetric(p.data.active,
prometheus.GaugeValue, float64(s.Active), p.ingressClass, p.watchNamespace)
ch <- prometheus.MustNewConstMetric(p.data.accepted,
prometheus.GaugeValue, float64(s.Accepted), p.ingressClass, p.watchNamespace)
ch <- prometheus.MustNewConstMetric(p.data.handled,
prometheus.GaugeValue, float64(s.Handled), p.ingressClass, p.watchNamespace)
ch <- prometheus.MustNewConstMetric(p.data.requests,
prometheus.GaugeValue, float64(s.Requests), p.ingressClass, p.watchNamespace)
ch <- prometheus.MustNewConstMetric(p.data.reading,
prometheus.GaugeValue, float64(s.Reading), p.ingressClass, p.watchNamespace)
ch <- prometheus.MustNewConstMetric(p.data.writing,
prometheus.GaugeValue, float64(s.Writing), p.ingressClass, p.watchNamespace)
ch <- prometheus.MustNewConstMetric(p.data.waiting,
prometheus.GaugeValue, float64(s.Waiting), p.ingressClass, p.watchNamespace)
}

View file

@ -0,0 +1,174 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package collector
import (
"path/filepath"
"github.com/golang/glog"
common "github.com/ncabatoff/process-exporter"
"github.com/ncabatoff/process-exporter/proc"
"github.com/prometheus/client_golang/prometheus"
)
// BinaryNameMatcher ...
type BinaryNameMatcher struct {
Name string
Binary string
}
// MatchAndName returns false if the match failed, otherwise
// true and the resulting name.
func (em BinaryNameMatcher) MatchAndName(nacl common.NameAndCmdline) (bool, string) {
if len(nacl.Cmdline) == 0 {
return false, ""
}
cmd := filepath.Base(em.Binary)
return em.Name == cmd, ""
}
type namedProcessData struct {
numProcs *prometheus.Desc
cpuSecs *prometheus.Desc
readBytes *prometheus.Desc
writeBytes *prometheus.Desc
memResidentbytes *prometheus.Desc
memVirtualbytes *prometheus.Desc
startTime *prometheus.Desc
}
type namedProcess struct {
*proc.Grouper
scrapeChan chan scrapeRequest
fs *proc.FS
data namedProcessData
}
// NewNamedProcess returns a new prometheus collector for the nginx process
func NewNamedProcess(children bool, mn common.MatchNamer) (prometheus.Collector, error) {
fs, err := proc.NewFS("/proc")
if err != nil {
return nil, err
}
p := namedProcess{
scrapeChan: make(chan scrapeRequest),
Grouper: proc.NewGrouper(children, mn),
fs: fs,
}
_, err = p.Update(p.fs.AllProcs())
if err != nil {
return nil, err
}
p.data = namedProcessData{
numProcs: prometheus.NewDesc(
"num_procs",
"number of processes",
nil, nil),
cpuSecs: prometheus.NewDesc(
"cpu_seconds_total",
"Cpu usage in seconds",
nil, nil),
readBytes: prometheus.NewDesc(
"read_bytes_total",
"number of bytes read",
nil, nil),
writeBytes: prometheus.NewDesc(
"write_bytes_total",
"number of bytes written",
nil, nil),
memResidentbytes: prometheus.NewDesc(
"resident_memory_bytes",
"number of bytes of memory in use",
nil, nil),
memVirtualbytes: prometheus.NewDesc(
"virtual_memory_bytes",
"number of bytes of memory in use",
nil, nil),
startTime: prometheus.NewDesc(
"oldest_start_time_seconds",
"start time in seconds since 1970/01/01",
nil, nil),
}
go p.start()
return p, nil
}
// Describe implements prometheus.Collector.
func (p namedProcess) Describe(ch chan<- *prometheus.Desc) {
ch <- p.data.cpuSecs
ch <- p.data.numProcs
ch <- p.data.readBytes
ch <- p.data.writeBytes
ch <- p.data.memResidentbytes
ch <- p.data.memVirtualbytes
ch <- p.data.startTime
}
// Collect implements prometheus.Collector.
func (p namedProcess) Collect(ch chan<- prometheus.Metric) {
req := scrapeRequest{results: ch, done: make(chan struct{})}
p.scrapeChan <- req
<-req.done
}
func (p namedProcess) start() {
for req := range p.scrapeChan {
ch := req.results
p.scrape(ch)
req.done <- struct{}{}
}
}
func (p namedProcess) Stop() {
close(p.scrapeChan)
}
func (p namedProcess) scrape(ch chan<- prometheus.Metric) {
_, err := p.Update(p.fs.AllProcs())
if err != nil {
glog.Warningf("unexpected error obtaining nginx process info: %v", err)
return
}
for _, gcounts := range p.Groups() {
ch <- prometheus.MustNewConstMetric(p.data.numProcs,
prometheus.GaugeValue, float64(gcounts.Procs))
ch <- prometheus.MustNewConstMetric(p.data.memResidentbytes,
prometheus.GaugeValue, float64(gcounts.Memresident))
ch <- prometheus.MustNewConstMetric(p.data.memVirtualbytes,
prometheus.GaugeValue, float64(gcounts.Memvirtual))
ch <- prometheus.MustNewConstMetric(p.data.startTime,
prometheus.GaugeValue, float64(gcounts.OldestStartTime.Unix()))
ch <- prometheus.MustNewConstMetric(p.data.cpuSecs,
prometheus.CounterValue, gcounts.Cpu)
ch <- prometheus.MustNewConstMetric(p.data.readBytes,
prometheus.CounterValue, float64(gcounts.ReadBytes))
ch <- prometheus.MustNewConstMetric(p.data.writeBytes,
prometheus.CounterValue, float64(gcounts.WriteBytes))
}
}

View file

@ -0,0 +1,30 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package collector
import "github.com/prometheus/client_golang/prometheus"
// Stopable defines a prometheus collector that can be stopped
type Stopable interface {
prometheus.Collector
Stop()
}
type scrapeRequest struct {
results chan<- prometheus.Metric
done chan struct{}
}

View file

@ -0,0 +1,225 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package collector
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"regexp"
"strconv"
"github.com/golang/glog"
)
var (
ac = regexp.MustCompile(`Active connections: (\d+)`)
sahr = regexp.MustCompile(`(\d+)\s(\d+)\s(\d+)`)
reading = regexp.MustCompile(`Reading: (\d+)`)
writing = regexp.MustCompile(`Writing: (\d+)`)
waiting = regexp.MustCompile(`Waiting: (\d+)`)
)
type basicStatus struct {
// Active total number of active connections
Active int
// Accepted total number of accepted client connections
Accepted int
// Handled total number of handled connections. Generally, the parameter value is the same as accepts unless some resource limits have been reached (for example, the worker_connections limit).
Handled int
// Requests total number of client requests.
Requests int
// Reading current number of connections where nginx is reading the request header.
Reading int
// Writing current number of connections where nginx is writing the response back to the client.
Writing int
// Waiting current number of idle client connections waiting for a request.
Waiting int
}
// https://github.com/vozlt/nginx-module-vts
type vts struct {
NginxVersion string `json:"nginxVersion"`
LoadMsec int `json:"loadMsec"`
NowMsec int `json:"nowMsec"`
// Total connections and requests(same as stub_status_module in NGINX)
Connections connections `json:"connections"`
// Traffic(in/out) and request and response counts and cache hit ratio per each server zone
ServerZones map[string]serverZone `json:"serverZones"`
// Traffic(in/out) and request and response counts and cache hit ratio per each server zone filtered through
// the vhost_traffic_status_filter_by_set_key directive
FilterZones map[string]map[string]filterZone `json:"filterZones"`
// Traffic(in/out) and request and response counts per server in each upstream group
UpstreamZones map[string][]upstreamZone `json:"upstreamZones"`
}
type serverZone struct {
RequestCounter float64 `json:"requestCounter"`
InBytes float64 `json:"inBytes"`
OutBytes float64 `json:"outBytes"`
Responses response `json:"responses"`
Cache cache `json:"cache"`
}
type filterZone struct {
RequestCounter float64 `json:"requestCounter"`
InBytes float64 `json:"inBytes"`
OutBytes float64 `json:"outBytes"`
Cache cache `json:"cache"`
Responses response `json:"responses"`
}
type upstreamZone struct {
Responses response `json:"responses"`
Server string `json:"server"`
RequestCounter float64 `json:"requestCounter"`
InBytes float64 `json:"inBytes"`
OutBytes float64 `json:"outBytes"`
ResponseMsec float64 `json:"responseMsec"`
Weight float64 `json:"weight"`
MaxFails float64 `json:"maxFails"`
FailTimeout float64 `json:"failTimeout"`
Backup BoolToFloat64 `json:"backup"`
Down BoolToFloat64 `json:"down"`
}
type cache struct {
Miss float64 `json:"miss"`
Bypass float64 `json:"bypass"`
Expired float64 `json:"expired"`
Stale float64 `json:"stale"`
Updating float64 `json:"updating"`
Revalidated float64 `json:"revalidated"`
Hit float64 `json:"hit"`
Scarce float64 `json:"scarce"`
}
type response struct {
OneXx float64 `json:"1xx"`
TwoXx float64 `json:"2xx"`
TheeXx float64 `json:"3xx"`
FourXx float64 `json:"4xx"`
FiveXx float64 `json:"5xx"`
}
type connections struct {
Active float64 `json:"active"`
Reading float64 `json:"reading"`
Writing float64 `json:"writing"`
Waiting float64 `json:"waiting"`
Accepted float64 `json:"accepted"`
Handled float64 `json:"handled"`
Requests float64 `json:"requests"`
}
// BoolToFloat64 ...
type BoolToFloat64 float64
// UnmarshalJSON ...
func (bit BoolToFloat64) UnmarshalJSON(data []byte) error {
asString := string(data)
if asString == "1" || asString == "true" {
bit = 1
} else if asString == "0" || asString == "false" {
bit = 0
} else {
return fmt.Errorf(fmt.Sprintf("boolean unmarshal error: invalid input %s", asString))
}
return nil
}
func getNginxStatus(port int, path string) (*basicStatus, error) {
url := fmt.Sprintf("http://0.0.0.0:%v%v", port, path)
glog.V(3).Infof("start scraping url: %v", url)
data, err := httpBody(url)
if err != nil {
return nil, fmt.Errorf("unexpected error scraping nginx status page: %v", err)
}
return parse(string(data)), nil
}
func httpBody(url string) ([]byte, error) {
resp, err := http.DefaultClient.Get(url)
if err != nil {
return nil, fmt.Errorf("unexpected error scraping nginx : %v", err)
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("unexpected error scraping nginx (%v)", err)
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 400 {
return nil, fmt.Errorf("unexpected error scraping nginx (status %v)", resp.StatusCode)
}
return data, nil
}
func getNginxVtsMetrics(port int, path string) (*vts, error) {
url := fmt.Sprintf("http://0.0.0.0:%v%v", port, path)
glog.V(3).Infof("start scraping url: %v", url)
data, err := httpBody(url)
if err != nil {
return nil, fmt.Errorf("unexpected error scraping nginx vts (%v)", err)
}
var vts *vts
err = json.Unmarshal(data, &vts)
if err != nil {
return nil, fmt.Errorf("unexpected error json unmarshal (%v)", err)
}
glog.V(3).Infof("scrape returned : %v", vts)
return vts, nil
}
func parse(data string) *basicStatus {
acr := ac.FindStringSubmatch(data)
sahrr := sahr.FindStringSubmatch(data)
readingr := reading.FindStringSubmatch(data)
writingr := writing.FindStringSubmatch(data)
waitingr := waiting.FindStringSubmatch(data)
return &basicStatus{
toInt(acr, 1),
toInt(sahrr, 1),
toInt(sahrr, 2),
toInt(sahrr, 3),
toInt(readingr, 1),
toInt(writingr, 1),
toInt(waitingr, 1),
}
}
func toInt(data []string, pos int) int {
if len(data) == 0 {
return 0
}
if pos > len(data) {
return 0
}
if v, err := strconv.Atoi(data[pos]); err == nil {
return v
}
return 0
}

View file

@ -0,0 +1,72 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package collector
import (
"testing"
"github.com/kylelemons/godebug/pretty"
)
func TestParseStatus(t *testing.T) {
tests := []struct {
in string
out *basicStatus
}{
{`Active connections: 43
server accepts handled requests
7368 7368 10993
Reading: 0 Writing: 5 Waiting: 38`,
&basicStatus{43, 7368, 7368, 10993, 0, 5, 38},
},
{`Active connections: 0
server accepts handled requests
1 7 0
Reading: A Writing: B Waiting: 38`,
&basicStatus{0, 1, 7, 0, 0, 0, 38},
},
}
for _, test := range tests {
r := parse(test.in)
if diff := pretty.Compare(r, test.out); diff != "" {
t.Logf("%v", diff)
t.Fatalf("expected %v but returned %v", test.out, r)
}
}
}
func TestToint(t *testing.T) {
tests := []struct {
in []string
pos int
exp int
}{
{[]string{}, 0, 0},
{[]string{}, 1, 0},
{[]string{"A"}, 0, 0},
{[]string{"1"}, 0, 1},
{[]string{"a", "2"}, 1, 2},
}
for _, test := range tests {
v := toInt(test.in, test.pos)
if v != test.exp {
t.Fatalf("expected %v but returned %v", test.exp, v)
}
}
}

View file

@ -0,0 +1,273 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package collector
import (
"reflect"
"github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
)
const ns = "nginx"
type (
vtsCollector struct {
scrapeChan chan scrapeRequest
port int
path string
data *vtsData
watchNamespace string
ingressClass string
}
vtsData struct {
bytes *prometheus.Desc
cache *prometheus.Desc
connections *prometheus.Desc
responses *prometheus.Desc
requests *prometheus.Desc
filterZoneBytes *prometheus.Desc
filterZoneResponses *prometheus.Desc
filterZoneCache *prometheus.Desc
upstreamBackup *prometheus.Desc
upstreamBytes *prometheus.Desc
upstreamDown *prometheus.Desc
upstreamFailTimeout *prometheus.Desc
upstreamMaxFails *prometheus.Desc
upstreamResponses *prometheus.Desc
upstreamRequests *prometheus.Desc
upstreamResponseMsec *prometheus.Desc
upstreamWeight *prometheus.Desc
}
)
// NewNGINXVTSCollector returns a new prometheus collector for the VTS module
func NewNGINXVTSCollector(watchNamespace, ingressClass string, port int, path string) Stopable {
p := vtsCollector{
scrapeChan: make(chan scrapeRequest),
port: port,
path: path,
watchNamespace: watchNamespace,
ingressClass: ingressClass,
}
p.data = &vtsData{
bytes: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "bytes_total"),
"Nginx bytes count",
[]string{"ingress_class", "namespace", "server_zone", "direction"}, nil),
cache: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "cache_total"),
"Nginx cache count",
[]string{"ingress_class", "namespace", "server_zone", "type"}, nil),
connections: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "connections_total"),
"Nginx connections count",
[]string{"ingress_class", "namespace", "type"}, nil),
responses: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "responses_total"),
"The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.",
[]string{"ingress_class", "namespace", "server_zone", "status_code"}, nil),
requests: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "requests_total"),
"The total number of requested client connections.",
[]string{"ingress_class", "namespace", "server_zone"}, nil),
filterZoneBytes: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "filterzone_bytes_total"),
"Nginx bytes count",
[]string{"ingress_class", "namespace", "server_zone", "key", "direction"}, nil),
filterZoneResponses: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "filterzone_responses_total"),
"The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.",
[]string{"ingress_class", "namespace", "server_zone", "key", "status_code"}, nil),
filterZoneCache: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "filterzone_cache_total"),
"Nginx cache count",
[]string{"ingress_class", "namespace", "server_zone", "key", "type"}, nil),
upstreamBackup: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "upstream_backup"),
"Current backup setting of the server.",
[]string{"ingress_class", "namespace", "upstream", "server"}, nil),
upstreamBytes: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "upstream_bytes_total"),
"The total number of bytes sent to this server.",
[]string{"ingress_class", "namespace", "upstream", "server", "direction"}, nil),
upstreamDown: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "vts_upstream_down_total"),
"Current down setting of the server.",
[]string{"ingress_class", "namespace", "upstream", "server"}, nil),
upstreamFailTimeout: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "upstream_fail_timeout"),
"Current fail_timeout setting of the server.",
[]string{"ingress_class", "namespace", "upstream", "server"}, nil),
upstreamMaxFails: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "upstream_maxfails"),
"Current max_fails setting of the server.",
[]string{"ingress_class", "namespace", "upstream", "server"}, nil),
upstreamResponses: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "upstream_responses_total"),
"The number of upstream responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.",
[]string{"ingress_class", "namespace", "upstream", "server", "status_code"}, nil),
upstreamRequests: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "upstream_requests_total"),
"The total number of client connections forwarded to this server.",
[]string{"ingress_class", "namespace", "upstream", "server"}, nil),
upstreamResponseMsec: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "upstream_response_msecs_avg"),
"The average of only upstream response processing times in milliseconds.",
[]string{"ingress_class", "namespace", "upstream", "server"}, nil),
upstreamWeight: prometheus.NewDesc(
prometheus.BuildFQName(ns, "", "upstream_weight"),
"Current upstream weight setting of the server.",
[]string{"ingress_class", "namespace", "upstream", "server"}, nil),
}
go p.start()
return p
}
// Describe implements prometheus.Collector.
func (p vtsCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- p.data.bytes
ch <- p.data.cache
ch <- p.data.connections
ch <- p.data.requests
ch <- p.data.responses
ch <- p.data.upstreamBackup
ch <- p.data.upstreamBytes
ch <- p.data.upstreamDown
ch <- p.data.upstreamFailTimeout
ch <- p.data.upstreamMaxFails
ch <- p.data.upstreamRequests
ch <- p.data.upstreamResponseMsec
ch <- p.data.upstreamResponses
ch <- p.data.upstreamWeight
ch <- p.data.filterZoneBytes
ch <- p.data.filterZoneCache
ch <- p.data.filterZoneResponses
}
// Collect implements prometheus.Collector.
func (p vtsCollector) Collect(ch chan<- prometheus.Metric) {
req := scrapeRequest{results: ch, done: make(chan struct{})}
p.scrapeChan <- req
<-req.done
}
func (p vtsCollector) start() {
for req := range p.scrapeChan {
ch := req.results
p.scrapeVts(ch)
req.done <- struct{}{}
}
}
func (p vtsCollector) Stop() {
close(p.scrapeChan)
}
// scrapeVts scrape nginx vts metrics
func (p vtsCollector) scrapeVts(ch chan<- prometheus.Metric) {
nginxMetrics, err := getNginxVtsMetrics(p.port, p.path)
if err != nil {
glog.Warningf("unexpected error obtaining nginx status info: %v", err)
return
}
reflectMetrics(&nginxMetrics.Connections, p.data.connections, ch, p.ingressClass, p.watchNamespace)
for name, zones := range nginxMetrics.UpstreamZones {
for pos, value := range zones {
reflectMetrics(&zones[pos].Responses, p.data.upstreamResponses, ch, p.ingressClass, p.watchNamespace, name, value.Server)
ch <- prometheus.MustNewConstMetric(p.data.upstreamRequests,
prometheus.CounterValue, zones[pos].RequestCounter, p.ingressClass, p.watchNamespace, name, value.Server)
ch <- prometheus.MustNewConstMetric(p.data.upstreamDown,
prometheus.CounterValue, float64(zones[pos].Down), p.ingressClass, p.watchNamespace, name, value.Server)
ch <- prometheus.MustNewConstMetric(p.data.upstreamWeight,
prometheus.CounterValue, zones[pos].Weight, p.ingressClass, p.watchNamespace, name, value.Server)
ch <- prometheus.MustNewConstMetric(p.data.upstreamResponseMsec,
prometheus.CounterValue, zones[pos].ResponseMsec, p.ingressClass, p.watchNamespace, name, value.Server)
ch <- prometheus.MustNewConstMetric(p.data.upstreamBackup,
prometheus.CounterValue, float64(zones[pos].Backup), p.ingressClass, p.watchNamespace, name, value.Server)
ch <- prometheus.MustNewConstMetric(p.data.upstreamFailTimeout,
prometheus.CounterValue, zones[pos].FailTimeout, p.ingressClass, p.watchNamespace, name, value.Server)
ch <- prometheus.MustNewConstMetric(p.data.upstreamMaxFails,
prometheus.CounterValue, zones[pos].MaxFails, p.ingressClass, p.watchNamespace, name, value.Server)
ch <- prometheus.MustNewConstMetric(p.data.upstreamBytes,
prometheus.CounterValue, zones[pos].InBytes, p.ingressClass, p.watchNamespace, name, value.Server, "in")
ch <- prometheus.MustNewConstMetric(p.data.upstreamBytes,
prometheus.CounterValue, zones[pos].OutBytes, p.ingressClass, p.watchNamespace, name, value.Server, "out")
}
}
for name, zone := range nginxMetrics.ServerZones {
reflectMetrics(&zone.Responses, p.data.responses, ch, p.ingressClass, p.watchNamespace, name)
reflectMetrics(&zone.Cache, p.data.cache, ch, p.ingressClass, p.watchNamespace, name)
ch <- prometheus.MustNewConstMetric(p.data.requests,
prometheus.CounterValue, zone.RequestCounter, p.ingressClass, p.watchNamespace, name)
ch <- prometheus.MustNewConstMetric(p.data.bytes,
prometheus.CounterValue, zone.InBytes, p.ingressClass, p.watchNamespace, name, "in")
ch <- prometheus.MustNewConstMetric(p.data.bytes,
prometheus.CounterValue, zone.OutBytes, p.ingressClass, p.watchNamespace, name, "out")
}
for serverZone, keys := range nginxMetrics.FilterZones {
for name, zone := range keys {
reflectMetrics(&zone.Responses, p.data.filterZoneResponses, ch, p.ingressClass, p.watchNamespace, serverZone, name)
reflectMetrics(&zone.Cache, p.data.filterZoneCache, ch, p.ingressClass, p.watchNamespace, serverZone, name)
ch <- prometheus.MustNewConstMetric(p.data.filterZoneBytes,
prometheus.CounterValue, zone.InBytes, p.ingressClass, p.watchNamespace, serverZone, name, "in")
ch <- prometheus.MustNewConstMetric(p.data.filterZoneBytes,
prometheus.CounterValue, zone.OutBytes, p.ingressClass, p.watchNamespace, serverZone, name, "out")
}
}
}
func reflectMetrics(value interface{}, desc *prometheus.Desc, ch chan<- prometheus.Metric, labels ...string) {
val := reflect.ValueOf(value).Elem()
for i := 0; i < val.NumField(); i++ {
tag := val.Type().Field(i).Tag
l := append(labels, tag.Get("json"))
ch <- prometheus.MustNewConstMetric(desc,
prometheus.CounterValue, val.Field(i).Interface().(float64),
l...)
}
}

View file

@ -34,7 +34,6 @@ func init() {
prometheus.MustRegister(reloadOperation)
prometheus.MustRegister(reloadOperationErrors)
prometheus.MustRegister(sslExpireTime)
}
var (
@ -74,11 +73,9 @@ func incReloadErrorCount() {
}
func setSSLExpireTime(servers []*ingress.Server) {
for _, s := range servers {
if s.Hostname != defServerName {
sslExpireTime.WithLabelValues(s.Hostname).Set(float64(s.SSLExpireTime.Unix()))
}
}
}

View file

@ -0,0 +1,686 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controller
import (
"bytes"
"encoding/base64"
"errors"
"fmt"
"io/ioutil"
"net"
"os"
"os/exec"
"strconv"
"strings"
"sync"
"syscall"
"time"
"github.com/golang/glog"
apiv1 "k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes/scheme"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/flowcontrol"
"k8s.io/ingress-nginx/pkg/ingress"
"k8s.io/ingress-nginx/pkg/ingress/annotations/class"
"k8s.io/ingress-nginx/pkg/ingress/annotations/parser"
ngx_config "k8s.io/ingress-nginx/pkg/ingress/controller/config"
"k8s.io/ingress-nginx/pkg/ingress/controller/process"
ngx_template "k8s.io/ingress-nginx/pkg/ingress/controller/template"
"k8s.io/ingress-nginx/pkg/ingress/defaults"
"k8s.io/ingress-nginx/pkg/ingress/status"
ing_net "k8s.io/ingress-nginx/pkg/net"
"k8s.io/ingress-nginx/pkg/net/dns"
"k8s.io/ingress-nginx/pkg/net/ssl"
"k8s.io/ingress-nginx/pkg/task"
)
type statusModule string
const (
ngxHealthPath = "/healthz"
defaultStatusModule statusModule = "default"
vtsStatusModule statusModule = "vts"
)
var (
tmplPath = "/etc/nginx/template/nginx.tmpl"
cfgPath = "/etc/nginx/nginx.conf"
nginxBinary = "/usr/sbin/nginx"
defIngressClass = "nginx"
)
// NewNGINXController creates a new NGINX Ingress controller.
// If the environment variable NGINX_BINARY exists it will be used
// as source for nginx commands
func NewNGINXController(config *Configuration) *NGINXController {
ngx := os.Getenv("NGINX_BINARY")
if ngx == "" {
ngx = nginxBinary
}
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartLogging(glog.Infof)
eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{
Interface: config.Client.CoreV1().Events(config.Namespace),
})
h, err := dns.GetSystemNameServers()
if err != nil {
glog.Warningf("unexpected error reading system nameservers: %v", err)
}
n := &NGINXController{
backendDefaults: ngx_config.NewDefault().Backend,
binary: ngx,
configmap: &apiv1.ConfigMap{},
isIPV6Enabled: ing_net.IsIPv6Enabled(),
resolver: h,
cfg: config,
sslCertTracker: newSSLCertTracker(),
syncRateLimiter: flowcontrol.NewTokenBucketRateLimiter(0.3, 1),
recorder: eventBroadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{
Component: "nginx-ingress-controller",
}),
stopCh: make(chan struct{}),
stopLock: &sync.Mutex{},
}
n.stats = newStatsCollector(config.Namespace, config.IngressClass, n.binary, n.cfg.ListenPorts.Status)
n.syncQueue = task.NewTaskQueue(n.syncIngress)
n.listers = n.createListers(config.DisableNodeList, n.stopCh)
if config.UpdateStatus {
n.syncStatus = status.NewStatusSyncer(status.Config{
Client: config.Client,
PublishService: n.cfg.PublishService,
IngressLister: n.listers.Ingress,
ElectionID: config.ElectionID,
IngressClass: config.IngressClass,
DefaultIngressClass: config.DefaultIngressClass,
UpdateStatusOnShutdown: config.UpdateStatusOnShutdown,
UseNodeInternalIP: n.cfg.UseNodeInternalIP,
})
} else {
glog.Warning("Update of ingress status is disabled (flag --update-status=false was specified)")
}
n.annotations = newAnnotationExtractor(n)
var onChange func()
onChange = func() {
template, err := ngx_template.NewTemplate(tmplPath, onChange)
if err != nil {
// this error is different from the rest because it must be clear why nginx is not working
glog.Errorf(`
-------------------------------------------------------------------------------
Error loading new template : %v
-------------------------------------------------------------------------------
`, err)
return
}
n.t.Close()
n.t = template
glog.Info("new NGINX template loaded")
n.SetForceReload(true)
}
ngxTpl, err := ngx_template.NewTemplate(tmplPath, onChange)
if err != nil {
glog.Fatalf("invalid NGINX template: %v", err)
}
n.t = ngxTpl
return n
}
// NGINXController ...
type NGINXController struct {
cfg *Configuration
listers *ingress.StoreLister
annotations annotationExtractor
recorder record.EventRecorder
syncQueue *task.Queue
syncStatus status.Sync
// local store of SSL certificates
// (only certificates used in ingress)
sslCertTracker *sslCertTracker
syncRateLimiter flowcontrol.RateLimiter
// stopLock is used to enforce only a single call to Stop is active.
// Needed because we allow stopping through an http endpoint and
// allowing concurrent stoppers leads to stack traces.
stopLock *sync.Mutex
stopCh chan struct{}
// ngxErrCh channel used to detect errors with the nginx processes
ngxErrCh chan error
// runningConfig contains the running configuration in the Backend
runningConfig *ingress.Configuration
forceReload int32
t *ngx_template.Template
configmap *apiv1.ConfigMap
storeLister *ingress.StoreLister
binary string
resolver []net.IP
stats *statsCollector
statusModule statusModule
// returns true if IPV6 is enabled in the pod
isIPV6Enabled bool
// returns true if proxy protocol es enabled
IsProxyProtocolEnabled bool
isSSLPassthroughEnabled bool
isShuttingDown bool
Proxy *TCPProxy
backendDefaults defaults.Backend
}
// Start start a new NGINX master process running in foreground.
func (n *NGINXController) Start() {
glog.Infof("starting Ingress controller")
// initial sync of secrets to avoid unnecessary reloads
glog.Info("running initial sync of secrets")
for _, obj := range n.listers.Ingress.List() {
ing := obj.(*extensions.Ingress)
if !class.IsValid(ing, n.cfg.IngressClass, n.cfg.DefaultIngressClass) {
a, _ := parser.GetStringAnnotation(class.IngressKey, ing)
glog.Infof("ignoring add for ingress %v based on annotation %v with value %v", ing.Name, class.IngressKey, a)
continue
}
n.readSecrets(ing)
}
go n.syncQueue.Run(time.Second, n.stopCh)
if n.syncStatus != nil {
go n.syncStatus.Run(n.stopCh)
}
go wait.Until(n.checkMissingSecrets, 30*time.Second, n.stopCh)
done := make(chan error, 1)
cmd := exec.Command(n.binary, "-c", cfgPath)
// put nginx in another process group to prevent it
// to receive signals meant for the controller
cmd.SysProcAttr = &syscall.SysProcAttr{
Setpgid: true,
Pgid: 0,
}
glog.Info("starting NGINX process...")
n.start(cmd)
// force initial sync
n.syncQueue.Enqueue(&extensions.Ingress{})
for {
select {
case err := <-done:
if n.isShuttingDown {
break
}
// if the nginx master process dies the workers continue to process requests,
// passing checks but in case of updates in ingress no updates will be
// reflected in the nginx configuration which can lead to confusion and report
// issues because of this behavior.
// To avoid this issue we restart nginx in case of errors.
if process.IsRespawnIfRequired(err) {
process.WaitUntilPortIsAvailable(n.cfg.ListenPorts.HTTP)
// release command resources
cmd.Process.Release()
cmd = exec.Command(n.binary, "-c", cfgPath)
// start a new nginx master process if the controller is not being stopped
n.start(cmd)
}
case <-n.stopCh:
break
}
}
}
// Stop gracefully stops the NGINX master process.
func (n *NGINXController) Stop() error {
n.isShuttingDown = true
n.stopLock.Lock()
defer n.stopLock.Unlock()
// Only try draining the workqueue if we haven't already.
if n.syncQueue.IsShuttingDown() {
return fmt.Errorf("shutdown already in progress")
}
glog.Infof("shutting down controller queues")
close(n.stopCh)
go n.syncQueue.Shutdown()
if n.syncStatus != nil {
n.syncStatus.Shutdown()
}
// Send stop signal to Nginx
glog.Info("stopping NGINX process...")
cmd := exec.Command(n.binary, "-c", cfgPath, "-s", "quit")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Run()
if err != nil {
return err
}
// Wait for the Nginx process disappear
timer := time.NewTicker(time.Second * 1)
for t := range timer.C {
glog.V(3).Infof("tick at", t)
if !process.IsNginxRunning() {
glog.Info("NGINX process has stopped")
timer.Stop()
break
}
}
return nil
}
func (n *NGINXController) start(cmd *exec.Cmd) {
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Start(); err != nil {
glog.Fatalf("nginx error: %v", err)
n.ngxErrCh <- err
return
}
go func() {
n.ngxErrCh <- cmd.Wait()
}()
}
// DefaultEndpoint returns the default endpoint to be use as default server that returns 404.
func (n NGINXController) DefaultEndpoint() ingress.Endpoint {
return ingress.Endpoint{
Address: "127.0.0.1",
Port: fmt.Sprintf("%v", n.cfg.ListenPorts.Default),
Target: &apiv1.ObjectReference{},
}
}
// testTemplate checks if the NGINX configuration inside the byte array is valid
// running the command "nginx -t" using a temporal file.
func (n NGINXController) testTemplate(cfg []byte) error {
if len(cfg) == 0 {
return fmt.Errorf("invalid nginx configuration (empty)")
}
tmpfile, err := ioutil.TempFile("", "nginx-cfg")
if err != nil {
return err
}
defer tmpfile.Close()
err = ioutil.WriteFile(tmpfile.Name(), cfg, 0644)
if err != nil {
return err
}
out, err := exec.Command(n.binary, "-t", "-c", tmpfile.Name()).CombinedOutput()
if err != nil {
// this error is different from the rest because it must be clear why nginx is not working
oe := fmt.Sprintf(`
-------------------------------------------------------------------------------
Error: %v
%v
-------------------------------------------------------------------------------
`, err, string(out))
return errors.New(oe)
}
os.Remove(tmpfile.Name())
return nil
}
// SetConfig sets the configured configmap
func (n *NGINXController) SetConfig(cmap *apiv1.ConfigMap) {
n.configmap = cmap
n.IsProxyProtocolEnabled = false
m := map[string]string{}
if cmap != nil {
m = cmap.Data
}
val, ok := m["use-proxy-protocol"]
if ok {
b, err := strconv.ParseBool(val)
if err == nil {
n.IsProxyProtocolEnabled = b
}
}
c := ngx_template.ReadConfig(m)
if c.SSLSessionTicketKey != "" {
d, err := base64.StdEncoding.DecodeString(c.SSLSessionTicketKey)
if err != nil {
glog.Warningf("unexpected error decoding key ssl-session-ticket-key: %v", err)
c.SSLSessionTicketKey = ""
}
ioutil.WriteFile("/etc/nginx/tickets.key", d, 0644)
}
n.backendDefaults = c.Backend
}
// OnUpdate is called by syncQueue in https://github.com/kubernetes/ingress-nginx/blob/master/pkg/ingress/controller/controller.go#L426
// periodically to keep the configuration in sync.
//
// convert configmap to custom configuration object (different in each implementation)
// write the custom template (the complexity depends on the implementation)
// write the configuration file
// returning nill implies the backend will be reloaded.
// if an error is returned means requeue the update
func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
cfg := ngx_template.ReadConfig(n.configmap.Data)
cfg.Resolver = n.resolver
servers := []*TCPServer{}
for _, pb := range ingressCfg.PassthroughBackends {
svc := pb.Service
if svc == nil {
glog.Warningf("missing service for PassthroughBackends %v", pb.Backend)
continue
}
port, err := strconv.Atoi(pb.Port.String())
if err != nil {
for _, sp := range svc.Spec.Ports {
if sp.Name == pb.Port.String() {
port = int(sp.Port)
break
}
}
} else {
for _, sp := range svc.Spec.Ports {
if sp.Port == int32(port) {
port = int(sp.Port)
break
}
}
}
//TODO: Allow PassthroughBackends to specify they support proxy-protocol
servers = append(servers, &TCPServer{
Hostname: pb.Hostname,
IP: svc.Spec.ClusterIP,
Port: port,
ProxyProtocol: false,
})
}
if n.isSSLPassthroughEnabled {
n.Proxy.ServerList = servers
}
// we need to check if the status module configuration changed
if cfg.EnableVtsStatus {
n.setupMonitor(vtsStatusModule)
} else {
n.setupMonitor(defaultStatusModule)
}
// NGINX cannot resize the hash tables used to store server names.
// For this reason we check if the defined size defined is correct
// for the FQDN defined in the ingress rules adjusting the value
// if is required.
// https://trac.nginx.org/nginx/ticket/352
// https://trac.nginx.org/nginx/ticket/631
var longestName int
var serverNameBytes int
redirectServers := make(map[string]string)
for _, srv := range ingressCfg.Servers {
if longestName < len(srv.Hostname) {
longestName = len(srv.Hostname)
}
serverNameBytes += len(srv.Hostname)
if srv.RedirectFromToWWW {
var n string
if strings.HasPrefix(srv.Hostname, "www.") {
n = strings.TrimLeft(srv.Hostname, "www.")
} else {
n = fmt.Sprintf("www.%v", srv.Hostname)
}
glog.V(3).Infof("creating redirect from %v to %v", srv.Hostname, n)
if _, ok := redirectServers[n]; !ok {
found := false
for _, esrv := range ingressCfg.Servers {
if esrv.Hostname == n {
found = true
break
}
}
if !found {
redirectServers[n] = srv.Hostname
}
}
}
}
if cfg.ServerNameHashBucketSize == 0 {
nameHashBucketSize := nginxHashBucketSize(longestName)
glog.V(3).Infof("adjusting ServerNameHashBucketSize variable to %v", nameHashBucketSize)
cfg.ServerNameHashBucketSize = nameHashBucketSize
}
serverNameHashMaxSize := nextPowerOf2(serverNameBytes)
if cfg.ServerNameHashMaxSize < serverNameHashMaxSize {
glog.V(3).Infof("adjusting ServerNameHashMaxSize variable to %v", serverNameHashMaxSize)
cfg.ServerNameHashMaxSize = serverNameHashMaxSize
}
// the limit of open files is per worker process
// and we leave some room to avoid consuming all the FDs available
wp, err := strconv.Atoi(cfg.WorkerProcesses)
glog.V(3).Infof("number of worker processes: %v", wp)
if err != nil {
wp = 1
}
maxOpenFiles := (sysctlFSFileMax() / wp) - 1024
glog.V(3).Infof("maximum number of open file descriptors : %v", sysctlFSFileMax())
if maxOpenFiles < 1024 {
// this means the value of RLIMIT_NOFILE is too low.
maxOpenFiles = 1024
}
setHeaders := map[string]string{}
if cfg.ProxySetHeaders != "" {
cmap, exists, err := n.storeLister.ConfigMap.GetByKey(cfg.ProxySetHeaders)
if err != nil {
glog.Warningf("unexpected error reading configmap %v: %v", cfg.ProxySetHeaders, err)
}
if exists {
setHeaders = cmap.(*apiv1.ConfigMap).Data
}
}
addHeaders := map[string]string{}
if cfg.AddHeaders != "" {
cmap, exists, err := n.storeLister.ConfigMap.GetByKey(cfg.AddHeaders)
if err != nil {
glog.Warningf("unexpected error reading configmap %v: %v", cfg.AddHeaders, err)
}
if exists {
addHeaders = cmap.(*apiv1.ConfigMap).Data
}
}
sslDHParam := ""
if cfg.SSLDHParam != "" {
secretName := cfg.SSLDHParam
s, exists, err := n.storeLister.Secret.GetByKey(secretName)
if err != nil {
glog.Warningf("unexpected error reading secret %v: %v", secretName, err)
}
if exists {
secret := s.(*apiv1.Secret)
nsSecName := strings.Replace(secretName, "/", "-", -1)
dh, ok := secret.Data["dhparam.pem"]
if ok {
pemFileName, err := ssl.AddOrUpdateDHParam(nsSecName, dh)
if err != nil {
glog.Warningf("unexpected error adding or updating dhparam %v file: %v", nsSecName, err)
} else {
sslDHParam = pemFileName
}
}
}
}
cfg.SSLDHParam = sslDHParam
tc := ngx_config.TemplateConfig{
ProxySetHeaders: setHeaders,
AddHeaders: addHeaders,
MaxOpenFiles: maxOpenFiles,
BacklogSize: sysctlSomaxconn(),
Backends: ingressCfg.Backends,
PassthroughBackends: ingressCfg.PassthroughBackends,
Servers: ingressCfg.Servers,
TCPBackends: ingressCfg.TCPEndpoints,
UDPBackends: ingressCfg.UDPEndpoints,
HealthzURI: ngxHealthPath,
CustomErrors: len(cfg.CustomHTTPErrors) > 0,
Cfg: cfg,
IsIPV6Enabled: n.isIPV6Enabled && !cfg.DisableIpv6,
RedirectServers: redirectServers,
IsSSLPassthroughEnabled: n.isSSLPassthroughEnabled,
ListenPorts: n.cfg.ListenPorts,
PublishService: n.GetPublishService(),
}
content, err := n.t.Write(tc)
if err != nil {
return err
}
err = n.testTemplate(content)
if err != nil {
return err
}
if glog.V(2) {
src, _ := ioutil.ReadFile(cfgPath)
if !bytes.Equal(src, content) {
tmpfile, err := ioutil.TempFile("", "new-nginx-cfg")
if err != nil {
return err
}
defer tmpfile.Close()
err = ioutil.WriteFile(tmpfile.Name(), content, 0644)
if err != nil {
return err
}
diffOutput, err := exec.Command("diff", "-u", cfgPath, tmpfile.Name()).CombinedOutput()
if err != nil {
return err
}
glog.Infof("NGINX configuration diff\n")
glog.Infof("%v\n", string(diffOutput))
// Do not use defer to remove the temporal file.
// This is helpful when there is an error in the
// temporal configuration (we can manually inspect the file).
// Only remove the file when no error occured.
os.Remove(tmpfile.Name())
}
}
err = ioutil.WriteFile(cfgPath, content, 0644)
if err != nil {
return err
}
o, err := exec.Command(n.binary, "-s", "reload", "-c", cfgPath).CombinedOutput()
if err != nil {
return fmt.Errorf("%v\n%v", err, string(o))
}
return nil
}
// nginxHashBucketSize computes the correct nginx hash_bucket_size for a hash with the given longest key
func nginxHashBucketSize(longestString int) int {
// See https://github.com/kubernetes/ingress-nginxs/issues/623 for an explanation
wordSize := 8 // Assume 64 bit CPU
n := longestString + 2
aligned := (n + wordSize - 1) & ^(wordSize - 1)
rawSize := wordSize + wordSize + aligned
return nextPowerOf2(rawSize)
}
// http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
// https://play.golang.org/p/TVSyCcdxUh
func nextPowerOf2(v int) int {
v--
v |= v >> 1
v |= v >> 2
v |= v >> 4
v |= v >> 8
v |= v >> 16
v++
return v
}

View file

@ -0,0 +1,115 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controller
import "testing"
func TestNginxHashBucketSize(t *testing.T) {
tests := []struct {
n int
expected int
}{
{0, 32},
{1, 32},
{2, 32},
{3, 32},
// ...
{13, 32},
{14, 32},
{15, 64},
{16, 64},
// ...
{45, 64},
{46, 64},
{47, 128},
{48, 128},
// ...
// ...
{109, 128},
{110, 128},
{111, 256},
{112, 256},
// ...
{237, 256},
{238, 256},
{239, 512},
{240, 512},
}
for _, test := range tests {
actual := nginxHashBucketSize(test.n)
if actual != test.expected {
t.Errorf("Test nginxHashBucketSize(%d): expected %d but returned %d", test.n, test.expected, actual)
}
}
}
func TestNextPowerOf2(t *testing.T) {
// Powers of 2
actual := nextPowerOf2(2)
if actual != 2 {
t.Errorf("TestNextPowerOf2: expected %d but returned %d.", 2, actual)
}
actual = nextPowerOf2(4)
if actual != 4 {
t.Errorf("TestNextPowerOf2: expected %d but returned %d.", 4, actual)
}
actual = nextPowerOf2(32)
if actual != 32 {
t.Errorf("TestNextPowerOf2: expected %d but returned %d.", 32, actual)
}
actual = nextPowerOf2(256)
if actual != 256 {
t.Errorf("TestNextPowerOf2: expected %d but returned %d.", 256, actual)
}
// Not Powers of 2
actual = nextPowerOf2(7)
if actual != 8 {
t.Errorf("TestNextPowerOf2: expected %d but returned %d.", 8, actual)
}
actual = nextPowerOf2(9)
if actual != 16 {
t.Errorf("TestNextPowerOf2: expected %d but returned %d.", 16, actual)
}
actual = nextPowerOf2(15)
if actual != 16 {
t.Errorf("TestNextPowerOf2: expected %d but returned %d.", 16, actual)
}
actual = nextPowerOf2(17)
if actual != 32 {
t.Errorf("TestNextPowerOf2: expected %d but returned %d.", 32, actual)
}
actual = nextPowerOf2(250)
if actual != 256 {
t.Errorf("TestNextPowerOf2: expected %d but returned %d.", 256, actual)
}
// Other
actual = nextPowerOf2(0)
if actual != 0 {
t.Errorf("TestNextPowerOf2: expected %d but returned %d.", 0, actual)
}
actual = nextPowerOf2(-1)
if actual != 0 {
t.Errorf("TestNextPowerOf2: expected %d but returned %d.", 0, actual)
}
actual = nextPowerOf2(-2)
if actual != 0 {
t.Errorf("TestNextPowerOf2: expected %d but returned %d.", 0, actual)
}
}

View file

@ -0,0 +1,71 @@
package process
import (
"fmt"
"net"
"os"
"os/exec"
"syscall"
"time"
"github.com/golang/glog"
ps "github.com/mitchellh/go-ps"
"github.com/ncabatoff/process-exporter/proc"
)
func IsRespawnIfRequired(err error) bool {
exitError, ok := err.(*exec.ExitError)
if !ok {
return false
}
waitStatus := exitError.Sys().(syscall.WaitStatus)
glog.Warningf(`
-------------------------------------------------------------------------------
NGINX master process died (%v): %v
-------------------------------------------------------------------------------
`, waitStatus.ExitStatus(), err)
return true
}
func WaitUntilPortIsAvailable(port int) {
// we wait until the workers are killed
for {
conn, err := net.DialTimeout("tcp", fmt.Sprintf("0.0.0.0:%v", port), 1*time.Second)
if err != nil {
break
}
conn.Close()
// kill nginx worker processes
fs, err := proc.NewFS("/proc")
procs, _ := fs.FS.AllProcs()
for _, p := range procs {
pn, err := p.Comm()
if err != nil {
glog.Errorf("unexpected error obtaining process information: %v", err)
continue
}
if pn == "nginx" {
osp, err := os.FindProcess(p.PID)
if err != nil {
glog.Errorf("unexpected error obtaining process information: %v", err)
continue
}
osp.Signal(syscall.SIGQUIT)
}
}
time.Sleep(100 * time.Millisecond)
}
}
// IsNginxRunning returns true if a process with the name 'nginx' is found
func IsNginxRunning() bool {
processes, _ := ps.Processes()
for _, p := range processes {
if p.Executable() == "nginx" {
return true
}
}
return false
}

View file

@ -0,0 +1,97 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controller
import (
"github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/ingress-nginx/pkg/ingress/controller/metric/collector"
)
const (
ngxStatusPath = "/nginx_status"
ngxVtsPath = "/nginx_status/format/json"
)
func (n *NGINXController) setupMonitor(sm statusModule) {
csm := n.statusModule
if csm != sm {
glog.Infof("changing prometheus collector from %v to %v", csm, sm)
n.stats.stop(csm)
n.stats.start(sm)
n.statusModule = sm
}
}
type statsCollector struct {
process prometheus.Collector
basic collector.Stopable
vts collector.Stopable
namespace string
watchClass string
port int
}
func (s *statsCollector) stop(sm statusModule) {
switch sm {
case defaultStatusModule:
s.basic.Stop()
prometheus.Unregister(s.basic)
case vtsStatusModule:
s.vts.Stop()
prometheus.Unregister(s.vts)
}
}
func (s *statsCollector) start(sm statusModule) {
switch sm {
case defaultStatusModule:
s.basic = collector.NewNginxStatus(s.namespace, s.watchClass, s.port, ngxStatusPath)
prometheus.Register(s.basic)
break
case vtsStatusModule:
s.vts = collector.NewNGINXVTSCollector(s.namespace, s.watchClass, s.port, ngxVtsPath)
prometheus.Register(s.vts)
break
}
}
func newStatsCollector(ns, class, binary string, port int) *statsCollector {
glog.Infof("starting new nginx stats collector for Ingress controller running in namespace %v (class %v)", ns, class)
glog.Infof("collector extracting information from port %v", port)
pc, err := collector.NewNamedProcess(true, collector.BinaryNameMatcher{
Name: "nginx",
Binary: binary,
})
if err != nil {
glog.Fatalf("unexpected error registering nginx collector: %v", err)
}
err = prometheus.Register(pc)
if err != nil {
glog.Fatalf("unexpected error registering nginx collector: %v", err)
}
return &statsCollector{
namespace: ns,
watchClass: class,
process: pc,
port: port,
}
}

View file

@ -0,0 +1,110 @@
package controller
import (
"fmt"
"io"
"net"
"github.com/golang/glog"
"github.com/paultag/sniff/parser"
)
type TCPServer struct {
Hostname string
IP string
Port int
ProxyProtocol bool
}
type TCPProxy struct {
ServerList []*TCPServer
Default *TCPServer
}
func (p *TCPProxy) Get(host string) *TCPServer {
if p.ServerList == nil {
return p.Default
}
for _, s := range p.ServerList {
if s.Hostname == host {
return s
}
}
return p.Default
}
func (p *TCPProxy) Handle(conn net.Conn) {
defer conn.Close()
data := make([]byte, 4096)
length, err := conn.Read(data)
if err != nil {
glog.V(4).Infof("error reading the first 4k of the connection: %s", err)
return
}
proxy := p.Default
hostname, err := parser.GetHostname(data[:])
if err == nil {
glog.V(4).Infof("parsed hostname from TLS Client Hello: %s", hostname)
proxy = p.Get(hostname)
}
if proxy == nil {
glog.V(4).Infof("there is no configured proxy for SSL connections")
return
}
clientConn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", proxy.IP, proxy.Port))
if err != nil {
return
}
defer clientConn.Close()
if proxy.ProxyProtocol {
//Write out the proxy-protocol header
localAddr := conn.LocalAddr().(*net.TCPAddr)
remoteAddr := conn.RemoteAddr().(*net.TCPAddr)
protocol := "UNKNOWN"
if remoteAddr.IP.To4() != nil {
protocol = "TCP4"
} else if remoteAddr.IP.To16() != nil {
protocol = "TCP6"
}
proxyProtocolHeader := fmt.Sprintf("PROXY %s %s %s %d %d\r\n", protocol, remoteAddr.IP.String(), localAddr.IP.String(), remoteAddr.Port, localAddr.Port)
glog.V(4).Infof("Writing proxy protocol header - %s", proxyProtocolHeader)
_, err = fmt.Fprintf(clientConn, proxyProtocolHeader)
}
if err != nil {
glog.Errorf("unexpected error writing proxy-protocol header: %s", err)
clientConn.Close()
} else {
_, err = clientConn.Write(data[:length])
if err != nil {
glog.Errorf("unexpected error writing first 4k of proxy data: %s", err)
clientConn.Close()
}
}
pipe(clientConn, conn)
}
func pipe(client, server net.Conn) {
doCopy := func(s, c net.Conn, cancel chan<- bool) {
io.Copy(s, c)
cancel <- true
}
cancel := make(chan bool, 2)
go doCopy(server, client, cancel)
go doCopy(client, server, cancel)
select {
case <-cancel:
return
}
}

View file

@ -0,0 +1,135 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package template
import (
"fmt"
"net"
"strconv"
"strings"
"github.com/golang/glog"
"github.com/mitchellh/mapstructure"
"k8s.io/ingress-nginx/pkg/ingress/controller/config"
ing_net "k8s.io/ingress-nginx/pkg/net"
)
const (
customHTTPErrors = "custom-http-errors"
skipAccessLogUrls = "skip-access-log-urls"
whitelistSourceRange = "whitelist-source-range"
proxyRealIPCIDR = "proxy-real-ip-cidr"
bindAddress = "bind-address"
)
// ReadConfig obtains the configuration defined by the user merged with the defaults.
func ReadConfig(src map[string]string) config.Configuration {
conf := map[string]string{}
// we need to copy the configmap data because the content is altered
for k, v := range src {
conf[k] = v
}
errors := make([]int, 0)
skipUrls := make([]string, 0)
whitelist := make([]string, 0)
proxylist := make([]string, 0)
bindAddressIpv4List := make([]string, 0)
bindAddressIpv6List := make([]string, 0)
if val, ok := conf[customHTTPErrors]; ok {
delete(conf, customHTTPErrors)
for _, i := range strings.Split(val, ",") {
j, err := strconv.Atoi(i)
if err != nil {
glog.Warningf("%v is not a valid http code: %v", i, err)
} else {
errors = append(errors, j)
}
}
}
if val, ok := conf[skipAccessLogUrls]; ok {
delete(conf, skipAccessLogUrls)
skipUrls = strings.Split(val, ",")
}
if val, ok := conf[whitelistSourceRange]; ok {
delete(conf, whitelistSourceRange)
whitelist = append(whitelist, strings.Split(val, ",")...)
}
if val, ok := conf[proxyRealIPCIDR]; ok {
delete(conf, proxyRealIPCIDR)
proxylist = append(proxylist, strings.Split(val, ",")...)
} else {
proxylist = append(proxylist, "0.0.0.0/0")
}
if val, ok := conf[bindAddress]; ok {
delete(conf, bindAddress)
for _, i := range strings.Split(val, ",") {
ns := net.ParseIP(i)
if ns != nil {
if ing_net.IsIPV6(ns) {
bindAddressIpv6List = append(bindAddressIpv6List, fmt.Sprintf("[%v]", ns))
} else {
bindAddressIpv4List = append(bindAddressIpv4List, fmt.Sprintf("%v", ns))
}
} else {
glog.Warningf("%v is not a valid textual representation of an IP address", i)
}
}
}
to := config.NewDefault()
to.CustomHTTPErrors = filterErrors(errors)
to.SkipAccessLogURLs = skipUrls
to.WhitelistSourceRange = whitelist
to.ProxyRealIPCIDR = proxylist
to.BindAddressIpv4 = bindAddressIpv4List
to.BindAddressIpv6 = bindAddressIpv6List
config := &mapstructure.DecoderConfig{
Metadata: nil,
WeaklyTypedInput: true,
Result: &to,
TagName: "json",
}
decoder, err := mapstructure.NewDecoder(config)
if err != nil {
glog.Warningf("unexpected error merging defaults: %v", err)
}
err = decoder.Decode(conf)
if err != nil {
glog.Warningf("unexpected error merging defaults: %v", err)
}
return to
}
func filterErrors(codes []int) []int {
var fa []int
for _, code := range codes {
if code > 299 && code < 600 {
fa = append(fa, code)
} else {
glog.Warningf("error code %v is not valid for custom error pages", code)
}
}
return fa
}

View file

@ -0,0 +1,95 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package template
import (
"testing"
"github.com/kylelemons/godebug/pretty"
"k8s.io/ingress-nginx/pkg/ingress/controller/config"
)
func TestFilterErrors(t *testing.T) {
e := filterErrors([]int{200, 300, 345, 500, 555, 999})
if len(e) != 4 {
t.Errorf("expected 4 elements but %v returned", len(e))
}
}
func TestMergeConfigMapToStruct(t *testing.T) {
conf := map[string]string{
"custom-http-errors": "300,400,demo",
"proxy-read-timeout": "1",
"proxy-send-timeout": "2",
"skip-access-log-urls": "/log,/demo,/test",
"use-proxy-protocol": "true",
"disable-access-log": "true",
"access-log-path": "/var/log/test/access.log",
"error-log-path": "/var/log/test/error.log",
"use-gzip": "true",
"enable-dynamic-tls-records": "false",
"gzip-types": "text/html",
"proxy-real-ip-cidr": "1.1.1.1/8,2.2.2.2/24",
"bind-address": "1.1.1.1,2.2.2.2,3.3.3,2001:db8:a0b:12f0::1,3731:54:65fe:2::a7,33:33:33::33::33",
"worker-shutdown-timeout": "99s",
}
def := config.NewDefault()
def.CustomHTTPErrors = []int{300, 400}
def.DisableAccessLog = true
def.AccessLogPath = "/var/log/test/access.log"
def.ErrorLogPath = "/var/log/test/error.log"
def.SkipAccessLogURLs = []string{"/log", "/demo", "/test"}
def.ProxyReadTimeout = 1
def.ProxySendTimeout = 2
def.EnableDynamicTLSRecords = false
def.UseProxyProtocol = true
def.GzipTypes = "text/html"
def.ProxyRealIPCIDR = []string{"1.1.1.1/8", "2.2.2.2/24"}
def.BindAddressIpv4 = []string{"1.1.1.1", "2.2.2.2"}
def.BindAddressIpv6 = []string{"[2001:db8:a0b:12f0::1]", "[3731:54:65fe:2::a7]"}
def.WorkerShutdownTimeout = "99s"
to := ReadConfig(conf)
if diff := pretty.Compare(to, def); diff != "" {
t.Errorf("unexpected diff: (-got +want)\n%s", diff)
}
def = config.NewDefault()
to = ReadConfig(map[string]string{})
if diff := pretty.Compare(to, def); diff != "" {
t.Errorf("unexpected diff: (-got +want)\n%s", diff)
}
def = config.NewDefault()
def.WhitelistSourceRange = []string{"1.1.1.1/32"}
to = ReadConfig(map[string]string{
"whitelist-source-range": "1.1.1.1/32",
})
if diff := pretty.Compare(to, def); diff != "" {
t.Errorf("unexpected diff: (-got +want)\n%s", diff)
}
}
func TestDefaultLoadBalance(t *testing.T) {
conf := map[string]string{}
to := ReadConfig(conf)
if to.LoadBalanceAlgorithm != "least_conn" {
t.Errorf("default load balance algorithm wrong")
}
}

View file

@ -0,0 +1,700 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package template
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"net"
"net/url"
"os"
"os/exec"
"strconv"
"strings"
text_template "text/template"
"github.com/golang/glog"
"github.com/pborman/uuid"
extensions "k8s.io/api/extensions/v1beta1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/ingress-nginx/pkg/ingress"
"k8s.io/ingress-nginx/pkg/ingress/annotations/ratelimit"
"k8s.io/ingress-nginx/pkg/ingress/controller/config"
ing_net "k8s.io/ingress-nginx/pkg/net"
"k8s.io/ingress-nginx/pkg/watch"
)
const (
slash = "/"
nonIdempotent = "non_idempotent"
defBufferSize = 65535
)
// Template ...
type Template struct {
tmpl *text_template.Template
fw watch.FileWatcher
s int
}
//NewTemplate returns a new Template instance or an
//error if the specified template file contains errors
func NewTemplate(file string, onChange func()) (*Template, error) {
tmpl, err := text_template.New("nginx.tmpl").Funcs(funcMap).ParseFiles(file)
if err != nil {
return nil, err
}
fw, err := watch.NewFileWatcher(file, onChange)
if err != nil {
return nil, err
}
return &Template{
tmpl: tmpl,
fw: fw,
s: defBufferSize,
}, nil
}
// Close removes the file watcher
func (t *Template) Close() {
t.fw.Close()
}
// Write populates a buffer using a template with NGINX configuration
// and the servers and upstreams created by Ingress rules
func (t *Template) Write(conf config.TemplateConfig) ([]byte, error) {
tmplBuf := bytes.NewBuffer(make([]byte, 0, t.s))
outCmdBuf := bytes.NewBuffer(make([]byte, 0, t.s))
defer func() {
if t.s < tmplBuf.Cap() {
glog.V(2).Infof("adjusting template buffer size from %v to %v", t.s, tmplBuf.Cap())
t.s = tmplBuf.Cap()
}
}()
if glog.V(3) {
b, err := json.Marshal(conf)
if err != nil {
glog.Errorf("unexpected error: %v", err)
}
glog.Infof("NGINX configuration: %v", string(b))
}
err := t.tmpl.Execute(tmplBuf, conf)
if err != nil {
return nil, err
}
// squeezes multiple adjacent empty lines to be single
// spaced this is to avoid the use of regular expressions
cmd := exec.Command("/ingress-controller/clean-nginx-conf.sh")
cmd.Stdin = tmplBuf
cmd.Stdout = outCmdBuf
if err := cmd.Run(); err != nil {
glog.Warningf("unexpected error cleaning template: %v", err)
return tmplBuf.Bytes(), nil
}
return outCmdBuf.Bytes(), nil
}
var (
funcMap = text_template.FuncMap{
"empty": func(input interface{}) bool {
check, ok := input.(string)
if ok {
return len(check) == 0
}
return true
},
"buildLocation": buildLocation,
"buildAuthLocation": buildAuthLocation,
"buildAuthResponseHeaders": buildAuthResponseHeaders,
"buildProxyPass": buildProxyPass,
"filterRateLimits": filterRateLimits,
"buildRateLimitZones": buildRateLimitZones,
"buildRateLimit": buildRateLimit,
"buildResolvers": buildResolvers,
"buildUpstreamName": buildUpstreamName,
"isLocationAllowed": isLocationAllowed,
"buildLogFormatUpstream": buildLogFormatUpstream,
"buildDenyVariable": buildDenyVariable,
"getenv": os.Getenv,
"contains": strings.Contains,
"hasPrefix": strings.HasPrefix,
"hasSuffix": strings.HasSuffix,
"toUpper": strings.ToUpper,
"toLower": strings.ToLower,
"formatIP": formatIP,
"buildNextUpstream": buildNextUpstream,
"getIngressInformation": getIngressInformation,
"serverConfig": func(all config.TemplateConfig, server *ingress.Server) interface{} {
return struct{ First, Second interface{} }{all, server}
},
"isValidClientBodyBufferSize": isValidClientBodyBufferSize,
"buildForwardedFor": buildForwardedFor,
"buildAuthSignURL": buildAuthSignURL,
}
)
// formatIP will wrap IPv6 addresses in [] and return IPv4 addresses
// without modification. If the input cannot be parsed as an IP address
// it is returned without modification.
func formatIP(input string) string {
ip := net.ParseIP(input)
if ip == nil {
return input
}
if v4 := ip.To4(); v4 != nil {
return input
}
return fmt.Sprintf("[%s]", input)
}
// buildResolvers returns the resolvers reading the /etc/resolv.conf file
func buildResolvers(input interface{}) string {
// NGINX need IPV6 addresses to be surrounded by brackets
nss, ok := input.([]net.IP)
if !ok {
glog.Errorf("expected a '[]net.IP' type but %T was returned", input)
return ""
}
if len(nss) == 0 {
return ""
}
r := []string{"resolver"}
for _, ns := range nss {
if ing_net.IsIPV6(ns) {
r = append(r, fmt.Sprintf("[%v]", ns))
} else {
r = append(r, fmt.Sprintf("%v", ns))
}
}
r = append(r, "valid=30s;")
return strings.Join(r, " ")
}
// buildLocation produces the location string, if the ingress has redirects
// (specified through the ingress.kubernetes.io/rewrite-to annotation)
func buildLocation(input interface{}) string {
location, ok := input.(*ingress.Location)
if !ok {
glog.Errorf("expected an '*ingress.Location' type but %T was returned", input)
return slash
}
path := location.Path
if len(location.Rewrite.Target) > 0 && location.Rewrite.Target != path {
if path == slash {
return fmt.Sprintf("~* %s", path)
}
// baseuri regex will parse basename from the given location
baseuri := `(?<baseuri>.*)`
if !strings.HasSuffix(path, slash) {
// Not treat the slash after "location path" as a part of baseuri
baseuri = fmt.Sprintf(`\/?%s`, baseuri)
}
return fmt.Sprintf(`~* ^%s%s`, path, baseuri)
}
return path
}
// TODO: Needs Unit Tests
func buildAuthLocation(input interface{}) string {
location, ok := input.(*ingress.Location)
if !ok {
glog.Errorf("expected an '*ingress.Location' type but %T was returned", input)
return ""
}
if location.ExternalAuth.URL == "" {
return ""
}
str := base64.URLEncoding.EncodeToString([]byte(location.Path))
// avoid locations containing the = char
str = strings.Replace(str, "=", "", -1)
return fmt.Sprintf("/_external-auth-%v", str)
}
func buildAuthResponseHeaders(input interface{}) []string {
location, ok := input.(*ingress.Location)
res := []string{}
if !ok {
glog.Errorf("expected an '*ingress.Location' type but %T was returned", input)
return res
}
if len(location.ExternalAuth.ResponseHeaders) == 0 {
return res
}
for i, h := range location.ExternalAuth.ResponseHeaders {
hvar := strings.ToLower(h)
hvar = strings.NewReplacer("-", "_").Replace(hvar)
res = append(res, fmt.Sprintf("auth_request_set $authHeader%v $upstream_http_%v;", i, hvar))
res = append(res, fmt.Sprintf("proxy_set_header '%v' $authHeader%v;", h, i))
}
return res
}
func buildLogFormatUpstream(input interface{}) string {
cfg, ok := input.(config.Configuration)
if !ok {
glog.Errorf("expected a 'config.Configuration' type but %T was returned", input)
return ""
}
return cfg.BuildLogFormatUpstream()
}
// buildProxyPass produces the proxy pass string, if the ingress has redirects
// (specified through the ingress.kubernetes.io/rewrite-to annotation)
// If the annotation ingress.kubernetes.io/add-base-url:"true" is specified it will
// add a base tag in the head of the response from the service
func buildProxyPass(host string, b interface{}, loc interface{}) string {
backends, ok := b.([]*ingress.Backend)
if !ok {
glog.Errorf("expected an '[]*ingress.Backend' type but %T was returned", b)
return ""
}
location, ok := loc.(*ingress.Location)
if !ok {
glog.Errorf("expected a '*ingress.Location' type but %T was returned", loc)
return ""
}
path := location.Path
proto := "http"
upstreamName := location.Backend
for _, backend := range backends {
if backend.Name == location.Backend {
if backend.Secure || backend.SSLPassthrough {
proto = "https"
}
if isSticky(host, location, backend.SessionAffinity.CookieSessionAffinity.Locations) {
upstreamName = fmt.Sprintf("sticky-%v", upstreamName)
}
break
}
}
// defProxyPass returns the default proxy_pass, just the name of the upstream
defProxyPass := fmt.Sprintf("proxy_pass %s://%s;", proto, upstreamName)
// if the path in the ingress rule is equals to the target: no special rewrite
if path == location.Rewrite.Target {
return defProxyPass
}
if !strings.HasSuffix(path, slash) {
path = fmt.Sprintf("%s/", path)
}
if len(location.Rewrite.Target) > 0 {
abu := ""
if location.Rewrite.AddBaseURL {
// path has a slash suffix, so that it can be connected with baseuri directly
bPath := fmt.Sprintf("%s%s", path, "$baseuri")
regex := `(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)`
if len(location.Rewrite.BaseURLScheme) > 0 {
abu = fmt.Sprintf(`subs_filter '%v' '$1<base href="%v://$http_host%v">' ro;
`, regex, location.Rewrite.BaseURLScheme, bPath)
} else {
abu = fmt.Sprintf(`subs_filter '%v' '$1<base href="$scheme://$http_host%v">' ro;
`, regex, bPath)
}
}
if location.Rewrite.Target == slash {
// special case redirect to /
// ie /something to /
return fmt.Sprintf(`
rewrite %s(.*) /$1 break;
rewrite %s / break;
proxy_pass %s://%s;
%v`, path, location.Path, proto, upstreamName, abu)
}
return fmt.Sprintf(`
rewrite %s(.*) %s/$1 break;
proxy_pass %s://%s;
%v`, path, location.Rewrite.Target, proto, upstreamName, abu)
}
// default proxy_pass
return defProxyPass
}
// TODO: Needs Unit Tests
func filterRateLimits(input interface{}) []ratelimit.RateLimit {
ratelimits := []ratelimit.RateLimit{}
found := sets.String{}
servers, ok := input.([]*ingress.Server)
if !ok {
glog.Errorf("expected a '[]ratelimit.RateLimit' type but %T was returned", input)
return ratelimits
}
for _, server := range servers {
for _, loc := range server.Locations {
if loc.RateLimit.ID != "" && !found.Has(loc.RateLimit.ID) {
found.Insert(loc.RateLimit.ID)
ratelimits = append(ratelimits, loc.RateLimit)
}
}
}
return ratelimits
}
// TODO: Needs Unit Tests
// buildRateLimitZones produces an array of limit_conn_zone in order to allow
// rate limiting of request. Each Ingress rule could have up to three zones, one
// for connection limit by IP address, one for limiting requests per minute, and
// one for limiting requests per second.
func buildRateLimitZones(input interface{}) []string {
zones := sets.String{}
servers, ok := input.([]*ingress.Server)
if !ok {
glog.Errorf("expected a '[]*ingress.Server' type but %T was returned", input)
return zones.List()
}
for _, server := range servers {
for _, loc := range server.Locations {
if loc.RateLimit.Connections.Limit > 0 {
zone := fmt.Sprintf("limit_conn_zone $limit_%s zone=%v:%vm;",
loc.RateLimit.ID,
loc.RateLimit.Connections.Name,
loc.RateLimit.Connections.SharedSize)
if !zones.Has(zone) {
zones.Insert(zone)
}
}
if loc.RateLimit.RPM.Limit > 0 {
zone := fmt.Sprintf("limit_req_zone $limit_%s zone=%v:%vm rate=%vr/m;",
loc.RateLimit.ID,
loc.RateLimit.RPM.Name,
loc.RateLimit.RPM.SharedSize,
loc.RateLimit.RPM.Limit)
if !zones.Has(zone) {
zones.Insert(zone)
}
}
if loc.RateLimit.RPS.Limit > 0 {
zone := fmt.Sprintf("limit_req_zone $limit_%s zone=%v:%vm rate=%vr/s;",
loc.RateLimit.ID,
loc.RateLimit.RPS.Name,
loc.RateLimit.RPS.SharedSize,
loc.RateLimit.RPS.Limit)
if !zones.Has(zone) {
zones.Insert(zone)
}
}
}
}
return zones.List()
}
// buildRateLimit produces an array of limit_req to be used inside the Path of
// Ingress rules. The order: connections by IP first, then RPS, and RPM last.
func buildRateLimit(input interface{}) []string {
limits := []string{}
loc, ok := input.(*ingress.Location)
if !ok {
glog.Errorf("expected an '*ingress.Location' type but %T was returned", input)
return limits
}
if loc.RateLimit.Connections.Limit > 0 {
limit := fmt.Sprintf("limit_conn %v %v;",
loc.RateLimit.Connections.Name, loc.RateLimit.Connections.Limit)
limits = append(limits, limit)
}
if loc.RateLimit.RPS.Limit > 0 {
limit := fmt.Sprintf("limit_req zone=%v burst=%v nodelay;",
loc.RateLimit.RPS.Name, loc.RateLimit.RPS.Burst)
limits = append(limits, limit)
}
if loc.RateLimit.RPM.Limit > 0 {
limit := fmt.Sprintf("limit_req zone=%v burst=%v nodelay;",
loc.RateLimit.RPM.Name, loc.RateLimit.RPM.Burst)
limits = append(limits, limit)
}
if loc.RateLimit.LimitRateAfter > 0 {
limit := fmt.Sprintf("limit_rate_after %vk;",
loc.RateLimit.LimitRateAfter)
limits = append(limits, limit)
}
if loc.RateLimit.LimitRate > 0 {
limit := fmt.Sprintf("limit_rate %vk;",
loc.RateLimit.LimitRate)
limits = append(limits, limit)
}
return limits
}
func isLocationAllowed(input interface{}) bool {
loc, ok := input.(*ingress.Location)
if !ok {
glog.Errorf("expected an '*ingress.Location' type but %T was returned", input)
return false
}
return loc.Denied == nil
}
var (
denyPathSlugMap = map[string]string{}
)
// buildDenyVariable returns a nginx variable for a location in a
// server to be used in the whitelist check
// This method uses a unique id generator library to reduce the
// size of the string to be used as a variable in nginx to avoid
// issue with the size of the variable bucket size directive
func buildDenyVariable(a interface{}) string {
l, ok := a.(string)
if !ok {
glog.Errorf("expected a 'string' type but %T was returned", a)
return ""
}
if _, ok := denyPathSlugMap[l]; !ok {
denyPathSlugMap[l] = buildRandomUUID()
}
return fmt.Sprintf("$deny_%v", denyPathSlugMap[l])
}
// TODO: Needs Unit Tests
func buildUpstreamName(host string, b interface{}, loc interface{}) string {
backends, ok := b.([]*ingress.Backend)
if !ok {
glog.Errorf("expected an '[]*ingress.Backend' type but %T was returned", b)
return ""
}
location, ok := loc.(*ingress.Location)
if !ok {
glog.Errorf("expected a '*ingress.Location' type but %T was returned", loc)
return ""
}
upstreamName := location.Backend
for _, backend := range backends {
if backend.Name == location.Backend {
if backend.SessionAffinity.AffinityType == "cookie" &&
isSticky(host, location, backend.SessionAffinity.CookieSessionAffinity.Locations) {
upstreamName = fmt.Sprintf("sticky-%v", upstreamName)
}
break
}
}
return upstreamName
}
// TODO: Needs Unit Tests
func isSticky(host string, loc *ingress.Location, stickyLocations map[string][]string) bool {
if _, ok := stickyLocations[host]; ok {
for _, sl := range stickyLocations[host] {
if sl == loc.Path {
return true
}
}
}
return false
}
func buildNextUpstream(i, r interface{}) string {
nextUpstream, ok := i.(string)
if !ok {
glog.Errorf("expected a 'string' type but %T was returned", i)
return ""
}
retryNonIdempotent := r.(bool)
parts := strings.Split(nextUpstream, " ")
nextUpstreamCodes := make([]string, 0, len(parts))
for _, v := range parts {
if v != "" && v != nonIdempotent {
nextUpstreamCodes = append(nextUpstreamCodes, v)
}
if v == nonIdempotent {
retryNonIdempotent = true
}
}
if retryNonIdempotent {
nextUpstreamCodes = append(nextUpstreamCodes, nonIdempotent)
}
return strings.Join(nextUpstreamCodes, " ")
}
// buildRandomUUID return a random string to be used in the template
func buildRandomUUID() string {
s := uuid.New()
return strings.Replace(s, "-", "", -1)
}
func isValidClientBodyBufferSize(input interface{}) bool {
s, ok := input.(string)
if !ok {
glog.Errorf("expected an 'string' type but %T was returned", input)
return false
}
if s == "" {
return false
}
_, err := strconv.Atoi(s)
if err != nil {
sLowercase := strings.ToLower(s)
kCheck := strings.TrimSuffix(sLowercase, "k")
_, err := strconv.Atoi(kCheck)
if err == nil {
return true
}
mCheck := strings.TrimSuffix(sLowercase, "m")
_, err = strconv.Atoi(mCheck)
if err == nil {
return true
}
glog.Errorf("client-body-buffer-size '%v' was provided in an incorrect format, hence it will not be set.", s)
return false
}
return true
}
type ingressInformation struct {
Namespace string
Rule string
Service string
Annotations map[string]string
}
func getIngressInformation(i, p interface{}) *ingressInformation {
ing, ok := i.(*extensions.Ingress)
if !ok {
glog.Errorf("expected an '*extensions.Ingress' type but %T was returned", i)
return &ingressInformation{}
}
path, ok := p.(string)
if !ok {
glog.Errorf("expected a 'string' type but %T was returned", p)
return &ingressInformation{}
}
if ing == nil {
return &ingressInformation{}
}
info := &ingressInformation{
Namespace: ing.GetNamespace(),
Rule: ing.GetName(),
Annotations: ing.Annotations,
}
if ing.Spec.Backend != nil {
info.Service = ing.Spec.Backend.ServiceName
}
for _, rule := range ing.Spec.Rules {
if rule.HTTP == nil {
continue
}
for _, rPath := range rule.HTTP.Paths {
if path == rPath.Path {
info.Service = rPath.Backend.ServiceName
return info
}
}
}
return info
}
func buildForwardedFor(input interface{}) string {
s, ok := input.(string)
if !ok {
glog.Errorf("expected a 'string' type but %T was returned", input)
return ""
}
ffh := strings.Replace(s, "-", "_", -1)
ffh = strings.ToLower(ffh)
return fmt.Sprintf("$http_%v", ffh)
}
func buildAuthSignURL(input interface{}) string {
s, ok := input.(string)
if !ok {
glog.Errorf("expected an 'string' type but %T was returned", input)
return ""
}
u, _ := url.Parse(s)
q := u.Query()
if len(q) == 0 {
return fmt.Sprintf("%v?rd=$pass_access_scheme://$http_host$request_uri", s)
}
if q.Get("rd") != "" {
return s
}
return fmt.Sprintf("%v&rd=$pass_access_scheme://$http_host$request_uri", s)
}

View file

@ -0,0 +1,394 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package template
import (
"encoding/json"
"io/ioutil"
"net"
"os"
"path"
"reflect"
"strings"
"testing"
"k8s.io/ingress-nginx/pkg/ingress"
"k8s.io/ingress-nginx/pkg/ingress/annotations/authreq"
"k8s.io/ingress-nginx/pkg/ingress/annotations/rewrite"
"k8s.io/ingress-nginx/pkg/ingress/controller/config"
)
var (
// TODO: add tests for secure endpoints
tmplFuncTestcases = map[string]struct {
Path string
Target string
Location string
ProxyPass string
AddBaseURL bool
BaseURLScheme string
}{
"invalid redirect / to /": {"/", "/", "/", "proxy_pass http://upstream-name;", false, ""},
"redirect / to /jenkins": {"/", "/jenkins", "~* /",
`
rewrite /(.*) /jenkins/$1 break;
proxy_pass http://upstream-name;
`, false, ""},
"redirect /something to /": {"/something", "/", `~* ^/something\/?(?<baseuri>.*)`, `
rewrite /something/(.*) /$1 break;
rewrite /something / break;
proxy_pass http://upstream-name;
`, false, ""},
"redirect /end-with-slash/ to /not-root": {"/end-with-slash/", "/not-root", "~* ^/end-with-slash/(?<baseuri>.*)", `
rewrite /end-with-slash/(.*) /not-root/$1 break;
proxy_pass http://upstream-name;
`, false, ""},
"redirect /something-complex to /not-root": {"/something-complex", "/not-root", `~* ^/something-complex\/?(?<baseuri>.*)`, `
rewrite /something-complex/(.*) /not-root/$1 break;
proxy_pass http://upstream-name;
`, false, ""},
"redirect / to /jenkins and rewrite": {"/", "/jenkins", "~* /", `
rewrite /(.*) /jenkins/$1 break;
proxy_pass http://upstream-name;
subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="$scheme://$http_host/$baseuri">' ro;
`, true, ""},
"redirect /something to / and rewrite": {"/something", "/", `~* ^/something\/?(?<baseuri>.*)`, `
rewrite /something/(.*) /$1 break;
rewrite /something / break;
proxy_pass http://upstream-name;
subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="$scheme://$http_host/something/$baseuri">' ro;
`, true, ""},
"redirect /end-with-slash/ to /not-root and rewrite": {"/end-with-slash/", "/not-root", `~* ^/end-with-slash/(?<baseuri>.*)`, `
rewrite /end-with-slash/(.*) /not-root/$1 break;
proxy_pass http://upstream-name;
subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="$scheme://$http_host/end-with-slash/$baseuri">' ro;
`, true, ""},
"redirect /something-complex to /not-root and rewrite": {"/something-complex", "/not-root", `~* ^/something-complex\/?(?<baseuri>.*)`, `
rewrite /something-complex/(.*) /not-root/$1 break;
proxy_pass http://upstream-name;
subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="$scheme://$http_host/something-complex/$baseuri">' ro;
`, true, ""},
"redirect /something to / and rewrite with specific scheme": {"/something", "/", `~* ^/something\/?(?<baseuri>.*)`, `
rewrite /something/(.*) /$1 break;
rewrite /something / break;
proxy_pass http://upstream-name;
subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="http://$http_host/something/$baseuri">' ro;
`, true, "http"},
}
)
func TestFormatIP(t *testing.T) {
cases := map[string]struct {
Input, Output string
}{
"ipv4-localhost": {"127.0.0.1", "127.0.0.1"},
"ipv4-internet": {"8.8.8.8", "8.8.8.8"},
"ipv6-localhost": {"::1", "[::1]"},
"ipv6-internet": {"2001:4860:4860::8888", "[2001:4860:4860::8888]"},
"invalid-ip": {"nonsense", "nonsense"},
"empty-ip": {"", ""},
}
for k, tc := range cases {
res := formatIP(tc.Input)
if res != tc.Output {
t.Errorf("%s: called formatIp('%s'); expected '%v' but returned '%v'", k, tc.Input, tc.Output, res)
}
}
}
func TestBuildLocation(t *testing.T) {
for k, tc := range tmplFuncTestcases {
loc := &ingress.Location{
Path: tc.Path,
Rewrite: rewrite.Redirect{Target: tc.Target, AddBaseURL: tc.AddBaseURL},
}
newLoc := buildLocation(loc)
if tc.Location != newLoc {
t.Errorf("%s: expected '%v' but returned %v", k, tc.Location, newLoc)
}
}
}
func TestBuildProxyPass(t *testing.T) {
for k, tc := range tmplFuncTestcases {
loc := &ingress.Location{
Path: tc.Path,
Rewrite: rewrite.Redirect{Target: tc.Target, AddBaseURL: tc.AddBaseURL, BaseURLScheme: tc.BaseURLScheme},
Backend: "upstream-name",
}
pp := buildProxyPass("", []*ingress.Backend{}, loc)
if !strings.EqualFold(tc.ProxyPass, pp) {
t.Errorf("%s: expected \n'%v'\nbut returned \n'%v'", k, tc.ProxyPass, pp)
}
}
}
func TestBuildAuthResponseHeaders(t *testing.T) {
loc := &ingress.Location{
ExternalAuth: authreq.External{ResponseHeaders: []string{"h1", "H-With-Caps-And-Dashes"}},
}
headers := buildAuthResponseHeaders(loc)
expected := []string{
"auth_request_set $authHeader0 $upstream_http_h1;",
"proxy_set_header 'h1' $authHeader0;",
"auth_request_set $authHeader1 $upstream_http_h_with_caps_and_dashes;",
"proxy_set_header 'H-With-Caps-And-Dashes' $authHeader1;",
}
if !reflect.DeepEqual(expected, headers) {
t.Errorf("Expected \n'%v'\nbut returned \n'%v'", expected, headers)
}
}
func TestTemplateWithData(t *testing.T) {
pwd, _ := os.Getwd()
f, err := os.Open(path.Join(pwd, "../../../../test/data/config.json"))
if err != nil {
t.Errorf("unexpected error reading json file: %v", err)
}
defer f.Close()
data, err := ioutil.ReadFile(f.Name())
if err != nil {
t.Error("unexpected error reading json file: ", err)
}
var dat config.TemplateConfig
if err := json.Unmarshal(data, &dat); err != nil {
t.Errorf("unexpected error unmarshalling json: %v", err)
}
if dat.ListenPorts == nil {
dat.ListenPorts = &config.ListenPorts{}
}
tf, err := os.Open(path.Join(pwd, "../../../../rootfs/etc/nginx/template/nginx.tmpl"))
if err != nil {
t.Errorf("unexpected error reading json file: %v", err)
}
defer tf.Close()
ngxTpl, err := NewTemplate(tf.Name(), func() {})
if err != nil {
t.Errorf("invalid NGINX template: %v", err)
}
_, err = ngxTpl.Write(dat)
if err != nil {
t.Errorf("invalid NGINX template: %v", err)
}
}
func BenchmarkTemplateWithData(b *testing.B) {
pwd, _ := os.Getwd()
f, err := os.Open(path.Join(pwd, "../../../../test/data/config.json"))
if err != nil {
b.Errorf("unexpected error reading json file: %v", err)
}
defer f.Close()
data, err := ioutil.ReadFile(f.Name())
if err != nil {
b.Error("unexpected error reading json file: ", err)
}
var dat config.TemplateConfig
if err := json.Unmarshal(data, &dat); err != nil {
b.Errorf("unexpected error unmarshalling json: %v", err)
}
tf, err := os.Open(path.Join(pwd, "../../../rootfs/etc/nginx/template/nginx.tmpl"))
if err != nil {
b.Errorf("unexpected error reading json file: %v", err)
}
defer tf.Close()
ngxTpl, err := NewTemplate(tf.Name(), func() {})
if err != nil {
b.Errorf("invalid NGINX template: %v", err)
}
for i := 0; i < b.N; i++ {
ngxTpl.Write(dat)
}
}
func TestBuildDenyVariable(t *testing.T) {
a := buildDenyVariable("host1.example.com_/.well-known/acme-challenge")
b := buildDenyVariable("host1.example.com_/.well-known/acme-challenge")
if !reflect.DeepEqual(a, b) {
t.Errorf("Expected '%v' but returned '%v'", a, b)
}
}
func TestBuildClientBodyBufferSize(t *testing.T) {
a := isValidClientBodyBufferSize("1000")
if a != true {
t.Errorf("Expected '%v' but returned '%v'", true, a)
}
b := isValidClientBodyBufferSize("1000k")
if b != true {
t.Errorf("Expected '%v' but returned '%v'", true, b)
}
c := isValidClientBodyBufferSize("1000m")
if c != true {
t.Errorf("Expected '%v' but returned '%v'", true, c)
}
d := isValidClientBodyBufferSize("1000km")
if d != false {
t.Errorf("Expected '%v' but returned '%v'", false, d)
}
e := isValidClientBodyBufferSize("1000mk")
if e != false {
t.Errorf("Expected '%v' but returned '%v'", false, e)
}
f := isValidClientBodyBufferSize("1000kk")
if f != false {
t.Errorf("Expected '%v' but returned '%v'", false, f)
}
g := isValidClientBodyBufferSize("1000mm")
if g != false {
t.Errorf("Expected '%v' but returned '%v'", false, g)
}
h := isValidClientBodyBufferSize(nil)
if h != false {
t.Errorf("Expected '%v' but returned '%v'", false, h)
}
i := isValidClientBodyBufferSize("")
if i != false {
t.Errorf("Expected '%v' but returned '%v'", false, i)
}
}
func TestIsLocationAllowed(t *testing.T) {
loc := ingress.Location{
Denied: nil,
}
isAllowed := isLocationAllowed(&loc)
if !isAllowed {
t.Errorf("Expected '%v' but returned '%v'", true, isAllowed)
}
}
func TestBuildForwardedFor(t *testing.T) {
inputStr := "X-Forwarded-For"
outputStr := buildForwardedFor(inputStr)
validStr := "$http_x_forwarded_for"
if outputStr != validStr {
t.Errorf("Expected '%v' but returned '%v'", validStr, outputStr)
}
}
func TestBuildResolvers(t *testing.T) {
ipOne := net.ParseIP("192.0.0.1")
ipTwo := net.ParseIP("2001:db8:1234:0000:0000:0000:0000:0000")
ipList := []net.IP{ipOne, ipTwo}
validResolver := "resolver 192.0.0.1 [2001:db8:1234::] valid=30s;"
resolver := buildResolvers(ipList)
if resolver != validResolver {
t.Errorf("Expected '%v' but returned '%v'", validResolver, resolver)
}
}
func TestBuildNextUpstream(t *testing.T) {
cases := map[string]struct {
NextUpstream string
NonIdempotent bool
Output string
}{
"default": {
"timeout http_500 http_502",
false,
"timeout http_500 http_502",
},
"global": {
"timeout http_500 http_502",
true,
"timeout http_500 http_502 non_idempotent",
},
"local": {
"timeout http_500 http_502 non_idempotent",
false,
"timeout http_500 http_502 non_idempotent",
},
}
for k, tc := range cases {
nextUpstream := buildNextUpstream(tc.NextUpstream, tc.NonIdempotent)
if nextUpstream != tc.Output {
t.Errorf(
"%s: called buildNextUpstream('%s', %v); expected '%v' but returned '%v'",
k,
tc.NextUpstream,
tc.NonIdempotent,
tc.Output,
nextUpstream,
)
}
}
}
func TestBuildRateLimit(t *testing.T) {
loc := &ingress.Location{}
loc.RateLimit.Connections.Name = "con"
loc.RateLimit.Connections.Limit = 1
loc.RateLimit.RPS.Name = "rps"
loc.RateLimit.RPS.Limit = 1
loc.RateLimit.RPS.Burst = 1
loc.RateLimit.RPM.Name = "rpm"
loc.RateLimit.RPM.Limit = 2
loc.RateLimit.RPM.Burst = 2
loc.RateLimit.LimitRateAfter = 1
loc.RateLimit.LimitRate = 1
validLimits := []string{
"limit_conn con 1;",
"limit_req zone=rps burst=1 nodelay;",
"limit_req zone=rpm burst=2 nodelay;",
"limit_rate_after 1k;",
"limit_rate 1k;",
}
limits := buildRateLimit(loc)
for i, limit := range limits {
if limit != validLimits[i] {
t.Errorf("Expected '%v' but returned '%v'", validLimits, limits)
}
}
}
func TestBuildAuthSignURL(t *testing.T) {
cases := map[string]struct {
Input, Output string
}{
"default url": {"http://google.com", "http://google.com?rd=$pass_access_scheme://$http_host$request_uri"},
"with random field": {"http://google.com?cat=0", "http://google.com?cat=0&rd=$pass_access_scheme://$http_host$request_uri"},
"with rd field": {"http://google.com?cat&rd=$request", "http://google.com?cat&rd=$request"},
}
for k, tc := range cases {
res := buildAuthSignURL(tc.Input)
if res != tc.Output {
t.Errorf("%s: called buildAuthSignURL('%s'); expected '%v' but returned '%v'", k, tc.Input, tc.Output, res)
}
}
}

View file

@ -0,0 +1,51 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controller
import (
"syscall"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/util/sysctl"
)
// sysctlSomaxconn returns the value of net.core.somaxconn, i.e.
// maximum number of connections that can be queued for acceptance
// http://nginx.org/en/docs/http/ngx_http_core_module.html#listen
func sysctlSomaxconn() int {
maxConns, err := sysctl.New().GetSysctl("net/core/somaxconn")
if err != nil || maxConns < 512 {
glog.V(3).Infof("system net.core.somaxconn=%v (using system default)", maxConns)
return 511
}
return maxConns
}
// sysctlFSFileMax returns the value of fs.file-max, i.e.
// maximum number of open file descriptors
func sysctlFSFileMax() int {
var rLimit syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
if err != nil {
glog.Errorf("unexpected error reading system maximum number of open file descriptors (RLIMIT_NOFILE): %v", err)
// returning 0 means don't render the value
return 0
}
return int(rLimit.Max)
}