Merge pull request #3198 from aledbf/only-dynamic

Only support dynamic configuration
This commit is contained in:
k8s-ci-robot 2018-10-10 05:07:34 -07:00 committed by GitHub
commit 3edf11b85f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 134 additions and 328 deletions

View file

@ -44,15 +44,13 @@ func (n *NGINXController) Check(_ *http.Request) error {
return fmt.Errorf("ingress controller is not healthy")
}
if n.cfg.DynamicConfigurationEnabled {
res, err := http.Get(fmt.Sprintf("http://127.0.0.1:%v/is-dynamic-lb-initialized", n.cfg.ListenPorts.Status))
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return fmt.Errorf("dynamic load balancer not started")
}
res, err = http.Get(fmt.Sprintf("http://127.0.0.1:%v/is-dynamic-lb-initialized", n.cfg.ListenPorts.Status))
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return fmt.Errorf("dynamic load balancer not started")
}
// check the nginx master process is running

View file

@ -685,26 +685,24 @@ func (cfg Configuration) BuildLogFormatUpstream() string {
// TemplateConfig contains the nginx configuration to render the file nginx.conf
type TemplateConfig struct {
ProxySetHeaders map[string]string
AddHeaders map[string]string
MaxOpenFiles int
BacklogSize int
Backends []*ingress.Backend
PassthroughBackends []*ingress.SSLPassthroughBackend
Servers []*ingress.Server
HealthzURI string
CustomErrors bool
Cfg Configuration
IsIPV6Enabled bool
IsSSLPassthroughEnabled bool
NginxStatusIpv4Whitelist []string
NginxStatusIpv6Whitelist []string
RedirectServers map[string]string
ListenPorts *ListenPorts
PublishService *apiv1.Service
DynamicConfigurationEnabled bool
DynamicCertificatesEnabled bool
DisableLua bool
ProxySetHeaders map[string]string
AddHeaders map[string]string
MaxOpenFiles int
BacklogSize int
Backends []*ingress.Backend
PassthroughBackends []*ingress.SSLPassthroughBackend
Servers []*ingress.Server
HealthzURI string
CustomErrors bool
Cfg Configuration
IsIPV6Enabled bool
IsSSLPassthroughEnabled bool
NginxStatusIpv4Whitelist []string
NginxStatusIpv6Whitelist []string
RedirectServers map[string]string
ListenPorts *ListenPorts
PublishService *apiv1.Service
DynamicCertificatesEnabled bool
}
// ListenPorts describe the ports required to run the

View file

@ -86,10 +86,6 @@ type Configuration struct {
SyncRateLimit float32
DynamicConfigurationEnabled bool
DisableLua bool
DynamicCertificatesEnabled bool
}
@ -162,9 +158,7 @@ func (n *NGINXController) syncIngress(interface{}) error {
return nil
}
if n.cfg.DynamicConfigurationEnabled && n.IsDynamicConfigurationEnough(pcfg) {
glog.Infof("Changes handled by the dynamic configuration, skipping backend reload.")
} else {
if !n.IsDynamicConfigurationEnough(pcfg) {
glog.Infof("Configuration changes detected, backend reload required.")
hash, _ := hashstructure.Hash(pcfg, &hashstructure.HashOptions{
@ -189,23 +183,21 @@ func (n *NGINXController) syncIngress(interface{}) error {
n.metricCollector.SetSSLExpireTime(servers)
}
if n.cfg.DynamicConfigurationEnabled {
isFirstSync := n.runningConfig.Equal(&ingress.Configuration{})
go func(isFirstSync bool) {
if isFirstSync {
glog.Infof("Initial synchronization of the NGINX configuration.")
isFirstSync := n.runningConfig.Equal(&ingress.Configuration{})
go func(isFirstSync bool) {
if isFirstSync {
glog.Infof("Initial synchronization of the NGINX configuration.")
// it takes time for NGINX to start listening on the configured ports
time.Sleep(1 * time.Second)
}
err := configureDynamically(pcfg, n.cfg.ListenPorts.Status, n.cfg.DynamicCertificatesEnabled)
if err == nil {
glog.Infof("Dynamic reconfiguration succeeded.")
} else {
glog.Warningf("Dynamic reconfiguration failed: %v", err)
}
}(isFirstSync)
}
// it takes time for NGINX to start listening on the configured ports
time.Sleep(1 * time.Second)
}
err := configureDynamically(pcfg, n.cfg.ListenPorts.Status, n.cfg.DynamicCertificatesEnabled)
if err == nil {
glog.Infof("Dynamic reconfiguration succeeded.")
} else {
glog.Warningf("Dynamic reconfiguration failed: %v", err)
}
}(isFirstSync)
ri := getRemovedIngresses(n.runningConfig, pcfg)
re := getRemovedHosts(n.runningConfig, pcfg)

View file

@ -571,26 +571,24 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
cfg.SSLDHParam = sslDHParam
tc := ngx_config.TemplateConfig{
ProxySetHeaders: setHeaders,
AddHeaders: addHeaders,
MaxOpenFiles: maxOpenFiles,
BacklogSize: sysctlSomaxconn(),
Backends: ingressCfg.Backends,
PassthroughBackends: ingressCfg.PassthroughBackends,
Servers: ingressCfg.Servers,
HealthzURI: ngxHealthPath,
CustomErrors: len(cfg.CustomHTTPErrors) > 0,
Cfg: cfg,
IsIPV6Enabled: n.isIPV6Enabled && !cfg.DisableIpv6,
NginxStatusIpv4Whitelist: cfg.NginxStatusIpv4Whitelist,
NginxStatusIpv6Whitelist: cfg.NginxStatusIpv6Whitelist,
RedirectServers: redirectServers,
IsSSLPassthroughEnabled: n.cfg.EnableSSLPassthrough,
ListenPorts: n.cfg.ListenPorts,
PublishService: n.GetPublishService(),
DynamicConfigurationEnabled: n.cfg.DynamicConfigurationEnabled,
DynamicCertificatesEnabled: n.cfg.DynamicCertificatesEnabled,
DisableLua: n.cfg.DisableLua,
ProxySetHeaders: setHeaders,
AddHeaders: addHeaders,
MaxOpenFiles: maxOpenFiles,
BacklogSize: sysctlSomaxconn(),
Backends: ingressCfg.Backends,
PassthroughBackends: ingressCfg.PassthroughBackends,
Servers: ingressCfg.Servers,
HealthzURI: ngxHealthPath,
CustomErrors: len(cfg.CustomHTTPErrors) > 0,
Cfg: cfg,
IsIPV6Enabled: n.isIPV6Enabled && !cfg.DisableIpv6,
NginxStatusIpv4Whitelist: cfg.NginxStatusIpv4Whitelist,
NginxStatusIpv6Whitelist: cfg.NginxStatusIpv6Whitelist,
RedirectServers: redirectServers,
IsSSLPassthroughEnabled: n.cfg.EnableSSLPassthrough,
ListenPorts: n.cfg.ListenPorts,
PublishService: n.GetPublishService(),
DynamicCertificatesEnabled: n.cfg.DynamicCertificatesEnabled,
}
tc.Cfg.Checksum = ingressCfg.ConfigurationChecksum

View file

@ -198,24 +198,20 @@ func shouldConfigureLuaRestyWAF(disableLuaRestyWAF bool, mode string) bool {
return false
}
func buildLuaSharedDictionaries(s interface{}, dynamicConfigurationEnabled bool, disableLuaRestyWAF bool) string {
func buildLuaSharedDictionaries(s interface{}, disableLuaRestyWAF bool) string {
servers, ok := s.([]*ingress.Server)
if !ok {
glog.Errorf("expected an '[]*ingress.Server' type but %T was returned", s)
return ""
}
out := []string{}
if dynamicConfigurationEnabled {
out = append(out,
"lua_shared_dict configuration_data 5M",
"lua_shared_dict certificate_data 16M",
"lua_shared_dict locks 512k",
"lua_shared_dict balancer_ewma 1M",
"lua_shared_dict balancer_ewma_last_touched_at 1M",
"lua_shared_dict sticky_sessions 1M",
)
out := []string{
"lua_shared_dict configuration_data 5M",
"lua_shared_dict certificate_data 16M",
"lua_shared_dict locks 512k",
"lua_shared_dict balancer_ewma 1M",
"lua_shared_dict balancer_ewma_last_touched_at 1M",
"lua_shared_dict sticky_sessions 1M",
}
if !disableLuaRestyWAF {
@ -439,7 +435,7 @@ func buildLoadBalancingConfig(b interface{}, fallbackLoadBalancing string) strin
// (specified through the nginx.ingress.kubernetes.io/rewrite-target annotation)
// If the annotation nginx.ingress.kubernetes.io/add-base-url:"true" is specified it will
// add a base tag in the head of the response from the service
func buildProxyPass(host string, b interface{}, loc interface{}, dynamicConfigurationEnabled bool) string {
func buildProxyPass(host string, b interface{}, loc interface{}) string {
backends, ok := b.([]*ingress.Backend)
if !ok {
glog.Errorf("expected an '[]*ingress.Backend' type but %T was returned", b)
@ -473,10 +469,6 @@ func buildProxyPass(host string, b interface{}, loc interface{}, dynamicConfigur
upstreamName := "upstream_balancer"
if !dynamicConfigurationEnabled {
upstreamName = location.Backend
}
for _, backend := range backends {
if backend.Name == location.Backend {
if backend.SSLPassthrough {
@ -487,10 +479,6 @@ func buildProxyPass(host string, b interface{}, loc interface{}, dynamicConfigur
}
}
if !dynamicConfigurationEnabled && isSticky(host, location, backend.SessionAffinity.CookieSessionAffinity.Locations) {
upstreamName = fmt.Sprintf("sticky-%v", upstreamName)
}
break
}
}
@ -724,14 +712,7 @@ func buildDenyVariable(a interface{}) string {
return fmt.Sprintf("$deny_%v", denyPathSlugMap[l])
}
func buildUpstreamName(host string, b interface{}, loc interface{}, dynamicConfigurationEnabled bool) string {
backends, ok := b.([]*ingress.Backend)
if !ok {
glog.Errorf("expected an '[]*ingress.Backend' type but %T was returned", b)
return ""
}
func buildUpstreamName(loc interface{}) string {
location, ok := loc.(*ingress.Location)
if !ok {
glog.Errorf("expected a '*ingress.Location' type but %T was returned", loc)
@ -740,19 +721,6 @@ func buildUpstreamName(host string, b interface{}, loc interface{}, dynamicConfi
upstreamName := location.Backend
if !dynamicConfigurationEnabled {
for _, backend := range backends {
if backend.Name == location.Backend {
if backend.SessionAffinity.AffinityType == "cookie" &&
isSticky(host, location, backend.SessionAffinity.CookieSessionAffinity.Locations) {
upstreamName = fmt.Sprintf("sticky-%v", upstreamName)
}
break
}
}
}
return upstreamName
}

View file

@ -40,41 +40,26 @@ import (
var (
// TODO: add tests for SSLPassthrough
tmplFuncTestcases = map[string]struct {
Path string
Target string
Location string
ProxyPass string
AddBaseURL bool
BaseURLScheme string
Sticky bool
XForwardedPrefix bool
DynamicConfigurationEnabled bool
SecureBackend bool
enforceRegex bool
Path string
Target string
Location string
ProxyPass string
AddBaseURL bool
BaseURLScheme string
Sticky bool
XForwardedPrefix bool
SecureBackend bool
enforceRegex bool
}{
"when secure backend enabled": {
"/",
"/",
"/",
"proxy_pass https://upstream-name;",
"proxy_pass https://upstream_balancer;",
false,
"",
false,
false,
false,
true,
false,
},
"when secure backend and stickeness enabled": {
"/",
"/",
"/",
"proxy_pass https://sticky-upstream-name;",
false,
"",
true,
false,
false,
true,
false,
},
@ -88,8 +73,8 @@ var (
false,
false,
true,
true,
false},
false,
},
"when secure backend, stickeness and dynamic config enabled": {
"/",
"/",
@ -100,7 +85,6 @@ var (
true,
false,
true,
true,
false,
},
"invalid redirect / to / with dynamic config enabled": {
@ -112,7 +96,6 @@ var (
"",
false,
false,
true,
false,
false,
},
@ -120,14 +103,13 @@ var (
"/",
"/",
"/",
"proxy_pass http://upstream-name;",
"proxy_pass http://upstream_balancer;",
false,
"",
false,
false,
false,
false,
false,
},
"redirect / to /jenkins": {
"/",
@ -136,14 +118,13 @@ var (
`
rewrite "(?i)/(.*)" /jenkins/$1 break;
rewrite "(?i)/$" /jenkins/ break;
proxy_pass http://upstream-name;
proxy_pass http://upstream_balancer;
`,
false,
"",
false,
false,
false,
false,
true,
},
"redirect /something to /": {
@ -153,14 +134,13 @@ proxy_pass http://upstream-name;
`
rewrite "(?i)/something/(.*)" /$1 break;
rewrite "(?i)/something$" / break;
proxy_pass http://upstream-name;
proxy_pass http://upstream_balancer;
`,
false,
"",
false,
false,
false,
false,
true,
},
"redirect /end-with-slash/ to /not-root": {
@ -170,14 +150,13 @@ proxy_pass http://upstream-name;
`
rewrite "(?i)/end-with-slash/(.*)" /not-root/$1 break;
rewrite "(?i)/end-with-slash/$" /not-root/ break;
proxy_pass http://upstream-name;
proxy_pass http://upstream_balancer;
`,
false,
"",
false,
false,
false,
false,
true,
},
"redirect /something-complex to /not-root": {
@ -187,14 +166,13 @@ proxy_pass http://upstream-name;
`
rewrite "(?i)/something-complex/(.*)" /not-root/$1 break;
rewrite "(?i)/something-complex$" /not-root/ break;
proxy_pass http://upstream-name;
proxy_pass http://upstream_balancer;
`,
false,
"",
false,
false,
false,
false,
true,
},
"redirect / to /jenkins and rewrite": {
@ -204,7 +182,7 @@ proxy_pass http://upstream-name;
`
rewrite "(?i)/(.*)" /jenkins/$1 break;
rewrite "(?i)/$" /jenkins/ break;
proxy_pass http://upstream-name;
proxy_pass http://upstream_balancer;
set_escape_uri $escaped_base_uri $baseuri;
subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="$scheme://$http_host/$escaped_base_uri">' ro;
@ -214,7 +192,6 @@ subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="
false,
false,
false,
false,
true,
},
"redirect /something to / and rewrite": {
@ -224,7 +201,7 @@ subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="
`
rewrite "(?i)/something/(.*)" /$1 break;
rewrite "(?i)/something$" / break;
proxy_pass http://upstream-name;
proxy_pass http://upstream_balancer;
set_escape_uri $escaped_base_uri $baseuri;
subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="$scheme://$http_host/something/$escaped_base_uri">' ro;
@ -234,7 +211,6 @@ subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="
false,
false,
false,
false,
true,
},
"redirect /end-with-slash/ to /not-root and rewrite": {
@ -244,7 +220,7 @@ subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="
`
rewrite "(?i)/end-with-slash/(.*)" /not-root/$1 break;
rewrite "(?i)/end-with-slash/$" /not-root/ break;
proxy_pass http://upstream-name;
proxy_pass http://upstream_balancer;
set_escape_uri $escaped_base_uri $baseuri;
subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="$scheme://$http_host/end-with-slash/$escaped_base_uri">' ro;
@ -254,7 +230,6 @@ subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="
false,
false,
false,
false,
true,
},
"redirect /something-complex to /not-root and rewrite": {
@ -264,7 +239,7 @@ subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="
`
rewrite "(?i)/something-complex/(.*)" /not-root/$1 break;
rewrite "(?i)/something-complex$" /not-root/ break;
proxy_pass http://upstream-name;
proxy_pass http://upstream_balancer;
set_escape_uri $escaped_base_uri $baseuri;
subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="$scheme://$http_host/something-complex/$escaped_base_uri">' ro;
@ -274,7 +249,6 @@ subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="
false,
false,
false,
false,
true,
},
"redirect /something to / and rewrite with specific scheme": {
@ -284,7 +258,7 @@ subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="
`
rewrite "(?i)/something/(.*)" /$1 break;
rewrite "(?i)/something$" / break;
proxy_pass http://upstream-name;
proxy_pass http://upstream_balancer;
set_escape_uri $escaped_base_uri $baseuri;
subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="http://$http_host/something/$escaped_base_uri">' ro;
@ -294,7 +268,6 @@ subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="
false,
false,
false,
false,
true,
},
"redirect / to /something with sticky enabled": {
@ -304,14 +277,13 @@ subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="
`
rewrite "(?i)/(.*)" /something/$1 break;
rewrite "(?i)/$" /something/ break;
proxy_pass http://sticky-upstream-name;
proxy_pass http://upstream_balancer;
`,
false,
"http",
true,
false,
false,
false,
true,
},
"redirect / to /something with sticky and dynamic config enabled": {
@ -327,7 +299,6 @@ proxy_pass http://upstream_balancer;
"http",
true,
false,
true,
false,
true,
},
@ -339,27 +310,25 @@ proxy_pass http://upstream_balancer;
rewrite "(?i)/there/(.*)" /something/$1 break;
rewrite "(?i)/there$" /something/ break;
proxy_set_header X-Forwarded-Prefix "/there/";
proxy_pass http://sticky-upstream-name;
proxy_pass http://upstream_balancer;
`,
false,
"http",
true,
true,
false,
false,
true,
},
"use ~* location modifier when ingress does not use rewrite/regex target but at least one other ingress does": {
"/something",
"/something",
`~* "^/something"`,
"proxy_pass http://upstream-name;",
"proxy_pass http://upstream_balancer;",
false,
"",
false,
false,
false,
false,
true,
},
}
@ -377,11 +346,7 @@ func TestBuildLuaSharedDictionaries(t *testing.T) {
},
}
config := buildLuaSharedDictionaries(servers, false, false)
if config != "" {
t.Errorf("expected to not configure any lua shared dictionary, but generated %s", config)
}
config = buildLuaSharedDictionaries(servers, true, false)
config := buildLuaSharedDictionaries(servers, false)
if !strings.Contains(config, "lua_shared_dict configuration_data") {
t.Errorf("expected to include 'configuration_data' but got %s", config)
}
@ -390,19 +355,10 @@ func TestBuildLuaSharedDictionaries(t *testing.T) {
}
servers[1].Locations[0].LuaRestyWAF = luarestywaf.Config{Mode: "ACTIVE"}
config = buildLuaSharedDictionaries(servers, false, false)
config = buildLuaSharedDictionaries(servers, false)
if !strings.Contains(config, "lua_shared_dict waf_storage") {
t.Errorf("expected to configure 'waf_storage', but got %s", config)
}
config = buildLuaSharedDictionaries(servers, true, false)
if !strings.Contains(config, "lua_shared_dict waf_storage") {
t.Errorf("expected to configure 'waf_storage', but got %s", config)
}
config = buildLuaSharedDictionaries(servers, false, true)
if config != "" {
t.Errorf("expected to not configure any lua shared dictionary, but generated %s", config)
}
}
func TestFormatIP(t *testing.T) {
@ -471,7 +427,7 @@ func TestBuildProxyPass(t *testing.T) {
backends := []*ingress.Backend{backend}
pp := buildProxyPass(defaultHost, backends, loc, tc.DynamicConfigurationEnabled)
pp := buildProxyPass(defaultHost, backends, loc)
if !strings.EqualFold(tc.ProxyPass, pp) {
t.Errorf("%s: expected \n'%v'\nbut returned \n'%v'", k, tc.ProxyPass, pp)
}
@ -836,10 +792,6 @@ func TestBuildUpstreamName(t *testing.T) {
expected := defaultBackend
if tc.Sticky {
if !tc.DynamicConfigurationEnabled {
expected = fmt.Sprintf("sticky-" + expected)
}
backend.SessionAffinity = ingress.SessionAffinityConfig{
AffinityType: "cookie",
CookieSessionAffinity: ingress.CookieSessionAffinity{
@ -850,9 +802,7 @@ func TestBuildUpstreamName(t *testing.T) {
}
}
backends := []*ingress.Backend{backend}
pp := buildUpstreamName(defaultHost, backends, loc, tc.DynamicConfigurationEnabled)
pp := buildUpstreamName(loc)
if !strings.EqualFold(expected, pp) {
t.Errorf("%s: expected \n'%v'\nbut returned \n'%v'", k, expected, pp)
}