DevFW-CICD/ingress-nginx-helm
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #3341 from Shopify/canary_upstream

Browse source 
Add canary annotation and alternative backends for traffic shaping
This commit is contained in:
k8s-ci-robot 2018-11-06 12:22:16 -08:00 • committed by GitHub
commit 17cad51e47
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 859 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

3
internal/ingress/annotations/annotations.go
View file

@ -19,6 +19,7 @@ package annotations
import (
"github.com/golang/glog"
"github.com/imdario/mergo"
"k8s.io/ingress-nginx/internal/ingress/annotations/canary"
"k8s.io/ingress-nginx/internal/ingress/annotations/sslcipher"
apiv1 "k8s.io/api/core/v1"
@ -68,6 +69,7 @@ type Ingress struct {
BackendProtocol string
Alias string
BasicDigestAuth auth.Config
Canary canary.Config
CertificateAuth authtls.Config
ClientBodyBufferSize string
ConfigurationSnippet string
@ -109,6 +111,7 @@ func NewAnnotationExtractor(cfg resolver.Resolver) Extractor {
map[string]parser.IngressAnnotation{
"Alias": alias.NewParser(cfg),
"BasicDigestAuth": auth.NewParser(auth.AuthDirectory, cfg),
"Canary": canary.NewParser(cfg),
"CertificateAuth": authtls.NewParser(cfg),
"ClientBodyBufferSize": clientbodybuffersize.NewParser(cfg),
"ConfigurationSnippet": snippet.NewParser(cfg),

75
internal/ingress/annotations/canary/main.go Normal file
View file

@ -0,0 +1,75 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package canary
import (
extensions "k8s.io/api/extensions/v1beta1"
"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
"k8s.io/ingress-nginx/internal/ingress/errors"
"k8s.io/ingress-nginx/internal/ingress/resolver"
)
type canary struct {
r resolver.Resolver
}
// Config returns the configuration rules for setting up the Canary
type Config struct {
Enabled bool
Weight int
Header string
Cookie string
}
// NewParser parses the ingress for canary related annotations
func NewParser(r resolver.Resolver) parser.IngressAnnotation {
return canary{r}
}
// Parse parses the annotations contained in the ingress
// rule used to indicate if the canary should be enabled and with what config
func (c canary) Parse(ing *extensions.Ingress) (interface{}, error) {
config := &Config{}
var err error
config.Enabled, err = parser.GetBoolAnnotation("canary", ing)
if err != nil {
config.Enabled = false
}
config.Weight, err = parser.GetIntAnnotation("canary-weight", ing)
if err != nil {
config.Weight = 0
}
config.Header, err = parser.GetStringAnnotation("canary-by-header", ing)
if err != nil {
config.Header = ""
}
config.Cookie, err = parser.GetStringAnnotation("canary-by-cookie", ing)
if err != nil {
config.Cookie = ""
}
if !config.Enabled && (config.Weight > 0 || len(config.Header) > 0 || len(config.Cookie) > 0) {
return nil, errors.NewInvalidAnnotationConfiguration("canary", "configured but not enabled")
}
return config, nil
}

126
internal/ingress/annotations/canary/main_test.go Normal file
View file

@ -0,0 +1,126 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package canary
import (
api "k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
"testing"
"k8s.io/ingress-nginx/internal/ingress/resolver"
"strconv"
)
func buildIngress() *extensions.Ingress {
defaultBackend := extensions.IngressBackend{
ServiceName: "default-backend",
ServicePort: intstr.FromInt(80),
}
return &extensions.Ingress{
ObjectMeta: metaV1.ObjectMeta{
Name: "foo",
Namespace: api.NamespaceDefault,
},
Spec: extensions.IngressSpec{
Backend: &extensions.IngressBackend{
ServiceName: "default-backend",
ServicePort: intstr.FromInt(80),
},
Rules: []extensions.IngressRule{
{
Host: "foo.bar.com",
IngressRuleValue: extensions.IngressRuleValue{
HTTP: &extensions.HTTPIngressRuleValue{
Paths: []extensions.HTTPIngressPath{
{
Path: "/foo",
Backend: defaultBackend,
},
},
},
},
},
},
},
}
}
func TestAnnotations(t *testing.T) {
ing := buildIngress()
data := map[string]string{}
ing.SetAnnotations(data)
tests := []struct {
title string
canaryEnabled bool
canaryWeight int
canaryHeader string
canaryCookie string
expErr bool
}{
{"canary disabled and no weight", false, 0, "", "", false},
{"canary disabled and weight", false, 20, "", "", true},
{"canary disabled and header", false, 0, "X-Canary", "", true},
{"canary disabled and cookie", false, 0, "", "canary_enabled", true},
{"canary enabled and weight", true, 20, "", "", false},
{"canary enabled and no weight", true, 0, "", "", false},
{"canary enabled by header", true, 20, "X-Canary", "", false},
{"canary enabled by cookie", true, 20, "", "canary_enabled", false},
}
for _, test := range tests {
data[parser.GetAnnotationWithPrefix("canary")] = strconv.FormatBool(test.canaryEnabled)
data[parser.GetAnnotationWithPrefix("canary-weight")] = strconv.Itoa(test.canaryWeight)
data[parser.GetAnnotationWithPrefix("canary-by-header")] = test.canaryHeader
data[parser.GetAnnotationWithPrefix("canary-by-cookie")] = test.canaryCookie
i, err := NewParser(&resolver.Mock{}).Parse(ing)
if test.expErr {
if err == nil {
t.Errorf("%v: expected error but returned nil", test.title)
}
continue
} else {
if err != nil {
t.Errorf("%v: expected nil but returned error %v", test.title, err)
}
}
canaryConfig, ok := i.(*Config)
if !ok {
t.Errorf("%v: expected an External type", test.title)
}
if canaryConfig.Enabled != test.canaryEnabled {
t.Errorf("%v: expected \"%v\", but \"%v\" was returned", test.title, test.canaryEnabled, canaryConfig.Enabled)
}
if canaryConfig.Weight != test.canaryWeight {
t.Errorf("%v: expected \"%v\", but \"%v\" was returned", test.title, test.canaryWeight, canaryConfig.Weight)
}
if canaryConfig.Header != test.canaryHeader {
t.Errorf("%v: expected \"%v\", but \"%v\" was returned", test.title, test.canaryHeader, canaryConfig.Header)
}
if canaryConfig.Cookie != test.canaryCookie {
t.Errorf("%v: expected \"%v\", but \"%v\" was returned", test.title, test.canaryCookie, canaryConfig.Cookie)
}
}
}

103
internal/ingress/controller/controller.go
View file

@ -268,6 +268,7 @@ func (n *NGINXController) getBackendServers(ingresses []*extensions.Ingress) ([]
if host == "" {
host = defServerName
}
server := servers[host]
if server == nil {
server = servers[defServerName]
@ -300,13 +301,15 @@ func (n *NGINXController) getBackendServers(ingresses []*extensions.Ingress) ([]
}
for _, path := range rule.HTTP.Paths {
upsName := fmt.Sprintf("%v-%v-%v",
ing.Namespace,
path.Backend.ServiceName,
path.Backend.ServicePort.String())
upsName := upstreamName(ing.Namespace, path.Backend.ServiceName, path.Backend.ServicePort)
ups := upstreams[upsName]
// Backend is not referenced to by a server
if ups.NoServer {
continue
}
nginxPath := rootLocation
if path.Path != "" {
nginxPath = path.Path
@ -420,6 +423,11 @@ func (n *NGINXController) getBackendServers(ingresses []*extensions.Ingress) ([]
}
}
}
if anns.Canary.Enabled {
glog.Infof("Canary ingress %v detected. Finding eligible backends to merge into.", ing.Name)
mergeAlternativeBackends(ing, upstreams, servers)
}
}
aUpstreams := make([]*ingress.Backend, 0, len(upstreams))
@ -508,10 +516,7 @@ func (n *NGINXController) createUpstreams(data []*extensions.Ingress, du *ingres
var defBackend string
if ing.Spec.Backend != nil {
defBackend = fmt.Sprintf("%v-%v-%v",
ing.Namespace,
ing.Spec.Backend.ServiceName,
ing.Spec.Backend.ServicePort.String())
defBackend = upstreamName(ing.Namespace, ing.Spec.Backend.ServiceName, ing.Spec.Backend.ServicePort)
glog.V(3).Infof("Creating upstream %q", defBackend)
upstreams[defBackend] = newUpstream(defBackend)
@ -537,6 +542,16 @@ func (n *NGINXController) createUpstreams(data []*extensions.Ingress, du *ingres
}
}
// configure traffic shaping for canary
if anns.Canary.Enabled {
upstreams[defBackend].NoServer = true
upstreams[defBackend].TrafficShapingPolicy = ingress.TrafficShapingPolicy{
Weight: anns.Canary.Weight,
Header: anns.Canary.Header,
Cookie: anns.Canary.Cookie,
}
}
if len(upstreams[defBackend].Endpoints) == 0 {
endps, err := n.serviceEndpoints(svcKey, ing.Spec.Backend.ServicePort.String())
upstreams[defBackend].Endpoints = append(upstreams[defBackend].Endpoints, endps...)
@ -558,10 +573,7 @@ func (n *NGINXController) createUpstreams(data []*extensions.Ingress, du *ingres
}
for _, path := range rule.HTTP.Paths {
name := fmt.Sprintf("%v-%v-%v",
ing.Namespace,
path.Backend.ServiceName,
path.Backend.ServicePort.String())
name := upstreamName(ing.Namespace, path.Backend.ServiceName, path.Backend.ServicePort)
if _, ok := upstreams[name]; ok {
continue
@ -595,6 +607,16 @@ func (n *NGINXController) createUpstreams(data []*extensions.Ingress, du *ingres
}
}
// configure traffic shaping for canary
if anns.Canary.Enabled {
upstreams[name].NoServer = true
upstreams[name].TrafficShapingPolicy = ingress.TrafficShapingPolicy{
Weight: anns.Canary.Weight,
Header: anns.Canary.Header,
Cookie: anns.Canary.Cookie,
}
}
if len(upstreams[name].Endpoints) == 0 {
endp, err := n.serviceEndpoints(svcKey, path.Backend.ServicePort.String())
if err != nil {
@ -970,6 +992,63 @@ func (n *NGINXController) createServers(data []*extensions.Ingress,
return servers
}
// Compares an Ingress of a potential alternative backend's rules with each existing server and finds matching host + path pairs.
// If a match is found, we know that this server should back the alternative backend and add the alternative backend
// to a backend's alternative list.
// If no match is found, then the serverless backend is deleted.
func mergeAlternativeBackends(ing *extensions.Ingress, upstreams map[string]*ingress.Backend,
servers map[string]*ingress.Server) {
// merge catch-all alternative backends
if ing.Spec.Backend != nil {
upsName := upstreamName(ing.Namespace, ing.Spec.Backend.ServiceName, ing.Spec.Backend.ServicePort)
ups := upstreams[upsName]
defLoc := servers[defServerName].Locations[0]
glog.Infof("matching backend %v found for alternative backend %v",
upstreams[defLoc.Backend].Name, ups.Name)
upstreams[defLoc.Backend].AlternativeBackends =
append(upstreams[defLoc.Backend].AlternativeBackends, ups.Name)
}
for _, rule := range ing.Spec.Rules {
for _, path := range rule.HTTP.Paths {
upsName := upstreamName(ing.Namespace, path.Backend.ServiceName, path.Backend.ServicePort)
ups := upstreams[upsName]
merged := false
server := servers[rule.Host]
// find matching paths
for _, location := range server.Locations {
if location.Backend == defUpstreamName {
continue
}
if location.Path == path.Path && !upstreams[location.Backend].NoServer {
glog.Infof("matching backend %v found for alternative backend %v",
upstreams[location.Backend].Name, ups.Name)
upstreams[location.Backend].AlternativeBackends =
append(upstreams[location.Backend].AlternativeBackends, ups.Name)
merged = true
}
}
if !merged {
glog.Warningf("unable to find real backend for alternative backend %v. Deleting.", ups.Name)
delete(upstreams, ups.Name)
}
}
}
}
// extractTLSSecretName returns the name of the Secret containing a SSL
// certificate for the given host name, or an empty string.
func extractTLSSecretName(host string, ing *extensions.Ingress,

179
internal/ingress/controller/controller_test.go
View file

@ -20,6 +20,7 @@ import (
"crypto/x509"
"crypto/x509/pkix"
"encoding/asn1"
"k8s.io/apimachinery/pkg/util/intstr"
"testing"
extensions "k8s.io/api/extensions/v1beta1"
@ -27,6 +28,184 @@ import (
"k8s.io/ingress-nginx/internal/ingress"
)
func TestMergeAlternativeBackends(t *testing.T) {
testCases := map[string]struct {
ingress *extensions.Ingress
upstreams map[string]*ingress.Backend
servers map[string]*ingress.Server
expNumAlternativeBackends int
expNumLocations int
}{
"alternative backend has no server and embeds into matching real backend": {
&extensions.Ingress{
ObjectMeta: metav1.ObjectMeta{
Namespace: "example",
},
Spec: extensions.IngressSpec{
Rules: []extensions.IngressRule{
{
Host: "example.com",
IngressRuleValue: extensions.IngressRuleValue{
HTTP: &extensions.HTTPIngressRuleValue{
Paths: []extensions.HTTPIngressPath{
{
Path: "/",
Backend: extensions.IngressBackend{
ServiceName: "http-svc-canary",
ServicePort: intstr.IntOrString{
Type: intstr.Int,
IntVal: 80,
},
},
},
},
},
},
},
},
},
},
map[string]*ingress.Backend{
"example-http-svc-80": {
Name: "example-http-svc-80",
NoServer: false,
},
"example-http-svc-canary-80": {
Name: "example-http-svc-canary-80",
NoServer: true,
TrafficShapingPolicy: ingress.TrafficShapingPolicy{
Weight: 20,
},
},
},
map[string]*ingress.Server{
"example.com": {
Hostname: "example.com",
Locations: []*ingress.Location{
{
Path: "/",
Backend: "example-http-svc-80",
},
},
},
},
1,
1,
},
"merging a alternative backend matches with the correct host": {
&extensions.Ingress{
ObjectMeta: metav1.ObjectMeta{
Namespace: "example",
},
Spec: extensions.IngressSpec{
Rules: []extensions.IngressRule{
{
Host: "foo.bar",
IngressRuleValue: extensions.IngressRuleValue{
HTTP: &extensions.HTTPIngressRuleValue{
Paths: []extensions.HTTPIngressPath{
{
Path: "/",
Backend: extensions.IngressBackend{
ServiceName: "foo-http-svc-canary",
ServicePort: intstr.IntOrString{
Type: intstr.Int,
IntVal: 80,
},
},
},
},
},
},
},
{
Host: "example.com",
IngressRuleValue: extensions.IngressRuleValue{
HTTP: &extensions.HTTPIngressRuleValue{
Paths: []extensions.HTTPIngressPath{
{
Path: "/",
Backend: extensions.IngressBackend{
ServiceName: "http-svc-canary",
ServicePort: intstr.IntOrString{
Type: intstr.Int,
IntVal: 80,
},
},
},
},
},
},
},
},
},
},
map[string]*ingress.Backend{
"example-foo-http-svc-80": {
Name: "example-foo-http-svc-80",
NoServer: false,
},
"example-foo-http-svc-canary-80": {
Name: "example-foo-http-svc-canary-80",
NoServer: true,
TrafficShapingPolicy: ingress.TrafficShapingPolicy{
Weight: 20,
},
},
"example-http-svc-80": {
Name: "example-http-svc-80",
NoServer: false,
},
"example-http-svc-canary-80": {
Name: "example-http-svc-canary-80",
NoServer: true,
TrafficShapingPolicy: ingress.TrafficShapingPolicy{
Weight: 20,
},
},
},
map[string]*ingress.Server{
"foo.bar": {
Hostname: "foo.bar",
Locations: []*ingress.Location{
{
Path: "/",
Backend: "example-foo-http-svc-80",
},
},
},
"example.com": {
Hostname: "example.com",
Locations: []*ingress.Location{
{
Path: "/",
Backend: "example-http-svc-80",
},
},
},
},
1,
1,
},
}
for title, tc := range testCases {
t.Run(title, func(t *testing.T) {
mergeAlternativeBackends(tc.ingress, tc.upstreams, tc.servers)
numAlternativeBackends := len(tc.upstreams["example-http-svc-80"].AlternativeBackends)
if numAlternativeBackends != tc.expNumAlternativeBackends {
t.Errorf("expected %d alternative backends (got %d)", tc.expNumAlternativeBackends, numAlternativeBackends)
}
numLocations := len(tc.servers["example.com"].Locations)
if numLocations != tc.expNumLocations {
t.Errorf("expected %d locations (got %d)", tc.expNumLocations, numLocations)
}
})
}
}
func TestExtractTLSSecretName(t *testing.T) {
testCases := map[string]struct {
host string

17
internal/ingress/controller/nginx.go
View file

@ -759,13 +759,16 @@ func configureDynamically(pcfg *ingress.Configuration, port int, isDynamicCertif
service = &apiv1.Service{Spec: backend.Service.Spec}
}
luaBackend := &ingress.Backend{
Name: backend.Name,
Port: backend.Port,
SSLPassthrough: backend.SSLPassthrough,
SessionAffinity: backend.SessionAffinity,
UpstreamHashBy: backend.UpstreamHashBy,
LoadBalancing: backend.LoadBalancing,
Service: service,
Name: backend.Name,
Port: backend.Port,
SSLPassthrough: backend.SSLPassthrough,
SessionAffinity: backend.SessionAffinity,
UpstreamHashBy: backend.UpstreamHashBy,
LoadBalancing: backend.LoadBalancing,
Service: service,
NoServer: backend.NoServer,
TrafficShapingPolicy: backend.TrafficShapingPolicy,
AlternativeBackends: backend.AlternativeBackends,
}
var endpoints []ingress.Endpoint

8
internal/ingress/controller/util.go
View file

@ -17,10 +17,13 @@ limitations under the License.
package controller
import (
"k8s.io/apimachinery/pkg/util/intstr"
"os"
"os/exec"
"syscall"
"fmt"
"github.com/golang/glog"
api "k8s.io/api/core/v1"
@ -43,6 +46,11 @@ func newUpstream(name string) *ingress.Backend {
}
}
// upstreamName returns a formatted upstream name based on namespace, service, and port
func upstreamName(namespace string, service string, port intstr.IntOrString) string {
return fmt.Sprintf("%v-%v-%v", namespace, service, port.String())
}
// sysctlSomaxconn returns the maximum number of connections that can be queued
// for acceptance (value of net.core.somaxconn)
// http://nginx.org/en/docs/http/ngx_http_core_module.html#listen

17
internal/ingress/errors/errors.go
View file

@ -32,6 +32,14 @@ var (
ErrInvalidAnnotationName = errors.New("invalid annotation name")
)
// NewInvalidAnnotationConfiguration returns a new InvalidConfiguration error for use when
// annotations are not correctly configured
func NewInvalidAnnotationConfiguration(name string, reason string) error {
return InvalidConfiguration{
Name: fmt.Sprintf("the annotation %v does not contain a valid configuration: %v", name, reason),
}
}
// NewInvalidAnnotationContent returns a new InvalidContent error
func NewInvalidAnnotationContent(name string, val interface{}) error {
return InvalidContent{
@ -46,6 +54,15 @@ func NewLocationDenied(reason string) error {
}
}
// InvalidConfiguration Error
type InvalidConfiguration struct {
Name string
}
func (e InvalidConfiguration) Error() string {
return e.Name
}
// InvalidContent error
type InvalidContent struct {
Name string

24
internal/ingress/types.go
View file

@ -85,6 +85,30 @@ type Backend struct {
UpstreamHashBy string `json:"upstream-hash-by,omitempty"`
// LB algorithm configuration per ingress
LoadBalancing string `json:"load-balance,omitempty"`
// Denotes if a backend has no server. The backend instead shares a server with another backend and acts as an
// alternative backend.
// This can be used to share multiple upstreams in the sam nginx server block.
NoServer bool `json:"noServer"`
// Policies to describe the characteristics of an alternative backend.
// +optional
TrafficShapingPolicy TrafficShapingPolicy `json:"trafficShapingPolicy,omitempty"`
// Contains a list of backends without servers that are associated with this backend.
// +optional
AlternativeBackends []string `json:"alternativeBackends,omitempty"`
}
// TrafficShapingPolicy describes the policies to put in place when a backend has no server and is used as an
// alternative backend
// +k8s:deepcopy-gen=true
type TrafficShapingPolicy struct {
// Weight (0-100) of traffic to redirect to the backend.
// e.g. Weight 20 means 20% of traffic will be redirected to the backend and 80% will remain
// with the other backend. 0 weight will not send any traffic to this backend
Weight int `json:"weight"`
// Header on which to redirect requests to this backend
Header string `json:"header"`
// Cookie on which to redirect requests to this backend
Cookie string `json:"cookie"`
}
// HashInclude defines if a field should be used or not to calculate the hash

35
internal/ingress/types_equals.go
View file

@ -84,6 +84,9 @@ func (b1 *Backend) Equal(b2 *Backend) bool {
if b1.Name != b2.Name {
return false
}
if b1.NoServer != b2.NoServer {
return false
}
if b1.Service != b2.Service {
if b1.Service == nil || b2.Service == nil {
@ -133,6 +136,23 @@ func (b1 *Backend) Equal(b2 *Backend) bool {
}
}
if !b1.TrafficShapingPolicy.Equal(b2.TrafficShapingPolicy) {
return false
}
for _, vb1 := range b1.AlternativeBackends {
found := false
for _, vb2 := range b2.AlternativeBackends {
if vb1 == vb2 {
found = true
break
}
}
if !found {
return false
}
}
return true
}
@ -202,6 +222,21 @@ func (e1 *Endpoint) Equal(e2 *Endpoint) bool {
return true
}
// Equal checks for equality between two TrafficShapingPolicies
func (tsp1 TrafficShapingPolicy) Equal(tsp2 TrafficShapingPolicy) bool {
if tsp1.Weight != tsp2.Weight {
return false
}
if tsp1.Header != tsp2.Header {
return false
}
if tsp1.Cookie != tsp2.Cookie {
return false
}
return true
}
// Equal tests for equality between two Server types
func (s1 *Server) Equal(s2 *Server) bool {
if s1 == s2 {