DevFW-CICD/ingress-nginx-helm
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

implement canary annotation and alternative backends

Browse source 
Adds the ability to create alternative backends. Alternative backends enable
traffic shaping by sharing a single location but routing to different
backends depending on the TrafficShapingPolicy defined by AlternativeBackends.

When the list of upstreams and servers are retrieved, we then call
mergeAlternativeBackends which iterates through the paths of every ingress
and checks if the backend supporting the path is a AlternativeBackend. If
so, we then iterate through the map of servers and find the real backend
that the AlternativeBackend should fall under. Once found, the
AlternativeBackend is embedded in the list of VirtualBackends for the real
backend.

If no matching real backend for a AlternativeBackend is found, then the
AlternativeBackend is deleted as it cannot be backed by any server.
This commit is contained in:
Conor Landry 2018-09-18 14:05:32 -04:00
parent 5ceb723963
commit 412cd70d3a
18 changed files with 859 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

103
internal/ingress/controller/controller.go
View file

@ -268,6 +268,7 @@ func (n *NGINXController) getBackendServers(ingresses []*extensions.Ingress) ([]
if host == "" {
host = defServerName
}
server := servers[host]
if server == nil {
server = servers[defServerName]
@ -300,13 +301,15 @@ func (n *NGINXController) getBackendServers(ingresses []*extensions.Ingress) ([]
}
for _, path := range rule.HTTP.Paths {
upsName := fmt.Sprintf("%v-%v-%v",
ing.Namespace,
path.Backend.ServiceName,
path.Backend.ServicePort.String())
upsName := upstreamName(ing.Namespace, path.Backend.ServiceName, path.Backend.ServicePort)
ups := upstreams[upsName]
// Backend is not referenced to by a server
if ups.NoServer {
continue
}
nginxPath := rootLocation
if path.Path != "" {
nginxPath = path.Path
@ -416,6 +419,11 @@ func (n *NGINXController) getBackendServers(ingresses []*extensions.Ingress) ([]
}
}
}
if anns.Canary.Enabled {
glog.Infof("Canary ingress %v detected. Finding eligible backends to merge into.", ing.Name)
mergeAlternativeBackends(ing, upstreams, servers)
}
}
aUpstreams := make([]*ingress.Backend, 0, len(upstreams))
@ -513,10 +521,7 @@ func (n *NGINXController) createUpstreams(data []*extensions.Ingress, du *ingres
var defBackend string
if ing.Spec.Backend != nil {
defBackend = fmt.Sprintf("%v-%v-%v",
ing.Namespace,
ing.Spec.Backend.ServiceName,
ing.Spec.Backend.ServicePort.String())
defBackend = upstreamName(ing.Namespace, ing.Spec.Backend.ServiceName, ing.Spec.Backend.ServicePort)
glog.V(3).Infof("Creating upstream %q", defBackend)
upstreams[defBackend] = newUpstream(defBackend)
@ -542,6 +547,16 @@ func (n *NGINXController) createUpstreams(data []*extensions.Ingress, du *ingres
}
}
// configure traffic shaping for canary
if anns.Canary.Enabled {
upstreams[defBackend].NoServer = true
upstreams[defBackend].TrafficShapingPolicy = ingress.TrafficShapingPolicy{
Weight: anns.Canary.Weight,
Header: anns.Canary.Header,
Cookie: anns.Canary.Cookie,
}
}
if len(upstreams[defBackend].Endpoints) == 0 {
endps, err := n.serviceEndpoints(svcKey, ing.Spec.Backend.ServicePort.String())
upstreams[defBackend].Endpoints = append(upstreams[defBackend].Endpoints, endps...)
@ -558,10 +573,7 @@ func (n *NGINXController) createUpstreams(data []*extensions.Ingress, du *ingres
}
for _, path := range rule.HTTP.Paths {
name := fmt.Sprintf("%v-%v-%v",
ing.Namespace,
path.Backend.ServiceName,
path.Backend.ServicePort.String())
name := upstreamName(ing.Namespace, path.Backend.ServiceName, path.Backend.ServicePort)
if _, ok := upstreams[name]; ok {
continue
@ -595,6 +607,16 @@ func (n *NGINXController) createUpstreams(data []*extensions.Ingress, du *ingres
}
}
// configure traffic shaping for canary
if anns.Canary.Enabled {
upstreams[name].NoServer = true
upstreams[name].TrafficShapingPolicy = ingress.TrafficShapingPolicy{
Weight: anns.Canary.Weight,
Header: anns.Canary.Header,
Cookie: anns.Canary.Cookie,
}
}
if len(upstreams[name].Endpoints) == 0 {
endp, err := n.serviceEndpoints(svcKey, path.Backend.ServicePort.String())
if err != nil {
@ -970,6 +992,63 @@ func (n *NGINXController) createServers(data []*extensions.Ingress,
return servers
}
// Compares an Ingress of a potential alternative backend's rules with each existing server and finds matching host + path pairs.
// If a match is found, we know that this server should back the alternative backend and add the alternative backend
// to a backend's alternative list.
// If no match is found, then the serverless backend is deleted.
func mergeAlternativeBackends(ing *extensions.Ingress, upstreams map[string]*ingress.Backend,
servers map[string]*ingress.Server) {
// merge catch-all alternative backends
if ing.Spec.Backend != nil {
upsName := upstreamName(ing.Namespace, ing.Spec.Backend.ServiceName, ing.Spec.Backend.ServicePort)
ups := upstreams[upsName]
defLoc := servers[defServerName].Locations[0]
glog.Infof("matching backend %v found for alternative backend %v",
upstreams[defLoc.Backend].Name, ups.Name)
upstreams[defLoc.Backend].AlternativeBackends =
append(upstreams[defLoc.Backend].AlternativeBackends, ups.Name)
}
for _, rule := range ing.Spec.Rules {
for _, path := range rule.HTTP.Paths {
upsName := upstreamName(ing.Namespace, path.Backend.ServiceName, path.Backend.ServicePort)
ups := upstreams[upsName]
merged := false
server := servers[rule.Host]
// find matching paths
for _, location := range server.Locations {
if location.Backend == defUpstreamName {
continue
}
if location.Path == path.Path && !upstreams[location.Backend].NoServer {
glog.Infof("matching backend %v found for alternative backend %v",
upstreams[location.Backend].Name, ups.Name)
upstreams[location.Backend].AlternativeBackends =
append(upstreams[location.Backend].AlternativeBackends, ups.Name)
merged = true
}
}
if !merged {
glog.Warningf("unable to find real backend for alternative backend %v. Deleting.", ups.Name)
delete(upstreams, ups.Name)
}
}
}
}
// extractTLSSecretName returns the name of the Secret containing a SSL
// certificate for the given host name, or an empty string.
func extractTLSSecretName(host string, ing *extensions.Ingress,

179
internal/ingress/controller/controller_test.go
View file

@ -20,6 +20,7 @@ import (
"crypto/x509"
"crypto/x509/pkix"
"encoding/asn1"
"k8s.io/apimachinery/pkg/util/intstr"
"testing"
extensions "k8s.io/api/extensions/v1beta1"
@ -27,6 +28,184 @@ import (
"k8s.io/ingress-nginx/internal/ingress"
)
func TestMergeAlternativeBackends(t *testing.T) {
testCases := map[string]struct {
ingress *extensions.Ingress
upstreams map[string]*ingress.Backend
servers map[string]*ingress.Server
expNumAlternativeBackends int
expNumLocations int
}{
"alternative backend has no server and embeds into matching real backend": {
&extensions.Ingress{
ObjectMeta: metav1.ObjectMeta{
Namespace: "example",
},
Spec: extensions.IngressSpec{
Rules: []extensions.IngressRule{
{
Host: "example.com",
IngressRuleValue: extensions.IngressRuleValue{
HTTP: &extensions.HTTPIngressRuleValue{
Paths: []extensions.HTTPIngressPath{
{
Path: "/",
Backend: extensions.IngressBackend{
ServiceName: "http-svc-canary",
ServicePort: intstr.IntOrString{
Type: intstr.Int,
IntVal: 80,
},
},
},
},
},
},
},
},
},
},
map[string]*ingress.Backend{
"example-http-svc-80": {
Name: "example-http-svc-80",
NoServer: false,
},
"example-http-svc-canary-80": {
Name: "example-http-svc-canary-80",
NoServer: true,
TrafficShapingPolicy: ingress.TrafficShapingPolicy{
Weight: 20,
},
},
},
map[string]*ingress.Server{
"example.com": {
Hostname: "example.com",
Locations: []*ingress.Location{
{
Path: "/",
Backend: "example-http-svc-80",
},
},
},
},
1,
1,
},
"merging a alternative backend matches with the correct host": {
&extensions.Ingress{
ObjectMeta: metav1.ObjectMeta{
Namespace: "example",
},
Spec: extensions.IngressSpec{
Rules: []extensions.IngressRule{
{
Host: "foo.bar",
IngressRuleValue: extensions.IngressRuleValue{
HTTP: &extensions.HTTPIngressRuleValue{
Paths: []extensions.HTTPIngressPath{
{
Path: "/",
Backend: extensions.IngressBackend{
ServiceName: "foo-http-svc-canary",
ServicePort: intstr.IntOrString{
Type: intstr.Int,
IntVal: 80,
},
},
},
},
},
},
},
{
Host: "example.com",
IngressRuleValue: extensions.IngressRuleValue{
HTTP: &extensions.HTTPIngressRuleValue{
Paths: []extensions.HTTPIngressPath{
{
Path: "/",
Backend: extensions.IngressBackend{
ServiceName: "http-svc-canary",
ServicePort: intstr.IntOrString{
Type: intstr.Int,
IntVal: 80,
},
},
},
},
},
},
},
},
},
},
map[string]*ingress.Backend{
"example-foo-http-svc-80": {
Name: "example-foo-http-svc-80",
NoServer: false,
},
"example-foo-http-svc-canary-80": {
Name: "example-foo-http-svc-canary-80",
NoServer: true,
TrafficShapingPolicy: ingress.TrafficShapingPolicy{
Weight: 20,
},
},
"example-http-svc-80": {
Name: "example-http-svc-80",
NoServer: false,
},
"example-http-svc-canary-80": {
Name: "example-http-svc-canary-80",
NoServer: true,
TrafficShapingPolicy: ingress.TrafficShapingPolicy{
Weight: 20,
},
},
},
map[string]*ingress.Server{
"foo.bar": {
Hostname: "foo.bar",
Locations: []*ingress.Location{
{
Path: "/",
Backend: "example-foo-http-svc-80",
},
},
},
"example.com": {
Hostname: "example.com",
Locations: []*ingress.Location{
{
Path: "/",
Backend: "example-http-svc-80",
},
},
},
},
1,
1,
},
}
for title, tc := range testCases {
t.Run(title, func(t *testing.T) {
mergeAlternativeBackends(tc.ingress, tc.upstreams, tc.servers)
numAlternativeBackends := len(tc.upstreams["example-http-svc-80"].AlternativeBackends)
if numAlternativeBackends != tc.expNumAlternativeBackends {
t.Errorf("expected %d alternative backends (got %d)", tc.expNumAlternativeBackends, numAlternativeBackends)
}
numLocations := len(tc.servers["example.com"].Locations)
if numLocations != tc.expNumLocations {
t.Errorf("expected %d locations (got %d)", tc.expNumLocations, numLocations)
}
})
}
}
func TestExtractTLSSecretName(t *testing.T) {
testCases := map[string]struct {
host string

17
internal/ingress/controller/nginx.go
View file

@ -759,13 +759,16 @@ func configureDynamically(pcfg *ingress.Configuration, port int, isDynamicCertif
service = &apiv1.Service{Spec: backend.Service.Spec}
}
luaBackend := &ingress.Backend{
Name: backend.Name,
Port: backend.Port,
SSLPassthrough: backend.SSLPassthrough,
SessionAffinity: backend.SessionAffinity,
UpstreamHashBy: backend.UpstreamHashBy,
LoadBalancing: backend.LoadBalancing,
Service: service,
Name: backend.Name,
Port: backend.Port,
SSLPassthrough: backend.SSLPassthrough,
SessionAffinity: backend.SessionAffinity,
UpstreamHashBy: backend.UpstreamHashBy,
LoadBalancing: backend.LoadBalancing,
Service: service,
NoServer: backend.NoServer,
TrafficShapingPolicy: backend.TrafficShapingPolicy,
AlternativeBackends: backend.AlternativeBackends,
}
var endpoints []ingress.Endpoint

8
internal/ingress/controller/util.go
View file

@ -17,10 +17,13 @@ limitations under the License.
package controller
import (
"k8s.io/apimachinery/pkg/util/intstr"
"os"
"os/exec"
"syscall"
"fmt"
"github.com/golang/glog"
api "k8s.io/api/core/v1"
@ -43,6 +46,11 @@ func newUpstream(name string) *ingress.Backend {
}
}
// upstreamName returns a formatted upstream name based on namespace, service, and port
func upstreamName(namespace string, service string, port intstr.IntOrString) string {
return fmt.Sprintf("%v-%v-%v", namespace, service, port.String())
}
// sysctlSomaxconn returns the maximum number of connections that can be queued
// for acceptance (value of net.core.somaxconn)
// http://nginx.org/en/docs/http/ngx_http_core_module.html#listen