Adds CustomHTTPErrors ingress annotation and test

Adds per-server/location error-catch functionality to nginx template

Adds documentation

Reduces template duplication with helper function for CUSTOM_ERRORS data

Updates documentation

Adds e2e test for customerrors

Removes AllCustomHTTPErrors, replaces with template function with deduplication and adds e2e test of deduplication

Fixes copy-paste error in test, adds additional test cases

Reverts noop change in controller.go (unused now)

internal/ingress/annotations/annotations.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/ingress-nginx/internal/ingress/annotations/clientbodybuffersize"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/connection"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/cors"
+	"k8s.io/ingress-nginx/internal/ingress/annotations/customhttperrors"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/defaultbackend"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/influxdb"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/ipwhitelist"
@@ -72,6 +73,7 @@ type Ingress struct {
 	ConfigurationSnippet string
 	Connection           connection.Config
 	CorsConfig           cors.Config
+	CustomHTTPErrors     []int
 	DefaultBackend       *apiv1.Service
 	Denied               error
 	ExternalAuth         authreq.Config
@@ -112,6 +114,7 @@ func NewAnnotationExtractor(cfg resolver.Resolver) Extractor {
 			"ConfigurationSnippet": snippet.NewParser(cfg),
 			"Connection":           connection.NewParser(cfg),
 			"CorsConfig":           cors.NewParser(cfg),
+			"CustomHTTPErrors":     customhttperrors.NewParser(cfg),
 			"DefaultBackend":       defaultbackend.NewParser(cfg),
 			"ExternalAuth":         authreq.NewParser(cfg),
 			"Proxy":                proxy.NewParser(cfg),

internal/ingress/annotations/annotations_test.go

@@ -43,6 +43,7 @@ var (
 	annotationAffinityCookieName   = parser.GetAnnotationWithPrefix("session-cookie-name")
 	annotationAffinityCookieHash   = parser.GetAnnotationWithPrefix("session-cookie-hash")
 	annotationUpstreamHashBy       = parser.GetAnnotationWithPrefix("upstream-hash-by")
+	annotationCustomHTTPErrors     = parser.GetAnnotationWithPrefix("custom-http-errors")
 )
 
 type mockCfg struct {
@@ -270,6 +271,41 @@ func TestCors(t *testing.T) {
 
 	}
 }
+func TestCustomHTTPErrors(t *testing.T) {
+	ec := NewAnnotationExtractor(mockCfg{})
+	ing := buildIngress()
+
+	fooAnns := []struct {
+		annotations map[string]string
+		er          []int
+	}{
+		{map[string]string{annotationCustomHTTPErrors: "404,415"}, []int{404, 415}},
+		{map[string]string{annotationCustomHTTPErrors: "404"}, []int{404}},
+		{map[string]string{annotationCustomHTTPErrors: ""}, []int{}},
+		{map[string]string{annotationCustomHTTPErrors + "_no": "404"}, []int{}},
+		{map[string]string{}, []int{}},
+		{nil, []int{}},
+	}
+
+	for _, foo := range fooAnns {
+		ing.SetAnnotations(foo.annotations)
+		r := ec.Extract(ing).CustomHTTPErrors
+
+		// Check that expected codes were created
+		for i := range foo.er {
+			if r[i] != foo.er[i] {
+				t.Errorf("Returned %v but expected %v", r, foo.er)
+			}
+		}
+
+		// Check that no unexpected codes were also created
+		for i := range r {
+			if r[i] != foo.er[i] {
+				t.Errorf("Returned %v but expected %v", r, foo.er)
+			}
+		}
+	}
+}
 
 /*
 func TestMergeLocationAnnotations(t *testing.T) {

internal/ingress/annotations/customhttperrors/main.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package customhttperrors
+
+import (
+	"strconv"
+	"strings"
+
+	extensions "k8s.io/api/extensions/v1beta1"
+
+	"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
+	"k8s.io/ingress-nginx/internal/ingress/resolver"
+)
+
+type customhttperrors struct {
+	r resolver.Resolver
+}
+
+// NewParser creates a new custom http errors annotation parser
+func NewParser(r resolver.Resolver) parser.IngressAnnotation {
+	return customhttperrors{r}
+}
+
+// Parse parses the annotations contained in the ingress to use
+// custom http errors
+func (e customhttperrors) Parse(ing *extensions.Ingress) (interface{}, error) {
+	c, err := parser.GetStringAnnotation("custom-http-errors", ing)
+	if err != nil {
+		return nil, err
+	}
+
+	cSplit := strings.Split(c, ",")
+	var codes []int
+	for _, i := range cSplit {
+		num, err := strconv.Atoi(i)
+		if err != nil {
+			return nil, err
+		}
+		codes = append(codes, num)
+	}
+
+	return codes, nil
+}

internal/ingress/controller/controller.go

@@ -351,6 +351,7 @@ func (n *NGINXController) getBackendServers(ingresses []*extensions.Ingress) ([]
 						loc.InfluxDB = anns.InfluxDB
 						loc.DefaultBackend = anns.DefaultBackend
 						loc.BackendProtocol = anns.BackendProtocol
+						loc.CustomHTTPErrors = anns.CustomHTTPErrors
 
 						if loc.Redirect.FromToWWW {
 							server.RedirectFromToWWW = true
@@ -391,6 +392,7 @@ func (n *NGINXController) getBackendServers(ingresses []*extensions.Ingress) ([]
 						InfluxDB:             anns.InfluxDB,
 						DefaultBackend:       anns.DefaultBackend,
 						BackendProtocol:      anns.BackendProtocol,
+						CustomHTTPErrors:     anns.CustomHTTPErrors,
 					}
 
 					if loc.Redirect.FromToWWW {

internal/ingress/controller/template/template.go

@@ -151,14 +151,16 @@ var (
 		"serverConfig": func(all config.TemplateConfig, server *ingress.Server) interface{} {
 			return struct{ First, Second interface{} }{all, server}
 		},
-		"isValidClientBodyBufferSize": isValidClientBodyBufferSize,
-		"buildForwardedFor":           buildForwardedFor,
-		"buildAuthSignURL":            buildAuthSignURL,
-		"buildOpentracing":            buildOpentracing,
-		"proxySetHeader":              proxySetHeader,
-		"buildInfluxDB":               buildInfluxDB,
-		"enforceRegexModifier":        enforceRegexModifier,
-		"stripLocationModifer":        stripLocationModifer,
+		"isValidClientBodyBufferSize":  isValidClientBodyBufferSize,
+		"buildForwardedFor":            buildForwardedFor,
+		"buildAuthSignURL":             buildAuthSignURL,
+		"buildOpentracing":             buildOpentracing,
+		"proxySetHeader":               proxySetHeader,
+		"buildInfluxDB":                buildInfluxDB,
+		"enforceRegexModifier":         enforceRegexModifier,
+		"stripLocationModifer":         stripLocationModifer,
+		"buildCustomErrorDeps":         buildCustomErrorDeps,
+		"collectCustomErrorsPerServer": collectCustomErrorsPerServer,
 	}
 )
 
@@ -941,3 +943,40 @@ func proxySetHeader(loc interface{}) string {
 
 	return "proxy_set_header"
 }
+
+// buildCustomErrorDeps is a utility function returning a struct wrapper with
+// the data required to build the 'CUSTOM_ERRORS' template
+func buildCustomErrorDeps(proxySetHeaders map[string]string, errorCodes []int) interface{} {
+	return struct {
+		ProxySetHeaders map[string]string
+		ErrorCodes      []int
+	}{
+		ProxySetHeaders: proxySetHeaders,
+		ErrorCodes:      errorCodes,
+	}
+}
+
+// collectCustomErrorsPerServer is a utility function which will collect all
+// custom error codes for all locations of a server block, deduplicates them,
+// and returns a unique set (for the template to create @custom_xxx locations)
+func collectCustomErrorsPerServer(input interface{}) []int {
+	server, ok := input.(*ingress.Server)
+	if !ok {
+		glog.Errorf("expected a '*ingress.Server' type but %T was returned", input)
+		return nil
+	}
+
+	codesMap := make(map[int]bool)
+	for _, loc := range server.Locations {
+		for _, code := range loc.CustomHTTPErrors {
+			codesMap[code] = true
+		}
+	}
+
+	uniqueCodes := make([]int, 0, len(codesMap))
+	for key := range codesMap {
+		uniqueCodes = append(uniqueCodes, key)
+	}
+
+	return uniqueCodes
+}

internal/ingress/types.go

@@ -253,6 +253,9 @@ type Location struct {
 	// BackendProtocol indicates which protocol should be used to communicate with the service
 	// By default this is HTTP
 	BackendProtocol string `json:"backend-protocol"`
+	// CustomHTTPErrors specifies the error codes that should be intercepted.
+	// +optional
+	CustomHTTPErrors []int `json:"custom-http-errors"`
 }
 
 // SSLPassthroughBackend describes a SSL upstream server configured