Allow the usage of Services as Upstream on a global level (#7469)

It is possible to change this behavior on an ingress level, which works
well when you only have a few of them. When running several dozen
ingress and with a high change rate of running pods it makes it easier
to define this configuration on a global level.

This change is completely backwards compatible, only adding the
possibility of defining a new key in the configmap.

internal/ingress/annotations/serviceupstream/main.go

@@ -20,6 +20,7 @@ import (
 	networking "k8s.io/api/networking/v1"
 
 	"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
+	"k8s.io/ingress-nginx/internal/ingress/errors"
 	"k8s.io/ingress-nginx/internal/ingress/resolver"
 )
 
@@ -33,5 +34,13 @@ func NewParser(r resolver.Resolver) parser.IngressAnnotation {
 }
 
 func (s serviceUpstream) Parse(ing *networking.Ingress) (interface{}, error) {
-	return parser.GetBoolAnnotation("service-upstream", ing)
+	defBackend := s.r.GetDefaultBackend()
+
+	val, err := parser.GetBoolAnnotation("service-upstream", ing)
+	// A missing annotation is not a problem, just use the default
+	if err == errors.ErrMissingAnnotations {
+		return defBackend.ServiceUpstream, nil
+	}
+
+	return val, nil
 }

internal/ingress/annotations/serviceupstream/main_test.go

@@ -23,6 +23,7 @@ import (
 	networking "k8s.io/api/networking/v1"
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
+	"k8s.io/ingress-nginx/internal/ingress/defaults"
 	"k8s.io/ingress-nginx/internal/ingress/resolver"
 )
 
@@ -119,3 +120,52 @@ func TestIngressAnnotationServiceUpstreamSetFalse(t *testing.T) {
 		t.Errorf("expected annotation value to be false, got true")
 	}
 }
+
+type mockBackend struct {
+	resolver.Mock
+}
+
+// GetDefaultBackend returns the backend that must be used as default
+func (m mockBackend) GetDefaultBackend() defaults.Backend {
+	return defaults.Backend{
+		ServiceUpstream: true,
+	}
+}
+
+// Test that when we have a default configuration set on the Backend that is used
+// when we don't have the annotation
+func TestParseAnnotationsWithDefaultConfig(t *testing.T) {
+	ing := buildIngress()
+
+	val, _ := NewParser(mockBackend{}).Parse(ing)
+	enabled, ok := val.(bool)
+
+	if !ok {
+		t.Errorf("expected a bool type")
+	}
+
+	if !enabled {
+		t.Errorf("expected annotation value to be true, got false")
+	}
+}
+
+// Test that the annotation will disable the service upstream when enabled
+// in the default configuration
+func TestParseAnnotationsOverridesDefaultConfig(t *testing.T) {
+	ing := buildIngress()
+
+	data := map[string]string{}
+	data[parser.GetAnnotationWithPrefix("service-upstream")] = "false"
+	ing.SetAnnotations(data)
+
+	val, _ := NewParser(mockBackend{}).Parse(ing)
+	enabled, ok := val.(bool)
+
+	if !ok {
+		t.Errorf("expected a bool type")
+	}
+
+	if enabled {
+		t.Errorf("expected annotation value to be false, got true")
+	}
+}

internal/ingress/controller/config/config.go

@@ -860,6 +860,7 @@ func NewDefault() Configuration {
 			ProxyBuffering:           "off",
 			ProxyHTTPVersion:         "1.1",
 			ProxyMaxTempFileSize:     "1024m",
+			ServiceUpstream:          false,
 		},
 		UpstreamKeepaliveConnections:           320,
 		UpstreamKeepaliveTimeout:               60,

internal/ingress/defaults/main.go

@@ -161,4 +161,8 @@ type Backend struct {
 	// Sets the maximum temp file size when proxy-buffers capacity is exceeded.
 	// http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_max_temp_file_size
 	ProxyMaxTempFileSize string `json:"proxy-max-temp-file-size"`
+
+	// By default, the NGINX ingress controller uses a list of all endpoints (Pod IP/port) in the NGINX upstream configuration.
+	// It disables that behavior and instead uses a single upstream in NGINX, the service's Cluster IP and port.
+	ServiceUpstream bool `json:"service-upstream"`
 }