Add keepalive support for auth requests (#8219)

* Add keepalive support for auth requests

* Fix typo

* Address PR comments

* Log warning when auth-url contains variable in its host:port
* Generate upstream name without replacing dots to underscores in server name
* Add comment in the nginx template when the keepalive upstream block is referenced

* Workaround for auth_request module ignores keepalive in upstream block

* The `auth_request` module does not support HTTP keepalives in upstream block:
  https://trac.nginx.org/nginx/ticket/1579
* As a workaround we use ngx.location.capture but unfortunately it does not
  support HTTP/2 so `use-http2` configuration parameter is needed.

* Handle PR comments

* Address PR comments

* Handle invalid values for int parameters

* Handle PR comments

* Fix e2e test

internal/ingress/annotations/authreq/main.go

@@ -44,12 +44,22 @@ type Config struct {
 	AuthSnippet            string            `json:"authSnippet"`
 	AuthCacheKey           string            `json:"authCacheKey"`
 	AuthCacheDuration      []string          `json:"authCacheDuration"`
+	KeepaliveConnections   int               `json:"keepaliveConnections"`
+	KeepaliveRequests      int               `json:"keepaliveRequests"`
+	KeepaliveTimeout       int               `json:"keepaliveTimeout"`
 	ProxySetHeaders        map[string]string `json:"proxySetHeaders,omitempty"`
 }
 
 // DefaultCacheDuration is the fallback value if no cache duration is provided
 const DefaultCacheDuration = "200 202 401 5m"
 
+// fallback values when no keepalive parameters are set
+const (
+	defaultKeepaliveConnections = 0
+	defaultKeepaliveRequests    = 1000
+	defaultKeepaliveTimeout     = 60
+)
+
 // Equal tests for equality between two Config types
 func (e1 *Config) Equal(e2 *Config) bool {
 	if e1 == e2 {
@@ -90,6 +100,18 @@ func (e1 *Config) Equal(e2 *Config) bool {
 		return false
 	}
 
+	if e1.KeepaliveConnections != e2.KeepaliveConnections {
+		return false
+	}
+
+	if e1.KeepaliveRequests != e2.KeepaliveRequests {
+		return false
+	}
+
+	if e1.KeepaliveTimeout != e2.KeepaliveTimeout {
+		return false
+	}
+
 	return sets.StringElementsMatch(e1.AuthCacheDuration, e2.AuthCacheDuration)
 }
 
@@ -193,6 +215,43 @@ func (a authReq) Parse(ing *networking.Ingress) (interface{}, error) {
 		klog.V(3).InfoS("auth-cache-key annotation is undefined and will not be set")
 	}
 
+	keepaliveConnections, err := parser.GetIntAnnotation("auth-keepalive", ing)
+	if err != nil {
+		klog.V(3).InfoS("auth-keepalive annotation is undefined and will be set to its default value")
+		keepaliveConnections = defaultKeepaliveConnections
+	}
+	switch {
+	case keepaliveConnections < 0:
+		klog.Warningf("auth-keepalive annotation (%s) contains a negative value, setting auth-keepalive to 0", authURL.Host)
+		keepaliveConnections = 0
+	case keepaliveConnections > 0:
+		// NOTE: upstream block cannot reference a variable in the server directive
+		if strings.IndexByte(authURL.Host, '$') != -1 {
+			klog.Warningf("auth-url annotation (%s) contains $ in the host:port part, setting auth-keepalive to 0", authURL.Host)
+			keepaliveConnections = 0
+		}
+	}
+
+	keepaliveRequests, err := parser.GetIntAnnotation("auth-keepalive-requests", ing)
+	if err != nil {
+		klog.V(3).InfoS("auth-keepalive-requests annotation is undefined and will be set to its default value")
+		keepaliveRequests = defaultKeepaliveRequests
+	}
+	if keepaliveRequests <= 0 {
+		klog.Warningf("auth-keepalive-requests annotation (%s) should be greater than zero, setting auth-keepalive to 0", authURL.Host)
+		keepaliveConnections = 0
+	}
+
+	keepaliveTimeout, err := parser.GetIntAnnotation("auth-keepalive-timeout", ing)
+	if err != nil {
+		klog.V(3).InfoS("auth-keepalive-timeout annotation is undefined and will be set to its default value")
+		keepaliveTimeout = defaultKeepaliveTimeout
+	}
+	if keepaliveTimeout <= 0 {
+		klog.Warningf("auth-keepalive-timeout annotation (%s) should be greater than zero, setting auth-keepalive 0", authURL.Host)
+		keepaliveConnections = 0
+	}
+
 	durstr, _ := parser.GetStringAnnotation("auth-cache-duration", ing)
 	authCacheDuration, err := ParseStringToCacheDurations(durstr)
 	if err != nil {
@@ -249,6 +308,9 @@ func (a authReq) Parse(ing *networking.Ingress) (interface{}, error) {
 		AuthSnippet:            authSnippet,
 		AuthCacheKey:           authCacheKey,
 		AuthCacheDuration:      authCacheDuration,
+		KeepaliveConnections:   keepaliveConnections,
+		KeepaliveRequests:      keepaliveRequests,
+		KeepaliveTimeout:       keepaliveTimeout,
 		ProxySetHeaders:        proxySetHeaders,
 	}, nil
 }

internal/ingress/annotations/authreq/main_test.go

@@ -243,6 +243,71 @@ func TestCacheDurationAnnotations(t *testing.T) {
 	}
 }
 
+func TestKeepaliveAnnotations(t *testing.T) {
+	ing := buildIngress()
+
+	data := map[string]string{}
+	ing.SetAnnotations(data)
+
+	tests := []struct {
+		title                string
+		url                  string
+		keepaliveConnections string
+		keepaliveRequests    string
+		keepaliveTimeout     string
+		expectedConnections  int
+		expectedRequests     int
+		expectedTimeout      int
+	}{
+		{"all set", "http://goog.url", "5", "500", "50", 5, 500, 50},
+		{"no annotation", "http://goog.url", "", "", "", defaultKeepaliveConnections, defaultKeepaliveRequests, defaultKeepaliveTimeout},
+		{"default for connections", "http://goog.url", "x", "500", "50", defaultKeepaliveConnections, 500, 50},
+		{"default for requests", "http://goog.url", "5", "x", "50", 5, defaultKeepaliveRequests, 50},
+		{"default for invalid timeout", "http://goog.url", "5", "500", "x", 5, 500, defaultKeepaliveTimeout},
+		{"variable in host", "http://$host:5000/a/b", "5", "", "", 0, defaultKeepaliveRequests, defaultKeepaliveTimeout},
+		{"variable in path", "http://goog.url:5000/$path", "5", "", "", 5, defaultKeepaliveRequests, defaultKeepaliveTimeout},
+		{"negative connections", "http://goog.url", "-2", "", "", 0, defaultKeepaliveRequests, defaultKeepaliveTimeout},
+		{"negative requests", "http://goog.url", "5", "-1", "", 0, -1, defaultKeepaliveTimeout},
+		{"negative timeout", "http://goog.url", "5", "", "-1", 0, defaultKeepaliveRequests, -1},
+		{"negative request and timeout", "http://goog.url", "5", "-2", "-3", 0, -2, -3},
+	}
+
+	for _, test := range tests {
+		data[parser.GetAnnotationWithPrefix("auth-url")] = test.url
+		data[parser.GetAnnotationWithPrefix("auth-keepalive")] = test.keepaliveConnections
+		data[parser.GetAnnotationWithPrefix("auth-keepalive-timeout")] = test.keepaliveTimeout
+		data[parser.GetAnnotationWithPrefix("auth-keepalive-requests")] = test.keepaliveRequests
+
+		i, err := NewParser(&resolver.Mock{}).Parse(ing)
+		if err != nil {
+			t.Errorf("%v: unexpected error: %v", test.title, err)
+			continue
+		}
+
+		u, ok := i.(*Config)
+		if !ok {
+			t.Errorf("%v: expected an External type", test.title)
+			continue
+		}
+
+		if u.URL != test.url {
+			t.Errorf("%v: expected \"%v\" but \"%v\" was returned", test.title, test.url, u.URL)
+		}
+
+		if u.KeepaliveConnections != test.expectedConnections {
+			t.Errorf("%v: expected \"%v\" but \"%v\" was returned", test.title, test.expectedConnections, u.KeepaliveConnections)
+		}
+
+		if u.KeepaliveRequests != test.expectedRequests {
+			t.Errorf("%v: expected \"%v\" but \"%v\" was returned", test.title, test.expectedRequests, u.KeepaliveRequests)
+		}
+
+		if u.KeepaliveTimeout != test.expectedTimeout {
+			t.Errorf("%v: expected \"%v\" but \"%v\" was returned", test.title, test.expectedTimeout, u.KeepaliveTimeout)
+		}
+	}
+}
+
 func TestParseStringToCacheDurations(t *testing.T) {
 
 	tests := []struct {