DevFW-CICD/ingress-nginx-helm
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: support topology aware hints (#9165)

Browse source 
* support topology aware hints

Signed-off-by: tombokombo <tombo@sysart.tech>

* add flag to enable topology and fixes

Signed-off-by: tombokombo <tombo@sysart.tech>

* update readme

Signed-off-by: tombokombo <tombo@sysart.tech>

* add e2e test

Signed-off-by: tombokombo <tombo@sysart.tech>

* isolate topology test

Signed-off-by: tombokombo <tombo@sysart.tech>

* gofmt fix

Signed-off-by: tombokombo <tombo@sysart.tech>

Signed-off-by: tombokombo <tombo@sysart.tech>
This commit is contained in:
Tomas Hulata 2023-01-16 03:46:50 +01:00 • committed by GitHub
parent ada114315e
commit 5b2a9475dc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 564 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

55
internal/ingress/controller/controller.go
View file

@ -53,6 +53,7 @@ const (
defUpstreamName = "upstream-default-backend"
defServerName = "_"
rootLocation = "/"
emptyZone = ""
)
// Configuration contains all the settings required by an Ingress controller
@ -131,6 +132,21 @@ type Configuration struct {
DynamicConfigurationRetries int
DisableSyncEvents bool
EnableTopologyAwareRouting bool
}
func getIngressPodZone(svc *apiv1.Service) string {
svcKey := k8s.MetaNamespaceKey(svc)
if svcZoneAnnotation, ok := svc.ObjectMeta.GetAnnotations()[apiv1.AnnotationTopologyAwareHints]; ok {
if strings.ToLower(svcZoneAnnotation) == "auto" {
if foundZone, ok := k8s.IngressNodeDetails.GetLabels()[apiv1.LabelTopologyZone]; ok {
klog.V(3).Infof("Svc has topology aware annotation enabled, try to use zone %q where controller pod is running for Service %q ", foundZone, svcKey)
return foundZone
}
}
}
return emptyZone
}
// GetPublishService returns the Service used to set the load-balancer status of Ingresses.
@ -429,6 +445,13 @@ func (n *NGINXController) getStreamServices(configmapName string, proto apiv1.Pr
var endps []ingress.Endpoint
/* #nosec */
targetPort, err := strconv.Atoi(svcPort) // #nosec
var zone string
if n.cfg.EnableTopologyAwareRouting {
zone = getIngressPodZone(svc)
} else {
zone = emptyZone
}
if err != nil {
// not a port number, fall back to using port name
klog.V(3).Infof("Searching Endpoints with %v port name %q for Service %q", proto, svcPort, nsName)
@ -436,7 +459,7 @@ func (n *NGINXController) getStreamServices(configmapName string, proto apiv1.Pr
sp := svc.Spec.Ports[i]
if sp.Name == svcPort {
if sp.Protocol == proto {
endps = getEndpointsFromSlices(svc, &sp, proto, n.store.GetServiceEndpointsSlices)
endps = getEndpointsFromSlices(svc, &sp, proto, zone, n.store.GetServiceEndpointsSlices)
break
}
}
@ -447,7 +470,7 @@ func (n *NGINXController) getStreamServices(configmapName string, proto apiv1.Pr
sp := svc.Spec.Ports[i]
if sp.Port == int32(targetPort) {
if sp.Protocol == proto {
endps = getEndpointsFromSlices(svc, &sp, proto, n.store.GetServiceEndpointsSlices)
endps = getEndpointsFromSlices(svc, &sp, proto, zone, n.store.GetServiceEndpointsSlices)
break
}
}
@ -498,8 +521,13 @@ func (n *NGINXController) getDefaultUpstream() *ingress.Backend {
upstream.Endpoints = append(upstream.Endpoints, n.DefaultEndpoint())
return upstream
}
endps := getEndpointsFromSlices(svc, &svc.Spec.Ports[0], apiv1.ProtocolTCP, n.store.GetServiceEndpointsSlices)
var zone string
if n.cfg.EnableTopologyAwareRouting {
zone = getIngressPodZone(svc)
} else {
zone = emptyZone
}
endps := getEndpointsFromSlices(svc, &svc.Spec.Ports[0], apiv1.ProtocolTCP, zone, n.store.GetServiceEndpointsSlices)
if len(endps) == 0 {
klog.Warningf("Service %q does not have any active Endpoint", svcKey)
endps = []ingress.Endpoint{n.DefaultEndpoint()}
@ -827,7 +855,13 @@ func (n *NGINXController) getBackendServers(ingresses []*ingress.Ingress) ([]*in
}
sp := location.DefaultBackend.Spec.Ports[0]
endps := getEndpointsFromSlices(location.DefaultBackend, &sp, apiv1.ProtocolTCP, n.store.GetServiceEndpointsSlices)
var zone string
if n.cfg.EnableTopologyAwareRouting {
zone = getIngressPodZone(location.DefaultBackend)
} else {
zone = emptyZone
}
endps := getEndpointsFromSlices(location.DefaultBackend, &sp, apiv1.ProtocolTCP, zone, n.store.GetServiceEndpointsSlices)
// custom backend is valid only if contains at least one endpoint
if len(endps) > 0 {
name := fmt.Sprintf("custom-default-backend-%v-%v", location.DefaultBackend.GetNamespace(), location.DefaultBackend.GetName())
@ -1083,7 +1117,12 @@ func (n *NGINXController) serviceEndpoints(svcKey, backendPort string) ([]ingres
if err != nil {
return upstreams, err
}
var zone string
if n.cfg.EnableTopologyAwareRouting {
zone = getIngressPodZone(svc)
} else {
zone = emptyZone
}
klog.V(3).Infof("Obtaining ports information for Service %q", svcKey)
// Ingress with an ExternalName Service and no port defined for that Service
if svc.Spec.Type == apiv1.ServiceTypeExternalName {
@ -1092,7 +1131,7 @@ func (n *NGINXController) serviceEndpoints(svcKey, backendPort string) ([]ingres
return upstreams, nil
}
servicePort := externalNamePorts(backendPort, svc)
endps := getEndpointsFromSlices(svc, servicePort, apiv1.ProtocolTCP, n.store.GetServiceEndpointsSlices)
endps := getEndpointsFromSlices(svc, servicePort, apiv1.ProtocolTCP, zone, n.store.GetServiceEndpointsSlices)
if len(endps) == 0 {
klog.Warningf("Service %q does not have any active Endpoint.", svcKey)
return upstreams, nil
@ -1109,7 +1148,7 @@ func (n *NGINXController) serviceEndpoints(svcKey, backendPort string) ([]ingres
servicePort.TargetPort.String() == backendPort ||
servicePort.Name == backendPort {
endps := getEndpointsFromSlices(svc, &servicePort, apiv1.ProtocolTCP, n.store.GetServiceEndpointsSlices)
endps := getEndpointsFromSlices(svc, &servicePort, apiv1.ProtocolTCP, zone, n.store.GetServiceEndpointsSlices)
if len(endps) == 0 {
klog.Warningf("Service %q does not have any active Endpoint.", svcKey)
}

31
internal/ingress/controller/endpointslices.go
View file

@ -35,7 +35,7 @@ import (
)
// getEndpoints returns a list of Endpoint structs for a given service/target port combination.
func getEndpointsFromSlices(s *corev1.Service, port *corev1.ServicePort, proto corev1.Protocol,
func getEndpointsFromSlices(s *corev1.Service, port *corev1.ServicePort, proto corev1.Protocol, zoneForHints string,
getServiceEndpointsSlices func(string) ([]*discoveryv1.EndpointSlice, error)) []ingress.Endpoint {
upsServers := []ingress.Endpoint{}
@ -49,6 +49,7 @@ func getEndpointsFromSlices(s *corev1.Service, port *corev1.ServicePort, proto c
processedUpstreamServers := make(map[string]struct{})
svcKey := k8s.MetaNamespaceKey(s)
var useTopologyHints bool
// ExternalName services
if s.Spec.Type == corev1.ServiceTypeExternalName {
@ -111,12 +112,38 @@ func getEndpointsFromSlices(s *corev1.Service, port *corev1.ServicePort, proto c
ports = append(ports, targetPort)
}
}
useTopologyHints = false
if zoneForHints != emptyZone {
useTopologyHints = true
// check if all endpointslices has zone hints
for _, ep := range eps.Endpoints {
if ep.Hints == nil || len(ep.Hints.ForZones) == 0 {
useTopologyHints = false
break
}
}
if useTopologyHints {
klog.V(3).Infof("All endpoint slices has zone hint, using zone %q for Service %q", zoneForHints, svcKey)
}
}
for _, ep := range eps.Endpoints {
if !(*ep.Conditions.Ready) {
continue
}
epHasZone := false
if useTopologyHints {
for _, epzone := range ep.Hints.ForZones {
if epzone.Name == zoneForHints {
epHasZone = true
break
}
}
}
// ep.Hints
if useTopologyHints && !epHasZone {
continue
}
for _, epPort := range ports {
for _, epAddress := range ep.Addresses {

261
internal/ingress/controller/endpointslices_test.go
View file

@ -33,6 +33,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
svc *corev1.Service
port *corev1.ServicePort
proto corev1.Protocol
zone string
fn func(string) ([]*discoveryv1.EndpointSlice, error)
result []ingress.Endpoint
}{
@ -41,6 +42,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
nil,
nil,
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return nil, nil
},
@ -51,6 +53,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
&corev1.Service{},
nil,
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return nil, nil
},
@ -61,6 +64,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
&corev1.Service{},
&corev1.ServicePort{Name: "default"},
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{}, nil
},
@ -75,6 +79,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
},
&corev1.ServicePort{Name: "default"},
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{}, nil
},
@ -99,6 +104,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
TargetPort: intstr.FromInt(80),
},
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{}, nil
},
@ -123,6 +129,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
TargetPort: intstr.FromInt(80),
},
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{}, nil
},
@ -147,6 +154,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
TargetPort: intstr.FromInt(80),
},
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{}, nil
},
@ -176,6 +184,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
TargetPort: intstr.FromInt(80),
},
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{}, nil
},
@ -205,6 +214,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
TargetPort: intstr.FromInt(80),
},
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{}, nil
},
@ -229,6 +239,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
TargetPort: intstr.FromInt(80),
},
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return nil, fmt.Errorf("unexpected error")
},
@ -253,6 +264,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
TargetPort: intstr.FromInt(80),
},
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{{
ObjectMeta: metav1.ObjectMeta{
@ -296,6 +308,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
TargetPort: intstr.FromInt(80),
},
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{{
ObjectMeta: metav1.ObjectMeta{
@ -339,6 +352,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
TargetPort: intstr.FromString("port-1"),
},
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{{
ObjectMeta: metav1.ObjectMeta{
@ -382,6 +396,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
TargetPort: intstr.FromInt(80),
},
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{{
ObjectMeta: metav1.ObjectMeta{
@ -430,6 +445,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
TargetPort: intstr.FromInt(80),
},
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{{
ObjectMeta: metav1.ObjectMeta{
@ -478,6 +494,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
TargetPort: intstr.FromString("port-1"),
},
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{
{
@ -552,6 +569,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
TargetPort: intstr.FromString("port-1"),
},
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{
{
@ -622,6 +640,7 @@ func TestGetEndpointsFromSlices(t *testing.T) {
TargetPort: intstr.FromString("port-1"),
},
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{{
ObjectMeta: metav1.ObjectMeta{
@ -656,11 +675,251 @@ func TestGetEndpointsFromSlices(t *testing.T) {
},
},
},
{
"should return one endpoint which belongs to zone",
&corev1.Service{
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
ClusterIP: "1.1.1.1",
Ports: []corev1.ServicePort{
{
Name: "default",
TargetPort: intstr.FromString("port-1"),
},
},
},
},
&corev1.ServicePort{
Name: "port-1",
TargetPort: intstr.FromString("port-1"),
},
corev1.ProtocolTCP,
"eu-west-1b",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{discoveryv1.LabelServiceName: "default"},
},
Endpoints: []discoveryv1.Endpoint{
{
Addresses: []string{"1.1.1.1"},
Conditions: discoveryv1.EndpointConditions{
Ready: &[]bool{true}[0],
},
Hints: &[]discoveryv1.EndpointHints{{
ForZones: []discoveryv1.ForZone{{
Name: "eu-west-1b",
}},
}}[0],
},
{
Addresses: []string{"1.1.1.2"},
Conditions: discoveryv1.EndpointConditions{
Ready: &[]bool{true}[0],
},
Hints: &[]discoveryv1.EndpointHints{{
ForZones: []discoveryv1.ForZone{{
Name: "eu-west-1a",
}},
}}[0],
},
{
Addresses: []string{"1.1.1.3"},
Conditions: discoveryv1.EndpointConditions{
Ready: &[]bool{true}[0],
},
Hints: &[]discoveryv1.EndpointHints{{
ForZones: []discoveryv1.ForZone{{
Name: "eu-west-1c",
}},
}}[0],
},
},
Ports: []discoveryv1.EndpointPort{
{
Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0],
Port: &[]int32{80}[0],
Name: &[]string{"port-1"}[0],
},
},
}}, nil
},
[]ingress.Endpoint{
{
Address: "1.1.1.1",
Port: "80",
},
},
},
{
"should return all endpoints because one is missing zone hint",
&corev1.Service{
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
ClusterIP: "1.1.1.1",
Ports: []corev1.ServicePort{
{
Name: "default",
TargetPort: intstr.FromString("port-1"),
},
},
},
},
&corev1.ServicePort{
Name: "port-1",
TargetPort: intstr.FromString("port-1"),
},
corev1.ProtocolTCP,
"eu-west-1b",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{discoveryv1.LabelServiceName: "default"},
},
Endpoints: []discoveryv1.Endpoint{
{
Addresses: []string{"1.1.1.1"},
Conditions: discoveryv1.EndpointConditions{
Ready: &[]bool{true}[0],
},
Hints: &[]discoveryv1.EndpointHints{{
ForZones: []discoveryv1.ForZone{{
Name: "eu-west-1b",
}},
}}[0],
},
{
Addresses: []string{"1.1.1.2"},
Conditions: discoveryv1.EndpointConditions{
Ready: &[]bool{true}[0],
},
Hints: &[]discoveryv1.EndpointHints{{
ForZones: []discoveryv1.ForZone{{
Name: "eu-west-1b",
}},
}}[0],
},
{
Addresses: []string{"1.1.1.3"},
Conditions: discoveryv1.EndpointConditions{
Ready: &[]bool{true}[0],
},
Hints: &[]discoveryv1.EndpointHints{{}}[0],
},
},
Ports: []discoveryv1.EndpointPort{
{
Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0],
Port: &[]int32{80}[0],
Name: &[]string{"port-1"}[0],
},
},
}}, nil
},
[]ingress.Endpoint{
{
Address: "1.1.1.1",
Port: "80",
},
{
Address: "1.1.1.2",
Port: "80",
},
{
Address: "1.1.1.3",
Port: "80",
},
},
},
{
"should return all endpoints because no zone from controller node",
&corev1.Service{
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
ClusterIP: "1.1.1.1",
Ports: []corev1.ServicePort{
{
Name: "default",
TargetPort: intstr.FromString("port-1"),
},
},
},
},
&corev1.ServicePort{
Name: "port-1",
TargetPort: intstr.FromString("port-1"),
},
corev1.ProtocolTCP,
"",
func(string) ([]*discoveryv1.EndpointSlice, error) {
return []*discoveryv1.EndpointSlice{{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{discoveryv1.LabelServiceName: "default"},
},
Endpoints: []discoveryv1.Endpoint{
{
Addresses: []string{"1.1.1.1"},
Conditions: discoveryv1.EndpointConditions{
Ready: &[]bool{true}[0],
},
Hints: &[]discoveryv1.EndpointHints{{
ForZones: []discoveryv1.ForZone{{
Name: "eu-west-1a",
}},
}}[0],
},
{
Addresses: []string{"1.1.1.2"},
Conditions: discoveryv1.EndpointConditions{
Ready: &[]bool{true}[0],
},
Hints: &[]discoveryv1.EndpointHints{{
ForZones: []discoveryv1.ForZone{{
Name: "eu-west-1b",
}},
}}[0],
},
{
Addresses: []string{"1.1.1.3"},
Conditions: discoveryv1.EndpointConditions{
Ready: &[]bool{true}[0],
},
Hints: &[]discoveryv1.EndpointHints{{
ForZones: []discoveryv1.ForZone{{
Name: "eu-west-1c",
}},
}}[0],
},
},
Ports: []discoveryv1.EndpointPort{
{
Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0],
Port: &[]int32{80}[0],
Name: &[]string{"port-1"}[0],
},
},
}}, nil
},
[]ingress.Endpoint{
{
Address: "1.1.1.1",
Port: "80",
},
{
Address: "1.1.1.2",
Port: "80",
},
{
Address: "1.1.1.3",
Port: "80",
},
},
},
}
for _, testCase := range tests {
t.Run(testCase.name, func(t *testing.T) {
result := getEndpointsFromSlices(testCase.svc, testCase.port, testCase.proto, testCase.fn)
result := getEndpointsFromSlices(testCase.svc, testCase.port, testCase.proto, testCase.zone, testCase.fn)
if len(testCase.result) != len(result) {
t.Errorf("Expected %d Endpoints but got %d", len(testCase.result), len(result))
}

20
internal/k8s/main.go
View file

@ -78,6 +78,8 @@ func GetNodeIPOrName(kubeClient clientset.Interface, name string, useInternalIP
var (
// IngressPodDetails hold information about the ingress-nginx pod
IngressPodDetails *PodInfo
// IngressNodeDetails old information about the node running ingress-nginx pod
IngressNodeDetails *NodeInfo
)
// PodInfo contains runtime information about the pod running the Ingres controller
@ -87,6 +89,12 @@ type PodInfo struct {
metav1.ObjectMeta
}
// NodeInfo contains runtime information about the node pod running the Ingres controller, eg. zone where pod is running
type NodeInfo struct {
metav1.TypeMeta
metav1.ObjectMeta
}
// GetIngressPod load the ingress-nginx pod
func GetIngressPod(kubeClient clientset.Interface) error {
podName := os.Getenv("POD_NAME")
@ -108,6 +116,18 @@ func GetIngressPod(kubeClient clientset.Interface) error {
pod.ObjectMeta.DeepCopyInto(&IngressPodDetails.ObjectMeta)
IngressPodDetails.SetLabels(pod.GetLabels())
IngressNodeDetails = &NodeInfo{
TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Node"},
}
// Try to get node info/labels to determine topology zone where pod is running
node, err := kubeClient.CoreV1().Nodes().Get(context.TODO(), pod.Spec.NodeName, metav1.GetOptions{})
if err != nil {
klog.Warningf("Unable to get NODE information: %v", err)
} else {
node.ObjectMeta.DeepCopyInto(&IngressNodeDetails.ObjectMeta)
IngressNodeDetails.SetLabels(node.GetLabels())
}
return nil
}