DevFW-CICD/ingress-nginx-helm
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: OpenTelemetry module integration (#9062)

Browse source 
* OpenTelemetry module integration

* e2e test

* e2e test fix

* default OpentelemetryConfig

* e2e values

* mount otel module for otel test only

* propagate IS_CHROOT

* propagate IS_CHROOT e2e test

* code doc

* comments

* golint

* opentelemetry doc

* zipkin

* zipkin

* typo

* update e2e test OpenTelemetry value

* use opentelemetry value

* revert merge conflict

* fix

* format

* review comments

* clean
This commit is contained in:
Ehsan Saei 2023-03-22 19:58:22 +01:00 • committed by GitHub
parent c075793ae5
commit c8cb9167d3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 1131 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

96
internal/ingress/controller/template/template.go
View file

@ -265,6 +265,7 @@ var (
"buildAuthSignURL": buildAuthSignURL,
"buildAuthSignURLLocation": buildAuthSignURLLocation,
"buildOpentracing": buildOpentracing,
"buildOpentelemetry": buildOpentelemetry,
"proxySetHeader": proxySetHeader,
"buildInfluxDB": buildInfluxDB,
"enforceRegexModifier": enforceRegexModifier,
@ -274,7 +275,9 @@ var (
"buildHTTPListener": buildHTTPListener,
"buildHTTPSListener": buildHTTPSListener,
"buildOpentracingForLocation": buildOpentracingForLocation,
"buildOpentelemetryForLocation": buildOpentelemetryForLocation,
"shouldLoadOpentracingModule": shouldLoadOpentracingModule,
"shouldLoadOpentelemetryModule": shouldLoadOpentelemetryModule,
"buildModSecurityForLocation": buildModSecurityForLocation,
"buildMirrorLocations": buildMirrorLocations,
"shouldLoadAuthDigestModule": shouldLoadAuthDigestModule,
@ -1239,6 +1242,33 @@ func buildOpentracing(c interface{}, s interface{}) string {
return buf.String()
}
func buildOpentelemetry(c interface{}, s interface{}) string {
cfg, ok := c.(config.Configuration)
if !ok {
klog.Errorf("expected a 'config.Configuration' type but %T was returned", c)
return ""
}
servers, ok := s.([]*ingress.Server)
if !ok {
klog.Errorf("expected an '[]*ingress.Server' type but %T was returned", s)
return ""
}
if !shouldLoadOpentelemetryModule(cfg, servers) {
return ""
}
buf := bytes.NewBufferString("")
buf.WriteString("\r\n")
if cfg.OpentelemetryOperationName != "" {
buf.WriteString(fmt.Sprintf("opentelemetry_operation_name \"%s\";\n", cfg.OpentelemetryOperationName))
}
return buf.String()
}
// buildInfluxDB produces the single line configuration
// needed by the InfluxDB module to send request's metrics
// for the current resource
@ -1360,6 +1390,13 @@ func opentracingPropagateContext(location *ingress.Location) string {
return "opentracing_propagate_context;"
}
func opentelemetryPropagateContext(location *ingress.Location) string {
if location == nil {
return ""
}
return "opentelemetry_propagate;"
}
// shouldLoadModSecurityModule determines whether or not the ModSecurity module needs to be loaded.
// First, it checks if `enable-modsecurity` is set in the ConfigMap. If it is not, it iterates over all locations to
// check if ModSecurity is enabled by the annotation `nginx.ingress.kubernetes.io/enable-modsecurity`.
@ -1575,6 +1612,36 @@ func buildOpentracingForLocation(isOTEnabled bool, isOTTrustSet bool, location *
return opc
}
func buildOpentelemetryForLocation(isOTEnabled bool, isOTTrustSet bool, location *ingress.Location) string {
isOTEnabledInLoc := location.Opentelemetry.Enabled
isOTSetInLoc := location.Opentelemetry.Set
if isOTEnabled {
if isOTSetInLoc && !isOTEnabledInLoc {
return "opentelemetry off;"
}
} else if !isOTSetInLoc || !isOTEnabledInLoc {
return ""
}
opc := opentelemetryPropagateContext(location)
if opc != "" {
opc = fmt.Sprintf("opentelemetry on;\n%v", opc)
}
if location.Opentelemetry.OperationName != "" {
opc = opc + "\nopentelemetry_operation_name " + location.Opentelemetry.OperationName + ";"
}
if (!isOTTrustSet && !location.Opentelemetry.TrustSet) ||
(location.Opentelemetry.TrustSet && !location.Opentelemetry.TrustEnabled) {
opc = opc + "\nopentelemetry_trust_incoming_spans off;"
} else {
opc = opc + "\nopentelemetry_trust_incoming_spans on;"
}
return opc
}
// shouldLoadOpentracingModule determines whether or not the Opentracing module needs to be loaded.
// First, it checks if `enable-opentracing` is set in the ConfigMap. If it is not, it iterates over all locations to
// check if Opentracing is enabled by the annotation `nginx.ingress.kubernetes.io/enable-opentracing`.
@ -1606,6 +1673,35 @@ func shouldLoadOpentracingModule(c interface{}, s interface{}) bool {
return false
}
// shouldLoadOpentelemetryModule determines whether or not the Opentelemetry module needs to be loaded.
// It checks if `enable-opentelemetry` is set in the ConfigMap.
func shouldLoadOpentelemetryModule(c interface{}, s interface{}) bool {
cfg, ok := c.(config.Configuration)
if !ok {
klog.Errorf("expected a 'config.Configuration' type but %T was returned", c)
return false
}
servers, ok := s.([]*ingress.Server)
if !ok {
klog.Errorf("expected an '[]*ingress.Server' type but %T was returned", s)
return false
}
if cfg.EnableOpentelemetry {
return true
}
for _, server := range servers {
for _, location := range server.Locations {
if location.Opentelemetry.Enabled {
return true
}
}
}
return false
}
func buildModSecurityForLocation(cfg config.Configuration, location *ingress.Location) string {
isMSEnabledInLoc := location.ModSecurity.Enable
isMSEnableSetInLoc := location.ModSecurity.EnableSet

153
internal/ingress/controller/template/template_test.go
View file

@ -37,6 +37,7 @@ import (
"k8s.io/ingress-nginx/internal/ingress/annotations/authreq"
"k8s.io/ingress-nginx/internal/ingress/annotations/influxdb"
"k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity"
"k8s.io/ingress-nginx/internal/ingress/annotations/opentelemetry"
"k8s.io/ingress-nginx/internal/ingress/annotations/opentracing"
"k8s.io/ingress-nginx/internal/ingress/annotations/ratelimit"
"k8s.io/ingress-nginx/internal/ingress/annotations/rewrite"
@ -1150,6 +1151,26 @@ func TestOpentracingPropagateContext(t *testing.T) {
}
}
func TestOpentelemetryPropagateContext(t *testing.T) {
tests := map[*ingress.Location]string{
{BackendProtocol: "HTTP"}: "opentelemetry_propagate;",
{BackendProtocol: "HTTPS"}: "opentelemetry_propagate;",
{BackendProtocol: "AUTO_HTTP"}: "opentelemetry_propagate;",
{BackendProtocol: "GRPC"}: "opentelemetry_propagate;",
{BackendProtocol: "GRPCS"}: "opentelemetry_propagate;",
{BackendProtocol: "AJP"}: "opentelemetry_propagate;",
{BackendProtocol: "FCGI"}: "opentelemetry_propagate;",
nil: "",
}
for loc, expectedDirective := range tests {
actualDirective := opentelemetryPropagateContext(loc)
if actualDirective != expectedDirective {
t.Errorf("Expected %v but returned %v", expectedDirective, actualDirective)
}
}
}
func TestGetIngressInformation(t *testing.T) {
testcases := map[string]struct {
@ -1723,6 +1744,37 @@ func TestBuildOpenTracing(t *testing.T) {
}
func TestBuildOpenTelemetry(t *testing.T) {
invalidType := &ingress.Ingress{}
expected := ""
actual := buildOpentelemetry(invalidType, []*ingress.Server{})
if expected != actual {
t.Errorf("Expected '%v' but returned '%v'", expected, actual)
}
cfgNoHost := config.Configuration{
EnableOpentelemetry: true,
}
expected = "\r\n"
actual = buildOpentelemetry(cfgNoHost, []*ingress.Server{})
if expected != actual {
t.Errorf("Expected '%v' but returned '%v'", expected, actual)
}
cfgOpenTelemetry := config.Configuration{
EnableOpentelemetry: true,
OpentelemetryOperationName: "my-operation-name",
}
expected = "\r\n"
expected += "opentelemetry_operation_name \"my-operation-name\";\n"
actual = buildOpentelemetry(cfgOpenTelemetry, []*ingress.Server{})
if expected != actual {
t.Errorf("Expected '%v' but returned '%v'", expected, actual)
}
}
func TestEnforceRegexModifier(t *testing.T) {
invalidType := &ingress.Ingress{}
expected := false
@ -1903,6 +1955,107 @@ func TestShouldLoadOpentracingModule(t *testing.T) {
}
}
func TestOpentelemetryForLocation(t *testing.T) {
trueVal := true
falseVal := false
loadOT := `opentelemetry on;
opentelemetry_propagate;
opentelemetry_trust_incoming_spans on;`
loadOTUntrustedSpan := `opentelemetry on;
opentelemetry_propagate;
opentelemetry_trust_incoming_spans off;`
testCases := []struct {
description string
globalOT bool
isSetInLoc bool
isOTInLoc *bool
globalTrust bool
isTrustSetInLoc bool
isTrustInLoc *bool
expected string
}{
{"globally enabled, without annotation", true, false, nil, true, false, nil, loadOT},
{"globally enabled and enabled in location", true, true, &trueVal, true, false, nil, loadOT},
{"globally disabled and not enabled in location", false, false, nil, true, false, nil, ""},
{"globally disabled but enabled in location", false, true, &trueVal, true, false, nil, loadOT},
{"globally trusted, not trusted in location", true, false, nil, true, true, &falseVal, loadOTUntrustedSpan},
{"not globally trusted, trust set in location", true, false, nil, false, true, &trueVal, loadOT},
{"not globally trusted, trust not set in location", true, false, nil, false, false, nil, loadOTUntrustedSpan},
}
for _, testCase := range testCases {
il := &ingress.Location{
Opentelemetry: opentelemetry.Config{Set: testCase.isSetInLoc, TrustSet: testCase.isTrustSetInLoc},
}
if il.Opentelemetry.Set {
il.Opentelemetry.Enabled = *testCase.isOTInLoc
}
if il.Opentelemetry.TrustSet {
il.Opentelemetry.TrustEnabled = *testCase.isTrustInLoc
}
actual := buildOpentelemetryForLocation(testCase.globalOT, testCase.globalTrust, il)
if testCase.expected != actual {
t.Errorf("%v: expected '%v' but returned '%v'", testCase.description, testCase.expected, actual)
}
}
}
func TestShouldLoadOpentelemetryModule(t *testing.T) {
// ### Invalid argument type tests ###
// The first tests should return false.
expected := false
invalidType := &ingress.Ingress{}
actual := shouldLoadOpentelemetryModule(config.Configuration{}, invalidType)
if expected != actual {
t.Errorf("Expected '%v' but returned '%v'", expected, actual)
}
actual = shouldLoadOpentelemetryModule(invalidType, []*ingress.Server{})
if expected != actual {
t.Errorf("Expected '%v' but returned '%v'", expected, actual)
}
// ### Functional tests ###
actual = shouldLoadOpentelemetryModule(config.Configuration{}, []*ingress.Server{})
if expected != actual {
t.Errorf("Expected '%v' but returned '%v'", expected, actual)
}
// All further tests should return true.
expected = true
configuration := config.Configuration{EnableOpentelemetry: true}
actual = shouldLoadOpentelemetryModule(configuration, []*ingress.Server{})
if expected != actual {
t.Errorf("Expected '%v' but returned '%v'", expected, actual)
}
servers := []*ingress.Server{
{
Locations: []*ingress.Location{
{
Opentelemetry: opentelemetry.Config{
Enabled: true,
},
},
},
},
}
actual = shouldLoadOpentelemetryModule(config.Configuration{}, servers)
if expected != actual {
t.Errorf("Expected '%v' but returned '%v'", expected, actual)
}
actual = shouldLoadOpentelemetryModule(configuration, servers)
if expected != actual {
t.Errorf("Expected '%v' but returned '%v'", expected, actual)
}
}
func TestModSecurityForLocation(t *testing.T) {
loadModule := `modsecurity on;
`