feat: OpenTelemetry module integration (#9062)

* OpenTelemetry module integration

* e2e test

* e2e test fix

* default OpentelemetryConfig

* e2e values

* mount otel module for otel test only

* propagate IS_CHROOT

* propagate IS_CHROOT e2e test

* code doc

* comments

* golint

* opentelemetry doc

* zipkin

* zipkin

* typo

* update e2e test OpenTelemetry value

* use opentelemetry value

* revert merge conflict

* fix

* format

* review comments

* clean

internal/ingress/controller/config/config.go

@@ -568,6 +568,54 @@ type Configuration struct {
 	// Default: true
 	OpentracingTrustIncomingSpan bool `json:"opentracing-trust-incoming-span"`
 
+	// EnableOpentelemetry enables the nginx Opentelemetry extension
+	// By default this is disabled
+	EnableOpentelemetry bool `json:"enable-opentelemetry"`
+
+	// OpentelemetryConfig sets the opentelemetry config file
+	// Default: /etc/nginx/opentelemetry.toml
+	OpentelemetryConfig string `json:"opentelemetry-config"`
+
+	// OpentelemetryOperationName specifies a custom name for the server span
+	OpentelemetryOperationName string `json:"opentelemetry-operation-name"`
+
+	// OpentelemetryTrustIncomingSpan sets whether or not to trust incoming trace spans
+	// If false, incoming span headers will be rejected
+	// Default: true
+	OpentelemetryTrustIncomingSpan bool `json:"opentelemetry-trust-incoming-span"`
+
+	// OtlpCollectorHost specifies the host to use when uploading traces
+	OtlpCollectorHost string `json:"otlp-collector-host"`
+
+	// OtlpCollectorPort specifies the port to use when uploading traces
+	// Default: 4317
+	OtlpCollectorPort string `json:"otlp-collector-port"`
+
+	// OtelServiceName specifies the service name to use for any traces created
+	// Default: nginx
+	OtelServiceName string `json:"otel-service-name"`
+
+	// OtelSampler specifies the sampler to use for any traces created
+	// Default: AlwaysOff
+	OtelSampler string `json:"otel-sampler"`
+
+	// OtelSamplerRatio specifies the sampler ratio to use for any traces created
+	// Default: 0.01
+	OtelSamplerRatio float32 `json:"otel-sampler-ratio"`
+
+	//OtelSamplerParentBased specifies the parent based sampler to be use for any traces created
+	// Default: false
+	OtelSamplerParentBased bool `json:"otel-sampler-parent-based"`
+
+	// MaxQueueSize specifies the max queue size for uploading traces
+	OtelMaxQueueSize int32 `json:"otel-max-queuesize"`
+
+	// ScheduleDelayMillis specifies the max delay between uploading traces
+	OtelScheduleDelayMillis int32 `json:"otel-schedule-delay-millis"`
+
+	// MaxExportBatchSize specifies the max export batch size to used when uploading traces
+	OtelMaxExportBatchSize int32 `json:"otel-max-export-batch-size"`
+
 	// ZipkinCollectorHost specifies the host to use when uploading traces
 	ZipkinCollectorHost string `json:"zipkin-collector-host"`
 
@@ -917,6 +965,13 @@ func NewDefault() Configuration {
 		BindAddressIpv4:                        defBindAddress,
 		BindAddressIpv6:                        defBindAddress,
 		OpentracingTrustIncomingSpan:           true,
+		OpentelemetryTrustIncomingSpan:         true,
+		OpentelemetryConfig:                    "/etc/nginx/opentelemetry.toml",
+		OtlpCollectorPort:                      "4317",
+		OtelServiceName:                        "nginx",
+		OtelSampler:                            "AlwaysOff",
+		OtelSamplerRatio:                       0.01,
+		OtelSamplerParentBased:                 false,
 		ZipkinCollectorPort:                    9411,
 		ZipkinServiceName:                      "nginx",
 		ZipkinSampleRate:                       1.0,

internal/ingress/controller/controller.go

@@ -1444,6 +1444,7 @@ func locationApplyAnnotations(loc *ingress.Location, anns *annotations.Ingress)
 	loc.EnableGlobalAuth = anns.EnableGlobalAuth
 	loc.HTTP2PushPreload = anns.HTTP2PushPreload
 	loc.Opentracing = anns.Opentracing
+	loc.Opentelemetry = anns.Opentelemetry
 	loc.Proxy = anns.Proxy
 	loc.ProxySSL = anns.ProxySSL
 	loc.RateLimit = anns.RateLimit

internal/ingress/controller/nginx.go

@@ -673,6 +673,11 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
 		return err
 	}
 
+	err = createOpentelemetryCfg(cfg)
+	if err != nil {
+		return err
+	}
+
 	err = n.testTemplate(content)
 	if err != nil {
 		return err
@@ -1056,6 +1061,29 @@ const datadogTmpl = `{
   "dd.priority.sampling": {{ .DatadogPrioritySampling }}
 }`
 
+const otelTmpl = `
+exporter = "otlp"
+processor = "batch"
+
+[exporters.otlp]
+# Alternatively the OTEL_EXPORTER_OTLP_ENDPOINT environment variable can also be used.
+host = "{{ .OtlpCollectorHost }}"
+port = {{ .OtlpCollectorPort }}
+
+[processors.batch]
+max_queue_size = {{ .OtelMaxQueueSize }}
+schedule_delay_millis = {{ .OtelScheduleDelayMillis }}
+max_export_batch_size = {{ .OtelMaxExportBatchSize }}
+
+[service]
+name = "{{ .OtelServiceName }}" # Opentelemetry resource name
+
+[sampler]
+name = "{{ .OtelSampler }}" # Also: AlwaysOff, TraceIdRatioBased
+ratio = {{ .OtelSamplerRatio }}
+parent_based = {{ .OtelSamplerParentBased }}
+`
+
 func createOpentracingCfg(cfg ngx_config.Configuration) error {
 	var tmpl *template.Template
 	var err error
@@ -1091,6 +1119,21 @@ func createOpentracingCfg(cfg ngx_config.Configuration) error {
 	return os.WriteFile("/etc/nginx/opentracing.json", []byte(expanded), file.ReadWriteByUser)
 }
 
+func createOpentelemetryCfg(cfg ngx_config.Configuration) error {
+
+	tmpl, err := template.New("otel").Parse(otelTmpl)
+	if err != nil {
+		return err
+	}
+	tmplBuf := bytes.NewBuffer(make([]byte, 0))
+	err = tmpl.Execute(tmplBuf, cfg)
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(cfg.OpentelemetryConfig, tmplBuf.Bytes(), file.ReadWriteByUser)
+}
+
 func cleanTempNginxCfg() error {
 	var files []string
 

internal/ingress/controller/template/template.go

@@ -265,6 +265,7 @@ var (
 		"buildAuthSignURL":                   buildAuthSignURL,
 		"buildAuthSignURLLocation":           buildAuthSignURLLocation,
 		"buildOpentracing":                   buildOpentracing,
+		"buildOpentelemetry":                 buildOpentelemetry,
 		"proxySetHeader":                     proxySetHeader,
 		"buildInfluxDB":                      buildInfluxDB,
 		"enforceRegexModifier":               enforceRegexModifier,
@@ -274,7 +275,9 @@ var (
 		"buildHTTPListener":                  buildHTTPListener,
 		"buildHTTPSListener":                 buildHTTPSListener,
 		"buildOpentracingForLocation":        buildOpentracingForLocation,
+		"buildOpentelemetryForLocation":      buildOpentelemetryForLocation,
 		"shouldLoadOpentracingModule":        shouldLoadOpentracingModule,
+		"shouldLoadOpentelemetryModule":      shouldLoadOpentelemetryModule,
 		"buildModSecurityForLocation":        buildModSecurityForLocation,
 		"buildMirrorLocations":               buildMirrorLocations,
 		"shouldLoadAuthDigestModule":         shouldLoadAuthDigestModule,
@@ -1239,6 +1242,33 @@ func buildOpentracing(c interface{}, s interface{}) string {
 	return buf.String()
 }
 
+func buildOpentelemetry(c interface{}, s interface{}) string {
+	cfg, ok := c.(config.Configuration)
+	if !ok {
+		klog.Errorf("expected a 'config.Configuration' type but %T was returned", c)
+		return ""
+	}
+
+	servers, ok := s.([]*ingress.Server)
+	if !ok {
+		klog.Errorf("expected an '[]*ingress.Server' type but %T was returned", s)
+		return ""
+	}
+
+	if !shouldLoadOpentelemetryModule(cfg, servers) {
+		return ""
+	}
+
+	buf := bytes.NewBufferString("")
+
+	buf.WriteString("\r\n")
+
+	if cfg.OpentelemetryOperationName != "" {
+		buf.WriteString(fmt.Sprintf("opentelemetry_operation_name \"%s\";\n", cfg.OpentelemetryOperationName))
+	}
+	return buf.String()
+}
+
 // buildInfluxDB produces the single line configuration
 // needed by the InfluxDB module to send request's metrics
 // for the current resource
@@ -1360,6 +1390,13 @@ func opentracingPropagateContext(location *ingress.Location) string {
 	return "opentracing_propagate_context;"
 }
 
+func opentelemetryPropagateContext(location *ingress.Location) string {
+	if location == nil {
+		return ""
+	}
+	return "opentelemetry_propagate;"
+}
+
 // shouldLoadModSecurityModule determines whether or not the ModSecurity module needs to be loaded.
 // First, it checks if `enable-modsecurity` is set in the ConfigMap. If it is not, it iterates over all locations to
 // check if ModSecurity is enabled by the annotation `nginx.ingress.kubernetes.io/enable-modsecurity`.
@@ -1575,6 +1612,36 @@ func buildOpentracingForLocation(isOTEnabled bool, isOTTrustSet bool, location *
 	return opc
 }
 
+func buildOpentelemetryForLocation(isOTEnabled bool, isOTTrustSet bool, location *ingress.Location) string {
+	isOTEnabledInLoc := location.Opentelemetry.Enabled
+	isOTSetInLoc := location.Opentelemetry.Set
+
+	if isOTEnabled {
+		if isOTSetInLoc && !isOTEnabledInLoc {
+			return "opentelemetry off;"
+		}
+	} else if !isOTSetInLoc || !isOTEnabledInLoc {
+		return ""
+	}
+
+	opc := opentelemetryPropagateContext(location)
+	if opc != "" {
+		opc = fmt.Sprintf("opentelemetry on;\n%v", opc)
+	}
+
+	if location.Opentelemetry.OperationName != "" {
+		opc = opc + "\nopentelemetry_operation_name " + location.Opentelemetry.OperationName + ";"
+	}
+
+	if (!isOTTrustSet && !location.Opentelemetry.TrustSet) ||
+		(location.Opentelemetry.TrustSet && !location.Opentelemetry.TrustEnabled) {
+		opc = opc + "\nopentelemetry_trust_incoming_spans off;"
+	} else {
+		opc = opc + "\nopentelemetry_trust_incoming_spans on;"
+	}
+	return opc
+}
+
 // shouldLoadOpentracingModule determines whether or not the Opentracing module needs to be loaded.
 // First, it checks if `enable-opentracing` is set in the ConfigMap. If it is not, it iterates over all locations to
 // check if Opentracing is enabled by the annotation `nginx.ingress.kubernetes.io/enable-opentracing`.
@@ -1606,6 +1673,35 @@ func shouldLoadOpentracingModule(c interface{}, s interface{}) bool {
 	return false
 }
 
+// shouldLoadOpentelemetryModule determines whether or not the Opentelemetry module needs to be loaded.
+// It checks if `enable-opentelemetry` is set in the ConfigMap.
+func shouldLoadOpentelemetryModule(c interface{}, s interface{}) bool {
+	cfg, ok := c.(config.Configuration)
+	if !ok {
+		klog.Errorf("expected a 'config.Configuration' type but %T was returned", c)
+		return false
+	}
+
+	servers, ok := s.([]*ingress.Server)
+	if !ok {
+		klog.Errorf("expected an '[]*ingress.Server' type but %T was returned", s)
+		return false
+	}
+
+	if cfg.EnableOpentelemetry {
+		return true
+	}
+
+	for _, server := range servers {
+		for _, location := range server.Locations {
+			if location.Opentelemetry.Enabled {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 func buildModSecurityForLocation(cfg config.Configuration, location *ingress.Location) string {
 	isMSEnabledInLoc := location.ModSecurity.Enable
 	isMSEnableSetInLoc := location.ModSecurity.EnableSet

internal/ingress/controller/template/template_test.go

@@ -37,6 +37,7 @@ import (
 	"k8s.io/ingress-nginx/internal/ingress/annotations/authreq"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/influxdb"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity"
+	"k8s.io/ingress-nginx/internal/ingress/annotations/opentelemetry"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/opentracing"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/ratelimit"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/rewrite"
@@ -1150,6 +1151,26 @@ func TestOpentracingPropagateContext(t *testing.T) {
 	}
 }
 
+func TestOpentelemetryPropagateContext(t *testing.T) {
+	tests := map[*ingress.Location]string{
+		{BackendProtocol: "HTTP"}:      "opentelemetry_propagate;",
+		{BackendProtocol: "HTTPS"}:     "opentelemetry_propagate;",
+		{BackendProtocol: "AUTO_HTTP"}: "opentelemetry_propagate;",
+		{BackendProtocol: "GRPC"}:      "opentelemetry_propagate;",
+		{BackendProtocol: "GRPCS"}:     "opentelemetry_propagate;",
+		{BackendProtocol: "AJP"}:       "opentelemetry_propagate;",
+		{BackendProtocol: "FCGI"}:      "opentelemetry_propagate;",
+		nil:                            "",
+	}
+
+	for loc, expectedDirective := range tests {
+		actualDirective := opentelemetryPropagateContext(loc)
+		if actualDirective != expectedDirective {
+			t.Errorf("Expected %v but returned %v", expectedDirective, actualDirective)
+		}
+	}
+}
+
 func TestGetIngressInformation(t *testing.T) {
 
 	testcases := map[string]struct {
@@ -1723,6 +1744,37 @@ func TestBuildOpenTracing(t *testing.T) {
 
 }
 
+func TestBuildOpenTelemetry(t *testing.T) {
+	invalidType := &ingress.Ingress{}
+	expected := ""
+	actual := buildOpentelemetry(invalidType, []*ingress.Server{})
+
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+
+	cfgNoHost := config.Configuration{
+		EnableOpentelemetry: true,
+	}
+	expected = "\r\n"
+	actual = buildOpentelemetry(cfgNoHost, []*ingress.Server{})
+
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+
+	cfgOpenTelemetry := config.Configuration{
+		EnableOpentelemetry:        true,
+		OpentelemetryOperationName: "my-operation-name",
+	}
+	expected = "\r\n"
+	expected += "opentelemetry_operation_name \"my-operation-name\";\n"
+	actual = buildOpentelemetry(cfgOpenTelemetry, []*ingress.Server{})
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+}
+
 func TestEnforceRegexModifier(t *testing.T) {
 	invalidType := &ingress.Ingress{}
 	expected := false
@@ -1903,6 +1955,107 @@ func TestShouldLoadOpentracingModule(t *testing.T) {
 	}
 }
 
+func TestOpentelemetryForLocation(t *testing.T) {
+	trueVal := true
+	falseVal := false
+
+	loadOT := `opentelemetry on;
+opentelemetry_propagate;
+opentelemetry_trust_incoming_spans on;`
+	loadOTUntrustedSpan := `opentelemetry on;
+opentelemetry_propagate;
+opentelemetry_trust_incoming_spans off;`
+	testCases := []struct {
+		description     string
+		globalOT        bool
+		isSetInLoc      bool
+		isOTInLoc       *bool
+		globalTrust     bool
+		isTrustSetInLoc bool
+		isTrustInLoc    *bool
+		expected        string
+	}{
+		{"globally enabled, without annotation", true, false, nil, true, false, nil, loadOT},
+		{"globally enabled and enabled in location", true, true, &trueVal, true, false, nil, loadOT},
+		{"globally disabled and not enabled in location", false, false, nil, true, false, nil, ""},
+		{"globally disabled but enabled in location", false, true, &trueVal, true, false, nil, loadOT},
+		{"globally trusted, not trusted in location", true, false, nil, true, true, &falseVal, loadOTUntrustedSpan},
+		{"not globally trusted, trust set in location", true, false, nil, false, true, &trueVal, loadOT},
+		{"not globally trusted, trust not set in location", true, false, nil, false, false, nil, loadOTUntrustedSpan},
+	}
+
+	for _, testCase := range testCases {
+		il := &ingress.Location{
+			Opentelemetry: opentelemetry.Config{Set: testCase.isSetInLoc, TrustSet: testCase.isTrustSetInLoc},
+		}
+		if il.Opentelemetry.Set {
+			il.Opentelemetry.Enabled = *testCase.isOTInLoc
+		}
+		if il.Opentelemetry.TrustSet {
+			il.Opentelemetry.TrustEnabled = *testCase.isTrustInLoc
+		}
+
+		actual := buildOpentelemetryForLocation(testCase.globalOT, testCase.globalTrust, il)
+
+		if testCase.expected != actual {
+			t.Errorf("%v: expected '%v' but returned '%v'", testCase.description, testCase.expected, actual)
+		}
+	}
+}
+
+func TestShouldLoadOpentelemetryModule(t *testing.T) {
+	// ### Invalid argument type tests ###
+	// The first tests should return false.
+	expected := false
+
+	invalidType := &ingress.Ingress{}
+	actual := shouldLoadOpentelemetryModule(config.Configuration{}, invalidType)
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+
+	actual = shouldLoadOpentelemetryModule(invalidType, []*ingress.Server{})
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+
+	// ### Functional tests ###
+	actual = shouldLoadOpentelemetryModule(config.Configuration{}, []*ingress.Server{})
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+
+	// All further tests should return true.
+	expected = true
+
+	configuration := config.Configuration{EnableOpentelemetry: true}
+	actual = shouldLoadOpentelemetryModule(configuration, []*ingress.Server{})
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+
+	servers := []*ingress.Server{
+		{
+			Locations: []*ingress.Location{
+				{
+					Opentelemetry: opentelemetry.Config{
+						Enabled: true,
+					},
+				},
+			},
+		},
+	}
+	actual = shouldLoadOpentelemetryModule(config.Configuration{}, servers)
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+
+	actual = shouldLoadOpentelemetryModule(configuration, servers)
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+}
+
 func TestModSecurityForLocation(t *testing.T) {
 	loadModule := `modsecurity on;
 `