First sidecar module: OpenTelemetry (#8013)

* remove opentelemetry from main nginx image

* add opentelemetry sidecar image

* handle extra modules in helm chart

* fix running helm chart

* mount the modules volume in the init container

* merge the mounted folder

* fix the otel image

* fix licence year

* fix cloudbuild image

* use the same nginx version as in the main image

* only retrieve /etc/nginx/modules for now

images/opentelemetry/Makefile

@@ -0,0 +1,57 @@
+# Copyright 2021 The Kubernetes Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+.DEFAULT_GOAL:=build
+
+# set default shell
+SHELL=/bin/bash -o pipefail -o errexit
+
+DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))))
+INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh
+
+# 0.0.0 shouldn't clobber any released builds
+TAG ?= 0.0
+REGISTRY ?= gcr.io/k8s-staging-ingress-nginx
+
+IMAGE = $(REGISTRY)/opentelemetry
+
+# required to enable buildx
+export DOCKER_CLI_EXPERIMENTAL=enabled
+
+# build with buildx
+PLATFORMS?=linux/amd64,linux/arm
+OUTPUT=
+PROGRESS=plain
+build: ensure-buildx
+	docker buildx build \
+		--platform=${PLATFORMS} $(OUTPUT) \
+		--progress=$(PROGRESS) \
+		--pull \
+		--tag $(IMAGE):$(TAG) rootfs
+
+# push the cross built image
+push: OUTPUT=--push
+push: build
+
+# enable buildx
+ensure-buildx:
+# this is required for cloudbuild
+ifeq ("$(wildcard $(INIT_BUILDX))","")
+	@curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/hack/init-buildx.sh | bash
+else
+	@exec $(INIT_BUILDX)
+endif
+	@echo "done"
+
+.PHONY: build push ensure-buildx

images/opentelemetry/README.md

@@ -0,0 +1,5 @@
+# OpenTelemetry library builder
+
+**How to use this image:**
+This image only contains the necessary files in /usr/local and /etc/nginx/opentelemetry to 
+be copied to Ingress Controller deployment when OpenTelemetry is enabled

images/opentelemetry/cloudbuild.yaml

@@ -0,0 +1,24 @@
+timeout: 10800s
+options:
+  substitution_option: ALLOW_LOOSE
+  # job builds a multi-arch docker image for amd64,arm,arm64 and s390x.
+  machineType: N1_HIGHCPU_32
+steps:
+  - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20211118-2f2d816b90
+    entrypoint: bash
+    env:
+      - DOCKER_CLI_EXPERIMENTAL=enabled
+      - TAG=$_GIT_TAG
+      - BASE_REF=$_PULL_BASE_REF
+      - REGISTRY=gcr.io/k8s-staging-ingress-nginx
+      # default cloudbuild has HOME=/builder/home and docker buildx is in /root/.docker/cli-plugins/docker-buildx
+      # set the home to /root explicitly to if using docker buildx
+      - HOME=/root
+    args:
+    - -c
+    - |
+      gcloud auth configure-docker \
+      && make push
+substitutions:
+  _GIT_TAG: "12345"
+  _PULL_BASE_REF: "master"

images/opentelemetry/rootfs/Dockerfile

@@ -0,0 +1,28 @@
+# Copyright 2021 The Kubernetes Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+FROM alpine:3.14.2 as builder
+
+COPY . /
+
+RUN apk update \
+	&& apk upgrade \
+	&& apk add -U bash \
+	&& /build.sh
+
+FROM busybox:latest 
+
+COPY --from=builder init_module.sh /usr/local/bin/init_module.sh
+COPY --from=builder /etc/nginx/modules /etc/nginx/modules

images/opentelemetry/rootfs/build.sh

@@ -0,0 +1,111 @@
+#!/bin/bash
+
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export NGINX_VERSION=1.19.9
+
+# Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp/compare/v1.0.0...main
+export OPENTELEMETRY_CPP_VERSION=1.0.0
+
+# Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp-contrib/compare/f4850...main
+export OPENTELEMETRY_CONTRIB_COMMIT=f48500884b1b32efc456790bbcdc2e6cf7a8e630
+
+export BUILD_PATH=/tmp/build
+
+rm -rf \
+   /var/cache/debconf/* \
+   /var/lib/apt/lists/* \
+   /var/log/* \
+   /tmp/* \
+   /var/tmp/*
+
+
+mkdir -p /etc/nginx
+mkdir --verbose -p "$BUILD_PATH"
+cd "$BUILD_PATH"
+
+apk add \
+  curl \
+  git \
+  build-base
+
+get_src()
+{
+  hash="$1"
+  url="$2"
+  f=$(basename "$url")
+
+  echo "Downloading $url"
+
+  curl -sSL "$url" -o "$f"
+  echo "$hash  $f" | sha256sum -c - || exit 10
+  tar xzf "$f"
+  rm -rf "$f"
+}
+
+
+get_src e462e11533d5c30baa05df7652160ff5979591d291736cfa5edb9fd2edb48c49 \
+        "https://nginx.org/download/nginx-$NGINX_VERSION.tar.gz"
+
+get_src 45c52498788e47131b20a4786dbb08f4390b8cb419bd3d61c88b503cafff3324 \
+        "https://github.com/open-telemetry/opentelemetry-cpp-contrib/archive/$OPENTELEMETRY_CONTRIB_COMMIT.tar.gz"
+
+# improve compilation times
+CORES=$(($(grep -c ^processor /proc/cpuinfo) - 1))
+
+export MAKEFLAGS=-j${CORES}
+
+apk add \
+  protobuf-dev \
+  grpc \
+  grpc-dev \
+  gtest-dev \
+  c-ares-dev \
+  pcre-dev
+
+cd $BUILD_PATH
+git clone --recursive https://github.com/open-telemetry/opentelemetry-cpp opentelemetry-cpp-$OPENTELEMETRY_CPP_VERSION
+cd "opentelemetry-cpp-$OPENTELEMETRY_CPP_VERSION"
+git checkout v$OPENTELEMETRY_CPP_VERSION
+mkdir .build
+cd .build
+
+cmake -DCMAKE_BUILD_TYPE=Release \
+      -DBUILD_TESTING=OFF \
+      -DWITH_EXAMPLES=OFF \
+      -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+      -DWITH_OTLP=ON \
+      -DWITH_OTLP_HTTP=OFF \
+      ..
+make
+make install
+
+# build nginx
+cd "$BUILD_PATH/nginx-$NGINX_VERSION"
+./configure \
+  --prefix=/usr/local/nginx \
+  --with-compat \
+  --add-dynamic-module=$BUILD_PATH/opentelemetry-cpp-contrib-$OPENTELEMETRY_CONTRIB_COMMIT/instrumentation/nginx
+
+make modules
+mkdir -p /etc/nginx/modules
+cp objs/otel_ngx_module.so /etc/nginx/modules/otel_ngx_module.so
+
+# remove .a files
+find /usr/local -name "*.a" -print | xargs /bin/rm

images/opentelemetry/rootfs/init_module.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+cp -R /etc/nginx/modules /modules_mount/etc/nginx/modules