Add OTEL build test and for NGINX v1.25 (#10889)

* Add OTEL build test * Simplify otel compilation * Remove http2 deprecated arg * Move image build to CI * Turn image from scratch to optimize usage * rollback image from scratch * Final reviews on nginx v1.25 image * Remove s390x from final image
2024-01-27 12:33:50 -03:00 · 2024-01-27 12:33:50 -03:00 · 02e6ebc95a
commit 02e6ebc95a
parent c295cd1c4b
20 changed files with 247 additions and 409 deletions
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@ -9,6 +9,7 @@ on:
      - 'deploy/**'
      - '**.md'
      - 'images/**' # Images changes should be tested on their own workflow
+      - '!images/nginx-1.25/**'

  push:
    branches:
@ -41,6 +42,7 @@ jobs:
    outputs:
      go: ${{ steps.filter.outputs.go }}
      charts: ${{ steps.filter.outputs.charts }}
+      baseimage: ${{ steps.filter.outputs.baseimage }}

    steps:

@ -64,6 +66,8 @@ jobs:
              - 'charts/ingress-nginx/Chart.yaml'
              - 'charts/ingress-nginx/**/*'
              - 'NGINX_BASE'
+            baseimage:
+              - 'images/nginx-1.25/**'

  test-go:
    runs-on: ubuntu-latest
@ -89,8 +93,10 @@ jobs:
    runs-on: ubuntu-latest
    needs: changes
    if: |
-      (needs.changes.outputs.go == 'true') || (needs.changes.outputs.charts == 'true')
-
+      (needs.changes.outputs.go == 'true') || (needs.changes.outputs.charts == 'true') || (needs.changes.outputs.baseimage == 'true')
+     
+    env:
+        PLATFORMS: linux/amd64
    steps:
      - name: Checkout
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@ -119,7 +125,14 @@ jobs:
          curl -LO https://dl.k8s.io/release/v1.27.3/bin/linux/amd64/kubectl
          chmod +x ./kubectl
          sudo mv ./kubectl /usr/local/bin/kubectl
-
+      
+      - name: Build NGINX Base image
+        if: |
+          needs.changes.outputs.baseimage == 'true'
+        run: |
+            export TAG=$(cat images/nginx-1.25/TAG)
+            cd images/nginx-1.25/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --load -t gcr.io/k8s-staging-ingress-nginx/nginx-1.25:${TAG} .
+    
      - name: Build images
        env:
          TAG: 1.0.0-dev
@ -127,7 +140,8 @@ jobs:
          REGISTRY: ingress-controller
        run: |
          echo "building images..."
-          make clean-image build image image-chroot
+          export TAGNGINX=$(cat images/nginx-1.25/TAG)
+          make BASE_IMAGE=gcr.io/k8s-staging-ingress-nginx/nginx-1.25:${TAGNGINX} clean-image build image image-chroot
          make -C test/e2e-image image

          echo "creating images cache..."
@ -150,11 +164,11 @@ jobs:
      - changes
      - build
    if: |
-      (needs.changes.outputs.charts == 'true')
+      (needs.changes.outputs.charts == 'true') || (needs.changes.outputs.baseimage == 'true')

    strategy:
      matrix:
-        k8s: [v1.25.11, v1.26.6, v1.27.3, v1.28.0, v1.29.0]
+        k8s: [v1.26.6, v1.27.3, v1.28.0, v1.29.0]

    steps:
      - name: Checkout
@ -222,6 +236,7 @@ jobs:
          KIND_CLUSTER_NAME: kind
          SKIP_CLUSTER_CREATION: true
          SKIP_IMAGE_CREATION: true
+          SKIP_INGRESS_IMAGE_CREATION: true
        run: |
          kind get kubeconfig > $HOME/.kube/kind-config-kind
          make kind-e2e-chart-tests
@ -232,10 +247,10 @@ jobs:
      - changes
      - build
    if: |
-      (needs.changes.outputs.go == 'true')
+      (needs.changes.outputs.go == 'true') || (needs.changes.outputs.baseimage == 'true')
    strategy:
      matrix:
-        k8s: [v1.25.11, v1.26.6, v1.27.3, v1.28.0, v1.29.0]
+        k8s: [v1.26.6, v1.27.3, v1.28.0, v1.29.0]
    uses: ./.github/workflows/zz-tmpl-k8s-e2e.yaml
    with:
      k8s-version: ${{ matrix.k8s }}
@ -246,10 +261,10 @@ jobs:
      - changes
      - build
    if: |
-      (needs.changes.outputs.go == 'true')
+      (needs.changes.outputs.go == 'true') || (needs.changes.outputs.baseimage == 'true')
    strategy:
      matrix:
-        k8s: [v1.25.11, v1.26.6, v1.27.3, v1.28.0, v1.29.0]
+        k8s: [v1.26.6, v1.27.3, v1.28.0, v1.29.0]
    uses: ./.github/workflows/zz-tmpl-k8s-e2e.yaml
    with:
      k8s-version: ${{ matrix.k8s }}
@ -261,10 +276,10 @@ jobs:
      - changes
      - build
    if: |
-      (needs.changes.outputs.go == 'true')
+      (needs.changes.outputs.go == 'true') || (needs.changes.outputs.baseimage == 'true')
    strategy:
      matrix:
-        k8s: [v1.25.11, v1.26.6, v1.27.3, v1.28.0, v1.29.0]
+        k8s: [v1.26.6, v1.27.3, v1.28.0, v1.29.0]
    uses: ./.github/workflows/zz-tmpl-k8s-e2e.yaml
    with:
      k8s-version: ${{ matrix.k8s }}
--- a/.github/workflows/images.yaml
+++ b/.github/workflows/images.yaml
@ -36,6 +36,8 @@ jobs:
      kube-webhook-certgen: ${{ steps.filter.outputs.kube-webhook-certgen }}
      ext-auth-example-authsvc: ${{ steps.filter.outputs.ext-auth-example-authsvc }}
      nginx: ${{ steps.filter.outputs.nginx }}
+      nginx125: ${{ steps.filter.outputs.nginx125 }}
+      opentelemetry: ${{ steps.filter.outputs.opentelemetry }}

    steps:
      - name: Checkout
@ -63,6 +65,10 @@ jobs:
              - 'images/ext-auth-example-authsvc/**'
            nginx:
              - 'images/nginx/**'
+            opentelemetry:
+              - 'images/opentelemetry/**'
+            nginx125:
+              - 'images/nginx-1.25/TAG'
 
  #### TODO: Make the below jobs 'less dumb' and use the job name as parameter (the github.job context does not work here)
  cfssl:
@ -179,3 +185,53 @@ jobs:
      uses: github/codeql-action/upload-sarif@v3.23.1
      with:
        sarif_file: 'trivy-results.sarif'
+
+  opentelemetry:
+    runs-on: ubuntu-latest
+    env:
+      PLATFORMS: linux/amd64,linux/arm,linux/arm64
+    needs: changes
+    if: |
+      (needs.changes.outputs.opentelemetry == 'true')
+    strategy:
+      matrix:
+        nginx: ['1.25.3', '1.21.6']
+    steps:
+    - name: Checkout
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+    - name: image build
+      run: |
+        cd images/opentelemetry && make NGINX_VERSION=${{ matrix.nginx }} build
+
+  nginx125:
+    permissions:
+      contents: write
+      packages: write
+    runs-on: ubuntu-latest
+    needs: changes
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.changes.outputs.nginx125 == 'true')
+    env:
+      PLATFORMS: linux/amd64,linux/arm,linux/arm64
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+        with:
+          version: latest
+          platforms: ${{ env.PLATFORMS }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: build-image
+        run: |
+          export TAG=$(cat images/nginx-1.25/TAG)
+          cd images/nginx-1.25/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --push --load -t ingressnginx/nginx-1.25:${TAG} .
+    
+    
--- a/.github/workflows/nginx125.yaml
+++ b/.github/workflows/nginx125.yaml
@ -1,167 +0,0 @@
-name: NGINX v1.25 Image
-
-on:
-  pull_request:
-    branches:
-    - "*"
-    paths:
-    - 'images/nginx-1.25/**'
-  push:
-    branches:
-    - main
-    paths:
-    - 'images/nginx-1.25/**'
-
-permissions:
-    contents: read
-
-jobs:
-  changes:
-    permissions:
-      contents: read  # for dorny/paths-filter to fetch a list of changed files
-      pull-requests: read  # for dorny/paths-filter to read pull requests
-    runs-on: ubuntu-latest
-    outputs:
-      nginx: ${{ steps.filter.outputs.nginx }}
-      tag: ${{ steps.filter.outputs.tag }}
-    steps:
-    - name: Checkout
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-    - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
-      id: filter
-      with:
-        token: ${{ secrets.GITHUB_TOKEN }}
-        filters: |
-          nginx:
-          - 'images/nginx-1.25/**'
-          tag:
-          - 'images/nginx-1.25/TAG'
-
-  build:
-    permissions:
-      contents: read  # for dorny/paths-filter to fetch a list of changed files
-      pull-requests: read  # for dorny/paths-filter to read pull requests
-    runs-on: ubuntu-latest
-    needs: changes
-    if: |
-      (github.event_name != 'push' && github.ref != 'refs/heads/main' && needs.changes.outputs.nginx == 'true')
-    env:
-      PLATFORMS: linux/amd64
-    steps:
-    - name: Checkout
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-    - name: Set up Go
-      id: go
-      uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
-      with:
-        go-version: '1.21.5'
-        check-latest: true
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
-    - name: Set up Docker Buildx
-      id: buildx
-      uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
-      with:
-        version: latest
-        platforms: ${{ env.PLATFORMS }}
-    - name: Prepare Host
-      run: |
-        curl -LO https://dl.k8s.io/release/v1.27.3/bin/linux/amd64/kubectl
-        chmod +x ./kubectl
-        sudo mv ./kubectl /usr/local/bin/kubectl
-    - name: build-image
-      run: |
-        cd images/nginx-1.25/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --load -t nginx-1.25:1.0.0-dev .
-    - name: load-image
-      run: |
-        make clean-image build
-        make -C test/e2e-image image
-        docker build \
-          --platform linux \
-          --no-cache \
-          --build-arg BASE_IMAGE="nginx-1.25:1.0.0-dev" \
-          --build-arg VERSION="0.0.1-${{ github.sha }}" \
-          --build-arg TARGETARCH="amd64" \
-          --build-arg COMMIT_SHA="git-${{ github.sha }}" \
-          --build-arg BUILD_ID=""UNSET"" \
-          -t ingress-controller/controller:1.0.0-dev rootfs
-        docker save \
-          nginx-ingress-controller:e2e \
-          ingress-controller/controller:1.0.0-dev \
-          nginx-1.25:1.0.0-dev \
-          | gzip > docker.tar.gz
-    - name: cache
-      uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0
-      with:
-        name: docker.tar.gz
-        path: docker.tar.gz
-        retention-days: 2
-
-  e2e-test:
-    name: Kubernetes
-    runs-on: ubuntu-latest
-    needs:
-      - build
-    strategy:
-      matrix:
-        k8s: [v1.27.3, v1.28.0, v1.29.0]
-    steps:
-    - name: Checkout
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-  
-    - name: cache
-      uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
-      with:
-        name: docker.tar.gz
-  
-    - name: Create Kubernetes ${{ matrix.k8s }} cluster
-      id: kind
-      run: |
-        kind create cluster --image=kindest/node:${{ matrix.k8s }} --config test/e2e/kind.yaml
-  
-    - name: Load images from cache
-      run: |
-        echo "loading docker images..."
-        gzip -dc docker.tar.gz | docker load
-  
-    - name: Run e2e tests
-      env:
-        KIND_CLUSTER_NAME: kind
-        SKIP_CLUSTER_CREATION: true
-        SKIP_IMAGE_CREATION: true
-        SKIP_OPENTELEMETRY_TESTS: true
-      run: |
-        kind get kubeconfig > $HOME/.kube/kind-config-kind
-        make NGINX_BASE_IMAGE="nginx-1.25:1.0.0-dev" kind-e2e-test
-        
-  push:
-    permissions:
-      contents: write
-      packages: write
-    runs-on: ubuntu-latest
-    needs: changes
-    if: |
-      (github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.changes.outputs.tag == 'true')
-    env:
-      PLATFORMS: linux/amd64,linux/arm,linux/arm64,linux/s390x
-    steps:
-    - name: Checkout
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
-    - name: Set up Docker Buildx
-      id: buildx
-      uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
-      with:
-        version: latest
-        platforms: ${{ env.PLATFORMS }}
-    - name: Login to GitHub Container Registry
-      uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - name: build-image
-      run: |
-        export TAG=$(cat images/nginx-1.25/TAG)
-        cd images/nginx-1.25/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --push --load -t ingressnginx/nginx-1.25:${TAG} .
-
--- a/.github/workflows/zz-tmpl-k8s-e2e.yaml
+++ b/.github/workflows/zz-tmpl-k8s-e2e.yaml
@ -41,7 +41,8 @@ jobs:
        env:
          KIND_CLUSTER_NAME: kind
          SKIP_CLUSTER_CREATION: true
-          SKIP_IMAGE_CREATION: true
+          SKIP_INGRESS_IMAGE_CREATION: true
+          SKIP_E2E_IMAGE_CREATION: true
          ENABLE_VALIDATIONS: ${{ inputs.variation == 'VALIDATIONS' }}
          IS_CHROOT: ${{ inputs.variation == 'CHROOT' }}
        run: |