diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 000000000..f91e5c1fe --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,8 @@ +coverage: + status: + project: + default: + target: 40% + threshold: null + patch: false + changes: false diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 07770e47c..095f079d6 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -10,8 +10,7 @@ assignees: '' -**What happened**: - - - -**What you expected to happen**: - - - - -**NGINX Ingress controller version** (exec into the pod and run `/nginx-ingress-controller --version`): - +**NGINX Ingress controller version**: **Kubernetes version** (use `kubectl version`): @@ -53,36 +37,17 @@ kubectl exec -it $POD_NAME -n $POD_NAMESPACE -- /nginx-ingress-controller --vers - **OS** (e.g. from /etc/os-release): - **Kernel** (e.g. `uname -a`): - **Install tools**: - - `Please mention how/where was the cluster created like kubeadm/kops/minikube/kind etc. ` -- **Basic cluster related info**: - - `kubectl version` - - `kubectl get nodes -o wide` - -- **How was the ingress-nginx-controller installed**: - - If helm was used then please show output of `helm ls -A | grep -i ingress` - - If helm was used then please show output of `helm -n get values ` - - If helm was not used, then copy/paste the complete precise command used to install the controller, along with the flags and options used - - if you have more than one instance of the ingress-nginx-controller installed in the same cluster, please provide details for all the instances - -- **Current State of the controller**: - - `kubectl describe ingressclasses` - - `kubectl -n get all -A -o wide` - - `kubectl -n describe po ` - - `kubectl -n describe svc ` - -- **Current state of ingress object, if applicable**: - - `kubectl -n get all,ing -o wide` - - `kubectl -n describe ing ` - - If applicable, then, your complete and exact curl/grpcurl command (redacted if required) and the reponse to the curl/grpcurl command with the -v flag - - **Others**: - - Any other related information like ; - - copy/paste of the snippet (if applicable) - - `kubectl describe ...` of any custom configmap(s) created and in use - - Any other related information that may help +**What happened**: -**How to reproduce this issue**: + + +**What you expected to happen**: + + + +**How to reproduce it**: + - +/kind bug diff --git a/.github/ISSUE_TEMPLATE/cve_report.md b/.github/ISSUE_TEMPLATE/cve_report.md deleted file mode 100644 index d6e598c88..000000000 --- a/.github/ISSUE_TEMPLATE/cve_report.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: CVE Finding Report -about: CVE reporting for ingress-nginx -title: '' -labels: kind/bug -assignees: - - Gacko - - strongjz ---- - - - - - - - - - - diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 406a7ad0a..771804b54 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -7,15 +7,24 @@ assignees: '' --- + + - + - +/kind feature diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c1c815678..8b499fe8d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,5 @@ - ## What this PR does / why we need it: @@ -10,9 +9,7 @@ - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) -- [ ] CVE Report (Scanner found CVE and adding report) - [ ] Breaking change (fix or feature that would cause existing functionality to change) -- [ ] Documentation only ## Which issue/s this PR fixes - [ ] My change requires a change to the documentation. - [ ] I have updated the documentation accordingly. -- [ ] I've read the [CONTRIBUTION](https://github.com/kubernetes/ingress-nginx/blob/main/CONTRIBUTING.md) guide -- [ ] I have added unit and/or e2e tests to cover my changes. +- [ ] I've read the [CONTRIBUTION](https://github.com/kubernetes/ingress-nginx/blob/master/CONTRIBUTING.md) guide +- [ ] I have added tests to cover my changes. - [ ] All new and existing tests passed. diff --git a/.github/actions/mkdocs/Dockerfile b/.github/actions/mkdocs/Dockerfile index f00584d32..b5b564d3e 100644 --- a/.github/actions/mkdocs/Dockerfile +++ b/.github/actions/mkdocs/Dockerfile @@ -1,4 +1,6 @@ -FROM squidfunk/mkdocs-material:9.4.5 +FROM squidfunk/mkdocs-material:6.2.4 + +RUN pip install mkdocs-awesome-pages-plugin COPY action.sh /action.sh diff --git a/.github/actions/mkdocs/action.sh b/.github/actions/mkdocs/action.sh index d9f8ecbfc..07ddbddb9 100644 --- a/.github/actions/mkdocs/action.sh +++ b/.github/actions/mkdocs/action.sh @@ -16,7 +16,7 @@ set -e -REQUIREMENTS="${GITHUB_WORKSPACE}/docs/requirements.txt" +REQUIREMENTS="${GITHUB_WORKSPACE}/requirements.txt" if [ -f "${REQUIREMENTS}" ]; then pip install -r "${REQUIREMENTS}" diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index deb434675..000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,43 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" - labels: - - "area/dependency" - - "release-note-none" - - "ok-to-test" - groups: - actions: - update-types: - - "minor" - - "patch" - - package-ecosystem: "docker" - directories: - - "**/rootfs" - schedule: - interval: "weekly" - labels: - - "area/dependency" - - "release-note-none" - - "ok-to-test" - groups: - docker: - update-types: - - "minor" - - "patch" - - package-ecosystem: "gomod" - directories: - - "/" - - "**/rootfs" - schedule: - interval: "weekly" - labels: - - "area/dependency" - - "release-note-none" - - "ok-to-test" - groups: - go: - update-types: - - "patch" diff --git a/.github/workflows/chart.yaml b/.github/workflows/chart.yaml deleted file mode 100644 index 7c37447af..000000000 --- a/.github/workflows/chart.yaml +++ /dev/null @@ -1,64 +0,0 @@ -name: Chart - -on: - push: - branches: - - main - - release-* - paths: - - charts/ingress-nginx/Chart.yaml - - workflow_dispatch: - -permissions: - contents: read - -jobs: - release: - name: Release - runs-on: ubuntu-latest - - permissions: - contents: write - - steps: - - name: Set up Python - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 - with: - python-version: 3.x - - - name: Set up Helm - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 - - - name: Set up Helm Chart Testing - uses: helm/chart-testing-action@0d28d3144d3a25ea2cc349d6e59901c4ff469b3b # v2.7.0 - - - name: Set up Artifact Hub - run: | - curl --fail --location https://github.com/artifacthub/hub/releases/download/v1.20.0/ah_1.20.0_linux_amd64.tar.gz --output /tmp/ah.tar.gz - echo "9027626f19ff9f3ac668f222917130ac885e289e922e1428bfd2e7f066324e31 /tmp/ah.tar.gz" | shasum --check - sudo tar --extract --file /tmp/ah.tar.gz --directory /usr/local/bin ah - - - name: Set up Git - run: | - git config --global user.name "${GITHUB_ACTOR}" - git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" - - - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - - name: Lint chart - run: | - ct lint --config .ct.yaml - ah lint --path charts/ingress-nginx - - - name: Release chart - uses: helm/chart-releaser-action@cae68fefc6b5f367a0275617c9f83181ba54714f # v1.7.0 - env: - CR_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CR_RELEASE_NAME_TEMPLATE: helm-chart-{{ .Version }} - CR_SKIP_EXISTING: true - with: - charts_dir: charts diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b06cef33c..612bd4b38 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,52 +4,25 @@ on: pull_request: branches: - "*" - paths-ignore: - - 'docs/**' - - 'deploy/**' - - '**.md' - - 'images/**' # Images changes should be tested on their own workflow - - '!images/nginx/**' push: branches: - - main - - release-* - paths-ignore: - - 'docs/**' - - 'deploy/**' - - '**.md' - - 'images/**' # Images changes should be tested on their own workflow - - workflow_dispatch: - inputs: - run_e2e: - description: 'Force e2e to run' - required: false - type: boolean - - -permissions: - contents: read + - master jobs: changes: - permissions: - contents: read # for dorny/paths-filter to fetch a list of changed files - pull-requests: read # for dorny/paths-filter to read pull requests runs-on: ubuntu-latest outputs: go: ${{ steps.filter.outputs.go }} charts: ${{ steps.filter.outputs.charts }} - baseimage: ${{ steps.filter.outputs.baseimage }} steps: - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v2 - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + - uses: dorny/paths-filter@v2 id: filter with: token: ${{ secrets.GITHUB_TOKEN }} @@ -61,128 +34,61 @@ jobs: - 'rootfs/**/*' - 'TAG' - 'test/e2e/**/*' - - 'NGINX_BASE' charts: - 'charts/ingress-nginx/Chart.yaml' - 'charts/ingress-nginx/**/*' - - 'NGINX_BASE' - baseimage: - - 'NGINX_BASE' - - 'images/nginx/**' - docs: - - '**/*.md' - lua: - - '**/*.lua' - lua-lint: + security: runs-on: ubuntu-latest - needs: changes - if: | - (needs.changes.outputs.lua == 'true') || ${{ github.event.workflow_dispatch.run_e2e == 'true' }} steps: + - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Lint Lua - uses: lunarmodules/luacheck@v1 + uses: actions/checkout@v2 + + - name: Run Gosec Security Scanner + uses: securego/gosec@master with: - args: --codes --globals lua_ingress --globals configuration --globals balancer --globals monitor --globals certificate --globals tcp_udp_configuration --globals tcp_udp_balancer --no-max-comment-line-length -q rootfs/etc/nginx/lua/ - - test-go: - runs-on: ubuntu-latest - needs: changes - if: | - (needs.changes.outputs.go == 'true') || ${{ github.event.workflow_dispatch.run_e2e == 'true' }} - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Get go version - run: echo "GOLANG_VERSION=$(cat GOLANG_VERSION)" >> $GITHUB_ENV - - - name: Set up Go - id: go - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 - with: - go-version: ${{ env.GOLANG_VERSION }} - check-latest: true - - - name: Run test - run: make test - - - verify-docs: - name: Verify Doc generation - runs-on: ubuntu-latest - needs: changes - if: | - (needs.changes.outputs.go == 'true') || (needs.changes.outputs.docs == 'true') || ${{ github.event.workflow_dispatch.run_e2e == 'true' }} - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Get go version - run: echo "GOLANG_VERSION=$(cat GOLANG_VERSION)" >> $GITHUB_ENV - - name: Set up Go - id: go - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 - with: - go-version: ${{ env.GOLANG_VERSION }} - check-latest: true - - name: Verify Docs - run: make verify-docs + # G601 for zz_generated.deepcopy.go + # G306 TODO: Expect WriteFile permissions to be 0600 or less + # G307 TODO: Deferring unsafe method "Close" + args: -exclude=G601,G104,G204,G304,G306,G307 -tests=false -exclude-dir=test -exclude-dir=images/ -exclude-dir=docs/ ./... build: name: Build runs-on: ubuntu-latest needs: changes - outputs: - golangversion: ${{ steps.golangversion.outputs.version }} if: | - (needs.changes.outputs.go == 'true') || (needs.changes.outputs.charts == 'true') || (needs.changes.outputs.baseimage == 'true') || ${{ github.event.workflow_dispatch.run_e2e == 'true' }} + (needs.changes.outputs.go == 'true') - env: - PLATFORMS: linux/amd64 steps: + - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v2 - - name: Get go version - id: golangversion - run: | - echo "version=$(cat GOLANG_VERSION)" >> "$GITHUB_OUTPUT" - - - name: Set up Go + - name: Set up Go 1.15 id: go - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 + uses: actions/setup-go@v2 with: - go-version: ${{ steps.golangversion.outputs.version }} - check-latest: true - - - name: Set up QEMU - uses: docker/setup-qemu-action@4574d27a4764455b42196d70a065bc6853246a25 # v3.4.0 + go-version: 1.15 - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3.9.0 + uses: crazy-max/ghaction-docker-buildx@v1 with: - version: latest + buildx-version: latest + qemu-version: latest - name: Available platforms run: echo ${{ steps.buildx.outputs.platforms }} - name: Prepare Host run: | - curl -LO https://dl.k8s.io/release/v1.32.2/bin/linux/amd64/kubectl + sudo apt-get -qq update || true + sudo apt-get install -y pigz + curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.17.0/bin/linux/amd64/kubectl chmod +x ./kubectl sudo mv ./kubectl /usr/local/bin/kubectl - - name: Build NGINX Base image - if: | - needs.changes.outputs.baseimage == 'true' - run: | - export TAG=$(cat images/nginx/TAG) - cd images/nginx/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --load -t registry.k8s.io/ingress-nginx/nginx:${TAG} . - - name: Build images env: TAG: 1.0.0-dev @@ -190,133 +96,112 @@ jobs: REGISTRY: ingress-controller run: | echo "building images..." - export TAGNGINX=$(cat images/nginx/TAG) - make BASE_IMAGE=registry.k8s.io/ingress-nginx/nginx:${TAGNGINX} clean-image build image image-chroot + make clean-image build image make -C test/e2e-image image echo "creating images cache..." docker save \ nginx-ingress-controller:e2e \ ingress-controller/controller:1.0.0-dev \ - ingress-controller/controller-chroot:1.0.0-dev \ - | gzip > docker.tar.gz + | pigz > docker.tar.gz - name: cache - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + uses: actions/upload-artifact@v2 with: name: docker.tar.gz path: docker.tar.gz - retention-days: 5 - chart-lint: - name: Chart / Lint + helm: + name: Helm chart runs-on: ubuntu-latest needs: - changes - - if: fromJSON(needs.changes.outputs.charts) || fromJSON(needs.changes.outputs.baseimage) || fromJSON(github.event.workflow_dispatch.run_e2e) + if: | + (needs.changes.outputs.charts == 'true') steps: - - name: Set up Python - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 - with: - python-version: 3.x - - name: Set up Helm - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 + - name: Checkout + uses: actions/checkout@v2 - - name: Set up Helm Chart Testing - uses: helm/chart-testing-action@0d28d3144d3a25ea2cc349d6e59901c4ff469b3b # v2.7.0 - - - name: Set up Artifact Hub + - name: Lint run: | - curl --fail --location https://github.com/artifacthub/hub/releases/download/v1.20.0/ah_1.20.0_linux_amd64.tar.gz --output /tmp/ah.tar.gz - echo "9027626f19ff9f3ac668f222917130ac885e289e922e1428bfd2e7f066324e31 /tmp/ah.tar.gz" | shasum --check - sudo tar --extract --file /tmp/ah.tar.gz --directory /usr/local/bin ah + ./build/run-in-docker.sh ./hack/verify-chart-lint.sh - - name: Set up Helm Docs - uses: gabe565/setup-helm-docs-action@d5c35bdc9133cfbea3b671acadf50a29029e87c2 # v1.0.4 - - - name: Set up Helm Unit Test - run: helm plugin install https://github.com/helm-unittest/helm-unittest - - - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - - name: Lint chart + - name: fix permissions run: | - ct lint --config .ct.yaml - ah lint --path charts/ingress-nginx + sudo mkdir -p $HOME/.kube + sudo chmod -R 777 $HOME/.kube - - name: Check docs - run: | - helm-docs --chart-search-root charts - git diff --exit-code charts/ingress-nginx/README.md - - - name: Run tests - run: helm unittest charts/ingress-nginx --file "tests/**/*_test.yaml" - - chart-test: - name: Chart / Test - runs-on: ubuntu-latest - needs: - - changes - - build - - chart-lint - - if: fromJSON(needs.changes.outputs.charts) || fromJSON(needs.changes.outputs.baseimage) || fromJSON(github.event.workflow_dispatch.run_e2e) - - strategy: - matrix: - k8s: [v1.28.15, v1.29.12, v1.30.8, v1.31.4, v1.32.0] - - steps: - - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Download cache - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + - name: Create Kubernetes cluster + id: kind + uses: engineerd/setup-kind@v0.5.0 with: - name: docker.tar.gz + version: v0.10.0 + image: kindest/node:v1.20.2 - - name: Load cache - run: gzip --decompress --stdout docker.tar.gz | docker load - - - name: Run tests + - name: Test env: - K8S_VERSION: ${{ matrix.k8s }} - SKIP_IMAGE_CREATION: true + KIND_CLUSTER_NAME: kind + SKIP_CLUSTER_CREATION: true run: | - sudo mkdir -pm 777 "${HOME}/.kube" + kind get kubeconfig > $HOME/.kube/kind-config-kind make kind-e2e-chart-tests kubernetes: name: Kubernetes + runs-on: ubuntu-latest needs: - changes - build if: | - (needs.changes.outputs.go == 'true') || (needs.changes.outputs.baseimage == 'true') || ${{ github.event.workflow_dispatch.run_e2e == 'true' }} - strategy: - matrix: - k8s: [v1.28.15, v1.29.12, v1.30.8, v1.31.4, v1.32.0] - uses: ./.github/workflows/zz-tmpl-k8s-e2e.yaml - with: - k8s-version: ${{ matrix.k8s }} + (needs.changes.outputs.go == 'true') - kubernetes-chroot: - name: Kubernetes chroot - needs: - - changes - - build - if: | - (needs.changes.outputs.go == 'true') || (needs.changes.outputs.baseimage == 'true') || ${{ github.event.workflow_dispatch.run_e2e == 'true' }} strategy: matrix: - k8s: [v1.28.15, v1.29.12, v1.30.8, v1.31.4, v1.32.0] - uses: ./.github/workflows/zz-tmpl-k8s-e2e.yaml - with: - k8s-version: ${{ matrix.k8s }} - variation: "CHROOT" + k8s: [v1.16.15, v1.17.17, v1.18.15, v1.19.7, v1.20.2] + + steps: + + - name: Checkout + uses: actions/checkout@v2 + + - name: cache + uses: actions/download-artifact@v2 + with: + name: docker.tar.gz + + - name: Create Kubernetes ${{ matrix.k8s }} cluster + id: kind + uses: engineerd/setup-kind@v0.5.0 + with: + version: v0.10.0 + config: test/e2e/kind.yaml + image: kindest/node:${{ matrix.k8s }} + + - uses: geekyeggo/delete-artifact@v1 + with: + name: docker.tar.gz + failOnError: false + + - name: Prepare cluster for testing + id: local-path + run: | + kubectl version + echo + echo "installing helm 3..." + curl -sSL https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash + + - name: Load images from cache + run: | + echo "loading docker images..." + pigz -dc docker.tar.gz | docker load + + - name: Run e2e tests + env: + KIND_CLUSTER_NAME: kind + SKIP_CLUSTER_CREATION: true + SKIP_IMAGE_CREATION: true + run: | + kind get kubeconfig > $HOME/.kube/kind-config-kind + make kind-e2e-test diff --git a/.github/workflows/depreview.yaml b/.github/workflows/depreview.yaml deleted file mode 100644 index 6d7e44608..000000000 --- a/.github/workflows/depreview.yaml +++ /dev/null @@ -1,14 +0,0 @@ -name: 'Dependency Review' -on: [pull_request] - -permissions: - contents: read - -jobs: - dependency-review: - runs-on: ubuntu-latest - steps: - - name: 'Checkout Repository' - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: 'Dependency Review' - uses: actions/dependency-review-action@3b139cfc5fae8b618d3eae3675e383bb1769c019 # v4.5.0 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 40ccba278..3d180dc54 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -3,17 +3,11 @@ name: Documentation on: push: branches: - - main - -permissions: - contents: read + - master jobs: changes: - permissions: - contents: read # for dorny/paths-filter to fetch a list of changed files - pull-requests: read # for dorny/paths-filter to read pull requests runs-on: ubuntu-latest if: | (github.repository == 'kubernetes/ingress-nginx') @@ -22,16 +16,17 @@ jobs: charts: ${{ steps.filter.outputs.charts }} steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + - name: Checkout + uses: actions/checkout@v2 + + - uses: dorny/paths-filter@v2 id: filter with: token: ${{ secrets.GITHUB_TOKEN }} filters: | docs: - - 'docs/**/*' + - 'docs/**/*' docs: name: Update @@ -42,14 +37,12 @@ jobs: (github.repository == 'kubernetes/ingress-nginx') && (needs.changes.outputs.docs == 'true') - permissions: - contents: write # needed to write releases - steps: + - name: Checkout master - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@v2 - name: Deploy uses: ./.github/actions/mkdocs env: - PERSONAL_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + PERSONAL_TOKEN: ${{ secrets.PERSONAL_TOKEN }} diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml deleted file mode 100644 index f57878084..000000000 --- a/.github/workflows/golangci-lint.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: golangci-lint - -on: - pull_request: - paths: - - '**/*.go' - - '.github/workflows/golangci-lint.yml' - -permissions: - contents: read - -jobs: - golangci: - name: lint - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Get go version - run: echo "GOLANG_VERSION=$(cat GOLANG_VERSION)" >> $GITHUB_ENV - - - name: Set up Go - id: go - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 - with: - go-version: ${{ env.GOLANG_VERSION }} - check-latest: true - - - name: golangci-lint - uses: golangci/golangci-lint-action@2226d7cb06a077cd73e56eedd38eecad18e5d837 # v6.5.0 - with: - version: v1.62 - only-new-issues: true diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml new file mode 100644 index 000000000..af8b01448 --- /dev/null +++ b/.github/workflows/helm.yaml @@ -0,0 +1,61 @@ +name: Helm + +on: + push: + branches: + - master + +jobs: + + changes: + runs-on: ubuntu-latest + if: | + (github.repository == 'kubernetes/ingress-nginx') + outputs: + docs: ${{ steps.filter.outputs.docs }} + charts: ${{ steps.filter.outputs.charts }} + + steps: + + - name: Checkout + uses: actions/checkout@v2 + + - uses: dorny/paths-filter@v2 + id: filter + with: + token: ${{ secrets.GITHUB_TOKEN }} + filters: | + charts: + - 'charts/ingress-nginx/Chart.yaml' + - 'charts/ingress-nginx/**/*' + + chart: + name: Release Chart + runs-on: ubuntu-latest + needs: + - changes + if: | + (github.repository == 'kubernetes/ingress-nginx') && + (needs.changes.outputs.charts == 'true') + + steps: + + - name: Checkout master + uses: actions/checkout@v2 + with: + # Fetch entire history. Required for chart-releaser; see https://github.com/helm/chart-releaser-action/issues/13#issuecomment-602063896 + fetch-depth: 0 + + - name: Setup + shell: bash + run: | + git config --global user.name "$GITHUB_ACTOR" + git config --global user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.1.0 + env: + CR_TOKEN: "${{ secrets.PERSONAL_TOKEN }}" + CR_RELEASE_NAME_TEMPLATE: "helm-chart-{{ .Version }}" + with: + charts_dir: charts diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml deleted file mode 100644 index 6d32f56f6..000000000 --- a/.github/workflows/images.yaml +++ /dev/null @@ -1,192 +0,0 @@ -name: Container Images - -on: - pull_request: - branches: - - "*" - paths: - - 'images/**' - - push: - branches: - - main - paths: - - 'images/**' - -permissions: - contents: write - packages: write - -env: - PLATFORMS: linux/amd64 - -jobs: - changes: - permissions: - contents: read # for dorny/paths-filter to fetch a list of changed files - pull-requests: read # for dorny/paths-filter to read pull requests - runs-on: ubuntu-latest - outputs: - custom-error-pages: ${{ steps.filter.outputs.custom-error-pages }} - cfssl: ${{ steps.filter.outputs.cfssl }} - fastcgi-helloserver: ${{ steps.filter.outputs.fastcgi-helloserver }} - e2e-test-echo: ${{ steps.filter.outputs.e2e-test-echo }} - go-grpc-greeter-server: ${{ steps.filter.outputs.go-grpc-greeter-server }} - httpbun: ${{ steps.filter.outputs.httpbun }} - kube-webhook-certgen: ${{ steps.filter.outputs.kube-webhook-certgen }} - ext-auth-example-authsvc: ${{ steps.filter.outputs.ext-auth-example-authsvc }} - nginx: ${{ steps.filter.outputs.nginx }} - - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 - id: filter - with: - token: ${{ secrets.GITHUB_TOKEN }} - filters: | - custom-error-pages: - - 'images/custom-error-pages/**' - cfssl: - - 'images/cfssl/**' - fastcgi-helloserver: - - 'images/fastcgi-helloserver/**' - e2e-test-echo: - - 'images/e2e-test-echo/**' - go-grpc-greeter-server: - - 'images/go-grpc-greeter-server/**' - httpbun: - - 'images/httpbun/**' - kube-webhook-certgen: - - 'images/kube-webhook-certgen/**' - ext-auth-example-authsvc: - - 'images/ext-auth-example-authsvc/**' - nginx: - - 'images/nginx/**' - - #### TODO: Make the below jobs 'less dumb' and use the job name as parameter (the github.job context does not work here) - cfssl: - needs: changes - if: | - (needs.changes.outputs.cfssl == 'true') - uses: ./.github/workflows/zz-tmpl-images.yaml - with: - name: cfssl - secrets: inherit - - custom-error-pages: - needs: changes - if: | - (needs.changes.outputs.custom-error-pages == 'true') - uses: ./.github/workflows/zz-tmpl-images.yaml - with: - name: custom-error-pages - secrets: inherit - - e2e-test-echo: - needs: changes - if: | - (needs.changes.outputs.e2e-test-echo == 'true') - uses: ./.github/workflows/zz-tmpl-images.yaml - with: - name: e2e-test-echo - secrets: inherit - - ext-auth-example-authsvc: - needs: changes - if: | - (needs.changes.outputs.ext-auth-example-authsvc == 'true') - uses: ./.github/workflows/zz-tmpl-images.yaml - with: - name: ext-auth-example-authsvc - secrets: inherit - - fastcgi-helloserver: - needs: changes - if: | - (needs.changes.outputs.fastcgi-helloserver == 'true') - uses: ./.github/workflows/zz-tmpl-images.yaml - with: - name: fastcgi-helloserver - secrets: inherit - - go-grpc-greeter-server: - needs: changes - if: | - (needs.changes.outputs.go-grpc-greeter-server == 'true') - uses: ./.github/workflows/zz-tmpl-images.yaml - with: - name: go-grpc-greeter-server - secrets: inherit - - httpbun: - needs: changes - if: | - (needs.changes.outputs.httpbun == 'true') - uses: ./.github/workflows/zz-tmpl-images.yaml - with: - name: httpbun - secrets: inherit - - kube-webhook-certgen: - runs-on: ubuntu-latest - needs: changes - if: | - (needs.changes.outputs.kube-webhook-certgen == 'true') - strategy: - matrix: - k8s: [v1.28.15, v1.29.12, v1.30.8, v1.31.4, v1.32.0] - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Get go version - run: echo "GOLANG_VERSION=$(cat GOLANG_VERSION)" >> $GITHUB_ENV - - - name: Set up Go - id: go - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 - with: - go-version: ${{ env.GOLANG_VERSION }} - check-latest: true - - name: image build - run: | - cd images/ && make NAME=kube-webhook-certgen build - - name: Create Kubernetes cluster - id: kind - run: | - kind create cluster --image=kindest/node:${{ matrix.k8s }} - - name: image test - run: | - cd images/ && make NAME=kube-webhook-certgen test test-e2e - - nginx: - permissions: - contents: write - packages: write - runs-on: ubuntu-latest - needs: changes - if: | - (github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.changes.outputs.nginx == 'true') - env: - PLATFORMS: linux/amd64,linux/arm,linux/arm64 - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Set up QEMU - uses: docker/setup-qemu-action@4574d27a4764455b42196d70a065bc6853246a25 # v3.4.0 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca # v3.9.0 - with: - version: latest - platforms: ${{ env.PLATFORMS }} - - name: Login to GitHub Container Registry - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: build-image - run: | - export TAG=$(cat images/nginx/TAG) - cd images/nginx/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --push -t ingressnginx/nginx:${TAG} . diff --git a/.github/workflows/junit-reports.yaml b/.github/workflows/junit-reports.yaml deleted file mode 100644 index e2a82910e..000000000 --- a/.github/workflows/junit-reports.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: 'E2E Test Report' - -on: - workflow_run: - workflows: ['CI'] # runs after CI workflow - types: - - completed - -permissions: - checks: write - -jobs: - report: - runs-on: ubuntu-latest - steps: - - uses: dorny/test-reporter@31a54ee7ebcacc03a09ea97a7e5465a47b84aea5 # v1.9.1 - with: - artifact: /e2e-test-reports-(.*)/ - name: JEST Tests $1 # Name of the check run which will be created - path: 'report*.xml' # Path to test results (inside artifact .zip) - reporter: jest-junit # Format of test results - fail-on-empty: 'true' diff --git a/.github/workflows/perftest.yaml b/.github/workflows/perftest.yaml deleted file mode 100644 index de22d53d9..000000000 --- a/.github/workflows/perftest.yaml +++ /dev/null @@ -1,72 +0,0 @@ -name: Performance Test - -on: - workflow_dispatch: - inputs: - logLevel: - description: 'Log level' - required: true - default: 'warning' - tags: - description: 'K6 Load Test' - -permissions: - contents: read - -jobs: - k6_test_run: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Install K6 - run: | - wget https://github.com/grafana/k6/releases/download/v0.38.2/k6-v0.38.2-linux-amd64.tar.gz - echo '7c9e5a26aaa2c638c042f6dfda7416161b8d2e0d4cb930721a38083b8be109ab *k6-v0.38.2-linux-amd64.tar.gz' | shasum -c - tar -xvf k6-v0.38.2-linux-amd64.tar.gz k6-v0.38.2-linux-amd64/k6 - mv k6-v0.38.2-linux-amd64/k6 . - ./k6 - - - name: Make dev-env - run: | - mkdir $HOME/.kube - make dev-env - podName=`kubectl -n ingress-nginx get po | grep -i controller | awk '{print $1}'` - if [[ -z ${podName} ]] ; then - sleep 5 - fi - kubectl wait pod -n ingress-nginx --for condition=Ready $podName - kubectl get all -A - - - name: Deploy workload - run: | - kubectl create deploy k6 --image kennethreitz/httpbin --port 80 && \ - kubectl expose deploy k6 --port 80 && \ - kubectl create ing k6 --class nginx \ - --rule test.ingress-nginx-controller.ga/*=k6:80 - podName=`kubectl get po | grep -i k6 | awk '{print $1}'` - if [[ -z ${podName} ]] ; then - sleep 5 - fi - kubectl wait pod --for condition=Ready $podName - kubectl get all,secrets,ing - - - name: Tune OS - run : | - sudo sysctl -A 2>/dev/null | egrep -i "local_port_range|tw_reuse|tcp_timestamps" - sudo sh -c "ulimit" - sudo sysctl -w net.ipv4.ip_local_port_range="1024 65535" - sudo sysctl -w net.ipv4.tcp_tw_reuse=1 - sudo sysctl -w net.ipv4.tcp_timestamps=1 - sudo sh -c "ulimit " - - - name: Run smoke test - run: | - vmstat -at 5 | tee vmstat_report & - #./k6 login cloud -t $K6_TOKEN - #./k6 run -o cloud ./smoketest.js - ./k6 run test/k6/smoketest.js - pkill vmstat - cat vmstat_report diff --git a/.github/workflows/plugin.yaml b/.github/workflows/plugin.yaml deleted file mode 100644 index 20f2caeae..000000000 --- a/.github/workflows/plugin.yaml +++ /dev/null @@ -1,50 +0,0 @@ -name: kubectl plugin - -on: - release: - types: [published] - -permissions: - contents: write # for goreleaser/goreleaser-action - -jobs: - release-plugin: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - - name: Get go version - run: echo "GOLANG_VERSION=$(cat GOLANG_VERSION)" >> $GITHUB_ENV - - - name: Set up Go - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 - with: - go-version: ${{ env.GOLANG_VERSION }} - check-latest: true - - - name: Run GoReleaser Snapshot - if: ${{ ! startsWith(github.ref, 'refs/tags/') }} - uses: goreleaser/goreleaser-action@90a3faa9d0182683851fbfa97ca1a2cb983bfca3 # v6.2.1 - with: - version: "~> v2" - args: release --snapshot --clean - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Run GoReleaser - if: ${{ startsWith(github.ref, 'refs/tags/') }} - uses: goreleaser/goreleaser-action@90a3faa9d0182683851fbfa97ca1a2cb983bfca3 # v6.2.1 - with: - version: "~> v2" - args: release --clean - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Update new version in krew-index - if: ${{ startsWith(github.ref, 'refs/tags/') }} - uses: rajatjindal/krew-release-bot@3d9faef30a82761d610544f62afddca00993eef9 # v0.0.47 - with: - krew_template_file: cmd/plugin/krew.yaml diff --git a/.github/workflows/project.yml b/.github/workflows/project.yml deleted file mode 100644 index 9babf234e..000000000 --- a/.github/workflows/project.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Adds all issues - -on: - issues: - types: - - opened - -jobs: - add-to-project: - name: Add issue to project - runs-on: ubuntu-latest - permissions: - repository-projects: write - issues: write - steps: - - uses: actions/add-to-project@244f685bbc3b7adfa8466e08b698b5577571133e # v1.0.2 - with: - project-url: https://github.com/orgs/kubernetes/projects/104 - github-token: ${{ secrets.PROJECT_WRITER }} diff --git a/.github/workflows/release.txt b/.github/workflows/release.txt new file mode 100644 index 000000000..18039311e --- /dev/null +++ b/.github/workflows/release.txt @@ -0,0 +1,128 @@ +name: Release + +on: + watch: + types: [started] + +jobs: + release: + runs-on: ubuntu-latest + if: github.actor == 'aledbf' + steps: + - name: Checkout master + uses: actions/checkout@v2 + + - name: Set up Go 1.14 + uses: actions/setup-go@master + with: + go-version: 1.14 + + - name: Build krew plugin + shell: bash + run: | + export PATH=${PATH}:`go env GOPATH`/bin + make build-plugin + +# - name: Build and publish images +# shell: bash +# run: | +# cat < build/images/nginx/aws.tfvars +# access_key = "${{ secrets.AWS_ACCESS_KEY }}" +# secret_key = "${{ secrets.AWS_ACCESS_SECRET_KEY }}" +# EOF +# +# cat < build/images/nginx/env.tfvars +# docker_username="${{ secrets.DOCKER_USERNAME }}" +# docker_password="${{ secrets.DOCKER_PASSWORD }}" +# EOF +# +# ./build/build-ingress-controller.sh + + - name: Create env variables + shell: bash + run: | + # $(cat cmd/plugin/release/ingress-nginx.yaml) + echo "::set-env name=RELEASE_VERSION::$(make show-version)" + echo "::set-env name=RELEASE_CHANGELOG::##CHANGELOG##" + + - name: Create release + id: create_release + uses: actions/create-release@v1 + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/controller') + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: controller-${{ env.RELEASE_VERSION }} + release_name: "NGINX: ${{ env.RELEASE_VERSION }}" + body: | + **Image:** `quay.io/kubernetes-ingress-controller/nginx-ingress-controller:${{ env.RELEASE_VERSION }}` + + ${{ env.RELEASE_CHANGELOG }} + + Complete changelog [here](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md) + + draft: false + prerelease: false + + - name: Upload krew plugin Darwin + uses: actions/upload-release-asset@v1 + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/controller') + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: cmd/plugin/release/kubectl-ingress_nginx-darwin-amd64.tar.gz + asset_name: kubectl-ingress_nginx-darwin-amd64.tar.gz + asset_content_type: application/tar+gzip + + - name: Upload krew plugin Linux + uses: actions/upload-release-asset@v1 + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/controller') + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: cmd/plugin/release/kubectl-ingress_nginx-linux-amd64.tar.gz + asset_name: kubectl-ingress_nginx-linux-amd64.tar.gz + asset_content_type: application/tar+gzip + + - name: Upload krew plugin Windows + uses: actions/upload-release-asset@v1 + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/controller') + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: cmd/plugin/release/kubectl-ingress_nginx-windows-amd64.tar.gz + asset_name: kubectl-ingress_nginx-windows-amd64.tar.gz + asset_content_type: application/tar+gzip + + - name: Upload krew plugin + uses: actions/upload-release-asset@v1 + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/controller') + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: cmd/plugin/release/ingress-nginx.yaml + asset_name: ingress-nginx.yaml + asset_content_type: text/x-yaml + + - name: Create KREW_PLUGIN_BODY_PR env variable + shell: bash + run: | + echo "::set-env name=KREW_PLUGIN_BODY_PR::$(cat cmd/plugin/release/ingress-nginx.yaml)" + +# - name: Open PR for new release of Krew Plugin +# uses: gha-utilities/init-pull-request@v0.1.2 +# env: +# GITHUB_REPOSITORY: maintainer/repo-name +# with: +# verbose: true +# draft: false +# pull_request_token: ${{ secrets.GITHUB_TOKEN }} +# head: ingress-nginx:krew +# base: krew +# title: 'Update ingress-nginx krew plugin to ${{ env.RELEASE_VERSION }}' +# body: | +# ${{ env.KREW_PLUGIN_BODY_PR }} diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml deleted file mode 100644 index a4473710d..000000000 --- a/.github/workflows/scorecards.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Scorecards supply-chain security - -on: - # Only the default branch is supported. - branch_protection_rule: - schedule: - - cron: '20 11 * * 5' - push: - branches: - - "main" - -# Declare default permissions as read only. -permissions: read-all - -jobs: - analysis: - name: Scorecards analysis - runs-on: ubuntu-latest - permissions: - # Needed to upload the results to code-scanning dashboard. - security-events: write - # Used to receive a badge. (Upcoming feature) - id-token: write - # Needs for private repositories. - contents: read - actions: read - - steps: - - name: "Checkout code" - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - - name: "Run analysis" - uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 - with: - results_file: results.sarif - results_format: sarif - # (Optional) Read-only PAT token. Uncomment the `repo_token` line below if: - # - you want to enable the Branch-Protection check on a *public* repository, or - # - you are installing Scorecards on a *private* repository - # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. - # repo_token: ${{ secrets.SCORECARD_READ_TOKEN }} - - # Publish the results for public repositories to enable scorecard badges. For more details, see - # https://github.com/ossf/scorecard-action#publishing-results. - # For private repositories, `publish_results` will automatically be set to `false`, regardless - # of the value entered here. - publish_results: true - - # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF - # format to the repository Actions tab. - - name: "Upload artifact" - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 - with: - name: SARIF file - path: results.sarif - retention-days: 5 - - # Upload the results to GitHub's code scanning dashboard. - - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0 # v3.28.9 - with: - sarif_file: results.sarif diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml deleted file mode 100644 index 45a7cd320..000000000 --- a/.github/workflows/stale.yaml +++ /dev/null @@ -1,24 +0,0 @@ -name: 'Stale Issues and PRs' - -on: - schedule: - - cron: '30 1 * * *' - -jobs: - stale: - runs-on: ubuntu-latest - - permissions: - issues: write - pull-requests: write - - steps: - - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0 - with: - stale-issue-message: "This is stale, but we won't close it automatically, just bare in mind the maintainers may be busy with other tasks and will reach your issue ASAP. If you have any question or request to prioritize this, please reach `#ingress-nginx-dev` on Kubernetes Slack." - stale-pr-message: "This is stale, but we won't close it automatically, just bare in mind the maintainers may be busy with other tasks and will reach your issue ASAP. If you have any question or request to prioritize this, please reach `#ingress-nginx-dev` on Kubernetes Slack." - stale-issue-label: lifecycle/frozen - stale-pr-label: lifecycle/frozen - days-before-issue-stale: 30 - days-before-pr-stale: 45 - days-before-close: -1 # dont not close issues/prs diff --git a/.github/workflows/vulnerability-scans.yaml b/.github/workflows/vulnerability-scans.yaml deleted file mode 100644 index 4461d9757..000000000 --- a/.github/workflows/vulnerability-scans.yaml +++ /dev/null @@ -1,92 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Vulnerability Scan - -on: - workflow_dispatch: - release: - schedule: - - cron: '00 9 * * 1' - -permissions: - contents: read - security-events: write - -jobs: - version: - runs-on: ubuntu-latest - outputs: - versions: ${{ steps.version.outputs.TAGS }} - steps: - - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - - name: Latest Tag - id: version - shell: bash - run: | - readarray -t TAGS_ARRAY <<<"$(git tag --list 'controller-v*.*.*' --sort=-version:refname | grep -v 'beta\|alpha')" - FULL_TAGS=(${TAGS_ARRAY[0]} ${TAGS_ARRAY[1]} ${TAGS_ARRAY[2]}) - SHORT_TAGS=() - for i in ${FULL_TAGS[@]} - do - echo "tag: $i" - short=$(echo "$i" | cut -d - -f 2) - SHORT_TAGS+=($short) - done - echo "${SHORT_TAGS[0]},${SHORT_TAGS[1]},${SHORT_TAGS[2]}" - TAGS_JSON="[\"${SHORT_TAGS[0]}\",\"${SHORT_TAGS[1]}\",\"${SHORT_TAGS[2]}\"]" - echo "${TAGS_JSON}" - echo "TAGS=${TAGS_JSON}" >> $GITHUB_OUTPUT - - scan: - runs-on: ubuntu-latest - needs: version - strategy: - matrix: - versions: ${{ fromJSON(needs.version.outputs.versions) }} - steps: - - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - shell: bash - id: test - run: echo "Scanning registry.k8s.io/ingress-nginx/controller@${{ matrix.versions }}" - - - name: Scan image with AquaSec/Trivy - id: scan - uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0 # v0.29.0 - with: - image-ref: registry.k8s.io/ingress-nginx/controller:${{ matrix.versions }} - format: 'sarif' - output: trivy-results-${{ matrix.versions }}.sarif - exit-code: 0 - vuln-type: 'os,library' - severity: 'CRITICAL,HIGH,MEDIUM,LOW,UNKNOWN' - - - name: Output Sarif File - shell: bash - run: cat ${{ github.workspace }}/trivy-results-${{ matrix.versions }}.sarif - - # This step checks out a copy of your repository. - - name: Upload SARIF file - uses: github/codeql-action/upload-sarif@9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0 # v3.28.9 - with: - token: ${{ github.token }} - # Path to SARIF file relative to the root of the repository - sarif_file: ${{ github.workspace }}/trivy-results-${{ matrix.versions }}.sarif - - - name: Vulz Count - shell: bash - run: | - TRIVY_COUNT=$(cat ${{ github.workspace }}/trivy-results-${{ matrix.versions }}.sarif | jq '.runs[0].results | length') - echo "TRIVY_COUNT: $TRIVY_COUNT" - echo "Image Vulnerability scan output" >> $GITHUB_STEP_SUMMARY - echo "Image ID: registry.k8s.io/ingress-nginx/controller@${{ matrix.versions }}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Trivy Count: $TRIVY_COUNT" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/zz-tmpl-images.yaml b/.github/workflows/zz-tmpl-images.yaml deleted file mode 100644 index 5e98ddf70..000000000 --- a/.github/workflows/zz-tmpl-images.yaml +++ /dev/null @@ -1,81 +0,0 @@ -#### THIS IS A TEMPLATE #### -# This workflow is created to be a template for every time an e2e test is required, - -on: - workflow_call: - inputs: - name: - required: true - type: string - platforms-test: - type: string - default: linux/amd64 - platforms-publish: - type: string - default: linux/amd64 - -env: - PLATFORMS: ${{ inputs.platforms-test }} - -permissions: - contents: write - packages: write - -jobs: - changestag: - permissions: - contents: read # for dorny/paths-filter to fetch a list of changed files - runs-on: ubuntu-latest - outputs: - tag: ${{ steps.filter.outputs.tag }} - - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 - id: filter - with: - token: ${{ secrets.GITHUB_TOKEN }} - filters: | - tag: - - 'images/**/TAG' - - image-build: - name: Build - runs-on: ubuntu-latest - permissions: - contents: read - - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Build - run: | - cd images/ && make NAME=${{ inputs.name }} build - - image-push: - name: Push - needs: changestag - if: | - (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'kubernetes/ingress-nginx' && needs.changestag.outputs.tag == 'true') - runs-on: ubuntu-latest - permissions: - contents: write - packages: write - env: - PLATFORMS: ${{ inputs.platforms-publish }} - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Push - run: | - cd images/ && make REGISTRY=ingressnginx NAME=${{ inputs.name }} push - diff --git a/.github/workflows/zz-tmpl-k8s-e2e.yaml b/.github/workflows/zz-tmpl-k8s-e2e.yaml deleted file mode 100644 index c46e4a957..000000000 --- a/.github/workflows/zz-tmpl-k8s-e2e.yaml +++ /dev/null @@ -1,57 +0,0 @@ -#### THIS IS A TEMPLATE #### -# This workflow is created to be a template for every time an e2e test is required, - -on: - workflow_call: - inputs: - k8s-version: - required: true - type: string - variation: - type: string - -permissions: - contents: read - -jobs: - kubernetes: - name: Kubernetes ${{ inputs.variation }} - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: cache - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - with: - name: docker.tar.gz - - - name: Create Kubernetes ${{ inputs.k8s-version }} cluster - id: kind - run: | - kind create cluster --image=kindest/node:${{ inputs.k8s-version }} --config test/e2e/kind.yaml - - - name: Load images from cache - run: | - echo "loading docker images..." - gzip -dc docker.tar.gz | docker load - - - name: Run e2e tests ${{ inputs.variation }} - env: - KIND_CLUSTER_NAME: kind - SKIP_CLUSTER_CREATION: true - SKIP_INGRESS_IMAGE_CREATION: true - SKIP_E2E_IMAGE_CREATION: true - IS_CHROOT: ${{ inputs.variation == 'CHROOT' }} - run: | - kind get kubeconfig > $HOME/.kube/kind-config-kind - make kind-e2e-test - - - name: Upload e2e junit-reports ${{ inputs.variation }} - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 - if: success() || failure() - with: - name: e2e-test-reports-${{ inputs.k8s-version }}${{ inputs.variation }} - path: 'test/junitreports/report*.xml' - diff --git a/.gitignore b/.gitignore index 5eac1a800..257ff6594 100644 --- a/.gitignore +++ b/.gitignore @@ -46,7 +46,6 @@ test/e2e/e2e\.test bin test/e2e-image/wait-for-nginx.sh .cache -.modcache cover.out # secret terraform variables @@ -58,6 +57,3 @@ images/fastcgi-helloserver/rootfs/fastcgi-helloserver cmd/plugin/release/ingress-nginx.yaml cmd/plugin/release/*.tar.gz cmd/plugin/release/LICENSE -tmp/ -test/junitreports/ -tests/__snapshot__ diff --git a/.golangci.yml b/.golangci.yml deleted file mode 100644 index 729468711..000000000 --- a/.golangci.yml +++ /dev/null @@ -1,236 +0,0 @@ -run: - timeout: 10m - allow-parallel-runners: true - -issues: - # Maximum issues count per one linter. Set to 0 to disable. Default is 50. - max-issues-per-linter: 0 - - # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. - max-same-issues: 0 -linters: - disable-all: true - enable: - - asasalint - - asciicheck - - bidichk - - bodyclose - - contextcheck - - decorder - - dogsled - - dupl - - durationcheck - - errcheck - - errchkjson - - errname - - ginkgolinter - - gocheckcompilerdirectives - - goconst - - gocritic - - gocyclo - - godox - - gofmt - - gofumpt - - goheader - - goimports - - gomoddirectives - - gomodguard - - goprintffuncname - - gosec - - gosimple - - govet - - grouper - - importas - - ineffassign - - loggercheck - - makezero - - misspell - - musttag - - nakedret - - nolintlint - - nosprintfhostport - - prealloc - - predeclared - - promlinter - - reassign - - revive - - rowserrcheck - - sqlclosecheck - - staticcheck - - stylecheck - - tenv - - testableexamples - - typecheck - - unconvert - - unparam - - unused - - usestdlibvars - - whitespace - # - containedctx - # - cyclop - # - dupword - # - errorlint - # - exhaustive - # - exhaustruct - # - exportloopref - # - forbidigo - # - forcetypeassert - # - funlen - # - gci - # - gochecknoglobals - # - gochecknoinits - # - gocognit - # - godot - # - goerr113 - # - gomnd - # - interfacebloat - # - ireturn - # - lll - # - maintidx - # - nestif - # - nilerr - # - nilnil - # - nlreturn - # - noctx - # - nonamedreturns - # - paralleltest - # - tagliatelle - # - testpackage - # - thelper - # - tparallel - # - varnamelen - # - wastedassign - # - wrapcheck - # - wsl -linters-settings: - gocyclo: - min-complexity: 40 - godox: - keywords: - - BUG - - FIXME - - HACK - errcheck: - check-type-assertions: true - check-blank: true - gocritic: - enabled-checks: - # Diagnostic - - appendAssign - - argOrder - - badCall - - badCond - - badLock - - badRegexp - - badSorting - - builtinShadowDecl - - caseOrder - - codegenComment - - commentedOutCode - - deferInLoop - - deprecatedComment - - dupArg - - dupBranchBody - - dupCase - - dupSubExpr - - dynamicFmtString - - emptyDecl - - evalOrder - - exitAfterDefer - - externalErrorReassign - - filepathJoin - - flagDeref - - flagName - - mapKey - - nilValReturn - - offBy1 - - regexpPattern - - returnAfterHttpError - - sloppyReassign - - sloppyTypeAssert - - sortSlice - - sprintfQuotedString - - sqlQuery - - syncMapLoadAndDelete - - truncateCmp - - unnecessaryDefer - - weakCond - - # Performance - - appendCombine - - equalFold - - hugeParam - - indexAlloc - - preferDecodeRune - - preferFprint - - preferStringWriter - - preferWriteByte - - rangeExprCopy - - rangeValCopy - - sliceClear - - stringXbytes - - # Style - - assignOp - - boolExprSimplify - - captLocal - - commentFormatting - - commentedOutImport - - defaultCaseOrder - - deferUnlambda - - docStub - - dupImport - - elseif - - emptyFallthrough - - emptyStringTest - - exposedSyncMutex - - hexLiteral - - httpNoBody - - ifElseChain - - methodExprCall - - newDeref - - octalLiteral - - preferFilepathJoin - - redundantSprint - - regexpMust - - regexpSimplify - - ruleguard - - singleCaseSwitch - - sloppyLen - - stringConcatSimplify - - stringsCompare - - switchTrue - - timeExprSimplify - - tooManyResultsChecker - - typeAssertChain - - typeDefFirst - - typeSwitchVar - - underef - - unlabelStmt - - unlambda - - unslice - - valSwap - - whyNoLint - - wrapperFunc - - yodaStyleExpr - - # Opinionated - - builtinShadow - - importShadow - - initClause - - nestingReduce - - paramTypeCombine - - ptrToRefParam - - typeUnparen - - unnamedResult - - unnecessaryBlock - nolintlint: - # Enable to ensure that nolint directives are all used. Default is true. - allow-unused: false - # Exclude following linters from requiring an explanation. Default is []. - allow-no-explanation: [] - # Enable to require an explanation of nonzero length after each nolint directive. Default is false. - # TODO(lint): Enforce explanations for `nolint` directives - require-explanation: false - # Enable to require nolint directives to mention the specific linter being suppressed. Default is false. - require-specific: true diff --git a/.goreleaser.yaml b/.goreleaser.yaml deleted file mode 100644 index a0ef6eb3d..000000000 --- a/.goreleaser.yaml +++ /dev/null @@ -1,29 +0,0 @@ -project_name: ingress-nginx -release: - github: - owner: kubernetes - name: ingress-nginx -builds: - - id: ingress-nginx - goos: - - darwin - - linux - - windows - goarch: - - arm64 - - amd64 - env: - - CGO_ENABLED=0 - - GO111MODULE=on - main: cmd/plugin/main.go - binary: kubectl-ingress-nginx - ldflags: | - -s -w - -X k8s.io/ingress-nginx/version.COMMIT={{ .Commit }} - -X k8s.io/ingress-nginx/version.RELEASE={{ .Tag }} -archives: - - id: ingress-nginx - builds: - - ingress-nginx - name_template: "kubectl-{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}" - format: tar.gz diff --git a/.luacheckrc b/.luacheckrc index 5a1c249fe..a24399e26 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -1,6 +1,6 @@ std = 'ngx_lua' max_line_length = 100 -exclude_files = {'./rootfs/etc/nginx/lua/test/**/*.lua'} +exclude_files = {'./rootfs/etc/nginx/lua/test/**/*.lua', './rootfs/etc/nginx/lua/plugins/**/test/**/*.lua'} files["rootfs/etc/nginx/lua/lua_ingress.lua"] = { ignore = { "122" }, -- TODO(elvinefendi) figure out why this does not work diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a11435aef..5dd01830b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,8 +2,6 @@ Read the following guide if you're interested in contributing to Ingress. [Make Ingress-Nginx Work for you, and the Community](https://youtu.be/GDm-7BlmPPg) from KubeCon Europe 2018 is a great video to get you started!! -Note that this guide refers to contributing to actual sources of the repository. If you interested in contributing through issue triaging, have a look at [this guide](./ISSUE_TRIAGE.md). - ## Contributor License Agreements We'd love to accept your patches! Before we can take them, we have to jump a couple of legal hurdles. @@ -17,16 +15,14 @@ Follow either of the two links above to access the appropriate CLA and instructi ***NOTE***: Only original source code from you and other people that have signed the CLA can be accepted into the main repository. -## Finding Issues That Need Help +## Finding Things That Need Help -If you're new to the project and want to help, but don't know where to start, we have a semi-curated list of issues that should not need deep knowledge of the system. [Have a look and see if anything sounds interesting](https://github.com/kubernetes/ingress-nginx/issues?utf8=%E2%9C%93&q=is%3Aopen%20is%3Aissue%20label%3A%22help+wanted%22). - -Alternatively, search for the label [`triage-accepted`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+is%3Aissue+label%3Atriage%2Faccepted+) if you have some experience with ingress-nginx. Note, that it could make sense to grab issues with higher priority first. +If you're new to the project and want to help, but don't know where to start, we have a semi-curated list of issues that should not need deep knowledge of the system. [Have a look and see if anything sounds interesting](https://github.com/kubernetes/ingress-nginx/issues?utf8=%E2%9C%93&q=is%3Aopen%20is%3Aissue%20label%3A%22help+wanted%22). Alternatively, read some of the docs on other controllers and try to write your own, file and fix any/all issues that come up, including gaps in documentation! ## Contributing a Patch 1. If you haven't already done so, sign a Contributor License Agreement (see details above). -1. Read the [Ingress development guide](docs/developer-guide/getting-started.md). +1. Read the [Ingress development guide](docs/development.md). 1. Fork the desired repo, develop and test your code changes. 1. Submit a pull request. @@ -34,9 +30,7 @@ All changes must be code reviewed. Coding conventions and standards are explaine ### Merge Approval -Ingress Nginx collaborators may add "/lgtm" (Looks Good To Me) to indicate that a PR is acceptable. Any change requires at least one LGTM. No pull requests can be merged until at least one Ingress Nginx collaborator signs off with an LGTM. Adding the "/lgtm" comment result in the prow bot adding the `lgtm` label. Note that a pull request still needs an `approve` label from one of the owners. - -Reviewers or members who want to become reviewers according to the [k8s membership ladder](https://github.com/kubernetes/community/blob/master/community-membership.md), could actively search for [pull requests that need a review](https://github.com/kubernetes/ingress-nginx/pulls?q=is%3Aopen+is%3Apr+label%3Atriage%2Faccepted). +Ingress collaborators may add "LGTM" (Looks Good To Me) or an equivalent comment to indicate that a PR is acceptable. Any change requires at least one LGTM. No pull requests can be merged until at least one Ingress collaborator signs off with an LGTM. ## Support Channels @@ -47,6 +41,3 @@ Whether you are a user or contributor, official support channels include: - Post: [Kubernetes Forum](https://discuss.kubernetes.io) Before opening a new issue or submitting a new pull request, it's helpful to search the project - it's likely that another user has already reported the issue you're facing, or it's a known issue that we're already aware of. - -## New Contributor Tips -If you're a new contributor, you can follow the [New Contributor Tips guide](NEW_CONTRIBUTOR.md) diff --git a/Changelog.md b/Changelog.md index f049654ee..d5316fc85 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,965 +1,5 @@ # Changelog -All New change are in [Changelog](./changelog) - -### 1.5.1 - -* Upgrade NGINX to 1.21.6 -* Upgrade Golang 1.19.2 -* Fix Service Name length Bug [9245](https://github.com/kubernetes/ingress-nginx/pull/9245) -* CVE fixes CVE-2022-32149, CVE-2022-27664, CVE-2022-1996 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.5.1@sha256:4ba73c697770664c1e00e9f968de14e08f606ff961c76e5d7033a4a9c593c629 -* registry.k8s.io/ingress-nginx/controller-chroot:v1.5.1@sha256:c1c091b88a6c936a83bd7b098662760a87868d12452529bad0d178fb36147345 - -### All Changes: - -* chore Fixed to Support Versions table by @yutachaos in https://github.com/kubernetes/ingress-nginx/pull/9117 -* Updated incorrect version number in the Installation Guide by @afro-coder in https://github.com/kubernetes/ingress-nginx/pull/9120 -* Updated the Developer guide with New Contributor information by @afro-coder in https://github.com/kubernetes/ingress-nginx/pull/9114 -* Remove deprecated net dependency by @rikatz in https://github.com/kubernetes/ingress-nginx/pull/9110 -* Fixed docs helm-docs version by @yutachaos in https://github.com/kubernetes/ingress-nginx/pull/9121 -* Fix CVE 2022 27664 by @strongjz in https://github.com/kubernetes/ingress-nginx/pull/9109 -* upgrade to golang 1.19.2 by @strongjz in https://github.com/kubernetes/ingress-nginx/pull/9124 -* fix e2e resource leak when ginkgo exit before clear resource by @loveRhythm1990 in https://github.com/kubernetes/ingress-nginx/pull/9103 -* fix: handle 401 and 403 by external auth by @johanneswuerbach in https://github.com/kubernetes/ingress-nginx/pull/9131 -* Move bowei to emeritus owner by @rikatz in https://github.com/kubernetes/ingress-nginx/pull/9150 -* fix null ports by @tombokombo in https://github.com/kubernetes/ingress-nginx/pull/9149 -* Documentation added for implemented redirection in the proxy to ensure image pulling by @Sanghamitra-PERSONAL in https://github.com/kubernetes/ingress-nginx/pull/9098 -* updating runner with golang 1.19.2 by @strongjz in https://github.com/kubernetes/ingress-nginx/pull/9158 -* Add install command for OVHcloud by @scraly in https://github.com/kubernetes/ingress-nginx/pull/9171 -* GitHub Templates: Remove trailing whitespaces. by @Gacko in https://github.com/kubernetes/ingress-nginx/pull/9172 -* Update helm chart changelog to show that kubernetes v1.21.x is no longer supported by @cskinfill in https://github.com/kubernetes/ingress-nginx/pull/9147 -* Add section to troubleshooting docs for failure to listen on port by @jrhunger in https://github.com/kubernetes/ingress-nginx/pull/9185 -* Implement parseFloat for annotations by @kirs in https://github.com/kubernetes/ingress-nginx/pull/9195 -* fix typo in docs. by @guettli in https://github.com/kubernetes/ingress-nginx/pull/9167 -* add:(admission-webhooks) ability to set securityContext by @ybelMekk in https://github.com/kubernetes/ingress-nginx/pull/9186 -* Fix Markdown header level by @jaens in https://github.com/kubernetes/ingress-nginx/pull/9210 -* chore: bump NGINX version v1.21.4 by @tao12345666333 in https://github.com/kubernetes/ingress-nginx/pull/8889 -* chore: update NGINX to 1.21.6 by @tao12345666333 in https://github.com/kubernetes/ingress-nginx/pull/9231 -* fix svc long name by @tombokombo in https://github.com/kubernetes/ingress-nginx/pull/9245 -* update base image of nginx to 1.21.6 by @strongjz in https://github.com/kubernetes/ingress-nginx/pull/9257 -* Fix CVE-2022-32149 by @esigo in https://github.com/kubernetes/ingress-nginx/pull/9258 -* Fix CVE-2022-1996 by @esigo in https://github.com/kubernetes/ingress-nginx/pull/9244 -* Adding support for disabling liveness and readiness probes to the Helm chart by @njegosrailic in https://github.com/kubernetes/ingress-nginx/pull/9238 -* fix CVE-2022-27664 by @esigo in https://github.com/kubernetes/ingress-nginx/pull/9273 -* Add CVE-2022-27664 #9273 in latest release by @strongjz in https://github.com/kubernetes/ingress-nginx/pull/9275 - -### Dependencies updates: - -* Bump docker/setup-buildx-action from 2.0.0 to 2.1.0 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9180 -* Bump dorny/paths-filter from 2.10.2 to 2.11.1 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9183 -* Bump helm/chart-releaser-action from 1.4.0 to 1.4.1 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9136 -* Bump github/codeql-action from 2.1.25 to 2.1.27 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9137 -* Bump ossf/scorecard-action from 2.0.3 to 2.0.4 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9138 -* Bump google.golang.org/grpc from 1.49.0 to 1.50.0 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9134 -* Bump actions/checkout from 3.0.2 to 3.1.0 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9135 -* Bump actions/dependency-review-action from 2.5.0 to 2.5.1 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9237 -* Bump github/codeql-action from 2.1.28 to 2.1.29 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9236 -* Bump github.com/spf13/cobra from 1.6.0 to 1.6.1 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9233 -* Bump actions/upload-artifact from 3.1.0 to 3.1.1 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9234 -* Bump azure/setup-helm from 3.3 to 3.4 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9235 -* Bump github.com/onsi/ginkgo/v2 from 2.3.1 to 2.4.0 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9201 -* Bump goreleaser/goreleaser-action from 3.1.0 to 3.2.0 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9208 -* Bump github.com/stretchr/testify from 1.8.0 to 1.8.1 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9202 -* Bump ossf/scorecard-action from 2.0.4 to 2.0.6 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9203 -* Bump docker/setup-buildx-action from 2.1.0 to 2.2.1 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9204 -* Bump actions/setup-go from 3.3.0 to 3.3.1 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9205 -* Bump github/codeql-action from 2.1.27 to 2.1.28 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9206 -* Bump actions/download-artifact from 3.0.0 to 3.0.1 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9207 -* Bump github.com/prometheus/client_model from 0.2.0 to 0.3.0 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9200 -* Bump github.com/spf13/cobra from 1.5.0 to 1.6.0 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9173 -* Bump google.golang.org/grpc from 1.50.0 to 1.50.1 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9174 -* Bump k8s.io/component-base from 0.25.2 to 0.25.3 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9175 -* Bump github.com/fsnotify/fsnotify from 1.5.4 to 1.6.0 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9176 -* Bump github.com/onsi/ginkgo/v2 from 2.2.0 to 2.3.1 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9177 -* Bump geekyeggo/delete-artifact from 1.0.0 to 2.0.0 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9178 -* Bump actions/dependency-review-action from 2.4.0 to 2.5.0 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9179 -* Bump docker/setup-qemu-action from 2.0.0 to 2.1.0 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9181 -* Bump securego/gosec from 2.13.1 to 2.14.0 by @dependabot in https://github.com/kubernetes/ingress-nginx/pull/9182 - - -## New Contributors -* @yutachaos made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9117 -* @Gacko made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9123 -* @loveRhythm1990 made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9103 -* @johanneswuerbach made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9131 -* @FutureMatt made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9133 -* @Sanghamitra-PERSONAL made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9098 -* @scraly made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9171 -* @cskinfill made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9147 -* @jrhunger made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9185 -* @guettli made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9167 -* @ybelMekk made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9186 -* @jaens made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9210 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.4.0...controller-v1.5.1 - -### 1.4.0 - -### Community Updates - -We will discuss the results of our Community Survey, progress on the stabilization project, and ideas going -forward with the project at -[Kubecon NA 2022 in Detroit](https://events.linuxfoundation.org/kubecon-cloudnativecon-north-america/). Come join us -and let us hear what you'd like to see in the future for ingress-nginx. - -https://kccncna2022.sched.com/event/18lgl?iframe=no - -[**Kubernetes Registry change notice**](https://twitter.com/BenTheElder/status/1575898507235323904) -The [@kubernetesio](https://twitter.com/kubernetesio) container image host http://k8s.gcr.io is -*actually* getting redirected to the community controlled http://registry.k8s.io starting with a small portion of -traffic on October 3rd. - -If you notice any issues, *please* ping [Ben Elder](https://twitter.com/BenTheElder), -[@thockin](https://twitter.com/thockin), [@ameukam](https://twitter.com/ameukam),or report issues in slack to -[sig-k8s-infra slack channel](https://kubernetes.slack.com/archives/CCK68P2Q2). - -### What's Changed - -* 1.4.0 updates ingress-nginx to use Endpointslices instead of Endpoints. Thank you, @tombokombo, for your work in -[8890](https://github.com/kubernetes/ingress-nginx/pull/8890) -* Update to Prometheus metric names, more information [available here]( https://github.com/kubernetes/ingress-nginx/pull/8728 -) -* Deprecated Kubernetes versions 1.20-1.21, Added support for, 1.25, currently supported versions v1.22, v1.23, v1.24, v1.25 - -ADDED -* `_request_duration_seconds` Histogram -* `_connect_duration_seconds` Histogram -* `_header_duration_seconds` Histogram -* `_response_duration_seconds` Histogram - -Updated -* `_response_size` Histogram -* `_request_size` Histogram -* `_requests` Counter - -DEPRECATED -* `_bytes_sent` Histogram -* _ingress_upstream_latency_seconds` Summary - -REMOVED -* `ingress_upstream_header_seconds` Summary - -Also upgraded to golang 1.19.1 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.4.0@sha256:34ee929b111ffc7aa426ffd409af44da48e5a0eea1eb2207994d9e0c0882d143 -* registry.k8s.io/ingress-nginx/controller-chroot:v1.4.0@sha256:b67e889f1db8692de7e41d4d9aef8de56645bf048261f31fa7f8bfc6ea2222a0 - - -### All Changes: - -* [9104](https://github.com/kubernetes/ingress-nginx/pull/9104) Fix yaml formatting error with multiple annotations -* [9090](https://github.com/kubernetes/ingress-nginx/pull/9090) fix chroot module mount path -* [9088](https://github.com/kubernetes/ingress-nginx/pull/9088) Add annotation for setting sticky cookie domain -* [9086](https://github.com/kubernetes/ingress-nginx/pull/9086) Update Version ModSecurity and Coreruleset -* [9081](https://github.com/kubernetes/ingress-nginx/pull/9081) plugin - endpoints to slices -* [9078](https://github.com/kubernetes/ingress-nginx/pull/9078) expand CI testing for all stable versions of Kubernetes -* [9074](https://github.com/kubernetes/ingress-nginx/pull/9074) fix: do not apply job-patch psp on Kubernetes 1.25 and newer -* [9072](https://github.com/kubernetes/ingress-nginx/pull/9072) Added a Link to the New Contributors Tips -* [9069](https://github.com/kubernetes/ingress-nginx/pull/9069) Add missing space to error message -* [9059](https://github.com/kubernetes/ingress-nginx/pull/9059) kubewebhookcertgen sha change after go1191 -* [9058](https://github.com/kubernetes/ingress-nginx/pull/9058) updated testrunner image sha after bump to go1191 -* [9046](https://github.com/kubernetes/ingress-nginx/pull/9046) Parameterize metrics port name -* [9036](https://github.com/kubernetes/ingress-nginx/pull/9036) update OpenTelemetry image -* [9035](https://github.com/kubernetes/ingress-nginx/pull/9035) Added instructions for Rancher Desktop -* [9028](https://github.com/kubernetes/ingress-nginx/pull/9028) fix otel init_module -* [9023](https://github.com/kubernetes/ingress-nginx/pull/9023) updates for fixing 1.3.1 release -* [9018](https://github.com/kubernetes/ingress-nginx/pull/9018) Add v1.25 test and reduce amount of e2e tests -* [9017](https://github.com/kubernetes/ingress-nginx/pull/9017) fix LD_LIBRARY_PATH for opentelemetry - -### Dependencies updates: - -* [9085](https://github.com/kubernetes/ingress-nginx/pull/9085) Bump actions/dependency-review-action from 2.1.0 to 2.4.0 -* [9084](https://github.com/kubernetes/ingress-nginx/pull/9084) Bump actions/checkout from 1 to 3 -* [9083](https://github.com/kubernetes/ingress-nginx/pull/9083) Bump github/codeql-action from 2.1.24 to 2.1.25 -* [9089](https://github.com/kubernetes/ingress-nginx/pull/9089) Bump k8s.io/component-base from 0.25.1 to 0.25.2 -* [9066](https://github.com/kubernetes/ingress-nginx/pull/9066) Bump github/codeql-action from 2.1.23 to 2.1.24 -* [9065](https://github.com/kubernetes/ingress-nginx/pull/9065) Bump k8s.io/component-base from 0.25.0 to 0.25.1 -* [9064](https://github.com/kubernetes/ingress-nginx/pull/9064) Bump github.com/onsi/ginkgo/v2 from 2.1.6 to 2.2.0 -* [9057](https://github.com/kubernetes/ingress-nginx/pull/9057) bump go to v1.19.1 -* [9053](https://github.com/kubernetes/ingress-nginx/pull/9053) Bump ossf/scorecard-action from 2.0.2 to 2.0.3 -* [9052](https://github.com/kubernetes/ingress-nginx/pull/9052) Bump github/codeql-action from 2.1.22 to 2.1.23 -* [9045](https://github.com/kubernetes/ingress-nginx/pull/9045) Bump actions/upload-artifact from 3.0.0 to 3.1.0 -* [9044](https://github.com/kubernetes/ingress-nginx/pull/9044) Bump ossf/scorecard-action from 1.1.2 to 2.0.2 -* [9043](https://github.com/kubernetes/ingress-nginx/pull/9043) Bump k8s.io/klog/v2 from 2.80.0 to 2.80.1 -* [9022](https://github.com/kubernetes/ingress-nginx/pull/9022) Bump github.com/onsi/ginkgo/v2 from 2.1.4 to 2.1.6 -* [9021](https://github.com/kubernetes/ingress-nginx/pull/9021) Bump k8s.io/klog/v2 from 2.70.1 to 2.80.0 - -## New Contributors -* @gunamata made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9035 -* @afro-coder made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/8924 -* @wilmardo made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9074 -* @nicolasjulian made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9086 -* @mtneug made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9088 -* @knbnnate made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/8692 -* @mklauber made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9104 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.3.1...controller-v1.4.0 - -### 1.3.1 - -In v1.3.1 leader elections will be done entirely using the Lease API and no longer using configmaps. -v1.3.0 is a safe transition version, using v1.3.0 can automatically complete the merging of election locks, and then you can safely upgrade to v1.3.1. - -Also, *important note*, with the Release of Kubernetes v1.25 we are dropping support for the legacy branches, -Also, *important note*, with the release of Kubernetes v1.25, we are dropping support for the legacy edition, -that means all version <1.0.0 of the ingress-nginx-controller. - -## Image: -- registry.k8s.io/ingress-nginx/controller:v1.3.1@sha256:54f7fe2c6c5a9db9a0ebf1131797109bb7a4d91f56b9b362bde2abd237dd1974 -- registry.k8s.io/ingress-nginx/controller-chroot:v1.3.1@sha256:a8466b19c621bd550b1645e27a004a5cc85009c858a9ab19490216735ac432b1 - - -## What's Changed - -_IMPORTANT CHANGES:_ -- Update to golang 1.19 -- Started migration for Data and Control Plane splits -- Upgrade to Alpine 3.16.2 -- New kubectl plugin release workflow -- New CVE findings template - -All other Changes -- [9006](https://github.com/kubernetes/ingress-nginx/pull/9006) issue:8739 fix doc issue -- [9003](https://github.com/kubernetes/ingress-nginx/pull/9003) Bump github/codeql-action from 2.1.21 to 2.1.22 -- [9001](https://github.com/kubernetes/ingress-nginx/pull/9001) GitHub Workflows security hardening -- [8992](https://github.com/kubernetes/ingress-nginx/pull/8992) Bump github.com/opencontainers/runc from 1.1.3 to 1.1.4 -- [8991](https://github.com/kubernetes/ingress-nginx/pull/8991) Bump google.golang.org/grpc from 1.48.0 to 1.49.0 -- [8986](https://github.com/kubernetes/ingress-nginx/pull/8986) Bump goreleaser/goreleaser-action from 3.0.0 to 3.1.0 -- [8984](https://github.com/kubernetes/ingress-nginx/pull/8984) fixed deprecated ginkgo flags -- [8982](https://github.com/kubernetes/ingress-nginx/pull/8982) Bump github/codeql-action from 2.1.20 to 2.1.21 -- [8981](https://github.com/kubernetes/ingress-nginx/pull/8981) Bump actions/setup-go from 3.2.1 to 3.3.0 -- [8976](https://github.com/kubernetes/ingress-nginx/pull/8976) Update apiserver to 0.25 to remove v2 go-restful -- [8970](https://github.com/kubernetes/ingress-nginx/pull/8970) bump Golang to 1.19 #8932 -- [8969](https://github.com/kubernetes/ingress-nginx/pull/8969) fix: go-restful CVE #8745 -- [8967](https://github.com/kubernetes/ingress-nginx/pull/8967) updated to testrunnerimage with updated yamale yamllint -- [8966](https://github.com/kubernetes/ingress-nginx/pull/8966) added note on digitalocean annotations -- [8960](https://github.com/kubernetes/ingress-nginx/pull/8960) upgrade yamale and yamllint version -- [8959](https://github.com/kubernetes/ingress-nginx/pull/8959) revert changes to configmap resource permissions -- [8957](https://github.com/kubernetes/ingress-nginx/pull/8957) Bump github/codeql-action from 2.1.19 to 2.1.20 -- [8956](https://github.com/kubernetes/ingress-nginx/pull/8956) Bump azure/setup-helm from 2.1 to 3.3 -- [8954](https://github.com/kubernetes/ingress-nginx/pull/8954) Bump actions/dependency-review-action from 2.0.4 to 2.1.0 -- [8953](https://github.com/kubernetes/ingress-nginx/pull/8953) Bump aquasecurity/trivy-action from 0.5.1 to 0.7.1 -- [8952](https://github.com/kubernetes/ingress-nginx/pull/8952) Bump securego/gosec from b99b5f7838e43a4104354ad92a6a1774302ee1f9 to 2.13.1 -- [8951](https://github.com/kubernetes/ingress-nginx/pull/8951) Bump geekyeggo/delete-artifact from a6ab43859c960a8b74cbc6291f362c7fb51829ba to 1 -- [8950](https://github.com/kubernetes/ingress-nginx/pull/8950) Bump github/codeql-action from 2.1.18 to 2.1.19 -- [8948](https://github.com/kubernetes/ingress-nginx/pull/8948) updated testrunner and testecho images -- [8946](https://github.com/kubernetes/ingress-nginx/pull/8946) Clean old code and move helper functions -- [8944](https://github.com/kubernetes/ingress-nginx/pull/8944) Make keep-alive documentation more explicit for clarity -- [8939](https://github.com/kubernetes/ingress-nginx/pull/8939) bump baseimage alpine to v3.16.2 for zlib CVE fix - -## New Contributors -* @mtnezm made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/8817 -* @tamcore made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/8821 -* @guilhem made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/8827 -* @lilien1010 made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/8830 -* @qilongqiu made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/8855 -* @dgoffredo made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/8848 -* @Volatus made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/8859 -* @europ made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/8841 -* @mrksngl made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/7892 -* @omichels made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/8895 -* @zeeZ made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/8881 -* @mjudeikis made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/8928 -* @NissesSenap made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/8873 -* @anders-swanson made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/8665 -* @aslafy-z made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/8905 -* @harry1064 made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/8825 -* @sashashura made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9001 -* @sreelakshminarayananm made their first contribution in https://github.com/kubernetes/ingress-nginx/pull/9006 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.3.0...controller-v1.3.1 - -### 1.3.0 - -Image: -- registry.k8s.io/ingress-nginx/controller:v1.3.0@sha256:d1707ca76d3b044ab8a28277a2466a02100ee9f58a86af1535a3edf9323ea1b5 -- registry.k8s.io/ingress-nginx/controller-chroot:v1.3.0@sha256:0fcb91216a22aae43b374fc2e6a03b8afe9e8c78cbf07a09d75636dc4ea3c191 - -_IMPORTANT CHANGES:_ -* This release removes support for Kubernetes v1.19.0 -* This release adds support for Kubernetes v1.24.0 -* Starting with this release, we will need permissions on the `coordination.k8s.io/leases` resource for leaderelection lock - -_KNOWN ISSUES:_ -* This release reports a false positive on go-restful library that will be fixed with Kubernetes v1.25 release - Issue #8745 - -_Changes:_ -- "[8810](https://github.com/kubernetes/ingress-nginx/pull/8810) Prepare for v1.3.0" -- "[8808](https://github.com/kubernetes/ingress-nginx/pull/8808) revert arch var name" -- "[8805](https://github.com/kubernetes/ingress-nginx/pull/8805) Bump k8s.io/klog/v2 from 2.60.1 to 2.70.1" -- "[8803](https://github.com/kubernetes/ingress-nginx/pull/8803) Update to nginx base with alpine v3.16" -- "[8802](https://github.com/kubernetes/ingress-nginx/pull/8802) chore: start v1.3.0 release process" -- "[8798](https://github.com/kubernetes/ingress-nginx/pull/8798) Add v1.24.0 to test matrix" -- "[8796](https://github.com/kubernetes/ingress-nginx/pull/8796) fix: add MAC_OS variable for static-check" -- "[8793](https://github.com/kubernetes/ingress-nginx/pull/8793) changed to alpine-v3.16" -- "[8781](https://github.com/kubernetes/ingress-nginx/pull/8781) Bump github.com/stretchr/testify from 1.7.5 to 1.8.0" -- "[8778](https://github.com/kubernetes/ingress-nginx/pull/8778) chore: remove stable.txt from release process" -- "[8775](https://github.com/kubernetes/ingress-nginx/pull/8775) Remove stable" -- "[8773](https://github.com/kubernetes/ingress-nginx/pull/8773) Bump github/codeql-action from 2.1.14 to 2.1.15" -- "[8772](https://github.com/kubernetes/ingress-nginx/pull/8772) Bump ossf/scorecard-action from 1.1.1 to 1.1.2" -- "[8771](https://github.com/kubernetes/ingress-nginx/pull/8771) fix bullet md format" -- "[8770](https://github.com/kubernetes/ingress-nginx/pull/8770) Add condition for monitoring.coreos.com/v1 API" -- "[8769](https://github.com/kubernetes/ingress-nginx/pull/8769) Fix typos and add links to developer guide" -- "[8767](https://github.com/kubernetes/ingress-nginx/pull/8767) change v1.2.0 to v1.2.1 in deploy doc URLs" -- "[8765](https://github.com/kubernetes/ingress-nginx/pull/8765) Bump github/codeql-action from 1.0.26 to 2.1.14" -- "[8752](https://github.com/kubernetes/ingress-nginx/pull/8752) Bump github.com/spf13/cobra from 1.4.0 to 1.5.0" -- "[8751](https://github.com/kubernetes/ingress-nginx/pull/8751) Bump github.com/stretchr/testify from 1.7.2 to 1.7.5" -- "[8750](https://github.com/kubernetes/ingress-nginx/pull/8750) added announcement" -- "[8740](https://github.com/kubernetes/ingress-nginx/pull/8740) change sha e2etestrunner and echoserver" -- "[8738](https://github.com/kubernetes/ingress-nginx/pull/8738) Update docs to make it easier for noobs to follow step by step" -- "[8737](https://github.com/kubernetes/ingress-nginx/pull/8737) updated baseimage sha" -- "[8736](https://github.com/kubernetes/ingress-nginx/pull/8736) set ld-musl-path" -- "[8733](https://github.com/kubernetes/ingress-nginx/pull/8733) feat: migrate leaderelection lock to leases" -- "[8726](https://github.com/kubernetes/ingress-nginx/pull/8726) prometheus metric: upstream_latency_seconds" -- "[8720](https://github.com/kubernetes/ingress-nginx/pull/8720) Ci pin deps" -- "[8719](https://github.com/kubernetes/ingress-nginx/pull/8719) Working OpenTelemetry sidecar (base nginx image)" -- "[8714](https://github.com/kubernetes/ingress-nginx/pull/8714) Create Openssf scorecard" -- "[8708](https://github.com/kubernetes/ingress-nginx/pull/8708) Bump github.com/prometheus/common from 0.34.0 to 0.35.0" -- "[8703](https://github.com/kubernetes/ingress-nginx/pull/8703) Bump actions/dependency-review-action from 1 to 2" -- "[8701](https://github.com/kubernetes/ingress-nginx/pull/8701) Fix several typos" -- "[8699](https://github.com/kubernetes/ingress-nginx/pull/8699) fix the gosec test and a make target for it" -- "[8698](https://github.com/kubernetes/ingress-nginx/pull/8698) Bump actions/upload-artifact from 2.3.1 to 3.1.0" -- "[8697](https://github.com/kubernetes/ingress-nginx/pull/8697) Bump actions/setup-go from 2.2.0 to 3.2.0" -- "[8695](https://github.com/kubernetes/ingress-nginx/pull/8695) Bump actions/download-artifact from 2 to 3" -- "[8694](https://github.com/kubernetes/ingress-nginx/pull/8694) Bump crazy-max/ghaction-docker-buildx from 1.6.2 to 3.3.1" - -### 1.2.1 - -Image: -- k8s.gcr.io/ingress-nginx/controller:v1.2.1@sha256:5516d103a9c2ecc4f026efbd4b40662ce22dc1f824fb129ed121460aaa5c47f8 -- k8s.gcr.io/ingress-nginx/controller-chroot:v1.2.1@sha256:d301551cf62bc3fb75c69fa56f7aa1d9e87b5079333adaf38afe84d9b7439355 - -This release removes the root and alias directives in NGINX, this can avoid some potential security attacks. - -_Changes:_ - -- [8459](https://github.com/kubernetes/ingress-nginx/pull/8459) Update default allowed CORS headers -- [8202](https://github.com/kubernetes/ingress-nginx/pull/8202) disable modsecurity on error page -- [8178](https://github.com/kubernetes/ingress-nginx/pull/8178) Add header Host into mirror annotations -- [8458](https://github.com/kubernetes/ingress-nginx/pull/8458) Add portNamePreffix Helm chart parameter -- [8587](https://github.com/kubernetes/ingress-nginx/pull/8587) Add CAP_SYS_CHROOT to DS/PSP when needed -- [8213](https://github.com/kubernetes/ingress-nginx/pull/8213) feat: always set auth cookie -- [8548](https://github.com/kubernetes/ingress-nginx/pull/8548) Implement reporting status classes in metrics -- [8612](https://github.com/kubernetes/ingress-nginx/pull/8612) move so files under /etc/nginx/modules -- [8624](https://github.com/kubernetes/ingress-nginx/pull/8624) Add patch to remove root and alias directives -- [8623](https://github.com/kubernetes/ingress-nginx/pull/8623) Improve path rule - -### 1.2.0 - -Image: -- k8s.gcr.io/ingress-nginx/controller:v1.2.0@sha256:d8196e3bc1e72547c5dec66d6556c0ff92a23f6d0919b206be170bc90d5f9185 -- k8s.gcr.io/ingress-nginx/controller-chroot:v1.2.0@sha256:fb17f1700b77d4fcc52ca6f83ffc2821861ae887dbb87149cf5cbc52bea425e5 - -This minor version release, introduces 2 breaking changes. For the first time, an option to jail/chroot the nginx process, inside the controller container, is being introduced. This provides an additional layer of security, for sensitive information like K8S serviceaccounts. This release also brings a special new feature of deep inspection into objects. The inspection is a walk through of all the spec, checking for possible attempts to escape configs. Currently such an inspection only occurs for `networking.Ingress`. Additionally there are fixes for the recently announced CVEs on busybox & ssl_client. And there is a fix to a recently introduced redirection related bug, that was setting the protocol on URLs to "nil". - -_Changes:_ - -- [8481](https://github.com/kubernetes/ingress-nginx/pull/8481) Fix log creation in chroot script -- [8479](https://github.com/kubernetes/ingress-nginx/pull/8479) changed nginx base img tag to img built with alpine3.14.6 -- [8478](https://github.com/kubernetes/ingress-nginx/pull/8478) update base images and protobuf gomod -- [8468](https://github.com/kubernetes/ingress-nginx/pull/8468) Fallback to ngx.var.scheme for redirectScheme with use-forward-headers when X-Forwarded-Proto is empty -- [8456](https://github.com/kubernetes/ingress-nginx/pull/8456) Implement object deep inspector -- [8455](https://github.com/kubernetes/ingress-nginx/pull/8455) Update dependencies -- [8454](https://github.com/kubernetes/ingress-nginx/pull/8454) Update index.md -- [8447](https://github.com/kubernetes/ingress-nginx/pull/8447) typo fixing -- [8446](https://github.com/kubernetes/ingress-nginx/pull/8446) Fix suggested annotation-value-word-blocklist -- [8444](https://github.com/kubernetes/ingress-nginx/pull/8444) replace deprecated topology key in example with current one -- [8443](https://github.com/kubernetes/ingress-nginx/pull/8443) Add dependency review enforcement -- [8434](https://github.com/kubernetes/ingress-nginx/pull/8434) added new auth-tls-match-cn annotation -- [8426](https://github.com/kubernetes/ingress-nginx/pull/8426) Bump github.com/prometheus/common from 0.32.1 to 0.33.0 -- [8417](https://github.com/kubernetes/ingress-nginx/pull/8417) force helm release to artifact hub -- [8405](https://github.com/kubernetes/ingress-nginx/pull/8405) kubectl-plugin code overview info -- [8399](https://github.com/kubernetes/ingress-nginx/pull/8399) Darwin arm64 -- [8443](https://github.com/kubernetes/ingress-nginx/pull/8443) Add dependency review enforcement -- [8444](https://github.com/kubernetes/ingress-nginx/pull/8444) replace deprecated topology key in example with current one -- [8446](https://github.com/kubernetes/ingress-nginx/pull/8446) Fix suggested annotation-value-word-blocklist -- [8219](https://github.com/kubernetes/ingress-nginx/pull/8219) Add keepalive support for auth requests -- [8455](https://github.com/kubernetes/ingress-nginx/pull/8455) Update dependencies -- [8456](https://github.com/kubernetes/ingress-nginx/pull/8456) Implement object deep inspector -- [8325](https://github.com/kubernetes/ingress-nginx/pull/8325) Fix for buggy ingress sync with retries -- [8322](https://github.com/kubernetes/ingress-nginx/pull/8322) Improve req handling dashboard - -### 1.2.0-beta.0 - -**Image:** -- k8s.gcr.io/ingress-nginx/controller:v1.2.0-beta.0@sha256:92115f5062568ebbcd450cd2cf9bffdef8df9fc61e7d5868ba8a7c9d773e0961 -- k8s.gcr.io/ingress-nginx/controller-chroot:v1.2.0-beta.0@sha256:0082f0f547b147a30ad85a5d6d2ceb3edbf0848b2008ed754365b6678bdea9a5 - -This release introduces Jail/chroot nginx process inside controller container for the first time - -_Changes:_ - -- [8417](https://github.com/kubernetes/ingress-nginx/pull/8417) force helm release to artifact hub -- [8421](https://github.com/kubernetes/ingress-nginx/pull/8421) fix change log changes list -- [8405](https://github.com/kubernetes/ingress-nginx/pull/8405) kubectl-plugin code overview info -- [8399](https://github.com/kubernetes/ingress-nginx/pull/8399) Darwin arm64 -- [8443](https://github.com/kubernetes/ingress-nginx/pull/8443) Add dependency review enforcement -- [8426](https://github.com/kubernetes/ingress-nginx/pull/8426) Bump github.com/prometheus/common from 0.32.1 to 0.33.0 -- [8444](https://github.com/kubernetes/ingress-nginx/pull/8444) replace deprecated topology key in example with current one -- [8447](https://github.com/kubernetes/ingress-nginx/pull/8447) typo fixing -- [8446](https://github.com/kubernetes/ingress-nginx/pull/8446) Fix suggested annotation-value-word-blocklist -- [8219](https://github.com/kubernetes/ingress-nginx/pull/8219) Add keepalive support for auth requests -- [8337](https://github.com/kubernetes/ingress-nginx/pull/8337) Jail/chroot nginx process inside controller container -- [8454](https://github.com/kubernetes/ingress-nginx/pull/8454) Update index.md -- [8455](https://github.com/kubernetes/ingress-nginx/pull/8455) Update dependencies -- [8456](https://github.com/kubernetes/ingress-nginx/pull/8456) Implement object deep inspector -- [8325](https://github.com/kubernetes/ingress-nginx/pull/8325) Fix for buggy ingress sync with retries -- [8322](https://github.com/kubernetes/ingress-nginx/pull/8322) Improve req handling dashboard -- [8464](https://github.com/kubernetes/ingress-nginx/pull/8464) Prepare v1.2.0-beta.0 release - - -### 1.1.3 - -**Image:** -- k8s.gcr.io/ingress-nginx/controller:v1.1.3@sha256:31f47c1e202b39fadecf822a9b76370bd4baed199a005b3e7d4d1455f4fd3fe2 - -This release upgrades Alpine to 3.14.4 and nginx to 1.19.10 - -Patches [OpenSSL CVE-2022-0778](https://github.com/kubernetes/ingress-nginx/issues/8339) - -Patches [Libxml2 CVE-2022-23308](https://github.com/kubernetes/ingress-nginx/issues/8321) - -_Breaking Changes:_ - -- https://github.com/nginx/nginx/commit/d18e066d650bff39f1705d3038804873584007af Deprecated http2_recv_timeout in favor of client_header_timeout (client-header-timeout) -- https://github.com/nginx/nginx/commit/51fea093e4374dbd857dc437ff9588060ef56471 Deprecated http2_max_field_size (http2-max-field-size) and http2_max_header_size (http2-max-header-size) in favor of large_client_header_buffers (large-client-header-buffers) -- https://github.com/nginx/nginx/commit/49ab3312448495f0ee8e00143a29624dde46ef5c Deprecated http2_idle_timeout and http2_max_requests (http2-max-requests) in favor of keepalive_timeout (upstream-keepalive-timeout?) and keepalive_requests (upstream-keepalive-requests?) respectively - -_Changes:_ - -- [8415](https://github.com/kubernetes/ingress-nginx/pull/8415) base img update for e2e-test-runner & opentelemetry -- [8403](https://github.com/kubernetes/ingress-nginx/pull/8403) Add execute permissions to nginx image entrypoint.sh -- [8392](https://github.com/kubernetes/ingress-nginx/pull/8392) fix document for monitoring -- [8386](https://github.com/kubernetes/ingress-nginx/pull/8386) downgrade to 3.14.4 and fix tag -- [8379](https://github.com/kubernetes/ingress-nginx/pull/8379) bump luarocks to 3.8.0 -- [8368](https://github.com/kubernetes/ingress-nginx/pull/8368) Updated semver in install docs URLs -- [8360](https://github.com/kubernetes/ingress-nginx/pull/8360) Bump github.com/stretchr/testify from 1.7.0 to 1.7.1 -- [8334](https://github.com/kubernetes/ingress-nginx/pull/8334) Pinned GitHub workflows by SHA -- [8324](https://github.com/kubernetes/ingress-nginx/pull/8324) Added missing "repo" option on "helm upgrade" command -- [8315](https://github.com/kubernetes/ingress-nginx/pull/8315) Fix 50% split between canary and mainline tests -- [8311](https://github.com/kubernetes/ingress-nginx/pull/8311) leaving it the git tag -- [8307](https://github.com/kubernetes/ingress-nginx/pull/8307) Nginx v1.19.10 -- [8302](https://github.com/kubernetes/ingress-nginx/pull/8302) docs: fix changelog formatting for 1.1.2 -- [8300](https://github.com/kubernetes/ingress-nginx/pull/8300) Names cannot contain _ (underscore)! So I changed it to -. -- [8288](https://github.com/kubernetes/ingress-nginx/pull/8288) [docs] Missing annotations -- [8287](https://github.com/kubernetes/ingress-nginx/pull/8287) Add the shareProcessNamespace as a configurable setting in the helm chart -- [8286](https://github.com/kubernetes/ingress-nginx/pull/8286) Fix OpenTelemetry sidecar image build -- [8281](https://github.com/kubernetes/ingress-nginx/pull/8281) force prow job by changing something in images/ot dir -- [8273](https://github.com/kubernetes/ingress-nginx/pull/8273) Issue#8241 -- [8267](https://github.com/kubernetes/ingress-nginx/pull/8267) Add fsGroup value to admission-webhooks/job-patch charts -- [8262](https://github.com/kubernetes/ingress-nginx/pull/8262) Updated confusing error -- [8258](https://github.com/kubernetes/ingress-nginx/pull/8258) remove 0.46.0 from supported versions table -- [8256](https://github.com/kubernetes/ingress-nginx/pull/8256) fix: deny locations with invalid auth-url annotation -- [8253](https://github.com/kubernetes/ingress-nginx/pull/8253) Add a certificate info metric - -### 1.1.2 - -**Image:** -- k8s.gcr.io/ingress-nginx/controller:v1.1.2@sha256:28b11ce69e57843de44e3db6413e98d09de0f6688e33d4bd384002a44f78405c - -This release bumps grpc version to 1.44.0 & runc to version 1.1.0. The release also re-introduces the ingress.class annotation, which was previously declared as deprecated. Besides that, several bug fixes and improvements are listed below. - -_Changes:_ - -- [8291](https://github.com/kubernetes/ingress-nginx/pull/8291) remove git tag env from cloud build -- [8286](https://github.com/kubernetes/ingress-nginx/pull/8286) Fix OpenTelemetry sidecar image build -- [8277](https://github.com/kubernetes/ingress-nginx/pull/8277) Add OpenSSF Best practices badge -- [8273](https://github.com/kubernetes/ingress-nginx/pull/8273) Issue#8241 -- [8267](https://github.com/kubernetes/ingress-nginx/pull/8267) Add fsGroup value to admission-webhooks/job-patch charts -- [8262](https://github.com/kubernetes/ingress-nginx/pull/8262) Updated confusing error -- [8256](https://github.com/kubernetes/ingress-nginx/pull/8256) fix: deny locations with invalid auth-url annotation -- [8253](https://github.com/kubernetes/ingress-nginx/pull/8253) Add a certificate info metric -- [8236](https://github.com/kubernetes/ingress-nginx/pull/8236) webhook: remove useless code. -- [8227](https://github.com/kubernetes/ingress-nginx/pull/8227) Update libraries in webhook image -- [8225](https://github.com/kubernetes/ingress-nginx/pull/8225) fix inconsistent-label-cardinality for prometheus metrics: nginx_ingress_controller_requests -- [8221](https://github.com/kubernetes/ingress-nginx/pull/8221) Do not validate ingresses with unknown ingress class in admission webhook endpoint -- [8210](https://github.com/kubernetes/ingress-nginx/pull/8210) Bump github.com/prometheus/client_golang from 1.11.0 to 1.12.1 -- [8209](https://github.com/kubernetes/ingress-nginx/pull/8209) Bump google.golang.org/grpc from 1.43.0 to 1.44.0 -- [8204](https://github.com/kubernetes/ingress-nginx/pull/8204) Add Artifact Hub lint -- [8203](https://github.com/kubernetes/ingress-nginx/pull/8203) Fix Indentation of example and link to cert-manager tutorial -- [8201](https://github.com/kubernetes/ingress-nginx/pull/8201) feat(metrics): add path and method labels to requests countera -- [8199](https://github.com/kubernetes/ingress-nginx/pull/8199) use functional options to reduce number of methods creating an EchoDeployment -- [8196](https://github.com/kubernetes/ingress-nginx/pull/8196) docs: fix inconsistent controller annotation -- [8191](https://github.com/kubernetes/ingress-nginx/pull/8191) Using Go install for misspell -- [8186](https://github.com/kubernetes/ingress-nginx/pull/8186) prometheus+grafana using servicemonitor -- [8185](https://github.com/kubernetes/ingress-nginx/pull/8185) Append elements on match, instead of removing for cors-annotations -- [8179](https://github.com/kubernetes/ingress-nginx/pull/8179) Bump github.com/opencontainers/runc from 1.0.3 to 1.1.0 -- [8173](https://github.com/kubernetes/ingress-nginx/pull/8173) Adding annotations to the controller service account -- [8163](https://github.com/kubernetes/ingress-nginx/pull/8163) Update the $req_id placeholder description -- [8162](https://github.com/kubernetes/ingress-nginx/pull/8162) Versioned static manifests -- [8159](https://github.com/kubernetes/ingress-nginx/pull/8159) Adding some geoip variables and default values -- [8155](https://github.com/kubernetes/ingress-nginx/pull/8155) #7271 feat: avoid-pdb-creation-when-default-backend-disabled-and-replicas-gt-1 -- [8151](https://github.com/kubernetes/ingress-nginx/pull/8151) Automatically generate helm docs -- [8143](https://github.com/kubernetes/ingress-nginx/pull/8143) Allow to configure delay before controller exits -- [8136](https://github.com/kubernetes/ingress-nginx/pull/8136) add ingressClass option to helm chart - back compatibility with ingress.class annotations - - -### 1.1.1 - -**Image:** -- k8s.gcr.io/ingress-nginx/controller:v1.1.1@sha256:0bc88eb15f9e7f84e8e56c14fa5735aaa488b840983f87bd79b1054190e660de - -This release contains several fixes and improvements. This image is now built using Go v1.17.6 and gRPC v1.43.0. See detailed list below. - -_Changes:_ - -- [8120](https://github.com/kubernetes/ingress-nginx/pull/8120) Update go in runner and release v1.1.1 -- [8119](https://github.com/kubernetes/ingress-nginx/pull/8119) Update to go v1.17.6 -- [8118](https://github.com/kubernetes/ingress-nginx/pull/8118) Remove deprecated libraries, update other libs -- [8117](https://github.com/kubernetes/ingress-nginx/pull/8117) Fix codegen errors -- [8115](https://github.com/kubernetes/ingress-nginx/pull/8115) chart/ghaction: set the correct permission to have access to push a release -- [8098](https://github.com/kubernetes/ingress-nginx/pull/8098) generating SHA for CA only certs in backend_ssl.go + comparision of P… -- [8088](https://github.com/kubernetes/ingress-nginx/pull/8088) Fix Edit this page link to use main branch -- [8072](https://github.com/kubernetes/ingress-nginx/pull/8072) Expose GeoIP2 Continent code as variable -- [8061](https://github.com/kubernetes/ingress-nginx/pull/8061) docs(charts): using helm-docs for chart -- [8058](https://github.com/kubernetes/ingress-nginx/pull/8058) Bump github.com/spf13/cobra from 1.2.1 to 1.3.0 -- [8054](https://github.com/kubernetes/ingress-nginx/pull/8054) Bump google.golang.org/grpc from 1.41.0 to 1.43.0 -- [8051](https://github.com/kubernetes/ingress-nginx/pull/8051) align bug report with feature request regarding kind documentation -- [8046](https://github.com/kubernetes/ingress-nginx/pull/8046) Report expired certificates (#8045) -- [8044](https://github.com/kubernetes/ingress-nginx/pull/8044) remove G109 check till gosec resolves issues -- [8042](https://github.com/kubernetes/ingress-nginx/pull/8042) docs_multiple_instances_one_cluster_ticket_7543 -- [8041](https://github.com/kubernetes/ingress-nginx/pull/8041) docs: fix typo'd executible name -- [8035](https://github.com/kubernetes/ingress-nginx/pull/8035) Comment busy owners -- [8029](https://github.com/kubernetes/ingress-nginx/pull/8029) Add stream-snippet as a ConfigMap and Annotation option -- [8023](https://github.com/kubernetes/ingress-nginx/pull/8023) fix nginx compilation flags -- [8021](https://github.com/kubernetes/ingress-nginx/pull/8021) Disable default modsecurity_rules_file if modsecurity-snippet is specified -- [8019](https://github.com/kubernetes/ingress-nginx/pull/8019) Revise main documentation page -- [8018](https://github.com/kubernetes/ingress-nginx/pull/8018) Preserve order of plugin invocation -- [8015](https://github.com/kubernetes/ingress-nginx/pull/8015) Add newline indenting to admission webhook annotations -- [8014](https://github.com/kubernetes/ingress-nginx/pull/8014) Add link to example error page manifest in docs -- [8009](https://github.com/kubernetes/ingress-nginx/pull/8009) Fix spelling in documentation and top-level files -- [8008](https://github.com/kubernetes/ingress-nginx/pull/8008) Add relabelings in controller-servicemonitor.yaml -- [8003](https://github.com/kubernetes/ingress-nginx/pull/8003) Minor improvements (formatting, consistency) in install guide -- [8001](https://github.com/kubernetes/ingress-nginx/pull/8001) fix: go-grpc Dockerfile -- [7999](https://github.com/kubernetes/ingress-nginx/pull/7999) images: use k8s-staging-test-infra/gcb-docker-gcloud -- [7996](https://github.com/kubernetes/ingress-nginx/pull/7996) doc: improvement -- [7983](https://github.com/kubernetes/ingress-nginx/pull/7983) Fix a couple of misspellings in the annotations documentation. -- [7979](https://github.com/kubernetes/ingress-nginx/pull/7979) allow set annotations for admission Jobs -- [7977](https://github.com/kubernetes/ingress-nginx/pull/7977) Add ssl_reject_handshake to defaul server -- [7975](https://github.com/kubernetes/ingress-nginx/pull/7975) add legacy version update v0.50.0 to main changelog -- [7972](https://github.com/kubernetes/ingress-nginx/pull/7972) updated service upstream definition -- [7963](https://github.com/kubernetes/ingress-nginx/pull/7963) Change sanitization message from error to warning - -### 1.1.0 - -**Image:** -- k8s.gcr.io/ingress-nginx/controller:v1.1.0@sha256:f766669fdcf3dc26347ed273a55e754b427eb4411ee075a53f30718b4499076a - -This release makes the annotation `annotation-value-word-blocklist` backwards compatible by being an empty list instead of prescribed defaults. -Effectively reverting [7874](https://github.com/kubernetes/ingress-nginx/pull/7874) but keeping the functionality of `annotation-value-word-blocklist` - -See Issue [7939](https://github.com/kubernetes/ingress-nginx/pull/7939) for more discussion - -Admins should still consider putting a reasonable block list in place, more information on why can be found [here](https://github.com/kubernetes/ingress-nginx/issues/7837) and how in our documentation [here](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#annotation-value-word-blocklist) - -_Changes:_ -- [7963](https://github.com/kubernetes/ingress-nginx/pull/7963) Change sanitization message from error to warning (#7963) -- [7942](https://github.com/kubernetes/ingress-nginx/pull/7942) update default block list,docs, tests (#7942) -- [7948](https://github.com/kubernetes/ingress-nginx/pull/7948) applied allowPrivilegeEscalation=false (#7948) -- [7941](https://github.com/kubernetes/ingress-nginx/pull/7941) update build for darwin arm64 (#7941) - -### 1.0.5 - -**Image:** -- k8s.gcr.io/ingress-nginx/controller:v1.0.5@sha256:55a1fcda5b7657c372515fe402c3e39ad93aa59f6e4378e82acd99912fe6028d - -_Possible Breaking Change_ -We now implement string sanitization in annotation values. This means that words like "location", "by_lua" and -others will drop the reconciliation of an Ingress object. - -Users from mod_security and other features should be aware that some blocked values may be used by those features -and must be manually unblocked by the Ingress Administrator. - -For more details please check [https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#annotation-value-word-blocklist] - -_Changes:_ -- [7874](https://github.com/kubernetes/ingress-nginx/pull/7874) Add option to sanitize annotation inputs -- [7854](https://github.com/kubernetes/ingress-nginx/pull/7854) Add brotli-min-length configuration option -- [7800](https://github.com/kubernetes/ingress-nginx/pull/7800) Fix thread synchronization issue -- [7711](https://github.com/kubernetes/ingress-nginx/pull/7711) Added AdmissionController metrics -- [7614](https://github.com/kubernetes/ingress-nginx/pull/7614) Support cors-allow-origin with multiple origins -- [7472](https://github.com/kubernetes/ingress-nginx/pull/7472) Support watch multiple namespaces matched witch namespace selector - -### 1.0.4 - -**Image:** -- k8s.gcr.io/ingress-nginx/controller:v1.0.4@sha256:545cff00370f28363dad31e3b59a94ba377854d3a11f18988f5f9e56841ef9ef - -_Possible Breaking Change_ -We have disabled the builtin ssl_session_cache due to possible memory fragmentation. This should not impact the majority of users, but please let us know -if you face any problem - -_Changes:_ - -- [7777](https://github.com/kubernetes/ingress-nginx/pull/7777) Disable builtin ssl_session_cache -- [7727](https://github.com/kubernetes/ingress-nginx/pull/7727) Print warning only instead of error if no IngressClass permission is available -- Bump internal libraries versions -- Fix diverse documentation - -### 1.0.3 - -**Image:** -- k8s.gcr.io/ingress-nginx/controller:v1.0.3@sha256:4ade87838eb8256b094fbb5272d7dda9b6c7fa8b759e6af5383c1300996a7452 - -**Known Issues** -* Ingress controller now (starting from v1.0.0) mandates cluster scoped access to IngressClass. This leads to problems when updating old Ingress controller to newest version, as described [here](https://github.com/kubernetes/ingress-nginx/issues/7510). We plan to fix it in v1.0.4, see [this](https://github.com/kubernetes/ingress-nginx/pull/7578). - -_New Features:_ - -_Changes:_ - -- [X] [#7727](https://github.com/kubernetes/ingress-nginx/pull/7727) Fix selector for shutting down Pods status -- [X] [#7719](https://github.com/kubernetes/ingress-nginx/pull/7719) Fix overlap check when ingress is configured as canary -- Diverse docs fixes and libraries bumping - -### 1.0.2 - -**Image:** -- k8s.gcr.io/ingress-nginx/controller:v1.0.2@sha256:85b53b493d6d658d8c013449223b0ffd739c76d76dc9bf9000786669ec04e049 - -**Known Issues** -* Ingress controller now (starting from v1.0.0) mandates cluster scoped access to IngressClass. This leads to problems when updating old Ingress controller to newest version, as described [here](https://github.com/kubernetes/ingress-nginx/issues/7510). We plan to fix it in v1.0.3, see [this](https://github.com/kubernetes/ingress-nginx/pull/7578). - -_New Features:_ - -_Changes:_ - -- [X] [#7706](https://github.com/kubernetes/ingress-nginx/pull/7706) Add build info on prometheus metrics -- [X] [#7702](https://github.com/kubernetes/ingress-nginx/pull/7702) Upgrade lua-resty-balancer to v0.04 -- [X] [#7696](https://github.com/kubernetes/ingress-nginx/pull/7696) Add canary backend name for requests metrics - -### 1.0.1 - -**Image:** -- k8s.gcr.io/ingress-nginx/controller:v1.0.1@sha256:26bbd57f32bac3b30f90373005ef669aae324a4de4c19588a13ddba399c6664e - -**Known Issues** -* Ingress controller now (starting from v1.0.0) mandates cluster scoped access to IngressClass. This leads to problems when updating old Ingress controller to newest version, as described [here](https://github.com/kubernetes/ingress-nginx/issues/7510). We plan to fix it in v1.0.2, see [this](https://github.com/kubernetes/ingress-nginx/pull/7578). - -_New Features:_ - -_Changes:_ - -- [X] [#7670](https://github.com/kubernetes/ingress-nginx/pull/7670) Change enable-snippet to allow-snippet-annotation (#7670) -- [X] [#7672](https://github.com/kubernetes/ingress-nginx/pull/7672) add option for documentiony only to pr template (#7672) -- [X] [#7668](https://github.com/kubernetes/ingress-nginx/pull/7668) added example multiple controller install to faq (#7668) -- [X] [#7665](https://github.com/kubernetes/ingress-nginx/pull/7665) Add option to force enabling snippet directives (#7665) -- [X] [#7659](https://github.com/kubernetes/ingress-nginx/pull/7659) Replace kube-lego docs with cert-manager (#7659) -- [X] [#7656](https://github.com/kubernetes/ingress-nginx/pull/7656) Changelog.md: Update references to sigs.k8s.io/promo-tools (#7656) -- [X] [#7603](https://github.com/kubernetes/ingress-nginx/pull/7603) additional info for the custom-headers documentation page (#7603) -- [X] [#7630](https://github.com/kubernetes/ingress-nginx/pull/7630) images/kube-webhook-certgen/rootfs: improvements (#7630) -- [X] [#7636](https://github.com/kubernetes/ingress-nginx/pull/7636) Add github action for building images (#7636) -- [X] [#7648](https://github.com/kubernetes/ingress-nginx/pull/7648) Update e2e-test-runner image (#7648) -- [X] [#7643](https://github.com/kubernetes/ingress-nginx/pull/7643) Update NGINX base image to v1.19 (#7643) -- [X] [#7642](https://github.com/kubernetes/ingress-nginx/pull/7642) Add security contacts (#7642) -- [X] [#7640](https://github.com/kubernetes/ingress-nginx/pull/7640) fix typos. (#7640) -- [X] [#7639](https://github.com/kubernetes/ingress-nginx/pull/7536) Downgrade nginx to v1.19 (#7639) -- [X] [#7575](https://github.com/kubernetes/ingress-nginx/pull/7575) Add e2e tests for secure cookie annotations (#7575) (#7619) -- [X] [#7625](https://github.com/kubernetes/ingress-nginx/pull/7625) Only build nginx-errors for linux/amd64 (#7625) -- [X] [#7609](https://github.com/kubernetes/ingress-nginx/pull/7609) Add new flag to watch ingressclass by name instead of spec (#7609) -- [X] [#7604](https://github.com/kubernetes/ingress-nginx/pull/7604) Change the cloudbuild timeout (#7604) -- [X] [#7460](https://github.com/kubernetes/ingress-nginx/pull/7460) Fix old tag of custom error pages used in example (#7460) -- [X] [#7577](https://github.com/kubernetes/ingress-nginx/pull/7577) Added command to get Nginx versionq! (#7577) -- [X] [#7611](https://github.com/kubernetes/ingress-nginx/pull/7611) Helm notes outputs non nil value for ingress.class annotation (#7611) -- [X] [#7469](https://github.com/kubernetes/ingress-nginx/pull/7469) Allow the usage of Services as Upstream on a global level (#7469) -- [X] [#7393](https://github.com/kubernetes/ingress-nginx/pull/7393) getEndpoints uses service target port directly if it's a number and mismatch with port name in endpoint (#7393) -- [X] [#7514](https://github.com/kubernetes/ingress-nginx/pull/7514) nginx flag: --disable-full-test flag to allow testing configfile of the current single ingress (#7514) -- [X] [#7374](https://github.com/kubernetes/ingress-nginx/pull/7374) Trigger syncIngress on Service addition/deletion #7346 (#7374) -- [X] [#7464](https://github.com/kubernetes/ingress-nginx/pull/7464) Sync Hostname and IP address from service to ingress status (#7464) -- [X] [#7560](https://github.com/kubernetes/ingress-nginx/pull/7560) put modsecurity e2e tests into their own packages (#7560) -- [X] [#7202](https://github.com/kubernetes/ingress-nginx/pull/7202) Additional AuthTLS assertions and doc change to demonstrate auth-tls-secret enables the other AuthTLS annotations (#7202) -- [X] [#7606](https://github.com/kubernetes/ingress-nginx/pull/7606) fix cli flag typo in faq (#7606) -- [X] [#7601](https://github.com/kubernetes/ingress-nginx/pull/7601) fix charts README.md to give additional detail on prometheus metrics … (#7601) -- [X] [#7596](https://github.com/kubernetes/ingress-nginx/pull/7596) Update e2e test runner image (#7596) -- [X] [#7604](https://github.com/kubernetes/ingress-nginx/pull/7604) Update cloudbuild timeout (#7604) -- [X] [#7440](https://github.com/kubernetes/ingress-nginx/pull/7440) remove timestamp when requeuing Element (#7440) -- [X] [#7598](https://github.com/kubernetes/ingress-nginx/pull/7598) correct docs on ingressClass resource being disabled (#7598) -- [X] [#7597](https://github.com/kubernetes/ingress-nginx/pull/7597) Update to the base nginx image (#7597) -- [X] [#7540](https://github.com/kubernetes/ingress-nginx/pull/7540) improve faq for migration to ingress api v1 (#7540) -- [X] [#7594](https://github.com/kubernetes/ingress-nginx/pull/7594) Remove addgroup directive from alpine building (#7594) -- [X] [#7592](https://github.com/kubernetes/ingress-nginx/pull/7592) Change builder in a new attempt to make it run (#7592) -- [X] [#7584](https://github.com/kubernetes/ingress-nginx/pull/7584) Changing gcb builder (#7584) -- [X] [#7583](https://github.com/kubernetes/ingress-nginx/pull/7583) update alpine and remove buildx restriction (#7583) -- [X] [#7561](https://github.com/kubernetes/ingress-nginx/pull/7561) Add doc ref for preserve-trailing-slash annotation (#7561) -- [X] [#7581](https://github.com/kubernetes/ingress-nginx/pull/7581) Default KinD manifest to watch ingresses without class (#7581) -- [X] [#7511](https://github.com/kubernetes/ingress-nginx/pull/7511) add same tcp and udp ports to internal load balancer (#7511) -- [X] [#7399](https://github.com/kubernetes/ingress-nginx/pull/7399) feat: add session-cookie-secure annotation (#7399) -- [X] [#7556](https://github.com/kubernetes/ingress-nginx/pull/7556) Fix YAML indentation issue (#7556) -- [X] [#7558](https://github.com/kubernetes/ingress-nginx/pull/7558) Revert "Update base nginx" (#7558) -- [X] [#7552](https://github.com/kubernetes/ingress-nginx/pull/7552) Update base nginx (#7552) -- [X] [#7541](https://github.com/kubernetes/ingress-nginx/pull/7541) Add a flag to specify address to bind the healthz server (#7541) -- [X] [#7503](https://github.com/kubernetes/ingress-nginx/pull/7503) Document the keep-alive 0 effect on http/2 requests (#7503) -- [X] [#7264](https://github.com/kubernetes/ingress-nginx/pull/7264) Update docs for new ingress api in cluster version >=1.19 (#7264) -- [X] [#7545](https://github.com/kubernetes/ingress-nginx/pull/7545) Improving e2e tests for non-service backends #7544 (#7545) -- [X] [#7537](https://github.com/kubernetes/ingress-nginx/pull/7537) improve docs for release - added step to edit README for support matrix (#7537) -- [X] [#7536](https://github.com/kubernetes/ingress-nginx/pull/7536) add known issues in changelog.md for release v1.0.0 (#7536) - -### 1.0.0 -**This is a breaking change** - -This release only supports Kubernetes versions >= v1.19. The support for Ingress Object in `networking.k8s.io/v1beta` is being dropped and manifests should now use `networking.k8s.io/v1`. - -**Image:** - -- `k8s.gcr.io/ingress-nginx/controller:v1.0.0@sha256:0851b34f69f69352bf168e6ccf30e1e20714a264ab1ecd1933e4d8c0fc3215c6` - -**Known Issues** -Ingress Controller only supports cluster scoped IngressClass and needs cluster wide permission for this object, otherwise it is not going to start. -We plan to fix this in v1.0.1 and issues #7510 and #7502 are tracking this. - -Changes: -- [X] [#7529](https://github.com/kubernetes/ingress-nginx/pull/7529) End-to-end tests for canary affinity -- [X] [#7489](https://github.com/kubernetes/ingress-nginx/pull/7489) docs: Clarify default-backend behavior -- [X] [#7524](https://github.com/kubernetes/ingress-nginx/pull/7524) docs for migration to apiVersion networking.k8s.io/v1 -- [X] [#7443](https://github.com/kubernetes/ingress-nginx/pull/7443) fix ingress-nginx panic when the certificate format is wrong. -- [X] [#7521](https://github.com/kubernetes/ingress-nginx/pull/7521) Update ingress to go 1.17 -- [X] [#7493](https://github.com/kubernetes/ingress-nginx/pull/7493) Add appProtocol field to all ServicePorts -- [X] [#7525](https://github.com/kubernetes/ingress-nginx/pull/7525) improve RELEASE.md -- [X] [#7203](https://github.com/kubernetes/ingress-nginx/pull/7203) Make HPA behavior configurable -- [X] [#7487](https://github.com/kubernetes/ingress-nginx/pull/7487)[Cherry - Pick] - Fix default backend annotation and tests -- [X] [#7459](https://github.com/kubernetes/ingress-nginx/pull/7459) Add controller.watchIngressWithoutClass config option -- [X] [#7478](https://github.com/kubernetes/ingress-nginx/pull/7478) Release new helm chart with certgen fixed -- [X] [#7341](https://github.com/kubernetes/ingress-nginx/pull/7341) Fix IngressClass logic for newer releases (#7341) -- [X] [#7355](https://github.com/kubernetes/ingress-nginx/pull/7355) Downgrade Lua modules for s390x (#7355) -- [X] [#7319](https://github.com/kubernetes/ingress-nginx/pull/7319) Lower webhook timeout for digital ocean (#7319) -- [X] [#7161](https://github.com/kubernetes/ingress-nginx/pull/7161) fix: allow scope/tcp/udp configmap namespace to altered (#7161) -- [X] [#7331](https://github.com/kubernetes/ingress-nginx/pull/7331) Fix forwarding of auth-response-headers to gRPC backends (#7331) -- [X] [#7332](https://github.com/kubernetes/ingress-nginx/pull/7332) controller: ignore non-service backends (#7332) -- [X] [#7314](https://github.com/kubernetes/ingress-nginx/pull/7314) Add configuration to disable external name service feature -- [X] [#7313](https://github.com/kubernetes/ingress-nginx/pull/7313) Add file containing stable release -- [X] [#7311](https://github.com/kubernetes/ingress-nginx/pull/7311) Handle named (non-numeric) ports correctly -- [X] [#7308](https://github.com/kubernetes/ingress-nginx/pull/7308) Updated v1beta1 to v1 as its deprecated -- [X] [#7298](https://github.com/kubernetes/ingress-nginx/pull/7298) Speed up admission hook by eliminating deep copy of Ingresses in CheckIngress -- [X] [#7242](https://github.com/kubernetes/ingress-nginx/pull/7242) Retry to download maxmind DB if it fails -- [X] [#7228](https://github.com/kubernetes/ingress-nginx/pull/7228) Discover mounted geoip db files -- [X] [#7208](https://github.com/kubernetes/ingress-nginx/pull/7208) ingress/tcp: add additional error logging on failed -- [X] [#7190](https://github.com/kubernetes/ingress-nginx/pull/7190) chart: using Helm builtin capabilities check -- [X] [#7146](https://github.com/kubernetes/ingress-nginx/pull/7146) Use ENV expansion for namespace in args -- [X] [#7107](https://github.com/kubernetes/ingress-nginx/pull/7107) Fix MaxWorkerOpenFiles calculation on high cores nodes -- [X] [#7076](https://github.com/kubernetes/ingress-nginx/pull/7076) Rewrite clean-nginx-conf.sh in Go to speed up admission webhook -- [X] [#7031](https://github.com/kubernetes/ingress-nginx/pull/7031) Remove mercurial from build -- [X] [#6990](https://github.com/kubernetes/ingress-nginx/pull/6990) Use listen to ensure the port is free -- [X] [#6944](https://github.com/kubernetes/ingress-nginx/pull/6944) Update proper default value for HTTP2MaxConcurrentStreams in Docs -- [X] [#6940](https://github.com/kubernetes/ingress-nginx/pull/6940) Fix definition order of modsecurity directives -- [X] [#7156] Drops support for Ingress Object v1beta1 - -### 0.50.0 - -**Image:** - -- `k8s.gcr.io/ingress-nginx/controller:v0.50.0@sha256:f46fc2d161c97a9d950635acb86fb3f8d4adcfb03ee241ea89c6cde16aa3fdf8` - -This release makes the annotation `annotation-value-word-blocklist` backwards compatible by being an empty list instead of prescribed defaults. -Effectively reverting [7874](https://github.com/kubernetes/ingress-nginx/pull/7874) but keeping the functionality of `annotation-value-word-blocklist` - -See Issue [7939](https://github.com/kubernetes/ingress-nginx/pull/7939) for more discussion - -Admins should still consider putting a reasonable block list in place, more information on why can be found [here](https://github.com/kubernetes/ingress-nginx/issues/7837) and how in our documentation [here](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#annotation-value-word-blocklist) - -_Changes:_ -- [7963](https://github.com/kubernetes/ingress-nginx/pull/7963) Change sanitization message from error to warning (#7963) -- [7942](https://github.com/kubernetes/ingress-nginx/pull/7942) update default block list,docs, tests (#7942) - -### 0.49.3 - -**Image:** - -- `k8s.gcr.io/ingress-nginx/controller:v0.49.3@sha256:35fe394c82164efa8f47f3ed0be981b3f23da77175bbb8268a9ae438851c8324` - -_Changes:_ - -- [X] [#7731](https://github.com/kubernetes/ingress-nginx/pull/7727) Fix selector for shutting down Pods status -- [X] [#7741](https://github.com/kubernetes/ingress-nginx/pull/7719) Fix overlap check when ingress is configured as canary - -### 0.49.2 - -**Image:** - -- `k8s.gcr.io/ingress-nginx/controller:v0.49.2@sha256:84e351228337bb7b09f0e90e6b6f5e2f8f4cf7d618c1ddc1e966f23902d273db` - -_Changes:_ - -- [x] [#7702](https://github.com/kubernetes/ingress-nginx/pull/#7702) upgrade lua-resty-balancer to v0.04 - -### 0.49.1 - -**Image:** - -- `k8s.gcr.io/ingress-nginx/controller:v0.49.1@sha256:4080849490fd4f61416ad7bdba6fdd0ee58164f2e13df1128b407ce82d5f3986` - -_New Features:_ - - -_Changes:_ - -- [x] [#7670](https://github.com/kubernetes/ingress-nginx/pull/7670) Change enable-snippet to allow-snippet-annotation (#7670) (#7677) -- [x] [#7676](https://github.com/kubernetes/ingress-nginx/pull/7676) Fix opentracing in v0.X (#7676) -- [x] [#7643](https://github.com/kubernetes/ingress-nginx/pull/7643) Update NGINX base image to v1.19 (#7643) -- [x] [#7665](https://github.com/kubernetes/ingress-nginx/pull/7665) Add option to force enabling snippet directives (7665) -- [x] [#7521](https://github.com/kubernetes/ingress-nginx/pull/7521) Update ingress to go 1.17 (#7521) - -### 0.49.0 - -**Image:** - -- `k8s.gcr.io/ingress-nginx/controller:v0.49.0@sha256:e9707504ad0d4c119036b6d41ace4a33596139d3feb9ccb6617813ce48c3eeef` - -_New Features:_ - - -_Changes:_ - -- [x] [#7486](https://github.com/kubernetes/ingress-nginx/pull/7486) Fix default backend annotation test -- [x] [#7481](https://github.com/kubernetes/ingress-nginx/pull/7481) Add linux node selector as default -- [x] [#6750](https://github.com/kubernetes/ingress-nginx/pull/6750) allow kb granularity for lua shared dicts -- [x] [#7463](https://github.com/kubernetes/ingress-nginx/pull/7463) Improved disableaccesslog tests -- [x] [#7485](https://github.com/kubernetes/ingress-nginx/pull/7485) update e2e test images to newest promoted one -- [x] [#7479](https://github.com/kubernetes/ingress-nginx/pull/7479) Make custom-default-backend upstream name more unique -- [x] [#7477](https://github.com/kubernetes/ingress-nginx/pull/7477) Trigger webhook image generation -- [x] [#7475](https://github.com/kubernetes/ingress-nginx/pull/7475) Migrate the webhook-certgen program to inside ingress repo -- [x] [#7331](https://github.com/kubernetes/ingress-nginx/pull/7331) Fix forwarding of auth-response-headers to gRPC backends -- [x] [#7228](https://github.com/kubernetes/ingress-nginx/pull/7228) fix: discover mounted geoip db files -- [x] [#7242](https://github.com/kubernetes/ingress-nginx/pull/7242) Retry to download maxmind DB if it fails -- [x] [#7473](https://github.com/kubernetes/ingress-nginx/pull/7473) update to newest image -- [x] [#7386](https://github.com/kubernetes/ingress-nginx/pull/7386) Add hostname value to override pod's hostname -- [x] [#7467](https://github.com/kubernetes/ingress-nginx/pull/7467) use listen to ensure the port is free -- [x] [#7411](https://github.com/kubernetes/ingress-nginx/pull/7411) Update versions of components for base image, including `nginx-http-auth-digest`, `ngx_http_substitutions_filter_module`, `nginx-opentracing`, `opentracing-cpp`, `ModSecurity-nginx`, `yaml-cpp`, `msgpack-c`, `lua-nginx-module`, `stream-lua-nginx-module`, `lua-upstream-nginx-module`, `luajit2`, `dd-opentracing-cpp`, `ngx_http_geoip2_module`, `nginx_ajp_module`, `lua-resty-string`, `lua-resty-balancer`, `lua-resty-core`, `lua-cjson`, `lua-resty-cookie`, `lua-resty-lrucache`, `lua-resty-dns`, `lua-resty-http`, `lua-resty-memcached`, `lua-resty-ipmatcher`. -- [x] [#7462](https://github.com/kubernetes/ingress-nginx/pull/7462) Update configmap.md -- [x] [#7369](https://github.com/kubernetes/ingress-nginx/pull/7369) Change all master reference to main -- [x] [#7245](https://github.com/kubernetes/ingress-nginx/pull/7245) Allow overriding of the default response format -- [x] [#7449](https://github.com/kubernetes/ingress-nginx/pull/7449) Fix cap for NET_BIND_SERVICE -- [x] [#7450](https://github.com/kubernetes/ingress-nginx/pull/7450) correct ingress-controller naming -- [x] [#7437](https://github.com/kubernetes/ingress-nginx/pull/7437) added K8s v1.22 tip for kind cluster,bug-report -- [x] [#7455](https://github.com/kubernetes/ingress-nginx/pull/7455) Add documentation for monitoring without helm -- [x] [#7454](https://github.com/kubernetes/ingress-nginx/pull/7454) Update go version to v1.16, modules and remove ioutil -- [x] [#7452](https://github.com/kubernetes/ingress-nginx/pull/7452) run k8s job ci pipeline with 1.21.2 in main branch -- [x] [#7451](https://github.com/kubernetes/ingress-nginx/pull/7451) Prepare for go v1.16 -- [x] [#7434](https://github.com/kubernetes/ingress-nginx/pull/7434) Helm - Enable configuring request and limit for containers in webhook jobs -- [x] [#6864](https://github.com/kubernetes/ingress-nginx/pull/6864) Add scope configuration check -- [x] [#7421](https://github.com/kubernetes/ingress-nginx/pull/7421) Bump PDB API version to v1 -- [x] [#7431](https://github.com/kubernetes/ingress-nginx/pull/7431) Add http request test to annotaion ssl cipher test -- [x] [#7426](https://github.com/kubernetes/ingress-nginx/pull/7426) Removed tabs and one extra-space -- [x] [#7423](https://github.com/kubernetes/ingress-nginx/pull/7423) Fixed chart version -- [x] [#7424](https://github.com/kubernetes/ingress-nginx/pull/7424) Add dev-v1 branch into helm releaser -- [x] [#7415](https://github.com/kubernetes/ingress-nginx/pull/7415) added checks to verify backend works with the given configs -- [x] [#7371](https://github.com/kubernetes/ingress-nginx/pull/7371) Enable session affinity for canaries -- [x] [#6985](https://github.com/kubernetes/ingress-nginx/pull/6985) auto backend protocol for HTTP/HTTPS -- [x] [#7394](https://github.com/kubernetes/ingress-nginx/pull/7394) reorder contributing infos -- [x] [#7224](https://github.com/kubernetes/ingress-nginx/pull/7224) docs：update troubleshooting.md -- [x] [#7353](https://github.com/kubernetes/ingress-nginx/pull/7353) aws-load-balancer-internal is a boolean value -- [x] [#7387](https://github.com/kubernetes/ingress-nginx/pull/7387) Automatically add area labels to help triaging -- [x] [#7360](https://github.com/kubernetes/ingress-nginx/pull/7360) grpc - replaced fortune-builder app with official greeter app -- [x] [#7361](https://github.com/kubernetes/ingress-nginx/pull/7361) doc: fix monitoring usage docs -- [x] [#7365](https://github.com/kubernetes/ingress-nginx/pull/7365) update OWNERS and aliases files -- [x] [#7039](https://github.com/kubernetes/ingress-nginx/pull/7039) Add missing tests for store/endpoint -- [x] [#7364](https://github.com/kubernetes/ingress-nginx/pull/7364) Add cpanato as Helm chart approver -- [x] [#7362](https://github.com/kubernetes/ingress-nginx/pull/7362) changed syntax from v1beta1 to v1 - -### 0.48.1 - -**Image:** - -- `k8s.gcr.io/ingress-nginx/controller:v0.48.1@sha256:e9fb216ace49dfa4a5983b183067e97496e7a8b307d2093f4278cd550c303899` - -_New Features:_ - - -_Changes:_ - -- [X] [#7298](https://github.com/kubernetes/ingress-nginx/pull/ ) Speed up admission hook by eliminating deep - copy of Ingresses in CheckIngress -- [X] [#6940](https://github.com/kubernetes/ingress-nginx/pull/6940) Fix definition order of modsecurity - directives for controller -- [X] [#7314](https://github.com/kubernetes/ingress-nginx/pull/7314) Add configuration to disable external name service feature #7314 -- [X] [#7076](https://github.com/kubernetes/ingress-nginx/pull/7076) Rewrite clean-nginx-conf.sh in Go to speed up - admission webhook -- [X] [#7255](https://github.com/kubernetes/ingress-nginx/pull/7255) Fix nilpointer in admission and remove failing - test #7255 -- [X] [#7216](https://github.com/kubernetes/ingress-nginx/pull/7216) Admission: Skip validation checks if an ingress - is marked as deleted #7216 - -### 1.0.0-beta.3 -** This is a breaking change** - -This release only supports Kubernetes versions >= v1.19. The support for Ingress Object in `networking.k8s.io/v1beta` is being dropped and manifests should now use `networking.k8s.io/v1`. - -**Image:** - -- `k8s.gcr.io/ingress-nginx/controller:v1.0.0-beta.3@sha256:44a7a06b71187a4529b0a9edee5cc22bdf71b414470eff696c3869ea8d90a695` - -Changes: - -- [X] [#7487](https://github.com/kubernetes/ingress-nginx/pull/7487)[Cherry - Pick] - Fix default backend annotation and tests -- [X] [#7459](https://github.com/kubernetes/ingress-nginx/pull/7459) Add controller.watchIngressWithoutClass config option -- [X] [#7478](https://github.com/kubernetes/ingress-nginx/pull/7478) Release new helm chart with certgen fixed - -### 1.0.0-beta.1 -**THIS IS A BREAKING CHANGE** - -This release only supports Kubernetes versions >= v1.19. The support for Ingress Object in `networking.k8s.io/v1beta` is being dropped and manifests should now use `networking.k8s.io/v1`. - -**Image:** - -- `k8s.gcr.io/ingress-nginx/controller:v1.0.0-beta.1@sha256:f058f3fdc940095957695829745956c6acddcaef839907360965e27fd3348e2e` - -_ New Features:_ - -_Changes:_ - -- [X] [#7341](https://github.com/kubernetes/ingress-nginx/pull/7341) Fix IngressClass logic for newer releases (#7341) -- [X] [#7355](https://github.com/kubernetes/ingress-nginx/pull/7355) Downgrade Lua modules for s390x (#7355) -- [X] [#7319](https://github.com/kubernetes/ingress-nginx/pull/7319) Lower webhook timeout for digital ocean (#7319) -- [X] [#7161](https://github.com/kubernetes/ingress-nginx/pull/7161) fix: allow scope/tcp/udp configmap namespace to altered (#7161) -- [X] [#7331](https://github.com/kubernetes/ingress-nginx/pull/7331) Fix forwarding of auth-response-headers to gRPC backends (#7331) -- [X] [#7332](https://github.com/kubernetes/ingress-nginx/pull/7332) controller: ignore non-service backends (#7332) - -### 1.0.0-alpha.2 -**THIS IS A BREAKING CHANGE** - -This release only supports Kubernetes versions >= v1.19. The support for Ingress Object in `networking.k8s.io/v1beta` is being dropped and manifests should now use `networking.k8s.io/v1`. - -**Image:** - -- `k8s.gcr.io/ingress-nginx/controller:v1.0.0-alpha.2@sha256:04a0ad3a1279c2a58898e789eed767eafa138ee1e5b9b23a988c6e8485cf958d` - -_ New Features:_ - -- [X] [#7314](https://github.com/kubernetes/ingress-nginx/pull/7314) Add configuration to disable external name service feature -- [X] [#7313](https://github.com/kubernetes/ingress-nginx/pull/7313) Add file containing stable release -- [X] [#7311](https://github.com/kubernetes/ingress-nginx/pull/7311) Handle named (non-numeric) ports correctly -- [X] [#7308](https://github.com/kubernetes/ingress-nginx/pull/7308) Updated v1beta1 to v1 as its deprecated -- [X] [#7298](https://github.com/kubernetes/ingress-nginx/pull/7298) Speed up admission hook by eliminating deep copy of Ingresses in CheckIngress -- [X] [#7242](https://github.com/kubernetes/ingress-nginx/pull/7242) Retry to download maxmind DB if it fails -- [X] [#7228](https://github.com/kubernetes/ingress-nginx/pull/7228) Discover mounted geoip db files -- [X] [#7208](https://github.com/kubernetes/ingress-nginx/pull/7208) ingress/tcp: add additional error logging on failed -- [X] [#7190](https://github.com/kubernetes/ingress-nginx/pull/7190) chart: using Helm builtin capabilities check -- [X] [#7146](https://github.com/kubernetes/ingress-nginx/pull/7146) Use ENV expansion for namespace in args -- [X] [#7107](https://github.com/kubernetes/ingress-nginx/pull/7107) Fix MaxWorkerOpenFiles calculation on high cores nodes -- [X] [#7076](https://github.com/kubernetes/ingress-nginx/pull/7076) Rewrite clean-nginx-conf.sh in Go to speed up admission webhook -- [X] [#7031](https://github.com/kubernetes/ingress-nginx/pull/7031) Remove mercurial from build -- [X] [#6990](https://github.com/kubernetes/ingress-nginx/pull/6990) Use listen to ensure the port is free -- [X] [#6944](https://github.com/kubernetes/ingress-nginx/pull/6944) Update proper default value for HTTP2MaxConcurrentStreams in Docs -- [X] [#6940](https://github.com/kubernetes/ingress-nginx/pull/6940) Fix definition order of modsecurity directives - -### 1.0.0-alpha.1 -**THIS IS A BREAKING CHANGE** - -This release only supports Kubernetes versions >= v1.19. The support for Ingress Object in `networking.k8s.io/v1beta` is being dropped and manifests should now use `networking.k8s.io/v1`. - -**Image:** - -- `k8s.gcr.io/ingress-nginx/controller:v1.0.0-alpha.1@sha256:32f3f02a038c0d7cf33b71a14028c3a4ddee6f4c3fe5fadfa14b915e5e0d9faf` - -_ New Features:_ - -- [X] [#7156] Drops support for Ingress Object v1beta1 - ### 0.47.0 **Image:** @@ -972,15 +12,15 @@ _New Features:_ _Changes:_ -- [X] [#7179](https://github.com/kubernetes/ingress-nginx/pull/7179) Upgrade Nginx to 1.20.1 -- [X] [#7101](https://github.com/kubernetes/ingress-nginx/pull/7101) Removed Codecov -- [X] [#6993](https://github.com/kubernetes/ingress-nginx/pull/6993) Fix cookieAffinity log printing error -- [X] [#7046](https://github.com/kubernetes/ingress-nginx/pull/7046) Allow configuring controller container name -- [X] [#6994](https://github.com/kubernetes/ingress-nginx/pull/6994) Fixed oauth2 callback url -- [X] [#6740](https://github.com/kubernetes/ingress-nginx/pull/6740) non-host canary ingress use default server name as host to merge -- [X] [#7126](https://github.com/kubernetes/ingress-nginx/pull/7126) set x-forwarded-scheme to be the same as x-forwarded-proto -- [X] [#6734](https://github.com/kubernetes/ingress-nginx/pull/6734) Update controller-poddisruptionbudget.yaml -- [X] [#7117](https://github.com/kubernetes/ingress-nginx/pull/7117) Adding annotations for HPA +- [X] [#7179] Upgrade Nginx to 1.20.1 +- [X] [#7101] Removed Codecov +- [X] [#6993] Fix cookieAffinity log printing error +- [X] [#7046] Allow configuring controller container name +- [X] [#6994] Fixed oauth2 callback url +- [X] [#6740] non-host canary ingress use default server name as host to merge +- [X] [#7126] set x-forwarded-scheme to be the same as x-forwarded-proto +- [X] [#6734] Update controller-poddisruptionbudget.yaml +- [X] [#7117] Adding annotations for HPA ### 0.46.0 @@ -1421,7 +461,7 @@ _Documentation:_ _New Features:_ - NGINX 1.19.2 -- New configmap option `enable-real-ip` to enable [realip_module](https://nginx.org/en/docs/http/ngx_http_realip_module.html) +- New configmap option `enable-real-ip` to enable [realip_module](http://nginx.org/en/docs/http/ngx_http_realip_module.html) - Use k8s.gcr.io vanity domain - Go 1.15 - client-go v0.18.6 @@ -1515,7 +555,7 @@ _New Features:_ - Configure User-Agent for [client-go](https://github.com/kubernetes/ingress-nginx/pull/5700) - Switch to [gcr.io](https://cloud.google.com/container-registry/) as container registry - Use cloud-build to build [container images](https://console.cloud.google.com/gcr/images/k8s-artifacts-prod/US/ingress-nginx) -- Publish images using [artifact promotion tooling](https://sigs.k8s.io/promo-tools) +- Publish images using [Container Image Promoter](https://github.com/kubernetes-sigs/k8s-container-image-promoter) - Go 1.14.4 - client-go v0.18.5 @@ -1743,7 +783,7 @@ _New Features:_ - NGINX 1.17.10 - OpenSSL 1.1.1g - [CVE-2020-1967](https://cve.mitre.org/cgi-bin/cvename.cgi?name=2020-1967) - OCSP stapling -- Helm chart [stable/nginx-ingress](https://github.com/helm/charts/tree/master/stable/nginx-ingress) is now maintained in the [ingress-nginx](https://github.com/kubernetes/ingress-nginx/tree/main/charts/ingress-nginx) repository +- Helm chart [stable/nginx-ingress](https://github.com/helm/charts/tree/master/stable/nginx-ingress) is now maintained in the [ingress-nginx](https://github.com/kubernetes/ingress-nginx/tree/master/charts/ingress-nginx) repository - Support for custom Maxmind GeoLite2 Databases [flag --maxmind-edition-ids](https://kubernetes.github.io/ingress-nginx/user-guide/cli-arguments/) - New [PathType](https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types) and [IngressClass](https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class) fields. Requires Kubernetes v1.18 or higher - Enable configuration of lua plugins using the configuration configmap @@ -2057,7 +1097,7 @@ _Breaking Changes:_ ``` Due to upcoming data privacy regulations, we are making significant changes to how you access free GeoLite2 databases starting December 30, 2019. - Learn more on our blog https://blog.maxmind.com/2019/12/significant-changes-to-accessing-and-using-geolite2-databases/ + Learn more on our blog https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases/ ``` Because of this change, it is not clear we can provide the databases directly from the docker image. @@ -2193,7 +1233,7 @@ _New Features:_ If the active connections end before that, the pod will terminate gracefully at that time. - To effectively take advantage of this feature, the Configmap feature [worker-shutdown-timeout](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#worker-shutdown-timeout) new value is `240s` instead of `10s`. + To efectively take advantage of this feature, the Configmap feature [worker-shutdown-timeout](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#worker-shutdown-timeout) new value is `240s` instead of `10s`. **IMPORTANT:** this value has a side effect during reloads, consuming more memory until the old NGINX workers are replaced. @@ -2595,7 +1635,7 @@ _Documentation:_ _New Features:_ - NGINX 1.15.9 -- New `canary-by-header-value` [annotation](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/annotations.md#canary). +- New `canary-by-header-value` [annotation](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/annotations.md#canary). - New debug binary to get runtime information from lua [3686](https://github.com/kubernetes/ingress-nginx/pull/3686) - Support for Opentracing with Datadog - New [kubectl plugin](https://github.com/kubernetes/ingress-nginx/pull/3779) **Alpha** @@ -2603,7 +1643,7 @@ _New Features:_ _Breaking changes:_ - The NGINX server listening in port 18080 was removed. It was replaced by a server using an unix socket as port [#3684](https://github.com/kubernetes/ingress-nginx/pull/3684) - This server was internal to the ingress controller. In case this was being acceded from the outside, you can restore the old server using the `http-snippet` feature in the configuration configmap like: + This server was internal to the ingress controller. In case this was being acceded from the outside, you can restore the old server using the `http-snipet` feature in the configuration configmap like: ```yaml http-snippet: | @@ -2964,7 +2004,7 @@ _Documentation:_ _New Features:_ - NGINX 1.15.5 -- Support for *regular expressions* in paths https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/ingress-path-matching.md +- Support for *regular expressions* in paths https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/ingress-path-matching.md - Provide possibility to block IPs, User-Agents and Referers globally - Remove --default-backend-service requirement. Use the flag only for custom default backends - Valgrind and Openresty gdb tools @@ -3165,7 +2205,7 @@ _Documentation:_ _New Features:_ -- [Grafana dashboards](https://github.com/kubernetes/ingress-nginx/tree/main/deploy/grafana/dashboards) +- [Grafana dashboards](https://github.com/kubernetes/ingress-nginx/tree/master/deploy/grafana/dashboards) _Changes:_ @@ -3487,7 +2527,7 @@ _New Features:_ - The annotation `nginx.ingress.kubernetes.io/grpc-backend: "true"` enable this feature - If the gRPC service requires TLS `nginx.ingress.kubernetes.io/secure-backends: "true"` - Configurable load balancing with EWMA -- Support for [lua-resty-waf](https://github.com/p0pr0ck5/lua-resty-waf) as alternative to ModSecurity. [Check configuration guide](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/annotations.md#lua-resty-waf) +- Support for [lua-resty-waf](https://github.com/p0pr0ck5/lua-resty-waf) as alternative to ModSecurity. [Check configuration guide](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/annotations.md#lua-resty-waf) - Support for session affinity when dynamic configuration is enabled. - Add NoAuthLocations and default it to "/.well-known/acme-challenge" @@ -4524,7 +3564,7 @@ _New Features:_ - Custom log formats using `log-format-upstream` directive in the configuration configmap. - Force redirect to SSL using the annotation `ingress.kubernetes.io/force-ssl-redirect` - Prometheus metric for VTS status module (transparent, just enable vts stats) -- Improved external authentication adding `ingress.kubernetes.io/auth-signin` annotation. Please check this [example](https://github.com/kubernetes/ingress/tree/main/examples/external-auth/nginx) +- Improved external authentication adding `ingress.kubernetes.io/auth-signin` annotation. Please check this [example](https://github.com/kubernetes/ingress/tree/master/examples/external-auth/nginx) _Breaking changes:_ @@ -4588,9 +3628,9 @@ _Changes:_ _New Features:_ -- New configuration flag `proxy-set-headers` to allow set custom headers before send traffic to backends. [Example here](https://github.com/kubernetes/ingress/tree/main/examples/customization/custom-headers/nginx) +- New configuration flag `proxy-set-headers` to allow set custom headers before send traffic to backends. [Example here](https://github.com/kubernetes/ingress/tree/master/examples/customization/custom-headers/nginx) - Disable directive access_log globally using `disable-access-log: "true"` in the configuration ConfigMap. -- Sticky session per Ingress rule using the annotation `ingress.kubernetes.io/affinity`. [Example here](https://github.com/kubernetes/ingress/tree/main/examples/affinity/cookie/nginx) +- Sticky session per Ingress rule using the annotation `ingress.kubernetes.io/affinity`. [Example here](https://github.com/kubernetes/ingress/tree/master/examples/affinity/cookie/nginx) _Changes:_ diff --git a/GOLANG_VERSION b/GOLANG_VERSION deleted file mode 100644 index d8c40e539..000000000 --- a/GOLANG_VERSION +++ /dev/null @@ -1 +0,0 @@ -1.23.6 diff --git a/ISSUE_TRIAGE.md b/ISSUE_TRIAGE.md deleted file mode 100644 index e6d2c1d8e..000000000 --- a/ISSUE_TRIAGE.md +++ /dev/null @@ -1,89 +0,0 @@ -# Triage Process - -As any kind of contributor (triage, reviewer ...), always have in mind that if a user came to us and raised an issue, the user may have a real problem. We must assume that, and not the opposite (the user needs to prove to us that this is a bug). Keeping that in mind, **be nice with users, even if you don’t agree with them** - -Note that this guide refers to contributing through issue triaging. If you are interested in contributing to actual sources of the repository, see [this guide](./CONTRIBUTING.md). - -## General Information - -The triage process of the ingress-nginx maintainers is based on the [triage process guidelines](https://github.com/kubernetes/community/blob/master/contributors/guide/issue-triage.md) of the Kubernetes community - -However the exact process of the ingress-nginx maintainers may differ in certain aspects. This doc gives a more precise overview on how the ingress-nginx maintainers approach the issue triage process and other processes that are related. - -## Triage Flow (Issues) - -This section describes the different stages of the triage flow for issues. - -### Prepare Issues -New issues come in with the labels `needs-triage` and `needs-priority` and one of: `kind/bug`, `kind/feature` or `kind/support`. Unfortunately there are also some legacy issues that only have a `kind/*` label but neither `needs-triage` nor `needs-priority` . However for every issue that does not have the `triage-accepted` label the following steps have to be done to prepare them for further processing: - -* Filter for issues [without the `triage-accepted`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+-label%3Atriage%2Faccepted+is%3Aissue) label. -* Check if all necessary information are available. This is basically true, if people filled out the issue template correctly. If necessary information is missing, ask the author to add the missing information and add the label `triage/needs-information` if not already present. If already present, send the author a friendly reminder to add those. -* Check if the used versions of ingress-nginx and Kubernetes is supported. Note that [we only support n-3 versions](https://github.com/kubernetes/ingress-nginx#support-versions-table). If the version is not supported, ask the author to upgrade to newer versions and see if the error still persists. -* Read through the issue description and comments briefly to understand what the issue is about. Also check if the kind and area is correct, and adjust it if necessary. If the issue is understandable add the label `triage-accepted`. -* If at any point you don't know how to proceed with an issue during the triage process, tag one of the [core maintainers](OWNERS_ALIASES) in the issue to raise attention or alternatively come to [this slack channel](https://kubernetes.slack.com/archives/C021E147ZA4) which may be the quicker way as people tend to miss github notifications. - -Note: Issues that are stale for 90 days are being closed automatically. However we could be missing a bug here, so from time to time it makes sense to go over the closed ones and see if there is something important. Use [this filter](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aclosed+is%3Aissue+label%3Alifecycle%2Frotten+) to find those. - -Who and When? -* Basically everyone who wants to contribute can do the mentioned steps at any time. - -### Issue Prioritization -For all issues, where all necessary information is available thus triage is accepted, we need to do some prioritization: - -* Go through all issues with label [`triage-accepted`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+is%3Aissue+label%3Atriage%2Faccepted+). -* Add appropriate priority label: `priority/backlog`, `priority/critical-urgent`, `priority/awaiting-more-evidence`, `priority/important-longterm`, `priority/important-soon` or `good first issue` - -Who and When? -* Basically every contributor should be able to do that. -* Tricky/important ones could be brought up during community meetings - -## Triage Flow (Pull Requests) - -This section describes the different stages of the triage flow for pull requests. - -### Prepare Pull Requests -Pull requests come in with the labels `needs-triage`, `needs-priority` and `needs-kind` and one that indicates the size(`size/*`). Unfortunately there are also some legacy pull requests that only have a `size/*` label but neither `needs-triage` nor `needs-priority` . However for every pull request that does not have the `triage-accepted` label the following steps should be done to prepare them for further processing: - -* Filter for pull requests [without the `triage-accepted`](https://github.com/kubernetes/ingress-nginx/pulls?q=is%3Aopen+-label%3Atriage%2Faccepted+is%3Apr) label. -* Check if the cla is signed and all necessary information are available. This is basically true, if people filled out the pull request template correctly. If everything is fine add the `triage-accepted` label. -* If at any point you don't know how to proceed with an issue during the triage process, tag one of the [core maintainers](OWNERS_ALIASES) in the issue to raise attention or alternatively come to [this slack channel](https://kubernetes.slack.com/archives/C021E147ZA4) which may be the quicker way as people tend to miss github notifications. - -Who and When? -* Basically everyone who wants to contribute can do the mentioned steps at any time. - -### Pull Request Prioritization -For all pull requests, where all necessary information is available and cla is signed thus triage is accepted, we need to do some prioritization: - -* Go through all pull requests with label [`triage-accepted`](https://github.com/kubernetes/ingress-nginx/pulls?q=is%3Aopen+is%3Apr+label%3Atriage%2Faccepted). -* Sync the `kind/*` and `priority/*` label from the linked issue for the pull request. If the pull request does not have any issue associated (which normally should not be the case), add an appropriate priority and kind label (one of: `priority/backlog`, `priority/critical-urgent`, `priority/important-longterm`, `priority/important-soon`) - -Who and When? -* Basically every contributor should be able to do that. -* Tricky/important ones could be brought up during community meetings - -## Labels -Labels are helpful for issues or pull requests to indicate in which lifecycle state they are currently and to categorize them. This section describes the most important ones with the additional info about how to add those. A complete label list of the Kubernetes community can be found [here](https://github.com/kubernetes/kubernetes/labels) while a complete label list for this project can be found [here](https://github.com/kubernetes/ingress-nginx/labels). However, here the most important ones: - -* Triage: - * [`needs-triage`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+is%3Aissue+label%3Aneeds-triage): Indicates that the issue or pull request needs triage. Automatically added. - * [`triage/accepted`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+label%3Atriage%2Faccepted+is%3Aissue+): Indicates that the issue is ready for further processing. Add with `/triage accepted`. - * [`triage/needs-information`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+label%3Atriage%2Fneeds-information+is%3Aissue+): Indicates that the issue lacks information. Add with `/triage needs-information`. -* Kind: - * [`kind/bug`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+label%3Akind%2Fbug+is%3Aissue): Indicates that the issue is assumed to be a bug. Add with `/kind bug`. Remove with `/remove-kind bug`. - * [`kind/feature`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+label%3Akind%2Ffeature+is%3Aissue+): Indicates that the issue is a feature request. Add with `/kind feature`. Remove with `/remove-kind feature`. - * [`kind/documentation`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+label%3Akind%2Fdocumentation+is%3Aissue+): Indicates that the issue is documentation related. Add with `/kind documentation`. Remove with `/remove-kind documentation`. - * [`kind/support`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+label%3Akind%2Fsupport+is%3Aissue+): Indicates the the issue is a support request. Add with `/kind support`. Remove with `/remove-kind support`. -* Area: - * [`area/helm`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+label%3Aarea%2Fhelm+is%3Aissue+): Indicates that the issue is related to helm charts. Add with `/area helm`. - * [`area/lua`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+label%3Aarea%2Flua+is%3Aissue+): Indicates that the issue is related to lua. Add with `/area lua`. - * [`area/docs`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+label%3Aarea%2Fdocs+is%3Aissue): Indicates that the issue is related to documentation. Add with `/area docs` . -* Priority: - * [`needs-priority`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+is%3Aissue+label%3Aneeds-priority): Indicates that the issue has no prioritization yet. Automatically added. - * [`priority/critical-urgent`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+label%3Apriority%2Fcritical-urgent+is%3Aissue+): indicates that the issue has highest priority. Add with `/priority critical-urgent`. - * [`priority/important-soon`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+label%3Apriority%2Fimportant-soon+is%3Aissue+): indicates that the issue should be worked on either currently soon, ideally in time for the next release. Add with `/priority important-soon`. - * [`priority/important-longterm`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+label%3Apriority%2Fimportant-longterm+is%3Aissue+): indicates that the issue is not important for now, but should be worked on in one of the upcoming releases. Add with `/priority important-longterm`. - * [`priority/backlog`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+label%3Apriority%2Fbacklog+is%3Aissue+): Indicates that the issue has the lowest priority. Add with `/priority backlog`. -* Other: - * [`help wanted`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22): indicates that the issue needs help from a contributor. Add with `/help`. - * [`good first issue`](https://github.com/kubernetes/ingress-nginx/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22): indicates that the issue needs help from a contributor and is a good first issue for new contributors. Add with `/good-first-issue`. \ No newline at end of file diff --git a/MANUAL_RELEASE.md b/MANUAL_RELEASE.md deleted file mode 100644 index 0ae7a4e37..000000000 --- a/MANUAL_RELEASE.md +++ /dev/null @@ -1,300 +0,0 @@ -# RELEASE PROCESS - -## 1. BUILD the new Ingress-Nginx-Controller image - -### a. Make changes in codebase - -- Make changes as per issue - -### b. Make changes to appropriate files in [images directory ](images) - -- Make changes in /images - -### c. Create Pull Request - -- Open a Pull Request for your changes considering the following steps to fire cloudbuild of a new image for the Ingress-Nginx-Controller: - - - In case of rare CVE fix or other reason to rebuild the nginx-base-image itself, look at the /images directory [NGINX Base Image](https://github.com/kubernetes/ingress-nginx/tree/main/images/nginx). - - - Example [NGINX_VERSION](images/nginx/rootfs/build.sh#L21), [SHA256](images/nginx/rootfs/build.sh#L124). - - - If you are updating any component in [build.sh](images/nginx/rootfs/build.sh) please also update the SHA256 checksum of that component as well, the cloud build will fail with an exit 10 if not. - -### d. Merge - -- Merging will fire cloudbuild, which will result in images being promoted to the [staging container registry](https://console.cloud.google.com/gcr/images/k8s-staging-ingress-nginx). - -### e. Make sure cloudbuild is a success - -- Wait for [cloud build](https://console.cloud.google.com/cloud-build/builds?project=k8s-staging-ingress-nginx). If you don't have access to cloudbuild, you can also have a look at [this](https://prow.k8s.io/?repo=kubernetes%2Fingress-nginx&job=post-*), to see the progress of the build. - -- Proceed only after cloud-build is successful in building a new Ingress-Nginx-Controller image. - - -## 2. If applicable, BUILD other images - -- If applicable, then build a new image of any other related component, ONLY IF APPLICABLE TO THE RELEASE - -### a. If applicable then make changes in relevant codebase - -- Change code as per issue - -### b. Make changes to appropriate files in [images directory ](images) - -- Sometimes, you may also be needing to rebuild, images for one or multiple other related components of the Ingress-Nginx-Controller ecosystem. Make changes to the required files in the /images directory, if/as applicable, in the context of the release you are attempting. : - - - [e2e](https://github.com/kubernetes/ingress-nginx/tree/main/test/e2e-image) - - - Update references to e2e-test-runner image [If applicable] : - - - [e2e-image](https://github.com/kubernetes/ingress-nginx/blob/main/test/e2e-image/Dockerfile#L1) - - [run-in-docker.sh](https://github.com/kubernetes/ingress-nginx/blob/main/build/run-in-docker.sh#L37) - - - [test-runner](https://github.com/kubernetes/ingress-nginx/tree/main/images/test-runner) - - - [echo](https://github.com/kubernetes/ingress-nginx/tree/main/images/echo) - - - [cfssl](https://github.com/kubernetes/ingress-nginx/tree/main/images/cfssl) - - - [fastcgi-helloserver](https://github.com/kubernetes/ingress-nginx/tree/main/images/fastcgi-helloserver) - - - [httpbin](https://github.com/kubernetes/ingress-nginx/tree/main/images/httpbin) - - - [kube-webhook-certgen](https://github.com/kubernetes/ingress-nginx/tree/main/images/kube-webhook-certgen) - -### c. Create PR - -- Open pull request(s) accordingly, to fire cloudbuild for rebuilding the component's image (if applicable). - -### d. Merge - -- Merging will fire cloudbuild, which will result in images being promoted to the [staging container registry](https://console.cloud.google.com/gcr/images/k8s-staging-ingress-nginx). - -### e. Make sure cloudbuild is a success - -- Wait for [cloud build](https://console.cloud.google.com/cloud-build/builds?project=k8s-staging-ingress-nginx). If you don't have access to cloudbuild, you can also have a look at [this](https://prow.k8s.io/?repo=kubernetes%2Fingress-nginx&job=post-*), to see the progress of the build. - -- Proceed only after cloud-build is successful in building a new Ingress-Nginx-Controller image. - - -## 3. PROMOTE the Image(s): - -Promoting the images basically means that images, that were pushed to staging container registry in the steps above, now are also pushed to the public container registry. Thus are publicly available. Follow these steps to promote images: - -### a. Get the sha - -- Get the sha of the new image(s) of the controller, (and any other component image IF APPLICABLE to release), from the cloudbuild, from steps above - - - The sha is available in output from [cloud build](https://console.cloud.google.com/cloud-build/builds?project=k8s-staging-ingress-nginx) - - - The sha is also visible here https://console.cloud.google.com/gcr/images/k8s-staging-ingress-nginx/global/controller - - - The sha is also visible [here](https://prow.k8s.io/?repo=kubernetes%2Fingress-nginx&job=post-*), after cloud build is finished. Click on the respective job, go to `Artifacts` section in the UI, then again `artifacts` in the directory browser. In the `build.log` at the very bottom you see something like this: - - ``` - ... - pushing manifest for us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx/controller:v1.0.2@sha256:e15fac6e8474d77e1f017edc33d804ce72a184e3c0a30963b2a0d7f0b89f6b16 - ... - ``` - -### b. Add the new image to [k8s.io](http://github.com/kubernetes/k8s.io) - -- The sha(s) from the step before (and the tag(s) for the new image(s) have to be added, as a new line, in a file, of the [k8s.io](http://github.com/kubernetes/k8s.io) project of Kubernetes organization. - -- Fork that other project (if you don't have a fork already). - -- Other project to fork [GitHub repo kubernetes/k8s.io](http://github.com/kubernetes/k8s.io) - -- Fetch --all and rebase to upstream if already forked. - -- Create a branch in your fork, named as the issue number for this release - -- In the related branch, of your fork, edit the file /registry.k8s.io/images/k8s-staging-ingress-nginx/images.yaml. - -- For making, it easier, you can edit your branch directly in the browser. But be careful about making any mistake. - -- Insert the sha(s) & the tag(s), in a new line, in this file [Project kubernetes/k8s.io Ingress-Nginx-Controller Images](https://github.com/kubernetes/k8s.io/blob/main/registry.k8s.io/images/k8s-staging-ingress-nginx/images.yaml) Look at this [example PR and the diff](https://github.com/kubernetes/k8s.io/pull/2536) to see how it was done before - -- Save and commit - -### c. Create PR - -- Open pull request to promote the new controller image. - -### d. Merge - -- Merge success is required for next step - -- Proceed only after cloud-build is successful in building a new Ingress-Nginx-Controller image. - - -## 4. PREPARE for a new Release - -- Make sure to get the tag and sha of the promoted image from the step before, either from cloudbuild or from [here](https://console.cloud.google.com/gcr/images/k8s-artifacts-prod/us/ingress-nginx/controller). - -- This involves editing of several files. So carefully follow the steps below and double check all changes with diff/grep etc., repeatedly. Mistakes here impact endusers. - -### a. Make sure your git workspace is ready - -- Get your git workspace ready - - - If not using a pre-existing fork, then Fork the repo kubernetes/ingress-nginx - - - Clone (to laptop or wherever) - - - Add upstream - - - Set upstream url to no_push - - - Checkout & switch to branch, named as per related new-release-issue-number - - - If already forked, and upstream already added, then `git fetch --all` and `git rebase upstream/main` (not origin) - - - Checkout a branch in your fork's clone - - - Perform any other diligence as needed - -- Prefer to edit only and only in your branch, in your Fork - -### b. Edit the semver tag - - [TAG](https://github.com/kubernetes/ingress-nginx/blob/main/TAG#L1) - -### c. Edit the helm Chart - - Change the below-mentioned [Fields in Chart.yaml](https://github.com/kubernetes/ingress-nginx/blob/main/charts/ingress-nginx/Chart.yaml) - - version - - appVersion - - kubeVersion (**ONLY if applicable**) - - annotations - - artifacthub.io/prerelease: "true" - - artifacthub.io/changes: | - - Replace this line and other lines under this annotation with the Changelog. One process to generate the Changelog is described below - - Install and configure GitHub cli as per the docs of gh-cli https://cli.github.com/, - - Change dir to your clone, of your fork, of the ingress-nginx project - - Run the below command and save the output to a txt file - - ``` - gh pr list -R kubernetes/ingress-nginx -s merged -L 38 -B main | cut -f1,2 | tee ~/Downloads/prlist.txt - ``` - - The -L 38 was used for 2 reasons. - - Default number of results is 30 and there were more than 30 PRs merged while releasing v1.1.1. If you see the current/soon-to-be-old changelog, you can look at the most recent PR number that has been accounted for already, and start from after that last accounted for PR. - - The other reason to use -L 38 was to omit the 39th, the 40th and the 41st line in the resulting list. These were non-relevant PRs. - - If you save the output of above command to a file called prlist.txt. It looks somewhat like this ; - - ``` - % cat ~/Downloads/prlist.txt - 8129 fix syntax in docs for multi-tls example - 8120 Update go in runner and release v1.1.1 - 8119 Update to go v1.17.6 - 8118 Remove deprecated libraries, update other libs - 8117 Fix codegen errors - 8115 chart/ghaction: set the correct permission to have access to push a release - .... - ``` - You can delete the lines, that refer to PRs of the release process itself. We only need to list the feature/bugfix PRs. You can also delete the lines that are housekeeping or not really worth mentioning in the changelog. - - you use some easy automation in bash/python/other, to get the PR-List that can be used in the changelog. For example, it's possible to use a bash scripty way, seen below, to convert those plaintext PR numbers into clickable links. - - ``` - #!/usr/bin/bash - - file="$1" - - while read -r line; do - pr_num=`echo "$line" | cut -f1` - pr_title=`echo "$line" | cut -f2` - echo "[$pr_num](https://github.com/kubernetes/ingress-nginx/pull/$pr_num) $pr_title" - done <$file - - ``` - - There was a parsing issue and path issue on MacOS, so above script had to be modified and MacOS monterey compatible script is below ; - - ``` - #!/bin/bash - - file="$1" - - while read -r line; do - pr_num=`echo "$line" | cut -f1` - pr_title=`echo "$line" | cut -f2` - echo \""[$pr_num](https://github.com/kubernetes/ingress-nginx/pull/$pr_num) $pr_title"\" - done <$file - - ``` - - If you saved the bash script content above, in a file like `$HOME/bin/prlist_to_changelog.sh`, then you could execute a command like this to get your prlist in a text file called changelog_content.txt;` - - ``` - prlist_to_changelog.sh ~/Downloads/prlist.txt | tee ~/Downloads//changelog_content.txt - ``` - -### d. Edit the values.yaml and run helm-docs - - - [Fields to edit in values.yaml](https://github.com/kubernetes/ingress-nginx/blob/main/charts/ingress-nginx/values.yaml) - - - tag - - digest - - - [helm-docs](https://github.com/norwoodj/helm-docs) is a tool that generates the README.md for a Helm chart automatically. In the CI pipeline workflow of GitHub actions (.github/workflows/ci.yaml), you can see how helm-docs is used. The CI pipeline is not designed to make commits back into the project, so we need to run helm-docs manually and commit the resulting generated README.md. You can obtain a recent version of the helm-docs binary here: https://github.com/norwoodj/helm-docs/releases. - ``` - helm-docs --chart-search-root charts - git diff charts/ingress-nginx/README.md - ``` - Take care of not leaving the helm-docs executable in your clone workspace or not committing the new README.md. - -### e. Edit the static manifests - - - Prepare to use a script to update the edit the static manifests and set the "image", "digest", "version" etc. fields to the desired value. - - - This script depends on kustomize and helm. The versions are pinned in `hack/.tool-versions` and you can use [asdf](https://github.com/asdf-vm/asdf#asdf) to install them - - - Execute the script to update static manifests using that script [hack/generate-deploy-scripts.sh](https://github.com/kubernetes/ingress-nginx/blob/main/hack/generate-deploy-scripts.sh) - - Open some of the manifests and check if the script worked properly - - - Use `grep -ir image: | less` on the deploy directory, to view for any misses by the script on image digest value or other undesired changes. The script should properly set the image and the digest fields to the desired tag and semver - - -### f. Edit the changelog - - [Changelog.md](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md) -- Each time a release is made, a new section is added to the Changelog.md file -- A new section in the Changelog.md file consists of 3 components listed below - - the "Image" - - the "Description" - - the "PRs list" -- Look at the previous content to understand what the 3 components look like. -- You can easily get the "Image" from a yaml manifest but be sure to look at a manifest in your git clone now and not the upstream on github. This is because, if you are following this documentation, then you generated manifests with new updated digest for the image, in step 4e above. You also most likely promoted the new image in a step above. Look at the previous release section in Changelog.md. The format looks like `registry.k8s.io/ingress-nginx/controller:.......`. One example of a yaml file to look at is /deploy/static/provider/baremetal/deploy.yaml (in your git clone branch and not on the upstream). -- Next, you need to have a good overview of the changes introduced in this release and based on that you write a description. Look at previous descriptions. Ask the ingress-nginx-dev channel if required. -- And then you need to add a list of the PRs merged, since the previous release. -- One process to generate this list of PRs is already described above in step 4c. So if you are following this document, then you have done this already and very likely have retained the file containing the list of PRs, in the format that is needed. - -### g. Edit the Documentation: - -- Update the version in [docs/deploy/index.md](docs/deploy/index.md) -- Update Supported versions in the Support Versions table in the README.md -- Execute the script to update e2e docs [hack/generate-e2e-suite-doc.sh](https://github.com/kubernetes/ingress-nginx/blob/main/hack/generate-e2e-suite-doc.sh) - -### h. Update README.md - -- Update the table in README.md in the root of the project to reflect the support matrix. Add the new release version and details in there. - -## 5. RELEASE new version - -### a. Create PR - -- Open PR for releasing the new version of the Ingress-Nginx-Controller ; - - Look at this PR for how it was done before [example PR](https://github.com/kubernetes/ingress-nginx/pull/7490) - - Create a PR - -### b. Merge - -- Merge should produce manifests as well as chart -- Check - - `helm repo update` - - `helm search repo ingress-nginx` - -## 6. GitHub release - -- Release to github - -- Edit the ghpages file as needed - -## TODO -- Automate & simplify as much as possible, whenever possible, however possible diff --git a/Makefile b/Makefile index 0b8f1f5c2..8a449ede8 100644 --- a/Makefile +++ b/Makefile @@ -29,14 +29,9 @@ SHELL=/bin/bash -o pipefail -o errexit # Use the 0.0 tag for testing, it shouldn't clobber any release builds TAG ?= $(shell cat TAG) -# The env below is called GO_VERSION and not GOLANG_VERSION because -# the gcb image we use to build already defines GOLANG_VERSION and is a -# really old version -GO_VERSION ?= $(shell cat GOLANG_VERSION) - # e2e settings # Allow limiting the scope of the e2e tests. By default run everything -FOCUS ?= +FOCUS ?= .* # number of parallel test E2E_NODES ?= 7 # run e2e test suite with tests that check for memory leaks? (default is false) @@ -54,13 +49,9 @@ ifeq ($(ARCH),) $(error mandatory variable ARCH is empty, either set it when calling the command or make sure 'go env GOARCH' works) endif -ifneq ($(PLATFORM),) - PLATFORM_FLAG="--platform" -endif +REGISTRY ?= gcr.io/k8s-staging-ingress-nginx -REGISTRY ?= us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx - -BASE_IMAGE ?= $(shell cat NGINX_BASE) +BASE_IMAGE ?= k8s.gcr.io/ingress-nginx/nginx:v20210530-g6aab4c291@sha256:a7356029dd0c26cc3466bf7a27daec0f4df73aa14ca6c8b871a767022a812c0b GOARCH=$(ARCH) @@ -70,8 +61,7 @@ help: ## Display this help .PHONY: image image: clean-image ## Build image for a particular arch. echo "Building docker image ($(ARCH))..." - docker build \ - ${PLATFORM_FLAG} ${PLATFORM} \ + @docker build \ --no-cache \ --build-arg BASE_IMAGE="$(BASE_IMAGE)" \ --build-arg VERSION="$(TAG)" \ @@ -80,110 +70,71 @@ image: clean-image ## Build image for a particular arch. --build-arg BUILD_ID="$(BUILD_ID)" \ -t $(REGISTRY)/controller:$(TAG) rootfs -.PHONY: gosec -gosec: - docker run --rm -it -w /source/ -v "$(pwd)"/:/source securego/gosec:2.11.0 -exclude=G109,G601,G104,G204,G304,G306,G307 -tests=false -exclude-dir=test -exclude-dir=images/ -exclude-dir=docs/ /source/... - -.PHONY: image-chroot -image-chroot: clean-chroot-image ## Build image for a particular arch. - echo "Building docker image ($(ARCH))..." - docker build \ - --no-cache \ - --build-arg BASE_IMAGE="$(BASE_IMAGE)" \ - --build-arg VERSION="$(TAG)" \ - --build-arg TARGETARCH="$(ARCH)" \ - --build-arg COMMIT_SHA="$(COMMIT_SHA)" \ - --build-arg BUILD_ID="$(BUILD_ID)" \ - -t $(REGISTRY)/controller-chroot:$(TAG) rootfs -f rootfs/Dockerfile-chroot - .PHONY: clean-image clean-image: ## Removes local image echo "removing old image $(REGISTRY)/controller:$(TAG)" @docker rmi -f $(REGISTRY)/controller:$(TAG) || true - -.PHONY: clean-chroot-image -clean-chroot-image: ## Removes local image - echo "removing old image $(REGISTRY)/controller-chroot:$(TAG)" - @docker rmi -f $(REGISTRY)/controller-chroot:$(TAG) || true - - .PHONY: build build: ## Build ingress controller, debug tool and pre-stop hook. - E2E_IMAGE=golang:$(GO_VERSION)-alpine3.21 USE_SHELL=/bin/sh build/run-in-docker.sh \ - MAC_OS=$(MAC_OS) \ + @build/run-in-docker.sh \ PKG=$(PKG) \ ARCH=$(ARCH) \ COMMIT_SHA=$(COMMIT_SHA) \ REPO_INFO=$(REPO_INFO) \ TAG=$(TAG) \ + GOBUILD_FLAGS=$(GOBUILD_FLAGS) \ build/build.sh +.PHONY: build-plugin +build-plugin: ## Build ingress-nginx krew plugin. + @build/run-in-docker.sh \ + PKG=$(PKG) \ + ARCH=$(ARCH) \ + COMMIT_SHA=$(COMMIT_SHA) \ + REPO_INFO=$(REPO_INFO) \ + TAG=$(TAG) \ + GOBUILD_FLAGS=$(GOBUILD_FLAGS) \ + build/build-plugin.sh .PHONY: clean clean: ## Remove .gocache directory. rm -rf bin/ .gocache/ .cache/ -.PHONY: verify-docs -verify-docs: ## Verify doc generation - hack/verify-annotation-docs.sh - .PHONY: static-check static-check: ## Run verification script for boilerplate, codegen, gofmt, golint, lualint and chart-lint. @build/run-in-docker.sh \ - MAC_OS=$(MAC_OS) \ hack/verify-all.sh -.PHONY: golint-check -golint-check: - @build/run-in-docker.sh \ - MAC_OS=$(MAC_OS) \ - hack/verify-golint.sh - -############################### -# Tests for ingress-nginx -############################### - .PHONY: test test: ## Run go unit tests. @build/run-in-docker.sh \ PKG=$(PKG) \ - MAC_OS=$(MAC_OS) \ ARCH=$(ARCH) \ COMMIT_SHA=$(COMMIT_SHA) \ REPO_INFO=$(REPO_INFO) \ TAG=$(TAG) \ - GOFLAGS="-buildvcs=false" \ - test/test.sh + GOBUILD_FLAGS=$(GOBUILD_FLAGS) \ + build/test.sh .PHONY: lua-test lua-test: ## Run lua unit tests. @build/run-in-docker.sh \ - MAC_OS=$(MAC_OS) \ - test/test-lua.sh + BUSTED_ARGS=$(BUSTED_ARGS) \ + build/test-lua.sh .PHONY: e2e-test e2e-test: ## Run e2e tests (expects access to a working Kubernetes cluster). - @test/e2e/run-e2e-suite.sh - -.PHONY: kind-e2e-test -kind-e2e-test: ## Run e2e tests using kind. - @test/e2e/run-kind-e2e.sh - -.PHONY: kind-e2e-chart-tests -kind-e2e-chart-tests: ## Run helm chart e2e tests - @test/e2e/run-chart-test.sh + @build/run-e2e-suite.sh .PHONY: e2e-test-binary e2e-test-binary: ## Build binary for e2e tests. @build/run-in-docker.sh \ - MAC_OS=$(MAC_OS) \ ginkgo build ./test/e2e .PHONY: print-e2e-suite print-e2e-suite: e2e-test-binary ## Prints information about the suite of e2e tests. @build/run-in-docker.sh \ - MAC_OS=$(MAC_OS) \ hack/print-e2e-suite.sh .PHONY: vet @@ -192,11 +143,10 @@ vet: .PHONY: check_dead_links check_dead_links: ## Check if the documentation contains dead links. - @docker run ${PLATFORM_FLAG} ${PLATFORM} -t \ - -w /tmp \ - -v $$PWD:/tmp dkhamsing/awesome_bot:1.20.0 \ + @docker run -t \ + -v $$PWD:/tmp aledbf/awesome_bot:0.1 \ --allow-dupe \ - --allow-redirect $(shell find $$PWD -mindepth 1 -name vendor -prune -o -name .modcache -prune -o -iname Changelog.md -prune -o -name "*.md" | sed -e "s#$$PWD/##") + --allow-redirect $(shell find $$PWD -mindepth 1 -name "*.md" -printf '%P\n' | grep -v vendor | grep -v Changelog.md) .PHONY: dev-env dev-env: ## Starts a local Kubernetes cluster using kind, building and deploying the ingress controller. @@ -206,28 +156,31 @@ dev-env: ## Starts a local Kubernetes cluster using kind, building and deployin dev-env-stop: ## Deletes local Kubernetes cluster created by kind. @kind delete cluster --name ingress-nginx-dev - - .PHONY: live-docs live-docs: ## Build and launch a local copy of the documentation website in http://localhost:8000 - @docker build ${PLATFORM_FLAG} ${PLATFORM} \ - --no-cache \ - -t ingress-nginx-docs .github/actions/mkdocs - @docker run ${PLATFORM_FLAG} ${PLATFORM} --rm -it \ + @docker build -t ingress-nginx-docs .github/actions/mkdocs + @docker run --rm -it \ -p 8000:8000 \ -v ${PWD}:/docs \ - --entrypoint /bin/bash \ - ingress-nginx-docs \ - -c "pip install -r /docs/docs/requirements.txt && mkdocs serve --dev-addr=0.0.0.0:8000" + --entrypoint mkdocs \ + ingress-nginx-docs serve --dev-addr=0.0.0.0:8000 .PHONY: misspell misspell: ## Check for spelling errors. - @go install github.com/client9/misspell/cmd/misspell@latest + @go get github.com/client9/misspell/cmd/misspell misspell \ -locale US \ -error \ cmd/* internal/* deploy/* docs/* design/* test/* README.md +.PHONY: kind-e2e-test +kind-e2e-test: ## Run e2e tests using kind. + @test/e2e/run.sh + +.PHONY: kind-e2e-chart-tests +kind-e2e-chart-tests: ## Run helm chart e2e tests + @test/e2e/run-chart-test.sh + .PHONY: run-ingress-controller run-ingress-controller: ## Run the ingress controller locally using a kubectl proxy connection. @build/run-ingress-controller.sh @@ -240,43 +193,25 @@ ensure-buildx: show-version: echo -n $(TAG) -PLATFORMS ?= amd64 arm arm64 -BUILDX_PLATFORMS ?= linux/amd64,linux/arm,linux/arm64 +PLATFORMS ?= amd64 arm arm64 s390x + +EMPTY := +SPACE := $(EMPTY) $(EMPTY) +COMMA := , .PHONY: release # Build a multi-arch docker image release: ensure-buildx clean echo "Building binaries..." $(foreach PLATFORM,$(PLATFORMS), echo -n "$(PLATFORM)..."; ARCH=$(PLATFORM) make build;) - echo "Building and pushing ingress-nginx image...$(BUILDX_PLATFORMS)" - - docker buildx build \ + echo "Building and pushing ingress-nginx image..." + @docker buildx build \ --no-cache \ - $(MAC_DOCKER_FLAGS) \ --push \ - --pull \ --progress plain \ - --platform $(BUILDX_PLATFORMS) \ + --platform $(subst $(SPACE),$(COMMA),$(PLATFORMS)) \ --build-arg BASE_IMAGE="$(BASE_IMAGE)" \ --build-arg VERSION="$(TAG)" \ --build-arg COMMIT_SHA="$(COMMIT_SHA)" \ --build-arg BUILD_ID="$(BUILD_ID)" \ -t $(REGISTRY)/controller:$(TAG) rootfs - - docker buildx build \ - --no-cache \ - $(MAC_DOCKER_FLAGS) \ - --push \ - --pull \ - --progress plain \ - --platform $(BUILDX_PLATFORMS) \ - --build-arg BASE_IMAGE="$(BASE_IMAGE)" \ - --build-arg VERSION="$(TAG)" \ - --build-arg COMMIT_SHA="$(COMMIT_SHA)" \ - --build-arg BUILD_ID="$(BUILD_ID)" \ - -t $(REGISTRY)/controller-chroot:$(TAG) rootfs -f rootfs/Dockerfile-chroot - -.PHONY: build-docs -build-docs: - pip install -r docs/requirements.txt - mkdocs build --config-file mkdocs.yml diff --git a/NEW_CONTRIBUTOR.md b/NEW_CONTRIBUTOR.md deleted file mode 100644 index c9668430c..000000000 --- a/NEW_CONTRIBUTOR.md +++ /dev/null @@ -1,852 +0,0 @@ -## New Contributor Tips - -Welcome to the Ingress Nginx new contributor tips. -This guide briefly outlines the necessary knowledge & tools, required to start working on Ingress-NGINX Issues. - -### Prerequisites -- Basic understanding of linux -- Familiarity with the command line on linux -- OSI Model(Links below) - -### Introduction -It all starts with the OSI model... -> The Open Systems Interconnection (OSI) model describes seven layers that computer systems use to communicate over a network. It was the first standard model for network communications, adopted by all major computer and telecommunication companies - -![Describes the 7 Layers of the OSI Model](https://i.imgur.com/qF0KjBq.png) - -#### Reading material for OSI Model -[OSI Model CertificationKits](https://www.certificationkits.com/cisco-certification/cisco-ccna-640-802-exam-certification-guide/cisco-ccna-the-osi-model/) - -### Approaching the problem - - -Not everybody knows everything. But the factors that help are a love/passion for this to begin. But to move forward, it's the approach and not the knowledge that sustains prolonged joy, while working on issues. If the approach is simple and powered by good-wishes-for-community, then info & tools are forthcoming and easy. - -Here we take a bird's eye-view of the hops in the network plumbing, that a packet takes, from source to destination, when we run `curl`, from a laptop to a nginx webserver process, running in a container, inside a pod, inside a Kubernetes cluster, created using `kind` or `minikube` or any other cluster-management tool. - -### [Kind](https://kind.sigs.k8s.io/) cluster example on a Linux Host - -#### TL;DR -The destination of the packet from the curl command, is looked up, in the `routing table`. Based on the route, the packet first travels to the virtual bridge `172.18.0.1` interface, created by docker, when we created the kind cluster on a laptop. Next the packet is forwarded to `172.18.0.2`(See below on how we got this IP address), within the kind cluster. The `kube-proxy` container creates iptables rules that make sure the packet goes to the correct pod ip in this case `10.244.0.5` - -Command: -``` -# docker ps -CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -230e7246a32c kindest/node:v1.24.1 "/usr/local/bin/entr…" 2 weeks ago Up 54 seconds 127.0.0.1:38143->6443/tcp kind-control-plane - -# docker inspect kind-control-plane -f '{{ .NetworkSettings.Networks.kind.IPAddress }}' -172.18.0.2 - -``` - - - -If this part is confusing, you would first need to understand what a [bridge](https://tldp.org/HOWTO/BRIDGE-STP-HOWTO/what-is-a-bridge.html) is and what [docker network](https://docs.docker.com/network/) is. - - - -#### The journey of a curl packet. -Let's begin with creating a [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/) Cluster on your laptop -``` -# kind create cluster -``` -This will create a cluster called `kind`, to view the clusters type -``` -# kind get clusters    -kind -``` -Kind ships with `kubectl`, so we can use that to communicate with our clusters. -``` -# kubectl get no -o wide    -NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME -kind-control-plane Ready control-plane 5d23h v1.24.1 172.18.0.2 Ubuntu 21.10 5.18.12-arch1-1 containerd://1.6.4 -``` -Kind creates a cluster using docker container as nodes, it does this using [containerd](https://containerd.io/) within the docker container. -The concept of Docker in Docker is very important here. - -To start with simply create a nginx deployment using `kubectl`. -``` -# kubectl create deployment nginx --image nginx:alpine --port=80 -deployment.apps/nginx created -``` -Then we expose this as a NodePort Service. -``` -# kubectl expose deployment/nginx --type=NodePort -service/nginx-new exposed -``` -Command: Now we can see that the service has been exposed. -``` -# kubectl get svc -o wide -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR -nginx NodePort 10.96.176.241 80:32329/TCP 4d8h app=nginx -``` -Output Relevance: From the above output, we can see that our nginx pod is being exposed as the `NodePort` service type, and now we can curl the Node IP `172.18.0.2` with the exposed port `32329` - -Command: The pod has an IP as shown below -``` -# kubectl get po -o wide   -NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES -nginx-6c8b449b8f-pdvdk 1/1 Running 1 (32h ago) 4d8h 10.244.0.5 kind-control-plane -``` - -Command: We can use `curl` on the laptop to view the nginx container that is running on port `32329`. - -``` -# curl 172.18.0.2:32329 - - - - -Welcome to nginx! - - - -

Welcome to nginx!

-

If you see this page, the nginx web server is successfully installed and -working. Further configuration is required.

- -

For online documentation and support please refer to -nginx.org.
-Commercial support is available at -nginx.com.

- -

Thank you for using nginx.

- - -``` -Now, we can check the ip interfaces as well subnets for our system is connected to: - -``` -$ ifconfig -ethbr0: flags=4163 mtu 1500 - inet 192.168.31.9 netmask 255.255.255.0 broadcast 192.168.31.255 - inet6 fe80::7530:9ae5:3e8d:e45a prefixlen 64 scopeid 0x20 - ether 2e:90:b3:e8:52:5b txqueuelen 1000 (Ethernet) - RX packets 31220566 bytes 44930589084 (41.8 GiB) - RX errors 0 dropped 0 overruns 0 frame 0 - TX packets 18104006 bytes 1757183680 (1.6 GiB) - TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 - -br-2fffe5cd5d9e: flags=4163 mtu 1500 - inet 172.18.0.1 netmask 255.255.0.0 broadcast 172.18.255.255 - inet6 fc00:f853:ccd:e793::1 prefixlen 64 scopeid 0x0 - inet6 fe80::42:12ff:fed3:8fb0 prefixlen 64 scopeid 0x20 - inet6 fe80::1 prefixlen 64 scopeid 0x20 - ether 02:42:12:d3:8f:b0 txqueuelen 0 (Ethernet) - RX packets 3547 bytes 414792 (405.0 KiB) - RX errors 0 dropped 0 overruns 0 frame 0 - TX packets 6267 bytes 8189931 (7.8 MiB) - TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 -docker0: flags=4099 mtu 1500 - inet 172.17.0.1 netmask 255.255.0.0 broadcast 172.17.255.255 - inet6 fe80::42:a2ff:fe09:5edb prefixlen 64 scopeid 0x20 - ether 02:42:a2:09:5e:db txqueuelen 0 (Ethernet) - RX packets 14 bytes 2143 (2.0 KiB) - RX errors 0 dropped 0 overruns 0 frame 0 - TX packets 40 bytes 6406 (6.2 KiB) - TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 -``` -From the above output we can see that, there are two bridges connected to our systems network interface,one is the docker default bridge`docker0` and the other created by kind -`br-2fffe5cd5d9e`. - -Since kind creates nodes as containers, this is easily accessible via `docker ps`. -``` -$ docker ps - -CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -230e7246a32c kindest/node:v1.24.1 "/usr/local/bin/entr…" 6 days ago Up 33 hours 127.0.0.1:38143->6443/tcp kind-control-plane -``` -If we do a docker `exec` we can enter the container, we can also see the network interfaces within the container. -``` -# docker exec -it 230e7246a32c bash - -# root@kind-control-plane:/# ip a -1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 - link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 - inet 127.0.0.1/8 scope host lo - valid_lft forever preferred_lft forever - inet6 ::1/128 scope host - valid_lft forever preferred_lft forever -2: vethdb0d1da1@if2: mtu 1500 qdisc noqueue state UP group default - link/ether a2:a1:ce:08:d2:39 brd ff:ff:ff:ff:ff:ff link-netns cni-ddc25710-030a-cc05-c600-5a183fae01f7 - inet 10.244.0.1/32 scope global vethdb0d1da1 - valid_lft forever preferred_lft forever -3: veth4d76603f@if2: mtu 1500 qdisc noqueue state UP group default - link/ether 9a:9b:6b:3e:d1:53 brd ff:ff:ff:ff:ff:ff link-netns cni-f2270000-8fc8-6f89-e56b-4759ae10a084 - inet 10.244.0.1/32 scope global veth4d76603f - valid_lft forever preferred_lft forever -4: vethcc2586d6@if2: mtu 1500 qdisc noqueue state UP group default - link/ether 52:f9:20:63:62:a2 brd ff:ff:ff:ff:ff:ff link-netns cni-97e337cd-1322-c1fa-7523-789af94f397f - inet 10.244.0.1/32 scope global vethcc2586d6 - valid_lft forever preferred_lft forever -5: veth783189a9@if2: mtu 1500 qdisc noqueue state UP group default - link/ether ba:e1:55:1f:6f:12 brd ff:ff:ff:ff:ff:ff link-netns cni-90849001-668a-03d2-7d9e-192de79ccc59 - inet 10.244.0.1/32 scope global veth783189a9 - valid_lft forever preferred_lft forever -6: veth79c98c12@if2: mtu 1500 qdisc noqueue state UP group default - link/ether 22:05:55:c7:86:e9 brd ff:ff:ff:ff:ff:ff link-netns cni-734dfac9-9f70-ab33-265b-21569d90312a - inet 10.244.0.1/32 scope global veth79c98c12 - valid_lft forever preferred_lft forever -7: veth5b221c83@if2: mtu 1500 qdisc noqueue state UP group default - link/ether 92:3f:04:54:72:5a brd ff:ff:ff:ff:ff:ff link-netns cni-d8f6666b-1cfb-ef08-4bf8-237a7fc32da2 - inet 10.244.0.1/32 scope global veth5b221c83 - valid_lft forever preferred_lft forever -8: vethad630fb8@if2: mtu 1500 qdisc noqueue state UP group default - link/ether 32:78:ec:f6:01:ea brd ff:ff:ff:ff:ff:ff link-netns cni-6cb3c179-cb17-3b81-2051-27231c44a3c4 - inet 10.244.0.1/32 scope global vethad630fb8 - valid_lft forever preferred_lft forever -9: veth573a629b@if2: mtu 1500 qdisc noqueue state UP group default - link/ether e2:57:f8:c9:bc:94 brd ff:ff:ff:ff:ff:ff link-netns cni-d2dbb903-8310-57b4-7ba4-9f353dbc79dc - inet 10.244.0.1/32 scope global veth573a629b - valid_lft forever preferred_lft forever -10: eth0@if11: mtu 1500 qdisc noqueue state UP group default - link/ether 02:42:ac:12:00:02 brd ff:ff:ff:ff:ff:ff link-netnsid 0 - inet 172.18.0.2/16 brd 172.18.255.255 scope global eth0 - valid_lft forever preferred_lft forever - inet6 fc00:f853:ccd:e793::2/64 scope global nodad - valid_lft forever preferred_lft forever - inet6 fe80::42:acff:fe12:2/64 scope link - valid_lft forever preferred_lft forever -11: vethd7368e27@if2: mtu 1500 qdisc noqueue state UP group default - link/ether 8a:74:ec:f6:d6:c9 brd ff:ff:ff:ff:ff:ff link-netns cni-7c7eb9cd-bbb1-65b0-0480-b8f1265f2f36 - inet 10.244.0.1/32 scope global vethd7368e27 - valid_lft forever preferred_lft forever -12: veth7cadbf2b@if2: mtu 1500 qdisc noqueue state UP group default - link/ether 12:48:10:b7:b8:f5 brd ff:ff:ff:ff:ff:ff link-netns cni-b39e37b5-1bc8-626a-a553-a0be2f94a117 - inet 10.244.0.1/32 scope global veth7cadbf2b - valid_lft forever preferred_lft forever - -``` -When we run `curl 172.18.0.2:32329` on the laptop it first needs to figure out where `172.18.0.2`, to do this it refers to the host routing table. -``` -sudo netstat -rn    main  -Kernel IP routing table -Destination Gateway Genmask Flags MSS Window irtt Iface -0.0.0.0 192.168.31.1 0.0.0.0 UG 0 0 0 ethbr0 -172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0 -172.18.0.0 0.0.0.0 255.255.0.0 U 0 0 0 br-2fffe5cd5d9e -172.19.0.0 0.0.0.0 255.255.0.0 U 0 0 0 br-be5b544733a3 -192.168.31.0 0.0.0.0 255.255.255.0 U 0 0 0 ethbr0 -192.168.31.0 0.0.0.0 255.255.255.0 U 0 0 0 ethbr0 -192.168.39.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr2 -192.168.122.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr0 -``` -Output Relevance: From the above output, you can see that the `iface`(Interface) for `172.18.0.0` is `br-2fffe5cd5d9e`, which means traffic that needs to go to `172.18.0.0` will go through `br-2fffe5cd5d9e` which is created by docker for the kind container (this is the node in case of kind cluster). - -Now we need to understand how the packet travels from the container interface to the pod with IP `10.244.0.5`. The component that handles this is called kube-proxy - -So what exactly is [kube-proxy](https://kubernetes.io/docs/concepts/overview/components/#kube-proxy): -> Kube-Proxy is a network proxy that runs on each node in your cluster, implementing part of the Kubernetes Service concept. -kube-proxy maintains network rules on nodes. These network rules allow network communication to your Pods from network sessions inside or outside of your cluster - -So, as we can see that kube proxy handles the network rules required to aid the communication to the pods, we will look at the [iptables](https://linux.die.net/man/8/iptables) -> `iptables` is a command line interface used to set up and maintain tables for the Netfilter firewall for IPv4, included in the Linux kernel. The firewall matches packets with rules defined in these tables and then takes the specified action on a possible match. Tables is the name for a set of chains - -Command: -``` -# iptables -t nat -L PREROUTING -n -Chain PREROUTING (policy ACCEPT) -target prot opt source destination -KUBE-SERVICES all -- 0.0.0.0/0 0.0.0.0/0 /* kubernetes service portals */ -DOCKER_OUTPUT all -- 0.0.0.0/0 172.18.0.1 -CNI-HOSTPORT-DNAT all -- 0.0.0.0/0 0.0.0.0/0 ADDRTYPE match dst-type LOCAL -``` - -``` -# iptables-save | grep PREROUTING --A PREROUTING -m comment --comment "kubernetes service portals" -j KUBE-SERVICES -``` -Output Relevance: -> -A: append new iptable rule -> -j: jump to the target -> KUBE-SERVICES: target - -> The above output appends a new rule for PREROUTING which every network packet will go through first as they try to access any kubernetes service - - -What is `PREROUTING` in iptables? ->PREROUTING: This chain is used to make any routing related decisions before (PRE) sending any packets - -To dig in further we need to go to the target, `KUBE-SERVICES` for our nginx service. -``` -# iptables -t nat -L KUBE-SERVICES -n| grep nginx -KUBE-SVC-2CMXP7HKUVJN7L6M tcp -- 0.0.0.0/0 10.96.176.241 /* default/nginx cluster IP */ tcp dpt:80 -``` -Command: -``` -# iptables -t nat -L KUBE-SVC-2CMXP7HKUVJN7L6M -n -Chain KUBE-SVC-2CMXP7HKUVJN7L6M (2 references) -target prot opt source destination -KUBE-MARK-MASQ tcp -- !10.244.0.0/16 10.96.176.241 /* default/nginx cluster IP */ tcp dpt:80 -KUBE-SEP-4IEO3WJHPKXV3AOH all -- 0.0.0.0/0 0.0.0.0/0 /* default/nginx -> 10.244.0.5:80 */ - -# iptables -t nat -L KUBE-MARK-MASQ -n -Chain KUBE-MARK-MASQ (31 references) -target prot opt source destination -MARK all -- 0.0.0.0/0 0.0.0.0/0 MARK or 0x4000 - -# iptables -t nat -L KUBE-SEP-4IEO3WJHPKXV3AOH -n -Chain KUBE-SEP-4IEO3WJHPKXV3AOH (1 references) -target prot opt source destination -KUBE-MARK-MASQ all -- 10.244.0.5 0.0.0.0/0 /* default/nginx */ -DNAT tcp -- 0.0.0.0/0 0.0.0.0/0 /* default/nginx */ tcp to:10.244.0.5:80 -``` - - -``` -iptables-save | grep 10.96.176.241 - --A KUBE-SERVICES -d 10.96.176.241/32 -p tcp -m comment --comment "default/nginx cluster IP" -m tcp --dport 80 -j KUBE-SVC-2CMXP7HKUVJN7L6M --A KUBE-SVC-2CMXP7HKUVJN7L6M ! -s 10.244.0.0/16 -d 10.96.176.241/32 -p tcp -m comment --comment "default/nginx cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ -``` - -As you can see the rules added by `kube-proxy` helps the packet reach to the destination service. - -### Minikube KVM VM Example on Linux - -#### TL;DR -Now we look at the curl packet journey on minikube. The `routing table` is looked up to know the destination of the curl packet. The packet then first travels to the virtual bridge `192.168.39.1`, created by minikube kvm2 driver, when we created the minikube cluster, on a linux laptop. Then this packet is forwarded to `192.168.39.57`, within the minikube VM. We have docker containers running in the VM. Among them, the `kube-proxy` container creates iptables rules that make sure the packet goes to the correct pod ip, in this case `172.17.0.4`. - - -To begin with the minikube example, we first need to create a minikube cluster on a linux laptop. In this example I'll be using the `kvm2` driver option for `minikube start` command, as default. - -``` -minikube start -😄 minikube v1.26.0 on Arch "rolling" -🆕 Kubernetes 1.24.2 is now available. If you would like to upgrade, specify: --kubernetes-version=v1.24.2 -✨ Using the kvm2 driver based on existing profile -👍 Starting control plane node minikube in cluster minikube -🏃 Updating the running kvm2 "minikube" VM ... -🐳 Preparing Kubernetes v1.23.3 on Docker 20.10.12 ... - ▪ kubelet.housekeeping-interval=5m -🔎 Verifying Kubernetes components... - ▪ Using image registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.1.1 - ▪ Using image registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.1.1 - ▪ Using image registry.k8s.io/ingress-nginx/controller:v1.2.1 - ▪ Using image gcr.io/k8s-minikube/storage-provisioner:v5 -🔎 Verifying ingress addon... -🌟 Enabled addons: ingress, storage-provisioner, default-storageclass -🏄 Done! kubectl is now configured to use "minikube" cluster and "default" namespace by default -``` -**Note**: The KVM driver provides a lot of options on customizing the cluster, however that is currently beyond the scope of this guide. - -Next we will get the Node IP. -``` -$ kubectl get no -o wide   -NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME -minikube Ready control-plane,master 25d v1.23.3 192.168.39.57 Buildroot 2021.02.4 4.19.202 docker://20.10.12 -``` -Minikube creates a Virtual Machine using the KVM2 driver(Other drivers such as Virtualbox do exist see `minikube start --help` for more information ), you should be able to see this with the following output(You may have to use sudo to get this output) - -``` -$ virsh --connect qemu:///system list - Id Name State --------------------------- - 1 minikube running - - or - - $ sudo virsh list - Id Name State --------------------------- - 1 minikube running - -``` - -Moving on, simply create a nginx deployment using `kubectl`. -``` -# kubectl create deployment nginx --image nginx:alpine --port=80 -deployment.apps/nginx created -``` -Then we expose this as a NodePort Service. -``` -# kubectl expose deployment/nginx --type=NodePort -service/nginx-new exposed -``` -Command: Now we can see that the service has been exposed. -``` -# kubectl get svc -o wide    main  -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR -kubernetes ClusterIP 10.96.0.1 443/TCP 25d -nginx-minikube NodePort 10.97.44.4 80:32007/TCP 45h app=nginx-minikube -``` -Output Relevance: From the above output, we can see that our nginx pod is being exposed as the `NodePort` service type, and now we can curl the Node IP `192.168.39.57` with the exposed port `32007` - -Command: The pod has an IP as shown below -``` -# kubectl get po -o wide -NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES -nginx-minikube-7546f79bd8-x88bt 1/1 Running 3 (43m ago) 45h 172.17.0.4 minikube - -``` - -Command: We can use `curl` on the laptop to view the nginx container that is running on port `32007`. -``` -curl 192.168.39.57:32007 - - - -Welcome to nginx! - - - -

Welcome to nginx!

-

If you see this page, the nginx web server is successfully installed and -working. Further configuration is required.

- -

For online documentation and support please refer to -nginx.org.
-Commercial support is available at -nginx.com.

- -

Thank you for using nginx.

- - -``` - -So, how does this packet travel, lets dive in. -We can check the ip interfaces as well subnets for our system is connected to: -``` -$ ifconfig -virbr2: flags=4163 mtu 1500 - inet 192.168.39.1 netmask 255.255.255.0 broadcast 192.168.39.255 - ether 52:54:00:19:29:93 txqueuelen 1000 (Ethernet) - RX packets 5132 bytes 1777099 (1.6 MiB) - RX errors 0 dropped 0 overruns 0 frame 0 - TX packets 6113 bytes 998530 (975.1 KiB) - TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 - -virbr0: flags=4163 mtu 1500 - inet 192.168.122.1 netmask 255.255.255.0 broadcast 192.168.122.255 - ether 52:54:00:48:ee:35 txqueuelen 1000 (Ethernet) - RX packets 23648 bytes 1265196 (1.2 MiB) - RX errors 0 dropped 0 overruns 0 frame 0 - TX packets 40751 bytes 60265308 (57.4 MiB) - TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 -``` -Output Relevance: From the above output you can see there are two Virtual Bridges created by minikube when we created the cluster on the network. Here, `virbr0` is the default NAT network bridge while `virbr2` is a isolated network bridge on which the pods run. - -Minikube creates a Virtual Machine, to enter the virtual machine we can simply do: -``` -# minikube ssh -``` - -The interfaces within the Virtual Machine are as follows. -``` -docker0 Link encap:Ethernet HWaddr 02:42:03:24:26:78 - inet addr:172.17.0.1 Bcast:172.17.255.255 Mask:255.255.0.0 - UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 - RX packets:31478 errors:0 dropped:0 overruns:0 frame:0 - TX packets:36704 errors:0 dropped:0 overruns:0 carrier:0 - collisions:0 txqueuelen:0 - RX bytes:3264056 (3.1 MiB) TX bytes:14061883 (13.4 MiB) - -eth0 Link encap:Ethernet HWaddr 52:54:00:C9:3A:73 - inet addr:192.168.39.57 Bcast:192.168.39.255 Mask:255.255.255.0 - UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 - RX packets:8245 errors:0 dropped:9 overruns:0 frame:0 - TX packets:3876 errors:0 dropped:0 overruns:0 carrier:0 - collisions:0 txqueuelen:1000 - RX bytes:812006 (792.9 KiB) TX bytes:1044724 (1020.2 KiB) - -eth1 Link encap:Ethernet HWaddr 52:54:00:7B:37:79 - inet addr:192.168.122.35 Bcast:192.168.122.255 Mask:255.255.255.0 - UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 - RX packets:4459 errors:0 dropped:9 overruns:0 frame:0 - TX packets:201 errors:0 dropped:0 overruns:0 carrier:0 - collisions:0 txqueuelen:1000 - RX bytes:298528 (291.5 KiB) TX bytes:25813 (25.2 KiB) - -lo Link encap:Local Loopback - inet addr:127.0.0.1 Mask:255.0.0.0 - UP LOOPBACK RUNNING MTU:65536 Metric:1 - RX packets:946772 errors:0 dropped:0 overruns:0 frame:0 - TX packets:946772 errors:0 dropped:0 overruns:0 carrier:0 - collisions:0 txqueuelen:1000 - RX bytes:213465460 (203.5 MiB) TX bytes:213465460 (203.5 MiB) - -vetha4f1dc5 Link encap:Ethernet HWaddr 3E:1C:FE:C9:75:86 - UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 - RX packets:10 errors:0 dropped:0 overruns:0 frame:0 - TX packets:16 errors:0 dropped:0 overruns:0 carrier:0 - collisions:0 txqueuelen:0 - RX bytes:1413 (1.3 KiB) TX bytes:955 (955.0 B) - -vethbf35613 Link encap:Ethernet HWaddr BA:31:7D:AE:2A:BF - UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 - RX packets:3526 errors:0 dropped:0 overruns:0 frame:0 - TX packets:3934 errors:0 dropped:0 overruns:0 carrier:0 - collisions:0 txqueuelen:0 - RX bytes:342408 (334.3 KiB) TX bytes:380193 (371.2 KiB) - -vethe092a51 Link encap:Ethernet HWaddr 8A:37:D3:D9:D9:0E - UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 - RX packets:9603 errors:0 dropped:0 overruns:0 frame:0 - TX packets:11151 errors:0 dropped:0 overruns:0 carrier:0 - collisions:0 txqueuelen:0 - RX bytes:1199235 (1.1 MiB) TX bytes:5449408 (5.1 MiB) -``` -Output Relevance: Here we have the Virtual Ethernet and we have docker bridges too since docker runs within the Virtual Machine. - -When we do a `curl` to `192.168.39.57:32007` on the laptop the packet first goes to the route table -``` -Destination Gateway Genmask Flags MSS Window irtt Iface -0.0.0.0 192.168.31.1 0.0.0.0 UG 0 0 0 ethbr0 -172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0 -172.18.0.0 0.0.0.0 255.255.0.0 U 0 0 0 br-2fffe5cd5d9e -172.19.0.0 0.0.0.0 255.255.0.0 U 0 0 0 br-be5b544733a3 -192.168.31.0 0.0.0.0 255.255.255.0 U 0 0 0 ethbr0 -192.168.31.0 0.0.0.0 255.255.255.0 U 0 0 0 ethbr0 -192.168.39.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr2 -192.168.122.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr0 -``` -Output Relevance: As you can see multiple routes are defined here, of which our Virtual Machine Node IP(192.168.39.57) is also shown in the table, so the packet now knows where it has to go. - -With that clear we now know how the packet goes from the laptop to the virtual bridge and then enters the Virtual Machine. - -Inside the virtual machine, [kube-proxy](https://kubernetes.io/docs/concepts/overview/components/#kube-proxy) handles the routing using iptables. - -So what exactly is [kube-proxy](https://kubernetes.io/docs/concepts/overview/components/#kube-proxy)(For those who skipped the kind example): -> Kube-Proxy is a network proxy that runs on each node in your cluster, implementing part of the Kubernetes Service concept. -kube-proxy maintains network rules on nodes. These network rules allow network communication to your Pods from network sessions inside or outside of your cluster - -So, as we can see that kube proxy handles the network rules required to aid the communication to the pods, we will look at the [iptables](https://linux.die.net/man/8/iptables) -> `iptables` is a command line interface used to set up and maintain tables for the Netfilter firewall for IPv4, included in the Linux kernel. The firewall matches packets with rules defined in these tables and then takes the specified action on a possible match. Tables is the name for a set of chains - -Command: - -``` -# minikube ssh   - _ _ - _ _ ( ) ( ) - ___ ___ (_) ___ (_)| |/') _ _ | |_ __ -/' _ ` _ `\| |/' _ `\| || , < ( ) ( )| '_`\ /'__`\ -| ( ) ( ) || || ( ) || || |\`\ | (_) || |_) )( ___/ -(_) (_) (_)(_)(_) (_)(_)(_) (_)`\___/'(_,__/'`\____) - -$ sudo iptables -t nat -L PREROUTING -n -Chain PREROUTING (policy ACCEPT) -target prot opt source destination -KUBE-SERVICES all -- 0.0.0.0/0 0.0.0.0/0 /* kubernetes service portals */ -DOCKER all -- 0.0.0.0/0 0.0.0.0/0 ADDRTYPE match dst-type LOCAL - -$ iptables-save | grep PREROUTING --A PREROUTING -m comment --comment "kubernetes service portals" -j KUBE-SERVICES - -``` - -Output Relevance: -> -A: append new iptable rule -> -j: jump to the target -> KUBE-SERVICES: target - -> The above output appends a new rule for PREROUTING which every network packet will go through first as they try to access any kubernetes service - - -What is `PREROUTING` in iptables? ->PREROUTING: This chain is used to make any routing related decisions before (PRE) sending any packets - -To dig in further we need to go to the target, `KUBE-SERVICES` for our nginx service. -``` -# iptables -t nat -L KUBE-SERVICES -n| grep nginx -KUBE-SVC-NRDCJV6H42SDXARP tcp -- 0.0.0.0/0 10.97.44.4 /* default/nginx-minikube cluster IP */ tcp dpt:80 -``` -Command: -``` -$ sudo iptables -t nat -L| grep KUBE-SVC-NRDCJV6H42SDXARP -KUBE-SVC-NRDCJV6H42SDXARP tcp -- 0.0.0.0/0 0.0.0.0/0 /* default/nginx-minikube */ tcp dpt:32007 -KUBE-SVC-NRDCJV6H42SDXARP tcp -- 0.0.0.0/0 10.97.44.4 /* default/nginx-minikube cluster IP */ tcp dpt:80 - -$ sudo iptables -t nat -L KUBE-MARK-MASQ -n -Chain KUBE-MARK-MASQ (19 references) -target prot opt source destination -MARK all -- 0.0.0.0/0 0.0.0.0/0 MARK or 0x4000 - -sudo iptables-save | grep 172.17.0.4 --A KUBE-SEP-AHQQ7ZFXMEBNX76B -s 172.17.0.4/32 -m comment --comment "default/nginx-minikube" -j KUBE-MARK-MASQ --A KUBE-SEP-AHQQ7ZFXMEBNX76B -p tcp -m comment --comment "default/nginx-minikube" -m tcp -j DNAT --to-destination 172.17.0.4:80 -``` -As you can see the rules added by kube-proxy helps the packet reach to the destination service. - - -### Connection termination -Connection termination is a type of event that occurs when there are load balancers present, the information for this is quite scarce, however I've found the following article, [IBM - Network Termination](https://www.ibm.com/docs/en/sva/9.0.4?topic=balancer-network-termination) that describes what it means by connection termination between clients(laptop) and server(load balancer) and the various other services. - -### Different types of connection errors. -The following article on [TCP/IP errors](https://www.ibm.com/docs/en/db2/11.1?topic=message-tcpip-errors) has a list of the important tcp timeout errors that we need to know. - - -| Common TCP/IP errors | Meaning | -| -------- | -------- | -| Resource temporarily unavailable.| Self-explanatory. | -| No space is left on a device or system table.|The disk partition is full| -|No route to the host is available.|The routing table doesn't know where to route the packet.| -|Connection was reset by the partner.|This usually means the packet was dropped as soon as it reached the server can be due to a firewall.| -|The connection was timed out.|This indicates the firewall blocking your connection or the connection took too long.| - -## OSI Model Layer 7 (Application Layer) - -[What is layer 7?](https://www.cloudflare.com/learning/ddos/what-is-layer-7/) -#### Summary -Layer 7 refers to the seventh and topmost layer of the Open Systems Interconnect (OSI) Model known as the application layer. This is the highest layer which supports end-user processes and applications. Layer 7 identifies the communicating parties and the quality of service between them, considers privacy and user authentication, as well as identifies any constraints on the data syntax. This layer is wholly application-specific. - - -## Setting up Ingress-Nginx Controller - -Since we are doing this on our local laptop, we are going to use the following tools: -- [Minikube using KVM driver](https://minikube.sigs.k8s.io/docs/start/) - The host is linux-based in our example -- [Metallb](https://metallb.universe.tf/) - Baremetal load-balancer. -- [KVM](https://www.linux-kvm.org/page/Main_Page) / [Oracle VirtualBox](https://www.virtualbox.org/wiki/Downloads) / [VMWare](https://www.vmware.com/in/products/workstation-pro.html) - - -### So let's begin with Metallb and Ingress-Nginx setup. - -For setting up metallb, we are going to follow the below steps: - - - To begin the installation, we will execute: -``` -minikube start -``` -- To install Metallb, one can install it using the [manifest](https://metallb.universe.tf/installation/#installation-by-manifest) or by using [helm](https://metallb.universe.tf/installation/#installation-with-helm), for now we will use the Manifest method: -``` -kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.4/config/manifests/metallb-native.yaml -``` - -- We need to now configure Metallb, we are using [Layer 2 configuration](https://metallb.universe.tf/configuration/#announce-the-service-ips), let's head over to the [Metallb Configuration](https://metallb.universe.tf/configuration/) website, here you will see how to setup metallb. ->Layer 2 mode does not require the IPs to be bound to the network interfaces of your worker nodes. It works by responding to ARP requests on your local network directly, to give the machine’s MAC address to clients. -In order to advertise the IP coming from an IPAddressPool, an L2Advertisement instance must be associated to the IPAddressPool. -- We have modified the IP address pool so that our loadbalancer knows which subnet to choose an IP from.Since we have only one minikube IP we need to modify the code given in the documentation. -Save this as `metallb-config.yaml`: -``` -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: first-pool - namespace: metallb-system -spec: - addresses: - # The configuration website show's you this - - #- 192.168.10.0/24 - #- 192.168.9.1-192.168.9.5 - #- fc00:f853:0ccd:e799::/124 - - # We are going to change this to `minikube ip` as such - - 192.168.39.57/32 -``` -Now deploy it using `kubectl` -``` -kubectl apply -f metallb-config.yaml -``` -- Now that metallb is setup, let's install [ingress-nginx](https://kubernetes.github.io/ingress-nginx/deploy/#quick-start) on the laptop. -Note: We are using the install by manifest option from the Installation manual -``` -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.3.0/deploy/static/provider/cloud/deploy.yaml -``` -or one can also install it using the minikube addons: -``` -minikube addons enable ingress -``` - - Once your Ingress-Nginx controller is created you can run the following commands to see the output of the setup done. -``` -kubectl get pods -n ingress-nginx -NAME READY STATUS RESTARTS AGE -ingress-nginx-admission-create-65bld 0/1 Completed 0 14m -ingress-nginx-admission-patch-rwq4x 0/1 Completed 0 14m -ingress-nginx-controller-6dc865cd86-7c5zd 1/1 Running 0 14m -``` -The Ingress controller creates a Service with the type LoadBalancer and metallb provides the IP address. - -``` -kubectl -n ingress-nginx get svc - -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -ingress-nginx-controller LoadBalancer 10.108.154.53 192.168.39.223 80:30367/TCP,443:31491/TCP 4d15h -ingress-nginx-controller-admission ClusterIP 10.98.54.3 443/TCP 4d15h -``` - -#### Creating an Ingress - -We will deploy a `httpd` service in a `httpd` namespace and create a ingress for it. - -First, let's create a namespace. -``` -kubectl create namespace httpd -``` - -Next we will create a deployment -``` -kubectl create deployment httpd -n httpd --image=httpd:alpine -``` - -Now, In order to create a service, let's expose this deployment -``` -kubectl expose deployment -n httpd httpd --port 80 -``` -Let's check the `pod` that is created - -``` -kubectl get po -n httpd -NAME READY STATUS RESTARTS AGE -httpd-fb7fcdc77-w287c 1/1 Running 0 64s -``` - -Let's list the services in the `httpd` namespace -``` -kubectl get svc -n httpd -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -httpd ClusterIP 10.104.111.0 80/TCP 13s -``` - -Once we have this we can now create an ingress using the following -``` -kubectl -n httpd create ingress httpd --class nginx --rule httpd.dev.leonnunes.com/"*"=httpd:80 -``` -The above output, creates an ingress, for us with the rule to match the service if the host is `httpd.dev.leonnunes.com`. The class here is retrieved from the below command. - -To list the `ingressclasses` use -``` -kubectl get ingressclasses -NAME CONTROLLER PARAMETERS AGE -nginx k8s.io/ingress-nginx 6h49m -``` - -The following command shows the ingress created -``` -$ kubectl get ingress -A -o wide - -NAMESPACE NAME CLASS HOSTS ADDRESS PORTS AGE -httpd httpd nginx httpd.dev.leonnunes.com 192.168.39.223 80 11d -``` - -To test if the rule works we can now do -``` -$ minikube ip -192.168.39.223 - -$ curl --resolve httpd.dev.leonnunes.com:80:192.168.39.223 httpd.dev.leonnunes.com -

It works!

- -or - -curl -H "Host: httpd.dev.leonnunes.com" 192.168.39.223 -``` - -#### Example of Information found on layer 7 -We have setup `Ingress-Nginx`, using `nginx` as a class and `httpd` for this example. - -In order to display the info on Layer - 7, we have extracted the Layer 7 information from a simple `curl` request, and then using `tcpdump` command within the `httpd` pod we extracted the network packets and opened it using the `Wireshark` utility. - -Below given is the output that is important: -```bash -Frame 4: 391 bytes on wire (3128 bits), 391 bytes captured (3128 bits) -Linux cooked capture v2 -Internet Protocol Version 4, Src: 172.17.0.4, Dst: 172.17.0.3 -Transmission Control Protocol, Src Port: 49074, Dst Port: 80, Seq: 1, Ack: 1, Len: 319 -Hypertext Transfer Protocol - GET / HTTP/1.1\r\n - Host: httpd.dev.leonnunes.com\r\n - X-Request-ID: 6e1a790412a0d1615dc0231358dc9c8b\r\n - X-Real-IP: 172.17.0.1\r\n - X-Forwarded-For: 172.17.0.1\r\n - X-Forwarded-Host: httpd.dev.leonnunes.com\r\n - X-Forwarded-Port: 80\r\n - X-Forwarded-Proto: http\r\n - X-Forwarded-Scheme: http\r\n - X-Scheme: http\r\n - User-Agent: curl/7.84.0\r\n - Accept: */*\r\n - \r\n - [Full request URI: http://httpd.dev.leonnunes.com/] - [HTTP request 1/1] - [Response in frame: 6] - -``` -The above output shows the information that the `httpd` pod receives. The `curl` command sends the host header, `Host: httpd.dev.leonnunes.com`, to the nginx controller, that then matches the rule and sends the information to the right controller - -The following output shows what is sent via the laptop. -``` -curl --resolve httpd.dev.leonnunes.com:80:192.168.39.57 -H "Host: httpd.dev.leonnunes.com" 192.168.39.57 -vL -* Added httpd.dev.leonnunes.com:80:192.168.39.57 to DNS cache -* Trying 192.168.39.57:80... -* Connected to 192.168.39.57 (192.168.39.57) port 80 (#0) -> GET / HTTP/1.1 -> Host: httpd.dev.leonnunes.com -> User-Agent: curl/7.84.0 -> Accept: */* -> -* Mark bundle as not supporting multiuse -< HTTP/1.1 200 OK -< Date: Mon, 22 Aug 2022 16:05:27 GMT -< Content-Type: text/html -< Content-Length: 45 -< Connection: keep-alive -< Last-Modified: Mon, 11 Jun 2007 18:53:14 GMT -< ETag: "2d-432a5e4a73a80" -< Accept-Ranges: bytes -< -

It works!

-* Connection #0 to host 192.168.39.57 left intact -``` -As you can see from the above output there are several headers added to the curl output after it reaches the `httpd` pod, these headers are added by the Ingress Nginx Controller. - - -### References -#### Basics of Networking - - https://www.cisco.com/en/US/docs/security/vpn5000/manager/reference/guide/appA.html - - http://web.stanford.edu/class/cs101/ - - https://www.geeksforgeeks.org/basics-computer-networking/ - - Subnetting - - https://www.computernetworkingnotes.com/ccna-study-guide/subnetting-tutorial-subnetting-explained-with-examples.html - -#### Video Links - - https://www.youtube.com/playlist?list=PLhfrWIlLOoKPc2RecyiM_A9nf3fUU3e6g - - https://www.youtube.com/watch?v=S7MNX_UD7vY&list=PLIhvC56v63IJVXv0GJcl9vO5Z6znCVb1P - -### Topics to read about - - Docker in Docker - - [Docker/Containers](https://www.oreilly.com/library/view/docker-deep-dive/9781800565135/) - - Containers - -### Basics of Kubernetes -#### Reading Material -- https://nubenetes.com/kubernetes-tutorials/ -- https://kubernetes.io/docs/concepts/ -#### Video Material -- [Techworld with Nana 101](https://www.youtube.com/playlist?list=PLy7NrYWoggjziYQIDorlXjTvvwweTYoNC) -- [Jeff Geerling Kubernetes 101](https://www.youtube.com/watch?v=IcslsH7OoYo&list=PL2_OBreMn7FoYmfx27iSwocotjiikS5BD) - -#### Hands-On Kubernetes -- https://kube.academy/ -- https://www.civo.com/academy - -### Networking in Kubernetes -- [Kubernetes Networking 101](https://youtu.be/CYnwBIpvSlM?t=284) -- [CNCF Kubernetes 101](https://www.youtube.com/watch?v=cUGXu2tiZMc) - -### Tools/Commands to help with troubleshooting. -- [mtr](https://www.redhat.com/sysadmin/linux-mtr-command) - Tracing the packet from the source to destination -- [tcpdump](https://linuxconfig.org/how-to-use-tcpdump-command-on-linux) - Monitor packets -- [wireshark](https://www.lifewire.com/wireshark-tutorial-4143298) - Read/Sniff packets -- [nslookup](https://phoenixnap.com/kb/nslookup-command) - Lookup Nameservers -- [netstat](https://www.lifewire.com/netstat-command-2618098) - List network details -- [curl](https://linuxhandbook.com/curl-command-examples/) - Curl a website from the command line -- [ifconfig](https://www.tecmint.com/ifconfig-command-examples/)/[ip](https://www.geeksforgeeks.org/ip-command-in-linux-with-examples/) - Show ip address configuration -- [dig](https://www.geeksforgeeks.org/dig-command-in-linux-with-examples/) - Query Nameservers -- [ipcalc](https://www.linux.com/topic/networking/how-calculate-network-addresses-ipcalc/) - Calculate IP addresses -- Advanced Tools for troubleshooting - - [Netshoot](https://github.com/nicolaka/netshoot) - Troubleshoot Networks -- Cluster Creation tools - - [kind](https://kind.sigs.k8s.io/docs/user/quick-start/) - - [minikube](https://minikube.sigs.k8s.io/docs/start/) -- MacOS users - - [docker-mac-net-connect](https://github.com/chipmk/docker-mac-net-connect) - See this [issue](https://github.com/kubernetes/minikube/issues/7332) diff --git a/NEW_RELEASE_PROCESS.md b/NEW_RELEASE_PROCESS.md deleted file mode 100644 index cdb683abf..000000000 --- a/NEW_RELEASE_PROCESS.md +++ /dev/null @@ -1,9 +0,0 @@ -# Semi-Automated Release Process - -1. Update TAG -2. Cloud Build -3. k8s.io PR -4. git pull origin main -5. git checkout -b $RELEASE_VERSION -6. mage release:newrelease $RELEASE_VERSION -7. Wait for PR \ No newline at end of file diff --git a/NGINX_BASE b/NGINX_BASE deleted file mode 100644 index dd8d6c586..000000000 --- a/NGINX_BASE +++ /dev/null @@ -1 +0,0 @@ -registry.k8s.io/ingress-nginx/nginx:v2.0.0@sha256:3e7bda4cf5111d283ed1e4ff5cc9a2b5cdc5ebe62d50ba67473d3e25b1389133 diff --git a/OWNERS b/OWNERS index bafe6b3f7..250bce4e8 100644 --- a/OWNERS +++ b/OWNERS @@ -1,13 +1,15 @@ -# See the OWNERS docs: https://www.kubernetes.dev/docs/guide/owners +# See the OWNERS docs: https://github.com/kubernetes/community/blob/master/contributors/guide/owners.md approvers: -- ingress-nginx-maintainers + - ingress-nginx-admins + - ingress-nginx-maintainers + - ElvinEfendi + - rikatz reviewers: -- ingress-nginx-reviewers + - ElvinEfendi + - cmluciano + - rikatz emeritus_approvers: -- aledbf # 2020-04-02 -- bowei # 2022-10-12 -- ElvinEfendi # 2023-04-23 -- rikatz # 2024-12-15 + - aledbf # 2020-04-02 diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 314202c10..30f781cfc 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -1,17 +1,10 @@ -# See the OWNERS docs: https://www.kubernetes.dev/docs/guide/owners +# See the OWNERS docs: https://git.k8s.io/community/docs/devel/owners.md aliases: + sig-network-leads: + - caseydavenport + - dcbw + - thockin + ingress-nginx-admins: + - bowei ingress-nginx-maintainers: - - cpanato - - Gacko - - strongjz - - tao12345666333 - - ingress-nginx-reviewers: - - cpanato - - Gacko - - strongjz - - tao12345666333 - - ingress-nginx-docs-maintainers: - - longwuyuan diff --git a/README.md b/README.md index c95e644de..6a5d7605b 100644 --- a/README.md +++ b/README.md @@ -1,94 +1,57 @@ -# Ingress NGINX Controller +## Help us to improve the NGINX Ingress controller [completing the survey](https://docs.google.com/forms/d/15ULTOvYDsV920V0GWrspew4yyjEmTAi740Wr34UgKwA/viewform) + +--- + +# NGINX Ingress Controller -[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/5691/badge)](https://bestpractices.coreinfrastructure.org/projects/5691) [![Go Report Card](https://goreportcard.com/badge/github.com/kubernetes/ingress-nginx)](https://goreportcard.com/report/github.com/kubernetes/ingress-nginx) -[![GitHub license](https://img.shields.io/github/license/kubernetes/ingress-nginx.svg)](https://github.com/kubernetes/ingress-nginx/blob/main/LICENSE) +[![GitHub license](https://img.shields.io/github/license/kubernetes/ingress-nginx.svg)](https://github.com/kubernetes/ingress-nginx/blob/master/LICENSE) [![GitHub stars](https://img.shields.io/github/stars/kubernetes/ingress-nginx.svg)](https://github.com/kubernetes/ingress-nginx/stargazers) -[![GitHub stars](https://img.shields.io/badge/contributions-welcome-orange.svg)](https://github.com/kubernetes/ingress-nginx/blob/main/CONTRIBUTING.md) +[![GitHub stars](https://img.shields.io/badge/contributions-welcome-orange.svg)](https://github.com/kubernetes/ingress-nginx/blob/master/CONTRIBUTING.md) +[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fkubernetes%2Fingress-nginx.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fkubernetes%2Fingress-nginx?ref=badge_shield) ## Overview -ingress-nginx is an Ingress controller for Kubernetes using [NGINX](https://www.nginx.org/) as a reverse proxy and load -balancer. +ingress-nginx is an Ingress controller for Kubernetes using [NGINX](https://www.nginx.org/) as a reverse proxy and load balancer. -[Learn more about Ingress on the Kubernetes documentation site](https://kubernetes.io/docs/concepts/services-networking/ingress/). +[Learn more about Ingress on the main Kubernetes documentation site](https://kubernetes.io/docs/concepts/services-networking/ingress/). ## Get started See the [Getting Started](https://kubernetes.github.io/ingress-nginx/deploy/) document. -Do not use in multi-tenant Kubernetes production installations. This project assumes that users that can create Ingress objects are administrators of the cluster. See the [FAQ](https://kubernetes.github.io/ingress-nginx/faq/#faq) for more. - ## Troubleshooting -If you encounter issues, review the [troubleshooting docs](docs/troubleshooting.md), -[file an issue](https://github.com/kubernetes/ingress-nginx/issues), or talk to us on the -[#ingress-nginx channel](https://kubernetes.slack.com/messages/ingress-nginx) on the Kubernetes Slack server. +If you encounter issues, review the [troubleshooting docs](docs/troubleshooting.md), [file an issue](https://github.com/kubernetes/ingress-nginx/issues), or talk to us on the [#ingress-nginx channel](https://kubernetes.slack.com/messages/ingress-nginx) on the Kubernetes Slack server. -## Changelog - -See [the list of releases](https://github.com/kubernetes/ingress-nginx/releases) for all changes. -For detailed changes for each release, please check the [changelog-$version.md](./changelog) file for the release version. -For detailed changes on the `ingress-nginx` helm chart, please check the changelog folder for a specific version. -[CHANGELOG-$current-version.md](./charts/ingress-nginx/changelog) file. - -### Supported Versions table - -Supported versions for the ingress-nginx project mean that we have completed E2E tests, and they are passing for -the versions listed. Ingress-Nginx versions **may** work on older versions, but the project does not make that guarantee. - -| Supported | Ingress-NGINX version | k8s supported version | Alpine Version | Nginx Version | Helm Chart Version | -| :-------: | --------------------- | ----------------------------- | -------------- | ------------- | ------------------ | -| 🔄 | **v1.12.0** | 1.32, 1.31, 1.30, 1.29, 1.28 | 3.21.0 | 1.25.5 | 4.12.0 | -| 🔄 | **v1.12.0-beta.0** | 1.32, 1.31, 1.30, 1.29, 1.28 | 3.20.3 | 1.25.5 | 4.12.0-beta.0 | -| 🔄 | **v1.11.4** | 1.30, 1.29, 1.28, 1.27, 1.26 | 3.21.0 | 1.25.5 | 4.11.4 | -| 🔄 | **v1.11.3** | 1.30, 1.29, 1.28, 1.27, 1.26 | 3.20.3 | 1.25.5 | 4.11.3 | -| 🔄 | **v1.11.2** | 1.30, 1.29, 1.28, 1.27, 1.26 | 3.20.0 | 1.25.5 | 4.11.2 | -| 🔄 | **v1.11.1** | 1.30, 1.29, 1.28, 1.27, 1.26 | 3.20.0 | 1.25.5 | 4.11.1 | -| 🔄 | **v1.11.0** | 1.30, 1.29, 1.28, 1.27, 1.26 | 3.20.0 | 1.25.5 | 4.11.0 | -| | **v1.10.6** | 1.30, 1.29, 1.28, 1.27, 1.26 | 3.21.0 | 1.25.5 | 4.10.6 | -| | **v1.10.5** | 1.30, 1.29, 1.28, 1.27, 1.26 | 3.20.3 | 1.25.5 | 4.10.5 | -| | **v1.10.4** | 1.30, 1.29, 1.28, 1.27, 1.26 | 3.20.0 | 1.25.5 | 4.10.4 | -| | **v1.10.3** | 1.30, 1.29, 1.28, 1.27, 1.26 | 3.20.0 | 1.25.5 | 4.10.3 | -| | **v1.10.2** | 1.30, 1.29, 1.28, 1.27, 1.26 | 3.20.0 | 1.25.5 | 4.10.2 | -| | **v1.10.1** | 1.30, 1.29, 1.28, 1.27, 1.26 | 3.19.1 | 1.25.3 | 4.10.1 | -| | **v1.10.0** | 1.29, 1.28, 1.27, 1.26 | 3.19.1 | 1.25.3 | 4.10.0 | -| | v1.9.6 | 1.29, 1.28, 1.27, 1.26, 1.25 | 3.19.0 | 1.21.6 | 4.9.1 | -| | v1.9.5 | 1.28, 1.27, 1.26, 1.25 | 3.18.4 | 1.21.6 | 4.9.0 | -| | v1.9.4 | 1.28, 1.27, 1.26, 1.25 | 3.18.4 | 1.21.6 | 4.8.3 | -| | v1.9.3 | 1.28, 1.27, 1.26, 1.25 | 3.18.4 | 1.21.6 | 4.8.* | -| | v1.9.1 | 1.28, 1.27, 1.26, 1.25 | 3.18.4 | 1.21.6 | 4.8.* | -| | v1.9.0 | 1.28, 1.27, 1.26, 1.25 | 3.18.2 | 1.21.6 | 4.8.* | -| | v1.8.4 | 1.27, 1.26, 1.25, 1.24 | 3.18.2 | 1.21.6 | 4.7.* | -| | v1.7.1 | 1.27, 1.26, 1.25, 1.24 | 3.17.2 | 1.21.6 | 4.6.* | -| | v1.6.4 | 1.26, 1.25, 1.24, 1.23 | 3.17.0 | 1.21.6 | 4.5.* | -| | v1.5.1 | 1.25, 1.24, 1.23 | 3.16.2 | 1.21.6 | 4.4.* | -| | v1.4.0 | 1.25, 1.24, 1.23, 1.22 | 3.16.2 | 1.19.10† | 4.3.0 | -| | v1.3.1 | 1.24, 1.23, 1.22, 1.21, 1.20 | 3.16.2 | 1.19.10† | 4.2.5 | - -See [this article](https://kubernetes.io/blog/2021/07/26/update-with-ingress-nginx/) if you want upgrade to the stable -Ingress API. - -## Get Involved +## Contributing Thanks for taking the time to join our community and start contributing! -- This project adheres to the [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md). - By participating in this project, you agree to abide by its terms. -- **Contributing**: Contributions of all kinds are welcome! +- This project adheres to the [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md). By participating in this project, you agree to abide by its terms. +- See [CONTRIBUTING.md](CONTRIBUTING.md) for information about setting up your environment, the workflow that we expect, and instructions on the developer certificate of origin that we require. +- Check out the [open issues](https://github.com/kubernetes/ingress-nginx). +- Join our Kubernetes Slack channel: [#ingress-nginx](https://kubernetes.slack.com/messages/CANQGM8BA/). - - Read [`CONTRIBUTING.md`](CONTRIBUTING.md) for information about setting up your environment, the workflow that we - expect, and instructions on the developer certificate of origin that we require. - - Join our Kubernetes Slack channel for developer discussion : [#ingress-nginx-dev](https://kubernetes.slack.com/archives/C021E147ZA4). - - Submit GitHub issues for any feature enhancements, bugs, or documentation problems. - - Please make sure to read the [Issue Reporting Checklist](https://github.com/kubernetes/ingress-nginx/blob/main/CONTRIBUTING.md#issue-reporting-guidelines) before opening an issue. Issues not conforming to the guidelines **may be closed immediately**. - - Join our [ingress-nginx-dev mailing list](https://groups.google.com/a/kubernetes.io/g/ingress-nginx-dev/c/ebbBMo-zX-w) -- **Support**: +## Changelog - - Join the [#ingress-nginx-users](https://kubernetes.slack.com/messages/CANQGM8BA/) channel inside the [Kubernetes Slack](http://slack.kubernetes.io/) to ask questions or get support from the maintainers and other users. - - The [GitHub issues](https://github.com/kubernetes/ingress-nginx/issues) in the repository are **exclusively** for bug reports and feature requests. - - **Discuss**: Tweet using the `#IngressNginx` hashtag or sharing with us [@IngressNginx](https://twitter.com/IngressNGINX). +See [the list of releases](https://github.com/kubernetes/ingress-nginx/releases) to find out about feature changes. +For detailed changes for each release; please check the [Changelog.md](Changelog.md) file. +For detailed changes on the `ingress-nginx` helm chart, please check the following [CHANGELOG.md](charts/ingress-nginx/CHANGELOG.md) file. + +# Get Involved + +- **Contributing**: Pull requests are welcome! + - Read [`CONTRIBUTING.md`](CONTRIBUTING.md) and check out [help-wanted](https://github.com/kubernetes/ingress-nginx/labels/help%20wanted) issues. + - Submit github issues for any feature enhancements, bugs or documentation problems. +- **Support**: Join to [Kubernetes Slack](http://slack.kubernetes.io/) in the [#ingress-nginx](https://kubernetes.slack.com/messages/CANQGM8BA/) channel to ask questions to get support from the maintainers and other users. + - The [github issues](https://github.com/kubernetes/ingress-nginx/issues) in the repository are **exclusively** for bug reports and feature requests. +- **Discuss**: Tweet using the `#IngressNginx` hashtag. + +## Issues + +Please make sure to read the [Issue Reporting Checklist](https://github.com/kubernetes/ingress-nginx/blob/master/CONTRIBUTING.md#issue-reporting-guidelines) before opening an issue. Issues not conforming to the guidelines **may be closed immediately**. ## License -[Apache License 2.0](https://github.com/kubernetes/ingress-nginx/blob/main/LICENSE) +[Apache License 2.0](https://github.com/kubernetes/ingress-nginx/blob/master/LICENSE) diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 000000000..f9a90a3da --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,76 @@ +1. [NGINX](https://github.com/kubernetes/ingress-nginx/tree/master/images/nginx) + +* Open pull request + +If you are updating any component in [build.sh](images/nginx/rootfs/build.sh) please also update the SHA256 checksum of that component as + well, the cloud build will fail with an exit 10 if not. +Example [NGINX_VERSION](images/nginx/rootfs/build.sh#L21), +[SHA256](images/nginx/rootfs/build.sh#L124) +* Merge +* Wait for [cloud build](https://console.cloud.google.com/cloud-build/builds?project=k8s-staging-ingress-nginx) + +1a. Promote images: + +Open pull request to promote staging image: +[add sha - version](https://github.com/kubernetes/k8s.io/blob/main/k8s.gcr.io/images/k8s-staging-ingress-nginx/images.yaml#L1) + +The sha is available in output from [cloud build](https://console.cloud.google.com/cloud-build/builds?project=k8s-staging-ingress-nginx) + +2. Change to images: + +* [e2e](https://github.com/kubernetes/ingress-nginx/tree/master/images/test-runner) + + * [test-runner](https://github.com/kubernetes/ingress-nginx/tree/master/images/echo) + * [echo](https://github.com/kubernetes/ingress-nginx/tree/master/images/echo) + * [cfssl](https://github.com/kubernetes/ingress-nginx/tree/master/images/cfssl) + * [fastcgi-helloserver](https://github.com/kubernetes/ingress-nginx/tree/master/images/fastcgi-helloserver) + * [httpbin](https://github.com/kubernetes/ingress-nginx/tree/master/images/httpbin) + +* Open pull request +* Merge +* Wait for [cloud build](https://console.cloud.google.com/cloud-build/builds?project=k8s-staging-ingress-nginx) + +2a. Promote images: + +* Open pull request to promote [staging image](https://github.com/kubernetes/k8s.io/blob/master/k8s.gcr.io/images/k8s-staging-ingress-nginx/images.yaml) + * e2e-test-runner + * e2e-test-cfssl + * e2e-test-echo + * e2e-test-fastcgi-helloserver + * e2e-test-httpbin + +3. Update references to e2e-test-runner image: + +* [e2e-image](https://github.com/kubernetes/ingress-nginx/blob/master/test/e2e-image/Dockerfile#L1) +* [run-in-docker.sh](https://github.com/kubernetes/ingress-nginx/blob/ff60aa9e2b5377db1544091b98f475a90a630297/build/run-in-docker.sh#L37) + +4. Prepare for a new release: + +* Change [TAG](https://github.com/kubernetes/ingress-nginx/blob/master/TAG#L1) +* Open pull request +* Merge +* [Wait for cloud build](https://console.cloud.google.com/cloud-build/builds?project=k8s-staging-ingress-nginx) + +4a. Promote images: + +* Open pull request to promote [staging image](https://github.com/kubernetes/k8s.io/blob/master/k8s.gcr.io/images/k8s-staging-ingress-nginx/images.yaml) + * controller + +5. Release helm chart: + +* Open pull request updating [Chart.yaml](https://github.com/kubernetes/ingress-nginx/blob/master/charts/ingress-nginx/Chart.yaml#L3-L4) +* Merge +* [New helm chart is available](https://github.com/kubernetes/ingress-nginx/blob/master/.github/workflows/main.yaml#L47-L68) + +6. New release: + +* Update static scripts: + * [generate-deploy-scripts.sh](https://github.com/kubernetes/ingress-nginx/blob/master/hack/generate-deploy-scripts.sh) + * Open pull request with the updates + * Merge + +* Update changelog: + * Open pull request updating [Changelog.md](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md) + * Merge + +7. Github release \ No newline at end of file diff --git a/SECURITY_CONTACTS b/SECURITY_CONTACTS index 587c7e016..fd437b717 100644 --- a/SECURITY_CONTACTS +++ b/SECURITY_CONTACTS @@ -9,5 +9,4 @@ # # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE # INSTRUCTIONS AT https://kubernetes.io/security/ -Gacko -strongjz + diff --git a/TAG b/TAG new file mode 100644 index 000000000..0131a133c --- /dev/null +++ b/TAG @@ -0,0 +1 @@ +v0.47.0 diff --git a/build/build-plugin.sh b/build/build-plugin.sh new file mode 100755 index 000000000..578957eb4 --- /dev/null +++ b/build/build-plugin.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ -n "$DEBUG" ]; then + set -x +fi + +set -o errexit +set -o nounset +set -o pipefail + +declare -a mandatory +mandatory=( + PKG + ARCH + COMMIT_SHA + REPO_INFO + TAG +) + +missing=false +for var in "${mandatory[@]}"; do + if [[ -z "${!var:-}" ]]; then + echo "Environment variable $var must be set" + missing=true + fi +done + +if [ "$missing" = true ]; then + exit 1 +fi + +export CGO_ENABLED=0 + +release=cmd/plugin/release + +function build_for_arch(){ + os=$1 + arch=$2 + extension=$3 + + env GOOS="${os}" GOARCH="${arch}" go build \ + "${GOBUILD_FLAGS}" \ + -trimpath -ldflags="-buildid= -w -s" \ + -X ${PKG}/version.RELEASE=${TAG} \ + -X ${PKG}/version.COMMIT=${COMMIT_SHA} \ + -X ${PKG}/version.REPO=${REPO_INFO}" \ + -o "${release}/kubectl-ingress_nginx${extension}" "${PKG}/cmd/plugin" + + cp LICENSE ${release} + tar -C "${release}" -zcvf "${release}/kubectl-ingress_nginx-${os}-${arch}.tar.gz" "kubectl-ingress_nginx${extension}" LICENSE + rm "${release}/kubectl-ingress_nginx${extension}" + hash=$(sha256sum "${release}/kubectl-ingress_nginx-${os}-${arch}.tar.gz" | awk '{ print $1 }') + sed -i "s/%%%shasum_${os}_${arch}%%%/${hash}/g" "${release}/ingress-nginx.yaml" +} + +rm -rf "${release}" +mkdir "${release}" + +cp cmd/plugin/ingress-nginx.yaml.tmpl "${release}/ingress-nginx.yaml" + +sed -i "s/%%%tag%%%/${TAG}/g" ${release}/ingress-nginx.yaml + +build_for_arch darwin amd64 '' +build_for_arch linux amd64 '' +build_for_arch windows amd64 '.exe' diff --git a/build/build.sh b/build/build.sh index bbcaf78e8..914716feb 100755 --- a/build/build.sh +++ b/build/build.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Copyright 2018 The Kubernetes Authors. # @@ -14,56 +14,55 @@ # See the License for the specific language governing permissions and # limitations under the License. -GO_BUILD_CMD="go build" - -#if [ -n "$DEBUG" ]; then -# set -x -# GO_BUILD_CMD="go build -v" -#fi +if [ -n "$DEBUG" ]; then + set -x +fi set -o errexit set -o nounset set -o pipefail +declare -a mandatory +mandatory=( + PKG + ARCH + COMMIT_SHA + REPO_INFO + TAG +) -if [ -z "$PKG" ] || [ -z "$ARCH" ] || [ -z "$COMMIT_SHA" ] || [ -z "$REPO_INFO" ] || [ -z "$TAG" ]; then - echo "Environments PKG, ARCH, COMMIT_SHA, REPO_INFO and TAG are required" - exit 1 +missing=false +for var in "${mandatory[@]}"; do + if [[ -z "${!var:-}" ]]; then + echo "Environment variable $var must be set" + missing=true + fi +done + +if [ "$missing" = true ]; then + exit 1 fi - export CGO_ENABLED=0 -export GOARCH="${ARCH}" +export GOARCH=${ARCH} -TARGETS_DIR="rootfs/bin/${ARCH}" -echo "Building targets for ${ARCH}, generated targets in ${TARGETS_DIR} directory." - -echo "Building ${PKG}/cmd/nginx" - -${GO_BUILD_CMD} \ +go build \ -trimpath -ldflags="-buildid= -w -s \ - -X ${PKG}/version.RELEASE=${TAG} \ - -X ${PKG}/version.COMMIT=${COMMIT_SHA} \ - -X ${PKG}/version.REPO=${REPO_INFO}" \ - -buildvcs=false \ - -o "${TARGETS_DIR}/nginx-ingress-controller" "${PKG}/cmd/nginx" + -X ${PKG}/version.RELEASE=${TAG} \ + -X ${PKG}/version.COMMIT=${COMMIT_SHA} \ + -X ${PKG}/version.REPO=${REPO_INFO}" \ + -o "rootfs/bin/${ARCH}/nginx-ingress-controller" "${PKG}/cmd/nginx" -echo "Building ${PKG}/cmd/dbg" - -${GO_BUILD_CMD} \ +go build \ -trimpath -ldflags="-buildid= -w -s \ - -X ${PKG}/version.RELEASE=${TAG} \ - -X ${PKG}/version.COMMIT=${COMMIT_SHA} \ - -X ${PKG}/version.REPO=${REPO_INFO}" \ - -buildvcs=false \ - -o "${TARGETS_DIR}/dbg" "${PKG}/cmd/dbg" + -X ${PKG}/version.RELEASE=${TAG} \ + -X ${PKG}/version.COMMIT=${COMMIT_SHA} \ + -X ${PKG}/version.REPO=${REPO_INFO}" \ + -o "rootfs/bin/${ARCH}/dbg" "${PKG}/cmd/dbg" -echo "Building ${PKG}/cmd/waitshutdown" - -${GO_BUILD_CMD} \ +go build \ -trimpath -ldflags="-buildid= -w -s \ - -X ${PKG}/version.RELEASE=${TAG} \ - -X ${PKG}/version.COMMIT=${COMMIT_SHA} \ - -X ${PKG}/version.REPO=${REPO_INFO}" \ - -buildvcs=false \ - -o "${TARGETS_DIR}/wait-shutdown" "${PKG}/cmd/waitshutdown" \ No newline at end of file + -X ${PKG}/version.RELEASE=${TAG} \ + -X ${PKG}/version.COMMIT=${COMMIT_SHA} \ + -X ${PKG}/version.REPO=${REPO_INFO}" \ + -o "rootfs/bin/${ARCH}/wait-shutdown" "${PKG}/cmd/waitshutdown" diff --git a/build/dev-env.sh b/build/dev-env.sh index dd1518ee0..b089de206 100755 --- a/build/dev-env.sh +++ b/build/dev-env.sh @@ -36,7 +36,7 @@ if ! command -v kind &> /dev/null; then fi if ! command -v kubectl &> /dev/null; then - echo "Please install kubectl 1.24.0 or higher" + echo "Please install kubectl 1.15 or higher" exit 1 fi @@ -45,18 +45,15 @@ if ! command -v helm &> /dev/null; then exit 1 fi -function ver { printf "%d%03d%03d" $(echo "$1" | tr '.' ' '); } - -HELM_VERSION=$(helm version 2>&1 | cut -f1 -d"," | grep -oE '[0-9]+\.[0-9]+\.[0-9]+') || true -echo $HELM_VERSION -if [[ $(ver $HELM_VERSION) -lt $(ver "3.10.0") ]]; then - echo "Please upgrade helm to v3.10.0 or higher" +HELM_VERSION=$(helm version 2>&1 | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+') || true +if [[ ${HELM_VERSION} < "v3.0.0" ]]; then + echo "Please upgrade helm to v3.0.0 or higher" exit 1 fi -KUBE_CLIENT_VERSION=$(kubectl version --client -oyaml 2>/dev/null | grep "minor:" | awk '{print $2}' | tr -d '"') || true -if [[ ${KUBE_CLIENT_VERSION} -lt 24 ]]; then - echo "Please update kubectl to 1.24.2 or higher" +KUBE_CLIENT_VERSION=$(kubectl version --client --short | awk '{print $3}' | cut -d. -f2) || true +if [[ ${KUBE_CLIENT_VERSION} -lt 14 ]]; then + echo "Please update kubectl to 1.15 or higher" exit 1 fi @@ -64,13 +61,32 @@ echo "[dev-env] building image" make build image docker tag "${REGISTRY}/controller:${TAG}" "${DEV_IMAGE}" -export K8S_VERSION=${K8S_VERSION:-v1.32.0@sha256:c48c62eac5da28cdadcf560d1d8616cfa6783b58f0d94cf63ad1bf49600cb027} +export K8S_VERSION=${K8S_VERSION:-v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab} KIND_CLUSTER_NAME="ingress-nginx-dev" if ! kind get clusters -q | grep -q ${KIND_CLUSTER_NAME}; then - echo "[dev-env] creating Kubernetes cluster with kind" - kind create cluster --name ${KIND_CLUSTER_NAME} --image "kindest/node:${K8S_VERSION}" --config ${DIR}/kind.yaml +echo "[dev-env] creating Kubernetes cluster with kind" +cat </dev/null || true -} - set -o errexit set -o nounset set -o pipefail @@ -49,9 +43,16 @@ if [ "$missing" = true ]; then exit 1 fi -BASEDIR=$(dirname "$0") -NGINX_BASE_IMAGE=$(cat $BASEDIR/../../NGINX_BASE) -HTTPBUN_IMAGE=$(cat $BASEDIR/HTTPBUN_IMAGE) +function cleanup { + kubectl delete pod e2e 2>/dev/null || true +} +trap cleanup EXIT + +E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS:-} +FOCUS=${FOCUS:-.*} + +export E2E_CHECK_LEAKS +export FOCUS echo -e "${BGREEN}Granting permissions to ingress-nginx e2e service account...${NC}" kubectl create serviceaccount ingress-nginx-e2e || true @@ -61,14 +62,11 @@ kubectl create clusterrolebinding permissive-binding \ --user=kubelet \ --serviceaccount=default:ingress-nginx-e2e || true -VER=$(kubectl version --client=false -o json |jq '.serverVersion.minor |tonumber') -if [ $VER -lt 24 ]; then - echo -e "${BGREEN}Waiting service account...${NC}"; \ - until kubectl get secret | grep -q -e ^ingress-nginx-e2e-token; do \ - echo -e "waiting for api token"; \ - sleep 3; \ - done -fi +echo -e "${BGREEN}Waiting service account...${NC}"; \ +until kubectl get secret | grep -q -e ^ingress-nginx-e2e-token; do \ + echo -e "waiting for api token"; \ + sleep 3; \ +done echo -e "Starting the e2e test pod" @@ -77,19 +75,6 @@ kubectl run --rm \ --restart=Never \ --env="E2E_NODES=${E2E_NODES}" \ --env="FOCUS=${FOCUS}" \ - --env="IS_CHROOT=${IS_CHROOT:-false}"\ - --env="SKIP_OPENTELEMETRY_TESTS=${SKIP_OPENTELEMETRY_TESTS:-false}"\ --env="E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS}" \ - --env="NGINX_BASE_IMAGE=${NGINX_BASE_IMAGE}" \ - --env="HTTPBUN_IMAGE=${HTTPBUN_IMAGE}" \ --overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "ingress-nginx-e2e"}}' \ e2e --image=nginx-ingress-controller:e2e - -# Get the junit-reports stored in the configMaps created during e2etests -echo "Getting the report file out now.." -reportsDir="test/junitreports" -reportFile="report-e2e-test-suite.xml.gz" -mkdir -p $reportsDir -cd $reportsDir -kubectl get cm $reportFile -o "jsonpath={.binaryData['${reportFile//\./\\.}']}" | base64 -d | gunzip > ${reportFile%\.gz} -echo "done getting the report file out.." diff --git a/build/run-in-docker.sh b/build/run-in-docker.sh index c4f3b34e0..d6723c7a1 100755 --- a/build/run-in-docker.sh +++ b/build/run-in-docker.sh @@ -14,39 +14,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -DEBUG=${DEBUG:-"false"} -if [ "$DEBUG" == "true" ]; then +if [ -n "$DEBUG" ]; then set -x fi -RUNTIME=${RUNTIME:-"docker"} - set -o errexit set -o nounset set -o pipefail # temporal directory for the /etc/ingress-controller directory -if [[ "$OSTYPE" == darwin* ]]; then - mkdir -p "tmp" - INGRESS_VOLUME=$(pwd)/$(mktemp -d tmp/XXXXXX) -else - INGRESS_VOLUME=$(mktemp -d) -fi +INGRESS_VOLUME=$(mktemp -d) -# make sure directory for SSL cert storage exists under ingress volume -mkdir "${INGRESS_VOLUME}/ssl" +if [[ "$OSTYPE" == darwin* ]]; then + INGRESS_VOLUME=/private$INGRESS_VOLUME +fi function cleanup { rm -rf "${INGRESS_VOLUME}" } trap cleanup EXIT -E2E_IMAGE=${E2E_IMAGE:-registry.k8s.io/ingress-nginx/e2e-test-runner:v20250112-a188f4eb@sha256:043038b1e30e5a0b64f3f919f096c5c9488ac3f617ac094b07fb9db8215f9441} - -if [[ "$RUNTIME" == podman ]]; then - # Podman does not support both tag and digest - E2E_IMAGE=$(echo $E2E_IMAGE | awk -F "@sha" '{print $1}') -fi +E2E_IMAGE=${E2E_IMAGE:-k8s.gcr.io/ingress-nginx/e2e-test-runner:v20210601-g96a87c79b@sha256:f84dcddc84e5cba220260f315e18cd47fc8c6b7f3f4f57b7b3e9cc2ea25324b7} DOCKER_OPTS=${DOCKER_OPTS:-} DOCKER_IN_DOCKER_ENABLED=${DOCKER_IN_DOCKER_ENABLED:-} @@ -64,33 +52,21 @@ fi # create output directory as current user to avoid problem with docker. mkdir -p "${KUBE_ROOT}/bin" "${KUBE_ROOT}/bin/${ARCH}" -PLATFORM="${PLATFORM:-}" -if [[ -n "$PLATFORM" ]]; then - PLATFORM_FLAG=--platform -else - PLATFORM_FLAG= -fi - -USER=${USER:-nobody} - -USE_SHELL=${USE_SHELL:-"/bin/bash"} - if [[ "$DOCKER_IN_DOCKER_ENABLED" == "true" ]]; then - echo "..reached DIND check TRUE block, inside run-in-docker.sh" - echo "FLAGS=$FLAGS" - #go env - go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo@v2.22.2 - find / -type f -name ginkgo 2>/dev/null - which ginkgo /bin/bash -c "${FLAGS}" else - echo "Reached DIND check ELSE block, inside run-in-docker.sh" - - args="${PLATFORM_FLAG} ${PLATFORM} --tty --rm ${DOCKER_OPTS} -e DEBUG=${DEBUG} -e GOCACHE="/go/src/${PKG}/.cache" -e GOMODCACHE="/go/src/${PKG}/.modcache" -e DOCKER_IN_DOCKER_ENABLED="true" -v "${HOME}/.kube:${HOME}/.kube" -v "${KUBE_ROOT}:/go/src/${PKG}" -v "${KUBE_ROOT}/bin/${ARCH}:/go/bin/linux_${ARCH}" -v "${INGRESS_VOLUME}:/etc/ingress-controller/" -w "/go/src/${PKG}"" - - if [[ "$RUNTIME" == "docker" ]]; then - args="$args -v /var/run/docker.sock:/var/run/docker.sock" - fi - - ${RUNTIME} run $args ${E2E_IMAGE} ${USE_SHELL} -c "${FLAGS}" + docker run \ + --tty \ + --rm \ + ${DOCKER_OPTS} \ + -e GOCACHE="/go/src/${PKG}/.cache" \ + -e DOCKER_IN_DOCKER_ENABLED="true" \ + -v "${HOME}/.kube:${HOME}/.kube" \ + -v "${KUBE_ROOT}:/go/src/${PKG}" \ + -v "${KUBE_ROOT}/bin/${ARCH}:/go/bin/linux_${ARCH}" \ + -v "/var/run/docker.sock:/var/run/docker.sock" \ + -v "${INGRESS_VOLUME}:/etc/ingress-controller/" \ + -w "/go/src/${PKG}" \ + -u $(id -u ${USER}):$(id -g ${USER}) \ + ${E2E_IMAGE} /bin/bash -c "${FLAGS}" fi diff --git a/build/run-ingress-controller.sh b/build/run-ingress-controller.sh index 99f598b1f..29338241f 100755 --- a/build/run-ingress-controller.sh +++ b/build/run-ingress-controller.sh @@ -49,7 +49,7 @@ fi SSL_VOLUME=$(mktemp -d) function cleanup { - echo -e "${BGREEN}Stopping kubectl proxy${NC}" + echo -e "${BGREEN}Stoping kubectl proxy${NC}" rm -rf "${SSL_VOLUME}" kill "$proxy_pid" } diff --git a/test/test-lua.sh b/build/test-lua.sh similarity index 54% rename from test/test-lua.sh rename to build/test-lua.sh index e7ee5843e..3b00cf35a 100755 --- a/test/test-lua.sh +++ b/build/test-lua.sh @@ -25,22 +25,14 @@ set -o pipefail BUSTED_ARGS="-v --pattern=_test" -SHDICT_ARGS=( - "-I" "./rootfs/etc/nginx/lua" - "--shdict" "configuration_data 5M" - "--shdict" "certificate_data 16M" - "--shdict" "certificate_servers 1M" - "--shdict" "ocsp_response_cache 1M" - "--shdict" "balancer_ewma 1M" - "--shdict" "quota_tracker 1M" - "--shdict" "high_throughput_tracker 1M" - "--shdict" "balancer_ewma_last_touched_at 1M" - "--shdict" "balancer_ewma_locks 512k" - "./rootfs/etc/nginx/lua/test/run.lua" -) - -if [ $# -eq 0 ]; then - resty "${SHDICT_ARGS[@]}" ./rootfs/etc/nginx/lua/test/ ${BUSTED_ARGS} -else - resty "${SHDICT_ARGS[@]}" $@ ${BUSTED_ARGS} -fi +resty \ + -I ./rootfs/etc/nginx/lua \ + --shdict "configuration_data 5M" \ + --shdict "certificate_data 16M" \ + --shdict "certificate_servers 1M" \ + --shdict "ocsp_response_cache 1M" \ + --shdict "balancer_ewma 1M" \ + --shdict "balancer_ewma_last_touched_at 1M" \ + --shdict "balancer_ewma_locks 512k" \ + --shdict "global_throttle_cache 5M" \ + ./rootfs/etc/nginx/lua/test/run.lua ${BUSTED_ARGS} ./rootfs/etc/nginx/lua/test/ ./rootfs/etc/nginx/lua/plugins/**/test diff --git a/test/test.sh b/build/test.sh similarity index 97% rename from test/test.sh rename to build/test.sh index 674b9484a..bb7ccdb7e 100755 --- a/test/test.sh +++ b/build/test.sh @@ -23,7 +23,6 @@ set -o errexit set -o nounset set -o pipefail -mkdir -p /tmp/nginx if [ -z "${PKG}" ]; then echo "PKG must be set" exit 1 diff --git a/changelog/controller-1.10.0.md b/changelog/controller-1.10.0.md deleted file mode 100644 index e0a6c9a10..000000000 --- a/changelog/controller-1.10.0.md +++ /dev/null @@ -1,68 +0,0 @@ -# Changelog - -This release is the first using NGINX v1.25.0! - -## Breaking changes -* This version does not support chroot image, this will be fixed on a future minor patch release -* This version dropped Opentracing and zipkin modules, just Opentelemetry is supported -* This version dropped support for PodSecurityPolicy -* This version dropped support for GeoIP (legacy). Only GeoIP2 is supported - -### controller-v1.10.0 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.10.0@sha256:42b3f0e5d0846876b1791cd3afeb5f1cbbe4259d6f35651dcc1b5c980925379c - -### All changes: - -* Start the release of v1.10.0 (#11038) -* bump nginx and Go, remove tag file and old CI jobs (#11037) -* Fix kubewebhook image tag (#11033) -* add missing backend-protocol annotation option (#9545) -* Update controller-prometheusrules.yaml (#8902) -* Stop reporting interrupted tests (#11027) -* test(gzip): reach ingress (#9541) -* fix datasource, $exported_namespace variable in grafana nginx dashboard (#9092) -* Properly support a TLS-wrapped OCSP responder (#10164) -* Fix print-e2e-suite (#9536) -* chore(deps): upgrade headers-more module to 0.37 (#10991) -* Update ingress-path-matching.md (#11008) -* Update ingress-path-matching.md (#11007) -* E2E Tests: Explicitly enable metrics. (#10962) -* Chart: Set `--enable-metrics` depending on `controller.metrics.enabled`. (#10959) -* Chart: Remove useless `default` from `_params.tpl`. (#10957) -* Fix golang makefile var name (#10932) -* Fixing image push (#10931) -* fix: live-docs script (#10928) -* docs: Add vouch-proxy OAuth example (#10929) -* Add OTEL build test and for NGINX v1.25 (#10889) -* docs: update annotations docs with missing session-cookie section (#10917) -* Release controller 1.9.6 and helm 4.9.1 (#10919) - -### Dependency updates: - -* Bump kubewebhook certgen (#11034) -* Bump go libraries (#11023) -* Bump modsecurity on nginx 1.25 (#11024) -* Bump grpc and reintroduce OTEL compilation (#11021) -* Bump github/codeql-action from 3.24.0 to 3.24.5 (#11017) -* Bump actions/dependency-review-action from 4.0.0 to 4.1.3 (#11016) -* Bump dorny/paths-filter from 3.0.0 to 3.0.1 (#10994) -* Bump github.com/prometheus/client_model from 0.5.0 to 0.6.0 (#10998) -* Bump actions/upload-artifact from 4.3.0 to 4.3.1 (#10978) -* Bump actions/download-artifact from 4.1.1 to 4.1.2 (#10981) -* Bump aquasecurity/trivy-action from 0.16.1 to 0.17.0 (#10979) -* Bump golangci/golangci-lint-action from 3.7.0 to 4.0.0 (#10980) -* Bump golang.org/x/crypto from 0.18.0 to 0.19.0 (#10976) -* Bump github/codeql-action from 3.23.2 to 3.24.0 (#10971) -* Bump github.com/opencontainers/runc from 1.1.11 to 1.1.12 (#10951) -* Bump google.golang.org/grpc from 1.60.1 to 1.61.0 (#10938) -* Bump actions/upload-artifact from 4.2.0 to 4.3.0 (#10937) -* Bump dorny/test-reporter from 1.7.0 to 1.8.0 (#10936) -* Bump github/codeql-action from 3.23.1 to 3.23.2 (#10935) -* Bump dorny/paths-filter from 2.11.1 to 3.0.0 (#10934) -* Bump alpine to 3.19.1 (#10930) -* Bump go to v1.21.6 and set a single source of truth (#10926) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.9.6...controller-v1.10.0 diff --git a/changelog/controller-1.10.1.md b/changelog/controller-1.10.1.md deleted file mode 100644 index 40fc321fa..000000000 --- a/changelog/controller-1.10.1.md +++ /dev/null @@ -1,57 +0,0 @@ -# Changelog - -### controller-v1.10.1 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.10.1@sha256:e24f39d3eed6bcc239a56f20098878845f62baa34b9f2be2fd2c38ce9fb0f29e -* registry.k8s.io/ingress-nginx/controller-chroot:v1.10.1@sha256:c155954116b397163c88afcb3252462771bd7867017e8a17623e83601bab7ac7 - -### All changes: - -* start 1.10.1 build (#11246) -* force nginx rebuild (#11245) -* update k8s version to latest kind release (#11241) -* remove _ssl_expire_time_seconds metric by identifier (#11239) -* update post submit helm ci and clean up (#11221) -* Chart: Add unit tests for default backend & topology spread constraints. (#11219) -* sort default backend hpa metrics (#11217) -* updated certgen image shatag (#11216) -* changed testrunner image sha (#11211) -* bumped certgeimage tag (#11213) -* updated baseimage & deleted a useless file (#11209) -* bump ginkgo to 2-17-1 in testrunner (#11204) -* chunking related faq update (#11205) -* Fix-semver (#11199) -* refactor helm ci tests part I (#11188) -* Proposal: e2e tests for regex patterns (#11185) -* bump ginkgo to v2.17.1 (#11186) -* fixes brotli build issue (#11187) -* fix geoip2 configuration docs (#11151) -* Fix typos in OTel doc (#11081) (#11129) -* Chart: Render `controller.ingressClassResource.parameters` natively. (#11126) -* Fix admission controller logging of `admissionTime` and `testedConfigurationSize` (#11114) -* Chart: Align HPA & KEDA conditions. (#11113) -* Chart: Improve IngressClass documentation. (#11111) -* Chart: Add Gacko to maintainers. Again. (#11112) -* Chart: Deploy `PodDisruptionBudget` with KEDA. (#11105) -* Chores: Pick patches from main. (#11103) - -### Dependency updates: - -* Bump google.golang.org/grpc from 1.63.0 to 1.63.2 (#11238) -* Bump google.golang.org/grpc from 1.62.1 to 1.63.0 (#11234) -* Bump github.com/prometheus/common from 0.51.1 to 0.52.2 (#11233) -* Bump golang.org/x/crypto from 0.21.0 to 0.22.0 (#11232) -* Bump github.com/prometheus/client_model in the all group (#11231) -* Bump the all group with 3 updates (#11230) -* Bump the all group with 2 updates (#11190) -* Bump actions/add-to-project from 0.6.1 to 1.0.0 (#11189) -* Bump the all group with 3 updates (#11166) -* Bump github.com/prometheus/common from 0.50.0 to 0.51.1 (#11160) -* Bump the all group with 4 updates (#11140) -* Bump the all group with 1 update (#11136) -* Bump google.golang.org/protobuf from 1.32.0 to 1.33.0 in /magefiles (#11127) -* Bump google.golang.org/protobuf in /images/custom-error-pages/rootfs (#11128) -* Bump google.golang.org/protobuf in /images/kube-webhook-certgen/rootfs (#11122) - diff --git a/changelog/controller-1.10.2.md b/changelog/controller-1.10.2.md deleted file mode 100644 index 384768a24..000000000 --- a/changelog/controller-1.10.2.md +++ /dev/null @@ -1,130 +0,0 @@ -# Changelog - -### controller-v1.10.2 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.10.2@sha256:e3311b3d9671bc52d90572bcbfb7ee5b71c985d6d6cffd445c241f1e2703363c -* registry.k8s.io/ingress-nginx/controller-chroot:v1.10.2@sha256:c4395cba98f9721e3381d3c06e7994371bae20f5ab30e457cd7debe44a8c8c54 - -### All changes: - -* update test runner to latest build (#11557) -* add k8s 1.30 to ci build (#11553) -* update test runner go base to 3.20 (#11550) -* tag new test runner image with new nginx base 0.0.8 (#11549) -* bump NGINX_BASE to v0.0.8 (#11543) -* trigger build for NGINX-1.25 v0.0.8 (#11542) -* Upgrade OWASP_MODSECURITY_CRS_VERSION 3.3.5 to 4.4.0 and update docs (#11548) -* [feature] bump nginx to 1.25.5 and add http3 module (#11541) -* add ssl patches to nginx-1.25 image for coroutines to work in lua client hello and cert ssl blocks (#11534) -* bump alpine version to 3.20 to custom-error-pages (#11537) -* fix: Ensure changes in MatchCN annotation are detected (#11528) -* Docs: Add information about HTTP/3 support. (#11525) -* Docs: Specify `ingressClass` for multi-controller setup. (#11520) -* Docs: Improve default certificate usage. (#11519) -* docs: Update Ingress-NGINX v1.10.1 compatibility with Kubernetes v1.30 (#11500) -* Update getting-started.md with new prerequisites (#11487) -* Fix boolean configuration (#11484) -* Chores: Align security contacts & chart maintainers to actual owners. (#11480) -* CI: Bump forgotten Ginkgo versions. (#11469) -* Tests: Replace deprecated `grpc.Dial` by `grpc.NewClient`. (#11468) -* Owners: Promote Gacko to admin. (#11464) -* fixed fastcgi userguide (#11455) -* Remove unnecessary space character (#11451) -* fix for docs issue 11432 (#11446) -* Update index.md (#11445) -* upgrade to alpine 3.20 (#11438) -* update golang to 1.22.4 (#11431) -* Adapt dashboards for Grafana 11 compatibility (#11414) -* Rename variable to fix typo (#11413) -* Fix helm install on cloud provider admonition block (#11412) -* edited helm-install tips (#11411) -* added info for aws helm install (#11410) -* added multiplecontrollers-howto to faq (#11409) -* removed tlsv1 & tlsv1.1 (#11408) -* Docs: Remove opentracing and zipkin from docs (#11405) -* Go: Sync modules from `main`. (#11398) -* add workflow to helm release and update ct for branch (#11317) -* Merge pull request #11277 from strongjz/chart-1.10.1 (#11314) -* Release Helm Chart on branch update (#11306) -* Release controller 1.10.1 (#11298) -* fix path in file changed detected message (#11286) -* chore: fix function names in comment (#11281) -* fix: update kube version requirement to 1.21 (#11279) -* release helm chart from release branch (#11278) -* start 1.10.1 build (#11246) -* force nginx rebuild (#11245) -* update k8s version to latest kind release (#11241) -* remove _ssl_expire_time_seconds metric by identifier (#11239) -* update post submit helm ci and clean up (#11221) -* Chart: Add unit tests for default backend & topology spread constraints. (#11219) -* sort default backend hpa metrics (#11217) -* updated certgen image shatag (#11216) -* changed testrunner image sha (#11211) -* bumped certgeimage tag (#11213) -* updated baseimage & deleted a useless file (#11209) -* bump ginkgo to 2-17-1 in testrunner (#11204) -* chunking related faq update (#11205) -* Fix-semver (#11199) -* refactor helm ci tests part I (#11188) -* Proposal: e2e tests for regex patterns (#11185) -* bump ginkgo to v2.17.1 (#11186) -* fixes brotli build issue (#11187) -* fix geoip2 configuration docs (#11151) -* Fix typos in OTel doc (#11081) (#11129) -* Chart: Render `controller.ingressClassResource.parameters` natively. (#11126) -* Fix admission controller logging of `admissionTime` and `testedConfigurationSize` (#11114) -* Chart: Align HPA & KEDA conditions. (#11113) -* Chart: Improve IngressClass documentation. (#11111) -* Chart: Add Gacko to maintainers. Again. (#11112) -* Chart: Deploy `PodDisruptionBudget` with KEDA. (#11105) -* Chores: Pick patches from main. (#11103) -* Start the release of v1.10.0 (#11038) - -### Dependency updates: - -* Bump the all group with 2 updates (#11524) -* Bump k8s.io/klog/v2 from 2.130.0 to 2.130.1 in the all group (#11521) -* Bump aquasecurity/trivy-action from 0.22.0 to 0.23.0 in the all group (#11501) -* Bump k8s.io/klog/v2 from 2.120.1 to 2.130.0 (#11479) -* Bump the all group with 3 updates (#11478) -* Bump the all group with 2 updates (#11477) -* Bump golang.org/x/crypto from 0.23.0 to 0.24.0 (#11471) -* Bump sigs.k8s.io/controller-runtime in the all group (#11449) -* Bump github.com/prometheus/common from 0.53.0 to 0.54.0 (#11447) -* Bump the all group with 3 updates (#11450) -* Bump goreleaser/goreleaser-action from 5.1.0 to 6.0.0 (#11448) -* Bump github.com/onsi/ginkgo/v2 from 2.17.2 to 2.19.0 (#11422) -* Bump the all group with 2 updates (#11421) -* Bump google.golang.org/grpc from 1.63.2 to 1.64.0 (#11423) -* Bump the all group across 1 directory with 6 updates (#11407) -* Bump golangci/golangci-lint-action from 5.3.0 to 6.0.1 (#11406) -* Bump the all group with 3 updates (#11404) -* Bump Kubernetes version on images (#11403) -* Bump golangci/golangci-lint-action from 4.0.0 to 5.0.0 (#11402) -* Bump the all group with 4 updates (#11380) -* Bump k8s.io/component-base from 0.29.3 to 0.30.0 (#11301) -* Bump github.com/prometheus/common from 0.52.3 to 0.53.0 (#11300) -* Bump golang.org/x/net from 0.22.0 to 0.23.0 (#11285) -* Bump golang.org/x/net in /images/kube-webhook-certgen/rootfs (#11284) -* Bump the all group with 2 updates (#11266) -* Bump azure/setup-helm from 3.5 to 4 (#11265) -* Bump actions/add-to-project from 1.0.0 to 1.0.1 in the all group (#11264) -* Bump google.golang.org/grpc from 1.63.0 to 1.63.2 (#11238) -* Bump google.golang.org/grpc from 1.62.1 to 1.63.0 (#11234) -* Bump github.com/prometheus/common from 0.51.1 to 0.52.2 (#11233) -* Bump golang.org/x/crypto from 0.21.0 to 0.22.0 (#11232) -* Bump github.com/prometheus/client_model in the all group (#11231) -* Bump the all group with 3 updates (#11230) -* Bump the all group with 2 updates (#11190) -* Bump actions/add-to-project from 0.6.1 to 1.0.0 (#11189) -* Bump the all group with 3 updates (#11166) -* Bump github.com/prometheus/common from 0.50.0 to 0.51.1 (#11160) -* Bump the all group with 4 updates (#11140) -* Bump the all group with 1 update (#11136) -* Bump google.golang.org/protobuf from 1.32.0 to 1.33.0 in /magefiles (#11127) -* Bump google.golang.org/protobuf in /images/custom-error-pages/rootfs (#11128) -* Bump google.golang.org/protobuf in /images/kube-webhook-certgen/rootfs (#11122) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.10.1...controller-v1.10.2 diff --git a/changelog/controller-1.10.3.md b/changelog/controller-1.10.3.md deleted file mode 100644 index 1cbc77951..000000000 --- a/changelog/controller-1.10.3.md +++ /dev/null @@ -1,37 +0,0 @@ -# Changelog - -### controller-v1.10.3 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.10.3@sha256:b5a5082f8e508cc1aac1c0ef101dc2f87b63d51598a5747d81d6cf6e7ba058fd -* registry.k8s.io/ingress-nginx/controller-chroot:v1.10.3@sha256:9033e04bd3cd01f92414f8d5999c5095734d4caceb4923942298152a38373d4b - -### All changes: - -* Images: Trigger `controller` v1.10.3 build. (#11648) -* Tests: Bump `test-runner` to v20240717-1fe74b5f. (#11646) -* Images: Re-run `test-runner` build. (#11643) -* Images: Trigger `test-runner` build. (#11639) -* Images: Bump `NGINX_BASE` to v0.0.10. (#11637) -* Images: Trigger NGINX build. (#11631) -* bump testing runner (#11626) -* remove modsecurity coreruleset test files from nginx image (#11619) -* unskip the ocsp tests and update images to fix cfssl bug (#11615) -* Fix indent in YAML for example pod (#11609) -* Images: Bump `test-runner`. (#11604) -* Images: Bump `NGINX_BASE` to v0.0.9. (#11601) -* revert module upgrade (#11595) -* README: Fix support matrix. (#11593) -* Mage: Stop mutating release notes. (#11582) -* Images: Bump `kube-webhook-certgen`. (#11583) - -### Dependency updates: - -* Bump github.com/prometheus/common from 0.54.0 to 0.55.0 (#11622) -* Bump the all group with 5 updates (#11613) -* Bump golang.org/x/crypto from 0.24.0 to 0.25.0 (#11579) -* Bump google.golang.org/grpc from 1.64.0 to 1.65.0 (#11577) -* Bump the all group with 4 updates (#11574) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.10.2...controller-v1.10.3 diff --git a/changelog/controller-1.10.4.md b/changelog/controller-1.10.4.md deleted file mode 100644 index 9a4e8025b..000000000 --- a/changelog/controller-1.10.4.md +++ /dev/null @@ -1,53 +0,0 @@ -# Changelog - -### controller-v1.10.4 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.10.4@sha256:505b9048c02dde3d6c8667bf0b52aba7b36adf7b03da34c47d5fa312d2d4c6fc -* registry.k8s.io/ingress-nginx/controller-chroot:v1.10.4@sha256:bf71acf6e71830a4470e2183e3bc93c4f006b954f8a05fb434242ef0f8a24858 - -### All changes: - -* Chart: Bump Kube Webhook CertGen & OpenTelemetry. (#11811) -* Images: Trigger controller build. (#11808) -* Tests & Docs: Bump images. (#11804) -* Images: Trigger failed builds. (#11801) -* Images: Trigger other builds. (#11797) -* Controller: Fix panic in alternative backend merging. (#11793) -* Tests: Bump `e2e-test-runner` to v20240812-3f0129aa. (#11791) -* Images: Trigger `test-runner` build. (#11786) -* Images: Bump `NGINX_BASE` to v0.0.12. (#11783) -* Images: Trigger NGINX build. (#11780) -* Cloud Build: Add missing config, remove unused ones. (#11776) -* Generate correct output on NumCPU() when using cgroups2 (#11775) -* Cloud Build: Tweak timeouts. (#11762) -* Cloud Build: Fix substitutions. (#11759) -* Cloud Build: Some chores. (#11756) -* Go: Bump to v1.22.6. (#11748) -* Images: Bump `NGINX_BASE` to v0.0.11. (#11744) -* Images: Trigger NGINX build. (#11736) -* docs: update OpenSSL Roadmap link (#11734) -* Go: Bump to v1.22.5. (#11731) -* Docs: Fix typo in AWS LB Controller reference (#11724) -* Perform some cleaning operations on line breaks. (#11722) -* Missing anchors in regular expression. (#11718) -* Docs: Fix `from-to-www` redirect description. (#11715) -* Chart: Remove `isControllerTagValid`. (#11714) -* Tests: Bump `e2e-test-runner` to v20240729-04899b27. (#11704) -* Docs: Clarify `from-to-www` redirect direction. (#11692) -* added real-client-ip faq (#11665) -* Docs: Format NGINX configuration table. (#11660) - -### Dependency updates: - -* Bump github.com/onsi/ginkgo/v2 from 2.19.1 to 2.20.0 (#11772) -* Bump the all group with 2 updates (#11770) -* Bump golang.org/x/crypto from 0.25.0 to 0.26.0 (#11768) -* Bump the all group with 3 updates (#11729) -* Bump github.com/onsi/ginkgo/v2 from 2.19.0 to 2.19.1 in the all group (#11700) -* Bump the all group with 2 updates (#11697) -* Bump the all group with 4 updates (#11676) -* Bump the all group with 2 updates (#11674) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.10.3...controller-v1.10.4 diff --git a/changelog/controller-1.10.5.md b/changelog/controller-1.10.5.md deleted file mode 100644 index 82be0a608..000000000 --- a/changelog/controller-1.10.5.md +++ /dev/null @@ -1,90 +0,0 @@ -# Changelog - -### controller-v1.10.5 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.10.5@sha256:c84d11b1f7bd14ebbf49918a7f0dc01b31c0c6e757e0129520ea93453096315c -* registry.k8s.io/ingress-nginx/controller-chroot:v1.10.5@sha256:030a43bdd5f0212a7e135cc4da76b15a6706ef65a6824eb4cc401f87a81c2987 - -### All changes: - -* Images: Trigger controller build. (#12133) -* Tests & Docs: Bump `e2e-test-echo` to v1.0.1. (#12146) -* Images: Trigger `e2e-test-echo` build. (#12142) -* Images: Drop `s390x`. (#12139) -* Images: Build `s390x` controller. (#12128) -* Chart: Bump Kube Webhook CertGen. (#12122) -* Tests & Docs: Bump images. (#12120) -* Cloud Build: Bump `gcb-docker-gcloud` to v20240718-5ef92b5c36. (#12116) -* Images: Trigger other builds. (#12111) -* Tests: Bump `e2e-test-runner` to v20241004-114a6abb. (#12104) -* Images: Trigger `test-runner` build. (#12101) -* Docs: Add a multi-tenant warning. (#12098) -* Go: Bump to v1.22.8. (#12093) -* Images: Bump `NGINX_BASE` to v0.1.0. (#12079) -* Images: Trigger NGINX build. (#12077) -* Images: Remove NGINX v1.21. (#12057) -* GitHub: Improve Dependabot. (#12037) -* Chart: Improve CI. (#12029) -* Chart: Extend image tests. (#12026) -* Docs: Add health check annotations for AWS. (#12021) -* Docs: Convert `opentelemetry.md` from CRLF to LF. (#12007) -* Chart: Test `controller.minAvailable` & `controller.maxUnavailable`. (#12001) -* Chart: Align default backend `PodDisruptionBudget`. (#11998) -* Metrics: Fix namespace in `nginx_ingress_controller_ssl_expire_time_seconds`. (#11985) -* Chart: Improve default backend service account. (#11973) -* Go: Bump to v1.22.7. (#11969) -* Images: Bump OpenTelemetry C++ Contrib. (#11950) -* Docs: Add note about `--watch-namespace`. (#11948) -* Images: Use latest Alpine 3.20 everywhere. (#11945) -* Fix minor typos (#11940) -* Chart: Implement `controller.admissionWebhooks.service.servicePort`. (#11933) -* Tests: Bump `e2e-test-runner` to v20240829-2c421762. (#11920) -* Images: Trigger `test-runner` build. (#11918) -* Chart: Add tests for `PrometheusRule` & `ServiceMonitor`. (#11888) -* Annotations: Allow commas in URLs. (#11886) -* CI: Grant checks write permissions to E2E Test Report. (#11884) -* Update maxmind post link about geolite2 license changes (#11880) -* Go: Sync `go.work.sum`. (#11876) -* Replace deprecated queue method (#11858) -* Auto-generate annotation docs (#11835) - -### Dependency updates: - -* Bump the actions group with 3 updates (#12150) -* Bump golang.org/x/crypto from 0.27.0 to 0.28.0 (#12108) -* Bump the actions group with 3 updates (#12096) -* Bump sigs.k8s.io/mdtoc from 1.1.0 to 1.4.0 (#12088) -* Bump github.com/prometheus/common from 0.59.1 to 0.60.0 (#12086) -* Bump google.golang.org/grpc from 1.67.0 to 1.67.1 in the go group across 1 directory (#12084) -* Bump k8s.io/cli-runtime from 0.30.0 to 0.31.1 (#12082) -* Bump github/codeql-action from 3.26.9 to 3.26.10 in the actions group (#12054) -* Bump the go group across 1 directory with 3 updates (#12052) -* Bump k8s.io/kube-aggregator from 0.29.3 to 0.31.1 in /images/kube-webhook-certgen/rootfs (#12048) -* Bump k8s.io/apimachinery from 0.23.1 to 0.31.1 in /images/ext-auth-example-authsvc/rootfs (#12044) -* Bump github.com/prometheus/client_golang from 1.11.1 to 1.20.4 in /images/custom-error-pages/rootfs (#12045) -* Bump the all group with 2 updates (#12035) -* Bump github/codeql-action from 3.26.7 to 3.26.8 in the all group (#12015) -* Bump google.golang.org/grpc from 1.66.2 to 1.67.0 (#12013) -* Bump github.com/prometheus/client_golang from 1.20.3 to 1.20.4 in the all group (#12011) -* Bump the all group with 2 updates (#11979) -* Bump github/codeql-action from 3.26.6 to 3.26.7 in the all group (#11978) -* Bump github.com/prometheus/common from 0.57.0 to 0.59.1 (#11960) -* Bump golang.org/x/crypto from 0.26.0 to 0.27.0 (#11959) -* Bump github.com/prometheus/client_golang from 1.20.2 to 1.20.3 in the all group (#11956) -* Bump github.com/opencontainers/runc from 1.1.13 to 1.1.14 (#11929) -* Bump the all group with 2 updates (#11924) -* Bump github.com/onsi/ginkgo/v2 from 2.20.1 to 2.20.2 in the all group (#11912) -* Bump google.golang.org/grpc from 1.65.0 to 1.66.0 (#11907) -* Bump github.com/prometheus/common from 0.55.0 to 0.57.0 (#11906) -* Bump github/codeql-action from 3.26.5 to 3.26.6 in the all group (#11905) -* Bump the all group with 2 updates (#11870) -* Bump github/codeql-action from 3.26.2 to 3.26.5 in the all group (#11869) -* Bump github.com/prometheus/client_golang from 1.19.1 to 1.20.1 (#11848) -* Bump sigs.k8s.io/controller-runtime from 0.18.4 to 0.19.0 (#11847) -* Bump dario.cat/mergo from 1.0.0 to 1.0.1 in the all group (#11846) -* Bump k8s.io/component-base from 0.30.3 to 0.31.0 (#11841) -* Bump github/codeql-action from 3.26.0 to 3.26.2 in the all group (#11833) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.10.4...controller-v1.10.5 diff --git a/changelog/controller-1.10.6.md b/changelog/controller-1.10.6.md deleted file mode 100644 index 695e418d8..000000000 --- a/changelog/controller-1.10.6.md +++ /dev/null @@ -1,92 +0,0 @@ -# Changelog - -### controller-v1.10.6 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.10.6@sha256:b6fbd102255edb3ba8e5421feebe14fd3e94cf53d199af9e40687f536152189c -* registry.k8s.io/ingress-nginx/controller-chroot:v1.10.6@sha256:44ceedafc0e04a75521b5d472c1b6b5cc08afb8038b5bbfd79c21d066ccf300e - -### All changes: - -* Images: Trigger controller build. (#12611) -* Chart: Bump Kube Webhook CertGen. (#12608) -* Tests & Docs: Bump images. (#12605) -* Images: Trigger other builds (2/2). (#12598) -* Images: Trigger other builds (1/2). (#12597) -* Tests: Bump `e2e-test-runner` to v20241224-68ed4e7b. (#12592) -* Images: Trigger `test-runner` build. (#12586) -* Images: Bump `NGINX_BASE` to v0.2.0. (#12584) -* Images: Trigger NGINX build. (#12578) -* Go: Clean `go.work.sum`. (#12575) -* Repository: Update owners. (#12570) -* Images: Bump `gcb-docker-gcloud` to v20241217-ff46a068cd. (#12563) -* CI: Update KIND images. (#12559) -* Images: Bump Alpine to v3.21. (#12530) -* Docs: Add guide on how to set a Maintenance Page. (#12527) -* rikatz is stepping down (#12518) -* rikatz is stepping down (#12497) -* Go: Bump to v1.23.4. (#12485) -* Plugin: Bump `goreleaser` to v2. (#12442) -* GitHub: Fix `exec` in issue template. (#12389) -* CI: Update KIND images. (#12368) -* Images: Bump `gcb-docker-gcloud` to v20241110-72bb0b1665. (#12341) -* Go: Bump to v1.23.3. (#12339) -* Auth TLS: Add `_` to redirect RegEx. (#12328) -* Auth TLS: Improve redirect RegEx. (#12321) -* Tests: Bump `e2e-test-runner` to v20241104-02a3933e. (#12314) -* Images: Trigger `test-runner` build. (#12307) -* Config: Fix panic on invalid `lua-shared-dict`. (#12282) -* Docs: fix limit-rate-after references (#12280) -* Chart: Rework ServiceMonitor. (#12268) -* Chart: Add ServiceAccount tests. (#12266) -* CI: Fix chart testing. (#12260) -* [fix] fix nginx temp configs cleanup (#12224) -* Chart: Suggest `matchLabelKeys` in Topology Spread Constraints. (#12204) -* Docs: Add Pod Security Admission. (#12198) -* Docs: Clarify external & service port in TCP/UDP services explanation. (#12194) - -### Dependency updates: - -* Bump k8s.io/apiextensions-apiserver from 0.31.3 to 0.32.0 (#12565) -* Bump github.com/onsi/ginkgo/v2 from 2.22.0 to 2.22.1 (#12557) -* Bump k8s.io/code-generator from 0.31.3 to 0.32.0 (#12552) -* Bump k8s.io/cli-runtime from 0.31.3 to 0.32.0 (#12549) -* Bump k8s.io/apiserver from 0.31.3 to 0.32.0 (#12546) -* Bump the actions group with 2 updates (#12543) -* Bump google.golang.org/grpc from 1.68.1 to 1.69.2 (#12540) -* Bump k8s.io/client-go from 0.31.3 to 0.32.0 (#12514) -* Bump github.com/opencontainers/runc from 1.2.2 to 1.2.3 in the go group across 1 directory (#12511) -* Bump the actions group with 3 updates (#12508) -* Bump k8s.io/kube-aggregator from 0.31.3 to 0.32.0 in /images/kube-webhook-certgen/rootfs (#12504) -* Bump k8s.io/apimachinery from 0.31.3 to 0.32.0 in /images/ext-auth-example-authsvc/rootfs (#12501) -* Bump golang.org/x/crypto from 0.30.0 to 0.31.0 (#12478) -* Bump golang.org/x/crypto from 0.21.0 to 0.31.0 in /magefiles (#12473) -* Bump github.com/prometheus/common from 0.60.1 to 0.61.0 (#12466) -* Bump github/codeql-action from 3.27.5 to 3.27.6 in the actions group (#12463) -* Bump the go group across 1 directory with 2 updates (#12459) -* Bump github.com/onsi/ginkgo/v2 from 2.21.0 to 2.22.0 (#12425) -* Bump github.com/stretchr/testify from 1.9.0 to 1.10.0 (#12416) -* Bump the go group across 3 directories with 10 updates (#12414) -* Bump the actions group with 3 updates (#12410) -* Bump github.com/opencontainers/runc from 1.2.1 to 1.2.2 in the go group across 1 directory (#12382) -* Bump github/codeql-action from 3.27.1 to 3.27.4 in the actions group (#12375) -* Bump golangci-lint on actions and disable deprecated linters (#12363) -* Bump google.golang.org/grpc from 1.67.1 to 1.68.0 (#12356) -* Bump the actions group with 3 updates (#12353) -* Bump golang.org/x/crypto from 0.28.0 to 0.29.0 (#12351) -* Bump github.com/fsnotify/fsnotify from 1.7.0 to 1.8.0 (#12297) -* Bump github.com/opencontainers/runc from 1.2.0 to 1.2.1 in the go group across 1 directory (#12294) -* Bump github.com/onsi/ginkgo/v2 from 2.20.2 to 2.21.0 (#12290) -* Bump actions/dependency-review-action from 4.3.5 to 4.4.0 in the actions group (#12275) -* Bump the go group across 3 directories with 11 updates (#12246) -* Bump github.com/opencontainers/runc from 1.1.15 to 1.2.0 (#12241) -* Bump the actions group with 5 updates (#12243) -* Bump github.com/ncabatoff/process-exporter from 0.8.3 to 0.8.4 in the go group across 1 directory (#12219) -* Bump aquasecurity/trivy-action from 0.27.0 to 0.28.0 in the actions group (#12215) -* Bump github/codeql-action from 3.26.12 to 3.26.13 in the actions group (#12191) -* Bump the go group across 2 directories with 1 update (#12189) -* Bump the actions group with 2 updates (#12185) -* Bump github.com/opencontainers/runc from 1.1.14 to 1.1.15 in the go group across 1 directory (#12184) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.10.5...controller-v1.10.6 diff --git a/changelog/controller-1.11.0.md b/changelog/controller-1.11.0.md deleted file mode 100644 index d462f9574..000000000 --- a/changelog/controller-1.11.0.md +++ /dev/null @@ -1,164 +0,0 @@ -# Changelog - -### controller-v1.11.0 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.11.0@sha256:a886e56d532d1388c77c8340261149d974370edca1093af4c97a96fb1467cb39 -* registry.k8s.io/ingress-nginx/controller-chroot:v1.11.0@sha256:f16dfed1c94d216b65e5dcb7508ab46148641a99649c5a700749db6f01a7039e - -### All changes: - -* update test runner to latest build (#11558) -* add k8s 1.30 to ci build (#11554) -* update test runner go base to 3.20 (#11552) -* tag new test runner image with new nginx base 0.0.8 (#11551) -* bump NGINX_BASE to v0.0.8 (#11544) -* add ssl patches to nginx-1.25 image for coroutines to work in lua client hello and cert ssl blocks (#11535) -* trigger build for NGINX-1.25 v0.0.8 (#11539) -* bump alpine version to 3.20 to custom-error-pages (#11538) -* fix: Ensure changes in MatchCN annotation are detected (#11529) -* Docs: Add information about HTTP/3 support. (#11513) -* Docs: Specify `ingressClass` for multi-controller setup. (#11493) -* Docs: Improve default certificate usage. (#11504) -* Upgrade OWASP_MODSECURITY_CRS_VERSION 3.3.5 to 4.4.0 and update docs (#11511) -* docs: Update Ingress-NGINX v1.10.1 compatibility with Kubernetes v1.30 (#11495) -* Update getting-started.md with new prerequisites (#11486) -* [feature] bump nginx to 1.25.5 and add http3 module (#11470) -* Fix boolean configuration (#11483) -* Chores: Align security contacts & chart maintainers to actual owners. (#11465) -* CI: Bump forgotten Ginkgo versions. (#11467) -* Tests: Replace deprecated `grpc.Dial` by `grpc.NewClient`. (#11462) -* Owners: Promote Gacko to admin. (#11463) -* Chart: Make pod affinity templatable. (#11453) -* fixed fastcgi userguide (#11454) -* Remove unnecessary space character (#11434) -* upgrade to alpine 3.20 (#11428) -* fix for docs issue 11432 (#11433) -* Update index.md (#11437) -* update golang to 1.22.4 (#11427) -* Chart: Fix `IngressClass` annotations. (#11416) -* Chart: Make admission webhook patch job RBAC configurable. (#11376) -* Merge pull request #11277 from strongjz/chart-1.10.1 (#11415) -* Chart: Remove `controller.enableWorkerSerialReloads`. (#11400) -* Adapt dashboards for Grafana 11 compatibility (#11399) -* Rename variable to fix typo (#11395) -* Fix helm install on cloud provider admonition block (#11394) -* edited helm-install tips (#11393) -* added info for aws helm install (#11390) -* added multiplecontrollers-howto to faq (#11389) -* removed tlsv1 & tlsv1.1 (#11343) -* feat: Add grpc timeouts annotations (#11258) -* sfix position of options (#11379) -* add workflow to helm release and update ct for branch (#11378) -* Accept user defined annotations in IngressClass (#11362) -* Docs: Remove opentracing and zipkin from docs (#11361) -* Allow configuring nginx worker reload behaviour, to prevent multiple concurrent worker reloads which can lead to high resource usage and OOMKill (#10884) -* chore(deps): group update k8s.io packages to v0.30.0 (#11344) -* Fix function name in comment (#11296) -* fix path in file changed detected message (#11271) -* chore: fix function names in comment (#11280) -* fix: update kube version requirement to 1.21 (#11275) -* release helm chart from release branch (#11276) -* update k8s version to latest kind release (#11240) -* feat: add annotation to allow to add custom response headers (#9742) -* remove _ssl_expire_time_seconds metric by identifier (#9706) -* update post submit helm ci and clean up (#11220) -* Chart: Add unit tests for default backend & topology spread constraints. (#11218) -* sort default backend hpa metrics (#11215) -* updated certgen image shatag (#11214) -* feature(default_backend): topologySpreadConstraints on default backend (#11197) -* bumped certgeimage tag (#11212) -* changed testrunner image sha (#11207) -* updated baseimage & deleted a useless file (#11208) -* Chart: Make `controller.config` templatable. (#11181) -* chunking related faq update (#11196) -* bump ginkgo to 2-17-1 in testrunner (#11202) -* Owners: Promote Gacko to `ingress-nginx-maintainers` & `ingress-nginx-reviewers`. (#11165) -* Fix-semver (#11193) -* refactor helm ci tests part I (#11178) -* fixes brotli build issue (#10484) -* bump ginkgo to v2.17.1 (#11177) -* Proposal: e2e tests for regex patterns (#11174) -* Controller: Make Leader Election TTL configurable. (#11142) -* Chores: Remove recently added whitespaces. (#11156) -* Add GRPC Buffer Size to the Configmap (#11155) -* fix geoip2 configuration docs (#11150) -* feature(geoip2_autoreload): Enable GeoIP2 auto_reload config (#11079) -* Chart: Add IngressClass aliases. (#11109) -* Fix typos in OTel doc (#11081) -* Chart: Render `controller.ingressClassResource.parameters` natively. (#11108) -* Fix admission controller logging of `admissionTime` and `testedConfigurationSize` (#11089) -* Chart: Align HPA & KEDA conditions. (#11110) -* Chart: Add Gacko to maintainers. Again. (#11107) -* Chart: Improve IngressClass documentation. (#11104) -* Chart: Deploy `PodDisruptionBudget` with KEDA. (#11032) -* Undo #11062 since it breaks the nginx config (#11082) -* [mTLS] Fix acme verification when mTLS and Client CN verification is enabled (#11062) -* golangci-lint update, ci cleanup, group dependabot updates (#11071) -* bump golang (#11070) -* feature(leader_election): flag to disable leader election feature on controller (#11064) -* docs: update the 404 link to FAQ (#11069) -* Update README.md (#11065) -* quotes around numbers fort ports definitions (#11052) - -### Dependency updates: - -* Bump the all group with 2 updates (#11523) -* Bump k8s.io/klog/v2 from 2.130.0 to 2.130.1 in the all group (#11499) -* Bump aquasecurity/trivy-action from 0.22.0 to 0.23.0 in the all group (#11497) -* Bump k8s.io/klog/v2 from 2.120.1 to 2.130.0 (#11475) -* Bump the all group with 3 updates (#11474) -* Bump the all group with 2 updates (#11476) -* Bump golang.org/x/crypto from 0.23.0 to 0.24.0 (#11442) -* Bump the all group with 3 updates (#11443) -* Bump sigs.k8s.io/controller-runtime in the all group (#11440) -* Bump goreleaser/goreleaser-action from 5.1.0 to 6.0.0 (#11444) -* Bump github.com/prometheus/common from 0.53.0 to 0.54.0 (#11441) -* Bump the all group with 2 updates (#11419) -* Bump github.com/onsi/ginkgo/v2 from 2.17.2 to 2.19.0 (#11418) -* Bump google.golang.org/grpc from 1.63.2 to 1.64.0 (#11417) -* Bump the all group across 1 directory with 3 updates (#11384) -* Bump the all group across 1 directory with 6 updates (#11383) -* Bump golang.org/x/crypto from 0.22.0 to 0.23.0 (#11357) -* Bump golangci/golangci-lint-action from 5.3.0 to 6.0.1 (#11355) -* Bump the all group with 3 updates (#11348) -* Bump Kubernetes version on images (#11346) -* Bump sigs.k8s.io/controller-runtime from 0.17.3 to 0.18.1 (#11345) -* Bump golangci/golangci-lint-action from 4.0.0 to 5.0.0 (#11328) -* Bump the all group with 4 updates (#11327) -* Bump k8s.io/component-base from 0.29.3 to 0.30.0 (#11291) -* Bump github.com/prometheus/common from 0.52.3 to 0.53.0 (#11290) -* Bump golang.org/x/net from 0.22.0 to 0.23.0 (#11282) -* Bump golang.org/x/net in /images/kube-webhook-certgen/rootfs (#11283) -* Bump the all group with 2 updates (#11261) -* Bump azure/setup-helm from 3.5 to 4 (#11263) -* Bump actions/add-to-project from 1.0.0 to 1.0.1 in the all group (#11262) -* Bump google.golang.org/grpc from 1.63.0 to 1.63.2 (#11237) -* Bump google.golang.org/grpc from 1.62.1 to 1.63.0 (#11228) -* Bump github.com/prometheus/common from 0.51.1 to 0.52.2 (#11227) -* Bump golang.org/x/crypto from 0.21.0 to 0.22.0 (#11229) -* Bump github.com/prometheus/client_model in the all group (#11226) -* Bump the all group with 3 updates (#11225) -* Bump the all group with 2 updates (#11183) -* Bump actions/add-to-project from 0.6.1 to 1.0.0 (#11184) -* Bump the all group with 3 updates (#11157) -* Bump github.com/prometheus/common from 0.50.0 to 0.51.1 (#11159) -* Bump the all group with 4 updates (#11133) -* Bump the all group with 1 update (#11134) -* Bump google.golang.org/protobuf in /images/custom-error-pages/rootfs (#11119) -* Bump google.golang.org/protobuf from 1.32.0 to 1.33.0 in /magefiles (#11121) -* Bump google.golang.org/protobuf in /images/kube-webhook-certgen/rootfs (#11120) -* Bump github.com/onsi/ginkgo/v2 from 2.15.0 to 2.16.0 (#11076) -* Bump the all group with 1 update (#11073) -* Bump the all group with 1 update (#11072) -* Bump github.com/prometheus/common from 0.49.0 to 0.50.0 (#11075) -* Bump actions/download-artifact from 4.1.2 to 4.1.4 (#11059) -* Bump github.com/stretchr/testify from 1.8.4 to 1.9.0 (#11055) -* Bump docker/setup-buildx-action from 3.0.0 to 3.1.0 (#11057) -* Bump github.com/prometheus/common from 0.48.0 to 0.49.0 (#11056) -* Bump github/codeql-action from 3.24.5 to 3.24.6 (#11060) -* Bump aquasecurity/trivy-action from 0.17.0 to 0.18.0 (#11058) -* Bump dorny/paths-filter from 3.0.1 to 3.0.2 (#11061) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.10.2...controller-v1.11.0 diff --git a/changelog/controller-1.11.1.md b/changelog/controller-1.11.1.md deleted file mode 100644 index a93e02e40..000000000 --- a/changelog/controller-1.11.1.md +++ /dev/null @@ -1,45 +0,0 @@ -# Changelog - -### controller-v1.11.1 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.11.1@sha256:e6439a12b52076965928e83b7b56aae6731231677b01e81818bce7fa5c60161a -* registry.k8s.io/ingress-nginx/controller-chroot:v1.11.1@sha256:7cabe4bd7558bfdf5b707976d7be56fd15ffece735d7c90fc238b6eda290fd8d - -### All changes: - -* Tests: Bump `test-runner` to v20240717-1fe74b5f. (#11647) -* Images: Re-run `test-runner` build. (#11644) -* Images: Trigger `test-runner` build. (#11640) -* Images: Bump `NGINX_BASE` to v0.0.10. (#11638) -* Images: Trigger NGINX build. (#11632) -* bump testing runner (#11627) -* remove modsecurity coreruleset test files from nginx image (#11620) -* unskip the ocsp tests and update images to fix cfssl bug (#11616) -* Fix indent in YAML for example pod (#11610) -* Images: Bump `test-runner`. (#11605) -* Images: Bump `NGINX_BASE` to v0.0.9. (#11602) -* revert module upgrade (#11597) -* Release: Apply changes from `main`. (#11589) -* Mage: Stop mutating release notes. (#11581) -* Images: Bump `kube-webhook-certgen`. (#11584) -* update test runner to latest build (#11558) -* add k8s 1.30 to ci build (#11554) -* update test runner go base to 3.20 (#11552) -* tag new test runner image with new nginx base 0.0.8 (#11551) -* bump NGINX_BASE to v0.0.8 (#11544) -* add ssl patches to nginx-1.25 image for coroutines to work in lua client hello and cert ssl blocks (#11535) -* trigger build for NGINX-1.25 v0.0.8 (#11539) -* bump alpine version to 3.20 to custom-error-pages (#11538) -* fix: Ensure changes in MatchCN annotation are detected (#11529) - -### Dependency updates: - -* Bump github.com/prometheus/common from 0.54.0 to 0.55.0 (#11621) -* Bump the all group with 5 updates (#11614) -* Bump golang.org/x/crypto from 0.24.0 to 0.25.0 (#11580) -* Bump google.golang.org/grpc from 1.64.0 to 1.65.0 (#11576) -* Bump the all group with 4 updates (#11575) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.11.0...controller-v1.11.1 diff --git a/changelog/controller-1.11.2.md b/changelog/controller-1.11.2.md deleted file mode 100644 index 0979af15d..000000000 --- a/changelog/controller-1.11.2.md +++ /dev/null @@ -1,54 +0,0 @@ -# Changelog - -### controller-v1.11.2 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.11.2@sha256:d5f8217feeac4887cb1ed21f27c2674e58be06bd8f5184cacea2a69abaf78dce -* registry.k8s.io/ingress-nginx/controller-chroot:v1.11.2@sha256:21b55a2f0213a18b91612a8c0850167e00a8e34391fd595139a708f9c047e7a8 - -### All changes: - -* Chart: Bump Kube Webhook CertGen & OpenTelemetry. (#11812) -* Images: Trigger controller build. (#11807) -* Tests & Docs: Bump images. (#11805) -* Images: Trigger failed builds. (#11802) -* Images: Trigger other builds. (#11798) -* Controller: Fix panic in alternative backend merging. (#11794) -* Tests: Bump `e2e-test-runner` to v20240812-3f0129aa. (#11792) -* Images: Trigger `test-runner` build. (#11787) -* Images: Bump `NGINX_BASE` to v0.0.12. (#11784) -* Images: Trigger NGINX build. (#11781) -* Cloud Build: Add missing config, remove unused ones. (#11777) -* Generate correct output on NumCPU() when using cgroups2 (#11778) -* Cloud Build: Tweak timeouts. (#11763) -* Cloud Build: Fix substitutions. (#11760) -* Cloud Build: Some chores. (#11757) -* Go: Bump to v1.22.6. (#11749) -* Images: Bump `NGINX_BASE` to v0.0.11. (#11743) -* Images: Trigger NGINX build. (#11737) -* docs: update OpenSSL Roadmap link (#11733) -* Go: Bump to v1.22.5. (#11732) -* Docs: Fix typo in AWS LB Controller reference (#11725) -* Perform some cleaning operations on line breaks. (#11721) -* Missing anchors in regular expression. (#11719) -* Docs: Fix `from-to-www` redirect description. (#11716) -* Chart: Remove `isControllerTagValid`. (#11713) -* Tests: Bump `e2e-test-runner` to v20240729-04899b27. (#11705) -* Docs: Clarify `from-to-www` redirect direction. (#11693) -* added real-client-ip faq (#11664) -* Docs: Format NGINX configuration table. (#11662) -* Docs: Update version in `deploy/index.md`. (#11652) - -### Dependency updates: - -* Bump github.com/onsi/ginkgo/v2 from 2.19.1 to 2.20.0 (#11773) -* Bump the all group with 2 updates (#11771) -* Bump golang.org/x/crypto from 0.25.0 to 0.26.0 (#11769) -* Bump the all group with 3 updates (#11728) -* Bump github.com/onsi/ginkgo/v2 from 2.19.0 to 2.19.1 in the all group (#11701) -* Bump the all group with 2 updates (#11698) -* Bump the all group with 4 updates (#11677) -* Bump the all group with 2 updates (#11675) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.11.1...controller-v1.11.2 diff --git a/changelog/controller-1.11.3.md b/changelog/controller-1.11.3.md deleted file mode 100644 index f5c373015..000000000 --- a/changelog/controller-1.11.3.md +++ /dev/null @@ -1,91 +0,0 @@ -# Changelog - -### controller-v1.11.3 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.11.3@sha256:d56f135b6462cfc476447cfe564b83a45e8bb7da2774963b00d12161112270b7 -* registry.k8s.io/ingress-nginx/controller-chroot:v1.11.3@sha256:22701f0fc0f2dd209ef782f4e281bfe2d8cccd50ededa00aec88e0cdbe7edd14 - -### All changes: - -* Images: Trigger controller build. (#12134) -* Tests & Docs: Bump `e2e-test-echo` to v1.0.1. (#12145) -* Images: Trigger `e2e-test-echo` build. (#12141) -* Images: Drop `s390x`. (#12138) -* Images: Build `s390x` controller. (#12127) -* Chart: Bump Kube Webhook CertGen. (#12123) -* Tests & Docs: Bump images. (#12121) -* Cloud Build: Bump `gcb-docker-gcloud` to v20240718-5ef92b5c36. (#12117) -* Images: Trigger other builds. (#12112) -* Tests: Bump `e2e-test-runner` to v20241004-114a6abb. (#12105) -* Images: Trigger `test-runner` build. (#12102) -* Docs: Add a multi-tenant warning. (#12099) -* Go: Bump to v1.22.8. (#12094) -* Images: Bump `NGINX_BASE` to v0.1.0. (#12080) -* Images: Trigger NGINX build. (#12076) -* Images: Remove NGINX v1.21. (#12058) -* GitHub: Improve Dependabot. (#12038) -* Chart: Improve CI. (#12030) -* Chart: Extend image tests. (#12027) -* Docs: Add health check annotations for AWS. (#12020) -* Docs: Convert `opentelemetry.md` from CRLF to LF. (#12006) -* Chart: Test `controller.minAvailable` & `controller.maxUnavailable`. (#12002) -* Chart: Align default backend `PodDisruptionBudget`. (#11999) -* Metrics: Fix namespace in `nginx_ingress_controller_ssl_expire_time_seconds`. (#11986) -* Chart: Improve default backend service account. (#11974) -* Go: Bump to v1.22.7. (#11970) -* Images: Bump OpenTelemetry C++ Contrib. (#11951) -* Docs: Add note about `--watch-namespace`. (#11949) -* Images: Use latest Alpine 3.20 everywhere. (#11946) -* Fix minor typos (#11941) -* Chart: Implement `controller.admissionWebhooks.service.servicePort`. (#11934) -* Tests: Bump `e2e-test-runner` to v20240829-2c421762. (#11921) -* Images: Trigger `test-runner` build. (#11917) -* Chart: Add tests for `PrometheusRule` & `ServiceMonitor`. (#11889) -* Annotations: Allow commas in URLs. (#11887) -* CI: Grant checks write permissions to E2E Test Report. (#11885) -* Chart: Use generic values for `ConfigMap` test. (#11879) -* Update maxmind post link about geolite2 license changes (#11881) -* Go: Sync `go.work.sum`. (#11875) -* Replace deprecated queue method (#11859) -* Auto-generate annotation docs (#11831) - -### Dependency updates: - -* Bump the actions group with 3 updates (#12149) -* Bump golang.org/x/crypto from 0.27.0 to 0.28.0 (#12109) -* Bump the actions group with 3 updates (#12097) -* Bump sigs.k8s.io/mdtoc from 1.1.0 to 1.4.0 (#12089) -* Bump github.com/prometheus/common from 0.59.1 to 0.60.0 (#12087) -* Bump google.golang.org/grpc from 1.67.0 to 1.67.1 in the go group across 1 directory (#12085) -* Bump k8s.io/cli-runtime from 0.30.0 to 0.31.1 (#12083) -* Bump github/codeql-action from 3.26.9 to 3.26.10 in the actions group (#12055) -* Bump the go group across 1 directory with 3 updates (#12053) -* Bump k8s.io/kube-aggregator from 0.29.3 to 0.31.1 in /images/kube-webhook-certgen/rootfs (#12049) -* Bump k8s.io/apimachinery from 0.23.1 to 0.31.1 in /images/ext-auth-example-authsvc/rootfs (#12047) -* Bump github.com/prometheus/client_golang from 1.11.1 to 1.20.4 in /images/custom-error-pages/rootfs (#12046) -* Bump the all group with 2 updates (#12036) -* Bump github/codeql-action from 3.26.7 to 3.26.8 in the all group (#12016) -* Bump google.golang.org/grpc from 1.66.2 to 1.67.0 (#12014) -* Bump github.com/prometheus/client_golang from 1.20.3 to 1.20.4 in the all group (#12012) -* Bump the all group with 2 updates (#11981) -* Bump github/codeql-action from 3.26.6 to 3.26.7 in the all group (#11980) -* Bump github.com/prometheus/common from 0.57.0 to 0.59.1 (#11961) -* Bump golang.org/x/crypto from 0.26.0 to 0.27.0 (#11958) -* Bump github.com/prometheus/client_golang from 1.20.2 to 1.20.3 in the all group (#11957) -* Bump github.com/opencontainers/runc from 1.1.13 to 1.1.14 (#11930) -* Bump the all group with 2 updates (#11925) -* Bump github.com/onsi/ginkgo/v2 from 2.20.1 to 2.20.2 in the all group (#11913) -* Bump google.golang.org/grpc from 1.65.0 to 1.66.0 (#11910) -* Bump github.com/prometheus/common from 0.55.0 to 0.57.0 (#11909) -* Bump github/codeql-action from 3.26.5 to 3.26.6 in the all group (#11908) -* Bump the all group with 2 updates (#11871) -* Bump github/codeql-action from 3.26.2 to 3.26.5 in the all group (#11868) -* Bump github.com/prometheus/client_golang from 1.19.1 to 1.20.1 (#11840) -* Bump sigs.k8s.io/controller-runtime from 0.18.4 to 0.19.0 (#11839) -* Bump dario.cat/mergo from 1.0.0 to 1.0.1 in the all group (#11837) -* Bump k8s.io/component-base from 0.30.3 to 0.31.0 (#11836) -* Bump github/codeql-action from 3.26.0 to 3.26.2 in the all group (#11834) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.11.2...controller-v1.11.3 diff --git a/changelog/controller-1.11.4.md b/changelog/controller-1.11.4.md deleted file mode 100644 index a0870d060..000000000 --- a/changelog/controller-1.11.4.md +++ /dev/null @@ -1,94 +0,0 @@ -# Changelog - -### controller-v1.11.4 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.11.4@sha256:981a97d78bee3109c0b149946c07989f8f1478a9265031d2d23dea839ba05b52 -* registry.k8s.io/ingress-nginx/controller-chroot:v1.11.4@sha256:f29d0f9e7a9ef4947eda59ed0c09ec13380b13639d1518cf1ab8ec09c3e22ef8 - -### All changes: - -* Images: Trigger controller build. (#12610) -* Chart: Bump Kube Webhook CertGen. (#12607) -* Tests & Docs: Bump images. (#12604) -* Images: Trigger other builds (2/2). (#12600) -* Images: Trigger other builds (1/2). (#12596) -* Tests: Bump `e2e-test-runner` to v20241224-68ed4e7b. (#12591) -* Images: Trigger `test-runner` build. (#12588) -* Images: Bump `NGINX_BASE` to v0.2.0. (#12583) -* Images: Trigger NGINX build. (#12577) -* Go: Clean `go.work.sum`. (#12574) -* Repository: Update owners. (#12569) -* Images: Bump `gcb-docker-gcloud` to v20241217-ff46a068cd. (#12562) -* CI: Update KIND images. (#12558) -* Images: Bump Alpine to v3.21. (#12529) -* Docs: Add guide on how to set a Maintenance Page. (#12526) -* rikatz is stepping down (#12517) -* rikatz is stepping down (#12495) -* Go: Bump to v1.23.4. (#12484) -* Plugin: Bump `goreleaser` to v2. (#12441) -* GitHub: Fix `exec` in issue template. (#12388) -* CI: Update KIND images. (#12365) -* Images: Bump `gcb-docker-gcloud` to v20241110-72bb0b1665. (#12343) -* Go: Bump to v1.23.3. (#12338) -* Auth TLS: Add `_` to redirect RegEx. (#12327) -* Auth TLS: Improve redirect RegEx. (#12322) -* Update custom headers annotation documentation (#12319) -* Tests: Bump `e2e-test-runner` to v20241104-02a3933e. (#12313) -* Images: Trigger `test-runner` build. (#12306) -* Config: Fix panic on invalid `lua-shared-dict`. (#12284) -* Docs: fix limit-rate-after references (#12279) -* Chart: Rework ServiceMonitor. (#12270) -* Chart: Add ServiceAccount tests. (#12264) -* CI: Fix chart testing. (#12259) -* [fix] fix nginx temp configs cleanup (#12223) -* Chart: Suggest `matchLabelKeys` in Topology Spread Constraints. (#12203) -* Docs: Add Pod Security Admission. (#12197) -* Docs: Clarify external & service port in TCP/UDP services explanation. (#12193) -* Docs: Goodbye, v1.10. (#12159) - -### Dependency updates: - -* Bump k8s.io/apiextensions-apiserver from 0.31.3 to 0.32.0 (#12567) -* Bump github.com/onsi/ginkgo/v2 from 2.22.0 to 2.22.1 (#12556) -* Bump k8s.io/code-generator from 0.31.3 to 0.32.0 (#12551) -* Bump k8s.io/cli-runtime from 0.31.3 to 0.32.0 (#12548) -* Bump k8s.io/apiserver from 0.31.3 to 0.32.0 (#12545) -* Bump the actions group with 2 updates (#12542) -* Bump google.golang.org/grpc from 1.68.1 to 1.69.2 (#12539) -* Bump k8s.io/client-go from 0.31.3 to 0.32.0 (#12513) -* Bump github.com/opencontainers/runc from 1.2.2 to 1.2.3 in the go group across 1 directory (#12510) -* Bump the actions group with 3 updates (#12507) -* Bump k8s.io/kube-aggregator from 0.31.3 to 0.32.0 in /images/kube-webhook-certgen/rootfs (#12503) -* Bump k8s.io/apimachinery from 0.31.3 to 0.32.0 in /images/ext-auth-example-authsvc/rootfs (#12500) -* Bump golang.org/x/crypto from 0.30.0 to 0.31.0 (#12477) -* Bump golang.org/x/crypto from 0.21.0 to 0.31.0 in /magefiles (#12475) -* Bump github.com/prometheus/common from 0.60.1 to 0.61.0 (#12465) -* Bump github/codeql-action from 3.27.5 to 3.27.6 in the actions group (#12462) -* Bump the go group across 1 directory with 2 updates (#12458) -* Bump github.com/onsi/ginkgo/v2 from 2.21.0 to 2.22.0 (#12427) -* Bump github.com/stretchr/testify from 1.9.0 to 1.10.0 (#12417) -* Bump the go group across 3 directories with 10 updates (#12415) -* Bump the actions group with 3 updates (#12411) -* Bump github.com/opencontainers/runc from 1.2.1 to 1.2.2 in the go group across 1 directory (#12381) -* Bump github/codeql-action from 3.27.1 to 3.27.4 in the actions group (#12374) -* Bump golangci-lint on actions and disable deprecated linters (#12362) -* Bump google.golang.org/grpc from 1.67.1 to 1.68.0 (#12355) -* Bump the actions group with 3 updates (#12352) -* Bump golang.org/x/crypto from 0.28.0 to 0.29.0 (#12350) -* Bump github.com/fsnotify/fsnotify from 1.7.0 to 1.8.0 (#12298) -* Bump github.com/opencontainers/runc from 1.2.0 to 1.2.1 in the go group across 1 directory (#12295) -* Bump github.com/onsi/ginkgo/v2 from 2.20.2 to 2.21.0 (#12289) -* Bump actions/dependency-review-action from 4.3.5 to 4.4.0 in the actions group (#12274) -* Bump the go group across 3 directories with 11 updates (#12245) -* Bump github.com/opencontainers/runc from 1.1.15 to 1.2.0 (#12239) -* Bump the actions group with 5 updates (#12240) -* Bump github.com/ncabatoff/process-exporter from 0.8.3 to 0.8.4 in the go group across 1 directory (#12220) -* Bump aquasecurity/trivy-action from 0.27.0 to 0.28.0 in the actions group (#12216) -* Bump github/codeql-action from 3.26.12 to 3.26.13 in the actions group (#12190) -* Bump the go group across 2 directories with 1 update (#12187) -* Bump the actions group with 2 updates (#12181) -* Bump github.com/opencontainers/runc from 1.1.14 to 1.1.15 in the go group across 1 directory (#12179) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.11.3...controller-v1.11.4 diff --git a/changelog/controller-1.12.0-beta.0.md b/changelog/controller-1.12.0-beta.0.md deleted file mode 100644 index 5f2fa9d74..000000000 --- a/changelog/controller-1.12.0-beta.0.md +++ /dev/null @@ -1,216 +0,0 @@ -# Changelog - -### controller-v1.12.0-beta.0 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.12.0-beta.0@sha256:9724476b928967173d501040631b23ba07f47073999e80e34b120e8db5f234d5 -* registry.k8s.io/ingress-nginx/controller-chroot:v1.12.0-beta.0@sha256:6e2f8f52e1f2571ff65bc4fc4826d5282d5def5835ec4ab433dcb8e659b2fbac - -### All changes: - -* Images: Trigger controller build. (#12154) -* ⚠️ Metrics: Disable by default. (#12153) ⚠️ - - This changes the default of the following CLI arguments: - - * `--enable-metrics` gets disabled by default. - -* Tests & Docs: Bump `e2e-test-echo` to v1.0.1. (#12147) -* Images: Trigger `e2e-test-echo` build. (#12140) -* ⚠️ Images: Drop `s390x`. (#12137) ⚠️ - - Support for the `s390x` architecture has already been removed from the controller image. This also removes it from the NGINX base image and CI relevant images. - -* Images: Build `s390x` controller. (#12126) -* Chart: Bump Kube Webhook CertGen. (#12119) -* Tests & Docs: Bump images. (#12118) -* Cloud Build: Bump `gcb-docker-gcloud` to v20240718-5ef92b5c36. (#12113) -* Images: Trigger other builds. (#12110) -* Tests: Bump `e2e-test-runner` to v20241004-114a6abb. (#12103) -* Images: Trigger `test-runner` build. (#12100) -* Docs: Add a multi-tenant warning. (#12091) -* Go: Bump to v1.22.8. (#12069) -* Images: Bump `NGINX_BASE` to v1.0.0. (#12066) -* Images: Trigger NGINX build. (#12063) -* Images: Remove NGINX v1.21. (#12031) -* Chart: Add `controller.metrics.service.enabled`. (#12056) -* GitHub: Improve Dependabot. (#12033) -* Chart: Add `global.image.registry`. (#12028) -* ⚠️ Images: Remove OpenTelemetry. (#12024) ⚠️ - - OpenTelemetry is still supported, but since the module is built into the controller image since v1.10, we hereby remove the init container and image which were used to install it upon controller startup. - -* Chart: Improve CI. (#12003) -* Chart: Extend image tests. (#12025) -* Chart: Add `controller.progressDeadlineSeconds`. (#12017) -* Docs: Add health check annotations for AWS. (#12018) -* Docs: Convert `opentelemetry.md` from CRLF to LF. (#12005) -* Chart: Implement `unhealthyPodEvictionPolicy`. (#11992) -* Chart: Add `defaultBackend.maxUnavailable`. (#11995) -* Chart: Test `controller.minAvailable` & `controller.maxUnavailable`. (#12000) -* Chart: Align default backend `PodDisruptionBudget`. (#11993) -* Metrics: Fix namespace in `nginx_ingress_controller_ssl_expire_time_seconds`. (#10274) -* ⚠️ Chart: Remove Pod Security Policy. (#11971) ⚠️ - - This removes Pod Security Policies and related resources from the chart. - -* Chart: Improve default backend service account. (#11972) -* Go: Bump to v1.22.7. (#11943) -* NGINX: Remove inline Lua from template. (#11806) -* Images: Bump OpenTelemetry C++ Contrib. (#11629) -* Docs: Add note about `--watch-namespace`. (#11947) -* Images: Use latest Alpine 3.20 everywhere. (#11944) -* Fix minor typos (#11935) -* Chart: Implement `controller.admissionWebhooks.service.servicePort`. (#11931) -* Allow any protocol for cors origins (#11153) -* Tests: Bump `e2e-test-runner` to v20240829-2c421762. (#11919) -* Images: Trigger `test-runner` build. (#11916) -* Chart: Add `controller.metrics.prometheusRule.annotations`. (#11849) -* Chart: Add tests for `PrometheusRule` & `ServiceMonitor`. (#11883) -* Annotations: Allow commas in URLs. (#11882) -* CI: Grant checks write permissions to E2E Test Report. (#11862) -* Chart: Use generic values for `ConfigMap` test. (#11877) -* Security: Follow-up on recent changes. (#11874) -* Lua: Remove plugins from `.luacheckrc` & E2E docs. (#11872) -* Dashboard: Remove `ingress_upstream_latency_seconds`. (#11878) -* Metrics: Add `--metrics-per-undefined-host` argument. (#11818) -* Update maxmind post link about geolite2 license changes (#11861) -* ⚠️ Remove global-rate-limit feature (#11851) ⚠️ - - This removes the following configuration options: - - * `global-rate-limit-memcached-host` - * `global-rate-limit-memcached-port` - * `global-rate-limit-memcached-connect-timeout` - * `global-rate-limit-memcached-max-idle-timeout` - * `global-rate-limit-memcached-pool-size` - * `global-rate-limit-status-code` - - It also removes the following annotations: - - * `global-rate-limit` - * `global-rate-limit-window` - * `global-rate-limit-key` - * `global-rate-limit-ignored-cidrs` - -* Revert "docs: Add deployment for AWS NLB Proxy." (#11857) -* Add custom code handling for temporal redirect (#10651) -* Add native histogram support for histogram metrics (#9971) -* Replace deprecated queue method (#11853) -* ⚠️ Enable security features by default (#11819) ⚠️ - - This changes the default of the following CLI arguments: - - * `--enable-annotation-validation` gets enabled by default. - - It also changes the default of the following configuration options: - - * `allow-cross-namespace-resources` gets disabled by default. - * `annotations-risk-level` gets lowered to "High" by default. - * `strict-validate-path-type` gets enabled by default. - -* docs: Add deployment for AWS NLB Proxy. (#9565) -* ⚠️ Remove 3rd party lua plugin support (#11821) ⚠️ - - This removes the following configuration options: - - * `plugins` - - It also removes support for user provided Lua plugins in the `/etc/nginx/lua/plugins` directory. - -* Auto-generate annotation docs (#11820) -* ⚠️ Metrics: Remove `ingress_upstream_latency_seconds`. (#11795) ⚠️ - - This metric has already been deprecated and is now getting removed. - -* Release controller v1.11.2/v1.10.4 & chart v4.11.2/v4.10.4. (#11816) -* Chart: Bump Kube Webhook CertGen & OpenTelemetry. (#11809) -* Tests & Docs: Bump images. (#11803) -* Images: Trigger failed builds. (#11800) -* Images: Trigger other builds. (#11796) -* Controller: Fix panic in alternative backend merging. (#11789) -* Tests: Bump `e2e-test-runner` to v20240812-3f0129aa. (#11788) -* Images: Trigger `test-runner` build. (#11785) -* Images: Bump `NGINX_BASE` to v0.0.12. (#11782) -* Images: Trigger NGINX build. (#11779) -* Cloud Build: Add missing config, remove unused ones. (#11774) -* Cloud Build: Tweak timeouts. (#11761) -* Cloud Build: Fix substitutions. (#11758) -* Cloud Build: Some chores. (#11633) -* Go: Bump to v1.22.6. (#11747) -* Images: Bump `NGINX_BASE` to v0.0.11. (#11741) -* Images: Trigger NGINX build. (#11735) -* docs: update OpenSSL Roadmap link (#11730) -* Go: Bump to v1.22.5. (#11634) -* Docs: Fix typo in AWS LB Controller reference (#11723) -* Perform some cleaning operations on line breaks. (#11720) -* Missing anchors in regular expression. (#11717) -* Docs: Fix `from-to-www` redirect description. (#11712) -* Chart: Remove `isControllerTagValid`. (#11710) -* Tests: Bump `e2e-test-runner` to v20240729-04899b27. (#11702) -* Chart: Explicitly set `runAsGroup`. (#11679) -* Docs: Clarify `from-to-www` redirect direction. (#11682) -* added real-client-ip faq (#11663) -* Docs: Format NGINX configuration table. (#11659) -* Release controller v1.11.1/v1.10.3 & chart v4.11.1/v4.10.3. (#11654) -* Tests: Bump `test-runner` to v20240717-1fe74b5f. (#11645) -* Images: Trigger `test-runner` build. (#11636) -* Images: Bump `NGINX_BASE` to v0.0.10. (#11635) -* remove modsecurity coreruleset test files from nginx image (#11617) -* unskip the ocsp tests and update images to fix cfssl bug (#11606) -* Fix indent in YAML for example pod (#11598) -* Images: Bump `test-runner`. (#11600) -* Images: Bump `NGINX_BASE` to v0.0.9. (#11599) -* revert module upgrade (#11594) -* README: Fix support matrix. (#11586) -* Repository: Add changelogs from `release-v1.10`. (#11587) - -### Dependency updates: - -* Bump the actions group with 3 updates (#12152) -* Bump golang.org/x/crypto from 0.27.0 to 0.28.0 (#12107) -* Bump the actions group with 3 updates (#12092) -* Bump sigs.k8s.io/mdtoc from 1.1.0 to 1.4.0 (#12062) -* Bump github.com/prometheus/common from 0.59.1 to 0.60.0 (#12060) -* Bump google.golang.org/grpc from 1.67.0 to 1.67.1 in the go group across 1 directory (#12059) -* Bump k8s.io/cli-runtime from 0.30.0 to 0.31.1 (#12061) -* Bump github/codeql-action from 3.26.9 to 3.26.10 in the actions group (#12051) -* Bump the go group across 1 directory with 3 updates (#12050) -* Bump k8s.io/kube-aggregator from 0.29.3 to 0.31.1 in /images/kube-webhook-certgen/rootfs (#12043) -* Bump k8s.io/apimachinery from 0.23.1 to 0.31.1 in /images/ext-auth-example-authsvc/rootfs (#12041) -* Bump github.com/prometheus/client_golang from 1.11.1 to 1.20.4 in /images/custom-error-pages/rootfs (#12040) -* Bump the all group with 2 updates (#12032) -* Bump github/codeql-action from 3.26.7 to 3.26.8 in the all group (#12010) -* Bump google.golang.org/grpc from 1.66.2 to 1.67.0 (#12009) -* Bump github.com/prometheus/client_golang from 1.20.3 to 1.20.4 in the all group (#12008) -* Bump the all group with 2 updates (#11977) -* Bump github/codeql-action from 3.26.6 to 3.26.7 in the all group (#11976) -* Bump github.com/prometheus/common from 0.57.0 to 0.59.1 (#11954) -* Bump golang.org/x/crypto from 0.26.0 to 0.27.0 (#11955) -* Bump github.com/prometheus/client_golang from 1.20.2 to 1.20.3 in the all group (#11953) -* Bump github.com/opencontainers/runc from 1.1.13 to 1.1.14 (#11928) -* Bump the all group with 2 updates (#11922) -* Bump github.com/onsi/ginkgo/v2 from 2.20.1 to 2.20.2 in the all group (#11901) -* Bump google.golang.org/grpc from 1.65.0 to 1.66.0 (#11902) -* Bump github.com/prometheus/common from 0.55.0 to 0.57.0 (#11903) -* Bump github/codeql-action from 3.26.5 to 3.26.6 in the all group (#11904) -* Bump the all group with 2 updates (#11865) -* Bump github/codeql-action from 3.26.2 to 3.26.5 in the all group (#11867) -* Bump github.com/prometheus/client_golang from 1.19.1 to 1.20.1 (#11832) -* Bump sigs.k8s.io/controller-runtime from 0.18.4 to 0.19.0 (#11823) -* Bump dario.cat/mergo from 1.0.0 to 1.0.1 in the all group (#11822) -* Bump k8s.io/component-base from 0.30.3 to 0.31.0 (#11825) -* Bump github/codeql-action from 3.26.0 to 3.26.2 in the all group (#11826) -* Bump github.com/onsi/ginkgo/v2 from 2.19.1 to 2.20.0 (#11766) -* Bump the all group with 2 updates (#11767) -* Bump golang.org/x/crypto from 0.25.0 to 0.26.0 (#11765) -* Bump the all group with 3 updates (#11727) -* Bump github.com/onsi/ginkgo/v2 from 2.19.0 to 2.19.1 in the all group (#11696) -* Bump the all group with 2 updates (#11695) -* Bump the all group with 4 updates (#11673) -* Bump the all group with 2 updates (#11672) -* Bump github.com/prometheus/common from 0.54.0 to 0.55.0 (#11522) -* Bump the all group with 5 updates (#11611) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.11.0...controller-v1.12.0-beta.0 diff --git a/changelog/controller-1.12.0.md b/changelog/controller-1.12.0.md deleted file mode 100644 index dc7155546..000000000 --- a/changelog/controller-1.12.0.md +++ /dev/null @@ -1,294 +0,0 @@ -# Changelog - -### controller-v1.12.0 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.12.0@sha256:e6b8de175acda6ca913891f0f727bca4527e797d52688cbe9fec9040d6f6b6fa -* registry.k8s.io/ingress-nginx/controller-chroot:v1.12.0@sha256:87c88e1c38a6c8d4483c8f70b69e2cca49853bb3ec3124b9b1be648edf139af3 - -### All changes: - -* Images: Trigger controller build. (#12609) -* Chart: Bump Kube Webhook CertGen. (#12606) -* Tests & Docs: Bump images. (#12603) -* Images: Trigger other builds (2/2). (#12599) -* Images: Trigger other builds (1/2). (#12595) -* Tests: Bump `e2e-test-runner` to v20241224-68ed4e7b. (#12590) -* Images: Trigger `test-runner` build. (#12587) -* Images: Bump `NGINX_BASE` to v1.1.0. (#12582) -* Images: Trigger NGINX build. (#12579) -* Go: Clean `go.work.sum`. (#12573) -* Repository: Update owners. (#12568) -* Images: Bump `gcb-docker-gcloud` to v20241217-ff46a068cd. (#12561) -* CI: Update KIND images. (#12560) -* Images: Bump Alpine to v3.21. (#12528) -* Docs: Add guide on how to set a Maintenance Page. (#12525) -* rikatz is stepping down (#12516) -* rikatz is stepping down (#12494) -* Go: Bump to v1.23.4. (#12483) -* Plugin: Bump `goreleaser` to v2. (#12440) -* GitHub: Fix `exec` in issue template. (#12387) -* CI: Update KIND images. (#12367) -* Images: Bump `gcb-docker-gcloud` to v20241110-72bb0b1665. (#12342) -* Go: Bump to v1.23.3. (#12337) -* Auth TLS: Add `_` to redirect RegEx. (#12326) -* Auth TLS: Improve redirect RegEx. (#12323) -* Update custom headers annotation documentation (#12318) -* Tests: Bump `e2e-test-runner` to v20241104-02a3933e. (#12312) -* Docs: Add CPU usage note for `--metrics-per-undefined-host`. (#12310) -* Images: Trigger `test-runner` build. (#12308) -* Config: Fix panic on invalid `lua-shared-dict`. (#12283) -* Docs: fix limit-rate-after references (#12278) -* Chart: Rework ServiceMonitor. (#12269) -* Chart: Add ServiceAccount tests. (#12263) -* CI: Fix chart testing. (#12258) -* [fix] fix nginx temp configs cleanup (#12225) -* Chart: Suggest `matchLabelKeys` in Topology Spread Constraints. (#12202) -* Docs: Add Pod Security Admission. (#12195) -* Docs: Clarify external & service port in TCP/UDP services explanation. (#12192) -* Images: Trigger controller build. (#12154) -* ⚠️ Metrics: Disable by default. (#12153) ⚠️ - - This changes the default of the following CLI arguments: - - * `--enable-metrics` gets disabled by default. - -* Tests & Docs: Bump `e2e-test-echo` to v1.0.1. (#12147) -* Images: Trigger `e2e-test-echo` build. (#12140) -* ⚠️ Images: Drop `s390x`. (#12137) ⚠️ - - Support for the `s390x` architecture has already been removed from the controller image. This also removes it from the NGINX base image and CI relevant images. - -* Images: Build `s390x` controller. (#12126) -* Chart: Bump Kube Webhook CertGen. (#12119) -* Tests & Docs: Bump images. (#12118) -* Cloud Build: Bump `gcb-docker-gcloud` to v20240718-5ef92b5c36. (#12113) -* Images: Trigger other builds. (#12110) -* Tests: Bump `e2e-test-runner` to v20241004-114a6abb. (#12103) -* Images: Trigger `test-runner` build. (#12100) -* Docs: Add a multi-tenant warning. (#12091) -* Go: Bump to v1.22.8. (#12069) -* Images: Bump `NGINX_BASE` to v1.0.0. (#12066) -* Images: Trigger NGINX build. (#12063) -* Images: Remove NGINX v1.21. (#12031) -* Chart: Add `controller.metrics.service.enabled`. (#12056) -* GitHub: Improve Dependabot. (#12033) -* Chart: Add `global.image.registry`. (#12028) -* ⚠️ Images: Remove OpenTelemetry. (#12024) ⚠️ - - OpenTelemetry is still supported, but since the module is built into the controller image since v1.10, we hereby remove the init container and image which were used to install it upon controller startup. - -* Chart: Improve CI. (#12003) -* Chart: Extend image tests. (#12025) -* Chart: Add `controller.progressDeadlineSeconds`. (#12017) -* Docs: Add health check annotations for AWS. (#12018) -* Docs: Convert `opentelemetry.md` from CRLF to LF. (#12005) -* Chart: Implement `unhealthyPodEvictionPolicy`. (#11992) -* Chart: Add `defaultBackend.maxUnavailable`. (#11995) -* Chart: Test `controller.minAvailable` & `controller.maxUnavailable`. (#12000) -* Chart: Align default backend `PodDisruptionBudget`. (#11993) -* Metrics: Fix namespace in `nginx_ingress_controller_ssl_expire_time_seconds`. (#10274) -* ⚠️ Chart: Remove Pod Security Policy. (#11971) ⚠️ - - This removes Pod Security Policies and related resources from the chart. - -* Chart: Improve default backend service account. (#11972) -* Go: Bump to v1.22.7. (#11943) -* NGINX: Remove inline Lua from template. (#11806) -* Images: Bump OpenTelemetry C++ Contrib. (#11629) -* Docs: Add note about `--watch-namespace`. (#11947) -* Images: Use latest Alpine 3.20 everywhere. (#11944) -* Fix minor typos (#11935) -* Chart: Implement `controller.admissionWebhooks.service.servicePort`. (#11931) -* Allow any protocol for cors origins (#11153) -* Tests: Bump `e2e-test-runner` to v20240829-2c421762. (#11919) -* Images: Trigger `test-runner` build. (#11916) -* Chart: Add `controller.metrics.prometheusRule.annotations`. (#11849) -* Chart: Add tests for `PrometheusRule` & `ServiceMonitor`. (#11883) -* Annotations: Allow commas in URLs. (#11882) -* CI: Grant checks write permissions to E2E Test Report. (#11862) -* Chart: Use generic values for `ConfigMap` test. (#11877) -* Security: Follow-up on recent changes. (#11874) -* Lua: Remove plugins from `.luacheckrc` & E2E docs. (#11872) -* Dashboard: Remove `ingress_upstream_latency_seconds`. (#11878) -* Metrics: Add `--metrics-per-undefined-host` argument. (#11818) -* Update maxmind post link about geolite2 license changes (#11861) -* ⚠️ Remove global-rate-limit feature (#11851) ⚠️ - - This removes the following configuration options: - - * `global-rate-limit-memcached-host` - * `global-rate-limit-memcached-port` - * `global-rate-limit-memcached-connect-timeout` - * `global-rate-limit-memcached-max-idle-timeout` - * `global-rate-limit-memcached-pool-size` - * `global-rate-limit-status-code` - - It also removes the following annotations: - - * `global-rate-limit` - * `global-rate-limit-window` - * `global-rate-limit-key` - * `global-rate-limit-ignored-cidrs` - -* Revert "docs: Add deployment for AWS NLB Proxy." (#11857) -* Add custom code handling for temporal redirect (#10651) -* Add native histogram support for histogram metrics (#9971) -* Replace deprecated queue method (#11853) -* ⚠️ Enable security features by default (#11819) ⚠️ - - This changes the default of the following CLI arguments: - - * `--enable-annotation-validation` gets enabled by default. - - It also changes the default of the following configuration options: - - * `allow-cross-namespace-resources` gets disabled by default. - * `annotations-risk-level` gets lowered to "High" by default. - * `strict-validate-path-type` gets enabled by default. - -* docs: Add deployment for AWS NLB Proxy. (#9565) -* ⚠️ Remove 3rd party lua plugin support (#11821) ⚠️ - - This removes the following configuration options: - - * `plugins` - - It also removes support for user provided Lua plugins in the `/etc/nginx/lua/plugins` directory. - -* Auto-generate annotation docs (#11820) -* ⚠️ Metrics: Remove `ingress_upstream_latency_seconds`. (#11795) ⚠️ - - This metric has already been deprecated and is now getting removed. - -* Release controller v1.11.2/v1.10.4 & chart v4.11.2/v4.10.4. (#11816) -* Chart: Bump Kube Webhook CertGen & OpenTelemetry. (#11809) -* Tests & Docs: Bump images. (#11803) -* Images: Trigger failed builds. (#11800) -* Images: Trigger other builds. (#11796) -* Controller: Fix panic in alternative backend merging. (#11789) -* Tests: Bump `e2e-test-runner` to v20240812-3f0129aa. (#11788) -* Images: Trigger `test-runner` build. (#11785) -* Images: Bump `NGINX_BASE` to v0.0.12. (#11782) -* Images: Trigger NGINX build. (#11779) -* Cloud Build: Add missing config, remove unused ones. (#11774) -* Cloud Build: Tweak timeouts. (#11761) -* Cloud Build: Fix substitutions. (#11758) -* Cloud Build: Some chores. (#11633) -* Go: Bump to v1.22.6. (#11747) -* Images: Bump `NGINX_BASE` to v0.0.11. (#11741) -* Images: Trigger NGINX build. (#11735) -* docs: update OpenSSL Roadmap link (#11730) -* Go: Bump to v1.22.5. (#11634) -* Docs: Fix typo in AWS LB Controller reference (#11723) -* Perform some cleaning operations on line breaks. (#11720) -* Missing anchors in regular expression. (#11717) -* Docs: Fix `from-to-www` redirect description. (#11712) -* Chart: Remove `isControllerTagValid`. (#11710) -* Tests: Bump `e2e-test-runner` to v20240729-04899b27. (#11702) -* Chart: Explicitly set `runAsGroup`. (#11679) -* Docs: Clarify `from-to-www` redirect direction. (#11682) -* added real-client-ip faq (#11663) -* Docs: Format NGINX configuration table. (#11659) -* Release controller v1.11.1/v1.10.3 & chart v4.11.1/v4.10.3. (#11654) -* Tests: Bump `test-runner` to v20240717-1fe74b5f. (#11645) -* Images: Trigger `test-runner` build. (#11636) -* Images: Bump `NGINX_BASE` to v0.0.10. (#11635) -* remove modsecurity coreruleset test files from nginx image (#11617) -* unskip the ocsp tests and update images to fix cfssl bug (#11606) -* Fix indent in YAML for example pod (#11598) -* Images: Bump `test-runner`. (#11600) -* Images: Bump `NGINX_BASE` to v0.0.9. (#11599) -* revert module upgrade (#11594) -* README: Fix support matrix. (#11586) -* Repository: Add changelogs from `release-v1.10`. (#11587) - -### Dependency updates: - -* Bump k8s.io/apiextensions-apiserver from 0.31.3 to 0.32.0 (#12566) -* Bump github.com/onsi/ginkgo/v2 from 2.22.0 to 2.22.1 (#12555) -* Bump k8s.io/code-generator from 0.31.3 to 0.32.0 (#12550) -* Bump k8s.io/cli-runtime from 0.31.3 to 0.32.0 (#12547) -* Bump k8s.io/apiserver from 0.31.3 to 0.32.0 (#12544) -* Bump the actions group with 2 updates (#12541) -* Bump google.golang.org/grpc from 1.68.1 to 1.69.2 (#12538) -* Bump k8s.io/client-go from 0.31.3 to 0.32.0 (#12512) -* Bump github.com/opencontainers/runc from 1.2.2 to 1.2.3 in the go group across 1 directory (#12509) -* Bump the actions group with 3 updates (#12506) -* Bump k8s.io/kube-aggregator from 0.31.3 to 0.32.0 in /images/kube-webhook-certgen/rootfs (#12505) -* Bump k8s.io/apimachinery from 0.31.3 to 0.32.0 in /images/ext-auth-example-authsvc/rootfs (#12502) -* Bump golang.org/x/crypto from 0.30.0 to 0.31.0 (#12476) -* Bump golang.org/x/crypto from 0.21.0 to 0.31.0 in /magefiles (#12472) -* Bump github.com/prometheus/common from 0.60.1 to 0.61.0 (#12464) -* Bump github/codeql-action from 3.27.5 to 3.27.6 in the actions group (#12461) -* Bump the go group across 1 directory with 2 updates (#12460) -* Bump github.com/onsi/ginkgo/v2 from 2.21.0 to 2.22.0 (#12426) -* Bump github.com/stretchr/testify from 1.9.0 to 1.10.0 (#12418) -* Bump the go group across 3 directories with 10 updates (#12413) -* Bump the actions group with 3 updates (#12412) -* Bump github.com/opencontainers/runc from 1.2.1 to 1.2.2 in the go group across 1 directory (#12380) -* Bump github/codeql-action from 3.27.1 to 3.27.4 in the actions group (#12373) -* Bump golangci-lint on actions and disable deprecated linters (#12361) -* Bump google.golang.org/grpc from 1.67.1 to 1.68.0 (#12357) -* Bump the actions group with 3 updates (#12354) -* Bump golang.org/x/crypto from 0.28.0 to 0.29.0 (#12349) -* Bump github.com/fsnotify/fsnotify from 1.7.0 to 1.8.0 (#12299) -* Bump github.com/opencontainers/runc from 1.2.0 to 1.2.1 in the go group across 1 directory (#12296) -* Bump github.com/onsi/ginkgo/v2 from 2.20.2 to 2.21.0 (#12288) -* Bump actions/dependency-review-action from 4.3.5 to 4.4.0 in the actions group (#12273) -* Bump the go group across 3 directories with 11 updates (#12244) -* Bump github.com/opencontainers/runc from 1.1.15 to 1.2.0 (#12242) -* Bump the actions group with 5 updates (#12236) -* Bump github.com/ncabatoff/process-exporter from 0.8.3 to 0.8.4 in the go group across 1 directory (#12218) -* Bump aquasecurity/trivy-action from 0.27.0 to 0.28.0 in the actions group (#12217) -* Bump github/codeql-action from 3.26.12 to 3.26.13 in the actions group (#12188) -* Bump the go group across 2 directories with 1 update (#12186) -* Bump the actions group with 2 updates (#12180) -* Bump github.com/opencontainers/runc from 1.1.14 to 1.1.15 in the go group across 1 directory (#12178) -* Bump the actions group with 3 updates (#12152) -* Bump golang.org/x/crypto from 0.27.0 to 0.28.0 (#12107) -* Bump the actions group with 3 updates (#12092) -* Bump sigs.k8s.io/mdtoc from 1.1.0 to 1.4.0 (#12062) -* Bump github.com/prometheus/common from 0.59.1 to 0.60.0 (#12060) -* Bump google.golang.org/grpc from 1.67.0 to 1.67.1 in the go group across 1 directory (#12059) -* Bump k8s.io/cli-runtime from 0.30.0 to 0.31.1 (#12061) -* Bump github/codeql-action from 3.26.9 to 3.26.10 in the actions group (#12051) -* Bump the go group across 1 directory with 3 updates (#12050) -* Bump k8s.io/kube-aggregator from 0.29.3 to 0.31.1 in /images/kube-webhook-certgen/rootfs (#12043) -* Bump k8s.io/apimachinery from 0.23.1 to 0.31.1 in /images/ext-auth-example-authsvc/rootfs (#12041) -* Bump github.com/prometheus/client_golang from 1.11.1 to 1.20.4 in /images/custom-error-pages/rootfs (#12040) -* Bump the all group with 2 updates (#12032) -* Bump github/codeql-action from 3.26.7 to 3.26.8 in the all group (#12010) -* Bump google.golang.org/grpc from 1.66.2 to 1.67.0 (#12009) -* Bump github.com/prometheus/client_golang from 1.20.3 to 1.20.4 in the all group (#12008) -* Bump the all group with 2 updates (#11977) -* Bump github/codeql-action from 3.26.6 to 3.26.7 in the all group (#11976) -* Bump github.com/prometheus/common from 0.57.0 to 0.59.1 (#11954) -* Bump golang.org/x/crypto from 0.26.0 to 0.27.0 (#11955) -* Bump github.com/prometheus/client_golang from 1.20.2 to 1.20.3 in the all group (#11953) -* Bump github.com/opencontainers/runc from 1.1.13 to 1.1.14 (#11928) -* Bump the all group with 2 updates (#11922) -* Bump github.com/onsi/ginkgo/v2 from 2.20.1 to 2.20.2 in the all group (#11901) -* Bump google.golang.org/grpc from 1.65.0 to 1.66.0 (#11902) -* Bump github.com/prometheus/common from 0.55.0 to 0.57.0 (#11903) -* Bump github/codeql-action from 3.26.5 to 3.26.6 in the all group (#11904) -* Bump the all group with 2 updates (#11865) -* Bump github/codeql-action from 3.26.2 to 3.26.5 in the all group (#11867) -* Bump github.com/prometheus/client_golang from 1.19.1 to 1.20.1 (#11832) -* Bump sigs.k8s.io/controller-runtime from 0.18.4 to 0.19.0 (#11823) -* Bump dario.cat/mergo from 1.0.0 to 1.0.1 in the all group (#11822) -* Bump k8s.io/component-base from 0.30.3 to 0.31.0 (#11825) -* Bump github/codeql-action from 3.26.0 to 3.26.2 in the all group (#11826) -* Bump github.com/onsi/ginkgo/v2 from 2.19.1 to 2.20.0 (#11766) -* Bump the all group with 2 updates (#11767) -* Bump golang.org/x/crypto from 0.25.0 to 0.26.0 (#11765) -* Bump the all group with 3 updates (#11727) -* Bump github.com/onsi/ginkgo/v2 from 2.19.0 to 2.19.1 in the all group (#11696) -* Bump the all group with 2 updates (#11695) -* Bump the all group with 4 updates (#11673) -* Bump the all group with 2 updates (#11672) -* Bump github.com/prometheus/common from 0.54.0 to 0.55.0 (#11522) -* Bump the all group with 5 updates (#11611) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.11.0...controller-v1.12.0 diff --git a/changelog/controller-1.6.4.md b/changelog/controller-1.6.4.md deleted file mode 100644 index 15b3f671a..000000000 --- a/changelog/controller-1.6.4.md +++ /dev/null @@ -1,138 +0,0 @@ -# Changelog - -### 1.6.4 - -Images: - -* registry.k8s.io/controller:controller-v1.6.4@sha256:15be4666c53052484dd2992efacf2f50ea77a78ae8aa21ccd91af6baaa7ea22f -* registry.k8s.io/controller-chroot:controller-v1.6.4@sha256:0de01e2c316c3ca7847ca13b32d077af7910d07f21a4a82f81061839764f8f81 - -### All changes: - -* remove tests and regex path checks (#9626) -* Fix incorrect annotation name in upstream hashing configuration (#9617) -* Release docs for Controller v1.6.3 and Helm v4.5.0 (#9614) -* Revert Implement pathType validation (#9511) (#9607) -* update history and allow to pass a target test (#9605) -* Allow to pass a target test (#9542) -* Replace deprecated command with environment file (#9581) -* build 1.6.2 to fix (#9569) -* add lint on chart before release (#9570) -* tcpproxy: increase buffer size to 16K (#9548) -* Move and spell-check Kubernetes 1.22 migration FAQ (#9544) -* Add CORS template check inside location for externalAuth.SignURL (#8814) -* fix(grafana-dashboard): remove hardcoded namespace references (#9523) -* Replace deprecated command with environment file (#9581) -* add lint on chart before release (#9570) -* Switch logic on path type validation and setting it to false (#9543) -* tcpproxy: increase buffer size to 16K (#9548) -* Move and spell-check Kubernetes 1.22 migration FAQ (#9544) -* Add CORS template check inside location for externalAuth.SignURL (#8814) -* fix(grafana-dashboard): remove hardcoded namespace references (#9523) -* Align default value for keepalive_request with NGINX default (#9518) -* feat(configmap): expose gzip-disable (#9505) -* Values: Add missing `controller.metrics.service.labels`. (#9501) -* Add docs about orphan_ingress metric (#9514) -* Add new prometheus metric for orphaned ingress (#8230) -* Sanitise request metrics in monitoring docs (#9384) -* Change default value of enable-brotli (#9500) -* feat: support topology aware hints (#9165) -* Remove 1.5.2 from readme (#9498) -* Remove nonexistent load flag from docker build commands (#9122) -* added option to disable sync event creation (#8528) -* Add buildResolvers to the stream module (#9184) -* fix: disable auth access logs (#9049) -* Adding ipdenylist annotation (#8795) -* Add update updateStrategy and minReadySeconds for defaultBackend (#8506) -* Fix indentation on serviceAccount annotation (#9129) -* Update monitoring.md (#9269) -* add github actions stale bot (#9439) -* Admission Webhooks/Job: Add `NetworkPolicy`. (#9218) -* update OpenTelemetry image (#9491) -* bump OpenTelemetry (#9489) -* Optional podman support (#9294) -* fix change images (#9463) -* move tests to gh actions (#9461) -* Automated Release Controller 1.5.2 (#9455) -* Add sslpassthrough tests (#9457) -* updated the link in RELEASE.md file (#9456) -* restart 1.5.2 release process (#9450) -* Update command line arguments documentation (#9224) -* start release 1.5.2 (#9445) -* upgrade nginx base image (#9436) -* test the new e2e test images (#9444) -* avoid builds and tests for non-code changes (#9392) -* CI updates (#9440) -* HPA: Add `controller.autoscaling.annotations` to `values.yaml`. (#9253) -* update the nginx run container for alpine:3.17.0 (#9430) -* cleanup: remove ioutil for new go version (#9427) -* start upgrade to golang 1.19.4 and alpine 3.17.0 (#9417) -* ci: remove setup-helm step (#9404) -* ci: remove setup-kind step (#9401) -* Add reporter for all tests (#9395) -* added action for issues to project (#9386) -* doc: update NEW_CONTRIBUTOR.md (#9381) -* feat(helm): Optionally use cert-manager instead admission patch (#9279) -* integrated junit-reports with ghactions (#9361) -* [user-guide configmap] fix doc for global-auth-snippet (#9372) -* update OpenTelemetry image (#9308) -* fix: missing CORS headers when auth fails (#9251) -* Fix styling in canary annotation docs. (#9259) -* resolved ginkgo deprecation message (#9365) -* Enable profiler-address to be configured (#9311) -* ModSecurity dependencies update to avoid Memory Leaks (#9330) -* fix(hpa): deprecated api version, bump to v2 (#9348) -* fix(typo): pluralize provider (#9346) -* removed deprecation message for ingressClass annotation (#9357) -* added ginkgo junit reports (#9350) -* Fix typos found by codespell (#9353) -* bumped ginkgo to v2.5.1 in testrunner (#9340) -* create nsswitch-conf if missing (#9339) -* remove the configmap related permissions (#9310) -* remove hardcoded datasource from grafana dashboard (#9284) -* update gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b to 3.0.0 (#9277) -* added SAN to cert create command (#9295) -* Missing controller.ingressClass (#9304) -* OpenTelemetry static linking (#9286) -* Fixed indentation in commented-out autoscaling (#9225) -* run helm release on main only and when the chart/value changes only (#9290) -* fix broken annotation yaml (#9243) -* PDB: Add `maxUnavailable`. (#9278) -* add containerSecurityContext to extraModules init containers (kubernetes#9016) (#9242) - -### Dependency updates: - -* Bump google.golang.org/grpc from 1.52.0 to 1.52.3 (#9555) -* Bump k8s.io/klog/v2 from 2.80.1 to 2.90.0 (#9553) -* Bump sigs.k8s.io/controller-runtime from 0.13.1 to 0.14.2 (#9552) -* Bump google.golang.org/grpc from 1.51.0 to 1.52.0 (#9512) -* Bump `client-go` to remove dependence on go-autorest dependency (#9488) -* Bump google.golang.org/grpc from 1.52.0 to 1.52.3 (#9555) -* Bump k8s.io/klog/v2 from 2.80.1 to 2.90.0 (#9553) -* Bump sigs.k8s.io/controller-runtime from 0.13.1 to 0.14.2 (#9552) -* Bump google.golang.org/grpc from 1.51.0 to 1.52.0 (#9512) -* Bump `client-go` to remove dependence on go-autorest dependency (#9488) -* Bump golang.org/x/crypto from 0.4.0 to 0.5.0 (#9494) -* Bump golang.org/x/crypto from 0.3.0 to 0.4.0 (#9397) -* Bump github.com/onsi/ginkgo/v2 from 2.6.0 to 2.6.1 (#9432) -* Bump github.com/onsi/ginkgo/v2 from 2.6.0 to 2.6.1 (#9421) -* Bump github/codeql-action from 2.1.36 to 2.1.37 (#9423) -* Bump actions/checkout from 3.1.0 to 3.2.0 (#9425) -* Bump goreleaser/goreleaser-action from 3.2.0 to 4.1.0 (#9426) -* Bump actions/dependency-review-action from 3.0.1 to 3.0.2 (#9424) -* Bump ossf/scorecard-action from 2.0.6 to 2.1.0 (#9422) -* Bump github.com/prometheus/common from 0.37.0 to 0.39.0 (#9416) -* Bump github.com/onsi/ginkgo/v2 from 2.5.1 to 2.6.0 (#9408) -* Bump github.com/onsi/ginkgo/v2 from 2.5.1 to 2.6.0 (#9398) -* Bump github/codeql-action from 2.1.35 to 2.1.36 (#9400) -* Bump actions/setup-go from 3.3.1 to 3.4.0 (#9370) -* Bump github/codeql-action from 2.1.31 to 2.1.35 (#9369) -* Bump google.golang.org/grpc from 1.50.1 to 1.51.0 (#9316) -* Bump github.com/prometheus/client_golang from 1.13.1 to 1.14.0 (#9298) -* Bump actions/dependency-review-action from 3.0.0 to 3.0.1 (#9319) -* Bump golang.org/x/crypto from 0.1.0 to 0.3.0 (#9318) -* Bump github.com/onsi/ginkgo/v2 from 2.4.0 to 2.5.1 (#9317) -* Bump actions/dependency-review-action from 2.5.1 to 3.0.0 (#9301) -* Bump k8s.io/component-base from 0.25.3 to 0.25.4 (#9300) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-controller-v1.5.1...controller-controller-v1.6.4 diff --git a/changelog/controller-1.7.0.md b/changelog/controller-1.7.0.md deleted file mode 100644 index 97f0ebf22..000000000 --- a/changelog/controller-1.7.0.md +++ /dev/null @@ -1,80 +0,0 @@ -# Changelog - -### 1.7.0 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.7.0@sha256:7612338342a1e7b8090bef78f2a04fffcadd548ccaabe8a47bf7758ff549a5f7 -* registry.k8s.io/ingress-nginx/controller-chroot:v1.7.0@sha256:e84ef3b44c8efeefd8b0aa08770a886bfea1f04c53b61b4ba9a7204e9f1a7edc - -### All changes: - -* kick off 1.7.0 build (#9775) -* Update exposing-tcp-udp-services.md (#9777) -* feat: OpenTelemetry module integration (#9062) -* drop k8s 1.23 support (#9772) -* Fix canary-weight-total annotation ignored in rule backends (#9729) -* fix: controller psp's volume config (#9740) -* Fix several Helm YAML issues with extraModules and extraInitContainers (#9709) -* docs(helm): fix value key in readme for enabling certManager (#9640) -* updated digest and sha for e2e-test-echo (#9760) -* updated digest and sha for e2e-test-fastcgi-helloserver (#9759) -* updated digest and sha for opentelemetry (#9758) -* updated digest and sha for e2e-test-cfssl (#9757) -* updated kube-webhook-certgen digest and tags (#9756) -* updated nginx-error digest and tags (#9755) -* added upgrade ginkgo documentation for contributors (#9753) -* changes Makefile of echo folder to trigger code-build (#9754) -* Chart: Drop `controller.headers`, rework DH param secret. (#9659) -* updated NGINX_BASE image with latest tag (#9747) -* Deployment/DaemonSet: Label pods using `ingress-nginx.labels`. (#9732) -* bumped ginkgo to v2.9.0 (#9722) -* HPA: autoscaling/v2beta1 deprecated, bump apiVersion to v2 for defaultBackend (#9731) -* update to golang 1.20 (#9690) -* Indent values.yaml using 2 instead of 4 spaces (#9656) -* fix some comments (#9688) -* migrate mitchellh/hashstructure to v2 (#9651) -* changed v1.6.3 to v1.6.4 on deploy docs (#9647) -* controller: Don't panic when ready condition in a endpointslice is missing (#9550) -* Rework Ginkgo usage (#9522) -* code clean for fsnotify (#9571) -* Optimize the document for readability (#9551) -* sets.String is deprecated: use generic Set instead. new ways: s1 := Set[string]{} s2 := New[string]() (#9589) -* Adjust the import package order and use http library variables (#9587) -* Optimize the judgment mode to remove redundant transformations (#9588) -* Fix rewrite example (#9633) -* remove tests and regex path checks (#9626) -* Fix incorrect annotation name in upstream hashing configuration (#9617) -* Release docs for Controller v1.6.3 and Helm v4.5.0 (#9614) - -### Dependency updates: - -* Bump aquasecurity/trivy-action from 0.8.0 to 0.9.2 (#9767) -* Bump k8s.io/component-base from 0.26.2 to 0.26.3 (#9764) -* Bump actions/dependency-review-action from 3.0.3 to 3.0.4 (#9766) -* Bump actions/add-to-project from 0.4.0 to 0.4.1 (#9765) -* Bump actions/dependency-review-action from 3.0.2 to 3.0.3 (#9727) -* Bump github.com/prometheus/common from 0.41.0 to 0.42.0 (#9724) -* Bump golang.org/x/crypto from 0.6.0 to 0.7.0 (#9723) -* Bump actions/download-artifact from 3.0.1 to 3.0.2 (#9721) -* Bump goreleaser/goreleaser-action from 4.1.0 to 4.2.0 (#9718) -* Bump actions/upload-artifact from 3.1.1 to 3.1.2 (#9717) -* Bump docker/setup-buildx-action from 2.2.1 to 2.5.0 (#9719) -* Bump helm/chart-releaser-action from 1.4.1 to 1.5.0 (#9720) -* Bump github.com/onsi/ginkgo/v2 from 2.6.1 to 2.9.0 (#9695) -* Bump k8s.io/klog/v2 from 2.90.0 to 2.90.1 (#9694) -* Bump golang.org/x/crypto in /magefiles (#9691) -* Bump k8s.io/component-base from 0.26.1 to 0.26.2 (#9696) -* Bump github.com/prometheus/common from 0.40.0 to 0.41.0 (#9698) -* Bump sigs.k8s.io/controller-runtime from 0.14.2 to 0.14.5 (#9697) -* Bump golang.org/x/net in /magefiles (#9692) -* Bump golang.org/x/sys in /images/custom-error-pages/rootfs (#9671) -* Bump github.com/stretchr/testify from 1.8.1 to 1.8.2 (#9675) -* Bump github.com/prometheus/common from 0.39.0 to 0.40.0 (#9653) -* Bump golang.org/x/net from 0.6.0 to 0.7.0 (#9646) -* Bump golang.org/x/net in /images/kube-webhook-certgen/rootfs (#9645) -* Bump google.golang.org/grpc from 1.52.3 to 1.53.0 (#9610) -* Bump github.com/prometheus/client_golang (#9630) -* Bump golang.org/x/crypto from 0.5.0 to 0.6.0 (#9609) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-controller-v1.6.3...controller-controller-v1.7.0 diff --git a/changelog/controller-1.7.1.md b/changelog/controller-1.7.1.md deleted file mode 100644 index a7a5c4bbf..000000000 --- a/changelog/controller-1.7.1.md +++ /dev/null @@ -1,54 +0,0 @@ -# Changelog - -### 1.7.1 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.7.1@sha256:7244b95ea47bddcb8267c1e625fb163fc183ef55448855e3ac52a7b260a60407 -* registry.k8s.io/ingress-nginx/controller-chroot:v1.7.1@sha256:e35d5ab487861b9d419c570e3530589229224a0762c7b4d2e2222434abb8d988 - -### All changes: - -* Update TAG - 1.7.1 (#9922) -* Update dependabot to watch docker images (#9600) -* [helm] Support custom port configuration for internal service (#9846) -* Add support for --container flag (#9703) -* Fix typo in OpenTelemetry (#9903) -* ensure make lua-test runs locally (#9902) -* update k8s.io dependencies to v0.26.4 (#9893) -* Adding resource type to default HPA configuration to resolve issues with Terraform helm chart usage (#9803) -* I have not been able to fulfill my maintainer responsibilities for a while already, making it official now. (#9883) -* Update k8s versions (#9879) -* README: Update `external-dns` link. (#9866) -* Fastcgi configmap should be on the same namespace of ingress (#9863) -* Deprecate and remove influxdb feature (#9861) -* Remove deprecated annotation secure-upstream (#9862) -* Exclude socket metrics (#9770) -* Chart: Improve `README.md`. (#9831) -* update all container tags with date and sha, upgrade all containers (#9834) -* updated NGINX_BASE image in project (#9829) -* ISO 8601 date format (#9682) -* Values: Fix indention of commented values. (#9812) -* The Ingress-Nginx project recently released version 1.7.0 of the controller, but the deployment documentation still referenced version 1.6.4. This commit updates the documentation to reference the latest version, ensuring that users have access to the most up-to-date information. Fixes#9787 (#9788) - -### Dependency updates: - -* Bump github.com/opencontainers/runc from 1.1.6 to 1.1.7 (#9912) -* Bump github.com/prometheus/client_golang from 1.14.0 to 1.15.0 (#9868) -* Bump aquasecurity/trivy-action from 0.9.2 to 0.10.0 (#9888) -* Bump github.com/opencontainers/runc from 1.1.5 to 1.1.6 (#9867) -* Bump actions/checkout from 3.5.0 to 3.5.2 (#9870) -* Bump golang.org/x/crypto from 0.7.0 to 0.8.0 (#9838) -* Bump github.com/spf13/cobra from 1.6.1 to 1.7.0 (#9839) -* Bump actions/add-to-project from 0.4.1 to 0.5.0 (#9840) -* Bump actions/checkout from 3.4.0 to 3.5.0 (#9798) -* Bump ossf/scorecard-action from 2.1.2 to 2.1.3 (#9823) -* Bump github.com/opencontainers/runc from 1.1.4 to 1.1.5 (#9806) -* Bump actions/stale from 7.0.0 to 8.0.0 (#9799) -* Bump rajatjindal/krew-release-bot from 0.0.43 to 0.0.46 (#9797) -* Bump actions/setup-go from 3.5.0 to 4.0.0 (#9796) -* Bump github.com/imdario/mergo from 0.3.13 to 0.3.15 (#9795) -* Bump google.golang.org/grpc from 1.53.0 to 1.54.0 (#9794) -* Bump sigs.k8s.io/controller-runtime from 0.14.5 to 0.14.6 (#9822) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-controller-v1.7.0...controller-controller-v1.7.1 diff --git a/changelog/controller-1.8.0.md b/changelog/controller-1.8.0.md deleted file mode 100644 index f335777d6..000000000 --- a/changelog/controller-1.8.0.md +++ /dev/null @@ -1,64 +0,0 @@ -# Changelog - -### 1.8.0 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.8.0@sha256:744ae2afd433a395eeb13dc03d3313facba92e96ad71d9feaafc85925493fee3 -* registry.k8s.io/ingress-nginx/controller-chroot:v1.8.0@sha256:a45e41cd2b7670adf829759878f512d4208d0aec1869dae593a0fecd09a5e49e - -### Important changes: - -* Validate path types (#9967) -* images: upgrade to Alpine 3.18 (#9997) -* Update documentation to reflect project name; Ingress-Nginx Controller - -For improving security, our 1.8.0 release includes a [new, **optional** validation ](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#strict-validate-path-type) that limits the characters accepted on ".spec paths.path" when pathType=Exact or athType=Prefix, to alphanumeric characters only. - -More information can be found on our [Google doc](https://docs.google.com/document/d/1HPvaEwHRuMSkXYkVIJ-w7IpijKdHfNynm_4N2Akt0CQ/edit?usp=sharing), our new [ingress-nginx-dev mailing list](https://groups.google.com/a/kubernetes.io/g/ingress-nginx-dev/c/ebbBMo-zX-w) or in our [docs](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#strict-validate-path-type) - -### Community Updates - -We are now posting updates and release to our twitter handle, [@IngressNginx](https://twitter.com/IngressNGINX) and -on our new [ingress-nginx-dev mailing list](https://groups.google.com/a/kubernetes.io/g/ingress-nginx-dev/c/ebbBMo-zX-w) - -### All changes: - -* Add legacy to OpenTelemetry migration doc (#10011) -* changed tagsha to recent builds (#10001) -* change to alpine318 baseimage (#10000) -* images: upgrade to Alpine 3.18 (#9997) -* openssl CVE fix (#9996) -* PodDisruptionBudget spec logic update (#9904) -* Admission warning (#9975) -* Add OPA examples on pathType restrictions (#9992) -* updated testrunner image tag+sha (#9987) -* bumped ginkgo to v2.9.5 (#9985) -* helm: Fix opentelemetry module installation for daemonset (#9792) -* OpenTelemetry default config (#9978) -* Correct annotations in monitoring docs (#9976) -* fix: avoid builds and tests for changes to markdown (#9962) -* Validate path types (#9967) -* HPA: Use capabilities & align manifests. (#9521) -* Use dl.k8s.io instead of hardcoded GCS URIs (#9946) -* add option for annotations in PodDisruptionBudget (#9843) -* chore: update httpbin to httpbun (#9919) -* image_update (#9942) -* Add geoname id value into $geoip2_*_geoname_id variables (#9527) -* Update annotations.md (#9933) -* Update charts/* to keep project name display aligned (#9931) -* Keep project name display aligned (#9920) - -### Dependencies updates: -* Bump github.com/imdario/mergo from 0.3.15 to 0.3.16 (#10008) -* Bump github.com/prometheus/common from 0.43.0 to 0.44.0 (#10007) -* Bump k8s.io/klog/v2 from 2.90.1 to 2.100.1 (#9913) -* Bump github.com/onsi/ginkgo/v2 from 2.9.0 to 2.9.5 (#9980) -* Bump golang.org/x/crypto from 0.8.0 to 0.9.0 (#9982) -* Bump actions/setup-go from 4.0.0 to 4.0.1 (#9984) -* Bump securego/gosec from 2.15.0 to 2.16.0 (#9983) -* Bump github.com/prometheus/common from 0.42.0 to 0.43.0 (#9981) -* Bump github.com/prometheus/client_model from 0.3.0 to 0.4.0 (#9937) -* Bump google.golang.org/grpc from 1.54.0 to 1.55.0 (#9936) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-controller-v1.7.1...controller-controller-v1.8.0 diff --git a/changelog/controller-1.8.1.md b/changelog/controller-1.8.1.md deleted file mode 100644 index 16d4a8409..000000000 --- a/changelog/controller-1.8.1.md +++ /dev/null @@ -1,69 +0,0 @@ -# Changelog - -### 1.8.1 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.8.1@sha256:e5c4824e7375fcf2a393e1c03c293b69759af37a9ca6abdb91b13d78a93da8bd -* registry.k8s.io/ingress-nginx/controller-chroot:v1.8.1@sha256:e0d4121e3c5e39de9122e55e331a32d5ebf8d4d257227cb93ab54a1b912a7627 - -### All changes: - -* netlify: Only trigger preview when there are changes in docs. (#10144) -* changed to updated baseimage and reverted tag (#10143) -* Fix loadBalancerClass value (#10139) -* Added a doc line to the missing helm value service.internal.loadBalancerIP (#9406) -* Set grpc :authority header from request header (#8912) -* bump pinned golang to 1.20.5 (#10127) -* update test runner (#10125) -* chore: remove echo from snippet tests (#10110) -* Update typo in docs for lb scheme (#10117) -* golang 1.20.5 bump (#10120) -* feat(helm): Add loadBalancerClass (#9562) -* chore: remove echo friom canary tests (#10089) -* fix: obsolete warnings (#10029) -* docs: change Dockefile url ref main (#10087) -* Revert "Remove fastcgi feature" (#10081) -* docs: add netlify configuration (#10073) -* add distroless otel init (#10035) -* chore: move httpbun to be part of framework (#9955) -* Remove fastcgi feature (#9864) -* Fix mirror-target values without path separator and port (#9889) -* Adding feature to upgrade Oracle Cloud Infrastructure's Flexible Load Balancer and adjusting Health Check that were critical in the previous configuration (#9961) -* add support for keda fallback settings (#9993) -* unnecessary use of fmt.Sprint (S1039) (#10049) -* chore: pkg imported more than once (#10048) -* tracing: upgrade to dd-opentracing-cpp v1.3.7 (#10031) -* fix: add canary to sidebar in examples (#10068) -* docs: add lua testing documentation (#10060) -* docs: canary weighted deployments example (#10067) -* Update Internal Load Balancer docs (#10062) -* fix broken kubernetes.io/user-guide/ docs links (#10055) -* docs: Updated the content of deploy/rbac.md (#10054) -* ensured hpa mem spec before cpu spec (#10043) -* Fix typo in controller_test (#10034) -* chore(dep): upgrade github.com/emicklei/go-restful/v3 to 3.10 (#10028) -* Upgrade to Golang 1.20.4 (#10016) -* perf: avoid unnecessary byte/string conversion (#10012) -* added note on dns for localtesting (#10021) -* added helmshowvalues example (#10019) -* release controller 1.8.0 and chart 4.7.0 (#10017) - -### Dependency updates: - -* Bump ossf/scorecard-action from 2.1.3 to 2.2.0 (#10133) -* Bump google.golang.org/grpc from 1.56.0 to 1.56.1 (#10134) -* Bump github.com/prometheus/client_golang from 1.15.1 to 1.16.0 (#10106) -* Bump golang.org/x/crypto from 0.9.0 to 0.10.0 (#10105) -* Bump google.golang.org/grpc from 1.55.0 to 1.56.0 (#10103) -* Bump goreleaser/goreleaser-action from 4.2.0 to 4.3.0 (#10101) -* Bump docker/setup-buildx-action from 2.6.0 to 2.7.0 (#10102) -* Bump actions/checkout from 3.5.2 to 3.5.3 (#10076) -* Bump docker/setup-qemu-action from 2.1.0 to 2.2.0 (#10075) -* Bump aquasecurity/trivy-action from 0.10.0 to 0.11.2 (#10078) -* Bump docker/setup-buildx-action from 2.5.0 to 2.6.0 (#10077) -* Bump actions/dependency-review-action from 3.0.4 to 3.0.6 (#10042) -* Bump github.com/stretchr/testify from 1.8.3 to 1.8.4 (#10041) -* Bump github.com/stretchr/testify from 1.8.2 to 1.8.3 (#10005) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-controller-v1.8.0...controller-controller-v1.8.1 diff --git a/changelog/controller-1.8.2.md b/changelog/controller-1.8.2.md deleted file mode 100644 index 0c20decc9..000000000 --- a/changelog/controller-1.8.2.md +++ /dev/null @@ -1,19 +0,0 @@ -# Changelog - -### 1.8.2 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.8.2@sha256:74834d3d25b336b62cabeb8bf7f1d788706e2cf1cfd64022de4137ade8881ff2 -* registry.k8s.io/ingress-nginx/controller-chroot:v1.8.2@sha256:1317a563219f755a6094d990057c78e5c4dcea5e31f4ce1db8641e732a7d6133 - -### All changes: - -* Release v1.8.2 and Update Go to v1.21.1 (#10379) -* Making auth access logs optional (#10380) -* [release-1.8] Disable Modsecurity from internal processing which affects large ingresses (#10375) -* promote distroless otel init image (#10270) -* [release-1.8] Update images tags after adding git data in gcloud (#10233) -* [release-1.8] Golang 1.20.6 for test runner (#10231) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-controller-v1.8.1...controller-controller-v1.8.2 diff --git a/changelog/controller-1.9.0-beta.0.md b/changelog/controller-1.9.0-beta.0.md deleted file mode 100644 index 5ca5cfeeb..000000000 --- a/changelog/controller-1.9.0-beta.0.md +++ /dev/null @@ -1,95 +0,0 @@ -# Changelog - -### 1.9.0-beta.0 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.9.0-beta.0@sha256:531377e4cc9dc62af40d742402222603259673f5a755a64d74122f256dfad8f9 -* registry.k8s.io/ingress-nginx/controller-chroot:v1.9.0-beta.0@sha256:60b4c95349ce2a81a3b2a76423ee483b847b89d3fa8cb148468434f606f3fa0c - -### All changes: - -* Rework mage (#10418) -* Start release of v1.9.0 beta0 (#10407) -* Update k8s versions on CI (#10406) -* Add a flag to enable or disable aio_write (#10394) -* Update external-articles.md - advanced setup with GKE/Cloud Armor/IAP (#10372) -* Fix e2e test suite doc (#10396) -* Disable user snippets per default (#10393) -* Deployment/DaemonSet: Fix templating & value. (#10240) -* Fix deferInLoop error (#10387) -* Remove gofmt (#10385) -* Deployment/DaemonSet: Template `topologySpreadConstraints`. (#10259) -* release notes 1.8.2 (#10389) -* fix: remove curl on base container #9716 (#10306) -* Fix http default backend test (#10382) -* Add golangci github action and replace the deprecated golint (#10187) -* BUGFIX incorrect indentation (#10254) -* Upgrade OpenTelemetry to v1.11.0 and gRPC to v1.57.0 (#10352) -* fix: path with special characters warning #10281 #10308 (#10330) -* Fix golangci-lint errors (#10196) -* chore(build): Fix Run make dev-env syntax error (#10294) -* Add firewall configuration to quick start documentation (#10357) -* Making auth access logs optional (#10335) -* Fix “dev-env” Makefile target to work with kubectl 1.28+ (#10350) -* fix: update action file to auto release plugin #10197 (#10321) -* Use gzip instead of pigz in CI (#10348) -* Disable Modsecurity from internal processing which affects large ingresses (#10316) -* fix: add /etc/mime.types #10309 (#10310) -* Remove curl dependencies in e2e tests #9716 (#10296) -* docs: swap explanation to match example (#10220) -* ci(helm): fix Helm Chart release action 422 error (#10237) -* helm: Use .Release.Namespace as default for ServiceMonitor namespace (#10249) -* Updated index.md - Fix typos (#10256) -* Handle request_id variable correctly in auth requests (#9219) -* test kind updates (#10272) -* promote distroless otel init image (#10257) -* [helm] configure allow to configure hostAliases (#10180) -* Add rolling update strategy to each static deployment file (#10129) -* Implement annotation validation (#9673) -* Golang 1.20.6 for test runner (#10230) -* [helm] pass service annotations through helm tpl engine (#10084) -* Ignore deployment template's replicas if KEDA is enabled (#9534) -* chore: bump OpenResty to v1.21.4.2 (#10219) -* Scanning port 10247 lead to tcp connection 502 error (#9815) -* revise Datadog trace sampling configuration (#10151) -* Clarify TCP/UDP service docs (#10146) -* Exposed continent data as variable in the case of Maxmind city files (#10157) -* Cleanup errcheck code (#10166) -* Fix golang-ci linter errors (#10128) -* Deprecate and remove AJP support (#10158) -* release notes 1.8.1 (#10161) - -### Dependency updates: - -* Bump github.com/onsi/ginkgo/v2 from 2.9.5 to 2.12.0 (#10355) -* Bump golang.org/x/crypto from 0.12.0 to 0.13.0 (#10399) -* Bump actions/setup-go from 4.0.1 to 4.1.0 (#10403) -* Bump goreleaser/goreleaser-action from 4.4.0 to 4.6.0 (#10402) -* Bump actions/upload-artifact from 3.1.2 to 3.1.3 (#10404) -* Bump golangci/golangci-lint-action from 3.6.0 to 3.7.0 (#10400) -* Bump google.golang.org/grpc from 1.57.0 to 1.58.0 (#10398) -* Bump actions/dependency-review-action from 3.0.8 to 3.1.0 (#10401) -* Bump github.com/cyphar/filepath-securejoin from 0.2.3 to 0.2.4 (#10373) -* Bump github.com/cyphar/filepath-securejoin in /magefiles (#10374) -* Bump Go version to 1.21.1 (#10377) -* Bump Go version to 1.21.1 on testrunner (#10378) -* Bump aquasecurity/trivy-action from 0.11.2 to 0.12.0 (#10365) -* Bump docker/setup-buildx-action from 2.9.1 to 2.10.0 (#10353) -* Bump actions/checkout from 3.5.3 to 3.6.0 (#10354) -* Bump actions/dependency-review-action from 3.0.6 to 3.0.8 (#10333) -* Bump actions/setup-go from 4.0.1 to 4.1.0 (#10313) -* Bump securego/gosec from 2.16.0 to 2.17.0 (#10332) -* Bump goreleaser/goreleaser-action from 4.3.0 to 4.4.0 (#10314) -* Bump github.com/opencontainers/runc from 1.1.8 to 1.1.9 (#10298) -* Bump k8s.io/component-base from 0.26.4 to 0.27.4 (Replace Topology Aware Hints with Topology Aware Routing) (#10282) -* Bump google.golang.org/grpc from 1.56.2 to 1.57.0 (#10258) -* Bump golang.org/x/crypto from 0.11.0 to 0.12.0 (#10280) -* Bump github.com/opencontainers/runc from 1.1.7 to 1.1.8 (#10244) -* Bump google.golang.org/grpc from 1.56.1 to 1.56.2 (#10193) -* Bump docker/setup-buildx-action from 2.9.0 to 2.9.1 (#10207) -* Bump golang.org/x/crypto from 0.10.0 to 0.11.0 (#10192) -* Bump docker/setup-buildx-action from 2.8.0 to 2.9.0 (#10191) -* Bump docker/setup-buildx-action from 2.7.0 to 2.8.0 (#10165) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-release-1.8...controller-controller-v1.9.0-beta.0 diff --git a/changelog/controller-1.9.0.md b/changelog/controller-1.9.0.md deleted file mode 100644 index d6b000acc..000000000 --- a/changelog/controller-1.9.0.md +++ /dev/null @@ -1,95 +0,0 @@ -# Changelog - -### 1.9.0 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.9.0@sha256:c15d1a617858d90fb8f8a2dd60b0676f2bb85c54e3ed11511794b86ec30c8c60 -* registry.k8s.io/ingress-nginx/controller-chroot:v1.9.0@sha256:d9fa7a136de2104fb2ecfcf1666978bfab927f4a125b75c0fb471e6104366ab5 - -### All changes: - -* Rework mage (#10418) -* Start release of v1.9.0 beta0 (#10407) -* Update k8s versions on CI (#10406) -* Add a flag to enable or disable aio_write (#10394) -* Update external-articles.md - advanced setup with GKE/Cloud Armor/IAP (#10372) -* Fix e2e test suite doc (#10396) -* Disable user snippets per default (#10393) -* Deployment/DaemonSet: Fix templating & value. (#10240) -* Fix deferInLoop error (#10387) -* Remove gofmt (#10385) -* Deployment/DaemonSet: Template `topologySpreadConstraints`. (#10259) -* release notes 1.8.2 (#10389) -* fix: remove curl on base container #9716 (#10306) -* Fix http default backend test (#10382) -* Add golangci github action and replace the deprecated golint (#10187) -* BUGFIX incorrect indentation (#10254) -* Upgrade OpenTelemetry to v1.11.0 and gRPC to v1.57.0 (#10352) -* fix: path with special characters warning #10281 #10308 (#10330) -* Fix golangci-lint errors (#10196) -* chore(build): Fix Run make dev-env syntax error (#10294) -* Add firewall configuration to quick start documentation (#10357) -* Making auth access logs optional (#10335) -* Fix “dev-env” Makefile target to work with kubectl 1.28+ (#10350) -* fix: update action file to auto release plugin #10197 (#10321) -* Use gzip instead of pigz in CI (#10348) -* Disable Modsecurity from internal processing which affects large ingresses (#10316) -* fix: add /etc/mime.types #10309 (#10310) -* Remove curl dependencies in e2e tests #9716 (#10296) -* docs: swap explanation to match example (#10220) -* ci(helm): fix Helm Chart release action 422 error (#10237) -* helm: Use .Release.Namespace as default for ServiceMonitor namespace (#10249) -* Updated index.md - Fix typos (#10256) -* Handle request_id variable correctly in auth requests (#9219) -* test kind updates (#10272) -* promote distroless otel init image (#10257) -* [helm] configure allow to configure hostAliases (#10180) -* Add rolling update strategy to each static deployment file (#10129) -* Implement annotation validation (#9673) -* Golang 1.20.6 for test runner (#10230) -* [helm] pass service annotations through helm tpl engine (#10084) -* Ignore deployment template's replicas if KEDA is enabled (#9534) -* chore: bump OpenResty to v1.21.4.2 (#10219) -* Scanning port 10247 lead to tcp connection 502 error (#9815) -* revise Datadog trace sampling configuration (#10151) -* Clarify TCP/UDP service docs (#10146) -* Exposed continent data as variable in the case of Maxmind city files (#10157) -* Cleanup errcheck code (#10166) -* Fix golang-ci linter errors (#10128) -* Deprecate and remove AJP support (#10158) -* release notes 1.8.1 (#10161) - -### Dependency updates: - -* Bump github.com/onsi/ginkgo/v2 from 2.9.5 to 2.12.0 (#10355) -* Bump golang.org/x/crypto from 0.12.0 to 0.13.0 (#10399) -* Bump actions/setup-go from 4.0.1 to 4.1.0 (#10403) -* Bump goreleaser/goreleaser-action from 4.4.0 to 4.6.0 (#10402) -* Bump actions/upload-artifact from 3.1.2 to 3.1.3 (#10404) -* Bump golangci/golangci-lint-action from 3.6.0 to 3.7.0 (#10400) -* Bump google.golang.org/grpc from 1.57.0 to 1.58.0 (#10398) -* Bump actions/dependency-review-action from 3.0.8 to 3.1.0 (#10401) -* Bump github.com/cyphar/filepath-securejoin from 0.2.3 to 0.2.4 (#10373) -* Bump github.com/cyphar/filepath-securejoin in /magefiles (#10374) -* Bump Go version to 1.21.1 (#10377) -* Bump Go version to 1.21.1 on testrunner (#10378) -* Bump aquasecurity/trivy-action from 0.11.2 to 0.12.0 (#10365) -* Bump docker/setup-buildx-action from 2.9.1 to 2.10.0 (#10353) -* Bump actions/checkout from 3.5.3 to 3.6.0 (#10354) -* Bump actions/dependency-review-action from 3.0.6 to 3.0.8 (#10333) -* Bump actions/setup-go from 4.0.1 to 4.1.0 (#10313) -* Bump securego/gosec from 2.16.0 to 2.17.0 (#10332) -* Bump goreleaser/goreleaser-action from 4.3.0 to 4.4.0 (#10314) -* Bump github.com/opencontainers/runc from 1.1.8 to 1.1.9 (#10298) -* Bump k8s.io/component-base from 0.26.4 to 0.27.4 (Replace Topology Aware Hints with Topology Aware Routing) (#10282) -* Bump google.golang.org/grpc from 1.56.2 to 1.57.0 (#10258) -* Bump golang.org/x/crypto from 0.11.0 to 0.12.0 (#10280) -* Bump github.com/opencontainers/runc from 1.1.7 to 1.1.8 (#10244) -* Bump google.golang.org/grpc from 1.56.1 to 1.56.2 (#10193) -* Bump docker/setup-buildx-action from 2.9.0 to 2.9.1 (#10207) -* Bump golang.org/x/crypto from 0.10.0 to 0.11.0 (#10192) -* Bump docker/setup-buildx-action from 2.8.0 to 2.9.0 (#10191) -* Bump docker/setup-buildx-action from 2.7.0 to 2.8.0 (#10165) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-release-1.8...controller-controller-v1.9.0 diff --git a/changelog/controller-1.9.1.md b/changelog/controller-1.9.1.md deleted file mode 100644 index b05c88eae..000000000 --- a/changelog/controller-1.9.1.md +++ /dev/null @@ -1,21 +0,0 @@ -# Changelog - -### 1.9.1 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.9.1@sha256:605a737877de78969493a4b1213b21de4ee425d2926906857b98050f57a95b25 -* registry.k8s.io/ingress-nginx/controller-chroot:v1.9.1@sha256:2ac744ef08850ee86ad7162451a6879f47c1a41c6a757f6b6f913c52103b8836 - -### All changes: - -* upgrade owasp modsecurity core rule set to v3.3.5 (#10437) -* Start v1.9.1 release (#10463) -* Accept backend protocol on any case (#10461) -* Chart: Rework network policies. (#10438) - -### Dependency updates: - -* Bump google.golang.org/grpc from 1.58.0 to 1.58.1 (#10436) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-controller-v1.9.0...controller-controller-v1.9.1 diff --git a/changelog/controller-1.9.3.md b/changelog/controller-1.9.3.md deleted file mode 100644 index 59ba545d9..000000000 --- a/changelog/controller-1.9.3.md +++ /dev/null @@ -1,29 +0,0 @@ -# Changelog - -### 1.9.3 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.9.3@sha256:8fd21d59428507671ce0fb47f818b1d859c92d2ad07bb7c947268d433030ba98 -* registry.k8s.io/ingress-nginx/controller-chroot:v1.9.3@sha256:df4931fd6859fbf1a71e785f02a44b2f9a16f010ae852c442e9bb779cbefdc86 - -### All changes: - -* update nginx base, httpbun, e2e, helm webhook cert gen (#10506) -* added warning for configuration-snippets usage (#10492) -* Remove legacy GeoIP from controller (#10495) -* add upstream patch for CVE-2023-44487 (#10494) -* Revert "Remove curl from nginx base image (#10477)" (#10479) -* update error and otel to have all the arch we support (#10476) -* Remove curl from nginx base image (#10477) - -### Dependency updates: - -* Bump x/net (#10514) -* Bump curl and Go version (#10503) -* Bump google.golang.org/grpc from 1.58.2 to 1.58.3 (#10496) -* Bump github.com/prometheus/client_model (#10486) -* Bump ossf/scorecard-action from 2.2.0 to 2.3.0 (#10487) -* Bump golang.org/x/crypto from 0.13.0 to 0.14.0 (#10485) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-controller-v1.9.1...controller-controller-v1.9.3 diff --git a/changelog/controller-1.9.4.md b/changelog/controller-1.9.4.md deleted file mode 100644 index 69c6ad2e3..000000000 --- a/changelog/controller-1.9.4.md +++ /dev/null @@ -1,14 +0,0 @@ -# Changelog - -### 1.9.4 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.9.4@sha256:5b161f051d017e55d358435f295f5e9a297e66158f136321d9b04520ec6c48a3 -* registry.k8s.io/ingress-nginx/controller-chroot:v1.9.4@sha256:5976b1067cfbca8a21d0ba53d71f83543a73316a61ea7f7e436d6cf84ddf9b26 - -### All changes: - -* Cherry pick fcgi fix and release v1.9.4 (#10544) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-controller-v1.9.3...controller-controller-v1.9.4 diff --git a/changelog/controller-1.9.5.md b/changelog/controller-1.9.5.md deleted file mode 100644 index 51b23ba64..000000000 --- a/changelog/controller-1.9.5.md +++ /dev/null @@ -1,37 +0,0 @@ -# Changelog - -### controller-v1.9.5 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.9.5@sha256:b3aba22b1da80e7acfc52b115cae1d4c687172cbf2b742d5b502419c25ff340e -* registry.k8s.io/ingress-nginx/controller-chroot:v1.9.5@sha256:9a8d7b25a846a6461cd044b9aea9cf6cad972bcf2e64d9fd246c0279979aad2d - -### All changes: - -* update nginx build (#10781) -* update images from golang upgrade (#10762) -* fix: remove tcpproxy copy error handling (#10715) -* Ignore fake certificate for NGINXCertificateExpiry (#10694) -* Comment NGINXCertificateExpiry alert label matcher (#10692) -* chart: allow setting allocateLoadBalancerNodePorts (#10693) -* [release-1.9] feat(helm): add documentation about metric args (#10695) -* chore(dep): change lua-resty-cookie's repo (#10691) -* annotation validation - extended URLWithNginxVariableRegex from alphaNumericChars to extendedAlphaNumeric (#10656) -* fix: adjust unfulfillable validation check for session-cookie-samesite annotation (#10604) -* fix: Validate x-forwarded-prefix annotation with RegexPathWithCapture (#10603) -* Increase HSTS max-age to default to one year (#10580) -* [release-1.9] update nginx base, httpbun, e2e, helm webhook cert gen (#10507) -* [release-1.9] add upstream patch for CVE-2023-44487 (#10499) -* fix brotli build issues (#10468) -* upgrade owasp modsecurity core rule set to v3.3.5 (#10437) -* Accept backend protocol on any case (#10461) -* Chart: Rework network policies. (#10438) -* Rework mage (#10418) - -### Dependency updates: - -* Bump x/net (#10517) -* Bump google.golang.org/grpc from 1.58.0 to 1.58.1 (#10436) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.9.4...controller-v1.9.5 diff --git a/changelog/controller-1.9.6.md b/changelog/controller-1.9.6.md deleted file mode 100644 index ccc382644..000000000 --- a/changelog/controller-1.9.6.md +++ /dev/null @@ -1,21 +0,0 @@ -# Changelog - -### controller-v1.9.6 - -Images: - -* registry.k8s.io/ingress-nginx/controller:v1.9.6@sha256:1405cc613bd95b2c6edd8b2a152510ae91c7e62aea4698500d23b2145960ab9c -* registry.k8s.io/ingress-nginx/controller-chroot:v1.9.6@sha256:7eb46ff733429e0e46892903c7394aff149ac6d284d92b3946f3baf7ff26a096 - -### All changes: - -* update web hook cert gen to latest release v20231226-1a7112e06 -* annotation validation: validate regex in common name annotation (#10880) -* change MODSECURITY_VERSION_LIB to 3.0.11 (#10879) -* Include SECLEVEL and STRENGTH as part of ssl-cipher list validation (#10871) - -### Dependency updates: - -* Bump github.com/opencontainers/runc from 1.1.10 to 1.1.11 (#10878) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/controller-v1.9.5...controller-v1.9.6 diff --git a/changelog/controller.md.gotmpl b/changelog/controller.md.gotmpl deleted file mode 100644 index 0d7ad5588..000000000 --- a/changelog/controller.md.gotmpl +++ /dev/null @@ -1,22 +0,0 @@ -# Changelog - -### {{ .NewControllerVersion }} -{{ with .ControllerImages }} -Images: -{{ range . }} -* {{ .Registry }}/{{ .Name }}:{{ .Tag}}@{{ .Digest }} -{{- end }} -{{ end }} -{{- with .Updates }} -### All changes: -{{ range . }} -* {{ . }} -{{- end }} -{{ end }} -{{- with .DepUpdates }} -### Dependency updates: -{{ range . }} -* {{ . }} -{{- end }} -{{ end }} -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/{{ .PreviousControllerVersion }}...{{ .NewControllerVersion }} diff --git a/charts/ingress-nginx/.helmignore b/charts/ingress-nginx/.helmignore index 109b40811..50af03172 100644 --- a/charts/ingress-nginx/.helmignore +++ b/charts/ingress-nginx/.helmignore @@ -20,4 +20,3 @@ .idea/ *.tmproj .vscode/ -__snapshot__ diff --git a/charts/ingress-nginx/CHANGELOG.md b/charts/ingress-nginx/CHANGELOG.md new file mode 100644 index 000000000..1711f4aa8 --- /dev/null +++ b/charts/ingress-nginx/CHANGELOG.md @@ -0,0 +1,242 @@ +# Changelog + +This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). + +### 3.32.0 + +- [7117] https://github.com/kubernetes/ingress-nginx/pull/7117 Add annotations for HPA + +### 3.31.0 + +- [7137] https://github.com/kubernetes/ingress-nginx/pull/7137 Add support for custom probes + +### 3.30.0 + +- [#7092](https://github.com/kubernetes/ingress-nginx/pull/7092) Removes the possibility of using localhost in ExternalNames as endpoints + +### 3.29.0 + +- [X] [#6945](https://github.com/kubernetes/ingress-nginx/pull/7020) Add option to specify job label for ServiceMonitor + +### 3.28.0 + +- [ ] [#6900](https://github.com/kubernetes/ingress-nginx/pull/6900) Support existing PSPs + +### 3.27.0 + +- Update ingress-nginx v0.45.0 + +### 3.26.0 + +- [X] [#6979](https://github.com/kubernetes/ingress-nginx/pull/6979) Changed servicePort value for metrics + +### 3.25.0 + +- [X] [#6957](https://github.com/kubernetes/ingress-nginx/pull/6957) Add ability to specify automountServiceAccountToken + +### 3.24.0 + +- [X] [#6908](https://github.com/kubernetes/ingress-nginx/pull/6908) Add volumes to default-backend deployment + +### 3.23.0 + +- Update ingress-nginx v0.44.0 + +### 3.22.0 + +- [X] [#6802](https://github.com/kubernetes/ingress-nginx/pull/6802) Add value for configuring a custom Diffie-Hellman parameters file +- [X] [#6815](https://github.com/kubernetes/ingress-nginx/pull/6815) Allow use of numeric namespaces in helm chart + +### 3.21.0 + +- [X] [#6783](https://github.com/kubernetes/ingress-nginx/pull/6783) Add custom annotations to ScaledObject +- [X] [#6761](https://github.com/kubernetes/ingress-nginx/pull/6761) Adding quotes in the serviceAccount name in Helm values +- [X] [#6767](https://github.com/kubernetes/ingress-nginx/pull/6767) Remove ClusterRole when scope option is enabled +- [X] [#6785](https://github.com/kubernetes/ingress-nginx/pull/6785) Update kube-webhook-certgen image to v1.5.1 + +### 3.20.1 + +- Do not create KEDA in case of DaemonSets. +- Fix KEDA v2 definition + +### 3.20.0 + +- [X] [#6730](https://github.com/kubernetes/ingress-nginx/pull/6730) Do not create HPA for defaultBackend if not enabled. + +### 3.19.0 + +- Update ingress-nginx v0.43.0 + +### 3.18.0 + +- [X] [#6688](https://github.com/kubernetes/ingress-nginx/pull/6688) Allow volume-type emptyDir in controller podsecuritypolicy +- [X] [#6691](https://github.com/kubernetes/ingress-nginx/pull/6691) Improve parsing of helm parameters + +### 3.17.0 + +- Update ingress-nginx v0.42.0 + +### 3.16.1 + +- Fix chart-releaser action + +### 3.16.0 + +- [X] [#6646](https://github.com/kubernetes/ingress-nginx/pull/6646) Added LoadBalancerIP value for internal service + +### 3.15.1 + +- Fix chart-releaser action + +### 3.15.0 + +- [X] [#6586](https://github.com/kubernetes/ingress-nginx/pull/6586) Fix 'maxmindLicenseKey' location in values.yaml + +### 3.14.0 + +- [X] [#6469](https://github.com/kubernetes/ingress-nginx/pull/6469) Allow custom service names for controller and backend + +### 3.13.0 + +- [X] [#6544](https://github.com/kubernetes/ingress-nginx/pull/6544) Fix default backend HPA name variable + +### 3.12.0 + +- [X] [#6514](https://github.com/kubernetes/ingress-nginx/pull/6514) Remove helm2 support and update docs + +### 3.11.1 + +- [X] [#6505](https://github.com/kubernetes/ingress-nginx/pull/6505) Reorder HPA resource list to work with GitOps tooling + +### 3.11.0 + +- Support Keda Autoscaling + +### 3.10.1 + +- Fix regression introduced in 0.41.0 with external authentication + +### 3.10.0 + +- Fix routing regression introduced in 0.41.0 with PathType Exact + +### 3.9.0 + +- [X] [#6423](https://github.com/kubernetes/ingress-nginx/pull/6423) Add Default backend HPA autoscaling + +### 3.8.0 + +- [X] [#6395](https://github.com/kubernetes/ingress-nginx/pull/6395) Update jettech/kube-webhook-certgen image +- [X] [#6377](https://github.com/kubernetes/ingress-nginx/pull/6377) Added loadBalancerSourceRanges for internal lbs +- [X] [#6356](https://github.com/kubernetes/ingress-nginx/pull/6356) Add securitycontext settings on defaultbackend +- [X] [#6401](https://github.com/kubernetes/ingress-nginx/pull/6401) Fix controller service annotations +- [X] [#6403](https://github.com/kubernetes/ingress-nginx/pull/6403) Initial helm chart changelog + +### 3.7.1 + +- [X] [#6326](https://github.com/kubernetes/ingress-nginx/pull/6326) Fix liveness and readiness probe path in daemonset chart + +### 3.7.0 + +- [X] [#6316](https://github.com/kubernetes/ingress-nginx/pull/6316) Numerals in podAnnotations in quotes [#6315](https://github.com/kubernetes/ingress-nginx/issues/6315) + +### 3.6.0 + +- [X] [#6305](https://github.com/kubernetes/ingress-nginx/pull/6305) Add default linux nodeSelector + +### 3.5.1 + +- [X] [#6299](https://github.com/kubernetes/ingress-nginx/pull/6299) Fix helm chart release + +### 3.5.0 + +- [X] [#6260](https://github.com/kubernetes/ingress-nginx/pull/6260) Allow Helm Chart to customize admission webhook's annotations, timeoutSeconds, namespaceSelector, objectSelector and cert files locations + +### 3.4.0 + +- [X] [#6268](https://github.com/kubernetes/ingress-nginx/pull/6268) Update to 0.40.2 in helm chart #6288 + +### 3.3.1 + +- [X] [#6259](https://github.com/kubernetes/ingress-nginx/pull/6259) Release helm chart +- [X] [#6258](https://github.com/kubernetes/ingress-nginx/pull/6258) Fix chart markdown link +- [X] [#6253](https://github.com/kubernetes/ingress-nginx/pull/6253) Release v0.40.0 + +### 3.3.1 + +- [X] [#6233](https://github.com/kubernetes/ingress-nginx/pull/6233) Add admission controller e2e test + +### 3.3.0 + +- [X] [#6203](https://github.com/kubernetes/ingress-nginx/pull/6203) Refactor parsing of key values +- [X] [#6162](https://github.com/kubernetes/ingress-nginx/pull/6162) Add helm chart options to expose metrics service as NodePort +- [X] [#6180](https://github.com/kubernetes/ingress-nginx/pull/6180) Fix helm chart admissionReviewVersions regression +- [X] [#6169](https://github.com/kubernetes/ingress-nginx/pull/6169) Fix Typo in example prometheus rules + +### 3.0.0 + +- [X] [#6167](https://github.com/kubernetes/ingress-nginx/pull/6167) Update chart requirements + +### 2.16.0 + +- [X] [#6154](https://github.com/kubernetes/ingress-nginx/pull/6154) add `topologySpreadConstraint` to controller + +### 2.15.0 + +- [X] [#6087](https://github.com/kubernetes/ingress-nginx/pull/6087) Adding parameter for externalTrafficPolicy in internal controller service spec + +### 2.14.0 + +- [X] [#6104](https://github.com/kubernetes/ingress-nginx/pull/6104) Misc fixes for nginx-ingress chart for better keel and prometheus-operator integration + +### 2.13.0 + +- [X] [#6093](https://github.com/kubernetes/ingress-nginx/pull/6093) Release v0.35.0 + +### 2.13.0 + +- [X] [#6093](https://github.com/kubernetes/ingress-nginx/pull/6093) Release v0.35.0 +- [X] [#6080](https://github.com/kubernetes/ingress-nginx/pull/6080) Switch images to k8s.gcr.io after Vanity Domain Flip + +### 2.12.1 + +- [X] [#6075](https://github.com/kubernetes/ingress-nginx/pull/6075) Sync helm chart affinity examples + +### 2.12.0 + +- [X] [#6039](https://github.com/kubernetes/ingress-nginx/pull/6039) Add configurable serviceMonitor metricRelabelling and targetLabels +- [X] [#6044](https://github.com/kubernetes/ingress-nginx/pull/6044) Fix YAML linting + +### 2.11.3 + +- [X] [#6038](https://github.com/kubernetes/ingress-nginx/pull/6038) Bump chart version PATCH + +### 2.11.2 + +- [X] [#5951](https://github.com/kubernetes/ingress-nginx/pull/5951) Bump chart patch version + +### 2.11.1 + +- [X] [#5900](https://github.com/kubernetes/ingress-nginx/pull/5900) Release helm chart for v0.34.1 + +### 2.11.0 + +- [X] [#5879](https://github.com/kubernetes/ingress-nginx/pull/5879) Update helm chart for v0.34.0 +- [X] [#5671](https://github.com/kubernetes/ingress-nginx/pull/5671) Make liveness probe more fault tolerant than readiness probe + +### 2.10.0 + +- [X] [#5843](https://github.com/kubernetes/ingress-nginx/pull/5843) Update jettech/kube-webhook-certgen image + +### 2.9.1 + +- [X] [#5823](https://github.com/kubernetes/ingress-nginx/pull/5823) Add quoting to sysctls because numeric values need to be presented as strings (#5823) + +### 2.9.0 + +- [X] [#5795](https://github.com/kubernetes/ingress-nginx/pull/5795) Use fully qualified images to avoid cri-o issues + + +### TODO + +Keep building the changelog using *git log charts* checking the tag diff --git a/charts/ingress-nginx/Chart.yaml b/charts/ingress-nginx/Chart.yaml index 11000f43d..6cecb04ab 100644 --- a/charts/ingress-nginx/Chart.yaml +++ b/charts/ingress-nginx/Chart.yaml @@ -1,24 +1,24 @@ -annotations: - artifacthub.io/changes: | - - 'CI: Fix chart testing. (#12258)' - - Update Ingress-Nginx version controller-v1.12.0 - artifacthub.io/prerelease: "false" apiVersion: v2 -appVersion: 1.12.0 -description: Ingress controller for Kubernetes using NGINX as a reverse proxy and - load balancer +name: ingress-nginx +# When the version is modified, make sure the artifacthub.io/changes list is updated +# Also update CHANGELOG.md +version: 3.33.0 +appVersion: 0.47.0 home: https://github.com/kubernetes/ingress-nginx +description: Ingress controller for Kubernetes using NGINX as a reverse proxy and load balancer icon: https://upload.wikimedia.org/wikipedia/commons/thumb/c/c5/Nginx_logo.svg/500px-Nginx_logo.svg.png keywords: -- ingress -- nginx -kubeVersion: '>=1.21.0-0' -maintainers: -- name: cpanato -- name: Gacko -- name: strongjz -- name: tao12345666333 -name: ingress-nginx + - ingress + - nginx sources: -- https://github.com/kubernetes/ingress-nginx -version: 4.12.0 + - https://github.com/kubernetes/ingress-nginx +type: application +maintainers: + - name: ChiefAlexander +engine: gotpl +kubeVersion: ">=1.16.0-0" +annotations: + # List of changes for the release in artifacthub.io + # https://artifacthub.io/packages/helm/ingress-nginx/ingress-nginx?modal=changelog + artifacthub.io/changes: | + - Update nginx to v1.20.1 diff --git a/charts/ingress-nginx/OWNERS b/charts/ingress-nginx/OWNERS index 428474f63..7aadb8dc2 100644 --- a/charts/ingress-nginx/OWNERS +++ b/charts/ingress-nginx/OWNERS @@ -1,4 +1,5 @@ -# See the OWNERS docs: https://www.kubernetes.dev/docs/guide/owners +approvers: + - ChiefAlexander -labels: -- area/helm +reviewers: + - ChiefAlexander diff --git a/charts/ingress-nginx/README.md b/charts/ingress-nginx/README.md index 7d8c1e74f..53657e56f 100644 --- a/charts/ingress-nginx/README.md +++ b/charts/ingress-nginx/README.md @@ -2,15 +2,13 @@ [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Ingress controller for Kubernetes using NGINX as a reverse proxy and load balancer -![Version: 4.12.0](https://img.shields.io/badge/Version-4.12.0-informational?style=flat-square) ![AppVersion: 1.12.0](https://img.shields.io/badge/AppVersion-1.12.0-informational?style=flat-square) - -To use, add `ingressClassName: nginx` spec field or the `kubernetes.io/ingress.class: nginx` annotation to your Ingress resources. +To use, add the `kubernetes.io/ingress.class: nginx` annotation to your Ingress resources. This chart bootstraps an ingress-nginx deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. -## Requirements +## Prerequisites -Kubernetes: `>=1.21.0-0` +- Kubernetes v1.16+ ## Get Repo Info @@ -51,6 +49,10 @@ helm upgrade [RELEASE_NAME] [CHART] --install _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ +### Upgrading With Zero Downtime in Production + +By default the ingress-nginx controller has service interruptions whenever it's pods are restarted or redeployed. In order to fix that, see the excellent blog post by Lindsay Landry from Codecademy: [Kubernetes: Nginx and Zero Downtime in Production](https://medium.com/codecademy-engineering/kubernetes-nginx-and-zero-downtime-in-production-2c910c6a5ed8). + ### Migrating from stable/nginx-ingress There are two main ways to migrate a release from `stable/nginx-ingress` to `ingress-nginx/ingress-nginx` chart: @@ -61,6 +63,7 @@ There are two main ways to migrate a release from `stable/nginx-ingress` to `ing 1. Redirect your DNS traffic from the old controller to the new controller 1. Log traffic from both controllers during this changeover 1. [Uninstall](#uninstall-chart) the old controller once traffic has fully drained from it + 1. For details on all of these steps see [Upgrading With Zero Downtime in Production](#upgrading-with-zero-downtime-in-production) Note that there are some different and upgraded configurations between the two charts, described by Rimas Mocevicius from JFrog in the "Upgrading to ingress-nginx Helm chart" section of [Migrating from Helm chart nginx-ingress to ingress-nginx](https://rimusz.net/migrating-to-ingress-nginx). As the `ingress-nginx/ingress-nginx` chart continues to update, you will want to check current differences by running [helm configuration](#configuration) commands on both charts. @@ -79,22 +82,21 @@ else it would make it impossible to evacuate a node. See [gh issue #7127](https: ### Prometheus Metrics -The Ingress-Nginx Controller can export Prometheus metrics, by setting `controller.metrics.enabled` to `true`. +The Nginx ingress controller can export Prometheus metrics, by setting `controller.metrics.enabled` to `true`. -You can add Prometheus annotations to the metrics service using `controller.metrics.service.annotations`. -Alternatively, if you use the Prometheus Operator, you can enable ServiceMonitor creation using `controller.metrics.serviceMonitor.enabled`. And set `controller.metrics.serviceMonitor.additionalLabels.release="prometheus"`. "release=prometheus" should match the label configured in the prometheus servicemonitor ( see `kubectl get servicemonitor prometheus-kube-prom-prometheus -oyaml -n prometheus`) +You can add Prometheus annotations to the metrics service using `controller.metrics.service.annotations`. Alternatively, if you use the Prometheus Operator, you can enable ServiceMonitor creation using `controller.metrics.serviceMonitor.enabled`. ### ingress-nginx nginx\_status page/stats server -Previous versions of this chart had a `controller.stats.*` configuration block, which is now obsolete due to the following changes in Ingress-Nginx Controller: +Previous versions of this chart had a `controller.stats.*` configuration block, which is now obsolete due to the following changes in nginx ingress controller: -- In [0.16.1](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0161), the vts (virtual host traffic status) dashboard was removed -- In [0.23.0](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0230), the status page at port 18080 is now a unix socket webserver only available at localhost. - You can use `curl --unix-socket /tmp/nginx-status-server.sock http://localhost/nginx_status` inside the controller container to access it locally, or use the snippet from [nginx-ingress changelog](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0230) to re-enable the http server +- In [0.16.1](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md#0161), the vts (virtual host traffic status) dashboard was removed +- In [0.23.0](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md#0230), the status page at port 18080 is now a unix socket webserver only available at localhost. + You can use `curl --unix-socket /tmp/nginx-status-server.sock http://localhost/nginx_status` inside the controller container to access it locally, or use the snippet from [nginx-ingress changelog](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md#0230) to re-enable the http server ### ExternalDNS Service Configuration -Add an [ExternalDNS](https://github.com/kubernetes-sigs/external-dns) annotation to the LoadBalancer service: +Add an [ExternalDNS](https://github.com/kubernetes-incubator/external-dns) annotation to the LoadBalancer service: ```yaml controller: @@ -105,7 +107,7 @@ controller: ### AWS L7 ELB with SSL Termination -Annotate the controller as shown in the [nginx-ingress l7 patch](https://github.com/kubernetes/ingress-nginx/blob/ab3a789caae65eec4ad6e3b46b19750b481b6bce/deploy/aws/l7/service-l7.yaml): +Annotate the controller as shown in the [nginx-ingress l7 patch](https://github.com/kubernetes/ingress-nginx/blob/master/deploy/aws/l7/service-l7.yaml): ```yaml controller: @@ -120,6 +122,19 @@ controller: service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: '3600' ``` +### AWS route53-mapper + +To configure the LoadBalancer service with the [route53-mapper addon](https://github.com/kubernetes/kops/tree/master/addons/route53-mapper), add the `domainName` annotation and `dns` label: + +```yaml +controller: + service: + labels: + dns: "route53" + annotations: + domainName: "kubernetes-example.com" +``` + ### Additional Internal Load Balancer This setup is useful when you need both external and internal load balancers but don't want to have multiple ingress controllers and multiple ingress objects per application. @@ -143,10 +158,8 @@ controller: internal: enabled: true annotations: - # Create internal NLB - service.beta.kubernetes.io/aws-load-balancer-scheme: "internal" - # Create internal ELB(Deprecated) - # service.beta.kubernetes.io/aws-load-balancer-internal: "true" + # Create internal ELB + service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0 # Any other annotation can be declared here. ``` @@ -158,13 +171,13 @@ controller: internal: enabled: true annotations: - # Create internal LB. More information: https://cloud.google.com/kubernetes-engine/docs/how-to/internal-load-balancing + # Create internal LB. More informations: https://cloud.google.com/kubernetes-engine/docs/how-to/internal-load-balancing # For GKE versions 1.17 and later networking.gke.io/load-balancer-type: "Internal" # For earlier versions # cloud.google.com/load-balancer-type: "Internal" - - # Any other annotation can be declared here. + + # Any other annotation can be declared here. ``` Example for Azure: @@ -189,34 +202,17 @@ controller: # Any other annotation can be declared here. ``` -The load balancer annotations of more cloud service providers can be found: [Internal load balancer](https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer). - An use case for this scenario is having a split-view DNS setup where the public zone CNAME records point to the external balancer URL while the private zone CNAME records point to the internal balancer URL. This way, you only need one ingress kubernetes object. Optionally you can set `controller.service.loadBalancerIP` if you need a static IP for the resulting `LoadBalancer`. ### Ingress Admission Webhooks -With nginx-ingress-controller version 0.25+, the Ingress-Nginx Controller pod exposes an endpoint that will integrate with the `validatingwebhookconfiguration` Kubernetes feature to prevent bad ingress from being added to the cluster. +With nginx-ingress-controller version 0.25+, the nginx ingress controller pod exposes an endpoint that will integrate with the `validatingwebhookconfiguration` Kubernetes feature to prevent bad ingress from being added to the cluster. **This feature is enabled by default since 0.31.0.** With nginx-ingress-controller in 0.25.* work only with kubernetes 1.14+, 0.26 fix [this issue](https://github.com/kubernetes/ingress-nginx/pull/4521) -#### How the Chart Configures the Hooks -A validating and configuration requires the endpoint to which the request is sent to use TLS. It is possible to set up custom certificates to do this, but in most cases, a self-signed certificate is enough. The setup of this component requires some more complex orchestration when using helm. The steps are created to be idempotent and to allow turning the feature on and off without running into helm quirks. - -1. A pre-install hook provisions a certificate into the same namespace using a format compatible with provisioning using end user certificates. If the certificate already exists, the hook exits. -2. The Ingress-Nginx Controller pod is configured to use a TLS proxy container, which will load that certificate. -3. Validating and Mutating webhook configurations are created in the cluster. -4. A post-install hook reads the CA from the secret created by step 1 and patches the Validating and Mutating webhook configurations. This process will allow a custom CA provisioned by some other process to also be patched into the webhook configurations. The chosen failure policy is also patched into the webhook configurations - -#### Alternatives -It should be possible to use [cert-manager/cert-manager](https://github.com/cert-manager/cert-manager) if a more complete solution is required. - -You can enable automatic self-signed TLS certificate provisioning via cert-manager by setting the `controller.admissionWebhooks.certManager.enabled` value to true. - -Please ensure that cert-manager is correctly installed and configured. - ### Helm Error When Upgrading: spec.clusterIP: Invalid value: "" If you are upgrading this chart from a version between 0.31.0 and 1.2.2 then you may get an error like this: @@ -228,349 +224,3 @@ Error: UPGRADE FAILED: Service "?????-controller" is invalid: spec.clusterIP: In Detail of how and why are in [this issue](https://github.com/helm/charts/pull/13646) but to resolve this you can set `xxxx.service.omitClusterIP` to `true` where `xxxx` is the service referenced in the error. As of version `1.26.0` of this chart, by simply not providing any clusterIP value, `invalid: spec.clusterIP: Invalid value: "": field is immutable` will no longer occur since `clusterIP: ""` will not be rendered. - -### Pod Security Admission - -You can use Pod Security Admission by applying labels to the `ingress-nginx` namespace as instructed by the [documentation](https://kubernetes.io/docs/tasks/configure-pod-container/enforce-standards-namespace-labels). - -Example: - -```yaml -apiVersion: v1 -kind: Namespace -metadata: - name: ingress-nginx - labels: - kubernetes.io/metadata.name: ingress-nginx - name: ingress-nginx - pod-security.kubernetes.io/enforce: restricted - pod-security.kubernetes.io/enforce-version: v1.31 -``` - -## Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| commonLabels | object | `{}` | | -| controller.addHeaders | object | `{}` | Will add custom headers before sending response traffic to the client according to: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#add-headers | -| controller.admissionWebhooks.annotations | object | `{}` | | -| controller.admissionWebhooks.certManager.admissionCert.duration | string | `""` | | -| controller.admissionWebhooks.certManager.enabled | bool | `false` | | -| controller.admissionWebhooks.certManager.rootCert.duration | string | `""` | | -| controller.admissionWebhooks.certificate | string | `"/usr/local/certificates/cert"` | | -| controller.admissionWebhooks.createSecretJob.name | string | `"create"` | | -| controller.admissionWebhooks.createSecretJob.resources | object | `{}` | | -| controller.admissionWebhooks.createSecretJob.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532,"seccompProfile":{"type":"RuntimeDefault"}}` | Security context for secret creation containers | -| controller.admissionWebhooks.enabled | bool | `true` | | -| controller.admissionWebhooks.extraEnvs | list | `[]` | Additional environment variables to set | -| controller.admissionWebhooks.failurePolicy | string | `"Fail"` | Admission Webhook failure policy to use | -| controller.admissionWebhooks.key | string | `"/usr/local/certificates/key"` | | -| controller.admissionWebhooks.labels | object | `{}` | Labels to be added to admission webhooks | -| controller.admissionWebhooks.name | string | `"admission"` | | -| controller.admissionWebhooks.namespaceSelector | object | `{}` | | -| controller.admissionWebhooks.objectSelector | object | `{}` | | -| controller.admissionWebhooks.patch.enabled | bool | `true` | | -| controller.admissionWebhooks.patch.image.digest | string | `"sha256:0de05718b59dc33b57ddfb4d8ad5f637cefd13eafdec0e1579d782b3483c27c3"` | | -| controller.admissionWebhooks.patch.image.image | string | `"ingress-nginx/kube-webhook-certgen"` | | -| controller.admissionWebhooks.patch.image.pullPolicy | string | `"IfNotPresent"` | | -| controller.admissionWebhooks.patch.image.tag | string | `"v1.5.1"` | | -| controller.admissionWebhooks.patch.labels | object | `{}` | Labels to be added to patch job resources | -| controller.admissionWebhooks.patch.networkPolicy.enabled | bool | `false` | Enable 'networkPolicy' or not | -| controller.admissionWebhooks.patch.nodeSelector."kubernetes.io/os" | string | `"linux"` | | -| controller.admissionWebhooks.patch.podAnnotations | object | `{}` | | -| controller.admissionWebhooks.patch.priorityClassName | string | `""` | Provide a priority class name to the webhook patching job # | -| controller.admissionWebhooks.patch.rbac | object | `{"create":true}` | Admission webhook patch job RBAC | -| controller.admissionWebhooks.patch.rbac.create | bool | `true` | Create RBAC or not | -| controller.admissionWebhooks.patch.securityContext | object | `{}` | Security context for secret creation & webhook patch pods | -| controller.admissionWebhooks.patch.serviceAccount | object | `{"automountServiceAccountToken":true,"create":true,"name":""}` | Admission webhook patch job service account | -| controller.admissionWebhooks.patch.serviceAccount.automountServiceAccountToken | bool | `true` | Auto-mount service account token or not | -| controller.admissionWebhooks.patch.serviceAccount.create | bool | `true` | Create a service account or not | -| controller.admissionWebhooks.patch.serviceAccount.name | string | `""` | Custom service account name | -| controller.admissionWebhooks.patch.tolerations | list | `[]` | | -| controller.admissionWebhooks.patchWebhookJob.name | string | `"patch"` | | -| controller.admissionWebhooks.patchWebhookJob.resources | object | `{}` | | -| controller.admissionWebhooks.patchWebhookJob.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsGroup":65532,"runAsNonRoot":true,"runAsUser":65532,"seccompProfile":{"type":"RuntimeDefault"}}` | Security context for webhook patch containers | -| controller.admissionWebhooks.port | int | `8443` | | -| controller.admissionWebhooks.service.annotations | object | `{}` | | -| controller.admissionWebhooks.service.externalIPs | list | `[]` | | -| controller.admissionWebhooks.service.loadBalancerSourceRanges | list | `[]` | | -| controller.admissionWebhooks.service.servicePort | int | `443` | | -| controller.admissionWebhooks.service.type | string | `"ClusterIP"` | | -| controller.affinity | object | `{}` | Affinity and anti-affinity rules for server scheduling to nodes # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity # | -| controller.allowSnippetAnnotations | bool | `false` | This configuration defines if Ingress Controller should allow users to set their own *-snippet annotations, otherwise this is forbidden / dropped when users add those annotations. Global snippets in ConfigMap are still respected | -| controller.annotations | object | `{}` | Annotations to be added to the controller Deployment or DaemonSet # | -| controller.autoscaling.annotations | object | `{}` | | -| controller.autoscaling.behavior | object | `{}` | | -| controller.autoscaling.enabled | bool | `false` | | -| controller.autoscaling.maxReplicas | int | `11` | | -| controller.autoscaling.minReplicas | int | `1` | | -| controller.autoscaling.targetCPUUtilizationPercentage | int | `50` | | -| controller.autoscaling.targetMemoryUtilizationPercentage | int | `50` | | -| controller.autoscalingTemplate | list | `[]` | | -| controller.config | object | `{}` | Global configuration passed to the ConfigMap consumed by the controller. Values may contain Helm templates. Ref.: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/ | -| controller.configAnnotations | object | `{}` | Annotations to be added to the controller config configuration configmap. | -| controller.configMapNamespace | string | `""` | Allows customization of the configmap / nginx-configmap namespace; defaults to $(POD_NAMESPACE) | -| controller.containerName | string | `"controller"` | Configures the controller container name | -| controller.containerPort | object | `{"http":80,"https":443}` | Configures the ports that the nginx-controller listens on | -| controller.containerSecurityContext | object | `{}` | Security context for controller containers | -| controller.customTemplate.configMapKey | string | `""` | | -| controller.customTemplate.configMapName | string | `""` | | -| controller.disableLeaderElection | bool | `false` | This configuration disable Nginx Controller Leader Election | -| controller.dnsConfig | object | `{}` | Optionally customize the pod dnsConfig. | -| controller.dnsPolicy | string | `"ClusterFirst"` | Optionally change this to ClusterFirstWithHostNet in case you have 'hostNetwork: true'. By default, while using host network, name resolution uses the host's DNS. If you wish nginx-controller to keep resolving names inside the k8s network, use ClusterFirstWithHostNet. | -| controller.electionID | string | `""` | Election ID to use for status update, by default it uses the controller name combined with a suffix of 'leader' | -| controller.electionTTL | string | `""` | Duration a leader election is valid before it's getting re-elected, e.g. `15s`, `10m` or `1h`. (Default: 30s) | -| controller.enableAnnotationValidations | bool | `true` | | -| controller.enableMimalloc | bool | `true` | Enable mimalloc as a drop-in replacement for malloc. # ref: https://github.com/microsoft/mimalloc # | -| controller.enableTopologyAwareRouting | bool | `false` | This configuration enables Topology Aware Routing feature, used together with service annotation service.kubernetes.io/topology-mode="auto" Defaults to false | -| controller.extraArgs | object | `{}` | Additional command line arguments to pass to Ingress-Nginx Controller E.g. to specify the default SSL certificate you can use | -| controller.extraContainers | list | `[]` | Additional containers to be added to the controller pod. See https://github.com/lemonldap-ng-controller/lemonldap-ng-controller as example. | -| controller.extraEnvs | list | `[]` | Additional environment variables to set | -| controller.extraInitContainers | list | `[]` | Containers, which are run before the app containers are started. | -| controller.extraModules | list | `[]` | Modules, which are mounted into the core nginx image. | -| controller.extraVolumeMounts | list | `[]` | Additional volumeMounts to the controller main container. | -| controller.extraVolumes | list | `[]` | Additional volumes to the controller pod. | -| controller.healthCheckHost | string | `""` | Address to bind the health check endpoint. It is better to set this option to the internal node address if the Ingress-Nginx Controller is running in the `hostNetwork: true` mode. | -| controller.healthCheckPath | string | `"/healthz"` | Path of the health check endpoint. All requests received on the port defined by the healthz-port parameter are forwarded internally to this path. | -| controller.hostAliases | list | `[]` | Optionally customize the pod hostAliases. | -| controller.hostNetwork | bool | `false` | Required for use with CNI based kubernetes installations (such as ones set up by kubeadm), since CNI and hostport don't mix yet. Can be deprecated once https://github.com/kubernetes/kubernetes/issues/23920 is merged | -| controller.hostPort.enabled | bool | `false` | Enable 'hostPort' or not | -| controller.hostPort.ports.http | int | `80` | 'hostPort' http port | -| controller.hostPort.ports.https | int | `443` | 'hostPort' https port | -| controller.hostname | object | `{}` | Optionally customize the pod hostname. | -| controller.image.allowPrivilegeEscalation | bool | `false` | | -| controller.image.chroot | bool | `false` | | -| controller.image.digest | string | `"sha256:e6b8de175acda6ca913891f0f727bca4527e797d52688cbe9fec9040d6f6b6fa"` | | -| controller.image.digestChroot | string | `"sha256:87c88e1c38a6c8d4483c8f70b69e2cca49853bb3ec3124b9b1be648edf139af3"` | | -| controller.image.image | string | `"ingress-nginx/controller"` | | -| controller.image.pullPolicy | string | `"IfNotPresent"` | | -| controller.image.readOnlyRootFilesystem | bool | `false` | | -| controller.image.runAsGroup | int | `82` | This value must not be changed using the official image. uid=101(www-data) gid=82(www-data) groups=82(www-data) | -| controller.image.runAsNonRoot | bool | `true` | | -| controller.image.runAsUser | int | `101` | This value must not be changed using the official image. uid=101(www-data) gid=82(www-data) groups=82(www-data) | -| controller.image.seccompProfile.type | string | `"RuntimeDefault"` | | -| controller.image.tag | string | `"v1.12.0"` | | -| controller.ingressClass | string | `"nginx"` | For backwards compatibility with ingress.class annotation, use ingressClass. Algorithm is as follows, first ingressClassName is considered, if not present, controller looks for ingress.class annotation | -| controller.ingressClassByName | bool | `false` | Process IngressClass per name (additionally as per spec.controller). | -| controller.ingressClassResource | object | `{"aliases":[],"annotations":{},"controllerValue":"k8s.io/ingress-nginx","default":false,"enabled":true,"name":"nginx","parameters":{}}` | This section refers to the creation of the IngressClass resource. IngressClasses are immutable and cannot be changed after creation. We do not support namespaced IngressClasses, yet, so a ClusterRole and a ClusterRoleBinding is required. | -| controller.ingressClassResource.aliases | list | `[]` | Aliases of this IngressClass. Creates copies with identical settings but the respective alias as name. Useful for development environments with only one Ingress Controller but production-like Ingress resources. `default` gets enabled on the original IngressClass only. | -| controller.ingressClassResource.annotations | object | `{}` | Annotations to be added to the IngressClass resource. | -| controller.ingressClassResource.controllerValue | string | `"k8s.io/ingress-nginx"` | Controller of the IngressClass. An Ingress Controller looks for IngressClasses it should reconcile by this value. This value is also being set as the `--controller-class` argument of this Ingress Controller. Ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class | -| controller.ingressClassResource.default | bool | `false` | If true, Ingresses without `ingressClassName` get assigned to this IngressClass on creation. Ingress creation gets rejected if there are multiple default IngressClasses. Ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#default-ingress-class | -| controller.ingressClassResource.enabled | bool | `true` | Create the IngressClass or not | -| controller.ingressClassResource.name | string | `"nginx"` | Name of the IngressClass | -| controller.ingressClassResource.parameters | object | `{}` | A link to a custom resource containing additional configuration for the controller. This is optional if the controller consuming this IngressClass does not require additional parameters. Ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class | -| controller.keda.apiVersion | string | `"keda.sh/v1alpha1"` | | -| controller.keda.behavior | object | `{}` | | -| controller.keda.cooldownPeriod | int | `300` | | -| controller.keda.enabled | bool | `false` | | -| controller.keda.maxReplicas | int | `11` | | -| controller.keda.minReplicas | int | `1` | | -| controller.keda.pollingInterval | int | `30` | | -| controller.keda.restoreToOriginalReplicaCount | bool | `false` | | -| controller.keda.scaledObject.annotations | object | `{}` | | -| controller.keda.triggers | list | `[]` | | -| controller.kind | string | `"Deployment"` | Use a `DaemonSet` or `Deployment` | -| controller.labels | object | `{}` | Labels to be added to the controller Deployment or DaemonSet and other resources that do not have option to specify labels # | -| controller.lifecycle | object | `{"preStop":{"exec":{"command":["/wait-shutdown"]}}}` | Improve connection draining when ingress controller pod is deleted using a lifecycle hook: With this new hook, we increased the default terminationGracePeriodSeconds from 30 seconds to 300, allowing the draining of connections up to five minutes. If the active connections end before that, the pod will terminate gracefully at that time. To effectively take advantage of this feature, the Configmap feature worker-shutdown-timeout new value is 240s instead of 10s. # | -| controller.livenessProbe.failureThreshold | int | `5` | | -| controller.livenessProbe.httpGet.path | string | `"/healthz"` | | -| controller.livenessProbe.httpGet.port | int | `10254` | | -| controller.livenessProbe.httpGet.scheme | string | `"HTTP"` | | -| controller.livenessProbe.initialDelaySeconds | int | `10` | | -| controller.livenessProbe.periodSeconds | int | `10` | | -| controller.livenessProbe.successThreshold | int | `1` | | -| controller.livenessProbe.timeoutSeconds | int | `1` | | -| controller.maxmindLicenseKey | string | `""` | Maxmind license key to download GeoLite2 Databases. # https://blog.maxmind.com/2019/12/significant-changes-to-accessing-and-using-geolite2-databases/ | -| controller.metrics.enabled | bool | `false` | | -| controller.metrics.port | int | `10254` | | -| controller.metrics.portName | string | `"metrics"` | | -| controller.metrics.prometheusRule.additionalLabels | object | `{}` | | -| controller.metrics.prometheusRule.annotations | object | `{}` | Annotations to be added to the PrometheusRule. | -| controller.metrics.prometheusRule.enabled | bool | `false` | | -| controller.metrics.prometheusRule.rules | list | `[]` | | -| controller.metrics.service.annotations | object | `{}` | | -| controller.metrics.service.enabled | bool | `true` | Enable the metrics service or not. | -| controller.metrics.service.externalIPs | list | `[]` | List of IP addresses at which the stats-exporter service is available # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-ips # | -| controller.metrics.service.labels | object | `{}` | Labels to be added to the metrics service resource | -| controller.metrics.service.loadBalancerSourceRanges | list | `[]` | | -| controller.metrics.service.servicePort | int | `10254` | | -| controller.metrics.service.type | string | `"ClusterIP"` | | -| controller.metrics.serviceMonitor.additionalLabels | object | `{}` | | -| controller.metrics.serviceMonitor.annotations | object | `{}` | Annotations to be added to the ServiceMonitor. | -| controller.metrics.serviceMonitor.enabled | bool | `false` | | -| controller.metrics.serviceMonitor.labelLimit | int | `0` | Per-scrape limit on number of labels that will be accepted for a sample. | -| controller.metrics.serviceMonitor.labelNameLengthLimit | int | `0` | Per-scrape limit on length of labels name that will be accepted for a sample. | -| controller.metrics.serviceMonitor.labelValueLengthLimit | int | `0` | Per-scrape limit on length of labels value that will be accepted for a sample. | -| controller.metrics.serviceMonitor.metricRelabelings | list | `[]` | | -| controller.metrics.serviceMonitor.namespace | string | `""` | | -| controller.metrics.serviceMonitor.namespaceSelector | object | `{}` | | -| controller.metrics.serviceMonitor.relabelings | list | `[]` | | -| controller.metrics.serviceMonitor.sampleLimit | int | `0` | Defines a per-scrape limit on the number of scraped samples that will be accepted. | -| controller.metrics.serviceMonitor.scrapeInterval | string | `"30s"` | | -| controller.metrics.serviceMonitor.targetLabels | list | `[]` | | -| controller.metrics.serviceMonitor.targetLimit | int | `0` | Defines a limit on the number of scraped targets that will be accepted. | -| controller.minAvailable | int | `1` | Minimum available pods set in PodDisruptionBudget. Define either 'minAvailable' or 'maxUnavailable', never both. | -| controller.minReadySeconds | int | `0` | `minReadySeconds` to avoid killing pods before we are ready # | -| controller.name | string | `"controller"` | | -| controller.networkPolicy.enabled | bool | `false` | Enable 'networkPolicy' or not | -| controller.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for controller pod assignment # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/ # | -| controller.podAnnotations | object | `{}` | Annotations to be added to controller pods # | -| controller.podLabels | object | `{}` | Labels to add to the pod container metadata | -| controller.podSecurityContext | object | `{}` | Security context for controller pods | -| controller.priorityClassName | string | `""` | | -| controller.progressDeadlineSeconds | int | `0` | Specifies the number of seconds you want to wait for the controller deployment to progress before the system reports back that it has failed. Ref.: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#progress-deadline-seconds | -| controller.proxySetHeaders | object | `{}` | Will add custom headers before sending traffic to backends according to https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/custom-headers | -| controller.publishService | object | `{"enabled":true,"pathOverride":""}` | Allows customization of the source of the IP address or FQDN to report in the ingress status field. By default, it reads the information provided by the service. If disable, the status field reports the IP address of the node or nodes where an ingress controller pod is running. | -| controller.publishService.enabled | bool | `true` | Enable 'publishService' or not | -| controller.publishService.pathOverride | string | `""` | Allows overriding of the publish service to bind to Must be / | -| controller.readinessProbe.failureThreshold | int | `3` | | -| controller.readinessProbe.httpGet.path | string | `"/healthz"` | | -| controller.readinessProbe.httpGet.port | int | `10254` | | -| controller.readinessProbe.httpGet.scheme | string | `"HTTP"` | | -| controller.readinessProbe.initialDelaySeconds | int | `10` | | -| controller.readinessProbe.periodSeconds | int | `10` | | -| controller.readinessProbe.successThreshold | int | `1` | | -| controller.readinessProbe.timeoutSeconds | int | `1` | | -| controller.replicaCount | int | `1` | | -| controller.reportNodeInternalIp | bool | `false` | Bare-metal considerations via the host network https://kubernetes.github.io/ingress-nginx/deploy/baremetal/#via-the-host-network Ingress status was blank because there is no Service exposing the Ingress-Nginx Controller in a configuration using the host network, the default --publish-service flag used in standard cloud setups does not apply | -| controller.resources.requests.cpu | string | `"100m"` | | -| controller.resources.requests.memory | string | `"90Mi"` | | -| controller.scope.enabled | bool | `false` | Enable 'scope' or not | -| controller.scope.namespace | string | `""` | Namespace to limit the controller to; defaults to $(POD_NAMESPACE) | -| controller.scope.namespaceSelector | string | `""` | When scope.enabled == false, instead of watching all namespaces, we watching namespaces whose labels only match with namespaceSelector. Format like foo=bar. Defaults to empty, means watching all namespaces. | -| controller.service.annotations | object | `{}` | Annotations to be added to the external controller service. See `controller.service.internal.annotations` for annotations to be added to the internal controller service. | -| controller.service.appProtocol | bool | `true` | Declare the app protocol of the external HTTP and HTTPS listeners or not. Supersedes provider-specific annotations for declaring the backend protocol. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#application-protocol | -| controller.service.clusterIP | string | `""` | Pre-defined cluster internal IP address of the external controller service. Take care of collisions with existing services. This value is immutable. Set once, it can not be changed without deleting and re-creating the service. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address | -| controller.service.clusterIPs | list | `[]` | Pre-defined cluster internal IP addresses of the external controller service. Take care of collisions with existing services. This value is immutable. Set once, it can not be changed without deleting and re-creating the service. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address | -| controller.service.enableHttp | bool | `true` | Enable the HTTP listener on both controller services or not. | -| controller.service.enableHttps | bool | `true` | Enable the HTTPS listener on both controller services or not. | -| controller.service.enabled | bool | `true` | Enable controller services or not. This does not influence the creation of either the admission webhook or the metrics service. | -| controller.service.external.enabled | bool | `true` | Enable the external controller service or not. Useful for internal-only deployments. | -| controller.service.external.labels | object | `{}` | Labels to be added to the external controller service. | -| controller.service.externalIPs | list | `[]` | List of node IP addresses at which the external controller service is available. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-ips | -| controller.service.externalTrafficPolicy | string | `""` | External traffic policy of the external controller service. Set to "Local" to preserve source IP on providers supporting it. Ref: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip | -| controller.service.internal.annotations | object | `{}` | Annotations to be added to the internal controller service. Mandatory for the internal controller service to be created. Varies with the cloud service. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer | -| controller.service.internal.appProtocol | bool | `true` | Declare the app protocol of the internal HTTP and HTTPS listeners or not. Supersedes provider-specific annotations for declaring the backend protocol. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#application-protocol | -| controller.service.internal.clusterIP | string | `""` | Pre-defined cluster internal IP address of the internal controller service. Take care of collisions with existing services. This value is immutable. Set once, it can not be changed without deleting and re-creating the service. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address | -| controller.service.internal.clusterIPs | list | `[]` | Pre-defined cluster internal IP addresses of the internal controller service. Take care of collisions with existing services. This value is immutable. Set once, it can not be changed without deleting and re-creating the service. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address | -| controller.service.internal.enabled | bool | `false` | Enable the internal controller service or not. Remember to configure `controller.service.internal.annotations` when enabling this. | -| controller.service.internal.externalIPs | list | `[]` | List of node IP addresses at which the internal controller service is available. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-ips | -| controller.service.internal.externalTrafficPolicy | string | `""` | External traffic policy of the internal controller service. Set to "Local" to preserve source IP on providers supporting it. Ref: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip | -| controller.service.internal.ipFamilies | list | `["IPv4"]` | List of IP families (e.g. IPv4, IPv6) assigned to the internal controller service. This field is usually assigned automatically based on cluster configuration and the `ipFamilyPolicy` field. Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/#services | -| controller.service.internal.ipFamilyPolicy | string | `"SingleStack"` | Represents the dual-stack capabilities of the internal controller service. Possible values are SingleStack, PreferDualStack or RequireDualStack. Fields `ipFamilies` and `clusterIP` depend on the value of this field. Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/#services | -| controller.service.internal.labels | object | `{}` | Labels to be added to the internal controller service. | -| controller.service.internal.loadBalancerClass | string | `""` | Load balancer class of the internal controller service. Used by cloud providers to select a load balancer implementation other than the cloud provider default. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-class | -| controller.service.internal.loadBalancerIP | string | `""` | Deprecated: Pre-defined IP address of the internal controller service. Used by cloud providers to connect the resulting load balancer service to a pre-existing static IP. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer | -| controller.service.internal.loadBalancerSourceRanges | list | `[]` | Restrict access to the internal controller service. Values must be CIDRs. Allows any source address by default. | -| controller.service.internal.nodePorts.http | string | `""` | Node port allocated for the internal HTTP listener. If left empty, the service controller allocates one from the configured node port range. | -| controller.service.internal.nodePorts.https | string | `""` | Node port allocated for the internal HTTPS listener. If left empty, the service controller allocates one from the configured node port range. | -| controller.service.internal.nodePorts.tcp | object | `{}` | Node port mapping for internal TCP listeners. If left empty, the service controller allocates them from the configured node port range. Example: tcp: 8080: 30080 | -| controller.service.internal.nodePorts.udp | object | `{}` | Node port mapping for internal UDP listeners. If left empty, the service controller allocates them from the configured node port range. Example: udp: 53: 30053 | -| controller.service.internal.ports | object | `{}` | | -| controller.service.internal.sessionAffinity | string | `""` | Session affinity of the internal controller service. Must be either "None" or "ClientIP" if set. Defaults to "None". Ref: https://kubernetes.io/docs/reference/networking/virtual-ips/#session-affinity | -| controller.service.internal.targetPorts | object | `{}` | | -| controller.service.internal.trafficDistribution | string | `""` | Traffic distribution policy of the internal controller service. Set to "PreferClose" to route traffic to endpoints that are topologically closer to the client. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#traffic-distribution | -| controller.service.internal.type | string | `""` | Type of the internal controller service. Defaults to the value of `controller.service.type`. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types | -| controller.service.ipFamilies | list | `["IPv4"]` | List of IP families (e.g. IPv4, IPv6) assigned to the external controller service. This field is usually assigned automatically based on cluster configuration and the `ipFamilyPolicy` field. Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/#services | -| controller.service.ipFamilyPolicy | string | `"SingleStack"` | Represents the dual-stack capabilities of the external controller service. Possible values are SingleStack, PreferDualStack or RequireDualStack. Fields `ipFamilies` and `clusterIP` depend on the value of this field. Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/#services | -| controller.service.labels | object | `{}` | Labels to be added to both controller services. | -| controller.service.loadBalancerClass | string | `""` | Load balancer class of the external controller service. Used by cloud providers to select a load balancer implementation other than the cloud provider default. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-class | -| controller.service.loadBalancerIP | string | `""` | Deprecated: Pre-defined IP address of the external controller service. Used by cloud providers to connect the resulting load balancer service to a pre-existing static IP. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer | -| controller.service.loadBalancerSourceRanges | list | `[]` | Restrict access to the external controller service. Values must be CIDRs. Allows any source address by default. | -| controller.service.nodePorts.http | string | `""` | Node port allocated for the external HTTP listener. If left empty, the service controller allocates one from the configured node port range. | -| controller.service.nodePorts.https | string | `""` | Node port allocated for the external HTTPS listener. If left empty, the service controller allocates one from the configured node port range. | -| controller.service.nodePorts.tcp | object | `{}` | Node port mapping for external TCP listeners. If left empty, the service controller allocates them from the configured node port range. Example: tcp: 8080: 30080 | -| controller.service.nodePorts.udp | object | `{}` | Node port mapping for external UDP listeners. If left empty, the service controller allocates them from the configured node port range. Example: udp: 53: 30053 | -| controller.service.ports.http | int | `80` | Port the external HTTP listener is published with. | -| controller.service.ports.https | int | `443` | Port the external HTTPS listener is published with. | -| controller.service.sessionAffinity | string | `""` | Session affinity of the external controller service. Must be either "None" or "ClientIP" if set. Defaults to "None". Ref: https://kubernetes.io/docs/reference/networking/virtual-ips/#session-affinity | -| controller.service.targetPorts.http | string | `"http"` | Port of the ingress controller the external HTTP listener is mapped to. | -| controller.service.targetPorts.https | string | `"https"` | Port of the ingress controller the external HTTPS listener is mapped to. | -| controller.service.trafficDistribution | string | `""` | Traffic distribution policy of the external controller service. Set to "PreferClose" to route traffic to endpoints that are topologically closer to the client. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#traffic-distribution | -| controller.service.type | string | `"LoadBalancer"` | Type of the external controller service. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types | -| controller.shareProcessNamespace | bool | `false` | | -| controller.sysctls | object | `{}` | sysctls for controller pods # Ref: https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ | -| controller.tcp.annotations | object | `{}` | Annotations to be added to the tcp config configmap | -| controller.tcp.configMapNamespace | string | `""` | Allows customization of the tcp-services-configmap; defaults to $(POD_NAMESPACE) | -| controller.terminationGracePeriodSeconds | int | `300` | `terminationGracePeriodSeconds` to avoid killing pods before we are ready # wait up to five minutes for the drain of connections # | -| controller.tolerations | list | `[]` | Node tolerations for server scheduling to nodes with taints # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ # | -| controller.topologySpreadConstraints | list | `[]` | Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in. # Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ # | -| controller.udp.annotations | object | `{}` | Annotations to be added to the udp config configmap | -| controller.udp.configMapNamespace | string | `""` | Allows customization of the udp-services-configmap; defaults to $(POD_NAMESPACE) | -| controller.unhealthyPodEvictionPolicy | string | `""` | Eviction policy for unhealthy pods guarded by PodDisruptionBudget. Ref: https://kubernetes.io/blog/2023/01/06/unhealthy-pod-eviction-policy-for-pdbs/ | -| controller.updateStrategy | object | `{}` | The update strategy to apply to the Deployment or DaemonSet # | -| controller.watchIngressWithoutClass | bool | `false` | Process Ingress objects without ingressClass annotation/ingressClassName field Overrides value for --watch-ingress-without-class flag of the controller binary Defaults to false | -| defaultBackend.affinity | object | `{}` | Affinity and anti-affinity rules for server scheduling to nodes # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity | -| defaultBackend.autoscaling.annotations | object | `{}` | | -| defaultBackend.autoscaling.enabled | bool | `false` | | -| defaultBackend.autoscaling.maxReplicas | int | `2` | | -| defaultBackend.autoscaling.minReplicas | int | `1` | | -| defaultBackend.autoscaling.targetCPUUtilizationPercentage | int | `50` | | -| defaultBackend.autoscaling.targetMemoryUtilizationPercentage | int | `50` | | -| defaultBackend.containerSecurityContext | object | `{}` | Security context for default backend containers | -| defaultBackend.enabled | bool | `false` | | -| defaultBackend.extraArgs | object | `{}` | | -| defaultBackend.extraConfigMaps | list | `[]` | | -| defaultBackend.extraEnvs | list | `[]` | Additional environment variables to set for defaultBackend pods | -| defaultBackend.extraVolumeMounts | list | `[]` | | -| defaultBackend.extraVolumes | list | `[]` | | -| defaultBackend.image.allowPrivilegeEscalation | bool | `false` | | -| defaultBackend.image.image | string | `"defaultbackend-amd64"` | | -| defaultBackend.image.pullPolicy | string | `"IfNotPresent"` | | -| defaultBackend.image.readOnlyRootFilesystem | bool | `true` | | -| defaultBackend.image.runAsGroup | int | `65534` | | -| defaultBackend.image.runAsNonRoot | bool | `true` | | -| defaultBackend.image.runAsUser | int | `65534` | | -| defaultBackend.image.seccompProfile.type | string | `"RuntimeDefault"` | | -| defaultBackend.image.tag | string | `"1.5"` | | -| defaultBackend.labels | object | `{}` | Labels to be added to the default backend resources | -| defaultBackend.livenessProbe.failureThreshold | int | `3` | | -| defaultBackend.livenessProbe.initialDelaySeconds | int | `30` | | -| defaultBackend.livenessProbe.periodSeconds | int | `10` | | -| defaultBackend.livenessProbe.successThreshold | int | `1` | | -| defaultBackend.livenessProbe.timeoutSeconds | int | `5` | | -| defaultBackend.minAvailable | int | `1` | Minimum available pods set in PodDisruptionBudget. Define either 'minAvailable' or 'maxUnavailable', never both. | -| defaultBackend.minReadySeconds | int | `0` | `minReadySeconds` to avoid killing pods before we are ready # | -| defaultBackend.name | string | `"defaultbackend"` | | -| defaultBackend.networkPolicy.enabled | bool | `false` | Enable 'networkPolicy' or not | -| defaultBackend.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for default backend pod assignment # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/ # | -| defaultBackend.podAnnotations | object | `{}` | Annotations to be added to default backend pods # | -| defaultBackend.podLabels | object | `{}` | Labels to add to the pod container metadata | -| defaultBackend.podSecurityContext | object | `{}` | Security context for default backend pods | -| defaultBackend.port | int | `8080` | | -| defaultBackend.priorityClassName | string | `""` | | -| defaultBackend.readinessProbe.failureThreshold | int | `6` | | -| defaultBackend.readinessProbe.initialDelaySeconds | int | `0` | | -| defaultBackend.readinessProbe.periodSeconds | int | `5` | | -| defaultBackend.readinessProbe.successThreshold | int | `1` | | -| defaultBackend.readinessProbe.timeoutSeconds | int | `5` | | -| defaultBackend.replicaCount | int | `1` | | -| defaultBackend.resources | object | `{}` | | -| defaultBackend.service.annotations | object | `{}` | | -| defaultBackend.service.clusterIPs | list | `[]` | Pre-defined cluster internal IP addresses of the default backend service. Take care of collisions with existing services. This value is immutable. Set once, it can not be changed without deleting and re-creating the service. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address | -| defaultBackend.service.externalIPs | list | `[]` | List of IP addresses at which the default backend service is available # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-ips # | -| defaultBackend.service.loadBalancerSourceRanges | list | `[]` | | -| defaultBackend.service.servicePort | int | `80` | | -| defaultBackend.service.type | string | `"ClusterIP"` | | -| defaultBackend.serviceAccount.automountServiceAccountToken | bool | `true` | | -| defaultBackend.serviceAccount.create | bool | `true` | | -| defaultBackend.serviceAccount.name | string | `""` | | -| defaultBackend.tolerations | list | `[]` | Node tolerations for server scheduling to nodes with taints # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ # | -| defaultBackend.topologySpreadConstraints | list | `[]` | Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in. Ref.: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ | -| defaultBackend.unhealthyPodEvictionPolicy | string | `""` | Eviction policy for unhealthy pods guarded by PodDisruptionBudget. Ref: https://kubernetes.io/blog/2023/01/06/unhealthy-pod-eviction-policy-for-pdbs/ | -| defaultBackend.updateStrategy | object | `{}` | The update strategy to apply to the Deployment or DaemonSet # | -| dhParam | string | `""` | A base64-encoded Diffie-Hellman parameter. This can be generated with: `openssl dhparam 4096 2> /dev/null | base64` # Ref: https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/ssl-dh-param | -| global.image.registry | string | `"registry.k8s.io"` | Registry host to pull images from. | -| imagePullSecrets | list | `[]` | Optional array of imagePullSecrets containing private registry credentials # Ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ | -| namespaceOverride | string | `""` | Override the deployment namespace; defaults to .Release.Namespace | -| portNamePrefix | string | `""` | Prefix for TCP and UDP ports names in ingress controller service # Some cloud providers, like Yandex Cloud may have a requirements for a port name regex to support cloud load balancer integration | -| rbac.create | bool | `true` | | -| rbac.scope | bool | `false` | | -| revisionHistoryLimit | int | `10` | Rollback limit # | -| serviceAccount.annotations | object | `{}` | Annotations for the controller service account | -| serviceAccount.automountServiceAccountToken | bool | `true` | | -| serviceAccount.create | bool | `true` | | -| serviceAccount.name | string | `""` | | -| tcp | object | `{}` | TCP service key-value pairs # Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md # | -| udp | object | `{}` | UDP service key-value pairs # Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md # | diff --git a/charts/ingress-nginx/README.md.gotmpl b/charts/ingress-nginx/README.md.gotmpl deleted file mode 100644 index 3cb9d5651..000000000 --- a/charts/ingress-nginx/README.md.gotmpl +++ /dev/null @@ -1,247 +0,0 @@ -{{ template "chart.header" . }} -[ingress-nginx](https://github.com/kubernetes/ingress-nginx) Ingress controller for Kubernetes using NGINX as a reverse proxy and load balancer - -{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} - -To use, add `ingressClassName: nginx` spec field or the `kubernetes.io/ingress.class: nginx` annotation to your Ingress resources. - -This chart bootstraps an ingress-nginx deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. - -{{ template "chart.requirementsSection" . }} - -## Get Repo Info - -```console -helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx -helm repo update -``` - -## Install Chart - -**Important:** only helm3 is supported - -```console -helm install [RELEASE_NAME] ingress-nginx/ingress-nginx -``` - -The command deploys ingress-nginx on the Kubernetes cluster in the default configuration. - -_See [configuration](#configuration) below._ - -_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ - -## Uninstall Chart - -```console -helm uninstall [RELEASE_NAME] -``` - -This removes all the Kubernetes components associated with the chart and deletes the release. - -_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ - -## Upgrading Chart - -```console -helm upgrade [RELEASE_NAME] [CHART] --install -``` - -_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ - -### Migrating from stable/nginx-ingress - -There are two main ways to migrate a release from `stable/nginx-ingress` to `ingress-nginx/ingress-nginx` chart: - -1. For Nginx Ingress controllers used for non-critical services, the easiest method is to [uninstall](#uninstall-chart) the old release and [install](#install-chart) the new one -1. For critical services in production that require zero-downtime, you will want to: - 1. [Install](#install-chart) a second Ingress controller - 1. Redirect your DNS traffic from the old controller to the new controller - 1. Log traffic from both controllers during this changeover - 1. [Uninstall](#uninstall-chart) the old controller once traffic has fully drained from it - -Note that there are some different and upgraded configurations between the two charts, described by Rimas Mocevicius from JFrog in the "Upgrading to ingress-nginx Helm chart" section of [Migrating from Helm chart nginx-ingress to ingress-nginx](https://rimusz.net/migrating-to-ingress-nginx). As the `ingress-nginx/ingress-nginx` chart continues to update, you will want to check current differences by running [helm configuration](#configuration) commands on both charts. - -## Configuration - -See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: - -```console -helm show values ingress-nginx/ingress-nginx -``` - -### PodDisruptionBudget - -Note that the PodDisruptionBudget resource will only be defined if the replicaCount is greater than one, -else it would make it impossible to evacuate a node. See [gh issue #7127](https://github.com/helm/charts/issues/7127) for more info. - -### Prometheus Metrics - -The Ingress-Nginx Controller can export Prometheus metrics, by setting `controller.metrics.enabled` to `true`. - -You can add Prometheus annotations to the metrics service using `controller.metrics.service.annotations`. -Alternatively, if you use the Prometheus Operator, you can enable ServiceMonitor creation using `controller.metrics.serviceMonitor.enabled`. And set `controller.metrics.serviceMonitor.additionalLabels.release="prometheus"`. "release=prometheus" should match the label configured in the prometheus servicemonitor ( see `kubectl get servicemonitor prometheus-kube-prom-prometheus -oyaml -n prometheus`) - -### ingress-nginx nginx\_status page/stats server - -Previous versions of this chart had a `controller.stats.*` configuration block, which is now obsolete due to the following changes in Ingress-Nginx Controller: - -- In [0.16.1](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0161), the vts (virtual host traffic status) dashboard was removed -- In [0.23.0](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0230), the status page at port 18080 is now a unix socket webserver only available at localhost. - You can use `curl --unix-socket /tmp/nginx-status-server.sock http://localhost/nginx_status` inside the controller container to access it locally, or use the snippet from [nginx-ingress changelog](https://github.com/kubernetes/ingress-nginx/blob/main/Changelog.md#0230) to re-enable the http server - -### ExternalDNS Service Configuration - -Add an [ExternalDNS](https://github.com/kubernetes-sigs/external-dns) annotation to the LoadBalancer service: - -```yaml -controller: - service: - annotations: - external-dns.alpha.kubernetes.io/hostname: kubernetes-example.com. -``` - -### AWS L7 ELB with SSL Termination - -Annotate the controller as shown in the [nginx-ingress l7 patch](https://github.com/kubernetes/ingress-nginx/blob/ab3a789caae65eec4ad6e3b46b19750b481b6bce/deploy/aws/l7/service-l7.yaml): - -```yaml -controller: - service: - targetPorts: - http: http - https: http - annotations: - service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:XX-XXXX-X:XXXXXXXXX:certificate/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXX - service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "http" - service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "https" - service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: '3600' -``` - -### Additional Internal Load Balancer - -This setup is useful when you need both external and internal load balancers but don't want to have multiple ingress controllers and multiple ingress objects per application. - -By default, the ingress object will point to the external load balancer address, but if correctly configured, you can make use of the internal one if the URL you are looking up resolves to the internal load balancer's URL. - -You'll need to set both the following values: - -`controller.service.internal.enabled` -`controller.service.internal.annotations` - -If one of them is missing the internal load balancer will not be deployed. Example you may have `controller.service.internal.enabled=true` but no annotations set, in this case no action will be taken. - -`controller.service.internal.annotations` varies with the cloud service you're using. - -Example for AWS: - -```yaml -controller: - service: - internal: - enabled: true - annotations: - # Create internal NLB - service.beta.kubernetes.io/aws-load-balancer-scheme: "internal" - # Create internal ELB(Deprecated) - # service.beta.kubernetes.io/aws-load-balancer-internal: "true" - # Any other annotation can be declared here. -``` - -Example for GCE: - -```yaml -controller: - service: - internal: - enabled: true - annotations: - # Create internal LB. More information: https://cloud.google.com/kubernetes-engine/docs/how-to/internal-load-balancing - # For GKE versions 1.17 and later - networking.gke.io/load-balancer-type: "Internal" - # For earlier versions - # cloud.google.com/load-balancer-type: "Internal" - - # Any other annotation can be declared here. -``` - -Example for Azure: - -```yaml -controller: - service: - annotations: - # Create internal LB - service.beta.kubernetes.io/azure-load-balancer-internal: "true" - # Any other annotation can be declared here. -``` - -Example for Oracle Cloud Infrastructure: - -```yaml -controller: - service: - annotations: - # Create internal LB - service.beta.kubernetes.io/oci-load-balancer-internal: "true" - # Any other annotation can be declared here. -``` - -The load balancer annotations of more cloud service providers can be found: [Internal load balancer](https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer). - -An use case for this scenario is having a split-view DNS setup where the public zone CNAME records point to the external balancer URL while the private zone CNAME records point to the internal balancer URL. This way, you only need one ingress kubernetes object. - -Optionally you can set `controller.service.loadBalancerIP` if you need a static IP for the resulting `LoadBalancer`. - -### Ingress Admission Webhooks - -With nginx-ingress-controller version 0.25+, the Ingress-Nginx Controller pod exposes an endpoint that will integrate with the `validatingwebhookconfiguration` Kubernetes feature to prevent bad ingress from being added to the cluster. -**This feature is enabled by default since 0.31.0.** - -With nginx-ingress-controller in 0.25.* work only with kubernetes 1.14+, 0.26 fix [this issue](https://github.com/kubernetes/ingress-nginx/pull/4521) - -#### How the Chart Configures the Hooks -A validating and configuration requires the endpoint to which the request is sent to use TLS. It is possible to set up custom certificates to do this, but in most cases, a self-signed certificate is enough. The setup of this component requires some more complex orchestration when using helm. The steps are created to be idempotent and to allow turning the feature on and off without running into helm quirks. - -1. A pre-install hook provisions a certificate into the same namespace using a format compatible with provisioning using end user certificates. If the certificate already exists, the hook exits. -2. The Ingress-Nginx Controller pod is configured to use a TLS proxy container, which will load that certificate. -3. Validating and Mutating webhook configurations are created in the cluster. -4. A post-install hook reads the CA from the secret created by step 1 and patches the Validating and Mutating webhook configurations. This process will allow a custom CA provisioned by some other process to also be patched into the webhook configurations. The chosen failure policy is also patched into the webhook configurations - -#### Alternatives -It should be possible to use [cert-manager/cert-manager](https://github.com/cert-manager/cert-manager) if a more complete solution is required. - -You can enable automatic self-signed TLS certificate provisioning via cert-manager by setting the `controller.admissionWebhooks.certManager.enabled` value to true. - -Please ensure that cert-manager is correctly installed and configured. - -### Helm Error When Upgrading: spec.clusterIP: Invalid value: "" - -If you are upgrading this chart from a version between 0.31.0 and 1.2.2 then you may get an error like this: - -```console -Error: UPGRADE FAILED: Service "?????-controller" is invalid: spec.clusterIP: Invalid value: "": field is immutable -``` - -Detail of how and why are in [this issue](https://github.com/helm/charts/pull/13646) but to resolve this you can set `xxxx.service.omitClusterIP` to `true` where `xxxx` is the service referenced in the error. - -As of version `1.26.0` of this chart, by simply not providing any clusterIP value, `invalid: spec.clusterIP: Invalid value: "": field is immutable` will no longer occur since `clusterIP: ""` will not be rendered. - -### Pod Security Admission - -You can use Pod Security Admission by applying labels to the `ingress-nginx` namespace as instructed by the [documentation](https://kubernetes.io/docs/tasks/configure-pod-container/enforce-standards-namespace-labels). - -Example: - -```yaml -apiVersion: v1 -kind: Namespace -metadata: - name: ingress-nginx - labels: - kubernetes.io/metadata.name: ingress-nginx - name: ingress-nginx - pod-security.kubernetes.io/enforce: restricted - pod-security.kubernetes.io/enforce-version: v1.31 -``` - -{{ template "chart.valuesSection" . }} diff --git a/charts/ingress-nginx/changelog/helm-chart-2.10.0.md b/charts/ingress-nginx/changelog/helm-chart-2.10.0.md deleted file mode 100644 index b42d6c28b..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-2.10.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 2.10.0 - -* [#5843](https://github.com/kubernetes/ingress-nginx/pull/5843) Update jettech/kube-webhook-certgen image - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-2.9.1...ingress-nginx-2.10.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-2.11.0.md b/charts/ingress-nginx/changelog/helm-chart-2.11.0.md deleted file mode 100644 index e549b3867..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-2.11.0.md +++ /dev/null @@ -1,10 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 2.11.0 - -* [#5879](https://github.com/kubernetes/ingress-nginx/pull/5879) Update helm chart for v0.34.0 -* [#5671](https://github.com/kubernetes/ingress-nginx/pull/5671) Make liveness probe more fault tolerant than readiness probe - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-2.10.0...ingress-nginx-2.11.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-2.11.1.md b/charts/ingress-nginx/changelog/helm-chart-2.11.1.md deleted file mode 100644 index d910d3bf4..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-2.11.1.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 2.11.1 - -* [#5900](https://github.com/kubernetes/ingress-nginx/pull/5900) Release helm chart for v0.34.1 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-2.11.0...ingress-nginx-2.11.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-2.11.2.md b/charts/ingress-nginx/changelog/helm-chart-2.11.2.md deleted file mode 100644 index 9f7821005..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-2.11.2.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 2.11.2 - -* [#5951](https://github.com/kubernetes/ingress-nginx/pull/5951) Bump chart patch version - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-2.11.1...ingress-nginx-2.11.2 diff --git a/charts/ingress-nginx/changelog/helm-chart-2.11.3.md b/charts/ingress-nginx/changelog/helm-chart-2.11.3.md deleted file mode 100644 index 344769163..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-2.11.3.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 2.11.3 - -* [#6038](https://github.com/kubernetes/ingress-nginx/pull/6038) Bump chart version PATCH - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-2.11.2...ingress-nginx-2.11.3 diff --git a/charts/ingress-nginx/changelog/helm-chart-2.12.0.md b/charts/ingress-nginx/changelog/helm-chart-2.12.0.md deleted file mode 100644 index 5cb3888aa..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-2.12.0.md +++ /dev/null @@ -1,10 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 2.12.0 - -* [#6039](https://github.com/kubernetes/ingress-nginx/pull/6039) Add configurable serviceMonitor metricRelabelling and targetLabels -* [#6044](https://github.com/kubernetes/ingress-nginx/pull/6044) Fix YAML linting - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-2.11.3...ingress-nginx-2.12.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-2.12.1.md b/charts/ingress-nginx/changelog/helm-chart-2.12.1.md deleted file mode 100644 index 94d121db5..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-2.12.1.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 2.12.1 - -* [#6075](https://github.com/kubernetes/ingress-nginx/pull/6075) Sync helm chart affinity examples - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-2.12.0...ingress-nginx-2.12.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-2.13.0.md b/charts/ingress-nginx/changelog/helm-chart-2.13.0.md deleted file mode 100644 index 01fe0b15d..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-2.13.0.md +++ /dev/null @@ -1,10 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 2.13.0 - -* [#6093](https://github.com/kubernetes/ingress-nginx/pull/6093) Release v0.35.0 -* [#6080](https://github.com/kubernetes/ingress-nginx/pull/6080) Switch images to k8s.gcr.io after Vanity Domain Flip - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-2.12.1...ingress-nginx-2.13.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-2.14.0.md b/charts/ingress-nginx/changelog/helm-chart-2.14.0.md deleted file mode 100644 index 2fb7a5a76..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-2.14.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 2.14.0 - -* [#6104](https://github.com/kubernetes/ingress-nginx/pull/6104) Misc fixes for nginx-ingress chart for better keel and prometheus-operator integration - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-2.13.0...ingress-nginx-2.14.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-2.15.0.md b/charts/ingress-nginx/changelog/helm-chart-2.15.0.md deleted file mode 100644 index 543a55927..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-2.15.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 2.15.0 - -* [#6087](https://github.com/kubernetes/ingress-nginx/pull/6087) Adding parameter for externalTrafficPolicy in internal controller service spec - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-2.14.0...ingress-nginx-2.15.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-2.16.0.md b/charts/ingress-nginx/changelog/helm-chart-2.16.0.md deleted file mode 100644 index 996f4489e..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-2.16.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 2.16.0 - -* [#6154](https://github.com/kubernetes/ingress-nginx/pull/6154) add `topologySpreadConstraint` to controller - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-2.15.0...ingress-nginx-2.16.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-2.9.0.md b/charts/ingress-nginx/changelog/helm-chart-2.9.0.md deleted file mode 100644 index 11c5f5fed..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-2.9.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 2.9.0 - -* [#5795](https://github.com/kubernetes/ingress-nginx/pull/5795) Use fully qualified images to avoid cri-o issues - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-TODO...ingress-nginx-2.9.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-2.9.1.md b/charts/ingress-nginx/changelog/helm-chart-2.9.1.md deleted file mode 100644 index 7d4314d9c..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-2.9.1.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 2.9.1 - -* [#5823](https://github.com/kubernetes/ingress-nginx/pull/5823) Add quoting to sysctls because numeric values need to be presented as strings (#5823) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-2.9.0...ingress-nginx-2.9.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.0.0.md b/charts/ingress-nginx/changelog/helm-chart-3.0.0.md deleted file mode 100644 index a7d50ee3a..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.0.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.0.0 - -* [#6167](https://github.com/kubernetes/ingress-nginx/pull/6167) Update chart requirements - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-2.16.0...ingress-nginx-3.0.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.10.0.md b/charts/ingress-nginx/changelog/helm-chart-3.10.0.md deleted file mode 100644 index 3369bed03..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.10.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.10.0 - -* Fix routing regression introduced in 0.41.0 with PathType Exact - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.9.0...ingress-nginx-3.10.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.10.1.md b/charts/ingress-nginx/changelog/helm-chart-3.10.1.md deleted file mode 100644 index 6ff682e52..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.10.1.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.10.1 - -* Fix regression introduced in 0.41.0 with external authentication - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.10.0...ingress-nginx-3.10.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.11.0.md b/charts/ingress-nginx/changelog/helm-chart-3.11.0.md deleted file mode 100644 index 69ba5506b..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.11.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.11.0 - -* Support Keda Autoscaling - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.10.1...ingress-nginx-3.11.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.11.1.md b/charts/ingress-nginx/changelog/helm-chart-3.11.1.md deleted file mode 100644 index 4e81f4b41..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.11.1.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.11.1 - -* [#6505](https://github.com/kubernetes/ingress-nginx/pull/6505) Reorder HPA resource list to work with GitOps tooling - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.11.0...ingress-nginx-3.11.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.12.0.md b/charts/ingress-nginx/changelog/helm-chart-3.12.0.md deleted file mode 100644 index 41b9744de..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.12.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.12.0 - -* [#6514](https://github.com/kubernetes/ingress-nginx/pull/6514) Remove helm2 support and update docs - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.11.1...ingress-nginx-3.12.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.13.0.md b/charts/ingress-nginx/changelog/helm-chart-3.13.0.md deleted file mode 100644 index 0855a7913..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.13.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.13.0 - -* [#6544](https://github.com/kubernetes/ingress-nginx/pull/6544) Fix default backend HPA name variable - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.12.0...ingress-nginx-3.13.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.14.0.md b/charts/ingress-nginx/changelog/helm-chart-3.14.0.md deleted file mode 100644 index e07880bf4..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.14.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.14.0 - -* [#6469](https://github.com/kubernetes/ingress-nginx/pull/6469) Allow custom service names for controller and backend - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.13.0...ingress-nginx-3.14.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.15.0.md b/charts/ingress-nginx/changelog/helm-chart-3.15.0.md deleted file mode 100644 index 3053a3548..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.15.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.15.0 - -* [#6586](https://github.com/kubernetes/ingress-nginx/pull/6586) Fix 'maxmindLicenseKey' location in values.yaml - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.14.0...ingress-nginx-3.15.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.15.1.md b/charts/ingress-nginx/changelog/helm-chart-3.15.1.md deleted file mode 100644 index f11ee0a76..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.15.1.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.15.1 - -* Fix chart-releaser action - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.15.0...ingress-nginx-3.15.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.16.0.md b/charts/ingress-nginx/changelog/helm-chart-3.16.0.md deleted file mode 100644 index fba30b171..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.16.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.16.0 - -* [#6646](https://github.com/kubernetes/ingress-nginx/pull/6646) Added LoadBalancerIP value for internal service - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.15.1...helm-chart-3.16.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.16.1.md b/charts/ingress-nginx/changelog/helm-chart-3.16.1.md deleted file mode 100644 index 650d1b8fa..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.16.1.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.16.1 - -* Fix chart-releaser action - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.16.0...helm-chart-3.16.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.17.0.md b/charts/ingress-nginx/changelog/helm-chart-3.17.0.md deleted file mode 100644 index 175c7a264..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.17.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.17.0 - -* Update ingress-nginx v0.42.0 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.16.1...helm-chart-3.17.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.18.0.md b/charts/ingress-nginx/changelog/helm-chart-3.18.0.md deleted file mode 100644 index 31b815e4e..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.18.0.md +++ /dev/null @@ -1,10 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.18.0 - -* [#6688](https://github.com/kubernetes/ingress-nginx/pull/6688) Allow volume-type emptyDir in controller podsecuritypolicy -* [#6691](https://github.com/kubernetes/ingress-nginx/pull/6691) Improve parsing of helm parameters - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.17.0...helm-chart-3.18.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.19.0.md b/charts/ingress-nginx/changelog/helm-chart-3.19.0.md deleted file mode 100644 index 0970bf02c..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.19.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.19.0 - -* Update ingress-nginx v0.43.0 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.18.0...helm-chart-3.19.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.20.0.md b/charts/ingress-nginx/changelog/helm-chart-3.20.0.md deleted file mode 100644 index 4b81ae42f..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.20.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.20.0 - -* [#6730](https://github.com/kubernetes/ingress-nginx/pull/6730) Do not create HPA for defaultBackend if not enabled. - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.19.0...helm-chart-3.20.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.20.1.md b/charts/ingress-nginx/changelog/helm-chart-3.20.1.md deleted file mode 100644 index 952bf2bd3..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.20.1.md +++ /dev/null @@ -1,10 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.20.1 - -* Do not create KEDA in case of DaemonSets. -* Fix KEDA v2 definition - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.20.0...helm-chart-3.20.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.21.0.md b/charts/ingress-nginx/changelog/helm-chart-3.21.0.md deleted file mode 100644 index 25edbefd9..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.21.0.md +++ /dev/null @@ -1,12 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.21.0 - -* [#6783](https://github.com/kubernetes/ingress-nginx/pull/6783) Add custom annotations to ScaledObject -* [#6761](https://github.com/kubernetes/ingress-nginx/pull/6761) Adding quotes in the serviceAccount name in Helm values -* [#6767](https://github.com/kubernetes/ingress-nginx/pull/6767) Remove ClusterRole when scope option is enabled -* [#6785](https://github.com/kubernetes/ingress-nginx/pull/6785) Update kube-webhook-certgen image to v1.5.1 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.20.1...helm-chart-3.21.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.22.0.md b/charts/ingress-nginx/changelog/helm-chart-3.22.0.md deleted file mode 100644 index 147d66421..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.22.0.md +++ /dev/null @@ -1,10 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.22.0 - -* [#6802](https://github.com/kubernetes/ingress-nginx/pull/6802) Add value for configuring a custom Diffie-Hellman parameters file -* [#6815](https://github.com/kubernetes/ingress-nginx/pull/6815) Allow use of numeric namespaces in helm chart - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.21.0...helm-chart-3.22.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.23.0.md b/charts/ingress-nginx/changelog/helm-chart-3.23.0.md deleted file mode 100644 index 5dcb50fa8..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.23.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.23.0 - -* Update ingress-nginx v0.44.0 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.22.0...helm-chart-3.23.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.24.0.md b/charts/ingress-nginx/changelog/helm-chart-3.24.0.md deleted file mode 100644 index d7db808b1..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.24.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.24.0 - -* [#6908](https://github.com/kubernetes/ingress-nginx/pull/6908) Add volumes to default-backend deployment - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.23.0...helm-chart-3.24.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.25.0.md b/charts/ingress-nginx/changelog/helm-chart-3.25.0.md deleted file mode 100644 index f9679a124..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.25.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.25.0 - -* [#6957](https://github.com/kubernetes/ingress-nginx/pull/6957) Add ability to specify automountServiceAccountToken - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.24.0...helm-chart-3.25.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.26.0.md b/charts/ingress-nginx/changelog/helm-chart-3.26.0.md deleted file mode 100644 index 0c3a1df68..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.26.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.26.0 - -* [#6979](https://github.com/kubernetes/ingress-nginx/pull/6979) Changed servicePort value for metrics - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.25.0...helm-chart-3.26.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.27.0.md b/charts/ingress-nginx/changelog/helm-chart-3.27.0.md deleted file mode 100644 index 8113d7b9b..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.27.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.27.0 - -* Update ingress-nginx v0.45.0 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.26.0...helm-chart-3.27.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.28.0.md b/charts/ingress-nginx/changelog/helm-chart-3.28.0.md deleted file mode 100644 index eee0ccbec..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.28.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.28.0 - -* [#6900](https://github.com/kubernetes/ingress-nginx/pull/6900) Support existing PSPs - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.27.0...helm-chart-3.28.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.29.0.md b/charts/ingress-nginx/changelog/helm-chart-3.29.0.md deleted file mode 100644 index f0fabdce1..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.29.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.29.0 - -* [#6945](https://github.com/kubernetes/ingress-nginx/pull/7020) Add option to specify job label for ServiceMonitor - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.28.0...helm-chart-3.29.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.3.0.md b/charts/ingress-nginx/changelog/helm-chart-3.3.0.md deleted file mode 100644 index 09fab3756..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.3.0.md +++ /dev/null @@ -1,12 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.3.0 - -* [#6203](https://github.com/kubernetes/ingress-nginx/pull/6203) Refactor parsing of key values -* [#6162](https://github.com/kubernetes/ingress-nginx/pull/6162) Add helm chart options to expose metrics service as NodePort -* [#6180](https://github.com/kubernetes/ingress-nginx/pull/6180) Fix helm chart admissionReviewVersions regression -* [#6169](https://github.com/kubernetes/ingress-nginx/pull/6169) Fix Typo in example prometheus rules - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.0.0...ingress-nginx-3.3.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.3.1.md b/charts/ingress-nginx/changelog/helm-chart-3.3.1.md deleted file mode 100644 index 81f44fdbd..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.3.1.md +++ /dev/null @@ -1,12 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.3.1 - -* [#6259](https://github.com/kubernetes/ingress-nginx/pull/6259) Release helm chart -* [#6258](https://github.com/kubernetes/ingress-nginx/pull/6258) Fix chart markdown link -* [#6253](https://github.com/kubernetes/ingress-nginx/pull/6253) Release v0.40.0 -* [#6233](https://github.com/kubernetes/ingress-nginx/pull/6233) Add admission controller e2e test - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.3.0...ingress-nginx-3.3.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.30.0.md b/charts/ingress-nginx/changelog/helm-chart-3.30.0.md deleted file mode 100644 index 77ad6b41b..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.30.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.30.0 - -* [#7092](https://github.com/kubernetes/ingress-nginx/pull/7092) Removes the possibility of using localhost in ExternalNames as endpoints - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.29.0...helm-chart-3.30.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.31.0.md b/charts/ingress-nginx/changelog/helm-chart-3.31.0.md deleted file mode 100644 index bc07fed76..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.31.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.31.0 - -* [7137] https://github.com/kubernetes/ingress-nginx/pull/7137 Add support for custom probes - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.30.0...helm-chart-3.31.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.32.0.md b/charts/ingress-nginx/changelog/helm-chart-3.32.0.md deleted file mode 100644 index 68f7ed6b2..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.32.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.32.0 - -* [7117] https://github.com/kubernetes/ingress-nginx/pull/7117 Add annotations for HPA - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.31.0...helm-chart-3.32.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.33.0.md b/charts/ingress-nginx/changelog/helm-chart-3.33.0.md deleted file mode 100644 index b56c5fc1e..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.33.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.33.0 - -* [7164] https://github.com/kubernetes/ingress-nginx/pull/7164 Update nginx to v1.20.1 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.32.0...helm-chart-3.33.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.34.0.md b/charts/ingress-nginx/changelog/helm-chart-3.34.0.md deleted file mode 100644 index a28cd0282..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.34.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.34.0 - -* [7256] https://github.com/kubernetes/ingress-nginx/pull/7256 Add namespace field in the namespace scoped resource templates - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.33.0...helm-chart-3.34.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.4.0.md b/charts/ingress-nginx/changelog/helm-chart-3.4.0.md deleted file mode 100644 index 3b4ca9353..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.4.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.4.0 - -* [#6268](https://github.com/kubernetes/ingress-nginx/pull/6268) Update to 0.40.2 in helm chart #6288 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.3.1...ingress-nginx-3.4.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.5.0.md b/charts/ingress-nginx/changelog/helm-chart-3.5.0.md deleted file mode 100644 index 44991b3bb..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.5.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.5.0 - -* [#6260](https://github.com/kubernetes/ingress-nginx/pull/6260) Allow Helm Chart to customize admission webhook's annotations, timeoutSeconds, namespaceSelector, objectSelector and cert files locations - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.4.0...ingress-nginx-3.5.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.5.1.md b/charts/ingress-nginx/changelog/helm-chart-3.5.1.md deleted file mode 100644 index 740f00c45..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.5.1.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.5.1 - -* [#6299](https://github.com/kubernetes/ingress-nginx/pull/6299) Fix helm chart release - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.5.0...ingress-nginx-3.5.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.6.0.md b/charts/ingress-nginx/changelog/helm-chart-3.6.0.md deleted file mode 100644 index 4af7f11a1..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.6.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.6.0 - -* [#6305](https://github.com/kubernetes/ingress-nginx/pull/6305) Add default linux nodeSelector - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.5.1...ingress-nginx-3.6.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.7.0.md b/charts/ingress-nginx/changelog/helm-chart-3.7.0.md deleted file mode 100644 index a6b12994b..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.7.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.7.0 - -* [#6316](https://github.com/kubernetes/ingress-nginx/pull/6316) Numerals in podAnnotations in quotes [#6315](https://github.com/kubernetes/ingress-nginx/issues/6315) - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.6.0...ingress-nginx-3.7.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.7.1.md b/charts/ingress-nginx/changelog/helm-chart-3.7.1.md deleted file mode 100644 index 6ba12df91..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.7.1.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.7.1 - -* [#6326](https://github.com/kubernetes/ingress-nginx/pull/6326) Fix liveness and readiness probe path in daemonset chart - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.7.0...ingress-nginx-3.7.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.8.0.md b/charts/ingress-nginx/changelog/helm-chart-3.8.0.md deleted file mode 100644 index 8df250a98..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.8.0.md +++ /dev/null @@ -1,13 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.8.0 - -* [#6395](https://github.com/kubernetes/ingress-nginx/pull/6395) Update jettech/kube-webhook-certgen image -* [#6377](https://github.com/kubernetes/ingress-nginx/pull/6377) Added loadBalancerSourceRanges for internal lbs -* [#6356](https://github.com/kubernetes/ingress-nginx/pull/6356) Add securitycontext settings on defaultbackend -* [#6401](https://github.com/kubernetes/ingress-nginx/pull/6401) Fix controller service annotations -* [#6403](https://github.com/kubernetes/ingress-nginx/pull/6403) Initial helm chart changelog - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.7.1...ingress-nginx-3.8.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-3.9.0.md b/charts/ingress-nginx/changelog/helm-chart-3.9.0.md deleted file mode 100644 index e8c9def51..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-3.9.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 3.9.0 - -* [#6423](https://github.com/kubernetes/ingress-nginx/pull/6423) Add Default backend HPA autoscaling - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/ingress-nginx-3.8.0...ingress-nginx-3.9.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.0.1.md b/charts/ingress-nginx/changelog/helm-chart-4.0.1.md deleted file mode 100644 index 7a187b350..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.0.1.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.0.1 - -* [7535] https://github.com/kubernetes/ingress-nginx/pull/7535 Release v1.0.0 ingress-nginx - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-3.34.0...helm-chart-4.0.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.0.10.md b/charts/ingress-nginx/changelog/helm-chart-4.0.10.md deleted file mode 100644 index c5d651670..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.0.10.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.0.10 - -* [7964] https://github.com/kubernetes/ingress-nginx/pull/7964 Update controller version to v1.1.0 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.0.9...helm-chart-4.0.10 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.0.11.md b/charts/ingress-nginx/changelog/helm-chart-4.0.11.md deleted file mode 100644 index 554182355..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.0.11.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.0.11 - -* [7873] https://github.com/kubernetes/ingress-nginx/pull/7873 Makes the [appProtocol](https://kubernetes.io/docs/concepts/services-networking/_print/#application-protocol) field optional. - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.0.10...helm-chart-4.0.11 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.0.12.md b/charts/ingress-nginx/changelog/helm-chart-4.0.12.md deleted file mode 100644 index 320f6f546..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.0.12.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.0.12 - -* [7978] https://github.com/kubernetes/ingress-nginx/pull/7979 Support custom annotations in admissions Jobs - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.0.11...helm-chart-4.0.12 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.0.13.md b/charts/ingress-nginx/changelog/helm-chart-4.0.13.md deleted file mode 100644 index edd922814..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.0.13.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.0.13 - -* [8008] https://github.com/kubernetes/ingress-nginx/pull/8008 Add relabelings in controller-servicemonitor.yaml - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.0.12...helm-chart-4.0.13 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.0.14.md b/charts/ingress-nginx/changelog/helm-chart-4.0.14.md deleted file mode 100644 index e92701039..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.0.14.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.0.14 - -* [8061] https://github.com/kubernetes/ingress-nginx/pull/8061 Using helm-docs to populate values table in README.md - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.0.13...helm-chart-4.0.14 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.0.15.md b/charts/ingress-nginx/changelog/helm-chart-4.0.15.md deleted file mode 100644 index d3d14a98d..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.0.15.md +++ /dev/null @@ -1,43 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.0.15 - -* [8120] https://github.com/kubernetes/ingress-nginx/pull/8120 Update go in runner and release v1.1.1 -* [8119] https://github.com/kubernetes/ingress-nginx/pull/8119 Update to go v1.17.6 -* [8118] https://github.com/kubernetes/ingress-nginx/pull/8118 Remove deprecated libraries, update other libs -* [8117] https://github.com/kubernetes/ingress-nginx/pull/8117 Fix codegen errors -* [8115] https://github.com/kubernetes/ingress-nginx/pull/8115 chart/ghaction: set the correct permission to have access to push a release -* [8098] https://github.com/kubernetes/ingress-nginx/pull/8098 generating SHA for CA only certs in backend_ssl.go + comparison of P… -* [8088] https://github.com/kubernetes/ingress-nginx/pull/8088 Fix Edit this page link to use main branch -* [8072] https://github.com/kubernetes/ingress-nginx/pull/8072 Expose GeoIP2 Continent code as variable -* [8061] https://github.com/kubernetes/ingress-nginx/pull/8061 docs(charts): using helm-docs for chart -* [8058] https://github.com/kubernetes/ingress-nginx/pull/8058 Bump github.com/spf13/cobra from 1.2.1 to 1.3.0 -* [8054] https://github.com/kubernetes/ingress-nginx/pull/8054 Bump google.golang.org/grpc from 1.41.0 to 1.43.0 -* [8051] https://github.com/kubernetes/ingress-nginx/pull/8051 align bug report with feature request regarding kind documentation -* [8046] https://github.com/kubernetes/ingress-nginx/pull/8046 Report expired certificates (#8045) -* [8044] https://github.com/kubernetes/ingress-nginx/pull/8044 remove G109 check till gosec resolves issues -* [8042] https://github.com/kubernetes/ingress-nginx/pull/8042 docs_multiple_instances_one_cluster_ticket_7543 -* [8041] https://github.com/kubernetes/ingress-nginx/pull/8041 docs: fix typo'd executable name -* [8035] https://github.com/kubernetes/ingress-nginx/pull/8035 Comment busy owners -* [8029] https://github.com/kubernetes/ingress-nginx/pull/8029 Add stream-snippet as a ConfigMap and Annotation option -* [8023] https://github.com/kubernetes/ingress-nginx/pull/8023 fix nginx compilation flags -* [8021] https://github.com/kubernetes/ingress-nginx/pull/8021 Disable default modsecurity_rules_file if modsecurity-snippet is specified -* [8019] https://github.com/kubernetes/ingress-nginx/pull/8019 Revise main documentation page -* [8018] https://github.com/kubernetes/ingress-nginx/pull/8018 Preserve order of plugin invocation -* [8015] https://github.com/kubernetes/ingress-nginx/pull/8015 Add newline indenting to admission webhook annotations -* [8014] https://github.com/kubernetes/ingress-nginx/pull/8014 Add link to example error page manifest in docs -* [8009] https://github.com/kubernetes/ingress-nginx/pull/8009 Fix spelling in documentation and top-level files -* [8008] https://github.com/kubernetes/ingress-nginx/pull/8008 Add relabelings in controller-servicemonitor.yaml -* [8003] https://github.com/kubernetes/ingress-nginx/pull/8003 Minor improvements (formatting, consistency) in install guide -* [8001] https://github.com/kubernetes/ingress-nginx/pull/8001 fix: go-grpc Dockerfile -* [7999] https://github.com/kubernetes/ingress-nginx/pull/7999 images: use k8s-staging-test-infra/gcb-docker-gcloud -* [7996] https://github.com/kubernetes/ingress-nginx/pull/7996 doc: improvement -* [7983] https://github.com/kubernetes/ingress-nginx/pull/7983 Fix a couple of misspellings in the annotations documentation. -* [7979] https://github.com/kubernetes/ingress-nginx/pull/7979 allow set annotations for admission Jobs -* [7977] https://github.com/kubernetes/ingress-nginx/pull/7977 Add ssl_reject_handshake to default server -* [7975] https://github.com/kubernetes/ingress-nginx/pull/7975 add legacy version update v0.50.0 to main changelog -* [7972] https://github.com/kubernetes/ingress-nginx/pull/7972 updated service upstream definition - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.0.14...helm-chart-4.0.15 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.0.18.md b/charts/ingress-nginx/changelog/helm-chart-4.0.18.md deleted file mode 100644 index 30a8f75c1..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.0.18.md +++ /dev/null @@ -1,40 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.0.18 - -* [8291](https://github.com/kubernetes/ingress-nginx/pull/8291) remove git tag env from cloud build -* [8286](https://github.com/kubernetes/ingress-nginx/pull/8286) Fix OpenTelemetry sidecar image build -* [8277](https://github.com/kubernetes/ingress-nginx/pull/8277) Add OpenSSF Best practices badge -* [8273](https://github.com/kubernetes/ingress-nginx/pull/8273) Issue#8241 -* [8267](https://github.com/kubernetes/ingress-nginx/pull/8267) Add fsGroup value to admission-webhooks/job-patch charts -* [8262](https://github.com/kubernetes/ingress-nginx/pull/8262) Updated confusing error -* [8256](https://github.com/kubernetes/ingress-nginx/pull/8256) fix: deny locations with invalid auth-url annotation -* [8253](https://github.com/kubernetes/ingress-nginx/pull/8253) Add a certificate info metric -* [8236](https://github.com/kubernetes/ingress-nginx/pull/8236) webhook: remove useless code. -* [8227](https://github.com/kubernetes/ingress-nginx/pull/8227) Update libraries in webhook image -* [8225](https://github.com/kubernetes/ingress-nginx/pull/8225) fix inconsistent-label-cardinality for prometheus metrics: nginx_ingress_controller_requests -* [8221](https://github.com/kubernetes/ingress-nginx/pull/8221) Do not validate ingresses with unknown ingress class in admission webhook endpoint -* [8210](https://github.com/kubernetes/ingress-nginx/pull/8210) Bump github.com/prometheus/client_golang from 1.11.0 to 1.12.1 -* [8209](https://github.com/kubernetes/ingress-nginx/pull/8209) Bump google.golang.org/grpc from 1.43.0 to 1.44.0 -* [8204](https://github.com/kubernetes/ingress-nginx/pull/8204) Add Artifact Hub lint -* [8203](https://github.com/kubernetes/ingress-nginx/pull/8203) Fix Indentation of example and link to cert-manager tutorial -* [8201](https://github.com/kubernetes/ingress-nginx/pull/8201) feat(metrics): add path and method labels to requests countera -* [8199](https://github.com/kubernetes/ingress-nginx/pull/8199) use functional options to reduce number of methods creating an EchoDeployment -* [8196](https://github.com/kubernetes/ingress-nginx/pull/8196) docs: fix inconsistent controller annotation -* [8191](https://github.com/kubernetes/ingress-nginx/pull/8191) Using Go install for misspell -* [8186](https://github.com/kubernetes/ingress-nginx/pull/8186) prometheus+grafana using servicemonitor -* [8185](https://github.com/kubernetes/ingress-nginx/pull/8185) Append elements on match, instead of removing for cors-annotations -* [8179](https://github.com/kubernetes/ingress-nginx/pull/8179) Bump github.com/opencontainers/runc from 1.0.3 to 1.1.0 -* [8173](https://github.com/kubernetes/ingress-nginx/pull/8173) Adding annotations to the controller service account -* [8163](https://github.com/kubernetes/ingress-nginx/pull/8163) Update the $req_id placeholder description -* [8162](https://github.com/kubernetes/ingress-nginx/pull/8162) Versioned static manifests -* [8159](https://github.com/kubernetes/ingress-nginx/pull/8159) Adding some geoip variables and default values -* [8155](https://github.com/kubernetes/ingress-nginx/pull/8155) #7271 feat: avoid-pdb-creation-when-default-backend-disabled-and-replicas-gt-1 -* [8151](https://github.com/kubernetes/ingress-nginx/pull/8151) Automatically generate helm docs -* [8143](https://github.com/kubernetes/ingress-nginx/pull/8143) Allow to configure delay before controller exits -* [8136](https://github.com/kubernetes/ingress-nginx/pull/8136) add ingressClass option to helm chart - back compatibility with ingress.class annotations -* [8126](https://github.com/kubernetes/ingress-nginx/pull/8126) Example for JWT - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.0.15...helm-chart-4.0.18 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.0.2.md b/charts/ingress-nginx/changelog/helm-chart-4.0.2.md deleted file mode 100644 index 9dbd003a9..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.0.2.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.0.2 - -* [7681] https://github.com/kubernetes/ingress-nginx/pull/7681 Release v1.0.1 of ingress-nginx - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.0.1...helm-chart-4.0.2 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.0.3.md b/charts/ingress-nginx/changelog/helm-chart-4.0.3.md deleted file mode 100644 index 09b89f66c..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.0.3.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.0.3 - -* [7707] https://github.com/kubernetes/ingress-nginx/pull/7707 Release v1.0.2 of ingress-nginx - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.0.2...helm-chart-4.0.3 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.0.5.md b/charts/ingress-nginx/changelog/helm-chart-4.0.5.md deleted file mode 100644 index be67704ba..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.0.5.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.0.5 - -* [7740] https://github.com/kubernetes/ingress-nginx/pull/7740 Release v1.0.3 of ingress-nginx - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.0.3...helm-chart-4.0.5 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.0.6.md b/charts/ingress-nginx/changelog/helm-chart-4.0.6.md deleted file mode 100644 index 25276e2be..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.0.6.md +++ /dev/null @@ -1,12 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.0.6 - -* [7804] https://github.com/kubernetes/ingress-nginx/pull/7804 Release v1.0.4 of ingress-nginx -* [7651] https://github.com/kubernetes/ingress-nginx/pull/7651 Support ipFamilyPolicy and ipFamilies fields in Helm Chart -* [7798] https://github.com/kubernetes/ingress-nginx/pull/7798 Exoscale: use HTTP Healthcheck mode -* [7793] https://github.com/kubernetes/ingress-nginx/pull/7793 Update kube-webhook-certgen to v1.1.1 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.0.5...helm-chart-4.0.6 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.0.7.md b/charts/ingress-nginx/changelog/helm-chart-4.0.7.md deleted file mode 100644 index 50fd9227c..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.0.7.md +++ /dev/null @@ -1,10 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.0.7 - -* [7923] https://github.com/kubernetes/ingress-nginx/pull/7923 Release v1.0.5 of ingress-nginx -* [7806] https://github.com/kubernetes/ingress-nginx/pull/7806 Choice option for internal/external loadbalancer type service - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.0.6...helm-chart-4.0.7 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.0.9.md b/charts/ingress-nginx/changelog/helm-chart-4.0.9.md deleted file mode 100644 index f2f725c93..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.0.9.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.0.9 - -* [6992] https://github.com/kubernetes/ingress-nginx/pull/6992 Add ability to specify labels for all resources - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.0.7...helm-chart-4.0.9 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.1.0.md b/charts/ingress-nginx/changelog/helm-chart-4.1.0.md deleted file mode 100644 index 24aaf49ae..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.1.0.md +++ /dev/null @@ -1,21 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.1.0 - -* [8481](https://github.com/kubernetes/ingress-nginx/pull/8481) Fix log creation in chroot script -* [8479](https://github.com/kubernetes/ingress-nginx/pull/8479) changed nginx base img tag to img built with alpine3.14.6 -* [8478](https://github.com/kubernetes/ingress-nginx/pull/8478) update base images and protobuf gomod -* [8468](https://github.com/kubernetes/ingress-nginx/pull/8468) Fallback to ngx.var.scheme for redirectScheme with use-forward-headers when X-Forwarded-Proto is empty -* [8456](https://github.com/kubernetes/ingress-nginx/pull/8456) Implement object deep inspector -* [8455](https://github.com/kubernetes/ingress-nginx/pull/8455) Update dependencies -* [8454](https://github.com/kubernetes/ingress-nginx/pull/8454) Update index.md -* [8447](https://github.com/kubernetes/ingress-nginx/pull/8447) typo fixing -* [8446](https://github.com/kubernetes/ingress-nginx/pull/8446) Fix suggested annotation-value-word-blocklist -* [8444](https://github.com/kubernetes/ingress-nginx/pull/8444) replace deprecated topology key in example with current one -* [8443](https://github.com/kubernetes/ingress-nginx/pull/8443) Add dependency review enforcement -* [8434](https://github.com/kubernetes/ingress-nginx/pull/8434) added new auth-tls-match-cn annotation -* [8426](https://github.com/kubernetes/ingress-nginx/pull/8426) Bump github.com/prometheus/common from 0.32.1 to 0.33.0 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.0.18...helm-chart-4.1.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.1.2.md b/charts/ingress-nginx/changelog/helm-chart-4.1.2.md deleted file mode 100644 index 20618557f..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.1.2.md +++ /dev/null @@ -1,11 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.1.2 - -* [8587](https://github.com/kubernetes/ingress-nginx/pull/8587) Add CAP_SYS_CHROOT to DS/PSP when needed -* [8458](https://github.com/kubernetes/ingress-nginx/pull/8458) Add portNamePrefix Helm chart parameter -* [8522](https://github.com/kubernetes/ingress-nginx/pull/8522) Add documentation for controller.service.loadBalancerIP in Helm chart - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.1.0...helm-chart-4.1.2 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.10.0.md b/charts/ingress-nginx/changelog/helm-chart-4.10.0.md deleted file mode 100644 index a1f1847ec..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.10.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.10.0 - -* - "Update Ingress-Nginx version controller-v1.10.0" - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.9.1...helm-chart-4.10.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.10.1.md b/charts/ingress-nginx/changelog/helm-chart-4.10.1.md deleted file mode 100644 index 3a28de00c..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.10.1.md +++ /dev/null @@ -1,11 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.10.1 - -* - "update post submit helm ci and clean up (#11221)" -* - "refactor helm ci tests part I (#11188)" -* - "Update Ingress-Nginx version controller-v1.10.1" - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.10.0...helm-chart-4.10.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.10.2.md b/charts/ingress-nginx/changelog/helm-chart-4.10.2.md deleted file mode 100644 index 399bd98d6..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.10.2.md +++ /dev/null @@ -1,18 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.10.2 - -* Chores: Align security contacts & chart maintainers to actual owners. (#11480) -* Fix helm install on cloud provider admonition block (#11412) -* edited helm-install tips (#11411) -* added info for aws helm install (#11410) -* add workflow to helm release and update ct for branch (#11317) -* Merge pull request #11277 from strongjz/chart-1.10.1 (#11314) -* release helm chart from release branch (#11278) -* update post submit helm ci and clean up (#11221) -* refactor helm ci tests part I (#11188) -* Update Ingress-Nginx version controller-v1.10.2 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.10.1...helm-chart-4.10.2 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.10.3.md b/charts/ingress-nginx/changelog/helm-chart-4.10.3.md deleted file mode 100644 index 3f77d405b..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.10.3.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.10.3 - -* Update Ingress-Nginx version controller-v1.10.3 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.10.2...helm-chart-4.10.3 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.10.4.md b/charts/ingress-nginx/changelog/helm-chart-4.10.4.md deleted file mode 100644 index 661d3c9bb..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.10.4.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.10.4 - -* Update Ingress-Nginx version controller-v1.10.4 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.10.3...helm-chart-4.10.4 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.10.5.md b/charts/ingress-nginx/changelog/helm-chart-4.10.5.md deleted file mode 100644 index 72c72c720..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.10.5.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.10.5 - -* Update Ingress-Nginx version controller-v1.10.5 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.10.4...helm-chart-4.10.5 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.10.6.md b/charts/ingress-nginx/changelog/helm-chart-4.10.6.md deleted file mode 100644 index c26c8817b..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.10.6.md +++ /dev/null @@ -1,10 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.10.6 - -* CI: Fix chart testing. (#12260) -* Update Ingress-Nginx version controller-v1.10.6 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.10.5...helm-chart-4.10.6 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.11.0.md b/charts/ingress-nginx/changelog/helm-chart-4.11.0.md deleted file mode 100644 index 64108c04e..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.11.0.md +++ /dev/null @@ -1,18 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.11.0 - -* Chores: Align security contacts & chart maintainers to actual owners. (#11465) -* Merge pull request #11277 from strongjz/chart-1.10.1 (#11415) -* Fix helm install on cloud provider admonition block (#11394) -* edited helm-install tips (#11393) -* added info for aws helm install (#11390) -* add workflow to helm release and update ct for branch (#11378) -* release helm chart from release branch (#11276) -* update post submit helm ci and clean up (#11220) -* refactor helm ci tests part I (#11178) -* Update Ingress-Nginx version controller-v1.11.0 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.10.2...helm-chart-4.11.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.11.1.md b/charts/ingress-nginx/changelog/helm-chart-4.11.1.md deleted file mode 100644 index 281513e5f..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.11.1.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.11.1 - -* Update Ingress-Nginx version controller-v1.11.1 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.11.0...helm-chart-4.11.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.11.2.md b/charts/ingress-nginx/changelog/helm-chart-4.11.2.md deleted file mode 100644 index c7645a5b6..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.11.2.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.11.2 - -* Update Ingress-Nginx version controller-v1.11.2 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.11.1...helm-chart-4.11.2 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.11.3.md b/charts/ingress-nginx/changelog/helm-chart-4.11.3.md deleted file mode 100644 index 18ec6ba82..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.11.3.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.11.3 - -* Update Ingress-Nginx version controller-v1.11.3 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.11.2...helm-chart-4.11.3 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.11.4.md b/charts/ingress-nginx/changelog/helm-chart-4.11.4.md deleted file mode 100644 index 003c787d3..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.11.4.md +++ /dev/null @@ -1,10 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.11.4 - -* CI: Fix chart testing. (#12259) -* Update Ingress-Nginx version controller-v1.11.4 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.11.3...helm-chart-4.11.4 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.12.0-beta.0.md b/charts/ingress-nginx/changelog/helm-chart-4.12.0-beta.0.md deleted file mode 100644 index fa980f1fb..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.12.0-beta.0.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.12.0-beta.0 - -* Update Ingress-Nginx version controller-v1.12.0-beta.0 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.11.0...helm-chart-4.12.0-beta.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.12.0.md b/charts/ingress-nginx/changelog/helm-chart-4.12.0.md deleted file mode 100644 index f8f36d499..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.12.0.md +++ /dev/null @@ -1,10 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.12.0 - -* CI: Fix chart testing. (#12258) -* Update Ingress-Nginx version controller-v1.12.0 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.11.0...helm-chart-4.12.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.2.0.md b/charts/ingress-nginx/changelog/helm-chart-4.2.0.md deleted file mode 100644 index 2074a0953..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.2.0.md +++ /dev/null @@ -1,47 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.2.0 - -* Support for Kubernetes v1.19.0 was removed -* [8810](https://github.com/kubernetes/ingress-nginx/pull/8810) Prepare for v1.3.0 -* [8808](https://github.com/kubernetes/ingress-nginx/pull/8808) revert arch var name -* [8805](https://github.com/kubernetes/ingress-nginx/pull/8805) Bump k8s.io/klog/v2 from 2.60.1 to 2.70.1 -* [8803](https://github.com/kubernetes/ingress-nginx/pull/8803) Update to nginx base with alpine v3.16 -* [8802](https://github.com/kubernetes/ingress-nginx/pull/8802) chore: start v1.3.0 release process -* [8798](https://github.com/kubernetes/ingress-nginx/pull/8798) Add v1.24.0 to test matrix -* [8796](https://github.com/kubernetes/ingress-nginx/pull/8796) fix: add MAC_OS variable for static-check -* [8793](https://github.com/kubernetes/ingress-nginx/pull/8793) changed to alpine-v3.16 -* [8781](https://github.com/kubernetes/ingress-nginx/pull/8781) Bump github.com/stretchr/testify from 1.7.5 to 1.8.0 -* [8778](https://github.com/kubernetes/ingress-nginx/pull/8778) chore: remove stable.txt from release process -* [8775](https://github.com/kubernetes/ingress-nginx/pull/8775) Remove stable -* [8773](https://github.com/kubernetes/ingress-nginx/pull/8773) Bump github/codeql-action from 2.1.14 to 2.1.15 -* [8772](https://github.com/kubernetes/ingress-nginx/pull/8772) Bump ossf/scorecard-action from 1.1.1 to 1.1.2 -* [8771](https://github.com/kubernetes/ingress-nginx/pull/8771) fix bullet md format -* [8770](https://github.com/kubernetes/ingress-nginx/pull/8770) Add condition for monitoring.coreos.com/v1 API -* [8769](https://github.com/kubernetes/ingress-nginx/pull/8769) Fix typos and add links to developer guide -* [8767](https://github.com/kubernetes/ingress-nginx/pull/8767) change v1.2.0 to v1.2.1 in deploy doc URLs -* [8765](https://github.com/kubernetes/ingress-nginx/pull/8765) Bump github/codeql-action from 1.0.26 to 2.1.14 -* [8752](https://github.com/kubernetes/ingress-nginx/pull/8752) Bump github.com/spf13/cobra from 1.4.0 to 1.5.0 -* [8751](https://github.com/kubernetes/ingress-nginx/pull/8751) Bump github.com/stretchr/testify from 1.7.2 to 1.7.5 -* [8750](https://github.com/kubernetes/ingress-nginx/pull/8750) added announcement -* [8740](https://github.com/kubernetes/ingress-nginx/pull/8740) change sha e2etestrunner and echoserver -* [8738](https://github.com/kubernetes/ingress-nginx/pull/8738) Update docs to make it easier for noobs to follow step by step -* [8737](https://github.com/kubernetes/ingress-nginx/pull/8737) updated baseimage sha -* [8736](https://github.com/kubernetes/ingress-nginx/pull/8736) set ld-musl-path -* [8733](https://github.com/kubernetes/ingress-nginx/pull/8733) feat: migrate leaderelection lock to leases -* [8726](https://github.com/kubernetes/ingress-nginx/pull/8726) prometheus metric: upstream_latency_seconds -* [8720](https://github.com/kubernetes/ingress-nginx/pull/8720) Ci pin deps -* [8719](https://github.com/kubernetes/ingress-nginx/pull/8719) Working OpenTelemetry sidecar (base nginx image) -* [8714](https://github.com/kubernetes/ingress-nginx/pull/8714) Create Openssf scorecard -* [8708](https://github.com/kubernetes/ingress-nginx/pull/8708) Bump github.com/prometheus/common from 0.34.0 to 0.35.0 -* [8703](https://github.com/kubernetes/ingress-nginx/pull/8703) Bump actions/dependency-review-action from 1 to 2 -* [8701](https://github.com/kubernetes/ingress-nginx/pull/8701) Fix several typos -* [8699](https://github.com/kubernetes/ingress-nginx/pull/8699) fix the gosec test and a make target for it -* [8698](https://github.com/kubernetes/ingress-nginx/pull/8698) Bump actions/upload-artifact from 2.3.1 to 3.1.0 -* [8697](https://github.com/kubernetes/ingress-nginx/pull/8697) Bump actions/setup-go from 2.2.0 to 3.2.0 -* [8695](https://github.com/kubernetes/ingress-nginx/pull/8695) Bump actions/download-artifact from 2 to 3 -* [8694](https://github.com/kubernetes/ingress-nginx/pull/8694) Bump crazy-max/ghaction-docker-buildx from 1.6.2 to 3.3.1 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.1.2...helm-chart-4.2.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.2.1.md b/charts/ingress-nginx/changelog/helm-chart-4.2.1.md deleted file mode 100644 index 7965bb1c2..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.2.1.md +++ /dev/null @@ -1,10 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.2.1 - -* The sha of kube-webhook-certgen image & the opentelemetry image, in values file, was changed to new images built on alpine-v3.16.1 -* [8896](https://github.com/kubernetes/ingress-nginx/pull/8896) updated to new images built today - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.2.0...helm-chart-4.2.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.3.0.md b/charts/ingress-nginx/changelog/helm-chart-4.3.0.md deleted file mode 100644 index f9dca22d9..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.3.0.md +++ /dev/null @@ -1,14 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.3.0 - -* Support for Kubernetes v.1.25.0 was added and support for endpoint slices -* Support for Kubernetes v1.20.0 and v1.21.0 was removed -* [8890](https://github.com/kubernetes/ingress-nginx/pull/8890) migrate to endpointslices -* [9059](https://github.com/kubernetes/ingress-nginx/pull/9059) kubewebhookcertgen sha change after go1191 -* [9046](https://github.com/kubernetes/ingress-nginx/pull/9046) Parameterize metrics port name -* [9104](https://github.com/kubernetes/ingress-nginx/pull/9104) Fix yaml formatting error with multiple annotations - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.2.1...helm-chart-4.3.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.4.0.md b/charts/ingress-nginx/changelog/helm-chart-4.4.0.md deleted file mode 100644 index 20f9e2336..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.4.0.md +++ /dev/null @@ -1,12 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.4.0 - -* Adding support for disabling liveness and readiness probes to the Helm chart by @njegosrailic in https://github.com/kubernetes/ingress-nginx/pull/9238 -* add:(admission-webhooks) ability to set securityContext by @ybelMekk in https://github.com/kubernetes/ingress-nginx/pull/9186 -* #7652 - Updated Helm chart to use the fullname for the electionID if not specified. by @FutureMatt in https://github.com/kubernetes/ingress-nginx/pull/9133 -* Rename controller-wehbooks-networkpolicy.yaml. by @Gacko in https://github.com/kubernetes/ingress-nginx/pull/9123 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.3.0...helm-chart-4.4.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.5.2.md b/charts/ingress-nginx/changelog/helm-chart-4.5.2.md deleted file mode 100644 index b6d8a3b0c..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.5.2.md +++ /dev/null @@ -1,13 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.5.2 - -* add lint on chart before release (#9570) -* ci: remove setup-helm step (#9404) -* feat(helm): Optionally use cert-manager instead admission patch (#9279) -* run helm release on main only and when the chart/value changes only (#9290) -* Update Ingress-Nginx version controller-v1.6.4 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.4.3...helm-chart-4.5.2 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.6.0.md b/charts/ingress-nginx/changelog/helm-chart-4.6.0.md deleted file mode 100644 index 469aaba8c..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.6.0.md +++ /dev/null @@ -1,24 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.5.3 - -* docs(helm): fix value key in readme for enabling certManager (#9640) -* Upgrade alpine 3.17.2 -* Upgrade golang 1.20 -* Drop testing/support for Kubernetes 1.23 -* docs(helm): fix value key in readme for enabling certManager (#9640) -* Update Ingress-Nginx version controller-v1.7.0 -* feat: OpenTelemetry module integration (#9062) -* canary-weight-total annotation ignored in rule backends (#9729) -* fix controller psp's volume config (#9740) -* Fix several Helm YAML issues with extraModules and extraInitContainers (#9709) -* Chart: Drop `controller.headers`, rework DH param secret. (#9659) -* Deployment/DaemonSet: Label pods using `ingress-nginx.labels`. (#9732) -* HPA: autoscaling/v2beta1 deprecated, bump apiVersion to v2 for defaultBackend (#9731) -* Fix incorrect annotation name in upstream hashing configuration (#9617) - -* Update Ingress-Nginx version controller-v1.7.0 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.5.2...helm-chart-4.6.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.6.1.md b/charts/ingress-nginx/changelog/helm-chart-4.6.1.md deleted file mode 100644 index 57d99b8db..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.6.1.md +++ /dev/null @@ -1,11 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.6.1 - -* [helm] Support custom port configuration for internal service (#9846) -* Adding resource type to default HPA configuration to resolve issues with Terraform helm chart usage (#9803) -* Update Ingress-Nginx version controller-v1.7.1 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.6.0...helm-chart-4.6.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.7.0.md b/charts/ingress-nginx/changelog/helm-chart-4.7.0.md deleted file mode 100644 index 9d5407053..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.7.0.md +++ /dev/null @@ -1,14 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.7.0 - -* helm: Fix opentelemetry module installation for daemonset (#9792) -* Update charts/* to keep project name display aligned (#9931) -* HPA: Use capabilities & align manifests. (#9521) -* PodDisruptionBudget spec logic update (#9904) -* add option for annotations in PodDisruptionBudget (#9843) -* Update Ingress-Nginx version controller-v1.8.0 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.6.1...helm-chart-4.7.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.7.1.md b/charts/ingress-nginx/changelog/helm-chart-4.7.1.md deleted file mode 100644 index 4d69a7117..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.7.1.md +++ /dev/null @@ -1,12 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.7.1 - -* Added a doc line to the missing helm value service.internal.loadBalancerIP (#9406) -* feat(helm): Add loadBalancerClass (#9562) -* added helmshowvalues example (#10019) -* Update Ingress-Nginx version controller-v1.8.1 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.7.0...helm-chart-4.7.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.7.2.md b/charts/ingress-nginx/changelog/helm-chart-4.7.2.md deleted file mode 100644 index 57b17b982..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.7.2.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.7.2 - -* Update Ingress-Nginx version controller-v1.8.2 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.7.1...helm-chart-4.7.2 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.8.0-beta.0.md b/charts/ingress-nginx/changelog/helm-chart-4.8.0-beta.0.md deleted file mode 100644 index 9072a75b4..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.8.0-beta.0.md +++ /dev/null @@ -1,13 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.8.0-beta.0 - -* ci(helm): fix Helm Chart release action 422 error (#10237) -* helm: Use .Release.Namespace as default for ServiceMonitor namespace (#10249) -* [helm] configure allow to configure hostAliases (#10180) -* [helm] pass service annotations through helm tpl engine (#10084) -* Update Ingress-Nginx version controller-v1.9.0-beta.0 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.7.2...helm-chart-4.8.0-beta.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.8.0.md b/charts/ingress-nginx/changelog/helm-chart-4.8.0.md deleted file mode 100644 index af8f1241f..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.8.0.md +++ /dev/null @@ -1,13 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.8.0 - -* ci(helm): fix Helm Chart release action 422 error (#10237) -* helm: Use .Release.Namespace as default for ServiceMonitor namespace (#10249) -* [helm] configure allow to configure hostAliases (#10180) -* [helm] pass service annotations through helm tpl engine (#10084) -* Update Ingress-Nginx version controller-v1.9.0 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.7.2...helm-chart-4.8.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.8.1.md b/charts/ingress-nginx/changelog/helm-chart-4.8.1.md deleted file mode 100644 index 53a4493de..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.8.1.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.8.1 - -* Update Ingress-Nginx version controller-v1.9.1 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.8.0...helm-chart-4.8.1 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.8.2.md b/charts/ingress-nginx/changelog/helm-chart-4.8.2.md deleted file mode 100644 index 9957c1a85..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.8.2.md +++ /dev/null @@ -1,10 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.8.2 - -* update nginx base, httpbun, e2e, helm webhook cert gen (#10506) -* Update Ingress-Nginx version controller-v1.9.3 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.8.1...helm-chart-4.8.2 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.8.3.md b/charts/ingress-nginx/changelog/helm-chart-4.8.3.md deleted file mode 100644 index b8d4d56b3..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.8.3.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.8.3 - -* Update Ingress-Nginx version controller-v1.9.4 - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.8.2...helm-chart-4.8.3 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.9.0.md b/charts/ingress-nginx/changelog/helm-chart-4.9.0.md deleted file mode 100644 index 5c7729866..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.9.0.md +++ /dev/null @@ -1,13 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.9.0 - -* - "Add controller.metrics.serviceMonitor.annotations in Helm chart" -* - "fix(labels): use complete labels variable on default-backend deployment" -* - "chart: allow setting allocateLoadBalancerNodePorts (#10693)" -* - "[release-1.9] feat(helm): add documentation about metric args (#10695)" -* - "Update Ingress-Nginx version controller-v1.9.5" - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.8.3...helm-chart-4.9.0 diff --git a/charts/ingress-nginx/changelog/helm-chart-4.9.1.md b/charts/ingress-nginx/changelog/helm-chart-4.9.1.md deleted file mode 100644 index c6120e736..000000000 --- a/charts/ingress-nginx/changelog/helm-chart-4.9.1.md +++ /dev/null @@ -1,10 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### 4.9.1 - -* - "update web hook cert gen to latest release v20231226-1a7112e06" -* - "Update Ingress-Nginx version controller-v1.9.6" - -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-4.9.0...helm-chart-4.9.1 diff --git a/charts/ingress-nginx/changelog/helm-chart.md.gotmpl b/charts/ingress-nginx/changelog/helm-chart.md.gotmpl deleted file mode 100644 index ef5add55d..000000000 --- a/charts/ingress-nginx/changelog/helm-chart.md.gotmpl +++ /dev/null @@ -1,11 +0,0 @@ -# Changelog - -This file documents all notable changes to [ingress-nginx](https://github.com/kubernetes/ingress-nginx) Helm Chart. The release numbering uses [semantic versioning](http://semver.org). - -### {{ .NewHelmChartVersion }} -{{ with .HelmUpdates }} -{{- range . }} -* {{ . }} -{{- end }} -{{ end }} -**Full Changelog**: https://github.com/kubernetes/ingress-nginx/compare/helm-chart-{{ .PreviousHelmChartVersion }}...helm-chart-{{ .NewHelmChartVersion }} diff --git a/charts/ingress-nginx/ci/admission-webhooks-cert-manager-values.yaml b/charts/ingress-nginx/ci/admission-webhooks-cert-manager-values.yaml deleted file mode 100644 index 7eafd0c5b..000000000 --- a/charts/ingress-nginx/ci/admission-webhooks-cert-manager-values.yaml +++ /dev/null @@ -1,12 +0,0 @@ -controller: - image: - repository: ingress-controller/controller - tag: 1.0.0-dev - digest: null - - service: - type: ClusterIP - - admissionWebhooks: - certManager: - enabled: true diff --git a/charts/ingress-nginx/ci/controller-configmap-addheaders-values.yaml b/charts/ingress-nginx/ci/controller-configmap-addheaders-values.yaml deleted file mode 100644 index 460a610ba..000000000 --- a/charts/ingress-nginx/ci/controller-configmap-addheaders-values.yaml +++ /dev/null @@ -1,11 +0,0 @@ -controller: - image: - repository: ingress-controller/controller - tag: 1.0.0-dev - digest: null - - service: - type: ClusterIP - - addHeaders: - X-Frame-Options: deny diff --git a/charts/ingress-nginx/ci/controller-configmap-proxyheaders-values.yaml b/charts/ingress-nginx/ci/controller-configmap-proxyheaders-values.yaml deleted file mode 100644 index e23a13c0c..000000000 --- a/charts/ingress-nginx/ci/controller-configmap-proxyheaders-values.yaml +++ /dev/null @@ -1,11 +0,0 @@ -controller: - image: - repository: ingress-controller/controller - tag: 1.0.0-dev - digest: null - - service: - type: ClusterIP - - proxySetHeaders: - X-Forwarded-Proto: https diff --git a/charts/ingress-nginx/ci/controller-configmap-values.yaml b/charts/ingress-nginx/ci/controller-configmap-values.yaml deleted file mode 100644 index a7029895c..000000000 --- a/charts/ingress-nginx/ci/controller-configmap-values.yaml +++ /dev/null @@ -1,11 +0,0 @@ -controller: - image: - repository: ingress-controller/controller - tag: 1.0.0-dev - digest: null - - service: - type: ClusterIP - - config: - use-proxy-protocol: "true" diff --git a/charts/ingress-nginx/ci/controller-daemonset-metrics-values.yaml b/charts/ingress-nginx/ci/controller-daemonset-metrics-values.yaml deleted file mode 100644 index 7a98580cd..000000000 --- a/charts/ingress-nginx/ci/controller-daemonset-metrics-values.yaml +++ /dev/null @@ -1,13 +0,0 @@ -controller: - image: - repository: ingress-controller/controller - tag: 1.0.0-dev - digest: null - - service: - type: ClusterIP - - kind: DaemonSet - - metrics: - enabled: true diff --git a/charts/ingress-nginx/ci/controller-daemonset-values.yaml b/charts/ingress-nginx/ci/controller-daemonset-values.yaml deleted file mode 100644 index d34025c80..000000000 --- a/charts/ingress-nginx/ci/controller-daemonset-values.yaml +++ /dev/null @@ -1,10 +0,0 @@ -controller: - image: - repository: ingress-controller/controller - tag: 1.0.0-dev - digest: null - - service: - type: ClusterIP - - kind: DaemonSet diff --git a/charts/ingress-nginx/ci/controller-deployment-metrics-values.yaml b/charts/ingress-nginx/ci/controller-deployment-metrics-values.yaml deleted file mode 100644 index 9c95d347c..000000000 --- a/charts/ingress-nginx/ci/controller-deployment-metrics-values.yaml +++ /dev/null @@ -1,13 +0,0 @@ -controller: - image: - repository: ingress-controller/controller - tag: 1.0.0-dev - digest: null - - service: - type: ClusterIP - - kind: Deployment - - metrics: - enabled: true diff --git a/charts/ingress-nginx/ci/controller-deployment-values.yaml b/charts/ingress-nginx/ci/controller-deployment-values.yaml deleted file mode 100644 index 1b092dc0c..000000000 --- a/charts/ingress-nginx/ci/controller-deployment-values.yaml +++ /dev/null @@ -1,10 +0,0 @@ -controller: - image: - repository: ingress-controller/controller - tag: 1.0.0-dev - digest: null - - service: - type: ClusterIP - - kind: Deployment diff --git a/charts/ingress-nginx/ci/controller-hpa-values.yaml b/charts/ingress-nginx/ci/controller-hpa-values.yaml deleted file mode 100644 index 54a0d2f75..000000000 --- a/charts/ingress-nginx/ci/controller-hpa-values.yaml +++ /dev/null @@ -1,18 +0,0 @@ -controller: - image: - repository: ingress-controller/controller - tag: 1.0.0-dev - digest: null - - service: - type: ClusterIP - - autoscaling: - enabled: true - behavior: - scaleDown: - stabilizationWindowSeconds: 300 - policies: - - type: Pods - value: 1 - periodSeconds: 180 diff --git a/charts/ingress-nginx/ci/controller-ingressclass-values.yaml b/charts/ingress-nginx/ci/controller-ingressclass-values.yaml deleted file mode 100644 index c06429f97..000000000 --- a/charts/ingress-nginx/ci/controller-ingressclass-values.yaml +++ /dev/null @@ -1,15 +0,0 @@ -controller: - image: - repository: ingress-controller/controller - tag: 1.0.0-dev - digest: null - - service: - type: ClusterIP - - ingressClassResource: - name: custom-nginx - default: true - controllerValue: k8s.io/custom-nginx - - watchIngressWithoutClass: true diff --git a/charts/ingress-nginx/ci/controller-service-internal-values.yaml b/charts/ingress-nginx/ci/controller-service-internal-values.yaml deleted file mode 100644 index 01635e339..000000000 --- a/charts/ingress-nginx/ci/controller-service-internal-values.yaml +++ /dev/null @@ -1,15 +0,0 @@ -controller: - image: - repository: ingress-controller/controller - tag: 1.0.0-dev - digest: null - - service: - type: ClusterIP - - internal: - enabled: true - labels: - external-dns.alpha.kubernetes.io/hostname: internal.example.com - annotations: - service.beta.kubernetes.io/aws-load-balancer-internal: "true" diff --git a/charts/ingress-nginx/ci/controller-service-values.yaml b/charts/ingress-nginx/ci/controller-service-values.yaml deleted file mode 100644 index 4ba3debba..000000000 --- a/charts/ingress-nginx/ci/controller-service-values.yaml +++ /dev/null @@ -1,26 +0,0 @@ -controller: - image: - repository: ingress-controller/controller - tag: 1.0.0-dev - digest: null - - service: - type: NodePort - - external: - labels: - external-dns.alpha.kubernetes.io/hostname: external.example.com - - nodePorts: - tcp: - 9000: 30090 - udp: - 9001: 30091 - -portNamePrefix: port - -tcp: - 9000: default/test:8080 - -udp: - 9001: default/test:8080 diff --git a/charts/ingress-nginx/ci/daemonset-customconfig-values.yaml b/charts/ingress-nginx/ci/daemonset-customconfig-values.yaml new file mode 100644 index 000000000..e12b53421 --- /dev/null +++ b/charts/ingress-nginx/ci/daemonset-customconfig-values.yaml @@ -0,0 +1,9 @@ +controller: + kind: DaemonSet + admissionWebhooks: + enabled: false + service: + type: ClusterIP + + config: + use-proxy-protocol: "true" diff --git a/charts/ingress-nginx/ci/daemonset-customnodeport-values.yaml b/charts/ingress-nginx/ci/daemonset-customnodeport-values.yaml new file mode 100644 index 000000000..cfc545f69 --- /dev/null +++ b/charts/ingress-nginx/ci/daemonset-customnodeport-values.yaml @@ -0,0 +1,18 @@ +controller: + kind: DaemonSet + admissionWebhooks: + enabled: false + + service: + type: NodePort + nodePorts: + tcp: + 9000: 30090 + udp: + 9001: 30091 + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/charts/ingress-nginx/ci/daemonset-headers-values.yaml b/charts/ingress-nginx/ci/daemonset-headers-values.yaml new file mode 100644 index 000000000..ff82cd9c7 --- /dev/null +++ b/charts/ingress-nginx/ci/daemonset-headers-values.yaml @@ -0,0 +1,10 @@ +controller: + kind: DaemonSet + admissionWebhooks: + enabled: false + addHeaders: + X-Frame-Options: deny + proxySetHeaders: + X-Forwarded-Proto: https + service: + type: ClusterIP diff --git a/charts/ingress-nginx/ci/daemonset-internal-lb-values.yaml b/charts/ingress-nginx/ci/daemonset-internal-lb-values.yaml new file mode 100644 index 000000000..443e39d8b --- /dev/null +++ b/charts/ingress-nginx/ci/daemonset-internal-lb-values.yaml @@ -0,0 +1,10 @@ +controller: + kind: DaemonSet + admissionWebhooks: + enabled: false + service: + type: ClusterIP + internal: + enabled: true + annotations: + service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0 diff --git a/charts/ingress-nginx/ci/daemonset-nodeport-values.yaml b/charts/ingress-nginx/ci/daemonset-nodeport-values.yaml new file mode 100644 index 000000000..6d6605f0e --- /dev/null +++ b/charts/ingress-nginx/ci/daemonset-nodeport-values.yaml @@ -0,0 +1,6 @@ +controller: + kind: DaemonSet + admissionWebhooks: + enabled: false + service: + type: NodePort diff --git a/charts/ingress-nginx/ci/controller-daemonset-podannotations-values.yaml b/charts/ingress-nginx/ci/daemonset-podannotations-values.yaml similarity index 68% rename from charts/ingress-nginx/ci/controller-daemonset-podannotations-values.yaml rename to charts/ingress-nginx/ci/daemonset-podannotations-values.yaml index 405992ef3..04ac58dbd 100644 --- a/charts/ingress-nginx/ci/controller-daemonset-podannotations-values.yaml +++ b/charts/ingress-nginx/ci/daemonset-podannotations-values.yaml @@ -1,16 +1,13 @@ controller: - image: - repository: ingress-controller/controller - tag: 1.0.0-dev - digest: null - + kind: DaemonSet + admissionWebhooks: + enabled: false + metrics: + enabled: true service: type: ClusterIP - - kind: DaemonSet - podAnnotations: - prometheus.io/scrape: "true" + prometheus.io/path: /metrics prometheus.io/port: "10254" prometheus.io/scheme: http - prometheus.io/path: /metrics + prometheus.io/scrape: "true" diff --git a/charts/ingress-nginx/ci/daemonset-tcp-udp-configMapNamespace-values.yaml b/charts/ingress-nginx/ci/daemonset-tcp-udp-configMapNamespace-values.yaml new file mode 100644 index 000000000..afb5487c5 --- /dev/null +++ b/charts/ingress-nginx/ci/daemonset-tcp-udp-configMapNamespace-values.yaml @@ -0,0 +1,16 @@ +controller: + kind: DaemonSet + admissionWebhooks: + enabled: false + service: + type: ClusterIP + tcp: + configMapNamespace: default + udp: + configMapNamespace: default + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/charts/ingress-nginx/ci/daemonset-tcp-udp-values.yaml b/charts/ingress-nginx/ci/daemonset-tcp-udp-values.yaml new file mode 100644 index 000000000..7b4d7cbe7 --- /dev/null +++ b/charts/ingress-nginx/ci/daemonset-tcp-udp-values.yaml @@ -0,0 +1,12 @@ +controller: + kind: DaemonSet + admissionWebhooks: + enabled: false + service: + type: ClusterIP + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/charts/ingress-nginx/ci/daemonset-tcp-values.yaml b/charts/ingress-nginx/ci/daemonset-tcp-values.yaml new file mode 100644 index 000000000..a359a6a40 --- /dev/null +++ b/charts/ingress-nginx/ci/daemonset-tcp-values.yaml @@ -0,0 +1,10 @@ +controller: + kind: DaemonSet + admissionWebhooks: + enabled: false + service: + type: ClusterIP + +tcp: + 9000: "default/test:8080" + 9001: "default/test:8080" diff --git a/charts/ingress-nginx/ci/deamonset-default-values.yaml b/charts/ingress-nginx/ci/deamonset-default-values.yaml new file mode 100644 index 000000000..e63a7f5db --- /dev/null +++ b/charts/ingress-nginx/ci/deamonset-default-values.yaml @@ -0,0 +1,6 @@ +controller: + kind: DaemonSet + admissionWebhooks: + enabled: false + service: + type: ClusterIP diff --git a/charts/ingress-nginx/ci/deamonset-metrics-values.yaml b/charts/ingress-nginx/ci/deamonset-metrics-values.yaml new file mode 100644 index 000000000..1e5190afc --- /dev/null +++ b/charts/ingress-nginx/ci/deamonset-metrics-values.yaml @@ -0,0 +1,8 @@ +controller: + kind: DaemonSet + admissionWebhooks: + enabled: false + metrics: + enabled: true + service: + type: ClusterIP diff --git a/charts/ingress-nginx/ci/deamonset-psp-values.yaml b/charts/ingress-nginx/ci/deamonset-psp-values.yaml new file mode 100644 index 000000000..017b60a9c --- /dev/null +++ b/charts/ingress-nginx/ci/deamonset-psp-values.yaml @@ -0,0 +1,9 @@ +controller: + kind: DaemonSet + admissionWebhooks: + enabled: false + service: + type: ClusterIP + +podSecurityPolicy: + enabled: true diff --git a/charts/ingress-nginx/ci/deamonset-webhook-and-psp-values.yaml b/charts/ingress-nginx/ci/deamonset-webhook-and-psp-values.yaml new file mode 100644 index 000000000..88aafc66f --- /dev/null +++ b/charts/ingress-nginx/ci/deamonset-webhook-and-psp-values.yaml @@ -0,0 +1,9 @@ +controller: + kind: DaemonSet + admissionWebhooks: + enabled: true + service: + type: ClusterIP + +podSecurityPolicy: + enabled: true diff --git a/charts/ingress-nginx/ci/deamonset-webhook-values.yaml b/charts/ingress-nginx/ci/deamonset-webhook-values.yaml new file mode 100644 index 000000000..6e3b371da --- /dev/null +++ b/charts/ingress-nginx/ci/deamonset-webhook-values.yaml @@ -0,0 +1,6 @@ +controller: + kind: DaemonSet + admissionWebhooks: + enabled: true + service: + type: ClusterIP diff --git a/charts/ingress-nginx/ci/deployment-autoscaling-values.yaml b/charts/ingress-nginx/ci/deployment-autoscaling-values.yaml new file mode 100644 index 000000000..5314cecb3 --- /dev/null +++ b/charts/ingress-nginx/ci/deployment-autoscaling-values.yaml @@ -0,0 +1,7 @@ +controller: + autoscaling: + enabled: true + admissionWebhooks: + enabled: false + service: + type: ClusterIP diff --git a/charts/ingress-nginx/ci/deployment-customconfig-values.yaml b/charts/ingress-nginx/ci/deployment-customconfig-values.yaml new file mode 100644 index 000000000..f232531ac --- /dev/null +++ b/charts/ingress-nginx/ci/deployment-customconfig-values.yaml @@ -0,0 +1,7 @@ +controller: + config: + use-proxy-protocol: "true" + admissionWebhooks: + enabled: false + service: + type: ClusterIP diff --git a/charts/ingress-nginx/ci/deployment-customnodeport-values.yaml b/charts/ingress-nginx/ci/deployment-customnodeport-values.yaml new file mode 100644 index 000000000..9eda282b1 --- /dev/null +++ b/charts/ingress-nginx/ci/deployment-customnodeport-values.yaml @@ -0,0 +1,16 @@ +controller: + admissionWebhooks: + enabled: false + service: + type: NodePort + nodePorts: + tcp: + 9000: 30090 + udp: + 9001: 30091 + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/charts/ingress-nginx/ci/deployment-default-values.yaml b/charts/ingress-nginx/ci/deployment-default-values.yaml new file mode 100644 index 000000000..93a393c97 --- /dev/null +++ b/charts/ingress-nginx/ci/deployment-default-values.yaml @@ -0,0 +1,4 @@ +# Left blank to test default values +controller: + service: + type: ClusterIP diff --git a/charts/ingress-nginx/ci/deployment-headers-values.yaml b/charts/ingress-nginx/ci/deployment-headers-values.yaml new file mode 100644 index 000000000..665fd48d3 --- /dev/null +++ b/charts/ingress-nginx/ci/deployment-headers-values.yaml @@ -0,0 +1,9 @@ +controller: + admissionWebhooks: + enabled: false + addHeaders: + X-Frame-Options: deny + proxySetHeaders: + X-Forwarded-Proto: https + service: + type: ClusterIP diff --git a/charts/ingress-nginx/ci/deployment-internal-lb-values.yaml b/charts/ingress-nginx/ci/deployment-internal-lb-values.yaml new file mode 100644 index 000000000..892f6de3f --- /dev/null +++ b/charts/ingress-nginx/ci/deployment-internal-lb-values.yaml @@ -0,0 +1,9 @@ +controller: + admissionWebhooks: + enabled: false + service: + type: ClusterIP + internal: + enabled: true + annotations: + service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0 diff --git a/charts/ingress-nginx/ci/deployment-metrics-values.yaml b/charts/ingress-nginx/ci/deployment-metrics-values.yaml new file mode 100644 index 000000000..887ed0f62 --- /dev/null +++ b/charts/ingress-nginx/ci/deployment-metrics-values.yaml @@ -0,0 +1,7 @@ +controller: + admissionWebhooks: + enabled: false + metrics: + enabled: true + service: + type: ClusterIP diff --git a/charts/ingress-nginx/ci/deployment-nodeport-values.yaml b/charts/ingress-nginx/ci/deployment-nodeport-values.yaml new file mode 100644 index 000000000..84f1f7582 --- /dev/null +++ b/charts/ingress-nginx/ci/deployment-nodeport-values.yaml @@ -0,0 +1,5 @@ +controller: + admissionWebhooks: + enabled: false + service: + type: NodePort diff --git a/charts/ingress-nginx/ci/controller-deployment-podannotations-values.yaml b/charts/ingress-nginx/ci/deployment-podannotations-values.yaml similarity index 62% rename from charts/ingress-nginx/ci/controller-deployment-podannotations-values.yaml rename to charts/ingress-nginx/ci/deployment-podannotations-values.yaml index cf1f2611e..b65a0910b 100644 --- a/charts/ingress-nginx/ci/controller-deployment-podannotations-values.yaml +++ b/charts/ingress-nginx/ci/deployment-podannotations-values.yaml @@ -1,16 +1,12 @@ controller: - image: - repository: ingress-controller/controller - tag: 1.0.0-dev - digest: null - + admissionWebhooks: + enabled: false + metrics: + enabled: true service: type: ClusterIP - - kind: Deployment - podAnnotations: - prometheus.io/scrape: "true" + prometheus.io/path: /metrics prometheus.io/port: "10254" prometheus.io/scheme: http - prometheus.io/path: /metrics + prometheus.io/scrape: "true" diff --git a/charts/ingress-nginx/ci/deployment-psp-values.yaml b/charts/ingress-nginx/ci/deployment-psp-values.yaml new file mode 100644 index 000000000..e339c69c3 --- /dev/null +++ b/charts/ingress-nginx/ci/deployment-psp-values.yaml @@ -0,0 +1,6 @@ +controller: + service: + type: ClusterIP + +podSecurityPolicy: + enabled: true diff --git a/charts/ingress-nginx/ci/deployment-tcp-udp-configMapNamespace-values.yaml b/charts/ingress-nginx/ci/deployment-tcp-udp-configMapNamespace-values.yaml new file mode 100644 index 000000000..141e06b68 --- /dev/null +++ b/charts/ingress-nginx/ci/deployment-tcp-udp-configMapNamespace-values.yaml @@ -0,0 +1,15 @@ +controller: + admissionWebhooks: + enabled: false + service: + type: ClusterIP + tcp: + configMapNamespace: default + udp: + configMapNamespace: default + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/charts/ingress-nginx/ci/deployment-tcp-udp-values.yaml b/charts/ingress-nginx/ci/deployment-tcp-udp-values.yaml new file mode 100644 index 000000000..bc29abeba --- /dev/null +++ b/charts/ingress-nginx/ci/deployment-tcp-udp-values.yaml @@ -0,0 +1,11 @@ +controller: + admissionWebhooks: + enabled: false + service: + type: ClusterIP + +tcp: + 9000: "default/test:8080" + +udp: + 9001: "default/test:8080" diff --git a/charts/ingress-nginx/ci/deployment-tcp-values.yaml b/charts/ingress-nginx/ci/deployment-tcp-values.yaml new file mode 100644 index 000000000..b7f54c09f --- /dev/null +++ b/charts/ingress-nginx/ci/deployment-tcp-values.yaml @@ -0,0 +1,7 @@ +controller: + service: + type: ClusterIP + +tcp: + 9000: "default/test:8080" + 9001: "default/test:8080" diff --git a/charts/ingress-nginx/ci/deployment-webhook-and-psp-values.yaml b/charts/ingress-nginx/ci/deployment-webhook-and-psp-values.yaml new file mode 100644 index 000000000..a829c3614 --- /dev/null +++ b/charts/ingress-nginx/ci/deployment-webhook-and-psp-values.yaml @@ -0,0 +1,8 @@ +controller: + admissionWebhooks: + enabled: true + service: + type: ClusterIP + +podSecurityPolicy: + enabled: true diff --git a/charts/ingress-nginx/ci/deployment-webhook-values.yaml b/charts/ingress-nginx/ci/deployment-webhook-values.yaml new file mode 100644 index 000000000..4f18a70b9 --- /dev/null +++ b/charts/ingress-nginx/ci/deployment-webhook-values.yaml @@ -0,0 +1,5 @@ +controller: + admissionWebhooks: + enabled: true + service: + type: ClusterIP diff --git a/charts/ingress-nginx/templates/NOTES.txt b/charts/ingress-nginx/templates/NOTES.txt index f4923007e..60fb2c1f6 100644 --- a/charts/ingress-nginx/templates/NOTES.txt +++ b/charts/ingress-nginx/templates/NOTES.txt @@ -6,59 +6,50 @@ Get the application URL by running these commands: {{- if (not (empty .Values.controller.service.nodePorts.http)) }} export HTTP_NODE_PORT={{ .Values.controller.service.nodePorts.http }} {{- else }} - export HTTP_NODE_PORT=$(kubectl get service --namespace {{ include "ingress-nginx.namespace" . }} {{ include "ingress-nginx.controller.fullname" . }} --output jsonpath="{.spec.ports[0].nodePort}") + export HTTP_NODE_PORT=$(kubectl --namespace {{ .Release.Namespace }} get services -o jsonpath="{.spec.ports[0].nodePort}" {{ include "ingress-nginx.controller.fullname" . }}) {{- end }} {{- if (not (empty .Values.controller.service.nodePorts.https)) }} export HTTPS_NODE_PORT={{ .Values.controller.service.nodePorts.https }} {{- else }} - export HTTPS_NODE_PORT=$(kubectl get service --namespace {{ include "ingress-nginx.namespace" . }} {{ include "ingress-nginx.controller.fullname" . }} --output jsonpath="{.spec.ports[1].nodePort}") + export HTTPS_NODE_PORT=$(kubectl --namespace {{ .Release.Namespace }} get services -o jsonpath="{.spec.ports[1].nodePort}" {{ include "ingress-nginx.controller.fullname" . }}) {{- end }} - export NODE_IP="$(kubectl get nodes --output jsonpath="{.items[0].status.addresses[1].address}")" + export NODE_IP=$(kubectl --namespace {{ .Release.Namespace }} get nodes -o jsonpath="{.items[0].status.addresses[1].address}") - echo "Visit http://${NODE_IP}:${HTTP_NODE_PORT} to access your application via HTTP." - echo "Visit https://${NODE_IP}:${HTTPS_NODE_PORT} to access your application via HTTPS." + echo "Visit http://$NODE_IP:$HTTP_NODE_PORT to access your application via HTTP." + echo "Visit https://$NODE_IP:$HTTPS_NODE_PORT to access your application via HTTPS." {{- else if contains "LoadBalancer" .Values.controller.service.type }} -It may take a few minutes for the load balancer IP to be available. -You can watch the status by running 'kubectl get service --namespace {{ include "ingress-nginx.namespace" . }} {{ include "ingress-nginx.controller.fullname" . }} --output wide --watch' +It may take a few minutes for the LoadBalancer IP to be available. +You can watch the status by running 'kubectl --namespace {{ .Release.Namespace }} get services -o wide -w {{ include "ingress-nginx.controller.fullname" . }}' {{- else if contains "ClusterIP" .Values.controller.service.type }} Get the application URL by running these commands: - export POD_NAME="$(kubectl get pods --namespace {{ include "ingress-nginx.namespace" . }} --selector app.kubernetes.io/name={{ include "ingress-nginx.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=controller --output jsonpath="{.items[0].metadata.name}")" - kubectl port-forward --namespace {{ include "ingress-nginx.namespace" . }} "${POD_NAME}" 8080:80 + export POD_NAME=$(kubectl --namespace {{ .Release.Namespace }} get pods -o jsonpath="{.items[0].metadata.name}" -l "app={{ template "ingress-nginx.name" . }},component={{ .Values.controller.name }},release={{ .Release.Name }}") + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:80 echo "Visit http://127.0.0.1:8080 to access your application." {{- end }} An example Ingress that makes use of the controller: -{{- $isV1 := semverCompare ">=1" .Chart.AppVersion}} - apiVersion: networking.k8s.io/v1 + apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: - name: example - namespace: foo - {{- if eq $isV1 false }} annotations: kubernetes.io/ingress.class: {{ .Values.controller.ingressClass }} - {{- end }} + name: example + namespace: foo spec: - {{- if $isV1 }} - ingressClassName: {{ .Values.controller.ingressClassResource.name }} - {{- end }} rules: - host: www.example.com http: paths: - - pathType: Prefix - backend: - service: - name: exampleService - port: - number: 80 + - backend: + serviceName: exampleService + servicePort: 80 path: / # This section is only required if TLS is to be enabled for the Ingress tls: - - hosts: - - www.example.com - secretName: example-tls + - hosts: + - www.example.com + secretName: example-tls If TLS is enabled for the Ingress, a Secret containing the certificate and key must also be provided: @@ -71,3 +62,10 @@ If TLS is enabled for the Ingress, a Secret containing the certificate and key m tls.crt: tls.key: type: kubernetes.io/tls + +{{- if .Values.controller.headers }} +################################################################################# +###### WARNING: `controller.headers` has been deprecated! ##### +###### It has been renamed to `controller.proxySetHeaders`. ##### +################################################################################# +{{- end }} diff --git a/charts/ingress-nginx/templates/_helpers.tpl b/charts/ingress-nginx/templates/_helpers.tpl index 6cbda2d4d..b48bf4a4a 100644 --- a/charts/ingress-nginx/templates/_helpers.tpl +++ b/charts/ingress-nginx/templates/_helpers.tpl @@ -30,69 +30,6 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- end -}} {{- end -}} -{{/* -Expand the namespace of the release. -Allows overriding it for multi-namespace deployments in combined charts. -*/}} -{{- define "ingress-nginx.namespace" -}} -{{- default .Release.Namespace .Values.namespaceOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Controller container security context. -*/}} -{{- define "ingress-nginx.controller.containerSecurityContext" -}} -{{- if .Values.controller.containerSecurityContext -}} -{{- toYaml .Values.controller.containerSecurityContext -}} -{{- else -}} -runAsNonRoot: {{ .Values.controller.image.runAsNonRoot }} -runAsUser: {{ .Values.controller.image.runAsUser }} -runAsGroup: {{ .Values.controller.image.runAsGroup }} -allowPrivilegeEscalation: {{ or .Values.controller.image.allowPrivilegeEscalation .Values.controller.image.chroot }} -{{- if .Values.controller.image.seccompProfile }} -seccompProfile: {{ toYaml .Values.controller.image.seccompProfile | nindent 2 }} -{{- end }} -capabilities: - drop: - - ALL - add: - - NET_BIND_SERVICE - {{- if .Values.controller.image.chroot }} - {{- if .Values.controller.image.seccompProfile }} - - SYS_ADMIN - {{- end }} - - SYS_CHROOT - {{- end }} -readOnlyRootFilesystem: {{ .Values.controller.image.readOnlyRootFilesystem }} -{{- end -}} -{{- end -}} - -{{/* -Get specific image -*/}} -{{- define "ingress-nginx.image" -}} -{{- if .chroot -}} -{{- printf "%s-chroot" .image -}} -{{- else -}} -{{- printf "%s" .image -}} -{{- end }} -{{- end -}} - -{{/* -Get specific image digest -*/}} -{{- define "ingress-nginx.imageDigest" -}} -{{- if .chroot -}} -{{- if .digestChroot -}} -{{- printf "@%s" .digestChroot -}} -{{- end }} -{{- else -}} -{{ if .digest -}} -{{- printf "@%s" .digest -}} -{{- end -}} -{{- end -}} -{{- end -}} - {{/* Create a default fully qualified controller name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). @@ -101,16 +38,6 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- printf "%s-%s" (include "ingress-nginx.fullname" .) .Values.controller.name | trunc 63 | trimSuffix "-" -}} {{- end -}} -{{/* -Construct a unique electionID. -Users can provide an override for an explicit electionID if they want via `.Values.controller.electionID` -*/}} -{{- define "ingress-nginx.controller.electionID" -}} -{{- $defElectionID := printf "%s-leader" (include "ingress-nginx.fullname" .) -}} -{{- $electionID := default $defElectionID .Values.controller.electionID -}} -{{- print $electionID -}} -{{- end -}} - {{/* Construct the path for the publish-service. @@ -118,6 +45,7 @@ By convention this will simply use the / to match th service generated. Users can provide an override for an explicit service they want bound via `.Values.controller.publishService.pathOverride` + */}} {{- define "ingress-nginx.controller.publishServicePath" -}} {{- $defServiceName := printf "%s/%s" "$(POD_NAMESPACE)" (include "ingress-nginx.controller.fullname" .) -}} @@ -125,6 +53,14 @@ Users can provide an override for an explicit service they want bound via `.Valu {{- print $servicePath | trimSuffix "-" -}} {{- end -}} +{{/* +Create a default fully qualified default backend name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "ingress-nginx.defaultBackend.fullname" -}} +{{- printf "%s-%s" (include "ingress-nginx.fullname" .) .Values.defaultBackend.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{/* Common labels */}} @@ -134,11 +70,7 @@ helm.sh/chart: {{ include "ingress-nginx.chart" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} -app.kubernetes.io/part-of: {{ template "ingress-nginx.name" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- if .Values.commonLabels}} -{{ toYaml .Values.commonLabels }} -{{- end }} {{- end -}} {{/* @@ -161,50 +93,7 @@ Create the name of the controller service account to use {{- end -}} {{/* -Create a default fully qualified admission webhook name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "ingress-nginx.admissionWebhooks.fullname" -}} -{{- printf "%s-%s" (include "ingress-nginx.fullname" .) .Values.controller.admissionWebhooks.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create the name of the admission webhook patch job service account to use -*/}} -{{- define "ingress-nginx.admissionWebhooks.patch.serviceAccountName" -}} -{{- if .Values.controller.admissionWebhooks.patch.serviceAccount.create -}} - {{ default (include "ingress-nginx.admissionWebhooks.fullname" .) .Values.controller.admissionWebhooks.patch.serviceAccount.name }} -{{- else -}} - {{ default "default" .Values.controller.admissionWebhooks.patch.serviceAccount.name }} -{{- end -}} -{{- end -}} - -{{/* -Create a default fully qualified admission webhook secret creation job name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "ingress-nginx.admissionWebhooks.createSecretJob.fullname" -}} -{{- printf "%s-%s" (include "ingress-nginx.admissionWebhooks.fullname" .) .Values.controller.admissionWebhooks.createSecretJob.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified admission webhook patch job name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "ingress-nginx.admissionWebhooks.patchWebhookJob.fullname" -}} -{{- printf "%s-%s" (include "ingress-nginx.admissionWebhooks.fullname" .) .Values.controller.admissionWebhooks.patchWebhookJob.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified default backend name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "ingress-nginx.defaultBackend.fullname" -}} -{{- printf "%s-%s" (include "ingress-nginx.fullname" .) .Values.defaultBackend.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create the name of the default backend service account to use +Create the name of the backend service account to use - only used when podsecuritypolicy is also enabled */}} {{- define "ingress-nginx.defaultBackend.serviceAccountName" -}} {{- if .Values.defaultBackend.serviceAccount.create -}} @@ -215,49 +104,21 @@ Create the name of the default backend service account to use {{- end -}} {{/* -Default backend container security context. +Return the appropriate apiGroup for PodSecurityPolicy. */}} -{{- define "ingress-nginx.defaultBackend.containerSecurityContext" -}} -{{- if .Values.defaultBackend.containerSecurityContext -}} -{{- toYaml .Values.defaultBackend.containerSecurityContext -}} +{{- define "podSecurityPolicy.apiGroup" -}} +{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- print "policy" -}} {{- else -}} -runAsNonRoot: {{ .Values.defaultBackend.image.runAsNonRoot }} -runAsUser: {{ .Values.defaultBackend.image.runAsUser }} -runAsGroup: {{ .Values.defaultBackend.image.runAsGroup }} -allowPrivilegeEscalation: {{ .Values.defaultBackend.image.allowPrivilegeEscalation }} -{{- if .Values.defaultBackend.image.seccompProfile }} -seccompProfile: {{ toYaml .Values.defaultBackend.image.seccompProfile | nindent 2 }} -{{- end }} -capabilities: - drop: - - ALL -readOnlyRootFilesystem: {{ .Values.defaultBackend.image.readOnlyRootFilesystem }} +{{- print "extensions" -}} {{- end -}} {{- end -}} {{/* -Extra modules. +Check the ingress controller version tag is at most three versions behind the last release */}} -{{- define "extraModules" -}} -- name: {{ .name }} - {{- with .image }} - image: {{ if .repository }}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{ end }}:{{ .tag }}{{ if .digest }}@{{ .digest }}{{ end }} - command: - {{- if .distroless }} - - /init_module - {{- else }} - - sh - - -c - - /usr/local/bin/init_module.sh - {{- end }} - {{- end }} - {{- if .containerSecurityContext }} - securityContext: {{ toYaml .containerSecurityContext | nindent 4 }} - {{- end }} - {{- if .resources }} - resources: {{ toYaml .resources | nindent 4 }} - {{- end }} - volumeMounts: - - name: modules - mountPath: /modules_mount +{{- define "isControllerTagValid" -}} +{{- if not (semverCompare ">=0.27.0-0" .Values.controller.image.tag) -}} +{{- fail "Controller container image tag should be 0.27.0 or higher" -}} +{{- end -}} {{- end -}} diff --git a/charts/ingress-nginx/templates/_params.tpl b/charts/ingress-nginx/templates/_params.tpl deleted file mode 100644 index 0051dc9c0..000000000 --- a/charts/ingress-nginx/templates/_params.tpl +++ /dev/null @@ -1,77 +0,0 @@ -{{- define "ingress-nginx.params" -}} -- /nginx-ingress-controller -{{- if not .Values.controller.enableAnnotationValidations }} -- --enable-annotation-validation=false -{{- end }} -{{- if .Values.defaultBackend.enabled }} -- --default-backend-service=$(POD_NAMESPACE)/{{ include "ingress-nginx.defaultBackend.fullname" . }} -{{- end }} -{{- if and .Values.controller.publishService.enabled .Values.controller.service.enabled }} -{{- if .Values.controller.service.external.enabled }} -- --publish-service={{ template "ingress-nginx.controller.publishServicePath" . }} -{{- else if .Values.controller.service.internal.enabled }} -- --publish-service={{ template "ingress-nginx.controller.publishServicePath" . }}-internal -{{- end }} -{{- end }} -- --election-id={{ include "ingress-nginx.controller.electionID" . }} -- --controller-class={{ .Values.controller.ingressClassResource.controllerValue }} -{{- if .Values.controller.ingressClass }} -- --ingress-class={{ .Values.controller.ingressClass }} -{{- end }} -- --configmap={{ default "$(POD_NAMESPACE)" .Values.controller.configMapNamespace }}/{{ include "ingress-nginx.controller.fullname" . }} -{{- if .Values.tcp }} -- --tcp-services-configmap={{ default "$(POD_NAMESPACE)" .Values.controller.tcp.configMapNamespace }}/{{ include "ingress-nginx.fullname" . }}-tcp -{{- end }} -{{- if .Values.udp }} -- --udp-services-configmap={{ default "$(POD_NAMESPACE)" .Values.controller.udp.configMapNamespace }}/{{ include "ingress-nginx.fullname" . }}-udp -{{- end }} -{{- if .Values.controller.scope.enabled }} -- --watch-namespace={{ default "$(POD_NAMESPACE)" .Values.controller.scope.namespace }} -{{- end }} -{{- if and (not .Values.controller.scope.enabled) .Values.controller.scope.namespaceSelector }} -- --watch-namespace-selector={{ .Values.controller.scope.namespaceSelector }} -{{- end }} -{{- if and .Values.controller.reportNodeInternalIp .Values.controller.hostNetwork }} -- --report-node-internal-ip-address={{ .Values.controller.reportNodeInternalIp }} -{{- end }} -{{- if .Values.controller.admissionWebhooks.enabled }} -- --validating-webhook=:{{ .Values.controller.admissionWebhooks.port }} -- --validating-webhook-certificate={{ .Values.controller.admissionWebhooks.certificate }} -- --validating-webhook-key={{ .Values.controller.admissionWebhooks.key }} -{{- end }} -{{- if .Values.controller.maxmindLicenseKey }} -- --maxmind-license-key={{ .Values.controller.maxmindLicenseKey }} -{{- end }} -{{- if .Values.controller.healthCheckHost }} -- --healthz-host={{ .Values.controller.healthCheckHost }} -{{- end }} -{{- if not (eq .Values.controller.healthCheckPath "/healthz") }} -- --health-check-path={{ .Values.controller.healthCheckPath }} -{{- end }} -{{- if .Values.controller.ingressClassByName }} -- --ingress-class-by-name=true -{{- end }} -{{- if .Values.controller.watchIngressWithoutClass }} -- --watch-ingress-without-class=true -{{- end }} -{{- if .Values.controller.metrics.enabled }} -- --enable-metrics={{ .Values.controller.metrics.enabled }} -{{- end }} -{{- if .Values.controller.enableTopologyAwareRouting }} -- --enable-topology-aware-routing=true -{{- end }} -{{- if .Values.controller.disableLeaderElection }} -- --disable-leader-election=true -{{- end }} -{{- if .Values.controller.electionTTL }} -- --election-ttl={{ .Values.controller.electionTTL }} -{{- end }} -{{- range $key, $value := .Values.controller.extraArgs }} -{{- /* Accept keys without values or with false as value */}} -{{- if eq ($value | quote | len) 2 }} -- --{{ $key }} -{{- else }} -- --{{ $key }}={{ $value }} -{{- end }} -{{- end }} -{{- end -}} diff --git a/charts/ingress-nginx/templates/admission-webhooks/cert-manager.yaml b/charts/ingress-nginx/templates/admission-webhooks/cert-manager.yaml deleted file mode 100644 index db2946c3d..000000000 --- a/charts/ingress-nginx/templates/admission-webhooks/cert-manager.yaml +++ /dev/null @@ -1,63 +0,0 @@ -{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.certManager.enabled -}} -{{- if not .Values.controller.admissionWebhooks.certManager.issuerRef -}} -# Create a selfsigned Issuer, in order to create a root CA certificate for -# signing webhook serving certificates -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: {{ include "ingress-nginx.fullname" . }}-self-signed-issuer - namespace: {{ include "ingress-nginx.namespace" . }} -spec: - selfSigned: {} ---- -# Generate a CA Certificate used to sign certificates for the webhook -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: {{ include "ingress-nginx.fullname" . }}-root-cert - namespace: {{ include "ingress-nginx.namespace" . }} -spec: - secretName: {{ include "ingress-nginx.fullname" . }}-root-cert - duration: {{ .Values.controller.admissionWebhooks.certManager.rootCert.duration | default "43800h0m0s" | quote }} - issuerRef: - name: {{ include "ingress-nginx.fullname" . }}-self-signed-issuer - commonName: "ca.webhook.ingress-nginx" - isCA: true - subject: - organizations: - - ingress-nginx ---- -# Create an Issuer that uses the above generated CA certificate to issue certs -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: {{ include "ingress-nginx.fullname" . }}-root-issuer - namespace: {{ include "ingress-nginx.namespace" . }} -spec: - ca: - secretName: {{ include "ingress-nginx.fullname" . }}-root-cert -{{- end }} ---- -# generate a server certificate for the apiservices to use -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: {{ include "ingress-nginx.admissionWebhooks.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} -spec: - secretName: {{ include "ingress-nginx.admissionWebhooks.fullname" . }} - duration: {{ .Values.controller.admissionWebhooks.certManager.admissionCert.duration | default "8760h0m0s" | quote }} - issuerRef: - {{- if .Values.controller.admissionWebhooks.certManager.issuerRef }} - {{- toYaml .Values.controller.admissionWebhooks.certManager.issuerRef | nindent 4 }} - {{- else }} - name: {{ include "ingress-nginx.fullname" . }}-root-issuer - {{- end }} - dnsNames: - - {{ include "ingress-nginx.controller.fullname" . }}-admission - - {{ include "ingress-nginx.controller.fullname" . }}-admission.{{ include "ingress-nginx.namespace" . }} - - {{ include "ingress-nginx.controller.fullname" . }}-admission.{{ include "ingress-nginx.namespace" . }}.svc - subject: - organizations: - - ingress-nginx-admission -{{- end -}} diff --git a/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml b/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml index 54af7abb6..fd762f935 100644 --- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml +++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml @@ -1,17 +1,14 @@ -{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled .Values.controller.admissionWebhooks.patch.rbac.create (not .Values.controller.admissionWebhooks.certManager.enabled) -}} +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ include "ingress-nginx.admissionWebhooks.fullname" . }} + name: {{ include "ingress-nginx.fullname" . }}-admission annotations: "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: admission-webhook - {{- with .Values.controller.admissionWebhooks.patch.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} rules: - apiGroups: - admissionregistration.k8s.io @@ -20,4 +17,15 @@ rules: verbs: - get - update +{{- if .Values.podSecurityPolicy.enabled }} + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + {{- with .Values.controller.admissionWebhooks.existingPsp }} + - {{ . }} + {{- else }} + - {{ include "ingress-nginx.fullname" . }}-admission + {{- end }} +{{- end }} {{- end }} diff --git a/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml b/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml index b89388433..4990fb1c3 100644 --- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml +++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml @@ -1,23 +1,20 @@ -{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled .Values.controller.admissionWebhooks.patch.rbac.create (not .Values.controller.admissionWebhooks.certManager.enabled) -}} +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: {{ include "ingress-nginx.admissionWebhooks.fullname" . }} + name: {{ include "ingress-nginx.fullname" . }}-admission annotations: "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: admission-webhook - {{- with .Values.controller.admissionWebhooks.patch.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: {{ include "ingress-nginx.admissionWebhooks.fullname" . }} + name: {{ include "ingress-nginx.fullname" . }}-admission subjects: - kind: ServiceAccount - name: {{ include "ingress-nginx.admissionWebhooks.patch.serviceAccountName" . }} - namespace: {{ include "ingress-nginx.namespace" . }} + name: {{ include "ingress-nginx.fullname" . }}-admission + namespace: {{ .Release.Namespace | quote }} {{- end }} diff --git a/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml b/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml index bb31e60ba..e715c15e5 100644 --- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml +++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml @@ -1,21 +1,14 @@ -{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled (not .Values.controller.admissionWebhooks.certManager.enabled) -}} +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} apiVersion: batch/v1 kind: Job metadata: - name: {{ include "ingress-nginx.admissionWebhooks.createSecretJob.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} + name: {{ include "ingress-nginx.fullname" . }}-admission-create annotations: "helm.sh/hook": pre-install,pre-upgrade "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - {{- with .Values.controller.admissionWebhooks.annotations }} - {{- toYaml . | nindent 4 }} - {{- end }} labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: admission-webhook - {{- with .Values.controller.admissionWebhooks.patch.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} spec: {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} # Alpha feature since k8s 1.12 @@ -23,16 +16,13 @@ spec: {{- end }} template: metadata: - name: {{ include "ingress-nginx.admissionWebhooks.createSecretJob.fullname" . }} + name: {{ include "ingress-nginx.fullname" . }}-admission-create {{- if .Values.controller.admissionWebhooks.patch.podAnnotations }} annotations: {{ toYaml .Values.controller.admissionWebhooks.patch.podAnnotations | nindent 8 }} {{- end }} labels: {{- include "ingress-nginx.labels" . | nindent 8 }} app.kubernetes.io/component: admission-webhook - {{- with .Values.controller.admissionWebhooks.patch.labels }} - {{- toYaml . | nindent 8 }} - {{- end }} spec: {{- if .Values.controller.admissionWebhooks.patch.priorityClassName }} priorityClassName: {{ .Values.controller.admissionWebhooks.patch.priorityClassName }} @@ -42,39 +32,29 @@ spec: {{- end }} containers: - name: create - {{- with (merge .Values.controller.admissionWebhooks.patch.image .Values.global.image) }} - image: {{ if .repository }}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{ end }}:{{ .tag }}{{ if .digest }}@{{ .digest }}{{ end }} + {{- with .Values.controller.admissionWebhooks.patch.image }} + image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}" {{- end }} imagePullPolicy: {{ .Values.controller.admissionWebhooks.patch.image.pullPolicy }} args: - create - --host={{ include "ingress-nginx.controller.fullname" . }}-admission,{{ include "ingress-nginx.controller.fullname" . }}-admission.$(POD_NAMESPACE).svc - --namespace=$(POD_NAMESPACE) - - --secret-name={{ include "ingress-nginx.admissionWebhooks.fullname" . }} + - --secret-name={{ include "ingress-nginx.fullname" . }}-admission env: - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - {{- if .Values.controller.admissionWebhooks.extraEnvs }} - {{- toYaml .Values.controller.admissionWebhooks.extraEnvs | nindent 12 }} - {{- end }} - {{- if .Values.controller.admissionWebhooks.createSecretJob.securityContext }} - securityContext: {{ toYaml .Values.controller.admissionWebhooks.createSecretJob.securityContext | nindent 12 }} - {{- end }} - {{- if .Values.controller.admissionWebhooks.createSecretJob.resources }} - resources: {{ toYaml .Values.controller.admissionWebhooks.createSecretJob.resources | nindent 12 }} - {{- end }} restartPolicy: OnFailure - serviceAccountName: {{ include "ingress-nginx.admissionWebhooks.patch.serviceAccountName" . }} - automountServiceAccountToken: {{ .Values.controller.admissionWebhooks.patch.serviceAccount.automountServiceAccountToken }} + serviceAccountName: {{ include "ingress-nginx.fullname" . }}-admission {{- if .Values.controller.admissionWebhooks.patch.nodeSelector }} nodeSelector: {{ toYaml .Values.controller.admissionWebhooks.patch.nodeSelector | nindent 8 }} {{- end }} {{- if .Values.controller.admissionWebhooks.patch.tolerations }} tolerations: {{ toYaml .Values.controller.admissionWebhooks.patch.tolerations | nindent 8 }} {{- end }} - {{- if .Values.controller.admissionWebhooks.patch.securityContext }} - securityContext: {{ toYaml .Values.controller.admissionWebhooks.patch.securityContext | nindent 8 }} - {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.controller.admissionWebhooks.patch.runAsUser }} {{- end }} diff --git a/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml b/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml index cf757c98b..b8585e2cf 100644 --- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml +++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -1,21 +1,14 @@ -{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled (not .Values.controller.admissionWebhooks.certManager.enabled) -}} +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} apiVersion: batch/v1 kind: Job metadata: - name: {{ include "ingress-nginx.admissionWebhooks.patchWebhookJob.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} + name: {{ include "ingress-nginx.fullname" . }}-admission-patch annotations: "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - {{- with .Values.controller.admissionWebhooks.annotations }} - {{- toYaml . | nindent 4 }} - {{- end }} labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: admission-webhook - {{- with .Values.controller.admissionWebhooks.patch.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} spec: {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} # Alpha feature since k8s 1.12 @@ -23,16 +16,13 @@ spec: {{- end }} template: metadata: - name: {{ include "ingress-nginx.admissionWebhooks.patchWebhookJob.fullname" . }} + name: {{ include "ingress-nginx.fullname" . }}-admission-patch {{- if .Values.controller.admissionWebhooks.patch.podAnnotations }} annotations: {{ toYaml .Values.controller.admissionWebhooks.patch.podAnnotations | nindent 8 }} {{- end }} labels: {{- include "ingress-nginx.labels" . | nindent 8 }} app.kubernetes.io/component: admission-webhook - {{- with .Values.controller.admissionWebhooks.patch.labels }} - {{- toYaml . | nindent 8 }} - {{- end }} spec: {{- if .Values.controller.admissionWebhooks.patch.priorityClassName }} priorityClassName: {{ .Values.controller.admissionWebhooks.patch.priorityClassName }} @@ -42,41 +32,31 @@ spec: {{- end }} containers: - name: patch - {{- with (merge .Values.controller.admissionWebhooks.patch.image .Values.global.image) }} - image: {{ if .repository }}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{ end }}:{{ .tag }}{{ if .digest }}@{{ .digest }}{{ end }} + {{- with .Values.controller.admissionWebhooks.patch.image }} + image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}" {{- end }} imagePullPolicy: {{ .Values.controller.admissionWebhooks.patch.image.pullPolicy }} args: - patch - - --webhook-name={{ include "ingress-nginx.admissionWebhooks.fullname" . }} + - --webhook-name={{ include "ingress-nginx.fullname" . }}-admission - --namespace=$(POD_NAMESPACE) - --patch-mutating=false - - --secret-name={{ include "ingress-nginx.admissionWebhooks.fullname" . }} + - --secret-name={{ include "ingress-nginx.fullname" . }}-admission - --patch-failure-policy={{ .Values.controller.admissionWebhooks.failurePolicy }} env: - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - {{- if .Values.controller.admissionWebhooks.extraEnvs }} - {{- toYaml .Values.controller.admissionWebhooks.extraEnvs | nindent 12 }} - {{- end }} - {{- if .Values.controller.admissionWebhooks.patchWebhookJob.securityContext }} - securityContext: {{ toYaml .Values.controller.admissionWebhooks.patchWebhookJob.securityContext | nindent 12 }} - {{- end }} - {{- if .Values.controller.admissionWebhooks.patchWebhookJob.resources }} - resources: {{ toYaml .Values.controller.admissionWebhooks.patchWebhookJob.resources | nindent 12 }} - {{- end }} restartPolicy: OnFailure - serviceAccountName: {{ include "ingress-nginx.admissionWebhooks.patch.serviceAccountName" . }} - automountServiceAccountToken: {{ .Values.controller.admissionWebhooks.patch.serviceAccount.automountServiceAccountToken }} + serviceAccountName: {{ include "ingress-nginx.fullname" . }}-admission {{- if .Values.controller.admissionWebhooks.patch.nodeSelector }} nodeSelector: {{ toYaml .Values.controller.admissionWebhooks.patch.nodeSelector | nindent 8 }} {{- end }} {{- if .Values.controller.admissionWebhooks.patch.tolerations }} tolerations: {{ toYaml .Values.controller.admissionWebhooks.patch.tolerations | nindent 8 }} {{- end }} - {{- if .Values.controller.admissionWebhooks.patch.securityContext }} - securityContext: {{ toYaml .Values.controller.admissionWebhooks.patch.securityContext | nindent 8 }} - {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: {{ .Values.controller.admissionWebhooks.patch.runAsUser }} {{- end }} diff --git a/charts/ingress-nginx/templates/admission-webhooks/job-patch/networkpolicy.yaml b/charts/ingress-nginx/templates/admission-webhooks/job-patch/networkpolicy.yaml deleted file mode 100644 index a8f38df96..000000000 --- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/networkpolicy.yaml +++ /dev/null @@ -1,26 +0,0 @@ -{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled .Values.controller.admissionWebhooks.patch.networkPolicy.enabled (not .Values.controller.admissionWebhooks.certManager.enabled) -}} -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: {{ include "ingress-nginx.admissionWebhooks.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "ingress-nginx.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook - {{- with .Values.controller.admissionWebhooks.patch.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - podSelector: - matchLabels: - {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: admission-webhook - policyTypes: - - Ingress - - Egress - egress: - - {} -{{- end }} diff --git a/charts/ingress-nginx/templates/admission-webhooks/job-patch/psp.yaml b/charts/ingress-nginx/templates/admission-webhooks/job-patch/psp.yaml new file mode 100644 index 000000000..d2c7de685 --- /dev/null +++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/psp.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled .Values.podSecurityPolicy.enabled (empty .Values.controller.admissionWebhooks.existingPsp) -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: admission-webhook +spec: + allowPrivilegeEscalation: false + fsGroup: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + requiredDropCapabilities: + - ALL + runAsUser: + rule: MustRunAsNonRoot + seLinux: + rule: RunAsAny + supplementalGroups: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + volumes: + - configMap + - emptyDir + - projected + - secret + - downwardAPI +{{- end }} diff --git a/charts/ingress-nginx/templates/admission-webhooks/job-patch/role.yaml b/charts/ingress-nginx/templates/admission-webhooks/job-patch/role.yaml index c4b23aa08..fe1c2ee7f 100644 --- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/role.yaml +++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/role.yaml @@ -1,18 +1,14 @@ -{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled .Values.controller.admissionWebhooks.patch.rbac.create (not .Values.controller.admissionWebhooks.certManager.enabled) -}} +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: {{ include "ingress-nginx.admissionWebhooks.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} + name: {{ include "ingress-nginx.fullname" . }}-admission annotations: "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: admission-webhook - {{- with .Values.controller.admissionWebhooks.patch.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} rules: - apiGroups: - "" diff --git a/charts/ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml b/charts/ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml index 425e8d821..b4af7fbcf 100644 --- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml +++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml @@ -1,24 +1,20 @@ -{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled .Values.controller.admissionWebhooks.patch.rbac.create (not .Values.controller.admissionWebhooks.certManager.enabled) -}} +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: {{ include "ingress-nginx.admissionWebhooks.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} + name: {{ include "ingress-nginx.fullname" . }}-admission annotations: "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: admission-webhook - {{- with .Values.controller.admissionWebhooks.patch.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: {{ include "ingress-nginx.admissionWebhooks.fullname" . }} + name: {{ include "ingress-nginx.fullname" . }}-admission subjects: - kind: ServiceAccount - name: {{ include "ingress-nginx.admissionWebhooks.patch.serviceAccountName" . }} - namespace: {{ include "ingress-nginx.namespace" . }} + name: {{ include "ingress-nginx.fullname" . }}-admission + namespace: {{ .Release.Namespace | quote }} {{- end }} diff --git a/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml b/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml index 52f94dcce..5dfdd345a 100644 --- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml +++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml @@ -1,17 +1,12 @@ -{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled .Values.controller.admissionWebhooks.patch.serviceAccount.create (not .Values.controller.admissionWebhooks.certManager.enabled) -}} +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "ingress-nginx.admissionWebhooks.patch.serviceAccountName" . }} - namespace: {{ include "ingress-nginx.namespace" . }} + name: {{ include "ingress-nginx.fullname" . }}-admission annotations: "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: admission-webhook - {{- with .Values.controller.admissionWebhooks.patch.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} -automountServiceAccountToken: {{ .Values.controller.admissionWebhooks.patch.serviceAccount.automountServiceAccountToken }} {{- end }} diff --git a/charts/ingress-nginx/templates/admission-webhooks/validating-webhook.yaml b/charts/ingress-nginx/templates/admission-webhooks/validating-webhook.yaml index 0949cea75..2f3dd7784 100644 --- a/charts/ingress-nginx/templates/admission-webhooks/validating-webhook.yaml +++ b/charts/ingress-nginx/templates/admission-webhooks/validating-webhook.yaml @@ -4,21 +4,13 @@ apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: - annotations: - {{- if .Values.controller.admissionWebhooks.certManager.enabled }} - certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s" (include "ingress-nginx.namespace" .) (include "ingress-nginx.admissionWebhooks.fullname" .) | quote }} - cert-manager.io/inject-ca-from: {{ printf "%s/%s" (include "ingress-nginx.namespace" .) (include "ingress-nginx.admissionWebhooks.fullname" .) | quote }} - {{- end }} {{- if .Values.controller.admissionWebhooks.annotations }} - {{- toYaml .Values.controller.admissionWebhooks.annotations | nindent 4 }} + annotations: {{ toYaml .Values.controller.admissionWebhooks.annotations | nindent 4 }} {{- end }} labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: admission-webhook - {{- with .Values.controller.admissionWebhooks.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - name: {{ include "ingress-nginx.admissionWebhooks.fullname" . }} + name: {{ include "ingress-nginx.fullname" . }}-admission webhooks: - name: validate.nginx.ingress.kubernetes.io matchPolicy: Equivalent @@ -26,7 +18,7 @@ webhooks: - apiGroups: - networking.k8s.io apiVersions: - - v1 + - v1beta1 operations: - CREATE - UPDATE @@ -36,12 +28,12 @@ webhooks: sideEffects: None admissionReviewVersions: - v1 + - v1beta1 clientConfig: service: + namespace: {{ .Release.Namespace | quote }} name: {{ include "ingress-nginx.controller.fullname" . }}-admission - namespace: {{ include "ingress-nginx.namespace" . }} - port: {{ .Values.controller.admissionWebhooks.service.servicePort }} - path: /networking/v1/ingresses + path: /networking/v1beta1/ingresses {{- if .Values.controller.admissionWebhooks.timeoutSeconds }} timeoutSeconds: {{ .Values.controller.admissionWebhooks.timeoutSeconds }} {{- end }} diff --git a/charts/ingress-nginx/templates/clusterrole.yaml b/charts/ingress-nginx/templates/clusterrole.yaml index 51bc5002c..8ec5f49fa 100644 --- a/charts/ingress-nginx/templates/clusterrole.yaml +++ b/charts/ingress-nginx/templates/clusterrole.yaml @@ -1,18 +1,9 @@ -{{- if .Values.rbac.create }} - -{{- if and .Values.rbac.scope (not .Values.controller.scope.enabled) -}} - {{ required "Invalid configuration: 'rbac.scope' should be equal to 'controller.scope.enabled' (true/false)." (index (dict) ".") }} -{{- end }} - -{{- if not .Values.rbac.scope -}} +{{- if and .Values.rbac.create (not .Values.rbac.scope) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} name: {{ include "ingress-nginx.fullname" . }} rules: - apiGroups: @@ -23,16 +14,6 @@ rules: - nodes - pods - secrets -{{- if not .Values.controller.scope.enabled }} - - namespaces -{{- end}} - verbs: - - list - - watch - - apiGroups: - - coordination.k8s.io - resources: - - leases verbs: - list - watch @@ -61,7 +42,8 @@ rules: - list - watch - apiGroups: - - networking.k8s.io + - extensions + - "networking.k8s.io" # k8s 1.14+ resources: - ingresses verbs: @@ -76,27 +58,18 @@ rules: - create - patch - apiGroups: - - networking.k8s.io + - extensions + - "networking.k8s.io" # k8s 1.14+ resources: - ingresses/status verbs: - update - apiGroups: - - networking.k8s.io + - "networking.k8s.io" # k8s 1.14+ resources: - ingressclasses verbs: - get - list - watch - - apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get -{{- end }} - {{- end }} diff --git a/charts/ingress-nginx/templates/clusterrolebinding.yaml b/charts/ingress-nginx/templates/clusterrolebinding.yaml index 8f91aac80..81be52b87 100644 --- a/charts/ingress-nginx/templates/clusterrolebinding.yaml +++ b/charts/ingress-nginx/templates/clusterrolebinding.yaml @@ -4,9 +4,6 @@ kind: ClusterRoleBinding metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} name: {{ include "ingress-nginx.fullname" . }} roleRef: apiGroup: rbac.authorization.k8s.io @@ -15,5 +12,5 @@ roleRef: subjects: - kind: ServiceAccount name: {{ template "ingress-nginx.serviceAccountName" . }} - namespace: {{ include "ingress-nginx.namespace" . }} + namespace: {{ .Release.Namespace | quote }} {{- end }} diff --git a/charts/ingress-nginx/templates/controller-configmap-addheaders.yaml b/charts/ingress-nginx/templates/controller-configmap-addheaders.yaml index 4e4bd1310..c06458958 100644 --- a/charts/ingress-nginx/templates/controller-configmap-addheaders.yaml +++ b/charts/ingress-nginx/templates/controller-configmap-addheaders.yaml @@ -5,10 +5,6 @@ metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} name: {{ include "ingress-nginx.fullname" . }}-custom-add-headers - namespace: {{ include "ingress-nginx.namespace" . }} data: {{ toYaml .Values.controller.addHeaders | nindent 2 }} {{- end }} diff --git a/charts/ingress-nginx/templates/controller-configmap-proxyheaders.yaml b/charts/ingress-nginx/templates/controller-configmap-proxyheaders.yaml index 0a22600db..5a1b25229 100644 --- a/charts/ingress-nginx/templates/controller-configmap-proxyheaders.yaml +++ b/charts/ingress-nginx/templates/controller-configmap-proxyheaders.yaml @@ -1,14 +1,15 @@ -{{- if .Values.controller.proxySetHeaders -}} +{{- if or .Values.controller.proxySetHeaders .Values.controller.headers -}} apiVersion: v1 kind: ConfigMap metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} name: {{ include "ingress-nginx.fullname" . }}-custom-proxy-headers - namespace: {{ include "ingress-nginx.namespace" . }} -data: {{ toYaml .Values.controller.proxySetHeaders | nindent 2 }} +data: +{{- if .Values.controller.proxySetHeaders }} +{{ toYaml .Values.controller.proxySetHeaders | indent 2 }} +{{ else if and .Values.controller.headers (not .Values.controller.proxySetHeaders) }} +{{ toYaml .Values.controller.headers | indent 2 }} +{{- end }} {{- end }} diff --git a/charts/ingress-nginx/templates/controller-configmap-tcp.yaml b/charts/ingress-nginx/templates/controller-configmap-tcp.yaml index 131a9ad51..bc972517c 100644 --- a/charts/ingress-nginx/templates/controller-configmap-tcp.yaml +++ b/charts/ingress-nginx/templates/controller-configmap-tcp.yaml @@ -5,13 +5,9 @@ metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} {{- if .Values.controller.tcp.annotations }} annotations: {{ toYaml .Values.controller.tcp.annotations | nindent 4 }} {{- end }} name: {{ include "ingress-nginx.fullname" . }}-tcp - namespace: {{ include "ingress-nginx.namespace" . }} data: {{ tpl (toYaml .Values.tcp) . | nindent 2 }} {{- end }} diff --git a/charts/ingress-nginx/templates/controller-configmap-udp.yaml b/charts/ingress-nginx/templates/controller-configmap-udp.yaml index 7137da9ad..a9dc388f1 100644 --- a/charts/ingress-nginx/templates/controller-configmap-udp.yaml +++ b/charts/ingress-nginx/templates/controller-configmap-udp.yaml @@ -5,13 +5,9 @@ metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} {{- if .Values.controller.udp.annotations }} annotations: {{ toYaml .Values.controller.udp.annotations | nindent 4 }} {{- end }} name: {{ include "ingress-nginx.fullname" . }}-udp - namespace: {{ include "ingress-nginx.namespace" . }} data: {{ tpl (toYaml .Values.udp) . | nindent 2 }} {{- end }} diff --git a/charts/ingress-nginx/templates/controller-configmap.yaml b/charts/ingress-nginx/templates/controller-configmap.yaml index b73cdc2d0..0706fa0eb 100644 --- a/charts/ingress-nginx/templates/controller-configmap.yaml +++ b/charts/ingress-nginx/templates/controller-configmap.yaml @@ -4,27 +4,21 @@ metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} {{- if .Values.controller.configAnnotations }} annotations: {{ toYaml .Values.controller.configAnnotations | nindent 4 }} {{- end }} name: {{ include "ingress-nginx.controller.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} data: -{{- if .Values.controller.allowSnippetAnnotations }} - allow-snippet-annotations: "true" -{{- end }} {{- if .Values.controller.addHeaders }} - add-headers: {{ include "ingress-nginx.namespace" . }}/{{ include "ingress-nginx.fullname" . }}-custom-add-headers + add-headers: {{ .Release.Namespace }}/{{ include "ingress-nginx.fullname" . }}-custom-add-headers {{- end }} -{{- if .Values.controller.proxySetHeaders }} - proxy-set-headers: {{ include "ingress-nginx.namespace" . }}/{{ include "ingress-nginx.fullname" . }}-custom-proxy-headers +{{- if or .Values.controller.proxySetHeaders .Values.controller.headers }} + proxy-set-headers: {{ .Release.Namespace }}/{{ include "ingress-nginx.fullname" . }}-custom-proxy-headers {{- end }} {{- if .Values.dhParam }} - ssl-dh-param: {{ include "ingress-nginx.namespace" . }}/{{ include "ingress-nginx.controller.fullname" . }} + ssl-dh-param: {{ printf "%s/%s" .Release.Namespace (include "ingress-nginx.controller.fullname" .) }} {{- end }} {{- range $key, $value := .Values.controller.config }} - {{- $key | nindent 2 }}: {{ tpl (toString $value) $ | quote }} + {{ $key | nindent 2 }}: {{ $value | quote }} {{- end }} + diff --git a/charts/ingress-nginx/templates/controller-daemonset.yaml b/charts/ingress-nginx/templates/controller-daemonset.yaml index 804ff56d9..991dc4f36 100644 --- a/charts/ingress-nginx/templates/controller-daemonset.yaml +++ b/charts/ingress-nginx/templates/controller-daemonset.yaml @@ -1,4 +1,5 @@ -{{- if eq .Values.controller.kind "DaemonSet" -}} +{{- if or (eq .Values.controller.kind "DaemonSet") (eq .Values.controller.kind "Both") -}} +{{- include "isControllerTagValid" . -}} apiVersion: apps/v1 kind: DaemonSet metadata: @@ -9,7 +10,6 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} name: {{ include "ingress-nginx.controller.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} {{- if .Values.controller.annotations }} annotations: {{ toYaml .Values.controller.annotations | nindent 4 }} {{- end }} @@ -32,58 +32,97 @@ spec: {{- end }} {{- end }} labels: - {{- include "ingress-nginx.labels" . | nindent 8 }} + {{- include "ingress-nginx.selectorLabels" . | nindent 8 }} app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 8 }} - {{- end }} {{- if .Values.controller.podLabels }} {{- toYaml .Values.controller.podLabels | nindent 8 }} {{- end }} spec: {{- if .Values.controller.dnsConfig }} dnsConfig: {{ toYaml .Values.controller.dnsConfig | nindent 8 }} - {{- end }} - {{- if .Values.controller.hostAliases }} - hostAliases: {{ tpl (toYaml .Values.controller.hostAliases) $ | nindent 8 }} - {{- end }} - {{- if .Values.controller.hostname }} - hostname: {{ toYaml .Values.controller.hostname | nindent 8 }} {{- end }} dnsPolicy: {{ .Values.controller.dnsPolicy }} {{- if .Values.imagePullSecrets }} imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }} {{- end }} {{- if .Values.controller.priorityClassName }} - priorityClassName: {{ .Values.controller.priorityClassName | quote }} + priorityClassName: {{ .Values.controller.priorityClassName }} {{- end }} {{- if or .Values.controller.podSecurityContext .Values.controller.sysctls }} securityContext: - {{- if .Values.controller.podSecurityContext }} - {{- toYaml .Values.controller.podSecurityContext | nindent 8 }} - {{- end }} - {{- if .Values.controller.sysctls }} - sysctls: - {{- range $sysctl, $value := .Values.controller.sysctls }} - - name: {{ $sysctl | quote }} - value: {{ $value | quote }} - {{- end }} - {{- end }} {{- end }} - {{- if .Values.controller.shareProcessNamespace }} - shareProcessNamespace: {{ .Values.controller.shareProcessNamespace }} + {{- if .Values.controller.podSecurityContext }} + {{- toYaml .Values.controller.podSecurityContext | nindent 8 }} + {{- end }} + {{- if .Values.controller.sysctls }} + sysctls: + {{- range $sysctl, $value := .Values.controller.sysctls }} + - name: {{ $sysctl | quote }} + value: {{ $value | quote }} + {{- end }} {{- end }} containers: - name: {{ .Values.controller.containerName }} - {{- with (merge .Values.controller.image .Values.global.image) }} - image: {{ if .repository }}{{ .repository }}{{ else }}{{ .registry }}/{{ include "ingress-nginx.image" . }}{{ end }}:{{ .tag }}{{ include "ingress-nginx.imageDigest" . }} + {{- with .Values.controller.image }} + image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}" {{- end }} imagePullPolicy: {{ .Values.controller.image.pullPolicy }} {{- if .Values.controller.lifecycle }} lifecycle: {{ toYaml .Values.controller.lifecycle | nindent 12 }} {{- end }} - args: {{ include "ingress-nginx.params" . | nindent 12 }} - securityContext: {{ include "ingress-nginx.controller.containerSecurityContext" . | nindent 12 }} + args: + - /nginx-ingress-controller + {{- if .Values.defaultBackend.enabled }} + - --default-backend-service={{ .Release.Namespace }}/{{ include "ingress-nginx.defaultBackend.fullname" . }} + {{- end }} + {{- if .Values.controller.publishService.enabled }} + - --publish-service={{ template "ingress-nginx.controller.publishServicePath" . }} + {{- end }} + - --election-id={{ .Values.controller.electionID }} + - --ingress-class={{ .Values.controller.ingressClass }} + - --configmap={{ .Release.Namespace }}/{{ include "ingress-nginx.controller.fullname" . }} + {{- if .Values.tcp }} + - --tcp-services-configmap={{ .Release.Namespace }}/{{ include "ingress-nginx.fullname" . }}-tcp + {{- end }} + {{- if .Values.udp }} + - --udp-services-configmap={{ .Release.Namespace }}/{{ include "ingress-nginx.fullname" . }}-udp + {{- end }} + {{- if .Values.controller.scope.enabled }} + - --watch-namespace={{ default .Release.Namespace .Values.controller.scope.namespace }} + {{- end }} + {{- if and .Values.controller.reportNodeInternalIp .Values.controller.hostNetwork }} + - --report-node-internal-ip-address={{ .Values.controller.reportNodeInternalIp }} + {{- end }} + {{- if .Values.controller.admissionWebhooks.enabled }} + - --validating-webhook=:{{ .Values.controller.admissionWebhooks.port }} + - --validating-webhook-certificate={{ .Values.controller.admissionWebhooks.certificate }} + - --validating-webhook-key={{ .Values.controller.admissionWebhooks.key }} + {{- end }} + {{- if .Values.controller.maxmindMirror }} + - --maxmind-mirror={{ .Values.controller.maxmindMirror }} + {{- end}} + {{- if .Values.controller.maxmindLicenseKey }} + - --maxmind-license-key={{ .Values.controller.maxmindLicenseKey }} + {{- end }} + {{- if not (eq .Values.controller.healthCheckPath "/healthz") }} + - --health-check-path={{ .Values.controller.healthCheckPath }} + {{- end }} + {{- range $key, $value := .Values.controller.extraArgs }} + {{- /* Accept keys without values or with false as value */}} + {{- if eq ($value | quote | len) 2 }} + - --{{ $key }} + {{- else }} + - --{{ $key }}={{ $value }} + {{- end }} + {{- end }} + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: {{ .Values.controller.image.runAsUser }} + allowPrivilegeEscalation: {{ .Values.controller.image.allowPrivilegeEscalation }} env: - name: POD_NAME valueFrom: @@ -103,12 +142,8 @@ spec: {{- if .Values.controller.startupProbe }} startupProbe: {{ toYaml .Values.controller.startupProbe | nindent 12 }} {{- end }} - {{- if .Values.controller.livenessProbe }} livenessProbe: {{ toYaml .Values.controller.livenessProbe | nindent 12 }} - {{- end }} - {{- if .Values.controller.readinessProbe }} readinessProbe: {{ toYaml .Values.controller.readinessProbe | nindent 12 }} - {{- end }} ports: {{- range $key, $value := .Values.controller.containerPort }} - name: {{ $key }} @@ -119,7 +154,7 @@ spec: {{- end }} {{- end }} {{- if .Values.controller.metrics.enabled }} - - name: {{ .Values.controller.metrics.portName }} + - name: metrics containerPort: {{ .Values.controller.metrics.port }} protocol: TCP {{- end }} @@ -129,7 +164,7 @@ spec: protocol: TCP {{- end }} {{- range $key, $value := .Values.tcp }} - - name: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-tcp + - name: {{ $key }}-tcp containerPort: {{ $key }} protocol: TCP {{- if $.Values.controller.hostPort.enabled }} @@ -137,23 +172,15 @@ spec: {{- end }} {{- end }} {{- range $key, $value := .Values.udp }} - - name: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-udp + - name: {{ $key }}-udp containerPort: {{ $key }} protocol: UDP {{- if $.Values.controller.hostPort.enabled }} hostPort: {{ $key }} {{- end }} {{- end }} - {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules) }} + {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled) }} volumeMounts: - {{- if .Values.controller.extraModules }} - - name: modules - {{- if .Values.controller.image.chroot }} - mountPath: /chroot/modules_mount - {{- else }} - mountPath: /modules_mount - {{- end }} - {{- end }} {{- if .Values.controller.customTemplate.configMapName }} - mountPath: /etc/nginx/template name: nginx-template-volume @@ -172,19 +199,10 @@ spec: resources: {{ toYaml .Values.controller.resources | nindent 12 }} {{- end }} {{- if .Values.controller.extraContainers }} - {{- toYaml .Values.controller.extraContainers | nindent 8 }} - {{- end }} - {{- if (or .Values.controller.extraInitContainers .Values.controller.extraModules) }} - initContainers: - {{- if .Values.controller.extraInitContainers }} - {{- toYaml .Values.controller.extraInitContainers | nindent 8 }} - {{- end }} - {{- if .Values.controller.extraModules }} - {{- range .Values.controller.extraModules }} - {{- $containerSecurityContext := .containerSecurityContext | default $.Values.controller.containerSecurityContext }} - {{- include "extraModules" (dict "name" .name "image" (merge .image $.Values.global.image) "containerSecurityContext" $containerSecurityContext "resources" .resources) | nindent 8 }} - {{- end }} + {{ toYaml .Values.controller.extraContainers | nindent 8 }} {{- end }} + {{- if .Values.controller.extraInitContainers }} + initContainers: {{ toYaml .Values.controller.extraInitContainers | nindent 8 }} {{- end }} {{- if .Values.controller.hostNetwork }} hostNetwork: {{ .Values.controller.hostNetwork }} @@ -196,20 +214,15 @@ spec: tolerations: {{ toYaml .Values.controller.tolerations | nindent 8 }} {{- end }} {{- if .Values.controller.affinity }} - affinity: {{ tpl (toYaml .Values.controller.affinity) $ | nindent 8 }} + affinity: {{ toYaml .Values.controller.affinity | nindent 8 }} {{- end }} {{- if .Values.controller.topologySpreadConstraints }} - topologySpreadConstraints: {{ tpl (toYaml .Values.controller.topologySpreadConstraints) $ | nindent 8 }} + topologySpreadConstraints: {{ toYaml .Values.controller.topologySpreadConstraints | nindent 8 }} {{- end }} serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }} - automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }} - {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules) }} + {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes) }} volumes: - {{- if .Values.controller.extraModules }} - - name: modules - emptyDir: {} - {{- end }} {{- if .Values.controller.customTemplate.configMapName }} - name: nginx-template-volume configMap: @@ -221,14 +234,7 @@ spec: {{- if .Values.controller.admissionWebhooks.enabled }} - name: webhook-cert secret: - secretName: {{ include "ingress-nginx.admissionWebhooks.fullname" . }} - {{- if .Values.controller.admissionWebhooks.certManager.enabled }} - items: - - key: tls.crt - path: cert - - key: tls.key - path: key - {{- end }} + secretName: {{ include "ingress-nginx.fullname" . }}-admission {{- end }} {{- if .Values.controller.extraVolumes }} {{ toYaml .Values.controller.extraVolumes | nindent 8 }} diff --git a/charts/ingress-nginx/templates/controller-deployment.yaml b/charts/ingress-nginx/templates/controller-deployment.yaml index da8ce97d3..fe3b56a73 100644 --- a/charts/ingress-nginx/templates/controller-deployment.yaml +++ b/charts/ingress-nginx/templates/controller-deployment.yaml @@ -1,4 +1,5 @@ -{{- if eq .Values.controller.kind "Deployment" -}} +{{- if or (eq .Values.controller.kind "Deployment") (eq .Values.controller.kind "Both") -}} +{{- include "isControllerTagValid" . -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -9,7 +10,6 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} name: {{ include "ingress-nginx.controller.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} {{- if .Values.controller.annotations }} annotations: {{ toYaml .Values.controller.annotations | nindent 4 }} {{- end }} @@ -18,15 +18,13 @@ spec: matchLabels: {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: controller - {{- if eq .Values.controller.autoscaling.enabled .Values.controller.keda.enabled }} + {{- if not .Values.controller.autoscaling.enabled }} replicas: {{ .Values.controller.replicaCount }} {{- end }} revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} - {{- if .Values.controller.progressDeadlineSeconds }} - progressDeadlineSeconds: {{ .Values.controller.progressDeadlineSeconds }} - {{- end }} {{- if .Values.controller.updateStrategy }} - strategy: {{ toYaml .Values.controller.updateStrategy | nindent 4 }} + strategy: + {{ toYaml .Values.controller.updateStrategy | nindent 4 }} {{- end }} minReadySeconds: {{ .Values.controller.minReadySeconds }} template: @@ -38,58 +36,94 @@ spec: {{- end }} {{- end }} labels: - {{- include "ingress-nginx.labels" . | nindent 8 }} + {{- include "ingress-nginx.selectorLabels" . | nindent 8 }} app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 8 }} - {{- end }} {{- if .Values.controller.podLabels }} {{- toYaml .Values.controller.podLabels | nindent 8 }} {{- end }} spec: {{- if .Values.controller.dnsConfig }} dnsConfig: {{ toYaml .Values.controller.dnsConfig | nindent 8 }} - {{- end }} - {{- if .Values.controller.hostAliases }} - hostAliases: {{ tpl (toYaml .Values.controller.hostAliases) $ | nindent 8 }} - {{- end }} - {{- if .Values.controller.hostname }} - hostname: {{ toYaml .Values.controller.hostname | nindent 8 }} {{- end }} dnsPolicy: {{ .Values.controller.dnsPolicy }} {{- if .Values.imagePullSecrets }} imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }} {{- end }} {{- if .Values.controller.priorityClassName }} - priorityClassName: {{ .Values.controller.priorityClassName | quote }} + priorityClassName: {{ .Values.controller.priorityClassName }} {{- end }} {{- if or .Values.controller.podSecurityContext .Values.controller.sysctls }} securityContext: - {{- if .Values.controller.podSecurityContext }} - {{- toYaml .Values.controller.podSecurityContext | nindent 8 }} - {{- end }} - {{- if .Values.controller.sysctls }} - sysctls: - {{- range $sysctl, $value := .Values.controller.sysctls }} - - name: {{ $sysctl | quote }} - value: {{ $value | quote }} - {{- end }} - {{- end }} {{- end }} - {{- if .Values.controller.shareProcessNamespace }} - shareProcessNamespace: {{ .Values.controller.shareProcessNamespace }} + {{- if .Values.controller.podSecurityContext }} + {{- toYaml .Values.controller.podSecurityContext | nindent 8 }} + {{- end }} + {{- if .Values.controller.sysctls }} + sysctls: + {{- range $sysctl, $value := .Values.controller.sysctls }} + - name: {{ $sysctl | quote }} + value: {{ $value | quote }} + {{- end }} {{- end }} containers: - name: {{ .Values.controller.containerName }} - {{- with (merge .Values.controller.image .Values.global.image) }} - image: {{ if .repository }}{{ .repository }}{{ else }}{{ .registry }}/{{ include "ingress-nginx.image" . }}{{ end }}:{{ .tag }}{{ include "ingress-nginx.imageDigest" . }} + {{- with .Values.controller.image }} + image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}" {{- end }} imagePullPolicy: {{ .Values.controller.image.pullPolicy }} {{- if .Values.controller.lifecycle }} lifecycle: {{ toYaml .Values.controller.lifecycle | nindent 12 }} {{- end }} - args: {{ include "ingress-nginx.params" . | nindent 12 }} - securityContext: {{ include "ingress-nginx.controller.containerSecurityContext" . | nindent 12 }} + args: + - /nginx-ingress-controller + {{- if .Values.defaultBackend.enabled }} + - --default-backend-service=$(POD_NAMESPACE)/{{ include "ingress-nginx.defaultBackend.fullname" . }} + {{- end }} + {{- if .Values.controller.publishService.enabled }} + - --publish-service={{ template "ingress-nginx.controller.publishServicePath" . }} + {{- end }} + - --election-id={{ .Values.controller.electionID }} + - --ingress-class={{ .Values.controller.ingressClass }} + - --configmap=$(POD_NAMESPACE)/{{ include "ingress-nginx.controller.fullname" . }} + {{- if .Values.tcp }} + - --tcp-services-configmap=$(POD_NAMESPACE)/{{ include "ingress-nginx.fullname" . }}-tcp + {{- end }} + {{- if .Values.udp }} + - --udp-services-configmap=$(POD_NAMESPACE)/{{ include "ingress-nginx.fullname" . }}-udp + {{- end }} + {{- if .Values.controller.scope.enabled }} + - --watch-namespace={{ default "$(POD_NAMESPACE)" .Values.controller.scope.namespace }} + {{- end }} + {{- if and .Values.controller.reportNodeInternalIp .Values.controller.hostNetwork }} + - --report-node-internal-ip-address={{ .Values.controller.reportNodeInternalIp }} + {{- end }} + {{- if .Values.controller.admissionWebhooks.enabled }} + - --validating-webhook=:{{ .Values.controller.admissionWebhooks.port }} + - --validating-webhook-certificate={{ .Values.controller.admissionWebhooks.certificate }} + - --validating-webhook-key={{ .Values.controller.admissionWebhooks.key }} + {{- end }} + {{- if .Values.controller.maxmindLicenseKey }} + - --maxmind-license-key={{ .Values.controller.maxmindLicenseKey }} + {{- end }} + {{- if not (eq .Values.controller.healthCheckPath "/healthz") }} + - --health-check-path={{ .Values.controller.healthCheckPath }} + {{- end }} + {{- range $key, $value := .Values.controller.extraArgs }} + {{- /* Accept keys without values or with false as value */}} + {{- if eq ($value | quote | len) 2 }} + - --{{ $key }} + {{- else }} + - --{{ $key }}={{ $value }} + {{- end }} + {{- end }} + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: {{ .Values.controller.image.runAsUser }} + allowPrivilegeEscalation: {{ .Values.controller.image.allowPrivilegeEscalation }} env: - name: POD_NAME valueFrom: @@ -105,16 +139,12 @@ spec: {{- end }} {{- if .Values.controller.extraEnvs }} {{- toYaml .Values.controller.extraEnvs | nindent 12 }} - {{- end }} + {{- end }} {{- if .Values.controller.startupProbe }} startupProbe: {{ toYaml .Values.controller.startupProbe | nindent 12 }} {{- end }} - {{- if .Values.controller.livenessProbe }} livenessProbe: {{ toYaml .Values.controller.livenessProbe | nindent 12 }} - {{- end }} - {{- if .Values.controller.readinessProbe }} readinessProbe: {{ toYaml .Values.controller.readinessProbe | nindent 12 }} - {{- end }} ports: {{- range $key, $value := .Values.controller.containerPort }} - name: {{ $key }} @@ -125,7 +155,7 @@ spec: {{- end }} {{- end }} {{- if .Values.controller.metrics.enabled }} - - name: {{ .Values.controller.metrics.portName }} + - name: metrics containerPort: {{ .Values.controller.metrics.port }} protocol: TCP {{- end }} @@ -135,7 +165,7 @@ spec: protocol: TCP {{- end }} {{- range $key, $value := .Values.tcp }} - - name: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-tcp + - name: {{ $key }}-tcp containerPort: {{ $key }} protocol: TCP {{- if $.Values.controller.hostPort.enabled }} @@ -143,23 +173,15 @@ spec: {{- end }} {{- end }} {{- range $key, $value := .Values.udp }} - - name: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-udp + - name: {{ $key }}-udp containerPort: {{ $key }} protocol: UDP {{- if $.Values.controller.hostPort.enabled }} hostPort: {{ $key }} {{- end }} {{- end }} - {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules) }} + {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled) }} volumeMounts: - {{- if .Values.controller.extraModules }} - - name: modules - {{- if .Values.controller.image.chroot }} - mountPath: /chroot/modules_mount - {{- else }} - mountPath: /modules_mount - {{- end }} - {{- end }} {{- if .Values.controller.customTemplate.configMapName }} - mountPath: /etc/nginx/template name: nginx-template-volume @@ -178,19 +200,10 @@ spec: resources: {{ toYaml .Values.controller.resources | nindent 12 }} {{- end }} {{- if .Values.controller.extraContainers }} - {{- toYaml .Values.controller.extraContainers | nindent 8 }} - {{- end }} - {{- if (or .Values.controller.extraInitContainers .Values.controller.extraModules) }} - initContainers: - {{- if .Values.controller.extraInitContainers }} - {{- toYaml .Values.controller.extraInitContainers | nindent 8 }} - {{- end }} - {{- if .Values.controller.extraModules }} - {{- range .Values.controller.extraModules }} - {{- $containerSecurityContext := .containerSecurityContext | default $.Values.controller.containerSecurityContext }} - {{- include "extraModules" (dict "name" .name "image" (merge .image $.Values.global.image) "containerSecurityContext" $containerSecurityContext "resources" .resources) | nindent 8 }} - {{- end }} + {{ toYaml .Values.controller.extraContainers | nindent 8 }} {{- end }} + {{- if .Values.controller.extraInitContainers }} + initContainers: {{ toYaml .Values.controller.extraInitContainers | nindent 8 }} {{- end }} {{- if .Values.controller.hostNetwork }} hostNetwork: {{ .Values.controller.hostNetwork }} @@ -202,20 +215,15 @@ spec: tolerations: {{ toYaml .Values.controller.tolerations | nindent 8 }} {{- end }} {{- if .Values.controller.affinity }} - affinity: {{ tpl (toYaml .Values.controller.affinity) $ | nindent 8 }} + affinity: {{ toYaml .Values.controller.affinity | nindent 8 }} {{- end }} {{- if .Values.controller.topologySpreadConstraints }} - topologySpreadConstraints: {{ tpl (toYaml .Values.controller.topologySpreadConstraints) $ | nindent 8 }} + topologySpreadConstraints: {{ toYaml .Values.controller.topologySpreadConstraints | nindent 8 }} {{- end }} serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }} - automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }} - {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules) }} + {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes) }} volumes: - {{- if .Values.controller.extraModules }} - - name: modules - emptyDir: {} - {{- end }} {{- if .Values.controller.customTemplate.configMapName }} - name: nginx-template-volume configMap: @@ -227,17 +235,10 @@ spec: {{- if .Values.controller.admissionWebhooks.enabled }} - name: webhook-cert secret: - secretName: {{ include "ingress-nginx.admissionWebhooks.fullname" . }} - {{- if .Values.controller.admissionWebhooks.certManager.enabled }} - items: - - key: tls.crt - path: cert - - key: tls.key - path: key - {{- end }} + secretName: {{ include "ingress-nginx.fullname" . }}-admission {{- end }} {{- if .Values.controller.extraVolumes }} {{ toYaml .Values.controller.extraVolumes | nindent 8 }} {{- end }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/ingress-nginx/templates/controller-hpa.yaml b/charts/ingress-nginx/templates/controller-hpa.yaml index ec9ad7380..9eab00ebb 100644 --- a/charts/ingress-nginx/templates/controller-hpa.yaml +++ b/charts/ingress-nginx/templates/controller-hpa.yaml @@ -1,18 +1,17 @@ -{{- if and (eq .Values.controller.kind "Deployment") .Values.controller.autoscaling.enabled (not .Values.controller.keda.enabled) -}} -apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }} +{{- if and .Values.controller.autoscaling.enabled (or (eq .Values.controller.kind "Deployment") (eq .Values.controller.kind "Both")) -}} +{{- if not .Values.controller.keda.enabled }} + +apiVersion: autoscaling/v2beta2 kind: HorizontalPodAutoscaler metadata: + annotations: {{- with .Values.controller.autoscaling.annotations }} - annotations: {{ toYaml . | nindent 4 }} + {{- toYaml . | trimSuffix "\n" | nindent 4 }} {{- end }} labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} name: {{ include "ingress-nginx.controller.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} spec: scaleTargetRef: apiVersion: apps/v1 @@ -22,9 +21,9 @@ spec: maxReplicas: {{ .Values.controller.autoscaling.maxReplicas }} metrics: {{- with .Values.controller.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory + - type: Resource + resource: + name: memory target: type: Utilization averageUtilization: {{ . }} @@ -38,10 +37,8 @@ spec: averageUtilization: {{ . }} {{- end }} {{- with .Values.controller.autoscalingTemplate }} - {{- toYaml . | nindent 2 }} - {{- end }} - {{- with .Values.controller.autoscaling.behavior }} - behavior: - {{- toYaml . | nindent 4 }} +{{- toYaml . | nindent 2 }} {{- end }} {{- end }} +{{- end }} + diff --git a/charts/ingress-nginx/templates/controller-ingressclass-aliases.yaml b/charts/ingress-nginx/templates/controller-ingressclass-aliases.yaml deleted file mode 100644 index ffe22310c..000000000 --- a/charts/ingress-nginx/templates/controller-ingressclass-aliases.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- if .Values.controller.ingressClassResource.enabled -}} -{{- range .Values.controller.ingressClassResource.aliases }} ---- -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - labels: - {{- include "ingress-nginx.labels" $ | nindent 4 }} - app.kubernetes.io/component: controller - {{- with $.Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - name: {{ . }} - {{- if $.Values.controller.ingressClassResource.annotations }} - annotations: {{ toYaml $.Values.controller.ingressClassResource.annotations | nindent 4 }} - {{- end }} -spec: - controller: {{ $.Values.controller.ingressClassResource.controllerValue }} - {{- with $.Values.controller.ingressClassResource.parameters }} - parameters: {{ toYaml . | nindent 4 }} - {{- end }} -{{- end }} -{{- end }} diff --git a/charts/ingress-nginx/templates/controller-ingressclass.yaml b/charts/ingress-nginx/templates/controller-ingressclass.yaml deleted file mode 100644 index 98479a529..000000000 --- a/charts/ingress-nginx/templates/controller-ingressclass.yaml +++ /dev/null @@ -1,26 +0,0 @@ -{{- if .Values.controller.ingressClassResource.enabled -}} -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - labels: - {{- include "ingress-nginx.labels" . | nindent 4 }} - app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - name: {{ .Values.controller.ingressClassResource.name }} - {{- if or .Values.controller.ingressClassResource.default .Values.controller.ingressClassResource.annotations }} - annotations: - {{- if .Values.controller.ingressClassResource.default }} - ingressclass.kubernetes.io/is-default-class: "true" - {{- end }} - {{- if .Values.controller.ingressClassResource.annotations }} - {{- toYaml .Values.controller.ingressClassResource.annotations | nindent 4 }} - {{- end }} - {{- end }} -spec: - controller: {{ .Values.controller.ingressClassResource.controllerValue }} - {{- with .Values.controller.ingressClassResource.parameters }} - parameters: {{ toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/charts/ingress-nginx/templates/controller-keda.yaml b/charts/ingress-nginx/templates/controller-keda.yaml index 24d30fa0a..c7eebf5c8 100644 --- a/charts/ingress-nginx/templates/controller-keda.yaml +++ b/charts/ingress-nginx/templates/controller-keda.yaml @@ -1,15 +1,13 @@ -{{- if and (eq .Values.controller.kind "Deployment") .Values.controller.keda.enabled (not .Values.controller.autoscaling.enabled) -}} +{{- if and .Values.controller.keda.enabled (or (eq .Values.controller.kind "Deployment") (eq .Values.controller.kind "Both")) -}} +# https://keda.sh/docs/ + apiVersion: {{ .Values.controller.keda.apiVersion }} kind: ScaledObject metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} name: {{ include "ingress-nginx.controller.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} {{- if .Values.controller.keda.scaledObject.annotations }} annotations: {{ toYaml .Values.controller.keda.scaledObject.annotations | nindent 4 }} {{- end }} @@ -24,11 +22,6 @@ spec: cooldownPeriod: {{ .Values.controller.keda.cooldownPeriod }} minReplicaCount: {{ .Values.controller.keda.minReplicas }} maxReplicaCount: {{ .Values.controller.keda.maxReplicas }} -{{- with .Values.controller.keda.fallback }} - fallback: - failureThreshold: {{ .failureThreshold | default 3 }} - replicas: {{ .replicas | default $.Values.controller.keda.maxReplicas }} -{{- end }} triggers: {{- with .Values.controller.keda.triggers }} {{ toYaml . | indent 2 }} diff --git a/charts/ingress-nginx/templates/controller-networkpolicy.yaml b/charts/ingress-nginx/templates/controller-networkpolicy.yaml deleted file mode 100644 index e68f9916d..000000000 --- a/charts/ingress-nginx/templates/controller-networkpolicy.yaml +++ /dev/null @@ -1,45 +0,0 @@ -{{- if .Values.controller.networkPolicy.enabled }} -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - labels: - {{- include "ingress-nginx.labels" . | nindent 4 }} - app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - name: {{ include "ingress-nginx.controller.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} -spec: - podSelector: - matchLabels: - {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: controller - policyTypes: - - Ingress - - Egress - ingress: - - ports: - {{- range $key, $value := .Values.controller.containerPort }} - - protocol: TCP - port: {{ $value }} - {{- end }} - {{- if .Values.controller.metrics.enabled }} - - protocol: TCP - port: {{ .Values.controller.metrics.port }} - {{- end }} - {{- if .Values.controller.admissionWebhooks.enabled }} - - protocol: TCP - port: {{ .Values.controller.admissionWebhooks.port }} - {{- end }} - {{- range $key, $value := .Values.tcp }} - - protocol: TCP - port: {{ $key }} - {{- end }} - {{- range $key, $value := .Values.udp }} - - protocol: UDP - port: {{ $key }} - {{- end }} - egress: - - {} -{{- end }} diff --git a/charts/ingress-nginx/templates/controller-poddisruptionbudget.yaml b/charts/ingress-nginx/templates/controller-poddisruptionbudget.yaml index a1f5fbba2..369c12bdb 100644 --- a/charts/ingress-nginx/templates/controller-poddisruptionbudget.yaml +++ b/charts/ingress-nginx/templates/controller-poddisruptionbudget.yaml @@ -1,39 +1,15 @@ -# PDB is not supported for DaemonSets. -# https://github.com/kubernetes/kubernetes/issues/108124 -{{- if eq .Values.controller.kind "Deployment" }} -{{- $replicas := .Values.controller.replicaCount }} -{{- if and .Values.controller.autoscaling.enabled (not .Values.controller.keda.enabled) }} -{{- $replicas = .Values.controller.autoscaling.minReplicas }} -{{- else if and .Values.controller.keda.enabled (not .Values.controller.autoscaling.enabled) }} -{{- $replicas = .Values.controller.keda.minReplicas }} -{{- end }} -{{- if gt ($replicas | int) 1 }} -apiVersion: {{ ternary "policy/v1" "policy/v1beta1" (semverCompare ">=1.21.0-0" .Capabilities.KubeVersion.Version) }} +{{- if or (and .Values.controller.autoscaling.enabled (gt (.Values.controller.autoscaling.minReplicas | int) 1)) (and (not .Values.controller.autoscaling.enabled) (gt (.Values.controller.replicaCount | int) 1)) }} +apiVersion: policy/v1beta1 kind: PodDisruptionBudget metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} name: {{ include "ingress-nginx.controller.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} - {{- if .Values.controller.annotations }} - annotations: {{ toYaml .Values.controller.annotations | nindent 4 }} - {{- end }} spec: selector: matchLabels: {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: controller - {{- if and .Values.controller.minAvailable (not (hasKey .Values.controller "maxUnavailable")) }} minAvailable: {{ .Values.controller.minAvailable }} - {{- else if .Values.controller.maxUnavailable }} - maxUnavailable: {{ .Values.controller.maxUnavailable }} - {{- end }} - {{- if .Values.controller.unhealthyPodEvictionPolicy }} - unhealthyPodEvictionPolicy: {{ .Values.controller.unhealthyPodEvictionPolicy }} - {{- end }} -{{- end }} {{- end }} diff --git a/charts/ingress-nginx/templates/controller-prometheusrule.yaml b/charts/ingress-nginx/templates/controller-prometheusrules.yaml similarity index 78% rename from charts/ingress-nginx/templates/controller-prometheusrule.yaml rename to charts/ingress-nginx/templates/controller-prometheusrules.yaml index 4a9357f71..ca5427523 100644 --- a/charts/ingress-nginx/templates/controller-prometheusrule.yaml +++ b/charts/ingress-nginx/templates/controller-prometheusrules.yaml @@ -4,9 +4,7 @@ kind: PrometheusRule metadata: name: {{ include "ingress-nginx.controller.fullname" . }} {{- if .Values.controller.metrics.prometheusRule.namespace }} - namespace: {{ .Values.controller.metrics.prometheusRule.namespace }} -{{- else }} - namespace: {{ include "ingress-nginx.namespace" . }} + namespace: {{ .Values.controller.metrics.prometheusRule.namespace | quote }} {{- end }} labels: {{- include "ingress-nginx.labels" . | nindent 4 }} @@ -14,9 +12,6 @@ metadata: {{- if .Values.controller.metrics.prometheusRule.additionalLabels }} {{- toYaml .Values.controller.metrics.prometheusRule.additionalLabels | nindent 4 }} {{- end }} - {{- if .Values.controller.metrics.prometheusRule.annotations }} - annotations: {{ toYaml .Values.controller.metrics.prometheusRule.annotations | nindent 4 }} - {{- end }} spec: {{- if .Values.controller.metrics.prometheusRule.rules }} groups: diff --git a/charts/ingress-nginx/templates/controller-psp.yaml b/charts/ingress-nginx/templates/controller-psp.yaml new file mode 100644 index 000000000..bdb856310 --- /dev/null +++ b/charts/ingress-nginx/templates/controller-psp.yaml @@ -0,0 +1,86 @@ +{{- if and .Values.podSecurityPolicy.enabled (empty .Values.controller.existingPsp) -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "ingress-nginx.fullname" . }} + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller +spec: + allowedCapabilities: + - NET_BIND_SERVICE +{{- if .Values.controller.sysctls }} + allowedUnsafeSysctls: + {{- range $sysctl, $value := .Values.controller.sysctls }} + - {{ $sysctl }} + {{- end }} +{{- end }} + privileged: false + allowPrivilegeEscalation: true + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + #- 'projected' + - 'secret' + #- 'downwardAPI' +{{- if .Values.controller.hostNetwork }} + hostNetwork: {{ .Values.controller.hostNetwork }} +{{- end }} +{{- if or .Values.controller.hostNetwork .Values.controller.hostPort.enabled }} + hostPorts: +{{- if .Values.controller.hostNetwork }} +{{- range $key, $value := .Values.controller.containerPort }} + # {{ $key }} + - min: {{ $value }} + max: {{ $value }} +{{- end }} +{{- else if .Values.controller.hostPort.enabled }} +{{- range $key, $value := .Values.controller.hostPort.ports }} + # {{ $key }} + - min: {{ $value }} + max: {{ $value }} +{{- end }} +{{- end }} +{{- if .Values.controller.metrics.enabled }} + # metrics + - min: {{ .Values.controller.metrics.port }} + max: {{ .Values.controller.metrics.port }} +{{- end }} +{{- if .Values.controller.admissionWebhooks.enabled }} + # admission webhooks + - min: {{ .Values.controller.admissionWebhooks.port }} + max: {{ .Values.controller.admissionWebhooks.port }} +{{- end }} +{{- range $key, $value := .Values.tcp }} + # {{ $key }}-tcp + - min: {{ $key }} + max: {{ $key }} +{{- end }} +{{- range $key, $value := .Values.udp }} + # {{ $key }}-udp + - min: {{ $key }} + max: {{ $key }} +{{- end }} +{{- end }} + hostIPC: false + hostPID: false + runAsUser: + # Require the container to run without root privileges. + rule: 'MustRunAsNonRoot' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + seLinux: + rule: 'RunAsAny' +{{- end }} diff --git a/charts/ingress-nginx/templates/controller-role.yaml b/charts/ingress-nginx/templates/controller-role.yaml index 127b368c4..52f830315 100644 --- a/charts/ingress-nginx/templates/controller-role.yaml +++ b/charts/ingress-nginx/templates/controller-role.yaml @@ -5,11 +5,7 @@ metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} name: {{ include "ingress-nginx.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} rules: - apiGroups: - "" @@ -37,24 +33,23 @@ rules: - list - watch - apiGroups: - - networking.k8s.io + - extensions + - "networking.k8s.io" # k8s 1.14+ resources: - ingresses verbs: - get - list - watch - # Omit Ingress status permissions if `--update-status` is disabled. - {{- if ne (index .Values.controller.extraArgs "update-status") "false" }} - apiGroups: - - networking.k8s.io + - extensions + - "networking.k8s.io" # k8s 1.14+ resources: - ingresses/status verbs: - update - {{- end }} - apiGroups: - - networking.k8s.io + - "networking.k8s.io" # k8s 1.14+ resources: - ingressclasses verbs: @@ -62,18 +57,18 @@ rules: - list - watch - apiGroups: - - coordination.k8s.io + - "" resources: - - leases + - configmaps resourceNames: - - {{ include "ingress-nginx.controller.electionID" . }} + - {{ .Values.controller.electionID }}-{{ .Values.controller.ingressClass }} verbs: - get - update - apiGroups: - - coordination.k8s.io + - "" resources: - - leases + - configmaps verbs: - create - apiGroups: @@ -83,12 +78,14 @@ rules: verbs: - create - patch - - apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get +{{- if .Values.podSecurityPolicy.enabled }} + - apiGroups: [{{ template "podSecurityPolicy.apiGroup" . }}] + resources: ['podsecuritypolicies'] + verbs: ['use'] + {{- with .Values.controller.existingPsp }} + resourceNames: [{{ . }}] + {{- else }} + resourceNames: [{{ include "ingress-nginx.fullname" . }}] + {{- end }} +{{- end }} {{- end }} diff --git a/charts/ingress-nginx/templates/controller-rolebinding.yaml b/charts/ingress-nginx/templates/controller-rolebinding.yaml index 153430aa2..9ab3b461c 100644 --- a/charts/ingress-nginx/templates/controller-rolebinding.yaml +++ b/charts/ingress-nginx/templates/controller-rolebinding.yaml @@ -5,11 +5,7 @@ metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} name: {{ include "ingress-nginx.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -17,5 +13,5 @@ roleRef: subjects: - kind: ServiceAccount name: {{ template "ingress-nginx.serviceAccountName" . }} - namespace: {{ include "ingress-nginx.namespace" . }} + namespace: {{ .Release.Namespace | quote }} {{- end }} diff --git a/charts/ingress-nginx/templates/controller-secret.yaml b/charts/ingress-nginx/templates/controller-secret.yaml deleted file mode 100644 index f20f53469..000000000 --- a/charts/ingress-nginx/templates/controller-secret.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if .Values.dhParam -}} -apiVersion: v1 -kind: Secret -metadata: - labels: - {{- include "ingress-nginx.labels" . | nindent 4 }} - app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - name: {{ include "ingress-nginx.controller.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} -data: - dhparam.pem: {{ .Values.dhParam }} -{{- end }} diff --git a/charts/ingress-nginx/templates/controller-service-internal.yaml b/charts/ingress-nginx/templates/controller-service-internal.yaml index 8d369526d..49b4ee1d6 100644 --- a/charts/ingress-nginx/templates/controller-service-internal.yaml +++ b/charts/ingress-nginx/templates/controller-service-internal.yaml @@ -1,10 +1,10 @@ -{{- if and .Values.controller.service.enabled .Values.controller.service.internal.enabled .Values.controller.service.internal.annotations -}} +{{- if and .Values.controller.service.enabled .Values.controller.service.internal.enabled .Values.controller.service.internal.annotations}} apiVersion: v1 kind: Service metadata: annotations: {{- range $key, $value := .Values.controller.service.internal.annotations }} - {{ $key }}: {{ tpl ($value | toString) $ | quote }} + {{ $key }}: {{ $value | quote }} {{- end }} labels: {{- include "ingress-nginx.labels" . | nindent 4 }} @@ -12,102 +12,36 @@ metadata: {{- if .Values.controller.service.labels }} {{- toYaml .Values.controller.service.labels | nindent 4 }} {{- end }} - {{- if .Values.controller.service.internal.labels }} - {{- toYaml .Values.controller.service.internal.labels | nindent 4 }} - {{- end }} name: {{ include "ingress-nginx.controller.fullname" . }}-internal - namespace: {{ include "ingress-nginx.namespace" . }} spec: - type: {{ .Values.controller.service.internal.type | default .Values.controller.service.type }} -{{- if .Values.controller.service.internal.clusterIP }} - clusterIP: {{ .Values.controller.service.internal.clusterIP }} -{{- end }} -{{- if .Values.controller.service.internal.clusterIPs }} - clusterIPs: {{ toYaml .Values.controller.service.internal.clusterIPs | nindent 4 }} -{{- end }} -{{- if .Values.controller.service.internal.externalIPs }} - externalIPs: {{ toYaml .Values.controller.service.internal.externalIPs | nindent 4 }} -{{- end }} + type: "{{ .Values.controller.service.type }}" {{- if .Values.controller.service.internal.loadBalancerIP }} loadBalancerIP: {{ .Values.controller.service.internal.loadBalancerIP }} {{- end }} {{- if .Values.controller.service.internal.loadBalancerSourceRanges }} loadBalancerSourceRanges: {{ toYaml .Values.controller.service.internal.loadBalancerSourceRanges | nindent 4 }} {{- end }} -{{- if .Values.controller.service.internal.loadBalancerClass }} - loadBalancerClass: {{ .Values.controller.service.internal.loadBalancerClass }} -{{- end }} -{{- if hasKey .Values.controller.service.internal "allocateLoadBalancerNodePorts" }} - allocateLoadBalancerNodePorts: {{ .Values.controller.service.internal.allocateLoadBalancerNodePorts }} -{{- end }} {{- if .Values.controller.service.internal.externalTrafficPolicy }} externalTrafficPolicy: {{ .Values.controller.service.internal.externalTrafficPolicy }} -{{- end }} -{{- if .Values.controller.service.internal.sessionAffinity }} - sessionAffinity: {{ .Values.controller.service.internal.sessionAffinity }} -{{- end }} -{{- if .Values.controller.service.internal.healthCheckNodePort }} - healthCheckNodePort: {{ .Values.controller.service.internal.healthCheckNodePort }} -{{- end }} -{{- if semverCompare ">=1.31.0-0" .Capabilities.KubeVersion.Version -}} -{{- if .Values.controller.service.internal.trafficDistribution }} - trafficDistribution: {{ .Values.controller.service.internal.trafficDistribution }} -{{- end }} -{{- end }} -{{- if semverCompare ">=1.21.0-0" .Capabilities.KubeVersion.Version -}} -{{- if .Values.controller.service.internal.ipFamilyPolicy }} - ipFamilyPolicy: {{ .Values.controller.service.internal.ipFamilyPolicy }} -{{- end }} -{{- if .Values.controller.service.internal.ipFamilies }} - ipFamilies: {{ toYaml .Values.controller.service.internal.ipFamilies | nindent 4 }} -{{- end }} {{- end }} ports: {{- $setNodePorts := (or (eq .Values.controller.service.type "NodePort") (eq .Values.controller.service.type "LoadBalancer")) }} {{- if .Values.controller.service.enableHttp }} - name: http - port: {{ .Values.controller.service.internal.ports.http | default .Values.controller.service.ports.http }} + port: {{ .Values.controller.service.ports.http }} protocol: TCP - targetPort: {{ .Values.controller.service.internal.targetPorts.http | default .Values.controller.service.targetPorts.http }} - {{- if and (semverCompare ">=1.20.0-0" .Capabilities.KubeVersion.Version) (.Values.controller.service.internal.appProtocol) }} - appProtocol: http - {{- end }} - {{- if (and $setNodePorts (not (empty .Values.controller.service.internal.nodePorts.http))) }} - nodePort: {{ .Values.controller.service.internal.nodePorts.http }} + targetPort: {{ .Values.controller.service.targetPorts.http }} + {{- if (and $setNodePorts (not (empty .Values.controller.service.nodePorts.http))) }} + nodePort: {{ .Values.controller.service.nodePorts.http }} {{- end }} {{- end }} {{- if .Values.controller.service.enableHttps }} - name: https - port: {{ .Values.controller.service.internal.ports.https | default .Values.controller.service.ports.https }} + port: {{ .Values.controller.service.ports.https }} protocol: TCP - targetPort: {{ .Values.controller.service.internal.targetPorts.https | default .Values.controller.service.targetPorts.https }} - {{- if and (semverCompare ">=1.20.0-0" .Capabilities.KubeVersion.Version) (.Values.controller.service.internal.appProtocol) }} - appProtocol: https - {{- end }} - {{- if (and $setNodePorts (not (empty .Values.controller.service.internal.nodePorts.https))) }} - nodePort: {{ .Values.controller.service.internal.nodePorts.https }} - {{- end }} - {{- end }} - {{- range $key, $value := .Values.tcp }} - - name: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-tcp - port: {{ $key }} - protocol: TCP - targetPort: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-tcp - {{- if $.Values.controller.service.internal.nodePorts.tcp }} - {{- if index $.Values.controller.service.internal.nodePorts.tcp $key }} - nodePort: {{ index $.Values.controller.service.internal.nodePorts.tcp $key }} - {{- end }} - {{- end }} - {{- end }} - {{- range $key, $value := .Values.udp }} - - name: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-udp - port: {{ $key }} - protocol: UDP - targetPort: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-udp - {{- if $.Values.controller.service.internal.nodePorts.udp }} - {{- if index $.Values.controller.service.internal.nodePorts.udp $key }} - nodePort: {{ index $.Values.controller.service.internal.nodePorts.udp $key }} - {{- end }} + targetPort: {{ .Values.controller.service.targetPorts.https }} + {{- if (and $setNodePorts (not (empty .Values.controller.service.nodePorts.https))) }} + nodePort: {{ .Values.controller.service.nodePorts.https }} {{- end }} {{- end }} selector: diff --git a/charts/ingress-nginx/templates/controller-service-metrics.yaml b/charts/ingress-nginx/templates/controller-service-metrics.yaml index 4b25a840e..b01f460af 100644 --- a/charts/ingress-nginx/templates/controller-service-metrics.yaml +++ b/charts/ingress-nginx/templates/controller-service-metrics.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.controller.metrics.enabled .Values.controller.metrics.service.enabled -}} +{{- if .Values.controller.metrics.enabled -}} apiVersion: v1 kind: Service metadata: @@ -12,7 +12,6 @@ metadata: {{- toYaml .Values.controller.metrics.service.labels | nindent 4 }} {{- end }} name: {{ include "ingress-nginx.controller.fullname" . }}-metrics - namespace: {{ include "ingress-nginx.namespace" . }} spec: type: {{ .Values.controller.metrics.service.type }} {{- if .Values.controller.metrics.service.clusterIP }} @@ -31,10 +30,9 @@ spec: externalTrafficPolicy: {{ .Values.controller.metrics.service.externalTrafficPolicy }} {{- end }} ports: - - name: {{ .Values.controller.metrics.portName }} + - name: metrics port: {{ .Values.controller.metrics.service.servicePort }} - protocol: TCP - targetPort: {{ .Values.controller.metrics.portName }} + targetPort: metrics {{- $setNodePorts := (or (eq .Values.controller.metrics.service.type "NodePort") (eq .Values.controller.metrics.service.type "LoadBalancer")) }} {{- if (and $setNodePorts (not (empty .Values.controller.metrics.service.nodePort))) }} nodePort: {{ .Values.controller.metrics.service.nodePort }} diff --git a/charts/ingress-nginx/templates/controller-service-webhook.yaml b/charts/ingress-nginx/templates/controller-service-webhook.yaml index 67aac0d9a..7a4dd51db 100644 --- a/charts/ingress-nginx/templates/controller-service-webhook.yaml +++ b/charts/ingress-nginx/templates/controller-service-webhook.yaml @@ -8,11 +8,7 @@ metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} name: {{ include "ingress-nginx.controller.fullname" . }}-admission - namespace: {{ include "ingress-nginx.namespace" . }} spec: type: {{ .Values.controller.admissionWebhooks.service.type }} {{- if .Values.controller.admissionWebhooks.service.clusterIP }} @@ -29,11 +25,8 @@ spec: {{- end }} ports: - name: https-webhook - port: {{ .Values.controller.admissionWebhooks.service.servicePort }} + port: 443 targetPort: webhook - {{- if semverCompare ">=1.20.0-0" .Capabilities.KubeVersion.Version }} - appProtocol: https - {{- end }} selector: {{- include "ingress-nginx.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: controller diff --git a/charts/ingress-nginx/templates/controller-service.yaml b/charts/ingress-nginx/templates/controller-service.yaml index 36d2e4884..9db734deb 100644 --- a/charts/ingress-nginx/templates/controller-service.yaml +++ b/charts/ingress-nginx/templates/controller-service.yaml @@ -1,10 +1,10 @@ -{{- if and .Values.controller.service.enabled .Values.controller.service.external.enabled -}} +{{- if .Values.controller.service.enabled -}} apiVersion: v1 kind: Service metadata: annotations: {{- range $key, $value := .Values.controller.service.annotations }} - {{ $key }}: {{ tpl ($value | toString) $ | quote }} + {{ $key }}: {{ $value | quote }} {{- end }} labels: {{- include "ingress-nginx.labels" . | nindent 4 }} @@ -12,19 +12,12 @@ metadata: {{- if .Values.controller.service.labels }} {{- toYaml .Values.controller.service.labels | nindent 4 }} {{- end }} - {{- if .Values.controller.service.external.labels }} - {{- toYaml .Values.controller.service.external.labels | nindent 4 }} - {{- end }} name: {{ include "ingress-nginx.controller.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} spec: type: {{ .Values.controller.service.type }} {{- if .Values.controller.service.clusterIP }} clusterIP: {{ .Values.controller.service.clusterIP }} {{- end }} -{{- if .Values.controller.service.clusterIPs }} - clusterIPs: {{ toYaml .Values.controller.service.clusterIPs | nindent 4 }} -{{- end }} {{- if .Values.controller.service.externalIPs }} externalIPs: {{ toYaml .Values.controller.service.externalIPs | nindent 4 }} {{- end }} @@ -34,12 +27,6 @@ spec: {{- if .Values.controller.service.loadBalancerSourceRanges }} loadBalancerSourceRanges: {{ toYaml .Values.controller.service.loadBalancerSourceRanges | nindent 4 }} {{- end }} -{{- if .Values.controller.service.loadBalancerClass }} - loadBalancerClass: {{ .Values.controller.service.loadBalancerClass }} -{{- end }} -{{- if hasKey .Values.controller.service "allocateLoadBalancerNodePorts" }} - allocateLoadBalancerNodePorts: {{ .Values.controller.service.allocateLoadBalancerNodePorts }} -{{- end }} {{- if .Values.controller.service.externalTrafficPolicy }} externalTrafficPolicy: {{ .Values.controller.service.externalTrafficPolicy }} {{- end }} @@ -48,19 +35,6 @@ spec: {{- end }} {{- if .Values.controller.service.healthCheckNodePort }} healthCheckNodePort: {{ .Values.controller.service.healthCheckNodePort }} -{{- end }} -{{- if semverCompare ">=1.31.0-0" .Capabilities.KubeVersion.Version -}} -{{- if .Values.controller.service.trafficDistribution }} - trafficDistribution: {{ .Values.controller.service.trafficDistribution }} -{{- end }} -{{- end }} -{{- if semverCompare ">=1.21.0-0" .Capabilities.KubeVersion.Version -}} -{{- if .Values.controller.service.ipFamilyPolicy }} - ipFamilyPolicy: {{ .Values.controller.service.ipFamilyPolicy }} -{{- end }} -{{- if .Values.controller.service.ipFamilies }} - ipFamilies: {{ toYaml .Values.controller.service.ipFamilies | nindent 4 }} -{{- end }} {{- end }} ports: {{- $setNodePorts := (or (eq .Values.controller.service.type "NodePort") (eq .Values.controller.service.type "LoadBalancer")) }} @@ -69,9 +43,6 @@ spec: port: {{ .Values.controller.service.ports.http }} protocol: TCP targetPort: {{ .Values.controller.service.targetPorts.http }} - {{- if and (semverCompare ">=1.20.0-0" .Capabilities.KubeVersion.Version) (.Values.controller.service.appProtocol) }} - appProtocol: http - {{- end }} {{- if (and $setNodePorts (not (empty .Values.controller.service.nodePorts.http))) }} nodePort: {{ .Values.controller.service.nodePorts.http }} {{- end }} @@ -81,18 +52,15 @@ spec: port: {{ .Values.controller.service.ports.https }} protocol: TCP targetPort: {{ .Values.controller.service.targetPorts.https }} - {{- if and (semverCompare ">=1.20.0-0" .Capabilities.KubeVersion.Version) (.Values.controller.service.appProtocol) }} - appProtocol: https - {{- end }} {{- if (and $setNodePorts (not (empty .Values.controller.service.nodePorts.https))) }} nodePort: {{ .Values.controller.service.nodePorts.https }} {{- end }} {{- end }} {{- range $key, $value := .Values.tcp }} - - name: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-tcp + - name: {{ $key }}-tcp port: {{ $key }} protocol: TCP - targetPort: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-tcp + targetPort: {{ $key }}-tcp {{- if $.Values.controller.service.nodePorts.tcp }} {{- if index $.Values.controller.service.nodePorts.tcp $key }} nodePort: {{ index $.Values.controller.service.nodePorts.tcp $key }} @@ -100,10 +68,10 @@ spec: {{- end }} {{- end }} {{- range $key, $value := .Values.udp }} - - name: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-udp + - name: {{ $key }}-udp port: {{ $key }} protocol: UDP - targetPort: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-udp + targetPort: {{ $key }}-udp {{- if $.Values.controller.service.nodePorts.udp }} {{- if index $.Values.controller.service.nodePorts.udp $key }} nodePort: {{ index $.Values.controller.service.nodePorts.udp $key }} diff --git a/charts/ingress-nginx/templates/controller-serviceaccount.yaml b/charts/ingress-nginx/templates/controller-serviceaccount.yaml index df83de3d0..f4b1278f6 100644 --- a/charts/ingress-nginx/templates/controller-serviceaccount.yaml +++ b/charts/ingress-nginx/templates/controller-serviceaccount.yaml @@ -5,13 +5,6 @@ metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} name: {{ template "ingress-nginx.serviceAccountName" . }} - namespace: {{ include "ingress-nginx.namespace" . }} - {{- if .Values.serviceAccount.annotations }} - annotations: {{ toYaml .Values.serviceAccount.annotations | nindent 4 }} - {{- end }} automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} {{- end }} diff --git a/charts/ingress-nginx/templates/controller-servicemonitor.yaml b/charts/ingress-nginx/templates/controller-servicemonitor.yaml index 85bb84186..066488a04 100644 --- a/charts/ingress-nginx/templates/controller-servicemonitor.yaml +++ b/charts/ingress-nginx/templates/controller-servicemonitor.yaml @@ -3,63 +3,43 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: {{ include "ingress-nginx.controller.fullname" . }} - {{- if .Values.controller.metrics.serviceMonitor.namespace }} - namespace: {{ .Values.controller.metrics.serviceMonitor.namespace }} - {{- else }} - namespace: {{ include "ingress-nginx.namespace" . }} - {{- end }} +{{- if .Values.controller.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.controller.metrics.serviceMonitor.namespace | quote }} +{{- end }} labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: controller - {{- if .Values.controller.metrics.serviceMonitor.additionalLabels }} + {{- if .Values.controller.metrics.serviceMonitor.additionalLabels }} {{- toYaml .Values.controller.metrics.serviceMonitor.additionalLabels | nindent 4 }} - {{- end }} - {{- if .Values.controller.metrics.serviceMonitor.annotations }} - annotations: {{ toYaml .Values.controller.metrics.serviceMonitor.annotations | nindent 4 }} {{- end }} spec: - {{- if .Values.controller.metrics.serviceMonitor.namespaceSelector }} + endpoints: + - port: metrics + interval: {{ .Values.controller.metrics.serviceMonitor.scrapeInterval }} + {{- if .Values.controller.metrics.serviceMonitor.honorLabels }} + honorLabels: true + {{- end }} + {{- if .Values.controller.metrics.serviceMonitor.metricRelabelings }} + metricRelabelings: {{ toYaml .Values.controller.metrics.serviceMonitor.metricRelabelings | nindent 8 }} + {{- end }} +{{- if .Values.controller.metrics.serviceMonitor.jobLabel }} + jobLabel: {{ .Values.controller.metrics.serviceMonitor.jobLabel | quote }} +{{- end }} +{{- if .Values.controller.metrics.serviceMonitor.namespaceSelector }} namespaceSelector: {{ toYaml .Values.controller.metrics.serviceMonitor.namespaceSelector | nindent 4 }} - {{- else }} +{{ else }} namespaceSelector: matchNames: - - {{ include "ingress-nginx.namespace" . }} + - {{ .Release.Namespace }} +{{- end }} +{{- if .Values.controller.metrics.serviceMonitor.targetLabels }} + targetLabels: + {{- range .Values.controller.metrics.serviceMonitor.targetLabels }} + - {{ . }} {{- end }} +{{- end }} selector: matchLabels: {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: controller - endpoints: - - port: {{ .Values.controller.metrics.portName }} - interval: {{ .Values.controller.metrics.serviceMonitor.scrapeInterval }} - {{- if .Values.controller.metrics.serviceMonitor.honorLabels }} - honorLabels: true - {{- end }} - {{- if .Values.controller.metrics.serviceMonitor.relabelings }} - relabelings: {{ toYaml .Values.controller.metrics.serviceMonitor.relabelings | nindent 4 }} - {{- end }} - {{- if .Values.controller.metrics.serviceMonitor.metricRelabelings }} - metricRelabelings: {{ toYaml .Values.controller.metrics.serviceMonitor.metricRelabelings | nindent 4 }} - {{- end }} - {{- if .Values.controller.metrics.serviceMonitor.jobLabel }} - jobLabel: {{ .Values.controller.metrics.serviceMonitor.jobLabel | quote }} - {{- end }} - {{- if .Values.controller.metrics.serviceMonitor.targetLabels }} - targetLabels: {{ toYaml .Values.controller.metrics.serviceMonitor.targetLabels | nindent 2 }} - {{- end }} - {{- if .Values.controller.metrics.serviceMonitor.labelLimit }} - labelLimit: {{ .Values.controller.metrics.serviceMonitor.labelLimit }} - {{- end }} - {{- if .Values.controller.metrics.serviceMonitor.labelNameLengthLimit }} - labelNameLengthLimit: {{ .Values.controller.metrics.serviceMonitor.labelNameLengthLimit }} - {{- end }} - {{- if .Values.controller.metrics.serviceMonitor.labelValueLengthLimit }} - labelValueLengthLimit: {{ .Values.controller.metrics.serviceMonitor.labelValueLengthLimit }} - {{- end }} - {{- if .Values.controller.metrics.serviceMonitor.sampleLimit }} - sampleLimit: {{ .Values.controller.metrics.serviceMonitor.sampleLimit }} - {{- end }} - {{- if .Values.controller.metrics.serviceMonitor.targetLimit }} - targetLimit: {{ .Values.controller.metrics.serviceMonitor.targetLimit }} - {{- end }} {{- end }} diff --git a/charts/ingress-nginx/templates/default-backend-deployment.yaml b/charts/ingress-nginx/templates/default-backend-deployment.yaml index 4a17f7444..ff8a6d9a6 100644 --- a/charts/ingress-nginx/templates/default-backend-deployment.yaml +++ b/charts/ingress-nginx/templates/default-backend-deployment.yaml @@ -5,11 +5,7 @@ metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: default-backend - {{- with .Values.defaultBackend.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} name: {{ include "ingress-nginx.defaultBackend.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} spec: selector: matchLabels: @@ -19,22 +15,14 @@ spec: replicas: {{ .Values.defaultBackend.replicaCount }} {{- end }} revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} - {{- if .Values.defaultBackend.updateStrategy }} - strategy: - {{ toYaml .Values.defaultBackend.updateStrategy | nindent 4 }} - {{- end }} - minReadySeconds: {{ .Values.defaultBackend.minReadySeconds }} template: metadata: {{- if .Values.defaultBackend.podAnnotations }} annotations: {{ toYaml .Values.defaultBackend.podAnnotations | nindent 8 }} {{- end }} labels: - {{- include "ingress-nginx.labels" . | nindent 8 }} + {{- include "ingress-nginx.selectorLabels" . | nindent 8 }} app.kubernetes.io/component: default-backend - {{- with .Values.defaultBackend.labels }} - {{- toYaml . | nindent 8 }} - {{- end }} {{- if .Values.defaultBackend.podLabels }} {{- toYaml .Values.defaultBackend.podLabels | nindent 8 }} {{- end }} @@ -50,8 +38,8 @@ spec: {{- end }} containers: - name: {{ template "ingress-nginx.name" . }}-default-backend - {{- with (merge .Values.defaultBackend.image .Values.global.image) }} - image: {{ if .repository }}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{ end }}:{{ .tag }}{{ if .digest }}@{{ .digest }}{{ end }} + {{- with .Values.defaultBackend.image }} + image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}" {{- end }} imagePullPolicy: {{ .Values.defaultBackend.image.pullPolicy }} {{- if .Values.defaultBackend.extraArgs }} @@ -65,7 +53,14 @@ spec: {{- end }} {{- end }} {{- end }} - securityContext: {{ include "ingress-nginx.defaultBackend.containerSecurityContext" . | nindent 12 }} + securityContext: + capabilities: + drop: + - ALL + runAsUser: {{ .Values.defaultBackend.image.runAsUser }} + runAsNonRoot: {{ .Values.defaultBackend.image.runAsNonRoot }} + allowPrivilegeEscalation: {{ .Values.defaultBackend.image.allowPrivilegeEscalation }} + readOnlyRootFilesystem: {{ .Values.defaultBackend.image.readOnlyRootFilesystem}} {{- if .Values.defaultBackend.extraEnvs }} env: {{ toYaml .Values.defaultBackend.extraEnvs | nindent 12 }} {{- end }} @@ -102,16 +97,12 @@ spec: {{- if .Values.defaultBackend.nodeSelector }} nodeSelector: {{ toYaml .Values.defaultBackend.nodeSelector | nindent 8 }} {{- end }} - serviceAccountName: {{ include "ingress-nginx.defaultBackend.serviceAccountName" . }} - automountServiceAccountToken: {{ .Values.defaultBackend.serviceAccount.automountServiceAccountToken }} + serviceAccountName: {{ template "ingress-nginx.defaultBackend.serviceAccountName" . }} {{- if .Values.defaultBackend.tolerations }} tolerations: {{ toYaml .Values.defaultBackend.tolerations | nindent 8 }} {{- end }} {{- if .Values.defaultBackend.affinity }} - affinity: {{ tpl (toYaml .Values.defaultBackend.affinity) $ | nindent 8 }} - {{- end }} - {{- if .Values.defaultBackend.topologySpreadConstraints }} - topologySpreadConstraints: {{ tpl (toYaml .Values.defaultBackend.topologySpreadConstraints) $ | nindent 8 }} + affinity: {{ toYaml .Values.defaultBackend.affinity | nindent 8 }} {{- end }} terminationGracePeriodSeconds: 60 {{- if .Values.defaultBackend.extraVolumes }} diff --git a/charts/ingress-nginx/templates/default-backend-extra-configmaps.yaml b/charts/ingress-nginx/templates/default-backend-extra-configmaps.yaml deleted file mode 100644 index 9af56cf38..000000000 --- a/charts/ingress-nginx/templates/default-backend-extra-configmaps.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- if .Values.defaultBackend.enabled }} -{{- range .Values.defaultBackend.extraConfigMaps }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - {{- include "ingress-nginx.labels" $ | nindent 4 }} - app.kubernetes.io/component: default-backend - {{- with $.Values.defaultBackend.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - name: {{ .name }} - namespace: {{ include "ingress-nginx.namespace" $ }} -data: - {{- with .data }} - {{- toYaml . | nindent 2 }} - {{- end }} -{{- end }} -{{- end }} diff --git a/charts/ingress-nginx/templates/default-backend-hpa.yaml b/charts/ingress-nginx/templates/default-backend-hpa.yaml index 49bcdcfdc..a007d0315 100644 --- a/charts/ingress-nginx/templates/default-backend-hpa.yaml +++ b/charts/ingress-nginx/templates/default-backend-hpa.yaml @@ -1,40 +1,29 @@ {{- if and .Values.defaultBackend.enabled .Values.defaultBackend.autoscaling.enabled }} -apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }} +apiVersion: autoscaling/v2beta1 kind: HorizontalPodAutoscaler metadata: - {{- with .Values.defaultBackend.autoscaling.annotations }} - annotations: {{ toYaml . | nindent 4 }} - {{- end }} labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: default-backend - {{- with .Values.defaultBackend.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - name: {{ include "ingress-nginx.defaultBackend.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} + name: {{ template "ingress-nginx.defaultBackend.fullname" . }} spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment - name: {{ include "ingress-nginx.defaultBackend.fullname" . }} + name: {{ template "ingress-nginx.defaultBackend.fullname" . }} minReplicas: {{ .Values.defaultBackend.autoscaling.minReplicas }} maxReplicas: {{ .Values.defaultBackend.autoscaling.maxReplicas }} metrics: - {{- with .Values.defaultBackend.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ . }} - {{- end }} - {{- with .Values.defaultBackend.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ . }} - {{- end }} +{{- with .Values.defaultBackend.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ . }} +{{- end }} +{{- with .Values.defaultBackend.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ . }} +{{- end }} {{- end }} diff --git a/charts/ingress-nginx/templates/default-backend-networkpolicy.yaml b/charts/ingress-nginx/templates/default-backend-networkpolicy.yaml deleted file mode 100644 index 90b3c2ba0..000000000 --- a/charts/ingress-nginx/templates/default-backend-networkpolicy.yaml +++ /dev/null @@ -1,25 +0,0 @@ -{{- if and .Values.defaultBackend.enabled .Values.defaultBackend.networkPolicy.enabled }} -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - labels: - {{- include "ingress-nginx.labels" . | nindent 4 }} - app.kubernetes.io/component: default-backend - {{- with .Values.defaultBackend.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - name: {{ include "ingress-nginx.defaultBackend.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} -spec: - podSelector: - matchLabels: - {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: default-backend - policyTypes: - - Ingress - - Egress - ingress: - - ports: - - protocol: TCP - port: {{ .Values.defaultBackend.port }} -{{- end }} diff --git a/charts/ingress-nginx/templates/default-backend-poddisruptionbudget.yaml b/charts/ingress-nginx/templates/default-backend-poddisruptionbudget.yaml index e399ea8a4..327d809b9 100644 --- a/charts/ingress-nginx/templates/default-backend-poddisruptionbudget.yaml +++ b/charts/ingress-nginx/templates/default-backend-poddisruptionbudget.yaml @@ -1,32 +1,15 @@ -{{- if .Values.defaultBackend.enabled -}} -{{- $replicas := .Values.defaultBackend.replicaCount }} -{{- if .Values.defaultBackend.autoscaling.enabled }} -{{- $replicas = .Values.defaultBackend.autoscaling.minReplicas }} -{{- end }} -{{- if gt ($replicas | int) 1 }} -apiVersion: {{ ternary "policy/v1" "policy/v1beta1" (semverCompare ">=1.21.0-0" .Capabilities.KubeVersion.Version) }} +{{- if or (gt (.Values.defaultBackend.replicaCount | int) 1) (gt (.Values.defaultBackend.autoscaling.minReplicas | int) 1) }} +apiVersion: policy/v1beta1 kind: PodDisruptionBudget metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: default-backend - {{- with .Values.defaultBackend.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} name: {{ include "ingress-nginx.defaultBackend.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} spec: selector: matchLabels: {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: default-backend - {{- if and .Values.defaultBackend.minAvailable (not (hasKey .Values.defaultBackend "maxUnavailable")) }} minAvailable: {{ .Values.defaultBackend.minAvailable }} - {{- else if .Values.defaultBackend.maxUnavailable }} - maxUnavailable: {{ .Values.defaultBackend.maxUnavailable }} - {{- end }} - {{- if .Values.defaultBackend.unhealthyPodEvictionPolicy }} - unhealthyPodEvictionPolicy: {{ .Values.defaultBackend.unhealthyPodEvictionPolicy }} - {{- end }} -{{- end }} {{- end }} diff --git a/charts/ingress-nginx/templates/default-backend-psp.yaml b/charts/ingress-nginx/templates/default-backend-psp.yaml new file mode 100644 index 000000000..716dbf16f --- /dev/null +++ b/charts/ingress-nginx/templates/default-backend-psp.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.podSecurityPolicy.enabled .Values.defaultBackend.enabled (empty .Values.defaultBackend.existingPsp) -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "ingress-nginx.fullname" . }}-backend + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend +spec: + allowPrivilegeEscalation: false + fsGroup: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + requiredDropCapabilities: + - ALL + runAsUser: + rule: MustRunAsNonRoot + seLinux: + rule: RunAsAny + supplementalGroups: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + volumes: + - configMap + - emptyDir + - projected + - secret + - downwardAPI +{{- end }} diff --git a/charts/ingress-nginx/templates/default-backend-role.yaml b/charts/ingress-nginx/templates/default-backend-role.yaml new file mode 100644 index 000000000..53b63b6bf --- /dev/null +++ b/charts/ingress-nginx/templates/default-backend-role.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.rbac.create .Values.podSecurityPolicy.enabled .Values.defaultBackend.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + name: {{ include "ingress-nginx.fullname" . }}-backend +rules: + - apiGroups: [{{ template "podSecurityPolicy.apiGroup" . }}] + resources: ['podsecuritypolicies'] + verbs: ['use'] + {{- with .Values.defaultBackend.existingPsp }} + resourceNames: [{{ . }}] + {{- else }} + resourceNames: [{{ include "ingress-nginx.fullname" . }}-backend] + {{- end }} +{{- end }} diff --git a/charts/ingress-nginx/templates/default-backend-rolebinding.yaml b/charts/ingress-nginx/templates/default-backend-rolebinding.yaml new file mode 100644 index 000000000..03eac869f --- /dev/null +++ b/charts/ingress-nginx/templates/default-backend-rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.rbac.create .Values.podSecurityPolicy.enabled .Values.defaultBackend.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: default-backend + name: {{ include "ingress-nginx.fullname" . }}-backend +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "ingress-nginx.fullname" . }}-backend +subjects: + - kind: ServiceAccount + name: {{ template "ingress-nginx.defaultBackend.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end }} diff --git a/charts/ingress-nginx/templates/default-backend-service.yaml b/charts/ingress-nginx/templates/default-backend-service.yaml index 5a836365b..e74714d92 100644 --- a/charts/ingress-nginx/templates/default-backend-service.yaml +++ b/charts/ingress-nginx/templates/default-backend-service.yaml @@ -8,19 +8,12 @@ metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: default-backend - {{- with .Values.defaultBackend.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} name: {{ include "ingress-nginx.defaultBackend.fullname" . }} - namespace: {{ include "ingress-nginx.namespace" . }} spec: type: {{ .Values.defaultBackend.service.type }} {{- if .Values.defaultBackend.service.clusterIP }} clusterIP: {{ .Values.defaultBackend.service.clusterIP }} {{- end }} -{{- if .Values.defaultBackend.service.clusterIPs }} - clusterIPs: {{ toYaml .Values.defaultBackend.service.clusterIPs | nindent 4 }} -{{- end }} {{- if .Values.defaultBackend.service.externalIPs }} externalIPs: {{ toYaml .Values.defaultBackend.service.externalIPs | nindent 4 }} {{- end }} @@ -35,9 +28,6 @@ spec: port: {{ .Values.defaultBackend.service.servicePort }} protocol: TCP targetPort: http - {{- if semverCompare ">=1.20.0-0" .Capabilities.KubeVersion.Version }} - appProtocol: http - {{- end }} selector: {{- include "ingress-nginx.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: default-backend diff --git a/charts/ingress-nginx/templates/default-backend-serviceaccount.yaml b/charts/ingress-nginx/templates/default-backend-serviceaccount.yaml index 6fd2d6234..a95826bda 100644 --- a/charts/ingress-nginx/templates/default-backend-serviceaccount.yaml +++ b/charts/ingress-nginx/templates/default-backend-serviceaccount.yaml @@ -1,14 +1,10 @@ -{{- if and .Values.defaultBackend.enabled .Values.defaultBackend.serviceAccount.create -}} +{{- if and .Values.defaultBackend.enabled .Values.defaultBackend.serviceAccount.create -}} apiVersion: v1 kind: ServiceAccount metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} app.kubernetes.io/component: default-backend - {{- with .Values.defaultBackend.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - name: {{ include "ingress-nginx.defaultBackend.serviceAccountName" . }} - namespace: {{ include "ingress-nginx.namespace" . }} + name: {{ template "ingress-nginx.defaultBackend.serviceAccountName" . }} automountServiceAccountToken: {{ .Values.defaultBackend.serviceAccount.automountServiceAccountToken }} {{- end }} diff --git a/charts/ingress-nginx/templates/dh-param-secret.yaml b/charts/ingress-nginx/templates/dh-param-secret.yaml new file mode 100644 index 000000000..12e7a4f63 --- /dev/null +++ b/charts/ingress-nginx/templates/dh-param-secret.yaml @@ -0,0 +1,10 @@ +{{- with .Values.dhParam -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "ingress-nginx.controller.fullname" $ }} + labels: + {{- include "ingress-nginx.labels" $ | nindent 4 }} +data: + dhparam.pem: {{ . }} +{{- end }} diff --git a/charts/ingress-nginx/tests/admission-webhooks/job-patch/clusterrole_test.yaml b/charts/ingress-nginx/tests/admission-webhooks/job-patch/clusterrole_test.yaml deleted file mode 100644 index d7a8b8852..000000000 --- a/charts/ingress-nginx/tests/admission-webhooks/job-patch/clusterrole_test.yaml +++ /dev/null @@ -1,11 +0,0 @@ -suite: Admission Webhooks > Patch Job > ClusterRole -templates: - - admission-webhooks/job-patch/clusterrole.yaml - -tests: - - it: should not create a ClusterRole if `controller.admissionWebhooks.patch.rbac.create` is false - set: - controller.admissionWebhooks.patch.rbac.create: false - asserts: - - hasDocuments: - count: 0 diff --git a/charts/ingress-nginx/tests/admission-webhooks/job-patch/clusterrolebinding_test.yaml b/charts/ingress-nginx/tests/admission-webhooks/job-patch/clusterrolebinding_test.yaml deleted file mode 100644 index d7c3266d2..000000000 --- a/charts/ingress-nginx/tests/admission-webhooks/job-patch/clusterrolebinding_test.yaml +++ /dev/null @@ -1,11 +0,0 @@ -suite: Admission Webhooks > Patch Job > ClusterRoleBinding -templates: - - admission-webhooks/job-patch/clusterrolebinding.yaml - -tests: - - it: should not create a ClusterRoleBinding if `controller.admissionWebhooks.patch.rbac.create` is false - set: - controller.admissionWebhooks.patch.rbac.create: false - asserts: - - hasDocuments: - count: 0 diff --git a/charts/ingress-nginx/tests/admission-webhooks/job-patch/job-createSecret_test.yaml b/charts/ingress-nginx/tests/admission-webhooks/job-patch/job-createSecret_test.yaml deleted file mode 100644 index b5272553b..000000000 --- a/charts/ingress-nginx/tests/admission-webhooks/job-patch/job-createSecret_test.yaml +++ /dev/null @@ -1,12 +0,0 @@ -suite: Admission Webhooks > Patch Job > Create Secret Job -templates: - - admission-webhooks/job-patch/job-createSecret.yaml - -tests: - - it: should create a Job with token auto-mounting disabled if `controller.admissionWebhooks.patch.serviceAccount.automountServiceAccountToken` is false - set: - controller.admissionWebhooks.patch.serviceAccount.automountServiceAccountToken: false - asserts: - - equal: - path: spec.template.spec.automountServiceAccountToken - value: false diff --git a/charts/ingress-nginx/tests/admission-webhooks/job-patch/job-patchWebhook_test.yaml b/charts/ingress-nginx/tests/admission-webhooks/job-patch/job-patchWebhook_test.yaml deleted file mode 100644 index ca4c6b4c2..000000000 --- a/charts/ingress-nginx/tests/admission-webhooks/job-patch/job-patchWebhook_test.yaml +++ /dev/null @@ -1,12 +0,0 @@ -suite: Admission Webhooks > Patch Job > Patch Webhook Job -templates: - - admission-webhooks/job-patch/job-patchWebhook.yaml - -tests: - - it: should create a Job with token auto-mounting disabled if `controller.admissionWebhooks.patch.serviceAccount.automountServiceAccountToken` is false - set: - controller.admissionWebhooks.patch.serviceAccount.automountServiceAccountToken: false - asserts: - - equal: - path: spec.template.spec.automountServiceAccountToken - value: false diff --git a/charts/ingress-nginx/tests/admission-webhooks/job-patch/role_test.yaml b/charts/ingress-nginx/tests/admission-webhooks/job-patch/role_test.yaml deleted file mode 100644 index a236f3d7b..000000000 --- a/charts/ingress-nginx/tests/admission-webhooks/job-patch/role_test.yaml +++ /dev/null @@ -1,11 +0,0 @@ -suite: Admission Webhooks > Patch Job > Role -templates: - - admission-webhooks/job-patch/role.yaml - -tests: - - it: should not create a Role if `controller.admissionWebhooks.patch.rbac.create` is false - set: - controller.admissionWebhooks.patch.rbac.create: false - asserts: - - hasDocuments: - count: 0 diff --git a/charts/ingress-nginx/tests/admission-webhooks/job-patch/rolebinding_test.yaml b/charts/ingress-nginx/tests/admission-webhooks/job-patch/rolebinding_test.yaml deleted file mode 100644 index 74abaa163..000000000 --- a/charts/ingress-nginx/tests/admission-webhooks/job-patch/rolebinding_test.yaml +++ /dev/null @@ -1,11 +0,0 @@ -suite: Admission Webhooks > Patch Job > RoleBinding -templates: - - admission-webhooks/job-patch/rolebinding.yaml - -tests: - - it: should not create a RoleBinding if `controller.admissionWebhooks.patch.rbac.create` is false - set: - controller.admissionWebhooks.patch.rbac.create: false - asserts: - - hasDocuments: - count: 0 diff --git a/charts/ingress-nginx/tests/admission-webhooks/job-patch/serviceaccount_test.yaml b/charts/ingress-nginx/tests/admission-webhooks/job-patch/serviceaccount_test.yaml deleted file mode 100644 index f72bc4383..000000000 --- a/charts/ingress-nginx/tests/admission-webhooks/job-patch/serviceaccount_test.yaml +++ /dev/null @@ -1,47 +0,0 @@ -suite: Admission Webhooks > Patch Job > ServiceAccount -templates: - - admission-webhooks/job-patch/serviceaccount.yaml - -tests: - - it: should not create a ServiceAccount if `controller.admissionWebhooks.patch.serviceAccount.create` is false - set: - controller.admissionWebhooks.patch.serviceAccount.create: false - asserts: - - hasDocuments: - count: 0 - - - it: should create a ServiceAccount if `controller.admissionWebhooks.patch.serviceAccount.create` is true - set: - controller.admissionWebhooks.patch.serviceAccount.create: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: ServiceAccount - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-admission - - - it: should create a ServiceAccount with specified name if `controller.admissionWebhooks.patch.serviceAccount.name` is set - set: - controller.admissionWebhooks.patch.serviceAccount.name: ingress-nginx-admission-test-sa - asserts: - - hasDocuments: - count: 1 - - isKind: - of: ServiceAccount - - equal: - path: metadata.name - value: ingress-nginx-admission-test-sa - - - it: should create a ServiceAccount with token auto-mounting disabled if `controller.admissionWebhooks.patch.serviceAccount.automountServiceAccountToken` is false - set: - controller.admissionWebhooks.patch.serviceAccount.automountServiceAccountToken: false - asserts: - - hasDocuments: - count: 1 - - isKind: - of: ServiceAccount - - equal: - path: automountServiceAccountToken - value: false diff --git a/charts/ingress-nginx/tests/admission-webhooks/validating-webhook_test.yaml b/charts/ingress-nginx/tests/admission-webhooks/validating-webhook_test.yaml deleted file mode 100644 index 47b6b6873..000000000 --- a/charts/ingress-nginx/tests/admission-webhooks/validating-webhook_test.yaml +++ /dev/null @@ -1,32 +0,0 @@ -suite: Admission Webhooks > ValidatingWebhookConfiguration -templates: - - admission-webhooks/validating-webhook.yaml - -tests: - - it: should not create a ValidatingWebhookConfiguration if `controller.admissionWebhooks.enabled` is false - set: - controller.admissionWebhooks.enabled: false - asserts: - - hasDocuments: - count: 0 - - - it: should create a ValidatingWebhookConfiguration if `controller.admissionWebhooks.enabled` is true - set: - controller.admissionWebhooks.enabled: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: ValidatingWebhookConfiguration - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-admission - - - it: should create a ValidatingWebhookConfiguration with a custom port if `controller.admissionWebhooks.service.servicePort` is set - set: - controller.admissionWebhooks.enabled: true - controller.admissionWebhooks.service.servicePort: 9443 - asserts: - - equal: - path: webhooks[0].clientConfig.service.port - value: 9443 diff --git a/charts/ingress-nginx/tests/controller-configmap-addheaders_test.yaml b/charts/ingress-nginx/tests/controller-configmap-addheaders_test.yaml deleted file mode 100644 index e831d50c0..000000000 --- a/charts/ingress-nginx/tests/controller-configmap-addheaders_test.yaml +++ /dev/null @@ -1,27 +0,0 @@ -suite: Controller > ConfigMap > Add Headers -templates: - - controller-configmap-addheaders.yaml - -tests: - - it: should not create a ConfigMap if `controller.addHeaders` is not set - set: - controller.addHeaders: null - asserts: - - hasDocuments: - count: 0 - - - it: should create a ConfigMap if `controller.addHeaders` is set - set: - controller.addHeaders: - X-Another-Custom-Header: Value - asserts: - - hasDocuments: - count: 1 - - isKind: - of: ConfigMap - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-custom-add-headers - - equal: - path: data.X-Another-Custom-Header - value: Value diff --git a/charts/ingress-nginx/tests/controller-configmap-proxyheaders_test.yaml b/charts/ingress-nginx/tests/controller-configmap-proxyheaders_test.yaml deleted file mode 100644 index 0634a3739..000000000 --- a/charts/ingress-nginx/tests/controller-configmap-proxyheaders_test.yaml +++ /dev/null @@ -1,27 +0,0 @@ -suite: Controller > ConfigMap > Proxy Headers -templates: - - controller-configmap-proxyheaders.yaml - -tests: - - it: should not create a ConfigMap if `controller.proxySetHeaders` is not set - set: - controller.proxySetHeaders: null - asserts: - - hasDocuments: - count: 0 - - - it: should create a ConfigMap if `controller.proxySetHeaders` is set - set: - controller.proxySetHeaders: - X-Custom-Header: Value - asserts: - - hasDocuments: - count: 1 - - isKind: - of: ConfigMap - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-custom-proxy-headers - - equal: - path: data.X-Custom-Header - value: Value diff --git a/charts/ingress-nginx/tests/controller-configmap_test.yaml b/charts/ingress-nginx/tests/controller-configmap_test.yaml deleted file mode 100644 index 168b657d6..000000000 --- a/charts/ingress-nginx/tests/controller-configmap_test.yaml +++ /dev/null @@ -1,31 +0,0 @@ -suite: Controller > ConfigMap -templates: - - controller-configmap.yaml - -tests: - - it: should create a ConfigMap - asserts: - - hasDocuments: - count: 1 - - isKind: - of: ConfigMap - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-controller - - - it: should create a ConfigMap with templated values if `controller.config` contains templates - set: - controller.config: - template: "test.{{ .Release.Namespace }}.svc.kubernetes.local" - integer: 12345 - boolean: true - asserts: - - equal: - path: data.template - value: test.NAMESPACE.svc.kubernetes.local - - equal: - path: data.integer - value: "12345" - - equal: - path: data.boolean - value: "true" diff --git a/charts/ingress-nginx/tests/controller-daemonset_test.yaml b/charts/ingress-nginx/tests/controller-daemonset_test.yaml deleted file mode 100644 index 0321fd376..000000000 --- a/charts/ingress-nginx/tests/controller-daemonset_test.yaml +++ /dev/null @@ -1,201 +0,0 @@ -suite: Controller > DaemonSet -templates: - - controller-daemonset.yaml - -tests: - - it: should create a DaemonSet if `controller.kind` is "DaemonSet" - set: - controller.kind: DaemonSet - asserts: - - hasDocuments: - count: 1 - - isKind: - of: DaemonSet - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-controller - - - it: should create a DaemonSet with argument `--enable-metrics=true` if `controller.metrics.enabled` is true - set: - controller.kind: DaemonSet - controller.metrics.enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].args - content: --enable-metrics=true - - - it: should create a DaemonSet without argument `--enable-metrics=true` if `controller.metrics.enabled` is false - set: - controller.kind: DaemonSet - controller.metrics.enabled: false - asserts: - - notContains: - path: spec.template.spec.containers[0].args - content: --enable-metrics=true - - - it: should create a DaemonSet with argument `--controller-class=k8s.io/ingress-nginx-internal` if `controller.ingressClassResource.controllerValue` is "k8s.io/ingress-nginx-internal" - set: - controller.kind: DaemonSet - controller.ingressClassResource.controllerValue: k8s.io/ingress-nginx-internal - asserts: - - contains: - path: spec.template.spec.containers[0].args - content: --controller-class=k8s.io/ingress-nginx-internal - - - it: should create a DaemonSet with resource limits if `controller.resources.limits` is set - set: - controller.kind: DaemonSet - controller.resources.limits.cpu: 500m - controller.resources.limits.memory: 512Mi - asserts: - - equal: - path: spec.template.spec.containers[0].resources.limits.cpu - value: 500m - - equal: - path: spec.template.spec.containers[0].resources.limits.memory - value: 512Mi - - - it: should create a DaemonSet with topology spread constraints if `controller.topologySpreadConstraints` is set - set: - controller.kind: DaemonSet - controller.topologySpreadConstraints: - - labelSelector: - matchLabels: - app.kubernetes.io/name: '{{ include "ingress-nginx.name" . }}' - app.kubernetes.io/instance: '{{ .Release.Name }}' - app.kubernetes.io/component: controller - topologyKey: topology.kubernetes.io/zone - maxSkew: 1 - whenUnsatisfiable: ScheduleAnyway - - labelSelector: - matchLabels: - app.kubernetes.io/name: '{{ include "ingress-nginx.name" . }}' - app.kubernetes.io/instance: '{{ .Release.Name }}' - app.kubernetes.io/component: controller - topologyKey: kubernetes.io/hostname - maxSkew: 1 - whenUnsatisfiable: ScheduleAnyway - asserts: - - equal: - path: spec.template.spec.topologySpreadConstraints - value: - - labelSelector: - matchLabels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: RELEASE-NAME - app.kubernetes.io/component: controller - topologyKey: topology.kubernetes.io/zone - maxSkew: 1 - whenUnsatisfiable: ScheduleAnyway - - labelSelector: - matchLabels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: RELEASE-NAME - app.kubernetes.io/component: controller - topologyKey: kubernetes.io/hostname - maxSkew: 1 - whenUnsatisfiable: ScheduleAnyway - - - it: should create a DaemonSet with affinity if `controller.affinity` is set - set: - controller.kind: DaemonSet - controller.affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app.kubernetes.io/name - operator: In - values: - - '{{ include "ingress-nginx.name" . }}' - - key: app.kubernetes.io/instance - operator: In - values: - - '{{ .Release.Name }}' - - key: app.kubernetes.io/component - operator: In - values: - - controller - topologyKey: kubernetes.io/hostname - asserts: - - equal: - path: spec.template.spec.affinity - value: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app.kubernetes.io/name - operator: In - values: - - ingress-nginx - - key: app.kubernetes.io/instance - operator: In - values: - - RELEASE-NAME - - key: app.kubernetes.io/component - operator: In - values: - - controller - topologyKey: kubernetes.io/hostname - - - it: should create a DaemonSet with `runAsGroup` if `controller.image.runAsGroup` is set - set: - controller.kind: DaemonSet - controller.image.runAsGroup: 1000 - asserts: - - equal: - path: spec.template.spec.containers[0].securityContext.runAsGroup - value: 1000 - - - it: should create a DaemonSet with a custom registry if `global.image.registry` is set - set: - global.image.registry: custom.registry.io - controller.kind: DaemonSet - controller.image.tag: v1.0.0-dev - controller.image.digest: sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: custom.registry.io/ingress-nginx/controller:v1.0.0-dev@sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - - - it: should create a DaemonSet with a custom registry if `controller.image.registry` is set - set: - controller.kind: DaemonSet - controller.image.registry: custom.registry.io - controller.image.tag: v1.0.0-dev - controller.image.digest: sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: custom.registry.io/ingress-nginx/controller:v1.0.0-dev@sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - - - it: should create a DaemonSet with a custom image if `controller.image.image` is set - set: - controller.kind: DaemonSet - controller.image.image: custom-repo/custom-image - controller.image.tag: v1.0.0-dev - controller.image.digest: sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: registry.k8s.io/custom-repo/custom-image:v1.0.0-dev@sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - - - it: should create a DaemonSet with a custom tag if `controller.image.tag` is set - set: - controller.kind: DaemonSet - controller.image.tag: custom-tag - controller.image.digest: sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: registry.k8s.io/ingress-nginx/controller:custom-tag@sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - - - it: should create a DaemonSet with token auto-mounting disabled if `serviceAccount.automountServiceAccountToken` is false - set: - controller.kind: DaemonSet - serviceAccount.automountServiceAccountToken: false - asserts: - - equal: - path: spec.template.spec.automountServiceAccountToken - value: false diff --git a/charts/ingress-nginx/tests/controller-deployment_test.yaml b/charts/ingress-nginx/tests/controller-deployment_test.yaml deleted file mode 100644 index 18306079e..000000000 --- a/charts/ingress-nginx/tests/controller-deployment_test.yaml +++ /dev/null @@ -1,225 +0,0 @@ -suite: Controller > Deployment -templates: - - controller-deployment.yaml - -tests: - - it: should create a Deployment - asserts: - - hasDocuments: - count: 1 - - isKind: - of: Deployment - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-controller - - - it: should create a Deployment with 3 replicas if `controller.replicaCount` is 3 - set: - controller.replicaCount: 3 - asserts: - - equal: - path: spec.replicas - value: 3 - - - it: should create a Deployment without replicas if `controller.autoscaling.enabled` is true - set: - controller.autoscaling.enabled: true - asserts: - - notExists: - path: spec.replicas - - - it: should create a Deployment without replicas if `controller.keda.enabled` is true - set: - controller.keda.enabled: true - asserts: - - notExists: - path: spec.replicas - - - it: should create a Deployment with replicas if `controller.autoscaling.enabled` is true and `controller.keda.enabled` is true - set: - controller.autoscaling.enabled: true - controller.keda.enabled: true - asserts: - - exists: - path: spec.replicas - - - it: should create a Deployment with argument `--enable-metrics=true` if `controller.metrics.enabled` is true - set: - controller.metrics.enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].args - content: --enable-metrics=true - - - it: should create a Deployment without argument `--enable-metrics=true` if `controller.metrics.enabled` is false - set: - controller.metrics.enabled: false - asserts: - - notContains: - path: spec.template.spec.containers[0].args - content: --enable-metrics=true - - - it: should create a Deployment with argument `--controller-class=k8s.io/ingress-nginx-internal` if `controller.ingressClassResource.controllerValue` is "k8s.io/ingress-nginx-internal" - set: - controller.ingressClassResource.controllerValue: k8s.io/ingress-nginx-internal - asserts: - - contains: - path: spec.template.spec.containers[0].args - content: --controller-class=k8s.io/ingress-nginx-internal - - - it: should create a Deployment with resource limits if `controller.resources.limits` is set - set: - controller.resources.limits.cpu: 500m - controller.resources.limits.memory: 512Mi - asserts: - - equal: - path: spec.template.spec.containers[0].resources.limits.cpu - value: 500m - - equal: - path: spec.template.spec.containers[0].resources.limits.memory - value: 512Mi - - - it: should create a Deployment with topology spread constraints if `controller.topologySpreadConstraints` is set - set: - controller.topologySpreadConstraints: - - labelSelector: - matchLabels: - app.kubernetes.io/name: '{{ include "ingress-nginx.name" . }}' - app.kubernetes.io/instance: '{{ .Release.Name }}' - app.kubernetes.io/component: controller - topologyKey: topology.kubernetes.io/zone - maxSkew: 1 - whenUnsatisfiable: ScheduleAnyway - - labelSelector: - matchLabels: - app.kubernetes.io/name: '{{ include "ingress-nginx.name" . }}' - app.kubernetes.io/instance: '{{ .Release.Name }}' - app.kubernetes.io/component: controller - topologyKey: kubernetes.io/hostname - maxSkew: 1 - whenUnsatisfiable: ScheduleAnyway - asserts: - - equal: - path: spec.template.spec.topologySpreadConstraints - value: - - labelSelector: - matchLabels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: RELEASE-NAME - app.kubernetes.io/component: controller - topologyKey: topology.kubernetes.io/zone - maxSkew: 1 - whenUnsatisfiable: ScheduleAnyway - - labelSelector: - matchLabels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: RELEASE-NAME - app.kubernetes.io/component: controller - topologyKey: kubernetes.io/hostname - maxSkew: 1 - whenUnsatisfiable: ScheduleAnyway - - - it: should create a Deployment with affinity if `controller.affinity` is set - set: - controller.affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app.kubernetes.io/name - operator: In - values: - - '{{ include "ingress-nginx.name" . }}' - - key: app.kubernetes.io/instance - operator: In - values: - - '{{ .Release.Name }}' - - key: app.kubernetes.io/component - operator: In - values: - - controller - topologyKey: kubernetes.io/hostname - asserts: - - equal: - path: spec.template.spec.affinity - value: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app.kubernetes.io/name - operator: In - values: - - ingress-nginx - - key: app.kubernetes.io/instance - operator: In - values: - - RELEASE-NAME - - key: app.kubernetes.io/component - operator: In - values: - - controller - topologyKey: kubernetes.io/hostname - - - it: should create a Deployment with `runAsGroup` if `controller.image.runAsGroup` is set - set: - controller.image.runAsGroup: 1000 - asserts: - - equal: - path: spec.template.spec.containers[0].securityContext.runAsGroup - value: 1000 - - - it: should create a Deployment with a custom registry if `global.image.registry` is set - set: - global.image.registry: custom.registry.io - controller.image.tag: v1.0.0-dev - controller.image.digest: sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: custom.registry.io/ingress-nginx/controller:v1.0.0-dev@sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - - - it: should create a Deployment with a custom registry if `controller.image.registry` is set - set: - controller.image.registry: custom.registry.io - controller.image.tag: v1.0.0-dev - controller.image.digest: sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: custom.registry.io/ingress-nginx/controller:v1.0.0-dev@sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - - - it: should create a Deployment with a custom image if `controller.image.image` is set - set: - controller.image.image: custom-repo/custom-image - controller.image.tag: v1.0.0-dev - controller.image.digest: sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: registry.k8s.io/custom-repo/custom-image:v1.0.0-dev@sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - - - it: should create a Deployment with a custom tag if `controller.image.tag` is set - set: - controller.image.tag: custom-tag - controller.image.digest: sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: registry.k8s.io/ingress-nginx/controller:custom-tag@sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - - - it: should create a Deployment with `progressDeadlineSeconds` if `controller.progressDeadlineSeconds` is set - set: - controller.progressDeadlineSeconds: 111 - asserts: - - equal: - path: spec.progressDeadlineSeconds - value: 111 - - - it: should create a Deployment with token auto-mounting disabled if `serviceAccount.automountServiceAccountToken` is false - set: - serviceAccount.automountServiceAccountToken: false - asserts: - - equal: - path: spec.template.spec.automountServiceAccountToken - value: false diff --git a/charts/ingress-nginx/tests/controller-hpa_test.yaml b/charts/ingress-nginx/tests/controller-hpa_test.yaml deleted file mode 100644 index 869d3a690..000000000 --- a/charts/ingress-nginx/tests/controller-hpa_test.yaml +++ /dev/null @@ -1,31 +0,0 @@ -suite: Controller > HPA -templates: - - controller-hpa.yaml - -tests: - - it: should create an HPA if `controller.autoscaling.enabled` is true - set: - controller.autoscaling.enabled: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: HorizontalPodAutoscaler - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-controller - - - it: should not create an HPA if `controller.autoscaling.enabled` is true and `controller.keda.enabled` is true - set: - controller.autoscaling.enabled: true - controller.keda.enabled: true - asserts: - - hasDocuments: - count: 0 - - - it: should not create an HPA if `controller.kind` is "DaemonSet" - set: - controller.kind: DaemonSet - asserts: - - hasDocuments: - count: 0 diff --git a/charts/ingress-nginx/tests/controller-ingressclass-aliases_test.yaml b/charts/ingress-nginx/tests/controller-ingressclass-aliases_test.yaml deleted file mode 100644 index 9a4a576b7..000000000 --- a/charts/ingress-nginx/tests/controller-ingressclass-aliases_test.yaml +++ /dev/null @@ -1,110 +0,0 @@ -suite: Controller > IngressClass > Aliases -templates: - - controller-ingressclass-aliases.yaml - -tests: - - it: should not create IngressClass aliases - asserts: - - hasDocuments: - count: 0 - - - it: should create an IngressClass alias with name "nginx-alias" if `controller.ingressClassResource.aliases` is set - set: - controller.ingressClassResource.aliases: - - nginx-alias - asserts: - - hasDocuments: - count: 1 - - isKind: - of: IngressClass - - equal: - path: metadata.name - value: nginx-alias - - - it: should create an IngressClass alias without annotation `ingressclass.kubernetes.io/is-default-class` if `controller.ingressClassResource.default` is true - set: - controller.ingressClassResource.aliases: - - nginx-alias - controller.ingressClassResource.default: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: IngressClass - - equal: - path: metadata.name - value: nginx-alias - - notExists: - path: metadata.annotations["ingressclass.kubernetes.io/is-default-class"] - - - it: should create an IngressClass alias with annotations if `controller.ingressClassResource.annotations` is set - set: - controller.ingressClassResource.aliases: - - nginx-alias - controller.ingressClassResource.annotations: - my-fancy-annotation: has-a-value - asserts: - - hasDocuments: - count: 1 - - isKind: - of: IngressClass - - equal: - path: metadata.name - value: nginx-alias - - equal: - path: metadata.annotations.my-fancy-annotation - value: has-a-value - - - it: should create an IngressClass alias with controller "k8s.io/ingress-nginx-internal" if `controller.ingressClassResource.controllerValue` is "k8s.io/ingress-nginx-internal" - set: - controller.ingressClassResource.aliases: - - nginx-alias - controller.ingressClassResource.controllerValue: k8s.io/ingress-nginx-internal - asserts: - - hasDocuments: - count: 1 - - isKind: - of: IngressClass - - equal: - path: metadata.name - value: nginx-alias - - equal: - path: spec.controller - value: k8s.io/ingress-nginx-internal - - - it: should create an IngressClass alias with parameters if `controller.ingressClassResource.parameters` is set - set: - controller.ingressClassResource.aliases: - - nginx-alias - controller.ingressClassResource.parameters: - apiGroup: k8s.example.com - kind: IngressParameters - name: external-lb - asserts: - - hasDocuments: - count: 1 - - isKind: - of: IngressClass - - equal: - path: metadata.name - value: nginx-alias - - equal: - path: spec.parameters - value: - apiGroup: k8s.example.com - kind: IngressParameters - name: external-lb - - - it: should create two IngressClass aliases if `controller.ingressClassResource.aliases` has two elements - set: - controller.ingressClassResource.aliases: - - nginx-alias-1 - - nginx-alias-2 - asserts: - - hasDocuments: - count: 2 - - isKind: - of: IngressClass - - matchRegex: - path: metadata.name - pattern: nginx-alias-(1|2) diff --git a/charts/ingress-nginx/tests/controller-ingressclass_test.yaml b/charts/ingress-nginx/tests/controller-ingressclass_test.yaml deleted file mode 100644 index b3384af32..000000000 --- a/charts/ingress-nginx/tests/controller-ingressclass_test.yaml +++ /dev/null @@ -1,93 +0,0 @@ -suite: Controller > IngressClass -templates: - - controller-ingressclass.yaml - -tests: - - it: should create an IngressClass - asserts: - - hasDocuments: - count: 1 - - isKind: - of: IngressClass - - equal: - path: metadata.name - value: nginx - - - it: should create an IngressClass with name "nginx-internal" if `controller.ingressClassResource.name` is "nginx-internal" - set: - controller.ingressClassResource.name: nginx-internal - asserts: - - hasDocuments: - count: 1 - - isKind: - of: IngressClass - - equal: - path: metadata.name - value: nginx-internal - - - it: "should create an IngressClass with annotation `ingressclass.kubernetes.io/is-default-class: \"true\"` if `controller.ingressClassResource.default` is true" - set: - controller.ingressClassResource.default: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: IngressClass - - equal: - path: metadata.name - value: nginx - - equal: - path: metadata.annotations["ingressclass.kubernetes.io/is-default-class"] - value: "true" - - - it: should create an IngressClass with annotations if `controller.ingressClassResource.annotations` is set - set: - controller.ingressClassResource.annotations: - my-fancy-annotation: has-a-value - asserts: - - hasDocuments: - count: 1 - - isKind: - of: IngressClass - - equal: - path: metadata.name - value: nginx - - equal: - path: metadata.annotations.my-fancy-annotation - value: has-a-value - - - it: should create an IngressClass with controller "k8s.io/ingress-nginx-internal" if `controller.ingressClassResource.controllerValue` is "k8s.io/ingress-nginx-internal" - set: - controller.ingressClassResource.controllerValue: k8s.io/ingress-nginx-internal - asserts: - - hasDocuments: - count: 1 - - isKind: - of: IngressClass - - equal: - path: metadata.name - value: nginx - - equal: - path: spec.controller - value: k8s.io/ingress-nginx-internal - - - it: should create an IngressClass with parameters if `controller.ingressClassResource.parameters` is set - set: - controller.ingressClassResource.parameters: - apiGroup: k8s.example.com - kind: IngressParameters - name: external-lb - asserts: - - hasDocuments: - count: 1 - - isKind: - of: IngressClass - - equal: - path: metadata.name - value: nginx - - equal: - path: spec.parameters - value: - apiGroup: k8s.example.com - kind: IngressParameters - name: external-lb diff --git a/charts/ingress-nginx/tests/controller-keda_test.yaml b/charts/ingress-nginx/tests/controller-keda_test.yaml deleted file mode 100644 index 800283483..000000000 --- a/charts/ingress-nginx/tests/controller-keda_test.yaml +++ /dev/null @@ -1,31 +0,0 @@ -suite: Controller > KEDA -templates: - - controller-keda.yaml - -tests: - - it: should create a ScaledObject if `controller.keda.enabled` is true - set: - controller.keda.enabled: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: ScaledObject - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-controller - - - it: should not create a ScaledObject if `controller.keda.enabled` is true and `controller.autoscaling.enabled` is true - set: - controller.keda.enabled: true - controller.autoscaling.enabled: true - asserts: - - hasDocuments: - count: 0 - - - it: should not create a ScaledObject if `controller.kind` is "DaemonSet" - set: - controller.kind: DaemonSet - asserts: - - hasDocuments: - count: 0 diff --git a/charts/ingress-nginx/tests/controller-networkpolicy_test.yaml b/charts/ingress-nginx/tests/controller-networkpolicy_test.yaml deleted file mode 100644 index 5de12e9c4..000000000 --- a/charts/ingress-nginx/tests/controller-networkpolicy_test.yaml +++ /dev/null @@ -1,23 +0,0 @@ -suite: Controller > NetworkPolicy -templates: - - controller-networkpolicy.yaml - -tests: - - it: should not create a NetworkPolicy if `controller.networkPolicy.enabled` is false - set: - controller.networkPolicy.enabled: false - asserts: - - hasDocuments: - count: 0 - - - it: should create a NetworkPolicy if `controller.networkPolicy.enabled` is true - set: - controller.networkPolicy.enabled: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: NetworkPolicy - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-controller diff --git a/charts/ingress-nginx/tests/controller-poddisruptionbudget_test.yaml b/charts/ingress-nginx/tests/controller-poddisruptionbudget_test.yaml deleted file mode 100644 index 5ac986fc7..000000000 --- a/charts/ingress-nginx/tests/controller-poddisruptionbudget_test.yaml +++ /dev/null @@ -1,102 +0,0 @@ -suite: Controller > PodDisruptionBudget -templates: - - controller-poddisruptionbudget.yaml - -tests: - - it: should create a PodDisruptionBudget if `controller.replicaCount` is greater than 1 - set: - controller.replicaCount: 2 - asserts: - - hasDocuments: - count: 1 - - isKind: - of: PodDisruptionBudget - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-controller - - - it: should not create a PodDisruptionBudget if `controller.replicaCount` is less than or equal 1 - set: - controller.replicaCount: 1 - asserts: - - hasDocuments: - count: 0 - - - it: should create a PodDisruptionBudget if `controller.autoscaling.enabled` is true and `controller.autoscaling.minReplicas` is greater than 1 - set: - controller.autoscaling.enabled: true - controller.autoscaling.minReplicas: 2 - asserts: - - hasDocuments: - count: 1 - - isKind: - of: PodDisruptionBudget - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-controller - - - it: should not create a PodDisruptionBudget if `controller.autoscaling.enabled` is true and `controller.autoscaling.minReplicas` is less than or equal 1 - set: - controller.autoscaling.enabled: true - controller.autoscaling.minReplicas: 1 - asserts: - - hasDocuments: - count: 0 - - - it: should create a PodDisruptionBudget if `controller.keda.enabled` is true and `controller.keda.minReplicas` is greater than 1 - set: - controller.keda.enabled: true - controller.keda.minReplicas: 2 - asserts: - - hasDocuments: - count: 1 - - isKind: - of: PodDisruptionBudget - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-controller - - - it: should not create a PodDisruptionBudget if `controller.keda.enabled` is true and `controller.keda.minReplicas` is less than or equal 1 - set: - controller.keda.enabled: true - controller.keda.minReplicas: 1 - asserts: - - hasDocuments: - count: 0 - - - it: should not create a PodDisruptionBudget if `controller.autoscaling.enabled` is true and `controller.keda.enabled` is true - set: - controller.autoscaling.enabled: true - controller.keda.enabled: true - asserts: - - hasDocuments: - count: 0 - - - it: should create a PodDisruptionBudget without `minAvailable` and with `maxUnavailable` if `controller.minAvailable` and `controller.maxUnavailable` are set - set: - controller.replicaCount: 2 - controller.minAvailable: 1 - controller.maxUnavailable: 1 - asserts: - - hasDocuments: - count: 1 - - isKind: - of: PodDisruptionBudget - - notExists: - path: spec.minAvailable - - equal: - path: spec.maxUnavailable - value: 1 - - - it: should create a PodDisruptionBudget with `unhealthyPodEvictionPolicy` if `controller.unhealthyPodEvictionPolicy` is set - set: - controller.replicaCount: 2 - controller.unhealthyPodEvictionPolicy: IfHealthyBudget - asserts: - - hasDocuments: - count: 1 - - isKind: - of: PodDisruptionBudget - - equal: - path: spec.unhealthyPodEvictionPolicy - value: IfHealthyBudget diff --git a/charts/ingress-nginx/tests/controller-prometheusrule_test.yaml b/charts/ingress-nginx/tests/controller-prometheusrule_test.yaml deleted file mode 100644 index 2d330919d..000000000 --- a/charts/ingress-nginx/tests/controller-prometheusrule_test.yaml +++ /dev/null @@ -1,29 +0,0 @@ -suite: Controller > PrometheusRule -templates: - - controller-prometheusrule.yaml - -tests: - - it: should create a PrometheusRule if `controller.metrics.prometheusRule.enabled` is true - set: - controller.metrics.enabled: true - controller.metrics.prometheusRule.enabled: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: PrometheusRule - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-controller - - - it: should create a PrometheusRule with annotations if `controller.metrics.prometheusRule.annotations` is set - set: - controller.metrics.enabled: true - controller.metrics.prometheusRule.enabled: true - controller.metrics.prometheusRule.annotations: - my-little-annotation: test-value - asserts: - - equal: - path: metadata.annotations - value: - my-little-annotation: test-value diff --git a/charts/ingress-nginx/tests/controller-service-internal_test.yaml b/charts/ingress-nginx/tests/controller-service-internal_test.yaml deleted file mode 100644 index c0ece07d5..000000000 --- a/charts/ingress-nginx/tests/controller-service-internal_test.yaml +++ /dev/null @@ -1,75 +0,0 @@ -suite: Controller > Service > Internal -templates: - - controller-service-internal.yaml - -tests: - - it: should not create an internal Service if `controller.service.internal.enabled` is false - set: - controller.service.internal.enabled: false - asserts: - - hasDocuments: - count: 0 - - - it: should create an internal Service if `controller.service.internal.enabled` is true and `controller.service.internal.annotations` are set - set: - controller.service.internal.enabled: true - controller.service.internal.annotations: - test.annotation: "true" - asserts: - - hasDocuments: - count: 1 - - isKind: - of: Service - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-controller-internal - - - it: should create a Service without `clusterIPs` if `controller.service.internal.clusterIPs` is not set - set: - controller.service.internal.enabled: true - controller.service.internal.annotations: - test.annotation: "true" - asserts: - - notExists: - path: spec.clusterIPs - - - it: should create a Service with `clusterIPs` if `controller.service.internal.clusterIPs` is set - set: - controller.service.internal.enabled: true - controller.service.internal.annotations: - test.annotation: "true" - controller.service.internal.clusterIPs: - - 10.0.0.1 - - fd00::1 - asserts: - - equal: - path: spec.clusterIPs - value: - - 10.0.0.1 - - fd00::1 - - - it: should create a Service with `trafficDistribution` if `controller.service.internal.trafficDistribution` is set - capabilities: - majorVersion: 1 - minorVersion: 31 - set: - controller.service.internal.enabled: true - controller.service.internal.annotations: - test.annotation: "true" - controller.service.internal.trafficDistribution: PreferClose - asserts: - - equal: - path: spec.trafficDistribution - value: PreferClose - - - it: should create a Service with labels if `controller.service.internal.labels` is set - set: - controller.service.internal.enabled: true - controller.service.internal.annotations: - test.annotation: "true" - controller.service.internal.labels: - external-dns.alpha.kubernetes.io/hostname: internal.example.com - asserts: - - equal: - path: metadata.labels["external-dns.alpha.kubernetes.io/hostname"] - value: internal.example.com diff --git a/charts/ingress-nginx/tests/controller-service-metrics_test.yaml b/charts/ingress-nginx/tests/controller-service-metrics_test.yaml deleted file mode 100644 index ddb412e5b..000000000 --- a/charts/ingress-nginx/tests/controller-service-metrics_test.yaml +++ /dev/null @@ -1,41 +0,0 @@ -suite: Controller > Service > Metrics -templates: - - controller-service-metrics.yaml - -tests: - - it: should not create a metrics Service if `controller.metrics.enabled` is false and `controller.metrics.service.enabled` is false - set: - controller.metrics.enabled: false - controller.metrics.service.enabled: false - asserts: - - hasDocuments: - count: 0 - - - it: should not create a metrics Service if `controller.metrics.enabled` is false and `controller.metrics.service.enabled` is true - set: - controller.metrics.enabled: false - controller.metrics.service.enabled: true - asserts: - - hasDocuments: - count: 0 - - - it: should not create a metrics Service if `controller.metrics.enabled` is true and `controller.metrics.service.enabled` is false - set: - controller.metrics.enabled: true - controller.metrics.service.enabled: false - asserts: - - hasDocuments: - count: 0 - - - it: should create a metrics Service if `controller.metrics.enabled` is true and `controller.metrics.service.enabled` is true - set: - controller.metrics.enabled: true - controller.metrics.service.enabled: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: Service - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-controller-metrics diff --git a/charts/ingress-nginx/tests/controller-service-webhook_test.yaml b/charts/ingress-nginx/tests/controller-service-webhook_test.yaml deleted file mode 100644 index 1c759edbe..000000000 --- a/charts/ingress-nginx/tests/controller-service-webhook_test.yaml +++ /dev/null @@ -1,32 +0,0 @@ -suite: Controller > Service > Webhook -templates: - - controller-service-webhook.yaml - -tests: - - it: should not create a webhook Service if `controller.admissionWebhooks.enabled` is false - set: - controller.admissionWebhooks.enabled: false - asserts: - - hasDocuments: - count: 0 - - - it: should create a webhook Service if `controller.admissionWebhooks.enabled` is true - set: - controller.admissionWebhooks.enabled: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: Service - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-controller-admission - - - it: should create a webhook Service with a custom port if `controller.admissionWebhooks.service.servicePort` is set - set: - controller.admissionWebhooks.enabled: true - controller.admissionWebhooks.service.servicePort: 9443 - asserts: - - equal: - path: spec.ports[0].port - value: 9443 diff --git a/charts/ingress-nginx/tests/controller-service_test.yaml b/charts/ingress-nginx/tests/controller-service_test.yaml deleted file mode 100644 index f3e8cf030..000000000 --- a/charts/ingress-nginx/tests/controller-service_test.yaml +++ /dev/null @@ -1,74 +0,0 @@ -suite: Controller > Service -templates: - - controller-service.yaml - -tests: - - it: should not create a Service if `controller.service.external.enabled` is false - set: - controller.service.external.enabled: false - asserts: - - hasDocuments: - count: 0 - - - it: should create a Service if `controller.service.external.enabled` is true - set: - controller.service.external.enabled: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: Service - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-controller - - - it: should create a Service of type "NodePort" if `controller.service.external.enabled` is true and `controller.service.type` is "NodePort" - set: - controller.service.external.enabled: true - controller.service.type: NodePort - asserts: - - equal: - path: spec.type - value: NodePort - - - it: should create a Service without `clusterIPs` if `controller.service.clusterIPs` is not set - set: - controller.service.external.enabled: true - asserts: - - notExists: - path: spec.clusterIPs - - - it: should create a Service with `clusterIPs` if `controller.service.clusterIPs` is set - set: - controller.service.external.enabled: true - controller.service.clusterIPs: - - 10.0.0.1 - - fd00::1 - asserts: - - equal: - path: spec.clusterIPs - value: - - 10.0.0.1 - - fd00::1 - - - it: should create a Service with `trafficDistribution` if `controller.service.trafficDistribution` is set - capabilities: - majorVersion: 1 - minorVersion: 31 - set: - controller.service.external.enabled: true - controller.service.trafficDistribution: PreferClose - asserts: - - equal: - path: spec.trafficDistribution - value: PreferClose - - - it: should create a Service with labels if `controller.service.external.labels` is set - set: - controller.service.external.enabled: true - controller.service.external.labels: - external-dns.alpha.kubernetes.io/hostname: external.example.com - asserts: - - equal: - path: metadata.labels["external-dns.alpha.kubernetes.io/hostname"] - value: external.example.com diff --git a/charts/ingress-nginx/tests/controller-serviceaccount_test.yaml b/charts/ingress-nginx/tests/controller-serviceaccount_test.yaml deleted file mode 100644 index 928e53772..000000000 --- a/charts/ingress-nginx/tests/controller-serviceaccount_test.yaml +++ /dev/null @@ -1,47 +0,0 @@ -suite: Controller > ServiceAccount -templates: - - controller-serviceaccount.yaml - -tests: - - it: should not create a ServiceAccount if `serviceAccount.create` is false - set: - serviceAccount.create: false - asserts: - - hasDocuments: - count: 0 - - - it: should create a ServiceAccount if `serviceAccount.create` is true - set: - serviceAccount.create: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: ServiceAccount - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx - - - it: should create a ServiceAccount with specified name if `serviceAccount.name` is set - set: - serviceAccount.name: ingress-nginx-admission-test-sa - asserts: - - hasDocuments: - count: 1 - - isKind: - of: ServiceAccount - - equal: - path: metadata.name - value: ingress-nginx-admission-test-sa - - - it: should create a ServiceAccount with token auto-mounting disabled if `serviceAccount.automountServiceAccountToken` is false - set: - serviceAccount.automountServiceAccountToken: false - asserts: - - hasDocuments: - count: 1 - - isKind: - of: ServiceAccount - - equal: - path: automountServiceAccountToken - value: false diff --git a/charts/ingress-nginx/tests/controller-servicemonitor_test.yaml b/charts/ingress-nginx/tests/controller-servicemonitor_test.yaml deleted file mode 100644 index 7edee98c5..000000000 --- a/charts/ingress-nginx/tests/controller-servicemonitor_test.yaml +++ /dev/null @@ -1,79 +0,0 @@ -suite: Controller > ServiceMonitor -templates: - - controller-servicemonitor.yaml - -tests: - - it: should create a ServiceMonitor if `controller.metrics.serviceMonitor.enabled` is true - set: - controller.metrics.enabled: true - controller.metrics.serviceMonitor.enabled: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: ServiceMonitor - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-controller - - - it: should create a ServiceMonitor with annotations if `controller.metrics.serviceMonitor.annotations` is set - set: - controller.metrics.enabled: true - controller.metrics.serviceMonitor.enabled: true - controller.metrics.serviceMonitor.annotations: - my-little-annotation: test-value - asserts: - - equal: - path: metadata.annotations - value: - my-little-annotation: test-value - - - it: should create a ServiceMonitor with `labelLimit` if `controller.metrics.serviceMonitor.labelLimit` is set - set: - controller.metrics.enabled: true - controller.metrics.serviceMonitor.enabled: true - controller.metrics.serviceMonitor.labelLimit: 20 - asserts: - - equal: - path: spec.labelLimit - value: 20 - - - it: should create a ServiceMonitor with `labelNameLengthLimit` if `controller.metrics.serviceMonitor.labelNameLengthLimit` is set - set: - controller.metrics.enabled: true - controller.metrics.serviceMonitor.enabled: true - controller.metrics.serviceMonitor.labelNameLengthLimit: 50 - asserts: - - equal: - path: spec.labelNameLengthLimit - value: 50 - - - it: should create a ServiceMonitor with `labelValueLengthLimit` if `controller.metrics.serviceMonitor.labelValueLengthLimit` is set - set: - controller.metrics.enabled: true - controller.metrics.serviceMonitor.enabled: true - controller.metrics.serviceMonitor.labelValueLengthLimit: 50 - asserts: - - equal: - path: spec.labelValueLengthLimit - value: 50 - - - it: should create a ServiceMonitor with `sampleLimit` if `controller.metrics.serviceMonitor.sampleLimit` is set - set: - controller.metrics.enabled: true - controller.metrics.serviceMonitor.enabled: true - controller.metrics.serviceMonitor.sampleLimit: 5000 - asserts: - - equal: - path: spec.sampleLimit - value: 5000 - - - it: should create a ServiceMonitor with `targetLimit` if `controller.metrics.serviceMonitor.targetLimit` is set - set: - controller.metrics.enabled: true - controller.metrics.serviceMonitor.enabled: true - controller.metrics.serviceMonitor.targetLimit: 100 - asserts: - - equal: - path: spec.targetLimit - value: 100 diff --git a/charts/ingress-nginx/tests/default-backend-deployment_test.yaml b/charts/ingress-nginx/tests/default-backend-deployment_test.yaml deleted file mode 100644 index 11d400c46..000000000 --- a/charts/ingress-nginx/tests/default-backend-deployment_test.yaml +++ /dev/null @@ -1,198 +0,0 @@ -suite: Default Backend > Deployment -templates: - - default-backend-deployment.yaml - -tests: - - it: should not create a Deployment if `defaultBackend.enabled` is false - set: - defaultBackend.enabled: false - asserts: - - hasDocuments: - count: 0 - - - it: should create a Deployment if `defaultBackend.enabled` is true - set: - defaultBackend.enabled: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: Deployment - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-defaultbackend - - - it: should create a Deployment with 3 replicas if `defaultBackend.replicaCount` is 3 - set: - defaultBackend.enabled: true - defaultBackend.replicaCount: 3 - asserts: - - equal: - path: spec.replicas - value: 3 - - - it: should create a Deployment without replicas if `defaultBackend.autoscaling.enabled` is true - set: - defaultBackend.enabled: true - defaultBackend.autoscaling.enabled: true - asserts: - - notExists: - path: spec.replicas - - - it: should create a Deployment with resource limits if `defaultBackend.resources.limits` is set - set: - defaultBackend.enabled: true - defaultBackend.resources.limits.cpu: 500m - defaultBackend.resources.limits.memory: 512Mi - asserts: - - equal: - path: spec.template.spec.containers[0].resources.limits.cpu - value: 500m - - equal: - path: spec.template.spec.containers[0].resources.limits.memory - value: 512Mi - - - it: should create a Deployment with topology spread constraints if `defaultBackend.topologySpreadConstraints` is set - set: - defaultBackend.enabled: true - defaultBackend.topologySpreadConstraints: - - labelSelector: - matchLabels: - app.kubernetes.io/name: '{{ include "ingress-nginx.name" . }}' - app.kubernetes.io/instance: '{{ .Release.Name }}' - app.kubernetes.io/component: default-backend - topologyKey: topology.kubernetes.io/zone - maxSkew: 1 - whenUnsatisfiable: ScheduleAnyway - - labelSelector: - matchLabels: - app.kubernetes.io/name: '{{ include "ingress-nginx.name" . }}' - app.kubernetes.io/instance: '{{ .Release.Name }}' - app.kubernetes.io/component: default-backend - topologyKey: kubernetes.io/hostname - maxSkew: 1 - whenUnsatisfiable: ScheduleAnyway - asserts: - - equal: - path: spec.template.spec.topologySpreadConstraints - value: - - labelSelector: - matchLabels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: RELEASE-NAME - app.kubernetes.io/component: default-backend - topologyKey: topology.kubernetes.io/zone - maxSkew: 1 - whenUnsatisfiable: ScheduleAnyway - - labelSelector: - matchLabels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: RELEASE-NAME - app.kubernetes.io/component: default-backend - topologyKey: kubernetes.io/hostname - maxSkew: 1 - whenUnsatisfiable: ScheduleAnyway - - - it: should create a Deployment with affinity if `defaultBackend.affinity` is set - set: - defaultBackend.enabled: true - defaultBackend.affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app.kubernetes.io/name - operator: In - values: - - '{{ include "ingress-nginx.name" . }}' - - key: app.kubernetes.io/instance - operator: In - values: - - '{{ .Release.Name }}' - - key: app.kubernetes.io/component - operator: In - values: - - default-backend - topologyKey: kubernetes.io/hostname - asserts: - - equal: - path: spec.template.spec.affinity - value: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app.kubernetes.io/name - operator: In - values: - - ingress-nginx - - key: app.kubernetes.io/instance - operator: In - values: - - RELEASE-NAME - - key: app.kubernetes.io/component - operator: In - values: - - default-backend - topologyKey: kubernetes.io/hostname - - - it: should create a Deployment with `runAsGroup` if `defaultBackend.image.runAsGroup` is set - set: - defaultBackend.enabled: true - defaultBackend.image.runAsGroup: 1000 - asserts: - - equal: - path: spec.template.spec.containers[0].securityContext.runAsGroup - value: 1000 - - - it: should create a Deployment with a custom registry if `global.image.registry` is set - set: - global.image.registry: custom.registry.io - defaultBackend.enabled: true - defaultBackend.image.tag: v1.0.0-dev - defaultBackend.image.digest: sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: custom.registry.io/defaultbackend-amd64:v1.0.0-dev@sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - - - it: should create a Deployment with a custom registry if `defaultBackend.image.registry` is set - set: - defaultBackend.enabled: true - defaultBackend.image.registry: custom.registry.io - defaultBackend.image.tag: v1.0.0-dev - defaultBackend.image.digest: sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: custom.registry.io/defaultbackend-amd64:v1.0.0-dev@sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - - - it: should create a Deployment with a custom image if `defaultBackend.image.image` is set - set: - defaultBackend.enabled: true - defaultBackend.image.image: custom-repo/custom-image - defaultBackend.image.tag: v1.0.0-dev - defaultBackend.image.digest: sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: registry.k8s.io/custom-repo/custom-image:v1.0.0-dev@sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - - - it: should create a Deployment with a custom tag if `defaultBackend.image.tag` is set - set: - defaultBackend.enabled: true - defaultBackend.image.tag: custom-tag - defaultBackend.image.digest: sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: registry.k8s.io/defaultbackend-amd64:custom-tag@sha256:faa2d18687f734994b6bd9e309e7a73852a81c30e1b8f63165fcd4f0a087e3cd - - - it: should create a Deployment with token auto-mounting disabled if `defaultBackend.serviceAccount.automountServiceAccountToken` is false - set: - defaultBackend.enabled: true - defaultBackend.serviceAccount.automountServiceAccountToken: false - asserts: - - equal: - path: spec.template.spec.automountServiceAccountToken - value: false diff --git a/charts/ingress-nginx/tests/default-backend-extra-configmaps_test.yaml b/charts/ingress-nginx/tests/default-backend-extra-configmaps_test.yaml deleted file mode 100644 index aa600e749..000000000 --- a/charts/ingress-nginx/tests/default-backend-extra-configmaps_test.yaml +++ /dev/null @@ -1,50 +0,0 @@ -suite: Default Backend > Extra ConfigMaps -templates: - - default-backend-extra-configmaps.yaml - -tests: - - it: should not create a ConfigMap if `defaultBackend.extraConfigMaps` is empty - set: - defaultBackend.enabled: true - defaultBackend.extraConfigMaps: [] - asserts: - - hasDocuments: - count: 0 - - - it: should create one ConfigMap if `defaultBackend.extraConfigMaps` has one element - set: - defaultBackend.enabled: true - defaultBackend.extraConfigMaps: - - name: my-configmap-1 - data: - key1: value1 - asserts: - - hasDocuments: - count: 1 - - isKind: - of: ConfigMap - - equal: - path: metadata.name - value: my-configmap-1 - - equal: - path: data.key1 - value: value1 - - - it: should create two ConfigMaps if `defaultBackend.extraConfigMaps` has two elements - set: - defaultBackend.enabled: true - defaultBackend.extraConfigMaps: - - name: my-configmap-1 - data: - key1: value1 - - name: my-configmap-2 - data: - key2: value2 - asserts: - - hasDocuments: - count: 2 - - isKind: - of: ConfigMap - - matchRegex: - path: metadata.name - pattern: my-configmap-(1|2) diff --git a/charts/ingress-nginx/tests/default-backend-poddisruptionbudget_test.yaml b/charts/ingress-nginx/tests/default-backend-poddisruptionbudget_test.yaml deleted file mode 100644 index bfe98e883..000000000 --- a/charts/ingress-nginx/tests/default-backend-poddisruptionbudget_test.yaml +++ /dev/null @@ -1,79 +0,0 @@ -suite: Default Backend > PodDisruptionBudget -templates: - - default-backend-poddisruptionbudget.yaml - -tests: - - it: should create a PodDisruptionBudget if `defaultBackend.replicaCount` is greater than 1 - set: - defaultBackend.enabled: true - defaultBackend.replicaCount: 2 - asserts: - - hasDocuments: - count: 1 - - isKind: - of: PodDisruptionBudget - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-defaultbackend - - - it: should not create a PodDisruptionBudget if `defaultBackend.replicaCount` is less than or equal 1 - set: - defaultBackend.enabled: true - defaultBackend.replicaCount: 1 - asserts: - - hasDocuments: - count: 0 - - - it: should create a PodDisruptionBudget if `defaultBackend.autoscaling.enabled` is true and `defaultBackend.autoscaling.minReplicas` is greater than 1 - set: - defaultBackend.enabled: true - defaultBackend.autoscaling.enabled: true - defaultBackend.autoscaling.minReplicas: 2 - asserts: - - hasDocuments: - count: 1 - - isKind: - of: PodDisruptionBudget - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-defaultbackend - - - it: should not create a PodDisruptionBudget if `defaultBackend.autoscaling.enabled` is true and `defaultBackend.autoscaling.minReplicas` is less than or equal 1 - set: - defaultBackend.enabled: true - defaultBackend.autoscaling.enabled: true - defaultBackend.autoscaling.minReplicas: 1 - asserts: - - hasDocuments: - count: 0 - - - it: should create a PodDisruptionBudget without `minAvailable` and with `maxUnavailable` if `defaultBackend.minAvailable` and `defaultBackend.maxUnavailable` are set - set: - defaultBackend.enabled: true - defaultBackend.replicaCount: 2 - defaultBackend.minAvailable: 1 - defaultBackend.maxUnavailable: 1 - asserts: - - hasDocuments: - count: 1 - - isKind: - of: PodDisruptionBudget - - notExists: - path: spec.minAvailable - - equal: - path: spec.maxUnavailable - value: 1 - - - it: should create a PodDisruptionBudget with `unhealthyPodEvictionPolicy` if `defaultBackend.unhealthyPodEvictionPolicy` is set - set: - defaultBackend.enabled: true - defaultBackend.replicaCount: 2 - defaultBackend.unhealthyPodEvictionPolicy: IfHealthyBudget - asserts: - - hasDocuments: - count: 1 - - isKind: - of: PodDisruptionBudget - - equal: - path: spec.unhealthyPodEvictionPolicy - value: IfHealthyBudget diff --git a/charts/ingress-nginx/tests/default-backend-service_test.yaml b/charts/ingress-nginx/tests/default-backend-service_test.yaml deleted file mode 100644 index 521d82091..000000000 --- a/charts/ingress-nginx/tests/default-backend-service_test.yaml +++ /dev/null @@ -1,52 +0,0 @@ -suite: Default Backend > Service -templates: - - default-backend-service.yaml - -tests: - - it: should not create a Service if `defaultBackend.enabled` is false - set: - defaultBackend.enabled: false - asserts: - - hasDocuments: - count: 0 - - - it: should create a Service if `defaultBackend.enabled` is true - set: - defaultBackend.enabled: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: Service - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-defaultbackend - - - it: should create a Service with port 80 if `defaultBackend.service.port` is 80 - set: - defaultBackend.enabled: true - defaultBackend.service.port: 80 - asserts: - - equal: - path: spec.ports[0].port - value: 80 - - - it: should create a Service without `clusterIPs` if `defaultBackend.service.clusterIPs` is not set - set: - defaultBackend.enabled: true - asserts: - - notExists: - path: spec.clusterIPs - - - it: should create a Service with `clusterIPs` if `defaultBackend.service.clusterIPs` is set - set: - defaultBackend.enabled: true - defaultBackend.service.clusterIPs: - - 10.0.0.1 - - fd00::1 - asserts: - - equal: - path: spec.clusterIPs - value: - - 10.0.0.1 - - fd00::1 diff --git a/charts/ingress-nginx/tests/default-backend-serviceaccount_test.yaml b/charts/ingress-nginx/tests/default-backend-serviceaccount_test.yaml deleted file mode 100644 index 05a815d0a..000000000 --- a/charts/ingress-nginx/tests/default-backend-serviceaccount_test.yaml +++ /dev/null @@ -1,51 +0,0 @@ -suite: Default Backend > ServiceAccount -templates: - - default-backend-serviceaccount.yaml - -tests: - - it: should not create a ServiceAccount if `defaultBackend.serviceAccount.create` is false - set: - defaultBackend.enabled: true - defaultBackend.serviceAccount.create: false - asserts: - - hasDocuments: - count: 0 - - - it: should create a ServiceAccount if `defaultBackend.serviceAccount.create` is true - set: - defaultBackend.enabled: true - defaultBackend.serviceAccount.create: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: ServiceAccount - - equal: - path: metadata.name - value: RELEASE-NAME-ingress-nginx-backend - - - it: should create a ServiceAccount with specified name if `defaultBackend.serviceAccount.name` is set - set: - defaultBackend.enabled: true - defaultBackend.serviceAccount.name: ingress-nginx-admission-test-sa - asserts: - - hasDocuments: - count: 1 - - isKind: - of: ServiceAccount - - equal: - path: metadata.name - value: ingress-nginx-admission-test-sa - - - it: should create a ServiceAccount with token auto-mounting disabled if `defaultBackend.serviceAccount.automountServiceAccountToken` is false - set: - defaultBackend.enabled: true - defaultBackend.serviceAccount.automountServiceAccountToken: false - asserts: - - hasDocuments: - count: 1 - - isKind: - of: ServiceAccount - - equal: - path: automountServiceAccountToken - value: false diff --git a/charts/ingress-nginx/values.yaml b/charts/ingress-nginx/values.yaml index ccebb9ceb..42fe0c77e 100644 --- a/charts/ingress-nginx/values.yaml +++ b/charts/ingress-nginx/values.yaml @@ -1,217 +1,146 @@ ## nginx configuration -## Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/index.md +## Ref: https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/index.md ## -global: - image: - # -- Registry host to pull images from. - registry: registry.k8s.io ## Overrides for generated resource names # See templates/_helpers.tpl # nameOverride: # fullnameOverride: -# -- Override the deployment namespace; defaults to .Release.Namespace -namespaceOverride: "" -## Labels to apply to all resources -## -commonLabels: {} -# scmhash: abc123 -# myLabel: aakkmd - controller: name: controller - enableAnnotationValidations: true image: - ## Keep false as default for now! - chroot: false - # registry: registry.k8s.io + registry: k8s.gcr.io image: ingress-nginx/controller - ## for backwards compatibility consider setting the full image url via the repository value below - ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail - ## repository: - tag: "v1.12.0" - digest: sha256:e6b8de175acda6ca913891f0f727bca4527e797d52688cbe9fec9040d6f6b6fa - digestChroot: sha256:87c88e1c38a6c8d4483c8f70b69e2cca49853bb3ec3124b9b1be648edf139af3 + # for backwards compatibility consider setting the full image url via the repository value below + # use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail + # repository: + tag: "v0.47.0" + digest: sha256:a1e4efc107be0bb78f32eaec37bef17d7a0c81bec8066cdf2572508d21351d0b pullPolicy: IfNotPresent - runAsNonRoot: true - # -- This value must not be changed using the official image. - # uid=101(www-data) gid=82(www-data) groups=82(www-data) + # www-data -> uid 101 runAsUser: 101 - # -- This value must not be changed using the official image. - # uid=101(www-data) gid=82(www-data) groups=82(www-data) - runAsGroup: 82 - allowPrivilegeEscalation: false - seccompProfile: - type: RuntimeDefault - readOnlyRootFilesystem: false - # -- Configures the controller container name + allowPrivilegeEscalation: true + + # Use an existing PSP instead of creating one + existingPsp: "" + + # Configures the controller container name containerName: controller - # -- Configures the ports that the nginx-controller listens on + + # Configures the ports the nginx-controller listens on containerPort: http: 80 https: 443 - # -- Global configuration passed to the ConfigMap consumed by the controller. Values may contain Helm templates. - # Ref.: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/ + + # Will add custom configuration options to Nginx https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/ config: {} - # -- Annotations to be added to the controller config configuration configmap. + + ## Annotations to be added to the controller config configuration configmap + ## configAnnotations: {} - # -- Will add custom headers before sending traffic to backends according to https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/custom-headers + + # Will add custom headers before sending traffic to backends according to https://github.com/kubernetes/ingress-nginx/tree/master/docs/examples/customization/custom-headers proxySetHeaders: {} - # -- Will add custom headers before sending response traffic to the client according to: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#add-headers + + # Will add custom headers before sending response traffic to the client according to: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#add-headers addHeaders: {} - # -- Optionally customize the pod dnsConfig. + + # Optionally customize the pod dnsConfig. dnsConfig: {} - # -- Optionally customize the pod hostAliases. - hostAliases: [] - # - ip: 127.0.0.1 - # hostnames: - # - foo.local - # - bar.local - # - ip: 10.1.2.3 - # hostnames: - # - foo.remote - # - bar.remote - # -- Optionally customize the pod hostname. - hostname: {} - # -- Optionally change this to ClusterFirstWithHostNet in case you have 'hostNetwork: true'. + + # Optionally change this to ClusterFirstWithHostNet in case you have 'hostNetwork: true'. # By default, while using host network, name resolution uses the host's DNS. If you wish nginx-controller # to keep resolving names inside the k8s network, use ClusterFirstWithHostNet. dnsPolicy: ClusterFirst - # -- Bare-metal considerations via the host network https://kubernetes.github.io/ingress-nginx/deploy/baremetal/#via-the-host-network - # Ingress status was blank because there is no Service exposing the Ingress-Nginx Controller in a configuration using the host network, the default --publish-service flag used in standard cloud setups does not apply + + # Bare-metal considerations via the host network https://kubernetes.github.io/ingress-nginx/deploy/baremetal/#via-the-host-network + # Ingress status was blank because there is no Service exposing the NGINX Ingress controller in a configuration using the host network, the default --publish-service flag used in standard cloud setups does not apply reportNodeInternalIp: false - # -- Process Ingress objects without ingressClass annotation/ingressClassName field - # Overrides value for --watch-ingress-without-class flag of the controller binary - # Defaults to false - watchIngressWithoutClass: false - # -- Process IngressClass per name (additionally as per spec.controller). - ingressClassByName: false - # -- This configuration enables Topology Aware Routing feature, used together with service annotation service.kubernetes.io/topology-mode="auto" - # Defaults to false - enableTopologyAwareRouting: false - # -- This configuration disable Nginx Controller Leader Election - disableLeaderElection: false - # -- Duration a leader election is valid before it's getting re-elected, e.g. `15s`, `10m` or `1h`. (Default: 30s) - electionTTL: "" - # -- This configuration defines if Ingress Controller should allow users to set - # their own *-snippet annotations, otherwise this is forbidden / dropped - # when users add those annotations. - # Global snippets in ConfigMap are still respected - allowSnippetAnnotations: false - # -- Required for use with CNI based kubernetes installations (such as ones set up by kubeadm), + + # Required for use with CNI based kubernetes installations (such as ones set up by kubeadm), # since CNI and hostport don't mix yet. Can be deprecated once https://github.com/kubernetes/kubernetes/issues/23920 # is merged hostNetwork: false + ## Use host ports 80 and 443 ## Disabled by default + ## hostPort: - # -- Enable 'hostPort' or not enabled: false ports: - # -- 'hostPort' http port http: 80 - # -- 'hostPort' https port https: 443 - # NetworkPolicy for controller component. - networkPolicy: - # -- Enable 'networkPolicy' or not - enabled: false - # -- Election ID to use for status update, by default it uses the controller name combined with a suffix of 'leader' - electionID: "" - # -- This section refers to the creation of the IngressClass resource. - # IngressClasses are immutable and cannot be changed after creation. - # We do not support namespaced IngressClasses, yet, so a ClusterRole and a ClusterRoleBinding is required. - ingressClassResource: - # -- Name of the IngressClass - name: nginx - # -- Create the IngressClass or not - enabled: true - # -- If true, Ingresses without `ingressClassName` get assigned to this IngressClass on creation. - # Ingress creation gets rejected if there are multiple default IngressClasses. - # Ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#default-ingress-class - default: false - # -- Annotations to be added to the IngressClass resource. - annotations: {} - # -- Controller of the IngressClass. An Ingress Controller looks for IngressClasses it should reconcile by this value. - # This value is also being set as the `--controller-class` argument of this Ingress Controller. - # Ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class - controllerValue: k8s.io/ingress-nginx - # -- Aliases of this IngressClass. Creates copies with identical settings but the respective alias as name. - # Useful for development environments with only one Ingress Controller but production-like Ingress resources. - # `default` gets enabled on the original IngressClass only. - aliases: [] - # aliases: - # - nginx-alias-1 - # - nginx-alias-2 - # -- A link to a custom resource containing additional configuration for the controller. - # This is optional if the controller consuming this IngressClass does not require additional parameters. - # Ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class - parameters: {} - # parameters: - # apiGroup: k8s.example.com - # kind: IngressParameters - # name: external-lb - # -- For backwards compatibility with ingress.class annotation, use ingressClass. - # Algorithm is as follows, first ingressClassName is considered, if not present, controller looks for ingress.class annotation + + ## Election ID to use for status update + ## + electionID: ingress-controller-leader + + ## Name of the ingress class to route through this controller + ## ingressClass: nginx - # -- Labels to add to the pod container metadata + + # labels to add to the pod container metadata podLabels: {} # key: value - # -- Security context for controller pods + ## Security Context policies for controller pods + ## podSecurityContext: {} - # -- sysctls for controller pods - ## Ref: https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ + + ## See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for + ## notes on enabling and using sysctls + ### sysctls: {} # sysctls: # "net.core.somaxconn": "8192" - # -- Security context for controller containers - containerSecurityContext: {} - # -- Allows customization of the source of the IP address or FQDN to report - # in the ingress status field. By default, it reads the information provided - # by the service. If disable, the status field reports the IP address of the - # node or nodes where an ingress controller pod is running. + + ## Allows customization of the source of the IP address or FQDN to report + ## in the ingress status field. By default, it reads the information provided + ## by the service. If disable, the status field reports the IP address of the + ## node or nodes where an ingress controller pod is running. publishService: - # -- Enable 'publishService' or not enabled: true - # -- Allows overriding of the publish service to bind to - # Must be / + ## Allows overriding of the publish service to bind to + ## Must be / + ## pathOverride: "" - # Limit the scope of the controller to a specific namespace + + ## Limit the scope of the controller + ## scope: - # -- Enable 'scope' or not enabled: false - # -- Namespace to limit the controller to; defaults to $(POD_NAMESPACE) - namespace: "" - # -- When scope.enabled == false, instead of watching all namespaces, we watching namespaces whose labels - # only match with namespaceSelector. Format like foo=bar. Defaults to empty, means watching all namespaces. - namespaceSelector: "" - # -- Allows customization of the configmap / nginx-configmap namespace; defaults to $(POD_NAMESPACE) - configMapNamespace: "" + namespace: "" # defaults to .Release.Namespace + + ## Allows customization of the configmap / nginx-configmap namespace + ## + configMapNamespace: "" # defaults to .Release.Namespace + + ## Allows customization of the tcp-services-configmap + ## tcp: - # -- Allows customization of the tcp-services-configmap; defaults to $(POD_NAMESPACE) - configMapNamespace: "" - # -- Annotations to be added to the tcp config configmap + configMapNamespace: "" # defaults to .Release.Namespace + ## Annotations to be added to the tcp config configmap annotations: {} + + ## Allows customization of the udp-services-configmap + ## udp: - # -- Allows customization of the udp-services-configmap; defaults to $(POD_NAMESPACE) - configMapNamespace: "" - # -- Annotations to be added to the udp config configmap + configMapNamespace: "" # defaults to .Release.Namespace + ## Annotations to be added to the udp config configmap annotations: {} - # -- Maxmind license key to download GeoLite2 Databases. - ## https://blog.maxmind.com/2019/12/significant-changes-to-accessing-and-using-geolite2-databases/ + + # Maxmind license key to download GeoLite2 Databases + # https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases maxmindLicenseKey: "" - # -- Additional command line arguments to pass to Ingress-Nginx Controller - # E.g. to specify the default SSL certificate you can use - extraArgs: {} + + ## Additional command line arguments to pass to nginx-ingress-controller + ## E.g. to specify the default SSL certificate you can use ## extraArgs: ## default-ssl-certificate: "/" - ## time-buckets: "0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10" - ## length-buckets: "10,20,30,40,50,60,70,80,90,100" - ## size-buckets: "10,100,1000,10000,100000,1e+06,1e+07" + extraArgs: {} - # -- Additional environment variables to set + ## Additional environment variables to set extraEnvs: [] # extraEnvs: # - name: FOO @@ -220,33 +149,35 @@ controller: # key: FOO # name: secret-resource - # -- Use a `DaemonSet` or `Deployment` + ## DaemonSet or Deployment + ## kind: Deployment - # -- Annotations to be added to the controller Deployment or DaemonSet + + ## Annotations to be added to the controller Deployment or DaemonSet ## annotations: {} # keel.sh/pollSchedule: "@every 60m" - # -- Labels to be added to the controller Deployment or DaemonSet and other resources that do not have option to specify labels + ## Labels to be added to the controller Deployment or DaemonSet ## labels: {} # keel.sh/policy: patch # keel.sh/trigger: poll - # -- The update strategy to apply to the Deployment or DaemonSet + + # The update strategy to apply to the Deployment or DaemonSet ## updateStrategy: {} # rollingUpdate: # maxUnavailable: 1 # type: RollingUpdate - # -- Specifies the number of seconds you want to wait for the controller deployment to progress before the system reports back that it has failed. - # Ref.: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#progress-deadline-seconds - progressDeadlineSeconds: 0 - # -- `minReadySeconds` to avoid killing pods before we are ready + # minReadySeconds to avoid killing pods before we are ready ## minReadySeconds: 0 - # -- Node tolerations for server scheduling to nodes with taints + + + ## Node tolerations for server scheduling to nodes with taints ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ ## tolerations: [] @@ -255,98 +186,86 @@ controller: # value: "value" # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" - # -- Affinity and anti-affinity rules for server scheduling to nodes + ## Affinity and anti-affinity ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity ## affinity: {} - # # An example of preferred pod anti-affinity, weight is in the range 1-100 - # podAntiAffinity: - # preferredDuringSchedulingIgnoredDuringExecution: - # - weight: 100 - # podAffinityTerm: - # labelSelector: - # matchExpressions: - # - key: app.kubernetes.io/name - # operator: In - # values: - # - '{{ include "ingress-nginx.name" . }}' - # - key: app.kubernetes.io/instance - # operator: In - # values: - # - '{{ .Release.Name }}' - # - key: app.kubernetes.io/component - # operator: In - # values: - # - controller - # topologyKey: kubernetes.io/hostname + # # An example of preferred pod anti-affinity, weight is in the range 1-100 + # podAntiAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - weight: 100 + # podAffinityTerm: + # labelSelector: + # matchExpressions: + # - key: app.kubernetes.io/name + # operator: In + # values: + # - ingress-nginx + # - key: app.kubernetes.io/instance + # operator: In + # values: + # - ingress-nginx + # - key: app.kubernetes.io/component + # operator: In + # values: + # - controller + # topologyKey: kubernetes.io/hostname - # # An example of required pod anti-affinity - # podAntiAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # - labelSelector: - # matchExpressions: - # - key: app.kubernetes.io/name - # operator: In - # values: - # - '{{ include "ingress-nginx.name" . }}' - # - key: app.kubernetes.io/instance - # operator: In - # values: - # - '{{ .Release.Name }}' - # - key: app.kubernetes.io/component - # operator: In - # values: - # - controller - # topologyKey: kubernetes.io/hostname + # # An example of required pod anti-affinity + # podAntiAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # - labelSelector: + # matchExpressions: + # - key: app.kubernetes.io/name + # operator: In + # values: + # - ingress-nginx + # - key: app.kubernetes.io/instance + # operator: In + # values: + # - ingress-nginx + # - key: app.kubernetes.io/component + # operator: In + # values: + # - controller + # topologyKey: "kubernetes.io/hostname" - # -- Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in. + ## Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in. ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ ## topologySpreadConstraints: [] - # - labelSelector: - # matchLabels: - # app.kubernetes.io/name: '{{ include "ingress-nginx.name" . }}' - # app.kubernetes.io/instance: '{{ .Release.Name }}' - # app.kubernetes.io/component: controller - # matchLabelKeys: - # - pod-template-hash - # topologyKey: topology.kubernetes.io/zone - # maxSkew: 1 - # whenUnsatisfiable: ScheduleAnyway - # - labelSelector: - # matchLabels: - # app.kubernetes.io/name: '{{ include "ingress-nginx.name" . }}' - # app.kubernetes.io/instance: '{{ .Release.Name }}' - # app.kubernetes.io/component: controller - # matchLabelKeys: - # - pod-template-hash - # topologyKey: kubernetes.io/hostname - # maxSkew: 1 - # whenUnsatisfiable: ScheduleAnyway + # - maxSkew: 1 + # topologyKey: failure-domain.beta.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app.kubernetes.io/instance: ingress-nginx-internal - # -- `terminationGracePeriodSeconds` to avoid killing pods before we are ready + ## terminationGracePeriodSeconds ## wait up to five minutes for the drain of connections ## terminationGracePeriodSeconds: 300 - # -- Node labels for controller pod assignment - ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/ + + ## Node labels for controller pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ ## nodeSelector: kubernetes.io/os: linux + ## Liveness and readiness probe values ## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes ## - ## startupProbe: - ## httpGet: - ## # should match container.healthCheckPath - ## path: "/healthz" - ## port: 10254 - ## scheme: HTTP - ## initialDelaySeconds: 5 - ## periodSeconds: 5 - ## timeoutSeconds: 2 - ## successThreshold: 1 - ## failureThreshold: 5 + # startupProbe: + # httpGet: + # # should match container.healthCheckPath + # path: "/healthz" + # port: 10254 + # scheme: HTTP + # initialDelaySeconds: 5 + # periodSeconds: 5 + # timeoutSeconds: 2 + # successThreshold: 1 + # failureThreshold: 5 livenessProbe: httpGet: # should match container.healthCheckPath @@ -369,57 +288,40 @@ controller: timeoutSeconds: 1 successThreshold: 1 failureThreshold: 3 - # -- Path of the health check endpoint. All requests received on the port defined by + + + # Path of the health check endpoint. All requests received on the port defined by # the healthz-port parameter are forwarded internally to this path. healthCheckPath: "/healthz" - # -- Address to bind the health check endpoint. - # It is better to set this option to the internal node address - # if the Ingress-Nginx Controller is running in the `hostNetwork: true` mode. - healthCheckHost: "" - # -- Annotations to be added to controller pods + + ## Annotations to be added to controller pods ## podAnnotations: {} + replicaCount: 1 - # -- Minimum available pods set in PodDisruptionBudget. - # Define either 'minAvailable' or 'maxUnavailable', never both. + minAvailable: 1 - # -- Maximum unavailable pods set in PodDisruptionBudget. If set, 'minAvailable' is ignored. - # maxUnavailable: 1 - # -- Eviction policy for unhealthy pods guarded by PodDisruptionBudget. - # Ref: https://kubernetes.io/blog/2023/01/06/unhealthy-pod-eviction-policy-for-pdbs/ - unhealthyPodEvictionPolicy: "" - ## Define requests resources to avoid probe issues due to CPU utilization in busy nodes - ## ref: https://github.com/kubernetes/ingress-nginx/issues/4735#issuecomment-551204903 - ## Ideally, there should be no limits. - ## https://engineering.indeedblog.com/blog/2019/12/cpu-throttling-regression-fix/ + + # Define requests resources to avoid probe issues due to CPU utilization in busy nodes + # ref: https://github.com/kubernetes/ingress-nginx/issues/4735#issuecomment-551204903 + # Ideally, there should be no limits. + # https://engineering.indeedblog.com/blog/2019/12/cpu-throttling-regression-fix/ resources: - ## limits: - ## cpu: 100m - ## memory: 90Mi + # limits: + # cpu: 100m + # memory: 90Mi requests: cpu: 100m memory: 90Mi + # Mutually exclusive with keda autoscaling autoscaling: enabled: false - annotations: {} minReplicas: 1 maxReplicas: 11 targetCPUUtilizationPercentage: 50 targetMemoryUtilizationPercentage: 50 - behavior: {} - # scaleDown: - # stabilizationWindowSeconds: 300 - # policies: - # - type: Pods - # value: 1 - # periodSeconds: 180 - # scaleUp: - # stabilizationWindowSeconds: 300 - # policies: - # - type: Pods - # value: 2 - # periodSeconds: 60 + autoscalingTemplate: [] # Custom or additional autoscaling metrics # ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics @@ -434,17 +336,14 @@ controller: # Mutually exclusive with hpa autoscaling keda: apiVersion: "keda.sh/v1alpha1" - ## apiVersion changes with keda 1.x vs 2.x - ## 2.x = keda.sh/v1alpha1 - ## 1.x = keda.k8s.io/v1alpha1 + # apiVersion changes with keda 1.x vs 2.x + # 2.x = keda.sh/v1alpha1 + # 1.x = keda.k8s.io/v1alpha1 enabled: false minReplicas: 1 maxReplicas: 11 pollingInterval: 30 cooldownPeriod: 300 - # fallback: - # failureThreshold: 3 - # replicas: 11 restoreToOriginalReplicaCount: false scaledObject: annotations: {} @@ -452,225 +351,110 @@ controller: # annotations: # key: value triggers: [] - # - type: prometheus - # metadata: - # serverAddress: http://:9090 - # metricName: http_requests_total - # threshold: '100' - # query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) + # - type: prometheus + # metadata: + # serverAddress: http://:9090 + # metricName: http_requests_total + # threshold: '100' + # query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) behavior: {} - # scaleDown: - # stabilizationWindowSeconds: 300 - # policies: - # - type: Pods - # value: 1 - # periodSeconds: 180 - # scaleUp: - # stabilizationWindowSeconds: 300 - # policies: - # - type: Pods - # value: 2 - # periodSeconds: 60 - # -- Enable mimalloc as a drop-in replacement for malloc. + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + + ## Enable mimalloc as a drop-in replacement for malloc. ## ref: https://github.com/microsoft/mimalloc ## enableMimalloc: true + ## Override NGINX template customTemplate: configMapName: "" configMapKey: "" - service: - # -- Enable controller services or not. This does not influence the creation of either the admission webhook or the metrics service. - enabled: true - external: - # -- Enable the external controller service or not. Useful for internal-only deployments. - enabled: true - # -- Labels to be added to the external controller service. - labels: {} - # -- Annotations to be added to the external controller service. See `controller.service.internal.annotations` for annotations to be added to the internal controller service. - annotations: {} - # -- Labels to be added to both controller services. - labels: {} - # -- Type of the external controller service. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types - type: LoadBalancer - # -- Pre-defined cluster internal IP address of the external controller service. Take care of collisions with existing services. - # This value is immutable. Set once, it can not be changed without deleting and re-creating the service. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address - clusterIP: "" - # -- Pre-defined cluster internal IP addresses of the external controller service. Take care of collisions with existing services. - # This value is immutable. Set once, it can not be changed without deleting and re-creating the service. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address - clusterIPs: [] - # -- List of node IP addresses at which the external controller service is available. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-ips - externalIPs: [] - # -- Deprecated: Pre-defined IP address of the external controller service. Used by cloud providers to connect the resulting load balancer service to a pre-existing static IP. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer - loadBalancerIP: "" - # -- Restrict access to the external controller service. Values must be CIDRs. Allows any source address by default. - loadBalancerSourceRanges: [] - # -- Load balancer class of the external controller service. Used by cloud providers to select a load balancer implementation other than the cloud provider default. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-class - loadBalancerClass: "" - # -- Enable node port allocation for the external controller service or not. Applies to type `LoadBalancer` only. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-nodeport-allocation - # allocateLoadBalancerNodePorts: true - # -- External traffic policy of the external controller service. Set to "Local" to preserve source IP on providers supporting it. - # Ref: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip - externalTrafficPolicy: "" - # -- Session affinity of the external controller service. Must be either "None" or "ClientIP" if set. Defaults to "None". - # Ref: https://kubernetes.io/docs/reference/networking/virtual-ips/#session-affinity - sessionAffinity: "" - # -- Specifies the health check node port (numeric port number) for the external controller service. - # If not specified, the service controller allocates a port from your cluster's node port range. + service: + enabled: true + + annotations: {} + labels: {} + # clusterIP: "" + + ## List of IP addresses at which the controller services are available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + # loadBalancerIP: "" + loadBalancerSourceRanges: [] + + enableHttp: true + enableHttps: true + + ## Set external traffic policy to: "Local" to preserve source IP on + ## providers supporting it + ## Ref: https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typeloadbalancer + # externalTrafficPolicy: "" + + # Must be either "None" or "ClientIP" if set. Kubernetes will default to "None". + # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + # sessionAffinity: "" + + # specifies the health check node port (numeric port number) for the service. If healthCheckNodePort isn’t specified, + # the service controller allocates a port from your cluster’s NodePort range. # Ref: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip # healthCheckNodePort: 0 - # -- Traffic distribution policy of the external controller service. Set to "PreferClose" to route traffic to endpoints that are topologically closer to the client. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#traffic-distribution - trafficDistribution: "" - - # -- Represents the dual-stack capabilities of the external controller service. Possible values are SingleStack, PreferDualStack or RequireDualStack. - # Fields `ipFamilies` and `clusterIP` depend on the value of this field. - # Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/#services - ipFamilyPolicy: SingleStack - # -- List of IP families (e.g. IPv4, IPv6) assigned to the external controller service. This field is usually assigned automatically based on cluster configuration and the `ipFamilyPolicy` field. - # Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/#services - ipFamilies: - - IPv4 - # -- Enable the HTTP listener on both controller services or not. - enableHttp: true - # -- Enable the HTTPS listener on both controller services or not. - enableHttps: true ports: - # -- Port the external HTTP listener is published with. http: 80 - # -- Port the external HTTPS listener is published with. https: 443 + targetPorts: - # -- Port of the ingress controller the external HTTP listener is mapped to. http: http - # -- Port of the ingress controller the external HTTPS listener is mapped to. https: https - # -- Declare the app protocol of the external HTTP and HTTPS listeners or not. Supersedes provider-specific annotations for declaring the backend protocol. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#application-protocol - appProtocol: true + + type: LoadBalancer + + # type: NodePort + # nodePorts: + # http: 32080 + # https: 32443 + # tcp: + # 8080: 32808 nodePorts: - # -- Node port allocated for the external HTTP listener. If left empty, the service controller allocates one from the configured node port range. http: "" - # -- Node port allocated for the external HTTPS listener. If left empty, the service controller allocates one from the configured node port range. https: "" - # -- Node port mapping for external TCP listeners. If left empty, the service controller allocates them from the configured node port range. - # Example: - # tcp: - # 8080: 30080 tcp: {} - # -- Node port mapping for external UDP listeners. If left empty, the service controller allocates them from the configured node port range. - # Example: - # udp: - # 53: 30053 udp: {} + + ## Enables an additional internal load balancer (besides the external one). + ## Annotations are mandatory for the load balancer to come up. Varies with the cloud service. internal: - # -- Enable the internal controller service or not. Remember to configure `controller.service.internal.annotations` when enabling this. enabled: false - # -- Labels to be added to the internal controller service. - labels: {} - # -- Annotations to be added to the internal controller service. Mandatory for the internal controller service to be created. Varies with the cloud service. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer annotations: {} - # -- Type of the internal controller service. - # Defaults to the value of `controller.service.type`. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types - type: "" - # -- Pre-defined cluster internal IP address of the internal controller service. Take care of collisions with existing services. - # This value is immutable. Set once, it can not be changed without deleting and re-creating the service. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address - clusterIP: "" - # -- Pre-defined cluster internal IP addresses of the internal controller service. Take care of collisions with existing services. - # This value is immutable. Set once, it can not be changed without deleting and re-creating the service. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address - clusterIPs: [] - # -- List of node IP addresses at which the internal controller service is available. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-ips - externalIPs: [] - # -- Deprecated: Pre-defined IP address of the internal controller service. Used by cloud providers to connect the resulting load balancer service to a pre-existing static IP. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer - loadBalancerIP: "" - # -- Restrict access to the internal controller service. Values must be CIDRs. Allows any source address by default. + + # loadBalancerIP: "" + + ## Restrict access For LoadBalancer service. Defaults to 0.0.0.0/0. loadBalancerSourceRanges: [] - # -- Load balancer class of the internal controller service. Used by cloud providers to select a load balancer implementation other than the cloud provider default. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-class - loadBalancerClass: "" - # -- Enable node port allocation for the internal controller service or not. Applies to type `LoadBalancer` only. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-nodeport-allocation - # allocateLoadBalancerNodePorts: true - # -- External traffic policy of the internal controller service. Set to "Local" to preserve source IP on providers supporting it. - # Ref: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip - externalTrafficPolicy: "" - # -- Session affinity of the internal controller service. Must be either "None" or "ClientIP" if set. Defaults to "None". - # Ref: https://kubernetes.io/docs/reference/networking/virtual-ips/#session-affinity - sessionAffinity: "" - # -- Specifies the health check node port (numeric port number) for the internal controller service. - # If not specified, the service controller allocates a port from your cluster's node port range. - # Ref: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip - # healthCheckNodePort: 0 + ## Set external traffic policy to: "Local" to preserve source IP on + ## providers supporting it + ## Ref: https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typeloadbalancer + # externalTrafficPolicy: "" - # -- Traffic distribution policy of the internal controller service. Set to "PreferClose" to route traffic to endpoints that are topologically closer to the client. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#traffic-distribution - trafficDistribution: "" - - # -- Represents the dual-stack capabilities of the internal controller service. Possible values are SingleStack, PreferDualStack or RequireDualStack. - # Fields `ipFamilies` and `clusterIP` depend on the value of this field. - # Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/#services - ipFamilyPolicy: SingleStack - # -- List of IP families (e.g. IPv4, IPv6) assigned to the internal controller service. This field is usually assigned automatically based on cluster configuration and the `ipFamilyPolicy` field. - # Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/#services - ipFamilies: - - IPv4 - ports: {} - # -- Port the internal HTTP listener is published with. - # Defaults to the value of `controller.service.ports.http`. - # http: 80 - # -- Port the internal HTTPS listener is published with. - # Defaults to the value of `controller.service.ports.https`. - # https: 443 - - targetPorts: {} - # -- Port of the ingress controller the internal HTTP listener is mapped to. - # Defaults to the value of `controller.service.targetPorts.http`. - # http: http - # -- Port of the ingress controller the internal HTTPS listener is mapped to. - # Defaults to the value of `controller.service.targetPorts.https`. - # https: https - - # -- Declare the app protocol of the internal HTTP and HTTPS listeners or not. Supersedes provider-specific annotations for declaring the backend protocol. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#application-protocol - appProtocol: true - nodePorts: - # -- Node port allocated for the internal HTTP listener. If left empty, the service controller allocates one from the configured node port range. - http: "" - # -- Node port allocated for the internal HTTPS listener. If left empty, the service controller allocates one from the configured node port range. - https: "" - # -- Node port mapping for internal TCP listeners. If left empty, the service controller allocates them from the configured node port range. - # Example: - # tcp: - # 8080: 30080 - tcp: {} - # -- Node port mapping for internal UDP listeners. If left empty, the service controller allocates them from the configured node port range. - # Example: - # udp: - # 53: 30053 - udp: {} - # shareProcessNamespace enables process namespace sharing within the pod. - # This can be used for example to signal log rotation using `kill -USR1` from a sidecar. - shareProcessNamespace: false - # -- Additional containers to be added to the controller pod. - # See https://github.com/lemonldap-ng-controller/lemonldap-ng-controller as example. extraContainers: [] + ## Additional containers to be added to the controller pod. + ## See https://github.com/lemonldap-ng-controller/lemonldap-ng-controller as example. # - name: my-sidecar # image: nginx:latest # - name: lemonldap-ng-controller @@ -692,69 +476,25 @@ controller: # - name: copy-portal-skins # mountPath: /srv/var/lib/lemonldap-ng/portal/skins - # -- Additional volumeMounts to the controller main container. extraVolumeMounts: [] + ## Additional volumeMounts to the controller main container. # - name: copy-portal-skins # mountPath: /var/lib/lemonldap-ng/portal/skins - # -- Additional volumes to the controller pod. extraVolumes: [] + ## Additional volumes to the controller pod. # - name: copy-portal-skins # emptyDir: {} - # -- Containers, which are run before the app containers are started. extraInitContainers: [] + ## Containers, which are run before the app containers are started. # - name: init-myservice # image: busybox # command: ['sh', '-c', 'until nslookup myservice; do echo waiting for myservice; sleep 2; done;'] - # -- Modules, which are mounted into the core nginx image. - extraModules: [] - # - name: mytestmodule - # image: - # # registry: registry.k8s.io - # image: ingress-nginx/mytestmodule - # ## for backwards compatibility consider setting the full image url via the repository value below - # ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail - # ## repository: - # tag: "v1.0.0" - # digest: "" - # distroless: false - # containerSecurityContext: - # runAsNonRoot: true - # runAsUser: - # runAsGroup: - # allowPrivilegeEscalation: false - # seccompProfile: - # type: RuntimeDefault - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # resources: {} - # - # The image must contain a `/usr/local/bin/init_module.sh` executable, which - # will be executed as initContainers, to move its config files within the - # mounted volume. - admissionWebhooks: - name: admission annotations: {} - # ignore-check.kube-linter.io/no-read-only-rootfs: "This deployment needs write access to root filesystem". - - ## Additional annotations to the admission webhooks. - ## These annotations will be added to the ValidatingWebhookConfiguration and - ## the Jobs Spec of the admission webhooks. enabled: true - # -- Additional environment variables to set - extraEnvs: [] - # extraEnvs: - # - name: FOO - # valueFrom: - # secretKeyRef: - # key: FOO - # name: secret-resource - # -- Admission Webhook failure policy to use failurePolicy: Fail # timeoutSeconds: 10 port: 8443 @@ -762,8 +502,10 @@ controller: key: "/usr/local/certificates/key" namespaceSelector: {} objectSelector: {} - # -- Labels to be added to admission webhooks - labels: {} + + # Use an existing PSP instead of creating one + existingPsp: "" + service: annotations: {} # clusterIP: "" @@ -772,237 +514,158 @@ controller: loadBalancerSourceRanges: [] servicePort: 443 type: ClusterIP - createSecretJob: - name: create - # -- Security context for secret creation containers - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 - allowPrivilegeEscalation: false - seccompProfile: - type: RuntimeDefault - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - resources: {} - # limits: - # cpu: 10m - # memory: 20Mi - # requests: - # cpu: 10m - # memory: 20Mi - patchWebhookJob: - name: patch - # -- Security context for webhook patch containers - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 - allowPrivilegeEscalation: false - seccompProfile: - type: RuntimeDefault - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - resources: {} + patch: enabled: true image: - # registry: registry.k8s.io - image: ingress-nginx/kube-webhook-certgen - ## for backwards compatibility consider setting the full image url via the repository value below - ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail - ## repository: + registry: docker.io + image: jettech/kube-webhook-certgen + # for backwards compatibility consider setting the full image url via the repository value below + # use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail + # repository: tag: v1.5.1 - digest: sha256:0de05718b59dc33b57ddfb4d8ad5f637cefd13eafdec0e1579d782b3483c27c3 pullPolicy: IfNotPresent - # -- Provide a priority class name to the webhook patching job + ## Provide a priority class name to the webhook patching job ## priorityClassName: "" podAnnotations: {} - # NetworkPolicy for webhook patch - networkPolicy: - # -- Enable 'networkPolicy' or not - enabled: false - nodeSelector: - kubernetes.io/os: linux + nodeSelector: {} tolerations: [] - # -- Labels to be added to patch job resources - labels: {} - # -- Security context for secret creation & webhook patch pods - securityContext: {} - # -- Admission webhook patch job RBAC - rbac: - # -- Create RBAC or not - create: true - # -- Admission webhook patch job service account - serviceAccount: - # -- Create a service account or not - create: true - # -- Custom service account name - name: "" - # -- Auto-mount service account token or not - automountServiceAccountToken: true - # Use certmanager to generate webhook certs - certManager: - enabled: false - # self-signed root certificate - rootCert: - # default to be 5y - duration: "" - admissionCert: - # default to be 1y - duration: "" - # issuerRef: - # name: "issuer" - # kind: "ClusterIssuer" + runAsUser: 2000 + metrics: port: 10254 - portName: metrics # if this port is changed, change healthz-port: in extraArgs: accordingly enabled: false + service: - # -- Enable the metrics service or not. - enabled: true annotations: {} # prometheus.io/scrape: "true" # prometheus.io/port: "10254" - # -- Labels to be added to the metrics service resource - labels: {} + # clusterIP: "" - # -- List of IP addresses at which the stats-exporter service is available - ## Ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-ips + ## List of IP addresses at which the stats-exporter service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips ## externalIPs: [] + # loadBalancerIP: "" loadBalancerSourceRanges: [] servicePort: 10254 type: ClusterIP # externalTrafficPolicy: "" # nodePort: "" + serviceMonitor: enabled: false additionalLabels: {} - # -- Annotations to be added to the ServiceMonitor. - annotations: {} - ## The label to use to retrieve the job name from. - ## jobLabel: "app.kubernetes.io/name" + # The label to use to retrieve the job name from. + # jobLabel: "app.kubernetes.io/name" namespace: "" namespaceSelector: {} - ## Default: scrape .Release.Namespace or namespaceOverride only - ## To scrape all, use the following: - ## namespaceSelector: - ## any: true + # Default: scrape .Release.Namespace only + # To scrape all, use the following: + # namespaceSelector: + # any: true scrapeInterval: 30s # honorLabels: true targetLabels: [] - relabelings: [] metricRelabelings: [] - # -- Per-scrape limit on number of labels that will be accepted for a sample. - labelLimit: 0 - # -- Per-scrape limit on length of labels name that will be accepted for a sample. - labelNameLengthLimit: 0 - # -- Per-scrape limit on length of labels value that will be accepted for a sample. - labelValueLengthLimit: 0 - # -- Defines a per-scrape limit on the number of scraped samples that will be accepted. - sampleLimit: 0 - # -- Defines a limit on the number of scraped targets that will be accepted. - targetLimit: 0 + prometheusRule: enabled: false additionalLabels: {} - # -- Annotations to be added to the PrometheusRule. - annotations: {} # namespace: "" rules: [] - # # These are just examples rules, please adapt them to your needs - # - alert: NGINXConfigFailed - # expr: count(nginx_ingress_controller_config_last_reload_successful == 0) > 0 - # for: 1s - # labels: - # severity: critical - # annotations: - # description: bad ingress config - nginx config test failed - # summary: uninstall the latest ingress changes to allow config reloads to resume - # # By default a fake self-signed certificate is generated as default and - # # it is fine if it expires. If `--default-ssl-certificate` flag is used - # # and a valid certificate passed please do not filter for `host` label! - # # (i.e. delete `{host!="_"}` so also the default SSL certificate is - # # checked for expiration) - # - alert: NGINXCertificateExpiry - # expr: (avg(nginx_ingress_controller_ssl_expire_time_seconds{host!="_"}) by (host) - time()) < 604800 - # for: 1s - # labels: - # severity: critical - # annotations: - # description: ssl certificate(s) will expire in less then a week - # summary: renew expiring certificates to avoid downtime - # - alert: NGINXTooMany500s - # expr: 100 * ( sum( nginx_ingress_controller_requests{status=~"5.+"} ) / sum(nginx_ingress_controller_requests) ) > 5 - # for: 1m - # labels: - # severity: warning - # annotations: - # description: Too many 5XXs - # summary: More than 5% of all requests returned 5XX, this requires your attention - # - alert: NGINXTooMany400s - # expr: 100 * ( sum( nginx_ingress_controller_requests{status=~"4.+"} ) / sum(nginx_ingress_controller_requests) ) > 5 - # for: 1m - # labels: - # severity: warning - # annotations: - # description: Too many 4XXs - # summary: More than 5% of all requests returned 4XX, this requires your attention - # -- Improve connection draining when ingress controller pod is deleted using a lifecycle hook: - # With this new hook, we increased the default terminationGracePeriodSeconds from 30 seconds - # to 300, allowing the draining of connections up to five minutes. - # If the active connections end before that, the pod will terminate gracefully at that time. - # To effectively take advantage of this feature, the Configmap feature - # worker-shutdown-timeout new value is 240s instead of 10s. + # # These are just examples rules, please adapt them to your needs + # - alert: NGINXConfigFailed + # expr: count(nginx_ingress_controller_config_last_reload_successful == 0) > 0 + # for: 1s + # labels: + # severity: critical + # annotations: + # description: bad ingress config - nginx config test failed + # summary: uninstall the latest ingress changes to allow config reloads to resume + # - alert: NGINXCertificateExpiry + # expr: (avg(nginx_ingress_controller_ssl_expire_time_seconds) by (host) - time()) < 604800 + # for: 1s + # labels: + # severity: critical + # annotations: + # description: ssl certificate(s) will expire in less then a week + # summary: renew expiring certificates to avoid downtime + # - alert: NGINXTooMany500s + # expr: 100 * ( sum( nginx_ingress_controller_requests{status=~"5.+"} ) / sum(nginx_ingress_controller_requests) ) > 5 + # for: 1m + # labels: + # severity: warning + # annotations: + # description: Too many 5XXs + # summary: More than 5% of all requests returned 5XX, this requires your attention + # - alert: NGINXTooMany400s + # expr: 100 * ( sum( nginx_ingress_controller_requests{status=~"4.+"} ) / sum(nginx_ingress_controller_requests) ) > 5 + # for: 1m + # labels: + # severity: warning + # annotations: + # description: Too many 4XXs + # summary: More than 5% of all requests returned 4XX, this requires your attention + + ## Improve connection draining when ingress controller pod is deleted using a lifecycle hook: + ## With this new hook, we increased the default terminationGracePeriodSeconds from 30 seconds + ## to 300, allowing the draining of connections up to five minutes. + ## If the active connections end before that, the pod will terminate gracefully at that time. + ## To effectively take advantage of this feature, the Configmap feature + ## worker-shutdown-timeout new value is 240s instead of 10s. ## lifecycle: preStop: exec: command: - /wait-shutdown + priorityClassName: "" -# -- Rollback limit + +## Rollback limit ## revisionHistoryLimit: 10 + ## Default 404 backend ## defaultBackend: ## enabled: false + name: defaultbackend image: - # registry: registry.k8s.io + registry: k8s.gcr.io image: defaultbackend-amd64 - ## for backwards compatibility consider setting the full image url via the repository value below - ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail - ## repository: + # for backwards compatibility consider setting the full image url via the repository value below + # use *either* current default registry/image or repository format or installing chart by providing the values.yaml will fail + # repository: tag: "1.5" pullPolicy: IfNotPresent - runAsNonRoot: true # nobody user -> uid 65534 runAsUser: 65534 - runAsGroup: 65534 - allowPrivilegeEscalation: false - seccompProfile: - type: RuntimeDefault + runAsNonRoot: true readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + + # Use an existing PSP instead of creating one + existingPsp: "" + extraArgs: {} + serviceAccount: create: true name: "" automountServiceAccountToken: true - # -- Additional environment variables to set for defaultBackend pods + ## Additional environment variables to set for defaultBackend pods extraEnvs: [] + port: 8080 + ## Readiness and liveness probes for default backend ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ ## @@ -1018,17 +681,8 @@ defaultBackend: periodSeconds: 5 successThreshold: 1 timeoutSeconds: 5 - # -- The update strategy to apply to the Deployment or DaemonSet - ## - updateStrategy: {} - # rollingUpdate: - # maxUnavailable: 1 - # type: RollingUpdate - # -- `minReadySeconds` to avoid killing pods before we are ready - ## - minReadySeconds: 0 - # -- Node tolerations for server scheduling to nodes with taints + ## Node tolerations for server scheduling to nodes with taints ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ ## tolerations: [] @@ -1037,97 +691,31 @@ defaultBackend: # value: "value" # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" - # -- Affinity and anti-affinity rules for server scheduling to nodes - ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: {} - # # An example of preferred pod anti-affinity, weight is in the range 1-100 - # podAntiAffinity: - # preferredDuringSchedulingIgnoredDuringExecution: - # - weight: 100 - # podAffinityTerm: - # labelSelector: - # matchExpressions: - # - key: app.kubernetes.io/name - # operator: In - # values: - # - '{{ include "ingress-nginx.name" . }}' - # - key: app.kubernetes.io/instance - # operator: In - # values: - # - '{{ .Release.Name }}' - # - key: app.kubernetes.io/component - # operator: In - # values: - # - default-backend - # topologyKey: kubernetes.io/hostname - # # An example of required pod anti-affinity - # podAntiAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # - labelSelector: - # matchExpressions: - # - key: app.kubernetes.io/name - # operator: In - # values: - # - '{{ include "ingress-nginx.name" . }}' - # - key: app.kubernetes.io/instance - # operator: In - # values: - # - '{{ .Release.Name }}' - # - key: app.kubernetes.io/component - # operator: In - # values: - # - default-backend - # topologyKey: kubernetes.io/hostname - - # -- Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in. - # Ref.: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ - topologySpreadConstraints: [] - # - labelSelector: - # matchLabels: - # app.kubernetes.io/name: '{{ include "ingress-nginx.name" . }}' - # app.kubernetes.io/instance: '{{ .Release.Name }}' - # app.kubernetes.io/component: default-backend - # matchLabelKeys: - # - pod-template-hash - # topologyKey: topology.kubernetes.io/zone - # maxSkew: 1 - # whenUnsatisfiable: ScheduleAnyway - # - labelSelector: - # matchLabels: - # app.kubernetes.io/name: '{{ include "ingress-nginx.name" . }}' - # app.kubernetes.io/instance: '{{ .Release.Name }}' - # app.kubernetes.io/component: default-backend - # matchLabelKeys: - # - pod-template-hash - # topologyKey: kubernetes.io/hostname - # maxSkew: 1 - # whenUnsatisfiable: ScheduleAnyway - # -- Security context for default backend pods + ## Security Context policies for controller pods + ## See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for + ## notes on enabling and using sysctls + ## podSecurityContext: {} - # -- Security context for default backend containers - containerSecurityContext: {} - # -- Labels to add to the pod container metadata + + # labels to add to the pod container metadata podLabels: {} # key: value - # -- Node labels for default backend pod assignment - ## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/ + ## Node labels for default backend pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ ## - nodeSelector: - kubernetes.io/os: linux - # -- Annotations to be added to default backend pods + nodeSelector: {} + + ## Annotations to be added to default backend pods ## podAnnotations: {} + replicaCount: 1 - # -- Minimum available pods set in PodDisruptionBudget. - # Define either 'minAvailable' or 'maxUnavailable', never both. + minAvailable: 1 - # -- Maximum unavailable pods set in PodDisruptionBudget. If set, 'minAvailable' is ignored. - # maxUnavailable: 1 - # -- Eviction policy for unhealthy pods guarded by PodDisruptionBudget. - # Ref: https://kubernetes.io/blog/2023/01/06/unhealthy-pod-eviction-policy-for-pdbs/ - unhealthyPodEvictionPolicy: "" + resources: {} # limits: # cpu: 10m @@ -1146,21 +734,6 @@ defaultBackend: # - name: copy-portal-skins # emptyDir: {} - extraConfigMaps: [] - ## Additional configmaps to the default backend pod. - # - name: my-extra-configmap-1 - # labels: - # type: config-1 - # data: - # extra_file_1.html: | - # - # - name: my-extra-configmap-2 - # labels: - # type: config-2 - # data: - # extra_file_2.html: | - # - autoscaling: annotations: {} enabled: false @@ -1168,60 +741,57 @@ defaultBackend: maxReplicas: 2 targetCPUUtilizationPercentage: 50 targetMemoryUtilizationPercentage: 50 - # NetworkPolicy for default backend component. - networkPolicy: - # -- Enable 'networkPolicy' or not - enabled: false + service: annotations: {} - # clusterIP: "" - # -- Pre-defined cluster internal IP addresses of the default backend service. Take care of collisions with existing services. - # This value is immutable. Set once, it can not be changed without deleting and re-creating the service. - # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address - clusterIPs: [] - # -- List of IP addresses at which the default backend service is available - ## Ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-ips + # clusterIP: "" + + ## List of IP addresses at which the default backend service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips ## externalIPs: [] + # loadBalancerIP: "" loadBalancerSourceRanges: [] servicePort: 80 type: ClusterIP + priorityClassName: "" - # -- Labels to be added to the default backend resources - labels: {} -## Enable RBAC as per https://github.com/kubernetes/ingress-nginx/blob/main/docs/deploy/rbac.md and https://github.com/kubernetes/ingress-nginx/issues/266 + +## Enable RBAC as per https://github.com/kubernetes/ingress/tree/master/examples/rbac/nginx and https://github.com/kubernetes/ingress/issues/266 rbac: create: true scope: false + +# If true, create & use Pod Security Policy resources +# https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +podSecurityPolicy: + enabled: false + serviceAccount: create: true name: "" automountServiceAccountToken: true - # -- Annotations for the controller service account - annotations: {} -# -- Optional array of imagePullSecrets containing private registry credentials + +## Optional array of imagePullSecrets containing private registry credentials ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: [] # - name: secretName -# -- TCP service key-value pairs -## Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md +# TCP service key:value pairs +# Ref: https://github.com/kubernetes/contrib/tree/master/ingress/controllers/nginx/examples/tcp ## tcp: {} -# "8080": "default/example-tcp-svc:9000" +# 8080: "default/example-tcp-svc:9000" -# -- UDP service key-value pairs -## Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md +# UDP service key:value pairs +# Ref: https://github.com/kubernetes/contrib/tree/master/ingress/controllers/nginx/examples/udp ## udp: {} -# "53": "kube-system/kube-dns:53" +# 53: "kube-system/kube-dns:53" -# -- Prefix for TCP and UDP ports names in ingress controller service -## Some cloud providers, like Yandex Cloud may have a requirements for a port name regex to support cloud load balancer integration -portNamePrefix: "" -# -- (string) A base64-encoded Diffie-Hellman parameter. -# This can be generated with: `openssl dhparam 4096 2> /dev/null | base64` -## Ref: https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/ssl-dh-param -dhParam: "" +# A base64ed Diffie-Hellman parameter +# This can be generated with: openssl dhparam 4096 2> /dev/null | base64 +# Ref: https://github.com/krmichel/ingress-nginx/blob/master/docs/examples/customization/ssl-dh-param +dhParam: diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 0bb2b60a4..0c2854fa8 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,14 +1,25 @@ +# See https://cloud.google.com/cloud-build/docs/build-config + +timeout: 1800s options: - # Ignore Prow provided substitutions. substitution_option: ALLOW_LOOSE steps: - - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20250116-2a05ea7e3d - env: - - REGISTRY=us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx - - REPO_INFO=https://github.com/kubernetes/ingress-nginx - - COMMIT_SHA=${_PULL_BASE_SHA} - - BUILD_ID=${BUILD_ID} + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20201130-750d12f' entrypoint: bash + env: + - DOCKER_CLI_EXPERIMENTAL=enabled + - REGISTRY=gcr.io/k8s-staging-ingress-nginx + - REPO_INFO=https://github.com/kubernetes/ingress-nginx + - COMMIT_SHA=$_PULL_BASE_SHA + - BUILD_ID=$BUILD_ID + - HOME=/root + - USER=root args: - - -c - - gcloud auth configure-docker && make release + - -c + - | + gcloud auth configure-docker \ + && make release +substitutions: + _GIT_TAG: "12345" + _PULL_BASE_REF: "master" + _PULL_BASE_SHA: '12345' diff --git a/cmd/annotations/annotations.tmpl b/cmd/annotations/annotations.tmpl deleted file mode 100644 index 91dd21de8..000000000 --- a/cmd/annotations/annotations.tmpl +++ /dev/null @@ -1,7 +0,0 @@ -# Annotations Scope and Risk - -|Group |Annotation | Risk | Scope | -|--------|------------------|------|-------| -{{- range $doc := . }} -| {{ $doc.Group }} | {{ $doc.Annotation }} | {{ $doc.Risk }} | {{ $doc.Scope }} | -{{- end }} diff --git a/cmd/annotations/main.go b/cmd/annotations/main.go deleted file mode 100644 index 78f26099c..000000000 --- a/cmd/annotations/main.go +++ /dev/null @@ -1,91 +0,0 @@ -/* -Copyright 2024 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "bytes" - "embed" - "flag" - "fmt" - "os" - "slices" - "strings" - "text/template" - - anns "k8s.io/ingress-nginx/internal/ingress/annotations" -) - -type Documentation struct { - Group string - Annotation string - Risk string - Scope string -} - -var output string - -//go:embed annotations.tmpl -var content embed.FS - -func main() { - flag.StringVar(&output, "output", "", "where to write documentation") - flag.Parse() - if output == "" { - panic(fmt.Errorf("output field is required")) - } - docEntries := make([]Documentation, 0) - annotationFactory := anns.NewAnnotationFactory(nil) - for group, val := range annotationFactory { - annotations := val.GetDocumentation() - intermediateDocs := make([]Documentation, len(annotations)) - i := 0 - for annotation, values := range annotations { - doc := Documentation{ - Group: group, - Annotation: annotation, - Scope: string(values.Scope), - Risk: values.Risk.ToString(), - } - intermediateDocs[i] = doc - i++ - } - slices.SortStableFunc(intermediateDocs, func(a, b Documentation) int { - return strings.Compare(a.Annotation, b.Annotation) - }) - docEntries = append(docEntries, intermediateDocs...) - } - slices.SortStableFunc(docEntries, func(a, b Documentation) int { - return strings.Compare(a.Group, b.Group) - }) - - tmpl, err := template.New("annotations.tmpl").ParseFS(content, "annotations.tmpl") - if err != nil { - panic(fmt.Errorf("error parsing template: %s", err)) - } - - tplBuffer := new(bytes.Buffer) - err = tmpl.Execute(tplBuffer, docEntries) - if err != nil { - panic(err) - } - tplBuffer.WriteString("\n") - - //nolint:gosec // no need to check file permission here - if err := os.WriteFile(output, tplBuffer.Bytes(), 0o755); err != nil { - panic(err) - } -} diff --git a/cmd/dataplane/main.go b/cmd/dataplane/main.go deleted file mode 100644 index e7e4dc38f..000000000 --- a/cmd/dataplane/main.go +++ /dev/null @@ -1,96 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "net/http" - "os" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" - - "k8s.io/klog/v2" - - "k8s.io/ingress-nginx/internal/ingress/controller" - "k8s.io/ingress-nginx/internal/ingress/metric" - "k8s.io/ingress-nginx/internal/nginx" - ingressflags "k8s.io/ingress-nginx/pkg/flags" - "k8s.io/ingress-nginx/pkg/metrics" - "k8s.io/ingress-nginx/pkg/util/file" - "k8s.io/ingress-nginx/pkg/util/process" - "k8s.io/ingress-nginx/version" -) - -func main() { - klog.InitFlags(nil) - - fmt.Println(version.String()) - var err error - showVersion, conf, err := ingressflags.ParseFlags() - if showVersion { - os.Exit(0) - } - - if err != nil { - klog.Fatal(err) - } - - err = file.CreateRequiredDirectories() - if err != nil { - klog.Fatal(err) - } - - reg := prometheus.NewRegistry() - - reg.MustRegister(collectors.NewGoCollector()) - reg.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{ - PidFn: func() (int, error) { return os.Getpid(), nil }, - ReportErrors: true, - })) - - mc := metric.NewDummyCollector() - if conf.EnableMetrics { - // TODO: Ingress class is not a part of dataplane anymore - mc, err = metric.NewCollector(conf.MetricsPerHost, conf.MetricsPerUndefinedHost, conf.ReportStatusClasses, reg, conf.IngressClassConfiguration.Controller, *conf.MetricsBuckets, conf.MetricsBucketFactor, conf.MetricsMaxBuckets, conf.ExcludeSocketMetrics) - if err != nil { - klog.Fatalf("Error creating prometheus collector: %v", err) - } - } - // Pass the ValidationWebhook status to determine if we need to start the collector - // for the admissionWebhook - // TODO: Dataplane does not contain validation webhook so the MetricCollector should not receive - // this as an argument - mc.Start(conf.ValidationWebhook) - - if conf.EnableProfiling { - go metrics.RegisterProfiler(nginx.ProfilerAddress, nginx.ProfilerPort) - } - - ngx := controller.NewNGINXController(conf, mc) - - mux := http.NewServeMux() - metrics.RegisterHealthz(nginx.HealthPath, mux) - metrics.RegisterMetrics(reg, mux) - - go metrics.StartHTTPServer(conf.HealthCheckHost, conf.ListenPorts.Health, mux) - go ngx.Start() - - process.HandleSigterm(ngx, conf.PostShutdownGracePeriod, func(code int) { - os.Exit(code) - }) -} diff --git a/cmd/dbg/main.go b/cmd/dbg/main.go index ab68001c9..41a3c2772 100644 --- a/cmd/dbg/main.go +++ b/cmd/dbg/main.go @@ -47,7 +47,7 @@ func main() { backendsAllCmd := &cobra.Command{ Use: "all", Short: "Output the all dynamic backend information as a JSON array", - Run: func(_ *cobra.Command, _ []string) { + Run: func(cmd *cobra.Command, args []string) { backendsAll() }, } @@ -56,7 +56,7 @@ func main() { backendsListCmd := &cobra.Command{ Use: "list", Short: "Output a newline-separated list of the backend names", - Run: func(_ *cobra.Command, _ []string) { + Run: func(cmd *cobra.Command, args []string) { backendsList() }, } @@ -66,7 +66,7 @@ func main() { Use: "get [backend name]", Short: "Output the backend information only for the backend that has this name", Args: cobra.ExactArgs(1), - Run: func(_ *cobra.Command, args []string) { + Run: func(cmd *cobra.Command, args []string) { backendsGet(args[0]) }, } @@ -81,7 +81,7 @@ func main() { Use: "get [hostname]", Short: "Get the dynamically-loaded certificate information for the given hostname", Args: cobra.ExactArgs(1), - RunE: func(_ *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, args []string) error { certGet(args[0]) return nil }, @@ -93,7 +93,7 @@ func main() { generalCmd := &cobra.Command{ Use: "general", Short: "Output the general dynamic lua state", - Run: func(_ *cobra.Command, _ []string) { + Run: func(cmd *cobra.Command, args []string) { general() }, } @@ -102,7 +102,7 @@ func main() { confCmd := &cobra.Command{ Use: "conf", Short: "Dump the contents of /etc/nginx/nginx.conf", - Run: func(_ *cobra.Command, _ []string) { + Run: func(cmd *cobra.Command, args []string) { readNginxConf() }, } @@ -114,6 +114,7 @@ func main() { fmt.Println(err) os.Exit(1) } + } func backendsAll() { @@ -154,16 +155,10 @@ func backendsList() { fmt.Println(unmarshalErr) return } - backends, ok := f.([]interface{}) - if !ok { - fmt.Printf("unexpected type: %T", f) - } + backends := f.([]interface{}) for _, backendi := range backends { - backend, ok := backendi.(map[string]interface{}) - if !ok { - fmt.Printf("unexpected type: %T", backendi) - } + backend := backendi.(map[string]interface{}) fmt.Println(backend["name"].(string)) } } @@ -185,22 +180,12 @@ func backendsGet(name string) { fmt.Println(unmarshalErr) return } - backends, ok := f.([]interface{}) - if !ok { - fmt.Printf("unexpected type: %T", f) - } + backends := f.([]interface{}) for _, backendi := range backends { - backend, ok := backendi.(map[string]interface{}) - if !ok { - fmt.Printf("unexpected type: %T", backendi) - } + backend := backendi.(map[string]interface{}) if backend["name"].(string) == name { - printed, err := json.MarshalIndent(backend, "", " ") - if err != nil { - fmt.Println(err) - return - } + printed, _ := json.MarshalIndent(backend, "", " ") fmt.Println(string(printed)) return } @@ -228,7 +213,18 @@ func certGet(host string) { } func general() { - // TODO: refactor to obtain ingress-nginx pod count from the api server + //TODO: refactor to obtain ingress-nginx pod count from the api server + /* + statusCode, body, requestErr := nginx.NewGetStatusRequest(generalPath) + if requestErr != nil { + fmt.Println(requestErr) + return + } + if statusCode != 200 { + fmt.Printf("Nginx returned code %v\n", statusCode) + return + } + */ var prettyBuffer bytes.Buffer indentErr := json.Indent(&prettyBuffer, []byte("{}"), "", " ") diff --git a/pkg/flags/flags.go b/cmd/nginx/flags.go similarity index 51% rename from pkg/flags/flags.go rename to cmd/nginx/flags.go index ce24160fd..5fb63b430 100644 --- a/pkg/flags/flags.go +++ b/cmd/nginx/flags.go @@ -14,36 +14,29 @@ See the License for the specific language governing permissions and limitations under the License. */ -package flags +package main import ( - "errors" "flag" "fmt" - "net" "os" "time" - "github.com/prometheus/client_golang/prometheus" "github.com/spf13/pflag" + apiv1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" + "k8s.io/klog/v2" + + "k8s.io/ingress-nginx/internal/ingress/annotations/class" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/controller" ngx_config "k8s.io/ingress-nginx/internal/ingress/controller/config" - "k8s.io/ingress-nginx/internal/ingress/controller/ingressclass" - "k8s.io/ingress-nginx/internal/ingress/metric/collectors" "k8s.io/ingress-nginx/internal/ingress/status" ing_net "k8s.io/ingress-nginx/internal/net" "k8s.io/ingress-nginx/internal/nginx" - klog "k8s.io/klog/v2" ) -// TODO: We should split the flags functions between common for all programs -// and specific for each component (like webhook, controller and configurer) -// ParseFlags generates a configuration for Ingress Controller based on the flags -// provided by users -func ParseFlags() (bool, *controller.Configuration, error) { +func parseFlags() (bool, *controller.Configuration, error) { var ( flags = pflag.NewFlagSet("", pflag.ExitOnError) @@ -64,21 +57,10 @@ only when the flag --apiserver-host is specified.`) Takes the form "namespace/name". The controller configures NGINX to forward requests to the first port of this Service.`) - ingressClassAnnotation = flags.String("ingress-class", ingressclass.DefaultAnnotationValue, - `[IN DEPRECATION] Name of the ingress class this controller satisfies. -The class of an Ingress object is set using the annotation "kubernetes.io/ingress.class" (deprecated). -The parameter --controller-class has precedence over this.`) - - ingressClassController = flags.String("controller-class", ingressclass.DefaultControllerName, - `Ingress Class Controller value this Ingress satisfies. -The class of an Ingress object is set using the field IngressClassName in Kubernetes clusters version v1.19.0 or higher. The .spec.controller value of the IngressClass -referenced in an Ingress Object should be the same value specified here to make this object be watched.`) - - watchWithoutClass = flags.Bool("watch-ingress-without-class", false, - `Define if Ingress Controller should also watch for Ingresses without an IngressClass or the annotation specified.`) - - ingressClassByName = flags.Bool("ingress-class-by-name", false, - `Define if Ingress Controller should watch for Ingress Class by Name together with Controller Class.`) + ingressClass = flags.String("ingress-class", "", + `Name of the ingress class this controller satisfies. +The class of an Ingress object is set using the field IngressClassName in Kubernetes clusters version v1.18.0 or higher or the annotation "kubernetes.io/ingress.class" (deprecated). +If this parameter is not set, or set to the default value of "nginx", it will handle ingresses with either an empty or "nginx" class name.`) configMap = flags.String("configmap", "", `Name of the ConfigMap containing custom global configurations for the controller.`) @@ -109,11 +91,8 @@ either be a port name or number.`) This includes Ingresses, Services and all configuration resources. All namespaces are watched if this parameter is left empty.`) - watchNamespaceSelector = flags.String("watch-namespace-selector", "", - `Selector selects namespaces the controller watches for updates to Kubernetes objects.`) - profiling = flags.Bool("profiling", true, - `Enable profiling via web interface host:port/debug/pprof/ .`) + `Enable profiling via web interface host:port/debug/pprof/`) defSSLCertificate = flags.String("default-ssl-certificate", "", `Secret containing a SSL certificate to be used by the default HTTPS server (catch-all). @@ -133,9 +112,6 @@ Requires setting the publish-service parameter to a valid Service reference.`) electionID = flags.String("election-id", "ingress-controller-leader", `Election id to use for Ingress status updates.`) - electionTTL = flags.Duration("election-ttl", 30*time.Second, - `Duration a leader election is valid before it's getting re-elected`) - updateStatusOnShutdown = flags.Bool("update-status-on-shutdown", true, `Update the load-balancer status of Ingress objects when the controller shuts down. Requires the update-status parameter.`) @@ -150,18 +126,9 @@ Requires the update-status parameter.`) enableSSLPassthrough = flags.Bool("enable-ssl-passthrough", false, `Enable SSL Passthrough.`) - disableLeaderElection = flags.Bool("disable-leader-election", false, - `Disable Leader Election on NGINX Controller.`) - - disableServiceExternalName = flags.Bool("disable-svc-external-name", false, - `Disable support for Services of type ExternalName.`) - annotationsPrefix = flags.String("annotations-prefix", parser.DefaultAnnotationsPrefix, `Prefix of the Ingress annotations specific to the NGINX controller.`) - enableAnnotationValidation = flags.Bool("enable-annotation-validation", true, - `If true, will enable the annotation validation feature. Defaults to true`) - enableSSLChainCompletion = flags.Bool("enable-ssl-chain-completion", false, `Autocomplete SSL certificate chains with missing intermediate CA certificates. Certificates uploaded to Kubernetes must have the "Authority Information Access" X.509 v3 @@ -174,22 +141,11 @@ extension for this to succeed.`) `Customized address (or addresses, separated by comma) to set as the load-balancer status of Ingress objects this controller satisfies. Requires the update-status parameter.`) - enableMetrics = flags.Bool("enable-metrics", false, - `Enables the collection of NGINX metrics.`) + enableMetrics = flags.Bool("enable-metrics", true, + `Enables the collection of NGINX metrics`) metricsPerHost = flags.Bool("metrics-per-host", true, - `Export metrics per-host.`) - metricsPerUndefinedHost = flags.Bool("metrics-per-undefined-host", false, - `Export metrics per-host even if the host is not defined in an ingress. Requires --metrics-per-host to be set to true.`) - reportStatusClasses = flags.Bool("report-status-classes", false, - `Use status classes (2xx, 3xx, 4xx and 5xx) instead of status codes in metrics.`) - - timeBuckets = flags.Float64Slice("time-buckets", prometheus.DefBuckets, "Set of buckets which will be used for prometheus histogram metrics such as RequestTime, ResponseTime.") - lengthBuckets = flags.Float64Slice("length-buckets", prometheus.LinearBuckets(10, 10, 10), "Set of buckets which will be used for prometheus histogram metrics such as RequestLength, ResponseLength.") - sizeBuckets = flags.Float64Slice("size-buckets", prometheus.ExponentialBuckets(10, 10, 7), "Set of buckets which will be used for prometheus histogram metrics such as BytesSent.") - bucketFactor = flags.Float64("bucket-factor", 0, "Bucket factor for native histograms. Value must be > 1 for enabling native histograms.") - maxBuckets = flags.Uint32("max-buckets", 100, "Maximum number of buckets for native histograms.") - excludeSocketMetrics = flags.StringSlice("exclude-socket-metrics", []string{}, "et of socket request metrics to exclude which won't be exported nor being calculated. E.g. 'nginx_ingress_controller_success,nginx_ingress_controller_header_duration_seconds'.") - monitorMaxBatchSize = flags.Int("monitor-max-batch-size", 10000, "Max batch size of NGINX metrics.") + `Export metrics per-host`) + monitorMaxBatchSize = flags.Int("monitor-max-batch-size", 10000, "Max batch size of NGINX metrics") httpPort = flags.Int("http-port", 80, `Port to use for servicing HTTP traffic.`) httpsPort = flags.Int("https-port", 443, `Port to use for servicing HTTPS traffic.`) @@ -197,10 +153,9 @@ Requires the update-status parameter.`) sslProxyPort = flags.Int("ssl-passthrough-proxy-port", 442, `Port to use internally for SSL Passthrough.`) defServerPort = flags.Int("default-server-port", 8181, `Port to use for exposing the default server (catch-all).`) healthzPort = flags.Int("healthz-port", 10254, "Port to use for the healthz endpoint.") - healthzHost = flags.String("healthz-host", "", "Address to bind the healthz endpoint.") disableCatchAll = flags.Bool("disable-catch-all", false, - `Disable support for catch-all Ingresses.`) + `Disable support for catch-all Ingresses`) validationWebhook = flags.String("validating-webhook", "", `The address to start an admission controller on to validate incoming ingresses. @@ -209,43 +164,30 @@ Takes the form ":port". If not provided, no admission controller is starte `The path of the validating webhook certificate PEM.`) validationWebhookKey = flags.String("validating-webhook-key", "", `The path of the validating webhook key PEM.`) - disableFullValidationTest = flags.Bool("disable-full-test", false, - `Disable full test of all merged ingresses at the admission stage and tests the template of the ingress being created or updated (full test of all ingresses is enabled by default).`) statusPort = flags.Int("status-port", 10246, `Port to use for the lua HTTP endpoint configuration.`) streamPort = flags.Int("stream-port", 10247, "Port to use for the lua TCP/UDP endpoint configuration.") - internalLoggerAddress = flags.String("internal-logger-address", "127.0.0.1:11514", "Address to be used when binding internal syslogger.") - - profilerPort = flags.Int("profiler-port", 10245, "Port to use for expose the ingress controller Go profiler when it is enabled.") - profilerAddress = flags.IP("profiler-address", net.ParseIP("127.0.0.1"), "IP address used by the ingress controller to expose the Go Profiler when it is enabled.") + profilerPort = flags.Int("profiler-port", 10245, "Port to use for expose the ingress controller Go profiler when it is enabled.") statusUpdateInterval = flags.Int("status-update-interval", status.UpdateInterval, "Time interval in seconds in which the status should check if an update is required. Default is 60 seconds") shutdownGracePeriod = flags.Int("shutdown-grace-period", 0, "Seconds to wait after receiving the shutdown signal, before stopping the nginx process.") - - postShutdownGracePeriod = flags.Int("post-shutdown-grace-period", 10, "Seconds to wait after the nginx process has stopped before controller exits.") - - deepInspector = flags.Bool("deep-inspect", true, "Enables ingress object security deep inspector") - - dynamicConfigurationRetries = flags.Int("dynamic-configuration-retries", 15, "Number of times to retry failed dynamic configuration before failing to sync an ingress.") - - disableSyncEvents = flags.Bool("disable-sync-events", false, "Disables the creation of 'Sync' event resources") - - enableTopologyAwareRouting = flags.Bool("enable-topology-aware-routing", false, "Enable topology aware routing feature, needs service object annotation service.kubernetes.io/topology-mode sets to auto.") ) - flags.StringVar(&nginx.MaxmindMirror, "maxmind-mirror", "", `Maxmind mirror url (example: http://geoip.local/databases.`) + flags.StringVar(&nginx.MaxmindMirror, "maxmind-mirror", "", `Maxmind mirror url (example: http://geoip.local/databases`) flags.StringVar(&nginx.MaxmindLicenseKey, "maxmind-license-key", "", `Maxmind license key to download GeoLite2 Databases. -https://blog.maxmind.com/2019/12/significant-changes-to-accessing-and-using-geolite2-databases/ .`) +https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases`) flags.StringVar(&nginx.MaxmindEditionIDs, "maxmind-edition-ids", "GeoLite2-City,GeoLite2-ASN", `Maxmind edition ids to download GeoLite2 Databases.`) - flags.IntVar(&nginx.MaxmindRetriesCount, "maxmind-retries-count", 1, "Number of attempts to download the GeoIP DB.") - flags.DurationVar(&nginx.MaxmindRetriesTimeout, "maxmind-retries-timeout", time.Second*0, "Maxmind downloading delay between 1st and 2nd attempt, 0s - do not retry to download if something went wrong.") + + flag.Set("logtostderr", "true") flags.AddGoFlagSet(flag.CommandLine) - if err := flags.Parse(os.Args); err != nil { - return false, nil, err - } + flags.Parse(os.Args) + + // Workaround for this issue: + // https://github.com/kubernetes/kubernetes/issues/17162 + flag.CommandLine.Parse([]string{}) pflag.VisitAll(func(flag *pflag.Flag) { klog.V(2).InfoS("FLAG", flag.Name, flag.Value) @@ -262,8 +204,19 @@ https://blog.maxmind.com/2019/12/significant-changes-to-accessing-and-using-geol status.UpdateInterval = *statusUpdateInterval } + if *ingressClass != "" { + klog.InfoS("Watching for Ingress", "class", *ingressClass) + + if *ingressClass != class.DefaultClass { + klog.Warningf("Only Ingresses with class %q will be processed by this Ingress controller", *ingressClass) + } else { + klog.Warning("Ingresses with an empty class will also be processed by this Ingress controller") + } + + class.IngressClass = *ingressClass + } + parser.AnnotationsPrefix = *annotationsPrefix - parser.EnableAnnotationValidation = *enableAnnotationValidation // check port collisions if !ing_net.IsPortAvailable(*httpPort) { @@ -293,7 +246,6 @@ https://blog.maxmind.com/2019/12/significant-changes-to-accessing-and-using-geol nginx.StatusPort = *statusPort nginx.StreamPort = *streamPort nginx.ProfilerPort = *profilerPort - nginx.ProfilerAddress = profilerAddress.String() if *enableSSLPassthrough && !ing_net.IsPortAvailable(*sslProxyPort) { return false, nil, fmt.Errorf("port %v is already in use. Please check the flag --ssl-passthrough-proxy-port", *sslProxyPort) @@ -309,74 +261,31 @@ https://blog.maxmind.com/2019/12/significant-changes-to-accessing-and-using-geol nginx.HealthCheckTimeout = time.Duration(*defHealthCheckTimeout) * time.Second } - if *watchNamespace != "" && *watchNamespaceSelector != "" { - return false, nil, fmt.Errorf("flags --watch-namespace and --watch-namespace-selector are mutually exclusive") - } - - var namespaceSelector labels.Selector - if *watchNamespaceSelector != "" { - var err error - namespaceSelector, err = labels.Parse(*watchNamespaceSelector) - if err != nil { - return false, nil, fmt.Errorf("failed to parse --watch-namespace-selector=%s, error: %v", *watchNamespaceSelector, err) - } - } - - if *metricsPerUndefinedHost && !*metricsPerHost { - return false, nil, errors.New("--metrics-per-undefined-host=true must be passed with --metrics-per-host=true") - } - - if *electionTTL <= 0 { - *electionTTL = 30 * time.Second - } - - histogramBuckets := &collectors.HistogramBuckets{ - TimeBuckets: *timeBuckets, - LengthBuckets: *lengthBuckets, - SizeBuckets: *sizeBuckets, - } - ngx_config.EnableSSLChainCompletion = *enableSSLChainCompletion config := &controller.Configuration{ - APIServerHost: *apiserverHost, - KubeConfigFile: *kubeConfigFile, - UpdateStatus: *updateStatus, - ElectionID: *electionID, - ElectionTTL: *electionTTL, - EnableProfiling: *profiling, - EnableMetrics: *enableMetrics, - MetricsPerHost: *metricsPerHost, - MetricsPerUndefinedHost: *metricsPerUndefinedHost, - MetricsBuckets: histogramBuckets, - MetricsBucketFactor: *bucketFactor, - MetricsMaxBuckets: *maxBuckets, - ReportStatusClasses: *reportStatusClasses, - ExcludeSocketMetrics: *excludeSocketMetrics, - MonitorMaxBatchSize: *monitorMaxBatchSize, - DisableServiceExternalName: *disableServiceExternalName, - EnableSSLPassthrough: *enableSSLPassthrough, - DisableLeaderElection: *disableLeaderElection, - ResyncPeriod: *resyncPeriod, - DefaultService: *defaultSvc, - Namespace: *watchNamespace, - WatchNamespaceSelector: namespaceSelector, - ConfigMapName: *configMap, - TCPConfigMapName: *tcpConfigMapName, - UDPConfigMapName: *udpConfigMapName, - DisableFullValidationTest: *disableFullValidationTest, - DefaultSSLCertificate: *defSSLCertificate, - DeepInspector: *deepInspector, - PublishService: *publishSvc, - PublishStatusAddress: *publishStatusAddress, - UpdateStatusOnShutdown: *updateStatusOnShutdown, - ShutdownGracePeriod: *shutdownGracePeriod, - PostShutdownGracePeriod: *postShutdownGracePeriod, - UseNodeInternalIP: *useNodeInternalIP, - SyncRateLimit: *syncRateLimit, - HealthCheckHost: *healthzHost, - DynamicConfigurationRetries: *dynamicConfigurationRetries, - EnableTopologyAwareRouting: *enableTopologyAwareRouting, + APIServerHost: *apiserverHost, + KubeConfigFile: *kubeConfigFile, + UpdateStatus: *updateStatus, + ElectionID: *electionID, + EnableProfiling: *profiling, + EnableMetrics: *enableMetrics, + MetricsPerHost: *metricsPerHost, + MonitorMaxBatchSize: *monitorMaxBatchSize, + EnableSSLPassthrough: *enableSSLPassthrough, + ResyncPeriod: *resyncPeriod, + DefaultService: *defaultSvc, + Namespace: *watchNamespace, + ConfigMapName: *configMap, + TCPConfigMapName: *tcpConfigMapName, + UDPConfigMapName: *udpConfigMapName, + DefaultSSLCertificate: *defSSLCertificate, + PublishService: *publishSvc, + PublishStatusAddress: *publishStatusAddress, + UpdateStatusOnShutdown: *updateStatusOnShutdown, + ShutdownGracePeriod: *shutdownGracePeriod, + UseNodeInternalIP: *useNodeInternalIP, + SyncRateLimit: *syncRateLimit, ListenPorts: &ngx_config.ListenPorts{ Default: *defServerPort, Health: *healthzPort, @@ -384,46 +293,26 @@ https://blog.maxmind.com/2019/12/significant-changes-to-accessing-and-using-geol HTTPS: *httpsPort, SSLProxy: *sslProxyPort, }, - IngressClassConfiguration: &ingressclass.Configuration{ - Controller: *ingressClassController, - AnnotationValue: *ingressClassAnnotation, - WatchWithoutClass: *watchWithoutClass, - IngressClassByName: *ingressClassByName, - }, DisableCatchAll: *disableCatchAll, ValidationWebhook: *validationWebhook, ValidationWebhookCertPath: *validationWebhookCert, ValidationWebhookKeyPath: *validationWebhookKey, - InternalLoggerAddress: *internalLoggerAddress, - DisableSyncEvents: *disableSyncEvents, } if *apiserverHost != "" { config.RootCAFile = *rootCAFile } - var err error - if nginx.MaxmindEditionIDs != "" { + if (nginx.MaxmindLicenseKey != "" || nginx.MaxmindMirror != "") && nginx.MaxmindEditionIDs != "" { if err := nginx.ValidateGeoLite2DBEditions(); err != nil { return false, nil, err } - if nginx.MaxmindLicenseKey != "" || nginx.MaxmindMirror != "" { - klog.InfoS("downloading maxmind GeoIP2 databases") - if err = nginx.DownloadGeoLite2DB(nginx.MaxmindRetriesCount, nginx.MaxmindRetriesTimeout); err != nil { - klog.ErrorS(err, "unexpected error downloading GeoIP2 database") - } + klog.InfoS("downloading maxmind GeoIP2 databases") + if err := nginx.DownloadGeoLite2DB(); err != nil { + klog.ErrorS(err, "unexpected error downloading GeoIP2 database") } - config.MaxmindEditionFiles = &nginx.MaxmindEditionFiles + config.MaxmindEditionFiles = nginx.MaxmindEditionFiles } - return false, config, err -} - -// ResetForTesting clears all flag state and sets the usage function as directed. -// After calling resetForTesting, parse errors in flag handling will not -// exit the program. -// Extracted from https://github.com/golang/go/blob/master/src/flag/export_test.go -func ResetForTesting(usage func()) { - flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ContinueOnError) - flag.Usage = usage + return false, config, nil } diff --git a/cmd/nginx/flags_test.go b/cmd/nginx/flags_test.go new file mode 100644 index 000000000..4286d4a20 --- /dev/null +++ b/cmd/nginx/flags_test.go @@ -0,0 +1,107 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "os" + "testing" +) + +// resetForTesting clears all flag state and sets the usage function as directed. +// After calling resetForTesting, parse errors in flag handling will not +// exit the program. +// Extracted from https://github.com/golang/go/blob/master/src/flag/export_test.go +func resetForTesting(usage func()) { + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + flag.Usage = usage +} + +func TestNoMandatoryFlag(t *testing.T) { + _, _, err := parseFlags() + if err != nil { + t.Fatalf("Expected no error but got: %s", err) + } +} + +func TestDefaults(t *testing.T) { + resetForTesting(func() { t.Fatal("Parsing failed") }) + + oldArgs := os.Args + defer func() { os.Args = oldArgs }() + os.Args = []string{"cmd", + "--default-backend-service", "namespace/test", + "--http-port", "0", + "--https-port", "0", + } + + showVersion, conf, err := parseFlags() + if err != nil { + t.Fatalf("Unexpected error parsing default flags: %v", err) + } + + if showVersion { + t.Fatal("Expected flag \"show-version\" to be false") + } + + if conf == nil { + t.Fatal("Expected a controller Configuration") + } +} + +func TestSetupSSLProxy(t *testing.T) { + // TODO +} + +func TestFlagConflict(t *testing.T) { + resetForTesting(func() { t.Fatal("Parsing failed") }) + + oldArgs := os.Args + defer func() { os.Args = oldArgs }() + os.Args = []string{"cmd", "--publish-service", "namespace/test", "--http-port", "0", "--https-port", "0", "--publish-status-address", "1.1.1.1"} + + _, _, err := parseFlags() + if err == nil { + t.Fatalf("Expected an error parsing flags but none returned") + } +} + +func TestMaxmindEdition(t *testing.T) { + resetForTesting(func() { t.Fatal("Parsing failed") }) + + oldArgs := os.Args + defer func() { os.Args = oldArgs }() + os.Args = []string{"cmd", "--publish-service", "namespace/test", "--http-port", "0", "--https-port", "0", "--maxmind-license-key", "0000000", "--maxmind-edition-ids", "GeoLite2-City, TestCheck"} + + _, _, err := parseFlags() + if err == nil { + t.Fatalf("Expected an error parsing flags but none returned") + } +} + +func TestMaxmindMirror(t *testing.T) { + resetForTesting(func() { t.Fatal("Parsing failed") }) + + oldArgs := os.Args + defer func() { os.Args = oldArgs }() + os.Args = []string{"cmd", "--publish-service", "namespace/test", "--http-port", "0", "--https-port", "0", "--maxmind-mirror", "http://geoip.local", "--maxmind-license-key", "0000000", "--maxmind-edition-ids", "GeoLite2-City, TestCheck"} + + _, _, err := parseFlags() + if err == nil { + t.Fatalf("Expected an error parsing flags but none returned") + } +} diff --git a/cmd/nginx/logger.go b/cmd/nginx/logger.go deleted file mode 100644 index 8f0db0257..000000000 --- a/cmd/nginx/logger.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - - "k8s.io/klog/v2" - - "gopkg.in/mcuadros/go-syslog.v2" -) - -func logger(address string) { - channel := make(syslog.LogPartsChannel) - handler := syslog.NewChannelHandler(channel) - - server := syslog.NewServer() - - server.SetFormat(syslog.RFC3164) - server.SetHandler(handler) - if err := server.ListenUDP(address); err != nil { - klog.Fatalf("failed bind internal syslog: %s", err.Error()) - } - - if err := server.Boot(); err != nil { - klog.Fatalf("failed to boot internal syslog: %s", err.Error()) - } - klog.Infof("Is Chrooted, starting logger") - - for logParts := range channel { - fmt.Printf("%s\n", logParts["content"]) - } - - server.Wait() - klog.Infof("Stopping logger") -} diff --git a/cmd/nginx/main.go b/cmd/nginx/main.go index 781f3a8eb..9c245baa1 100644 --- a/cmd/nginx/main.go +++ b/cmd/nginx/main.go @@ -19,43 +19,47 @@ package main import ( "context" "fmt" + "math/rand" // #nosec "net/http" + "net/http/pprof" "os" + "os/signal" "path/filepath" "runtime" + "syscall" "time" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" discovery "k8s.io/apimachinery/pkg/version" + "k8s.io/apiserver/pkg/server/healthz" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" certutil "k8s.io/client-go/util/cert" "k8s.io/klog/v2" + "k8s.io/ingress-nginx/internal/file" + "k8s.io/ingress-nginx/internal/ingress/annotations/class" "k8s.io/ingress-nginx/internal/ingress/controller" "k8s.io/ingress-nginx/internal/ingress/metric" "k8s.io/ingress-nginx/internal/k8s" "k8s.io/ingress-nginx/internal/net/ssl" "k8s.io/ingress-nginx/internal/nginx" - "k8s.io/ingress-nginx/pkg/util/file" "k8s.io/ingress-nginx/version" - - ingressflags "k8s.io/ingress-nginx/pkg/flags" - "k8s.io/ingress-nginx/pkg/metrics" - "k8s.io/ingress-nginx/pkg/util/process" ) func main() { klog.InitFlags(nil) + rand.Seed(time.Now().UnixNano()) + fmt.Println(version.String()) - showVersion, conf, err := ingressflags.ParseFlags() + showVersion, conf, err := parseFlags() if showVersion { os.Exit(0) } @@ -74,7 +78,7 @@ func main() { handleFatalInitError(err) } - if conf.DefaultService != "" { + if len(conf.DefaultService) > 0 { err := checkService(conf.DefaultService, kubeClient) if err != nil { klog.Fatal(err) @@ -83,7 +87,7 @@ func main() { klog.InfoS("Valid default backend", "service", conf.DefaultService) } - if conf.PublishService != "" { + if len(conf.PublishService) > 0 { err := checkService(conf.PublishService, kubeClient) if err != nil { klog.Fatal(err) @@ -100,19 +104,38 @@ func main() { conf.FakeCertificate = ssl.GetFakeSSLCert() klog.InfoS("SSL fake certificate created", "file", conf.FakeCertificate.PemFileName) - if !k8s.NetworkingIngressAvailable(kubeClient) { - klog.Fatalf("ingress-nginx requires Kubernetes v1.19.0 or higher") + var isNetworkingIngressAvailable bool + + isNetworkingIngressAvailable, k8s.IsIngressV1Beta1Ready, _ = k8s.NetworkingIngressAvailable(kubeClient) + if !isNetworkingIngressAvailable { + klog.Fatalf("ingress-nginx requires Kubernetes v1.14.0 or higher") } - _, err = kubeClient.NetworkingV1().IngressClasses().List(context.TODO(), metav1.ListOptions{}) - if err != nil { - if !errors.IsNotFound(err) { - if errors.IsForbidden(err) { - klog.Warningf("No permissions to list and get Ingress Classes: %v, IngressClass feature will be disabled", err) - conf.IngressClassConfiguration.IgnoreIngressClass = true + if k8s.IsIngressV1Beta1Ready { + klog.InfoS("Enabling new Ingress features available since Kubernetes v1.18") + k8s.IngressClass, err = kubeClient.NetworkingV1beta1().IngressClasses(). + Get(context.TODO(), class.IngressClass, metav1.GetOptions{}) + if err != nil { + if !errors.IsNotFound(err) { + if !errors.IsUnauthorized(err) && !errors.IsForbidden(err) { + klog.Fatalf("Error searching IngressClass: %v", err) + } + + klog.ErrorS(err, "Searching IngressClass", "class", class.IngressClass) } + + klog.Warningf("No IngressClass resource with name %v found. Only annotation will be used.", class.IngressClass) + + // TODO: remove once this is fixed in client-go + k8s.IngressClass = nil + } + + if k8s.IngressClass != nil && k8s.IngressClass.Spec.Controller != k8s.IngressNGINXController { + klog.Errorf(`Invalid IngressClass (Spec.Controller) value "%v". Should be "%v"`, k8s.IngressClass.Spec.Controller, k8s.IngressNGINXController) + klog.Fatalf("IngressClass with name %v is not valid for ingress-nginx (invalid Spec.Controller)", class.IngressClass) } } + conf.Client = kubeClient err = k8s.GetIngressPod(kubeClient) @@ -122,47 +145,60 @@ func main() { reg := prometheus.NewRegistry() - reg.MustRegister(collectors.NewGoCollector()) - reg.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{ + reg.MustRegister(prometheus.NewGoCollector()) + reg.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{ PidFn: func() (int, error) { return os.Getpid(), nil }, ReportErrors: true, })) mc := metric.NewDummyCollector() if conf.EnableMetrics { - mc, err = metric.NewCollector(conf.MetricsPerHost, conf.MetricsPerUndefinedHost, conf.ReportStatusClasses, reg, conf.IngressClassConfiguration.Controller, *conf.MetricsBuckets, conf.MetricsBucketFactor, conf.MetricsMaxBuckets, conf.ExcludeSocketMetrics) + mc, err = metric.NewCollector(conf.MetricsPerHost, reg) if err != nil { klog.Fatalf("Error creating prometheus collector: %v", err) } } - // Pass the ValidationWebhook status to determine if we need to start the collector - // for the admissionWebhook - mc.Start(conf.ValidationWebhook) + mc.Start() if conf.EnableProfiling { - go metrics.RegisterProfiler(nginx.ProfilerAddress, nginx.ProfilerPort) + go registerProfiler() } ngx := controller.NewNGINXController(conf, mc) mux := http.NewServeMux() - metrics.RegisterHealthz(nginx.HealthPath, mux, ngx) - metrics.RegisterMetrics(reg, mux) + registerHealthz(nginx.HealthPath, ngx, mux) + registerMetrics(reg, mux) - _, errExists := os.Stat("/chroot") - if errExists == nil { - conf.IsChroot = true - go logger(conf.InternalLoggerAddress) - } - - go metrics.StartHTTPServer(conf.HealthCheckHost, conf.ListenPorts.Health, mux) + go startHTTPServer(conf.ListenPorts.Health, mux) go ngx.Start() - process.HandleSigterm(ngx, conf.PostShutdownGracePeriod, func(code int) { + handleSigterm(ngx, func(code int) { os.Exit(code) }) } +type exiter func(code int) + +func handleSigterm(ngx *controller.NGINXController, exit exiter) { + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGTERM) + <-signalChan + klog.InfoS("Received SIGTERM, shutting down") + + exitCode := 0 + if err := ngx.Stop(); err != nil { + klog.Warningf("Error during shutdown: %v", err) + exitCode = 1 + } + + klog.InfoS("Handled quit, awaiting Pod deletion") + time.Sleep(10 * time.Second) + + klog.InfoS("Exiting", "code", exitCode) + exit(exitCode) +} + // createApiserverClient creates a new Kubernetes REST client. apiserverHost is // the URL of the API server in the format protocol://address:port/pathPrefix, // kubeConfig is the location of a kubeconfig file. If defined, the kubeconfig @@ -235,6 +271,7 @@ func createApiserverClient(apiserverHost, rootCAFile, kubeConfig string) (*kuber retries++ return false, nil }) + // err is returned in case of timeout in the exponential backoff (ErrWaitTimeout) if err != nil { return nil, lastErr @@ -267,6 +304,58 @@ func handleFatalInitError(err error) { err) } +func registerHealthz(healthPath string, ic *controller.NGINXController, mux *http.ServeMux) { + // expose health check endpoint (/healthz) + healthz.InstallPathHandler(mux, + healthPath, + healthz.PingHealthz, + ic, + ) +} + +func registerMetrics(reg *prometheus.Registry, mux *http.ServeMux) { + mux.Handle( + "/metrics", + promhttp.InstrumentMetricHandler( + reg, + promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), + ), + ) +} + +func registerProfiler() { + mux := http.NewServeMux() + + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/heap", pprof.Index) + mux.HandleFunc("/debug/pprof/mutex", pprof.Index) + mux.HandleFunc("/debug/pprof/goroutine", pprof.Index) + mux.HandleFunc("/debug/pprof/threadcreate", pprof.Index) + mux.HandleFunc("/debug/pprof/block", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + + server := &http.Server{ + Addr: fmt.Sprintf("127.0.0.1:%v", nginx.ProfilerPort), + Handler: mux, + } + klog.Fatal(server.ListenAndServe()) +} + +func startHTTPServer(port int, mux *http.ServeMux) { + server := &http.Server{ + Addr: fmt.Sprintf(":%v", port), + Handler: mux, + ReadTimeout: 10 * time.Second, + ReadHeaderTimeout: 10 * time.Second, + WriteTimeout: 300 * time.Second, + IdleTimeout: 120 * time.Second, + } + klog.Fatal(server.ListenAndServe()) +} + func checkService(key string, kubeClient *kubernetes.Clientset) error { ns, name, err := k8s.ParseNameNS(key) if err != nil { @@ -280,10 +369,10 @@ func checkService(key string, kubeClient *kubernetes.Clientset) error { } if errors.IsNotFound(err) { - return fmt.Errorf("no service with name %v found in namespace %v: %v", name, ns, err) + return fmt.Errorf("No service with name %v found in namespace %v: %v", name, ns, err) } - return fmt.Errorf("unexpected error searching service with name %v in namespace %v: %v", name, ns, err) + return fmt.Errorf("Unexpected error searching service with name %v in namespace %v: %v", name, ns, err) } return nil diff --git a/cmd/nginx/main_test.go b/cmd/nginx/main_test.go index 13f1e9eec..e6d24b301 100644 --- a/cmd/nginx/main_test.go +++ b/cmd/nginx/main_test.go @@ -33,8 +33,6 @@ import ( "k8s.io/ingress-nginx/internal/ingress/controller" "k8s.io/ingress-nginx/internal/k8s" "k8s.io/ingress-nginx/internal/nginx" - ingressflags "k8s.io/ingress-nginx/pkg/flags" - "k8s.io/ingress-nginx/pkg/util/process" ) func TestCreateApiserverClient(t *testing.T) { @@ -47,7 +45,7 @@ func TestCreateApiserverClient(t *testing.T) { func init() { // the default value of nginx.TemplatePath assumes the template exists in // the root filesystem and not in the rootfs directory - path, err := filepath.Abs(filepath.Join("..", "..", "rootfs", nginx.TemplatePath)) + path, err := filepath.Abs(filepath.Join("../../rootfs/", nginx.TemplatePath)) if err == nil { nginx.TemplatePath = path } @@ -85,21 +83,21 @@ func TestHandleSigterm(t *testing.T) { t.Fatalf("error creating pod %v: %v", pod, err) } - ingressflags.ResetForTesting(func() { t.Fatal("bad parse") }) + resetForTesting(func() { t.Fatal("bad parse") }) - t.Setenv("POD_NAME", podName) - t.Setenv("POD_NAMESPACE", namespace) + os.Setenv("POD_NAME", podName) + os.Setenv("POD_NAMESPACE", namespace) oldArgs := os.Args defer func() { - t.Setenv("POD_NAME", "") - t.Setenv("POD_NAMESPACE", "") + os.Setenv("POD_NAME", "") + os.Setenv("POD_NAMESPACE", "") os.Args = oldArgs }() os.Args = []string{"cmd", "--default-backend-service", "ingress-nginx/default-backend-http", "--http-port", "0", "--https-port", "0"} - _, conf, err := ingressflags.ParseFlags() + _, conf, err := parseFlags() if err != nil { t.Errorf("Unexpected error creating NGINX controller: %v", err) } @@ -107,7 +105,7 @@ func TestHandleSigterm(t *testing.T) { ngx := controller.NewNGINXController(conf, nil) - go process.HandleSigterm(ngx, 10, func(code int) { + go handleSigterm(ngx, func(code int) { if code != 1 { t.Errorf("Expected exit code 1 but %d received", code) } diff --git a/cmd/plugin/commands/backends/backends.go b/cmd/plugin/commands/backends/backends.go index e7bd42147..341c62a9c 100644 --- a/cmd/plugin/commands/backends/backends.go +++ b/cmd/plugin/commands/backends/backends.go @@ -30,11 +30,11 @@ import ( // CreateCommand creates and returns this cobra subcommand func CreateCommand(flags *genericclioptions.ConfigFlags) *cobra.Command { - var pod, deployment, selector, container *string + var pod, deployment, selector *string cmd := &cobra.Command{ Use: "backends", Short: "Inspect the dynamic backend information of an ingress-nginx instance", - RunE: func(cmd *cobra.Command, _ []string) error { + RunE: func(cmd *cobra.Command, args []string) error { backend, err := cmd.Flags().GetString("backend") if err != nil { return err @@ -47,7 +47,7 @@ func CreateCommand(flags *genericclioptions.ConfigFlags) *cobra.Command { return fmt.Errorf("--list and --backend cannot both be specified") } - util.PrintError(backends(flags, *pod, *deployment, *selector, *container, backend, onlyList)) + util.PrintError(backends(flags, *pod, *deployment, *selector, backend, onlyList)) return nil }, } @@ -55,7 +55,6 @@ func CreateCommand(flags *genericclioptions.ConfigFlags) *cobra.Command { pod = util.AddPodFlag(cmd) deployment = util.AddDeploymentFlag(cmd) selector = util.AddSelectorFlag(cmd) - container = util.AddContainerFlag(cmd) cmd.Flags().String("backend", "", "Output only the information for the given backend") cmd.Flags().Bool("list", false, "Output a newline-separated list of backend names") @@ -63,14 +62,13 @@ func CreateCommand(flags *genericclioptions.ConfigFlags) *cobra.Command { return cmd } -func backends(flags *genericclioptions.ConfigFlags, podName, deployment, selector, container, backend string, onlyList bool) error { +func backends(flags *genericclioptions.ConfigFlags, podName string, deployment string, selector string, backend string, onlyList bool) error { var command []string - switch { - case onlyList: + if onlyList { command = []string{"/dbg", "backends", "list"} - case backend != "": + } else if backend != "" { command = []string{"/dbg", "backends", "get", backend} - default: + } else { command = []string{"/dbg", "backends", "all"} } @@ -79,7 +77,7 @@ func backends(flags *genericclioptions.ConfigFlags, podName, deployment, selecto return err } - out, err := kubectl.PodExecString(flags, &pod, container, command) + out, err := kubectl.PodExecString(flags, &pod, command) if err != nil { return err } diff --git a/cmd/plugin/commands/certs/certs.go b/cmd/plugin/commands/certs/certs.go index 4a7fee1cf..07fd08ad3 100644 --- a/cmd/plugin/commands/certs/certs.go +++ b/cmd/plugin/commands/certs/certs.go @@ -18,7 +18,6 @@ package certs import ( "fmt" - "os" "github.com/spf13/cobra" @@ -31,35 +30,31 @@ import ( // CreateCommand creates and returns this cobra subcommand func CreateCommand(flags *genericclioptions.ConfigFlags) *cobra.Command { - var pod, deployment, selector, container *string + var pod, deployment, selector *string cmd := &cobra.Command{ Use: "certs", Short: "Output the certificate data stored in an ingress-nginx pod", - RunE: func(cmd *cobra.Command, _ []string) error { + RunE: func(cmd *cobra.Command, args []string) error { host, err := cmd.Flags().GetString("host") if err != nil { return err } - util.PrintError(certs(flags, *pod, *deployment, *selector, *container, host)) + util.PrintError(certs(flags, *pod, *deployment, *selector, host)) return nil }, } cmd.Flags().String("host", "", "Get the cert for this hostname") - if err := cobra.MarkFlagRequired(cmd.Flags(), "host"); err != nil { - util.PrintError(err) - os.Exit(1) - } + cobra.MarkFlagRequired(cmd.Flags(), "host") pod = util.AddPodFlag(cmd) deployment = util.AddDeploymentFlag(cmd) selector = util.AddSelectorFlag(cmd) - container = util.AddContainerFlag(cmd) return cmd } -func certs(flags *genericclioptions.ConfigFlags, podName, deployment, selector, container, host string) error { +func certs(flags *genericclioptions.ConfigFlags, podName string, deployment string, selector string, host string) error { command := []string{"/dbg", "certs", "get", host} pod, err := request.ChoosePod(flags, podName, deployment, selector) @@ -67,7 +62,7 @@ func certs(flags *genericclioptions.ConfigFlags, podName, deployment, selector, return err } - out, err := kubectl.PodExecString(flags, &pod, container, command) + out, err := kubectl.PodExecString(flags, &pod, command) if err != nil { return err } diff --git a/cmd/plugin/commands/conf/conf.go b/cmd/plugin/commands/conf/conf.go index 5821b11f1..5caa2a649 100644 --- a/cmd/plugin/commands/conf/conf.go +++ b/cmd/plugin/commands/conf/conf.go @@ -32,17 +32,17 @@ import ( // CreateCommand creates and returns this cobra subcommand func CreateCommand(flags *genericclioptions.ConfigFlags) *cobra.Command { - var pod, deployment, selector, container *string + var pod, deployment, selector *string cmd := &cobra.Command{ Use: "conf", Short: "Inspect the generated nginx.conf", - RunE: func(cmd *cobra.Command, _ []string) error { + RunE: func(cmd *cobra.Command, args []string) error { host, err := cmd.Flags().GetString("host") if err != nil { return err } - util.PrintError(conf(flags, host, *pod, *deployment, *selector, *container)) + util.PrintError(conf(flags, host, *pod, *deployment, *selector)) return nil }, } @@ -50,18 +50,17 @@ func CreateCommand(flags *genericclioptions.ConfigFlags) *cobra.Command { pod = util.AddPodFlag(cmd) deployment = util.AddDeploymentFlag(cmd) selector = util.AddSelectorFlag(cmd) - container = util.AddContainerFlag(cmd) return cmd } -func conf(flags *genericclioptions.ConfigFlags, host, podName, deployment, selector, container string) error { +func conf(flags *genericclioptions.ConfigFlags, host string, podName string, deployment string, selector string) error { pod, err := request.ChoosePod(flags, podName, deployment, selector) if err != nil { return err } - nginxConf, err := kubectl.PodExecString(flags, &pod, container, []string{"/dbg", "conf"}) + nginxConf, err := kubectl.PodExecString(flags, &pod, []string{"/dbg", "conf"}) if err != nil { return err } diff --git a/cmd/plugin/commands/exec/exec.go b/cmd/plugin/commands/exec/exec.go index a91bdb594..5f1a31913 100644 --- a/cmd/plugin/commands/exec/exec.go +++ b/cmd/plugin/commands/exec/exec.go @@ -29,21 +29,19 @@ import ( // CreateCommand creates and returns this cobra subcommand func CreateCommand(flags *genericclioptions.ConfigFlags) *cobra.Command { opts := execFlags{} - var pod, deployment, selector, container *string + var pod, deployment, selector *string cmd := &cobra.Command{ Use: "exec", Short: "Execute a command inside an ingress-nginx pod", - RunE: func(_ *cobra.Command, args []string) error { - util.PrintError(exec(flags, *pod, *deployment, *selector, *container, args, opts)) + RunE: func(cmd *cobra.Command, args []string) error { + util.PrintError(exec(flags, *pod, *deployment, *selector, args, opts)) return nil }, } pod = util.AddPodFlag(cmd) deployment = util.AddDeploymentFlag(cmd) selector = util.AddSelectorFlag(cmd) - container = util.AddContainerFlag(cmd) - cmd.Flags().BoolVarP(&opts.TTY, "tty", "t", false, "Stdin is a TTY") cmd.Flags().BoolVarP(&opts.Stdin, "stdin", "i", false, "Pass stdin to the container") @@ -55,7 +53,7 @@ type execFlags struct { Stdin bool } -func exec(flags *genericclioptions.ConfigFlags, podName, deployment, selector, container string, cmd []string, opts execFlags) error { +func exec(flags *genericclioptions.ConfigFlags, podName string, deployment string, selector string, cmd []string, opts execFlags) error { pod, err := request.ChoosePod(flags, podName, deployment, selector) if err != nil { return err @@ -69,7 +67,7 @@ func exec(flags *genericclioptions.ConfigFlags, podName, deployment, selector, c args = append(args, "-i") } - args = append(args, []string{"-n", pod.Namespace, "-c", container, pod.Name, "--"}...) + args = append(args, []string{"-n", pod.Namespace, pod.Name, "--"}...) args = append(args, cmd...) return kubectl.Exec(flags, args) } diff --git a/cmd/plugin/commands/general/general.go b/cmd/plugin/commands/general/general.go index d7b3b34f9..44e02ca88 100644 --- a/cmd/plugin/commands/general/general.go +++ b/cmd/plugin/commands/general/general.go @@ -30,30 +30,29 @@ import ( // CreateCommand creates and returns this cobra subcommand func CreateCommand(flags *genericclioptions.ConfigFlags) *cobra.Command { - var pod, deployment, selector, container *string + var pod, deployment, selector *string cmd := &cobra.Command{ Use: "general", Short: "Inspect the other dynamic ingress-nginx information", - RunE: func(_ *cobra.Command, _ []string) error { - util.PrintError(general(flags, *pod, *deployment, *selector, *container)) + RunE: func(cmd *cobra.Command, args []string) error { + util.PrintError(general(flags, *pod, *deployment, *selector)) return nil }, } pod = util.AddPodFlag(cmd) deployment = util.AddDeploymentFlag(cmd) selector = util.AddSelectorFlag(cmd) - container = util.AddContainerFlag(cmd) return cmd } -func general(flags *genericclioptions.ConfigFlags, podName, deployment, selector, container string) error { +func general(flags *genericclioptions.ConfigFlags, podName string, deployment string, selector string) error { pod, err := request.ChoosePod(flags, podName, deployment, selector) if err != nil { return err } - out, err := kubectl.PodExecString(flags, &pod, container, []string{"/dbg", "general"}) + out, err := kubectl.PodExecString(flags, &pod, []string{"/dbg", "general"}) if err != nil { return err } diff --git a/cmd/plugin/commands/info/info.go b/cmd/plugin/commands/info/info.go index e0fd3aaec..246046c3a 100644 --- a/cmd/plugin/commands/info/info.go +++ b/cmd/plugin/commands/info/info.go @@ -32,7 +32,7 @@ func CreateCommand(flags *genericclioptions.ConfigFlags) *cobra.Command { cmd := &cobra.Command{ Use: "info", Short: "Show information about the ingress-nginx service", - RunE: func(cmd *cobra.Command, _ []string) error { + RunE: func(cmd *cobra.Command, args []string) error { service, err := cmd.Flags().GetString("service") if err != nil { return err diff --git a/cmd/plugin/commands/ingresses/ingresses.go b/cmd/plugin/commands/ingresses/ingresses.go index 7cb90a570..38da62930 100644 --- a/cmd/plugin/commands/ingresses/ingresses.go +++ b/cmd/plugin/commands/ingresses/ingresses.go @@ -22,8 +22,7 @@ import ( "text/tabwriter" "github.com/spf13/cobra" - networking "k8s.io/api/networking/v1" - "k8s.io/apimachinery/pkg/util/intstr" + networking "k8s.io/api/networking/v1beta1" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/ingress-nginx/cmd/plugin/request" @@ -36,7 +35,7 @@ func CreateCommand(flags *genericclioptions.ConfigFlags) *cobra.Command { Use: "ingresses", Aliases: []string{"ingress", "ing"}, Short: "Provide a short summary of all of the ingress definitions", - RunE: func(cmd *cobra.Command, _ []string) error { + RunE: func(cmd *cobra.Command, args []string) error { host, err := cmd.Flags().GetString("host") if err != nil { return err @@ -74,9 +73,9 @@ func ingresses(flags *genericclioptions.ConfigFlags, host string, allNamespaces if host != "" { rowsWithHost := make([]ingressRow, 0) - for i := range rows { - if rows[i].Host == host { - rowsWithHost = append(rowsWithHost, rows[i]) + for _, row := range rows { + if row.Host == host { + rowsWithHost = append(rowsWithHost, row) } } rows = rowsWithHost @@ -91,8 +90,7 @@ func ingresses(flags *genericclioptions.ConfigFlags, host string, allNamespaces fmt.Fprintln(printer, "INGRESS NAME\tHOST+PATH\tADDRESSES\tTLS\tSERVICE\tSERVICE PORT\tENDPOINTS") } - for i := range rows { - row := &rows[i] + for _, row := range rows { var tlsMsg string if row.TLS { tlsMsg = "YES" @@ -135,18 +133,18 @@ type ingressRow struct { func getIngressRows(ingresses *[]networking.Ingress) []ingressRow { rows := make([]ingressRow, 0) - for i := range *ingresses { - ing := &(*ingresses)[i] + for _, ing := range *ingresses { + address := "" for _, lbIng := range ing.Status.LoadBalancer.Ingress { - if lbIng.IP != "" { + if len(lbIng.IP) > 0 { address = address + lbIng.IP + "," } - if lbIng.Hostname != "" { + if len(lbIng.Hostname) > 0 { address = address + lbIng.Hostname + "," } } - if address != "" { + if len(address) > 0 { address = address[:len(address)-1] } @@ -159,14 +157,13 @@ func getIngressRows(ingresses *[]networking.Ingress) []ingressRow { defaultBackendService := "" defaultBackendPort := "" - if ing.Spec.DefaultBackend != nil { - name, port := serviceToNameAndPort(ing.Spec.DefaultBackend.Service) - defaultBackendService = name - defaultBackendPort = port.String() + if ing.Spec.Backend != nil { + defaultBackendService = ing.Spec.Backend.ServiceName + defaultBackendPort = ing.Spec.Backend.ServicePort.String() } // Handle catch-all ingress - if len(ing.Spec.Rules) == 0 && defaultBackendService != "" { + if len(ing.Spec.Rules) == 0 && len(defaultBackendService) > 0 { row := ingressRow{ Namespace: ing.Namespace, IngressName: ing.Name, @@ -183,7 +180,7 @@ func getIngressRows(ingresses *[]networking.Ingress) []ingressRow { for _, rule := range ing.Spec.Rules { _, hasTLS := tlsHosts[rule.Host] - // Handle ingress with no paths + //Handle ingress with no paths if rule.HTTP == nil { row := ingressRow{ Namespace: ing.Namespace, @@ -200,15 +197,14 @@ func getIngressRows(ingresses *[]networking.Ingress) []ingressRow { } for _, path := range rule.HTTP.Paths { - svcName, svcPort := serviceToNameAndPort(path.Backend.Service) row := ingressRow{ Namespace: ing.Namespace, IngressName: ing.Name, Host: rule.Host, Path: path.Path, TLS: hasTLS, - ServiceName: svcName, - ServicePort: svcPort.String(), + ServiceName: path.Backend.ServiceName, + ServicePort: path.Backend.ServicePort.String(), Address: address, } @@ -219,17 +215,3 @@ func getIngressRows(ingresses *[]networking.Ingress) []ingressRow { return rows } - -func serviceToNameAndPort(svc *networking.IngressServiceBackend) (string, intstr.IntOrString) { - var svcName string - if svc != nil { - svcName = svc.Name - if svc.Port.Number > 0 { - return svcName, intstr.FromInt(int(svc.Port.Number)) - } - if svc.Port.Name != "" { - return svcName, intstr.FromString(svc.Port.Name) - } - } - return "", intstr.IntOrString{} -} diff --git a/cmd/plugin/commands/ingresses/ingresses_test.go b/cmd/plugin/commands/ingresses/ingresses_test.go deleted file mode 100644 index 7a90efe46..000000000 --- a/cmd/plugin/commands/ingresses/ingresses_test.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ingresses - -import ( - "testing" - - networking "k8s.io/api/networking/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -func TestGetIngressInformation(t *testing.T) { - testcases := map[string]struct { - ServiceBackend *networking.IngressServiceBackend - wantName string - wantPort intstr.IntOrString - }{ - "empty ingressServiceBackend": { - ServiceBackend: &networking.IngressServiceBackend{}, - wantName: "", - wantPort: intstr.IntOrString{}, - }, - "ingressServiceBackend with port 8080": { - ServiceBackend: &networking.IngressServiceBackend{ - Name: "test", - Port: networking.ServiceBackendPort{ - Number: 8080, - }, - }, - wantName: "test", - wantPort: intstr.IntOrString{ - Type: intstr.Int, - IntVal: 8080, - }, - }, - "ingressServiceBackend with port name a-svc": { - ServiceBackend: &networking.IngressServiceBackend{ - Name: "test", - Port: networking.ServiceBackendPort{ - Name: "a-svc", - }, - }, - wantName: "test", - wantPort: intstr.IntOrString{ - Type: intstr.String, - StrVal: "a-svc", - }, - }, - } - - for title, testCase := range testcases { - gotName, gotPort := serviceToNameAndPort(testCase.ServiceBackend) - if gotName != testCase.wantName { - t.Fatalf("%s: expected '%v' but returned %v", title, testCase.wantName, gotName) - } - if gotPort != testCase.wantPort { - t.Fatalf("%s: expected '%v' but returned %v", title, testCase.wantPort, gotPort) - } - } -} diff --git a/cmd/plugin/commands/lint/main.go b/cmd/plugin/commands/lint/main.go index 69ab984f5..d120e9311 100644 --- a/cmd/plugin/commands/lint/main.go +++ b/cmd/plugin/commands/lint/main.go @@ -22,7 +22,7 @@ import ( "github.com/spf13/cobra" appsv1 "k8s.io/api/apps/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/cli-runtime/pkg/genericclioptions" @@ -38,7 +38,7 @@ func CreateCommand(flags *genericclioptions.ConfigFlags) *cobra.Command { cmd := &cobra.Command{ Use: "lint", Short: "Inspect kubernetes resources for possible issues", - RunE: func(_ *cobra.Command, _ []string) error { + RunE: func(cmd *cobra.Command, args []string) error { err := opts.Validate() if err != nil { return err @@ -73,7 +73,7 @@ func createSubcommand(flags *genericclioptions.ConfigFlags, names []string, shor Use: names[0], Aliases: names[1:], Short: short, - RunE: func(_ *cobra.Command, _ []string) error { + RunE: func(cmd *cobra.Command, args []string) error { err := opts.Validate() if err != nil { return err @@ -111,13 +111,11 @@ type lintOptions struct { } func (opts *lintOptions) Validate() error { - //nolint:dogsled // Ignore 3 blank identifiers _, _, _, err := util.ParseVersionString(opts.versionFrom) if err != nil { return err } - //nolint:dogsled // Ignore 3 blank identifiers _, _, _, err = util.ParseVersionString(opts.versionTo) if err != nil { return err @@ -133,9 +131,9 @@ type lint interface { Version() string } -func checkObjectArray(allLints []lint, objects []kmeta.Object, opts lintOptions) { +func checkObjectArray(lints []lint, objects []kmeta.Object, opts lintOptions) { usedLints := make([]lint, 0) - for _, lint := range allLints { + for _, lint := range lints { lintVersion := lint.Version() if lint.Version() == "" { lintVersion = "0.0.0" @@ -191,7 +189,7 @@ func ingresses(opts lintOptions) error { return err } - iLints := lints.GetIngressLints() + var iLints []lints.IngressLint = lints.GetIngressLints() genericLints := make([]lint, len(iLints)) for i := range iLints { genericLints[i] = iLints[i] @@ -218,7 +216,7 @@ func deployments(opts lintOptions) error { return err } - iLints := lints.GetDeploymentLints() + var iLints []lints.DeploymentLint = lints.GetDeploymentLints() genericLints := make([]lint, len(iLints)) for i := range iLints { genericLints[i] = iLints[i] diff --git a/cmd/plugin/commands/logs/logs.go b/cmd/plugin/commands/logs/logs.go index d9f6e7bd8..55cd008dc 100644 --- a/cmd/plugin/commands/logs/logs.go +++ b/cmd/plugin/commands/logs/logs.go @@ -31,20 +31,19 @@ import ( // CreateCommand creates and returns this cobra subcommand func CreateCommand(flags *genericclioptions.ConfigFlags) *cobra.Command { o := logsFlags{} - var pod, deployment, selector, container *string + var pod, deployment, selector *string cmd := &cobra.Command{ Use: "logs", Short: "Get the kubernetes logs for an ingress-nginx pod", - RunE: func(_ *cobra.Command, _ []string) error { - util.PrintError(logs(flags, *pod, *deployment, *selector, *container, o)) + RunE: func(cmd *cobra.Command, args []string) error { + util.PrintError(logs(flags, *pod, *deployment, *selector, o)) return nil }, } pod = util.AddPodFlag(cmd) deployment = util.AddDeploymentFlag(cmd) selector = util.AddSelectorFlag(cmd) - container = util.AddContainerFlag(cmd) cmd.Flags().BoolVarP(&o.Follow, "follow", "f", o.Follow, "Specify if the logs should be streamed.") cmd.Flags().BoolVar(&o.Timestamps, "timestamps", o.Timestamps, "Include timestamps on each line in the log output") @@ -95,13 +94,13 @@ func (o *logsFlags) toStrings() []string { return r } -func logs(flags *genericclioptions.ConfigFlags, podName, deployment, selector, container string, opts logsFlags) error { +func logs(flags *genericclioptions.ConfigFlags, podName string, deployment string, selector string, opts logsFlags) error { pod, err := request.ChoosePod(flags, podName, deployment, selector) if err != nil { return err } - cmd := []string{"logs", "-n", pod.Namespace, "-c", container, pod.Name} + cmd := []string{"logs", "-n", pod.Namespace, pod.Name} cmd = append(cmd, opts.toStrings()...) return kubectl.Exec(flags, cmd) } diff --git a/cmd/plugin/commands/ssh/ssh.go b/cmd/plugin/commands/ssh/ssh.go index c9631d10b..5e8b49fac 100644 --- a/cmd/plugin/commands/ssh/ssh.go +++ b/cmd/plugin/commands/ssh/ssh.go @@ -28,28 +28,27 @@ import ( // CreateCommand creates and returns this cobra subcommand func CreateCommand(flags *genericclioptions.ConfigFlags) *cobra.Command { - var pod, deployment, selector, container *string + var pod, deployment, selector *string cmd := &cobra.Command{ Use: "ssh", Short: "ssh into a running ingress-nginx pod", - RunE: func(_ *cobra.Command, _ []string) error { - util.PrintError(ssh(flags, *pod, *deployment, *selector, *container)) + RunE: func(cmd *cobra.Command, args []string) error { + util.PrintError(ssh(flags, *pod, *deployment, *selector)) return nil }, } pod = util.AddPodFlag(cmd) deployment = util.AddDeploymentFlag(cmd) selector = util.AddSelectorFlag(cmd) - container = util.AddContainerFlag(cmd) return cmd } -func ssh(flags *genericclioptions.ConfigFlags, podName, deployment, selector, container string) error { +func ssh(flags *genericclioptions.ConfigFlags, podName string, deployment string, selector string) error { pod, err := request.ChoosePod(flags, podName, deployment, selector) if err != nil { return err } - return kubectl.Exec(flags, []string{"exec", "-it", "-n", pod.Namespace, "-c", container, pod.Name, "--", "/bin/bash"}) + return kubectl.Exec(flags, []string{"exec", "-it", "-n", pod.Namespace, pod.Name, "--", "/bin/bash"}) } diff --git a/cmd/plugin/ingress-nginx.yaml.tmpl b/cmd/plugin/ingress-nginx.yaml.tmpl new file mode 100644 index 000000000..f5e7c3541 --- /dev/null +++ b/cmd/plugin/ingress-nginx.yaml.tmpl @@ -0,0 +1,41 @@ +apiVersion: krew.googlecontainertools.github.com/v1alpha2 +kind: Plugin +metadata: + name: ingress-nginx +spec: + shortDescription: Interact with ingress-nginx + description: | + The official kubectl plugin for ingress-nginx. + version: v%%%tag%%% + homepage: https://kubernetes.github.io/ingress-nginx/kubectl-plugin/ + platforms: + - uri: https://github.com/kubernetes/ingress-nginx/releases/download/nginx-%%%tag%%%/kubectl-ingress_nginx-darwin-amd64.tar.gz + sha256: %%%shasum_darwin_amd64%%% + files: + - from: "*" + to: "." + bin: "./kubectl-ingress_nginx" + selector: + matchLabels: + os: darwin + arch: amd64 + - uri: https://github.com/kubernetes/ingress-nginx/releases/download/nginx-%%%tag%%%/kubectl-ingress_nginx-linux-amd64.tar.gz + sha256: %%%shasum_linux_amd64%%% + files: + - from: "*" + to: "." + bin: "./kubectl-ingress_nginx" + selector: + matchLabels: + os: linux + arch: amd64 + - uri: https://github.com/kubernetes/ingress-nginx/releases/download/nginx-%%%tag%%%/kubectl-ingress_nginx-windows-amd64.tar.gz + sha256: %%%shasum_windows_amd64%%% + files: + - from: "*" + to: "." + bin: "./kubectl-ingress_nginx.exe" + selector: + matchLabels: + os: windows + arch: amd64 diff --git a/cmd/plugin/krew.yaml b/cmd/plugin/krew.yaml deleted file mode 100644 index 0ef537d34..000000000 --- a/cmd/plugin/krew.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: krew.googlecontainertools.github.com/v1alpha2 -kind: Plugin -metadata: - name: ingress-nginx -spec: - shortDescription: Interact with ingress-nginx - description: | - The official kubectl plugin for ingress-nginx. - version: {{ .TagName }} - homepage: https://kubernetes.github.io/ingress-nginx/kubectl-plugin/ - platforms: - - selector: - matchLabels: - os: darwin - arch: arm64 - {{addURIAndSha "https://github.com/kubernetes/ingress-nginx/releases/download/{{ .TagName }}/kubectl-ingress-nginx_darwin_arm64.tar.gz" .TagName }} - bin: kubectl-ingress-nginx - - selector: - matchLabels: - os: darwin - arch: amd64 - {{addURIAndSha "https://github.com/kubernetes/ingress-nginx/releases/download/{{ .TagName }}/kubectl-ingress-nginx_darwin_amd64.tar.gz" .TagName }} - bin: kubectl-ingress-nginx - - selector: - matchLabels: - os: linux - arch: amd64 - {{addURIAndSha "https://github.com/kubernetes/ingress-nginx/releases/download/{{ .TagName }}/kubectl-ingress-nginx_linux_amd64.tar.gz" .TagName }} - bin: kubectl-ingress-nginx - - selector: - matchLabels: - os: linux - arch: arm64 - {{addURIAndSha "https://github.com/kubernetes/ingress-nginx/releases/download/{{ .TagName }}/kubectl-ingress-nginx_linux_arm64.tar.gz" .TagName }} - bin: kubectl-ingress-nginx - - selector: - matchLabels: - os: windows - arch: amd64 - {{addURIAndSha "https://github.com/kubernetes/ingress-nginx/releases/download/{{ .TagName }}/kubectl-ingress-nginx_windows_amd64.tar.gz" .TagName }} - bin: kubectl-ingress-nginx.exe diff --git a/cmd/plugin/kubectl/kubectl.go b/cmd/plugin/kubectl/kubectl.go index cb33243fc..c11ba5b77 100644 --- a/cmd/plugin/kubectl/kubectl.go +++ b/cmd/plugin/kubectl/kubectl.go @@ -31,18 +31,18 @@ import ( // PodExecString takes a pod and a command, uses kubectl exec to run the command in the pod // and returns stdout as a string -func PodExecString(flags *genericclioptions.ConfigFlags, pod *apiv1.Pod, container string, args []string) (string, error) { - args = append([]string{"exec", "-n", pod.Namespace, "-c", container, pod.Name}, args...) +func PodExecString(flags *genericclioptions.ConfigFlags, pod *apiv1.Pod, args []string) (string, error) { + args = append([]string{"exec", "-n", pod.Namespace, pod.Name}, args...) return ExecToString(flags, args) } // ExecToString runs a kubectl subcommand and returns stdout as a string func ExecToString(flags *genericclioptions.ConfigFlags, args []string) (string, error) { - kubectlArgs := getKubectlConfigFlags(flags) - kubectlArgs = append(kubectlArgs, args...) + kArgs := getKubectlConfigFlags(flags) + kArgs = append(kArgs, args...) buf := bytes.NewBuffer(make([]byte, 0)) - err := execToWriter(append([]string{"kubectl"}, kubectlArgs...), buf) + err := execToWriter(append([]string{"kubectl"}, kArgs...), buf) if err != nil { return "", err } @@ -51,9 +51,9 @@ func ExecToString(flags *genericclioptions.ConfigFlags, args []string) (string, // Exec replaces the current process with a kubectl invocation func Exec(flags *genericclioptions.ConfigFlags, args []string) error { - kubectlArgs := getKubectlConfigFlags(flags) - kubectlArgs = append(kubectlArgs, args...) - return execCommand(append([]string{"kubectl"}, kubectlArgs...)) + kArgs := getKubectlConfigFlags(flags) + kArgs = append(kArgs, args...) + return execCommand(append([]string{"kubectl"}, kArgs...)) } // Replaces the currently running process with the given command @@ -70,7 +70,6 @@ func execCommand(args []string) error { // Runs a command and returns stdout func execToWriter(args []string, writer io.Writer) error { - //nolint:gosec // Ignore G204 error cmd := exec.Command(args[0], args[1:]...) op, err := cmd.StdoutPipe() @@ -78,9 +77,7 @@ func execToWriter(args []string, writer io.Writer) error { return err } - go func() { - io.Copy(writer, op) //nolint:errcheck // Ignore the error - }() + go io.Copy(writer, op) err = cmd.Run() if err != nil { return err @@ -107,6 +104,7 @@ func getKubectlConfigFlags(flags *genericclioptions.ConfigFlags) []string { appendStringFlag(o, flags.Password, "password") appendStringFlag(o, flags.ClusterName, "cluster") appendStringFlag(o, flags.AuthInfoName, "user") + //appendStringFlag(o, flags.Namespace, "namespace") appendStringFlag(o, flags.Context, "context") appendStringFlag(o, flags.APIServer, "server") appendBoolFlag(o, flags.Insecure, "insecure-skip-tls-verify") @@ -128,7 +126,7 @@ func appendBoolFlag(out *[]string, in *bool, flag string) { } } -func appendStringArrayFlag(out, in *[]string, flag string) { +func appendStringArrayFlag(out *[]string, in *[]string, flag string) { if in != nil && len(*in) > 0 { *out = append(*out, fmt.Sprintf("--%v=%v'", flag, strings.Join(*in, ","))) } diff --git a/cmd/plugin/lints/deployment.go b/cmd/plugin/lints/deployment.go index ce1712284..a1c473f1e 100644 --- a/cmd/plugin/lints/deployment.go +++ b/cmd/plugin/lints/deployment.go @@ -35,10 +35,7 @@ type DeploymentLint struct { // Check returns true if the lint detects an issue func (lint DeploymentLint) Check(obj kmeta.Object) bool { - cmp, ok := obj.(*v1.Deployment) - if !ok { - util.PrintError(fmt.Errorf("unexpected type: %T", obj)) - } + cmp := obj.(*v1.Deployment) return lint.f(*cmp) } @@ -75,11 +72,11 @@ func removedFlag(flag string, issueNumber int, version string) DeploymentLint { issue: issueNumber, version: version, f: func(dep v1.Deployment) bool { - if !isIngressNginxDeployment(&dep) { + if !isIngressNginxDeployment(dep) { return false } - args := getNginxArgs(&dep) + args := getNginxArgs(dep) for _, arg := range args { if strings.HasPrefix(arg, fmt.Sprintf("--%v", flag)) { return true @@ -91,9 +88,8 @@ func removedFlag(flag string, issueNumber int, version string) DeploymentLint { } } -func getNginxArgs(dep *v1.Deployment) []string { - for i := range dep.Spec.Template.Spec.Containers { - container := &dep.Spec.Template.Spec.Containers[i] +func getNginxArgs(dep v1.Deployment) []string { + for _, container := range dep.Spec.Template.Spec.Containers { if len(container.Args) > 0 && container.Args[0] == "/nginx-ingress-controller" { return container.Args } @@ -101,10 +97,10 @@ func getNginxArgs(dep *v1.Deployment) []string { return make([]string, 0) } -func isIngressNginxDeployment(dep *v1.Deployment) bool { +func isIngressNginxDeployment(dep v1.Deployment) bool { containers := dep.Spec.Template.Spec.Containers - for i := range containers { - if len(containers[i].Args) > 0 && containers[i].Args[0] == "/nginx-ingress-controller" { + for _, container := range containers { + if len(container.Args) > 0 && container.Args[0] == "/nginx-ingress-controller" { return true } } diff --git a/cmd/plugin/lints/ingress.go b/cmd/plugin/lints/ingress.go index d5ad42e2c..0de4661f4 100644 --- a/cmd/plugin/lints/ingress.go +++ b/cmd/plugin/lints/ingress.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/cmd/plugin/util" ) @@ -30,16 +30,13 @@ type IngressLint struct { message string issue int version string - f func(ing *networking.Ingress) bool + f func(ing networking.Ingress) bool } // Check returns true if the lint detects an issue func (lint IngressLint) Check(obj kmeta.Object) bool { - ing, ok := obj.(*networking.Ingress) - if !ok { - util.PrintError(fmt.Errorf("unexpected type: %T", obj)) - } - return lint.f(ing) + ing := obj.(*networking.Ingress) + return lint.f(*ing) } // Message is a description of the lint @@ -97,7 +94,7 @@ func GetIngressLints() []IngressLint { } } -func xForwardedPrefixIsBool(ing *networking.Ingress) bool { +func xForwardedPrefixIsBool(ing networking.Ingress) bool { for name, val := range ing.Annotations { if strings.HasSuffix(name, "/x-forwarded-prefix") && (val == "true" || val == "false") { return true @@ -106,7 +103,7 @@ func xForwardedPrefixIsBool(ing *networking.Ingress) bool { return false } -func annotationPrefixIsNginxCom(ing *networking.Ingress) bool { +func annotationPrefixIsNginxCom(ing networking.Ingress) bool { for name := range ing.Annotations { if strings.HasPrefix(name, "nginx.com/") { return true @@ -115,7 +112,7 @@ func annotationPrefixIsNginxCom(ing *networking.Ingress) bool { return false } -func annotationPrefixIsNginxOrg(ing *networking.Ingress) bool { +func annotationPrefixIsNginxOrg(ing networking.Ingress) bool { for name := range ing.Annotations { if strings.HasPrefix(name, "nginx.org/") { return true @@ -124,7 +121,7 @@ func annotationPrefixIsNginxOrg(ing *networking.Ingress) bool { return false } -func rewriteTargetWithoutCaptureGroup(ing *networking.Ingress) bool { +func rewriteTargetWithoutCaptureGroup(ing networking.Ingress) bool { for name, val := range ing.Annotations { if strings.HasSuffix(name, "/rewrite-target") && !strings.Contains(val, "$1") { return true @@ -138,7 +135,7 @@ func removedAnnotation(annotationName string, issueNumber int, version string) I message: fmt.Sprintf("Contains the removed %v annotation.", annotationName), issue: issueNumber, version: version, - f: func(ing *networking.Ingress) bool { + f: func(ing networking.Ingress) bool { for annotation := range ing.Annotations { if strings.HasSuffix(annotation, "/"+annotationName) { return true @@ -149,7 +146,7 @@ func removedAnnotation(annotationName string, issueNumber int, version string) I } } -func satisfyDirective(ing *networking.Ingress) bool { +func satisfyDirective(ing networking.Ingress) bool { for name, val := range ing.Annotations { if strings.HasSuffix(name, "/configuration-snippet") { return strings.Contains(val, "satisfy") diff --git a/cmd/plugin/main.go b/cmd/plugin/main.go index e9a8ea59a..f3a809715 100644 --- a/cmd/plugin/main.go +++ b/cmd/plugin/main.go @@ -24,7 +24,7 @@ import ( "k8s.io/cli-runtime/pkg/genericclioptions" - // Just importing this is supposed to allow cloud authentication + //Just importing this is supposed to allow cloud authentication // eg GCP, AWS, Azure ... _ "k8s.io/client-go/plugin/pkg/client/auth" diff --git a/cmd/plugin/request/request.go b/cmd/plugin/request/request.go index 57b02827e..b8bc4eaae 100644 --- a/cmd/plugin/request/request.go +++ b/cmd/plugin/request/request.go @@ -22,20 +22,18 @@ import ( appsv1 "k8s.io/api/apps/v1" apiv1 "k8s.io/api/core/v1" - discoveryv1 "k8s.io/api/discovery/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/cli-runtime/pkg/genericclioptions" appsv1client "k8s.io/client-go/kubernetes/typed/apps/v1" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" - discoveryv1client "k8s.io/client-go/kubernetes/typed/discovery/v1" - typednetworking "k8s.io/client-go/kubernetes/typed/networking/v1" + typednetworking "k8s.io/client-go/kubernetes/typed/networking/v1beta1" "k8s.io/ingress-nginx/cmd/plugin/util" ) // ChoosePod finds a pod either by deployment or by name -func ChoosePod(flags *genericclioptions.ConfigFlags, podName, deployment, selector string) (apiv1.Pod, error) { +func ChoosePod(flags *genericclioptions.ConfigFlags, podName string, deployment string, selector string) (apiv1.Pod, error) { if podName != "" { return GetNamedPod(flags, podName) } @@ -54,9 +52,9 @@ func GetNamedPod(flags *genericclioptions.ConfigFlags, name string) (apiv1.Pod, return apiv1.Pod{}, err } - for i := range allPods { - if allPods[i].Name == name { - return allPods[i], nil + for _, pod := range allPods { + if pod.Name == name { + return pod, nil } } @@ -131,62 +129,55 @@ func GetIngressDefinitions(flags *genericclioptions.ConfigFlags, namespace strin return pods.Items, nil } -// GetNumEndpoints counts the number of endpointslices addresses for the service with the given name -func GetNumEndpoints(flags *genericclioptions.ConfigFlags, namespace, serviceName string) (*int, error) { - epss, err := GetEndpointSlicesByName(flags, namespace, serviceName) +// GetNumEndpoints counts the number of endpoints for the service with the given name +func GetNumEndpoints(flags *genericclioptions.ConfigFlags, namespace string, serviceName string) (*int, error) { + endpoints, err := GetEndpointsByName(flags, namespace, serviceName) if err != nil { return nil, err } - if len(epss) == 0 { + if endpoints == nil { return nil, nil } ret := 0 - for i := range epss { - eps := &epss[i] - for j := range eps.Endpoints { - ret += len(eps.Endpoints[j].Addresses) - } + for _, subset := range endpoints.Subsets { + ret += len(subset.Addresses) } return &ret, nil } -// GetEndpointSlicesByName returns the endpointSlices for the service with the given name -func GetEndpointSlicesByName(flags *genericclioptions.ConfigFlags, namespace, name string) ([]discoveryv1.EndpointSlice, error) { - allEndpointsSlices, err := getEndpointSlices(flags, namespace) +// GetEndpointsByName returns the endpoints for the service with the given name +func GetEndpointsByName(flags *genericclioptions.ConfigFlags, namespace string, name string) (*apiv1.Endpoints, error) { + allEndpoints, err := getEndpoints(flags, namespace) if err != nil { return nil, err } - var eps []discoveryv1.EndpointSlice - for i := range allEndpointsSlices { - if svcName, ok := allEndpointsSlices[i].ObjectMeta.GetLabels()[discoveryv1.LabelServiceName]; ok { - if svcName == name { - eps = append(eps, allEndpointsSlices[i]) - } + + for _, endpoints := range allEndpoints { + if endpoints.Name == name { + return &endpoints, nil } } - return eps, nil + return nil, nil } -var endpointSlicesCache = make(map[string]*[]discoveryv1.EndpointSlice) - -func getEndpointSlices(flags *genericclioptions.ConfigFlags, namespace string) ([]discoveryv1.EndpointSlice, error) { - cachedEndpointSlices, ok := endpointSlicesCache[namespace] +var endpointsCache = make(map[string]*[]apiv1.Endpoints) +func getEndpoints(flags *genericclioptions.ConfigFlags, namespace string) ([]apiv1.Endpoints, error) { + cachedEndpoints, ok := endpointsCache[namespace] if ok { - return *cachedEndpointSlices, nil + return *cachedEndpoints, nil } if namespace != "" { - tryAllNamespacesEndpointSlicesCache(flags) + tryAllNamespacesEndpointsCache(flags) } - cachedEndpointSlices = tryFilteringEndpointSlicesFromAllNamespacesCache(namespace) - - if cachedEndpointSlices != nil { - return *cachedEndpointSlices, nil + cachedEndpoints = tryFilteringEndpointsFromAllNamespacesCache(flags, namespace) + if cachedEndpoints != nil { + return *cachedEndpoints, nil } rawConfig, err := flags.ToRESTConfig() @@ -194,41 +185,42 @@ func getEndpointSlices(flags *genericclioptions.ConfigFlags, namespace string) ( return nil, err } - api, err := discoveryv1client.NewForConfig(rawConfig) + api, err := corev1.NewForConfig(rawConfig) if err != nil { return nil, err } - endpointSlicesList, err := api.EndpointSlices(namespace).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, err - } - endpointSlices := endpointSlicesList.Items - endpointSlicesCache[namespace] = &endpointSlices - return endpointSlices, nil + endpointsList, err := api.Endpoints(namespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + endpoints := endpointsList.Items + + endpointsCache[namespace] = &endpoints + return endpoints, nil } -func tryAllNamespacesEndpointSlicesCache(flags *genericclioptions.ConfigFlags) { - _, ok := endpointSlicesCache[""] +func tryAllNamespacesEndpointsCache(flags *genericclioptions.ConfigFlags) { + _, ok := endpointsCache[""] if !ok { - _, err := getEndpointSlices(flags, "") + _, err := getEndpoints(flags, "") if err != nil { - endpointSlicesCache[""] = nil + endpointsCache[""] = nil } } } -func tryFilteringEndpointSlicesFromAllNamespacesCache(namespace string) *[]discoveryv1.EndpointSlice { - allEndpointSlices := endpointSlicesCache[""] - if allEndpointSlices != nil { - endpointSlices := make([]discoveryv1.EndpointSlice, 0) - for i := range *allEndpointSlices { - if (*allEndpointSlices)[i].Namespace == namespace { - endpointSlices = append(endpointSlices, (*allEndpointSlices)[i]) +func tryFilteringEndpointsFromAllNamespacesCache(flags *genericclioptions.ConfigFlags, namespace string) *[]apiv1.Endpoints { + allEndpoints := endpointsCache[""] + if allEndpoints != nil { + endpoints := make([]apiv1.Endpoints, 0) + for _, thisEndpoints := range *allEndpoints { + if thisEndpoints.Namespace == namespace { + endpoints = append(endpoints, thisEndpoints) } } - endpointSlicesCache[namespace] = &endpointSlices - return &endpointSlices + endpointsCache[namespace] = &endpoints + return &endpoints } return nil } @@ -243,9 +235,9 @@ func GetServiceByName(flags *genericclioptions.ConfigFlags, name string, service services = &servicesArray } - for i := range *services { - if (*services)[i].Name == name { - return (*services)[i], nil + for _, svc := range *services { + if svc.Name == name { + return svc, nil } } @@ -289,6 +281,7 @@ func getLabeledPods(flags *genericclioptions.ConfigFlags, label string) ([]apiv1 pods, err := api.Pods(namespace).List(context.TODO(), metav1.ListOptions{ LabelSelector: label, }) + if err != nil { return make([]apiv1.Pod, 0), err } @@ -303,9 +296,9 @@ func getDeploymentPods(flags *genericclioptions.ConfigFlags, deployment string) } ingressPods := make([]apiv1.Pod, 0) - for i := range pods { - if util.PodInDeployment(&pods[i], deployment) { - ingressPods = append(ingressPods, pods[i]) + for _, pod := range pods { + if util.PodInDeployment(pod, deployment) { + ingressPods = append(ingressPods, pod) } } @@ -331,4 +324,5 @@ func getServices(flags *genericclioptions.ConfigFlags) ([]apiv1.Service, error) } return services.Items, nil + } diff --git a/cmd/plugin/util/util.go b/cmd/plugin/util/util.go index 7457b8c53..cc9882009 100644 --- a/cmd/plugin/util/util.go +++ b/cmd/plugin/util/util.go @@ -29,9 +29,8 @@ import ( // The default deployment and service names for ingress-nginx const ( - DefaultIngressDeploymentName = "ingress-nginx-controller" //#nosec G101 - DefaultIngressServiceName = "ingress-nginx-controller" //#nosec G101 - DefaultIngressContainerName = "controller" + DefaultIngressDeploymentName = "ingress-nginx-controller" + DefaultIngressServiceName = "ingress-nginx-controller" ) // IssuePrefix is the github url that we can append an issue number to to link to it @@ -47,25 +46,17 @@ func PrintError(e error) { } // ParseVersionString returns the major, minor, and patch numbers of a version string -func ParseVersionString(v string) (major, minor, patch int, err error) { +func ParseVersionString(v string) (int, int, int, error) { parts := versionRegex.FindStringSubmatch(v) if len(parts) != 4 { return 0, 0, 0, fmt.Errorf("could not parse %v as a version string (like 0.20.3)", v) } - major, err = strconv.Atoi(parts[1]) - if err != nil { - return 0, 0, 0, err - } - minor, err = strconv.Atoi(parts[2]) - if err != nil { - return 0, 0, 0, err - } - patch, err = strconv.Atoi(parts[3]) - if err != nil { - return 0, 0, 0, err - } + major, _ := strconv.Atoi(parts[1]) + minor, _ := strconv.Atoi(parts[2]) + patch, _ := strconv.Atoi(parts[3]) + return major, minor, patch, nil } @@ -98,7 +89,7 @@ func isVersionLessThan(a, b string) bool { // PodInDeployment returns whether a pod is part of a deployment with the given name // a pod is considered to be in {deployment} if it is owned by a replicaset with a name of format {deployment}-otherchars -func PodInDeployment(pod *apiv1.Pod, deployment string) bool { +func PodInDeployment(pod apiv1.Pod, deployment string) bool { for _, owner := range pod.OwnerReferences { if owner.Controller == nil || !*owner.Controller || owner.Kind != "ReplicaSet" { continue @@ -136,17 +127,10 @@ func AddSelectorFlag(cmd *cobra.Command) *string { return &v } -// AddContainerFlag adds a --container flag to a cobra command -func AddContainerFlag(cmd *cobra.Command) *string { - v := "" - cmd.Flags().StringVar(&v, "container", DefaultIngressContainerName, "The name of the ingress-nginx controller container") - return &v -} - // GetNamespace takes a set of kubectl flag values and returns the namespace we should be operating in func GetNamespace(flags *genericclioptions.ConfigFlags) string { namespace, _, err := flags.ToRawKubeConfigLoader().Namespace() - if err != nil || namespace == "" { + if err != nil || len(namespace) == 0 { namespace = apiv1.NamespaceDefault } return namespace diff --git a/deploy/grafana/dashboards/README.md b/deploy/grafana/dashboards/README.md index fc1e35406..26195583b 100644 --- a/deploy/grafana/dashboards/README.md +++ b/deploy/grafana/dashboards/README.md @@ -4,7 +4,7 @@ Ingress-nginx supports a rich collection of prometheus metrics. If you have pro This folder contains two dashboards that you can import. -## 1. Ingress-Nginx Controller +## 1. NGINX Ingress Controller ![Dashboard](screenshot.png) @@ -19,7 +19,7 @@ This folder contains two dashboards that you can import. ### Requirements -- **Grafana v10.4.3** (or newer) +- **Grafana v5.2.0** (or newer) ## 2. Request Handling Performance @@ -36,4 +36,4 @@ This folder contains two dashboards that you can import. ### Requirements -- **Grafana v10.4.3** (or newer) +- **Grafana v6.6.0** (or newer) diff --git a/deploy/grafana/dashboards/nginx.json b/deploy/grafana/dashboards/nginx.json index 5ebc4ae6c..a2c277047 100644 --- a/deploy/grafana/dashboards/nginx.json +++ b/deploy/grafana/dashboards/nginx.json @@ -9,53 +9,31 @@ "pluginName": "Prometheus" } ], - "__elements": {}, "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "10.4.3" - }, - { - "type": "panel", - "id": "heatmap", - "name": "Heatmap", - "version": "" + "version": "5.2.1" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", - "version": "1.0.0" + "version": "5.0.0" }, { "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" + "id": "singlestat", + "name": "Singlestat", + "version": "5.0.0" } ], "annotations": { "list": [ { "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -63,10 +41,7 @@ "type": "dashboard" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "datasource": "${DS_PROMETHEUS}", "enable": true, "expr": "sum(changes(nginx_ingress_controller_config_last_reload_successful_timestamp_seconds{instance!=\"unknown\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[30s])) by (controller_class)", "hide": false, @@ -83,75 +58,72 @@ ] }, "editable": true, - "fiscalYearStartMonth": 0, + "gnetId": null, "graphTooltip": 0, - "id": null, + "iteration": 1534359654832, "links": [], "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "rgb(31, 120, 193)", - "mode": "fixed" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 0 }, "id": 20, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true }, - "pluginVersion": "10.4.3", + "tableColumn": "", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m])), 0.001)", "format": "time_series", "intervalFactor": 1, @@ -159,47 +131,37 @@ "step": 4 } ], + "thresholds": "", "title": "Controller Request Volume", - "type": "stat" + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "rgb(31, 120, 193)", - "mode": "fixed" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true }, "gridPos": { "h": 3, @@ -208,30 +170,42 @@ "y": 0 }, "id": 82, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true }, - "pluginVersion": "10.4.3", + "tableColumn": "", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "expr": "sum(avg_over_time(nginx_ingress_controller_nginx_process_connections{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",state=\"active\"}[2m]))", "format": "time_series", "instant": false, @@ -240,51 +214,37 @@ "step": 4 } ], + "thresholds": "", "title": "Controller Connections", - "type": "stat" + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "rgb(31, 120, 193)", - "mode": "fixed" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 95 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 99 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false }, "gridPos": { "h": 3, @@ -293,30 +253,42 @@ "y": 0 }, "id": 21, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true }, - "pluginVersion": "10.4.3", + "tableColumn": "", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",status!~\"[4-5].*\"}[2m])) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m]))", "format": "time_series", "intervalFactor": 1, @@ -324,48 +296,38 @@ "step": 4 } ], + "thresholds": "95, 99, 99.5", "title": "Controller Success Rate (non-4|5xx responses)", - "type": "stat" + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "rgb(31, 120, 193)", - "mode": "fixed" - }, - "decimals": 0, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true }, "gridPos": { "h": 3, @@ -374,30 +336,42 @@ "y": 0 }, "id": 81, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true }, - "pluginVersion": "10.4.3", + "tableColumn": "", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "expr": "avg(irate(nginx_ingress_controller_success{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[1m])) * 60", "format": "time_series", "instant": false, @@ -406,48 +380,38 @@ "step": 4 } ], + "thresholds": "", "title": "Config Reloads", - "type": "stat" + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "rgb(31, 120, 193)", - "mode": "fixed" - }, - "decimals": 0, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true }, "gridPos": { "h": 3, @@ -456,30 +420,42 @@ "y": 0 }, "id": 83, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true }, - "pluginVersion": "10.4.3", + "tableColumn": "", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "expr": "count(nginx_ingress_controller_config_last_reload_successful{controller_pod=~\"$controller\",controller_namespace=~\"$namespace\"} == 0)", "format": "time_series", "instant": true, @@ -488,120 +464,73 @@ "step": 4 } ], + "thresholds": "", "title": "Last Config Failed", - "type": "stat" + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsZero", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] - }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 3 }, + "height": "200px", "id": 86, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true }, - "pluginVersion": "10.4.3", + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",exported_namespace=~\"$exported_namespace\",ingress=~\"$ingress\"}[2m])) by (ingress), 0.001)", + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress), 0.001)", "format": "time_series", "hide": false, "instant": false, @@ -613,135 +542,63 @@ "step": 10 } ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Ingress Request Volume", - "type": "timeseries" + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "max - istio-proxy" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890f02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "max - master" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#bf1b00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "max - prometheus" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#bf1b00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsNull", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": false - } - } - ] - } - ] + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, "gridPos": { "h": 7, "w": 12, @@ -749,28 +606,38 @@ "y": 3 }, "id": 87, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true }, - "pluginVersion": "10.4.3", + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",exported_namespace=~\"$exported_namespace\",ingress=~\"$ingress\",status!~\"[4-5].*\"}[2m])) by (ingress) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",exported_namespace=~\"$exported_namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\",status!~\"[4-5].*\"}[2m])) by (ingress) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", "format": "time_series", "instant": false, "interval": "10s", @@ -781,99 +648,95 @@ "step": 10 } ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Ingress Success Rate (non-4|5xx responses)", - "type": "timeseries" + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 10 }, + "height": "200px", "id": 32, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "width": 200 - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true }, - "pluginVersion": "10.4.3", + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "expr": "sum (irate (nginx_ingress_controller_request_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", "format": "time_series", "instant": false, @@ -885,10 +748,6 @@ "step": 10 }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "expr": "- sum (irate (nginx_ingress_controller_response_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", "format": "time_series", "hide": false, @@ -900,115 +759,63 @@ "step": 10 } ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Network I/O pressure", - "type": "timeseries" + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "max - istio-proxy" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#890f02", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "max - master" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#bf1b00", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "max - prometheus" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#bf1b00", - "mode": "fixed" - } - } - ] - } - ] + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, "gridPos": { "h": 6, "w": 8, @@ -1016,29 +823,35 @@ "y": 10 }, "id": 77, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "width": 200 - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true }, - "pluginVersion": "10.4.3", + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "expr": "avg(nginx_ingress_controller_nginx_process_resident_memory_bytes{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}) ", "format": "time_series", "instant": false, @@ -1050,94 +863,97 @@ "step": 10 } ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Average Memory Usage", - "type": "timeseries" + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "cores", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line+area" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00" }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 3, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 10 }, + "height": "", "id": 79, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sort": null, + "sortDesc": null, + "total": false, + "values": true }, - "pluginVersion": "10.4.3", + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "expr": "avg (rate (nginx_ingress_controller_nginx_process_cpu_seconds_total{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m])) ", "format": "time_series", "interval": "10s", @@ -1148,209 +964,59 @@ "step": 10 } ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt" + } + ], + "timeFrom": null, + "timeShift": null, "title": "Average CPU Usage", - "type": "timeseries" + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This data is real time, independent of dashboard time range", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "ingress" - }, - "properties": [ - { - "id": "displayName", - "value": "Ingress" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #A" - }, - "properties": [ - { - "id": "displayName", - "value": "P50 Latency" - }, - { - "id": "unit", - "value": "dtdurations" - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #B" - }, - "properties": [ - { - "id": "displayName", - "value": "P90 Latency" - }, - { - "id": "unit", - "value": "dtdurations" - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #C" - }, - "properties": [ - { - "id": "displayName", - "value": "P99 Latency" - }, - { - "id": "unit", - "value": "dtdurations" - }, - { - "id": "custom.align" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #D" - }, - "properties": [ - { - "id": "displayName", - "value": "IN" - }, - { - "id": "unit", - "value": "Bps" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)" - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - }, - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #E" - }, - "properties": [ - { - "id": "displayName", - "value": "OUT" - }, - { - "id": "unit", - "value": "Bps" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "custom.align" - } - ] - } - ] - }, + "columns": [], + "datasource": "${DS_PROMETHEUS}", + "fontSize": "100%", "gridPos": { "h": 8, "w": 24, @@ -1359,463 +1025,281 @@ }, "hideTimeOverride": false, "id": 75, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "10.4.3", + "links": [], + "pageSize": 7, + "repeat": null, "repeatDirection": "h", + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "Ingress", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "ingress", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Requests", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #A", + "thresholds": [ + "" + ], + "type": "number", + "unit": "ops" + }, + { + "alias": "Errors", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "ops" + }, + { + "alias": "P50 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P90 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P99 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "IN", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #F", + "thresholds": [ + "" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "OUT", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "Bps" + } + ], "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\",exported_namespace=~\"$exported_namespace\"}[2m])) by (le, ingress))", + "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", "format": "table", "hide": false, "instant": true, "intervalFactor": 1, "legendFormat": "{{ ingress }}", - "refId": "A" + "refId": "C" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.90, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\",exported_namespace=~\"$exported_namespace\"}[2m])) by (le, ingress))", + "expr": "histogram_quantile(0.90, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", "format": "table", "hide": false, "instant": true, "intervalFactor": 1, "legendFormat": "{{ ingress }}", - "refId": "B" + "refId": "D" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\",exported_namespace=~\"$exported_namespace\"}[2m])) by (le, ingress))", + "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", "format": "table", "hide": false, "instant": true, "intervalFactor": 1, "legendFormat": "{{ destination_service }}", - "refId": "C" + "refId": "E" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(irate(nginx_ingress_controller_request_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\",exported_namespace=~\"$exported_namespace\"}[2m])) by (ingress)", + "expr": "sum(irate(nginx_ingress_controller_request_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", "format": "table", "hide": false, "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "{{ ingress }}", - "refId": "D" + "refId": "F" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(irate(nginx_ingress_controller_response_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\",exported_namespace=~\"$exported_namespace\"}[2m])) by (ingress)", + "expr": "sum(irate(nginx_ingress_controller_response_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", "format": "table", "instant": true, "intervalFactor": 1, "legendFormat": "{{ ingress }}", - "refId": "E" + "refId": "G" } ], + "timeFrom": null, "title": "Ingress Percentile Response Times and Transfer Rates", - "transformations": [ - { - "id": "merge", - "options": { - "reducers": [] - } - } - ], + "transform": "table", + "transparent": false, "type": "table" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 24 - }, - "hideTimeOverride": false, - "id": 91, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.3.4", - "repeatDirection": "h", - "targets": [ + "columns": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "histogram_quantile(0.80, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\",exported_namespace=~\"$exported_namespace\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "P80", - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "histogram_quantile(0.90, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\",exported_namespace=~\"$exported_namespace\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "P90", - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\",exported_namespace=~\"$exported_namespace\"}[2m])) by (le))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "P99", - "refId": "E" + "text": "Current", + "value": "current" } ], - "title": "Ingress Percentile Response Times (Ingress Namespaces)", - "type": "timeseries" - }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateWarm", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 24 - }, - "heatmap": {}, - "hideZeroBuckets": false, - "highlightCards": true, - "id": 89, - "legend": { - "show": true - }, - "options": { - "calculate": false, - "calculation": {}, - "cellGap": 2, - "cellValues": {}, - "color": { - "exponent": 0.5, - "fill": "#b4ff00", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Warm", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "showValue": "never", - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "10.4.3", - "reverseYBuckets": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": true, - "expr": "sum(increase(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\",exported_namespace=~\"$exported_namespace\"}[2m])) by (le)", - "format": "heatmap", - "interval": "", - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Ingress Request Latency Heatmap (Ingress Namespaces)", - "tooltip": { - "show": true, - "showHistogram": true - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "decimals": 2, - "displayName": "", - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Last *" - }, - "properties": [ - { - "id": "displayName", - "value": "TTL" - }, - { - "id": "unit", - "value": "s" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 691200 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Field" - }, - "properties": [ - { - "id": "displayName", - "value": "Host" - } - ] - } - ] - }, + "datasource": "${DS_PROMETHEUS}", + "fontSize": "100%", "gridPos": { "h": 8, "w": 24, "x": 0, - "y": 31 + "y": 24 }, + "height": "1024", "id": 85, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true + "links": [], + "pageSize": 7, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": false }, - "pluginVersion": "10.4.3", + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "TTL", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Current", + "thresholds": [ + "0", + "691200" + ], + "type": "number", + "unit": "s" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, "expr": "avg(nginx_ingress_controller_ssl_expire_time_seconds{kubernetes_pod_name=~\"$controller\",namespace=~\"$namespace\",ingress=~\"$ingress\"}) by (host) - time()", "format": "time_series", "intervalFactor": 1, @@ -1826,55 +1310,35 @@ } ], "title": "Ingress Certificate Expiry", - "transformations": [ - { - "id": "reduce", - "options": { - "includeTimeField": false, - "labelsToFields": false, - "reducers": [ - "lastNotNull" - ] - } - } - ], + "transform": "timeseries_aggregations", "type": "table" } ], "refresh": "5s", - "schemaVersion": 39, + "schemaVersion": 16, + "style": "dark", "tags": [ "nginx" ], "templating": { "list": [ { - "current": { - "selected": false, - "text": "Prometheus", - "value": "${DS_PROMETHEUS}" - }, "hide": 0, - "includeAll": false, "label": "datasource", - "multi": false, "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", - "queryValue": "", "refresh": 1, "regex": "", - "skipUrlSync": false, "type": "datasource" }, { "allValue": ".*", - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "current": { + "text": "All", + "value": "$__all" }, - "definition": "", + "datasource": "${DS_PROMETHEUS}", "hide": 0, "includeAll": true, "label": "Namespace", @@ -1884,21 +1348,20 @@ "query": "label_values(nginx_ingress_controller_config_hash, controller_namespace)", "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", + "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "current": { + "text": "All", + "value": "$__all" }, - "definition": "", + "datasource": "${DS_PROMETHEUS}", "hide": 0, "includeAll": true, "label": "Controller Class", @@ -1908,21 +1371,20 @@ "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\"}, controller_class) ", "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", + "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "current": { + "text": "All", + "value": "$__all" }, - "definition": "", + "datasource": "${DS_PROMETHEUS}", "hide": 0, "includeAll": true, "label": "Controller", @@ -1932,57 +1394,33 @@ "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\",controller_class=~\"$controller_class\"}, controller_pod) ", "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 1, + "sort": 0, "tagValuesQuery": "", + "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "current": { + "tags": [], + "text": "All", + "value": "$__all" }, - "definition": "", - "hide": 0, - "includeAll": true, - "label": "Ingress Namespace", - "multi": false, - "name": "exported_namespace", - "options": [], - "query": "label_values(nginx_ingress_controller_requests{namespace=~\"$namespace\",controller_class=~\"$controller_class\",controller_pod=~\"$controller\"}, exported_namespace) ", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": ".*", - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "", + "datasource": "${DS_PROMETHEUS}", "hide": 0, "includeAll": true, "label": "Ingress", "multi": false, "name": "ingress", "options": [], - "query": "label_values(nginx_ingress_controller_requests{namespace=~\"$namespace\",controller_class=~\"$controller_class\",controller_pod=~\"$controller\",exported_namespace=~\"$exported_namespace\"}, ingress) ", + "query": "label_values(nginx_ingress_controller_requests{namespace=~\"$namespace\",controller_class=~\"$controller_class\",controller_pod=~\"$controller\"}, ingress) ", "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 1, + "sort": 2, "tagValuesQuery": "", + "tags": [], "tagsQuery": "", "type": "query", "useTags": false @@ -2021,6 +1459,5 @@ "timezone": "browser", "title": "NGINX Ingress controller", "uid": "nginx", - "version": 1, - "weekStart": "" -} \ No newline at end of file + "version": 1 +} diff --git a/deploy/grafana/dashboards/request-handling-performance.json b/deploy/grafana/dashboards/request-handling-performance.json index cde796384..e231129be 100644 --- a/deploy/grafana/dashboards/request-handling-performance.json +++ b/deploy/grafana/dashboards/request-handling-performance.json @@ -9,264 +9,191 @@ "pluginName": "Prometheus" } ], - "__elements": {}, "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "10.4.3" + "version": "6.6.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" } ], "annotations": { "list": [ { "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, + "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, "type": "dashboard" } ] }, "description": "", "editable": true, - "fiscalYearStartMonth": 0, "gnetId": 9614, "graphTooltip": 1, "id": null, + "iteration": 1582146566338, "links": [], - "liveNow": false, "panels": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total time for NGINX and upstream servers to process a request and send a response", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "Total time taken for nginx and upstream servers to process a request and send a response", + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "hiddenSeries": false, "id": 91, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "pluginVersion": "10.4.3", + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[5m]\n )\n )\n)", + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", "interval": "", "legendFormat": ".5", "refId": "D" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[5m]\n )\n )\n)", + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", "interval": "", "legendFormat": ".95", "refId": "B" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[5m]\n )\n )\n)", + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", "interval": "", "legendFormat": ".99", "refId": "A" } ], - "title": "Request Latency Percentiles", - "type": "timeseries" + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total request handling time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", "description": "The time spent on receiving the response from the upstream server", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "hiddenSeries": false, "id": 94, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "pluginVersion": "10.4.3", + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[5m]\n )\n )\n)", + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", "instant": false, "interval": "", "intervalFactor": 1, @@ -274,659 +201,730 @@ "refId": "D" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[5m]\n )\n )\n)", + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", "interval": "", "legendFormat": ".95", "refId": "B" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[5m]\n )\n )\n)", + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", "interval": "", "legendFormat": ".99", "refId": "A" } ], - "title": "Upstream Response Latency Percentiles", - "type": "timeseries" + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream response time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "hiddenSeries": false, "id": 93, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false }, - "pluginVersion": "10.4.3", + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": " sum by (method, host, path)(\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\"\n }[5m]\n )\n )\n", + "expr": " sum by (path)(\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n", "interval": "", "intervalFactor": 1, - "legendFormat": "{{ method }} {{ host }}{{path }}", + "legendFormat": "{{ path }}", "refId": "A" } ], - "title": "Request Rate by Method and Path", - "type": "timeseries" + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", "description": "For each path observed, its median upstream response time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "hiddenSeries": false, "id": 98, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false }, - "pluginVersion": "10.4.3", + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(\n .5,\n sum by (le, method, host, path)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[5m]\n )\n )\n)", + "expr": "histogram_quantile(\n .5,\n sum by (le, path)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", "interval": "", "intervalFactor": 1, - "legendFormat": "{{ method }} {{ host }}{{path }}", + "legendFormat": "{{ path }}", "refId": "A" } ], - "title": "Median Upstream Response Time by Method and Path", - "type": "timeseries" + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Median upstream response time by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", "description": "Percentage of 4xx and 5xx responses among all responses.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "hiddenSeries": false, "id": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false }, - "pluginVersion": "10.4.3", + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (method, host, path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~ \"[4-5].*\"\n}[5m])) / sum by (method, host, path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n}[5m]))", + "expr": "sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~ \"[4-5].*\"\n}[1m])) / sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n}[1m]))", "interval": "", "intervalFactor": 1, - "legendFormat": "{{ method }} {{ host }}{{path }}", + "legendFormat": "{{ path }}", "refId": "A" } ], - "title": "Response Error Rate by Method and Path", - "type": "timeseries" + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error rate by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", "description": "For each path observed, the sum of upstream request time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "hiddenSeries": false, "id": 102, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false }, - "pluginVersion": "10.4.3", + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (method, host, path) (rate(nginx_ingress_controller_response_duration_seconds_sum{ingress =~ \"$ingress\"}[5m]))", + "expr": "sum by (path) (rate(nginx_ingress_controller_response_duration_seconds_sum{ingress =~ \"$ingress\"}[1m]))", "interval": "", "intervalFactor": 1, - "legendFormat": "{{ method }} {{ host }}{{path }}", + "legendFormat": "{{ path }}", "refId": "A" } ], - "title": "Upstream Response Time by Method and Path", - "type": "timeseries" + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream time consumed by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "hiddenSeries": false, "id": 101, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false }, - "pluginVersion": "10.4.3", + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": " sum (\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~\"[4-5].*\",\n }[5m]\n )\n ) by(method, host, path, status)\n", + "expr": " sum (\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~\"[4-5].*\",\n }[1m]\n )\n ) by(path, status)\n", "interval": "", "intervalFactor": 1, - "legendFormat": "{{ method }} {{ host }}{{path }} {{ status }}", + "legendFormat": "{{ path }} {{ status }}", "refId": "A" } ], - "title": "Response Error Rate by Method and Path", - "type": "timeseries" + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "hiddenSeries": false, "id": 99, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false }, - "pluginVersion": "10.4.3", + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum (\n rate (\n nginx_ingress_controller_response_size_sum {\n ingress =~ \"$ingress\",\n }[5m]\n )\n) by (method, host, path) / sum (\n rate(\n nginx_ingress_controller_response_size_count {\n ingress =~ \"$ingress\",\n }[5m]\n )\n) by (method, host, path)\n", + "expr": "sum (\n rate (\n nginx_ingress_controller_response_size_sum {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path) / sum (\n rate(\n nginx_ingress_controller_response_size_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path)\n", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{ method }} {{ host }}{{path }}", + "legendFormat": "{{ path }}", "refId": "D" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": " sum (rate(nginx_ingress_controller_response_size_bucket{\n ingress =~ \"$ingress\",\n }[5m])) by (le)\n", + "expr": " sum (rate(nginx_ingress_controller_response_size_bucket{\n ingress =~ \"$ingress\",\n }[1m])) by (le)\n", "hide": true, "legendFormat": "{{le}}", "refId": "A" } ], - "title": "Average Response Size by Method and Path", - "type": "timeseries" + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average response size by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_sum {\n ingress =~ \"$ingress\",\n }[1m]\n)) / sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "average", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream service latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "refresh": "30s", - "schemaVersion": 39, + "schemaVersion": 22, + "style": "dark", "tags": [ "nginx" ], "templating": { "list": [ { - "current": { - "selected": false, - "text": "Prometheus", - "value": "${DS_PROMETHEUS}" - }, "hide": 0, - "includeAll": false, "label": "datasource", - "multi": false, "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "refresh": 1, "regex": "", - "skipUrlSync": false, "type": "datasource" }, { "allValue": ".*", "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "datasource": "${DS_PROMETHEUS}", "definition": "label_values(nginx_ingress_controller_requests, ingress) ", "hide": 0, "includeAll": true, @@ -934,15 +932,13 @@ "multi": false, "name": "ingress", "options": [], - "query": { - "query": "label_values(nginx_ingress_controller_requests, ingress) ", - "refId": "Prometheus-ingress-Variable-Query" - }, + "query": "label_values(nginx_ingress_controller_requests, ingress) ", "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 2, "tagValuesQuery": "", + "tags": [], "tagsQuery": "", "type": "query", "useTags": false @@ -981,6 +977,5 @@ "timezone": "browser", "title": "Request Handling Performance", "uid": "4GFbkOsZk", - "version": 1, - "weekStart": "" + "version": 1 } diff --git a/deploy/grafana/kustomization.yaml b/deploy/grafana/kustomization.yaml index e2744818e..2069c1a7a 100644 --- a/deploy/grafana/kustomization.yaml +++ b/deploy/grafana/kustomization.yaml @@ -9,3 +9,4 @@ resources: - service.yaml images: - name: grafana/grafana + newTag: 6.1.6 diff --git a/deploy/prometheus/kustomization.yaml b/deploy/prometheus/kustomization.yaml index a55bd8d2c..2d6e0a7e2 100644 --- a/deploy/prometheus/kustomization.yaml +++ b/deploy/prometheus/kustomization.yaml @@ -12,6 +12,7 @@ resources: - service.yaml images: - name: prom/prometheus + newTag: v2.3.2 configMapGenerator: - name: prometheus-configuration files: diff --git a/deploy/static/provider/aws/deploy-tls-termination.yaml b/deploy/static/provider/aws/deploy-tls-termination.yaml new file mode 100644 index 000000000..2cb23fe0e --- /dev/null +++ b/deploy/static/provider/aws/deploy-tls-termination.yaml @@ -0,0 +1,671 @@ + +apiVersion: v1 +kind: Namespace +metadata: + name: ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + +--- +# Source: ingress-nginx/templates/controller-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +automountServiceAccountToken: true +--- +# Source: ingress-nginx/templates/controller-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +data: + http-snippet: | + server { + listen 2443; + return 308 https://$host$request_uri; + } + proxy-real-ip-cidr: XXX.XXX.XXX/XX + use-forwarded-headers: 'true' +--- +# Source: ingress-nginx/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch +--- +# Source: ingress-nginx/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - namespaces + verbs: + - get + - apiGroups: + - '' + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - configmaps + resourceNames: + - ingress-controller-leader-nginx + verbs: + - get + - update + - apiGroups: + - '' + resources: + - configmaps + verbs: + - create + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +--- +# Source: ingress-nginx/templates/controller-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-service-webhook.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller-admission + namespace: ingress-nginx +spec: + type: ClusterIP + ports: + - name: https-webhook + port: 443 + targetPort: webhook + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-service.yaml +apiVersion: v1 +kind: Service +metadata: + annotations: + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: '60' + service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: 'true' + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:us-west-2:XXXXXXXX:certificate/XXXXXX-XXXXXXX-XXXXXXX-XXXXXXXX + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: https + service.beta.kubernetes.io/aws-load-balancer-type: elb + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + type: LoadBalancer + externalTrafficPolicy: Local + ports: + - name: http + port: 80 + protocol: TCP + targetPort: tohttps + - name: https + port: 443 + protocol: TCP + targetPort: http + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + selector: + matchLabels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + revisionHistoryLimit: 10 + minReadySeconds: 0 + template: + metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + spec: + dnsPolicy: ClusterFirst + containers: + - name: controller + image: k8s.gcr.io/ingress-nginx/controller:v0.46.0@sha256:52f0058bed0a17ab0fb35628ba97e8d52b5d32299fbc03cc0f6c7b9ff036b61a + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + args: + - /nginx-ingress-controller + - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller + - --election-id=ingress-controller-leader + - --ingress-class=nginx + - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: 101 + allowPrivilegeEscalation: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - name: http + containerPort: 80 + protocol: TCP + - name: https + containerPort: 80 + protocol: TCP + - name: tohttps + containerPort: 2443 + protocol: TCP + - name: webhook + containerPort: 8443 + protocol: TCP + volumeMounts: + - name: webhook-cert + mountPath: /usr/local/certificates/ + readOnly: true + resources: + requests: + cpu: 100m + memory: 90Mi + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: ingress-nginx + terminationGracePeriodSeconds: 300 + volumes: + - name: webhook-cert + secret: + secretName: ingress-nginx-admission +--- +# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml +# before changing this value, check the required kubernetes version +# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + name: ingress-nginx-admission +webhooks: + - name: validate.nginx.ingress.kubernetes.io + matchPolicy: Equivalent + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + namespace: ingress-nginx + name: ingress-nginx-controller-admission + path: /networking/v1beta1/ingresses +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - update +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - create +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-create + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-create + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: create + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - create + - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=ingress-nginx-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-patch + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-patch + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: patch + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - patch + - --webhook-name=ingress-nginx-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=ingress-nginx-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 diff --git a/deploy/static/provider/aws/deploy.yaml b/deploy/static/provider/aws/deploy.yaml index 1ed9b7471..9af4712a4 100644 --- a/deploy/static/provider/aws/deploy.yaml +++ b/deploy/static/provider/aws/deploy.yaml @@ -1,669 +1,658 @@ + apiVersion: v1 kind: Namespace metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx name: ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + --- +# Source: ingress-nginx/templates/controller-serviceaccount.yaml apiVersion: v1 -automountServiceAccountToken: true kind: ServiceAccount metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx namespace: ingress-nginx ---- -apiVersion: v1 automountServiceAccountToken: true -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get -- apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - coordination.k8s.io - resourceNames: - - ingress-nginx-leader - resources: - - leases - verbs: - - get - - update -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - - namespaces - verbs: - - list - - watch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -rules: -- apiGroups: - - admissionregistration.k8s.io - resources: - - validatingwebhookconfigurations - verbs: - - get - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx --- +# Source: ingress-nginx/templates/controller-configmap.yaml apiVersion: v1 -data: null kind: ConfigMap metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx +data: --- +# Source: ingress-nginx/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch +--- +# Source: ingress-nginx/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - namespaces + verbs: + - get + - apiGroups: + - '' + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - configmaps + resourceNames: + - ingress-controller-leader-nginx + verbs: + - get + - update + - apiGroups: + - '' + resources: + - configmaps + verbs: + - create + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +--- +# Source: ingress-nginx/templates/controller-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-service-webhook.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller-admission + namespace: ingress-nginx +spec: + type: ClusterIP + ports: + - name: https-webhook + port: 443 + targetPort: webhook + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-service.yaml apiVersion: v1 kind: Service metadata: annotations: service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp - service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true" + service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: 'true' service.beta.kubernetes.io/aws-load-balancer-type: nlb labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx spec: - externalTrafficPolicy: Local - ipFamilies: - - IPv4 - ipFamilyPolicy: SingleStack - ports: - - appProtocol: http - name: http - port: 80 - protocol: TCP - targetPort: http - - appProtocol: https - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-controller-admission - namespace: ingress-nginx -spec: + externalTrafficPolicy: Local ports: - - appProtocol: https - name: https-webhook - port: 443 - targetPort: webhook + - name: http + port: 80 + protocol: TCP + targetPort: http + - name: https + port: 443 + protocol: TCP + targetPort: https selector: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - type: ClusterIP + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller --- +# Source: ingress-nginx/templates/controller-deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx spec: - minReadySeconds: 0 - revisionHistoryLimit: 10 selector: matchLabels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - strategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + revisionHistoryLimit: 10 + minReadySeconds: 0 template: metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller spec: - containers: - - args: - - /nginx-ingress-controller - - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller - - --election-id=ingress-nginx-leader - - --controller-class=k8s.io/ingress-nginx - - --ingress-class=nginx - - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller - - --validating-webhook=:8443 - - --validating-webhook-certificate=/usr/local/certificates/cert - - --validating-webhook-key=/usr/local/certificates/key - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: LD_PRELOAD - value: /usr/local/lib/libmimalloc.so - image: registry.k8s.io/ingress-nginx/controller:v1.12.0@sha256:e6b8de175acda6ca913891f0f727bca4527e797d52688cbe9fec9040d6f6b6fa - imagePullPolicy: IfNotPresent - lifecycle: - preStop: - exec: - command: - - /wait-shutdown - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - name: controller - ports: - - containerPort: 80 - name: http - protocol: TCP - - containerPort: 443 - name: https - protocol: TCP - - containerPort: 8443 - name: webhook - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 100m - memory: 90Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: false - runAsGroup: 82 - runAsNonRoot: true - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /usr/local/certificates/ - name: webhook-cert - readOnly: true dnsPolicy: ClusterFirst + containers: + - name: controller + image: k8s.gcr.io/ingress-nginx/controller:v0.46.0@sha256:52f0058bed0a17ab0fb35628ba97e8d52b5d32299fbc03cc0f6c7b9ff036b61a + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + args: + - /nginx-ingress-controller + - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller + - --election-id=ingress-controller-leader + - --ingress-class=nginx + - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: 101 + allowPrivilegeEscalation: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - name: http + containerPort: 80 + protocol: TCP + - name: https + containerPort: 443 + protocol: TCP + - name: webhook + containerPort: 8443 + protocol: TCP + volumeMounts: + - name: webhook-cert + mountPath: /usr/local/certificates/ + readOnly: true + resources: + requests: + cpu: 100m + memory: 90Mi nodeSelector: kubernetes.io/os: linux serviceAccountName: ingress-nginx terminationGracePeriodSeconds: 300 volumes: - - name: webhook-cert - secret: - secretName: ingress-nginx-admission ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - spec: - containers: - - args: - - create - - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc - - --namespace=$(POD_NAMESPACE) - - --secret-name=ingress-nginx-admission - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: create - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - spec: - containers: - - args: - - patch - - --webhook-name=ingress-nginx-admission - - --namespace=$(POD_NAMESPACE) - - --patch-mutating=false - - --secret-name=ingress-nginx-admission - - --patch-failure-policy=Fail - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: patch - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: nginx -spec: - controller: k8s.io/ingress-nginx + - name: webhook-cert + secret: + secretName: ingress-nginx-admission --- +# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml +# before changing this value, check the required kubernetes version +# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook name: ingress-nginx-admission webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: ingress-nginx-controller-admission - namespace: ingress-nginx - path: /networking/v1/ingresses - port: 443 - failurePolicy: Fail - matchPolicy: Equivalent - name: validate.nginx.ingress.kubernetes.io - rules: + - name: validate.nginx.ingress.kubernetes.io + matchPolicy: Equivalent + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + namespace: ingress-nginx + name: ingress-nginx-controller-admission + path: /networking/v1beta1/ingresses +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +rules: - apiGroups: - - networking.k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE + - admissionregistration.k8s.io resources: - - ingresses - sideEffects: None + - validatingwebhookconfigurations + verbs: + - get + - update +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - create +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-create + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-create + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: create + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - create + - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=ingress-nginx-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-patch + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-patch + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: patch + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - patch + - --webhook-name=ingress-nginx-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=ingress-nginx-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 diff --git a/deploy/static/provider/aws/kustomization.yaml b/deploy/static/provider/aws/kustomization.yaml index 18c6bb6a3..d18ef3896 100644 --- a/deploy/static/provider/aws/kustomization.yaml +++ b/deploy/static/provider/aws/kustomization.yaml @@ -4,7 +4,7 @@ # ``` # namespace: ingress-nginx # bases: -# - github.com/kubernetes/ingress-nginx/tree/main/deploy/static/provider/aws +# - github.com/kubernetes/ingress-nginx/deploy/static/provider/aws?ref=master # ``` resources: diff --git a/deploy/static/provider/aws/nlb-with-tls-termination/deploy.yaml b/deploy/static/provider/aws/nlb-with-tls-termination/deploy.yaml deleted file mode 100644 index d048683d9..000000000 --- a/deploy/static/provider/aws/nlb-with-tls-termination/deploy.yaml +++ /dev/null @@ -1,681 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - name: ingress-nginx ---- -apiVersion: v1 -automountServiceAccountToken: true -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: v1 -automountServiceAccountToken: true -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get -- apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - coordination.k8s.io - resourceNames: - - ingress-nginx-leader - resources: - - leases - verbs: - - get - - update -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - - namespaces - verbs: - - list - - watch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -rules: -- apiGroups: - - admissionregistration.k8s.io - resources: - - validatingwebhookconfigurations - verbs: - - get - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: v1 -data: - http-snippet: | - server { - listen 2443; - return 308 https://$host$request_uri; - } - proxy-real-ip-cidr: XXX.XXX.XXX/XX - use-forwarded-headers: "true" -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-controller - namespace: ingress-nginx ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "60" - service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true" - service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:us-west-2:XXXXXXXX:certificate/XXXXXX-XXXXXXX-XXXXXXX-XXXXXXXX - service.beta.kubernetes.io/aws-load-balancer-ssl-ports: https - service.beta.kubernetes.io/aws-load-balancer-type: nlb - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-controller - namespace: ingress-nginx -spec: - externalTrafficPolicy: Local - ipFamilies: - - IPv4 - ipFamilyPolicy: SingleStack - ports: - - appProtocol: http - name: http - port: 80 - protocol: TCP - targetPort: tohttps - - appProtocol: https - name: https - port: 443 - protocol: TCP - targetPort: http - selector: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-controller-admission - namespace: ingress-nginx -spec: - ports: - - appProtocol: https - name: https-webhook - port: 443 - targetPort: webhook - selector: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - type: ClusterIP ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-controller - namespace: ingress-nginx -spec: - minReadySeconds: 0 - revisionHistoryLimit: 10 - selector: - matchLabels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - strategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate - template: - metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - spec: - containers: - - args: - - /nginx-ingress-controller - - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller - - --election-id=ingress-nginx-leader - - --controller-class=k8s.io/ingress-nginx - - --ingress-class=nginx - - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller - - --validating-webhook=:8443 - - --validating-webhook-certificate=/usr/local/certificates/cert - - --validating-webhook-key=/usr/local/certificates/key - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: LD_PRELOAD - value: /usr/local/lib/libmimalloc.so - image: registry.k8s.io/ingress-nginx/controller:v1.12.0@sha256:e6b8de175acda6ca913891f0f727bca4527e797d52688cbe9fec9040d6f6b6fa - imagePullPolicy: IfNotPresent - lifecycle: - preStop: - exec: - command: - - /wait-shutdown - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - name: controller - ports: - - containerPort: 80 - name: http - protocol: TCP - - containerPort: 80 - name: https - protocol: TCP - - containerPort: 2443 - name: tohttps - protocol: TCP - - containerPort: 8443 - name: webhook - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 100m - memory: 90Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: false - runAsGroup: 82 - runAsNonRoot: true - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /usr/local/certificates/ - name: webhook-cert - readOnly: true - dnsPolicy: ClusterFirst - nodeSelector: - kubernetes.io/os: linux - serviceAccountName: ingress-nginx - terminationGracePeriodSeconds: 300 - volumes: - - name: webhook-cert - secret: - secretName: ingress-nginx-admission ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - spec: - containers: - - args: - - create - - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc - - --namespace=$(POD_NAMESPACE) - - --secret-name=ingress-nginx-admission - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: create - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - spec: - containers: - - args: - - patch - - --webhook-name=ingress-nginx-admission - - --namespace=$(POD_NAMESPACE) - - --patch-mutating=false - - --secret-name=ingress-nginx-admission - - --patch-failure-policy=Fail - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: patch - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: nginx -spec: - controller: k8s.io/ingress-nginx ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: ingress-nginx-controller-admission - namespace: ingress-nginx - path: /networking/v1/ingresses - port: 443 - failurePolicy: Fail - matchPolicy: Equivalent - name: validate.nginx.ingress.kubernetes.io - rules: - - apiGroups: - - networking.k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - ingresses - sideEffects: None diff --git a/deploy/static/provider/aws/nlb-with-tls-termination/kustomization.yaml b/deploy/static/provider/aws/nlb-with-tls-termination/kustomization.yaml deleted file mode 100644 index 51c1513c9..000000000 --- a/deploy/static/provider/aws/nlb-with-tls-termination/kustomization.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# NOTE: kustomize is not supported. This file exists only to be able to reference it from bases. -# https://kubectl.docs.kubernetes.io/references/kustomize/bases/ -# -# ``` -# namespace: ingress-nginx -# bases: -# - github.com/kubernetes/ingress-nginx/tree/main/deploy/static/provider/aws/nlb-with-tls-termination -# ``` - -resources: - - deploy.yaml diff --git a/deploy/static/provider/baremetal/deploy.yaml b/deploy/static/provider/baremetal/deploy.yaml index 7248dfd9b..bd5cdab43 100644 --- a/deploy/static/provider/baremetal/deploy.yaml +++ b/deploy/static/provider/baremetal/deploy.yaml @@ -1,663 +1,653 @@ + apiVersion: v1 kind: Namespace metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx name: ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + --- +# Source: ingress-nginx/templates/controller-serviceaccount.yaml apiVersion: v1 -automountServiceAccountToken: true kind: ServiceAccount metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx namespace: ingress-nginx ---- -apiVersion: v1 automountServiceAccountToken: true -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get -- apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - coordination.k8s.io - resourceNames: - - ingress-nginx-leader - resources: - - leases - verbs: - - get - - update -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - - namespaces - verbs: - - list - - watch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -rules: -- apiGroups: - - admissionregistration.k8s.io - resources: - - validatingwebhookconfigurations - verbs: - - get - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx --- +# Source: ingress-nginx/templates/controller-configmap.yaml apiVersion: v1 -data: null kind: ConfigMap metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx +data: --- +# Source: ingress-nginx/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch +--- +# Source: ingress-nginx/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - namespaces + verbs: + - get + - apiGroups: + - '' + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - configmaps + resourceNames: + - ingress-controller-leader-nginx + verbs: + - get + - update + - apiGroups: + - '' + resources: + - configmaps + verbs: + - create + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +--- +# Source: ingress-nginx/templates/controller-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-service-webhook.yaml apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-controller - namespace: ingress-nginx -spec: - ipFamilies: - - IPv4 - ipFamilyPolicy: SingleStack - ports: - - appProtocol: http - name: http - port: 80 - protocol: TCP - targetPort: http - - appProtocol: https - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - app.kubernetes.io/component: controller app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - type: NodePort ---- -apiVersion: v1 -kind: Service -metadata: - labels: + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 name: ingress-nginx-controller-admission namespace: ingress-nginx spec: - ports: - - appProtocol: https - name: https-webhook - port: 443 - targetPort: webhook - selector: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx type: ClusterIP + ports: + - name: https-webhook + port: 443 + targetPort: webhook + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller --- +# Source: ingress-nginx/templates/controller-service.yaml +apiVersion: v1 +kind: Service +metadata: + annotations: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + type: NodePort + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx spec: - minReadySeconds: 0 - revisionHistoryLimit: 10 selector: matchLabels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - strategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + revisionHistoryLimit: 10 + minReadySeconds: 0 template: metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller spec: - containers: - - args: - - /nginx-ingress-controller - - --election-id=ingress-nginx-leader - - --controller-class=k8s.io/ingress-nginx - - --ingress-class=nginx - - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller - - --validating-webhook=:8443 - - --validating-webhook-certificate=/usr/local/certificates/cert - - --validating-webhook-key=/usr/local/certificates/key - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: LD_PRELOAD - value: /usr/local/lib/libmimalloc.so - image: registry.k8s.io/ingress-nginx/controller:v1.12.0@sha256:e6b8de175acda6ca913891f0f727bca4527e797d52688cbe9fec9040d6f6b6fa - imagePullPolicy: IfNotPresent - lifecycle: - preStop: - exec: - command: - - /wait-shutdown - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - name: controller - ports: - - containerPort: 80 - name: http - protocol: TCP - - containerPort: 443 - name: https - protocol: TCP - - containerPort: 8443 - name: webhook - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 100m - memory: 90Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: false - runAsGroup: 82 - runAsNonRoot: true - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /usr/local/certificates/ - name: webhook-cert - readOnly: true dnsPolicy: ClusterFirst + containers: + - name: controller + image: k8s.gcr.io/ingress-nginx/controller:v0.46.0@sha256:52f0058bed0a17ab0fb35628ba97e8d52b5d32299fbc03cc0f6c7b9ff036b61a + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + args: + - /nginx-ingress-controller + - --election-id=ingress-controller-leader + - --ingress-class=nginx + - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: 101 + allowPrivilegeEscalation: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - name: http + containerPort: 80 + protocol: TCP + - name: https + containerPort: 443 + protocol: TCP + - name: webhook + containerPort: 8443 + protocol: TCP + volumeMounts: + - name: webhook-cert + mountPath: /usr/local/certificates/ + readOnly: true + resources: + requests: + cpu: 100m + memory: 90Mi nodeSelector: kubernetes.io/os: linux serviceAccountName: ingress-nginx terminationGracePeriodSeconds: 300 volumes: - - name: webhook-cert - secret: - secretName: ingress-nginx-admission ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - spec: - containers: - - args: - - create - - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc - - --namespace=$(POD_NAMESPACE) - - --secret-name=ingress-nginx-admission - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: create - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - spec: - containers: - - args: - - patch - - --webhook-name=ingress-nginx-admission - - --namespace=$(POD_NAMESPACE) - - --patch-mutating=false - - --secret-name=ingress-nginx-admission - - --patch-failure-policy=Fail - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: patch - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: nginx -spec: - controller: k8s.io/ingress-nginx + - name: webhook-cert + secret: + secretName: ingress-nginx-admission --- +# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml +# before changing this value, check the required kubernetes version +# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook name: ingress-nginx-admission webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: ingress-nginx-controller-admission - namespace: ingress-nginx - path: /networking/v1/ingresses - port: 443 - failurePolicy: Fail - matchPolicy: Equivalent - name: validate.nginx.ingress.kubernetes.io - rules: + - name: validate.nginx.ingress.kubernetes.io + matchPolicy: Equivalent + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + namespace: ingress-nginx + name: ingress-nginx-controller-admission + path: /networking/v1beta1/ingresses +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +rules: - apiGroups: - - networking.k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE + - admissionregistration.k8s.io resources: - - ingresses - sideEffects: None + - validatingwebhookconfigurations + verbs: + - get + - update +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - create +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-create + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-create + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: create + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - create + - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=ingress-nginx-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-patch + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-patch + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: patch + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - patch + - --webhook-name=ingress-nginx-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=ingress-nginx-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 diff --git a/deploy/static/provider/baremetal/kustomization.yaml b/deploy/static/provider/baremetal/kustomization.yaml index d585f85c6..d13af0f82 100644 --- a/deploy/static/provider/baremetal/kustomization.yaml +++ b/deploy/static/provider/baremetal/kustomization.yaml @@ -4,7 +4,7 @@ # ``` # namespace: ingress-nginx # bases: -# - github.com/kubernetes/ingress-nginx/tree/main/deploy/static/provider/baremetal +# - github.com/kubernetes/ingress-nginx/deploy/static/provider/baremetal?ref=master # ``` resources: diff --git a/deploy/static/provider/cloud/deploy.yaml b/deploy/static/provider/cloud/deploy.yaml index 08bc7e4bf..1d2a459a4 100644 --- a/deploy/static/provider/cloud/deploy.yaml +++ b/deploy/static/provider/cloud/deploy.yaml @@ -1,665 +1,655 @@ + apiVersion: v1 kind: Namespace metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx name: ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + --- +# Source: ingress-nginx/templates/controller-serviceaccount.yaml apiVersion: v1 -automountServiceAccountToken: true kind: ServiceAccount metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx namespace: ingress-nginx ---- -apiVersion: v1 automountServiceAccountToken: true -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get -- apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - coordination.k8s.io - resourceNames: - - ingress-nginx-leader - resources: - - leases - verbs: - - get - - update -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - - namespaces - verbs: - - list - - watch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -rules: -- apiGroups: - - admissionregistration.k8s.io - resources: - - validatingwebhookconfigurations - verbs: - - get - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx --- +# Source: ingress-nginx/templates/controller-configmap.yaml apiVersion: v1 -data: null kind: ConfigMap metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx +data: --- +# Source: ingress-nginx/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch +--- +# Source: ingress-nginx/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - namespaces + verbs: + - get + - apiGroups: + - '' + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - configmaps + resourceNames: + - ingress-controller-leader-nginx + verbs: + - get + - update + - apiGroups: + - '' + resources: + - configmaps + verbs: + - create + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +--- +# Source: ingress-nginx/templates/controller-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-service-webhook.yaml apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-controller - namespace: ingress-nginx -spec: - externalTrafficPolicy: Local - ipFamilies: - - IPv4 - ipFamilyPolicy: SingleStack - ports: - - appProtocol: http - name: http - port: 80 - protocol: TCP - targetPort: http - - appProtocol: https - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - app.kubernetes.io/component: controller app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - labels: + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 name: ingress-nginx-controller-admission namespace: ingress-nginx spec: - ports: - - appProtocol: https - name: https-webhook - port: 443 - targetPort: webhook - selector: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx type: ClusterIP + ports: + - name: https-webhook + port: 443 + targetPort: webhook + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller --- +# Source: ingress-nginx/templates/controller-service.yaml +apiVersion: v1 +kind: Service +metadata: + annotations: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + type: LoadBalancer + externalTrafficPolicy: Local + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx spec: - minReadySeconds: 0 - revisionHistoryLimit: 10 selector: matchLabels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - strategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + revisionHistoryLimit: 10 + minReadySeconds: 0 template: metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller spec: - containers: - - args: - - /nginx-ingress-controller - - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller - - --election-id=ingress-nginx-leader - - --controller-class=k8s.io/ingress-nginx - - --ingress-class=nginx - - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller - - --validating-webhook=:8443 - - --validating-webhook-certificate=/usr/local/certificates/cert - - --validating-webhook-key=/usr/local/certificates/key - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: LD_PRELOAD - value: /usr/local/lib/libmimalloc.so - image: registry.k8s.io/ingress-nginx/controller:v1.12.0@sha256:e6b8de175acda6ca913891f0f727bca4527e797d52688cbe9fec9040d6f6b6fa - imagePullPolicy: IfNotPresent - lifecycle: - preStop: - exec: - command: - - /wait-shutdown - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - name: controller - ports: - - containerPort: 80 - name: http - protocol: TCP - - containerPort: 443 - name: https - protocol: TCP - - containerPort: 8443 - name: webhook - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 100m - memory: 90Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: false - runAsGroup: 82 - runAsNonRoot: true - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /usr/local/certificates/ - name: webhook-cert - readOnly: true dnsPolicy: ClusterFirst + containers: + - name: controller + image: k8s.gcr.io/ingress-nginx/controller:v0.46.0@sha256:52f0058bed0a17ab0fb35628ba97e8d52b5d32299fbc03cc0f6c7b9ff036b61a + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + args: + - /nginx-ingress-controller + - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller + - --election-id=ingress-controller-leader + - --ingress-class=nginx + - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: 101 + allowPrivilegeEscalation: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - name: http + containerPort: 80 + protocol: TCP + - name: https + containerPort: 443 + protocol: TCP + - name: webhook + containerPort: 8443 + protocol: TCP + volumeMounts: + - name: webhook-cert + mountPath: /usr/local/certificates/ + readOnly: true + resources: + requests: + cpu: 100m + memory: 90Mi nodeSelector: kubernetes.io/os: linux serviceAccountName: ingress-nginx terminationGracePeriodSeconds: 300 volumes: - - name: webhook-cert - secret: - secretName: ingress-nginx-admission ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - spec: - containers: - - args: - - create - - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc - - --namespace=$(POD_NAMESPACE) - - --secret-name=ingress-nginx-admission - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: create - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - spec: - containers: - - args: - - patch - - --webhook-name=ingress-nginx-admission - - --namespace=$(POD_NAMESPACE) - - --patch-mutating=false - - --secret-name=ingress-nginx-admission - - --patch-failure-policy=Fail - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: patch - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: nginx -spec: - controller: k8s.io/ingress-nginx + - name: webhook-cert + secret: + secretName: ingress-nginx-admission --- +# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml +# before changing this value, check the required kubernetes version +# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook name: ingress-nginx-admission webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: ingress-nginx-controller-admission - namespace: ingress-nginx - path: /networking/v1/ingresses - port: 443 - failurePolicy: Fail - matchPolicy: Equivalent - name: validate.nginx.ingress.kubernetes.io - rules: + - name: validate.nginx.ingress.kubernetes.io + matchPolicy: Equivalent + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + namespace: ingress-nginx + name: ingress-nginx-controller-admission + path: /networking/v1beta1/ingresses +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +rules: - apiGroups: - - networking.k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE + - admissionregistration.k8s.io resources: - - ingresses - sideEffects: None + - validatingwebhookconfigurations + verbs: + - get + - update +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - create +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-create + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-create + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: create + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - create + - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=ingress-nginx-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-patch + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-patch + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: patch + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - patch + - --webhook-name=ingress-nginx-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=ingress-nginx-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 diff --git a/deploy/static/provider/cloud/kustomization.yaml b/deploy/static/provider/cloud/kustomization.yaml index d477ec405..a92167fad 100644 --- a/deploy/static/provider/cloud/kustomization.yaml +++ b/deploy/static/provider/cloud/kustomization.yaml @@ -4,7 +4,7 @@ # ``` # namespace: ingress-nginx # bases: -# - github.com/kubernetes/ingress-nginx/tree/main/deploy/static/provider/cloud +# - github.com/kubernetes/ingress-nginx/deploy/static/provider/cloud?ref=master # ``` resources: diff --git a/deploy/static/provider/do/deploy.yaml b/deploy/static/provider/do/deploy.yaml index 4745c64fc..722f4199f 100644 --- a/deploy/static/provider/do/deploy.yaml +++ b/deploy/static/provider/do/deploy.yaml @@ -1,669 +1,657 @@ + apiVersion: v1 kind: Namespace metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx name: ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + --- +# Source: ingress-nginx/templates/controller-serviceaccount.yaml apiVersion: v1 -automountServiceAccountToken: true kind: ServiceAccount metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx namespace: ingress-nginx ---- -apiVersion: v1 automountServiceAccountToken: true -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get -- apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - coordination.k8s.io - resourceNames: - - ingress-nginx-leader - resources: - - leases - verbs: - - get - - update -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - - namespaces - verbs: - - list - - watch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -rules: -- apiGroups: - - admissionregistration.k8s.io - resources: - - validatingwebhookconfigurations - verbs: - - get - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx --- +# Source: ingress-nginx/templates/controller-configmap.yaml apiVersion: v1 -data: - use-proxy-protocol: "true" kind: ConfigMap metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx +data: + use-proxy-protocol: 'true' --- +# Source: ingress-nginx/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch +--- +# Source: ingress-nginx/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - namespaces + verbs: + - get + - apiGroups: + - '' + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - configmaps + resourceNames: + - ingress-controller-leader-nginx + verbs: + - get + - update + - apiGroups: + - '' + resources: + - configmaps + verbs: + - create + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +--- +# Source: ingress-nginx/templates/controller-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-service-webhook.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller-admission + namespace: ingress-nginx +spec: + type: ClusterIP + ports: + - name: https-webhook + port: 443 + targetPort: webhook + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-service.yaml apiVersion: v1 kind: Service metadata: annotations: - service.beta.kubernetes.io/do-loadbalancer-enable-proxy-protocol: "true" + service.beta.kubernetes.io/do-loadbalancer-enable-proxy-protocol: 'true' labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx spec: - externalTrafficPolicy: Local - ipFamilies: - - IPv4 - ipFamilyPolicy: SingleStack - ports: - - appProtocol: http - name: http - port: 80 - protocol: TCP - targetPort: http - - appProtocol: https - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-controller-admission - namespace: ingress-nginx -spec: + externalTrafficPolicy: Local ports: - - appProtocol: https - name: https-webhook - port: 443 - targetPort: webhook + - name: http + port: 80 + protocol: TCP + targetPort: http + - name: https + port: 443 + protocol: TCP + targetPort: https selector: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - type: ClusterIP + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller --- +# Source: ingress-nginx/templates/controller-deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx spec: - minReadySeconds: 0 - revisionHistoryLimit: 10 selector: matchLabels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - strategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + revisionHistoryLimit: 10 + minReadySeconds: 0 template: metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller spec: - containers: - - args: - - /nginx-ingress-controller - - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller - - --election-id=ingress-nginx-leader - - --controller-class=k8s.io/ingress-nginx - - --ingress-class=nginx - - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller - - --validating-webhook=:8443 - - --validating-webhook-certificate=/usr/local/certificates/cert - - --validating-webhook-key=/usr/local/certificates/key - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: LD_PRELOAD - value: /usr/local/lib/libmimalloc.so - image: registry.k8s.io/ingress-nginx/controller:v1.12.0@sha256:e6b8de175acda6ca913891f0f727bca4527e797d52688cbe9fec9040d6f6b6fa - imagePullPolicy: IfNotPresent - lifecycle: - preStop: - exec: - command: - - /wait-shutdown - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - name: controller - ports: - - containerPort: 80 - name: http - protocol: TCP - - containerPort: 443 - name: https - protocol: TCP - - containerPort: 8443 - name: webhook - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 100m - memory: 90Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: false - runAsGroup: 82 - runAsNonRoot: true - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /usr/local/certificates/ - name: webhook-cert - readOnly: true dnsPolicy: ClusterFirst + containers: + - name: controller + image: k8s.gcr.io/ingress-nginx/controller:v0.46.0@sha256:52f0058bed0a17ab0fb35628ba97e8d52b5d32299fbc03cc0f6c7b9ff036b61a + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + args: + - /nginx-ingress-controller + - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller + - --election-id=ingress-controller-leader + - --ingress-class=nginx + - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: 101 + allowPrivilegeEscalation: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - name: http + containerPort: 80 + protocol: TCP + - name: https + containerPort: 443 + protocol: TCP + - name: webhook + containerPort: 8443 + protocol: TCP + volumeMounts: + - name: webhook-cert + mountPath: /usr/local/certificates/ + readOnly: true + resources: + requests: + cpu: 100m + memory: 90Mi nodeSelector: kubernetes.io/os: linux serviceAccountName: ingress-nginx terminationGracePeriodSeconds: 300 volumes: - - name: webhook-cert - secret: - secretName: ingress-nginx-admission ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - spec: - containers: - - args: - - create - - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc - - --namespace=$(POD_NAMESPACE) - - --secret-name=ingress-nginx-admission - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: create - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - spec: - containers: - - args: - - patch - - --webhook-name=ingress-nginx-admission - - --namespace=$(POD_NAMESPACE) - - --patch-mutating=false - - --secret-name=ingress-nginx-admission - - --patch-failure-policy=Fail - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: patch - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: nginx -spec: - controller: k8s.io/ingress-nginx + - name: webhook-cert + secret: + secretName: ingress-nginx-admission --- +# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml +# before changing this value, check the required kubernetes version +# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook name: ingress-nginx-admission webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: ingress-nginx-controller-admission - namespace: ingress-nginx - path: /networking/v1/ingresses - port: 443 - failurePolicy: Fail - matchPolicy: Equivalent - name: validate.nginx.ingress.kubernetes.io - rules: + - name: validate.nginx.ingress.kubernetes.io + matchPolicy: Equivalent + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + namespace: ingress-nginx + name: ingress-nginx-controller-admission + path: /networking/v1beta1/ingresses +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +rules: - apiGroups: - - networking.k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE + - admissionregistration.k8s.io resources: - - ingresses - sideEffects: None - timeoutSeconds: 29 + - validatingwebhookconfigurations + verbs: + - get + - update +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - create +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-create + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-create + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: create + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - create + - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=ingress-nginx-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-patch + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-patch + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: patch + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - patch + - --webhook-name=ingress-nginx-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=ingress-nginx-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 diff --git a/deploy/static/provider/do/kustomization.yaml b/deploy/static/provider/do/kustomization.yaml index f20d445c7..c7bb8d250 100644 --- a/deploy/static/provider/do/kustomization.yaml +++ b/deploy/static/provider/do/kustomization.yaml @@ -4,7 +4,7 @@ # ``` # namespace: ingress-nginx # bases: -# - github.com/kubernetes/ingress-nginx/tree/main/deploy/static/provider/do +# - github.com/kubernetes/ingress-nginx/deploy/static/provider/do?ref=master # ``` resources: diff --git a/deploy/static/provider/exoscale/deploy.yaml b/deploy/static/provider/exoscale/deploy.yaml index 8ed5d002c..c05a916b5 100644 --- a/deploy/static/provider/exoscale/deploy.yaml +++ b/deploy/static/provider/exoscale/deploy.yaml @@ -1,674 +1,663 @@ + apiVersion: v1 kind: Namespace metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx name: ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + --- +# Source: ingress-nginx/templates/controller-serviceaccount.yaml apiVersion: v1 -automountServiceAccountToken: true kind: ServiceAccount metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx namespace: ingress-nginx ---- -apiVersion: v1 automountServiceAccountToken: true -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get -- apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - coordination.k8s.io - resourceNames: - - ingress-nginx-leader - resources: - - leases - verbs: - - get - - update -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - - namespaces - verbs: - - list - - watch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -rules: -- apiGroups: - - admissionregistration.k8s.io - resources: - - validatingwebhookconfigurations - verbs: - - get - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx --- +# Source: ingress-nginx/templates/controller-configmap.yaml apiVersion: v1 -data: null kind: ConfigMap metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx +data: --- +# Source: ingress-nginx/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch +--- +# Source: ingress-nginx/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - namespaces + verbs: + - get + - apiGroups: + - '' + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - configmaps + resourceNames: + - ingress-controller-leader-nginx + verbs: + - get + - update + - apiGroups: + - '' + resources: + - configmaps + verbs: + - create + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +--- +# Source: ingress-nginx/templates/controller-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-service-webhook.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller-admission + namespace: ingress-nginx +spec: + type: ClusterIP + ports: + - name: https-webhook + port: 443 + targetPort: webhook + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-service.yaml apiVersion: v1 kind: Service metadata: annotations: service.beta.kubernetes.io/exoscale-loadbalancer-description: NGINX Ingress Controller load balancer + service.beta.kubernetes.io/exoscale-loadbalancer-name: nginx-ingress-controller service.beta.kubernetes.io/exoscale-loadbalancer-service-healthcheck-interval: 10s - service.beta.kubernetes.io/exoscale-loadbalancer-service-healthcheck-mode: http - service.beta.kubernetes.io/exoscale-loadbalancer-service-healthcheck-retries: "1" + service.beta.kubernetes.io/exoscale-loadbalancer-service-healthcheck-mode: tcp + service.beta.kubernetes.io/exoscale-loadbalancer-service-healthcheck-retries: '1' service.beta.kubernetes.io/exoscale-loadbalancer-service-healthcheck-timeout: 3s - service.beta.kubernetes.io/exoscale-loadbalancer-service-healthcheck-uri: / service.beta.kubernetes.io/exoscale-loadbalancer-service-strategy: source-hash labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx spec: - externalTrafficPolicy: Local - ipFamilies: - - IPv4 - ipFamilyPolicy: SingleStack - ports: - - appProtocol: http - name: http - port: 80 - protocol: TCP - targetPort: http - - appProtocol: https - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-controller-admission - namespace: ingress-nginx -spec: + externalTrafficPolicy: Local ports: - - appProtocol: https - name: https-webhook - port: 443 - targetPort: webhook + - name: http + port: 80 + protocol: TCP + targetPort: http + - name: https + port: 443 + protocol: TCP + targetPort: https selector: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - type: ClusterIP + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller --- +# Source: ingress-nginx/templates/controller-daemonset.yaml apiVersion: apps/v1 kind: DaemonSet metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx spec: - minReadySeconds: 0 - revisionHistoryLimit: 10 selector: matchLabels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + revisionHistoryLimit: 10 + minReadySeconds: 0 template: metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller spec: - containers: - - args: - - /nginx-ingress-controller - - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller - - --election-id=ingress-nginx-leader - - --controller-class=k8s.io/ingress-nginx - - --ingress-class=nginx - - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller - - --validating-webhook=:8443 - - --validating-webhook-certificate=/usr/local/certificates/cert - - --validating-webhook-key=/usr/local/certificates/key - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: LD_PRELOAD - value: /usr/local/lib/libmimalloc.so - image: registry.k8s.io/ingress-nginx/controller:v1.12.0@sha256:e6b8de175acda6ca913891f0f727bca4527e797d52688cbe9fec9040d6f6b6fa - imagePullPolicy: IfNotPresent - lifecycle: - preStop: - exec: - command: - - /wait-shutdown - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - name: controller - ports: - - containerPort: 80 - name: http - protocol: TCP - - containerPort: 443 - name: https - protocol: TCP - - containerPort: 8443 - name: webhook - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 100m - memory: 90Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: false - runAsGroup: 82 - runAsNonRoot: true - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /usr/local/certificates/ - name: webhook-cert - readOnly: true dnsPolicy: ClusterFirst + containers: + - name: controller + image: k8s.gcr.io/ingress-nginx/controller:v0.46.0@sha256:52f0058bed0a17ab0fb35628ba97e8d52b5d32299fbc03cc0f6c7b9ff036b61a + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + args: + - /nginx-ingress-controller + - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller + - --election-id=ingress-controller-leader + - --ingress-class=nginx + - --configmap=ingress-nginx/ingress-nginx-controller + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: 101 + allowPrivilegeEscalation: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - name: http + containerPort: 80 + protocol: TCP + - name: https + containerPort: 443 + protocol: TCP + - name: webhook + containerPort: 8443 + protocol: TCP + volumeMounts: + - name: webhook-cert + mountPath: /usr/local/certificates/ + readOnly: true + resources: + requests: + cpu: 100m + memory: 90Mi nodeSelector: kubernetes.io/os: linux serviceAccountName: ingress-nginx terminationGracePeriodSeconds: 300 volumes: - - name: webhook-cert - secret: - secretName: ingress-nginx-admission - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - spec: - containers: - - args: - - create - - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc - - --namespace=$(POD_NAMESPACE) - - --secret-name=ingress-nginx-admission - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: create - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - spec: - containers: - - args: - - patch - - --webhook-name=ingress-nginx-admission - - --namespace=$(POD_NAMESPACE) - - --patch-mutating=false - - --secret-name=ingress-nginx-admission - - --patch-failure-policy=Fail - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: patch - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: nginx -spec: - controller: k8s.io/ingress-nginx + - name: webhook-cert + secret: + secretName: ingress-nginx-admission --- +# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml +# before changing this value, check the required kubernetes version +# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook name: ingress-nginx-admission webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: ingress-nginx-controller-admission - namespace: ingress-nginx - path: /networking/v1/ingresses - port: 443 - failurePolicy: Fail - matchPolicy: Equivalent - name: validate.nginx.ingress.kubernetes.io - rules: + - name: validate.nginx.ingress.kubernetes.io + matchPolicy: Equivalent + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + namespace: ingress-nginx + name: ingress-nginx-controller-admission + path: /networking/v1beta1/ingresses +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +rules: - apiGroups: - - networking.k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE + - admissionregistration.k8s.io resources: - - ingresses - sideEffects: None + - validatingwebhookconfigurations + verbs: + - get + - update +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - create +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-create + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-create + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: create + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - create + - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=ingress-nginx-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-patch + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-patch + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: patch + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - patch + - --webhook-name=ingress-nginx-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=ingress-nginx-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 diff --git a/deploy/static/provider/exoscale/kustomization.yaml b/deploy/static/provider/exoscale/kustomization.yaml index e79016cf3..8466627e2 100644 --- a/deploy/static/provider/exoscale/kustomization.yaml +++ b/deploy/static/provider/exoscale/kustomization.yaml @@ -4,7 +4,7 @@ # ``` # namespace: ingress-nginx # bases: -# - github.com/kubernetes/ingress-nginx/tree/main/deploy/static/provider/exoscale +# - github.com/kubernetes/ingress-nginx/deploy/static/provider/exoscale?ref=master # ``` resources: diff --git a/deploy/static/provider/kind/deploy.yaml b/deploy/static/provider/kind/deploy.yaml index b7488f046..066bbad85 100644 --- a/deploy/static/provider/kind/deploy.yaml +++ b/deploy/static/provider/kind/deploy.yaml @@ -1,675 +1,665 @@ + apiVersion: v1 kind: Namespace metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx name: ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + --- +# Source: ingress-nginx/templates/controller-serviceaccount.yaml apiVersion: v1 -automountServiceAccountToken: true kind: ServiceAccount metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx namespace: ingress-nginx ---- -apiVersion: v1 automountServiceAccountToken: true -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get -- apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - coordination.k8s.io - resourceNames: - - ingress-nginx-leader - resources: - - leases - verbs: - - get - - update -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - - namespaces - verbs: - - list - - watch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -rules: -- apiGroups: - - admissionregistration.k8s.io - resources: - - validatingwebhookconfigurations - verbs: - - get - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx --- +# Source: ingress-nginx/templates/controller-configmap.yaml apiVersion: v1 -data: null kind: ConfigMap metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx +data: --- +# Source: ingress-nginx/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch +--- +# Source: ingress-nginx/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - namespaces + verbs: + - get + - apiGroups: + - '' + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - configmaps + resourceNames: + - ingress-controller-leader-nginx + verbs: + - get + - update + - apiGroups: + - '' + resources: + - configmaps + verbs: + - create + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +--- +# Source: ingress-nginx/templates/controller-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-service-webhook.yaml apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-controller - namespace: ingress-nginx -spec: - ipFamilies: - - IPv4 - ipFamilyPolicy: SingleStack - ports: - - appProtocol: http - name: http - port: 80 - protocol: TCP - targetPort: http - - appProtocol: https - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - app.kubernetes.io/component: controller app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - type: NodePort ---- -apiVersion: v1 -kind: Service -metadata: - labels: + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 name: ingress-nginx-controller-admission namespace: ingress-nginx spec: - ports: - - appProtocol: https - name: https-webhook - port: 443 - targetPort: webhook - selector: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx type: ClusterIP + ports: + - name: https-webhook + port: 443 + targetPort: webhook + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller --- +# Source: ingress-nginx/templates/controller-service.yaml +apiVersion: v1 +kind: Service +metadata: + annotations: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + type: NodePort + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx spec: - minReadySeconds: 0 - revisionHistoryLimit: 10 selector: matchLabels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + revisionHistoryLimit: 10 strategy: rollingUpdate: maxUnavailable: 1 type: RollingUpdate + minReadySeconds: 0 template: metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller spec: - containers: - - args: - - /nginx-ingress-controller - - --election-id=ingress-nginx-leader - - --controller-class=k8s.io/ingress-nginx - - --ingress-class=nginx - - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller - - --validating-webhook=:8443 - - --validating-webhook-certificate=/usr/local/certificates/cert - - --validating-webhook-key=/usr/local/certificates/key - - --watch-ingress-without-class=true - - --publish-status-address=localhost - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: LD_PRELOAD - value: /usr/local/lib/libmimalloc.so - image: registry.k8s.io/ingress-nginx/controller:v1.12.0@sha256:e6b8de175acda6ca913891f0f727bca4527e797d52688cbe9fec9040d6f6b6fa - imagePullPolicy: IfNotPresent - lifecycle: - preStop: - exec: - command: - - /wait-shutdown - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - name: controller - ports: - - containerPort: 80 - hostPort: 80 - name: http - protocol: TCP - - containerPort: 443 - hostPort: 443 - name: https - protocol: TCP - - containerPort: 8443 - name: webhook - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 100m - memory: 90Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: false - runAsGroup: 82 - runAsNonRoot: true - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /usr/local/certificates/ - name: webhook-cert - readOnly: true dnsPolicy: ClusterFirst + containers: + - name: controller + image: k8s.gcr.io/ingress-nginx/controller:v0.46.0@sha256:52f0058bed0a17ab0fb35628ba97e8d52b5d32299fbc03cc0f6c7b9ff036b61a + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + args: + - /nginx-ingress-controller + - --election-id=ingress-controller-leader + - --ingress-class=nginx + - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key + - --publish-status-address=localhost + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: 101 + allowPrivilegeEscalation: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - name: http + containerPort: 80 + protocol: TCP + hostPort: 80 + - name: https + containerPort: 443 + protocol: TCP + hostPort: 443 + - name: webhook + containerPort: 8443 + protocol: TCP + volumeMounts: + - name: webhook-cert + mountPath: /usr/local/certificates/ + readOnly: true + resources: + requests: + cpu: 100m + memory: 90Mi nodeSelector: - ingress-ready: "true" + ingress-ready: 'true' kubernetes.io/os: linux + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal serviceAccountName: ingress-nginx terminationGracePeriodSeconds: 0 - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Equal - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Equal volumes: - - name: webhook-cert - secret: - secretName: ingress-nginx-admission ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - spec: - containers: - - args: - - create - - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc - - --namespace=$(POD_NAMESPACE) - - --secret-name=ingress-nginx-admission - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: create - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - spec: - containers: - - args: - - patch - - --webhook-name=ingress-nginx-admission - - --namespace=$(POD_NAMESPACE) - - --patch-mutating=false - - --secret-name=ingress-nginx-admission - - --patch-failure-policy=Fail - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: patch - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: nginx -spec: - controller: k8s.io/ingress-nginx + - name: webhook-cert + secret: + secretName: ingress-nginx-admission --- +# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml +# before changing this value, check the required kubernetes version +# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook name: ingress-nginx-admission webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: ingress-nginx-controller-admission - namespace: ingress-nginx - path: /networking/v1/ingresses - port: 443 - failurePolicy: Fail - matchPolicy: Equivalent - name: validate.nginx.ingress.kubernetes.io - rules: + - name: validate.nginx.ingress.kubernetes.io + matchPolicy: Equivalent + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + namespace: ingress-nginx + name: ingress-nginx-controller-admission + path: /networking/v1beta1/ingresses +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +rules: - apiGroups: - - networking.k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE + - admissionregistration.k8s.io resources: - - ingresses - sideEffects: None + - validatingwebhookconfigurations + verbs: + - get + - update +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - create +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-create + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-create + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: create + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - create + - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=ingress-nginx-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-patch + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-patch + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: patch + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - patch + - --webhook-name=ingress-nginx-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=ingress-nginx-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 diff --git a/deploy/static/provider/kind/kustomization.yaml b/deploy/static/provider/kind/kustomization.yaml index bd605a188..94b7c887d 100644 --- a/deploy/static/provider/kind/kustomization.yaml +++ b/deploy/static/provider/kind/kustomization.yaml @@ -4,7 +4,7 @@ # ``` # namespace: ingress-nginx # bases: -# - github.com/kubernetes/ingress-nginx/tree/main/deploy/static/provider/kind +# - github.com/kubernetes/ingress-nginx/deploy/static/provider/kind?ref=master # ``` resources: diff --git a/deploy/static/provider/oracle/deploy.yaml b/deploy/static/provider/oracle/deploy.yaml deleted file mode 100644 index 02d18b1bf..000000000 --- a/deploy/static/provider/oracle/deploy.yaml +++ /dev/null @@ -1,669 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - name: ingress-nginx ---- -apiVersion: v1 -automountServiceAccountToken: true -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: v1 -automountServiceAccountToken: true -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get -- apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - coordination.k8s.io - resourceNames: - - ingress-nginx-leader - resources: - - leases - verbs: - - get - - update -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - - namespaces - verbs: - - list - - watch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -rules: -- apiGroups: - - admissionregistration.k8s.io - resources: - - validatingwebhookconfigurations - verbs: - - get - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: v1 -data: null -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-controller - namespace: ingress-nginx ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - service.beta.kubernetes.io/oci-load-balancer-shape: flexible - service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "100" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "10" - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-controller - namespace: ingress-nginx -spec: - externalTrafficPolicy: Local - ipFamilies: - - IPv4 - ipFamilyPolicy: SingleStack - ports: - - appProtocol: http - name: http - port: 80 - protocol: TCP - targetPort: http - - appProtocol: https - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-controller-admission - namespace: ingress-nginx -spec: - ports: - - appProtocol: https - name: https-webhook - port: 443 - targetPort: webhook - selector: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - type: ClusterIP ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-controller - namespace: ingress-nginx -spec: - minReadySeconds: 0 - revisionHistoryLimit: 10 - selector: - matchLabels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - strategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate - template: - metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - spec: - containers: - - args: - - /nginx-ingress-controller - - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller - - --election-id=ingress-nginx-leader - - --controller-class=k8s.io/ingress-nginx - - --ingress-class=nginx - - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller - - --validating-webhook=:8443 - - --validating-webhook-certificate=/usr/local/certificates/cert - - --validating-webhook-key=/usr/local/certificates/key - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: LD_PRELOAD - value: /usr/local/lib/libmimalloc.so - image: registry.k8s.io/ingress-nginx/controller:v1.12.0@sha256:e6b8de175acda6ca913891f0f727bca4527e797d52688cbe9fec9040d6f6b6fa - imagePullPolicy: IfNotPresent - lifecycle: - preStop: - exec: - command: - - /wait-shutdown - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - name: controller - ports: - - containerPort: 80 - name: http - protocol: TCP - - containerPort: 443 - name: https - protocol: TCP - - containerPort: 8443 - name: webhook - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 100m - memory: 90Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: false - runAsGroup: 82 - runAsNonRoot: true - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /usr/local/certificates/ - name: webhook-cert - readOnly: true - dnsPolicy: ClusterFirst - nodeSelector: - kubernetes.io/os: linux - serviceAccountName: ingress-nginx - terminationGracePeriodSeconds: 300 - volumes: - - name: webhook-cert - secret: - secretName: ingress-nginx-admission ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - spec: - containers: - - args: - - create - - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc - - --namespace=$(POD_NAMESPACE) - - --secret-name=ingress-nginx-admission - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: create - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - spec: - containers: - - args: - - patch - - --webhook-name=ingress-nginx-admission - - --namespace=$(POD_NAMESPACE) - - --patch-mutating=false - - --secret-name=ingress-nginx-admission - - --patch-failure-policy=Fail - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: patch - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: nginx -spec: - controller: k8s.io/ingress-nginx ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: ingress-nginx-controller-admission - namespace: ingress-nginx - path: /networking/v1/ingresses - port: 443 - failurePolicy: Fail - matchPolicy: Equivalent - name: validate.nginx.ingress.kubernetes.io - rules: - - apiGroups: - - networking.k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - ingresses - sideEffects: None diff --git a/deploy/static/provider/oracle/kustomization.yaml b/deploy/static/provider/oracle/kustomization.yaml deleted file mode 100644 index 5c1dcff96..000000000 --- a/deploy/static/provider/oracle/kustomization.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# NOTE: kustomize is not supported. This file exists only to be able to reference it from bases. -# https://kubectl.docs.kubernetes.io/references/kustomize/bases/ -# -# ``` -# namespace: ingress-nginx -# bases: -# - github.com/kubernetes/ingress-nginx/tree/main/deploy/static/provider/oracle -# ``` - -resources: - - deploy.yaml diff --git a/deploy/static/provider/scw/deploy.yaml b/deploy/static/provider/scw/deploy.yaml index 8ed37d062..3776bb8d5 100644 --- a/deploy/static/provider/scw/deploy.yaml +++ b/deploy/static/provider/scw/deploy.yaml @@ -1,668 +1,657 @@ + apiVersion: v1 kind: Namespace metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx name: ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + --- +# Source: ingress-nginx/templates/controller-serviceaccount.yaml apiVersion: v1 -automountServiceAccountToken: true kind: ServiceAccount metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx namespace: ingress-nginx ---- -apiVersion: v1 automountServiceAccountToken: true -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get -- apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - coordination.k8s.io - resourceNames: - - ingress-nginx-leader - resources: - - leases - verbs: - - get - - update -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -rules: -- apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - - namespaces - verbs: - - list - - watch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -rules: -- apiGroups: - - admissionregistration.k8s.io - resources: - - validatingwebhookconfigurations - verbs: - - get - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx -subjects: -- kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx-admission -subjects: -- kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx --- +# Source: ingress-nginx/templates/controller-configmap.yaml apiVersion: v1 -data: - use-proxy-protocol: "true" kind: ConfigMap metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx +data: + use-proxy-protocol: 'true' --- +# Source: ingress-nginx/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch +--- +# Source: ingress-nginx/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - namespaces + verbs: + - get + - apiGroups: + - '' + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - configmaps + resourceNames: + - ingress-controller-leader-nginx + verbs: + - get + - update + - apiGroups: + - '' + resources: + - configmaps + verbs: + - create + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +--- +# Source: ingress-nginx/templates/controller-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-service-webhook.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller-admission + namespace: ingress-nginx +spec: + type: ClusterIP + ports: + - name: https-webhook + port: 443 + targetPort: webhook + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-service.yaml apiVersion: v1 kind: Service metadata: annotations: - service.beta.kubernetes.io/scw-loadbalancer-proxy-protocol-v2: "true" + service.beta.kubernetes.io/scw-loadbalancer-proxy-protocol-v2: 'true' labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx spec: - externalTrafficPolicy: Local - ipFamilies: - - IPv4 - ipFamilyPolicy: SingleStack - ports: - - appProtocol: http - name: http - port: 80 - protocol: TCP - targetPort: http - - appProtocol: https - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-controller-admission - namespace: ingress-nginx -spec: + externalTrafficPolicy: Local ports: - - appProtocol: https - name: https-webhook - port: 443 - targetPort: webhook + - name: http + port: 80 + protocol: TCP + targetPort: http + - name: https + port: 443 + protocol: TCP + targetPort: https selector: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - type: ClusterIP + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller --- +# Source: ingress-nginx/templates/controller-deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx spec: - minReadySeconds: 0 - revisionHistoryLimit: 10 selector: matchLabels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - strategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + revisionHistoryLimit: 10 + minReadySeconds: 0 template: metadata: labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller spec: - containers: - - args: - - /nginx-ingress-controller - - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller - - --election-id=ingress-nginx-leader - - --controller-class=k8s.io/ingress-nginx - - --ingress-class=nginx - - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller - - --validating-webhook=:8443 - - --validating-webhook-certificate=/usr/local/certificates/cert - - --validating-webhook-key=/usr/local/certificates/key - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: LD_PRELOAD - value: /usr/local/lib/libmimalloc.so - image: registry.k8s.io/ingress-nginx/controller:v1.12.0@sha256:e6b8de175acda6ca913891f0f727bca4527e797d52688cbe9fec9040d6f6b6fa - imagePullPolicy: IfNotPresent - lifecycle: - preStop: - exec: - command: - - /wait-shutdown - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - name: controller - ports: - - containerPort: 80 - name: http - protocol: TCP - - containerPort: 443 - name: https - protocol: TCP - - containerPort: 8443 - name: webhook - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 100m - memory: 90Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: false - runAsGroup: 82 - runAsNonRoot: true - runAsUser: 101 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /usr/local/certificates/ - name: webhook-cert - readOnly: true dnsPolicy: ClusterFirst + containers: + - name: controller + image: k8s.gcr.io/ingress-nginx/controller:v0.46.0@sha256:52f0058bed0a17ab0fb35628ba97e8d52b5d32299fbc03cc0f6c7b9ff036b61a + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + args: + - /nginx-ingress-controller + - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller + - --election-id=ingress-controller-leader + - --ingress-class=nginx + - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: 101 + allowPrivilegeEscalation: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - name: http + containerPort: 80 + protocol: TCP + - name: https + containerPort: 443 + protocol: TCP + - name: webhook + containerPort: 8443 + protocol: TCP + volumeMounts: + - name: webhook-cert + mountPath: /usr/local/certificates/ + readOnly: true + resources: + requests: + cpu: 100m + memory: 90Mi nodeSelector: kubernetes.io/os: linux serviceAccountName: ingress-nginx terminationGracePeriodSeconds: 300 volumes: - - name: webhook-cert - secret: - secretName: ingress-nginx-admission ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-create - spec: - containers: - - args: - - create - - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc - - --namespace=$(POD_NAMESPACE) - - --secret-name=ingress-nginx-admission - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: create - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - namespace: ingress-nginx -spec: - template: - metadata: - labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: ingress-nginx-admission-patch - spec: - containers: - - args: - - patch - - --webhook-name=ingress-nginx-admission - - --namespace=$(POD_NAMESPACE) - - --patch-mutating=false - - --secret-name=ingress-nginx-admission - - --patch-failure-policy=Fail - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.5.0@sha256:aaafd456bda110628b2d4ca6296f38731a3aaf0bf7581efae824a41c770a8fc4 - imagePullPolicy: IfNotPresent - name: patch - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsGroup: 65532 - runAsNonRoot: true - runAsUser: 65532 - seccompProfile: - type: RuntimeDefault - nodeSelector: - kubernetes.io/os: linux - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission ---- -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - labels: - app.kubernetes.io/component: controller - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 - name: nginx -spec: - controller: k8s.io/ingress-nginx + - name: webhook-cert + secret: + secretName: ingress-nginx-admission --- +# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml +# before changing this value, check the required kubernetes version +# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: labels: - app.kubernetes.io/component: admission-webhook - app.kubernetes.io/instance: ingress-nginx + helm.sh/chart: ingress-nginx-3.33.0 app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - app.kubernetes.io/version: 1.12.0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook name: ingress-nginx-admission webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: ingress-nginx-controller-admission - namespace: ingress-nginx - path: /networking/v1/ingresses - port: 443 - failurePolicy: Fail - matchPolicy: Equivalent - name: validate.nginx.ingress.kubernetes.io - rules: + - name: validate.nginx.ingress.kubernetes.io + matchPolicy: Equivalent + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + namespace: ingress-nginx + name: ingress-nginx-controller-admission + path: /networking/v1beta1/ingresses +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +rules: - apiGroups: - - networking.k8s.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE + - admissionregistration.k8s.io resources: - - ingresses - sideEffects: None + - validatingwebhookconfigurations + verbs: + - get + - update +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - create +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-create + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-create + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: create + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - create + - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=ingress-nginx-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-patch + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-patch + labels: + helm.sh/chart: ingress-nginx-3.33.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.47.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: patch + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - patch + - --webhook-name=ingress-nginx-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=ingress-nginx-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 diff --git a/deploy/static/provider/scw/kustomization.yaml b/deploy/static/provider/scw/kustomization.yaml index d8535dbde..da792b8f5 100644 --- a/deploy/static/provider/scw/kustomization.yaml +++ b/deploy/static/provider/scw/kustomization.yaml @@ -4,7 +4,7 @@ # ``` # namespace: ingress-nginx # bases: -# - github.com/kubernetes/ingress-nginx/tree/main/deploy/static/provider/scw +# - github.com/kubernetes/ingress-nginx/deploy/static/provider/scw?ref=master # ``` resources: diff --git a/docs/OWNERS b/docs/OWNERS deleted file mode 100644 index 245cb8688..000000000 --- a/docs/OWNERS +++ /dev/null @@ -1,7 +0,0 @@ -# See the OWNERS docs: https://www.kubernetes.dev/docs/guide/owners - -approvers: -- ingress-nginx-docs-maintainers - -labels: -- area/docs diff --git a/docs/deploy/baremetal.md b/docs/deploy/baremetal.md index 077d1e758..436fdc4ae 100644 --- a/docs/deploy/baremetal.md +++ b/docs/deploy/baremetal.md @@ -1,14 +1,14 @@ # Bare-metal considerations In traditional *cloud* environments, where network load balancers are available on-demand, a single Kubernetes manifest -suffices to provide a single point of contact to the Ingress-Nginx Controller to external clients and, indirectly, to +suffices to provide a single point of contact to the NGINX Ingress controller to external clients and, indirectly, to any application running inside the cluster. *Bare-metal* environments lack this commodity, requiring a slightly different setup to offer the same kind of access to external consumers. ![Cloud environment](../images/baremetal/cloud_overview.jpg) ![Bare-metal environment](../images/baremetal/baremetal_overview.jpg) -The rest of this document describes a few recommended approaches to deploying the Ingress-Nginx Controller inside a +The rest of this document describes a few recommended approaches to deploying the NGINX Ingress controller inside a Kubernetes cluster running on bare-metal. ## A pure software solution: MetalLB @@ -30,11 +30,10 @@ the traffic for the `ingress-nginx` Service IP. See [Traffic policies][metallb-t yourself by reading the official documentation thoroughly. MetalLB can be deployed either with a simple Kubernetes manifest or with Helm. The rest of this example assumes MetalLB -was deployed following the [Installation][metallb-install] instructions, and that the Ingress-Nginx Controller was installed -using the steps described in the [quickstart section of the installation guide][install-quickstart]. +was deployed following the [Installation][metallb-install] instructions. MetalLB requires a pool of IP addresses in order to be able to take ownership of the `ingress-nginx` Service. This pool -can be defined through `IPAddressPool` objects in the same namespace as the MetalLB controller. This pool of IPs **must** be dedicated to MetalLB's use, you can't reuse the Kubernetes node IPs or IPs handed out by a DHCP server. +can be defined in a ConfigMap named `config` located in the same namespace as the MetalLB controller. This pool of IPs **must** be dedicated to MetalLB's use, you can't reuse the Kubernetes node IPs or IPs handed out by a DHCP server. !!! example Given the following 3-node Kubernetes cluster (the external IP is added as an example, in most bare-metal @@ -48,29 +47,22 @@ can be defined through `IPAddressPool` objects in the same namespace as the Meta host-3 Ready node 203.0.113.3 ``` - After creating the following objects, MetalLB takes ownership of one of the IP addresses in the pool and updates + After creating the following ConfigMap, MetalLB takes ownership of one of the IP addresses in the pool and updates the *loadBalancer* IP field of the `ingress-nginx` Service accordingly. ```yaml - --- - apiVersion: metallb.io/v1beta1 - kind: IPAddressPool + apiVersion: v1 + kind: ConfigMap metadata: - name: default namespace: metallb-system - spec: - addresses: - - 203.0.113.10-203.0.113.15 - autoAssign: true - --- - apiVersion: metallb.io/v1beta1 - kind: L2Advertisement - metadata: - name: default - namespace: metallb-system - spec: - ipAddressPools: - - default + name: config + data: + config: | + address-pools: + - name: default + protocol: layer2 + addresses: + - 203.0.113.10-203.0.113.15 ``` ```console @@ -118,8 +110,6 @@ requests. ![NodePort request flow](../images/baremetal/nodeport.jpg) -You can **customize the exposed node port numbers** by setting the `controller.service.nodePorts.*` Helm values, but they still have to be in the 30000-32767 range. - !!! example Given the NodePort `30100` allocated to the `ingress-nginx` Service @@ -154,7 +144,7 @@ You can **customize the exposed node port numbers** by setting the `controller.s This approach has a few other limitations one ought to be aware of: -### Source IP address +* **Source IP address** Services of type NodePort perform [source address translation][nodeport-nat] by default. This means the source IP of a HTTP request is always **the IP address of the Kubernetes node that received the request** from the perspective of @@ -166,7 +156,7 @@ field of the `ingress-nginx` Service spec to `Local` ([example][preserve-ip]). !!! warning This setting effectively **drops packets** sent to Kubernetes nodes which are not running any instance of the NGINX Ingress controller. Consider [assigning NGINX Pods to specific nodes][pod-assign] in order to control on what nodes - the Ingress-Nginx Controller should be scheduled or not scheduled. + the NGINX Ingress controller should be scheduled or not scheduled. !!! example In a Kubernetes cluster composed of 3 nodes (the external IP is added as an example, in most bare-metal environments @@ -193,11 +183,9 @@ field of the `ingress-nginx` Service spec to `Local` ([example][preserve-ip]). Requests sent to `host-2` and `host-3` would be forwarded to NGINX and original client's IP would be preserved, while requests to `host-1` would get dropped because there is no NGINX replica running on that node. -Other ways to preserve the source IP in a NodePort setup are described here: [Source IP address](https://kubernetes.github.io/ingress-nginx/user-guide/miscellaneous/#source-ip-address). +* **Ingress status** -### Ingress status - -Because NodePort Services do not get a LoadBalancerIP assigned by definition, the Ingress-Nginx Controller **does not +Because NodePort Services do not get a LoadBalancerIP assigned by definition, the NGINX Ingress controller **does not update the status of Ingress objects it manages**. ```console @@ -206,12 +194,12 @@ NAME HOSTS ADDRESS PORTS test-ingress myapp.example.com 80 ``` -Despite the fact there is no load balancer providing a public IP address to the Ingress-Nginx Controller, it is possible +Despite the fact there is no load balancer providing a public IP address to the NGINX Ingress controller, it is possible to force the status update of all managed Ingress objects by setting the `externalIPs` field of the `ingress-nginx` Service. !!! warning - There is more to setting `externalIPs` than just enabling the Ingress-Nginx Controller to update the status of + There is more to setting `externalIPs` than just enabling the NGINX Ingress controller to update the status of Ingress objects. Please read about this option in the [Services][external-ips] page of official Kubernetes documentation as well as the section about [External IPs](#external-ips) in this document for more information. @@ -245,7 +233,7 @@ Service. test-ingress myapp.example.com 203.0.113.1,203.0.113.2,203.0.113.3 80 ``` -### Redirects +* **Redirects** As NGINX is **not aware of the port translation operated by the NodePort Service**, backend applications are responsible for generating redirect URLs that take into account the URL used by external clients, including the NodePort. @@ -262,21 +250,20 @@ for generating redirect URLs that take into account the URL used by external cli ``` [install-baremetal]: ./index.md#bare-metal -[install-quickstart]: ./index.md#quick-start -[nodeport-def]: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport +[nodeport-def]: https://kubernetes.io/docs/concepts/services-networking/service/#nodeport [nodeport-nat]: https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-type-nodeport [pod-assign]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ -[preserve-ip]: https://github.com/kubernetes/ingress-nginx/blob/ingress-nginx-3.15.2/deploy/static/provider/aws/deploy.yaml#L290 +[preserve-ip]: https://github.com/kubernetes/ingress-nginx/blob/nginx-0.19.0/deploy/provider/aws/service-nlb.yaml#L12-L14 ## Via the host network In a setup where there is no external load balancer available but using NodePorts is not an option, one can configure `ingress-nginx` Pods to use the network of the host they run on instead of a dedicated network namespace. The benefit of -this approach is that the Ingress-Nginx Controller can bind ports 80 and 443 directly to Kubernetes nodes' network +this approach is that the NGINX Ingress controller can bind ports 80 and 443 directly to Kubernetes nodes' network interfaces, without the extra network translation imposed by NodePort Services. !!! note - This approach does not leverage any Service object to expose the Ingress-Nginx Controller. If the `ingress-nginx` + This approach does not leverage any Service object to expose the NGINX Ingress controller. If the `ingress-nginx` Service exists in the target cluster, it is **recommended to delete it**. This can be achieved by enabling the `hostNetwork` option in the Pods' spec. @@ -288,7 +275,7 @@ template: ``` !!! danger "Security considerations" - Enabling this option **exposes every system daemon to the Ingress-Nginx Controller** on any network interface, + Enabling this option **exposes every system daemon to the NGINX Ingress controller** on any network interface, including the host's loopback. Please evaluate the impact this may have on the security of your system carefully. !!! example @@ -303,7 +290,7 @@ template: ingress-nginx-controller-5b4cf5fc6-lzrls 1/1 Running 203.0.113.2 host-2 ``` -One major limitation of this deployment approach is that only **a single Ingress-Nginx Controller Pod** may be scheduled +One major limitation of this deployment approach is that only **a single NGINX Ingress controller Pod** may be scheduled on each cluster node, because binding the same port multiple times on the same network interface is technically impossible. Pods that are unschedulable due to such situation fail with the following event: @@ -316,7 +303,7 @@ Events: Warning FailedScheduling default-scheduler 0/3 nodes are available: 3 node(s) didn't have free ports for the requested pod ports. ``` -One way to ensure only schedulable Pods are created is to deploy the Ingress-Nginx Controller as a *DaemonSet* instead +One way to ensure only schedulable Pods are created is to deploy the NGINX Ingress controller as a *DaemonSet* instead of a traditional Deployment. !!! info @@ -330,15 +317,15 @@ configuration of the corresponding manifest at the user's discretion. Like with NodePorts, this approach has a few quirks it is important to be aware of. -### DNS resolution +* **DNS resolution** Pods configured with `hostNetwork: true` do not use the internal DNS resolver (i.e. *kube-dns* or *CoreDNS*), unless their `dnsPolicy` spec field is set to [`ClusterFirstWithHostNet`][dnspolicy]. Consider using this setting if NGINX is expected to resolve internal names for any reason. -### Ingress status +* **Ingress status** -Because there is no Service exposing the Ingress-Nginx Controller in a configuration using the host network, the default +Because there is no Service exposing the NGINX Ingress controller in a configuration using the host network, the default `--publish-service` flag used in standard cloud setups **does not apply** and the status of all Ingress objects remains blank. @@ -350,7 +337,7 @@ test-ingress myapp.example.com 80 Instead, and because bare-metal nodes usually don't have an ExternalIP, one has to enable the [`--report-node-internal-ip-address`][cli-args] flag, which sets the status of all Ingress objects to the internal IP -address of all nodes running the Ingress-Nginx Controller. +address of all nodes running the NGINX Ingress controller. !!! example Given a `ingress-nginx-controller` DaemonSet composed of 2 replicas diff --git a/docs/deploy/hardening-guide.md b/docs/deploy/hardening-guide.md index 2726b1a07..5b44686ab 100644 --- a/docs/deploy/hardening-guide.md +++ b/docs/deploy/hardening-guide.md @@ -1,8 +1,6 @@ # Hardening Guide -Do not use in multi-tenant Kubernetes production installations. This project assumes that users that can create Ingress objects are administrators of the cluster. - ## Overview There are several ways to do hardening and securing of nginx. In this documentation two guides are used, the guides are overlapping in some points: @@ -56,14 +54,14 @@ This guide refers to chapters in the CIS Benchmark. For full explanation you sho | __2.4 Network Configuration__ ||| | | 2.4.1 Ensure NGINX only listens for network connections on authorized ports (Not Scored)| OK | Ensured by automatic nginx.conf configuration| | | 2.4.2 Ensure requests for unknown host names are rejected (Not Scored)| OK | They are not rejected but send to the "default backend" delivering appropriate errors (mostly 404)| | -| 2.4.3 Ensure keepalive_timeout is 10 seconds or less, but not 0 (Scored)| ACTION NEEDED| Default is 75s | configure keep-alive to 10 seconds [according to this documentation](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/configmap.md#keep-alive) | +| 2.4.3 Ensure keepalive_timeout is 10 seconds or less, but not 0 (Scored)| ACTION NEEDED| Default is 75s | configure keep-alive to 10 seconds [according to this documentation](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/configmap.md#keep-alive) | | 2.4.4 Ensure send_timeout is set to 10 seconds or less, but not 0 (Scored)| RISK TO BE ACCEPTED| Not configured, however the nginx default is 60s| Not configurable| | ||| | | __2.5 Information Disclosure__||| | | 2.5.1 Ensure server_tokens directive is set to `off` (Scored) | OK | server_tokens is configured to off by default| | | 2.5.2 Ensure default error and index.html pages do not reference NGINX (Scored) | ACTION NEEDED| 404 shows no version at all, 503 and 403 show "nginx", which is hardcoded [see this line in nginx source code](https://github.com/nginx/nginx/blob/master/src/http/ngx_http_special_response.c#L36) | configure custom error pages at least for 403, 404 and 503 and 500| | 2.5.3 Ensure hidden file serving is disabled (Not Scored) | ACTION NEEDED | config not set | configure a config.server-snippet Snippet, but beware of .well-known challenges or similar. Refer to the benchmark here please | -| 2.5.4 Ensure the NGINX reverse proxy does not enable information disclosure (Scored)| ACTION NEEDED| hide not configured| configure hide-headers with array of "X-Powered-By" and "Server": [according to this documentation](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/configmap.md#hide-headers) | +| 2.5.4 Ensure the NGINX reverse proxy does not enable information disclosure (Scored)| ACTION NEEDED| hide not configured| configure hide-headers with array of "X-Powered-By" and "Server": [according to this documentation](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/configmap.md#hide-headers) | | ||| | | __3 Logging__ ||| | | ||| | @@ -88,7 +86,7 @@ This guide refers to chapters in the CIS Benchmark. For full explanation you sho | 4.1.8 Ensure HTTP Strict Transport Security (HSTS) is enabled (Scored)| OK | HSTS is enabled by default | | | 4.1.9 Ensure HTTP Public Key Pinning is enabled (Not Scored)| ACTION NEEDED / RISK TO BE ACCEPTED | HKPK not enabled by default | If lets encrypt is not used, set correct HPKP header. There are several ways to implement this - with the helm charts it works via controller.add-headers. If lets encrypt is used, this is complicated, a solution here is yet unknown | | 4.1.10 Ensure upstream server traffic is authenticated with a client certificate (Scored) | DEPENDS ON BACKEND | Highly dependent on backends, not every backend allows configuring this, can also be mitigated via a service mesh| If backend allows it, [manual is here](https://kubernetes.github.io/ingress-nginx/examples/auth/client-certs/)| -| 4.1.11 Ensure the upstream traffic server certificate is trusted (Not Scored) | DEPENDS ON BACKEND | Highly dependent on backends, not every backend allows configuring this, can also be mitigated via a service mesh| If backend allows it, [see configuration here](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/annotations.md#backend-certificate-authentication) | +| 4.1.11 Ensure the upstream traffic server certificate is trusted (Not Scored) | DEPENDS ON BACKEND | Highly dependent on backends, not every backend allows configuring this, can also be mitigated via a service mesh| If backend allows it, [see configuration here](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/annotations.md#backend-certificate-authentication) | | 4.1.12 Ensure your domain is preloaded (Not Scored) | ACTION NEEDED| Preload is not active by default | Set controller.config.hsts-preload to true| | 4.1.13 Ensure session resumption is disabled to enable perfect forward security (Scored)| OK | Session tickets are disabled by default | | | 4.1.14 Ensure HTTP/2.0 is used (Not Scored) | OK | http2 is set by default| | @@ -96,13 +94,13 @@ This guide refers to chapters in the CIS Benchmark. For full explanation you sho | __5 Request Filtering and Restrictions__||| | | ||| | | __5.1 Access Control__||| | -| 5.1.1 Ensure allow and deny filters limit access to specific IP addresses (Not Scored)| OK/ACTION NEEDED | Depends on use case, geo ip module is compiled into Ingress-Nginx Controller, there are several ways to use it | If needed set IP restrictions via annotations or work with config snippets (be careful with lets-encrypt-http-challenge!) | +| 5.1.1 Ensure allow and deny filters limit access to specific IP addresses (Not Scored)| OK/ACTION NEEDED | Depends on use case, geo ip module is compiled into nginx ingress controller, there are several ways to use it | If needed set IP restrictions via annotations or work with config snippets (be careful with lets-encrypt-http-challenge!) | | 5.1.2 Ensure only whitelisted HTTP methods are allowed (Not Scored) | OK/ACTION NEEDED | Depends on use case| If required it can be set via config snippet| | ||| | | __5.2 Request Limits__||| | -| 5.2.1 Ensure timeout values for reading the client header and body are set correctly (Scored) | ACTION NEEDED| Default timeout is 60s | Set via [this configuration parameter](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/configmap.md#client-header-timeout) and respective body equivalent| -| 5.2.2 Ensure the maximum request body size is set correctly (Scored)| ACTION NEEDED| Default is 1m| set via [this configuration parameter](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/configmap.md#proxy-body-size)| -| 5.2.3 Ensure the maximum buffer size for URIs is defined (Scored) | ACTION NEEDED| Default is 4 8k| Set via [this configuration parameter](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/configmap.md#large-client-header-buffers)| +| 5.2.1 Ensure timeout values for reading the client header and body are set correctly (Scored) | ACTION NEEDED| Default timeout is 60s | Set via [this configuration parameter](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/configmap.md#client-header-timeout) and respective body equivalent| +| 5.2.2 Ensure the maximum request body size is set correctly (Scored)| ACTION NEEDED| Default is 1m| set via [this configuration parameter](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/configmap.md#proxy-body-size)| +| 5.2.3 Ensure the maximum buffer size for URIs is defined (Scored) | ACTION NEEDED| Default is 4 8k| Set via [this configuration parameter](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/configmap.md#large-client-header-buffers)| | 5.2.4 Ensure the number of connections per IP address is limited (Not Scored) | OK/ACTION NEEDED| No limit set| Depends on use case, limit can be set via [these annotations](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#rate-limiting)| | 5.2.5 Ensure rate limits by IP address are set (Not Scored) | OK/ACTION NEEDED| No limit set| Depends on use case, limit can be set via [these annotations](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#rate-limiting)| | ||| | diff --git a/docs/deploy/index.md b/docs/deploy/index.md index 717c479ac..a821357df 100644 --- a/docs/deploy/index.md +++ b/docs/deploy/index.md @@ -1,457 +1,21 @@ # Installation Guide -There are multiple ways to install the Ingress-Nginx Controller: - -- with [Helm](https://helm.sh), using the project repository chart; -- with `kubectl apply`, using YAML manifests; -- with specific addons (e.g. for [minikube](#minikube) or [MicroK8s](#microk8s)). - -On most Kubernetes clusters, the ingress controller will work without requiring any extra configuration. If you want to -get started as fast as possible, you can check the [quick start](#quick-start) instructions. However, in many -environments, you can improve the performance or get better logs by enabling extra features. We recommend that you -check the [environment-specific instructions](#environment-specific-instructions) for details about optimizing the -ingress controller for your particular environment or cloud provider. - -## Contents - - - -- [Quick start](#quick-start) - -- [Environment-specific instructions](#environment-specific-instructions) - - ... [Docker Desktop](#docker-desktop) - - ... [Rancher Desktop](#rancher-desktop) - - ... [minikube](#minikube) - - ... [MicroK8s](#microk8s) - - ... [AWS](#aws) - - ... [GCE - GKE](#gce-gke) - - ... [Azure](#azure) - - ... [Digital Ocean](#digital-ocean) - - ... [Scaleway](#scaleway) - - ... [Exoscale](#exoscale) - - ... [Oracle Cloud Infrastructure](#oracle-cloud-infrastructure) - - ... [OVHcloud](#ovhcloud) - - ... [Bare-metal](#bare-metal-clusters) -- [Miscellaneous](#miscellaneous) - - - -## Quick start - -**If you have Helm,** you can deploy the ingress controller with the following command: - -```console -helm upgrade --install ingress-nginx ingress-nginx \ - --repo https://kubernetes.github.io/ingress-nginx \ - --namespace ingress-nginx --create-namespace -``` - -It will install the controller in the `ingress-nginx` namespace, creating that namespace if it doesn't already exist. - -!!! info - This command is *idempotent*: - - - if the ingress controller is not installed, it will install it, - - if the ingress controller is already installed, it will upgrade it. - -**If you want a full list of values that you can set, while installing with Helm,** then run: - -```console -helm show values ingress-nginx --repo https://kubernetes.github.io/ingress-nginx -``` - -!!! attention "Helm install on AWS/GCP/Azure/Other providers" - The *ingress-nginx-controller helm-chart is a generic install out of the box*. The default set of helm values is **not** configured for installation on any infra provider. The annotations that are applicable to the cloud provider must be customized by the users.
- See [AWS LB Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.2/guide/service/annotations/).
- Examples of some annotations recommended (healthecheck ones are required for target-type IP) for the service resource of `--type LoadBalancer` on AWS are below: - ```yaml - annotations: - service.beta.kubernetes.io/aws-load-balancer-target-group-attributes: deregistration_delay.timeout_seconds=270 - service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip - service.beta.kubernetes.io/aws-load-balancer-healthcheck-path: /healthz - service.beta.kubernetes.io/aws-load-balancer-healthcheck-port: "10254" - service.beta.kubernetes.io/aws-load-balancer-healthcheck-protocol: http - service.beta.kubernetes.io/aws-load-balancer-healthcheck-success-codes: 200-299 - service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" - service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp - service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true" - service.beta.kubernetes.io/aws-load-balancer-type: nlb - service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules: "true" - service.beta.kubernetes.io/aws-load-balancer-access-log-enabled: "true" - service.beta.kubernetes.io/aws-load-balancer-security-groups: "sg-something1 sg-something2" - service.beta.kubernetes.io/aws-load-balancer-access-log-s3-bucket-name: "somebucket" - service.beta.kubernetes.io/aws-load-balancer-access-log-s3-bucket-prefix: "ingress-nginx" - service.beta.kubernetes.io/aws-load-balancer-access-log-emit-interval: "5" - ``` - -**If you don't have Helm** or if you prefer to use a YAML manifest, you can run the following command instead: - -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.0/deploy/static/provider/cloud/deploy.yaml -``` - -!!! info - The YAML manifest in the command above was generated with `helm template`, so you will end up with almost the same - resources as if you had used Helm to install the controller. - !!! attention - If you are running an old version of Kubernetes (1.18 or earlier), please read [this paragraph](#running-on-Kubernetes-versions-older-than-1.19) for specific instructions. - Because of api deprecations, the default manifest may not work on your cluster. - Specific manifests for supported Kubernetes versions are available within a sub-folder of each provider. + The default configuration watches Ingress object from **all namespaces**. -### Firewall configuration - -To check which ports are used by your installation of ingress-nginx, look at the output of `kubectl -n ingress-nginx get pod -o yaml`. In general, you need: - -- Port 8443 open between all hosts on which the kubernetes nodes are running. This is used for the ingress-nginx [admission controller](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/). -- Port 80 (for HTTP) and/or 443 (for HTTPS) open to the public on the kubernetes nodes to which the DNS of your apps are pointing. - -### Pre-flight check - -A few pods should start in the `ingress-nginx` namespace: - -```console -kubectl get pods --namespace=ingress-nginx -``` - -After a while, they should all be running. The following command will wait for the ingress controller pod to be up, -running, and ready: - -```console -kubectl wait --namespace ingress-nginx \ - --for=condition=ready pod \ - --selector=app.kubernetes.io/component=controller \ - --timeout=120s -``` - -### Local testing - -Let's create a simple web server and the associated service: - -```console -kubectl create deployment demo --image=httpd --port=80 -kubectl expose deployment demo -``` - -Then create an ingress resource. The following example uses a host that maps to `localhost`: - -```console -kubectl create ingress demo-localhost --class=nginx \ - --rule="demo.localdev.me/*=demo:80" -``` - -Now, forward a local port to the ingress controller: - -```console -kubectl port-forward --namespace=ingress-nginx service/ingress-nginx-controller 8080:80 -``` - -!!! info - A note on DNS & network-connection. - This documentation assumes that a user has awareness of the DNS and the network routing aspects involved in using ingress. - The port-forwarding mentioned above, is the easiest way to demo the working of ingress. The "kubectl port-forward..." command above has forwarded the port number 8080, on the localhost's tcp/ip stack, where the command was typed, to the port number 80, of the service created by the installation of ingress-nginx controller. So now, the traffic sent to port number 8080 on localhost will reach the port number 80, of the ingress-controller's service. - Port-forwarding is not for a production environment use-case. But here we use port-forwarding, to simulate a HTTP request, originating from outside the cluster, to reach the service of the ingress-nginx controller, that is exposed to receive traffic from outside the cluster. - [This issue](https://github.com/kubernetes/ingress-nginx/issues/10014#issuecomment-1567791549described) shows a typical DNS problem and its solution. - -At this point, you can access your deployment using curl ; - -```console -curl --resolve demo.localdev.me:8080:127.0.0.1 http://demo.localdev.me:8080 -``` - -You should see a HTML response containing text like **"It works!"**. - -### Online testing - -If your Kubernetes cluster is a "real" cluster that supports services of type `LoadBalancer`, it will have allocated an -external IP address or FQDN to the ingress controller. - -You can see that IP address or FQDN with the following command: - -```console -kubectl get service ingress-nginx-controller --namespace=ingress-nginx -``` - -It will be the `EXTERNAL-IP` field. If that field shows ``, this means that your Kubernetes cluster wasn't -able to provision the load balancer (generally, this is because it doesn't support services of type `LoadBalancer`). - -Once you have the external IP address (or FQDN), set up a DNS record pointing to it. Then you can create an ingress -resource. The following example assumes that you have set up a DNS record for `www.demo.io`: - -```console -kubectl create ingress demo --class=nginx \ - --rule="www.demo.io/*=demo:80" -``` - -Alternatively, the above command can be rewritten as follows for the ```--rule``` command and below. - -```console -kubectl create ingress demo --class=nginx \ - --rule www.demo.io/=demo:80 -``` - -You should then be able to see the "It works!" page when you connect to . Congratulations, -you are serving a public website hosted on a Kubernetes cluster! 🎉 - -## Environment-specific instructions - -### Local development clusters - -#### minikube - -The ingress controller can be installed through minikube's addons system: - -```console -minikube addons enable ingress -``` - -#### MicroK8s - -The ingress controller can be installed through MicroK8s's addons system: - -```console -microk8s enable ingress -``` - -Please check the MicroK8s [documentation page](https://microk8s.io/docs/addon-ingress) for details. - -#### Docker Desktop - -Kubernetes is available in Docker Desktop: - -- Mac, from [version 18.06.0-ce](https://docs.docker.com/docker-for-mac/release-notes/#stable-releases-of-2018) -- Windows, from [version 18.06.0-ce](https://docs.docker.com/docker-for-windows/release-notes/#docker-community-edition-18060-ce-win70-2018-07-25) - -First, make sure that Kubernetes is enabled in the Docker settings. The command `kubectl get nodes` should show a -single node called `docker-desktop`. - -The ingress controller can be installed on Docker Desktop using the default [quick start](#quick-start) instructions. - -On most systems, if you don't have any other service of type `LoadBalancer` bound to port 80, the ingress controller -will be assigned the `EXTERNAL-IP` of `localhost`, which means that it will be reachable on localhost:80. If that -doesn't work, you might have to fall back to the `kubectl port-forward` method described in the -[local testing section](#local-testing). - -#### Rancher Desktop - -Rancher Desktop provides Kubernetes and Container Management on the desktop. Kubernetes is enabled by default in Rancher Desktop. - -Rancher Desktop uses K3s under the hood, which in turn uses Traefik as the default ingress controller for the Kubernetes cluster. To use Ingress-Nginx Controller in place of the default Traefik, disable Traefik from Preference > Kubernetes menu. - -Once traefik is disabled, the Ingress-Nginx Controller can be installed on Rancher Desktop using the default [quick start](#quick-start) instructions. Follow the instructions described in the [local testing section](#local-testing) to try a sample. - -### Cloud deployments - -If the load balancers of your cloud provider do active healthchecks on their backends (most do), you can change the -`externalTrafficPolicy` of the ingress controller Service to `Local` (instead of the default `Cluster`) to save an -extra hop in some cases. If you're installing with Helm, this can be done by adding -`--set controller.service.externalTrafficPolicy=Local` to the `helm install` or `helm upgrade` command. - -Furthermore, if the load balancers of your cloud provider support the PROXY protocol, you can enable it, and it will -let the ingress controller see the real IP address of the clients. Otherwise, it will generally see the IP address of -the upstream load balancer. This must be done both in the ingress controller -(with e.g. `--set controller.config.use-proxy-protocol=true`) and in the cloud provider's load balancer configuration -to function correctly. - -In the following sections, we provide YAML manifests that enable these options when possible, using the specific -options of various cloud providers. - -#### AWS - -In AWS, we use a Network load balancer (NLB) to expose the Ingress-Nginx Controller behind a Service of `Type=LoadBalancer`. - -!!! info - The provided templates illustrate the setup for legacy in-tree service load balancer for AWS NLB. - AWS provides the documentation on how to use - [Network load balancing on Amazon EKS](https://docs.aws.amazon.com/eks/latest/userguide/network-load-balancing.html) - with [AWS Load Balancer Controller](https://github.com/kubernetes-sigs/aws-load-balancer-controller). - -##### Network Load Balancer (NLB) - -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.0/deploy/static/provider/aws/deploy.yaml -``` - -##### TLS termination in AWS Load Balancer (NLB) - -By default, TLS is terminated in the ingress controller. But it is also possible to terminate TLS in the Load Balancer. -This section explains how to do that on AWS using an NLB. - -1. Download the [deploy.yaml](https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.0/deploy/static/provider/aws/nlb-with-tls-termination/deploy.yaml) template - - ```console - wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.0/deploy/static/provider/aws/nlb-with-tls-termination/deploy.yaml - ``` - -2. Edit the file and change the VPC CIDR in use for the Kubernetes cluster: - - ``` - proxy-real-ip-cidr: XXX.XXX.XXX/XX - ``` - -3. Change the AWS Certificate Manager (ACM) ID as well: - - ``` - arn:aws:acm:us-west-2:XXXXXXXX:certificate/XXXXXX-XXXXXXX-XXXXXXX-XXXXXXXX - ``` - -4. Deploy the manifest: - - ```console - kubectl apply -f deploy.yaml - ``` - -##### NLB Idle Timeouts - -Idle timeout value for TCP flows is 350 seconds and -[cannot be modified](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#connection-idle-timeout). - -For this reason, you need to ensure the -[keepalive_timeout](https://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_timeout) -value is configured less than 350 seconds to work as expected. - -By default, NGINX `keepalive_timeout` is set to `75s`. - -More information with regard to timeouts can be found in the -[official AWS documentation](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#connection-idle-timeout) - -#### GCE-GKE - -First, your user needs to have `cluster-admin` permissions on the cluster. This can be done with the following command: - -```console -kubectl create clusterrolebinding cluster-admin-binding \ - --clusterrole cluster-admin \ - --user $(gcloud config get-value account) -``` - -Then, the ingress controller can be installed like this: - -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.0/deploy/static/provider/cloud/deploy.yaml -``` + To change this behavior use the flag `--watch-namespace` to limit the scope to a particular namespace. !!! warning - For private clusters, you will need to either add a firewall rule that allows master nodes access to - port `8443/tcp` on worker nodes, or change the existing rule that allows access to port `80/tcp`, `443/tcp` and - `10254/tcp` to also allow access to port `8443/tcp`. More information can be found in the - [Official GCP Documentation](https://cloud.google.com/load-balancing/docs/tcp/setting-up-tcp#config-hc-firewall). + If multiple Ingresses define paths for the same host, the ingress controller **merges the definitions**. - See the [GKE documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules) - on adding rules and the [Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/79739) for more detail. +!!! danger + The [admission webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) requires connectivity between Kubernetes API server and the ingress controller. -Proxy-protocol is supported in GCE check the [Official Documentations on how to enable.](https://cloud.google.com/load-balancing/docs/tcp/setting-up-tcp#proxy-protocol) - -#### Azure - -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.0/deploy/static/provider/cloud/deploy.yaml -``` - -More information with regard to Azure annotations for ingress controller can be found in the [official AKS documentation](https://docs.microsoft.com/en-us/azure/aks/ingress-internal-ip#create-an-ingress-controller). - -#### Digital Ocean - -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.0/deploy/static/provider/do/deploy.yaml -``` - -- By default the service object of the ingress-nginx-controller for Digital-Ocean, only configures one annotation. Its this one `service.beta.kubernetes.io/do-loadbalancer-enable-proxy-protocol: "true"`. While this makes the service functional, it was reported that the Digital-Ocean LoadBalancer graphs shows `no data`, unless a few other annotations are also configured. Some of these other annotations require values that can not be generic and hence not forced in a out-of-the-box installation. These annotations and a discussion on them is well documented in [this issue](https://github.com/kubernetes/ingress-nginx/issues/8965). Please refer to the issue to add annotations, with values specific to user, to get graphs of the DO-LB populated with data. - -#### Scaleway - -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.0/deploy/static/provider/scw/deploy.yaml -``` - -Refer to the [dedicated tutorial](https://www.scaleway.com/en/docs/tutorials/proxy-protocol-v2-load-balancer/#configuring-proxy-protocol-for-ingress-nginx) in the Scaleway documentation for configuring the proxy protocol for ingress-nginx with the Scaleway load balancer. - -#### Exoscale - -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/exoscale/deploy.yaml -``` - -The full list of annotations supported by Exoscale is available in the Exoscale Cloud Controller Manager -[documentation](https://github.com/exoscale/exoscale-cloud-controller-manager/blob/master/docs/service-loadbalancer.md). - -#### Oracle Cloud Infrastructure - -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.0/deploy/static/provider/cloud/deploy.yaml -``` - -A -[complete list of available annotations for Oracle Cloud Infrastructure](https://github.com/oracle/oci-cloud-controller-manager/blob/master/docs/load-balancer-annotations.md) -can be found in the [OCI Cloud Controller Manager](https://github.com/oracle/oci-cloud-controller-manager) documentation. - -#### OVHcloud - -```console -helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx -helm repo update -helm -n ingress-nginx install ingress-nginx ingress-nginx/ingress-nginx --create-namespace -``` - -You can find the [complete tutorial](https://docs.ovh.com/gb/en/kubernetes/installing-nginx-ingress/). - -### Bare metal clusters - -This section is applicable to Kubernetes clusters deployed on bare metal servers, as well as "raw" VMs where Kubernetes -was installed manually, using generic Linux distros (like CentOS, Ubuntu...) - -For quick testing, you can use a -[NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport). -This should work on almost every cluster, but it will typically use a port in the range 30000-32767. - -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.0/deploy/static/provider/baremetal/deploy.yaml -``` - -For more information about bare metal deployments (and how to use port 80 instead of a random port in the 30000-32767 range), -see [bare-metal considerations](./baremetal.md). - -## Miscellaneous - -### Checking ingress controller version - -Run `/nginx-ingress-controller --version` within the pod, for instance with `kubectl exec`: - -```console -POD_NAMESPACE=ingress-nginx -POD_NAME=$(kubectl get pods -n $POD_NAMESPACE -l app.kubernetes.io/name=ingress-nginx --field-selector=status.phase=Running -o name) -kubectl exec $POD_NAME -n $POD_NAMESPACE -- /nginx-ingress-controller --version -``` - -### Scope - -By default, the controller watches Ingress objects from all namespaces. If you want to change this behavior, -use the flag `--watch-namespace` or check the Helm chart value `controller.scope` to limit the controller to a single -namespace. Although the use of this flag is not popular, one important fact to note is that the secret containing the default-ssl-certificate needs to also be present in the watched namespace(s). - -See also -[“How to easily install multiple instances of the Ingress NGINX controller in the same cluster”](https://kubernetes.github.io/ingress-nginx/#how-to-easily-install-multiple-instances-of-the-ingress-nginx-controller-in-the-same-cluster) -for more details. - -### Webhook network access - -!!! warning - The controller uses an [admission webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) - to validate Ingress definitions. Make sure that you don't have - [Network policies](https://kubernetes.io/docs/concepts/services-networking/network-policies/) - or additional firewalls preventing connections from the API server to the `ingress-nginx-controller-admission` service. - -### Certificate generation + In case [Network policies](https://kubernetes.io/docs/concepts/services-networking/network-policies/) or additional firewalls, please allow access to port `8443`. !!! attention The first time the ingress controller starts, two [Jobs](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/) create the SSL Certificate used by the admission webhook. - -This can cause an initial delay of up to two minutes until it is possible to create and validate Ingress definitions. + For this reason, there is an initial delay of up to two minutes until it is possible to create and validate Ingress definitions. You can wait until it is ready to run the next command: @@ -462,26 +26,218 @@ You can wait until it is ready to run the next command: --timeout=120s ``` -### Running on Kubernetes versions older than 1.19 +## Contents -Ingress resources evolved over time. They started with `apiVersion: extensions/v1beta1`, -then moved to `apiVersion: networking.k8s.io/v1beta1` and more recently to `apiVersion: networking.k8s.io/v1`. +- [Provider Specific Steps](#provider-specific-steps) + - [Docker Desktop](#docker-desktop) + - [minikube](#minikube) + - [microk8s](#microk8s) + - [AWS](#aws) + - [GCE - GKE](#gce-gke) + - [Azure](#azure) + - [Digital Ocean](#digital-ocean) + - [Scaleway](#scaleway) + - [Exoscale](#exoscale) + - [Oracle Cloud Infrastructure](#oracle-cloud-infrastructure) + - [Bare-metal](#bare-metal) + - [Verify installation](#verify-installation) + - [Detect installed version](#detect-installed-version) +- [Using Helm](#using-helm) -Here is how these Ingress versions are supported in Kubernetes: +### Provider Specific Steps -- before Kubernetes 1.19, only `v1beta1` Ingress resources are supported -- from Kubernetes 1.19 to 1.21, both `v1beta1` and `v1` Ingress resources are supported -- in Kubernetes 1.22 and above, only `v1` Ingress resources are supported +#### Docker Desktop -And here is how these Ingress versions are supported in Ingress-Nginx Controller: +Kubernetes is available in Docker Desktop -- before version 1.0, only `v1beta1` Ingress resources are supported -- in version 1.0 and above, only `v1` Ingress resources are +- Mac, from [version 18.06.0-ce](https://docs.docker.com/docker-for-mac/release-notes/#stable-releases-of-2018) +- Windows, from [version 18.06.0-ce](https://docs.docker.com/docker-for-windows/release-notes/#docker-community-edition-18060-ce-win70-2018-07-25) -As a result, if you're running Kubernetes 1.19 or later, you should be able to use the latest version of the NGINX -Ingress Controller; but if you're using an old version of Kubernetes (1.18 or earlier) you will have to use version 0.X -of the Ingress-Nginx Controller (e.g. version 0.49). +```console +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v0.47.0/deploy/static/provider/cloud/deploy.yaml +``` -The Helm chart of the Ingress-Nginx Controller switched to version 1 in version 4 of the chart. In other words, if -you're running Kubernetes 1.19 or earlier, you should use version 3.X of the chart (this can be done by adding -`--version='<4'` to the `helm install` command ). +#### minikube + +For standard usage: + +```console +minikube addons enable ingress +``` + +#### microk8s + +For standard usage: + +```console +microk8s enable ingress +``` + +Please check the microk8s [documentation page](https://microk8s.io/docs/addon-ingress) + +#### AWS + +In AWS we use a Network load balancer (NLB) to expose the NGINX Ingress controller behind a Service of `Type=LoadBalancer`. + +##### Network Load Balancer (NLB) + +```console +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v0.47.0/deploy/static/provider/aws/deploy.yaml +``` + +##### TLS termination in AWS Load Balancer (ELB) + +In some scenarios is required to terminate TLS in the Load Balancer and not in the ingress controller. + +For this purpose we provide a template: + +- Download [deploy-tls-termination.yaml](https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v0.47.0/deploy/static/provider/aws/deploy-tls-termination.yaml) + +```console +wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v0.47.0/deploy/static/provider/aws/deploy-tls-termination.yaml +``` + +- Edit the file and change: + + - VPC CIDR in use for the Kubernetes cluster: + + `proxy-real-ip-cidr: XXX.XXX.XXX/XX` + + - AWS Certificate Manager (ACM) ID + + `arn:aws:acm:us-west-2:XXXXXXXX:certificate/XXXXXX-XXXXXXX-XXXXXXX-XXXXXXXX` + +- Deploy the manifest: + +```console +kubectl apply -f deploy-tls-termination.yaml +``` + +##### NLB Idle Timeouts + +Idle timeout value for TCP flows is 350 seconds and [cannot be modified](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#connection-idle-timeout). + +For this reason, you need to ensure the [keepalive_timeout](http://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_timeout) value is configured less than 350 seconds to work as expected. + +By default NGINX `keepalive_timeout` is set to `75s`. + +More information with regards to timeouts can be found in the [official AWS documentation](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#connection-idle-timeout) + +#### GCE-GKE + +!!! info + Initialize your user as a cluster-admin with the following command: + ```console + kubectl create clusterrolebinding cluster-admin-binding \ + --clusterrole cluster-admin \ + --user $(gcloud config get-value account) + ``` + +!!! danger + For private clusters, you will need to either add an additional firewall rule that allows master nodes access to port `8443/tcp` on worker nodes, or change the existing rule that allows access to ports `80/tcp`, `443/tcp` and `10254/tcp` to also allow access to port `8443/tcp`. + + See the [GKE documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules) on adding rules and the [Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/79739) for more detail. + + +```console +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v0.47.0/deploy/static/provider/cloud/deploy.yaml +``` + +!!! failure Important + Proxy protocol is not supported in GCE/GKE + +#### Azure + +```console +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v0.47.0/deploy/static/provider/cloud/deploy.yaml +``` + +More information with regards to Azure annotations for ingress controller can be found in the [official AKS documentation](https://docs.microsoft.com/en-us/azure/aks/ingress-internal-ip#create-an-ingress-controller). + +#### Digital Ocean + +```console +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v0.47.0/deploy/static/provider/do/deploy.yaml +``` + +#### Scaleway + +```console +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v0.47.0/deploy/static/provider/scw/deploy.yaml +``` + +#### Exoscale + +```console +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/exoscale/deploy.yaml +``` + +The full list of annotations supported by Exoscale is available in the Exoscale Cloud Controller Manager [documentation](https://github.com/exoscale/exoscale-cloud-controller-manager/blob/master/docs/service-loadbalancer.md). + +#### Oracle Cloud Infrastructure + +```console +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v0.44.0/deploy/static/provider/cloud/deploy.yaml +``` + +A [complete list of available annotations for Oracle Cloud Infrastructure](https://github.com/oracle/oci-cloud-controller-manager/blob/master/docs/load-balancer-annotations.md) can be found in the [OCI Cloud Controller Manager](https://github.com/oracle/oci-cloud-controller-manager) documentation. + +#### Bare-metal + +Using [NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport): + +```console +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v0.47.0/deploy/static/provider/baremetal/deploy.yaml +``` + +!!! tip + Applicable on kubernetes clusters deployed on bare-metal with generic Linux distro(Such as CentOs, Ubuntu ...). + +!!! info + For extended notes regarding deployments on bare-metal, see [Bare-metal considerations](./baremetal.md). + +### Verify installation + +To check if the ingress controller pods have started, run the following command: + +```console +kubectl get pods -n ingress-nginx \ + -l app.kubernetes.io/name=ingress-nginx --watch +``` + +Once the ingress controller pods are running, you can cancel the command typing `Ctrl+C`. + +Now, you are ready to create your first ingress. + +### Detect installed version + +To detect which version of the ingress controller is running, exec into the pod and run `nginx-ingress-controller --version`. + +```console +POD_NAMESPACE=ingress-nginx +POD_NAME=$(kubectl get pods -n $POD_NAMESPACE -l app.kubernetes.io/name=ingress-nginx --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}') + +kubectl exec -it $POD_NAME -n $POD_NAMESPACE -- /nginx-ingress-controller --version +``` + +## Using Helm + +!!! attention + Only Helm v3 is supported + +NGINX Ingress controller can be installed via [Helm](https://helm.sh/) using the chart from the project repository. +To install the chart with the release name `ingress-nginx`: + +```console +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update + +helm install ingress-nginx ingress-nginx/ingress-nginx +``` + +## Detect installed version: + +```console +POD_NAME=$(kubectl get pods -l app.kubernetes.io/name=ingress-nginx -o jsonpath='{.items[0].metadata.name}') +kubectl exec -it $POD_NAME -- /nginx-ingress-controller --version +``` diff --git a/docs/deploy/rbac.md b/docs/deploy/rbac.md index 70af8ba92..08c1a0291 100644 --- a/docs/deploy/rbac.md +++ b/docs/deploy/rbac.md @@ -2,7 +2,7 @@ ## Overview -This example applies to ingress-nginx-controllers being deployed in an environment with RBAC enabled. +This example applies to nginx-ingress-controllers being deployed in an environment with RBAC enabled. Role Based Access Control is comprised of four layers: @@ -11,65 +11,66 @@ Role Based Access Control is comprised of four layers: 3. `Role` - permissions assigned to a role that apply to a specific namespace 4. `RoleBinding` - binding a Role to a specific account -In order for RBAC to be applied to an ingress-nginx-controller, that controller +In order for RBAC to be applied to an nginx-ingress-controller, that controller should be assigned to a `ServiceAccount`. That `ServiceAccount` should be -bound to the `Role`s and `ClusterRole`s defined for the ingress-nginx-controller. +bound to the `Role`s and `ClusterRole`s defined for the nginx-ingress-controller. ## Service Accounts created in this example -One ServiceAccount is created in this example, `ingress-nginx`. +One ServiceAccount is created in this example, `nginx-ingress-serviceaccount`. ## Permissions Granted in this example There are two sets of permissions defined in this example. Cluster-wide -permissions defined by the `ClusterRole` named `ingress-nginx`, and -namespace specific permissions defined by the `Role` named `ingress-nginx`. +permissions defined by the `ClusterRole` named `nginx-ingress-clusterrole`, and +namespace specific permissions defined by the `Role` named `nginx-ingress-role`. ### Cluster Permissions -These permissions are granted in order for the ingress-nginx-controller to be +These permissions are granted in order for the nginx-ingress-controller to be able to function as an ingress across the cluster. These permissions are -granted to the `ClusterRole` named `ingress-nginx` +granted to the ClusterRole named `nginx-ingress-clusterrole` * `configmaps`, `endpoints`, `nodes`, `pods`, `secrets`: list, watch * `nodes`: get -* `services`, `ingresses`, `ingressclasses`, `endpointslices`: get, list, watch +* `services`, `ingresses`: get, list, watch * `events`: create, patch * `ingresses/status`: update -* `leases`: list, watch ### Namespace Permissions -These permissions are granted specific to the ingress-nginx namespace. These -permissions are granted to the `Role` named `ingress-nginx` +These permissions are granted specific to the nginx-ingress namespace. These +permissions are granted to the Role named `nginx-ingress-role` * `configmaps`, `pods`, `secrets`: get * `endpoints`: get -Furthermore to support leader-election, the ingress-nginx-controller needs to -have access to a `leases` using the resourceName `ingress-nginx-leader` +Furthermore to support leader-election, the nginx-ingress-controller needs to +have access to a `configmap` using the resourceName `ingress-controller-leader-nginx` > Note that resourceNames can NOT be used to limit requests using the “create” > verb because authorizers only have access to information that can be obtained > from the request URL, method, and headers (resource names in a “create” request > are part of the request body). -* `leases`: get, update (for resourceName `ingress-controller-leader`) -* `leases`: create +* `configmaps`: get, update (for resourceName `ingress-controller-leader-nginx`) +* `configmaps`: create -This resourceName is the `election-id` defined by the ingress-controller, which defaults to: +This resourceName is the concatenation of the `election-id` and the +`ingress-class` as defined by the ingress-controller, which defaults to: * `election-id`: `ingress-controller-leader` -* `resourceName` : `` +* `ingress-class`: `nginx` +* `resourceName` : `-` Please adapt accordingly if you overwrite either parameter when launching the -ingress-nginx-controller. +nginx-ingress-controller. ### Bindings -The ServiceAccount `ingress-nginx` is bound to the Role -`ingress-nginx` and the ClusterRole `ingress-nginx`. +The ServiceAccount `nginx-ingress-serviceaccount` is bound to the Role +`nginx-ingress-role` and the ClusterRole `nginx-ingress-clusterrole`. The serviceAccountName associated with the containers in the deployment must match the serviceAccount. The namespace references in the Deployment metadata, -container arguments, and POD_NAMESPACE should be in the ingress-nginx namespace. +container arguments, and POD_NAMESPACE should be in the nginx-ingress namespace. diff --git a/docs/deploy/upgrade.md b/docs/deploy/upgrade.md index e16992612..6d28f11eb 100644 --- a/docs/deploy/upgrade.md +++ b/docs/deploy/upgrade.md @@ -14,7 +14,7 @@ I.e. if your deployment resource looks like (partial example): ```yaml kind: Deployment metadata: - name: ingress-nginx-controller + name: nginx-ingress-controller namespace: ingress-nginx spec: replicas: 1 @@ -23,31 +23,31 @@ spec: metadata: ... spec: containers: - - name: ingress-nginx-controller - image: registry.k8s.io/ingress-nginx/controller:v1.0.4@sha256:545cff00370f28363dad31e3b59a94ba377854d3a11f18988f5f9e56841ef9ef + - name: nginx-ingress-controller + image: k8s.gcr.io/ingress-nginx/controller:v0.34.0@sha256:56633bd00dab33d92ba14c6e709126a762d54a75a6e72437adefeaaca0abb069 args: ... ``` -simply change the `v1.0.4` tag to the version you wish to upgrade to. +simply change the `0.34.0` tag to the version you wish to upgrade to. The easiest way to do this is e.g. (do note you may need to change the name parameter according to your installation): ``` -kubectl set image deployment/ingress-nginx-controller \ - controller=registry.k8s.io/ingress-nginx/controller:v1.0.5@sha256:55a1fcda5b7657c372515fe402c3e39ad93aa59f6e4378e82acd99912fe6028d \ +kubectl set image deployment/nginx-ingress-controller \ + nginx-ingress-controller=k8s.gcr.io/ingress-nginx/controller:v0.34.1@sha256:0e072dddd1f7f8fc8909a2ca6f65e76c5f0d2fcfb8be47935ae3457e8bbceb20 \ -n ingress-nginx ``` -For interactive editing, use `kubectl edit deployment ingress-nginx-controller -n ingress-nginx`. +For interactive editing, use `kubectl edit deployment nginx-ingress-controller -n ingress-nginx`. ## With Helm -If you installed ingress-nginx using the Helm command in the deployment docs so its name is `ingress-nginx`, +If you installed ingress-nginx using the Helm command in the deployment docs so its name is `ngx-ingress`, you should be able to upgrade using ```shell -helm upgrade --reuse-values ingress-nginx ingress-nginx/ingress-nginx +helm upgrade --reuse-values ngx-ingress ingress-nginx/ingress-nginx ``` ### Migrating from stable/nginx-ingress -See detailed steps in the upgrading section of the `ingress-nginx` chart [README](https://github.com/kubernetes/ingress-nginx/blob/main/charts/ingress-nginx/README.md#migrating-from-stablenginx-ingress). +See detailed steps in the upgrading section of the `ingress-nginx` chart [README](https://github.com/kubernetes/ingress-nginx/blob/master/charts/ingress-nginx/README.md#migrating-from-stablenginx-ingress). diff --git a/docs/developer-guide/code-overview.md b/docs/developer-guide/code-overview.md index 8e872211d..f75f948d7 100644 --- a/docs/developer-guide/code-overview.md +++ b/docs/developer-guide/code-overview.md @@ -11,10 +11,7 @@ This part of the code is responsible for the main logic of Ingress NGINX. It con ### Core Sync Logics: -Ingress-nginx has an internal model of the ingresses, secrets and endpoints in a given cluster. It maintains two copies of that: - -1. One copy is the currently running configuration model -2. Second copy is the one generated in response to some changes in the cluster +Ingress-nginx has an internal model of the ingresses, secrets and endpoints in a given cluster. It maintains two copy of that (1) currently running configuration model and (2) the one generated in response to some changes in the cluster. The sync logic diffs the two models and if there's a change it tries to converge the running configuration to the new one. @@ -28,41 +25,43 @@ The following parts of the code can be found: ### Entrypoint -The `main` package is responsible for starting ingress-nginx program, which can be found in [cmd/nginx](https://github.com/kubernetes/ingress-nginx/tree/main/cmd/nginx) directory. +Is the `main` package, responsible for starting ingress-nginx program. + +It can be found in [cmd/nginx](https://github.com/kubernetes/ingress-nginx/tree/master/cmd/nginx) directory. ### Version -Is the package of the code responsible for adding `version` subcommand, and can be found in [version](https://github.com/kubernetes/ingress-nginx/tree/main/version) directory. +Is the package of the code responsible for adding `version` subcommand, and can be found in [version](https://github.com/kubernetes/ingress-nginx/tree/master/version) directory. ### Internal code -This part of the code contains the internal logics that compose Ingress NGINX Controller, and it's split into: +This part of the code contains the internal logics that compose Ingress NGINX Controller, and it's split in: #### Admission Controller Contains the code of [Kubernetes Admission Controller](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/) which validates the syntax of ingress objects before accepting it. -This code can be found in [internal/admission/controller](https://github.com/kubernetes/ingress-nginx/tree/main/internal/admission/controller) directory. +This code can be found in [internal/admission/controller](https://github.com/kubernetes/ingress-nginx/tree/master/internal/admission/controller) directory. #### File functions Contains auxiliary codes that deal with files, such as generating the SHA1 checksum of a file, or creating required directories. -This code can be found in [internal/file](https://github.com/kubernetes/ingress-nginx/blob/main/internal/file) directory. +This code can be found in [internal/file](https://github.com/kubernetes/ingress-nginx/blob/master/internal/file) directory. #### Ingress functions -Contains all the logics from Ingress-Nginx Controller, with some examples being: +Contains all the logics from NGINX Ingress Controller, with some examples being: -* Expected Golang structures that will be used in templates and other parts of the code - [internal/ingress/types.go](https://github.com/kubernetes/ingress-nginx/blob/main/internal/ingress/types.go). -* supported annotations and its parsing logics - [internal/ingress/annotations](https://github.com/kubernetes/ingress-nginx/tree/main/internal/ingress/annotations). -* reconciliation loops and logics - [internal/ingress/controller](https://github.com/kubernetes/ingress-nginx/tree/main/internal/ingress/controller) -* defaults - define the default struct - [internal/ingress/defaults](https://github.com/kubernetes/ingress-nginx/tree/main/internal/ingress/defaults). -* Error interface and types implementation - [internal/ingress/errors](https://github.com/kubernetes/ingress-nginx/tree/main/internal/ingress/errors) -* Metrics collectors for Prometheus exporting - [internal/ingress/metric](https://github.com/kubernetes/ingress-nginx/tree/main/internal/ingress/metric). -* Resolver - Extracts information from a controller - [internal/ingress/resolver](https://github.com/kubernetes/ingress-nginx/tree/main/internal/ingress/resolver). -* Ingress Object status publisher - [internal/ingress/status](https://github.com/kubernetes/ingress-nginx/tree/main/internal/ingress/status). +* Expected Golang structures that will be used in templates and other parts of the codes - [internal/ingress/types.go](https://github.com/kubernetes/ingress-nginx/blob/master/internal/ingress/types.go). +* supported annotations and its parsing logics - [internal/ingress/annotations](https://github.com/kubernetes/ingress-nginx/tree/master/internal/ingress/annotations). +* reconciliation loops and logics - [internal/ingress/controller](https://github.com/kubernetes/ingress-nginx/tree/master/internal/ingress/controller) +* Defaults - define the default struct. +* Error interface and types implementation - [internal/ingress/errors](https://github.com/kubernetes/ingress-nginx/tree/master/internal/ingress/errors) +* Metrics collectors for Prometheus exporting - [internal/ingress/metric](https://github.com/kubernetes/ingress-nginx/tree/master/internal/ingress/metric). +* Resolver - Extracts information from a controller. +* Ingress Object status publisher - [internal/ingress/status](https://github.com/kubernetes/ingress-nginx/tree/master/internal/ingress/status). And other parts of the code that will be written in this document in a future. @@ -70,25 +69,25 @@ And other parts of the code that will be written in this document in a future. Contains helper functions for parsing Kubernetes objects. -This part of the code can be found in [internal/k8s](https://github.com/kubernetes/ingress-nginx/tree/main/internal/k8s) directory. +This part of the code can be found in [internal/k8s](https://github.com/kubernetes/ingress-nginx/tree/master/internal/k8s) directory. #### Networking functions Contains helper functions for networking, such as IPv4 and IPv6 parsing, SSL certificate parsing, etc. -This part of the code can be found in [internal/net](https://github.com/kubernetes/ingress-nginx/tree/main/internal/net) directory. +This part of the code can be found in [internal/net](https://github.com/kubernetes/ingress-nginx/tree/master/internal/net) directory. #### NGINX functions Contains helper function to deal with NGINX, such as verify if it's running and reading it's configuration file parts. -This part of the code can be found in [internal/nginx](https://github.com/kubernetes/ingress-nginx/tree/main/internal/nginx) directory. +This part of the code can be found in [internal/nginx](https://github.com/kubernetes/ingress-nginx/tree/master/internal/nginx) directory. #### Tasks / Queue Contains the functions responsible for the sync queue part of the controller. -This part of the code can be found in [internal/task](https://github.com/kubernetes/ingress-nginx/tree/main/internal/task) directory. +This part of the code can be found in [internal/task](https://github.com/kubernetes/ingress-nginx/tree/master/internal/task) directory. #### Other parts of internal @@ -96,35 +95,29 @@ Other parts of internal code might not be covered here, like runtime and watch b ## E2E Test -The e2e tests code is in [test](https://github.com/kubernetes/ingress-nginx/tree/main/test) directory. +The e2e tests code is in [test](https://github.com/kubernetes/ingress-nginx/tree/master/test) directory. ## Other programs Describe here `kubectl plugin`, `dbg`, `waitshutdown` and cover the hack scripts. -### kubectl plugin - -It contains kubectl plugin for inspecting your ingress-nginx deployments. -This part of code can be found in [cmd/plugin](https://github.com/kubernetes/ingress-nginx/tree/main/cmd/plugin) directory -Detail functions flow and available flow can be found in [kubectl-plugin](https://github.com/kubernetes/ingress-nginx/blob/main/docs/kubectl-plugin.md) - ## Deploy files -This directory contains the `yaml` deploy files used as examples or references in the docs to deploy Ingress NGINX and other components. +This directory contains the `yaml` deploy files used as examples or references in the docs to deploy Ingress NGINX and other componentes. -Those files are in [deploy](https://github.com/kubernetes/ingress-nginx/tree/main/deploy) directory. +Those files are in [deploy](https://github.com/kubernetes/ingress-nginx/tree/master/deploy) directory. ## Helm Chart Used to generate the Helm chart published. -Code is in [charts/ingress-nginx](https://github.com/kubernetes/ingress-nginx/tree/main/charts/ingress-nginx). +Code is in [charts/ingress-nginx](https://github.com/kubernetes/ingress-nginx/tree/master/charts/ingress-nginx). ## Documentation/Website The documentation used to generate the website https://kubernetes.github.io/ingress-nginx/ -This code is available in [docs](https://github.com/kubernetes/ingress-nginx/tree/main/docs) and it's main "language" is `Markdown`, used by [mkdocs](https://github.com/kubernetes/ingress-nginx/blob/main/mkdocs.yml) file to generate static pages. +This code is available in [docs](https://github.com/kubernetes/ingress-nginx/tree/master/docs) and it's main "language" is `Markdown`, used by [mkdocs](https://github.com/kubernetes/ingress-nginx/blob/master/mkdocs.yml) file to generate static pages. ## Container Images @@ -132,9 +125,9 @@ Container images used to run ingress-nginx, or to build the final image. ### Base Images -Contains the `Dockerfiles` and scripts used to build base images that are used in other parts of the repo. They are present in [images](https://github.com/kubernetes/ingress-nginx/tree/main/images) repo. Some examples: -* [nginx](https://github.com/kubernetes/ingress-nginx/tree/main/images/nginx) - The base NGINX image ingress-nginx uses is not a vanilla NGINX. It bundles many libraries together and it is a job in itself to maintain that and keep things up-to-date. -* [custom-error-pages](https://github.com/kubernetes/ingress-nginx/tree/main/images/custom-error-pages) - Used on the custom error page examples. +Contains the `Dockerfiles` and scripts used to build base images that are used in other parts of the repo. They are present in [images](https://github.com/kubernetes/ingress-nginx/tree/master/images) repo. Some examples: +* [nginx](https://github.com/kubernetes/ingress-nginx/tree/master/images/nginx) - The base NGINX image ingress-nginx uses is not a vanilla NGINX. It bundles many libraries together and it is a job in itself to maintain that and keep things up-to-date. +* [custom-error-pages](https://github.com/kubernetes/ingress-nginx/tree/master/images/custom-error-pages) - Used on the custom error page examples. There are other images inside this directory. @@ -144,20 +137,20 @@ The image used to build the final ingress controller, used in deploy scripts and This is NGINX with some Lua enhancement. We do dynamic certificate, endpoints handling, canary traffic split, custom load balancing etc at this component. One can also add new functionalities using Lua plugin system. -The files are in [rootfs](https://github.com/kubernetes/ingress-nginx/tree/main/rootfs) directory and contains: +The files are in [rootfs](https://github.com/kubernetes/ingress-nginx/tree/master/rootfs) directory and contains: * The Dockerfile -* [nginx config](https://github.com/kubernetes/ingress-nginx/tree/main/rootfs/etc/nginx) +* [Auxiliary scripts](https://github.com/kubernetes/ingress-nginx/tree/master/rootfs/ingress-controller) #### Ingress NGINX Lua Scripts Ingress NGINX uses Lua Scripts to enable features like hot reloading, rate limiting and monitoring. Some are written using the [OpenResty](https://openresty.org/en/) helper. -The directory containing Lua scripts is [rootfs/etc/nginx/lua](https://github.com/kubernetes/ingress-nginx/tree/main/rootfs/etc/nginx/lua). +The directory containing Lua scripts is [rootfs/etc/nginx/lua](https://github.com/kubernetes/ingress-nginx/tree/master/rootfs/etc/nginx/lua). #### Nginx Go template file One of the functions of Ingress NGINX is to turn [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) objects into nginx.conf file. -To do so, the final step is to apply those configurations in [nginx.tmpl](https://github.com/kubernetes/ingress-nginx/tree/main/rootfs/etc/nginx/template) turning it into a final nginx.conf file. +To do so, the final step is to apply those configurations in [nginx.tmpl](https://github.com/kubernetes/ingress-nginx/tree/master/rootfs/etc/nginx/template) turning it into a final nginx.conf file. diff --git a/docs/developer-guide/getting-started.md b/docs/developer-guide/getting-started.md index e4bb661d4..eb5e7cae5 100644 --- a/docs/developer-guide/getting-started.md +++ b/docs/developer-guide/getting-started.md @@ -1,14 +1,6 @@ - Developing for Ingress-Nginx Controller +# Developing for NGINX Ingress Controller -This document explains how to get started with developing for Ingress-Nginx Controller. - -For the really new contributors, who want to contribute to the INGRESS-NGINX project, but need help with understanding some basic concepts, -that are needed to work with the Kubernetes ingress resource, here is a link to the [New Contributors Guide](https://github.com/kubernetes/ingress-nginx/blob/main/NEW_CONTRIBUTOR.md). -This guide contains tips on how a http/https request travels, from a browser or a curl command, -to the webserver process running inside a container, in a pod, in a Kubernetes cluster, but enters the cluster via a ingress resource. -For those who are familiar with those basic networking concepts like routing of a packet with regards to a -http request, termination of connection, reverseproxy etc. etc., you can skip this and move on to the sections below. -(or read it anyways just for context and also provide feedbacks if any) +This document explains how to get started with developing for NGINX Ingress controller. ## Prerequisites @@ -19,10 +11,6 @@ Install [Go 1.14](https://golang.org/dl/) or later. Install [Docker](https://docs.docker.com/engine/install/) (v19.03.0 or later with experimental feature on) -Install [kubectl](https://kubernetes.io/docs/tasks/tools/) (1.24.0 or higher) - -Install [Kind](https://kind.sigs.k8s.io/) - !!! important The majority of make tasks run as docker containers @@ -41,7 +29,6 @@ Start a local Kubernetes cluster using [kind](https://kind.sigs.k8s.io/), build ```console make dev-env ``` -- If you are working on the v1.x.x version of this controller, and you want to create a cluster with kubernetes version 1.22, then please visit the [documentation for kind](https://kind.sigs.k8s.io/docs/user/configuration/#a-note-on-cli-parameters-and-configuration-files), and look for how to set a custom image for the kind node (image: kindest/node...), in the kind config file. ### Testing @@ -78,9 +65,9 @@ FOCUS="no-auth-locations" make kind-e2e-test !!! note The variable `FOCUS` defines Ginkgo [Focused Specs](https://onsi.github.io/ginkgo/#focused-specs) -Valid values are defined in the describe definition of the e2e tests like [Default Backend](https://github.com/kubernetes/ingress-nginx/blob/main/test/e2e/defaultbackend/default_backend.go#L29) +Valid values are defined in the describe definition of the e2e tests like [Default Backend](https://github.com/kubernetes/ingress-nginx/blob/master/test/e2e/defaultbackend/default_backend.go#L29) -The complete list of tests can be found [here](../e2e-tests.md) +The complete list of tests can be found [here](e2e-tests.md) ### Custom docker image diff --git a/docs/e2e-tests.md b/docs/e2e-tests.md index 43726f5a1..0a92e8ad1 100644 --- a/docs/e2e-tests.md +++ b/docs/e2e-tests.md @@ -1,563 +1,574 @@ - - -# e2e test suite for [Ingress NGINX Controller](https://github.com/kubernetes/ingress-nginx/tree/main/) -### [[Admission] admission controller](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/admission/admission.go#L39) -- [should not allow overlaps of host and paths without canary annotations](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/admission/admission.go#L47) -- [should allow overlaps of host and paths with canary annotation](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/admission/admission.go#L64) -- [should block ingress with invalid path](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/admission/admission.go#L85) -- [should return an error if there is an error validating the ingress definition](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/admission/admission.go#L102) -- [should return an error if there is an invalid value in some annotation](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/admission/admission.go#L116) -- [should return an error if there is a forbidden value in some annotation](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/admission/admission.go#L130) -- [should return an error if there is an invalid path and wrong pathType is set](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/admission/admission.go#L144) -- [should not return an error if the Ingress V1 definition is valid with Ingress Class](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/admission/admission.go#L178) -- [should not return an error if the Ingress V1 definition is valid with IngressClass annotation](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/admission/admission.go#L194) -- [should return an error if the Ingress V1 definition contains invalid annotations](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/admission/admission.go#L210) -- [should not return an error for an invalid Ingress when it has unknown class](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/admission/admission.go#L224) -### [affinity session-cookie-name](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinity.go#L43) -- [should set sticky cookie SERVERID](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinity.go#L50) -- [should change cookie name on ingress definition change](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinity.go#L72) -- [should set the path to /something on the generated cookie](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinity.go#L107) -- [does not set the path to / on the generated cookie if there's more than one rule referring to the same backend](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinity.go#L129) -- [should set cookie with expires](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinity.go#L202) -- [should set cookie with domain](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinity.go#L234) -- [should not set cookie without domain annotation](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinity.go#L257) -- [should work with use-regex annotation and session-cookie-path](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinity.go#L279) -- [should warn user when use-regex is true and session-cookie-path is not set](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinity.go#L303) -- [should not set affinity across all server locations when using separate ingresses](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinity.go#L329) -- [should set sticky cookie without host](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinity.go#L361) -- [should work with server-alias annotation](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinity.go#L381) -- [should set secure in cookie with provided true annotation on http](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinity.go#L421) -- [should not set secure in cookie with provided false annotation on http](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinity.go#L444) -- [should set secure in cookie with provided false annotation on https](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinity.go#L467) -### [affinitymode](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinitymode.go#L33) -- [Balanced affinity mode should balance](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinitymode.go#L36) -- [Check persistent affinity mode](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/affinitymode.go#L69) -### [server-alias](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/alias.go#L31) -- [should return status code 200 for host 'foo' and 404 for 'bar'](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/alias.go#L38) -- [should return status code 200 for host 'foo' and 'bar'](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/alias.go#L64) -- [should return status code 200 for hosts defined in two ingresses, different path with one alias](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/alias.go#L89) -### [app-root](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/approot.go#L28) -- [should redirect to /foo](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/approot.go#L35) -### [auth-*](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L45) -- [should return status code 200 when no authentication is configured](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L52) -- [should return status code 503 when authentication is configured with an invalid secret](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L71) -- [should return status code 401 when authentication is configured but Authorization header is not configured](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L95) -- [should return status code 401 when authentication is configured and Authorization header is sent with invalid credentials](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L122) -- [should return status code 401 and cors headers when authentication and cors is configured but Authorization header is not configured](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L150) -- [should return status code 200 when authentication is configured and Authorization header is sent](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L178) -- [should return status code 200 when authentication is configured with a map and Authorization header is sent](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L205) -- [should return status code 401 when authentication is configured with invalid content and Authorization header is sent](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L233) -- [proxy_set_header My-Custom-Header 42;](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L272) -- [proxy_set_header My-Custom-Header 42;](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L292) -- [proxy_set_header 'My-Custom-Header' '42';](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L311) -- [user retains cookie by default](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L420) -- [user does not retain cookie if upstream returns error status code](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L431) -- [user with annotated ingress retains cookie if upstream returns error status code](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L442) -- [should return status code 200 when signed in](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L481) -- [should redirect to signin url when not signed in](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L490) -- [keeps processing new ingresses even if one of the existing ingresses is misconfigured](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L501) -- [should overwrite Foo header with auth response](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L525) -- [should return status code 200 when signed in](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L701) -- [should redirect to signin url when not signed in](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L710) -- [keeps processing new ingresses even if one of the existing ingresses is misconfigured](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L721) -- [should return status code 200 when signed in after auth backend is deleted ](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L780) -- [should deny login for different location on same server](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L800) -- [should deny login for different servers](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L828) -- [should redirect to signin url when not signed in](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L857) -- [should return 503 (location was denied)](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L887) -- [should add error to the config](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/auth.go#L895) -### [auth-tls-*](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/authtls.go#L31) -- [should set sslClientCertificate, sslVerifyClient and sslVerifyDepth with auth-tls-secret](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/authtls.go#L38) -- [should set valid auth-tls-secret, sslVerify to off, and sslVerifyDepth to 2](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/authtls.go#L86) -- [should 302 redirect to error page instead of 400 when auth-tls-error-page is set](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/authtls.go#L116) -- [should pass URL-encoded certificate to upstream](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/authtls.go#L163) -- [should validate auth-tls-verify-client](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/authtls.go#L208) -- [should return 403 using auth-tls-match-cn with no matching CN from client](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/authtls.go#L267) -- [should return 200 using auth-tls-match-cn with matching CN from client](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/authtls.go#L296) -- [should reload the nginx config when auth-tls-match-cn is updated](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/authtls.go#L325) -- [should return 200 using auth-tls-match-cn where atleast one of the regex options matches CN from client](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/authtls.go#L368) -### [backend-protocol](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/backendprotocol.go#L29) -- [should set backend protocol to https:// and use proxy_pass](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/backendprotocol.go#L36) -- [should set backend protocol to https:// and use proxy_pass with lowercase annotation](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/backendprotocol.go#L51) -- [should set backend protocol to $scheme:// and use proxy_pass](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/backendprotocol.go#L66) -- [should set backend protocol to grpc:// and use grpc_pass](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/backendprotocol.go#L81) -- [should set backend protocol to grpcs:// and use grpc_pass](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/backendprotocol.go#L96) -- [should set backend protocol to '' and use fastcgi_pass](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/backendprotocol.go#L111) -### [canary-*](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L36) -- [should response with a 200 status from the mainline upstream when requests are made to the mainline ingress](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L45) -- [should return 404 status for requests to the canary if no matching ingress is found](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L89) -- [should return the correct status codes when endpoints are unavailable](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L120) -- [should route requests to the correct upstream if mainline ingress is created before the canary ingress](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L174) -- [should route requests to the correct upstream if mainline ingress is created after the canary ingress](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L232) -- [should route requests to the correct upstream if the mainline ingress is modified](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L289) -- [should route requests to the correct upstream if the canary ingress is modified](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L363) -- [should route requests to the correct upstream](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L445) -- [should route requests to the correct upstream](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L513) -- [should route requests to the correct upstream](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L594) -- [should route requests to the correct upstream](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L647) -- [should routes to mainline upstream when the given Regex causes error](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L692) -- [should route requests to the correct upstream](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L741) -- [respects always and never values](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L790) -- [should route requests only to mainline if canary weight is 0](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L862) -- [should route requests only to canary if canary weight is 100](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L910) -- [should route requests only to canary if canary weight is equal to canary weight total](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L952) -- [should route requests split between mainline and canary if canary weight is 50](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L995) -- [should route requests split between mainline and canary if canary weight is 100 and weight total is 200](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L1031) -- [should not use canary as a catch-all server](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L1070) -- [should not use canary with domain as a server](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L1104) -- [does not crash when canary ingress has multiple paths to the same non-matching backend](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L1138) -- [always routes traffic to canary if first request was affinitized to canary (default behavior)](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L1175) -- [always routes traffic to canary if first request was affinitized to canary (explicit sticky behavior)](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L1242) -- [routes traffic to either mainline or canary backend (legacy behavior)](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/canary.go#L1310) -### [client-body-buffer-size](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/clientbodybuffersize.go#L30) -- [should set client_body_buffer_size to 1000](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/clientbodybuffersize.go#L37) -- [should set client_body_buffer_size to 1K](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/clientbodybuffersize.go#L59) -- [should set client_body_buffer_size to 1k](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/clientbodybuffersize.go#L81) -- [should set client_body_buffer_size to 1m](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/clientbodybuffersize.go#L103) -- [should set client_body_buffer_size to 1M](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/clientbodybuffersize.go#L125) -- [should not set client_body_buffer_size to invalid 1b](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/clientbodybuffersize.go#L147) -### [connection-proxy-header](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/connection.go#L28) -- [set connection header to keep-alive](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/connection.go#L35) -### [cors-*](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L33) -- [should enable cors](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L40) -- [should set cors methods to only allow POST, GET](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L67) -- [should set cors max-age](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L83) -- [should disable cors allow credentials](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L99) -- [should allow origin for cors](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L115) -- [should allow headers for cors](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L142) -- [should expose headers for cors](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L158) -- [should allow - single origin for multiple cors values](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L174) -- [should not allow - single origin for multiple cors values](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L201) -- [should allow correct origins - single origin for multiple cors values](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L221) -- [should not break functionality](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L272) -- [should not break functionality - without `*`](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L296) -- [should not break functionality with extra domain](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L319) -- [should not match](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L343) -- [should allow - single origin with required port](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L363) -- [should not allow - single origin with port and origin without port](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L391) -- [should not allow - single origin without port and origin with required port](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L410) -- [should allow - matching origin with wildcard origin (2 subdomains)](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L430) -- [should not allow - unmatching origin with wildcard origin (2 subdomains)](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L473) -- [should allow - matching origin+port with wildcard origin](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L493) -- [should not allow - portless origin with wildcard origin](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L520) -- [should allow correct origins - missing subdomain + origin with wildcard origin and correct origin](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L540) -- [should allow - missing origins (should allow all origins)](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L576) -- [should allow correct origin but not others - cors allow origin annotations contain trailing comma](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L636) -- [should allow - origins with non-http[s] protocols](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/cors.go#L673) -### [custom-headers-*](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/customheaders.go#L33) -- [should return status code 200 when no custom-headers is configured](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/customheaders.go#L40) -- [should return status code 503 when custom-headers is configured with an invalid secret](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/customheaders.go#L57) -- [more_set_headers 'My-Custom-Header' '42';](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/customheaders.go#L78) -### [custom-http-errors](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/customhttperrors.go#L34) -- [configures Nginx correctly](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/customhttperrors.go#L41) -### [default-backend](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/default_backend.go#L29) -- [should use a custom default backend as upstream](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/default_backend.go#L37) -### [disable-access-log disable-http-access-log disable-stream-access-log](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/disableaccesslog.go#L28) -- [disable-access-log set access_log off](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/disableaccesslog.go#L35) -- [disable-http-access-log set access_log off](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/disableaccesslog.go#L53) -- [disable-stream-access-log set access_log off](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/disableaccesslog.go#L71) -### [disable-proxy-intercept-errors](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/disableproxyintercepterrors.go#L31) -- [configures Nginx correctly](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/disableproxyintercepterrors.go#L39) -### [backend-protocol - FastCGI](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/fastcgi.go#L30) -- [should use fastcgi_pass in the configuration file](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/fastcgi.go#L37) -- [should add fastcgi_index in the configuration file](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/fastcgi.go#L54) -- [should add fastcgi_param in the configuration file](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/fastcgi.go#L71) -- [should return OK for service with backend protocol FastCGI](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/fastcgi.go#L102) -### [force-ssl-redirect](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/forcesslredirect.go#L27) -- [should redirect to https](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/forcesslredirect.go#L34) -### [from-to-www-redirect](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/fromtowwwredirect.go#L31) -- [should redirect from www HTTP to HTTP](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/fromtowwwredirect.go#L38) -- [should redirect from www HTTPS to HTTPS](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/fromtowwwredirect.go#L64) -### [backend-protocol - GRPC](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/grpc.go#L45) -- [should use grpc_pass in the configuration file](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/grpc.go#L48) -- [should return OK for service with backend protocol GRPC](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/grpc.go#L71) -- [authorization metadata should be overwritten by external auth response headers](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/grpc.go#L132) -- [should return OK for service with backend protocol GRPCS](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/grpc.go#L193) -- [should return OK when request not exceed timeout](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/grpc.go#L260) -- [should return Error when request exceed timeout](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/grpc.go#L303) -### [http2-push-preload](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/http2pushpreload.go#L27) -- [enable the http2-push-preload directive](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/http2pushpreload.go#L34) -### [allowlist-source-range](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/ipallowlist.go#L27) -- [should set valid ip allowlist range](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/ipallowlist.go#L34) -### [denylist-source-range](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/ipdenylist.go#L28) -- [only deny explicitly denied IPs, allow all others](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/ipdenylist.go#L35) -- [only allow explicitly allowed IPs, deny all others](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/ipdenylist.go#L86) -### [Annotation - limit-connections](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/limitconnections.go#L31) -- [should limit-connections](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/limitconnections.go#L38) -### [limit-rate](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/limitrate.go#L29) -- [Check limit-rate annotation](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/limitrate.go#L37) -### [enable-access-log enable-rewrite-log](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/log.go#L27) -- [set access_log off](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/log.go#L34) -- [set rewrite_log on](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/log.go#L49) -### [mirror-*](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/mirror.go#L28) -- [should set mirror-target to http://localhost/mirror](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/mirror.go#L36) -- [should set mirror-target to https://test.env.com/$request_uri](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/mirror.go#L51) -- [should disable mirror-request-body](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/mirror.go#L67) -### [modsecurity owasp](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/modsecurity/modsecurity.go#L39) -- [should enable modsecurity](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/modsecurity/modsecurity.go#L46) -- [should enable modsecurity with transaction ID and OWASP rules](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/modsecurity/modsecurity.go#L64) -- [should disable modsecurity](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/modsecurity/modsecurity.go#L85) -- [should enable modsecurity with snippet](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/modsecurity/modsecurity.go#L102) -- [should enable modsecurity without using 'modsecurity on;'](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/modsecurity/modsecurity.go#L124) -- [should disable modsecurity using 'modsecurity off;'](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/modsecurity/modsecurity.go#L147) -- [should enable modsecurity with snippet and block requests](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/modsecurity/modsecurity.go#L169) -- [should enable modsecurity globally and with modsecurity-snippet block requests](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/modsecurity/modsecurity.go#L202) -- [should enable modsecurity when enable-owasp-modsecurity-crs is set to true](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/modsecurity/modsecurity.go#L235) -- [should enable modsecurity through the config map](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/modsecurity/modsecurity.go#L269) -- [should enable modsecurity through the config map but ignore snippet as disabled by admin](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/modsecurity/modsecurity.go#L309) -- [should disable default modsecurity conf setting when modsecurity-snippet is specified](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/modsecurity/modsecurity.go#L354) -### [preserve-trailing-slash](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/preservetrailingslash.go#L27) -- [should allow preservation of trailing slashes](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/preservetrailingslash.go#L34) -### [proxy-*](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxy.go#L30) -- [should set proxy_redirect to off](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxy.go#L38) -- [should set proxy_redirect to default](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxy.go#L54) -- [should set proxy_redirect to hello.com goodbye.com](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxy.go#L70) -- [should set proxy client-max-body-size to 8m](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxy.go#L87) -- [should not set proxy client-max-body-size to incorrect value](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxy.go#L102) -- [should set valid proxy timeouts](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxy.go#L117) -- [should not set invalid proxy timeouts](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxy.go#L138) -- [should turn on proxy-buffering](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxy.go#L159) -- [should turn off proxy-request-buffering](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxy.go#L184) -- [should build proxy next upstream](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxy.go#L199) -- [should setup proxy cookies](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxy.go#L220) -- [should change the default proxy HTTP version](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxy.go#L238) -### [proxy-ssl-*](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxyssl.go#L32) -- [should set valid proxy-ssl-secret](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxyssl.go#L39) -- [should set valid proxy-ssl-secret, proxy-ssl-verify to on, proxy-ssl-verify-depth to 2, and proxy-ssl-server-name to on](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxyssl.go#L66) -- [should set valid proxy-ssl-secret, proxy-ssl-ciphers to HIGH:!AES](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxyssl.go#L96) -- [should set valid proxy-ssl-secret, proxy-ssl-protocols](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxyssl.go#L124) -- [proxy-ssl-location-only flag should change the nginx config server part](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/proxyssl.go#L152) -### [permanent-redirect permanent-redirect-code](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/redirect.go#L30) -- [should respond with a standard redirect code](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/redirect.go#L33) -- [should respond with a custom redirect code](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/redirect.go#L61) -### [relative-redirects](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/relativeredirects.go#L35) -- [configures Nginx correctly](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/relativeredirects.go#L43) -- [should respond with absolute URL in Location](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/relativeredirects.go#L61) -- [should respond with relative URL in Location](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/relativeredirects.go#L85) -### [rewrite-target use-regex enable-rewrite-log](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/rewrite.go#L32) -- [should write rewrite logs](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/rewrite.go#L39) -- [should use correct longest path match](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/rewrite.go#L68) -- [should use ~* location modifier if regex annotation is present](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/rewrite.go#L113) -- [should fail to use longest match for documented warning](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/rewrite.go#L160) -- [should allow for custom rewrite parameters](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/rewrite.go#L192) -### [satisfy](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/satisfy.go#L33) -- [should configure satisfy directive correctly](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/satisfy.go#L40) -- [should allow multiple auth with satisfy any](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/satisfy.go#L82) -### [server-snippet](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/serversnippet.go#L28) -### [service-upstream](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/serviceupstream.go#L32) -- [should use the Service Cluster IP and Port ](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/serviceupstream.go#L41) -- [should use the Service Cluster IP and Port ](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/serviceupstream.go#L69) -- [should not use the Service Cluster IP and Port](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/serviceupstream.go#L97) -### [configuration-snippet](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/snippet.go#L28) -- [set snippet more_set_headers in all locations](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/snippet.go#L34) -- [drops snippet more_set_header in all locations if disabled by admin](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/snippet.go#L66) -### [ssl-ciphers](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/sslciphers.go#L28) -- [should change ssl ciphers](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/sslciphers.go#L35) -- [should keep ssl ciphers](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/sslciphers.go#L58) -### [stream-snippet](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/streamsnippet.go#L34) -- [should add value of stream-snippet to nginx config](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/streamsnippet.go#L41) -- [should add stream-snippet and drop annotations per admin config](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/streamsnippet.go#L88) -### [upstream-hash-by-*](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/upstreamhashby.go#L79) -- [should connect to the same pod](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/upstreamhashby.go#L86) -- [should connect to the same subset of pods](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/upstreamhashby.go#L95) -### [upstream-vhost](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/upstreamvhost.go#L27) -- [set host to upstreamvhost.bar.com](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/upstreamvhost.go#L34) -### [x-forwarded-prefix](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/xforwardedprefix.go#L28) -- [should set the X-Forwarded-Prefix to the annotation value](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/xforwardedprefix.go#L35) -- [should not add X-Forwarded-Prefix if the annotation value is empty](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/annotations/xforwardedprefix.go#L57) -### [[CGroups] cgroups](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/cgroups/cgroups.go#L32) -- [detects cgroups version v1](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/cgroups/cgroups.go#L40) -- [detect cgroups version v2](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/cgroups/cgroups.go#L83) -### [Debug CLI](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/dbg/main.go#L29) -- [should list the backend servers](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/dbg/main.go#L37) -- [should get information for a specific backend server](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/dbg/main.go#L56) -- [should produce valid JSON for /dbg general](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/dbg/main.go#L85) -### [[Default Backend] custom service](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_default_backend.go#L33) -- [uses custom default backend that returns 200 as status code](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_default_backend.go#L36) -### [[Default Backend]](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/default_backend.go#L30) -- [should return 404 sending requests when only a default backend is running](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/default_backend.go#L33) -- [enables access logging for default backend](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/default_backend.go#L88) -- [disables access logging for default backend](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/default_backend.go#L105) -### [[Default Backend] SSL](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/ssl.go#L26) -- [should return a self generated SSL certificate](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/ssl.go#L29) -### [[Default Backend] change default settings](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/with_hosts.go#L30) -- [should apply the annotation to the default backend](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/with_hosts.go#L38) -### [[Disable Leader] Routing works when leader election was disabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/disableleaderelection/disable_leader.go#L28) -- [should create multiple ingress routings rules when leader election has disabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/disableleaderelection/disable_leader.go#L35) -### [[Endpointslices] long service name](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/endpointslices/longname.go#L29) -- [should return 200 when service name has max allowed number of characters 63](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/endpointslices/longname.go#L38) -### [[TopologyHints] topology aware routing](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/endpointslices/topology.go#L34) -- [should return 200 when service has topology hints](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/endpointslices/topology.go#L42) -### [[Shutdown] Grace period shutdown](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/gracefulshutdown/grace_period.go#L32) -- [/healthz should return status code 500 during shutdown grace period](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/gracefulshutdown/grace_period.go#L35) -### [[Shutdown] ingress controller](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/gracefulshutdown/shutdown.go#L30) -- [should shutdown in less than 60 seconds without pending connections](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/gracefulshutdown/shutdown.go#L40) -### [[Shutdown] Graceful shutdown with pending request](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/gracefulshutdown/slow_requests.go#L25) -- [should let slow requests finish before shutting down](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/gracefulshutdown/slow_requests.go#L33) -### [[Ingress] DeepInspection](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ingress/deep_inspection.go#L27) -- [should drop whole ingress if one path matches invalid regex](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ingress/deep_inspection.go#L34) -### [single ingress - multiple hosts](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ingress/multiple_rules.go#L30) -- [should set the correct $service_name NGINX variable](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ingress/multiple_rules.go#L38) -### [[Ingress] [PathType] exact](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ingress/pathtype_exact.go#L30) -- [should choose exact location for /exact](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ingress/pathtype_exact.go#L37) -### [[Ingress] [PathType] mix Exact and Prefix paths](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ingress/pathtype_mixed.go#L30) -- [should choose the correct location](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ingress/pathtype_mixed.go#L39) -### [[Ingress] [PathType] prefix checks](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ingress/pathtype_prefix.go#L28) -- [should return 404 when prefix /aaa does not match request /aaaccc](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ingress/pathtype_prefix.go#L35) -- [should test prefix path using simple regex pattern for /id/{int}](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ingress/pathtype_prefix.go#L72) -- [should test prefix path using regex pattern for /id/{int} ignoring non-digits characters at end of string](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ingress/pathtype_prefix.go#L113) -- [should test prefix path using fixed path size regex pattern /id/{int}{3}](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ingress/pathtype_prefix.go#L142) -- [should correctly route multi-segment path patterns](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ingress/pathtype_prefix.go#L177) -### [[Ingress] definition without host](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ingress/without_host.go#L31) -- [should set ingress details variables for ingresses without a host](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ingress/without_host.go#L34) -- [should set ingress details variables for ingresses with host without IngressRuleValue, only Backend](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ingress/without_host.go#L55) -### [[Memory Leak] Dynamic Certificates](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/leaks/lua_ssl.go#L35) -- [should not leak memory from ingress SSL certificates or configuration updates](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/leaks/lua_ssl.go#L42) -### [[Load Balancer] load-balance](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/loadbalance/configmap.go#L30) -- [should apply the configmap load-balance setting](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/loadbalance/configmap.go#L37) -### [[Load Balancer] EWMA](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/loadbalance/ewma.go#L31) -- [does not fail requests](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/loadbalance/ewma.go#L43) -### [[Load Balancer] round-robin](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/loadbalance/round_robin.go#L31) -- [should evenly distribute requests with round-robin (default algorithm)](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/loadbalance/round_robin.go#L39) -### [[Lua] dynamic certificates](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/lua/dynamic_certificates.go#L37) -- [picks up the certificate when we add TLS spec to existing ingress](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/lua/dynamic_certificates.go#L45) -- [picks up the previously missing secret for a given ingress without reloading](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/lua/dynamic_certificates.go#L70) -- [supports requests with domain with trailing dot](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/lua/dynamic_certificates.go#L145) -- [picks up the updated certificate without reloading](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/lua/dynamic_certificates.go#L149) -- [falls back to using default certificate when secret gets deleted without reloading](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/lua/dynamic_certificates.go#L185) -- [picks up a non-certificate only change](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/lua/dynamic_certificates.go#L218) -- [removes HTTPS configuration when we delete TLS spec](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/lua/dynamic_certificates.go#L233) -### [[Lua] dynamic configuration](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/lua/dynamic_configuration.go#L41) -- [configures balancer Lua middleware correctly](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/lua/dynamic_configuration.go#L49) -- [handles endpoints only changes](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/lua/dynamic_configuration.go#L56) -- [handles endpoints only changes (down scaling of replicas)](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/lua/dynamic_configuration.go#L81) -- [handles endpoints only changes consistently (down scaling of replicas vs. empty service)](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/lua/dynamic_configuration.go#L119) -- [handles an annotation change](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/lua/dynamic_configuration.go#L165) -### [[metrics] exported prometheus metrics](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/metrics/metrics.go#L36) -- [exclude socket request metrics are absent](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/metrics/metrics.go#L51) -- [exclude socket request metrics are present](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/metrics/metrics.go#L73) -- [request metrics per undefined host are present when flag is set](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/metrics/metrics.go#L95) -- [request metrics per undefined host are not present when flag is not set](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/metrics/metrics.go#L128) -### [nginx-configuration](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/nginx/nginx.go#L99) -- [start nginx with default configuration](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/nginx/nginx.go#L102) -- [fails when using alias directive](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/nginx/nginx.go#L114) -- [fails when using root directive](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/nginx/nginx.go#L121) -### [[Security] request smuggling](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/security/request_smuggling.go#L32) -- [should not return body content from error_page](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/security/request_smuggling.go#L39) -### [[Service] backend status code 503](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/servicebackend/service_backend.go#L34) -- [should return 503 when backend service does not exist](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/servicebackend/service_backend.go#L37) -- [should return 503 when all backend service endpoints are unavailable](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/servicebackend/service_backend.go#L55) -### [[Service] Type ExternalName](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/servicebackend/service_externalname.go#L38) -- [works with external name set to incomplete fqdn](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/servicebackend/service_externalname.go#L41) -- [should return 200 for service type=ExternalName without a port defined](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/servicebackend/service_externalname.go#L78) -- [should return 200 for service type=ExternalName with a port defined](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/servicebackend/service_externalname.go#L118) -- [should return status 502 for service type=ExternalName with an invalid host](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/servicebackend/service_externalname.go#L148) -- [should return 200 for service type=ExternalName using a port name](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/servicebackend/service_externalname.go#L184) -- [should return 200 for service type=ExternalName using FQDN with trailing dot](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/servicebackend/service_externalname.go#L225) -- [should update the external name after a service update](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/servicebackend/service_externalname.go#L261) -- [should sync ingress on external name service addition/deletion](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/servicebackend/service_externalname.go#L344) -### [[Service] Nil Service Backend](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/servicebackend/service_nil_backend.go#L31) -- [should return 404 when backend service is nil](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/servicebackend/service_nil_backend.go#L38) -### [access-log](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/access_log.go#L27) -- [use the default configuration](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/access_log.go#L31) -- [use the specified configuration](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/access_log.go#L41) -- [use the specified configuration](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/access_log.go#L52) -- [use the specified configuration](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/access_log.go#L64) -- [use the specified configuration](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/access_log.go#L76) -### [aio-write](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/aio_write.go#L27) -- [should be enabled by default](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/aio_write.go#L30) -- [should be enabled when setting is true](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/aio_write.go#L37) -- [should be disabled when setting is false](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/aio_write.go#L46) -### [Bad annotation values](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/badannotationvalues.go#L29) -- [[BAD_ANNOTATIONS] should drop an ingress if there is an invalid character in some annotation](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/badannotationvalues.go#L36) -- [[BAD_ANNOTATIONS] should drop an ingress if there is a forbidden word in some annotation](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/badannotationvalues.go#L68) -- [[BAD_ANNOTATIONS] should allow an ingress if there is a default blocklist config in place](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/badannotationvalues.go#L105) -- [[BAD_ANNOTATIONS] should drop an ingress if there is a custom blocklist config in place and allow others to pass](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/badannotationvalues.go#L138) -### [brotli](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/brotli.go#L30) -- [should only compress responses that meet the `brotli-min-length` condition](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/brotli.go#L38) -### [Configmap change](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/configmap_change.go#L29) -- [should reload after an update in the configuration](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/configmap_change.go#L36) -### [add-headers](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/custom_header.go#L30) -- [Add a custom header](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/custom_header.go#L40) -- [Add multiple custom headers](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/custom_header.go#L65) -### [[SSL] [Flag] default-ssl-certificate](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/default_ssl_certificate.go#L35) -- [uses default ssl certificate for catch-all ingress](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/default_ssl_certificate.go#L66) -- [uses default ssl certificate for host based ingress when configured certificate does not match host](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/default_ssl_certificate.go#L82) -### [[Flag] disable-catch-all](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/disable_catch_all.go#L33) -- [should ignore catch all Ingress with backend](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/disable_catch_all.go#L50) -- [should ignore catch all Ingress with backend and rules](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/disable_catch_all.go#L69) -- [should delete Ingress updated to catch-all](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/disable_catch_all.go#L81) -- [should allow Ingress with rules](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/disable_catch_all.go#L123) -### [[Flag] disable-service-external-name](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/disable_service_external_name.go#L35) -- [should ignore services of external-name type](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/disable_service_external_name.go#L55) -### [[Flag] disable-sync-events](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/disable_sync_events.go#L32) -- [should create sync events (default)](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/disable_sync_events.go#L35) -- [should create sync events](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/disable_sync_events.go#L55) -- [should not create sync events](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/disable_sync_events.go#L83) -### [enable-real-ip](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/enable_real_ip.go#L30) -- [trusts X-Forwarded-For header only when setting is true](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/enable_real_ip.go#L40) -- [should not trust X-Forwarded-For header when setting is false](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/enable_real_ip.go#L79) -### [use-forwarded-headers](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/forwarded_headers.go#L31) -- [should trust X-Forwarded headers when setting is true](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/forwarded_headers.go#L41) -- [should not trust X-Forwarded headers when setting is false](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/forwarded_headers.go#L93) -### [Geoip2](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/geoip2.go#L36) -- [should include geoip2 line in config when enabled and db file exists](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/geoip2.go#L45) -- [should only allow requests from specific countries](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/geoip2.go#L69) -- [should up and running nginx controller using autoreload flag](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/geoip2.go#L122) -### [[Security] block-*](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/global_access_block.go#L28) -- [should block CIDRs defined in the ConfigMap](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/global_access_block.go#L38) -- [should block User-Agents defined in the ConfigMap](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/global_access_block.go#L55) -- [should block Referers defined in the ConfigMap](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/global_access_block.go#L88) -### [[Security] global-auth-url](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/global_external_auth.go#L39) -- [should return status code 401 when request any protected service](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/global_external_auth.go#L91) -- [should return status code 200 when request whitelisted (via no-auth-locations) service and 401 when request protected service](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/global_external_auth.go#L107) -- [should return status code 200 when request whitelisted (via ingress annotation) service and 401 when request protected service](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/global_external_auth.go#L130) -- [should still return status code 200 after auth backend is deleted using cache](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/global_external_auth.go#L158) -- [user retains cookie by default](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/global_external_auth.go#L322) -- [user does not retain cookie if upstream returns error status code](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/global_external_auth.go#L333) -- [user with global-auth-always-set-cookie key in configmap retains cookie if upstream returns error status code](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/global_external_auth.go#L344) -### [global-options](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/global_options.go#L28) -- [should have worker_rlimit_nofile option](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/global_options.go#L31) -- [should have worker_rlimit_nofile option and be independent on amount of worker processes](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/global_options.go#L37) -### [GRPC](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/grpc.go#L39) -- [should set the correct GRPC Buffer Size](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/grpc.go#L42) -### [gzip](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/gzip.go#L30) -- [should be disabled by default](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/gzip.go#L40) -- [should be enabled with default settings](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/gzip.go#L56) -- [should set gzip_comp_level to 4](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/gzip.go#L82) -- [should set gzip_disable to msie6](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/gzip.go#L102) -- [should set gzip_min_length to 100](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/gzip.go#L132) -- [should set gzip_types to text/html](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/gzip.go#L164) -### [hash size](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/hash-size.go#L27) -- [should set server_names_hash_bucket_size](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/hash-size.go#L39) -- [should set server_names_hash_max_size](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/hash-size.go#L47) -- [should set proxy-headers-hash-bucket-size](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/hash-size.go#L57) -- [should set proxy-headers-hash-max-size](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/hash-size.go#L65) -- [should set variables-hash-bucket-size](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/hash-size.go#L75) -- [should set variables-hash-max-size](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/hash-size.go#L83) -- [should set vmap-hash-bucket-size](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/hash-size.go#L93) -### [[Flag] ingress-class](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ingress_class.go#L41) -- [should ignore Ingress with a different class annotation](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ingress_class.go#L70) -- [should ignore Ingress with different controller class](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ingress_class.go#L106) -- [should accept both Ingresses with default IngressClassName and IngressClass annotation](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ingress_class.go#L134) -- [should ignore Ingress without IngressClass configuration](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ingress_class.go#L166) -- [should delete Ingress when class is removed](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ingress_class.go#L194) -- [should serve Ingress when class is added](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ingress_class.go#L259) -- [should serve Ingress when class is updated between annotation and ingressClassName](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ingress_class.go#L325) -- [should ignore Ingress with no class and accept the correctly configured Ingresses](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ingress_class.go#L414) -- [should watch Ingress with no class and ignore ingress with a different class](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ingress_class.go#L482) -- [should watch Ingress that uses the class name even if spec is different](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ingress_class.go#L538) -- [should watch Ingress with correct annotation](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ingress_class.go#L628) -- [should ignore Ingress with only IngressClassName](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ingress_class.go#L648) -### [keep-alive keep-alive-requests](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/keep-alive.go#L28) -- [should set keepalive_timeout](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/keep-alive.go#L40) -- [should set keepalive_requests](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/keep-alive.go#L48) -- [should set keepalive connection to upstream server](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/keep-alive.go#L58) -- [should set keep alive connection timeout to upstream server](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/keep-alive.go#L68) -- [should set keepalive time to upstream server](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/keep-alive.go#L78) -- [should set the request count to upstream server through one keep alive connection](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/keep-alive.go#L88) -### [Configmap - limit-rate](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/limit_rate.go#L28) -- [Check limit-rate config](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/limit_rate.go#L36) -### [[Flag] custom HTTP and HTTPS ports](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/listen_nondefault_ports.go#L30) -- [should set X-Forwarded-Port headers accordingly when listening on a non-default HTTP port](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/listen_nondefault_ports.go#L45) -- [should set X-Forwarded-Port header to 443](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/listen_nondefault_ports.go#L65) -- [should set the X-Forwarded-Port header to 443](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/listen_nondefault_ports.go#L93) -### [log-format-*](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/log-format.go#L28) -- [should not configure log-format escape by default](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/log-format.go#L39) -- [should enable the log-format-escape-json](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/log-format.go#L46) -- [should disable the log-format-escape-json](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/log-format.go#L54) -- [should enable the log-format-escape-none](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/log-format.go#L62) -- [should disable the log-format-escape-none](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/log-format.go#L70) -- [log-format-escape-json enabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/log-format.go#L80) -- [log-format default escape](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/log-format.go#L103) -- [log-format-escape-none enabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/log-format.go#L126) -### [[Lua] lua-shared-dicts](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/lua_shared_dicts.go#L26) -- [configures lua shared dicts](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/lua_shared_dicts.go#L29) -### [main-snippet](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/main_snippet.go#L27) -- [should add value of main-snippet setting to nginx config](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/main_snippet.go#L31) -### [[Security] modsecurity-snippet](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/modsecurity/modsecurity_snippet.go#L27) -- [should add value of modsecurity-snippet setting to nginx config](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/modsecurity/modsecurity_snippet.go#L30) -### [enable-multi-accept](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/multi_accept.go#L27) -- [should be enabled by default](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/multi_accept.go#L31) -- [should be enabled when set to true](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/multi_accept.go#L39) -- [should be disabled when set to false](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/multi_accept.go#L49) -### [[Flag] watch namespace selector](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/namespace_selector.go#L30) -- [should ignore Ingress of namespace without label foo=bar and accept those of namespace with label foo=bar](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/namespace_selector.go#L62) -### [[Security] no-auth-locations](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/no_auth_locations.go#L33) -- [should return status code 401 when accessing '/' unauthentication](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/no_auth_locations.go#L54) -- [should return status code 200 when accessing '/' authentication](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/no_auth_locations.go#L68) -- [should return status code 200 when accessing '/noauth' unauthenticated](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/no_auth_locations.go#L82) -### [Add no tls redirect locations](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/no_tls_redirect_locations.go#L27) -- [Check no tls redirect locations config](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/no_tls_redirect_locations.go#L30) -### [OCSP](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ocsp/ocsp.go#L43) -- [should enable OCSP and contain stapling information in the connection](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ocsp/ocsp.go#L50) -### [Configure Opentelemetry](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/opentelemetry.go#L39) -- [should not exists opentelemetry directive](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/opentelemetry.go#L49) -- [should exists opentelemetry directive when is enabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/opentelemetry.go#L62) -- [should include opentelemetry_trust_incoming_spans on directive when enabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/opentelemetry.go#L76) -- [should not exists opentelemetry_operation_name directive when is empty](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/opentelemetry.go#L91) -- [should exists opentelemetry_operation_name directive when is configured](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/opentelemetry.go#L106) -### [proxy-connect-timeout](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_connect_timeout.go#L29) -- [should set valid proxy timeouts using configmap values](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_connect_timeout.go#L37) -- [should not set invalid proxy timeouts using configmap values](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_connect_timeout.go#L53) -### [Dynamic $proxy_host](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_host.go#L28) -- [should exist a proxy_host](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_host.go#L36) -- [should exist a proxy_host using the upstream-vhost annotation value](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_host.go#L60) -### [proxy-next-upstream](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_next_upstream.go#L28) -- [should build proxy next upstream using configmap values](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_next_upstream.go#L36) -### [use-proxy-protocol](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_protocol.go#L38) -- [should respect port passed by the PROXY Protocol](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_protocol.go#L48) -- [should respect proto passed by the PROXY Protocol server port](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_protocol.go#L85) -- [should enable PROXY Protocol for HTTPS](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_protocol.go#L121) -- [should enable PROXY Protocol for TCP](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_protocol.go#L164) -### [proxy-read-timeout](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_read_timeout.go#L29) -- [should set valid proxy read timeouts using configmap values](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_read_timeout.go#L37) -- [should not set invalid proxy read timeouts using configmap values](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_read_timeout.go#L53) -### [proxy-send-timeout](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_send_timeout.go#L29) -- [should set valid proxy send timeouts using configmap values](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_send_timeout.go#L37) -- [should not set invalid proxy send timeouts using configmap values](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/proxy_send_timeout.go#L53) -### [reuse-port](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/reuse-port.go#L27) -- [reuse port should be enabled by default](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/reuse-port.go#L38) -- [reuse port should be disabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/reuse-port.go#L44) -- [reuse port should be enabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/reuse-port.go#L52) -### [configmap server-snippet](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/server_snippet.go#L28) -- [should add value of server-snippet setting to all ingress config](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/server_snippet.go#L35) -- [should add global server-snippet and drop annotations per admin config](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/server_snippet.go#L100) -### [server-tokens](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/server_tokens.go#L29) -- [should not exists Server header in the response](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/server_tokens.go#L38) -- [should exists Server header in the response when is enabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/server_tokens.go#L50) -### [ssl-ciphers](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ssl_ciphers.go#L28) -- [Add ssl ciphers](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ssl_ciphers.go#L31) -### [[Flag] enable-ssl-passthrough](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ssl_passthrough.go#L36) -### [With enable-ssl-passthrough enabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ssl_passthrough.go#L55) -- [should enable ssl-passthrough-proxy-port on a different port](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ssl_passthrough.go#L56) -- [should pass unknown traffic to default backend and handle known traffic](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/ssl_passthrough.go#L78) -### [configmap stream-snippet](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/stream_snippet.go#L35) -- [should add value of stream-snippet via config map to nginx config](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/stream_snippet.go#L42) -### [[SSL] TLS protocols, ciphers and headers](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/tls.go#L32) -- [setting cipher suite](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/tls.go#L66) -- [setting max-age parameter](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/tls.go#L110) -- [setting includeSubDomains parameter](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/tls.go#L127) -- [setting preload parameter](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/tls.go#L147) -- [overriding what's set from the upstream](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/tls.go#L168) -- [should not use ports during the HTTP to HTTPS redirection](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/tls.go#L190) -- [should not use ports or X-Forwarded-Host during the HTTP to HTTPS redirection](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/tls.go#L208) -### [annotation validations](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/validations/validations.go#L30) -- [should allow ingress based on their risk on webhooks](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/validations/validations.go#L33) -- [should allow ingress based on their risk on webhooks](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/settings/validations/validations.go#L68) -### [[SSL] redirect to HTTPS](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ssl/http_redirect.go#L29) -- [should redirect from HTTP to HTTPS when secret is missing](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ssl/http_redirect.go#L36) -### [[SSL] secret update](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ssl/secret_update.go#L33) -- [should not appear references to secret updates not used in ingress rules](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ssl/secret_update.go#L40) -- [should return the fake SSL certificate if the secret is invalid](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/ssl/secret_update.go#L83) -### [[Status] status update](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/status/update.go#L38) -- [should update status field after client-go reconnection](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/status/update.go#L43) -### [[TCP] tcp-services](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/tcpudp/tcp.go#L38) -- [should expose a TCP service](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/tcpudp/tcp.go#L46) -- [should expose an ExternalName TCP service](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/tcpudp/tcp.go#L80) -- [should reload after an update in the configuration](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/tcpudp/tcp.go#L169) \ No newline at end of file +# e2e test suite for [NGINX Ingress Controller](https://github.com/kubernetes/ingress-nginx/tree/master/) + + + +### [[Default Backend] change default settings](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/defaultbackend/with_hosts.go#L31) + +- [should apply the annotation to the default backend](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/defaultbackend/with_hosts.go#L39) + +### [[Default Backend]](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/defaultbackend/default_backend.go#L29) + +- [should return 404 sending requests when only a default backend is running](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/defaultbackend/default_backend.go#L32) +- [enables access logging for default backend](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/defaultbackend/default_backend.go#L89) +- [disables access logging for default backend](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/defaultbackend/default_backend.go#L106) + +### [[Default Backend] custom service](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/defaultbackend/custom_default_backend.go#L33) + +- [uses custom default backend that returns 200 as status code](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/defaultbackend/custom_default_backend.go#L36) + +### [[Default Backend] SSL](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/defaultbackend/ssl.go#L26) + +- [should return a self generated SSL certificate](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/defaultbackend/ssl.go#L29) + +### [[TCP] tcp-services](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/tcpudp/tcp.go#L37) + +- [should expose a TCP service](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/tcpudp/tcp.go#L40) +- [should expose an ExternalName TCP service](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/tcpudp/tcp.go#L98) + +### [auth-*](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L39) + +- [should return status code 200 when no authentication is configured](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L46) +- [should return status code 503 when authentication is configured with an invalid secret](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L65) +- [should return status code 401 when authentication is configured but Authorization header is not configured](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L89) +- [should return status code 401 when authentication is configured and Authorization header is sent with invalid credentials](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L116) +- [should return status code 200 when authentication is configured and Authorization header is sent](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L144) +- [should return status code 200 when authentication is configured with a map and Authorization header is sent](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L171) +- [should return status code 401 when authentication is configured with invalid content and Authorization header is sent](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L199) +- [proxy_set_header My-Custom-Header 42;](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L238) +- [proxy_set_header My-Custom-Header 42;](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L256) +- [proxy_set_header 'My-Custom-Header' '42';](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L273) +- [](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L294) +- [retains cookie set by external authentication server](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L316) +- [should return status code 200 when signed in](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L413) +- [should redirect to signin url when not signed in](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L422) +- [keeps processing new ingresses even if one of the existing ingresses is misconfigured](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L433) +- [should return status code 200 when signed in](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L490) +- [should redirect to signin url when not signed in](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L499) +- [keeps processing new ingresses even if one of the existing ingresses is misconfigured](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L510) +- [should return status code 200 when signed in after auth backend is deleted ](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L584) +- [should deny login for different location on same server](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L604) +- [should deny login for different servers](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L632) +- [should redirect to signin url when not signed in](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/auth.go#L661) + +### [affinitymode](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/affinitymode.go#L31) + +- [Balanced affinity mode should balance](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/affinitymode.go#L34) +- [Check persistent affinity mode](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/affinitymode.go#L64) + +### [proxy-*](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxy.go#L28) + +- [should set proxy_redirect to off](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxy.go#L36) +- [should set proxy_redirect to default](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxy.go#L52) +- [should set proxy_redirect to hello.com goodbye.com](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxy.go#L68) +- [should set proxy client-max-body-size to 8m](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxy.go#L85) +- [should not set proxy client-max-body-size to incorrect value](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxy.go#L100) +- [should set valid proxy timeouts](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxy.go#L115) +- [should not set invalid proxy timeouts](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxy.go#L136) +- [should turn on proxy-buffering](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxy.go#L157) +- [should turn off proxy-request-buffering](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxy.go#L179) +- [should build proxy next upstream](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxy.go#L194) +- [should setup proxy cookies](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxy.go#L215) +- [should change the default proxy HTTP version](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxy.go#L233) + +### [affinity session-cookie-name](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/affinity.go#L35) + +- [should set sticky cookie SERVERID](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/affinity.go#L42) +- [should change cookie name on ingress definition change](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/affinity.go#L64) +- [should set the path to /something on the generated cookie](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/affinity.go#L99) +- [does not set the path to / on the generated cookie if there's more than one rule referring to the same backend](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/affinity.go#L121) +- [should set cookie with expires](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/affinity.go#L182) +- [should work with use-regex annotation and session-cookie-path](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/affinity.go#L213) +- [should warn user when use-regex is true and session-cookie-path is not set](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/affinity.go#L237) +- [should not set affinity across all server locations when using separate ingresses](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/affinity.go#L263) +- [should set sticky cookie without host](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/affinity.go#L295) +- [should work with server-alias annotation](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/affinity.go#L315) + +### [mirror-*](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/mirror.go#L28) + +- [should set mirror-target to http://localhost/mirror](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/mirror.go#L36) +- [should set mirror-target to https://test.env.com/$request_uri](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/mirror.go#L51) +- [should disable mirror-request-body](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/mirror.go#L67) + +### [canary-*](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L35) + +- [should response with a 200 status from the mainline upstream when requests are made to the mainline ingress](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L47) +- [should return 404 status for requests to the canary if no matching ingress is found](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L79) +- [should return the correct status codes when endpoints are unavailable](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L106) +- [should route requests to the correct upstream if mainline ingress is created before the canary ingress](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L160) +- [should route requests to the correct upstream if mainline ingress is created after the canary ingress](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L205) +- [should route requests to the correct upstream if the mainline ingress is modified](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L249) +- [should route requests to the correct upstream if the canary ingress is modified](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L306) +- [should route requests to the correct upstream](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L361) +- [should route requests to the correct upstream](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L415) +- [should route requests to the correct upstream](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L479) +- [should route requests to the correct upstream](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L521) +- [should routes to mainline upstream when the given Regex causes error](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L555) +- [should route requests to the correct upstream](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L593) +- [should route requests to the correct upstream](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L632) +- [should route requests to the correct upstream](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L687) +- [should not use canary as a catch-all server](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L743) +- [should not use canary with domain as a server](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L771) +- [does not crash when canary ingress has multiple paths to the same non-matching backend](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/canary.go#L795) + +### [limit-rate](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/limitrate.go#L29) + +- [Check limit-rate annotation](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/limitrate.go#L37) + +### [force-ssl-redirect](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/forcesslredirect.go#L27) + +- [should redirect to https](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/forcesslredirect.go#L34) + +### [http2-push-preload](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/http2pushpreload.go#L27) + +- [enable the http2-push-preload directive](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/http2pushpreload.go#L34) + +### [proxy-ssl-*](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxyssl.go#L30) + +- [should set valid proxy-ssl-secret](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxyssl.go#L37) +- [should set valid proxy-ssl-secret, proxy-ssl-verify to on, proxy-ssl-verify-depth to 2, and proxy-ssl-server-name to on](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxyssl.go#L64) +- [should set valid proxy-ssl-secret, proxy-ssl-ciphers to HIGH:!AES](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxyssl.go#L94) +- [should set valid proxy-ssl-secret, proxy-ssl-protocols](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxyssl.go#L122) +- [proxy-ssl-location-only flag should change the nginx config server part](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/proxyssl.go#L150) + +### [modsecurity owasp](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/modsecurity.go#L27) + +- [should enable modsecurity](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/modsecurity.go#L34) +- [should enable modsecurity with transaction ID and OWASP rules](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/modsecurity.go#L52) +- [should disable modsecurity](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/modsecurity.go#L73) +- [should enable modsecurity with snippet](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/modsecurity.go#L90) +- [should enable modsecurity without using 'modsecurity on;'](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/modsecurity.go#L109) +- [should disable modsecurity using 'modsecurity off;'](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/modsecurity.go#L131) +- [should enable modsecurity with snippet and block requests](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/modsecurity.go#L152) +- [should enable modsecurity globally and with modsecurity-snippet block requests](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/modsecurity.go#L186) + +### [backend-protocol - GRPC](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/grpc.go#L38) + +- [should use grpc_pass in the configuration file](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/grpc.go#L41) +- [should return OK for service with backend protocol GRPC](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/grpc.go#L66) +- [should return OK for service with backend protocol GRPCS](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/grpc.go#L124) + +### [cors-*](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/cors.go#L28) + +- [should enable cors](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/cors.go#L35) +- [should set cors methods to only allow POST, GET](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/cors.go#L60) +- [should set cors max-age](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/cors.go#L76) +- [should disable cors allow credentials](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/cors.go#L92) +- [should allow origin for cors](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/cors.go#L108) +- [should allow headers for cors](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/cors.go#L124) +- [should expose headers for cors](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/cors.go#L140) + +### [influxdb-*](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/influxdb.go#L39) + +- [should send the request metric to the influxdb server](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/influxdb.go#L48) + +### [Annotation - limit-connections](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/limitconnections.go#L31) + +- [should limit-connections](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/limitconnections.go#L38) + +### [client-body-buffer-size](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/clientbodybuffersize.go#L28) + +- [should set client_body_buffer_size to 1000](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/clientbodybuffersize.go#L35) +- [should set client_body_buffer_size to 1K](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/clientbodybuffersize.go#L51) +- [should set client_body_buffer_size to 1k](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/clientbodybuffersize.go#L67) +- [should set client_body_buffer_size to 1m](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/clientbodybuffersize.go#L83) +- [should set client_body_buffer_size to 1M](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/clientbodybuffersize.go#L99) +- [should not set client_body_buffer_size to invalid 1b](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/clientbodybuffersize.go#L115) + +### [default-backend](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/default_backend.go#L29) + +- [should use a custom default backend as upstream](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/default_backend.go#L37) + +### [connection-proxy-header](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/connection.go#L29) + +- [set connection header to keep-alive](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/connection.go#L36) + +### [upstream-vhost](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/upstreamvhost.go#L27) + +- [set host to upstreamvhost.bar.com](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/upstreamvhost.go#L34) + +### [custom-http-errors](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/customhttperrors.go#L34) + +- [configures Nginx correctly](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/customhttperrors.go#L41) + +### [disable-access-log disable-http-access-log disable-stream-access-log](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/disableaccesslog.go#L27) + +- [disable-access-log set access_log off](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/disableaccesslog.go#L34) +- [disable-http-access-log set access_log off](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/disableaccesslog.go#L46) + +### [server-snippet](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/serversnippet.go#L27) + +- [](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/serversnippet.go#L34) + +### [rewrite-target use-regex enable-rewrite-log](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/rewrite.go#L30) + +- [should write rewrite logs](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/rewrite.go#L37) +- [should use correct longest path match](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/rewrite.go#L66) +- [should use ~* location modifier if regex annotation is present](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/rewrite.go#L111) +- [should fail to use longest match for documented warning](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/rewrite.go#L158) +- [should allow for custom rewrite parameters](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/rewrite.go#L190) + +### [app-root](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/approot.go#L28) + +- [should redirect to /foo](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/approot.go#L35) + +### [whitelist-source-range](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/ipwhitelist.go#L26) + +- [should set valid ip whitelist range](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/ipwhitelist.go#L33) + +### [enable-access-log enable-rewrite-log](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/log.go#L27) + +- [set access_log off](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/log.go#L34) +- [set rewrite_log on](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/log.go#L49) + +### [x-forwarded-prefix](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/xforwardedprefix.go#L28) + +- [should set the X-Forwarded-Prefix to the annotation value](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/xforwardedprefix.go#L35) +- [should not add X-Forwarded-Prefix if the annotation value is empty](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/xforwardedprefix.go#L57) + +### [configuration-snippet](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/snippet.go#L27) + +- [ in all locations](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/snippet.go#L34) + +### [backend-protocol - FastCGI](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/fastcgi.go#L31) + +- [should use fastcgi_pass in the configuration file](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/fastcgi.go#L38) +- [should add fastcgi_index in the configuration file](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/fastcgi.go#L55) +- [should add fastcgi_param in the configuration file](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/fastcgi.go#L72) +- [should return OK for service with backend protocol FastCGI](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/fastcgi.go#L105) + +### [from-to-www-redirect](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/fromtowwwredirect.go#L31) + +- [should redirect from www HTTP to HTTP](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/fromtowwwredirect.go#L38) +- [should redirect from www HTTPS to HTTPS](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/fromtowwwredirect.go#L64) + +### [permanent-redirect permanent-redirect-code](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/redirect.go#L30) + +- [should respond with a standard redirect code](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/redirect.go#L33) +- [should respond with a custom redirect code](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/redirect.go#L61) + +### [upstream-hash-by-*](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/upstreamhashby.go#L76) + +- [should connect to the same pod](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/upstreamhashby.go#L83) +- [should connect to the same subset of pods](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/upstreamhashby.go#L92) + +### [annotation-global-rate-limit](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/globalratelimit.go#L30) + +- [generates correct configuration](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/globalratelimit.go#L38) + +### [backend-protocol](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/backendprotocol.go#L27) + +- [should set backend protocol to https:// and use proxy_pass](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/backendprotocol.go#L34) +- [should set backend protocol to grpc:// and use grpc_pass](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/backendprotocol.go#L49) +- [should set backend protocol to grpcs:// and use grpc_pass](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/backendprotocol.go#L64) +- [should set backend protocol to '' and use fastcgi_pass](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/backendprotocol.go#L79) +- [should set backend protocol to '' and use ajp_pass](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/backendprotocol.go#L94) + +### [satisfy](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/satisfy.go#L35) + +- [should configure satisfy directive correctly](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/satisfy.go#L42) +- [should allow multiple auth with satisfy any](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/satisfy.go#L84) + +### [server-alias](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/alias.go#L29) + +- [should return status code 200 for host 'foo' and 404 for 'bar'](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/alias.go#L36) +- [should return status code 200 for host 'foo' and 'bar'](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/alias.go#L62) +- [should return status code 200 for hosts defined in two ingresses, different path with one alias](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/alias.go#L87) + +### [ssl-ciphers](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/sslciphers.go#L27) + +- [should change ssl ciphers](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/sslciphers.go#L34) + +### [auth-tls-*](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/authtls.go#L30) + +- [should set valid auth-tls-secret](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/authtls.go#L37) +- [should set valid auth-tls-secret, sslVerify to off, and sslVerifyDepth to 2](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/authtls.go#L73) +- [should set valid auth-tls-secret, pass certificate to upstream, and error page](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/authtls.go#L103) +- [should validate auth-tls-verify-client](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/annotations/authtls.go#L153) + +### [[Status] status update](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/status/update.go#L38) + +- [should update status field after client-go reconnection](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/status/update.go#L43) + +### [Debug CLI](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/dbg/main.go#L29) + +- [should list the backend servers](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/dbg/main.go#L37) +- [should get information for a specific backend server](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/dbg/main.go#L56) +- [should produce valid JSON for /dbg general](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/dbg/main.go#L85) + +### [[Memory Leak] Dynamic Certificates](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/leaks/lua_ssl.go#L35) + +- [should not leak memory from ingress SSL certificates or configuration updates](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/leaks/lua_ssl.go#L42) + +### [[Ingress] [PathType] mix Exact and Prefix paths](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/ingress/pathtype_mixed.go#L30) + +- [should choose the correct location](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/ingress/pathtype_mixed.go#L39) + +### [[Ingress] definition without host](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/ingress/without_host.go#L32) + +- [should set ingress details variables for ingresses without a host](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/ingress/without_host.go#L35) +- [should set ingress details variables for ingresses with host without IngressRuleValue, only Backend](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/ingress/without_host.go#L56) + +### [single ingress - multiple hosts](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/ingress/multiple_rules.go#L31) + +- [should set the correct $service_name NGINX variable](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/ingress/multiple_rules.go#L39) + +### [[Ingress] [PathType] exact](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/ingress/pathtype_exact.go#L30) + +- [should choose exact location for /exact](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/ingress/pathtype_exact.go#L37) + +### [[Ingress] [PathType] prefix checks](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/ingress/pathtype_prefix.go#L28) + +- [should return 404 when prefix /aaa does not match request /aaaccc](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/ingress/pathtype_prefix.go#L35) + +### [[Security] request smuggling](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/security/request_smuggling.go#L32) + +- [should not return body content from error_page](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/security/request_smuggling.go#L39) + +### [[SSL] [Flag] default-ssl-certificate](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/default_ssl_certificate.go#L33) + +- [uses default ssl certificate for catch-all ingress](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/default_ssl_certificate.go#L64) +- [uses default ssl certificate for host based ingress when configured certificate does not match host](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/default_ssl_certificate.go#L80) + +### [enable-real-ip](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/enable_real_ip.go#L30) + +- [trusts X-Forwarded-For header only when setting is true](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/enable_real_ip.go#L40) +- [should not trust X-Forwarded-For header when setting is false](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/enable_real_ip.go#L78) + +### [access-log](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/access_log.go#L26) + +- [use the default configuration](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/access_log.go#L31) +- [use the specified configuration](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/access_log.go#L39) +- [use the specified configuration](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/access_log.go#L51) +- [use the specified configuration](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/access_log.go#L63) +- [use the specified configuration](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/access_log.go#L75) + +### [[Lua] lua-shared-dicts](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/lua_shared_dicts.go#L26) + +- [configures lua shared dicts](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/lua_shared_dicts.go#L29) + +### [server-tokens](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/server_tokens.go#L30) + +- [should not exists Server header in the response](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/server_tokens.go#L38) +- [should exists Server header in the response when is enabled](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/server_tokens.go#L50) + +### [use-proxy-protocol](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_protocol.go#L36) + +- [should respect port passed by the PROXY Protocol](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_protocol.go#L46) +- [should respect proto passed by the PROXY Protocol server port](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_protocol.go#L79) +- [should enable PROXY Protocol for HTTPS](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_protocol.go#L112) +- [should enable PROXY Protocol for TCP](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_protocol.go#L155) + +### [[Flag] custom HTTP and HTTPS ports](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/listen_nondefault_ports.go#L32) + +- [should set X-Forwarded-Port headers accordingly when listening on a non-default HTTP port](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/listen_nondefault_ports.go#L48) +- [should set X-Forwarded-Port header to 443](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/listen_nondefault_ports.go#L70) +- [should set the X-Forwarded-Port header to 443](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/listen_nondefault_ports.go#L100) + +### [[Security] no-auth-locations](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/no_auth_locations.go#L34) + +- [should return status code 401 when accessing '/' unauthentication](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/no_auth_locations.go#L55) +- [should return status code 200 when accessing '/' authentication](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/no_auth_locations.go#L69) +- [should return status code 200 when accessing '/noauth' unauthenticated](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/no_auth_locations.go#L83) + +### [Dynamic $proxy_host](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_host.go#L28) + +- [should exist a proxy_host](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_host.go#L36) +- [should exist a proxy_host using the upstream-vhost annotation value](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_host.go#L57) + +### [proxy-connect-timeout](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_connect_timeout.go#L28) + +- [should set valid proxy timeouts using configmap values](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_connect_timeout.go#L36) +- [should not set invalid proxy timeouts using configmap values](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_connect_timeout.go#L52) + +### [[Security] Pod Security Policies](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/pod_security_policy.go#L40) + +- [should be running with a Pod Security Policy](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/pod_security_policy.go#L43) + +### [Geoip2](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/geoip2.go#L29) + +- [should only allow requests from specific countries](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/geoip2.go#L38) + +### [[Security] Pod Security Policies with volumes](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/pod_security_policy_volumes.go#L36) + +- [should be running with a Pod Security Policy](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/pod_security_policy_volumes.go#L39) + +### [enable-multi-accept](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/multi_accept.go#L27) + +- [should be enabled by default](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/multi_accept.go#L31) +- [should be enabled when set to true](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/multi_accept.go#L39) +- [should be disabled when set to false](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/multi_accept.go#L49) + +### [log-format-*](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/log-format.go#L28) + +- [should disable the log-format-escape-json by default](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/log-format.go#L40) +- [should enable the log-format-escape-json](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/log-format.go#L47) +- [should disable the log-format-escape-json](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/log-format.go#L55) +- [log-format-escape-json enabled](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/log-format.go#L66) +- [log-format-escape-json disabled](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/log-format.go#L89) + +### [[Flag] ingress-class](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/ingress_class.go#L39) + +- [should ignore Ingress with class](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/ingress_class.go#L91) +- [should ignore Ingress with no class](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/ingress_class.go#L143) +- [should delete Ingress when class is removed](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/ingress_class.go#L177) +- [check scenarios for IngressClass and ingress.class annotation](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/ingress_class.go#L216) + +### [ssl-ciphers](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/ssl_ciphers.go#L28) + +- [Add ssl ciphers](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/ssl_ciphers.go#L31) + +### [proxy-next-upstream](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_next_upstream.go#L28) + +- [should build proxy next upstream using configmap values](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_next_upstream.go#L36) + +### [[Security] global-auth-url](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/global_external_auth.go#L32) + +- [should return status code 401 when request any protected service](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/global_external_auth.go#L83) +- [should return status code 200 when request whitelisted (via no-auth-locations) service and 401 when request protected service](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/global_external_auth.go#L100) +- [should return status code 200 when request whitelisted (via ingress annotation) service and 401 when request protected service](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/global_external_auth.go#L124) +- [should still return status code 200 after auth backend is deleted using cache ](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/global_external_auth.go#L153) +- [](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/global_external_auth.go#L195) +- [](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/global_external_auth.go#L208) +- [](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/global_external_auth.go#L221) +- [](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/global_external_auth.go#L235) +- [](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/global_external_auth.go#L248) + +### [[Security] block-*](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/global_access_block.go#L28) + +- [should block CIDRs defined in the ConfigMap](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/global_access_block.go#L38) +- [should block User-Agents defined in the ConfigMap](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/global_access_block.go#L55) +- [should block Referers defined in the ConfigMap](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/global_access_block.go#L88) + +### [plugins](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/plugins.go#L28) + +- [should exist a x-hello-world header](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/plugins.go#L35) + +### [Configmap - limit-rate](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/limit_rate.go#L28) + +- [Check limit-rate config](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/limit_rate.go#L36) + +### [Configure OpenTracing](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/opentracing.go#L47) + +- [should not exists opentracing directive](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/opentracing.go#L57) +- [should exists opentracing directive when is enabled](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/opentracing.go#L70) +- [should not exists opentracing_operation_name directive when is empty](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/opentracing.go#L84) +- [should exists opentracing_operation_name directive when is configured](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/opentracing.go#L99) +- [should not exists opentracing_location_operation_name directive when is empty](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/opentracing.go#L114) +- [should exists opentracing_location_operation_name directive when is configured](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/opentracing.go#L129) +- [should enable opentracing using zipkin](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/opentracing.go#L144) +- [should enable opentracing using jaeger](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/opentracing.go#L156) +- [should enable opentracing using jaeger with sampler host](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/opentracing.go#L168) +- [should propagate the w3c header when configured with jaeger](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/opentracing.go#L181) +- [should enable opentracing using datadog](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/opentracing.go#L225) + +### [use-forwarded-headers](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/forwarded_headers.go#L30) + +- [should trust X-Forwarded headers when setting is true](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/forwarded_headers.go#L40) +- [should not trust X-Forwarded headers when setting is false](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/forwarded_headers.go#L90) + +### [proxy-send-timeout](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_send_timeout.go#L28) + +- [should set valid proxy send timeouts using configmap values](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_send_timeout.go#L36) +- [should not set invalid proxy send timeouts using configmap values](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_send_timeout.go#L52) + +### [Add no tls redirect locations](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/no_tls_redirect_locations.go#L28) + +- [Check no tls redirect locations config](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/no_tls_redirect_locations.go#L31) + +### [settings-global-rate-limit](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/globalratelimit.go#L30) + +- [generates correct NGINX configuration](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/globalratelimit.go#L38) + +### [add-headers](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/custom_header.go#L30) + +- [Add a custom header](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/custom_header.go#L40) +- [Add multiple custom headers](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/custom_header.go#L65) + +### [hash size](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/hash-size.go#L27) + +- [should set server_names_hash_bucket_size](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/hash-size.go#L40) +- [should set server_names_hash_max_size](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/hash-size.go#L48) +- [should set proxy-headers-hash-bucket-size](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/hash-size.go#L60) +- [should set proxy-headers-hash-max-size](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/hash-size.go#L68) +- [should set variables-hash-bucket-size](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/hash-size.go#L80) +- [should set variables-hash-max-size](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/hash-size.go#L88) +- [should set vmap-hash-bucket-size](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/hash-size.go#L100) + +### [keep-alive keep-alive-requests](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/keep-alive.go#L28) + +- [should set keepalive_timeout](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/keep-alive.go#L40) +- [should set keepalive_requests](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/keep-alive.go#L48) +- [should set keepalive connection to upstream server](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/keep-alive.go#L59) +- [should set keep alive connection timeout to upstream server](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/keep-alive.go#L68) +- [should set the request count to upstream server through one keep alive connection](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/keep-alive.go#L77) + +### [[Flag] disable-catch-all](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/disable_catch_all.go#L34) + +- [should ignore catch all Ingress](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/disable_catch_all.go#L51) +- [should delete Ingress updated to catch-all](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/disable_catch_all.go#L70) +- [should allow Ingress with both a default backend and rules](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/disable_catch_all.go#L108) + +### [main-snippet](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/main_snippet.go#L27) + +- [should add value of main-snippet setting to nginx config](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/main_snippet.go#L31) + +### [[SSL] TLS protocols, ciphers and headers)](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/tls.go#L31) + +- [setting cipher suite](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/tls.go#L65) +- [enforcing TLS v1.0](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/tls.go#L87) +- [setting max-age parameter](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/tls.go#L133) +- [setting includeSubDomains parameter](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/tls.go#L149) +- [setting preload parameter](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/tls.go#L168) +- [overriding what's set from the upstream](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/tls.go#L188) +- [should not use ports during the HTTP to HTTPS redirection](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/tls.go#L209) +- [should not use ports or X-Forwarded-Host during the HTTP to HTTPS redirection](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/tls.go#L227) + +### [Configmap change](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/configmap_change.go#L29) + +- [should reload after an update in the configuration](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/configmap_change.go#L36) + +### [proxy-read-timeout](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_read_timeout.go#L28) + +- [should set valid proxy read timeouts using configmap values](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_read_timeout.go#L36) +- [should not set invalid proxy read timeouts using configmap values](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/proxy_read_timeout.go#L52) + +### [[Security] modsecurity-snippet](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/modsecurity_snippet.go#L27) + +- [should add value of modsecurity-snippet setting to nginx config](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/modsecurity_snippet.go#L30) + +### [OCSP](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/ocsp/ocsp.go#L42) + +- [should enable OCSP and contain stapling information in the connection](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/ocsp/ocsp.go#L49) + +### [reuse-port](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/reuse-port.go#L27) + +- [reuse port should be enabled by default](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/reuse-port.go#L38) +- [reuse port should be disabled](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/reuse-port.go#L44) +- [reuse port should be enabled](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/settings/reuse-port.go#L52) + +### [[Shutdown] Graceful shutdown with pending request](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/gracefulshutdown/slow_requests.go#L28) + +- [should let slow requests finish before shutting down](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/gracefulshutdown/slow_requests.go#L36) + +### [[Shutdown] ingress controller](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/gracefulshutdown/shutdown.go#L33) + +- [should shutdown in less than 60 secons without pending connections](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/gracefulshutdown/shutdown.go#L43) +- [should shutdown after waiting 60 seconds for pending connections to be closed](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/gracefulshutdown/shutdown.go#L64) +- [should shutdown after waiting 150 seconds for pending connections to be closed](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/gracefulshutdown/shutdown.go#L109) + +### [[Service] backend status code 503](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/servicebackend/service_backend.go#L32) + +- [should return 503 when backend service does not exist](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/servicebackend/service_backend.go#L35) +- [should return 503 when all backend service endpoints are unavailable](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/servicebackend/service_backend.go#L53) + +### [[Service] Type ExternalName](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/servicebackend/service_externalname.go#L37) + +- [works with external name set to incomplete fqdn](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/servicebackend/service_externalname.go#L40) +- [should return 200 for service type=ExternalName without a port defined](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/servicebackend/service_externalname.go#L73) +- [should return 200 for service type=ExternalName with a port defined](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/servicebackend/service_externalname.go#L107) +- [should return status 502 for service type=ExternalName with an invalid host](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/servicebackend/service_externalname.go#L148) +- [should return 200 for service type=ExternalName using a port name](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/servicebackend/service_externalname.go#L179) +- [should return 200 for service type=ExternalName using FQDN with trailing dot](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/servicebackend/service_externalname.go#L221) +- [should update the external name after a service update](https://github.com/kubernetes/ingress-nginx/tree/master/test/e2e/servicebackend/service_externalname.go#L252) diff --git a/docs/enhancements/20190815-zone-aware-routing.md b/docs/enhancements/20190815-zone-aware-routing.md index 7000fb6fd..357e8ad92 100644 --- a/docs/enhancements/20190815-zone-aware-routing.md +++ b/docs/enhancements/20190815-zone-aware-routing.md @@ -17,15 +17,13 @@ status: implementable ## Table of Contents -- [Availability zone aware routing](#availability-zone-aware-routing) - - [Table of Contents](#table-of-contents) - - [Summary](#summary) - - [Motivation](#motivation) - - [Goals](#goals) - - [Non-Goals](#non-goals) - - [Proposal](#proposal) - - [Implementation History](#implementation-history) - - [Drawbacks [optional]](#drawbacks-optional) +- [Summary](#summary) +- [Motivation](#motivation) + - [Goals](#goals) + - [Non-Goals](#non-goals) +- [Proposal](#proposal) +- [Implementation History](#implementation-history) +- [Drawbacks [optional]](#drawbacks-optional) ## Summary @@ -41,7 +39,7 @@ inter-zone traffic and usually costs extra money. At the time of this writing, GCP charges $0.01 per GB of inter-zone egress traffic according to https://cloud.google.com/compute/network-pricing. -According to [https://datapath.io/resources/blog/what-are-aws-data-transfer-costs-and-how-to-minimize-them/](https://web.archive.org/web/20201008160149/https://datapath.io/resources/blog/what-are-aws-data-transfer-costs-and-how-to-minimize-them/) Amazon also charges the same amount of money as GCP for cross-zone, egress traffic. +According to https://datapath.io/resources/blog/what-are-aws-data-transfer-costs-and-how-to-minimize-them/ Amazon also charges the same amount of money as GCP for cross-zone, egress traffic. This can be a lot of money depending on once's traffic. By teaching ingress-nginx about zones we can eliminate or at least decrease this cost. diff --git a/docs/enhancements/20231001-split-containers.md b/docs/enhancements/20231001-split-containers.md deleted file mode 100644 index 3c2e85094..000000000 --- a/docs/enhancements/20231001-split-containers.md +++ /dev/null @@ -1,110 +0,0 @@ -# Proposal to split containers - -* All the NGINX files should live on one container - * No file other than NGINX files should exist on this container - * This includes not mounting the service account -* All the controller files should live on a different container - * Controller container should have bare minimum to work (just go program) - * ServiceAccount should be mounted just on controller - -* Inside nginx container, there should be a really small http listener just able -to start, stop and reload NGINX - -## Roadmap (what needs to be done) -* Map what needs to be done to mount the SA just on controller container -* Map all the required files for NGINX to work -* Map all the required network calls between controller and NGINX - * eg.: Dynamic lua reconfiguration -* Map problematic features that will need attention - * SSLPassthrough today happens on controller process and needs to happen on NGINX - -### Ports and endpoints on NGINX container -* Public HTTP/HTTPs port - 80 and 443 -* Lua configuration port - 10246 (HTTP) and 10247 (Stream) -* 3333 (temp) - Dataplane controller http server - * /reload - (POST) Reloads the configuration. - * "config" argument is the location of temporary file that should be used / moved to nginx.conf - * /test - (POST) Test the configuration of a given file location - * "config" argument is the location of temporary file that should be tested - -### Mounting empty SA on controller container - -```yaml -kind: Pod -apiVersion: v1 -metadata: - name: test -spec: - containers: - - name: nginx - image: nginx:latest - ports: - - containerPort: 80 - - name: othernginx - image: alpine:latest - command: ["/bin/sh"] - args: ["-c", "while true; do date; sleep 3; done"] - volumeMounts: - - mountPath: /var/run/secrets/kubernetes.io/serviceaccount - name: emptysecret - volumes: - - name: emptysecret - emptyDir: - sizeLimit: 1Mi -``` - -### Mapped folders on NGINX configuration -**WARNING** We need to be aware of inter mount containers and inode problems. If we -mount a file instead of a directory, it may take time to reflect the file value on -the target container - -* "/etc/nginx/lua/?.lua;/etc/nginx/lua/vendor/?.lua;;"; - Lua scripts -* "/var/log/nginx" - NGINX logs -* "/tmp/nginx (nginx.pid)" - NGINX pid directory / file, fcgi socket, etc -* " /etc/nginx/geoip" - GeoIP database directory - OK - /etc/ingress-controller/geoip -* /etc/nginx/mime.types - Mime types -* /etc/ingress-controller/ssl - SSL directory (fake cert, auth cert) -* /etc/ingress-controller/auth - Authentication files -* /etc/nginx/modsecurity - Modsecurity configuration -* /etc/nginx/owasp-modsecurity-crs - Modsecurity rules -* /etc/nginx/tickets.key - SSL tickets - OK - /etc/ingress-controller/tickets.key -* /etc/nginx/opentelemetry.toml - OTEL config - OK - /etc/ingress-controller/telemetry -* /etc/nginx/opentracing.json - Opentracing config - OK - /etc/ingress-controller/telemetry -* /etc/nginx/modules - NGINX modules -* /etc/nginx/fastcgi_params (maybe) - fcgi params -* /etc/nginx/template - Template, may be used by controller only - -##### List of modules -``` -ngx_http_auth_digest_module.so ngx_http_modsecurity_module.so -ngx_http_brotli_filter_module.so ngx_http_opentracing_module.so -ngx_http_brotli_static_module.so ngx_stream_geoip2_module.so -ngx_http_geoip2_module.so -``` - -##### List of files that may be removed -``` --rw-r--r-- 1 www-data www-data 1077 Jun 23 19:44 fastcgi.conf --rw-r--r-- 1 www-data www-data 1077 Jun 23 19:44 fastcgi.conf.default --rw-r--r-- 1 www-data www-data 1007 Jun 23 19:44 fastcgi_params --rw-r--r-- 1 www-data www-data 1007 Jun 23 19:44 fastcgi_params.default -drwxr-xr-x 2 www-data www-data 4096 Jun 23 19:34 geoip --rw-r--r-- 1 www-data www-data 2837 Jun 23 19:44 koi-utf --rw-r--r-- 1 www-data www-data 2223 Jun 23 19:44 koi-win -drwxr-xr-x 6 www-data www-data 4096 Sep 19 14:13 lua --rw-r--r-- 1 www-data www-data 5349 Jun 23 19:44 mime.types --rw-r--r-- 1 www-data www-data 5349 Jun 23 19:44 mime.types.default -drwxr-xr-x 2 www-data www-data 4096 Jun 23 19:44 modsecurity -drwxr-xr-x 2 www-data www-data 4096 Jun 23 19:44 modules --rw-r--r-- 1 www-data www-data 18275 Oct 1 21:28 nginx.conf --rw-r--r-- 1 www-data www-data 2656 Jun 23 19:44 nginx.conf.default --rwx------ 1 www-data www-data 420 Oct 1 21:28 opentelemetry.toml --rw-r--r-- 1 www-data www-data 2 Oct 1 21:28 opentracing.json -drwxr-xr-x 7 www-data www-data 4096 Jun 23 19:44 owasp-modsecurity-crs --rw-r--r-- 1 www-data www-data 636 Jun 23 19:44 scgi_params --rw-r--r-- 1 www-data www-data 636 Jun 23 19:44 scgi_params.default -drwxr-xr-x 2 www-data www-data 4096 Sep 19 14:13 template --rw-r--r-- 1 www-data www-data 664 Jun 23 19:44 uwsgi_params --rw-r--r-- 1 www-data www-data 664 Jun 23 19:44 uwsgi_params.default --rw-r--r-- 1 www-data www-data 3610 Jun 23 19:44 win-utf -``` diff --git a/docs/examples/PREREQUISITES.md b/docs/examples/PREREQUISITES.md index 4f9f80baf..c03dd095e 100644 --- a/docs/examples/PREREQUISITES.md +++ b/docs/examples/PREREQUISITES.md @@ -26,7 +26,7 @@ Note: If using CA Authentication, described below, you will need to sign the ser CA Authentication also known as Mutual Authentication allows both the server and client to verify each others identity via a common CA. -We have a CA Certificate which we usually obtain from a Certificate Authority and use that to sign +We have a CA Certificate which we obtain usually from a Certificate Authority and use that to sign both our server certificate and client certificate. Then every time we want to access our backend, we must pass the client certificate. @@ -54,15 +54,13 @@ openssl x509 -req -sha256 -days 365 -in client.csr -CA ca.crt -CAkey ca.key -set Once this is complete you can continue to follow the instructions [here](./auth/client-certs/README.md#creating-certificate-secrets) - - ## Test HTTP Service All examples that require a test HTTP Service use the standard http-svc pod, which you can deploy as follows ```console -$ kubectl create -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/docs/examples/http-svc.yaml +$ kubectl create -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/docs/examples/http-svc.yaml service "http-svc" created replicationcontroller "http-svc" created diff --git a/docs/examples/affinity/cookie/README.md b/docs/examples/affinity/cookie/README.md index e66503dd7..670d345e3 100644 --- a/docs/examples/affinity/cookie/README.md +++ b/docs/examples/affinity/cookie/README.md @@ -10,18 +10,15 @@ Session affinity can be configured using the following annotations: | --- | --- | --- | |nginx.ingress.kubernetes.io/affinity|Type of the affinity, set this to `cookie` to enable session affinity|string (NGINX only supports `cookie`)| |nginx.ingress.kubernetes.io/affinity-mode|The affinity mode defines how sticky a session is. Use `balanced` to redistribute some sessions when scaling pods or `persistent` for maximum stickiness.|`balanced` (default) or `persistent`| -|nginx.ingress.kubernetes.io/affinity-canary-behavior|Defines session affinity behavior of canaries. By default the behavior is `sticky`, and canaries respect session affinity configuration. Set this to `legacy` to restore original canary behavior, when session affinity parameters were not respected.|`sticky` (default) or `legacy`| |nginx.ingress.kubernetes.io/session-cookie-name|Name of the cookie that will be created|string (defaults to `INGRESSCOOKIE`)| -|nginx.ingress.kubernetes.io/session-cookie-secure|Set the cookie as secure regardless the protocol of the incoming request|`"true"` or `"false"`| |nginx.ingress.kubernetes.io/session-cookie-path|Path that will be set on the cookie (required if your [Ingress paths][ingress-paths] use regular expressions)|string (defaults to the currently [matched path][ingress-paths])| -|nginx.ingress.kubernetes.io/session-cookie-domain|Domain that will be set on the cookie|string| -|nginx.ingress.kubernetes.io/session-cookie-samesite|`SameSite` attribute to apply to the cookie|Browser accepted values are `None`, `Lax`, and `Strict`| +|nginx.ingress.kubernetes.io/session-cookie-samesite|SameSite attribute to apply to the cookie|Browser accepted values are `None`, `Lax`, and `Strict`| |nginx.ingress.kubernetes.io/session-cookie-conditional-samesite-none|Will omit `SameSite=None` attribute for older browsers which reject the more-recently defined `SameSite=None` value|`"true"` or `"false"` |nginx.ingress.kubernetes.io/session-cookie-max-age|Time until the cookie expires, corresponds to the `Max-Age` cookie directive|number of seconds| |nginx.ingress.kubernetes.io/session-cookie-expires|Legacy version of the previous annotation for compatibility with older browsers, generates an `Expires` cookie directive by adding the seconds to the current date|number of seconds| |nginx.ingress.kubernetes.io/session-cookie-change-on-failure|When set to `false` nginx ingress will send request to upstream pointed by sticky cookie even if previous attempt failed. When set to `true` and previous attempt failed, sticky cookie will be changed to point to another upstream.|`true` or `false` (defaults to `false`)| -You can create the [session affinity example Ingress](ingress.yaml) to test this: +You can create the [example Ingress](ingress.yaml) to test this: ```console kubectl create -f ingress.yaml @@ -50,7 +47,7 @@ Annotations: Events: FirstSeen LastSeen Count From SubObjectPath Type Reason Message --------- -------- ----- ---- ------------- -------- ------ ------- - 7s 7s 1 {ingress-nginx-controller } Normal CREATE default/nginx-test + 7s 7s 1 {nginx-ingress-controller } Normal CREATE default/nginx-test $ curl -I http://stickyingress.example.com @@ -67,15 +64,13 @@ Accept-Ranges: bytes ``` In the example above, you can see that the response contains a `Set-Cookie` header with the settings we have defined. -This cookie is created by the Ingress-Nginx Controller, it contains a randomly generated key corresponding to the upstream used for that request (selected using [consistent hashing][consistent-hashing]) and has an `Expires` directive. -If a client sends a cookie that doesn't correspond to an upstream, NGINX selects an upstream and creates a corresponding cookie. +This cookie is created by NGINX, it contains a randomly generated key corresponding to the upstream used for that request (selected using [consistent hashing][consistent-hashing]) and has an `Expires` directive. +If the user changes this cookie, NGINX creates a new one and redirects the user to another upstream. If the backend pool grows NGINX will keep sending the requests through the same server of the first request, even if it's overloaded. When the backend server is removed, the requests are re-routed to another upstream server. This does not require the cookie to be updated because the key's [consistent hash][consistent-hashing] will change. -## Caveats - When you have a Service pointing to more than one Ingress, with only one containing affinity configuration, the first created Ingress will be used. This means that you can face the situation that you've configured session affinity on one Ingress and it doesn't work because the Service is pointing to another Ingress that doesn't configure this. diff --git a/docs/examples/affinity/cookie/ingress-samesite.yaml b/docs/examples/affinity/cookie/ingress-samesite.yaml index 2f7022ef7..42d1c2e2d 100644 --- a/docs/examples/affinity/cookie/ingress-samesite.yaml +++ b/docs/examples/affinity/cookie/ingress-samesite.yaml @@ -1,30 +1,25 @@ -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: cookie-samesite-none annotations: nginx.ingress.kubernetes.io/affinity: "cookie" nginx.ingress.kubernetes.io/session-cookie-name: "SSNONE" - nginx.ingress.kubernetes.io/session-cookie-secure: "true" nginx.ingress.kubernetes.io/session-cookie-expires: "172800" nginx.ingress.kubernetes.io/session-cookie-max-age: "172800" nginx.ingress.kubernetes.io/session-cookie-samesite: "None" nginx.ingress.kubernetes.io/session-cookie-conditional-samesite-none: "true" # omits SameSite=None for older browsers which reject cookies with SameSite=None spec: - ingressClassName: nginx rules: - host: stickyingress-samesite-none.example.com http: paths: - - path: / - pathType: Prefix - backend: - service: - name: http-svc - port: - number: 80 + - backend: + serviceName: http-svc + servicePort: 80 + path: / --- -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: cookie-samesite-strict @@ -35,15 +30,11 @@ metadata: nginx.ingress.kubernetes.io/session-cookie-max-age: "172800" nginx.ingress.kubernetes.io/session-cookie-samesite: "Strict" spec: - ingressClassName: nginx rules: - host: stickyingress-samesite-strict.example.com http: paths: - - path: / - pathType: Prefix - backend: - service: - name: http-svc - port: - number: 80 + - backend: + serviceName: http-svc + servicePort: 80 + path: / diff --git a/docs/examples/affinity/cookie/ingress.yaml b/docs/examples/affinity/cookie/ingress.yaml index 4ca9fbc09..57edbdbd3 100644 --- a/docs/examples/affinity/cookie/ingress.yaml +++ b/docs/examples/affinity/cookie/ingress.yaml @@ -1,4 +1,4 @@ -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: nginx-test @@ -9,15 +9,11 @@ metadata: nginx.ingress.kubernetes.io/session-cookie-max-age: "172800" spec: - ingressClassName: nginx rules: - host: stickyingress.example.com http: paths: - - path: / - pathType: Prefix - backend: - service: - name: http-svc - port: - number: 80 + - backend: + serviceName: http-svc + servicePort: 80 + path: / diff --git a/docs/examples/auth/basic/README.md b/docs/examples/auth/basic/README.md index 5e890df80..194bacb3b 100644 --- a/docs/examples/auth/basic/README.md +++ b/docs/examples/auth/basic/README.md @@ -3,8 +3,6 @@ This example shows how to add authentication in a Ingress rule using a secret that contains a file generated with `htpasswd`. It's important the file generated is named `auth` (actually - that the secret has a key `data.auth`), otherwise the ingress-controller returns a 503. -## Create htpasswd file - ```console $ htpasswd -c auth foo New password: @@ -13,15 +11,11 @@ Re-type new password: Adding password for user foo ``` -## Convert htpasswd into a secret - ```console $ kubectl create secret generic basic-auth --from-file=auth secret "basic-auth" created ``` -## Examine secret - ```console $ kubectl get secret basic-auth -o yaml apiVersion: v1 @@ -34,11 +28,9 @@ metadata: type: Opaque ``` -## Using kubectl, create an ingress tied to the basic-auth secret - ```console -$ echo " -apiVersion: networking.k8s.io/v1 +echo " +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: ingress-with-auth @@ -50,23 +42,17 @@ metadata: # message to display with an appropriate context why the authentication is required nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - foo' spec: - ingressClassName: nginx rules: - host: foo.bar.com http: paths: - path: / - pathType: Prefix backend: - service: - name: http-svc - port: - number: 80 + serviceName: http-svc + servicePort: 80 " | kubectl create -f - ``` -## Use curl to confirm authorization is required by the ingress - ``` $ curl -v http://10.2.29.4/ -H 'Host: foo.bar.com' * Trying 10.2.29.4... @@ -94,8 +80,6 @@ $ curl -v http://10.2.29.4/ -H 'Host: foo.bar.com' * Connection #0 to host 10.2.29.4 left intact ``` -## Use curl with the correct credentials to connect to the ingress - ``` $ curl -v http://10.2.29.4/ -H 'Host: foo.bar.com' -u 'foo:bar' * Trying 10.2.29.4... diff --git a/docs/examples/auth/client-certs/README.md b/docs/examples/auth/client-certs/README.md index 94cf6c697..a60aa14e7 100644 --- a/docs/examples/auth/client-certs/README.md +++ b/docs/examples/auth/client-certs/README.md @@ -1,12 +1,11 @@ # Client Certificate Authentication It is possible to enable Client-Certificate Authentication by adding additional annotations to your Ingress Resource. +Before getting started you must have the following Certificates Setup: -Before getting started you must have the following Certificates configured: - -1. CA certificate and Key (Intermediate Certs need to be in CA) -2. Server Certificate (Signed by CA) and Key (CN should be equal the hostname you will use) -3. Client Certificate (Signed by CA) and Key +1. CA certificate and Key(Intermediate Certs need to be in CA) +2. Server Certificate(Signed by CA) and Key (CN should be equal the hostname you will use) +3. Client Certificate(Signed by CA) and Key For more details on the generation process, checkout the Prerequisite [docs](../../PREREQUISITES.md#client-certificate-authentication). @@ -16,13 +15,13 @@ You can have as many certificates as you want. If they're in the binary DER form openssl x509 -in certificate.der -inform der -out certificate.crt -outform pem ``` -Then, you can concatenate them all into one file, named 'ca.crt' with the following: +Then, you can concatenate them all in only one file, named 'ca.crt' as the following: ```bash cat certificate1.crt certificate2.crt certificate3.crt >> ca.crt ``` -**Note:** Make sure that the Key Size is greater than 1024 and Hashing Algorithm (Digest) is something better than md5 +**Note:** Make sure that the Key Size is greater than 1024 and Hashing Algorithm(Digest) is something better than md5 for each certificate generated. Otherwise you will receive an error. ## Creating Certificate Secrets @@ -30,7 +29,7 @@ for each certificate generated. Otherwise you will receive an error. There are many different ways of configuring your secrets to enable Client-Certificate Authentication to work properly. -* You can create a secret containing just the CA certificate and another +1. You can create a secret containing just the CA certificate and another Secret containing the Server Certificate which is Signed by the CA. ```bash @@ -38,14 +37,14 @@ Authentication to work properly. kubectl create secret generic tls-secret --from-file=tls.crt=server.crt --from-file=tls.key=server.key ``` -* You can create a secret containing CA certificate along with the Server - Certificate that can be used for both TLS and Client Auth. +2. You can create a secret containing CA certificate along with the Server + Certificate, that can be used for both TLS and Client Auth. ```bash kubectl create secret generic ca-secret --from-file=tls.crt=server.crt --from-file=tls.key=server.key --from-file=ca.crt=ca.crt ``` -* If you want to also enable Certificate Revocation List verification you can +3. If you want to also enable Certificate Revocation List verification you can create the secret also containing the CRL file in PEM format: ```bash kubectl create secret generic ca-secret --from-file=ca.crt=ca.crt --from-file=ca.crl=ca.crl diff --git a/docs/examples/auth/client-certs/ingress.yaml b/docs/examples/auth/client-certs/ingress.yaml index 0cd56fcbf..cf5f701b2 100644 --- a/docs/examples/auth/client-certs/ingress.yaml +++ b/docs/examples/auth/client-certs/ingress.yaml @@ -1,4 +1,4 @@ -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: annotations: @@ -15,18 +15,14 @@ metadata: name: nginx-test namespace: default spec: - ingressClassName: nginx rules: - host: mydomain.com http: paths: - - path: / - pathType: Prefix - backend: - service: - name: http-svc - port: - number: 80 + - backend: + serviceName: http-svc + servicePort: 80 + path: / tls: - hosts: - mydomain.com diff --git a/docs/examples/auth/external-auth/README.md b/docs/examples/auth/external-auth/README.md index ad3ee8b09..a08138373 100644 --- a/docs/examples/auth/external-auth/README.md +++ b/docs/examples/auth/external-auth/README.md @@ -1,6 +1,6 @@ # External Basic Authentication -### Example 1 +### Example 1: Use an external service (Basic Auth) located in `https://httpbin.org` @@ -13,7 +13,7 @@ NAME HOSTS ADDRESS PORTS AGE external-auth external-auth-01.sample.com 172.17.4.99 80 13s $ kubectl get ing external-auth -o yaml -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: annotations: @@ -23,20 +23,17 @@ metadata: name: external-auth namespace: default resourceVersion: "2068378" - selfLink: /apis/networking/v1/namespaces/default/ingresses/external-auth + selfLink: /apis/networking/v1beta1/namespaces/default/ingresses/external-auth uid: 5c388f1d-8970-11e6-9004-080027d2dc94 spec: rules: - host: external-auth-01.sample.com http: paths: - - path: / - pathType: Prefix - backend: - service: - name: http-svc - port: - number: 80 + - backend: + serviceName: http-svc + servicePort: 80 + path: / status: loadBalancer: ingress: @@ -44,7 +41,7 @@ status: $ ``` -## Test 1: no username/password (expect code 401) +Test 1: no username/password (expect code 401) ```console $ curl -k http://172.17.4.99 -v -H 'Host: external-auth-01.sample.com' @@ -74,8 +71,7 @@ $ curl -k http://172.17.4.99 -v -H 'Host: external-auth-01.sample.com' * Connection #0 to host 172.17.4.99 left intact ``` -## Test 2: valid username/password (expect code 200) - +Test 2: valid username/password (expect code 200) ``` $ curl -k http://172.17.4.99 -v -H 'Host: external-auth-01.sample.com' -u 'user:passwd' * Rebuilt URL to: http://172.17.4.99/ @@ -122,8 +118,7 @@ BODY: -no body in request- ``` -## Test 3: invalid username/password (expect code 401) - +Test 3: invalid username/password (expect code 401) ``` curl -k http://172.17.4.99 -v -H 'Host: external-auth-01.sample.com' -u 'user:user' * Rebuilt URL to: http://172.17.4.99/ diff --git a/docs/examples/auth/external-auth/ingress.yaml b/docs/examples/auth/external-auth/ingress.yaml index 1aa2f36e9..c7a87a240 100644 --- a/docs/examples/auth/external-auth/ingress.yaml +++ b/docs/examples/auth/external-auth/ingress.yaml @@ -1,19 +1,15 @@ -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: annotations: nginx.ingress.kubernetes.io/auth-url: "https://httpbin.org/basic-auth/user/passwd" name: external-auth spec: - ingressClassName: nginx rules: - host: external-auth-01.sample.com http: paths: - - path: / - pathType: Prefix - backend: - service: - name: http-svc - port: - number: 80 + - backend: + serviceName: http-svc + servicePort: 80 + path: / \ No newline at end of file diff --git a/docs/examples/auth/oauth-external-auth/README.md b/docs/examples/auth/oauth-external-auth/README.md index b64b0e70a..00f8d45dc 100644 --- a/docs/examples/auth/oauth-external-auth/README.md +++ b/docs/examples/auth/oauth-external-auth/README.md @@ -6,7 +6,7 @@ The `auth-url` and `auth-signin` annotations allow you to use an external authentication provider to protect your Ingress resources. !!! Important - This annotation requires `ingress-nginx-controller v0.9.0` or greater. + This annotation requires `nginx-ingress-controller v0.9.0` or greater.) ### Key Detail @@ -32,94 +32,45 @@ metadata: ### Example: OAuth2 Proxy + Kubernetes-Dashboard This example will show you how to deploy [`oauth2_proxy`](https://github.com/pusher/oauth2_proxy) -into a Kubernetes cluster and use it to protect the Kubernetes Dashboard using GitHub as the OAuth2 provider. +into a Kubernetes cluster and use it to protect the Kubernetes Dashboard using github as oAuth2 provider #### Prepare 1. Install the kubernetes dashboard - ```console - kubectl create -f https://raw.githubusercontent.com/kubernetes/kops/master/addons/kubernetes-dashboard/v1.10.1.yaml - ``` +```console +kubectl create -f https://raw.githubusercontent.com/kubernetes/kops/master/addons/kubernetes-dashboard/v1.10.1.yaml +``` -2. Create a [custom GitHub OAuth application](https://github.com/settings/applications/new) +2. Create a [custom Github OAuth application](https://github.com/settings/applications/new) - ![Register OAuth2 Application](images/register-oauth-app.png) +![Register OAuth2 Application](images/register-oauth-app.png) - - Homepage URL is the FQDN in the Ingress rule, like `https://foo.bar.com` - - Authorization callback URL is the same as the base FQDN plus `/oauth2/callback`, like `https://foo.bar.com/oauth2/callback` +- Homepage URL is the FQDN in the Ingress rule, like `https://foo.bar.com` +- Authorization callback URL is the same as the base FQDN plus `/oauth2/callback`, like `https://foo.bar.com/oauth2/callback` - ![Register OAuth2 Application](images/register-oauth-app-2.png) +![Register OAuth2 Application](images/register-oauth-app-2.png) -3. Configure values in the file [`oauth2-proxy.yaml`](https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/docs/examples/auth/oauth-external-auth/oauth2-proxy.yaml) with the values: +3. Configure oauth2_proxy values in the file [`oauth2-proxy.yaml`](https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/docs/examples/auth/oauth-external-auth/oauth2-proxy.yaml) with the values: - - OAUTH2_PROXY_CLIENT_ID with the github `` - - OAUTH2_PROXY_CLIENT_SECRET with the github `` - - OAUTH2_PROXY_COOKIE_SECRET with value of `python -c 'import os,base64; print(base64.b64encode(os.urandom(16)).decode("ascii"))'` - - (optional, but recommended) OAUTH2_PROXY_GITHUB_USERS with GitHub usernames to allow to login - - `__INGRESS_HOST__` with a valid FQDN (e.g. `foo.bar.com`) - - `__INGRESS_SECRET__` with a Secret with a valid SSL certificate +- OAUTH2_PROXY_CLIENT_ID with the github `` +- OAUTH2_PROXY_CLIENT_SECRET with the github `` +- OAUTH2_PROXY_COOKIE_SECRET with value of `python -c 'import os,base64; print(base64.b64encode(os.urandom(16)).decode("ascii"))'` -4. Deploy the oauth2 proxy and the ingress rules by running: +4. Customize the contents of the file [`dashboard-ingress.yaml`](https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/docs/examples/auth/oauth-external-auth/dashboard-ingress.yaml): - ```console - $ kubectl create -f oauth2-proxy.yaml - ``` +Replace `__INGRESS_HOST__` with a valid FQDN and `__INGRESS_SECRET__` with a Secret with a valid SSL certificate. -#### Test +5. Deploy the oauth2 proxy and the ingress rules running: -Test the integration by accessing the configured URL, e.g. `https://foo.bar.com` +```console +$ kubectl create -f oauth2-proxy.yaml,dashboard-ingress.yaml +``` + +Test the oauth integration accessing the configured URL, like `https://foo.bar.com` ![Register OAuth2 Application](images/github-auth.png) -![GitHub authentication](images/oauth-login.png) - -![Kubernetes dashboard](images/dashboard.png) - - -### Example: Vouch Proxy + Kubernetes-Dashboard - -This example will show you how to deploy [`Vouch Proxy`](https://github.com/vouch/vouch-proxy) -into a Kubernetes cluster and use it to protect the Kubernetes Dashboard using GitHub as the OAuth2 provider. - -#### Prepare - -1. Install the kubernetes dashboard - - ```console - kubectl create -f https://raw.githubusercontent.com/kubernetes/kops/master/addons/kubernetes-dashboard/v1.10.1.yaml - ``` - -2. Create a [custom GitHub OAuth application](https://github.com/settings/applications/new) - - ![Register OAuth2 Application](images/register-oauth-app.png) - - - Homepage URL is the FQDN in the Ingress rule, like `https://foo.bar.com` - - Authorization callback URL is the same as the base FQDN plus `/oauth2/auth`, like `https://foo.bar.com/oauth2/auth` - - ![Register OAuth2 Application](images/register-oauth-app-2.png) - -3. Configure Vouch Proxy values in the file [`vouch-proxy.yaml`](https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/docs/examples/auth/oauth-external-auth/vouch-proxy.yaml) with the values: - - - VOUCH_COOKIE_DOMAIN with value of `` - - OAUTH_CLIENT_ID with the github `` - - OAUTH_CLIENT_SECRET with the github `` - - (optional, but recommended) VOUCH_WHITELIST with GitHub usernames to allow to login - - `__INGRESS_HOST__` with a valid FQDN (e.g. `foo.bar.com`) - - `__INGRESS_SECRET__` with a Secret with a valid SSL certificate - -4. Deploy Vouch Proxy and the ingress rules by running: - - ```console - $ kubectl create -f vouch-proxy.yaml - ``` - -#### Test - -Test the integration by accessing the configured URL, e.g. `https://foo.bar.com` - -![Register OAuth2 Application](images/github-auth.png) - -![GitHub authentication](images/oauth-login.png) +![Github authentication](images/oauth-login.png) ![Kubernetes dashboard](images/dashboard.png) diff --git a/docs/examples/auth/oauth-external-auth/dashboard-ingress.yaml b/docs/examples/auth/oauth-external-auth/dashboard-ingress.yaml new file mode 100644 index 000000000..ade56a9e6 --- /dev/null +++ b/docs/examples/auth/oauth-external-auth/dashboard-ingress.yaml @@ -0,0 +1,38 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/auth-url: "https://$host/oauth2/auth" + nginx.ingress.kubernetes.io/auth-signin: "https://$host/oauth2/start?rd=$escaped_request_uri" + name: external-auth-oauth2 + namespace: kube-system +spec: + rules: + - host: __INGRESS_HOST__ + http: + paths: + - backend: + serviceName: kubernetes-dashboard + servicePort: 80 + path: / + +--- + +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: oauth2-proxy + namespace: kube-system +spec: + rules: + - host: __INGRESS_HOST__ + http: + paths: + - backend: + serviceName: oauth2-proxy + servicePort: 4180 + path: /oauth2 + tls: + - hosts: + - __INGRESS_HOST__ + secretName: __INGRESS_SECRET__ diff --git a/docs/examples/auth/oauth-external-auth/oauth2-proxy.yaml b/docs/examples/auth/oauth-external-auth/oauth2-proxy.yaml index 3d6d41297..b383ab95e 100644 --- a/docs/examples/auth/oauth-external-auth/oauth2-proxy.yaml +++ b/docs/examples/auth/oauth-external-auth/oauth2-proxy.yaml @@ -31,9 +31,6 @@ spec: # docker run -ti --rm python:3-alpine python -c 'import secrets,base64; print(base64.b64encode(base64.b64encode(secrets.token_bytes(16))));' - name: OAUTH2_PROXY_COOKIE_SECRET value: SECRET - # Recommended: remove email-domain=* in args and set an allowlist - # - name: OAUTH2_PROXY_GITHUB_USERS - # value: alice,bob image: quay.io/oauth2-proxy/oauth2-proxy:latest imagePullPolicy: Always name: oauth2-proxy @@ -58,52 +55,3 @@ spec: targetPort: 4180 selector: k8s-app: oauth2-proxy - ---- - -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: oauth2-proxy - namespace: kube-system -spec: - ingressClassName: nginx - rules: - - host: __INGRESS_HOST__ - http: - paths: - - path: /oauth2 - pathType: Prefix - backend: - service: - name: oauth2-proxy - port: - number: 4180 - tls: - - hosts: - - __INGRESS_HOST__ - secretName: __INGRESS_SECRET__ - ---- - -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - annotations: - nginx.ingress.kubernetes.io/auth-url: "https://$host/oauth2/auth" - nginx.ingress.kubernetes.io/auth-signin: "https://$host/oauth2/start?rd=$escaped_request_uri" - name: external-auth-oauth2 - namespace: kube-system -spec: - ingressClassName: nginx - rules: - - host: __INGRESS_HOST__ - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: kubernetes-dashboard - port: - number: 80 diff --git a/docs/examples/auth/oauth-external-auth/vouch-proxy.yaml b/docs/examples/auth/oauth-external-auth/vouch-proxy.yaml deleted file mode 100644 index 05eae6d11..000000000 --- a/docs/examples/auth/oauth-external-auth/vouch-proxy.yaml +++ /dev/null @@ -1,110 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - k8s-app: vouch-proxy - name: vouch-proxy - namespace: kube-system -spec: - replicas: 1 - selector: - matchLabels: - k8s-app: vouch-proxy - template: - metadata: - labels: - k8s-app: vouch-proxy - spec: - containers: - - env: - - name: VOUCH_ALLOWALLUSERS - value: true - # Recommended: remove VOUCH_ALLOWALLUSERS and set an allowlist - # - name: VOUCH_WHITELIST - # value: alice,bob - - name: VOUCH_COOKIE_DOMAIN - value: - - name: VOUCH_LISTEN - value: 0.0.0.0 - - name: VOUCH_DOCUMENT_ROOT - value: oauth2 - # See https://github.com/vouch/vouch-proxy/tree/master/config for different provider examples - - name: OAUTH_PROVIDER - value: github - - name: OAUTH_CLIENT_ID - value: - - name: OAUTH_CLIENT_SECRET - value: - image: quay.io/vouch/vouch-proxy:latest - imagePullPolicy: Always - name: vouch-proxy - ports: - - containerPort: 9090 - protocol: TCP - ---- - -apiVersion: v1 -kind: Service -metadata: - labels: - k8s-app: vouch-proxy - name: vouch-proxy - namespace: kube-system -spec: - ports: - - name: http - port: 9090 - protocol: TCP - targetPort: 9090 - selector: - k8s-app: vouch-proxy - ---- - -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: vouch-proxy - namespace: kube-system -spec: - ingressClassName: nginx - rules: - - host: __INGRESS_HOST__ - http: - paths: - - path: /oauth2 - pathType: Prefix - backend: - service: - name: vouch-proxy - port: - number: 9090 - tls: - - hosts: - - __INGRESS_HOST__ - secretName: __INGRESS_SECRET__ - ---- - -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - annotations: - nginx.ingress.kubernetes.io/auth-url: "https://$host/oauth2/validate" - nginx.ingress.kubernetes.io/auth-signin: "https://$host/oauth2/login?url=$scheme://$http_host$request_uri" - name: external-auth-oauth2 - namespace: kube-system -spec: - ingressClassName: nginx - rules: - - host: __INGRESS_HOST__ - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: kubernetes-dashboard - port: - number: 80 diff --git a/docs/examples/canary/README.md b/docs/examples/canary/README.md deleted file mode 100644 index 885991a3b..000000000 --- a/docs/examples/canary/README.md +++ /dev/null @@ -1,231 +0,0 @@ -# Canary - -Ingress Nginx Has the ability to handle canary routing by setting specific -annotations, the following is an example of how to configure a canary -deployment with weighted canary routing. - -## Create your main deployment and service - -This is the main deployment of your application with the service that will be -used to route to it - -```bash -echo " ---- -# Deployment -apiVersion: apps/v1 -kind: Deployment -metadata: - name: production - labels: - app: production -spec: - replicas: 1 - selector: - matchLabels: - app: production - template: - metadata: - labels: - app: production - spec: - containers: - - name: production - image: registry.k8s.io/ingress-nginx/e2e-test-echo:v1.1.1@sha256:a1e0152e2eeab26e3f6fd3986f3d82b17bc7711717cae5392dcd18dd447ba6ef - ports: - - containerPort: 80 - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP ---- -# Service -apiVersion: v1 -kind: Service -metadata: - name: production - labels: - app: production -spec: - ports: - - port: 80 - targetPort: 80 - protocol: TCP - name: http - selector: - app: production -" | kubectl apply -f - -``` - -## Create the canary deployment and service - -This is the canary deployment that will take a weighted amount of requests -instead of the main deployment - -```bash -echo " ---- -# Deployment -apiVersion: apps/v1 -kind: Deployment -metadata: - name: canary - labels: - app: canary -spec: - replicas: 1 - selector: - matchLabels: - app: canary - template: - metadata: - labels: - app: canary - spec: - containers: - - name: canary - image: registry.k8s.io/ingress-nginx/e2e-test-echo:v1.1.1@sha256:a1e0152e2eeab26e3f6fd3986f3d82b17bc7711717cae5392dcd18dd447ba6ef - ports: - - containerPort: 80 - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP ---- -# Service -apiVersion: v1 -kind: Service -metadata: - name: canary - labels: - app: canary -spec: - ports: - - port: 80 - targetPort: 80 - protocol: TCP - name: http - selector: - app: canary -" | kubectl apply -f - -``` - -## Create Ingress Pointing To Your Main Deployment - -Next you will need to expose your main deployment with an ingress resource, -note there are no canary specific annotations on this ingress - -```bash -echo " ---- -# Ingress -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: production - annotations: -spec: - ingressClassName: nginx - rules: - - host: echo.prod.mydomain.com - http: - paths: - - pathType: Prefix - path: / - backend: - service: - name: production - port: - number: 80 -" | kubectl apply -f - -``` - -## Create Ingress Pointing To Your Canary Deployment - -You will then create an Ingress that has the canary specific configuration, -please pay special notice of the following: - -- The host name is identical to the main ingress host name -- The `nginx.ingress.kubernetes.io/canary: "true"` annotation is required and - defines this as a canary annotation (if you do not have this the Ingresses - will clash) -- The `nginx.ingress.kubernetes.io/canary-weight: "50"` annotation dictates the - weight of the routing, in this case there is a "50%" chance a request will - hit the canary deployment over the main deployment -```bash -echo " ---- -# Ingress -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: canary - annotations: - nginx.ingress.kubernetes.io/canary: \"true\" - nginx.ingress.kubernetes.io/canary-weight: \"50\" -spec: - ingressClassName: nginx - rules: - - host: echo.prod.mydomain.com - http: - paths: - - pathType: Prefix - path: / - backend: - service: - name: canary - port: - number: 80 -" | kubectl apply -f - -``` - -## Testing your setup - -You can use the following command to test your setup (replacing -INGRESS_CONTROLLER_IP with your ingresse controllers IP Address) - -```bash -for i in $(seq 1 10); do curl -s --resolve echo.prod.mydomain.com:80:$INGRESS_CONTROLLER_IP echo.prod.mydomain.com | grep "Hostname"; done -``` - -You will get the following output showing that your canary setup is working as -expected: - -```bash -Hostname: production-5c5f65d859-phqzc -Hostname: canary-6697778457-zkfjf -Hostname: canary-6697778457-zkfjf -Hostname: production-5c5f65d859-phqzc -Hostname: canary-6697778457-zkfjf -Hostname: production-5c5f65d859-phqzc -Hostname: production-5c5f65d859-phqzc -Hostname: production-5c5f65d859-phqzc -Hostname: canary-6697778457-zkfjf -Hostname: production-5c5f65d859-phqzc -``` diff --git a/docs/examples/chashsubset/deployment.yaml b/docs/examples/chashsubset/deployment.yaml index 7eb7613ba..9b1bafcb1 100644 --- a/docs/examples/chashsubset/deployment.yaml +++ b/docs/examples/chashsubset/deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: nginxhello - image: registry.k8s.io/e2e-test-images/echoserver:2.3 + image: gcr.io/kubernetes-e2e-test-images/echoserver:2.2 ports: - containerPort: 8080 env: @@ -54,7 +54,7 @@ spec: targetPort: 8080 --- -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: annotations: @@ -64,15 +64,7 @@ metadata: name: nginxhello-ingress namespace: default spec: - ingressClassName: nginx - rules: - - host: foo.bar.com - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: nginxhello - port: - number: 80 + backend: + serviceName: nginxhello + servicePort: 80 + diff --git a/docs/examples/customization/configuration-snippets/README.md b/docs/examples/customization/configuration-snippets/README.md index 33e7c4bf0..d60a3d1a0 100644 --- a/docs/examples/customization/configuration-snippets/README.md +++ b/docs/examples/customization/configuration-snippets/README.md @@ -2,16 +2,13 @@ ## Ingress -The Ingress in [this example](ingress.yaml) adds a custom header to Nginx configuration that only applies to that specific Ingress. If you want to add headers that apply globally to all Ingresses, please have a look at [an example of specifying custom headers](../custom-headers/README.md). +The Ingress in [this example](ingress.yaml) adds a custom header to Nginx configuration that only applies to that specific Ingress. If you want to add headers that apply globally to all Ingresses, please have a look at [this example](../custom-headers/README.md). ```console -kubectl apply -f ingress.yaml +$ kubectl apply -f ingress.yaml ``` ## Test Check if the contents of the annotation are present in the nginx.conf file using: - -```console -kubectl exec ingress-nginx-controller-873061567-4n3k2 -n kube-system -- cat /etc/nginx/nginx.conf -``` +`kubectl exec ingress-nginx-controller-873061567-4n3k2 -n kube-system -- cat /etc/nginx/nginx.conf` diff --git a/docs/examples/customization/configuration-snippets/ingress.yaml b/docs/examples/customization/configuration-snippets/ingress.yaml index 1f0c4f24a..07af3552f 100644 --- a/docs/examples/customization/configuration-snippets/ingress.yaml +++ b/docs/examples/customization/configuration-snippets/ingress.yaml @@ -1,4 +1,4 @@ -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: nginx-configuration-snippet @@ -6,15 +6,11 @@ metadata: nginx.ingress.kubernetes.io/configuration-snippet: | more_set_headers "Request-Id: $req_id"; spec: - ingressClassName: nginx rules: - host: custom.configuration.com http: paths: - - path: / - pathType: Prefix - backend: - service: - name: http-svc - port: - number: 80 + - backend: + serviceName: http-svc + servicePort: 80 + path: / diff --git a/docs/examples/customization/custom-configuration/README.md b/docs/examples/customization/custom-configuration/README.md index 3fc6bbaf6..3ddefe245 100644 --- a/docs/examples/customization/custom-configuration/README.md +++ b/docs/examples/customization/custom-configuration/README.md @@ -17,7 +17,7 @@ metadata: ``` ``` -curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/docs/examples/customization/custom-configuration/configmap.yaml \ +curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/docs/examples/customization/custom-configuration/configmap.yaml \ | kubectl apply -f - ``` diff --git a/docs/examples/customization/custom-errors/README.md b/docs/examples/customization/custom-errors/README.md index 2d6e124bb..3a925bed7 100644 --- a/docs/examples/customization/custom-errors/README.md +++ b/docs/examples/customization/custom-errors/README.md @@ -2,15 +2,10 @@ This example demonstrates how to use a custom backend to render custom error pages. -If you are using the Helm Chart, look at [example values](https://github.com/kubernetes/ingress-nginx/blob/main/docs/examples/customization/custom-errors/custom-default-backend.helm.values.yaml) and don't forget to add the [ConfigMap](https://github.com/kubernetes/ingress-nginx/blob/main/docs/examples/customization/custom-errors/custom-default-backend-error_pages.configMap.yaml) to your deployment. Otherwise, continue with [Customized default backend](#customized-default-backend) manual deployment. - ## Customized default backend First, create the custom `default-backend`. It will be used by the Ingress controller later on. -To do that, you can take a look at the [example manifest](https://github.com/kubernetes/ingress-nginx/blob/main/docs/examples/customization/custom-errors/custom-default-backend.yaml) -in this project's GitHub repository. - ``` $ kubectl create -f custom-default-backend.yaml service "nginx-errors" created @@ -30,7 +25,7 @@ service/nginx-errors ClusterIP 10.0.0.12 80/TCP 10s ## Ingress controller configuration -If you do not already have an instance of the Ingress-Nginx Controller running, deploy it according to the +If you do not already have an instance of the NGINX Ingress controller running, deploy it according to the [deployment guide][deploy], then follow these steps: 1. Edit the `ingress-nginx-controller` Deployment and set the value of the `--default-backend-service` flag to the name of the @@ -38,12 +33,12 @@ If you do not already have an instance of the Ingress-Nginx Controller running, 2. Edit the `ingress-nginx-controller` ConfigMap and create the key `custom-http-errors` with a value of `404,503`. -3. Take note of the IP address assigned to the Ingress-Nginx Controller Service. - ``` - $ kubectl get svc ingress-nginx - NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE - ingress-nginx ClusterIP 10.0.0.13 80/TCP,443/TCP 10m - ``` +3. Take note of the IP address assigned to the NGINX Ingress controller Service. + ``` + $ kubectl get svc ingress-nginx + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + ingress-nginx ClusterIP 10.0.0.13 80/TCP,443/TCP 10m + ``` !!! note The `ingress-nginx` Service is of type `ClusterIP` in this example. This may vary depending on your environment. @@ -86,16 +81,3 @@ Vary: Accept-Encoding To go further with this example, feel free to deploy your own applications and Ingress objects, and validate that the responses are still in the correct format when a backend returns 503 (eg. if you scale a Deployment down to 0 replica). - -## Maintenance page - -You can also leverage custom error pages to set a **"_Service under maintenance_" page** for the whole cluster, useful to prevent users from accessing your services while you are performing planned scheduled maintenance. - -When enabled, the maintenance page is served to the clients with an HTTP [**503 Service Unavailable**](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503) response **status code**. - -To do that: - -- Enable a **custom error page for the 503 HTTP error**, by following the guide above -- Set the value of the `--watch-namespace-selector` flag to the name of some non-existent namespace, e.g. `nonexistent-namespace` - - This effectively prevents the NGINX Ingress Controller from reading `Ingress` resources from any namespace in the Kubernetes cluster -- Set your `location-snippet` to `return 503;`, to make the NGINX Ingress Controller always return the 503 HTTP error page for all the requests diff --git a/docs/examples/customization/custom-errors/custom-default-backend-error_pages.configMap.yaml b/docs/examples/customization/custom-errors/custom-default-backend-error_pages.configMap.yaml deleted file mode 100644 index 303734c90..000000000 --- a/docs/examples/customization/custom-errors/custom-default-backend-error_pages.configMap.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Custom error page configMap ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: custom-error-pages -data: - 404: | - - - PAGE NOT FOUND - PAGE NOT FOUND - - 503: | - - - CUSTOM SERVICE UNAVAILABLE - CUSTOM SERVICE UNAVAILABLE - diff --git a/docs/examples/customization/custom-errors/custom-default-backend.helm.values.yaml b/docs/examples/customization/custom-errors/custom-default-backend.helm.values.yaml deleted file mode 100644 index d72001d58..000000000 --- a/docs/examples/customization/custom-errors/custom-default-backend.helm.values.yaml +++ /dev/null @@ -1,21 +0,0 @@ -controller: - config: - custom-http-errors: "404,503" -defaultBackend: - enabled: true - image: - registry: registry.k8s.io - image: ingress-nginx/custom-error-pages - tag: v1.1.1@sha256:8c10776191ae44b5c387b8c7696d8bc17ceec90d7184a3a38b89ac8434b6c56b - extraVolumes: - - name: custom-error-pages - configMap: - name: custom-error-pages - items: - - key: "404" - path: "404.html" - - key: "503" - path: "503.html" - extraVolumeMounts: - - name: custom-error-pages - mountPath: /www diff --git a/docs/examples/customization/custom-errors/custom-default-backend.yaml b/docs/examples/customization/custom-errors/custom-default-backend.yaml index 088ca1374..70096bdbe 100644 --- a/docs/examples/customization/custom-errors/custom-default-backend.yaml +++ b/docs/examples/customization/custom-errors/custom-default-backend.yaml @@ -36,7 +36,7 @@ spec: spec: containers: - name: nginx-error-server - image: registry.k8s.io/ingress-nginx/custom-error-pages:v1.1.1@sha256:8c10776191ae44b5c387b8c7696d8bc17ceec90d7184a3a38b89ac8434b6c56b + image: quay.io/kubernetes-ingress-controller/custom-error-pages-amd64:0.3 ports: - containerPort: 8080 # Setting the environment variable DEBUG we can see the headers sent @@ -44,19 +44,3 @@ spec: # env: # - name: DEBUG # value: "true" - - # Mounting custom error page from configMap - # volumeMounts: - # - name: custom_error_pages - # mountPath: /www - - # Mounting custom error page from configMap - # volumes: - # - name: custom_error_pages - # configMap: - # name: custom_error_pages - # items: - # - key: "404" - # path: "404.html" - # - key: "503" - # path: "503.html" diff --git a/docs/examples/customization/custom-headers/README.md b/docs/examples/customization/custom-headers/README.md index 0da12582f..99af78408 100644 --- a/docs/examples/customization/custom-headers/README.md +++ b/docs/examples/customization/custom-headers/README.md @@ -1,40 +1,22 @@ # Custom Headers -## Caveats - -Changes to the custom header config maps do not force a reload of the ingress-nginx-controllers. - -### Workaround - -To work around this limitation, perform a rolling restart of the deployment. - -## Example - -This example demonstrates configuration of the Ingress-Nginx Controller via +This example demonstrates configuration of the nginx ingress controller via a ConfigMap to pass a custom list of headers to the upstream server. [custom-headers.yaml](custom-headers.yaml) defines a ConfigMap in the `ingress-nginx` namespace named `custom-headers`, holding several custom X-prefixed HTTP headers. ```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/docs/examples/customization/custom-headers/custom-headers.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/docs/examples/customization/custom-headers/custom-headers.yaml ``` [configmap.yaml](configmap.yaml) defines a ConfigMap in the `ingress-nginx` namespace named `ingress-nginx-controller`. This controls the [global configuration](../../../user-guide/nginx-configuration/configmap.md) of the ingress controller, and already exists in a standard installation. The key `proxy-set-headers` is set to cite the previously-created `ingress-nginx/custom-headers` ConfigMap. ```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/docs/examples/customization/custom-headers/configmap.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/docs/examples/customization/custom-headers/configmap.yaml ``` -The Ingress-Nginx Controller will read the `ingress-nginx/ingress-nginx-controller` ConfigMap, find the `proxy-set-headers` key, read HTTP headers from the `ingress-nginx/custom-headers` ConfigMap, and include those HTTP headers in all requests flowing from nginx to the backends. - - -The above example was for passing a custom list of headers to the upstream server. -To pass the custom headers before sending response traffic to the client, use the add-headers key: - -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/docs/examples/customization/custom-headers/configmap-client-response.yaml -``` +The nginx ingress controller will read the `ingress-nginx/ingress-nginx-controller` ConfigMap, find the `proxy-set-headers` key, read HTTP headers from the `ingress-nginx/custom-headers` ConfigMap, and include those HTTP headers in all requests flowing from nginx to the backends. ## Test diff --git a/docs/examples/customization/custom-headers/configmap-client-response.yaml b/docs/examples/customization/custom-headers/configmap-client-response.yaml deleted file mode 100644 index 3213de3f2..000000000 --- a/docs/examples/customization/custom-headers/configmap-client-response.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -data: - add-headers: "ingress-nginx/custom-headers" -kind: ConfigMap -metadata: - name: ingress-nginx-controller - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx \ No newline at end of file diff --git a/docs/examples/customization/external-auth-headers/Makefile b/docs/examples/customization/external-auth-headers/Makefile new file mode 100644 index 000000000..1ee7e06ac --- /dev/null +++ b/docs/examples/customization/external-auth-headers/Makefile @@ -0,0 +1,23 @@ +all: push + +TAG=0.1 +PREFIX?=electroma/ingress-demo- +ARCH?=amd64 +GOLANG_VERSION=1.9 +TEMP_DIR:=$(shell mktemp -d) + +build: clean + CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) go build -o authsvc/authsvc authsvc/authsvc.go + CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) go build -o echosvc/echosvc echosvc/echosvc.go + +container: build + docker build --pull -t $(PREFIX)authsvc-$(ARCH):$(TAG) authsvc + docker build --pull -t $(PREFIX)echosvc-$(ARCH):$(TAG) echosvc + +push: container + docker push $(PREFIX)authsvc-$(ARCH):$(TAG) + docker push $(PREFIX)echosvc-$(ARCH):$(TAG) + +clean: + rm -f authsvc/authsvc echosvc/echosvc + diff --git a/docs/examples/customization/external-auth-headers/README.md b/docs/examples/customization/external-auth-headers/README.md index 946088b4b..9aaf6864b 100644 --- a/docs/examples/customization/external-auth-headers/README.md +++ b/docs/examples/customization/external-auth-headers/README.md @@ -1,7 +1,7 @@ # External authentication, authentication service response headers propagation This example demonstrates propagation of selected authentication service response headers -to a backend service. +to backend service. Sample configuration includes: @@ -37,7 +37,7 @@ public-demo-echo-service public-demo-echo-service.kube.local 80 secure-demo-echo-service secure-demo-echo-service.kube.local 80 1m ``` -## Test 1: public service with no auth header +Test 1: public service with no auth header ```console $ curl -H 'Host: public-demo-echo-service.kube.local' -v 192.168.99.100 @@ -60,7 +60,7 @@ $ curl -H 'Host: public-demo-echo-service.kube.local' -v 192.168.99.100 UserID: , UserRole: ``` -## Test 2: secure service with no auth header +Test 2: secure service with no auth header ```console $ curl -H 'Host: secure-demo-echo-service.kube.local' -v 192.168.99.100 @@ -89,7 +89,7 @@ $ curl -H 'Host: secure-demo-echo-service.kube.local' -v 192.168.99.100 * Connection #0 to host 192.168.99.100 left intact ``` -## Test 3: public service with valid auth header +Test 3: public service with valid auth header ```console $ curl -H 'Host: public-demo-echo-service.kube.local' -H 'User:internal' -v 192.168.99.100 @@ -113,7 +113,7 @@ $ curl -H 'Host: public-demo-echo-service.kube.local' -H 'User:internal' -v 192. UserID: 1443635317331776148, UserRole: admin ``` -## Test 4: secure service with valid auth header +Test 4: secure service with valid auth header ```console $ curl -H 'Host: secure-demo-echo-service.kube.local' -H 'User:internal' -v 192.168.99.100 diff --git a/docs/examples/customization/external-auth-headers/authsvc/Dockerfile b/docs/examples/customization/external-auth-headers/authsvc/Dockerfile new file mode 100644 index 000000000..90e6ec2cb --- /dev/null +++ b/docs/examples/customization/external-auth-headers/authsvc/Dockerfile @@ -0,0 +1,5 @@ +FROM alpine:3.10 +MAINTAINER Roman Safronov +COPY authsvc / +EXPOSE 8080 +ENTRYPOINT ["/authsvc"] diff --git a/images/ext-auth-example-authsvc/rootfs/authsvc.go b/docs/examples/customization/external-auth-headers/authsvc/authsvc.go similarity index 100% rename from images/ext-auth-example-authsvc/rootfs/authsvc.go rename to docs/examples/customization/external-auth-headers/authsvc/authsvc.go diff --git a/docs/examples/customization/external-auth-headers/auth-service.yaml b/docs/examples/customization/external-auth-headers/deploy/auth-service.yaml similarity index 90% rename from docs/examples/customization/external-auth-headers/auth-service.yaml rename to docs/examples/customization/external-auth-headers/deploy/auth-service.yaml index faa345921..3a5dff12c 100644 --- a/docs/examples/customization/external-auth-headers/auth-service.yaml +++ b/docs/examples/customization/external-auth-headers/deploy/auth-service.yaml @@ -18,7 +18,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: auth-service - image: gcr.io/k8s-staging-ingress-nginx/ext-auth-example-authsvc:v1.0.0 + image: electroma/ingress-demo-authsvc-amd64:0.1 ports: - containerPort: 8080 resources: diff --git a/docs/examples/customization/external-auth-headers/echo-service.yaml b/docs/examples/customization/external-auth-headers/deploy/echo-service.yaml similarity index 72% rename from docs/examples/customization/external-auth-headers/echo-service.yaml rename to docs/examples/customization/external-auth-headers/deploy/echo-service.yaml index 10244458d..1c3667c7c 100644 --- a/docs/examples/customization/external-auth-headers/echo-service.yaml +++ b/docs/examples/customization/external-auth-headers/deploy/echo-service.yaml @@ -18,7 +18,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: echo-service - image: registry.k8s.io/ingress-nginx/e2e-test-echo:v1.1.1@sha256:a1e0152e2eeab26e3f6fd3986f3d82b17bc7711717cae5392dcd18dd447ba6ef + image: electroma/ingress-demo-echosvc-amd64:0.1 ports: - containerPort: 8080 resources: @@ -43,7 +43,7 @@ spec: selector: k8s-app: demo-echo-service --- -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: public-demo-echo-service @@ -52,20 +52,16 @@ metadata: nginx.ingress.kubernetes.io/auth-response-headers: UserID, UserRole namespace: default spec: - ingressClassName: nginx rules: - host: public-demo-echo-service.kube.local http: paths: - - path: / - pathType: Prefix - backend: - service: - name: demo-echo-service - port: - number: 80 + - backend: + serviceName: demo-echo-service + servicePort: 80 + path: / --- -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: secure-demo-echo-service @@ -74,15 +70,11 @@ metadata: nginx.ingress.kubernetes.io/auth-response-headers: UserID, UserRole namespace: default spec: - ingressClassName: nginx rules: - host: secure-demo-echo-service.kube.local http: paths: - - path: / - pathType: Prefix - backend: - service: - name: demo-echo-service - port: - number: 80 + - backend: + serviceName: demo-echo-service + servicePort: 80 + path: / diff --git a/docs/examples/customization/external-auth-headers/echosvc/Dockerfile b/docs/examples/customization/external-auth-headers/echosvc/Dockerfile new file mode 100644 index 000000000..04e4ead42 --- /dev/null +++ b/docs/examples/customization/external-auth-headers/echosvc/Dockerfile @@ -0,0 +1,5 @@ +FROM alpine:3.10 +MAINTAINER Roman Safronov +COPY echosvc / +EXPOSE 8080 +ENTRYPOINT ["/echosvc"] diff --git a/magefiles/exec.go b/docs/examples/customization/external-auth-headers/echosvc/echosvc.go similarity index 60% rename from magefiles/exec.go rename to docs/examples/customization/external-auth-headers/echosvc/echosvc.go index 0acd557c6..ba9702aa0 100644 --- a/magefiles/exec.go +++ b/docs/examples/customization/external-auth-headers/echosvc/echosvc.go @@ -1,7 +1,5 @@ -//go:build ignore - /* -Copyright 2023 The Kubernetes Authors. +Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,9 +17,16 @@ limitations under the License. package main import ( - "os" - - "github.com/magefile/mage/mage" + "fmt" + "net/http" ) -func main() { os.Exit(mage.Main()) } +func handler(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "UserID: %s, UserRole: %s", r.Header.Get("UserID"), r.Header.Get("UserRole")) +} + +// Sample "echo" service displaying UserID and UserRole HTTP request headers +func main() { + http.HandleFunc("/", handler) + http.ListenAndServe(":8080", nil) +} diff --git a/docs/examples/customization/jwt/README.md b/docs/examples/customization/jwt/README.md deleted file mode 100644 index a751ccb7b..000000000 --- a/docs/examples/customization/jwt/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Accommodation for JWT - -JWT (short for Json Web Token) is an authentication method widely used. Basically an authentication server generates -a JWT and you then use this token in every request you make to a backend service. The JWT can be quite big and is -present in every http headers. This means you may have to adapt the max-header size of your nginx-ingress in order -to support it. - -## Symptoms - -If you use JWT and you get http 502 error from your ingress, it may be a sign that the buffer size is not big enough. - -To be 100% sure look at the logs of the `ingress-nginx-controller` pod, you should see something like this: - -``` -upstream sent too big header while reading response header from upstream... -``` - - -## Increase buffer size for headers - -In nginx, we want to modify the property `proxy-buffer-size`. The size is arbitrary. It depends on your needs. Be aware -that a high value can lower the performance of your ingress proxy. In general a value of 16k should get you covered. - -### Using helm -If you're using helm you can simply use the [`config` properties](https://github.com/kubernetes/ingress-nginx/blob/main/charts/ingress-nginx/values.yaml#L37). -```yaml - # -- Will add custom configuration options to Nginx https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/ - config: - proxy-buffer-size: 16k -``` - -## Manually in kubernetes config files - -If you use an already generated config from for a provider, you will have to change the `controller-configmap.yaml` - -```yaml ---- -# Source: ingress-nginx/templates/controller-configmap.yaml -apiVersion: v1 -kind: ConfigMap -# ... -data: - #... - proxy-buffer-size: "16k" -``` - -References: - * [Custom Configuration](../custom-configuration/) \ No newline at end of file diff --git a/docs/examples/customization/ssl-dh-param/README.md b/docs/examples/customization/ssl-dh-param/README.md index fcd80757b..9029b834b 100644 --- a/docs/examples/customization/ssl-dh-param/README.md +++ b/docs/examples/customization/ssl-dh-param/README.md @@ -1,7 +1,7 @@ # Custom DH parameters for perfect forward secrecy -This example aims to demonstrate the deployment of an Ingress-Nginx Controller and -use a ConfigMap to configure a custom Diffie-Hellman parameters file to help with +This example aims to demonstrate the deployment of an nginx ingress controller and +use a ConfigMap to configure custom Diffie-Hellman parameters file to help with "Perfect Forward Secrecy". ## Custom configuration @@ -27,7 +27,7 @@ $ kubectl create -f configmap.yaml ## Custom DH parameters secret ```console -$ openssl dhparam 4096 2> /dev/null | base64 +$> openssl dhparam 4096 2> /dev/null | base64 LS0tLS1CRUdJTiBESCBQQVJBTUVURVJ... ``` @@ -52,6 +52,4 @@ $ kubectl create -f ssl-dh-param.yaml ## Test Check the contents of the configmap is present in the nginx.conf file using: -```console -$ kubectl exec ingress-nginx-controller-873061567-4n3k2 -n kube-system -- cat /etc/nginx/nginx.conf -``` +`kubectl exec ingress-nginx-controller-873061567-4n3k2 -n kube-system -- cat /etc/nginx/nginx.conf` diff --git a/docs/examples/customization/sysctl/README.md b/docs/examples/customization/sysctl/README.md index 54fbe4355..2de45b2da 100644 --- a/docs/examples/customization/sysctl/README.md +++ b/docs/examples/customization/sysctl/README.md @@ -1,10 +1,10 @@ # Sysctl tuning -This example aims to demonstrate the use of an Init Container to adjust sysctl default values using `kubectl patch`. +This example aims to demonstrate the use of an Init Container to adjust sysctl default values using `kubectl patch` ```console -kubectl patch deployment -n ingress-nginx ingress-nginx-controller \ - --patch="$(curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/docs/examples/customization/sysctl/patch.json)" +kubectl patch deployment -n ingress-nginx nginx-ingress-controller \ + --patch="$(curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/docs/examples/customization/sysctl/patch.json)" ``` **Changes:** diff --git a/docs/examples/customization/sysctl/patch.json b/docs/examples/customization/sysctl/patch.json index 69482c7be..d0cf1f9ae 100644 --- a/docs/examples/customization/sysctl/patch.json +++ b/docs/examples/customization/sysctl/patch.json @@ -4,7 +4,7 @@ "spec": { "initContainers": [{ "name": "sysctl", - "image": "alpine:3.21", + "image": "alpine:3.13", "securityContext": { "privileged": true }, @@ -13,4 +13,4 @@ } } } -} +} \ No newline at end of file diff --git a/docs/examples/docker-registry/README.md b/docs/examples/docker-registry/README.md index 8ba230f2c..495297a84 100644 --- a/docs/examples/docker-registry/README.md +++ b/docs/examples/docker-registry/README.md @@ -1,13 +1,13 @@ # Docker registry -This example demonstrates how to deploy a [docker registry](https://github.com/docker/distribution) in the cluster and configure Ingress to enable access from the Internet. +This example demonstrates how to deploy a [docker registry](https://github.com/docker/distribution) in the cluster and configure Ingress enable access from Internet ## Deployment First we deploy the docker registry in the cluster: ```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/docs/examples/docker-registry/deployment.yaml +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/docs/examples/docker-registry/deployment.yaml ``` !!! Important @@ -22,7 +22,7 @@ The next required step is creation of the ingress rules. To do this we have two Download and edit the yaml deployment replacing `registry.` with a valid DNS name pointing to the ingress controller: ```console -wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/docs/examples/docker-registry/ingress-without-tls.yaml +wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/docs/examples/docker-registry/ingress-without-tls.yaml ``` !!! Important @@ -35,7 +35,7 @@ Please check [deploy a plain http registry](https://docs.docker.com/registry/ins Download and edit the yaml deployment replacing `registry.` with a valid DNS name pointing to the ingress controller: ```console -wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/docs/examples/docker-registry/ingress-with-tls.yaml +wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/docs/examples/docker-registry/ingress-with-tls.yaml ``` Deploy [kube lego](https://github.com/jetstack/kube-lego) use [Let's Encrypt](https://letsencrypt.org/) certificates or edit the ingress rule to use a secret with an existing SSL certificate. diff --git a/docs/examples/docker-registry/ingress-with-tls.yaml b/docs/examples/docker-registry/ingress-with-tls.yaml index 1bdd9ed7a..fc277b20f 100644 --- a/docs/examples/docker-registry/ingress-with-tls.yaml +++ b/docs/examples/docker-registry/ingress-with-tls.yaml @@ -1,4 +1,4 @@ -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: annotations: @@ -9,7 +9,6 @@ metadata: name: docker-registry namespace: docker-registry spec: - ingressClassName: nginx tls: - hosts: - registry. @@ -18,10 +17,7 @@ spec: - host: registry. http: paths: - - path: / - pathType: Prefix - backend: - service: - name: docker-registry - port: - number: 5000 + - backend: + serviceName: docker-registry + servicePort: 5000 + path: / diff --git a/docs/examples/docker-registry/ingress-without-tls.yaml b/docs/examples/docker-registry/ingress-without-tls.yaml index d0ffc8baf..1ce1b98fb 100644 --- a/docs/examples/docker-registry/ingress-without-tls.yaml +++ b/docs/examples/docker-registry/ingress-without-tls.yaml @@ -1,4 +1,4 @@ -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: annotations: @@ -8,15 +8,11 @@ metadata: name: docker-registry namespace: docker-registry spec: - ingressClassName: nginx rules: - host: registry. http: paths: - - path: / - pathType: Prefix - backend: - service: - name: docker-registry - port: - number: 5000 \ No newline at end of file + - backend: + serviceName: docker-registry + servicePort: 5000 + path: / diff --git a/docs/examples/grpc/README.md b/docs/examples/grpc/README.md index 23126c345..60e696bfd 100644 --- a/docs/examples/grpc/README.md +++ b/docs/examples/grpc/README.md @@ -1,157 +1,98 @@ # gRPC -This example demonstrates how to route traffic to a gRPC service through the Ingress-NGINX controller. +This example demonstrates how to route traffic to a gRPC service through the +nginx controller. ## Prerequisites 1. You have a kubernetes cluster running. -2. You have a domain name such as `example.com` that is configured to route traffic to the Ingress-NGINX controller. -3. You have the ingress-nginx-controller installed as per docs. -4. You have a backend application running a gRPC server listening for TCP traffic. If you want, you can use as an example. -5. You're also responsible for provisioning an SSL certificate for the ingress. So you need to have a valid SSL certificate, deployed as a Kubernetes secret of type `tls`, in the same namespace as the gRPC application. +2. You have a domain name such as `example.com` that is configured to route + traffic to the ingress controller. Replace references to + `fortune-teller.stack.build` (the domain name used in this example) to your + own domain name (you're also responsible for provisioning an SSL certificate + for the ingress). +3. You have the nginx-ingress controller installed in typical fashion (must be + at least + [quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.30.0](https://quay.io/kubernetes-ingress-controller/nginx-ingress-controller) + for grpc support. +4. You have a backend application running a gRPC server and listening for TCP + traffic. If you prefer, you can use the + [fortune-teller](https://github.com/kubernetes/ingress-nginx/tree/master/images/grpc-fortune-teller) + application provided here as an example. -### Step 1: Create a Kubernetes `Deployment` for gRPC app +### Step 1: kubernetes `Deployment` -- Make sure your gRPC application pod is running and listening for connections. For example you can try a kubectl command like this below: - ```console - $ kubectl get po -A -o wide | grep go-grpc-greeter-server - ``` -- If you have a gRPC app deployed in your cluster, then skip further notes in this Step 1, and continue from Step 2 below. +```sh +$ kubectl create -f app.yaml +``` -- As an example gRPC application, we can use this app . +This is a standard kubernetes deployment object. It is running a grpc service +listening on port `50051`. -- To create a container image for this app, you can use [this Dockerfile](https://github.com/kubernetes/ingress-nginx/blob/main/images/go-grpc-greeter-server/rootfs/Dockerfile). +The sample application +[fortune-teller-app](https://github.com/kubernetes/ingress-nginx/tree/master/images/grpc-fortune-teller) +is a grpc server implemented in go. Here's the stripped-down implementation: -- If you use the Dockerfile mentioned above, to create a image, then you can use the following example Kubernetes manifest to create a deployment resource that uses that image. If necessary edit this manifest to suit your needs. +```go +func main() { + grpcServer := grpc.NewServer() + fortune.RegisterFortuneTellerServer(grpcServer, &FortuneTeller{}) + lis, _ := net.Listen("tcp", ":50051") + grpcServer.Serve(lis) +} +``` - ``` - cat </go-grpc-greeter-server # Edit this for your reponame - resources: - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 50m - memory: 50Mi - name: go-grpc-greeter-server - ports: - - containerPort: 50051 - EOF - ``` +The takeaway is that we are not doing any TLS configuration on the server (as we +are terminating TLS at the ingress level, grpc traffic will travel unencrypted +inside the cluster and arrive "insecure"). -### Step 2: Create the Kubernetes `Service` for the gRPC app +For your own application you may or may not want to do this. If you prefer to +forward encrypted traffic to your POD and terminate TLS at the gRPC server +itself, add the ingress annotation `nginx.ingress.kubernetes.io/backend-protocol: "GRPCS"`. -- You can use the following example manifest to create a service of type ClusterIP. Edit the name/namespace/label/port to match your deployment/pod. - ``` - cat < https://proto.stack.build, a protocol buffer / gRPC build service that can use > to help make it easier for your users to consume your API. -> See also the specific gRPC settings of NGINX: https://nginx.org/en/docs/http/ngx_http_grpc_module.html +> See also the specific GRPC settings of NGINX: https://nginx.org/en/docs/http/ngx_http_grpc_module.html ### Notes on using response/request streams -> `grpc_read_timeout` and `grpc_send_timeout` will be set as `proxy_read_timeout` and `proxy_send_timeout` when you set backend protocol to `GRPC` or `GRPCS`. - -1. If your server only does response streaming and you expect a stream to be open longer than 60 seconds, you will have to change the `grpc_read_timeout` to accommodate this. -2. If your service only does request streaming and you expect a stream to be open longer than 60 seconds, you have to change the +1. If your server does only response streaming and you expect a stream to be open longer than 60 seconds, you will have to change the `grpc_read_timeout` to accommodate for this. +2. If your service does only request streaming and you expect a stream to be open longer than 60 seconds, you have to change the `grpc_send_timeout` and the `client_body_timeout`. 3. If you do both response and request streaming with an open stream longer than 60 seconds, you have to change all three timeouts: `grpc_read_timeout`, `grpc_send_timeout` and `client_body_timeout`. + +Values for the timeouts must be specified as e.g. `"1200s"`. + +> On the most recent versions of nginx-ingress, changing these timeouts requires using the `nginx.ingress.kubernetes.io/server-snippet` annotation. There are plans for future releases to allow using the Kubernetes annotations to define each timeout separately. diff --git a/docs/examples/grpc/app.yaml b/docs/examples/grpc/app.yaml new file mode 100644 index 000000000..acc4060d0 --- /dev/null +++ b/docs/examples/grpc/app.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fortune-teller-app + labels: + k8s-app: fortune-teller-app + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: fortune-teller-app + template: + metadata: + labels: + k8s-app: fortune-teller-app + spec: + containers: + - name: fortune-teller-app + image: quay.io/kubernetes-ingress-controller/grpc-fortune-teller:0.1 + ports: + - containerPort: 50051 + name: grpc diff --git a/docs/examples/grpc/cert.yaml b/docs/examples/grpc/cert.yaml new file mode 100644 index 000000000..562c30313 --- /dev/null +++ b/docs/examples/grpc/cert.yaml @@ -0,0 +1,7 @@ +apiVersion: "stable.k8s.psg.io/v1" +kind: "Certificate" +metadata: + name: fortune-teller.stack.build + namespace: default +spec: + domain: "fortune-teller.stack.build" diff --git a/docs/examples/grpc/ingress.yaml b/docs/examples/grpc/ingress.yaml new file mode 100644 index 000000000..1d76476d1 --- /dev/null +++ b/docs/examples/grpc/ingress.yaml @@ -0,0 +1,24 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/backend-protocol: "GRPC" + name: fortune-ingress + namespace: default +spec: + rules: + - host: fortune-teller.stack.build + http: + paths: + - backend: + serviceName: fortune-teller-service + servicePort: grpc + tls: + # This secret must exist beforehand + # The cert must also contain the subj-name fortune-teller.stack.build + # https://github.com/kubernetes/ingress-nginx/blob/master/docs/examples/PREREQUISITES.md#tls-certificates + - secretName: fortune-teller.stack.build + hosts: + - fortune-teller.stack.build diff --git a/docs/examples/grpc/svc.yaml b/docs/examples/grpc/svc.yaml new file mode 100644 index 000000000..7bfa789fd --- /dev/null +++ b/docs/examples/grpc/svc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: fortune-teller-service + namespace: default +spec: + selector: + k8s-app: fortune-teller-app + ports: + - port: 50051 + targetPort: 50051 + name: grpc diff --git a/docs/examples/http-svc.yaml b/docs/examples/http-svc.yaml index 46815fa5d..bd2b34ce9 100644 --- a/docs/examples/http-svc.yaml +++ b/docs/examples/http-svc.yaml @@ -14,7 +14,7 @@ spec: spec: containers: - name: http-svc - image: registry.k8s.io/e2e-test-images/echoserver:2.3 + image: gcr.io/kubernetes-e2e-test-images/echoserver:2.1 ports: - containerPort: 8080 env: diff --git a/docs/examples/index.md b/docs/examples/index.md index 4efdae39f..04f42ea60 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -1,12 +1,8 @@ # Ingress examples -This directory contains a catalog of examples on how to run, configure and scale Ingress. +This directory contains a catalog of examples on how to run, configure and scale Ingress. Please review the [prerequisites](PREREQUISITES.md) before trying them. -The examples on these pages include the `spec.ingressClassName` field which replaces the deprecated `kubernetes.io/ingress.class: nginx` annotation. Users of ingress-nginx < 1.0.0 (Helm chart < 4.0.0) should use the [legacy documentation](https://github.com/kubernetes/ingress-nginx/tree/legacy/docs/examples). - -For more information, check out the [Migration to apiVersion networking.k8s.io/v1](../#faq-migration-to-apiversion-networkingk8siov1) guide. - Category | Name | Description | Complexity Level ---------| ---- | ----------- | ---------------- Apps | [Docker Registry](docker-registry/README.md) | TODO | TODO @@ -18,12 +14,11 @@ Customization | [Configuration snippets](customization/configuration-snippets/RE Customization | [Custom configuration](customization/custom-configuration/README.md) | TODO | TODO Customization | [Custom DH parameters for perfect forward secrecy](customization/ssl-dh-param/README.md) | TODO | TODO Customization | [Custom errors](customization/custom-errors/README.md) | serve custom error pages from the default backend | Intermediate -Customization | [Custom headers](customization/custom-headers/README.md) | set custom headers before sending traffic to backends | Advanced +Customization | [Custom headers](customization/custom-headers/README.md) | set custom headers before sending traffic to backends | Advanced Customization | [External authentication with response header propagation](customization/external-auth-headers/README.md) | TODO | TODO Customization | [Sysctl tuning](customization/sysctl/README.md) | TODO | TODO Features | [Rewrite](rewrite/README.md) | TODO | TODO Features | [Session stickiness](affinity/cookie/README.md) | route requests consistently to the same endpoint | Advanced -Features | [Canary Deployments](canary/README.md) | weighted canary routing to a separate deployment | Intermediate Scaling | [Static IP](static-ip/README.md) | a single ingress gets a single static IP | Intermediate TLS | [Multi TLS certificate termination](multi-tls/README.md) | TODO | TODO TLS | [TLS termination](tls-termination/README.md) | TODO | TODO diff --git a/docs/examples/multi-tls/README.md b/docs/examples/multi-tls/README.md index cc990151b..d1e7295d9 100644 --- a/docs/examples/multi-tls/README.md +++ b/docs/examples/multi-tls/README.md @@ -2,12 +2,13 @@ This example uses 2 different certificates to terminate SSL for 2 hostnames. -1. Create tls secrets for foo.bar.com and bar.baz.com as indicated in the yaml -2. Create [multi-tls.yaml](multi-tls.yaml) +1. Deploy the controller by creating the rc in the parent dir +2. Create tls secrets for foo.bar.com and bar.baz.com as indicated in the yaml +3. Create [multi-tls.yaml](multi-tls.yaml) This should generate a segment like: ```console -$ kubectl exec -it ingress-nginx-controller-6vwd1 -- cat /etc/nginx/nginx.conf | grep "foo.bar.com" -B 7 -A 35 +$ kubectl exec -it nginx-ingress-controller-6vwd1 -- cat /etc/nginx/nginx.conf | grep "foo.bar.com" -B 7 -A 35 server { listen 80; listen 443 ssl http2; diff --git a/docs/examples/multi-tls/multi-tls.yaml b/docs/examples/multi-tls/multi-tls.yaml index d52971326..c616501be 100644 --- a/docs/examples/multi-tls/multi-tls.yaml +++ b/docs/examples/multi-tls/multi-tls.yaml @@ -70,7 +70,7 @@ spec: spec: containers: - name: http-svc - image: registry.k8s.io/e2e-test-images/echoserver:2.3 + image: gcr.io/kubernetes-e2e-test-images/echoserver:2.1 ports: - containerPort: 8080 env: @@ -92,44 +92,37 @@ spec: fieldPath: status.podIP --- -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: foo-tls namespace: default spec: - ingressClassName: nginx tls: - hosts: - foo.bar.com # This secret must exist beforehand # The cert must also contain the subj-name foo.bar.com - # https://github.com/kubernetes/ingress-nginx/blob/main/docs/examples/PREREQUISITES.md#tls-certificates + # https://github.com/kubernetes/ingress-nginx/blob/master/docs/examples/PREREQUISITES.md#tls-certificates secretName: foobar - hosts: - bar.baz.com # This secret must exist beforehand # The cert must also contain the subj-name bar.baz.com - # https://github.com/kubernetes/ingress-nginx/blob/main/docs/examples/PREREQUISITES.md#tls-certificates + # https://github.com/kubernetes/ingress-nginx/blob/master/docs/examples/PREREQUISITES.md#tls-certificates secretName: barbaz rules: - host: foo.bar.com http: paths: - - path: / - pathType: Prefix - backend: - service: - name: http-svc - port: - number: 80 + - backend: + serviceName: http-svc + servicePort: 80 + path: / - host: bar.baz.com http: paths: - - path: / - pathType: Prefix - backend: - service: - name: nginx - port: - number: 80 + - backend: + serviceName: nginx + servicePort: 80 + path: / diff --git a/docs/examples/openpolicyagent/README.md b/docs/examples/openpolicyagent/README.md deleted file mode 100644 index 8d6337a38..000000000 --- a/docs/examples/openpolicyagent/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# OpenPolicyAgent and pathType enforcing - -Ingress API allows users to specify different [pathType](https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types) -on Ingress object. - -While pathType `Exact` and `Prefix` should allow only a small set of characters, pathType `ImplementationSpecific` -allows any characters, as it may contain regexes, variables and other features that may be specific of the Ingress -Controller being used. - -This means that the Ingress Admins (the persona who deployed the Ingress Controller) should trust the users -allowed to use `pathType: ImplementationSpecific`, as this may allow arbitrary configuration, and this -configuration may end on the proxy (aka Nginx) configuration. - -## Example -The example in this repo uses [Gatekeeper](https://open-policy-agent.github.io/gatekeeper/website/) to block the usage of `pathType: ImplementationSpecific`, -allowing just a specific list of namespaces to use it. - -It is recommended that the admin modifies this rules to enforce a specific set of characters when the usage of ImplementationSpecific -is allowed, or in ways that best suits their needs. - -First, the `ConstraintTemplate` from [template.yaml](template.yaml) will define a rule that validates if the Ingress object -is being created on an exempted namespace, and case not, will validate its pathType. - -Then, the rule `K8sBlockIngressPathType` contained in [rule.yaml](rule.yaml) will define the parameters: what kind of -object should be verified (Ingress), what are the exempted namespaces, and what kinds of pathType are blocked. diff --git a/docs/examples/openpolicyagent/rule.yaml b/docs/examples/openpolicyagent/rule.yaml deleted file mode 100644 index fce305241..000000000 --- a/docs/examples/openpolicyagent/rule.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: constraints.gatekeeper.sh/v1beta1 -kind: K8sBlockIngressPathType -metadata: - name: implspecificisblocked -spec: - match: - kinds: - - apiGroups: ["networking.k8s.io"] - kinds: ["Ingress"] - parameters: - namespacesExceptions: - - "privileged" - blockedTypes: - - "ImplementationSpecific" diff --git a/docs/examples/openpolicyagent/template.yaml b/docs/examples/openpolicyagent/template.yaml deleted file mode 100644 index 4302415a2..000000000 --- a/docs/examples/openpolicyagent/template.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: templates.gatekeeper.sh/v1 -kind: ConstraintTemplate -metadata: - name: k8sblockingresspathtype - annotations: - metadata.gatekeeper.sh/title: "Block a pathType usage" - description: >- - Users should not be able to use specific pathTypes -spec: - crd: - spec: - names: - kind: K8sBlockIngressPathType - validation: - openAPIV3Schema: - type: object - properties: - blockedTypes: - type: array - items: - type: string - namespacesExceptions: - type: array - items: - type: string - targets: - - target: admission.k8s.gatekeeper.sh - rego: | - package K8sBlockIngressPathType - - violation[{"msg": msg}] { - input.review.kind.kind == "Ingress" - ns := input.review.object.metadata.namespace - exemptNS := [good | exempts = input.parameters.namespacesExceptions[_] ; good = exempts == ns] - not any(exemptNS) - pathType := object.get(input.review.object.spec.rules[_].http.paths[_], "pathType", "") - blockedPath := [blocked | blockedTypes = input.parameters.blockedTypes[_] ; blocked = blockedTypes == pathType] - any(blockedPath) - msg := sprintf("pathType '%v' is not allowed in this namespace", [pathType]) - } diff --git a/docs/examples/openpolicyagent/tests/should-allow-ns-except.yaml b/docs/examples/openpolicyagent/tests/should-allow-ns-except.yaml deleted file mode 100644 index 974e83555..000000000 --- a/docs/examples/openpolicyagent/tests/should-allow-ns-except.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - creationTimestamp: null - name: simple - namespace: privileged -spec: - rules: - - host: foo1.com - http: - paths: - - backend: - service: - name: svc1 - port: - number: 8080 - path: /bar - pathType: ImplementationSpecific diff --git a/docs/examples/openpolicyagent/tests/should-allow.yaml b/docs/examples/openpolicyagent/tests/should-allow.yaml deleted file mode 100644 index 854aff14c..000000000 --- a/docs/examples/openpolicyagent/tests/should-allow.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - creationTimestamp: null - name: simple -spec: - rules: - - host: foo.com - http: - paths: - - backend: - service: - name: svc1 - port: - number: 8080 - path: /bar - pathType: Exact diff --git a/docs/examples/openpolicyagent/tests/should-deny.yaml b/docs/examples/openpolicyagent/tests/should-deny.yaml deleted file mode 100644 index b732fdc89..000000000 --- a/docs/examples/openpolicyagent/tests/should-deny.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - creationTimestamp: null - name: simple -spec: - rules: - - host: foo2.com - http: - paths: - - backend: - service: - name: svc1 - port: - number: 8080 - path: /bar - pathType: ImplementationSpecific diff --git a/docs/examples/psp/README.md b/docs/examples/psp/README.md new file mode 100644 index 000000000..4064ed218 --- /dev/null +++ b/docs/examples/psp/README.md @@ -0,0 +1,17 @@ +# Pod Security Policy (PSP) + +In most clusters today, by default, all resources (e.g. Deployments and ReplicatSets) +have permissions to create pods. +Kubernetes however provides a more fine-grained authorization policy called +[Pod Security Policy (PSP)](https://kubernetes.io/docs/concepts/policy/pod-security-policy/). + +PSP allows the cluster owner to define the permission of each object, for example creating a pod. +If you have PSP enabled on the cluster, and you deploy ingress-nginx, +you will need to provide the Deployment with the permissions to create pods. + +Before applying any objects, first apply the PSP permissions by running: +```console +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/docs/examples/psp/psp.yaml +``` + +Note: PSP permissions must be granted before to the creation of the Deployment and the ReplicaSet. diff --git a/docs/examples/psp/psp.yaml b/docs/examples/psp/psp.yaml new file mode 100644 index 000000000..0f859dece --- /dev/null +++ b/docs/examples/psp/psp.yaml @@ -0,0 +1,75 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ingress-nginx + +--- + +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: ingress-nginx + namespace: ingress-nginx +spec: + allowedCapabilities: + - NET_BIND_SERVICE + privileged: false + allowPrivilegeEscalation: true + # Allow core volume types. + volumes: + - configMap + - secret + hostIPC: false + hostPID: false + runAsUser: + # Require the container to run without root privileges. + rule: MustRunAsNonRoot + supplementalGroups: + rule: MustRunAs + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: MustRunAs + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + seLinux: + rule: RunAsAny + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ingress-nginx-psp + namespace: ingress-nginx +rules: +- apiGroups: [policy] + resources: [podsecuritypolicies] + verbs: [use] + resourceNames: [ingress-nginx] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ingress-nginx-psp + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-psp +subjects: +- kind: ServiceAccount + name: default +- kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +- kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx diff --git a/docs/examples/rewrite/README.md b/docs/examples/rewrite/README.md index 16889e0bc..3ad5e4fcd 100644 --- a/docs/examples/rewrite/README.md +++ b/docs/examples/rewrite/README.md @@ -1,6 +1,6 @@ # Rewrite -This example demonstrates how to use `Rewrite` annotations. +This example demonstrates how to use the Rewrite annotations ## Prerequisites @@ -15,9 +15,9 @@ Rewriting can be controlled using the following annotations: |Name|Description|Values| | --- | --- | --- | |nginx.ingress.kubernetes.io/rewrite-target|Target URI where the traffic must be redirected|string| -|nginx.ingress.kubernetes.io/ssl-redirect|Indicates if the location section is only accessible via SSL (defaults to True when Ingress contains a Certificate)|bool| +|nginx.ingress.kubernetes.io/ssl-redirect|Indicates if the location section is accessible SSL only (defaults to True when Ingress contains a Certificate)|bool| |nginx.ingress.kubernetes.io/force-ssl-redirect|Forces the redirection to HTTPS even if the Ingress is not TLS Enabled|bool| -|nginx.ingress.kubernetes.io/app-root|Defines the Application Root that the Controller must redirect if it's in `/` context|string| +|nginx.ingress.kubernetes.io/app-root|Defines the Application Root that the Controller must redirect if it's in '/' context|string| |nginx.ingress.kubernetes.io/use-regex|Indicates if the paths defined on an Ingress use regular expressions|bool| ## Examples @@ -30,34 +30,26 @@ Rewriting can be controlled using the following annotations: !!! note [Captured groups](https://www.regular-expressions.info/refcapture.html) are saved in numbered placeholders, chronologically, in the form `$1`, `$2` ... `$n`. These placeholders can be used as parameters in the `rewrite-target` annotation. -!!! note - Please see the [FAQ](../../faq.md#validation-of-path) for Validation Of __`path`__ - Create an Ingress rule with a rewrite annotation: ```console $ echo ' -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: annotations: - nginx.ingress.kubernetes.io/use-regex: "true" nginx.ingress.kubernetes.io/rewrite-target: /$2 name: rewrite namespace: default spec: - ingressClassName: nginx rules: - host: rewrite.bar.com http: paths: - - path: /something(/|$)(.*) - pathType: ImplementationSpecific - backend: - service: - name: http-svc - port: - number: 80 + - backend: + serviceName: http-svc + servicePort: 80 + path: /something(/|$)(.*) ' | kubectl create -f - ``` @@ -74,7 +66,7 @@ For example, the ingress definition above will result in the following rewrites: Create an Ingress rule with an app-root annotation: ``` $ echo " -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: annotations: @@ -82,18 +74,14 @@ metadata: name: approot namespace: default spec: - ingressClassName: nginx rules: - host: approot.bar.com http: paths: - - path: / - pathType: Prefix - backend: - service: - name: http-svc - port: - number: 80 + - backend: + serviceName: http-svc + servicePort: 80 + path: / " | kubectl create -f - ``` @@ -106,6 +94,6 @@ Server: nginx/1.11.10 Date: Mon, 13 Mar 2017 14:57:15 GMT Content-Type: text/html Content-Length: 162 -Location: http://approot.bar.com/app1 +Location: http://stickyingress.example.com/app1 Connection: keep-alive ``` diff --git a/docs/examples/static-ip/README.md b/docs/examples/static-ip/README.md index 992839a24..1f9ea5c96 100644 --- a/docs/examples/static-ip/README.md +++ b/docs/examples/static-ip/README.md @@ -1,6 +1,6 @@ # Static IPs -This example demonstrates how to assign a static-ip to an Ingress on through the Ingress-NGINX controller. +This example demonstrates how to assign a static-ip to an Ingress on through the Nginx controller. ## Prerequisites @@ -11,46 +11,46 @@ and that you have an ingress controller [running](../../deploy/) in your cluster ## Acquiring an IP -Since instances of the ingress nginx controller actually run on nodes in your cluster, +Since instances of the nginx controller actually run on nodes in your cluster, by default nginx Ingresses will only get static IPs if your cloudprovider supports static IP assignments to nodes. On GKE/GCE for example, even though -nodes get static IPs, the IPs are not retained across upgrades. +nodes get static IPs, the IPs are not retained across upgrade. -To acquire a static IP for the ingress-nginx-controller, simply put it +To acquire a static IP for the nginx ingress controller, simply put it behind a Service of `Type=LoadBalancer`. -First, create a loadbalancer Service and wait for it to acquire an IP: +First, create a loadbalancer Service and wait for it to acquire an IP ```console $ kubectl create -f static-ip-svc.yaml -service "ingress-nginx-lb" created +service "nginx-ingress-lb" created -$ kubectl get svc ingress-nginx-lb +$ kubectl get svc nginx-ingress-lb NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE -ingress-nginx-lb 10.0.138.113 104.154.109.191 80:31457/TCP,443:32240/TCP 15m +nginx-ingress-lb 10.0.138.113 104.154.109.191 80:31457/TCP,443:32240/TCP 15m ``` -Then, update the ingress controller so it adopts the static IP of the Service +then, update the ingress controller so it adopts the static IP of the Service by passing the `--publish-service` flag (the example yaml used in the next step -already has it set to "ingress-nginx-lb"). +already has it set to "nginx-ingress-lb"). ```console -$ kubectl create -f ingress-nginx-controller.yaml -deployment "ingress-nginx-controller" created +$ kubectl create -f nginx-ingress-controller.yaml +deployment "nginx-ingress-controller" created ``` ## Assigning the IP to an Ingress From here on every Ingress created with the `ingress.class` annotation set to -`nginx` will get the IP allocated in the previous step. +`nginx` will get the IP allocated in the previous step ```console -$ kubectl create -f ingress-nginx.yaml -ingress "ingress-nginx" created +$ kubectl create -f nginx-ingress.yaml +ingress "nginx-ingress" created $ kubectl get ing ingress-nginx NAME HOSTS ADDRESS PORTS AGE -ingress-nginx * 104.154.109.191 80, 443 13m +nginx-ingress * 104.154.109.191 80, 443 13m $ curl 104.154.109.191 -kL CLIENT VALUES: @@ -65,18 +65,18 @@ request_uri=http://104.154.109.191:8080/ ## Retaining the IP -You can test retention by deleting the Ingress: +You can test retention by deleting the Ingress ```console -$ kubectl delete ing ingress-nginx -ingress "ingress-nginx" deleted +$ kubectl delete ing nginx-ingress +ingress "nginx-ingress" deleted -$ kubectl create -f ingress-nginx.yaml -ingress "ingress-nginx" created +$ kubectl create -f nginx-ingress.yaml +ingress "nginx-ingress" created -$ kubectl get ing ingress-nginx +$ kubectl get ing nginx-ingress NAME HOSTS ADDRESS PORTS AGE -ingress-nginx * 104.154.109.191 80, 443 13m +nginx-ingress * 104.154.109.191 80, 443 13m ``` > Note that unlike the GCE Ingress, the same loadbalancer IP is shared amongst all @@ -85,28 +85,28 @@ ingress-nginx * 104.154.109.191 80, 443 13m ## Promote ephemeral to static IP -To promote the allocated IP to static, you can update the Service manifest: +To promote the allocated IP to static, you can update the Service manifest ```console -$ kubectl patch svc ingress-nginx-lb -p '{"spec": {"loadBalancerIP": "104.154.109.191"}}' -"ingress-nginx-lb" patched +$ kubectl patch svc nginx-ingress-lb -p '{"spec": {"loadBalancerIP": "104.154.109.191"}}' +"nginx-ingress-lb" patched ``` -... and promote the IP to static (promotion works differently for cloudproviders, -provided example is for GKE/GCE): - +and promote the IP to static (promotion works differently for cloudproviders, +provided example is for GKE/GCE) +` ```console -$ gcloud compute addresses create ingress-nginx-lb --addresses 104.154.109.191 --region us-central1 -Created [https://www.googleapis.com/compute/v1/projects/kubernetesdev/regions/us-central1/addresses/ingress-nginx-lb]. +$ gcloud compute addresses create nginx-ingress-lb --addresses 104.154.109.191 --region us-central1 +Created [https://www.googleapis.com/compute/v1/projects/kubernetesdev/regions/us-central1/addresses/nginx-ingress-lb]. --- address: 104.154.109.191 creationTimestamp: '2017-01-31T16:34:50.089-08:00' description: '' id: '5208037144487826373' kind: compute#address -name: ingress-nginx-lb +name: nginx-ingress-lb region: us-central1 -selfLink: https://www.googleapis.com/compute/v1/projects/kubernetesdev/regions/us-central1/addresses/ingress-nginx-lb +selfLink: https://www.googleapis.com/compute/v1/projects/kubernetesdev/regions/us-central1/addresses/nginx-ingress-lb status: IN_USE users: - us-central1/forwardingRules/a09f6913ae80e11e6a8c542010af0000 @@ -114,3 +114,4 @@ users: Now even if the Service is deleted, the IP will persist, so you can recreate the Service with `spec.loadBalancerIP` set to `104.154.109.191`. + diff --git a/docs/examples/static-ip/nginx-ingress-controller.yaml b/docs/examples/static-ip/nginx-ingress-controller.yaml index cf7759e1c..30885ec54 100644 --- a/docs/examples/static-ip/nginx-ingress-controller.yaml +++ b/docs/examples/static-ip/nginx-ingress-controller.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: ingress-nginx-controller + name: nginx-ingress-controller labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx @@ -18,14 +18,14 @@ spec: app.kubernetes.io/part-of: ingress-nginx spec: # hostNetwork makes it possible to use ipv6 and to preserve the source IP correctly regardless of docker configuration - # however, it is not a hard dependency of the ingress-nginx-controller itself and it may cause issues if port 10254 already is taken on the host + # however, it is not a hard dependency of the nginx-ingress-controller itself and it may cause issues if port 10254 already is taken on the host # that said, since hostPort is broken on CNI (https://github.com/kubernetes/kubernetes/issues/31307) we have to use hostNetwork where CNI is used # like with kubeadm # hostNetwork: true terminationGracePeriodSeconds: 60 containers: - - image: registry.k8s.io/ingress-nginx/controller:v1.0.5 - name: controller + - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.33.0 + name: nginx-ingress-controller readinessProbe: httpGet: path: /healthz @@ -54,4 +54,4 @@ spec: fieldPath: metadata.namespace args: - /nginx-ingress-controller - - --publish-service=$(POD_NAMESPACE)/ingress-nginx-lb + - --publish-service=$(POD_NAMESPACE)/nginx-ingress-lb diff --git a/docs/examples/static-ip/nginx-ingress.yaml b/docs/examples/static-ip/nginx-ingress.yaml index 5c8a3c920..aa4877e56 100644 --- a/docs/examples/static-ip/nginx-ingress.yaml +++ b/docs/examples/static-ip/nginx-ingress.yaml @@ -1,20 +1,15 @@ -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: ingress-nginx spec: - ingressClassName: nginx tls: # This assumes tls-secret exists. - secretName: tls-secret rules: - http: paths: - - path: / - pathType: Prefix - backend: + - backend: # This assumes http-svc exists and routes to healthy endpoints. - service: - name: http-svc - port: - number: 80 + serviceName: http-svc + servicePort: 80 diff --git a/docs/examples/static-ip/static-ip-svc.yaml b/docs/examples/static-ip/static-ip-svc.yaml index ee803951f..b64cf96cb 100644 --- a/docs/examples/static-ip/static-ip-svc.yaml +++ b/docs/examples/static-ip/static-ip-svc.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: ingress-nginx-lb + name: nginx-ingress-lb labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx @@ -18,6 +18,6 @@ spec: name: https targetPort: 443 selector: - # Selects ingress-nginx-controller pods + # Selects nginx-ingress-controller pods app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx diff --git a/docs/examples/tls-termination/README.md b/docs/examples/tls-termination/README.md index 25139b381..1a17d6d21 100644 --- a/docs/examples/tls-termination/README.md +++ b/docs/examples/tls-termination/README.md @@ -1,6 +1,6 @@ # TLS termination -This example demonstrates how to terminate TLS through the Ingress-Nginx Controller. +This example demonstrates how to terminate TLS through the nginx Ingress controller. ## Prerequisites @@ -11,7 +11,7 @@ You need a [TLS cert](../PREREQUISITES.md#tls-certificates) and a [test HTTP ser Create a `ingress.yaml` file. ```yaml -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: nginx-test @@ -22,19 +22,15 @@ spec: # This assumes tls-secret exists and the SSL # certificate contains a CN for foo.bar.com secretName: tls-secret - ingressClassName: nginx rules: - host: foo.bar.com http: paths: - path: / - pathType: Prefix backend: # This assumes http-svc exists and routes to healthy endpoints - service: - name: http-svc - port: - number: 80 + serviceName: http-svc + servicePort: 80 ``` The following command instructs the controller to terminate traffic using the provided @@ -65,10 +61,10 @@ Annotations: Events: FirstSeen LastSeen Count From SubObjectPath Type Reason Message --------- -------- ----- ---- ------------- -------- ------ ------- - 7s 7s 1 {ingress-nginx-controller } Normal CREATE default/nginx-test - 7s 7s 1 {ingress-nginx-controller } Normal UPDATE default/nginx-test - 7s 7s 1 {ingress-nginx-controller } Normal CREATE ip: 104.198.183.6 - 7s 7s 1 {ingress-nginx-controller } Warning MAPPING Ingress rule 'default/nginx-test' contains no path definition. Assuming / + 7s 7s 1 {nginx-ingress-controller } Normal CREATE default/nginx-test + 7s 7s 1 {nginx-ingress-controller } Normal UPDATE default/nginx-test + 7s 7s 1 {nginx-ingress-controller } Normal CREATE ip: 104.198.183.6 + 7s 7s 1 {nginx-ingress-controller } Warning MAPPING Ingress rule 'default/nginx-test' contains no path definition. Assuming / $ curl 104.198.183.6 -L curl: (60) SSL certificate problem: self signed certificate diff --git a/docs/examples/tls-termination/ingress.yaml b/docs/examples/tls-termination/ingress.yaml index 890247fd3..fc97b3707 100644 --- a/docs/examples/tls-termination/ingress.yaml +++ b/docs/examples/tls-termination/ingress.yaml @@ -1,9 +1,8 @@ -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: nginx-test spec: - ingressClassName: nginx tls: - hosts: - foo.bar.com @@ -15,10 +14,7 @@ spec: http: paths: - path: / - pathType: Prefix backend: # This assumes http-svc exists and routes to healthy endpoints - service: - name: http-svc - port: - number: 80 + serviceName: http-svc + servicePort: 80 diff --git a/docs/faq.md b/docs/faq.md deleted file mode 100644 index ea1e4d308..000000000 --- a/docs/faq.md +++ /dev/null @@ -1,200 +0,0 @@ - -# FAQ - -## Multi-tenant Kubernetes - -Do not use in multi-tenant Kubernetes production installations. This project assumes that users that can create Ingress objects are administrators of the cluster. - -For example, the Ingress NGINX control plane has global and per Ingress configuration options that make it insecure, if enabled, in a multi-tenant environment. - -For example, enabling snippets, a global configuration, allows any Ingress object to run arbitrary Lua code that could affect the security of all Ingress objects that a controller is running. - -We changed the default to allow snippets to `false` in https://github.com/kubernetes/ingress-nginx/pull/10393. - -## Multiple controller in one cluster - -Question - How can I easily install multiple instances of the ingress-nginx controller in the same cluster? - -You can install them in different namespaces. - -- Create a new namespace - - ``` - kubectl create namespace ingress-nginx-2 - ``` - -- Use Helm to install the additional instance of the ingress controller -- Ensure you have Helm working (refer to the [Helm documentation](https://helm.sh/docs/)) -- We have to assume that you have the helm repo for the ingress-nginx controller already added to your Helm config. - But, if you have not added the helm repo then you can do this to add the repo to your helm config; - - ``` - helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx - ``` - -- Make sure you have updated the helm repo data; - - ``` - helm repo update - ``` - -- Now, install an additional instance of the ingress-nginx controller like this: - - ``` - helm install ingress-nginx-2 ingress-nginx/ingress-nginx \ - --namespace ingress-nginx-2 \ - --set controller.ingressClassResource.name=nginx-two \ - --set controller.ingressClass=nginx-two \ - --set controller.ingressClassResource.controllerValue="example.com/ingress-nginx-2" \ - --set controller.ingressClassResource.enabled=true \ - --set controller.ingressClassByName=true - ``` - -If you need to install yet another instance, then repeat the procedure to create a new namespace, -change the values such as names & namespaces (for example from "-2" to "-3"), or anything else that meets your needs. - -Note that `controller.ingressClassResource.name` and `controller.ingressClass` have to be set correctly. -The first is to create the IngressClass object and the other is to modify the deployment of the actual ingress controller pod. - -### I can't use multiple namespaces, what should I do? - -If you need to install all instances in the same namespace, then you need to specify a different **election id**, like this: - -``` -helm install ingress-nginx-2 ingress-nginx/ingress-nginx \ ---namespace kube-system \ ---set controller.electionID=nginx-two-leader \ ---set controller.ingressClassResource.name=nginx-two \ ---set controller.ingressClass=nginx-two \ ---set controller.ingressClassResource.controllerValue="example.com/ingress-nginx-2" \ ---set controller.ingressClassResource.enabled=true \ ---set controller.ingressClassByName=true -``` - -## Retaining Client IPAddress - -Question - How to obtain the real-client-ipaddress ? - -The goto solution for retaining the real-client IPaddress is to enable PROXY protocol. - -Enabling PROXY protocol has to be done on both, the Ingress NGINX controller, as well as the L4 load balancer, in front of the controller. - -The real-client IP address is lost by default, when traffic is forwarded over the network. But enabling PROXY protocol ensures that the connection details are retained and hence the real-client IP address doesn't get lost. - -Enabling proxy-protocol on the controller is documented [here](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#use-proxy-protocol) . - -For enabling proxy-protocol on the LoadBalancer, please refer to the documentation of your infrastructure provider because that is where the LB is provisioned. - -Some more info available [here](https://kubernetes.github.io/ingress-nginx/user-guide/miscellaneous/#source-ip-address) - -Some more info on proxy-protocol is [here](https://kubernetes.github.io/ingress-nginx/user-guide/miscellaneous/#proxy-protocol) - -### client-ipaddress on single-node cluster - -Single node clusters are created for dev & test uses with tools like "kind" or "minikube". A trick to simulate a real use network with these clusters (kind or minikube) is to install Metallb and configure the ipaddress of the kind container or the minikube vm/container, as the starting and ending of the pool for Metallb in L2 mode. Then the host ip becomes a real client ipaddress, for curl requests sent from the host. - -After installing ingress-nginx controller on a kind or a minikube cluster with helm, you can configure it for real-client-ip with a simple change to the service that ingress-nginx controller creates. The service object of --type LoadBalancer has a field service.spec.externalTrafficPolicy. If you set the value of this field to "Local" then the real-ipaddress of a client is visible to the controller. - -``` -% kubectl explain service.spec.externalTrafficPolicy -KIND: Service -VERSION: v1 - -FIELD: externalTrafficPolicy - -DESCRIPTION: - externalTrafficPolicy describes how nodes distribute service traffic they - receive on one of the Service's "externally-facing" addresses (NodePorts, - ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will - configure the service in a way that assumes that external load balancers - will take care of balancing the service traffic between nodes, and so each - node will deliver traffic only to the node-local endpoints of the service, - without masquerading the client source IP. (Traffic mistakenly sent to a - node with no endpoints will be dropped.) The default value, "Cluster", uses - the standard behavior of routing to all endpoints evenly (possibly modified - by topology and other features). Note that traffic sent to an External IP or - LoadBalancer IP from within the cluster will always get "Cluster" semantics, - but clients sending to a NodePort from within the cluster may need to take - traffic policy into account when picking a node. - - Possible enum values: - - `"Cluster"` routes traffic to all endpoints. - - `"Local"` preserves the source IP of the traffic by routing only to - endpoints on the same node as the traffic was received on (dropping the - traffic if there are no local endpoints). -``` - -### client-ipaddress L7 - -The solution is to get the real client IPaddress from the ["X-Forward-For" HTTP header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For) - -Example : If your application pod behind Ingress NGINX controller, uses the NGINX webserver and the reverseproxy inside it, then you can do the following to preserve the remote client IP. - -- First you need to make sure that the X-Forwarded-For header reaches the backend pod. This is done by using a Ingress NGINX conftroller ConfigMap key. Its documented [here](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#use-forwarded-headers) - -- Next, edit `nginx.conf` file inside your app pod, to contain the directives shown below: - -``` -set_real_ip_from 0.0.0.0/0; # Trust all IPs (use your VPC CIDR block in production) -real_ip_header X-Forwarded-For; -real_ip_recursive on; - -log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" ' - 'host=$host x-forwarded-for=$http_x_forwarded_for'; - -access_log /var/log/nginx/access.log main; - -``` - -## Kubernetes v1.22 Migration - -If you are using Ingress objects in your cluster (running Kubernetes older than -version 1.22), and you plan to upgrade your Kubernetes version to K8S 1.22 or -above, then please read [the migration guide here](./user-guide/k8s-122-migration.md). - -## Validation Of **`path`** - -- For improving security and also following desired standards on Kubernetes API -spec, the next release, scheduled for v1.8.0, will include a new & optional -feature of validating the value for the key `ingress.spec.rules.http.paths.path`. - -- This behavior will be disabled by default on the 1.8.0 release and enabled by -default on the next breaking change release, set for 2.0.0. - -- When "`ingress.spec.rules.http.pathType=Exact`" or "`pathType=Prefix`", this -validation will limit the characters accepted on the field "`ingress.spec.rules.http.paths.path`", -to "`alphanumeric characters`", and "`/`", "`_`", "`-`". Also, in this case, -the path should start with "`/`". - -- When the ingress resource path contains other characters (like on rewrite -configurations), the pathType value should be "`ImplementationSpecific`". - -- API Spec on pathType is documented [here](https://kubernetes.io/docs/concepts/services-networking/ingress/#path-types) - -- When this option is enabled, the validation will happen on the Admission -Webhook. So if any new ingress object contains characters other than -alphanumeric characters, and, "`/`", "`_`", "`-`", in the `path` field, but -is not using `pathType` value as `ImplementationSpecific`, then the ingress -object will be denied admission. - -- The cluster admin should establish validation rules using mechanisms like -"`Open Policy Agent`", to validate that only authorized users can use -ImplementationSpecific pathType and that only the authorized characters can be -used. [The configmap value is here](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#strict-validate-path-type) - -- A complete example of an Openpolicyagent gatekeeper rule is available [here](https://kubernetes.github.io/ingress-nginx/examples/openpolicyagent/) - -- If you have any issues or concerns, please do one of the following: - - Open a GitHub issue - - Comment in our Dev Slack Channel - - Open a thread in our Google Group - -## Why is chunking not working since controller v1.10 ? - -- If your code is setting the HTTP header `"Transfer-Encoding: chunked"` and -the controller log messages show an error about duplicate header, it is -because of this change - -- More details are available in this issue diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 161803210..81d4ec2d6 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -1,6 +1,6 @@ # How it works -The objective of this document is to explain how the Ingress-NGINX controller works, in particular how the NGINX model is built and why we need one. +The objective of this document is to explain how the NGINX Ingress controller works, in particular how the NGINX model is built and why we need one. ## NGINX configuration @@ -10,7 +10,7 @@ The goal of this Ingress controller is the assembly of a configuration file (ngi Usually, a Kubernetes Controller utilizes the [synchronization loop pattern][1] to check if the desired state in the controller is updated or a change is required. To this purpose, we need to build a model using different objects from the cluster, in particular (in no special order) Ingresses, Services, Endpoints, Secrets, and Configmaps to generate a point in time configuration file that reflects the state of the cluster. -To get this object from the cluster, we use [Kubernetes Informers][2], in particular, `FilteredSharedInformer`. These informers allow reacting to change in using [callbacks][3] to individual changes when a new object is added, modified or removed. Unfortunately, there is no way to know if a particular change is going to affect the final configuration file. Therefore on every change, we have to rebuild a new model from scratch based on the state of cluster and compare it to the current model. If the new model equals to the current one, then we avoid generating a new NGINX configuration and triggering a reload. Otherwise, we check if the difference is only about Endpoints. If so we then send the new list of Endpoints to a Lua handler running inside Nginx using HTTP POST request and again avoid generating a new NGINX configuration and triggering a reload. If the difference between running and new model is about more than just Endpoints we create a new NGINX configuration based on the new model, replace the current model and trigger a reload. +To get this object from the cluster, we use [Kubernetes Informers][2], in particular, `FilteredSharedInformer`. This informers allows reacting to changes in using [callbacks][3] to individual changes when a new object is added, modified or removed. Unfortunately, there is no way to know if a particular change is going to affect the final configuration file. Therefore on every change, we have to rebuild a new model from scratch based on the state of cluster and compare it to the current model. If the new model equals to the current one, then we avoid generating a new NGINX configuration and triggering a reload. Otherwise, we check if the difference is only about Endpoints. If so we then send the new list of Endpoints to a Lua handler running inside Nginx using HTTP POST request and again avoid generating a new NGINX configuration and triggering a reload. If the difference between running and new model is about more than just Endpoints we create a new NGINX configuration based on the new model, replace the current model and trigger a reload. One of the uses of the model is to avoid unnecessary reloads when there's no change in the state and to detect conflicts in definitions. @@ -60,15 +60,15 @@ In a relatively big cluster with frequently deploying apps this feature saves si Because the ingress controller works using the [synchronization loop pattern](https://coreos.com/kubernetes/docs/latest/replication-controller.html#the-reconciliation-loop-in-detail), it is applying the configuration for all matching objects. In case some Ingress objects have a broken configuration, for example a syntax error in the `nginx.ingress.kubernetes.io/configuration-snippet` annotation, the generated configuration becomes invalid, does not reload and hence no more ingresses will be taken into account. -To prevent this situation to happen, the Ingress-Nginx Controller optionally exposes a [validating admission webhook server][8] to ensure the validity of incoming ingress objects. +To prevent this situation to happen, the nginx ingress controller optionally exposes a [validating admission webhook server][8] to ensure the validity of incoming ingress objects. This webhook appends the incoming ingress objects to the list of ingresses, generates the configuration and calls nginx to ensure the configuration has no syntax errors. [0]: https://github.com/openresty/lua-nginx-module/pull/1259 [1]: https://coreos.com/kubernetes/docs/latest/replication-controller.html#the-reconciliation-loop-in-detail [2]: https://godoc.org/k8s.io/client-go/informers#NewFilteredSharedInformerFactory [3]: https://godoc.org/k8s.io/client-go/tools/cache#ResourceEventHandlerFuncs -[4]: https://github.com/kubernetes/ingress-nginx/blob/main/internal/task/queue.go#L38 +[4]: https://github.com/kubernetes/ingress-nginx/blob/master/internal/task/queue.go#L38 [5]: https://golang.org/pkg/sync/#Mutex -[6]: https://github.com/kubernetes/ingress-nginx/blob/main/rootfs/etc/nginx/template/nginx.tmpl -[7]: https://nginx.org/en/docs/beginners_guide.html#control +[6]: https://github.com/kubernetes/ingress-nginx/blob/master/rootfs/etc/nginx/template/nginx.tmpl +[7]: http://nginx.org/en/docs/beginners_guide.html#control [8]: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#validatingadmissionwebhook diff --git a/docs/images/grafana-dashboard1.png b/docs/images/grafana-dashboard1.png deleted file mode 100644 index a4dfd81a7..000000000 Binary files a/docs/images/grafana-dashboard1.png and /dev/null differ diff --git a/docs/images/otel-grafana-demo.png b/docs/images/otel-grafana-demo.png deleted file mode 100644 index 61b50c861..000000000 Binary files a/docs/images/otel-grafana-demo.png and /dev/null differ diff --git a/docs/images/otel-jaeger-demo.png b/docs/images/otel-jaeger-demo.png deleted file mode 100644 index 006b23c34..000000000 Binary files a/docs/images/otel-jaeger-demo.png and /dev/null differ diff --git a/docs/images/otel-zipkin-demo.png b/docs/images/otel-zipkin-demo.png deleted file mode 100644 index a3af04e46..000000000 Binary files a/docs/images/otel-zipkin-demo.png and /dev/null differ diff --git a/docs/images/prometheus-dashboard1.png b/docs/images/prometheus-dashboard1.png deleted file mode 100644 index 86c76cbf2..000000000 Binary files a/docs/images/prometheus-dashboard1.png and /dev/null differ diff --git a/docs/index.md b/docs/index.md index bd6a825e1..8eeaaa080 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,12 +1,11 @@ -# Overview +# Welcome -This is the documentation for the Ingress NGINX Controller. +This is the documentation for the NGINX Ingress Controller. -It is built around the [Kubernetes Ingress resource](https://kubernetes.io/docs/concepts/services-networking/ingress/), using a [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) to store the controller configuration. +It is built around the [Kubernetes Ingress resource](http://kubernetes.io/docs/user-guide/ingress/), using a [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#understanding-configmaps-and-pods) to store the NGINX configuration. -You can learn more about using [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) in the official [Kubernetes documentation](https://docs.k8s.io). +Learn more about using Ingress on [k8s.io](http://kubernetes.io/docs/user-guide/ingress/). -# Getting Started +## Getting Started See [Deployment](./deploy/) for a whirlwind tour that will get you started. - diff --git a/docs/kubectl-plugin.md b/docs/kubectl-plugin.md index 9e5a5dcc6..801a187b7 100644 --- a/docs/kubectl-plugin.md +++ b/docs/kubectl-plugin.md @@ -1,7 +1,7 @@ @@ -68,7 +68,7 @@ Use "ingress-nginx [command] --help" for more information about a command. ## Common Flags - Every subcommand supports the basic `kubectl` configuration flags like `--namespace`, `--context`, `--client-key` and so on. -- Subcommands that act on a particular `ingress-nginx` pod (`backends`, `certs`, `conf`, `exec`, `general`, `logs`, `ssh`), support the `--deployment `, `--pod `, and `--container ` flags to select either a pod from a deployment with the given name, or a pod with the given name (and the given container name). The `--deployment` flag defaults to `ingress-nginx-controller`, and the `--container` flag defaults to `controller`. +- Subcommands that act on a particular `ingress-nginx` pod (`backends`, `certs`, `conf`, `exec`, `general`, `logs`, `ssh`), support the `--deployment ` and `--pod ` flags to select either a pod from a deployment with the given name, or a pod with the given name. The `--deployment` flag defaults to `nginx-ingress-controller`. - Subcommands that inspect resources (`ingresses`, `lint`) support the `--all-namespaces` flag, which causes them to inspect resources in every namespace. ## Subcommands @@ -208,7 +208,6 @@ modsecurity modules nginx.conf opentracing.json -opentelemetry.toml owasp-modsecurity-crs template ``` @@ -227,9 +226,7 @@ Use the `--service ` flag if your `ingress-nginx` `LoadBalancer` servic ### ingresses -`kubectl ingress-nginx ingresses`, alternately `kubectl ingress-nginx ing`, shows a more detailed view of the ingress definitions in a namespace. - -Compare: +`kubectl ingress-nginx ingresses`, alternately `kubectl ingress-nginx ing`, shows a more detailed view of the ingress definitions in a namespace. Compare: ```console $ kubectl get ingresses --all-namespaces @@ -238,7 +235,7 @@ default example-ingress1 testaddr.local,testaddr2.local localhost 80 default test-ingress-2 * localhost 80 5d ``` -vs. +vs ```console $ kubectl ingress-nginx ingresses --all-namespaces @@ -266,7 +263,7 @@ Checking ingresses... https://github.com/kubernetes/ingress-nginx/issues/3174 Checking deployments... -✗ namespace2/ingress-nginx-controller +✗ namespace2/nginx-ingress-controller - Uses removed config flag --sort-backends Lint added for version 0.22.0 https://github.com/kubernetes/ingress-nginx/issues/3655 @@ -275,7 +272,7 @@ Checking deployments... https://github.com/kubernetes/ingress-nginx/issues/3808 ``` -To show the lints added **only** for a particular `ingress-nginx` release, use the `--from-version` and `--to-version` flags: +to show the lints added **only** for a particular `ingress-nginx` release, use the `--from-version` and `--to-version` flags: ```console $ kubectl ingress-nginx lint --all-namespaces --verbose --from-version 0.24.0 --to-version 0.24.0 @@ -286,7 +283,7 @@ Checking ingresses... https://github.com/kubernetes/ingress-nginx/issues/3743 Checking deployments... -✗ namespace2/ingress-nginx-controller +✗ namespace2/nginx-ingress-controller - Uses removed config flag --enable-dynamic-certificates Lint added for version 0.24.0 https://github.com/kubernetes/ingress-nginx/issues/3808 @@ -321,5 +318,5 @@ I0405 16:53:46.193913 7 event.go:209] Event(v1.ObjectReference{Kind:"Confi ```console $ kubectl ingress-nginx ssh -n ingress-nginx -www-data@ingress-nginx-controller-7cbf77c976-wx5pn:/etc/nginx$ +www-data@nginx-ingress-controller-7cbf77c976-wx5pn:/etc/nginx$ ``` diff --git a/docs/lua_tests.md b/docs/lua_tests.md deleted file mode 100644 index 4d3d1fe70..000000000 --- a/docs/lua_tests.md +++ /dev/null @@ -1,19 +0,0 @@ -# Lua Tests - -## Running the Lua Tests - -To run the Lua tests you can run the following from the root directory: - -```bash -make lua-test -``` - -This command makes use of docker hence does not need any dependency -installations besides docker - -## Where are the Lua Tests? - -Lua Tests can be found in the [rootfs/etc/nginx/lua/test](../rootfs/etc/nginx/lua/test) directory - - -[1]: https://openresty.org/en/installation.html diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 5d5943b84..000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -mkdocs-material==9.4.5 -mkdocs-awesome-pages-plugin==2.9.2 -mkdocs-minify-plugin==0.7.1 -mkdocs-redirects==1.2.1 \ No newline at end of file diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index ffd8f839f..1707aa0e2 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1,7 +1,7 @@ @@ -13,7 +13,7 @@ Do not move it without providing redirects. There are many ways to troubleshoot the ingress-controller. The following are basic troubleshooting methods to obtain more information. -### Check the Ingress Resource Events +Check the Ingress Resource Events ```console $ kubectl get ing -n @@ -32,23 +32,23 @@ Rules: /tea tea-svc:80 () /coffee coffee-svc:80 () Annotations: - kubectl.kubernetes.io/last-applied-configuration: {"apiVersion":"networking.k8s.io/v1","kind":"Ingress","metadata":{"annotations":{},"name":"cafe-ingress","namespace":"default","selfLink":"/apis/networking/v1/namespaces/default/ingresses/cafe-ingress"},"spec":{"rules":[{"host":"cafe.com","http":{"paths":[{"backend":{"serviceName":"tea-svc","servicePort":80},"path":"/tea"},{"backend":{"serviceName":"coffee-svc","servicePort":80},"path":"/coffee"}]}}]},"status":{"loadBalancer":{"ingress":[{"ip":"169.48.142.110"}]}}} + kubectl.kubernetes.io/last-applied-configuration: {"apiVersion":"networking.k8s.io/v1beta1","kind":"Ingress","metadata":{"annotations":{},"name":"cafe-ingress","namespace":"default","selfLink":"/apis/networking/v1beta1/namespaces/default/ingresses/cafe-ingress"},"spec":{"rules":[{"host":"cafe.com","http":{"paths":[{"backend":{"serviceName":"tea-svc","servicePort":80},"path":"/tea"},{"backend":{"serviceName":"coffee-svc","servicePort":80},"path":"/coffee"}]}}]},"status":{"loadBalancer":{"ingress":[{"ip":"169.48.142.110"}]}}} Events: Type Reason Age From Message ---- ------ ---- ---- ------- - Normal CREATE 1m ingress-nginx-controller Ingress default/cafe-ingress - Normal UPDATE 58s ingress-nginx-controller Ingress default/cafe-ingress + Normal CREATE 1m nginx-ingress-controller Ingress default/cafe-ingress + Normal UPDATE 58s nginx-ingress-controller Ingress default/cafe-ingress ``` -### Check the Ingress Controller Logs +Check the Ingress Controller Logs ```console $ kubectl get pods -n NAME READY STATUS RESTARTS AGE -ingress-nginx-controller-67956bf89d-fv58j 1/1 Running 0 1m +nginx-ingress-controller-67956bf89d-fv58j 1/1 Running 0 1m -$ kubectl logs -n ingress-nginx-controller-67956bf89d-fv58j +$ kubectl logs -n nginx-ingress-controller-67956bf89d-fv58j ------------------------------------------------------------------------------- NGINX Ingress controller Release: 0.14.0 @@ -58,14 +58,14 @@ NGINX Ingress controller .... ``` -### Check the Nginx Configuration +Check the Nginx Configuration ```console $ kubectl get pods -n NAME READY STATUS RESTARTS AGE -ingress-nginx-controller-67956bf89d-fv58j 1/1 Running 0 1m +nginx-ingress-controller-67956bf89d-fv58j 1/1 Running 0 1m -$ kubectl exec -it -n ingress-nginx-controller-67956bf89d-fv58j -- cat /etc/nginx/nginx.conf +$ kubectl exec -it -n nginx-ingress-controller-67956bf89d-fv58j -- cat /etc/nginx/nginx.conf daemon off; worker_processes 2; pid /run/nginx.pid; @@ -80,7 +80,7 @@ http { .... ``` -### Check if used Services Exist +Check if used Services Exist ```console $ kubectl get svc --all-namespaces @@ -102,15 +102,15 @@ the deployment. $ kubectl get deploy -n NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE default-http-backend 1 1 1 1 35m -ingress-nginx-controller 1 1 1 1 35m +nginx-ingress-controller 1 1 1 1 35m -$ kubectl edit deploy -n ingress-nginx-controller +$ kubectl edit deploy -n nginx-ingress-controller # Add --v=X to "- args", where X is an integer ``` - `--v=2` shows details using `diff` about the changes in the configuration in nginx - `--v=3` shows details about the service, Ingress rule, endpoint changes and it dumps the nginx configuration in JSON format -- `--v=5` configures NGINX in [debug mode](https://nginx.org/en/docs/debugging_log.html) +- `--v=5` configures NGINX in [debug mode](http://nginx.org/en/docs/debugging_log.html) ## Authentication to the Kubernetes API Server @@ -130,14 +130,14 @@ Both authentications must work: **Service authentication** -The Ingress controller needs information from apiserver. Therefore, authentication is required, which can be achieved in a couple of ways: +The Ingress controller needs information from apiserver. Therefore, authentication is required, which can be achieved in two different ways: -* _Service Account:_ This is recommended, because nothing has to be configured. The Ingress controller will use information provided by the system to communicate with the API server. See 'Service Account' section for details. +1. _Service Account:_ This is recommended, because nothing has to be configured. The Ingress controller will use information provided by the system to communicate with the API server. See 'Service Account' section for details. -* _Kubeconfig file:_ In some Kubernetes environments service accounts are not available. In this case a manual configuration is required. The Ingress controller binary can be started with the `--kubeconfig` flag. The value of the flag is a path to a file specifying how to connect to the API server. Using the `--kubeconfig` does not requires the flag `--apiserver-host`. +2. _Kubeconfig file:_ In some Kubernetes environments service accounts are not available. In this case a manual configuration is required. The Ingress controller binary can be started with the `--kubeconfig` flag. The value of the flag is a path to a file specifying how to connect to the API server. Using the `--kubeconfig` does not requires the flag `--apiserver-host`. The format of the file is identical to `~/.kube/config` which is used by kubectl to connect to the API server. See 'kubeconfig' section for details. -* _Using the flag `--apiserver-host`:_ Using this flag `--apiserver-host=http://localhost:8080` it is possible to specify an unsecured API server or reach a remote kubernetes cluster using [kubectl proxy](https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#proxy). +3. _Using the flag `--apiserver-host`:_ Using this flag `--apiserver-host=http://localhost:8080` it is possible to specify an unsecured API server or reach a remote kubernetes cluster using [kubectl proxy](https://kubernetes.io/docs/user-guide/kubectl/kubectl_proxy/). Please do not use this approach in production. In the diagram below you can see the full authentication flow with all options, starting with the browser @@ -173,47 +173,68 @@ Verify with the following commands: ```console # start a container that contains curl -$ kubectl run -it --rm test --image=curlimages/curl --restart=Never -- /bin/sh +$ kubectl run test --image=tutum/curl -- sleep 10000 + +# check that container is running +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +test-701078429-s5kca 1/1 Running 0 16s # check if secret exists -/ $ ls /var/run/secrets/kubernetes.io/serviceaccount/ -ca.crt namespace token -/ $ +$ kubectl exec test-701078429-s5kca -- ls /var/run/secrets/kubernetes.io/serviceaccount/ +ca.crt +namespace +token + +# get service IP of master +$ kubectl get services +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +kubernetes 10.0.0.1 443/TCP 1d # check base connectivity from cluster inside -/ $ curl -k https://kubernetes.default.svc.cluster.local -{ - "kind": "Status", - "apiVersion": "v1", - "metadata": { - - }, - "status": "Failure", - "message": "forbidden: User \"system:anonymous\" cannot get path \"/\"", - "reason": "Forbidden", - "details": { - - }, - "code": 403 -}/ $ +$ kubectl exec test-701078429-s5kca -- curl -k https://10.0.0.1 +Unauthorized # connect using tokens -}/ $ curl --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" https://kubernetes.default.svc.cluster.local -&& echo +$ TOKEN_VALUE=$(kubectl exec test-701078429-s5kca -- cat /var/run/secrets/kubernetes.io/serviceaccount/token) +$ echo $TOKEN_VALUE +eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3Mi....9A +$ kubectl exec test-701078429-s5kca -- curl --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt -H "Authorization: Bearer $TOKEN_VALUE" https://10.0.0.1 { "paths": [ "/api", "/api/v1", "/apis", - "/apis/", - ... TRUNCATED - "/readyz/shutdown", + "/apis/apps", + "/apis/apps/v1alpha1", + "/apis/authentication.k8s.io", + "/apis/authentication.k8s.io/v1beta1", + "/apis/authorization.k8s.io", + "/apis/authorization.k8s.io/v1beta1", + "/apis/autoscaling", + "/apis/autoscaling/v1", + "/apis/batch", + "/apis/batch/v1", + "/apis/batch/v2alpha1", + "/apis/certificates.k8s.io", + "/apis/certificates.k8s.io/v1alpha1", + "/apis/networking", + "/apis/networking/v1beta1", + "/apis/policy", + "/apis/policy/v1alpha1", + "/apis/rbac.authorization.k8s.io", + "/apis/rbac.authorization.k8s.io/v1alpha1", + "/apis/storage.k8s.io", + "/apis/storage.k8s.io/v1beta1", + "/healthz", + "/healthz/ping", + "/logs", + "/metrics", + "/swaggerapi/", + "/ui/", "/version" ] } -/ $ - -# when you type `exit` or `^D` the test pod will be deleted. ``` If it is not working, there are two possible reasons: @@ -230,7 +251,7 @@ If it is not working, there are two possible reasons: More information: -- [User Guide: Service Accounts](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) +- [User Guide: Service Accounts](http://kubernetes.io/docs/user-guide/service-accounts/) - [Cluster Administrator Guide: Managing Service Accounts](http://kubernetes.io/docs/admin/service-accounts-admin/) ## Kube-Config @@ -247,235 +268,72 @@ Note: The below is based on the nginx [documentation](https://docs.nginx.com/ngi 1. SSH into the worker - ```console - $ ssh user@workerIP - ``` +```console +$ ssh user@workerIP +``` 2. Obtain the Docker Container Running nginx - ```console - $ docker ps | grep ingress-nginx-controller - CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - d9e1d243156a registry.k8s.io/ingress-nginx/controller "/usr/bin/dumb-init …" 19 minutes ago Up 19 minutes k8s_ingress-nginx-controller_ingress-nginx-controller-67956bf89d-mqxzt_kube-system_079f31ec-aa37-11e8-ad39-080027a227db_0 - ``` +```console +$ docker ps | grep nginx-ingress-controller +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +d9e1d243156a quay.io/kubernetes-ingress-controller/nginx-ingress-controller "/usr/bin/dumb-init …" 19 minutes ago Up 19 minutes k8s_nginx-ingress-controller_nginx-ingress-controller-67956bf89d-mqxzt_kube-system_079f31ec-aa37-11e8-ad39-080027a227db_0 +``` 3. Exec into the container - ```console - $ docker exec -it --user=0 --privileged d9e1d243156a bash - ``` +```console +$ docker exec -it --user=0 --privileged d9e1d243156a bash +``` 4. Make sure nginx is running in `--with-debug` - ```console - $ nginx -V 2>&1 | grep -- '--with-debug' - ``` +```console +$ nginx -V 2>&1 | grep -- '--with-debug' +``` 5. Get list of processes running on container - ```console - $ ps -ef - UID PID PPID C STIME TTY TIME CMD - root 1 0 0 20:23 ? 00:00:00 /usr/bin/dumb-init /nginx-ingres - root 5 1 0 20:23 ? 00:00:05 /ingress-nginx-controller --defa - root 21 5 0 20:23 ? 00:00:00 nginx: master process /usr/sbin/ - nobody 106 21 0 20:23 ? 00:00:00 nginx: worker process - nobody 107 21 0 20:23 ? 00:00:00 nginx: worker process - root 172 0 0 20:43 pts/0 00:00:00 bash - ``` - -6. Attach gdb to the nginx master process - - ```console - $ gdb -p 21 - .... - Attaching to process 21 - Reading symbols from /usr/sbin/nginx...done. - .... - (gdb) - ``` - -7. Copy and paste the following: - - ```console - set $cd = ngx_cycle->config_dump - set $nelts = $cd.nelts - set $elts = (ngx_conf_dump_t*)($cd.elts) - while ($nelts-- > 0) - set $name = $elts[$nelts]->name.data - printf "Dumping %s to nginx_conf.txt\n", $name - append memory nginx_conf.txt \ - $elts[$nelts]->buffer.start $elts[$nelts]->buffer.end - end - ``` - -8. Quit GDB by pressing CTRL+D - -9. Open nginx_conf.txt - - ```console - cat nginx_conf.txt - ``` - -## Image related issues faced on Nginx 4.2.5 or other versions (Helm chart versions) - -1. Incase you face below error while installing Nginx using helm chart (either by helm commands or helm_release terraform provider ) -``` -Warning Failed 5m5s (x4 over 6m34s) kubelet Failed to pull image "registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.3.0@sha256:549e71a6ca248c5abd51cdb73dbc3083df62cf92ed5e6147c780e30f7e007a47": rpc error: code = Unknown desc = failed to pull and unpack image "registry.k8s.io/ingress-nginx/kube-webhook-certgen@sha256:549e71a6ca248c5abd51cdb73dbc3083df62cf92ed5e6147c780e30f7e007a47": failed to resolve reference "registry.k8s.io/ingress-nginx/kube-webhook-certgen@sha256:549e71a6ca248c5abd51cdb73dbc3083df62cf92ed5e6147c780e30f7e007a47": failed to do request: Head "https://eu.gcr.io/v2/k8s-artifacts-prod/ingress-nginx/kube-webhook-certgen/manifests/sha256:549e71a6ca248c5abd51cdb73dbc3083df62cf92ed5e6147c780e30f7e007a47": EOF -``` - Then please follow the below steps. - -2. During troubleshooting you can also execute the below commands to test the connectivities from you local machines and repositories details - - a. curl registry.k8s.io/ingress-nginx/kube-webhook-certgen@sha256:549e71a6ca248c5abd51cdb73dbc3083df62cf92ed5e6147c780e30f7e007a47 > /dev/null - ``` - (⎈ |myprompt)➜ ~ curl registry.k8s.io/ingress-nginx/kube-webhook-certgen@sha256:549e71a6ca248c5abd51cdb73dbc3083df62cf92ed5e6147c780e30f7e007a47 > /dev/null - % Total % Received % Xferd Average Speed Time Time Time Current - Dload Upload Total Spent Left Speed - 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 - (⎈ |myprompt)➜ ~ - ``` - b. curl -I https://eu.gcr.io/v2/k8s-artifacts-prod/ingress-nginx/kube-webhook-certgen/manifests/sha256:549e71a6ca248c5abd51cdb73dbc3083df62cf92ed5e6147c780e30f7e007a47 - ``` - (⎈ |myprompt)➜ ~ curl -I https://eu.gcr.io/v2/k8s-artifacts-prod/ingress-nginx/kube-webhook-certgen/manifests/sha256:549e71a6ca248c5abd51cdb73dbc3083df62cf92ed5e6147c780e30f7e007a47 - HTTP/2 200 - docker-distribution-api-version: registry/2.0 - content-type: application/vnd.docker.distribution.manifest.list.v2+json - docker-content-digest: sha256:549e71a6ca248c5abd51cdb73dbc3083df62cf92ed5e6147c780e30f7e007a47 - content-length: 1384 - date: Wed, 28 Sep 2022 16:46:28 GMT - server: Docker Registry - x-xss-protection: 0 - x-frame-options: SAMEORIGIN - alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43" - - (⎈ |myprompt)➜ ~ - ``` - Redirection in the proxy is implemented to ensure the pulling of the images. - -3. This is the solution recommended to whitelist the below image repositories : - ``` - *.appspot.com - *.k8s.io - *.pkg.dev - *.gcr.io - - ``` - More details about the above repos : - a. *.k8s.io -> To ensure you can pull any images from registry.k8s.io - b. *.gcr.io -> GCP services are used for image hosting. This is part of the domains suggested by GCP to allow and ensure users can pull images from their container registry services. - c. *.appspot.com -> This a Google domain. part of the domain used for GCR. - -## Unable to listen on port (80/443) -One possible reason for this error is lack of permission to bind to the port. Ports 80, 443, and any other port < 1024 are Linux privileged ports which historically could only be bound by root. The ingress-nginx-controller uses the CAP_NET_BIND_SERVICE [linux capability](https://man7.org/linux/man-pages/man7/capabilities.7.html) to allow binding these ports as a normal user (www-data / 101). This involves two components: -1. In the image, the /nginx-ingress-controller file has the cap_net_bind_service capability added (e.g. via [setcap](https://man7.org/linux/man-pages/man8/setcap.8.html)) -2. The NET_BIND_SERVICE capability is added to the container in the containerSecurityContext of the deployment. - -If encountering this on one/some node(s) and not on others, try to purge and pull a fresh copy of the image to the affected node(s), in case there has been corruption of the underlying layers to lose the capability on the executable. - -### Create a test pod -The /nginx-ingress-controller process exits/crashes when encountering this error, making it difficult to troubleshoot what is happening inside the container. To get around this, start an equivalent container running "sleep 3600", and exec into it for further troubleshooting. For example: -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: ingress-nginx-sleep - namespace: default - labels: - app: nginx -spec: - containers: - - name: nginx - image: ##_CONTROLLER_IMAGE_## - resources: - requests: - memory: "512Mi" - cpu: "500m" - limits: - memory: "1Gi" - cpu: "1" - command: ["sleep"] - args: ["3600"] - ports: - - containerPort: 80 - name: http - protocol: TCP - - containerPort: 443 - name: https - protocol: TCP - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - runAsUser: 101 - restartPolicy: Never - nodeSelector: - kubernetes.io/hostname: ##_NODE_NAME_## - tolerations: - - key: "node.kubernetes.io/unschedulable" - operator: "Exists" - effect: NoSchedule -``` -* update the namespace if applicable/desired -* replace `##_NODE_NAME_##` with the problematic node (or remove nodeSelector section if problem is not confined to one node) -* replace `##_CONTROLLER_IMAGE_##` with the same image as in use by your ingress-nginx deployment -* confirm the securityContext section matches what is in place for ingress-nginx-controller pods in your cluster - -Apply the YAML and open a shell into the pod. -Try to manually run the controller process: ```console -$ /nginx-ingress-controller +$ ps -ef +UID PID PPID C STIME TTY TIME CMD +root 1 0 0 20:23 ? 00:00:00 /usr/bin/dumb-init /nginx-ingres +root 5 1 0 20:23 ? 00:00:05 /nginx-ingress-controller --defa +root 21 5 0 20:23 ? 00:00:00 nginx: master process /usr/sbin/ +nobody 106 21 0 20:23 ? 00:00:00 nginx: worker process +nobody 107 21 0 20:23 ? 00:00:00 nginx: worker process +root 172 0 0 20:43 pts/0 00:00:00 bash ``` -You should get the same error as from the ingress controller pod logs. -Confirm the capabilities are properly surfacing into the pod: +7. Attach gdb to the nginx master process + ```console -$ grep CapBnd /proc/1/status -CapBnd: 0000000000000400 +$ gdb -p 21 +.... +Attaching to process 21 +Reading symbols from /usr/sbin/nginx...done. +.... +(gdb) ``` -The above value has only net_bind_service enabled (per security context in YAML which adds that and drops all). If you get a different value, then you can decode it on another linux box (capsh not available in this container) like below, and then figure out why specified capabilities are not propagating into the pod/container. + +8. Copy and paste the following: + ```console -$ capsh --decode=0000000000000400 -0x0000000000000400=cap_net_bind_service +set $cd = ngx_cycle->config_dump +set $nelts = $cd.nelts +set $elts = (ngx_conf_dump_t*)($cd.elts) +while ($nelts-- > 0) +set $name = $elts[$nelts]->name.data +printf "Dumping %s to nginx_conf.txt\n", $name +append memory nginx_conf.txt \ + $elts[$nelts]->buffer.start $elts[$nelts]->buffer.end +end ``` -## Create a test pod as root -(Note, this may be restricted by PodSecurityAdmission/Standards, OPA Gatekeeper, etc. in which case you will need to do the appropriate workaround for testing, e.g. deploy in a new namespace without the restrictions.) -To test further you may want to install additional utilities, etc. Modify the pod yaml by: -* changing runAsUser from 101 to 0 -* removing the "drop..ALL" section from the capabilities. +9. Quit GDB by pressing CTRL+D -Some things to try after shelling into this container: +10. Open nginx_conf.txt -Try running the controller as the www-data (101) user: ```console -$ chmod 4755 /nginx-ingress-controller -$ /nginx-ingress-controller -``` -Examine the errors to see if there is still an issue listening on the port or if it passed that and moved on to other expected errors due to running out of context. - -Install the libcap package and check capabilities on the file: -```console -$ apk add libcap -(1/1) Installing libcap (2.50-r0) -Executing busybox-1.33.1-r7.trigger -OK: 26 MiB in 41 packages -$ getcap /nginx-ingress-controller -/nginx-ingress-controller cap_net_bind_service=ep -``` -(if missing, see above about purging image on the server and re-pulling) - -Strace the executable to see what system calls are being executed when it fails: -```console -$ apk add strace -(1/1) Installing strace (5.12-r0) -Executing busybox-1.33.1-r7.trigger -OK: 28 MiB in 42 packages -$ strace /nginx-ingress-controller -execve("/nginx-ingress-controller", ["/nginx-ingress-controller"], 0x7ffeb9eb3240 /* 131 vars */) = 0 -arch_prctl(ARCH_SET_FS, 0x29ea690) = 0 -... +cat nginx_conf.txt ``` diff --git a/docs/user-guide/basic-usage.md b/docs/user-guide/basic-usage.md index aee0c0fd3..82019d978 100644 --- a/docs/user-guide/basic-usage.md +++ b/docs/user-guide/basic-usage.md @@ -1,20 +1,18 @@ # Basic usage - host based routing -ingress-nginx can be used for many use cases, inside various cloud providers and supports a lot of configurations. In this section you can find a common usage scenario where a single load balancer powered by ingress-nginx will route traffic to 2 different HTTP backend services based on the host name. +ingress-nginx can be used for many use cases, inside various cloud provider and supports a lot of configurations. In this section you can find a common usage scenario where a single load balancer powered by ingress-nginx will route traffic to 2 different HTTP backend services based on the host name. -First of all follow the instructions to install ingress-nginx. Then imagine that you need to expose 2 HTTP services already installed, `myServiceA`, `myServiceB`, and configured as `type: ClusterIP`. - -Let's say that you want to expose the first at `myServiceA.foo.org` and the second at `myServiceB.foo.org`. - -If the cluster version is < 1.19, you can create two **ingress** resources like this: +First of all follow the instructions to install ingress-nginx. Then imagine that you need to expose 2 HTTP services already installed: `myServiceA`, `myServiceB`. Let's say that you want to expose the first at `myServiceA.foo.org` and the second at `myServiceB.foo.org`. One possible solution is to create two **ingress** resources: ``` apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: ingress-myservicea + annotations: + # use the shared ingress-nginx + kubernetes.io/ingress.class: "nginx" spec: - ingressClassName: nginx rules: - host: myservicea.foo.org http: @@ -42,53 +40,11 @@ spec: servicePort: 80 ``` -If the cluster uses Kubernetes version >= 1.19.x, then its suggested to create 2 ingress resources, using yaml examples shown below. These examples are in conformity with the `networking.kubernetes.io/v1` api. - -``` -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ingress-myservicea -spec: - rules: - - host: myservicea.foo.org - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: myservicea - port: - number: 80 - ingressClassName: nginx ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ingress-myserviceb -spec: - rules: - - host: myserviceb.foo.org - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: myserviceb - port: - number: 80 - ingressClassName: nginx -``` - -When you apply this yaml, 2 ingress resources will be created managed by the **ingress-nginx** instance. Nginx is configured to automatically discover all ingress with the `kubernetes.io/ingress.class: "nginx"` annotation or where `ingressClassName: nginx` is present. +When you apply this yaml, 2 ingress resources will be created managed by the **ingress-nginx** instance. Nginx is configured to automatically discover all ingress with the `kubernetes.io/ingress.class: "nginx"` annotation. Please note that the ingress resource should be placed inside the same namespace of the backend resource. -On many cloud providers ingress-nginx will also create the corresponding Load Balancer resource. All you have to do is get the external IP and add a DNS `A record` inside your DNS provider that point myservicea.foo.org and myserviceb.foo.org to the nginx external IP. Get the external IP by running: +On many cloud providers ingress-nginx will also create the corresponding Load Balancer resource. All you have to do is get the external IP and add a DNS `A record` inside your DNS provider that point myServiceA.foo.org and myServiceB.foo.org to the nginx external IP. Get the external IP by running: ``` kubectl get services -n ingress-nginx ``` - -To test inside minikube refer to this documentation: [Set up Ingress on Minikube with the NGINX Ingress Controller](https://kubernetes.io/docs/tasks/access-application-cluster/ingress-minikube/) diff --git a/docs/user-guide/cli-arguments.md b/docs/user-guide/cli-arguments.md index a33e75159..799ce7324 100644 --- a/docs/user-guide/cli-arguments.md +++ b/docs/user-guide/cli-arguments.md @@ -2,79 +2,62 @@ The following command line arguments are accepted by the Ingress controller executable. -They are set in the container spec of the `ingress-nginx-controller` Deployment manifest +They are set in the container spec of the `nginx-ingress-controller` Deployment manifest | Argument | Description | |----------|-------------| +| `--add_dir_header` | If true, adds the file directory to the header | +| `--alsologtostderr` | log to standard error as well as files | | `--annotations-prefix` | Prefix of the Ingress annotations specific to the NGINX controller. (default "nginx.ingress.kubernetes.io") | | `--apiserver-host` | Address of the Kubernetes API server. Takes the form "protocol://address:port". If not specified, it is assumed the program runs inside a Kubernetes cluster and local discovery is attempted. | -| `--bucket-factor` | Bucket factor for native histograms. Value must be > 1 for enabling native histograms. (default 0) | | `--certificate-authority` | Path to a cert file for the certificate authority. This certificate is used only when the flag --apiserver-host is specified. | | `--configmap` | Name of the ConfigMap containing custom global configurations for the controller. | -| `--controller-class` | Ingress Class Controller value this Ingress satisfies. The class of an Ingress object is set using the field IngressClassName in Kubernetes clusters version v1.19.0 or higher. The .spec.controller value of the IngressClass referenced in an Ingress Object should be the same value specified here to make this object be watched. | -| `--deep-inspect` | Enables ingress object security deep inspector. (default true) | | `--default-backend-service` | Service used to serve HTTP requests not matching any known server name (catch-all). Takes the form "namespace/name". The controller configures NGINX to forward requests to the first port of this Service. | | `--default-server-port` | Port to use for exposing the default server (catch-all). (default 8181) | | `--default-ssl-certificate` | Secret containing a SSL certificate to be used by the default HTTPS server (catch-all). Takes the form "namespace/name". | -| `--enable-annotation-validation` | If true, will enable the annotation validation feature. Defaults to true | -| `--disable-catch-all` | Disable support for catch-all Ingresses. (default false) | -| `--disable-full-test` | Disable full test of all merged ingresses at the admission stage and tests the template of the ingress being created or updated (full test of all ingresses is enabled by default). | -| `--disable-svc-external-name` | Disable support for Services of type ExternalName. (default false) | -| `--disable-sync-events` | Disables the creation of 'Sync' Event resources, but still logs them | -| `--dynamic-configuration-retries` | Number of times to retry failed dynamic configuration before failing to sync an ingress. (default 15) | +| `--disable-catch-all` | Disable support for catch-all Ingresses | | `--election-id` | Election id to use for Ingress status updates. (default "ingress-controller-leader") | -| `--election-ttl` | Duration a leader election is valid before it's getting re-elected, e.g. `15s`, `10m` or `1h`. (Default: 30s) | -| `--enable-metrics` | Enables the collection of NGINX metrics. (Default: false) | -| `--enable-ssl-chain-completion` | Autocomplete SSL certificate chains with missing intermediate CA certificates. Certificates uploaded to Kubernetes must have the "Authority Information Access" X.509 v3 extension for this to succeed. (default false)| -| `--enable-ssl-passthrough` | Enable SSL Passthrough. (default false) | -| `--disable-leader-election` | Disable Leader Election on Nginx Controller. (default false) | -| `--enable-topology-aware-routing` | Enable topology aware routing feature, needs service object annotation service.kubernetes.io/topology-mode sets to auto. (default false) | -| `--exclude-socket-metrics` | Set of socket request metrics to exclude which won't be exported nor being calculated. The possible socket request metrics to exclude are documented in the monitoring guide e.g. 'nginx_ingress_controller_request_duration_seconds,nginx_ingress_controller_response_size'| +| `--enable-metrics` | Enables the collection of NGINX metrics (default true) | +| `--enable-ssl-chain-completion` | Autocomplete SSL certificate chains with missing intermediate CA certificates. Certificates uploaded to Kubernetes must have the "Authority Information Access" X.509 v3 extension for this to succeed. | +| `--enable-ssl-passthrough` | Enable SSL Passthrough. | | `--health-check-path` | URL path of the health check endpoint. Configured inside the NGINX status server. All requests received on the port defined by the healthz-port parameter are forwarded internally to this path. (default "/healthz") | | `--health-check-timeout` | Time limit, in seconds, for a probe to health-check-path to succeed. (default 10) | | `--healthz-port` | Port to use for the healthz endpoint. (default 10254) | -| `--healthz-host` | Address to bind the healthz endpoint. | | `--http-port` | Port to use for servicing HTTP traffic. (default 80) | | `--https-port` | Port to use for servicing HTTPS traffic. (default 443) | | `--ingress-class` | Name of the ingress class this controller satisfies. The class of an Ingress object is set using the field IngressClassName in Kubernetes clusters version v1.18.0 or higher or the annotation "kubernetes.io/ingress.class" (deprecated). If this parameter is not set, or set to the default value of "nginx", it will handle ingresses with either an empty or "nginx" class name. | -| `--ingress-class-by-name` | Define if Ingress Controller should watch for Ingress Class by Name together with Controller Class. (default false). | -| `--internal-logger-address` | Address to be used when binding internal syslogger. (default 127.0.0.1:11514) | | `--kubeconfig` | Path to a kubeconfig file containing authorization and API server information. | -| `--length-buckets` | Set of buckets which will be used for prometheus histogram metrics such as RequestLength, ResponseLength. (default `[10, 20, 30, 40, 50, 60, 70, 80, 90, 100]`) | -| `--max-buckets` | Maximum number of buckets for native histograms. (default 100) | +| `--log_backtrace_at` | when logging hits line file:N, emit a stack trace (default :0) | +| `--log_dir` | If non-empty, write log files in this directory | +| `--log_file` | If non-empty, use this log file | +| `--log_file_max_size` | Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) | +| `--logtostderr` | log to standard error instead of files (default true) | | `--maxmind-edition-ids` | Maxmind edition ids to download GeoLite2 Databases. (default "GeoLite2-City,GeoLite2-ASN") | -| `--maxmind-retries-timeout` | Maxmind downloading delay between 1st and 2nd attempt, 0s - do not retry to download if something went wrong. (default 0s) | -| `--maxmind-retries-count` | Number of attempts to download the GeoIP DB. (default 1) | -| `--maxmind-license-key` | Maxmind license key to download GeoLite2 Databases. https://blog.maxmind.com/2019/12/significant-changes-to-accessing-and-using-geolite2-databases/ . | -| `--maxmind-mirror` | Maxmind mirror url (example: http://geoip.local/databases. | -| `--metrics-per-host` | Export metrics per-host. (default true) | -| `--metrics-per-undefined-host` | Export metrics per-host even if the host is not defined in an ingress. Requires --metrics-per-host to be set to true. (default false) | -| `--monitor-max-batch-size` | Max batch size of NGINX metrics. (default 10000)| -| `--post-shutdown-grace-period` | Additional delay in seconds before controller container exits. (default 10) | +| `--maxmind-license-key` | Maxmind license key to download GeoLite2 Databases. https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases | +| `--metrics-per-host` | Export metrics per-host (default true) | | `--profiler-port` | Port to use for expose the ingress controller Go profiler when it is enabled. (default 10245) | -| `--profiling` | Enable profiling via web interface host:port/debug/pprof/ . (default true) | +| `--profiling` | Enable profiling via web interface host:port/debug/pprof/ (default true) | | `--publish-service` | Service fronting the Ingress controller. Takes the form "namespace/name". When used together with update-status, the controller mirrors the address of this service's endpoints to the load-balancer status of all Ingress objects it satisfies. | | `--publish-status-address` | Customized address (or addresses, separated by comma) to set as the load-balancer status of Ingress objects this controller satisfies. Requires the update-status parameter. | -| `--report-node-internal-ip-address`| Set the load-balancer status of Ingress objects to internal Node addresses instead of external. Requires the update-status parameter. (default false) | -| `--report-status-classes` | If true, report status classes in metrics (2xx, 3xx, 4xx and 5xx) instead of full status codes. (default false) | +| `--report-node-internal-ip-address`| Set the load-balancer status of Ingress objects to internal Node addresses instead of external. Requires the update-status parameter. | +| `--skip_headers` | If true, avoid header prefixes in the log messages | +| `--skip_log_headers` | If true, avoid headers when opening log files | | `--ssl-passthrough-proxy-port` | Port to use internally for SSL Passthrough. (default 442) | | `--status-port` | Port to use for the lua HTTP endpoint configuration. (default 10246) | -| `--status-update-interval` | Time interval in seconds in which the status should check if an update is required. Default is 60 seconds. (default 60) | +| `--status-update-interval` | Time interval in seconds in which the status should check if an update is required. Default is 60 seconds (default 60) | +| `--stderrthreshold` | logs at or above this threshold go to stderr (default 2) | | `--stream-port` | Port to use for the lua TCP/UDP endpoint configuration. (default 10247) | | `--sync-period` | Period at which the controller forces the repopulation of its local object stores. Disabled by default. | -| `--sync-rate-limit` | Define the sync frequency upper limit. (default 0.3) | +| `--sync-rate-limit` | Define the sync frequency upper limit (default 0.3) | | `--tcp-services-configmap` | Name of the ConfigMap containing the definition of the TCP services to expose. The key in the map indicates the external port to be used. The value is a reference to a Service in the form "namespace/name:port", where "port" can either be a port number or name. TCP ports 80 and 443 are reserved by the controller for servicing HTTP traffic. | -| `--time-buckets` | Set of buckets which will be used for prometheus histogram metrics such as RequestTime, ResponseTime. (default `[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]`) | | `--udp-services-configmap` | Name of the ConfigMap containing the definition of the UDP services to expose. The key in the map indicates the external port to be used. The value is a reference to a Service in the form "namespace/name:port", where "port" can either be a port name or number. | | `--update-status` | Update the load-balancer status of Ingress objects this controller satisfies. Requires setting the publish-service parameter to a valid Service reference. (default true) | | `--update-status-on-shutdown` | Update the load-balancer status of Ingress objects when the controller shuts down. Requires the update-status parameter. (default true) | -| `--shutdown-grace-period` | Seconds to wait after receiving the shutdown signal, before stopping the nginx process. (default 0) | -| `--size-buckets` | Set of buckets which will be used for prometheus histogram metrics such as BytesSent. (default `[10, 100, 1000, 10000, 100000, 1e+06, 1e+07]`) | +| `--shutdown-grace-period` | Seconds to wait after receiving the shutdown signal, before stopping the nginx process. | | `-v, --v Level` | number for the log level verbosity | | `--validating-webhook` | The address to start an admission controller on to validate incoming ingresses. Takes the form ":port". If not provided, no admission controller is started. | | `--validating-webhook-certificate` | The path of the validating webhook certificate PEM. | | `--validating-webhook-key` | The path of the validating webhook key PEM. | -| `--version` | Show release information about the Ingress-Nginx Controller and exit. | -| `--watch-ingress-without-class` | Define if Ingress Controller should also watch for Ingresses without an IngressClass or the annotation specified. (default false) | +| `--version` | Show release information about the NGINX Ingress controller and exit. | +| `--vmodule` | comma-separated list of pattern=N settings for file-filtered logging | | `--watch-namespace` | Namespace the controller watches for updates to Kubernetes objects. This includes Ingresses, Services and all configuration resources. All namespaces are watched if this parameter is left empty. | -| `--watch-namespace-selector` | The controller will watch namespaces whose labels match the given selector. This flag only takes effective when `--watch-namespace` is empty. | diff --git a/docs/user-guide/convert_arguments_to_doc.py b/docs/user-guide/convert_arguments_to_doc.py index acd5ffc7b..e23e5693a 100755 --- a/docs/user-guide/convert_arguments_to_doc.py +++ b/docs/user-guide/convert_arguments_to_doc.py @@ -15,7 +15,7 @@ # limitations under the License. """ -Convert the output of `ingress-nginx-controller --help` to +Convert the output of `nginx-ingress-controller --help` to a Markdown table. """ diff --git a/docs/user-guide/custom-errors.md b/docs/user-guide/custom-errors.md index 159a82078..05678f9e6 100644 --- a/docs/user-guide/custom-errors.md +++ b/docs/user-guide/custom-errors.md @@ -27,5 +27,5 @@ An example of such custom backend is available inside the source repository at [ See also the [Custom errors][example-custom-errors] example. [cm-custom-http-errors]: ./nginx-configuration/configmap.md#custom-http-errors -[img-custom-error-pages]: https://github.com/kubernetes/ingress-nginx/tree/main/images/custom-error-pages +[img-custom-error-pages]: https://github.com/kubernetes/ingress-nginx/tree/master/images/custom-error-pages [example-custom-errors]: ../../examples/customization/custom-errors diff --git a/docs/user-guide/default-backend.md b/docs/user-guide/default-backend.md index f15561086..ccf11bb44 100644 --- a/docs/user-guide/default-backend.md +++ b/docs/user-guide/default-backend.md @@ -1,6 +1,6 @@ # Default backend -The default backend is a service which handles all URL paths and hosts the Ingress-NGINX controller doesn't understand +The default backend is a service which handles all URL paths and hosts the nginx controller doesn't understand (i.e., all the requests that are not mapped with an Ingress). Basically a default backend exposes two URLs: @@ -9,5 +9,5 @@ Basically a default backend exposes two URLs: - `/` that returns 404 !!! example - The sub-directory [`/images/custom-error-pages`](https://github.com/kubernetes/ingress-nginx/tree/main/images/custom-error-pages) + The sub-directory [`/images/custom-error-pages`](https://github.com/kubernetes/ingress-nginx/tree/master/images/custom-error-pages) provides an additional service for the purpose of customizing the error pages served via the default backend. diff --git a/docs/user-guide/exposing-tcp-udp-services.md b/docs/user-guide/exposing-tcp-udp-services.md index 9c14a7c39..6b595ffae 100644 --- a/docs/user-guide/exposing-tcp-udp-services.md +++ b/docs/user-guide/exposing-tcp-udp-services.md @@ -1,9 +1,7 @@ # Exposing TCP and UDP services -While the Kubernetes Ingress resource only officially supports routing external HTTP(s) traffic to services, ingress-nginx can be configured to receive external TCP/UDP traffic from non-HTTP protocols and route them to internal services using TCP/UDP port mappings that are specified within a ConfigMap. - -To support this, the `--tcp-services-configmap` and `--udp-services-configmap` flags can be used to point to an existing config map where the key is the external port to use and the value indicates the service to expose using the format: -`:::[PROXY]:[PROXY]` +Ingress does not support TCP or UDP services. For this reason this Ingress controller uses the flags `--tcp-services-configmap` and `--udp-services-configmap` to point to an existing config map where the key is the external port to use and the value indicates the service to expose using the format: +`::[PROXY]:[PROXY]` It is also possible to use a number or the name of the port. The two last fields are optional. Adding `PROXY` in either or both of the two last fields we can use [Proxy Protocol](https://www.nginx.com/resources/admin-guide/proxy-protocol) decoding (listen) and/or encoding (proxy_pass) in a TCP service. @@ -65,9 +63,3 @@ spec: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx ``` -Then, the configmap should be added into ingress controller's deployment args. -``` - args: - - /nginx-ingress-controller - - --tcp-services-configmap=ingress-nginx/tcp-services -``` diff --git a/docs/user-guide/external-articles.md b/docs/user-guide/external-articles.md index 8a60ad567..d9fd828b8 100644 --- a/docs/user-guide/external-articles.md +++ b/docs/user-guide/external-articles.md @@ -2,6 +2,5 @@ - [Pain(less) NGINX Ingress](http://danielfm.me/posts/painless-nginx-ingress.html) - [Accessing Kubernetes Pods from Outside of the Cluster](http://alesnosek.com/blog/2017/02/14/accessing-kubernetes-pods-from-outside-of-the-cluster) -- [Kubernetes - Redirect HTTP to HTTPS with ELB and the Ingress-Nginx Controller](https://dev.to/tomhoule/kubernetes---redirect-http-to-https-with-elb-and-the-nginx-ingress-controller) +- [Kubernetes - Redirect HTTP to HTTPS with ELB and the nginx ingress controller](https://dev.to/tomhoule/kubernetes---redirect-http-to-https-with-elb-and-the-nginx-ingress-controller) - [Configure Nginx Ingress Controller for TLS termination on Kubernetes on Azure](https://blogs.technet.microsoft.com/livedevopsinjapan/2017/02/28/configure-nginx-ingress-controller-for-tls-termination-on-kubernetes-on-azure-2/) -- [Secure your Nginx Ingress controller behind Google Cloud Armor or Identity-Aware Proxy (IAP)](https://medium.com/google-cloud/secure-your-nginx-ingress-controller-behind-cloud-armor-805d6109af86?sk=f64029bb5624b4ad5cd2828f4c358af3) diff --git a/docs/user-guide/fcgi-services.md b/docs/user-guide/fcgi-services.md index 9c222d9ad..7c9dd6138 100644 --- a/docs/user-guide/fcgi-services.md +++ b/docs/user-guide/fcgi-services.md @@ -1,3 +1,5 @@ + + # Exposing FastCGI Servers > **FastCGI** is a [binary protocol](https://en.wikipedia.org/wiki/Binary_protocol "Binary protocol") for interfacing interactive programs with a [web server](https://en.wikipedia.org/wiki/Web_server "Web server"). [...] (It's) aim is to reduce the overhead related to interfacing between web server and CGI programs, allowing a server to handle more web page requests per unit of time. @@ -6,60 +8,27 @@ The _ingress-nginx_ ingress controller can be used to directly expose [FastCGI](https://en.wikipedia.org/wiki/FastCGI) servers. Enabling FastCGI in your Ingress only requires setting the _backend-protocol_ annotation to `FCGI`, and with a couple more annotations you can customize the way _ingress-nginx_ handles the communication with your FastCGI _server_. -For most practical use-cases, php applications are a good example. PHP is not HTML so a FastCGI server like php-fpm processes a index.php script for the response to a request. See a working example below. -This [post in a FactCGI feature issue](https://github.com/kubernetes/ingress-nginx/issues/8207#issuecomment-2161405468) describes a test for the FastCGI feature. The same test is described below here. +## Example Objects to Expose a FastCGI Pod -## Example Objects to expose a FastCGI server pod - -### The FasctCGI server pod - -The _Pod_ object example below exposes port `9000`, which is the conventional FastCGI port. +The _Pod_ example object below exposes port `9000`, which is the conventional FastCGI port. ```yaml apiVersion: v1 kind: Pod metadata: name: example-app - labels: - app: example-app +labels: + app: example-app spec: containers: - name: example-app - image: php:fpm-alpine + image: example-app:1.0 ports: - containerPort: 9000 name: fastcgi ``` -- For this example to work, a HTML response should be received from the FastCGI server being exposed -- A HTTP request to the FastCGI server pod should be sent -- The response should be generated by a php script as that is what we are demonstrating here - -The image we are using here `php:fpm-alpine` does not ship with a ready to use php script inside it. So we need to provide the image with a simple php-script for this example to work. - -- Use `kubectl exec` to get into the example-app pod -- You will land at the path `/var/www/html` -- Create a simple php script there at the path /var/www/html called index.php -- Make the index.php file look like this - -``` - - - - PHP Test - - - FastCGI Test Worked!

'; ?> - - -``` - -- Save and exit from the shell in the pod -- If you delete the pod, then you will have to recreate the file as this method is not persistent - -### The FastCGI service - The _Service_ object example below matches port `9000` from the _Pod_ object above. ```yaml @@ -76,100 +45,38 @@ spec: name: fastcgi ``` -### The configMap object and the ingress object - -The _Ingress_ and _ConfigMap_ objects below demonstrate the supported _FastCGI_ specific annotations. - -!!! Important - NGINX actually has 50 [FastCGI directives](https://nginx.org/en/docs/http/ngx_http_fastcgi_module.html#directives) - All of the nginx directives have not been exposed in the ingress yet - -### The ConfigMap object - -This configMap object is required to set the parameters of [FastCGI directives](https://nginx.org/en/docs/http/ngx_http_fastcgi_module.html#directives) - -!!! Attention - - The _ConfigMap_ **must** be created before creating the ingress object - -- The _Ingress Controller_ needs to find the configMap when the _Ingress_ object with the FastCGI annotations is created -- So create the configMap before the ingress -- If the configMap is created after the ingress is created, then you will need to restart the _Ingress Controller_ pods. +And the _Ingress_ and _ConfigMap_ objects below demonstrates the supported _FastCGI_ specific annotations (NGINX actually has 50 FastCGI directives, all of which have not been exposed in the ingress yet), and matches the service `example-service`, and the port named `fastcgi` from above. The _ConfigMap_ **must** be created first for the _Ingress Controller_ to be able to find it when the _Ingress_ object is created, otherwise you will need to restart the _Ingress Controller_ pods. ```yaml +# The ConfigMap MUST be created first for the ingress controller to be able to +# find it when the Ingress object is created. + apiVersion: v1 kind: ConfigMap metadata: name: example-cm data: - SCRIPT_FILENAME: "/var/www/html/index.php" + SCRIPT_FILENAME: "/example/index.php" -``` +--- -### The ingress object - -- Do not create the ingress shown below until you have created the configMap seen above. -- You can see that this ingress matches the service `example-service`, and the port named `fastcgi` from above. - -``` -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: annotations: + kubernetes.io/ingress.class: "nginx" nginx.ingress.kubernetes.io/backend-protocol: "FCGI" nginx.ingress.kubernetes.io/fastcgi-index: "index.php" nginx.ingress.kubernetes.io/fastcgi-params-configmap: "example-cm" name: example-app spec: - ingressClassName: nginx rules: - host: app.example.com http: paths: - - path: / - pathType: Prefix - backend: - service: - name: example-service - port: - name: fastcgi -``` - -## Send a request to the exposed FastCGI server - -You will have to look at the external-ip of the ingress or you have to send the HTTP request to the ClusterIP address of the ingress-nginx controller pod. - -``` -% curl 172.19.0.2 -H "Host: app.example.com" -vik -* Trying 172.19.0.2:80... -* Connected to 172.19.0.2 (172.19.0.2) port 80 -> GET / HTTP/1.1 -> Host: app.example.com -> User-Agent: curl/8.6.0 -> Accept: */* -> -< HTTP/1.1 200 OK -HTTP/1.1 200 OK -< Date: Wed, 12 Jun 2024 07:11:59 GMT -Date: Wed, 12 Jun 2024 07:11:59 GMT -< Content-Type: text/html; charset=UTF-8 -Content-Type: text/html; charset=UTF-8 -< Transfer-Encoding: chunked -Transfer-Encoding: chunked -< Connection: keep-alive -Connection: keep-alive -< X-Powered-By: PHP/8.3.8 -X-Powered-By: PHP/8.3.8 - -< - - - - PHP Test - - -

FastCGI Test Worked

- - + - backend: + serviceName: example-service + servicePort: fastcgi ``` ## FastCGI Ingress Annotations @@ -182,13 +89,13 @@ To enable FastCGI, the `nginx.ingress.kubernetes.io/backend-protocol` annotation ### The `nginx.ingress.kubernetes.io/fastcgi-index` Annotation -To specify an index file, the `fastcgi-index` annotation value can optionally be set. In the example below, the value is set to `index.php`. This annotation corresponds to [the _NGINX_ `fastcgi_index` directive](https://nginx.org/en/docs/http/ngx_http_fastcgi_module.html#fastcgi_index). +To specify an index file, the `fastcgi-index` annotation value can optionally be set. In the example below, the value is set to `index.php`. This annotation corresponds to [the _NGINX_ `fastcgi_index` directive](http://nginx.org/en/docs/http/ngx_http_fastcgi_module.html#fastcgi_index). > `nginx.ingress.kubernetes.io/fastcgi-index: "index.php"` ### The `nginx.ingress.kubernetes.io/fastcgi-params-configmap` Annotation -To specify [_NGINX_ `fastcgi_param` directives](https://nginx.org/en/docs/http/ngx_http_fastcgi_module.html#fastcgi_param), the `fastcgi-params-configmap` annotation is used, which in turn must lead to a _ConfigMap_ object containing the _NGINX_ `fastcgi_param` directives as key/values. +To specify [_NGINX_ `fastcgi_param` directives](http://nginx.org/en/docs/http/ngx_http_fastcgi_module.html#fastcgi_param), the `fastcgi-params-configmap` annotation is used, which in turn must lead to a _ConfigMap_ object containing the _NGINX_ `fastcgi_param` directives as key/values. > `nginx.ingress.kubernetes.io/fastcgi-params-configmap: "example-configmap"` @@ -203,7 +110,6 @@ data: SCRIPT_FILENAME: "/example/index.php" HTTP_PROXY: "" ``` - Using the _namespace/_ prefix is also supported, for example: > `nginx.ingress.kubernetes.io/fastcgi-params-configmap: "example-namespace/example-configmap"` diff --git a/docs/user-guide/ingress-path-matching.md b/docs/user-guide/ingress-path-matching.md index 43d049043..092cf9264 100644 --- a/docs/user-guide/ingress-path-matching.md +++ b/docs/user-guide/ingress-path-matching.md @@ -3,10 +3,7 @@ ## Regular Expression Support !!! important - Regular expressions is not supported in the `spec.rules.host` field. The wildcard character '\*' must appear by itself as the first DNS label and matches only a single label. You cannot have a wildcard label by itself (e.g. Host == "\*"). - -!!! note - Please see the [FAQ](../faq.md#validation-of-path) for Validation Of __`path`__ + Regular expressions and wild cards are not supported in the `spec.rules.host` field. Full hostnames must be used. The ingress controller supports **case insensitive** regular expressions in the `spec.rules.http.paths.path` field. This can be enabled by setting the `nginx.ingress.kubernetes.io/use-regex` annotation to `true` (the default is false). @@ -18,25 +15,21 @@ This can be enabled by setting the `nginx.ingress.kubernetes.io/use-regex` annot See the [description](./nginx-configuration/annotations.md#use-regex) of the `use-regex` annotation for more details. ```yaml -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: test-ingress annotations: nginx.ingress.kubernetes.io/use-regex: "true" spec: - ingressClassName: nginx rules: - host: test.com http: paths: - path: /foo/.* - pathType: ImplementationSpecific backend: - service: - name: test - port: - number: 80 + serviceName: test + servicePort: 80 ``` The preceding ingress definition would translate to the following location block within the NGINX configuration for the `test.com` server: @@ -58,52 +51,41 @@ In NGINX, regular expressions follow a **first match** policy. In order to enabl Let the following two ingress definitions be created: ```yaml -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: test-ingress-1 spec: - ingressClassName: nginx rules: - host: test.com http: paths: - path: /foo/bar - pathType: Prefix backend: - service: - name: service1 - port: - number: 80 + serviceName: service1 + servicePort: 80 - path: /foo/bar/ - pathType: Prefix backend: - service: - name: service2 - port: - number: 80 + serviceName: service2 + servicePort: 80 ``` ```yaml -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: test-ingress-2 annotations: nginx.ingress.kubernetes.io/rewrite-target: /$1 spec: - ingressClassName: nginx rules: - host: test.com http: paths: - path: /foo/bar/(.+) - pathType: ImplementationSpecific backend: - service: - name: service3 - port: - number: 80 + serviceName: service3 + servicePort: 80 ``` The ingress controller would define the following location blocks, in order of descending length, within the NGINX template for the `test.com` server: @@ -143,32 +125,25 @@ This case is expected and a result of NGINX's a first match policy for paths tha Let the following ingress be defined: ```yaml -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: test-ingress-3 annotations: nginx.ingress.kubernetes.io/use-regex: "true" spec: - ingressClassName: nginx rules: - host: test.com http: paths: - path: /foo/bar/bar - pathType: Prefix backend: - service: - name: test - port: - number: 80 + serviceName: test + servicePort: 80 - path: /foo/bar/[A-Z0-9]{3} - pathType: ImplementationSpecific backend: - service: - name: test - port: - number: 80 + serviceName: test + servicePort: 80 ``` The ingress controller would define the following location blocks (in this order) within the NGINX template for the `test.com` server: diff --git a/docs/user-guide/k8s-122-migration.md b/docs/user-guide/k8s-122-migration.md deleted file mode 100644 index 0880f020e..000000000 --- a/docs/user-guide/k8s-122-migration.md +++ /dev/null @@ -1,194 +0,0 @@ -# FAQ - Migration to Kubernetes 1.22 and apiVersion `networking.k8s.io/v1` - -If you are using Ingress objects in your cluster (running Kubernetes older than v1.22), -and you plan to upgrade to Kubernetes v1.22, this page is relevant to you. - -- Please read this [official blog on deprecated Ingress API versions](https://kubernetes.io/blog/2021/07/26/update-with-ingress-nginx/) -- Please read this [official documentation on the IngressClass object](https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class) - -## What is an IngressClass and why is it important for users of ingress-nginx controller now? - -IngressClass is a Kubernetes resource. See the description below. -It's important because until now, a default install of the ingress-nginx controller did not require a IngressClass object. -From version 1.0.0 of the ingress-nginx controller, an IngressClass object is required. - -On clusters with more than one instance of the ingress-nginx controller, all instances of the controllers must be aware of which Ingress objects they serve. -The `ingressClassName` field of an Ingress is the way to let the controller know about that. - -```console -kubectl explain ingressclass -``` - -``` -KIND: IngressClass -VERSION: networking.k8s.io/v1 -DESCRIPTION: - IngressClass represents the class of the Ingress, referenced by the Ingress - Spec. The `ingressclass.kubernetes.io/is-default-class` annotation can be - used to indicate that an IngressClass should be considered default. When a - single IngressClass resource has this annotation set to true, new Ingress - resources without a class specified will be assigned this default class. -FIELDS: - apiVersion - APIVersion defines the versioned schema of this representation of an - object. Servers should convert recognized schemas to the latest internal - value, and may reject unrecognized values. More info: - https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - kind - Kind is a string value representing the REST resource this object - represents. Servers may infer this from the endpoint the client submits - requests to. Cannot be updated. In CamelCase. More info: - https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - metadata - Standard object's metadata. More info: - https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata - spec - Spec is the desired state of the IngressClass. More info: - https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status` -``` - -## What has caused this change in behavior? - -There are 2 primary reasons. - -### Reason 1 - -Until K8s version 1.21, it was possible to create an Ingress resource using deprecated versions of the Ingress API, such as: - -- `extensions/v1beta1` -- `networking.k8s.io/v1beta1` - You would get a message about deprecation, but the Ingress resource would get created. - -From K8s version 1.22 onwards, you can **only** access the Ingress API via the stable, `networking.k8s.io/v1` API. -The reason is explained in the [official blog on deprecated ingress API versions](https://kubernetes.io/blog/2021/07/26/update-with-ingress-nginx/). - -### Reason #2 - -If you are already using the ingress-nginx controller and then upgrade to Kubernetes 1.22, -there are several scenarios where your existing Ingress objects will not work how you expect. - -Read this FAQ to check which scenario matches your use case. - -## What is the `ingressClassName` field? - -`ingressClassName` is a field in the spec of an Ingress object. - -```shell -kubectl explain ingress.spec.ingressClassName -``` - -```console -KIND: Ingress -VERSION: networking.k8s.io/v1 -FIELD: ingressClassName -DESCRIPTION: - IngressClassName is the name of the IngressClass cluster resource. The - associated IngressClass defines which controller will implement the - resource. This replaces the deprecated `kubernetes.io/ingress.class` - annotation. For backwards compatibility, when that annotation is set, it - must be given precedence over this field. The controller may emit a warning - if the field and annotation have different values. Implementations of this - API should ignore Ingresses without a class specified. An IngressClass - resource may be marked as default, which can be used to set a default value - for this field. For more information, refer to the IngressClass - documentation. -``` - -The `.spec.ingressClassName` behavior has precedence over the deprecated `kubernetes.io/ingress.class` annotation. - -## I have only one ingress controller in my cluster. What should I do? - -If a single instance of the ingress-nginx controller is the sole Ingress controller running in your cluster, -you should add the annotation "ingressclass.kubernetes.io/is-default-class" in your IngressClass, -so any new Ingress objects will have this one as default IngressClass. - -When using Helm, you can enable this annotation by setting `.controller.ingressClassResource.default: true` in your Helm chart installation's values file. - -If you have any old Ingress objects remaining without an IngressClass set, you can do one or more of the following to make the ingress-nginx controller aware of the old objects: - -- You can manually set the [`.spec.ingressClassName`](https://kubernetes.io/docs/reference/kubernetes-api/service-resources/ingress-v1/#IngressSpec) field in the manifest of your own Ingress resources. -- You can re-create them after setting the `ingressclass.kubernetes.io/is-default-class` annotation to `true` on the IngressClass -- Alternatively you can make the ingress-nginx controller watch Ingress objects without the ingressClassName field set by starting your ingress-nginx with the flag [--watch-ingress-without-class=true](#what-is-the-flag-watch-ingress-without-class). - When using Helm, you can configure your Helm chart installation's values file with `.controller.watchIngressWithoutClass: true`. - -We recommend that you create the IngressClass as shown below: - -``` ---- -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - labels: - app.kubernetes.io/component: controller - name: nginx - annotations: - ingressclass.kubernetes.io/is-default-class: "true" -spec: - controller: k8s.io/ingress-nginx -``` - -and add the value `spec.ingressClassName=nginx` in your Ingress objects. - -## I have many ingress objects in my cluster. What should I do? - -If you have a lot of ingress objects without ingressClass configuration, -you can run the ingress controller with the flag `--watch-ingress-without-class=true`. - -### What is the flag `--watch-ingress-without-class`? - -It's a flag that is passed, as an argument, to the `nginx-ingress-controller` executable. -In the configuration, it looks like this: - -```yaml -# ... -args: - - /nginx-ingress-controller - - --watch-ingress-without-class=true - - --controller-class=k8s.io/ingress-nginx - # ... -# ... -``` - -## I have more than one controller in my cluster, and I'm already using the annotation - -No problem. This should still keep working, but we highly recommend you to test! -Even though `kubernetes.io/ingress.class` is deprecated, the ingress-nginx controller still understands that annotation. -If you want to follow good practice, you should consider migrating to use IngressClass and `.spec.ingressClassName`. - -## I have more than one controller running in my cluster, and I want to use the new API - -In this scenario, you need to create multiple IngressClasses (see the example above). - -Be aware that IngressClass works in a very specific way: you will need to change the `.spec.controller` value in your IngressClass and configure the controller to expect the exact same value. - -Let's see an example, supposing that you have three IngressClasses: - -- IngressClass `ingress-nginx-one`, with `.spec.controller` equal to `example.com/ingress-nginx1` -- IngressClass `ingress-nginx-two`, with `.spec.controller` equal to `example.com/ingress-nginx2` -- IngressClass `ingress-nginx-three`, with `.spec.controller` equal to `example.com/ingress-nginx1` - -For private use, you can also use a controller name that doesn't contain a `/`, e.g. `ingress-nginx1`. - -When deploying your ingress controllers, you will have to change the `--controller-class` field as follows: - -- Ingress-Nginx A, configured to use controller class name `example.com/ingress-nginx1` -- Ingress-Nginx B, configured to use controller class name `example.com/ingress-nginx2` - -When you create an Ingress object with its `ingressClassName` set to `ingress-nginx-two`, -only controllers looking for the `example.com/ingress-nginx2` controller class pay attention to the new object. - -Given that Ingress-Nginx B is set up that way, it will serve that object, whereas Ingress-Nginx A ignores the new Ingress. - -Bear in mind that if you start Ingress-Nginx B with the command line argument `--watch-ingress-without-class=true`, it will serve: - -1. Ingresses without any `ingressClassName` set -2. Ingresses where the deprecated annotation (`kubernetes.io/ingress.class`) matches the value set in the command line argument `--ingress-class` -3. Ingresses that refer to any IngressClass that has the same `spec.controller` as configured in `--controller-class` -4. If you start Ingress-Nginx B with the command line argument `--watch-ingress-without-class=true` and you run Ingress-Nginx A with the command line argument `--watch-ingress-without-class=false` then this is a supported configuration. - If you have two ingress-nginx controllers for the same cluster, both running with `--watch-ingress-without-class=true` then there is likely to be a conflict. - -## Why am I seeing "ingress class annotation is not equal to the expected by Ingress Controller" in my controller logs? - -It is highly likely that you will also see the name of the ingress resource in the same error message. -This error message has been observed on use the deprecated annotation (`kubernetes.io/ingress.class`) in an Ingress resource manifest. -It is recommended to use the `.spec.ingressClassName` field of the Ingress resource, to specify the name of the IngressClass of the Ingress you are defining. diff --git a/docs/user-guide/miscellaneous.md b/docs/user-guide/miscellaneous.md index ee3d63056..a991d91b9 100644 --- a/docs/user-guide/miscellaneous.md +++ b/docs/user-guide/miscellaneous.md @@ -4,11 +4,9 @@ By default NGINX uses the content of the header `X-Forwarded-For` as the source of truth to get information about the client IP address. This works without issues in L7 **if we configure the setting `proxy-real-ip-cidr`** with the correct information of the IP/network address of trusted external load balancer. -This setting can be enabled/disabled by setting [`use-forwarded-headers`](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#use-forwarded-headers). - If the ingress controller is running in AWS we need to use the VPC IPv4 CIDR. -Another option is to enable the **PROXY protocol** using [`use-proxy-protocol: "true"`](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#use-proxy-protocol). +Another option is to enable proxy protocol using `use-proxy-protocol: "true"`. In this mode NGINX does not use the content of the header to get the source IP address of the connection. @@ -19,7 +17,7 @@ By default NGINX path type is Prefix to not break existing definitions ## Proxy Protocol -If you are using a L4 proxy to forward the traffic to the Ingress NGINX pods and terminate HTTP/HTTPS there, you will lose the remote endpoint's IP address. To prevent this you could use the [PROXY Protocol](http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt) for forwarding traffic, this will send the connection details before forwarding the actual TCP connection itself. +If you are using a L4 proxy to forward the traffic to the NGINX pods and terminate HTTP/HTTPS there, you will lose the remote endpoint's IP address. To prevent this you could use the [Proxy Protocol](http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt) for forwarding traffic, this will send the connection details before forwarding the actual TCP connection itself. Amongst others [ELBs in AWS](http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/enable-proxy-protocol.html) and [HAProxy](http://www.haproxy.org/) support Proxy Protocol. @@ -29,16 +27,16 @@ Support for websockets is provided by NGINX out of the box. No special configura The only requirement to avoid the close of connections is the increase of the values of `proxy-read-timeout` and `proxy-send-timeout`. -The default value of these settings is `60 seconds`. +The default value of this settings is `60 seconds`. A more adequate value to support websockets is a value higher than one hour (`3600`). !!! Important - If the Ingress-Nginx Controller is exposed with a service `type=LoadBalancer` make sure the protocol between the loadbalancer and NGINX is TCP. + If the NGINX ingress controller is exposed with a service `type=LoadBalancer` make sure the protocol between the loadbalancer and NGINX is TCP. ## Optimizing TLS Time To First Byte (TTTFB) -NGINX provides the configuration option [ssl_buffer_size](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_buffer_size) to allow the optimization of the TLS record size. +NGINX provides the configuration option [ssl_buffer_size](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_buffer_size) to allow the optimization of the TLS record size. This improves the [TLS Time To First Byte](https://www.igvita.com/2013/12/16/optimizing-nginx-tls-time-to-first-byte/) (TTTFB). The default value in the Ingress controller is `4k` (NGINX default is `16k`). @@ -54,4 +52,4 @@ The previous behavior can be restored using `retry-non-idempotent=true` in the c ## Why endpoints and not services -The Ingress-Nginx Controller does not use [Services](http://kubernetes.io/docs/user-guide/services) to route traffic to the pods. Instead it uses the Endpoints API in order to bypass [kube-proxy](http://kubernetes.io/docs/admin/kube-proxy/) to allow NGINX features like session affinity and custom load balancing algorithms. It also removes some overhead, such as conntrack entries for iptables DNAT. +The NGINX ingress controller does not use [Services](http://kubernetes.io/docs/user-guide/services) to route traffic to the pods. Instead it uses the Endpoints API in order to bypass [kube-proxy](http://kubernetes.io/docs/admin/kube-proxy/) to allow NGINX features like session affinity and custom load balancing algorithms. It also removes some overhead, such as conntrack entries for iptables DNAT. diff --git a/docs/user-guide/monitoring.md b/docs/user-guide/monitoring.md index 4ad2795cf..4f6376a3b 100644 --- a/docs/user-guide/monitoring.md +++ b/docs/user-guide/monitoring.md @@ -1,476 +1,85 @@ -# Monitoring +# Prometheus and Grafana installation -Two different methods to install and configure Prometheus and Grafana are described in this doc. -* Prometheus and Grafana installation using Pod Annotations. This installs Prometheus and Grafana in the same namespace as NGINX Ingress -* Prometheus and Grafana installation using Service Monitors. This installs Prometheus and Grafana in two different namespaces. This is the preferred method, and helm charts supports this by default. - -## Prometheus and Grafana installation using Pod Annotations - -This tutorial will show you how to install [Prometheus](https://prometheus.io/) and [Grafana](https://grafana.com/) for scraping the metrics of the Ingress-Nginx Controller. +This tutorial will show you how to install [Prometheus](https://prometheus.io/) and [Grafana](https://grafana.com/) for scraping the metrics of the NGINX Ingress controller. !!! important This example uses `emptyDir` volumes for Prometheus and Grafana. This means once the pod gets terminated you will lose all the data. -### Before You Begin +## Before You Begin -- The Ingress-Nginx Controller should already be deployed according to the deployment instructions [here](../deploy/index.md). +The NGINX Ingress controller should already be deployed according to the deployment instructions [here](../deploy/index.md). -- The controller should be configured for exporting metrics. This requires 3 configurations to the controller. These configurations are : - 1. controller.metrics.enabled=true - 2. controller.podAnnotations."prometheus.io/scrape"="true" - 3. controller.podAnnotations."prometheus.io/port"="10254" +Note that the kustomize bases used in this tutorial are stored in the [deploy](https://github.com/kubernetes/ingress-nginx/tree/master/deploy) folder of the GitHub repository [kubernetes/ingress-nginx](https://github.com/kubernetes/ingress-nginx). - - The easiest way to configure the controller for metrics is via helm upgrade. Assuming you have installed the ingress-nginx controller as a helm release named ingress-nginx, then you can simply type the command shown below : - ``` - helm upgrade ingress-nginx ingress-nginx \ - --repo https://kubernetes.github.io/ingress-nginx \ - --namespace ingress-nginx \ - --set controller.metrics.enabled=true \ - --set-string controller.podAnnotations."prometheus\.io/scrape"="true" \ - --set-string controller.podAnnotations."prometheus\.io/port"="10254" - ``` - - You can validate that the controller is configured for metrics by looking at the values of the installed release, like this: - ``` - helm get values ingress-nginx --namespace ingress-nginx - ``` - - You should be able to see the values shown below: - ``` - .. - controller: - metrics: - enabled: true - podAnnotations: - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - .. - ``` - - If you are **not using helm**, you will have to edit your manifests like this: - - Service manifest: - ``` - apiVersion: v1 - kind: Service - .. - spec: - ports: - - name: prometheus - port: 10254 - targetPort: prometheus - .. +## Deploy and configure Prometheus Server - ``` - - Deployment manifest: - ``` - apiVersion: v1 - kind: Deployment - .. - spec: - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "10254" - spec: - containers: - - name: controller - ports: - - name: prometheus - containerPort: 10254 - .. - ``` +The Prometheus server must be configured so that it can discover endpoints of services. If a Prometheus server is already running in the cluster and if it is configured in a way that it can find the ingress controller pods, no extra configuration is needed. +If there is no existing Prometheus server running, the rest of this tutorial will guide you through the steps needed to deploy a properly configured Prometheus server. -### Deploy and configure Prometheus Server +Running the following command deploys prometheus in Kubernetes: -Note that the kustomize bases used in this tutorial are stored in the [deploy](https://github.com/kubernetes/ingress-nginx/tree/main/deploy) folder of the GitHub repository [kubernetes/ingress-nginx](https://github.com/kubernetes/ingress-nginx). +```console +kubectl apply --kustomize github.com/kubernetes/ingress-nginx/deploy/prometheus/ +``` -- The Prometheus server must be configured so that it can discover endpoints of services. If a Prometheus server is already running in the cluster and if it is configured in a way that it can find the ingress controller pods, no extra configuration is needed. +### Prometheus Dashboard -- If there is no existing Prometheus server running, the rest of this tutorial will guide you through the steps needed to deploy a properly configured Prometheus server. +Open Prometheus dashboard in a web browser: -- Running the following command deploys prometheus in Kubernetes: +```console +kubectl get svc -n ingress-nginx +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +default-http-backend ClusterIP 10.103.59.201 80/TCP 3d +ingress-nginx NodePort 10.97.44.72 80:30100/TCP,443:30154/TCP,10254:32049/TCP 5h +prometheus-server NodePort 10.98.233.86 9090:32630/TCP 1m +``` - ``` - kubectl apply --kustomize github.com/kubernetes/ingress-nginx/deploy/prometheus/ - ``` +Obtain the IP address of the nodes in the running cluster: -#### Prometheus Dashboard +```console +kubectl get nodes -o wide +``` -- Open Prometheus dashboard in a web browser: +In some cases where the node only have internal IP addresses we need to execute: - ```console - kubectl get svc -n ingress-nginx - NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE - default-http-backend ClusterIP 10.103.59.201 80/TCP 3d - ingress-nginx NodePort 10.97.44.72 80:30100/TCP,443:30154/TCP,10254:32049/TCP 5h - prometheus-server NodePort 10.98.233.86 9090:32630/TCP 1m - ``` +```console +kubectl get nodes --selector=kubernetes.io/role!=master -o jsonpath={.items[*].status.addresses[?\(@.type==\"InternalIP\"\)].address} +10.192.0.2 10.192.0.3 10.192.0.4 +``` - - Obtain the IP address of the nodes in the running cluster: +Open your browser and visit the following URL: _http://{node IP address}:{prometheus-svc-nodeport}_ to load the Prometheus Dashboard. - ```console - kubectl get nodes -o wide - ``` +According to the above example, this URL will be http://10.192.0.3:32630 - - In some cases where the node only have internal IP addresses we need to execute: +![Dashboard](../images/prometheus-dashboard.png) - ``` - kubectl get nodes --selector=kubernetes.io/role!=master -o jsonpath={.items[*].status.addresses[?\(@.type==\"InternalIP\"\)].address} - 10.192.0.2 10.192.0.3 10.192.0.4 - ``` +### Grafana - - Open your browser and visit the following URL: _http://{node IP address}:{prometheus-svc-nodeport}_ to load the Prometheus Dashboard. +```console +kubectl apply --kustomize github.com/kubernetes/ingress-nginx/deploy/grafana/ +``` - - According to the above example, this URL will be http://10.192.0.3:32630 +```console +kubectl get svc -n ingress-nginx +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +default-http-backend ClusterIP 10.103.59.201 80/TCP 3d +ingress-nginx NodePort 10.97.44.72 80:30100/TCP,443:30154/TCP,10254:32049/TCP 5h +prometheus-server NodePort 10.98.233.86 9090:32630/TCP 10m +grafana NodePort 10.98.233.87 3000:31086/TCP 10m +``` - ![Prometheus Dashboard](../images/prometheus-dashboard.png) - -#### Grafana - - Install grafana using the below command - ``` - kubectl apply --kustomize github.com/kubernetes/ingress-nginx/deploy/grafana/ - ``` - - Look at the services - ``` - kubectl get svc -n ingress-nginx - NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE - default-http-backend ClusterIP 10.103.59.201 80/TCP 3d - ingress-nginx NodePort 10.97.44.72 80:30100/TCP,443:30154/TCP,10254:32049/TCP 5h - prometheus-server NodePort 10.98.233.86 9090:32630/TCP 10m - grafana NodePort 10.98.233.87 3000:31086/TCP 10m - ``` - - - Open your browser and visit the following URL: _http://{node IP address}:{grafana-svc-nodeport}_ to load the Grafana Dashboard. +Open your browser and visit the following URL: _http://{node IP address}:{grafana-svc-nodeport}_ to load the Grafana Dashboard. According to the above example, this URL will be http://10.192.0.3:31086 - The username and password is `admin` +The username and password is `admin` - - After the login you can import the Grafana dashboard from [official dashboards](https://github.com/kubernetes/ingress-nginx/tree/main/deploy/grafana/dashboards), by following steps given below : +After the login you can import the Grafana dashboard from [official dashboards](https://github.com/kubernetes/ingress-nginx/tree/master/deploy/grafana/dashboards) - - Navigate to lefthand panel of grafana - - Hover on the gearwheel icon for Configuration and click "Data Sources" - - Click "Add data source" - - Select "Prometheus" - - Enter the details (note: I used http://CLUSTER_IP_PROMETHEUS_SVC:9090) - - Left menu (hover over +) -> Dashboard - - Click "Import" - - Enter the copy pasted json from https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/grafana/dashboards/nginx.json - - Click Import JSON - - Select the Prometheus data source - - Click "Import" +![Dashboard](../images/grafana.png) +## Caveats +### Wildcard ingresses - ![Grafana Dashboard](../images/grafana.png) - -### Caveats - -#### Wildcard ingresses - - - By default request metrics are labeled with the hostname. When you have a wildcard domain ingress, then there will be no metrics for that ingress (to prevent the metrics from exploding in cardinality). To get metrics in this case you have two options: - - Run the ingress controller with `--metrics-per-host=false`. You will lose labeling by hostname, but still have labeling by ingress. - - Run the ingress controller with `--metrics-per-undefined-host=true --metrics-per-host=true`. You will get labeling by hostname even if the hostname is not explicitly defined on an ingress. Be warned that cardinality could explode due to many hostnames and CPU usage could also increase. - -### Grafana dashboard using ingress resource - - If you want to expose the dashboard for grafana using an ingress resource, then you can : - - change the service type of the prometheus-server service and the grafana service to "ClusterIP" like this : - ``` - kubectl -n ingress-nginx edit svc grafana - ``` - - This will open the currently deployed service grafana in the default editor configured in your shell (vi/nvim/nano/other) - - scroll down to line 34 that looks like "type: NodePort" - - change it to look like "type: ClusterIP". Save and exit. - - create an ingress resource with backend as "grafana" and port as "3000" - - Similarly, you can edit the service "prometheus-server" and add an ingress resource. - -## Prometheus and Grafana installation using Service Monitors -This document assumes you're using helm and using the kube-prometheus-stack package to install Prometheus and Grafana. - -### Verify Ingress-Nginx Controller is installed - -- The Ingress-Nginx Controller should already be deployed according to the deployment instructions [here](../deploy/index.md). - -- To check if Ingress controller is deployed, - ``` - kubectl get pods -n ingress-nginx - ``` -- The result should look something like: - ``` - NAME READY STATUS RESTARTS AGE - ingress-nginx-controller-7c489dc7b7-ccrf6 1/1 Running 0 19h - ``` - -### Verify Prometheus is installed - -- To check if Prometheus is already deployed, run the following command: - - ``` - helm ls -A - ``` - ``` - NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION - ingress-nginx ingress-nginx 10 2022-01-20 18:08:55.267373 -0800 PST deployed ingress-nginx-4.0.16 1.1.1 - prometheus prometheus 1 2022-01-20 16:07:25.086828 -0800 PST deployed kube-prometheus-stack-30.1.0 0.53.1 - ``` -- Notice that prometheus is installed in a differenet namespace than ingress-nginx - -- If prometheus is not installed, then you can install from [here](https://artifacthub.io/packages/helm/prometheus-community/kube-prometheus-stack) - -### Re-configure Ingress-Nginx Controller - -- The Ingress NGINX controller needs to be reconfigured for exporting metrics. This requires 3 additional configurations to the controller. These configurations are : - ``` - controller.metrics.enabled=true - controller.metrics.serviceMonitor.enabled=true - controller.metrics.serviceMonitor.additionalLabels.release="prometheus" - ``` -- The easiest way of doing this is to helm upgrade - ``` - helm upgrade ingress-nginx ingress-nginx/ingress-nginx \ - --namespace ingress-nginx \ - --set controller.metrics.enabled=true \ - --set controller.metrics.serviceMonitor.enabled=true \ - --set controller.metrics.serviceMonitor.additionalLabels.release="prometheus" - ``` -- Here `controller.metrics.serviceMonitor.additionalLabels.release="prometheus"` should match the name of the helm release of the `kube-prometheus-stack` - -- You can validate that the controller has been successfully reconfigured to export metrics by looking at the values of the installed release, like this: - ``` - helm get values ingress-nginx --namespace ingress-nginx - ``` - ``` - controller: - metrics: - enabled: true - serviceMonitor: - additionalLabels: - release: prometheus - enabled: true - ``` -### Configure Prometheus - -- Since Prometheus is running in a different namespace and not in the ingress-nginx namespace, it would not be able to discover ServiceMonitors in other namespaces when installed. Reconfigure your kube-prometheus-stack Helm installation to set `serviceMonitorSelectorNilUsesHelmValues` flag to false. By default, Prometheus only discovers PodMonitors within its own namespace. This should be disabled by setting `podMonitorSelectorNilUsesHelmValues` to false -- The configurations required are: - ``` - prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false - prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false - ``` -- The easiest way of doing this is to use `helm upgrade ...` - ``` - helm upgrade prometheus prometheus-community/kube-prometheus-stack \ - --namespace prometheus \ - --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false \ - --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false - ``` -- You can validate that Prometheus has been reconfigured by looking at the values of the installed release, like this: - ``` - helm get values prometheus --namespace prometheus - ``` -- You should be able to see the values shown below: - ``` - prometheus: - prometheusSpec: - podMonitorSelectorNilUsesHelmValues: false - serviceMonitorSelectorNilUsesHelmValues: false - ``` - -### Connect and view Prometheus dashboard -- Port forward to Prometheus service. Find out the name of the prometheus service by using the following command: - ``` - kubectl get svc -n prometheus - ``` - - The result of this command would look like: - ``` - NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE - alertmanager-operated ClusterIP None 9093/TCP,9094/TCP,9094/UDP 7h46m - prometheus-grafana ClusterIP 10.106.28.162 80/TCP 7h46m - prometheus-kube-prometheus-alertmanager ClusterIP 10.108.125.245 9093/TCP 7h46m - prometheus-kube-prometheus-operator ClusterIP 10.110.220.1 443/TCP 7h46m - prometheus-kube-prometheus-prometheus ClusterIP 10.102.72.134 9090/TCP 7h46m - prometheus-kube-state-metrics ClusterIP 10.104.231.181 8080/TCP 7h46m - prometheus-operated ClusterIP None 9090/TCP 7h46m - prometheus-prometheus-node-exporter ClusterIP 10.96.247.128 9100/TCP 7h46m - ``` - prometheus-kube-prometheus-prometheus is the service we want to port forward to. We can do so using the following command: - ``` - kubectl port-forward svc/prometheus-kube-prometheus-prometheus -n prometheus 9090:9090 - ``` - When you run the above command, you should see something like: - ``` - Forwarding from 127.0.0.1:9090 -> 9090 - Forwarding from [::1]:9090 -> 9090 - ``` -- Open your browser and visit the following URL http://localhost:{port-forwarded-port} according to the above example it would be, http://localhost:9090 - - ![Prometheus Dashboard](../images/prometheus-dashboard1.png) - -### Connect and view Grafana dashboard -- Port forward to Grafana service. Find out the name of the Grafana service by using the following command: - ``` - kubectl get svc -n prometheus - ``` - - The result of this command would look like: - ``` - NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE - alertmanager-operated ClusterIP None 9093/TCP,9094/TCP,9094/UDP 7h46m - prometheus-grafana ClusterIP 10.106.28.162 80/TCP 7h46m - prometheus-kube-prometheus-alertmanager ClusterIP 10.108.125.245 9093/TCP 7h46m - prometheus-kube-prometheus-operator ClusterIP 10.110.220.1 443/TCP 7h46m - prometheus-kube-prometheus-prometheus ClusterIP 10.102.72.134 9090/TCP 7h46m - prometheus-kube-state-metrics ClusterIP 10.104.231.181 8080/TCP 7h46m - prometheus-operated ClusterIP None 9090/TCP 7h46m - prometheus-prometheus-node-exporter ClusterIP 10.96.247.128 9100/TCP 7h46m - ``` - prometheus-grafana is the service we want to port forward to. We can do so using the following command: - ``` - kubectl port-forward svc/prometheus-grafana 3000:80 -n prometheus - ``` - When you run the above command, you should see something like: - ``` - Forwarding from 127.0.0.1:3000 -> 3000 - Forwarding from [::1]:3000 -> 3000 - ``` -- Open your browser and visit the following URL http://localhost:{port-forwarded-port} according to the above example it would be, http://localhost:3000 - The default username/ password is admin/prom-operator -- After the login you can import the Grafana dashboard from [official dashboards](https://github.com/kubernetes/ingress-nginx/tree/main/deploy/grafana/dashboards), by following steps given below : - - - Navigate to lefthand panel of grafana - - Hover on the gearwheel icon for Configuration and click "Data Sources" - - Click "Add data source" - - Select "Prometheus" - - Enter the details (note: I used http://10.102.72.134:9090 which is the CLUSTER-IP for Prometheus service) - - Left menu (hover over +) -> Dashboard - - Click "Import" - - Enter the copy pasted json from https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/grafana/dashboards/nginx.json - - Click Import JSON - - Select the Prometheus data source - - Click "Import" - - ![Grafana Dashboard](../images/grafana-dashboard1.png) - - -## Exposed metrics - -Prometheus metrics are exposed on port 10254. - -### Request metrics - -* `nginx_ingress_controller_request_duration_seconds` Histogram\ - The request processing (time elapsed between the first bytes were read from the client and the log write after the last bytes were sent to the client) time in seconds (affected by client speed).\ - nginx var: `request_time` - -* `nginx_ingress_controller_response_duration_seconds` Histogram\ - The time spent on receiving the response from the upstream server in seconds (affected by client speed when the response is bigger than proxy buffers).\ - Note: can be up to several millis bigger than the `nginx_ingress_controller_request_duration_seconds` because of the different measuring method. - nginx var: `upstream_response_time` - -* `nginx_ingress_controller_header_duration_seconds` Histogram\ - The time spent on receiving first header from the upstream server\ - nginx var: `upstream_header_time` - -* `nginx_ingress_controller_connect_duration_seconds` Histogram\ - The time spent on establishing a connection with the upstream server\ - nginx var: `upstream_connect_time` - -* `nginx_ingress_controller_response_size` Histogram\ - The response length (including request line, header, and request body)\ - nginx var: `bytes_sent` - -* `nginx_ingress_controller_request_size` Histogram\ - The request length (including request line, header, and request body)\ - nginx var: `request_length` - -* `nginx_ingress_controller_requests` Counter\ - The total number of client requests - -* `nginx_ingress_controller_bytes_sent` Histogram\ - The number of bytes sent to a client. **Deprecated**, use `nginx_ingress_controller_response_size`\ - nginx var: `bytes_sent` - -``` -# HELP nginx_ingress_controller_bytes_sent The number of bytes sent to a client. DEPRECATED! Use nginx_ingress_controller_response_size -# TYPE nginx_ingress_controller_bytes_sent histogram -# HELP nginx_ingress_controller_connect_duration_seconds The time spent on establishing a connection with the upstream server -# TYPE nginx_ingress_controller_connect_duration_seconds nginx_ingress_controller_connect_duration_seconds -* HELP nginx_ingress_controller_header_duration_seconds The time spent on receiving first header from the upstream server -# TYPE nginx_ingress_controller_header_duration_seconds histogram -# HELP nginx_ingress_controller_request_duration_seconds The request processing time in milliseconds -# TYPE nginx_ingress_controller_request_duration_seconds histogram -# HELP nginx_ingress_controller_request_size The request length (including request line, header, and request body) -# TYPE nginx_ingress_controller_request_size histogram -# HELP nginx_ingress_controller_requests The total number of client requests. -# TYPE nginx_ingress_controller_requests counter -# HELP nginx_ingress_controller_response_duration_seconds The time spent on receiving the response from the upstream server -# TYPE nginx_ingress_controller_response_duration_seconds histogram -# HELP nginx_ingress_controller_response_size The response length (including request line, header, and request body) -# TYPE nginx_ingress_controller_response_size histogram -``` - - -### Nginx process metrics -``` -# HELP nginx_ingress_controller_nginx_process_connections current number of client connections with state {active, reading, writing, waiting} -# TYPE nginx_ingress_controller_nginx_process_connections gauge -# HELP nginx_ingress_controller_nginx_process_connections_total total number of connections with state {accepted, handled} -# TYPE nginx_ingress_controller_nginx_process_connections_total counter -# HELP nginx_ingress_controller_nginx_process_cpu_seconds_total Cpu usage in seconds -# TYPE nginx_ingress_controller_nginx_process_cpu_seconds_total counter -# HELP nginx_ingress_controller_nginx_process_num_procs number of processes -# TYPE nginx_ingress_controller_nginx_process_num_procs gauge -# HELP nginx_ingress_controller_nginx_process_oldest_start_time_seconds start time in seconds since 1970/01/01 -# TYPE nginx_ingress_controller_nginx_process_oldest_start_time_seconds gauge -# HELP nginx_ingress_controller_nginx_process_read_bytes_total number of bytes read -# TYPE nginx_ingress_controller_nginx_process_read_bytes_total counter -# HELP nginx_ingress_controller_nginx_process_requests_total total number of client requests -# TYPE nginx_ingress_controller_nginx_process_requests_total counter -# HELP nginx_ingress_controller_nginx_process_resident_memory_bytes number of bytes of memory in use -# TYPE nginx_ingress_controller_nginx_process_resident_memory_bytes gauge -# HELP nginx_ingress_controller_nginx_process_virtual_memory_bytes number of bytes of memory in use -# TYPE nginx_ingress_controller_nginx_process_virtual_memory_bytes gauge -# HELP nginx_ingress_controller_nginx_process_write_bytes_total number of bytes written -# TYPE nginx_ingress_controller_nginx_process_write_bytes_total counter -``` - -### Controller metrics -``` -# HELP nginx_ingress_controller_build_info A metric with a constant '1' labeled with information about the build. -# TYPE nginx_ingress_controller_build_info gauge -# HELP nginx_ingress_controller_check_success Cumulative number of Ingress controller syntax check operations -# TYPE nginx_ingress_controller_check_success counter -# HELP nginx_ingress_controller_config_hash Running configuration hash actually running -# TYPE nginx_ingress_controller_config_hash gauge -# HELP nginx_ingress_controller_config_last_reload_successful Whether the last configuration reload attempt was successful -# TYPE nginx_ingress_controller_config_last_reload_successful gauge -# HELP nginx_ingress_controller_config_last_reload_successful_timestamp_seconds Timestamp of the last successful configuration reload. -# TYPE nginx_ingress_controller_config_last_reload_successful_timestamp_seconds gauge -# HELP nginx_ingress_controller_ssl_certificate_info Hold all labels associated to a certificate -# TYPE nginx_ingress_controller_ssl_certificate_info gauge -# HELP nginx_ingress_controller_success Cumulative number of Ingress controller reload operations -# TYPE nginx_ingress_controller_success counter -# HELP nginx_ingress_controller_orphan_ingress Gauge reporting status of ingress orphanity, 1 indicates orphaned ingress. 'namespace' is the string used to identify namespace of ingress, 'ingress' for ingress name and 'type' for 'no-service' or 'no-endpoint' of orphanity -# TYPE nginx_ingress_controller_orphan_ingress gauge -``` - -### Admission metrics -``` -# HELP nginx_ingress_controller_admission_config_size The size of the tested configuration -# TYPE nginx_ingress_controller_admission_config_size gauge -# HELP nginx_ingress_controller_admission_render_duration The processing duration of ingresses rendering by the admission controller (float seconds) -# TYPE nginx_ingress_controller_admission_render_duration gauge -# HELP nginx_ingress_controller_admission_render_ingresses The length of ingresses rendered by the admission controller -# TYPE nginx_ingress_controller_admission_render_ingresses gauge -# HELP nginx_ingress_controller_admission_roundtrip_duration The complete duration of the admission controller at the time to process a new event (float seconds) -# TYPE nginx_ingress_controller_admission_roundtrip_duration gauge -# HELP nginx_ingress_controller_admission_tested_duration The processing duration of the admission controller tests (float seconds) -# TYPE nginx_ingress_controller_admission_tested_duration gauge -# HELP nginx_ingress_controller_admission_tested_ingresses The length of ingresses processed by the admission controller -# TYPE nginx_ingress_controller_admission_tested_ingresses gauge -``` - -### Histogram buckets - -You can configure buckets for histogram metrics using these command line options (here are their default values): -* `--time-buckets=[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]` -* `--length-buckets=[10, 20, 30, 40, 50, 60, 70, 80, 90, 100]` -* `--size-buckets=[10, 100, 1000, 10000, 100000, 1e+06, 1e+07]` +By default request metrics are labeled with the hostname. When you have a wildcard domain ingress, then there will be no metrics for that ingress (to prevent the metrics from exploding in cardinality). To get metrics in this case you need to run the ingress controller with `--metrics-per-host=false` (you will lose labeling by hostname, but still have labeling by ingress). diff --git a/docs/user-guide/multiple-ingress.md b/docs/user-guide/multiple-ingress.md index 2c7c8e712..2604f4e80 100644 --- a/docs/user-guide/multiple-ingress.md +++ b/docs/user-guide/multiple-ingress.md @@ -1,80 +1,7 @@ # Multiple Ingress controllers -By default, deploying multiple Ingress controllers (e.g., `ingress-nginx` & `gce`) will result in all controllers simultaneously racing to update Ingress status fields in confusing ways. - -To fix this problem, use [IngressClasses](https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-class). The `kubernetes.io/ingress.class` annotation is not being preferred or suggested to use as it can be deprecated in the future. Better to use the field `ingress.spec.ingressClassName`. -But, when user has deployed with `scope.enabled`, then the ingress class resource field is not used. - - -## Using IngressClasses - -If all ingress controllers respect IngressClasses (e.g. multiple instances of ingress-nginx v1.0), you can deploy two Ingress controllers by granting them control over two different IngressClasses, then selecting one of the two IngressClasses with `ingressClassName`. - -First, ensure the `--controller-class=` and `--ingress-class` are set to something different on each ingress controller, If your additional ingress controller is to be installed in a namespace, where there is/are one/more-than-one ingress-nginx-controller(s) already installed, then you need to specify a different unique `--election-id` for the new instance of the controller. - -```yaml -# ingress-nginx Deployment/Statefulset -spec: - template: - spec: - containers: - - name: ingress-nginx-internal-controller - args: - - /nginx-ingress-controller - - '--election-id=ingress-controller-leader' - - '--controller-class=k8s.io/internal-ingress-nginx' - - '--ingress-class=k8s.io/internal-nginx' - ... -``` - -Then use the same value in the IngressClass: - -```yaml -# ingress-nginx IngressClass -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - name: internal-nginx -spec: - controller: k8s.io/internal-ingress-nginx - ... -``` - -And refer to that IngressClass in your Ingress: - -```yaml -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: my-ingress -spec: - ingressClassName: internal-nginx - ... -``` - -or if installing with Helm: - -```yaml -controller: - electionID: ingress-controller-leader - ingressClass: internal-nginx # default: nginx - ingressClassResource: - name: internal-nginx # default: nginx - enabled: true - default: false - controllerValue: "k8s.io/internal-ingress-nginx" # default: k8s.io/ingress-nginx -``` - -!!! important - - When running multiple ingress-nginx controllers, it will only process an unset class annotation if one of the controllers uses the default - `--controller-class` value (see `IsValid` method in `internal/ingress/annotations/class/main.go`), otherwise the class annotation becomes required. - - If `--controller-class` is set to the default value of `k8s.io/ingress-nginx`, the controller will monitor Ingresses with no class annotation *and* Ingresses with annotation class set to `nginx`. Use a non-default value for `--controller-class`, to ensure that the controller only satisfied the specific class of Ingresses. - -## Using the kubernetes.io/ingress.class annotation (in deprecation) - -If you're running multiple ingress controllers where one or more do not support IngressClasses, you must specify the annotation `kubernetes.io/ingress.class: "nginx"` in all ingresses that you would like ingress-nginx to claim. +If you're running multiple ingress controllers, or running on a cloud provider that natively handles ingress such as GKE, +you need to specify the annotation `kubernetes.io/ingress.class: "nginx"` in all ingresses that you would like the ingress-nginx controller to claim. For instance, @@ -86,7 +13,7 @@ metadata: kubernetes.io/ingress.class: "gce" ``` -will target the GCE controller, forcing the Ingress-NGINX controller to ignore it, while an annotation like: +will target the GCE controller, forcing the nginx controller to ignore it, while an annotation like ```yaml metadata: @@ -95,24 +22,36 @@ metadata: kubernetes.io/ingress.class: "nginx" ``` -will target the Ingress-NGINX controller, forcing the GCE controller to ignore it. +will target the nginx controller, forcing the GCE controller to ignore it. -You can change the value "nginx" to something else by setting the `--ingress-class` flag: +To reiterate, setting the annotation to any value which does not match a valid ingress class will force the NGINX Ingress controller to ignore your Ingress. +If you are only running a single NGINX ingress controller, this can be achieved by setting the annotation to any value except "nginx" or an empty string. + +Do this if you wish to use one of the other Ingress controllers at the same time as the NGINX controller. + +## Multiple ingress-nginx controllers + +This mechanism also provides users the ability to run _multiple_ NGINX ingress controllers (e.g. one which serves public traffic, one which serves "internal" traffic). +To do this, the option `--ingress-class` must be changed to a value unique for the cluster within the definition of the replication controller. +Here is a partial example: ```yaml spec: template: spec: containers: - - name: ingress-nginx-internal-controller + - name: nginx-ingress-internal-controller args: - /nginx-ingress-controller - - --ingress-class=internal-nginx + - '--ingress-class=nginx-internal' + - '--configmap=ingress/nginx-ingress-internal-controller' ``` -then setting the corresponding `kubernetes.io/ingress.class: "internal-nginx"` annotation on your Ingresses. +!!! important + Deploying multiple Ingress controllers, of different types (e.g., `ingress-nginx` & `gce`), and not specifying a class annotation will + result in both or all controllers fighting to satisfy the Ingress, and all of them racing to update Ingress status field in confusing ways. -To reiterate, setting the annotation to any value which does not match a valid ingress class will force the Ingress-Nginx Controller to ignore your Ingress. -If you are only running a single Ingress-Nginx Controller, this can be achieved by setting the annotation to any value except "nginx" or an empty string. + When running multiple ingress-nginx controllers, it will only process an unset class annotation if one of the controllers uses the default + `--ingress-class` value (see `IsValid` method in `internal/ingress/annotations/class/main.go`), otherwise the class annotation become required. -Do this if you wish to use one of the other Ingress controllers at the same time as the NGINX controller. + If `--ingress-class` is set to the default value of `nginx`, the controller will monitor Ingresses with no class annotation *and* Ingresses with annotation class set to `nginx`. Use a non-default value for `--ingress-class`, to ensure that the controller only satisfied the specific class of Ingresses. diff --git a/docs/user-guide/nginx-configuration/annotations-risk.md b/docs/user-guide/nginx-configuration/annotations-risk.md deleted file mode 100755 index aff9357b8..000000000 --- a/docs/user-guide/nginx-configuration/annotations-risk.md +++ /dev/null @@ -1,141 +0,0 @@ -# Annotations Scope and Risk - -|Group |Annotation | Risk | Scope | -|--------|------------------|------|-------| -| Aliases | server-alias | High | ingress | -| Allowlist | allowlist-source-range | Medium | location | -| BackendProtocol | backend-protocol | Low | location | -| BasicDigestAuth | auth-realm | Medium | location | -| BasicDigestAuth | auth-secret | Medium | location | -| BasicDigestAuth | auth-secret-type | Low | location | -| BasicDigestAuth | auth-type | Low | location | -| Canary | canary | Low | ingress | -| Canary | canary-by-cookie | Medium | ingress | -| Canary | canary-by-header | Medium | ingress | -| Canary | canary-by-header-pattern | Medium | ingress | -| Canary | canary-by-header-value | Medium | ingress | -| Canary | canary-weight | Low | ingress | -| Canary | canary-weight-total | Low | ingress | -| CertificateAuth | auth-tls-error-page | High | location | -| CertificateAuth | auth-tls-match-cn | High | location | -| CertificateAuth | auth-tls-pass-certificate-to-upstream | Low | location | -| CertificateAuth | auth-tls-secret | Medium | location | -| CertificateAuth | auth-tls-verify-client | Medium | location | -| CertificateAuth | auth-tls-verify-depth | Low | location | -| ClientBodyBufferSize | client-body-buffer-size | Low | location | -| ConfigurationSnippet | configuration-snippet | Critical | location | -| Connection | connection-proxy-header | Low | location | -| CorsConfig | cors-allow-credentials | Low | ingress | -| CorsConfig | cors-allow-headers | Medium | ingress | -| CorsConfig | cors-allow-methods | Medium | ingress | -| CorsConfig | cors-allow-origin | Medium | ingress | -| CorsConfig | cors-expose-headers | Medium | ingress | -| CorsConfig | cors-max-age | Low | ingress | -| CorsConfig | enable-cors | Low | ingress | -| CustomHTTPErrors | custom-http-errors | Low | location | -| CustomHeaders | custom-headers | Medium | location | -| DefaultBackend | default-backend | Low | location | -| Denylist | denylist-source-range | Medium | location | -| DisableProxyInterceptErrors | disable-proxy-intercept-errors | Low | location | -| EnableGlobalAuth | enable-global-auth | Low | location | -| ExternalAuth | auth-always-set-cookie | Low | location | -| ExternalAuth | auth-cache-duration | Medium | location | -| ExternalAuth | auth-cache-key | Medium | location | -| ExternalAuth | auth-keepalive | Low | location | -| ExternalAuth | auth-keepalive-requests | Low | location | -| ExternalAuth | auth-keepalive-share-vars | Low | location | -| ExternalAuth | auth-keepalive-timeout | Low | location | -| ExternalAuth | auth-method | Low | location | -| ExternalAuth | auth-proxy-set-headers | Medium | location | -| ExternalAuth | auth-request-redirect | Medium | location | -| ExternalAuth | auth-response-headers | Medium | location | -| ExternalAuth | auth-signin | High | location | -| ExternalAuth | auth-signin-redirect-param | Medium | location | -| ExternalAuth | auth-snippet | Critical | location | -| ExternalAuth | auth-url | High | location | -| FastCGI | fastcgi-index | Medium | location | -| FastCGI | fastcgi-params-configmap | Medium | location | -| HTTP2PushPreload | http2-push-preload | Low | location | -| LoadBalancing | load-balance | Low | location | -| Logs | enable-access-log | Low | location | -| Logs | enable-rewrite-log | Low | location | -| Mirror | mirror-host | High | ingress | -| Mirror | mirror-request-body | Low | ingress | -| Mirror | mirror-target | High | ingress | -| ModSecurity | enable-modsecurity | Low | ingress | -| ModSecurity | enable-owasp-core-rules | Low | ingress | -| ModSecurity | modsecurity-snippet | Critical | ingress | -| ModSecurity | modsecurity-transaction-id | High | ingress | -| Opentelemetry | enable-opentelemetry | Low | location | -| Opentelemetry | opentelemetry-operation-name | Medium | location | -| Opentelemetry | opentelemetry-trust-incoming-span | Low | location | -| Proxy | proxy-body-size | Medium | location | -| Proxy | proxy-buffer-size | Low | location | -| Proxy | proxy-buffering | Low | location | -| Proxy | proxy-buffers-number | Low | location | -| Proxy | proxy-busy-buffers-size | Low | location | -| Proxy | proxy-connect-timeout | Low | location | -| Proxy | proxy-cookie-domain | Medium | location | -| Proxy | proxy-cookie-path | Medium | location | -| Proxy | proxy-http-version | Low | location | -| Proxy | proxy-max-temp-file-size | Low | location | -| Proxy | proxy-next-upstream | Medium | location | -| Proxy | proxy-next-upstream-timeout | Low | location | -| Proxy | proxy-next-upstream-tries | Low | location | -| Proxy | proxy-read-timeout | Low | location | -| Proxy | proxy-redirect-from | Medium | location | -| Proxy | proxy-redirect-to | Medium | location | -| Proxy | proxy-request-buffering | Low | location | -| Proxy | proxy-send-timeout | Low | location | -| ProxySSL | proxy-ssl-ciphers | Medium | ingress | -| ProxySSL | proxy-ssl-name | High | ingress | -| ProxySSL | proxy-ssl-protocols | Low | ingress | -| ProxySSL | proxy-ssl-secret | Medium | ingress | -| ProxySSL | proxy-ssl-server-name | Low | ingress | -| ProxySSL | proxy-ssl-verify | Low | ingress | -| ProxySSL | proxy-ssl-verify-depth | Low | ingress | -| RateLimit | limit-allowlist | Low | location | -| RateLimit | limit-burst-multiplier | Low | location | -| RateLimit | limit-connections | Low | location | -| RateLimit | limit-rate | Low | location | -| RateLimit | limit-rate-after | Low | location | -| RateLimit | limit-rpm | Low | location | -| RateLimit | limit-rps | Low | location | -| Redirect | from-to-www-redirect | Low | location | -| Redirect | permanent-redirect | Medium | location | -| Redirect | permanent-redirect-code | Low | location | -| Redirect | relative-redirects | Low | location | -| Redirect | temporal-redirect | Medium | location | -| Redirect | temporal-redirect-code | Low | location | -| Rewrite | app-root | Medium | location | -| Rewrite | force-ssl-redirect | Medium | location | -| Rewrite | preserve-trailing-slash | Medium | location | -| Rewrite | rewrite-target | Medium | ingress | -| Rewrite | ssl-redirect | Low | location | -| Rewrite | use-regex | Low | location | -| SSLCipher | ssl-ciphers | Low | ingress | -| SSLCipher | ssl-prefer-server-ciphers | Low | ingress | -| SSLPassthrough | ssl-passthrough | Low | ingress | -| Satisfy | satisfy | Low | location | -| ServerSnippet | server-snippet | Critical | ingress | -| ServiceUpstream | service-upstream | Low | ingress | -| SessionAffinity | affinity | Low | ingress | -| SessionAffinity | affinity-canary-behavior | Low | ingress | -| SessionAffinity | affinity-mode | Medium | ingress | -| SessionAffinity | session-cookie-change-on-failure | Low | ingress | -| SessionAffinity | session-cookie-conditional-samesite-none | Low | ingress | -| SessionAffinity | session-cookie-domain | Medium | ingress | -| SessionAffinity | session-cookie-expires | Medium | ingress | -| SessionAffinity | session-cookie-max-age | Medium | ingress | -| SessionAffinity | session-cookie-name | Medium | ingress | -| SessionAffinity | session-cookie-path | Medium | ingress | -| SessionAffinity | session-cookie-samesite | Low | ingress | -| SessionAffinity | session-cookie-secure | Low | ingress | -| StreamSnippet | stream-snippet | Critical | ingress | -| UpstreamHashBy | upstream-hash-by | High | location | -| UpstreamHashBy | upstream-hash-by-subset | Low | location | -| UpstreamHashBy | upstream-hash-by-subset-size | Low | location | -| UpstreamVhost | upstream-vhost | Low | location | -| UsePortInRedirects | use-port-in-redirects | Low | location | -| XForwardedPrefix | x-forwarded-prefix | Medium | location | - diff --git a/docs/user-guide/nginx-configuration/annotations.md b/docs/user-guide/nginx-configuration/annotations.md index b0ea3cce3..add5c5595 100755 --- a/docs/user-guide/nginx-configuration/annotations.md +++ b/docs/user-guide/nginx-configuration/annotations.md @@ -18,39 +18,31 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz |[nginx.ingress.kubernetes.io/app-root](#rewrite)|string| |[nginx.ingress.kubernetes.io/affinity](#session-affinity)|cookie| |[nginx.ingress.kubernetes.io/affinity-mode](#session-affinity)|"balanced" or "persistent"| -|[nginx.ingress.kubernetes.io/affinity-canary-behavior](#session-affinity)|"sticky" or "legacy"| |[nginx.ingress.kubernetes.io/auth-realm](#authentication)|string| |[nginx.ingress.kubernetes.io/auth-secret](#authentication)|string| |[nginx.ingress.kubernetes.io/auth-secret-type](#authentication)|string| -|[nginx.ingress.kubernetes.io/auth-type](#authentication)|"basic" or "digest"| +|[nginx.ingress.kubernetes.io/auth-type](#authentication)|basic or digest| |[nginx.ingress.kubernetes.io/auth-tls-secret](#client-certificate-authentication)|string| |[nginx.ingress.kubernetes.io/auth-tls-verify-depth](#client-certificate-authentication)|number| |[nginx.ingress.kubernetes.io/auth-tls-verify-client](#client-certificate-authentication)|string| |[nginx.ingress.kubernetes.io/auth-tls-error-page](#client-certificate-authentication)|string| |[nginx.ingress.kubernetes.io/auth-tls-pass-certificate-to-upstream](#client-certificate-authentication)|"true" or "false"| -|[nginx.ingress.kubernetes.io/auth-tls-match-cn](#client-certificate-authentication)|string| |[nginx.ingress.kubernetes.io/auth-url](#external-authentication)|string| |[nginx.ingress.kubernetes.io/auth-cache-key](#external-authentication)|string| |[nginx.ingress.kubernetes.io/auth-cache-duration](#external-authentication)|string| -|[nginx.ingress.kubernetes.io/auth-keepalive](#external-authentication)|number| -|[nginx.ingress.kubernetes.io/auth-keepalive-share-vars](#external-authentication)|"true" or "false"| -|[nginx.ingress.kubernetes.io/auth-keepalive-requests](#external-authentication)|number| -|[nginx.ingress.kubernetes.io/auth-keepalive-timeout](#external-authentication)|number| |[nginx.ingress.kubernetes.io/auth-proxy-set-headers](#external-authentication)|string| |[nginx.ingress.kubernetes.io/auth-snippet](#external-authentication)|string| |[nginx.ingress.kubernetes.io/enable-global-auth](#external-authentication)|"true" or "false"| -|[nginx.ingress.kubernetes.io/backend-protocol](#backend-protocol)|string| +|[nginx.ingress.kubernetes.io/backend-protocol](#backend-protocol)|string|HTTP,HTTPS,GRPC,GRPCS,AJP| |[nginx.ingress.kubernetes.io/canary](#canary)|"true" or "false"| |[nginx.ingress.kubernetes.io/canary-by-header](#canary)|string| |[nginx.ingress.kubernetes.io/canary-by-header-value](#canary)|string| |[nginx.ingress.kubernetes.io/canary-by-header-pattern](#canary)|string| |[nginx.ingress.kubernetes.io/canary-by-cookie](#canary)|string| |[nginx.ingress.kubernetes.io/canary-weight](#canary)|number| -|[nginx.ingress.kubernetes.io/canary-weight-total](#canary)|number| |[nginx.ingress.kubernetes.io/client-body-buffer-size](#client-body-buffer-size)|string| |[nginx.ingress.kubernetes.io/configuration-snippet](#configuration-snippet)|string| |[nginx.ingress.kubernetes.io/custom-http-errors](#custom-http-errors)|[]int| -|[nginx.ingress.kubernetes.io/custom-headers](#custom-headers)|string| |[nginx.ingress.kubernetes.io/default-backend](#default-backend)|string| |[nginx.ingress.kubernetes.io/enable-cors](#enable-cors)|"true" or "false"| |[nginx.ingress.kubernetes.io/cors-allow-origin](#enable-cors)|string| @@ -64,11 +56,13 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz |[nginx.ingress.kubernetes.io/http2-push-preload](#http2-push-preload)|"true" or "false"| |[nginx.ingress.kubernetes.io/limit-connections](#rate-limiting)|number| |[nginx.ingress.kubernetes.io/limit-rps](#rate-limiting)|number| +|[nginx.ingress.kubernetes.io/global-rate-limit](#global-rate-limiting)|number| +|[nginx.ingress.kubernetes.io/global-rate-limit-window](#global-rate-limiting)|duration| +|[nginx.ingress.kubernetes.io/global-rate-limit-key](#global-rate-limiting)|string| +|[nginx.ingress.kubernetes.io/global-rate-limit-ignored-cidrs](#global-rate-limiting)|string| |[nginx.ingress.kubernetes.io/permanent-redirect](#permanent-redirect)|string| |[nginx.ingress.kubernetes.io/permanent-redirect-code](#permanent-redirect-code)|number| |[nginx.ingress.kubernetes.io/temporal-redirect](#temporal-redirect)|string| -|[nginx.ingress.kubernetes.io/temporal-redirect-code](#temporal-redirect-code)|number| -|[nginx.ingress.kubernetes.io/preserve-trailing-slash](#server-side-https-enforcement-through-redirect)|"true" or "false"| |[nginx.ingress.kubernetes.io/proxy-body-size](#custom-max-body-size)|string| |[nginx.ingress.kubernetes.io/proxy-cookie-domain](#proxy-cookie-domain)|string| |[nginx.ingress.kubernetes.io/proxy-cookie-path](#proxy-cookie-path)|string| @@ -95,35 +89,32 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz |[nginx.ingress.kubernetes.io/server-alias](#server-alias)|string| |[nginx.ingress.kubernetes.io/server-snippet](#server-snippet)|string| |[nginx.ingress.kubernetes.io/service-upstream](#service-upstream)|"true" or "false"| -|[nginx.ingress.kubernetes.io/session-cookie-change-on-failure](#cookie-affinity)|"true" or "false"| -|[nginx.ingress.kubernetes.io/session-cookie-conditional-samesite-none](#cookie-affinity)|"true" or "false"| -|[nginx.ingress.kubernetes.io/session-cookie-domain](#cookie-affinity)|string| -|[nginx.ingress.kubernetes.io/session-cookie-expires](#cookie-affinity)|string| -|[nginx.ingress.kubernetes.io/session-cookie-max-age](#cookie-affinity)|string| -|[nginx.ingress.kubernetes.io/session-cookie-name](#cookie-affinity)|string|default "INGRESSCOOKIE"| +|[nginx.ingress.kubernetes.io/session-cookie-name](#cookie-affinity)|string| |[nginx.ingress.kubernetes.io/session-cookie-path](#cookie-affinity)|string| -|[nginx.ingress.kubernetes.io/session-cookie-samesite](#cookie-affinity)|string|"None", "Lax" or "Strict"| -|[nginx.ingress.kubernetes.io/session-cookie-secure](#cookie-affinity)|string| +|[nginx.ingress.kubernetes.io/session-cookie-change-on-failure](#cookie-affinity)|"true" or "false"| +|[nginx.ingress.kubernetes.io/session-cookie-samesite](#cookie-affinity)|string| +|[nginx.ingress.kubernetes.io/session-cookie-conditional-samesite-none](#cookie-affinity)|"true" or "false"| |[nginx.ingress.kubernetes.io/ssl-redirect](#server-side-https-enforcement-through-redirect)|"true" or "false"| |[nginx.ingress.kubernetes.io/ssl-passthrough](#ssl-passthrough)|"true" or "false"| -|[nginx.ingress.kubernetes.io/stream-snippet](#stream-snippet)|string| |[nginx.ingress.kubernetes.io/upstream-hash-by](#custom-nginx-upstream-hashing)|string| |[nginx.ingress.kubernetes.io/x-forwarded-prefix](#x-forwarded-prefix-header)|string| |[nginx.ingress.kubernetes.io/load-balance](#custom-nginx-load-balancing)|string| |[nginx.ingress.kubernetes.io/upstream-vhost](#custom-nginx-upstream-vhost)|string| -|[nginx.ingress.kubernetes.io/denylist-source-range](#denylist-source-range)|CIDR| |[nginx.ingress.kubernetes.io/whitelist-source-range](#whitelist-source-range)|CIDR| |[nginx.ingress.kubernetes.io/proxy-buffering](#proxy-buffering)|string| |[nginx.ingress.kubernetes.io/proxy-buffers-number](#proxy-buffers-number)|number| |[nginx.ingress.kubernetes.io/proxy-buffer-size](#proxy-buffer-size)|string| -|[nginx.ingress.kubernetes.io/proxy-busy-buffers-size](#proxy-busy-buffers-size)|string| |[nginx.ingress.kubernetes.io/proxy-max-temp-file-size](#proxy-max-temp-file-size)|string| |[nginx.ingress.kubernetes.io/ssl-ciphers](#ssl-ciphers)|string| |[nginx.ingress.kubernetes.io/ssl-prefer-server-ciphers](#ssl-ciphers)|"true" or "false"| |[nginx.ingress.kubernetes.io/connection-proxy-header](#connection-proxy-header)|string| |[nginx.ingress.kubernetes.io/enable-access-log](#enable-access-log)|"true" or "false"| -|[nginx.ingress.kubernetes.io/enable-opentelemetry](#enable-opentelemetry)|"true" or "false"| -|[nginx.ingress.kubernetes.io/opentelemetry-trust-incoming-span](#opentelemetry-trust-incoming-spans)|"true" or "false"| +|[nginx.ingress.kubernetes.io/enable-opentracing](#enable-opentracing)|"true" or "false"| +|[nginx.ingress.kubernetes.io/enable-influxdb](#influxdb)|"true" or "false"| +|[nginx.ingress.kubernetes.io/influxdb-measurement](#influxdb)|string| +|[nginx.ingress.kubernetes.io/influxdb-port](#influxdb)|string| +|[nginx.ingress.kubernetes.io/influxdb-host](#influxdb)|string| +|[nginx.ingress.kubernetes.io/influxdb-server-name](#influxdb)|string| |[nginx.ingress.kubernetes.io/use-regex](#use-regex)|bool| |[nginx.ingress.kubernetes.io/enable-modsecurity](#modsecurity)|bool| |[nginx.ingress.kubernetes.io/enable-owasp-core-rules](#modsecurity)|bool| @@ -131,7 +122,6 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz |[nginx.ingress.kubernetes.io/modsecurity-snippet](#modsecurity)|string| |[nginx.ingress.kubernetes.io/mirror-request-body](#mirror)|string| |[nginx.ingress.kubernetes.io/mirror-target](#mirror)|string| -|[nginx.ingress.kubernetes.io/mirror-host](#mirror)|string| ### Canary @@ -139,20 +129,18 @@ In some cases, you may want to "canary" a new set of changes by sending a small * `nginx.ingress.kubernetes.io/canary-by-header`: The header to use for notifying the Ingress to route the request to the service specified in the Canary Ingress. When the request header is set to `always`, it will be routed to the canary. When the header is set to `never`, it will never be routed to the canary. For any other value, the header will be ignored and the request compared against the other canary rules by precedence. -* `nginx.ingress.kubernetes.io/canary-by-header-value`: The header value to match for notifying the Ingress to route the request to the service specified in the Canary Ingress. When the request header is set to this value, it will be routed to the canary. For any other header value, the header will be ignored and the request compared against the other canary rules by precedence. This annotation has to be used together with `nginx.ingress.kubernetes.io/canary-by-header`. The annotation is an extension of the `nginx.ingress.kubernetes.io/canary-by-header` to allow customizing the header value instead of using hardcoded values. It doesn't have any effect if the `nginx.ingress.kubernetes.io/canary-by-header` annotation is not defined. +* `nginx.ingress.kubernetes.io/canary-by-header-value`: The header value to match for notifying the Ingress to route the request to the service specified in the Canary Ingress. When the request header is set to this value, it will be routed to the canary. For any other header value, the header will be ignored and the request compared against the other canary rules by precedence. This annotation has to be used together with . The annotation is an extension of the `nginx.ingress.kubernetes.io/canary-by-header` to allow customizing the header value instead of using hardcoded values. It doesn't have any effect if the `nginx.ingress.kubernetes.io/canary-by-header` annotation is not defined. * `nginx.ingress.kubernetes.io/canary-by-header-pattern`: This works the same way as `canary-by-header-value` except it does PCRE Regex matching. Note that when `canary-by-header-value` is set this annotation will be ignored. When the given Regex causes error during request processing, the request will be considered as not matching. * `nginx.ingress.kubernetes.io/canary-by-cookie`: The cookie to use for notifying the Ingress to route the request to the service specified in the Canary Ingress. When the cookie value is set to `always`, it will be routed to the canary. When the cookie is set to `never`, it will never be routed to the canary. For any other value, the cookie will be ignored and the request compared against the other canary rules by precedence. -* `nginx.ingress.kubernetes.io/canary-weight`: The integer based (0 - ) percent of random requests that should be routed to the service specified in the canary Ingress. A weight of 0 implies that no requests will be sent to the service in the Canary ingress by this canary rule. A weight of `` means implies all requests will be sent to the alternative service specified in the Ingress. `` defaults to 100, and can be increased via `nginx.ingress.kubernetes.io/canary-weight-total`. - -* `nginx.ingress.kubernetes.io/canary-weight-total`: The total weight of traffic. If unspecified, it defaults to 100. +* `nginx.ingress.kubernetes.io/canary-weight`: The integer based (0 - 100) percent of random requests that should be routed to the service specified in the canary Ingress. A weight of 0 implies that no requests will be sent to the service in the Canary ingress by this canary rule. A weight of 100 means implies all requests will be sent to the alternative service specified in the Ingress. Canary rules are evaluated in order of precedence. Precedence is as follows: `canary-by-header -> canary-by-cookie -> canary-weight` -**Note** that when you mark an ingress as canary, then all the other non-canary annotations will be ignored (inherited from the corresponding main ingress) except `nginx.ingress.kubernetes.io/load-balance`, `nginx.ingress.kubernetes.io/upstream-hash-by`, and [annotations related to session affinity](#session-affinity). If you want to restore the original behavior of canaries when session affinity was ignored, set `nginx.ingress.kubernetes.io/affinity-canary-behavior` annotation with value `legacy` on the canary ingress definition. +**Note** that when you mark an ingress as canary, then all the other non-canary annotations will be ignored (inherited from the corresponding main ingress) except `nginx.ingress.kubernetes.io/load-balance` and `nginx.ingress.kubernetes.io/upstream-hash-by`. **Known Limitations** @@ -175,8 +163,6 @@ The only affinity type available for NGINX is `cookie`. The annotation `nginx.ingress.kubernetes.io/affinity-mode` defines the stickiness of a session. Setting this to `balanced` (default) will redistribute some sessions if a deployment gets scaled up, therefore rebalancing the load on the servers. Setting this to `persistent` will not rebalance sessions to new servers, therefore providing maximum stickiness. -The annotation `nginx.ingress.kubernetes.io/affinity-canary-behavior` defines the behavior of canaries when session affinity is enabled. Setting this to `sticky` (default) will ensure that users that were served by canaries, will continue to be served by canaries. Setting this to `legacy` will restore original canary behavior, when session affinity was ignored. - !!! attention If more than one Ingress is defined for a host and at least one Ingress uses `nginx.ingress.kubernetes.io/affinity: cookie`, then only paths on the Ingress using `nginx.ingress.kubernetes.io/affinity` will use session cookie affinity. All paths defined on other Ingresses for the host will be load balanced through the random selection of a backend server. @@ -189,16 +175,8 @@ If you use the ``cookie`` affinity type you can also specify the name of the coo The NGINX annotation `nginx.ingress.kubernetes.io/session-cookie-path` defines the path that will be set on the cookie. This is optional unless the annotation `nginx.ingress.kubernetes.io/use-regex` is set to true; Session cookie paths do not support regex. -Use `nginx.ingress.kubernetes.io/session-cookie-domain` to set the `Domain` attribute of the sticky cookie. - Use `nginx.ingress.kubernetes.io/session-cookie-samesite` to apply a `SameSite` attribute to the sticky cookie. Browser accepted values are `None`, `Lax`, and `Strict`. Some browsers reject cookies with `SameSite=None`, including those created before the `SameSite=None` specification (e.g. Chrome 5X). Other browsers mistakenly treat `SameSite=None` cookies as `SameSite=Strict` (e.g. Safari running on OSX 14). To omit `SameSite=None` from browsers with these incompatibilities, add the annotation `nginx.ingress.kubernetes.io/session-cookie-conditional-samesite-none: "true"`. -Use `nginx.ingress.kubernetes.io/session-cookie-expires` to control the cookie expires, its value is a number of seconds until the cookie expires. - -Use `nginx.ingress.kubernetes.io/session-cookie-path` to control the cookie path when use-regex is set to true. - -Use `nginx.ingress.kubernetes.io/session-cookie-change-on-failure` to control the cookie change after request failure. - ### Authentication It is possible to add authentication by adding additional annotations in the Ingress rule. The source of the authentication is a secret that contains usernames and passwords. @@ -235,7 +213,7 @@ nginx.ingress.kubernetes.io/auth-realm: "realm string" ### Custom NGINX upstream hashing -NGINX supports load balancing by client-server mapping based on [consistent hashing](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#hash) for a given key. The key can contain text, variables or any combination thereof. This feature allows for request stickiness other than client IP or cookies. The [ketama](https://www.last.fm/user/RJ/journal/2007/04/10/rz_libketama_-_a_consistent_hashing_algo_for_memcache_clients) consistent hashing method will be used which ensures only a few keys would be remapped to different servers on upstream group changes. +NGINX supports load balancing by client-server mapping based on [consistent hashing](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#hash) for a given key. The key can contain text, variables or any combination thereof. This feature allows for request stickiness other than client IP or cookies. The [ketama](https://www.last.fm/user/RJ/journal/2007/04/10/rz_libketama_-_a_consistent_hashing_algo_for_memcache_clients) consistent hashing method will be used which ensures only a few keys would be remapped to different servers on upstream group changes. There is a special mode of upstream hashing called subset. In this mode, upstream servers are grouped into subsets, and stickiness works by mapping keys to a subset instead of individual upstream servers. Specific server is chosen uniformly at random from the selected sticky subset. It provides a balance between stickiness and load distribution. @@ -243,7 +221,7 @@ To enable consistent hashing for a backend: `nginx.ingress.kubernetes.io/upstream-hash-by`: the nginx variable, text value or any combination thereof to use for consistent hashing. For example: `nginx.ingress.kubernetes.io/upstream-hash-by: "$request_uri"` or `nginx.ingress.kubernetes.io/upstream-hash-by: "$request_uri$host"` or `nginx.ingress.kubernetes.io/upstream-hash-by: "${request_uri}-text-value"` to consistently hash upstream requests by the current request URI. -"subset" hashing can be enabled setting `nginx.ingress.kubernetes.io/upstream-hash-by-subset`: "true". This maps requests to subset of nodes instead of a single one. `nginx.ingress.kubernetes.io/upstream-hash-by-subset-size` determines the size of each subset (default 3). +"subset" hashing can be enabled setting `nginx.ingress.kubernetes.io/upstream-hash-by-subset`: "true". This maps requests to subset of nodes instead of a single one. `upstream-hash-by-subset-size` determines the size of each subset (default 3). Please check the [chashsubset](../../examples/chashsubset/deployment.yaml) example. @@ -262,19 +240,23 @@ It is possible to enable Client Certificate Authentication using additional anno Client Certificate Authentication is applied per host and it is not possible to specify rules that differ for individual paths. -To enable, add the annotation `nginx.ingress.kubernetes.io/auth-tls-secret: namespace/secretName`. This secret must have a file named `ca.crt` containing the full Certificate Authority chain `ca.crt` that is enabled to authenticate against this Ingress. +The annotations are: -You can further customize client certificate authentication and behavior with these annotations: - -* `nginx.ingress.kubernetes.io/auth-tls-verify-depth`: The validation depth between the provided client certificate and the Certification Authority chain. (default: 1) -* `nginx.ingress.kubernetes.io/auth-tls-verify-client`: Enables verification of client certificates. Possible values are: - * `on`: Request a client certificate that must be signed by a certificate that is included in the secret key `ca.crt` of the secret specified by `nginx.ingress.kubernetes.io/auth-tls-secret: namespace/secretName`. Failed certificate verification will result in a status code 400 (Bad Request) (default) - * `off`: Don't request client certificates and don't do client certificate verification. - * `optional`: Do optional client certificate validation against the CAs from `auth-tls-secret`. The request fails with status code 400 (Bad Request) when a certificate is provided that is not signed by the CA. When no or an otherwise invalid certificate is provided, the request does not fail, but instead the verification result is sent to the upstream service. - * `optional_no_ca`: Do optional client certificate validation, but do not fail the request when the client certificate is not signed by the CAs from `auth-tls-secret`. Certificate verification result is sent to the upstream service. -* `nginx.ingress.kubernetes.io/auth-tls-error-page`: The URL/Page that user should be redirected in case of a Certificate Authentication Error -* `nginx.ingress.kubernetes.io/auth-tls-pass-certificate-to-upstream`: Indicates if the received certificates should be passed or not to the upstream server in the header `ssl-client-cert`. Possible values are "true" or "false" (default). -* `nginx.ingress.kubernetes.io/auth-tls-match-cn`: Adds a sanity check for the CN of the client certificate that is sent over using a string / regex starting with "CN=", example: `"CN=myvalidclient"`. If the certificate CN sent during mTLS does not match your string / regex it will fail with status code 403. Another way of using this is by adding multiple options in your regex, example: `"CN=(option1|option2|myvalidclient)"`. In this case, as long as one of the options in the brackets matches the certificate CN then you will receive a 200 status code. +* `nginx.ingress.kubernetes.io/auth-tls-secret: secretName`: + The name of the Secret that contains the full Certificate Authority chain `ca.crt` that is enabled to authenticate against this Ingress. + This annotation expects the Secret name in the form "namespace/secretName". +* `nginx.ingress.kubernetes.io/auth-tls-verify-depth`: + The validation depth between the provided client certificate and the Certification Authority chain. +* `nginx.ingress.kubernetes.io/auth-tls-verify-client`: + Enables verification of client certificates. Possible values are: + * `off`: Don't request client certificates and don't do client certificate verification. (default) + * `on`: Request a client certificate that must be signed by a certificate that is included in the secret key `ca.crt` of the secret specified by `nginx.ingress.kubernetes.io/auth-tls-secret: secretName`. Failed certificate verification will result in a status code 400 (Bad Request). + * `optional`: Do optional client certificate validation against the CAs from `auth-tls-secret`. The request fails with status code 400 (Bad Request) when a certificate is provided that is not signed by the CA. When no or an otherwise invalid certificate is provided, the request does not fail, but instead the verification result is sent to the upstream service. + * `optional_no_ca`: Do optional client certificate validation, but do not fail the request when the client certificate is not signed by the CAs from `auth-tls-secret`. Certificate verification result is sent to the upstream service. +* `nginx.ingress.kubernetes.io/auth-tls-error-page`: + The URL/Page that user should be redirected in case of a Certificate Authentication Error +* `nginx.ingress.kubernetes.io/auth-tls-pass-certificate-to-upstream`: + Indicates if the received certificates should be passed or not to the upstream server in the header `ssl-client-cert`. Possible values are "true" or "false" (default). The following headers are sent to the upstream service according to the `auth-tls-*` annotations: @@ -291,7 +273,7 @@ The following headers are sent to the upstream service according to the `auth-tl Cloudflare only allows Authenticated Origin Pulls and is required to use their own certificate: [https://blog.cloudflare.com/protecting-the-origin-with-tls-authenticated-origin-pulls/](https://blog.cloudflare.com/protecting-the-origin-with-tls-authenticated-origin-pulls/) - Only Authenticated Origin Pulls are allowed and can be configured by following their tutorial: [https://support.cloudflare.com/hc/en-us/articles/204494148-Setting-up-NGINX-to-use-TLS-Authenticated-Origin-Pulls](https://web.archive.org/web/20200907143649/https://support.cloudflare.com/hc/en-us/articles/204899617-Setting-up-NGINX-to-use-TLS-Authenticated-Origin-Pulls#section5) + Only Authenticated Origin Pulls are allowed and can be configured by following their tutorial: [https://support.cloudflare.com/hc/en-us/articles/204494148-Setting-up-NGINX-to-use-TLS-Authenticated-Origin-Pulls](https://support.cloudflare.com/hc/en-us/articles/204494148-Setting-up-NGINX-to-use-TLS-Authenticated-Origin-Pulls) ### Backend Certificate Authentication @@ -305,11 +287,11 @@ It is possible to authenticate to a proxied HTTPS backend with certificate using * `nginx.ingress.kubernetes.io/proxy-ssl-verify-depth`: Sets the verification depth in the proxied HTTPS server certificates chain. (default: 1) * `nginx.ingress.kubernetes.io/proxy-ssl-ciphers`: - Specifies the enabled [ciphers](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_ssl_ciphers) for requests to a proxied HTTPS server. The ciphers are specified in the format understood by the OpenSSL library. + Specifies the enabled [ciphers](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_ssl_ciphers) for requests to a proxied HTTPS server. The ciphers are specified in the format understood by the OpenSSL library. * `nginx.ingress.kubernetes.io/proxy-ssl-name`: - Allows to set [proxy_ssl_name](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_ssl_name). This allows overriding the server name used to verify the certificate of the proxied HTTPS server. This value is also passed through SNI when a connection is established to the proxied HTTPS server. + Allows to set [proxy_ssl_name](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_ssl_name). This allows overriding the server name used to verify the certificate of the proxied HTTPS server. This value is also passed through SNI when a connection is established to the proxied HTTPS server. * `nginx.ingress.kubernetes.io/proxy-ssl-protocols`: - Enables the specified [protocols](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_ssl_protocols) for requests to a proxied HTTPS server. + Enables the specified [protocols](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_ssl_protocols) for requests to a proxied HTTPS server. * `nginx.ingress.kubernetes.io/proxy-ssl-server-name`: Enables passing of the server name through TLS Server Name Indication extension (SNI, RFC 6066) when establishing a connection with the proxied HTTPS server. @@ -322,8 +304,6 @@ nginx.ingress.kubernetes.io/configuration-snippet: | more_set_headers "Request-Id: $req_id"; ``` -Be aware this can be dangerous in multi-tenant clusters, as it can lead to people with otherwise limited permissions being able to retrieve all secrets on the cluster. The recommended mitigation for this threat is to disable this feature, so it may not work for you. See CVE-2021-25742 and the [related issue on github](https://github.com/kubernetes/ingress-nginx/issues/7837) for more information. - ### Custom HTTP Errors Like the [`custom-http-errors`](./configmap.md#custom-http-errors) value in the ConfigMap, this annotation will set NGINX `proxy-intercept-errors`, but only for the NGINX location associated with this ingress. If a [default backend annotation](#default-backend) is specified on the ingress, the errors will be routed to that annotation's default backend service (instead of the global default backend). @@ -335,34 +315,11 @@ Example usage: nginx.ingress.kubernetes.io/custom-http-errors: "404,415" ``` -### Custom Headers -This annotation is of the form `nginx.ingress.kubernetes.io/custom-headers: /` to specify a namespace and configmap name that contains custom headers. This annotation uses `more_set_headers` nginx directive. - -Example annotation for following example configmap: - -```yaml -nginx.ingress.kubernetes.io/custom-headers: default/custom-headers-configmap -``` - -Example configmap: -```yaml -apiVersion: v1 -data: - Content-Type: application/json -kind: ConfigMap -metadata: - name: custom-headers-configmap - namespace: default -``` - -!!! attention - First define the allowed response headers in [global-allowed-response-headers](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/configmap.md#global-allowed-response-headers). - ### Default Backend -This annotation is of the form `nginx.ingress.kubernetes.io/default-backend: ` to specify a custom default backend. This `` is a reference to a service inside of the same namespace in which you are applying this annotation. This annotation overrides the global default backend. In case the service has [multiple ports](https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services), the first one is the one which will receive the backend traffic. +This annotation is of the form `nginx.ingress.kubernetes.io/default-backend: ` to specify a custom default backend. This `` is a reference to a service inside of the same namespace in which you are applying this annotation. This annotation overrides the global default backend. -This service will be used to handle the response when the configured service in the Ingress rule does not have any active endpoints. It will also be used to handle the error responses if both this annotation and the [custom-http-errors annotation](#custom-http-errors) are set. +This service will be handle the response when the service in the Ingress rule does not have active endpoints. It will also handle the error responses if both this annotation and the [custom-http-errors annotation](#custom-http-errors) is set. ### Enable CORS @@ -372,46 +329,39 @@ location enabling this functionality. CORS can be controlled with the following annotations: -* `nginx.ingress.kubernetes.io/cors-allow-methods`: Controls which methods are accepted. +* `nginx.ingress.kubernetes.io/cors-allow-methods` + controls which methods are accepted. This is a multi-valued field, separated by ',' and + accepts only letters (upper and lower case). + - Default: `GET, PUT, POST, DELETE, PATCH, OPTIONS` + - Example: `nginx.ingress.kubernetes.io/cors-allow-methods: "PUT, GET, POST, OPTIONS"` - This is a multi-valued field, separated by ',' and accepts only letters (upper and lower case). +* `nginx.ingress.kubernetes.io/cors-allow-headers` + controls which headers are accepted. This is a multi-valued field, separated by ',' and accepts letters, + numbers, _ and -. + - Default: `DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization` + - Example: `nginx.ingress.kubernetes.io/cors-allow-headers: "X-Forwarded-For, X-app123-XPTO"` - - Default: `GET, PUT, POST, DELETE, PATCH, OPTIONS` - - Example: `nginx.ingress.kubernetes.io/cors-allow-methods: "PUT, GET, POST, OPTIONS"` +* `nginx.ingress.kubernetes.io/cors-expose-headers` + controls which headers are exposed to response. This is a multi-valued field, separated by ',' and accepts + letters, numbers, _, - and *. + - Default: *empty* + - Example: `nginx.ingress.kubernetes.io/cors-expose-headers: "*, X-CustomResponseHeader"` -* `nginx.ingress.kubernetes.io/cors-allow-headers`: Controls which headers are accepted. +* `nginx.ingress.kubernetes.io/cors-allow-origin` + controls what's the accepted Origin for CORS. + This is a single field value, with the following format: `http(s)://origin-site.com` or `http(s)://origin-site.com:port` + - Default: `*` + - Example: `nginx.ingress.kubernetes.io/cors-allow-origin: "https://origin-site.com:4443"` - This is a multi-valued field, separated by ',' and accepts letters, numbers, _ and -. +* `nginx.ingress.kubernetes.io/cors-allow-credentials` + controls if credentials can be passed during CORS operations. + - Default: `true` + - Example: `nginx.ingress.kubernetes.io/cors-allow-credentials: "false"` - - Default: `DNT,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization` - - Example: `nginx.ingress.kubernetes.io/cors-allow-headers: "X-Forwarded-For, X-app123-XPTO"` - -* `nginx.ingress.kubernetes.io/cors-expose-headers`: Controls which headers are exposed to response. - - This is a multi-valued field, separated by ',' and accepts letters, numbers, _, - and *. - - - Default: *empty* - - Example: `nginx.ingress.kubernetes.io/cors-expose-headers: "*, X-CustomResponseHeader"` - -* `nginx.ingress.kubernetes.io/cors-allow-origin`: Controls what's the accepted Origin for CORS. - - This is a multi-valued field, separated by ','. It must follow this format: `protocol://origin-site.com` or `protocol://origin-site.com:port` - - - Default: `*` - - Example: `nginx.ingress.kubernetes.io/cors-allow-origin: "https://origin-site.com:4443, http://origin-site.com, myprotocol://example.org:1199"` - - It also supports single level wildcard subdomains and follows this format: `protocol://*.foo.bar`, `protocol://*.bar.foo:8080` or `protocol://*.abc.bar.foo:9000` - - Example: `nginx.ingress.kubernetes.io/cors-allow-origin: "https://*.origin-site.com:4443, http://*.origin-site.com, myprotocol://example.org:1199"` - -* `nginx.ingress.kubernetes.io/cors-allow-credentials`: Controls if credentials can be passed during CORS operations. - - - Default: `true` - - Example: `nginx.ingress.kubernetes.io/cors-allow-credentials: "false"` - -* `nginx.ingress.kubernetes.io/cors-max-age`: Controls how long preflight requests can be cached. - - - Default: `1728000` - - Example: `nginx.ingress.kubernetes.io/cors-max-age: 600` +* `nginx.ingress.kubernetes.io/cors-max-age` + controls how long preflight requests can be cached. + Default: `1728000` + Example: `nginx.ingress.kubernetes.io/cors-max-age: 600` !!! note For more information please see [https://enable-cors.org](https://enable-cors.org/server_nginx.html) @@ -434,14 +384,14 @@ This will create a server with the same configuration, but adding new values to If a server-alias is created and later a new server with the same hostname is created, the new server configuration will take place over the alias configuration. -For more information please see [the `server_name` documentation](https://nginx.org/en/docs/http/ngx_http_core_module.html#server_name). +For more information please see [the `server_name` documentation](http://nginx.org/en/docs/http/ngx_http_core_module.html#server_name). ### Server snippet Using the annotation `nginx.ingress.kubernetes.io/server-snippet` it is possible to add custom configuration in the server configuration block. ```yaml -apiVersion: networking.k8s.io/v1 +apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: annotations: @@ -478,7 +428,7 @@ applied to each location provided in the ingress rule. * `nginx.ingress.kubernetes.io/client-body-buffer-size: 1m` # 1 megabyte * `nginx.ingress.kubernetes.io/client-body-buffer-size: 1M` # 1 megabyte -For more information please see [https://nginx.org](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_buffer_size) +For more information please see [http://nginx.org](http://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_buffer_size) ### External Authentication @@ -490,22 +440,6 @@ nginx.ingress.kubernetes.io/auth-url: "URL to the authentication service" Additionally it is possible to set: -* `nginx.ingress.kubernetes.io/auth-keepalive`: - `` to specify the maximum number of keepalive connections to `auth-url`. Only takes effect - when no variables are used in the host part of the URL. Defaults to `0` (keepalive disabled). - -> Note: does not work with HTTP/2 listener because of a limitation in Lua [subrequests](https://github.com/openresty/lua-nginx-module#spdy-mode-not-fully-supported). -> [UseHTTP2](./configmap.md#use-http2) configuration should be disabled! - -* `nginx.ingress.kubernetes.io/auth-keepalive-share-vars`: - Whether to share Nginx variables among the current request and the auth request. Example use case is to track requests: when set to "true" X-Request-ID HTTP header will be the same for the backend and the auth request. - Defaults to "false". -* `nginx.ingress.kubernetes.io/auth-keepalive-requests`: - `` to specify the maximum number of requests that can be served through one keepalive connection. - Defaults to `1000` and only applied if `auth-keepalive` is set to higher than `0`. -* `nginx.ingress.kubernetes.io/auth-keepalive-timeout`: - `` to specify a duration in seconds which an idle keepalive connection to an upstream server will stay open. - Defaults to `60` and only applied if `auth-keepalive` is set to higher than `0`. * `nginx.ingress.kubernetes.io/auth-method`: `` to specify the HTTP method to use. * `nginx.ingress.kubernetes.io/auth-signin`: @@ -521,9 +455,7 @@ Additionally it is possible to set: * `nginx.ingress.kubernetes.io/auth-cache-key`: `` this enables caching for auth requests. specify a lookup key for auth responses. e.g. `$remote_user$http_authorization`. Each server and location has it's own keyspace. Hence a cached response is only valid on a per-server and per-location basis. * `nginx.ingress.kubernetes.io/auth-cache-duration`: - `` to specify a caching time for auth responses based on their response codes, e.g. `200 202 30m`. See [proxy_cache_valid](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cache_valid) for details. You may specify multiple, comma-separated values: `200 202 10m, 401 5m`. defaults to `200 202 401 5m`. -* `nginx.ingress.kubernetes.io/auth-always-set-cookie`: - `` to set a cookie returned by auth request. By default, the cookie will be set only if an upstream reports with the code 200, 201, 204, 206, 301, 302, 303, 304, 307, or 308. + `` to specify a caching time for auth responses based on their response codes, e.g. `200 202 30m`. See [proxy_cache_valid](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cache_valid) for details. You may specify multiple, comma-separated values: `200 202 10m, 401 5m`. defaults to `200 202 401 5m`. * `nginx.ingress.kubernetes.io/auth-snippet`: `` to specify a custom snippet to use with external authentication, e.g. @@ -550,15 +482,10 @@ By default the controller redirects all requests to an existing service that pro These annotations define limits on connections and transmission rates. These can be used to mitigate [DDoS Attacks](https://www.nginx.com/blog/mitigating-ddos-attacks-with-nginx-and-nginx-plus). -!!! attention - Rate limits are applied per Ingress NGINX controller replica. - If you're running multiple replicas or using a horizontal pod autoscaler (HPA), the effective rate limit will be multiplied by the number of replicas. - When using HPA, the exact rate limit becomes dynamic as the number of replicas may change based on load. - -* `nginx.ingress.kubernetes.io/limit-connections`: number of concurrent connections allowed from a single IP address per controller replica. A 503 error is returned when exceeding this limit. -* `nginx.ingress.kubernetes.io/limit-rps`: number of requests accepted from a given IP each second per controller replica. The burst limit is set to this limit multiplied by the burst multiplier, the default multiplier is 5. When clients exceed this limit, [limit-req-status-code](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#limit-req-status-code) ***default:*** 503 is returned. -* `nginx.ingress.kubernetes.io/limit-rpm`: number of requests accepted from a given IP each minute per controller replica. The burst limit is set to this limit multiplied by the burst multiplier, the default multiplier is 5. When clients exceed this limit, [limit-req-status-code](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#limit-req-status-code) ***default:*** 503 is returned. -* `nginx.ingress.kubernetes.io/limit-burst-multiplier`: multiplier of the limit rate for burst size. The default burst multiplier is 5, this annotation override the default multiplier. When clients exceed this limit, [limit-req-status-code](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#limit-req-status-code) ***default:*** 503 is returned. +* `nginx.ingress.kubernetes.io/limit-connections`: number of concurrent connections allowed from a single IP address. A 503 error is returned when exceeding this limit. +* `nginx.ingress.kubernetes.io/limit-rps`: number of requests accepted from a given IP each second. The burst limit is set to this limit multiplied by the burst multiplier, the default multiplier is 5. When clients exceed this limit, [limit-req-status-code](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#limit-req-status-code) ***default:*** 503 is returned. +* `nginx.ingress.kubernetes.io/limit-rpm`: number of requests accepted from a given IP each minute. The burst limit is set to this limit multiplied by the burst multiplier, the default multiplier is 5. When clients exceed this limit, [limit-req-status-code](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#limit-req-status-code) ***default:*** 503 is returned. +* `nginx.ingress.kubernetes.io/limit-burst-multiplier`: multiplier of the limit rate for burst size. The default burst multiplier is 5, this annotation override the default multiplier. When clients exceed this limit, [limit-req-status-code](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#limit-req-status-code) ***default:*** 503 is returned. * `nginx.ingress.kubernetes.io/limit-rate-after`: initial number of kilobytes after which the further transmission of a response to a given connection will be rate limited. This feature must be used with [proxy-buffering](#proxy-buffering) enabled. * `nginx.ingress.kubernetes.io/limit-rate`: number of kilobytes per second allowed to send to a given connection. The zero value disables rate limiting. This feature must be used with [proxy-buffering](#proxy-buffering) enabled. * `nginx.ingress.kubernetes.io/limit-whitelist`: client IP source ranges to be excluded from rate-limiting. The value is a comma separated list of CIDRs. @@ -569,6 +496,46 @@ To configure settings globally for all Ingress rules, the `limit-rate-after` and The client IP address will be set based on the use of [PROXY protocol](./configmap.md#use-proxy-protocol) or from the `X-Forwarded-For` header value when [use-forwarded-headers](./configmap.md#use-forwarded-headers) is enabled. +### Global Rate Limiting + +**Note:** Be careful when configuring both (Local) Rate Limiting and Global Rate Limiting at the same time. +They are two completely different rate limiting implementations. Whichever limit exceeds first will reject the +requests. It might be a good idea to configure both of them to ease load on Global Rate Limiting backend +in cases of spike in traffic. + +The stock NGINX rate limiting does not share its counters among different NGINX instances. +Given that most ingress-nginx deployments are elastic and number of replicas can change any day +it is impossible to configure a proper rate limit using stock NGINX functionalities. +Global Rate Limiting overcome this by using [lua-resty-global-throttle](https://github.com/ElvinEfendi/lua-resty-global-throttle). `lua-resty-global-throttle` shares its counters via a central store such as `memcached`. +The obvious shortcoming of this is users have to deploy and operate a `memcached` instance +in order to benefit from this functionality. Configure the `memcached` +using [these configmap settings](./configmap.md#global-rate-limit). + +**Here are a few remarks for ingress-nginx integration of `lua-resty-global-throttle`:** + +1. We minimize `memcached` access by caching exceeding limit decisions. The expiry of +cache entry is the desired delay `lua-resty-global-throttle` calculates for us. +The Lua Shared Dictionary used for that is `global_throttle_cache`. Currently its size defaults to 10M. +Customize it as per your needs using [lua-shared-dicts](./configmap.md#lua-shared-dicts). +When we fail to cache the exceeding limit decision then we log an NGINX error. You can monitor +for that error to decide if you need to bump the cache size. Without cache the cost of processing a +request is two memcached commands: `GET`, and `INCR`. With the cache it is only `INCR`. +1. Log NGINX variable `$global_rate_limit_exceeding`'s value to have some visibility into +what portion of requests are rejected (value `y`), whether they are rejected using cached decision (value `c`), +or if they are not rejeced (default value `n`). You can use [log-format-upstream](./configmap.md#log-format-upstream) +to include that in access logs. +1. In case of an error it will log the error message and **fail open**. +1. The annotations below creates Global Rate Limiting instance per ingress. +That means if there are multuple paths configured under the same ingress, +the Global Rate Limiting will count requests to all the paths under the same counter. +Extract a path out into its own ingres if you need to isolate a certain path. + + +* `nginx.ingress.kubernetes.io/global-rate-limit`: Configures maximum allowed number of requests per window. Required. +* `nginx.ingress.kubernetes.io/global-rate-limit-window`: Configures a time window (i.e `1m`) that the limit is applied. Required. +* `nginx.ingress.kubernetes.io/global-rate-limit-key`: Configures a key for counting the samples. Defaults to `$remote_addr`. You can also combine multiple NGINX variables here, like `${remote_addr}-${http_x_api_client}` which would mean the limit will be applied to requests coming from the same API client (indicated by `X-API-Client` HTTP request header) with the same source IP address. +* `nginx.ingress.kubernetes.io/global-rate-limit-ignored-cidrs`: comma separated list of IPs and CIDRs to match client IP against. When there's a match request is not considered for rate limiting. + ### Permanent Redirect This annotation allows to return a permanent redirect (Return Code 301) instead of sending data to the upstream. For example `nginx.ingress.kubernetes.io/permanent-redirect: https://www.google.com` would redirect everything to Google. @@ -580,10 +547,6 @@ This annotation allows you to modify the status code used for permanent redirect ### Temporal Redirect This annotation allows you to return a temporal redirect (Return Code 302) instead of sending data to the upstream. For example `nginx.ingress.kubernetes.io/temporal-redirect: https://www.google.com` would redirect everything to Google with a Return Code of 302 (Moved Temporarily) -### Temporal Redirect Code - -This annotation allows you to modify the status code used for temporal redirects. For example `nginx.ingress.kubernetes.io/temporal-redirect-code: '307'` would return your temporal-redirect with a 307. - ### SSL Passthrough The annotation `nginx.ingress.kubernetes.io/ssl-passthrough` instructs the controller to send TLS connections directly @@ -600,11 +563,11 @@ the User guide. ### Service Upstream -By default the Ingress-Nginx Controller uses a list of all endpoints (Pod IP/port) in the NGINX upstream configuration. +By default the NGINX ingress controller uses a list of all endpoints (Pod IP/port) in the NGINX upstream configuration. The `nginx.ingress.kubernetes.io/service-upstream` annotation disables that behavior and instead uses a single upstream in NGINX, the service's Cluster IP and port. -This can be desirable for things like zero-downtime deployments . See issue [#257](https://github.com/kubernetes/ingress-nginx/issues/257). +This can be desirable for things like zero-downtime deployments as it reduces the need to reload NGINX configuration when Pods come up and down. See issue [#257](https://github.com/kubernetes/ingress-nginx/issues/257). #### Known Issues @@ -625,14 +588,9 @@ When using SSL offloading outside of cluster (e.g. AWS ELB) it may be useful to even when there is no TLS certificate available. This can be achieved by using the `nginx.ingress.kubernetes.io/force-ssl-redirect: "true"` annotation in the particular resource. -To preserve the trailing slash in the URI with `ssl-redirect`, set `nginx.ingress.kubernetes.io/preserve-trailing-slash: "true"` annotation for that particular resource. - ### Redirect from/to www -In some scenarios, it is required to redirect from `www.domain.com` to `domain.com` or vice versa, which way the redirect is performed depends on the configured `host` value in the Ingress object. - -For example, if `.spec.rules.host` is configured with a value like `www.example.com`, then this annotation will redirect from `example.com` to `www.example.com`. If `.spec.rules.host` is configured with a value like `example.com`, so without a `www`, then this annotation will redirect from `www.example.com` to `example.com` instead. - +In some scenarios is required to redirect from `www.domain.com` to `domain.com` or vice versa. To enable this feature use the annotation `nginx.ingress.kubernetes.io/from-to-www-redirect: "true"` !!! attention @@ -641,17 +599,6 @@ To enable this feature use the annotation `nginx.ingress.kubernetes.io/from-to-w !!! attention For HTTPS to HTTPS redirects is mandatory the SSL Certificate defined in the Secret, located in the TLS section of Ingress, contains both FQDN in the common name of the certificate. -### Denylist source range - -You can specify blocked client IP source ranges through the `nginx.ingress.kubernetes.io/denylist-source-range` annotation. -The value is a comma separated list of [CIDRs](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing), e.g. `10.0.0.0/24,172.10.0.1`. - -To configure this setting globally for all Ingress rules, the `denylist-source-range` value may be set in the [NGINX ConfigMap](./configmap.md#denylist-source-range). - -!!! note - Adding an annotation to an Ingress rule overrides any global restriction. - - ### Whitelist source range You can specify allowed client IP source ranges through the `nginx.ingress.kubernetes.io/whitelist-source-range` annotation. @@ -675,18 +622,12 @@ In some scenarios is required to have different values. To allow this we provide - `nginx.ingress.kubernetes.io/proxy-next-upstream-tries` - `nginx.ingress.kubernetes.io/proxy-request-buffering` -If you indicate [Backend Protocol](#backend-protocol) as `GRPC` or `GRPCS`, the following grpc values will be set and inherited from proxy timeouts: - -- [`grpc_connect_timeout=5s`](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_connect_timeout), from `nginx.ingress.kubernetes.io/proxy-connect-timeout` -- [`grpc_send_timeout=60s`](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_send_timeout), from `nginx.ingress.kubernetes.io/proxy-send-timeout` -- [`grpc_read_timeout=60s`](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_read_timeout), from `nginx.ingress.kubernetes.io/proxy-read-timeout` - Note: All timeout values are unitless and in seconds e.g. `nginx.ingress.kubernetes.io/proxy-read-timeout: "120"` sets a valid 120 seconds proxy read timeout. ### Proxy redirect -The annotations `nginx.ingress.kubernetes.io/proxy-redirect-from` and `nginx.ingress.kubernetes.io/proxy-redirect-to` will set the first and second parameters of NGINX's proxy_redirect directive respectively. It is possible to -set the text that should be changed in the `Location` and `Refresh` header fields of a [proxied server response](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_redirect) +With the annotations `nginx.ingress.kubernetes.io/proxy-redirect-from` and `nginx.ingress.kubernetes.io/proxy-redirect-to` it is possible to +set the text that should be changed in the `Location` and `Refresh` header fields of a [proxied server response](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_redirect) Setting "off" or "default" in the annotation `nginx.ingress.kubernetes.io/proxy-redirect-from` disables `nginx.ingress.kubernetes.io/proxy-redirect-to`, otherwise, both annotations must be used in unison. Note that each annotation must be a string without spaces. @@ -695,7 +636,7 @@ By default the value of each annotation is "off". ### Custom max body size -For NGINX, an 413 error will be returned to the client when the size in a request exceeds the maximum allowed size of the client request body. This size can be configured by the parameter [`client_max_body_size`](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size). +For NGINX, an 413 error will be returned to the client when the size in a request exceeds the maximum allowed size of the client request body. This size can be configured by the parameter [`client_max_body_size`](http://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size). To configure this setting globally for all Ingress rules, the `proxy-body-size` value may be set in the [NGINX ConfigMap](./configmap.md#proxy-body-size). To use custom values in an Ingress rule define these annotation: @@ -706,19 +647,19 @@ nginx.ingress.kubernetes.io/proxy-body-size: 8m ### Proxy cookie domain -Sets a text that [should be changed in the domain attribute](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cookie_domain) of the "Set-Cookie" header fields of a proxied server response. +Sets a text that [should be changed in the domain attribute](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cookie_domain) of the "Set-Cookie" header fields of a proxied server response. To configure this setting globally for all Ingress rules, the `proxy-cookie-domain` value may be set in the [NGINX ConfigMap](./configmap.md#proxy-cookie-domain). ### Proxy cookie path -Sets a text that [should be changed in the path attribute](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cookie_path) of the "Set-Cookie" header fields of a proxied server response. +Sets a text that [should be changed in the path attribute](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cookie_path) of the "Set-Cookie" header fields of a proxied server response. To configure this setting globally for all Ingress rules, the `proxy-cookie-path` value may be set in the [NGINX ConfigMap](./configmap.md#proxy-cookie-path). ### Proxy buffering -Enable or disable proxy buffering [`proxy_buffering`](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering). +Enable or disable proxy buffering [`proxy_buffering`](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering). By default proxy buffering is disabled in the NGINX config. To configure this setting globally for all Ingress rules, the `proxy-buffering` value may be set in the [NGINX ConfigMap](./configmap.md#proxy-buffering). @@ -730,7 +671,7 @@ nginx.ingress.kubernetes.io/proxy-buffering: "on" ### Proxy buffers Number -Sets the number of the buffers in [`proxy_buffers`](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) used for reading the first part of the response received from the proxied server. +Sets the number of the buffers in [`proxy_buffers`](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) used for reading the first part of the response received from the proxied server. By default proxy buffers number is set as 4 To configure this setting globally, set `proxy-buffers-number` in [NGINX ConfigMap](./configmap.md#proxy-buffers-number). To use custom values in an Ingress rule, define this annotation: @@ -740,7 +681,7 @@ nginx.ingress.kubernetes.io/proxy-buffers-number: "4" ### Proxy buffer size -Sets the size of the buffer [`proxy_buffer_size`](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) used for reading the first part of the response received from the proxied server. +Sets the size of the buffer [`proxy_buffer_size`](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) used for reading the first part of the response received from the proxied server. By default proxy buffer size is set as "4k" To configure this setting globally, set `proxy-buffer-size` in [NGINX ConfigMap](./configmap.md#proxy-buffer-size). To use custom values in an Ingress rule, define this annotation: @@ -748,21 +689,9 @@ To configure this setting globally, set `proxy-buffer-size` in [NGINX ConfigMap] nginx.ingress.kubernetes.io/proxy-buffer-size: "8k" ``` -### Proxy busy buffers size - -[Limits the total size of buffers that can be busy](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_busy_buffers_size) sending a response to the client while the response is not yet fully read. - -By default proxy busy buffers size is set as "8k". - -To configure this setting globally, set `proxy-busy-buffers-size` in the [ConfigMap](./configmap.md#proxy-busy-buffers-size). To use custom values in an Ingress rule, define this annotation: - -```yaml -nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "16k" -``` - ### Proxy max temp file size -When [`buffering`](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering) of responses from the proxied server is enabled, and the whole response does not fit into the buffers set by the [`proxy_buffer_size`](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) and [`proxy_buffers`](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) directives, a part of the response can be saved to a temporary file. This directive sets the maximum `size` of the temporary file setting the [`proxy_max_temp_file_size`](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_max_temp_file_size). The size of data written to the temporary file at a time is set by the [`proxy_temp_file_write_size`](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_temp_file_write_size) directive. +When [`buffering`](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering) of responses from the proxied server is enabled, and the whole response does not fit into the buffers set by the [`proxy_buffer_size`](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) and [`proxy_buffers`](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) directives, a part of the response can be saved to a temporary file. This directive sets the maximum `size` of the temporary file setting the [`proxy_max_temp_file_size`](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_max_temp_file_size). The size of data written to the temporary file at a time is set by the [`proxy_temp_file_write_size`](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_temp_file_write_size) directive. The zero value disables buffering of responses to temporary files. @@ -773,7 +702,7 @@ nginx.ingress.kubernetes.io/proxy-max-temp-file-size: "1024m" ### Proxy HTTP version -Using this annotation sets the [`proxy_http_version`](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_http_version) that the Nginx reverse proxy will use to communicate with the backend. +Using this annotation sets the [`proxy_http_version`](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_http_version) that the Nginx reverse proxy will use to communicate with the backend. By default this is set to "1.1". ```yaml @@ -782,7 +711,7 @@ nginx.ingress.kubernetes.io/proxy-http-version: "1.0" ### SSL ciphers -Specifies the [enabled ciphers](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_ciphers). +Specifies the [enabled ciphers](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_ciphers). Using this annotation will set the `ssl_ciphers` directive at the server level. This configuration is active for all the paths in the host. @@ -823,22 +752,13 @@ Note that rewrite logs are sent to the error_log file at the notice level. To en nginx.ingress.kubernetes.io/enable-rewrite-log: "true" ``` -### Enable Opentelemetry +### Enable Opentracing -Opentelemetry can be enabled or disabled globally through the ConfigMap but this will sometimes need to be overridden -to enable it or disable it for a specific ingress (e.g. to turn off telemetry of external health check endpoints) +Opentracing can be enabled or disabled globally through the ConfigMap but this will sometimes need to be overridden +to enable it or disable it for a specific ingress (e.g. to turn off tracing of external health check endpoints) ```yaml -nginx.ingress.kubernetes.io/enable-opentelemetry: "true" -``` - -### Opentelemetry Trust Incoming Span - -The option to trust incoming trace spans can be enabled or disabled globally through the ConfigMap but this will -sometimes need to be overridden to enable it or disable it for a specific ingress (e.g. only enable on a private endpoint) - -```yaml -nginx.ingress.kubernetes.io/opentelemetry-trust-incoming-spans: "true" +nginx.ingress.kubernetes.io/enable-opentracing: "true" ``` ### X-Forwarded-Prefix Header @@ -896,10 +816,33 @@ nginx.ingress.kubernetes.io/modsecurity-snippet: | Include /etc/nginx/owasp-modsecurity-crs/nginx-modsecurity.conf ``` +### InfluxDB + +Using `influxdb-*` annotations we can monitor requests passing through a Location by sending them to an InfluxDB backend exposing the UDP socket +using the [nginx-influxdb-module](https://github.com/influxdata/nginx-influxdb-module/). + +```yaml +nginx.ingress.kubernetes.io/enable-influxdb: "true" +nginx.ingress.kubernetes.io/influxdb-measurement: "nginx-reqs" +nginx.ingress.kubernetes.io/influxdb-port: "8089" +nginx.ingress.kubernetes.io/influxdb-host: "127.0.0.1" +nginx.ingress.kubernetes.io/influxdb-server-name: "nginx-ingress" +``` + +For the `influxdb-host` parameter you have two options: + +- Use an InfluxDB server configured with the [UDP protocol](https://docs.influxdata.com/influxdb/v1.5/supported_protocols/udp/) enabled. +- Deploy Telegraf as a sidecar proxy to the Ingress controller configured to listen UDP with the [socket listener input](https://github.com/influxdata/telegraf/tree/release-1.6/plugins/inputs/socket_listener) and to write using +anyone of the [outputs plugins](https://github.com/influxdata/telegraf/tree/release-1.7/plugins/outputs) like InfluxDB, Apache Kafka, +Prometheus, etc.. (recommended) + +It's important to remember that there's no DNS resolver at this stage so you will have to configure +an ip address to `nginx.ingress.kubernetes.io/influxdb-host`. If you deploy Influx or Telegraf as sidecar (another container in the same pod) this becomes straightforward since you can directly use `127.0.0.1`. + ### Backend Protocol Using `backend-protocol` annotations is possible to indicate how NGINX should communicate with the backend service. (Replaces `secure-backends` in older versions) -Valid Values: HTTP, HTTPS, AUTO_HTTP, GRPC, GRPCS and FCGI +Valid Values: HTTP, HTTPS, GRPC, GRPCS, AJP and FCGI By default NGINX uses `HTTP`. @@ -947,7 +890,7 @@ Enables a request to be mirrored to a mirror backend. Responses by mirror backen The mirror backend can be set by applying: ```yaml -nginx.ingress.kubernetes.io/mirror-target: https://test.env.com$request_uri +nginx.ingress.kubernetes.io/mirror-target: https://test.env.com/$request_uri ``` By default the request-body is sent to the mirror backend, but can be turned off by applying: @@ -956,32 +899,8 @@ By default the request-body is sent to the mirror backend, but can be turned off nginx.ingress.kubernetes.io/mirror-request-body: "off" ``` -Also by default header Host for mirrored requests will be set the same as a host part of uri in the "mirror-target" annotation. You can override it by "mirror-host" annotation: - -```yaml -nginx.ingress.kubernetes.io/mirror-target: https://1.2.3.4$request_uri -nginx.ingress.kubernetes.io/mirror-host: "test.env.com" -``` - **Note:** The mirror directive will be applied to all paths within the ingress resource. The request sent to the mirror is linked to the original request. If you have a slow mirror backend, then the original request will throttle. For more information on the mirror module see [ngx_http_mirror_module](https://nginx.org/en/docs/http/ngx_http_mirror_module.html) - - -### Stream snippet - -Using the annotation `nginx.ingress.kubernetes.io/stream-snippet` it is possible to add custom stream configuration. - -```yaml -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - annotations: - nginx.ingress.kubernetes.io/stream-snippet: | - server { - listen 8000; - proxy_pass 127.0.0.1:80; - } -``` diff --git a/docs/user-guide/nginx-configuration/configmap.md b/docs/user-guide/nginx-configuration/configmap.md old mode 100644 new mode 100755 index d8b4f6693..df357d5e9 --- a/docs/user-guide/nginx-configuration/configmap.md +++ b/docs/user-guide/nginx-configuration/configmap.md @@ -5,7 +5,7 @@ ConfigMaps allow you to decouple configuration artifacts from image content to k The ConfigMap API resource stores configuration data as key-value pairs. The data provides the configurations for system components for the nginx-controller. -In order to overwrite nginx-controller configuration values as seen in [config.go](https://github.com/kubernetes/ingress-nginx/blob/main/internal/ingress/controller/config/config.go), +In order to overwrite nginx-controller configuration values as seen in [config.go](https://github.com/kubernetes/ingress-nginx/blob/master/internal/ingress/controller/config/config.go), you can add key-value pairs to the data section of the config-map. For Example: ```yaml @@ -25,270 +25,204 @@ data: The following table shows a configuration option's name, type, and the default value: -| name | type | default | notes | -|:--------------------------------------------------------------------------------|:-------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------| -| [add-headers](#add-headers) | string | "" | | -| [allow-backend-server-header](#allow-backend-server-header) | bool | "false" | | -| [allow-cross-namespace-resources](#allow-cross-namespace-resources) | bool | "false" | | -| [allow-snippet-annotations](#allow-snippet-annotations) | bool | "false" | | -| [annotations-risk-level](#annotations-risk-level) | string | High | | -| [annotation-value-word-blocklist](#annotation-value-word-blocklist) | string array | "" | | -| [hide-headers](#hide-headers) | string array | empty | | -| [access-log-params](#access-log-params) | string | "" | | -| [access-log-path](#access-log-path) | string | "/var/log/nginx/access.log" | | -| [http-access-log-path](#http-access-log-path) | string | "" | | -| [stream-access-log-path](#stream-access-log-path) | string | "" | | -| [enable-access-log-for-default-backend](#enable-access-log-for-default-backend) | bool | "false" | | -| [error-log-path](#error-log-path) | string | "/var/log/nginx/error.log" | | -| [enable-modsecurity](#enable-modsecurity) | bool | "false" | | -| [modsecurity-snippet](#modsecurity-snippet) | string | "" | | -| [enable-owasp-modsecurity-crs](#enable-owasp-modsecurity-crs) | bool | "false" | | -| [client-header-buffer-size](#client-header-buffer-size) | string | "1k" | | -| [client-header-timeout](#client-header-timeout) | int | 60 | | -| [client-body-buffer-size](#client-body-buffer-size) | string | "8k" | | -| [client-body-timeout](#client-body-timeout) | int | 60 | | -| [disable-access-log](#disable-access-log) | bool | "false" | | -| [disable-ipv6](#disable-ipv6) | bool | "false" | | -| [disable-ipv6-dns](#disable-ipv6-dns) | bool | "false" | | -| [enable-underscores-in-headers](#enable-underscores-in-headers) | bool | "false" | | -| [enable-ocsp](#enable-ocsp) | bool | "false" | | -| [ignore-invalid-headers](#ignore-invalid-headers) | bool | "true" | | -| [retry-non-idempotent](#retry-non-idempotent) | bool | "false" | | -| [error-log-level](#error-log-level) | string | "notice" | | -| [http2-max-field-size](#http2-max-field-size) | string | "" | DEPRECATED in favour of [large_client_header_buffers](#large-client-header-buffers) | -| [http2-max-header-size](#http2-max-header-size) | string | "" | DEPRECATED in favour of [large_client_header_buffers](#large-client-header-buffers) | -| [http2-max-requests](#http2-max-requests) | int | 0 | DEPRECATED in favour of [keepalive_requests](#keepalive-requests) | -| [http2-max-concurrent-streams](#http2-max-concurrent-streams) | int | 128 | | -| [hsts](#hsts) | bool | "true" | | -| [hsts-include-subdomains](#hsts-include-subdomains) | bool | "true" | | -| [hsts-max-age](#hsts-max-age) | string | "31536000" | | -| [hsts-preload](#hsts-preload) | bool | "false" | | -| [keep-alive](#keep-alive) | int | 75 | | -| [keep-alive-requests](#keep-alive-requests) | int | 1000 | | -| [large-client-header-buffers](#large-client-header-buffers) | string | "4 8k" | | -| [log-format-escape-none](#log-format-escape-none) | bool | "false" | | -| [log-format-escape-json](#log-format-escape-json) | bool | "false" | | -| [log-format-upstream](#log-format-upstream) | string | `$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstream_name] [$proxy_alternative_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status $req_id` | | -| [log-format-stream](#log-format-stream) | string | `[$remote_addr] [$time_local] $protocol $status $bytes_sent $bytes_received $session_time` | | -| [enable-multi-accept](#enable-multi-accept) | bool | "true" | | -| [max-worker-connections](#max-worker-connections) | int | 16384 | | -| [max-worker-open-files](#max-worker-open-files) | int | 0 | | -| [map-hash-bucket-size](#max-hash-bucket-size) | int | 64 | | -| [nginx-status-ipv4-whitelist](#nginx-status-ipv4-whitelist) | []string | "127.0.0.1" | | -| [nginx-status-ipv6-whitelist](#nginx-status-ipv6-whitelist) | []string | "::1" | | -| [proxy-real-ip-cidr](#proxy-real-ip-cidr) | []string | "0.0.0.0/0" | | -| [proxy-set-headers](#proxy-set-headers) | string | "" | | -| [server-name-hash-max-size](#server-name-hash-max-size) | int | 1024 | | -| [server-name-hash-bucket-size](#server-name-hash-bucket-size) | int | `` | -| [proxy-headers-hash-max-size](#proxy-headers-hash-max-size) | int | 512 | | -| [proxy-headers-hash-bucket-size](#proxy-headers-hash-bucket-size) | int | 64 | | -| [reuse-port](#reuse-port) | bool | "true" | | -| [server-tokens](#server-tokens) | bool | "false" | | -| [ssl-ciphers](#ssl-ciphers) | string | "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384" | | -| [ssl-ecdh-curve](#ssl-ecdh-curve) | string | "auto" | | -| [ssl-dh-param](#ssl-dh-param) | string | "" | | -| [ssl-protocols](#ssl-protocols) | string | "TLSv1.2 TLSv1.3" | | -| [ssl-session-cache](#ssl-session-cache) | bool | "true" | | -| [ssl-session-cache-size](#ssl-session-cache-size) | string | "10m" | | -| [ssl-session-tickets](#ssl-session-tickets) | bool | "false" | | -| [ssl-session-ticket-key](#ssl-session-ticket-key) | string | `` | -| [ssl-session-timeout](#ssl-session-timeout) | string | "10m" | | -| [ssl-buffer-size](#ssl-buffer-size) | string | "4k" | | -| [use-proxy-protocol](#use-proxy-protocol) | bool | "false" | | -| [proxy-protocol-header-timeout](#proxy-protocol-header-timeout) | string | "5s" | | -| [enable-aio-write](#enable-aio-write) | bool | "true" | | -| [use-gzip](#use-gzip) | bool | "false" | | -| [use-geoip](#use-geoip) | bool | "true" | | -| [use-geoip2](#use-geoip2) | bool | "false" | | -| [geoip2-autoreload-in-minutes](#geoip2-autoreload-in-minutes) | int | "0" | | -| [enable-brotli](#enable-brotli) | bool | "false" | | -| [brotli-level](#brotli-level) | int | 4 | | -| [brotli-min-length](#brotli-min-length) | int | 20 | | -| [brotli-types](#brotli-types) | string | "application/xml+rss application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/javascript text/plain text/x-component" | | -| [use-http2](#use-http2) | bool | "true" | | -| [gzip-disable](#gzip-disable) | string | "" | | -| [gzip-level](#gzip-level) | int | 1 | | -| [gzip-min-length](#gzip-min-length) | int | 256 | | -| [gzip-types](#gzip-types) | string | "application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/javascript text/plain text/x-component" | | -| [worker-processes](#worker-processes) | string | `` | | -| [worker-cpu-affinity](#worker-cpu-affinity) | string | "" | | -| [worker-shutdown-timeout](#worker-shutdown-timeout) | string | "240s" | | -| [enable-serial-reloads](#enable-serial-reloads) | bool | "false" | | -| [load-balance](#load-balance) | string | "round_robin" | | -| [variables-hash-bucket-size](#variables-hash-bucket-size) | int | 128 | | -| [variables-hash-max-size](#variables-hash-max-size) | int | 2048 | | -| [upstream-keepalive-connections](#upstream-keepalive-connections) | int | 320 | | -| [upstream-keepalive-time](#upstream-keepalive-time) | string | "1h" | | -| [upstream-keepalive-timeout](#upstream-keepalive-timeout) | int | 60 | | -| [upstream-keepalive-requests](#upstream-keepalive-requests) | int | 10000 | | -| [limit-conn-zone-variable](#limit-conn-zone-variable) | string | "$binary_remote_addr" | | -| [proxy-stream-timeout](#proxy-stream-timeout) | string | "600s" | | -| [proxy-stream-next-upstream](#proxy-stream-next-upstream) | bool | "true" | | -| [proxy-stream-next-upstream-timeout](#proxy-stream-next-upstream-timeout) | string | "600s" | | -| [proxy-stream-next-upstream-tries](#proxy-stream-next-upstream-tries) | int | 3 | | -| [proxy-stream-responses](#proxy-stream-responses) | int | 1 | | -| [bind-address](#bind-address) | []string | "" | | -| [use-forwarded-headers](#use-forwarded-headers) | bool | "false" | | -| [enable-real-ip](#enable-real-ip) | bool | "false" | | -| [forwarded-for-header](#forwarded-for-header) | string | "X-Forwarded-For" | | -| [compute-full-forwarded-for](#compute-full-forwarded-for) | bool | "false" | | -| [proxy-add-original-uri-header](#proxy-add-original-uri-header) | bool | "false" | | -| [generate-request-id](#generate-request-id) | bool | "true" | | -| [jaeger-collector-host](#jaeger-collector-host) | string | "" | | -| [jaeger-collector-port](#jaeger-collector-port) | int | 6831 | | -| [jaeger-endpoint](#jaeger-endpoint) | string | "" | | -| [jaeger-service-name](#jaeger-service-name) | string | "nginx" | | -| [jaeger-propagation-format](#jaeger-propagation-format) | string | "jaeger" | | -| [jaeger-sampler-type](#jaeger-sampler-type) | string | "const" | | -| [jaeger-sampler-param](#jaeger-sampler-param) | string | "1" | | -| [jaeger-sampler-host](#jaeger-sampler-host) | string | "http://127.0.0.1" | | -| [jaeger-sampler-port](#jaeger-sampler-port) | int | 5778 | | -| [jaeger-trace-context-header-name](#jaeger-trace-context-header-name) | string | uber-trace-id | | -| [jaeger-debug-header](#jaeger-debug-header) | string | uber-debug-id | | -| [jaeger-baggage-header](#jaeger-baggage-header) | string | jaeger-baggage | | -| [jaeger-trace-baggage-header-prefix](#jaeger-trace-baggage-header-prefix) | string | uberctx- | | -| [datadog-collector-host](#datadog-collector-host) | string | "" | | -| [datadog-collector-port](#datadog-collector-port) | int | 8126 | | -| [datadog-service-name](#datadog-service-name) | string | "nginx" | | -| [datadog-environment](#datadog-environment) | string | "prod" | | -| [datadog-operation-name-override](#datadog-operation-name-override) | string | "nginx.handle" | | -| [datadog-priority-sampling](#datadog-priority-sampling) | bool | "true" | | -| [datadog-sample-rate](#datadog-sample-rate) | float | 1.0 | | -| [enable-opentelemetry](#enable-opentelemetry) | bool | "false" | | -| [opentelemetry-trust-incoming-span](#opentelemetry-trust-incoming-span) | bool | "true" | | -| [opentelemetry-operation-name](#opentelemetry-operation-name) | string | "" | | -| [opentelemetry-config](#/etc/nginx/opentelemetry.toml) | string | "/etc/nginx/opentelemetry.toml" | | -| [otlp-collector-host](#otlp-collector-host) | string | "" | | -| [otlp-collector-port](#otlp-collector-port) | int | 4317 | | -| [otel-max-queuesize](#otel-max-queuesize) | int | | | -| [otel-schedule-delay-millis](#otel-schedule-delay-millis) | int | | | -| [otel-max-export-batch-size](#otel-max-export-batch-size) | int | | | -| [otel-service-name](#otel-service-name) | string | "nginx" | | -| [otel-sampler](#otel-sampler) | string | "AlwaysOff" | | -| [otel-sampler-parent-based](#otel-sampler-parent-based) | bool | "false" | | -| [otel-sampler-ratio](#otel-sampler-ratio) | float | 0.01 | | -| [main-snippet](#main-snippet) | string | "" | | -| [http-snippet](#http-snippet) | string | "" | | -| [server-snippet](#server-snippet) | string | "" | | -| [stream-snippet](#stream-snippet) | string | "" | | -| [location-snippet](#location-snippet) | string | "" | | -| [custom-http-errors](#custom-http-errors) | []int | []int{} | | -| [proxy-body-size](#proxy-body-size) | string | "1m" | | -| [proxy-connect-timeout](#proxy-connect-timeout) | int | 5 | | -| [proxy-read-timeout](#proxy-read-timeout) | int | 60 | | -| [proxy-send-timeout](#proxy-send-timeout) | int | 60 | | -| [proxy-buffers-number](#proxy-buffers-number) | int | 4 | | -| [proxy-buffer-size](#proxy-buffer-size) | string | "4k" | | -| [proxy-busy-buffers-size](#proxy-busy-buffers-size) | string | "8k" | | -| [proxy-cookie-path](#proxy-cookie-path) | string | "off" | | -| [proxy-cookie-domain](#proxy-cookie-domain) | string | "off" | | -| [proxy-next-upstream](#proxy-next-upstream) | string | "error timeout" | | -| [proxy-next-upstream-timeout](#proxy-next-upstream-timeout) | int | 0 | | -| [proxy-next-upstream-tries](#proxy-next-upstream-tries) | int | 3 | | -| [proxy-redirect-from](#proxy-redirect-from) | string | "off" | | -| [proxy-request-buffering](#proxy-request-buffering) | string | "on" | | -| [ssl-redirect](#ssl-redirect) | bool | "true" | | -| [force-ssl-redirect](#force-ssl-redirect) | bool | "false" | | -| [denylist-source-range](#denylist-source-range) | []string | []string{} | | -| [whitelist-source-range](#whitelist-source-range) | []string | []string{} | | -| [skip-access-log-urls](#skip-access-log-urls) | []string | []string{} | | -| [limit-rate](#limit-rate) | int | 0 | | -| [limit-rate-after](#limit-rate-after) | int | 0 | | -| [lua-shared-dicts](#lua-shared-dicts) | string | "" | | -| [http-redirect-code](#http-redirect-code) | int | 308 | | -| [proxy-buffering](#proxy-buffering) | string | "off" | | -| [limit-req-status-code](#limit-req-status-code) | int | 503 | | -| [limit-conn-status-code](#limit-conn-status-code) | int | 503 | | -| [enable-syslog](#enable-syslog) | bool | "false" | | -| [syslog-host](#syslog-host) | string | "" | | -| [syslog-port](#syslog-port) | int | 514 | | -| [no-tls-redirect-locations](#no-tls-redirect-locations) | string | "/.well-known/acme-challenge" | | -| [global-allowed-response-headers](#global-allowed-response-headers) | string | "" | | -| [global-auth-url](#global-auth-url) | string | "" | | -| [global-auth-method](#global-auth-method) | string | "" | | -| [global-auth-signin](#global-auth-signin) | string | "" | | -| [global-auth-signin-redirect-param](#global-auth-signin-redirect-param) | string | "rd" | | -| [global-auth-response-headers](#global-auth-response-headers) | string | "" | | -| [global-auth-request-redirect](#global-auth-request-redirect) | string | "" | | -| [global-auth-snippet](#global-auth-snippet) | string | "" | | -| [global-auth-cache-key](#global-auth-cache-key) | string | "" | | -| [global-auth-cache-duration](#global-auth-cache-duration) | string | "200 202 401 5m" | | -| [no-auth-locations](#no-auth-locations) | string | "/.well-known/acme-challenge" | | -| [block-cidrs](#block-cidrs) | []string | "" | | -| [block-user-agents](#block-user-agents) | []string | "" | | -| [block-referers](#block-referers) | []string | "" | | -| [proxy-ssl-location-only](#proxy-ssl-location-only) | bool | "false" | | -| [default-type](#default-type) | string | "text/html" | | -| [service-upstream](#service-upstream) | bool | "false" | | -| [ssl-reject-handshake](#ssl-reject-handshake) | bool | "false" | | -| [debug-connections](#debug-connections) | []string | "127.0.0.1,1.1.1.1/24" | | -| [strict-validate-path-type](#strict-validate-path-type) | bool | "true" | | -| [grpc-buffer-size-kb](#grpc-buffer-size-kb) | int | 0 | | -| [relative-redirects](#relative-redirects) | bool | false | | +|name|type|default| +|:---|:---|:------| +|[add-headers](#add-headers)|string|""| +|[allow-backend-server-header](#allow-backend-server-header)|bool|"false"| +|[hide-headers](#hide-headers)|string array|empty| +|[access-log-params](#access-log-params)|string|""| +|[access-log-path](#access-log-path)|string|"/var/log/nginx/access.log"| +|[http-access-log-path](#http-access-log-path)|string|""| +|[stream-access-log-path](#stream-access-log-path)|string|""| +|[enable-access-log-for-default-backend](#enable-access-log-for-default-backend)|bool|"false"| +|[error-log-path](#error-log-path)|string|"/var/log/nginx/error.log"| +|[enable-modsecurity](#enable-modsecurity)|bool|"false"| +|[modsecurity-snippet](#modsecurity-snippet)|string|""| +|[enable-owasp-modsecurity-crs](#enable-owasp-modsecurity-crs)|bool|"false"| +|[client-header-buffer-size](#client-header-buffer-size)|string|"1k"| +|[client-header-timeout](#client-header-timeout)|int|60| +|[client-body-buffer-size](#client-body-buffer-size)|string|"8k"| +|[client-body-timeout](#client-body-timeout)|int|60| +|[disable-access-log](#disable-access-log)|bool|false| +|[disable-ipv6](#disable-ipv6)|bool|false| +|[disable-ipv6-dns](#disable-ipv6-dns)|bool|false| +|[enable-underscores-in-headers](#enable-underscores-in-headers)|bool|false| +|[enable-ocsp](#enable-ocsp)|bool|false| +|[ignore-invalid-headers](#ignore-invalid-headers)|bool|true| +|[retry-non-idempotent](#retry-non-idempotent)|bool|"false"| +|[error-log-level](#error-log-level)|string|"notice"| +|[http2-max-field-size](#http2-max-field-size)|string|"4k"| +|[http2-max-header-size](#http2-max-header-size)|string|"16k"| +|[http2-max-requests](#http2-max-requests)|int|1000| +|[http2-max-concurrent-streams](#http2-max-concurrent-streams)|int|1000| +|[hsts](#hsts)|bool|"true"| +|[hsts-include-subdomains](#hsts-include-subdomains)|bool|"true"| +|[hsts-max-age](#hsts-max-age)|string|"15724800"| +|[hsts-preload](#hsts-preload)|bool|"false"| +|[keep-alive](#keep-alive)|int|75| +|[keep-alive-requests](#keep-alive-requests)|int|100| +|[large-client-header-buffers](#large-client-header-buffers)|string|"4 8k"| +|[log-format-escape-json](#log-format-escape-json)|bool|"false"| +|[log-format-upstream](#log-format-upstream)|string|`$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstream_name] [$proxy_alternative_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status $req_id`| +|[log-format-stream](#log-format-stream)|string|`[$remote_addr] [$time_local] $protocol $status $bytes_sent $bytes_received $session_time`| +|[enable-multi-accept](#enable-multi-accept)|bool|"true"| +|[max-worker-connections](#max-worker-connections)|int|16384| +|[max-worker-open-files](#max-worker-open-files)|int|0| +|[map-hash-bucket-size](#max-hash-bucket-size)|int|64| +|[nginx-status-ipv4-whitelist](#nginx-status-ipv4-whitelist)|[]string|"127.0.0.1"| +|[nginx-status-ipv6-whitelist](#nginx-status-ipv6-whitelist)|[]string|"::1"| +|[proxy-real-ip-cidr](#proxy-real-ip-cidr)|[]string|"0.0.0.0/0"| +|[proxy-set-headers](#proxy-set-headers)|string|""| +|[server-name-hash-max-size](#server-name-hash-max-size)|int|1024| +|[server-name-hash-bucket-size](#server-name-hash-bucket-size)|int|`` +|[proxy-headers-hash-max-size](#proxy-headers-hash-max-size)|int|512| +|[proxy-headers-hash-bucket-size](#proxy-headers-hash-bucket-size)|int|64| +|[plugins](#plugins)|[]string| | +|[reuse-port](#reuse-port)|bool|"true"| +|[server-tokens](#server-tokens)|bool|"false"| +|[ssl-ciphers](#ssl-ciphers)|string|"ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384"| +|[ssl-ecdh-curve](#ssl-ecdh-curve)|string|"auto"| +|[ssl-dh-param](#ssl-dh-param)|string|""| +|[ssl-protocols](#ssl-protocols)|string|"TLSv1.2 TLSv1.3"| +|[ssl-session-cache](#ssl-session-cache)|bool|"true"| +|[ssl-session-cache-size](#ssl-session-cache-size)|string|"10m"| +|[ssl-session-tickets](#ssl-session-tickets)|bool|"false"| +|[ssl-session-ticket-key](#ssl-session-ticket-key)|string|`` +|[ssl-session-timeout](#ssl-session-timeout)|string|"10m"| +|[ssl-buffer-size](#ssl-buffer-size)|string|"4k"| +|[use-proxy-protocol](#use-proxy-protocol)|bool|"false"| +|[proxy-protocol-header-timeout](#proxy-protocol-header-timeout)|string|"5s"| +|[use-gzip](#use-gzip)|bool|"false"| +|[use-geoip](#use-geoip)|bool|"true"| +|[use-geoip2](#use-geoip2)|bool|"false"| +|[enable-brotli](#enable-brotli)|bool|"false"| +|[brotli-level](#brotli-level)|int|4| +|[brotli-types](#brotli-types)|string|"application/xml+rss application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/javascript text/plain text/x-component"| +|[use-http2](#use-http2)|bool|"true"| +|[gzip-level](#gzip-level)|int|1| +|[gzip-types](#gzip-types)|string|"application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/javascript text/plain text/x-component"| +|[worker-processes](#worker-processes)|string|``| +|[worker-cpu-affinity](#worker-cpu-affinity)|string|""| +|[worker-shutdown-timeout](#worker-shutdown-timeout)|string|"240s"| +|[load-balance](#load-balance)|string|"round_robin"| +|[variables-hash-bucket-size](#variables-hash-bucket-size)|int|128| +|[variables-hash-max-size](#variables-hash-max-size)|int|2048| +|[upstream-keepalive-connections](#upstream-keepalive-connections)|int|320| +|[upstream-keepalive-timeout](#upstream-keepalive-timeout)|int|60| +|[upstream-keepalive-requests](#upstream-keepalive-requests)|int|10000| +|[limit-conn-zone-variable](#limit-conn-zone-variable)|string|"$binary_remote_addr"| +|[proxy-stream-timeout](#proxy-stream-timeout)|string|"600s"| +|[proxy-stream-next-upstream](#proxy-stream-next-upstream)|bool|"true"| +|[proxy-stream-next-upstream-timeout](#proxy-stream-next-upstream-timeout)|string|"600s"| +|[proxy-stream-next-upstream-tries](#proxy-stream-next-upstream-tries)|int|3| +|[proxy-stream-responses](#proxy-stream-responses)|int|1| +|[bind-address](#bind-address)|[]string|""| +|[use-forwarded-headers](#use-forwarded-headers)|bool|"false"| +|[enable-real-ip](#enable-real-ip)|bool|"false"| +|[forwarded-for-header](#forwarded-for-header)|string|"X-Forwarded-For"| +|[compute-full-forwarded-for](#compute-full-forwarded-for)|bool|"false"| +|[proxy-add-original-uri-header](#proxy-add-original-uri-header)|bool|"false"| +|[generate-request-id](#generate-request-id)|bool|"true"| +|[enable-opentracing](#enable-opentracing)|bool|"false"| +|[opentracing-operation-name](#opentracing-operation-name)|string|""| +|[opentracing-location-operation-name](#opentracing-location-operation-name)|string|""| +|[zipkin-collector-host](#zipkin-collector-host)|string|""| +|[zipkin-collector-port](#zipkin-collector-port)|int|9411| +|[zipkin-service-name](#zipkin-service-name)|string|"nginx"| +|[zipkin-sample-rate](#zipkin-sample-rate)|float|1.0| +|[jaeger-collector-host](#jaeger-collector-host)|string|""| +|[jaeger-collector-port](#jaeger-collector-port)|int|6831| +|[jaeger-endpoint](#jaeger-endpoint)|string|""| +|[jaeger-service-name](#jaeger-service-name)|string|"nginx"| +|[jaeger-propagation-format](#jaeger-propagation-format)|string|"jaeger"| +|[jaeger-sampler-type](#jaeger-sampler-type)|string|"const"| +|[jaeger-sampler-param](#jaeger-sampler-param)|string|"1"| +|[jaeger-sampler-host](#jaeger-sampler-host)|string|"http://127.0.0.1"| +|[jaeger-sampler-port](#jaeger-sampler-port)|int|5778| +|[jaeger-trace-context-header-name](#jaeger-trace-context-header-name)|string|uber-trace-id| +|[jaeger-debug-header](#jaeger-debug-header)|string|uber-debug-id| +|[jaeger-baggage-header](#jaeger-baggage-header)|string|jaeger-baggage| +|[jaeger-trace-baggage-header-prefix](#jaeger-trace-baggage-header-prefix)|string|uberctx-| +|[datadog-collector-host](#datadog-collector-host)|string|""| +|[datadog-collector-port](#datadog-collector-port)|int|8126| +|[datadog-service-name](#datadog-service-name)|string|"nginx"| +|[datadog-environment](#datadog-environment)|string|"prod"| +|[datadog-operation-name-override](#datadog-operation-name-override)|string|"nginx.handle"| +|[datadog-priority-sampling](#datadog-priority-sampling)|bool|"true"| +|[datadog-sample-rate](#datadog-sample-rate)|float|1.0| +|[main-snippet](#main-snippet)|string|""| +|[http-snippet](#http-snippet)|string|""| +|[server-snippet](#server-snippet)|string|""| +|[location-snippet](#location-snippet)|string|""| +|[custom-http-errors](#custom-http-errors)|[]int|[]int{}| +|[proxy-body-size](#proxy-body-size)|string|"1m"| +|[proxy-connect-timeout](#proxy-connect-timeout)|int|5| +|[proxy-read-timeout](#proxy-read-timeout)|int|60| +|[proxy-send-timeout](#proxy-send-timeout)|int|60| +|[proxy-buffers-number](#proxy-buffers-number)|int|4| +|[proxy-buffer-size](#proxy-buffer-size)|string|"4k"| +|[proxy-cookie-path](#proxy-cookie-path)|string|"off"| +|[proxy-cookie-domain](#proxy-cookie-domain)|string|"off"| +|[proxy-next-upstream](#proxy-next-upstream)|string|"error timeout"| +|[proxy-next-upstream-timeout](#proxy-next-upstream-timeout)|int|0| +|[proxy-next-upstream-tries](#proxy-next-upstream-tries)|int|3| +|[proxy-redirect-from](#proxy-redirect-from)|string|"off"| +|[proxy-request-buffering](#proxy-request-buffering)|string|"on"| +|[ssl-redirect](#ssl-redirect)|bool|"true"| +|[whitelist-source-range](#whitelist-source-range)|[]string|[]string{}| +|[skip-access-log-urls](#skip-access-log-urls)|[]string|[]string{}| +|[limit-rate](#limit-rate)|int|0| +|[limit-rate-after](#limit-rate-after)|int|0| +|[lua-shared-dicts](#lua-shared-dicts)|string|""| +|[http-redirect-code](#http-redirect-code)|int|308| +|[proxy-buffering](#proxy-buffering)|string|"off"| +|[limit-req-status-code](#limit-req-status-code)|int|503| +|[limit-conn-status-code](#limit-conn-status-code)|int|503| +|[no-tls-redirect-locations](#no-tls-redirect-locations)|string|"/.well-known/acme-challenge"| +|[global-auth-url](#global-auth-url)|string|""| +|[global-auth-method](#global-auth-method)|string|""| +|[global-auth-signin](#global-auth-signin)|string|""| +|[global-auth-signin-redirect-param](#global-auth-signin-redirect-param)|string|"rd"| +|[global-auth-response-headers](#global-auth-response-headers)|string|""| +|[global-auth-request-redirect](#global-auth-request-redirect)|string|""| +|[global-auth-snippet](#global-auth-snippet)|string|""| +|[global-auth-cache-key](#global-auth-cache-key)|string|""| +|[global-auth-cache-duration](#global-auth-cache-duration)|string|"200 202 401 5m"| +|[no-auth-locations](#no-auth-locations)|string|"/.well-known/acme-challenge"| +|[block-cidrs](#block-cidrs)|[]string|""| +|[block-user-agents](#block-user-agents)|[]string|""| +|[block-referers](#block-referers)|[]string|""| +|[proxy-ssl-location-only](#proxy-ssl-location-only)|bool|"false"| +|[default-type](#default-type)|string|"text/html"| +|[global-rate-limit-memcached-host](#global-rate-limit)|string|""| +|[global-rate-limit-memcached-port](#global-rate-limit)|int|11211| +|[global-rate-limit-memcached-connect-timeout](#global-rate-limit)|int|50| +|[global-rate-limit-memcached-max-idle-timeout](#global-rate-limit)|int|10000| +|[global-rate-limit-memcached-pool-size](#global-rate-limit)|int|50| +|[global-rate-limit-status-code](#global-rate-limit)|int|429| ## add-headers -Sets custom headers from named configmap before sending traffic to the client. See [proxy-set-headers](#proxy-set-headers). [example](https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/custom-headers) +Sets custom headers from named configmap before sending traffic to the client. See [proxy-set-headers](#proxy-set-headers). [example](https://github.com/kubernetes/ingress-nginx/tree/master/docs/examples/customization/custom-headers) ## allow-backend-server-header Enables the return of the header Server from the backend instead of the generic nginx string. _**default:**_ is disabled -## allow-cross-namespace-resources - -Enables users to consume cross namespace resource on annotations, when was previously enabled . _**default:**_ false - -**Annotations that may be impacted with this change**: - -* `auth-secret` -* `auth-proxy-set-header` -* `auth-tls-secret` -* `fastcgi-params-configmap` -* `proxy-ssl-secret` - -## allow-snippet-annotations - -Enables Ingress to parse and add *-snippet annotations/directives created by the user. _**default:**_ `false` - -Warning: We recommend enabling this option only if you TRUST users with permission to create Ingress objects, as this -may allow a user to add restricted configurations to the final nginx.conf file - -## annotations-risk-level - -Represents the risk accepted on an annotation. If the risk is, for instance `Medium`, annotations with risk High and Critical will not be accepted. - -Accepted values are `Critical`, `High`, `Medium` and `Low`. - -_**default:**_ `High` - -## annotation-value-word-blocklist - -Contains a comma-separated value of chars/words that are well known of being used to abuse Ingress configuration -and must be blocked. Related to [CVE-2021-25742](https://github.com/kubernetes/ingress-nginx/issues/7837) - -When an annotation is detected with a value that matches one of the blocked bad words, the whole Ingress won't be configured. - -_**default:**_ `""` - -When doing this, the default blocklist is override, which means that the Ingress admin should add all the words -that should be blocked, here is a suggested block list. - -_**suggested:**_ `"load_module,lua_package,_by_lua,location,root,proxy_pass,serviceaccount,{,},',\""` - ## hide-headers Sets additional header that will not be passed from the upstream server to the client response. _**default:**_ empty _References:_ -[https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_hide_header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_hide_header) +[http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_hide_header](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_hide_header) ## access-log-params Additional params for access_log. For example, buffer=16k, gzip, flush=1m _References:_ -[https://nginx.org/en/docs/http/ngx_http_log_module.html#access_log](https://nginx.org/en/docs/http/ngx_http_log_module.html#access_log) +[http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log](http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log) ## access-log-path @@ -321,7 +255,7 @@ Error log path. Goes to `/var/log/nginx/error.log` by default. __Note:__ the file `/var/log/nginx/error.log` is a symlink to `/dev/stderr` _References:_ -[https://nginx.org/en/docs/ngx_core_module.html#error_log](https://nginx.org/en/docs/ngx_core_module.html#error_log) +[http://nginx.org/en/docs/ngx_core_module.html#error_log](http://nginx.org/en/docs/ngx_core_module.html#error_log) ## enable-modsecurity @@ -340,35 +274,35 @@ Adds custom rules to modsecurity section of nginx configuration Allows to configure a custom buffer size for reading client request header. _References:_ -[https://nginx.org/en/docs/http/ngx_http_core_module.html#client_header_buffer_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_header_buffer_size) +[http://nginx.org/en/docs/http/ngx_http_core_module.html#client_header_buffer_size](http://nginx.org/en/docs/http/ngx_http_core_module.html#client_header_buffer_size) ## client-header-timeout Defines a timeout for reading client request header, in seconds. _References:_ -[https://nginx.org/en/docs/http/ngx_http_core_module.html#client_header_timeout](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_header_timeout) +[http://nginx.org/en/docs/http/ngx_http_core_module.html#client_header_timeout](http://nginx.org/en/docs/http/ngx_http_core_module.html#client_header_timeout) ## client-body-buffer-size Sets buffer size for reading client request body. _References:_ -[https://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_buffer_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_buffer_size) +[http://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_buffer_size](http://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_buffer_size) ## client-body-timeout Defines a timeout for reading client request body, in seconds. _References:_ -[https://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_timeout](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_timeout) +[http://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_timeout](http://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_timeout) ## disable-access-log -Disables the Access Log from the entire Ingress Controller. _**default:**_ `false` +Disables the Access Log from the entire Ingress Controller. _**default:**_ '"false"' _References:_ -[https://nginx.org/en/docs/http/ngx_http_log_module.html#access_log](https://nginx.org/en/docs/http/ngx_http_log_module.html#access_log) +[http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log](http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log) ## disable-ipv6 @@ -401,13 +335,10 @@ Since 1.9.13 NGINX will not retry non-idempotent requests (POST, LOCK, PATCH) in Configures the logging level of errors. Log levels above are listed in the order of increasing severity. _References:_ -[https://nginx.org/en/docs/ngx_core_module.html#error_log](https://nginx.org/en/docs/ngx_core_module.html#error_log) +[http://nginx.org/en/docs/ngx_core_module.html#error_log](http://nginx.org/en/docs/ngx_core_module.html#error_log) ## http2-max-field-size -!!! warning - This feature was deprecated in 1.1.3 and will be removed in 1.3.0. Use [large-client-header-buffers](#large-client-header-buffers) instead. - Limits the maximum size of an HPACK-compressed request header field. _References:_ @@ -415,9 +346,6 @@ _References:_ ## http2-max-header-size -!!! warning - This feature was deprecated in 1.1.3 and will be removed in 1.3.0. Use [large-client-header-buffers](#large-client-header-buffers) instead. - Limits the maximum size of the entire request header list after HPACK decompression. _References:_ @@ -425,20 +353,17 @@ _References:_ ## http2-max-requests -!!! warning - This feature was deprecated in 1.1.3 and will be removed in 1.3.0. Use [upstream-keepalive-requests](#upstream-keepalive-requests) instead. - Sets the maximum number of requests (including push requests) that can be served through one HTTP/2 connection, after which the next client request will lead to connection closing and the need of establishing a new connection. _References:_ -[https://nginx.org/en/docs/http/ngx_http_v2_module.html#http2_max_requests](https://nginx.org/en/docs/http/ngx_http_v2_module.html#http2_max_requests) +[http://nginx.org/en/docs/http/ngx_http_v2_module.html#http2_max_requests](http://nginx.org/en/docs/http/ngx_http_v2_module.html#http2_max_requests) ## http2-max-concurrent-streams Sets the maximum number of concurrent HTTP/2 streams in a connection. _References:_ -[https://nginx.org/en/docs/http/ngx_http_v2_module.html#http2_max_concurrent_streams](https://nginx.org/en/docs/http/ngx_http_v2_module.html#http2_max_concurrent_streams) +[http://nginx.org/en/docs/http/ngx_http_v2_module.html#http2_max_concurrent_streams](http://nginx.org/en/docs/http/ngx_http_v2_module.html#http2_max_concurrent_streams) ## hsts @@ -460,63 +385,41 @@ Sets the time, in seconds, that the browser should remember that this site is on ## hsts-preload -Enables or disables the preload attribute in the HSTS feature (when it is enabled). +Enables or disables the preload attribute in the HSTS feature (when it is enabled) dd ## keep-alive -Sets the time, in seconds, during which a keep-alive client connection will stay open on the server side. The zero value disables keep-alive client connections. +Sets the time during which a keep-alive client connection will stay open on the server side. The zero value disables keep-alive client connections. _References:_ -[https://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_timeout](https://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_timeout) - -!!! important - Setting `keep-alive: '0'` will most likely break concurrent http/2 requests due to changes introduced with nginx 1.19.7 - -``` -Changes with nginx 1.19.7 16 Feb 2021 - - *) Change: connections handling in HTTP/2 has been changed to better - match HTTP/1.x; the "http2_recv_timeout", "http2_idle_timeout", and - "http2_max_requests" directives have been removed, the - "keepalive_timeout" and "keepalive_requests" directives should be - used instead. -``` - -_References:_ -[nginx change log](https://nginx.org/en/CHANGES) -[nginx issue tracker](https://trac.nginx.org/nginx/ticket/2155) -[nginx mailing list](https://mailman.nginx.org/pipermail/nginx/2021-May/060697.html) +[http://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_timeout](http://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_timeout) ## keep-alive-requests Sets the maximum number of requests that can be served through one keep-alive connection. _References:_ -[https://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_requests](https://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_requests) +[http://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_requests](http://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_requests) ## large-client-header-buffers Sets the maximum number and size of buffers used for reading large client request header. _**default:**_ 4 8k _References:_ -[https://nginx.org/en/docs/http/ngx_http_core_module.html#large_client_header_buffers](https://nginx.org/en/docs/http/ngx_http_core_module.html#large_client_header_buffers) - -## log-format-escape-none - -Sets if the escape parameter is disabled entirely for character escaping in variables ("true") or controlled by log-format-escape-json ("false") Sets the nginx [log format](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format). +[http://nginx.org/en/docs/http/ngx_http_core_module.html#large_client_header_buffers](http://nginx.org/en/docs/http/ngx_http_core_module.html#large_client_header_buffers) ## log-format-escape-json -Sets if the escape parameter allows JSON ("true") or default characters escaping in variables ("false") Sets the nginx [log format](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format). +Sets if the escape parameter allows JSON ("true") or default characters escaping in variables ("false") Sets the nginx [log format](http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format). ## log-format-upstream -Sets the nginx [log format](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format). +Sets the nginx [log format](http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format). Example for json output: ```json -log-format-upstream: '{"time": "$time_iso8601", "remote_addr": "$proxy_protocol_addr", "x_forwarded_for": "$proxy_add_x_forwarded_for", "request_id": "$req_id", +log-format-upstream: '{"time": "$time_iso8601", "remote_addr": "$proxy_protocol_addr", "x_forward_for": "$proxy_add_x_forwarded_for", "request_id": "$req_id", "remote_user": "$remote_user", "bytes_sent": $bytes_sent, "request_time": $request_time, "status": $status, "vhost": "$host", "request_proto": "$server_protocol", "path": "$uri", "request_query": "$args", "request_length": $request_length, "duration": $request_time,"method": "$request_method", "http_referrer": "$http_referer", "http_user_agent": "$http_user_agent" }' @@ -534,11 +437,11 @@ If disabled, a worker process will accept one new connection at a time. Otherwis _**default:**_ true _References:_ -[https://nginx.org/en/docs/ngx_core_module.html#multi_accept](https://nginx.org/en/docs/ngx_core_module.html#multi_accept) +[http://nginx.org/en/docs/ngx_core_module.html#multi_accept](http://nginx.org/en/docs/ngx_core_module.html#multi_accept) ## max-worker-connections -Sets the [maximum number of simultaneous connections](https://nginx.org/en/docs/ngx_core_module.html#worker_connections) that can be opened by each worker process. +Sets the [maximum number of simultaneous connections](http://nginx.org/en/docs/ngx_core_module.html#worker_connections) that can be opened by each worker process. 0 will use the value of [max-worker-open-files](#max-worker-open-files). _**default:**_ 16384 @@ -547,13 +450,13 @@ _**default:**_ 16384 ## max-worker-open-files -Sets the [maximum number of files](https://nginx.org/en/docs/ngx_core_module.html#worker_rlimit_nofile) that can be opened by each worker process. -The default of 0 means "max open files (system's limit) - 1024". +Sets the [maximum number of files](http://nginx.org/en/docs/ngx_core_module.html#worker_rlimit_nofile) that can be opened by each worker process. +The default of 0 means "max open files (system's limit) / [worker-processes](#worker-processes) - 1024". _**default:**_ 0 ## map-hash-bucket-size -Sets the bucket size for the [map variables hash tables](https://nginx.org/en/docs/http/ngx_http_map_module.html#map_hash_bucket_size). The details of setting up hash tables are provided in a separate [document](https://nginx.org/en/docs/hash.html). +Sets the bucket size for the [map variables hash tables](http://nginx.org/en/docs/http/ngx_http_map_module.html#map_hash_bucket_size). The details of setting up hash tables are provided in a separate [document](http://nginx.org/en/docs/hash.html). ## proxy-real-ip-cidr @@ -562,14 +465,14 @@ _**default:**_ "0.0.0.0/0" ## proxy-set-headers -Sets custom headers from named configmap before sending traffic to backends. The value format is namespace/name. See [example](https://kubernetes.github.io/ingress-nginx/examples/customization/custom-headers/) +Sets custom headers from named configmap before sending traffic to backends. The value format is namespace/name. See [example](https://github.com/kubernetes/ingress-nginx/tree/master/docs/examples/customization/custom-headers) ## server-name-hash-max-size -Sets the maximum size of the [server names hash tables](https://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_max_size) used in server names,map directive’s values, MIME types, names of request header strings, etc. +Sets the maximum size of the [server names hash tables](http://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_max_size) used in server names,map directive’s values, MIME types, names of request header strings, etc. _References:_ -[https://nginx.org/en/docs/hash.html](https://nginx.org/en/docs/hash.html) +[http://nginx.org/en/docs/hash.html](http://nginx.org/en/docs/hash.html) ## server-name-hash-bucket-size @@ -577,8 +480,8 @@ Sets the size of the bucket for the server names hash tables. _References:_ -- [https://nginx.org/en/docs/hash.html](https://nginx.org/en/docs/hash.html) -- [https://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_bucket_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_bucket_size) +- [http://nginx.org/en/docs/hash.html](http://nginx.org/en/docs/hash.html) +- [http://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_bucket_size](http://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_bucket_size) ## proxy-headers-hash-max-size @@ -586,7 +489,7 @@ Sets the maximum size of the proxy headers hash tables. _References:_ -- [https://nginx.org/en/docs/hash.html](https://nginx.org/en/docs/hash.html) +- [http://nginx.org/en/docs/hash.html](http://nginx.org/en/docs/hash.html) - [https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_headers_hash_max_size](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_headers_hash_max_size) ## reuse-port @@ -600,23 +503,27 @@ Sets the size of the bucket for the proxy headers hash tables. _References:_ -- [https://nginx.org/en/docs/hash.html](https://nginx.org/en/docs/hash.html) +- [http://nginx.org/en/docs/hash.html](http://nginx.org/en/docs/hash.html) - [https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_headers_hash_bucket_size](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_headers_hash_bucket_size) +## plugins + +Activates plugins installed in `/etc/nginx/lua/plugins`. Refer to [ingress-nginx plugins README](https://github.com/kubernetes/ingress-nginx/blob/master/rootfs/etc/nginx/lua/plugins/README.md) for more information on how to write and install a plugin. + ## server-tokens Send NGINX Server header in responses and display NGINX version in error pages. _**default:**_ is disabled ## ssl-ciphers -Sets the [ciphers](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_ciphers) list to enable. The ciphers are specified in the format understood by the OpenSSL library. +Sets the [ciphers](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_ciphers) list to enable. The ciphers are specified in the format understood by the OpenSSL library. The default cipher list is: `ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384`. The ordering of a ciphersuite is very important because it decides which algorithms are going to be selected in priority. The recommendation above prioritizes algorithms that provide perfect [forward secrecy](https://wiki.mozilla.org/Security/Server_Side_TLS#Forward_Secrecy). -DHE-based cyphers will not be available until DH parameter is configured [Custom DH parameters for perfect forward secrecy](https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/ssl-dh-param) +DHE-based cyphers will not be available until DH parameter is configured [Custom DH parameters for perfect forward secrecy](https://github.com/kubernetes/ingress-nginx/tree/master/docs/examples/customization/ssl-dh-param) Please check the [Mozilla SSL Configuration Generator](https://mozilla.github.io/server-side-tls/ssl-config-generator/). @@ -627,7 +534,7 @@ __Note:__ ssl_prefer_server_ciphers directive will be enabled by default for htt Specifies a curve for ECDHE ciphers. _References:_ -[https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_ecdh_curve](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_ecdh_curve) +[http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_ecdh_curve](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_ecdh_curve) ## ssl-dh-param @@ -637,11 +544,11 @@ _References:_ - [https://wiki.openssl.org/index.php/Diffie-Hellman_parameters](https://wiki.openssl.org/index.php/Diffie-Hellman_parameters) - [https://wiki.mozilla.org/Security/Server_Side_TLS#DHE_handshake_and_dhparam](https://wiki.mozilla.org/Security/Server_Side_TLS#DHE_handshake_and_dhparam) -- [https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_dhparam](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_dhparam) +- [http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_dhparam](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_dhparam) ## ssl-protocols -Sets the [SSL protocols](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_protocols) to use. The default is: `TLSv1.2 TLSv1.3`. +Sets the [SSL protocols](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_protocols) to use. The default is: `TLSv1.2 TLSv1.3`. Please check the result of the configuration using `https://ssllabs.com/ssltest/analyze.html` or `https://testssl.sh`. @@ -653,34 +560,34 @@ Time Resumption (0-RTT). This requires `ssl-protocols` to have `TLSv1.3` enabled. Enable this with caution, because requests sent within early data are subject to [replay attacks](https://tools.ietf.org/html/rfc8470). -[ssl_early_data](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_early_data). The default is: `false`. +[ssl_early_data](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_early_data). The default is: `false`. ## ssl-session-cache -Enables or disables the use of shared [SSL cache](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_cache) among worker processes. +Enables or disables the use of shared [SSL cache](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_cache) among worker processes. ## ssl-session-cache-size -Sets the size of the [SSL shared session cache](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_cache) between all worker processes. +Sets the size of the [SSL shared session cache](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_cache) between all worker processes. ## ssl-session-tickets -Enables or disables session resumption through [TLS session tickets](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_tickets). +Enables or disables session resumption through [TLS session tickets](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_tickets). ## ssl-session-ticket-key Sets the secret key used to encrypt and decrypt TLS session tickets. The value must be a valid base64 string. To create a ticket: `openssl rand 80 | openssl enc -A -base64` -[TLS session ticket-key](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_tickets), by default, a randomly generated key is used. +[TLS session ticket-key](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_tickets), by default, a randomly generated key is used. ## ssl-session-timeout -Sets the time during which a client may [reuse the session](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_timeout) parameters stored in a cache. +Sets the time during which a client may [reuse the session](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_timeout) parameters stored in a cache. ## ssl-buffer-size -Sets the size of the [SSL buffer](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_buffer_size) used for sending data. The default of 4k helps NGINX to improve TLS Time To First Byte (TTTFB). +Sets the size of the [SSL buffer](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_buffer_size) used for sending data. The default of 4k helps NGINX to improve TLS Time To First Byte (TTTFB). _References:_ [https://www.igvita.com/2013/12/16/optimizing-nginx-tls-time-to-first-byte/](https://www.igvita.com/2013/12/16/optimizing-nginx-tls-time-to-first-byte/) @@ -694,17 +601,13 @@ Enables or disables the [PROXY protocol](https://www.nginx.com/resources/admin-g Sets the timeout value for receiving the proxy-protocol headers. The default of 5 seconds prevents the TLS passthrough handler from waiting indefinitely on a dropped connection. _**default:**_ 5s -## enable-aio-write - -Enables or disables the directive [aio_write](https://nginx.org/en/docs/http/ngx_http_core_module.html#aio_write) that writes files asynchronously. _**default:**_ true - ## use-gzip -Enables or disables compression of HTTP responses using the ["gzip" module](https://nginx.org/en/docs/http/ngx_http_gzip_module.html). MIME types to compress are controlled by [gzip-types](#gzip-types). _**default:**_ false +Enables or disables compression of HTTP responses using the ["gzip" module](http://nginx.org/en/docs/http/ngx_http_gzip_module.html). MIME types to compress are controlled by [gzip-types](#gzip-types). _**default:**_ false ## use-geoip -Enables or disables ["geoip" module](https://nginx.org/en/docs/http/ngx_http_geoip_module.html) that creates variables with values depending on the client IP address, using the precompiled MaxMind databases. +Enables or disables ["geoip" module](http://nginx.org/en/docs/http/ngx_http_geoip_module.html) that creates variables with values depending on the client IP address, using the precompiled MaxMind databases. _**default:**_ true > __Note:__ MaxMind legacy databases are discontinued and will not receive updates after 2019-01-02, cf. [discontinuation notice](https://support.maxmind.com/geolite-legacy-discontinuation-notice/). Consider [use-geoip2](#use-geoip2) below. @@ -712,26 +615,19 @@ _**default:**_ true ## use-geoip2 Enables the [geoip2 module](https://github.com/leev/ngx_http_geoip2_module) for NGINX. -Since `0.27.0` and due to a [change in the MaxMind databases](https://blog.maxmind.com/2019/12/significant-changes-to-accessing-and-using-geolite2-databases/) a license is required to have access to the databases. +Since `0.27.0` and due to a [change in the MaxMind databases](https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases) a license is required to have access to the databases. For this reason, it is required to define a new flag `--maxmind-license-key` in the ingress controller deployment to download the databases needed during the initialization of the ingress controller. -Alternatively, it is possible to use a volume to mount the files `/etc/ingress-controller/geoip/GeoLite2-City.mmdb` and `/etc/ingress-controller/geoip/GeoLite2-ASN.mmdb`, avoiding the overhead of the download. +Alternatively, it is possible to use a volume to mount the files `/etc/nginx/geoip/GeoLite2-City.mmdb` and `/etc/nginx/geoip/GeoLite2-ASN.mmdb`, avoiding the overhead of the download. !!! important If the feature is enabled but the files are missing, GeoIP2 will not be enabled. _**default:**_ false -## geoip2-autoreload-in-minutes - -Enables the [geoip2 module](https://github.com/leev/ngx_http_geoip2_module) autoreload in MaxMind databases setting the interval in minutes. - -_**default:**_ 0 - ## enable-brotli Enables or disables compression of HTTP responses using the ["brotli" module](https://github.com/google/ngx_brotli). -The default mime type list to compress is: `application/xml+rss application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component`. -_**default:**_ false +The default mime type list to compress is: `application/xml+rss application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component`. _**default:**_ is disabled > __Note:__ Brotli does not works in Safari < 11. For more information see [https://caniuse.com/#feat=brotli](https://caniuse.com/#feat=brotli) @@ -739,10 +635,6 @@ _**default:**_ false Sets the Brotli Compression Level that will be used. _**default:**_ 4 -## brotli-min-length - -Minimum length of responses, in bytes, that will be eligible for brotli compression. _**default:**_ 20 - ## brotli-types Sets the MIME Types that will be compressed on-the-fly by brotli. @@ -750,11 +642,7 @@ _**default:**_ `application/xml+rss application/atom+xml application/javascript ## use-http2 -Enables or disables [HTTP/2](https://nginx.org/en/docs/http/ngx_http_v2_module.html) support in secure connections. - -## gzip-disable - -Disables [gzipping](http://nginx.org/en/docs/http/ngx_http_gzip_module.html#gzip_disable) of responses for requests with "User-Agent" header fields matching any of the specified regular expressions. +Enables or disables [HTTP/2](http://nginx.org/en/docs/http/ngx_http_v2_module.html) support in secure connections. ## gzip-level @@ -771,12 +659,12 @@ _**default:**_ `application/atom+xml application/javascript application/x-javasc ## worker-processes -Sets the number of [worker processes](https://nginx.org/en/docs/ngx_core_module.html#worker_processes). +Sets the number of [worker processes](http://nginx.org/en/docs/ngx_core_module.html#worker_processes). The default of "auto" means number of available CPU cores. ## worker-cpu-affinity -Binds worker processes to the sets of CPUs. [worker_cpu_affinity](https://nginx.org/en/docs/ngx_core_module.html#worker_cpu_affinity). +Binds worker processes to the sets of CPUs. [worker_cpu_affinity](http://nginx.org/en/docs/ngx_core_module.html#worker_cpu_affinity). By default worker processes are not bound to any specific CPUs. The value can be: - "": empty string indicate no affinity is applied. @@ -785,7 +673,7 @@ By default worker processes are not bound to any specific CPUs. The value can be ## worker-shutdown-timeout -Sets a timeout for Nginx to [wait for worker to gracefully shutdown](https://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout). _**default:**_ "240s" +Sets a timeout for Nginx to [wait for worker to gracefully shutdown](http://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout). _**default:**_ "240s" ## load-balance @@ -793,7 +681,7 @@ Sets the algorithm to use for load balancing. The value can either be: - round_robin: to use the default round robin loadbalancer -- ewma: to use the Peak EWMA method for routing ([implementation](https://github.com/kubernetes/ingress-nginx/blob/main/rootfs/etc/nginx/lua/balancer/ewma.lua)) +- ewma: to use the Peak EWMA method for routing ([implementation](https://github.com/kubernetes/ingress-nginx/blob/master/rootfs/etc/nginx/lua/balancer/ewma.lua)) The default is `round_robin`. @@ -801,21 +689,21 @@ The default is `round_robin`. - To load balance using session cookies, consider the `nginx.ingress.kubernetes.io/affinity` annotation. _References:_ -[https://nginx.org/en/docs/http/load_balancing.html](https://nginx.org/en/docs/http/load_balancing.html) +[http://nginx.org/en/docs/http/load_balancing.html](http://nginx.org/en/docs/http/load_balancing.html) ## variables-hash-bucket-size Sets the bucket size for the variables hash table. _References:_ -[https://nginx.org/en/docs/http/ngx_http_map_module.html#variables_hash_bucket_size](https://nginx.org/en/docs/http/ngx_http_map_module.html#variables_hash_bucket_size) +[http://nginx.org/en/docs/http/ngx_http_map_module.html#variables_hash_bucket_size](http://nginx.org/en/docs/http/ngx_http_map_module.html#variables_hash_bucket_size) ## variables-hash-max-size Sets the maximum size of the variables hash table. _References:_ -[https://nginx.org/en/docs/http/ngx_http_map_module.html#variables_hash_max_size](https://nginx.org/en/docs/http/ngx_http_map_module.html#variables_hash_max_size) +[http://nginx.org/en/docs/http/ngx_http_map_module.html#variables_hash_max_size](http://nginx.org/en/docs/http/ngx_http_map_module.html#variables_hash_max_size) ## upstream-keepalive-connections @@ -825,24 +713,16 @@ exceeded, the least recently used connections are closed. _**default:**_ 320 _References:_ -[https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) +[http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) -## upstream-keepalive-time - -Sets the maximum time during which requests can be processed through one keepalive connection. - _**default:**_ "1h" - -_References:_ -[http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_time](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_time) - ## upstream-keepalive-timeout Sets a timeout during which an idle keepalive connection to an upstream server will stay open. _**default:**_ 60 _References:_ -[https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_timeout](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_timeout) +[http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_timeout](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_timeout) ## upstream-keepalive-requests @@ -853,47 +733,47 @@ _**default:**_ 10000 _References:_ -[https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_requests](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_requests) +[http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_requests](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_requests) ## limit-conn-zone-variable -Sets parameters for a shared memory zone that will keep states for various keys of [limit_conn_zone](https://nginx.org/en/docs/http/ngx_http_limit_conn_module.html#limit_conn_zone). The default of "$binary_remote_addr" variable’s size is always 4 bytes for IPv4 addresses or 16 bytes for IPv6 addresses. +Sets parameters for a shared memory zone that will keep states for various keys of [limit_conn_zone](http://nginx.org/en/docs/http/ngx_http_limit_conn_module.html#limit_conn_zone). The default of "$binary_remote_addr" variable’s size is always 4 bytes for IPv4 addresses or 16 bytes for IPv6 addresses. ## proxy-stream-timeout Sets the timeout between two successive read or write operations on client or proxied server connections. If no data is transmitted within this time, the connection is closed. _References:_ -[https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_timeout](https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_timeout) +[http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_timeout](http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_timeout) ## proxy-stream-next-upstream When a connection to the proxied server cannot be established, determines whether a client connection will be passed to the next server. _References:_ -[https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream](https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream) +[http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream](http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream) ## proxy-stream-next-upstream-timeout Limits the time allowed to pass a connection to the next server. The 0 value turns off this limitation. _References:_ -[https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream_timeout](https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream_timeout) +[http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream_timeout](http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream_timeout) ## proxy-stream-next-upstream-tries Limits the number of possible tries a request should be passed to the next server. The 0 value turns off this limitation. _References:_ -[https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream_tries](https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream_timeout) +[http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream_tries](http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_next_upstream_timeout) ## proxy-stream-responses Sets the number of datagrams expected from the proxied server in response to the client request if the UDP protocol is used. _References:_ -[https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_responses](https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_responses) +[http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_responses](http://nginx.org/en/docs/stream/ngx_stream_proxy_module.html#proxy_responses) ## bind-address @@ -907,7 +787,7 @@ If false, NGINX ignores incoming `X-Forwarded-*` headers, filling them with the ## enable-real-ip -`enable-real-ip` enables the configuration of [https://nginx.org/en/docs/http/ngx_http_realip_module.html](https://nginx.org/en/docs/http/ngx_http_realip_module.html). Specific attributes of the module can be configured further by using `forwarded-for-header` and `proxy-real-ip-cidr` settings. +`enable-real-ip` enables the configuration of [http://nginx.org/en/docs/http/ngx_http_realip_module.html](http://nginx.org/en/docs/http/ngx_http_realip_module.html). Specific attributes of the module can be configured further by using `forwarded-for-header` and `proxy-real-ip-cidr` settings. ## forwarded-for-header @@ -925,6 +805,41 @@ Adds an X-Original-Uri header with the original request URI to the backend reque Ensures that X-Request-ID is defaulted to a random value, if no X-Request-ID is present in the request +## enable-opentracing + +Enables the nginx Opentracing extension. _**default:**_ is disabled + +_References:_ +[https://github.com/opentracing-contrib/nginx-opentracing](https://github.com/opentracing-contrib/nginx-opentracing) + +## opentracing-operation-name + +Specifies a custom name for the server span. _**default:**_ is empty + +For example, set to "HTTP $request_method $uri". + +## opentracing-location-operation-name + +Specifies a custom name for the location span. _**default:**_ is empty + +For example, set to "HTTP $request_method $uri". + +## zipkin-collector-host + +Specifies the host to use when uploading traces. It must be a valid URL. + +## zipkin-collector-port + +Specifies the port to use when uploading traces. _**default:**_ 9411 + +## zipkin-service-name + +Specifies the service name to use for any traces created. _**default:**_ nginx + +## zipkin-sample-rate + +Specifies sample rate for any traces created. _**default:**_ 1.0 + ## jaeger-collector-host Specifies the host to use when uploading traces. It must be a valid URL. @@ -997,7 +912,7 @@ Specifies the environment this trace belongs to. _**default:**_ prod ## datadog-operation-name-override -Overrides the operation name to use for any traces crated. _**default:**_ nginx.handle +Overrides the operation naem to use for any traces crated. _**default:**_ nginx.handle ## datadog-priority-sampling @@ -1009,46 +924,6 @@ If true disables client-side sampling (thus ignoring `sample_rate`) and enables Specifies sample rate for any traces created. This is effective only when `datadog-priority-sampling` is `false` _**default:**_ 1.0 -## enable-opentelemetry - -Enables the nginx OpenTelemetry extension. _**default:**_ is disabled - -_References:_ -[https://github.com/open-telemetry/opentelemetry-cpp-contrib](https://github.com/open-telemetry/opentelemetry-cpp-contrib/tree/main/instrumentation/nginx) - -## opentelemetry-operation-name - -Specifies a custom name for the server span. _**default:**_ is empty - -For example, set to "HTTP $request_method $uri". - -## otlp-collector-host - -Specifies the host to use when uploading traces. It must be a valid URL. - -## otlp-collector-port - -Specifies the port to use when uploading traces. _**default:**_ 4317 - -## otel-service-name - -Specifies the service name to use for any traces created. _**default:**_ nginx - -## opentelemetry-trust-incoming-span: "true" -Enables or disables using spans from incoming requests as parent for created ones. _**default:**_ true - -## otel-sampler-parent-based - -Uses sampler implementation which by default will take a sample if parent Activity is sampled. _**default:**_ false - -## otel-sampler-ratio - -Specifies sample rate for any traces created. _**default:**_ 0.01 - -## otel-sampler - -Specifies the sampler to be used when sampling traces. The available samplers are: AlwaysOff, AlwaysOn, TraceIdRatioBased, remote. _**default:**_ AlwaysOff - ## main-snippet Adds custom configuration to the main section of the nginx configuration. @@ -1061,10 +936,6 @@ Adds custom configuration to the http section of the nginx configuration. Adds custom configuration to all the servers in the nginx configuration. -## stream-snippet - -Adds custom configuration to the stream section of the nginx configuration. - ## location-snippet Adds custom configuration to all the locations in the nginx configuration. @@ -1073,96 +944,77 @@ You can not use this to add new locations that proxy to the Kubernetes pods, as ## custom-http-errors -Enables which HTTP codes should be passed for processing with the [error_page directive](https://nginx.org/en/docs/http/ngx_http_core_module.html#error_page) +Enables which HTTP codes should be passed for processing with the [error_page directive](http://nginx.org/en/docs/http/ngx_http_core_module.html#error_page) -Setting at least one code also enables [proxy_intercept_errors](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_intercept_errors) which are required to process error_page. +Setting at least one code also enables [proxy_intercept_errors](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_intercept_errors) which are required to process error_page. Example usage: `custom-http-errors: 404,415` ## proxy-body-size Sets the maximum allowed size of the client request body. -See NGINX [client_max_body_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size). +See NGINX [client_max_body_size](http://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size). ## proxy-connect-timeout -Sets the timeout for [establishing a connection with a proxied server](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout). It should be noted that this timeout cannot usually exceed 75 seconds. - -It will also set the [grpc_connect_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_connect_timeout) for gRPC connections. +Sets the timeout for [establishing a connection with a proxied server](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout). It should be noted that this timeout cannot usually exceed 75 seconds. ## proxy-read-timeout -Sets the timeout in seconds for [reading a response from the proxied server](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout). The timeout is set only between two successive read operations, not for the transmission of the whole response. - -It will also set the [grpc_read_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_read_timeout) for gRPC connections. +Sets the timeout in seconds for [reading a response from the proxied server](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout). The timeout is set only between two successive read operations, not for the transmission of the whole response. ## proxy-send-timeout -Sets the timeout in seconds for [transmitting a request to the proxied server](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_send_timeout). The timeout is set only between two successive write operations, not for the transmission of the whole request. - -It will also set the [grpc_send_timeout](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_send_timeout) for gRPC connections. +Sets the timeout in seconds for [transmitting a request to the proxied server](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_send_timeout). The timeout is set only between two successive write operations, not for the transmission of the whole request. ## proxy-buffers-number -Sets the number of the buffer used for [reading the first part of the response](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) received from the proxied server. This part usually contains a small response header. +Sets the number of the buffer used for [reading the first part of the response](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) received from the proxied server. This part usually contains a small response header. ## proxy-buffer-size -Sets the size of the buffer used for [reading the first part of the response](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) received from the proxied server. This part usually contains a small response header. - -## proxy-busy-buffers-size - -[Limits the total size of buffers that can be busy](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_busy_buffers_size) sending a response to the client while the response is not yet fully read. +Sets the size of the buffer used for [reading the first part of the response](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) received from the proxied server. This part usually contains a small response header. ## proxy-cookie-path -Sets a text that [should be changed in the path attribute](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cookie_path) of the “Set-Cookie” header fields of a proxied server response. +Sets a text that [should be changed in the path attribute](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cookie_path) of the “Set-Cookie” header fields of a proxied server response. ## proxy-cookie-domain -Sets a text that [should be changed in the domain attribute](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cookie_domain) of the “Set-Cookie” header fields of a proxied server response. +Sets a text that [should be changed in the domain attribute](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cookie_domain) of the “Set-Cookie” header fields of a proxied server response. ## proxy-next-upstream -Specifies in [which cases](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream) a request should be passed to the next server. +Specifies in [which cases](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream) a request should be passed to the next server. ## proxy-next-upstream-timeout -[Limits the time](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream_timeout) in seconds during which a request can be passed to the next server. +[Limits the time](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream_timeout) in seconds during which a request can be passed to the next server. ## proxy-next-upstream-tries -Limit the number of [possible tries](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream_tries) a request should be passed to the next server. +Limit the number of [possible tries](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream_tries) a request should be passed to the next server. ## proxy-redirect-from Sets the original text that should be changed in the "Location" and "Refresh" header fields of a proxied server response. _**default:**_ off _References:_ -[https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_redirect](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_redirect) +[http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_redirect](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_redirect) ## proxy-request-buffering -Enables or disables [buffering of a client request body](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_request_buffering). +Enables or disables [buffering of a client request body](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_request_buffering). ## ssl-redirect Sets the global value of redirects (301) to HTTPS if the server has a TLS certificate (defined in an Ingress rule). _**default:**_ "true" -## force-ssl-redirect -Sets the global value of redirects (308) to HTTPS if the server has a default TLS certificate (defined in extra-args). -_**default:**_ "false" - -## denylist-source-range - -Sets the default denylisted IPs for each `server` block. This can be overwritten by an annotation on an Ingress rule. -See [ngx_http_access_module](https://nginx.org/en/docs/http/ngx_http_access_module.html). - ## whitelist-source-range Sets the default whitelisted IPs for each `server` block. This can be overwritten by an annotation on an Ingress rule. -See [ngx_http_access_module](https://nginx.org/en/docs/http/ngx_http_access_module.html). +See [ngx_http_access_module](http://nginx.org/en/docs/http/ngx_http_access_module.html). ## skip-access-log-urls @@ -1173,15 +1025,12 @@ Sets a list of URLs that should not appear in the NGINX access log. This is usef Limits the rate of response transmission to a client. The rate is specified in bytes per second. The zero value disables rate limiting. The limit is set per a request, and so if a client simultaneously opens two connections, the overall rate will be twice as much as the specified limit. _References:_ -[https://nginx.org/en/docs/http/ngx_http_core_module.html#limit_rate](https://nginx.org/en/docs/http/ngx_http_core_module.html#limit_rate) +[http://nginx.org/en/docs/http/ngx_http_core_module.html#limit_rate](http://nginx.org/en/docs/http/ngx_http_core_module.html#limit_rate) ## limit-rate-after Sets the initial amount after which the further transmission of a response to a client will be rate limited. -_References:_ -[https://nginx.org/en/docs/http/ngx_http_core_module.html#limit_rate_after](https://nginx.org/en/docs/http/ngx_http_core_module.html#limit_rate_after) - ## lua-shared-dicts Customize default Lua shared dictionaries or define more. You can use the following syntax to do so: @@ -1197,11 +1046,8 @@ For example following will set default `certificate_data` dictionary to `100M` a lua-shared-dicts: "certificate_data: 100, my_custom_plugin: 5" ``` -You can optionally set a size unit to allow for kilobyte-granularity. Allowed units are 'm' or 'k' (case-insensitive), and it defaults to MB if no unit is provided. Here is a similar example, but the `my_custom_plugin` dict is only 512KB. - -``` -lua-shared-dicts: "certificate_data: 100, my_custom_plugin: 512k" -``` +_References:_ +[http://nginx.org/en/docs/http/ngx_http_core_module.html#limit_rate_after](http://nginx.org/en/docs/http/ngx_http_core_module.html#limit_rate_after) ## http-redirect-code @@ -1215,37 +1061,21 @@ _**default:**_ 308 ## proxy-buffering -Enables or disables [buffering of responses from the proxied server](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering). +Enables or disables [buffering of responses from the proxied server](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering). ## limit-req-status-code -Sets the [status code to return in response to rejected requests](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req_status). _**default:**_ 503 +Sets the [status code to return in response to rejected requests](http://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req_status). _**default:**_ 503 ## limit-conn-status-code -Sets the [status code to return in response to rejected connections](https://nginx.org/en/docs/http/ngx_http_limit_conn_module.html#limit_conn_status). _**default:**_ 503 - -## enable-syslog - -Enable [syslog](https://nginx.org/en/docs/syslog.html) feature for access log and error log. _**default:**_ false - -## syslog-host - -Sets the address of syslog server. The address can be specified as a domain name or IP address. - -## syslog-port - -Sets the port of syslog server. _**default:**_ 514 +Sets the [status code to return in response to rejected connections](http://nginx.org/en/docs/http/ngx_http_limit_conn_module.html#limit_conn_status). _**default:**_ 503 ## no-tls-redirect-locations A comma-separated list of locations on which http requests will never get redirected to their https counterpart. _**default:**_ "/.well-known/acme-challenge" -## global-allowed-response-headers - -A comma-separated list of allowed response headers inside the [custom headers annotations](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/annotations.md#custom-headers) - ## global-auth-url A url to an existing service that provides authentication for all the locations. @@ -1253,7 +1083,7 @@ Similar to the Ingress rule annotation `nginx.ingress.kubernetes.io/auth-url`. Locations that should not get authenticated can be listed using `no-auth-locations` See [no-auth-locations](#no-auth-locations). In addition, each service can be excluded from authentication via annotation `enable-global-auth` set to "false". _**default:**_ "" -_References:_ [https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/annotations.md#external-authentication](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/annotations.md#external-authentication) +_References:_ [https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/annotations.md#external-authentication](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/annotations.md#external-authentication) ## global-auth-method @@ -1288,7 +1118,7 @@ _**default:**_ "" ## global-auth-snippet Sets a custom snippet to use with external authentication. Applied to all the locations. -Similar to the Ingress rule annotation `nginx.ingress.kubernetes.io/auth-snippet`. +Similar to the Ingress rule annotation `nginx.ingress.kubernetes.io/auth-request-redirect`. _**default:**_ "" ## global-auth-cache-key @@ -1297,12 +1127,7 @@ Enables caching for global auth requests. Specify a lookup key for auth response ## global-auth-cache-duration -Set a caching time for auth responses based on their response codes, e.g. `200 202 30m`. See [proxy_cache_valid](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cache_valid) for details. You may specify multiple, comma-separated values: `200 202 10m, 401 5m`. defaults to `200 202 401 5m`. - -## global-auth-always-set-cookie - -Always set a cookie returned by auth request. By default, the cookie will be set only if an upstream reports with the code 200, 201, 204, 206, 301, 302, 303, 304, 307, or 308. -_**default:**_ false +Set a caching time for auth responses based on their response codes, e.g. `200 202 30m`. See [proxy_cache_valid](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cache_valid) for details. You may specify multiple, comma-separated values: `200 202 10m, 401 5m`. defaults to `200 202 401 5m`. ## no-auth-locations @@ -1314,7 +1139,7 @@ _**default:**_ "/.well-known/acme-challenge" A comma-separated list of IP addresses (or subnets), request from which have to be blocked globally. _References:_ -[https://nginx.org/en/docs/http/ngx_http_access_module.html#deny](https://nginx.org/en/docs/http/ngx_http_access_module.html#deny) +[http://nginx.org/en/docs/http/ngx_http_access_module.html#deny](http://nginx.org/en/docs/http/ngx_http_access_module.html#deny) ## block-user-agents @@ -1322,7 +1147,7 @@ A comma-separated list of User-Agent, request from which have to be blocked glob It's possible to use here full strings and regular expressions. More details about valid patterns can be found at `map` Nginx directive documentation. _References:_ -[https://nginx.org/en/docs/http/ngx_http_map_module.html#map](https://nginx.org/en/docs/http/ngx_http_map_module.html#map) +[http://nginx.org/en/docs/http/ngx_http_map_module.html#map](http://nginx.org/en/docs/http/ngx_http_map_module.html#map) ## block-referers @@ -1330,7 +1155,7 @@ A comma-separated list of Referers, request from which have to be blocked global It's possible to use here full strings and regular expressions. More details about valid patterns can be found at `map` Nginx directive documentation. _References:_ -[https://nginx.org/en/docs/http/ngx_http_map_module.html#map](https://nginx.org/en/docs/http/ngx_http_map_module.html#map) +[http://nginx.org/en/docs/http/ngx_http_map_module.html#map](http://nginx.org/en/docs/http/ngx_http_map_module.html#map) ## proxy-ssl-location-only @@ -1343,59 +1168,20 @@ Sets the default MIME type of a response. _**default:**_ text/html _References:_ -[https://nginx.org/en/docs/http/ngx_http_core_module.html#default_type](https://nginx.org/en/docs/http/ngx_http_core_module.html#default_type) +[http://nginx.org/en/docs/http/ngx_http_core_module.html#default_type](http://nginx.org/en/docs/http/ngx_http_core_module.html#default_type) -## service-upstream +## global-rate-limit -Set if the service's Cluster IP and port should be used instead of a list of all endpoints. This can be overwritten by an annotation on an Ingress rule. -_**default:**_ "false" +* `global-rate-limit-status-code`: configure HTTP status code to return when rejecting requests. Defaults to 429. -## ssl-reject-handshake +Configure `memcached` client for [Global Rate Limiting](https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/annotations.md#global-rate-limiting). -Set to reject SSL handshake to an unknown virtualhost. This parameter helps to mitigate the fingerprinting using default certificate of ingress. -_**default:**_ "false" +* `global-rate-limit-memcached-host`: IP/FQDN of memcached server to use. Required to enable Global Rate Limiting. +* `global-rate-limit-memcached-port`: port of memcached server to use. Defaults default memcached port of `11211`. +* `global-rate-limit-memcached-connect-timeout`: configure timeout for connect, send and receive operations. Unit is millisecond. Defaults to 50ms. +* `global-rate-limit-memcached-max-idle-timeout`: configure timeout for cleaning idle connections. Unit is millisecond. Defaults to 50ms. +* `global-rate-limit-memcached-pool-size`: configure number of max connections to keep alive. Make sure your `memcached` server can handle +`global-rate-limit-memcached-pool-size * worker-processes * ` simultaneous connections. -_References:_ -[https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_reject_handshake](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_reject_handshake) - -## debug-connections -Enables debugging log for selected client connections. -_**default:**_ "" - -_References:_ -[http://nginx.org/en/docs/ngx_core_module.html#debug_connection](http://nginx.org/en/docs/ngx_core_module.html#debug_connection) - -## strict-validate-path-type - -Ingress objects contains a field called pathType that defines the proxy behavior. It can be `Exact`, `Prefix` and `ImplementationSpecific`. - -When pathType is configured as `Exact` or `Prefix`, there should be a more strict validation, allowing only paths starting with "/" and -containing only alphanumeric characters and "-", "_" and additional "/". - -When this option is enabled, the validation will happen on the Admission Webhook, making any Ingress not using pathType `ImplementationSpecific` -and containing invalid characters to be denied. - -This means that Ingress objects that rely on paths containing regex characters should use `ImplementationSpecific` pathType. - -The cluster admin should establish validation rules using mechanisms like [Open Policy Agent](https://www.openpolicyagent.org/) to -validate that only authorized users can use `ImplementationSpecific` pathType and that only the authorized characters can be used. - -_**default:**_ "true" - -## grpc-buffer-size-kb - -Sets the configuration for the GRPC Buffer Size parameter. If not set it will use the default from NGINX. - -_References:_ -[https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_buffer_size](https://nginx.org/en/docs/http/ngx_http_grpc_module.html#grpc_buffer_size) - -## relative-redirects - -Use relative redirects instead of absolute redirects. Absolute redirects are the default in nginx. RFC7231 allows relative redirects since 2014. -Similar to the Ingress rule annotation `nginx.ingress.kubernetes.io/relative-redirects`. - -_**default:**_ "false" - -_References:_ -- [https://nginx.org/en/docs/http/ngx_http_core_module.html#absolute_redirect](https://nginx.org/en/docs/http/ngx_http_core_module.html#absolute_redirect) -- [https://datatracker.ietf.org/doc/html/rfc7231#section-7.1.2](https://datatracker.ietf.org/doc/html/rfc7231#section-7.1.2) +These settings get used by [lua-resty-global-throttle](https://github.com/ElvinEfendi/lua-resty-global-throttle) +that ingress-nginx includes. Refer to the link to learn more about `lua-resty-global-throttle`. diff --git a/docs/user-guide/nginx-configuration/index.md b/docs/user-guide/nginx-configuration/index.md index 232dd5cf8..bace1a086 100644 --- a/docs/user-guide/nginx-configuration/index.md +++ b/docs/user-guide/nginx-configuration/index.md @@ -4,4 +4,4 @@ There are three ways to customize NGINX: 1. [ConfigMap](./configmap.md): using a Configmap to set global configurations in NGINX. 2. [Annotations](./annotations.md): use this if you want a specific configuration for a particular Ingress rule. -3. [Custom template](./custom-template.md): when more specific settings are required, like [open_file_cache](https://nginx.org/en/docs/http/ngx_http_core_module.html#open_file_cache), adjust [listen](https://nginx.org/en/docs/http/ngx_http_core_module.html#listen) options as `rcvbuf` or when is not possible to change the configuration through the ConfigMap. +3. [Custom template](./custom-template.md): when more specific settings are required, like [open_file_cache](http://nginx.org/en/docs/http/ngx_http_core_module.html#open_file_cache), adjust [listen](http://nginx.org/en/docs/http/ngx_http_core_module.html#listen) options as `rcvbuf` or when is not possible to change the configuration through the ConfigMap. diff --git a/docs/user-guide/nginx-configuration/log-format.md b/docs/user-guide/nginx-configuration/log-format.md index 9a7506480..e39c5f695 100644 --- a/docs/user-guide/nginx-configuration/log-format.md +++ b/docs/user-guide/nginx-configuration/log-format.md @@ -29,7 +29,7 @@ log_format upstreaminfo | `$upstream_response_length` | the length of the response obtained from the upstream server | | `$upstream_response_time` | time spent on receiving the response from the upstream server as seconds with millisecond resolution | | `$upstream_status` | status code of the response obtained from the upstream server | -| `$req_id` | value of the `X-Request-ID` HTTP header. If the header is not set, a randomly generated ID. | +| `$req_id` | the randomly generated ID of the request | Additional available variables: @@ -43,5 +43,5 @@ Additional available variables: Sources: -- [Upstream variables](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#variables) -- [Embedded variables](https://nginx.org/en/docs/http/ngx_http_core_module.html#variables) +- [Upstream variables](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#variables) +- [Embedded variables](http://nginx.org/en/docs/http/ngx_http_core_module.html#variables) diff --git a/docs/user-guide/third-party-addons/modsecurity.md b/docs/user-guide/third-party-addons/modsecurity.md index 38d39888d..93a7f4097 100644 --- a/docs/user-guide/third-party-addons/modsecurity.md +++ b/docs/user-guide/third-party-addons/modsecurity.md @@ -14,97 +14,3 @@ The default `Serial` value in SecAuditLogType can impact performance. The OWASP ModSecurity Core Rule Set (CRS) is a set of generic attack detection rules for use with ModSecurity or compatible web application firewalls. The CRS aims to protect web applications from a wide range of attacks, including the OWASP Top Ten, with a minimum of false alerts. The directory `/etc/nginx/owasp-modsecurity-crs` contains the [OWASP ModSecurity Core Rule Set repository](https://github.com/coreruleset/coreruleset). Using `enable-owasp-modsecurity-crs: "true"` we enable the use of the rules. - -## Supported annotations - -For more info on supported annotations, please see [annotations/#modsecurity](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#modsecurity) - -## Example of using ModSecurity with plugins via the helm chart - -Suppose you have a ConfigMap that contains the contents of the [nextcloud-rule-exclusions plugin](https://github.com/coreruleset/nextcloud-rule-exclusions-plugin/blob/main/plugins/nextcloud-rule-exclusions-before.conf) like this: - -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: modsecurity-plugins -data: - empty-after.conf: | - # no data - empty-before.conf: | - # no data - empty-config.conf: | - # no data - nextcloud-rule-exclusions-before.conf: - # this is just a snippet - # find the full file at https://github.com/coreruleset/nextcloud-rule-exclusions-plugin - # - # [ File Manager ] - # The web interface uploads files, and interacts with the user. - SecRule REQUEST_FILENAME "@contains /remote.php/webdav" \ - "id:9508102,\ - phase:1,\ - pass,\ - t:none,\ - nolog,\ - ver:'nextcloud-rule-exclusions-plugin/1.2.0',\ - ctl:ruleRemoveById=920420,\ - ctl:ruleRemoveById=920440,\ - ctl:ruleRemoveById=941000-942999,\ - ctl:ruleRemoveById=951000-951999,\ - ctl:ruleRemoveById=953100-953130,\ - ctl:ruleRemoveByTag=attack-injection-php" -``` - -If you're using the helm chart, you can pass in the following parameters in your `values.yaml`: - -```yaml -controller: - config: - # Enables Modsecurity - enable-modsecurity: "true" - - # Update ModSecurity config and rules - modsecurity-snippet: | - # this enables the mod security nextcloud plugin - Include /etc/nginx/owasp-modsecurity-crs/plugins/nextcloud-rule-exclusions-before.conf - - # this enables the default OWASP Core Rule Set - Include /etc/nginx/owasp-modsecurity-crs/nginx-modsecurity.conf - - # Enable prevention mode. Options: DetectionOnly,On,Off (default is DetectionOnly) - SecRuleEngine On - - # Enable scanning of the request body - SecRequestBodyAccess On - - # Enable XML and JSON parsing - SecRule REQUEST_HEADERS:Content-Type "(?:text|application(?:/soap\+|/)|application/xml)/" \ - "id:200000,phase:1,t:none,t:lowercase,pass,nolog,ctl:requestBodyProcessor=XML" - - SecRule REQUEST_HEADERS:Content-Type "application/json" \ - "id:200001,phase:1,t:none,t:lowercase,pass,nolog,ctl:requestBodyProcessor=JSON" - - # Reject if larger (we could also let it pass with ProcessPartial) - SecRequestBodyLimitAction Reject - - # Send ModSecurity audit logs to the stdout (only for rejected requests) - SecAuditLog /dev/stdout - - # format the logs in JSON - SecAuditLogFormat JSON - - # could be On/Off/RelevantOnly - SecAuditEngine RelevantOnly - - # Add a volume for the plugins directory - extraVolumes: - - name: plugins - configMap: - name: modsecurity-plugins - - # override the /etc/nginx/enable-owasp-modsecurity-crs/plugins with your ConfigMap - extraVolumeMounts: - - name: plugins - mountPath: /etc/nginx/owasp-modsecurity-crs/plugins -``` diff --git a/docs/user-guide/third-party-addons/opentelemetry.md b/docs/user-guide/third-party-addons/opentelemetry.md deleted file mode 100644 index 255ba1ffa..000000000 --- a/docs/user-guide/third-party-addons/opentelemetry.md +++ /dev/null @@ -1,304 +0,0 @@ -# OpenTelemetry - -Enables requests served by NGINX for distributed telemetry via The OpenTelemetry Project. - -Using the third party module [opentelemetry-cpp-contrib/nginx](https://github.com/open-telemetry/opentelemetry-cpp-contrib/tree/main/instrumentation/nginx) the Ingress-Nginx Controller can configure NGINX to enable [OpenTelemetry](http://opentelemetry.io) instrumentation. -By default this feature is disabled. - -Check out this demo showcasing OpenTelemetry in Ingress NGINX. The video provides an overview and -practical demonstration of how OpenTelemetry can be utilized in Ingress NGINX for observability -and monitoring purposes. - -

- - Video Thumbnail - -

- -

Demo: OpenTelemetry in Ingress NGINX.

- -## Usage - -To enable the instrumentation we must enable OpenTelemetry in the configuration ConfigMap: -```yaml -data: - enable-opentelemetry: "true" -``` - -To enable or disable instrumentation for a single Ingress, use -the `enable-opentelemetry` annotation: -```yaml -kind: Ingress -metadata: - annotations: - nginx.ingress.kubernetes.io/enable-opentelemetry: "true" -``` - -We must also set the host to use when uploading traces: - -```yaml -otlp-collector-host: "otel-coll-collector.otel.svc" -``` -NOTE: While the option is called `otlp-collector-host`, you will need to point this to any backend that receives otlp-grpc. - -Next you will need to deploy a distributed telemetry system which uses OpenTelemetry. -[opentelemetry-collector](https://github.com/open-telemetry/opentelemetry-collector), [Jaeger](https://www.jaegertracing.io/) -[Tempo](https://github.com/grafana/tempo), and [zipkin](https://zipkin.io/) -have been tested. - -Other optional configuration options: -```yaml -# specifies the name to use for the server span -opentelemetry-operation-name - -# sets whether or not to trust incoming telemetry spans -opentelemetry-trust-incoming-span - -# specifies the port to use when uploading traces, Default: 4317 -otlp-collector-port - -# specifies the service name to use for any traces created, Default: nginx -otel-service-name - -# The maximum queue size. After the size is reached data are dropped. -otel-max-queuesize - -# The delay interval in milliseconds between two consecutive exports. -otel-schedule-delay-millis - -# How long the export can run before it is cancelled. -otel-schedule-delay-millis - -# The maximum batch size of every export. It must be smaller or equal to maxQueueSize. -otel-max-export-batch-size - -# specifies sample rate for any traces created, Default: 0.01 -otel-sampler-ratio - -# specifies the sampler to be used when sampling traces. -# The available samplers are: AlwaysOn, AlwaysOff, TraceIdRatioBased, Default: AlwaysOff -otel-sampler - -# Uses sampler implementation which by default will take a sample if parent Activity is sampled, Default: false -otel-sampler-parent-based -``` - -Note that you can also set whether to trust incoming spans (global default is true) per-location using annotations like the following: -```yaml -kind: Ingress -metadata: - annotations: - nginx.ingress.kubernetes.io/opentelemetry-trust-incoming-span: "true" -``` - -## Examples - -The following examples show how to deploy and test different distributed telemetry systems. These example can be performed using Docker Desktop. - -In the [esigo/nginx-example](https://github.com/esigo/nginx-example) -GitHub repository is an example of a simple hello service: - -```mermaid -graph TB - subgraph Browser - start["http://esigo.dev/hello/nginx"] - end - - subgraph app - sa[service-a] - sb[service-b] - sa --> |name: nginx| sb - sb --> |hello nginx!| sa - end - - subgraph otel - otc["Otel Collector"] - end - - subgraph observability - tempo["Tempo"] - grafana["Grafana"] - backend["Jaeger"] - zipkin["Zipkin"] - end - - subgraph ingress-nginx - ngx[nginx] - end - - subgraph ngx[nginx] - ng[nginx] - om[OpenTelemetry module] - end - - subgraph Node - app - otel - observability - ingress-nginx - om --> |otlp-gRPC| otc --> |jaeger| backend - otc --> |zipkin| zipkin - otc --> |otlp-gRPC| tempo --> grafana - sa --> |otlp-gRPC| otc - sb --> |otlp-gRPC| otc - start --> ng --> sa - end -``` - -To install the example and collectors run: - -1. Enable OpenTelemetry and set the otlp-collector-host: - - ```yaml - $ echo ' - apiVersion: v1 - kind: ConfigMap - data: - enable-opentelemetry: "true" - opentelemetry-config: "/etc/nginx/opentelemetry.toml" - opentelemetry-operation-name: "HTTP $request_method $service_name $uri" - opentelemetry-trust-incoming-span: "true" - otlp-collector-host: "otel-coll-collector.otel.svc" - otlp-collector-port: "4317" - otel-max-queuesize: "2048" - otel-schedule-delay-millis: "5000" - otel-max-export-batch-size: "512" - otel-service-name: "nginx-proxy" # Opentelemetry resource name - otel-sampler: "AlwaysOn" # Also: AlwaysOff, TraceIdRatioBased - otel-sampler-ratio: "1.0" - otel-sampler-parent-based: "false" - metadata: - name: ingress-nginx-controller - namespace: ingress-nginx - ' | kubectl replace -f - - ``` - -2. Deploy otel-collector, grafana and Jaeger backend: - - ```bash - # add helm charts needed for grafana and OpenTelemetry collector - helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts - helm repo add grafana https://grafana.github.io/helm-charts - helm repo update - # deploy cert-manager needed for OpenTelemetry collector operator - kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.3/cert-manager.yaml - # create observability namespace - kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/namespace.yaml - # install OpenTelemetry collector operator - helm upgrade --install otel-collector-operator -n otel --create-namespace open-telemetry/opentelemetry-operator - # deploy OpenTelemetry collector - kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/collector.yaml - # deploy Jaeger all-in-one - kubectl apply -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.37.0/jaeger-operator.yaml -n observability - kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/jaeger.yaml -n observability - # deploy zipkin - kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/zipkin.yaml -n observability - # deploy tempo and grafana - helm upgrade --install tempo grafana/tempo --create-namespace -n observability - helm upgrade -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/grafana/grafana-values.yaml --install grafana grafana/grafana --create-namespace -n observability - ``` - -3. Build and deploy demo app: - - ```bash - # build images - make images - - # deploy demo app: - make deploy-app - ``` - -4. Make a few requests to the Service: - - ```bash - kubectl port-forward --namespace=ingress-nginx service/ingress-nginx-controller 8090:80 - curl http://esigo.dev:8090/hello/nginx - - - StatusCode : 200 - StatusDescription : OK - Content : {"v":"hello nginx!"} - - RawContent : HTTP/1.1 200 OK - Connection: keep-alive - Content-Length: 21 - Content-Type: text/plain; charset=utf-8 - Date: Mon, 10 Oct 2022 17:43:33 GMT - - {"v":"hello nginx!"} - - Forms : {} - Headers : {[Connection, keep-alive], [Content-Length, 21], [Content-Type, text/plain; charset=utf-8], [Date, - Mon, 10 Oct 2022 17:43:33 GMT]} - Images : {} - InputFields : {} - Links : {} - ParsedHtml : System.__ComObject - RawContentLength : 21 - ``` - -5. View the Grafana UI: - - ```bash - kubectl port-forward --namespace=observability service/grafana 3000:80 - ``` - In the Grafana interface we can see the details: - ![grafana screenshot](../../images/otel-grafana-demo.png "grafana screenshot") - -6. View the Jaeger UI: - - ```bash - kubectl port-forward --namespace=observability service/jaeger-all-in-one-query 16686:16686 - ``` - In the Jaeger interface we can see the details: - ![Jaeger screenshot](../../images/otel-jaeger-demo.png "Jaeger screenshot") - -7. View the Zipkin UI: - - ```bash - kubectl port-forward --namespace=observability service/zipkin 9411:9411 - ``` - In the Zipkin interface we can see the details: - ![zipkin screenshot](../../images/otel-zipkin-demo.png "zipkin screenshot") - -## Migration from OpenTracing, Jaeger, Zipkin and Datadog - -If you are migrating from OpenTracing, Jaeger, Zipkin, or Datadog to OpenTelemetry, -you may need to update various annotations and configurations. Here are the mappings -for common annotations and configurations: - -### Annotations - -| Legacy | OpenTelemetry | -|---------------------------------------------------------------|-----------------------------------------------------------------| -| `nginx.ingress.kubernetes.io/enable-opentracing` | `nginx.ingress.kubernetes.io/enable-opentelemetry` | -| `nginx.ingress.kubernetes.io/opentracing-trust-incoming-span` | `nginx.ingress.kubernetes.io/opentelemetry-trust-incoming-span` | - -### Configs - -| Legacy | OpenTelemetry | -|---------------------------------------|----------------------------------------------| -| `opentracing-operation-name` | `opentelemetry-operation-name` | -| `opentracing-location-operation-name` | `opentelemetry-operation-name` | -| `opentracing-trust-incoming-span` | `opentelemetry-trust-incoming-span` | -| `zipkin-collector-port` | `otlp-collector-port` | -| `zipkin-service-name` | `otel-service-name` | -| `zipkin-sample-rate` | `otel-sampler-ratio` | -| `jaeger-collector-port` | `otlp-collector-port` | -| `jaeger-endpoint` | `otlp-collector-port`, `otlp-collector-host` | -| `jaeger-service-name` | `otel-service-name` | -| `jaeger-propagation-format` | `N/A` | -| `jaeger-sampler-type` | `otel-sampler` | -| `jaeger-sampler-param` | `otel-sampler` | -| `jaeger-sampler-host` | `N/A` | -| `jaeger-sampler-port` | `N/A` | -| `jaeger-trace-context-header-name` | `N/A` | -| `jaeger-debug-header` | `N/A` | -| `jaeger-baggage-header` | `N/A` | -| `jaeger-tracer-baggage-header-prefix` | `N/A` | -| `datadog-collector-port` | `otlp-collector-port` | -| `datadog-service-name` | `otel-service-name` | -| `datadog-environment` | `N/A` | -| `datadog-operation-name-override` | `N/A` | -| `datadog-priority-sampling` | `otel-sampler` | -| `datadog-sample-rate` | `otel-sampler-ratio` | diff --git a/docs/user-guide/third-party-addons/opentracing.md b/docs/user-guide/third-party-addons/opentracing.md new file mode 100644 index 000000000..3bef8332c --- /dev/null +++ b/docs/user-guide/third-party-addons/opentracing.md @@ -0,0 +1,247 @@ +# OpenTracing + +Enables requests served by NGINX for distributed tracing via The OpenTracing Project. + +Using the third party module [opentracing-contrib/nginx-opentracing](https://github.com/opentracing-contrib/nginx-opentracing) the NGINX ingress controller can configure NGINX to enable [OpenTracing](http://opentracing.io) instrumentation. +By default this feature is disabled. + +## Usage + +To enable the instrumentation we must enable OpenTracing in the configuration ConfigMap: +``` +data: + enable-opentracing: "true" +``` + +To enable or disable instrumentation for a single Ingress, use +the `enable-opentracing` annotation: +``` +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/enable-opentracing: "true" +``` + +We must also set the host to use when uploading traces: + +``` +zipkin-collector-host: zipkin.default.svc.cluster.local +jaeger-collector-host: jaeger-agent.default.svc.cluster.local +datadog-collector-host: datadog-agent.default.svc.cluster.local +``` +NOTE: While the option is called `jaeger-collector-host`, you will need to point this to a `jaeger-agent`, and not the `jaeger-collector` component. +Alternatively, you can set `jaeger-endpoint` and specify the full endpoint for uploading traces. This will use TCP and should be used for a collector rather than an agent. + +Next you will need to deploy a distributed tracing system which uses OpenTracing. +[Zipkin](https://github.com/openzipkin/zipkin) and +[Jaeger](https://github.com/jaegertracing/jaeger) and +[Datadog](https://github.com/DataDog/dd-opentracing-cpp) +have been tested. + +Other optional configuration options: +``` +# specifies the name to use for the server span +opentracing-operation-name + +# specifies specifies the name to use for the location span +opentracing-location-operation-name + +# specifies the port to use when uploading traces, Default: 9411 +zipkin-collector-port + +# specifies the service name to use for any traces created, Default: nginx +zipkin-service-name + +# specifies sample rate for any traces created, Default: 1.0 +zipkin-sample-rate + +# specifies the port to use when uploading traces, Default: 6831 +jaeger-collector-port + +# specifies the endpoint to use when uploading traces to a collector instead of an agent +jaeger-endpoint + +# specifies the service name to use for any traces created, Default: nginx +jaeger-service-name + +# specifies the traceparent/tracestate propagation format +jaeger-propagation-format + +# specifies the sampler to be used when sampling traces. +# The available samplers are: const, probabilistic, ratelimiting, remote, Default: const +jaeger-sampler-type + +# specifies the argument to be passed to the sampler constructor, Default: 1 +jaeger-sampler-param + +# Specifies the custom remote sampler host to be passed to the sampler constructor. Must be a valid URL. +# Default: http://127.0.0.1 +jaeger-sampler-host + +# Specifies the custom remote sampler port to be passed to the sampler constructor. Must be a number. Default: 5778 +jaeger-sampler-port + +# Specifies the header name used for passing trace context. Must be a string. Default: uber-trace-id +jaeger-trace-context-header-name + +# Specifies the header name used for force sampling. Must be a string. Default: jaeger-debug-id +jaeger-debug-header + +# Specifies the header name used to submit baggage if there is no root span. Must be a string. Default: jaeger-baggage +jaeger-baggage-header + +# Specifies the header prefix used to propagate baggage. Must be a string. Default: uberctx- +jaeger-tracer-baggage-header-prefix + +# specifies the port to use when uploading traces, Default 8126 +datadog-collector-port + +# specifies the service name to use for any traces created, Default: nginx +datadog-service-name + +# specifies the environment this trace belongs to, Default: prod +datadog-environment + +# specifies the operation name to use for any traces collected, Default: nginx.handle +datadog-operation-name-override + +# Specifies to use client-side sampling for distributed priority sampling and ignore sample rate, Default: true +datadog-priority-sampling + +# specifies sample rate for any traces created, Default: 1.0 +datadog-sample-rate +``` + +All these options (including host) allow environment variables, such as `$HOSTNAME` or `$HOST_IP`. In the case of Jaeger, if you have a Jaeger agent running on each machine in your cluster, you can use something like `$HOST_IP` (which can be 'mounted' with the `status.hostIP` fieldpath, as described [here](https://kubernetes.io/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/#capabilities-of-the-downward-api)) to make sure traces will be sent to the local agent. + +## Examples + +The following examples show how to deploy and test different distributed tracing systems. These example can be performed using Minikube. + +### Zipkin + +In the [rnburn/zipkin-date-server](https://github.com/rnburn/zipkin-date-server) +GitHub repository is an example of a dockerized date service. To install the example and Zipkin collector run: + +``` +kubectl create -f https://raw.githubusercontent.com/rnburn/zipkin-date-server/master/kubernetes/zipkin.yaml +kubectl create -f https://raw.githubusercontent.com/rnburn/zipkin-date-server/master/kubernetes/deployment.yaml +``` + +Also we need to configure the NGINX controller ConfigMap with the required values: + +``` +$ echo ' +apiVersion: v1 +kind: ConfigMap +data: + enable-opentracing: "true" + zipkin-collector-host: zipkin.default.svc.cluster.local +metadata: + name: ingress-nginx-controller + namespace: kube-system +' | kubectl replace -f - +``` + +In the Zipkin interface we can see the details: +![zipkin screenshot](../../images/zipkin-demo.png "zipkin collector screenshot") + +### Jaeger + +1. Enable Ingress addon in Minikube: + ``` + $ minikube addons enable ingress + ``` + +2. Add Minikube IP to /etc/hosts: + ``` + $ echo "$(minikube ip) example.com" | sudo tee -a /etc/hosts + ``` + +3. Apply a basic Service and Ingress Resource: + ``` + # Create Echoheaders Deployment + $ kubectl run echoheaders --image=k8s.gcr.io/echoserver:1.4 --replicas=1 --port=8080 + + # Expose as a Cluster-IP + $ kubectl expose deployment echoheaders --port=80 --target-port=8080 --name=echoheaders-x + + # Apply the Ingress Resource + $ echo ' + apiVersion: networking.k8s.io/v1beta1 + kind: Ingress + metadata: + name: echo-ingress + spec: + rules: + - host: example.com + http: + paths: + - backend: + serviceName: echoheaders-x + servicePort: 80 + path: /echo + ' | kubectl apply -f - + ``` + +4. Enable OpenTracing and set the jaeger-collector-host: + ``` + $ echo ' + apiVersion: v1 + kind: ConfigMap + data: + enable-opentracing: "true" + jaeger-collector-host: jaeger-agent.default.svc.cluster.local + metadata: + name: ingress-nginx-controller + namespace: kube-system + ' | kubectl replace -f - + ``` + +5. Apply the Jaeger All-In-One Template: + ``` + $ kubectl apply -f https://raw.githubusercontent.com/jaegertracing/jaeger-kubernetes/master/all-in-one/jaeger-all-in-one-template.yml + ``` + +6. Make a few requests to the Service: + ``` + $ curl example.com/echo -d "meow" + + CLIENT VALUES: + client_address=172.17.0.5 + command=POST + real path=/echo + query=nil + request_version=1.1 + request_uri=http://example.com:8080/echo + + SERVER VALUES: + server_version=nginx: 1.10.0 - lua: 10001 + + HEADERS RECEIVED: + accept=*/* + connection=close + content-length=4 + content-type=application/x-www-form-urlencoded + host=example.com + user-agent=curl/7.54.0 + x-forwarded-for=192.168.99.1 + x-forwarded-host=example.com + x-forwarded-port=80 + x-forwarded-proto=http + x-original-uri=/echo + x-real-ip=192.168.99.1 + x-scheme=http + BODY: + meow + ``` + +7. View the Jaeger UI: + ``` + $ minikube service jaeger-query --url + + http://192.168.99.100:30183 + ``` + + In the Jaeger interface we can see the details: + ![jaeger screenshot](../../images/jaeger-demo.png "jaeger collector screenshot") diff --git a/docs/user-guide/tls.md b/docs/user-guide/tls.md index eaf33e210..097c5c35e 100644 --- a/docs/user-guide/tls.md +++ b/docs/user-guide/tls.md @@ -4,13 +4,10 @@ Anytime we reference a TLS secret, we mean a PEM-encoded X.509, RSA (2048) secret. -!!! warning - Ensure that the certificate order is leaf->intermediate->root, otherwise the controller will not be able to import the certificate, and you'll see this error in the logs ```W1012 09:15:45.920000 6 backend_ssl.go:46] Error obtaining X.509 certificate: unexpected error creating SSL Cert: certificate and private key does not have a matching public key: tls: private key does not match public key``` - You can generate a self-signed certificate and private key with: ```bash -$ openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ${KEY_FILE} -out ${CERT_FILE} -subj "/CN=${HOST}/O=${HOST}" -addext "subjectAltName = DNS:${HOST}" +$ openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ${KEY_FILE} -out ${CERT_FILE} -subj "/CN=${HOST}/O=${HOST}" ``` Then create the secret in the cluster via: @@ -21,16 +18,12 @@ kubectl create secret tls ${CERT_NAME} --key ${KEY_FILE} --cert ${CERT_FILE} The resulting secret will be of type `kubernetes.io/tls`. -## Host names - -Ensure that the relevant [ingress rules specify a matching hostname](https://kubernetes.io/docs/concepts/services-networking/ingress/#tls). - ## Default SSL Certificate NGINX provides the option to configure a server as a catch-all with -[server_name](https://nginx.org/en/docs/http/server_names.html) +[server_name](http://nginx.org/en/docs/http/server_names.html) for requests that do not match any of the configured server names. -This configuration works out-of-the-box for HTTP traffic. +This configuration works without out-of-the-box for HTTP traffic. For HTTPS, a certificate is naturally required. For this reason the Ingress controller provides the flag `--default-ssl-certificate`. @@ -41,11 +34,8 @@ If this flag is not provided NGINX will use a self-signed certificate. For instance, if you have a TLS secret `foo-tls` in the `default` namespace, add `--default-ssl-certificate=default/foo-tls` in the `nginx-controller` deployment. -If the `tls:` section is not set, NGINX will provide the default certificate but will not force HTTPS redirect. - -On the other hand, if the `tls:` section is set - even without specifying a `secretName` option - NGINX will force HTTPS redirect. - -To force redirects for Ingresses that do not specify a TLS-block at all, take a look at `force-ssl-redirect` in [ConfigMap][ConfigMap]. +The default certificate will also be used for ingress `tls:` sections that do not +have a `secretName` option. ## SSL Passthrough @@ -93,38 +83,29 @@ annotation in the particular resource. This can be achieved by using the `nginx.ingress.kubernetes.io/force-ssl-redirect: "true"` annotation in the particular resource. -## Automated Certificate Management with cert-manager +## Automated Certificate Management with Kube-Lego -[cert-manager] automatically requests missing or expired certificates from a range of -[supported issuers][cert-manager-issuer-config] (including [Let's Encrypt]) by monitoring -ingress resources. +!!! tip + Kube-Lego has reached end-of-life and is being + replaced by [cert-manager](https://github.com/jetstack/cert-manager/). -To set up cert-manager you should take a look at this [full example][full-cert-manager-example]. +[Kube-Lego] automatically requests missing or expired certificates from [Let's Encrypt] +by monitoring ingress resources and their referenced secrets. -To enable it for an ingress resource you have to deploy cert-manager, configure a certificate -issuer update the manifest: +To enable this for an ingress resource you have to add an annotation: -```yaml -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ingress-demo - annotations: - cert-manager.io/issuer: "letsencrypt-staging" # Replace this with a production issuer once you've tested it - [..] -spec: - tls: - - hosts: - - ingress-demo.example.com - secretName: ingress-demo-tls - [...] +```console +kubectl annotate ing ingress-demo kubernetes.io/tls-acme="true" ``` +To setup Kube-Lego you can take a look at this [full example][full-kube-lego-example]. +The first version to fully support Kube-Lego is Nginx Ingress controller 0.8. + ## Default TLS Version and Ciphers To provide the most secure baseline configuration possible, -ingress-nginx defaults to using TLS 1.2 and 1.3 only, with a [secure set of TLS ciphers][ssl-ciphers]. +nginx-ingress defaults to using TLS 1.2 and 1.3 only, with a [secure set of TLS ciphers][ssl-ciphers]. ### Legacy TLS @@ -132,7 +113,7 @@ The default configuration, though secure, does not support some older browsers a For instance, TLS 1.1+ is only enabled by default from Android 5.0 on. At the time of writing, May 2018, [approximately 15% of Android devices](https://developer.android.com/about/dashboards/#Platform) -are not compatible with ingress-nginx's default configuration. +are not compatible with nginx-ingress's default configuration. To change this default behavior, use a [ConfigMap][ConfigMap]. @@ -146,16 +127,15 @@ metadata: name: nginx-config data: ssl-ciphers: "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA" - ssl-protocols: "TLSv1.2 TLSv1.3" + ssl-protocols: "TLSv1 TLSv1.1 TLSv1.2 TLSv1.3" ``` +[full-kube-lego-example]:https://github.com/jetstack/kube-lego/tree/master/examples +[Kube-Lego]:https://github.com/jetstack/kube-lego [Let's Encrypt]:https://letsencrypt.org [ConfigMap]: ./nginx-configuration/configmap.md [ssl-ciphers]: ./nginx-configuration/configmap.md#ssl-ciphers [SNI]: https://en.wikipedia.org/wiki/Server_Name_Indication [mozilla-ssl-config-old]: https://ssl-config.mozilla.org/#server=nginx&config=old -[cert-manager]: https://github.com/jetstack/cert-manager/ -[full-cert-manager-example]:https://cert-manager.io/docs/tutorials/acme/nginx-ingress/ -[cert-manager-issuer-config]:https://cert-manager.io/docs/configuration/ diff --git a/ginkgo_upgrade.md b/ginkgo_upgrade.md deleted file mode 100644 index 87e6bd9fd..000000000 --- a/ginkgo_upgrade.md +++ /dev/null @@ -1,131 +0,0 @@ -# GINKGO UPGRADE - -#### Bumping ginkgo in the project requires four PRs. - -## 1. Dependabot PR - -- Dependabot automatically updates `ginkgo version` but only in [go.mod ](go.mod) and [go.sum ](go.sum) files. -- This is an automatically generated PR by Dependabot but it needs approval from maintainers to get merged. - -## 2. Edit-hardcoded-version PR - -### a. Make changes to appropriate files in required directories - -- Make changes in files where ginkgo version is hardcoded. These files are : - - [run-in-docker.sh ](build/run-in-docker.sh) - - [Dockerfile ](images/test-runner/rootfs/Dockerfile) - - [run.sh ](test/e2e/run.sh) - - [run-chart-test.sh ](test/e2e/run-chart-test.sh) - -### b. Create PR - -- Open pull request(s) accordingly, to fire cloudbuild for building the component's image (if applicable). - -### c. Merge - -- Merging will fire cloudbuild, which will result in images being promoted to the [staging container registry](https://console.cloud.google.com/gcr/images/k8s-staging-ingress-nginx). - -### d. Make sure cloudbuild is a success - -- Wait for [cloud build](https://console.cloud.google.com/cloud-build/builds?project=k8s-staging-ingress-nginx). If you don't have access to cloudbuild, you can also have a look at [this](https://prow.k8s.io/?repo=kubernetes%2Fingress-nginx&job=post-*), to see the progress of the build. - -- Proceed only after cloud-build is successful in building a new Ingress-Nginx-Controller image. - - -## 3. PROMOTE the new-testrunner-image PR: - -Promoting the images basically means that images, that were pushed to staging container registry in the steps above, now are also pushed to the public container registry. Thus are publicly available. Follow these steps to promote images: -- When you make changes to the `Dockerfile` or other core content under [images directory ](images), it generates a new image in google cloudbuild. This is because kubernetes projects need to use the infra provided for the kubernetes projects. The new image is always only pushed to the staging repository of K8S. From the staging repo, the new image needs to be promoted to the production repo. And once promoted, its possible to use the sha of the new image in the code. - -### a. Get the sha - -- Get the sha of the new image(s) of the controller, from the cloudbuild, from steps above - - - The sha is available in output from [cloud build](https://console.cloud.google.com/cloud-build/builds?project=k8s-staging-ingress-nginx) - - - The sha is also visible [here](https://console.cloud.google.com/gcr/images/k8s-staging-ingress-nginx/global/e2e-test-runner) - - - The sha is also visible [here]((https://prow.k8s.io/?repo=kubernetes%2Fingress-nginx&job=post-*)), after cloud build is finished. Click on the respective job, go to `Artifacts` section in the UI, then again `artifacts` in the directory browser. In the `build.log` at the very bottom you see something like this: - - ``` - ... - pushing manifest for us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx/controller:v1.0.2@sha256:e15fac6e8474d77e1f017edc33d804ce72a184e3c0a30963b2a0d7f0b89f6b16 - ... - ``` - -### b. Add the new image to [k8s.io](http://github.com/kubernetes/k8s.io) - -- The sha(s) from the step before (and the tag(s) for the new image(s) have to be added, as a new line, in a file, of the [k8s.io](http://github.com/kubernetes/k8s.io) project of Kubernetes organization. - -- Fork that other project (if you don't have a fork already). - -- Other project to fork [Github repo kubernetes/k8s.io](http://github.com/kubernetes/k8s.io) - -- Fetch --all and rebase to upstream if already forked. - -- Create a branch in your fork, named as the issue number for this release - -- In the related branch, of your fork, edit the file registry.k8s.io/images/k8s-staging-ingress-nginx/images.yaml. - -- For making it easier, you can edit your branch directly in the browser. But be careful about making any mistake. - -- Insert the sha(s) & the tag(s), in a new line, in this file [Project kubernetes/k8s.io Ingress-Nginx-Controller Images](https://github.com/kubernetes/k8s.io/blob/main/registry.k8s.io/images/k8s-staging-ingress-nginx/images.yaml) Look at this [example PR and the diff](https://github.com/kubernetes/k8s.io/pull/4499) to see how it was done before - -- Save and commit - -### c. Create PR - -- Open pull request to promote the new e2e-test-runner image. - -### d. Merge - -- Merge success is required. - -- Proceed only after cloud-build is successful in building a new e2e-test-runner image. - - -## 4. Change testrunner-image-sha PR - -### a. Get the sha - -- Make sure to get the tag and sha of the promoted image from the step before, either from cloudbuild or from [here](https://console.cloud.google.com/gcr/images/k8s-artifacts-prod/us/ingress-nginx/e2e-test-runner). - -### a. Make sure your git workspace is ready - -- Get your git workspace ready - - - If not using a pre-existing fork, then Fork the repo kubernetes/ingress-nginx - - - Clone (to laptop or wherever) - - - Add upstream - - - Set upstream url to no_push - - - Checkout & switch to branch, named as per related new-release-issue-number - - - If already forked, and upstream already added, then `git fetch --all` and `git rebase upstream/main` (not origin) - - - Checkout a branch in your fork's clone - - - Perform any other diligence as needed - -- Prefer to edit only and only in your branch, in your Fork - -### b. Change testrunner-image-sha - -- You need update the testrunner-image-sha in the following files : - - - [run-in-docker.sh](https://github.com/kubernetes/ingress-nginx/blob/main/build/run-in-docker.sh#L41) - - [Makefile](https://github.com/kubernetes/ingress-nginx/blob/main/test/e2e-image/Makefile#L3) - -### c. Create PR - -- Look at this PR for how it was done before [example PR](https://github.com/kubernetes/ingress-nginx/pull/9444) -- Create a PR - -### d. Merge - -- Finally merge the PR. - -## END ## diff --git a/go.mod b/go.mod index 34584bbf9..efbf04769 100644 --- a/go.mod +++ b/go.mod @@ -1,137 +1,48 @@ module k8s.io/ingress-nginx -go 1.23.6 +go 1.15 require ( - dario.cat/mergo v1.0.1 - github.com/armon/go-proxyproto v0.1.0 + github.com/armon/go-proxyproto v0.0.0-20200108142055-f0b8253b1507 github.com/eapache/channels v1.1.0 - github.com/fsnotify/fsnotify v1.8.0 - github.com/json-iterator/go v1.1.12 - github.com/kylelemons/godebug v1.1.0 - github.com/mitchellh/go-ps v1.0.0 - github.com/mitchellh/hashstructure/v2 v2.0.2 - github.com/mitchellh/mapstructure v1.5.0 - github.com/moul/pb v0.0.0-20220425114252-bca18df4138c - github.com/ncabatoff/process-exporter v0.8.5 - github.com/onsi/ginkgo/v2 v2.22.2 - github.com/opencontainers/runc v1.2.5 - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 - github.com/prometheus/client_golang v1.20.5 - github.com/prometheus/client_model v0.6.1 - github.com/prometheus/common v0.62.0 - github.com/spf13/cobra v1.9.1 - github.com/spf13/pflag v1.0.6 - github.com/stretchr/testify v1.10.0 - github.com/yudai/gojsondiff v1.0.0 - github.com/zakjan/cert-chain-resolver v0.0.0-20221221105603-fcedb00c5b30 - golang.org/x/crypto v0.33.0 - google.golang.org/grpc v1.70.0 - google.golang.org/grpc/examples v0.0.0-20240223204917-5ccf176a08ab - gopkg.in/go-playground/pool.v3 v3.1.1 - gopkg.in/mcuadros/go-syslog.v2 v2.3.0 - k8s.io/api v0.32.2 - k8s.io/apiextensions-apiserver v0.32.2 - k8s.io/apimachinery v0.32.2 - k8s.io/apiserver v0.32.2 - k8s.io/cli-runtime v0.32.2 - k8s.io/client-go v0.32.2 - k8s.io/code-generator v0.32.2 - k8s.io/component-base v0.32.2 - k8s.io/klog/v2 v2.130.1 - pault.ag/go/sniff v0.0.0-20200207005214-cf7e4d167732 - sigs.k8s.io/controller-runtime v0.20.2 - sigs.k8s.io/mdtoc v1.4.0 -) - -require ( - github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect - github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/klauspost/compress v1.17.9 // indirect - github.com/moby/sys/userns v0.1.0 // indirect - github.com/x448/float16 v0.8.4 // indirect - go.opentelemetry.io/otel v1.32.0 // indirect - go.opentelemetry.io/otel/trace v1.32.0 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect - sigs.k8s.io/release-utils v0.8.3 // indirect -) - -require ( - github.com/Anddd7/pb v0.0.0-20240425032658-369b0f6a404c - github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect - github.com/BurntSushi/toml v1.3.2 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect - github.com/cyphar/filepath-securejoin v0.4.1 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/eapache/queue v1.1.0 // indirect - github.com/emicklei/go-restful/v3 v3.12.0 // indirect - github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/fsnotify/fsnotify v1.4.9 github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa // indirect - github.com/go-errors/errors v1.5.1 // indirect - github.com/go-logr/logr v1.4.2 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.21.0 // indirect - github.com/go-openapi/swag v0.23.0 // indirect - github.com/go-task/slim-sprig/v3 v3.0.0 // indirect - github.com/godbus/dbus/v5 v5.1.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/gomarkdown/markdown v0.0.0-20240328165702-4d01890c35c0 // indirect - github.com/google/btree v1.1.3 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect - github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect - github.com/mmarkdown/mmark v2.0.40+incompatible // indirect - github.com/moby/sys/mountinfo v0.7.1 // indirect - github.com/moby/term v0.5.0 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/ncabatoff/go-seq v0.0.0-20180805175032-b08ef85ed833 // indirect - github.com/onsi/ginkgo v1.16.5 // indirect - github.com/opencontainers/runtime-spec v1.2.0 // indirect - github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - github.com/sergi/go-diff v1.3.1 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect - github.com/xlab/treeprint v1.2.0 // indirect - github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect - github.com/yudai/pp v2.0.1+incompatible // indirect - golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/oauth2 v0.24.0 // indirect - golang.org/x/sync v0.11.0 // indirect - golang.org/x/sys v0.30.0 // indirect - golang.org/x/term v0.29.0 // indirect - golang.org/x/text v0.22.0 // indirect - golang.org/x/time v0.7.0 // indirect - golang.org/x/tools v0.28.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a // indirect - google.golang.org/protobuf v1.36.1 // indirect + github.com/gavv/httpexpect/v2 v2.1.0 + github.com/imdario/mergo v0.3.10 + github.com/json-iterator/go v1.1.10 + github.com/kylelemons/godebug v1.1.0 + github.com/mitchellh/copystructure v1.0.0 + github.com/mitchellh/go-ps v1.0.0 + github.com/mitchellh/hashstructure v1.0.0 + github.com/mitchellh/mapstructure v1.3.2 + github.com/moul/pb v0.0.0-20180404114147-54bdd96e6a52 + github.com/ncabatoff/process-exporter v0.7.2 + github.com/onsi/ginkgo v1.14.1 + github.com/opencontainers/runc v1.0.0-rc92 + github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.7.1 + github.com/prometheus/client_model v0.2.0 + github.com/prometheus/common v0.14.0 + github.com/spf13/cobra v1.1.1 + github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.6.1 + github.com/tallclair/mdtoc v1.0.0 + github.com/zakjan/cert-chain-resolver v0.0.0-20200729110141-6b99e360f97a + golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 + golang.org/x/net v0.0.0-20201110031124-69a78807bb2b + google.golang.org/grpc v1.27.1 gopkg.in/go-playground/assert.v1 v1.2.1 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 // indirect - k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect - k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect - sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect - sigs.k8s.io/kustomize/api v0.18.0 // indirect - sigs.k8s.io/kustomize/kyaml v0.18.1 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect + gopkg.in/go-playground/pool.v3 v3.1.1 + k8s.io/api v0.20.2 + k8s.io/apiextensions-apiserver v0.20.2 + k8s.io/apimachinery v0.20.2 + k8s.io/apiserver v0.20.2 + k8s.io/cli-runtime v0.20.2 + k8s.io/client-go v0.20.2 + k8s.io/code-generator v0.20.2 + k8s.io/component-base v0.20.2 + k8s.io/klog/v2 v2.4.0 + k8s.io/utils v0.0.0-20201110183641-67b214c5f920 + pault.ag/go/sniff v0.0.0-20200207005214-cf7e4d167732 + sigs.k8s.io/controller-runtime v0.8.0 ) diff --git a/go.sum b/go.sum index 6e933d349..4439cb737 100644 --- a/go.sum +++ b/go.sum @@ -1,219 +1,623 @@ -dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= -dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -github.com/Anddd7/pb v0.0.0-20240425032658-369b0f6a404c h1:uhBf0CHXi7nCFZXxHV7l1cBcYFEEVRK4FYxvm1l9lKg= -github.com/Anddd7/pb v0.0.0-20240425032658-369b0f6a404c/go.mod h1:vYWKbnXd2KAZHUECLPzSE0Er3FgiEmOdPtxwSIRihck= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/armon/go-proxyproto v0.1.0 h1:TWWcSsjco7o2itn6r25/5AqKBiWmsiuzsUDLT/MTl7k= -github.com/armon/go-proxyproto v0.1.0/go.mod h1:Xj90dce2VKbHzRAeiVQAMBtj4M5oidoXJ8lmgyW21mw= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.1 h1:eVvIXUKiTgv++6YnWb42DUA1YL7qDugnKP0HljexdnQ= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5 h1:Y3bBUV4rTuxenJJs41HU3qmqsb+auo+a3Lz+PlJPpL0= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-proxyproto v0.0.0-20200108142055-f0b8253b1507 h1:dmVRVC/MmuwC2edm/P6oWIP+9n+p9IgVgK0lq9mBQjU= +github.com/armon/go-proxyproto v0.0.0-20200108142055-f0b8253b1507/go.mod h1:QmP9hvJ91BbJmGVGSbutW19IC0Q9phDCLGaomwTJbgU= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ= -github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= -github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/containerd/console v1.0.0/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.1.0 h1:kq/SbG2BCKLkDKkjQf5OWwKWUKj1lgs3lFI4PxnR5lg= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= +github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k= github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= -github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= -github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/ekalinin/github-markdown-toc v0.0.0-20190514155158-83fadb60a7f1/go.mod h1:XfZS1iyC28CnllR54Ou2Ero6qs4Rmn7GpVumNSj1DZo= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= +github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fasthttp/websocket v1.4.2 h1:AU/zSiIIAuJjBMf5o+vO0syGOnEfvZRu40xIhW/3RuM= +github.com/fasthttp/websocket v1.4.2/go.mod h1:smsv/h4PBEBaU0XDTY5UwJTpZv69fQ0FfcLJr21mA6Y= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= +github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/fullsailor/pkcs7 v0.0.0-20160414161337-2585af45975b/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa h1:RDBNVkRviHZtvDvId8XSGPu3rmpmSe+wKRcEWNgsfWU= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= -github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= -github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= -github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= -github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= -github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= -github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= -github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= -github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/gavv/httpexpect/v2 v2.1.0 h1:Q7xnFuKqBY2si4DsqxdbWBt9rfrbVTT2/9YSomc9tEw= +github.com/gavv/httpexpect/v2 v2.1.0/go.mod h1:lnd0TqJLrP+wkJk3SFwtrpSlOAZQ7HaaIFuOYbgqgUM= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.3.0 h1:q4c+kbcR0d5rSurhBR8dIgieOaYpXtsdTYfx22Cu6rs= +github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/zapr v0.2.0 h1:v6Ji8yBW77pva6NkJKQdHLAJKrIJKRHz0RXwPqCHSR4= +github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/gomarkdown/markdown v0.0.0-20240328165702-4d01890c35c0 h1:4gjrh/PN2MuWCCElk8/I4OCKRKWCCo2zEct3VKCbibU= -github.com/gomarkdown/markdown v0.0.0-20240328165702-4d01890c35c0/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= -github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= -github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomarkdown/markdown v0.0.0-20190222000725-ee6a7931a1e4 h1:vELsocEzlhM4lk2nhxolEaQrMp25u7/i9IX8s9uLads= +github.com/gomarkdown/markdown v0.0.0-20190222000725-ee6a7931a1e4/go.mod h1:gmFANS06wAVmF0B9yi65QKsRmPQ97tze7FRLswua+OY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= -github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= -github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/gnostic v0.5.1 h1:A8Yhf6EtqTv9RMsU6MQTyrtV1TjWlR6xU9BsZIwuTCM= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.0.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.10 h1:6q5mVkdH/vYmqngx7kZQTjJ5HRsx+ImorDIEQ+beJgc= +github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imkira/go-interpol v1.0.0 h1:HrmLyvOLJyjR0YofMw8QGdCIuYOs4TJUBDNU5sJC09E= +github.com/imkira/go-interpol v1.0.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.8.2 h1:Bx0qjetmNjdFXASH02NSAREKpiaDwkO1DRZ3dV2KCcs= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= -github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= -github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y= +github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mmarkdown/mmark v2.0.40+incompatible h1:vMeUeDzBK3H+/mU0oMVfMuhSXJlIA+DE/DMPQNAj5C4= github.com/mmarkdown/mmark v2.0.40+incompatible/go.mod h1:Uvmoz7tvsWpr7bMVxIpqZPyN3FbOtzDmnsJDFp7ltJs= -github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= -github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= -github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= -github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/moby/sys/mountinfo v0.1.3/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= -github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/moul/pb v0.0.0-20220425114252-bca18df4138c h1:1STmblv9zmHLDpru4dbnf1PNL6wrrZNf7yBH+SfQU+s= -github.com/moul/pb v0.0.0-20220425114252-bca18df4138c/go.mod h1:jE2HT8eoucYyUPBFJMreiVlC3KPHkDMtN8wn+ef7Y64= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/moul/pb v0.0.0-20180404114147-54bdd96e6a52 h1:8zDEa5yAIWYBHSDpPbSgGIBL/SvPSE9/FlB3aQ54d/A= +github.com/moul/pb v0.0.0-20180404114147-54bdd96e6a52/go.mod h1:jE2HT8eoucYyUPBFJMreiVlC3KPHkDMtN8wn+ef7Y64= +github.com/mrunalp/fileutils v0.0.0-20200520151820-abd8a0e76976/go.mod h1:x8F1gnqOkIEiO4rqoeEEEqQbo7HjGMTvyoq3gej4iT0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/ncabatoff/fakescraper v0.0.0-20161023141611-15938421d91a/go.mod h1:Tx6UMSMyIsjLG/VU/F6xA1+0XI+/f9o1dGJnf1l+bPg= github.com/ncabatoff/go-seq v0.0.0-20180805175032-b08ef85ed833 h1:t4WWQ9I797y7QUgeEjeXnVb+oYuEDQc6gLvrZJTYo94= github.com/ncabatoff/go-seq v0.0.0-20180805175032-b08ef85ed833/go.mod h1:0CznHmXSjMEqs5Tezj/w2emQoM41wzYM9KpDKUHPYag= -github.com/ncabatoff/process-exporter v0.8.5 h1:Hk1sflgRWn0Xrh/OsupQLVVCTW01kv0YYrGxu7NvkmM= -github.com/ncabatoff/process-exporter v0.8.5/go.mod h1:IZndG/m2Y++D90y99NhDJfg0SOkpbx/Fl6MlnBr4SC0= +github.com/ncabatoff/process-exporter v0.7.2 h1:2UxJJ5fm9fiiUHqHgbusrfceCFQpEET/+wADEOfDSqI= +github.com/ncabatoff/process-exporter v0.7.2/go.mod h1:d7Yf/brhprE2fjCdtZ2Edt0so30RY5PmtRu6qD0gUOc= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= -github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= +github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4= +github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= -github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= -github.com/opencontainers/runc v1.2.5 h1:8KAkq3Wrem8bApgOHyhRI/8IeLXIfmZ6Qaw6DNSLnA4= -github.com/opencontainers/runc v1.2.5/go.mod h1:dOQeFo29xZKBNeRBI0B19mJtfHv68YgCTh1X+YphA+4= -github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= -github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= +github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opencontainers/runc v1.0.0-rc92 h1:+IczUKCRzDzFDnw99O/PAqrcBBCoRp9xN3cB1SYSNS4= +github.com/opencontainers/runc v1.0.0-rc92/go.mod h1:X1zlU4p7wOlX4+WRCz+hvlRv8phdL7UqbYD+vQwNMmE= +github.com/opencontainers/runtime-spec v1.0.3-0.20200728170252-4d89ac9fbff6 h1:NhsM2gc769rVWDqJvapK37r+7+CBXI8xHhnfnt8uQsg= +github.com/opencontainers/runtime-spec v1.0.3-0.20200728170252-4d89ac9fbff6/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= -github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.14.0 h1:RHRyE8UocrbjU+6UvRzwi6HjiDfxrrBU91TtbKzkGp4= +github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f h1:PgA+Olipyj258EIEYnpFFONrrCcAIWNUNoFhUfMqAGY= +github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.1.3/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/urfave/cli v1.17.1-0.20160602030128-01a33823596e/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= -github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= -github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/tallclair/mdtoc v1.0.0 h1:+FqBzRdFsgwrkzewUYC8GG6/hckREy9t4cDw4bWjx+M= +github.com/tallclair/mdtoc v1.0.0/go.mod h1:BjDk9nfX4091pXLHhvf6Ejr4/r//9NslWmweWb2Hkbs= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.9.0 h1:hNpmUdy/+ZXYpGy0OBfm7K0UQTzb73W0T0U4iJIVrMw= +github.com/valyala/fasthttp v1.9.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.1.0 h1:ngVtJC9TY/lg0AA/1k48FYhBrhRoFlEmWzsehpNAaZg= +github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= @@ -221,164 +625,424 @@ github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDf github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI= github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/zakjan/cert-chain-resolver v0.0.0-20221221105603-fcedb00c5b30 h1:rzHvkiukOVYcf840FqAsHqBMhfLofvQIxWtczkGRklU= -github.com/zakjan/cert-chain-resolver v0.0.0-20221221105603-fcedb00c5b30/go.mod h1:/Hzu8ych2oXCs1iNI+MeASyFzWTncQ6nlu/wgqbqC2A= -go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= -go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= -go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= -go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= -go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= -go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= -go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= -go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= -go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= -go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +github.com/zakjan/cert-chain-resolver v0.0.0-20200729110141-6b99e360f97a h1:Tj7Mkjj6sjeNwFBiRHM7ieg1CiPHCDNSMSmS0ooGcZo= +github.com/zakjan/cert-chain-resolver v0.0.0-20200729110141-6b99e360f97a/go.mod h1:KNkcm66cr4ilOiEcjydK+tc2ShPUhqmuoXCljXUBPu8= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0 h1:sFPn2GLc3poCkfrpIXGhBD2X0CMIo4Q/zSULXrj/+uc= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= +go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= -golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/cr4/aHLv9xU8Kkadrw= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= -golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= -golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= -golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= -golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200616133436-c1934b75d054 h1:HHeAlu5H9b71C+Fx0K+1dGgVFN1DM1/wz4aoGOA5qS8= +golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= -gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a h1:hgh8P4EuoxpsuKMXX/To36nOFD7vixReXgn8lPGnt+o= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= -google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= -google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= -google.golang.org/grpc/examples v0.0.0-20240223204917-5ccf176a08ab h1:tg8hvIl5RmFBuXlcJMuL0h4Psh1gx5Q5xEMwzBZIzWA= -google.golang.org/grpc/examples v0.0.0-20240223204917-5ccf176a08ab/go.mod h1:liVNnGuZDITxuksuZ+BBvdy7FcJfeNk+efF9qgqNUmc= +gomodules.xyz/jsonpatch/v2 v2.1.0 h1:Phva6wqu+xR//Njw6iorylFFgn/z547tw5Ne3HZPQ+k= +gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8= +google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= -google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/pool.v3 v3.1.1 h1:4Qcj91IsYTpIeRhe/eo6Fz+w6uKWPEghx8vHFTYMfhw= gopkg.in/go-playground/pool.v3 v3.1.1/go.mod h1:pUAGBximS/hccTTSzEop6wvvQhVa3QPDFFW+8REdutg= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/mcuadros/go-syslog.v2 v2.3.0 h1:kcsiS+WsTKyIEPABJBJtoG0KkOS6yzvJ+/eZlhD79kk= -gopkg.in/mcuadros/go-syslog.v2 v2.3.0/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.32.2 h1:bZrMLEkgizC24G9eViHGOPbW+aRo9duEISRIJKfdJuw= -k8s.io/api v0.32.2/go.mod h1:hKlhk4x1sJyYnHENsrdCWw31FEmCijNGPJO5WzHiJ6Y= -k8s.io/apiextensions-apiserver v0.32.2 h1:2YMk285jWMk2188V2AERy5yDwBYrjgWYggscghPCvV4= -k8s.io/apiextensions-apiserver v0.32.2/go.mod h1:GPwf8sph7YlJT3H6aKUWtd0E+oyShk/YHWQHf/OOgCA= -k8s.io/apimachinery v0.32.2 h1:yoQBR9ZGkA6Rgmhbp/yuT9/g+4lxtsGYwW6dR6BDPLQ= -k8s.io/apimachinery v0.32.2/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= -k8s.io/apiserver v0.32.2 h1:WzyxAu4mvLkQxwD9hGa4ZfExo3yZZaYzoYvvVDlM6vw= -k8s.io/apiserver v0.32.2/go.mod h1:PEwREHiHNU2oFdte7BjzA1ZyjWjuckORLIK/wLV5goM= -k8s.io/cli-runtime v0.32.2 h1:aKQR4foh9qeyckKRkNXUccP9moxzffyndZAvr+IXMks= -k8s.io/cli-runtime v0.32.2/go.mod h1:a/JpeMztz3xDa7GCyyShcwe55p8pbcCVQxvqZnIwXN8= -k8s.io/client-go v0.32.2 h1:4dYCD4Nz+9RApM2b/3BtVvBHw54QjMFUl1OLcJG5yOA= -k8s.io/client-go v0.32.2/go.mod h1:fpZ4oJXclZ3r2nDOv+Ux3XcJutfrwjKTCHz2H3sww94= -k8s.io/code-generator v0.32.2 h1:CIvyPrLWP7cMgrqval2qYT839YAwCDeSvGfXgWSNpHQ= -k8s.io/code-generator v0.32.2/go.mod h1:plh7bWk7JztAUkHM4zpbdy0KOMdrhsePcZL2HLWFH7Y= -k8s.io/component-base v0.32.2 h1:1aUL5Vdmu7qNo4ZsE+569PV5zFatM9hl+lb3dEea2zU= -k8s.io/component-base v0.32.2/go.mod h1:PXJ61Vx9Lg+P5mS8TLd7bCIr+eMJRQTyXe8KvkrvJq0= -k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 h1:si3PfKm8dDYxgfbeA6orqrtLkvvIeH8UqffFJDl0bz4= -k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= -k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= -k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= +k8s.io/api v0.20.2 h1:y/HR22XDZY3pniu9hIFDLpUCPq2w5eQ6aV/VFQ7uJMw= +k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= +k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= +k8s.io/apiextensions-apiserver v0.20.2 h1:rfrMWQ87lhd8EzQWRnbQ4gXrniL/yTRBgYH1x1+BLlo= +k8s.io/apiextensions-apiserver v0.20.2/go.mod h1:F6TXp389Xntt+LUq3vw6HFOLttPa0V8821ogLGwb6Zs= +k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg= +k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= +k8s.io/apiserver v0.20.2 h1:lGno2t3gcZnLtzsKH4oG0xA9/4GTiBzMO1DGp+K+Bak= +k8s.io/apiserver v0.20.2/go.mod h1:2nKd93WyMhZx4Hp3RfgH2K5PhwyTrprrkWYnI7id7jA= +k8s.io/cli-runtime v0.20.2 h1:W0/FHdbApnl9oB7xdG643c/Zaf7TZT+43I+zKxwqvhU= +k8s.io/cli-runtime v0.20.2/go.mod h1:FjH6uIZZZP3XmwrXWeeYCbgxcrD6YXxoAykBaWH0VdM= +k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= +k8s.io/client-go v0.20.2 h1:uuf+iIAbfnCSw8IGAv/Rg0giM+2bOzHLOsbbrwrdhNQ= +k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= +k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= +k8s.io/code-generator v0.20.2 h1:SQaysped4EtUDk3u1zphnUJiOAwFdhHx9xS3WKAE0x8= +k8s.io/code-generator v0.20.2/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= +k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= +k8s.io/component-base v0.20.2 h1:LMmu5I0pLtwjpp5009KLuMGFqSc2S2isGw8t1hpYKLE= +k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20201113003025-83324d819ded h1:JApXBKYyB7l9xx+DK7/+mFjC7A9Bt5A93FPvFD0HIFE= +k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e h1:C7q+e9M5nggAvWfVg9Nl66kebKeuJlP3FD58V4RR5wo= +moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e/go.mod h1:nejbQVfXh96n9dSF6cH3Jsk/QI1Z2oEL7sSI2ifXFNA= pault.ag/go/sniff v0.0.0-20200207005214-cf7e4d167732 h1:SAElp8THCfmBdM+4lmWX5gebiSSkEr7PAYDVF91qpfg= pault.ag/go/sniff v0.0.0-20200207005214-cf7e4d167732/go.mod h1:lpvCfhqEHNJSSpG5R5A2EgsVzG8RTt4RfPoQuRAcDmg= -sigs.k8s.io/controller-runtime v0.20.2 h1:/439OZVxoEc02psi1h4QO3bHzTgu49bb347Xp4gW1pc= -sigs.k8s.io/controller-runtime v0.20.2/go.mod h1:xg2XB0K5ShQzAgsoujxuKN4LNXR2LfwwHsPj7Iaw+XY= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= -sigs.k8s.io/kustomize/api v0.18.0 h1:hTzp67k+3NEVInwz5BHyzc9rGxIauoXferXyjv5lWPo= -sigs.k8s.io/kustomize/api v0.18.0/go.mod h1:f8isXnX+8b+SGLHQ6yO4JG1rdkZlvhaCf/uZbLVMb0U= -sigs.k8s.io/kustomize/kyaml v0.18.1 h1:WvBo56Wzw3fjS+7vBjN6TeivvpbW9GmRaWZ9CIVmt4E= -sigs.k8s.io/kustomize/kyaml v0.18.1/go.mod h1:C3L2BFVU1jgcddNBE1TxuVLgS46TjObMwW5FT9FcjYo= -sigs.k8s.io/mdtoc v1.4.0 h1:2pDEwJSjoVrGr5BPkG+LoLkYLKvgtGYurrBY8ul3SxQ= -sigs.k8s.io/mdtoc v1.4.0/go.mod h1:KVnRRtK1rX9aQ95qF0rt3x2ytTxf3r7W7N41H+0KF0k= -sigs.k8s.io/release-utils v0.8.3 h1:KtOtA4qDmzJyeQ2zkDsFVI25+NViwms/o5eL2NftFdA= -sigs.k8s.io/release-utils v0.8.3/go.mod h1:fp82Fma06OXBhEJ+GUJKqvcplDBomruK1R/1fWJnsrQ= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/controller-runtime v0.8.0 h1:s0dYdo7lQgJiAf+alP82PRwbz+oAqL3oSyMQ18XRDOc= +sigs.k8s.io/controller-runtime v0.8.0/go.mod h1:v9Lbj5oX443uR7GXYY46E0EE2o7k2YxQ58GxVNeXSW4= +sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= +sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/hack/.tool-versions b/hack/.tool-versions deleted file mode 100644 index 4bd118c7b..000000000 --- a/hack/.tool-versions +++ /dev/null @@ -1,2 +0,0 @@ -kustomize 4.5.4 -helm 3.8.2 diff --git a/hack/boilerplate/boilerplate.generated.go.txt b/hack/boilerplate/boilerplate.generated.go.txt deleted file mode 100644 index daba3a171..000000000 --- a/hack/boilerplate/boilerplate.generated.go.txt +++ /dev/null @@ -1,16 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - diff --git a/hack/boilerplate/boilerplate.py b/hack/boilerplate/boilerplate.py index 01a74d067..614c6ef67 100755 --- a/hack/boilerplate/boilerplate.py +++ b/hack/boilerplate/boilerplate.py @@ -193,7 +193,7 @@ def get_regexs(): '(%s)' % "|".join(map(lambda l: str(l), years))) # strip // +build \n\n build constraints regexs["go_build_constraints"] = re.compile( - r"^((// \+build.*\n)|(//go:build.*\n))+\n", re.MULTILINE) + r"^(// \+build.*\n)+\n", re.MULTILINE) # strip #!.* from shell scripts regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) return regexs diff --git a/hack/generate-deploy-scripts.sh b/hack/generate-deploy-scripts.sh index 6e22b8d88..e20596c1d 100755 --- a/hack/generate-deploy-scripts.sh +++ b/hack/generate-deploy-scripts.sh @@ -18,42 +18,193 @@ if [ -n "$DEBUG" ]; then set -x fi -#set -o errexit +set -o errexit set -o nounset set -o pipefail -K8S_VERSION="1.22" - DIR=$(cd $(dirname "${BASH_SOURCE}")/.. && pwd -P) -# clean -rm -rf ${DIR}/deploy/static/provider/* +RELEASE_NAME=ingress-nginx +NAMESPACE=ingress-nginx -TEMPLATE_DIR="${DIR}/hack/manifest-templates" +NAMESPACE_VAR=" +apiVersion: v1 +kind: Namespace +metadata: + name: $NAMESPACE + labels: + app.kubernetes.io/name: $RELEASE_NAME + app.kubernetes.io/instance: ingress-nginx +" -# each helm values file `values.yaml` under `hack/manifest-templates/provider` will be generated as provider/[/variant]/deploy.yaml -# TARGET is provider/[/variant] -TARGETS=$(dirname $(cd $DIR/hack/manifest-templates/ && find . -type f -name "values.yaml" ) | cut -d'/' -f2-) -for TARGET in ${TARGETS} -do - TARGET_DIR="${TEMPLATE_DIR}/${TARGET}" - MANIFEST="${TEMPLATE_DIR}/common/manifest.yaml" # intermediate manifest - OUTPUT_DIR="${DIR}/deploy/static/${TARGET}" - echo $OUTPUT_DIR +# Baremetal +OUTPUT_FILE="${DIR}/deploy/static/provider/baremetal/deploy.yaml" +cat << EOF | helm template $RELEASE_NAME ${DIR}/charts/ingress-nginx --namespace $NAMESPACE --values - | $DIR/hack/add-namespace.py $NAMESPACE > ${OUTPUT_FILE} +controller: + service: + type: NodePort - mkdir -p ${OUTPUT_DIR} - cd ${TARGET_DIR} - helm template ingress-nginx ${DIR}/charts/ingress-nginx \ - --values values.yaml \ - --namespace ingress-nginx \ - --kube-version ${K8S_VERSION} \ - > $MANIFEST - sed -i.bak '/app.kubernetes.io\/managed-by: Helm/d' $MANIFEST - sed -i.bak '/helm.sh/d' $MANIFEST + publishService: + enabled: false +EOF - kustomize --load-restrictor=LoadRestrictionsNone build . > ${OUTPUT_DIR}/deploy.yaml - rm $MANIFEST $MANIFEST.bak - cd ~- - # automatically generate the (unsupported) kustomization.yaml for each target - sed "s_{TARGET}_${TARGET}_" $TEMPLATE_DIR/static-kustomization-template.yaml > ${OUTPUT_DIR}/kustomization.yaml -done +echo "${NAMESPACE_VAR} +$(cat ${OUTPUT_FILE})" > ${OUTPUT_FILE} + +# Cloud - generic +OUTPUT_FILE="${DIR}/deploy/static/provider/cloud/deploy.yaml" +cat << EOF | helm template $RELEASE_NAME ${DIR}/charts/ingress-nginx --namespace $NAMESPACE --values - | $DIR/hack/add-namespace.py $NAMESPACE > ${OUTPUT_FILE} +controller: + service: + type: LoadBalancer + externalTrafficPolicy: Local +EOF + +echo "${NAMESPACE_VAR} +$(cat ${OUTPUT_FILE})" > ${OUTPUT_FILE} + + +# AWS - NLB +OUTPUT_FILE="${DIR}/deploy/static/provider/aws/deploy.yaml" +cat << EOF | helm template $RELEASE_NAME ${DIR}/charts/ingress-nginx --namespace $NAMESPACE --values - | $DIR/hack/add-namespace.py $NAMESPACE > ${OUTPUT_FILE} +controller: + service: + type: LoadBalancer + externalTrafficPolicy: Local + annotations: + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "tcp" + service.beta.kubernetes.io/aws-load-balancer-type: nlb + service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true" +EOF + +echo "${NAMESPACE_VAR} +$(cat ${OUTPUT_FILE})" > ${OUTPUT_FILE} + + +OUTPUT_FILE="${DIR}/deploy/static/provider/aws/deploy-tls-termination.yaml" +cat << EOF | helm template $RELEASE_NAME ${DIR}/charts/ingress-nginx --namespace $NAMESPACE --values - | $DIR/hack/add-namespace.py $NAMESPACE > ${OUTPUT_FILE} +controller: + service: + type: LoadBalancer + externalTrafficPolicy: Local + + annotations: + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http + service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: 'true' + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "https" + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "arn:aws:acm:us-west-2:XXXXXXXX:certificate/XXXXXX-XXXXXXX-XXXXXXX-XXXXXXXX" + service.beta.kubernetes.io/aws-load-balancer-type: elb + # Ensure the ELB idle timeout is less than nginx keep-alive timeout. By default, + # NGINX keep-alive is set to 75s. If using WebSockets, the value will need to be + # increased to '3600' to avoid any potential issues. + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "60" + + targetPorts: + http: tohttps + https: http + + # Configures the ports the nginx-controller listens on + containerPort: + http: 80 + https: 80 + tohttps: 2443 + + config: + proxy-real-ip-cidr: XXX.XXX.XXX/XX + use-forwarded-headers: "true" + http-snippet: | + server { + listen 2443; + return 308 https://\$host\$request_uri; + } +EOF + +echo "${NAMESPACE_VAR} +$(cat ${OUTPUT_FILE})" > ${OUTPUT_FILE} + +# Kind - https://kind.sigs.k8s.io/docs/user/ingress/ +OUTPUT_FILE="${DIR}/deploy/static/provider/kind/deploy.yaml" +cat << EOF | helm template $RELEASE_NAME ${DIR}/charts/ingress-nginx --namespace $NAMESPACE --values - | $DIR/hack/add-namespace.py $NAMESPACE > ${OUTPUT_FILE} +controller: + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + hostPort: + enabled: true + terminationGracePeriodSeconds: 0 + service: + type: NodePort + + nodeSelector: + ingress-ready: "true" + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Equal" + effect: "NoSchedule" + + publishService: + enabled: false + extraArgs: + publish-status-address: localhost +EOF + +# Digital Ocean +echo "${NAMESPACE_VAR} +$(cat ${OUTPUT_FILE})" > ${OUTPUT_FILE} + +OUTPUT_FILE="${DIR}/deploy/static/provider/do/deploy.yaml" +cat << EOF | helm template $RELEASE_NAME ${DIR}/charts/ingress-nginx --namespace $NAMESPACE --values - | $DIR/hack/add-namespace.py $NAMESPACE > ${OUTPUT_FILE} +controller: + service: + type: LoadBalancer + externalTrafficPolicy: Local + annotations: + service.beta.kubernetes.io/do-loadbalancer-enable-proxy-protocol: "true" + config: + use-proxy-protocol: "true" + +EOF + +# Scaleway +echo "${NAMESPACE_VAR} +$(cat ${OUTPUT_FILE})" > ${OUTPUT_FILE} + +OUTPUT_FILE="${DIR}/deploy/static/provider/scw/deploy.yaml" +cat << EOF | helm template $RELEASE_NAME ${DIR}/charts/ingress-nginx --namespace $NAMESPACE --values - | $DIR/hack/add-namespace.py $NAMESPACE > ${OUTPUT_FILE} +controller: + service: + type: LoadBalancer + externalTrafficPolicy: Local + annotations: + service.beta.kubernetes.io/scw-loadbalancer-proxy-protocol-v2: "true" + config: + use-proxy-protocol: "true" + +EOF + +# Exoscale +echo "${NAMESPACE_VAR} +$(cat ${OUTPUT_FILE})" > ${OUTPUT_FILE} + +OUTPUT_FILE="${DIR}/deploy/static/provider/exoscale/deploy.yaml" +cat << EOF | helm template $RELEASE_NAME ${DIR}/charts/ingress-nginx --namespace $NAMESPACE --values - | $DIR/hack/add-namespace.py $NAMESPACE > ${OUTPUT_FILE} +controller: + kind: DaemonSet + service: + type: LoadBalancer + externalTrafficPolicy: Local + annotations: + service.beta.kubernetes.io/exoscale-loadbalancer-name: "nginx-ingress-controller" + service.beta.kubernetes.io/exoscale-loadbalancer-description: "NGINX Ingress Controller load balancer" + service.beta.kubernetes.io/exoscale-loadbalancer-service-strategy: "source-hash" + service.beta.kubernetes.io/exoscale-loadbalancer-service-healthcheck-mode: "tcp" + service.beta.kubernetes.io/exoscale-loadbalancer-service-healthcheck-interval: "10s" + service.beta.kubernetes.io/exoscale-loadbalancer-service-healthcheck-timeout: "3s" + service.beta.kubernetes.io/exoscale-loadbalancer-service-healthcheck-retries: "1" + publishService: + enabled: true +EOF + +echo "${NAMESPACE_VAR} +$(cat ${OUTPUT_FILE})" > ${OUTPUT_FILE} diff --git a/hack/generate-e2e-suite-doc.sh b/hack/generate-e2e-suite-doc.sh new file mode 100755 index 000000000..cfcc55040 --- /dev/null +++ b/hack/generate-e2e-suite-doc.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ -n "$DEBUG" ]; then + set -x +fi + +set -o errexit +set -o nounset +set -o pipefail + +URL="https://github.com/kubernetes/ingress-nginx/tree/master/" +DIR=$(cd $(dirname "${BASH_SOURCE}")/.. && pwd -P) + +echo " + +# e2e test suite for [NGINX Ingress Controller]($URL) + +" + +for FILE in `find $DIR/test/e2e -name "*.go"`;do + # describe definition + DESCRIBE=$(cat $FILE | grep -n -oP 'Describe.*') + # line number + DESCRIBE_LINE=$(echo $DESCRIBE | cut -f1 -d ':') + # clean describe, extracting the string + DESCRIBE=$(echo $DESCRIBE | sed -En 's/.*"(.*)".*/\1/p') + + FILE_URL=$(echo $FILE | sed "s|${DIR}/|${URL}|g") + echo " +### [$DESCRIBE]($FILE_URL#L$DESCRIBE_LINE) +" + + # extract Tests + ITS=$(cat $FILE | grep -n -oP 'It\(.*') + while IFS= read -r line; do + IT_LINE=$(echo $line | cut -f1 -d ':') + IT=$(echo $line | sed -En 's/.*"(.*)".*/\1/p') + echo "- [$IT]($FILE_URL#L$IT_LINE)" + done <<< "$ITS" +done diff --git a/hack/init-buildx.sh b/hack/init-buildx.sh index bac68e1ae..a24b64956 100755 --- a/hack/init-buildx.sh +++ b/hack/init-buildx.sh @@ -29,28 +29,26 @@ if ! docker buildx 2>&1 >/dev/null; then exit 1 fi +# We can skip setup if the current builder already has multi-arch +# AND if it isn't the docker driver, which doesn't work +current_builder="$(docker buildx inspect)" +# linux/amd64, linux/arm64, linux/riscv64, linux/ppc64le, linux/s390x, linux/386, linux/arm/v7, linux/arm/v6 +if ! grep -q "^Driver: docker$" <<<"${current_builder}" && \ + grep -q "linux/amd64" <<<"${current_builder}" && \ + grep -q "linux/arm" <<<"${current_builder}" && \ + grep -q "linux/arm64" <<<"${current_builder}" && \ + grep -q "linux/s390x" <<<"${current_builder}"; then + exit 0 +fi + # Ensure qemu is in binfmt_misc # Docker desktop already has these in versions recent enough to have buildx # We only need to do this setup on linux hosts if [ "$(uname)" == 'Linux' ]; then # NOTE: this is pinned to a digest for a reason! - # Note2 (@rikatz) - Removing the pin, as apparently it's breaking new alpine builds - # docker run --rm --privileged multiarch/qemu-user-static@sha256:28ebe2e48220ae8fd5d04bb2c847293b24d7fbfad84f0b970246e0a4efd48ad6 --reset -p yes - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + docker run --rm --privileged multiarch/qemu-user-static@sha256:28ebe2e48220ae8fd5d04bb2c847293b24d7fbfad84f0b970246e0a4efd48ad6 --reset -p yes fi -# We can skip setup if the current builder already has multi-arch -# AND if it isn't the docker driver, which doesn't work -current_builder="$(docker buildx inspect)" -# linux/amd64, linux/arm, linux/arm64 -if ! grep -q "^Driver: docker$" <<<"${current_builder}" && \ - grep -q "linux/amd64" <<<"${current_builder}" && \ - grep -q "linux/arm" <<<"${current_builder}" && \ - grep -q "linux/arm64" <<<"${current_builder}"; then - exit 0 -fi - - # Ensure we use a builder that can leverage it (the default on linux will not) docker buildx rm ingress-nginx || true docker buildx create --use --name=ingress-nginx diff --git a/hack/manifest-templates/common/kustomization.yaml b/hack/manifest-templates/common/kustomization.yaml deleted file mode 100644 index a4b5f64cf..000000000 --- a/hack/manifest-templates/common/kustomization.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- manifest.yaml -- namespace.yaml - -patches: -- target: - group: apps - version: v1 - kind: Deployment - patch: |- - - op: remove - path: /spec/replicas diff --git a/hack/manifest-templates/common/namespace.yaml b/hack/manifest-templates/common/namespace.yaml deleted file mode 100644 index db890604b..000000000 --- a/hack/manifest-templates/common/namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx diff --git a/hack/manifest-templates/provider/aws/kustomization.yaml b/hack/manifest-templates/provider/aws/kustomization.yaml deleted file mode 100644 index cd6ef95be..000000000 --- a/hack/manifest-templates/provider/aws/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- ../../common diff --git a/hack/manifest-templates/provider/aws/nlb-with-tls-termination/kustomization.yaml b/hack/manifest-templates/provider/aws/nlb-with-tls-termination/kustomization.yaml deleted file mode 100644 index ecec1095c..000000000 --- a/hack/manifest-templates/provider/aws/nlb-with-tls-termination/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- ../../../common diff --git a/hack/manifest-templates/provider/aws/nlb-with-tls-termination/values.yaml b/hack/manifest-templates/provider/aws/nlb-with-tls-termination/values.yaml deleted file mode 100644 index 46f8417c7..000000000 --- a/hack/manifest-templates/provider/aws/nlb-with-tls-termination/values.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# AWS NLB with TLS termination -controller: - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - service: - type: LoadBalancer - externalTrafficPolicy: Local - - annotations: - # This example is for legacy in-tree service load balancer controller for AWS NLB, - # that has been phased out from Kubernetes mainline. - service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true" - service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "https" - service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "arn:aws:acm:us-west-2:XXXXXXXX:certificate/XXXXXX-XXXXXXX-XXXXXXX-XXXXXXXX" - service.beta.kubernetes.io/aws-load-balancer-type: nlb - # Ensure the ELB idle timeout is less than nginx keep-alive timeout. By default, - # NGINX keep-alive is set to 75s. If using WebSockets, the value will need to be - # increased to '3600' to avoid any potential issues. - service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "60" - - targetPorts: - http: tohttps - https: http - - # Configures the ports the nginx-controller listens on - containerPort: - http: 80 - https: 80 - tohttps: 2443 - - config: - proxy-real-ip-cidr: XXX.XXX.XXX/XX - use-forwarded-headers: "true" - http-snippet: | - server { - listen 2443; - return 308 https://$host$request_uri; - } diff --git a/hack/manifest-templates/provider/aws/values.yaml b/hack/manifest-templates/provider/aws/values.yaml deleted file mode 100644 index 37eac03c2..000000000 --- a/hack/manifest-templates/provider/aws/values.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# AWS - NLB -controller: - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - service: - type: LoadBalancer - externalTrafficPolicy: Local - annotations: - service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "tcp" - service.beta.kubernetes.io/aws-load-balancer-type: nlb - service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true" diff --git a/hack/manifest-templates/provider/baremetal/kustomization.yaml b/hack/manifest-templates/provider/baremetal/kustomization.yaml deleted file mode 100644 index cd6ef95be..000000000 --- a/hack/manifest-templates/provider/baremetal/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- ../../common diff --git a/hack/manifest-templates/provider/baremetal/values.yaml b/hack/manifest-templates/provider/baremetal/values.yaml deleted file mode 100644 index 205a7430c..000000000 --- a/hack/manifest-templates/provider/baremetal/values.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# Baremetal -controller: - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - service: - type: NodePort - - publishService: - enabled: false diff --git a/hack/manifest-templates/provider/cloud/kustomization.yaml b/hack/manifest-templates/provider/cloud/kustomization.yaml deleted file mode 100644 index cd6ef95be..000000000 --- a/hack/manifest-templates/provider/cloud/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- ../../common diff --git a/hack/manifest-templates/provider/cloud/values.yaml b/hack/manifest-templates/provider/cloud/values.yaml deleted file mode 100644 index edc5662dd..000000000 --- a/hack/manifest-templates/provider/cloud/values.yaml +++ /dev/null @@ -1,8 +0,0 @@ -controller: - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - service: - type: LoadBalancer - externalTrafficPolicy: Local diff --git a/hack/manifest-templates/provider/do/kustomization.yaml b/hack/manifest-templates/provider/do/kustomization.yaml deleted file mode 100644 index cd6ef95be..000000000 --- a/hack/manifest-templates/provider/do/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- ../../common diff --git a/hack/manifest-templates/provider/do/values.yaml b/hack/manifest-templates/provider/do/values.yaml deleted file mode 100644 index aeee47ad6..000000000 --- a/hack/manifest-templates/provider/do/values.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# Digital Ocean -controller: - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - service: - type: LoadBalancer - externalTrafficPolicy: Local - annotations: - service.beta.kubernetes.io/do-loadbalancer-enable-proxy-protocol: "true" - config: - use-proxy-protocol: "true" - admissionWebhooks: - timeoutSeconds: 29 diff --git a/hack/manifest-templates/provider/exoscale/kustomization.yaml b/hack/manifest-templates/provider/exoscale/kustomization.yaml deleted file mode 100644 index cd6ef95be..000000000 --- a/hack/manifest-templates/provider/exoscale/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- ../../common diff --git a/hack/manifest-templates/provider/exoscale/values.yaml b/hack/manifest-templates/provider/exoscale/values.yaml deleted file mode 100644 index 867ecc57e..000000000 --- a/hack/manifest-templates/provider/exoscale/values.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Exoscale -controller: - kind: DaemonSet - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - service: - type: LoadBalancer - externalTrafficPolicy: Local - annotations: - service.beta.kubernetes.io/exoscale-loadbalancer-description: "NGINX Ingress Controller load balancer" - service.beta.kubernetes.io/exoscale-loadbalancer-service-strategy: "source-hash" - service.beta.kubernetes.io/exoscale-loadbalancer-service-healthcheck-mode: "http" - service.beta.kubernetes.io/exoscale-loadbalancer-service-healthcheck-uri: "/" - service.beta.kubernetes.io/exoscale-loadbalancer-service-healthcheck-interval: "10s" - service.beta.kubernetes.io/exoscale-loadbalancer-service-healthcheck-timeout: "3s" - service.beta.kubernetes.io/exoscale-loadbalancer-service-healthcheck-retries: "1" - publishService: - enabled: true diff --git a/hack/manifest-templates/provider/kind/kustomization.yaml b/hack/manifest-templates/provider/kind/kustomization.yaml deleted file mode 100644 index cd6ef95be..000000000 --- a/hack/manifest-templates/provider/kind/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- ../../common diff --git a/hack/manifest-templates/provider/kind/values.yaml b/hack/manifest-templates/provider/kind/values.yaml deleted file mode 100644 index 6140f6500..000000000 --- a/hack/manifest-templates/provider/kind/values.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Kind - https://kind.sigs.k8s.io/docs/user/ingress/ -controller: - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - hostPort: - enabled: true - terminationGracePeriodSeconds: 0 - service: - type: LoadBalancer - watchIngressWithoutClass: true - - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Equal" - effect: "NoSchedule" - - key: "node-role.kubernetes.io/control-plane" - operator: "Equal" - effect: "NoSchedule" - - publishService: - enabled: false - extraArgs: - publish-status-address: localhost diff --git a/hack/manifest-templates/provider/oracle/kustomization.yaml b/hack/manifest-templates/provider/oracle/kustomization.yaml deleted file mode 100644 index cd6ef95be..000000000 --- a/hack/manifest-templates/provider/oracle/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- ../../common diff --git a/hack/manifest-templates/provider/oracle/values.yaml b/hack/manifest-templates/provider/oracle/values.yaml deleted file mode 100644 index 600dbfe5c..000000000 --- a/hack/manifest-templates/provider/oracle/values.yaml +++ /dev/null @@ -1,12 +0,0 @@ -controller: - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - service: - type: LoadBalancer - externalTrafficPolicy: Local - annotations: - service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "10" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "100" diff --git a/hack/manifest-templates/provider/scw/kustomization.yaml b/hack/manifest-templates/provider/scw/kustomization.yaml deleted file mode 100644 index cd6ef95be..000000000 --- a/hack/manifest-templates/provider/scw/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- ../../common diff --git a/hack/manifest-templates/provider/scw/values.yaml b/hack/manifest-templates/provider/scw/values.yaml deleted file mode 100644 index 56d351dcd..000000000 --- a/hack/manifest-templates/provider/scw/values.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Scaleway -controller: - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - service: - type: LoadBalancer - externalTrafficPolicy: Local - annotations: - service.beta.kubernetes.io/scw-loadbalancer-proxy-protocol-v2: "true" - config: - use-proxy-protocol: "true" diff --git a/hack/manifest-templates/static-kustomization-template.yaml b/hack/manifest-templates/static-kustomization-template.yaml deleted file mode 100644 index 18118a0ce..000000000 --- a/hack/manifest-templates/static-kustomization-template.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# NOTE: kustomize is not supported. This file exists only to be able to reference it from bases. -# https://kubectl.docs.kubernetes.io/references/kustomize/bases/ -# -# ``` -# namespace: ingress-nginx -# bases: -# - github.com/kubernetes/ingress-nginx/tree/main/deploy/static/{TARGET} -# ``` - -resources: - - deploy.yaml diff --git a/hack/print-e2e-suite.sh b/hack/print-e2e-suite.sh index 12b0efb13..e6b083d58 100755 --- a/hack/print-e2e-suite.sh +++ b/hack/print-e2e-suite.sh @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -if [ "$DEBUG" == "true" ]; then - set -x +if [ -n "$DEBUG" ]; then + set -x fi set -o errexit @@ -25,6 +25,5 @@ set -o pipefail DIR=$(cd $(dirname "${BASH_SOURCE}")/.. && pwd -P) $DIR/test/e2e/e2e.test \ - --ginkgo.no-color \ - --ginkgo.v \ - --ginkgo.dry-run | sed "s|$DIR/|File: |g" | sed 's/•//g' | tail -n+5 | head -n-3 + -ginkgo.noColor \ + -ginkgo.dryRun | sed "s|$DIR/|File: |g" | sed 's/•//g' | tail -n+5 | head -n-3 diff --git a/hack/tools.go b/hack/tools.go index 489a3ccb0..f2d455a94 100644 --- a/hack/tools.go +++ b/hack/tools.go @@ -1,4 +1,3 @@ -//go:build tools // +build tools /* @@ -22,6 +21,6 @@ limitations under the License. package tools import ( + _ "github.com/tallclair/mdtoc" _ "k8s.io/code-generator" - _ "sigs.k8s.io/mdtoc" ) diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index 47a104584..a80a03b3a 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -27,7 +27,7 @@ if [[ ! -d ${CODEGEN_PKG} ]]; then fi # Ensure we can execute. -chmod +x ${CODEGEN_PKG}/kube_codegen.sh +chmod +x ${CODEGEN_PKG}/generate-groups.sh # generate the code with: # --output-base because this script should also be able to run inside the vendor dir of @@ -37,8 +37,8 @@ chmod +x ${CODEGEN_PKG}/kube_codegen.sh # k8s.io/ingress-nginx/pkg/client k8s.io/ingress-nginx/pkg/apis \ # nginxingress:v1alpha1 \ # --output-base "$(dirname ${BASH_SOURCE})/../../.." -${CODEGEN_PKG}/kube_codegen.sh "deepcopy" \ - k8s.io/ingress-nginx/internal k8s.io/ingress-nginx/pkg/apis \ +${CODEGEN_PKG}/generate-groups.sh "deepcopy" \ + k8s.io/ingress-nginx/internal k8s.io/ingress-nginx/internal \ .:ingress \ --output-base "$(dirname ${BASH_SOURCE})/../../.." \ - --go-header-file ${SCRIPT_ROOT}/hack/boilerplate/boilerplate.generated.go.txt + --go-header-file ${SCRIPT_ROOT}/hack/boilerplate/boilerplate.go.txt diff --git a/hack/verify-annotation-docs.sh b/hack/verify-annotation-docs.sh deleted file mode 100755 index 54034539b..000000000 --- a/hack/verify-annotation-docs.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# Copyright 2024 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -SCRIPT_ROOT=$(dirname "${BASH_SOURCE}")/.. - -ANNOTATIONFILE="${SCRIPT_ROOT}/docs/user-guide/nginx-configuration/annotations-risk.md" -TMP_DIFFROOT="${SCRIPT_ROOT}/_tmp" -TMP_FILE="${TMP_DIFFROOT}/annotations-risk.md" - -cleanup() { - rm -rf "${TMP_DIFFROOT}" -} -trap "cleanup" EXIT SIGINT - -cleanup - -mkdir -p "${TMP_DIFFROOT}" - -go run cmd/annotations/main.go -output "${TMP_FILE}" -echo "diffing ${ANNOTATIONFILE} against freshly generated annotation doc" -ret=0 -diff -Naupr --no-dereference "${ANNOTATIONFILE}" "${TMP_FILE}" || ret=1 - -if [[ $ret -eq 0 ]]; then - echo "${ANNOTATIONFILE} up to date." -else - echo "${ANNOTATIONFILE} is out of date. Please run hack/update-annotation-doc.sh" - exit 1 -fi diff --git a/hack/update-annotation-doc.sh b/hack/verify-chart-lint.sh similarity index 70% rename from hack/update-annotation-doc.sh rename to hack/verify-chart-lint.sh index c4feb41ce..c18caaba2 100755 --- a/hack/update-annotation-doc.sh +++ b/hack/verify-chart-lint.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2024 The Kubernetes Authors. +# Copyright 2020 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,8 +16,8 @@ set -o errexit set -o nounset +set -o pipefail -SCRIPT_ROOT=$(dirname "${BASH_SOURCE}")/.. -ANNOTATIONFILE="${SCRIPT_ROOT}/docs/user-guide/nginx-configuration/annotations-risk.md" +KUBE_ROOT="$( cd "$(dirname "$0")../" >/dev/null 2>&1 ; pwd -P )" -go run "${SCRIPT_ROOT}"/cmd/annotations/main.go -output "${ANNOTATIONFILE}" \ No newline at end of file +ct lint --charts ${KUBE_ROOT}/charts/ingress-nginx --validate-maintainers=false diff --git a/hack/verify-golint.sh b/hack/verify-golint.sh index 17bcedd9f..7c9284880 100755 --- a/hack/verify-golint.sh +++ b/hack/verify-golint.sh @@ -22,13 +22,19 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. cd "${KUBE_ROOT}" -LINT=${LINT:-golangci-lint} - -if [[ -z "$(command -v ${LINT})" ]]; then - echo "${LINT} is missing. Installing it now." - # See: https://golangci-lint.run/usage/install/#local-installation - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.53.3 - LINT=$(go env GOPATH)/bin/golangci-lint +GOLINT=${GOLINT:-"golint"} +PACKAGES=($(go list ./internal/... | grep -v /vendor/)) +bad_files=() +for package in "${PACKAGES[@]}"; do + out=$("${GOLINT}" -min_confidence=0.9 "${package}" | grep -v -E '(should not use dot imports|internal/file/bindata.go)' || :) + if [[ -n "${out}" ]]; then + bad_files+=("${out}") + fi +done +if [[ "${#bad_files[@]}" -ne 0 ]]; then + echo "!!! '$GOLINT' problems: " + echo "${bad_files[@]}" + exit 1 fi -${LINT} run +# ex: ts=2 sw=2 et filetype=sh diff --git a/hack/verify-lualint.sh b/hack/verify-lualint.sh index 769d7a6cd..1f6048de0 100755 --- a/hack/verify-lualint.sh +++ b/hack/verify-lualint.sh @@ -18,13 +18,6 @@ set -o errexit set -o nounset set -o pipefail -luacheck --codes --globals lua_ingress \ - --globals configuration \ - --globals balancer \ - --globals monitor \ - --globals certificate \ - --globals tcp_udp_configuration \ - --globals tcp_udp_balancer \ - --no-max-comment-line-length -q rootfs/etc/nginx/lua/ +luacheck --codes -q rootfs/etc/nginx/lua/ find rootfs/etc/nginx/lua/ -name "*.lua" -not -path "*/test/*" -exec lj-releng -L -s {} + && echo "lj-releng validation is success!" diff --git a/images/OWNERS b/images/OWNERS new file mode 100644 index 000000000..b135a937c --- /dev/null +++ b/images/OWNERS @@ -0,0 +1,7 @@ +approvers: +- aledbf +- ElvinEfendi + +reviewers: +- ElvinEfendi +- aledbf diff --git a/images/README.md b/images/README.md index e7e5f71a6..13f6bfc3c 100644 --- a/images/README.md +++ b/images/README.md @@ -13,11 +13,10 @@ Directory | Purpose ------------ | ------------- -custom-error-pages | Example of Custom error pages for the Ingress-Nginx Controller +custom-error-pages | Example of Custom error pages for the NGINX Ingress controller e2e | Image to run e2e tests fastcgi-helloserver | FastCGI application for e2e tests grpc-fortune-teller | grpc server application for the nginx-ingress grpc example -httpbun | A simple HTTP Request & Response Service for e2e tests -httpbin | [Removed] we are no longer maintaining the httpbin image due to project being unmaintained +httpbin | A simple HTTP Request & Response Service for e2e tests nginx | NGINX base image using [alpine linux](https://www.alpinelinux.org) cfssl | Image to run cfssl commands diff --git a/images/cfssl/Makefile b/images/cfssl/Makefile new file mode 100644 index 000000000..344646462 --- /dev/null +++ b/images/cfssl/Makefile @@ -0,0 +1,55 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# set default shell +SHELL=/bin/bash -o pipefail -o errexit + +DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))) +INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh + +TAG ?=v$(shell date +%m%d%Y)-$(shell git rev-parse --short HEAD) +REGISTRY ?= local + +IMAGE = $(REGISTRY)/e2e-test-cfssl + +# required to enable buildx +export DOCKER_CLI_EXPERIMENTAL=enabled + +# build with buildx +PLATFORMS?=linux/amd64,linux/arm,linux/arm64,linux/s390x +OUTPUT= +PROGRESS=plain + +build: ensure-buildx + docker buildx build \ + --platform=${PLATFORMS} $(OUTPUT) \ + --progress=$(PROGRESS) \ + --pull \ + -t $(IMAGE):$(TAG) rootfs + +# push the cross built image +push: OUTPUT=--push +push: build + +# enable buildx +ensure-buildx: +# this is required for cloudbuild +ifeq ("$(wildcard $(INIT_BUILDX))","") + @curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/hack/init-buildx.sh | bash +else + @exec $(INIT_BUILDX) +endif + @echo "done" + +.PHONY: build push ensure-buildx diff --git a/images/cfssl/TAG b/images/cfssl/TAG deleted file mode 100644 index 56130fb3a..000000000 --- a/images/cfssl/TAG +++ /dev/null @@ -1 +0,0 @@ -v1.1.1 diff --git a/images/cfssl/cloudbuild.yaml b/images/cfssl/cloudbuild.yaml index 33fafdb08..d80d2aeaf 100644 --- a/images/cfssl/cloudbuild.yaml +++ b/images/cfssl/cloudbuild.yaml @@ -1,11 +1,22 @@ +timeout: 600s options: - # Ignore Prow provided substitutions. substitution_option: ALLOW_LOOSE steps: - - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20250116-2a05ea7e3d - env: - - REGISTRY=us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx + - name: gcr.io/k8s-testimages/gcb-docker-gcloud:v20201130-750d12f entrypoint: bash + env: + - DOCKER_CLI_EXPERIMENTAL=enabled + - TAG=$_GIT_TAG + - BASE_REF=$_PULL_BASE_REF + - REGISTRY=gcr.io/k8s-staging-ingress-nginx + # default cloudbuild has HOME=/builder/home and docker buildx is in /root/.docker/cli-plugins/docker-buildx + # set the home to /root explicitly to if using docker buildx + - HOME=/root args: - - -c - - gcloud auth configure-docker && cd images && make NAME=cfssl push + - -c + - | + gcloud auth configure-docker \ + && make push +substitutions: + _GIT_TAG: "12345" + _PULL_BASE_REF: "master" diff --git a/images/cfssl/rootfs/Dockerfile b/images/cfssl/rootfs/Dockerfile index 3978c8f4b..f12a1c776 100644 --- a/images/cfssl/rootfs/Dockerfile +++ b/images/cfssl/rootfs/Dockerfile @@ -12,12 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM alpine:3.21 +FROM alpine:3.13 -RUN echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories -RUN apk update \ - && apk upgrade && \ - apk add --no-cache \ +RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories +RUN apk add --no-cache \ bash \ cfssl@testing diff --git a/images/custom-error-pages/Makefile b/images/custom-error-pages/Makefile new file mode 100644 index 000000000..d08fa27e1 --- /dev/null +++ b/images/custom-error-pages/Makefile @@ -0,0 +1,105 @@ +all: all-container + +BUILDTAGS= + +# Use the 0.0 tag for testing, it shouldn't clobber any release builds +TAG?=0.4 +REGISTRY?=quay.io/kubernetes-ingress-controller +GOOS?=linux +DOCKER?=docker +SED_I?=sed -i +GOHOSTOS ?= $(shell go env GOHOSTOS) + +PKG=k8s.io/ingress-nginx/images/custom-error-pages + +ifeq ($(GOHOSTOS),darwin) + SED_I=sed -i '' +endif + +REPO_INFO=$(shell git config --get remote.origin.url) + +ifndef COMMIT + COMMIT := git-$(shell git rev-parse --short HEAD) +endif + +ARCH ?= $(shell go env GOARCH) +GOARCH = ${ARCH} + +BASEIMAGE?=alpine:3.10 + +ALL_ARCH = amd64 arm arm64 + +QEMUVERSION=v4.1.0-1 + +IMGNAME = custom-error-pages +IMAGE = $(REGISTRY)/$(IMGNAME) +MULTI_ARCH_IMG = $(IMAGE)-$(ARCH) + +ifeq ($(ARCH),arm) + QEMUARCH=arm + GOARCH=arm +endif +ifeq ($(ARCH),arm64) + QEMUARCH=aarch64 +endif + +TEMP_DIR := $(shell mktemp -d) + +DOCKERFILE := $(TEMP_DIR)/rootfs/Dockerfile + +sub-container-%: + $(MAKE) ARCH=$* build container + +sub-push-%: + $(MAKE) ARCH=$* push + +all-container: $(addprefix sub-container-,$(ALL_ARCH)) + +all-push: $(addprefix sub-push-,$(ALL_ARCH)) + +container: .container-$(ARCH) +.container-$(ARCH): + cp -r ./* $(TEMP_DIR) + $(SED_I) 's|BASEIMAGE|$(BASEIMAGE)|g' $(DOCKERFILE) + $(SED_I) "s|QEMUARCH|$(QEMUARCH)|g" $(DOCKERFILE) + +ifeq ($(ARCH),amd64) + # When building "normally" for amd64, remove the whole line, it has no part in the amd64 image + $(SED_I) "/CROSS_BUILD_/d" $(DOCKERFILE) +else + # When cross-building, only the placeholder "CROSS_BUILD_" should be removed + # Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel + # $(DOCKER) run --rm --privileged multiarch/qemu-user-static:register --reset + curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/$(QEMUVERSION)/x86_64_qemu-$(QEMUARCH)-static.tar.gz | tar -xz -C $(TEMP_DIR)/rootfs + $(SED_I) "s/CROSS_BUILD_//g" $(DOCKERFILE) +endif + + $(DOCKER) build -t $(MULTI_ARCH_IMG):$(TAG) $(TEMP_DIR)/rootfs + +ifeq ($(ARCH), amd64) + # This is for to maintain the backward compatibility + $(DOCKER) tag $(MULTI_ARCH_IMG):$(TAG) $(IMAGE):$(TAG) +endif + +push: .push-$(ARCH) +.push-$(ARCH): + $(DOCKER) push $(MULTI_ARCH_IMG):$(TAG) +ifeq ($(ARCH), amd64) + $(DOCKER) push $(IMAGE):$(TAG) +endif + +clean: + $(DOCKER) rmi -f $(MULTI_ARCH_IMG):$(TAG) || true + +build: clean + CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} go build -a -installsuffix cgo \ + -ldflags "-s -w" \ + -o ${TEMP_DIR}/rootfs/custom-error-pages ${PKG}/... + +release: all-container all-push + echo "done" + +.PHONY: register-qemu +register-qemu: + # Register /usr/bin/qemu-ARCH-static as the handler for binaries in multiple platforms + $(DOCKER) run --rm --privileged multiarch/qemu-user-static:register --reset diff --git a/images/custom-error-pages/README.md b/images/custom-error-pages/README.md new file mode 100644 index 000000000..88983591f --- /dev/null +++ b/images/custom-error-pages/README.md @@ -0,0 +1,3 @@ +# custom-error-pages + +Example of Custom error pages for the NGINX Ingress controller diff --git a/images/custom-error-pages/TAG b/images/custom-error-pages/TAG deleted file mode 100644 index 56130fb3a..000000000 --- a/images/custom-error-pages/TAG +++ /dev/null @@ -1 +0,0 @@ -v1.1.1 diff --git a/images/custom-error-pages/cloudbuild.yaml b/images/custom-error-pages/cloudbuild.yaml deleted file mode 100644 index 324a8f19a..000000000 --- a/images/custom-error-pages/cloudbuild.yaml +++ /dev/null @@ -1,11 +0,0 @@ -options: - # Ignore Prow provided substitutions. - substitution_option: ALLOW_LOOSE -steps: - - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20250116-2a05ea7e3d - env: - - REGISTRY=us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx - entrypoint: bash - args: - - -c - - gcloud auth configure-docker && cd images && make NAME=custom-error-pages push diff --git a/images/custom-error-pages/rootfs/main.go b/images/custom-error-pages/main.go similarity index 83% rename from images/custom-error-pages/rootfs/main.go rename to images/custom-error-pages/main.go index 7d3d73029..8f4e6b267 100644 --- a/images/custom-error-pages/rootfs/main.go +++ b/images/custom-error-pages/main.go @@ -62,12 +62,6 @@ const ( // ErrFilesPathVar is the name of the environment variable indicating // the location on disk of files served by the handler. ErrFilesPathVar = "ERROR_FILES_PATH" - - // DefaultFormatVar is the name of the environment variable indicating - // the default error MIME type that should be returned if either the - // client does not specify an Accept header, or the Accept header provided - // cannot be mapped to a file extension. - DefaultFormatVar = "DEFAULT_RESPONSE_FORMAT" ) func init() { @@ -81,12 +75,7 @@ func main() { errFilesPath = os.Getenv(ErrFilesPathVar) } - defaultFormat := "text/html" - if os.Getenv(DefaultFormatVar) != "" { - defaultFormat = os.Getenv(DefaultFormatVar) - } - - http.HandleFunc("/", errorHandler(errFilesPath, defaultFormat)) + http.HandleFunc("/", errorHandler(errFilesPath)) http.Handle("/metrics", promhttp.Handler()) @@ -97,16 +86,10 @@ func main() { http.ListenAndServe(fmt.Sprintf(":8080"), nil) } -func errorHandler(path, defaultFormat string) func(http.ResponseWriter, *http.Request) { - defaultExts, err := mime.ExtensionsByType(defaultFormat) - if err != nil || len(defaultExts) == 0 { - panic("couldn't get file extension for default format") - } - defaultExt := defaultExts[0] - +func errorHandler(path string) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { start := time.Now() - ext := defaultExt + ext := "html" if os.Getenv("DEBUG") != "" { w.Header().Set(FormatHeader, r.Header.Get(FormatHeader)) @@ -122,14 +105,14 @@ func errorHandler(path, defaultFormat string) func(http.ResponseWriter, *http.Re format := r.Header.Get(FormatHeader) if format == "" { - format = defaultFormat + format = "text/html" log.Printf("format not specified. Using %v", format) } cext, err := mime.ExtensionsByType(format) if err != nil { log.Printf("unexpected error reading media type extension: %v. Using %v", err, ext) - format = defaultFormat + format = "text/html" } else if len(cext) == 0 { log.Printf("couldn't get media type extension. Using %v", ext) } else { @@ -148,10 +131,6 @@ func errorHandler(path, defaultFormat string) func(http.ResponseWriter, *http.Re if !strings.HasPrefix(ext, ".") { ext = "." + ext } - // special case for compatibility - if ext == ".htm" { - ext = ".html" - } file := fmt.Sprintf("%v/%v%v", path, code, ext) f, err := os.Open(file) if err != nil { diff --git a/images/custom-error-pages/rootfs/metrics.go b/images/custom-error-pages/metrics.go similarity index 100% rename from images/custom-error-pages/rootfs/metrics.go rename to images/custom-error-pages/metrics.go diff --git a/images/custom-error-pages/rootfs/Dockerfile b/images/custom-error-pages/rootfs/Dockerfile index 478e4c22d..aaf60efb1 100755 --- a/images/custom-error-pages/rootfs/Dockerfile +++ b/images/custom-error-pages/rootfs/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2021 The Kubernetes Authors. All rights reserved. +# Copyright 2017 The Kubernetes Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,29 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GOLANG_VERSION +FROM BASEIMAGE -FROM golang:${GOLANG_VERSION}-alpine3.21 as builder +CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ -RUN apk update \ - && apk upgrade && apk add git +COPY . / -WORKDIR /go/src/k8s.io/ingress-nginx/images/custom-error-pages - -COPY . . - -RUN go get . && \ - CGO_ENABLED=0 go build -a -installsuffix cgo \ - -ldflags "-s -w" \ - -o nginx-errors . - -# Use distroless as minimal base image to package the binary -# Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot - -COPY --from=builder /go/src/k8s.io/ingress-nginx/images/custom-error-pages/nginx-errors / -COPY --from=builder /go/src/k8s.io/ingress-nginx/images/custom-error-pages/www /www -COPY --from=builder /go/src/k8s.io/ingress-nginx/images/custom-error-pages/etc /etc -USER nonroot:nonroot - -CMD ["/nginx-errors"] +CMD ["/custom-error-pages"] diff --git a/images/custom-error-pages/rootfs/go.mod b/images/custom-error-pages/rootfs/go.mod deleted file mode 100644 index 5c0377284..000000000 --- a/images/custom-error-pages/rootfs/go.mod +++ /dev/null @@ -1,17 +0,0 @@ -module k8s.io/ingress-nginx/custom-error-pages - -go 1.23.6 - -require github.com/prometheus/client_golang v1.20.5 - -require ( - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/klauspost/compress v1.17.9 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - golang.org/x/sys v0.22.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect -) diff --git a/images/custom-error-pages/rootfs/go.sum b/images/custom-error-pages/rootfs/go.sum deleted file mode 100644 index d5318cf86..000000000 --- a/images/custom-error-pages/rootfs/go.sum +++ /dev/null @@ -1,24 +0,0 @@ -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= -github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= diff --git a/images/e2e-test-echo/EXTRAARGS b/images/e2e-test-echo/EXTRAARGS deleted file mode 100644 index 3db8aaa5c..000000000 --- a/images/e2e-test-echo/EXTRAARGS +++ /dev/null @@ -1 +0,0 @@ ---build-arg LUAROCKS_VERSION=3.8.0 --build-arg LUAROCKS_SHA=ab6612ca9ab87c6984871d2712d05525775e8b50172701a0a1cabddf76de2be7 \ No newline at end of file diff --git a/images/e2e-test-echo/TAG b/images/e2e-test-echo/TAG deleted file mode 100644 index 56130fb3a..000000000 --- a/images/e2e-test-echo/TAG +++ /dev/null @@ -1 +0,0 @@ -v1.1.1 diff --git a/images/e2e-test-echo/cloudbuild.yaml b/images/e2e-test-echo/cloudbuild.yaml deleted file mode 100644 index 02bfc034a..000000000 --- a/images/e2e-test-echo/cloudbuild.yaml +++ /dev/null @@ -1,11 +0,0 @@ -options: - # Ignore Prow provided substitutions. - substitution_option: ALLOW_LOOSE -steps: - - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20250116-2a05ea7e3d - env: - - REGISTRY=us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx - entrypoint: bash - args: - - -c - - gcloud auth configure-docker && cd images && make NAME=e2e-test-echo push diff --git a/images/echo/Makefile b/images/echo/Makefile new file mode 100644 index 000000000..ca00528df --- /dev/null +++ b/images/echo/Makefile @@ -0,0 +1,58 @@ +# Copyright 2020 The Kubernetes Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# set default shell +SHELL=/bin/bash -o pipefail -o errexit + +DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))) +INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh + +TAG ?=v$(shell date +%m%d%Y)-$(shell git rev-parse --short HEAD) +REGISTRY ?= local + +IMAGE = $(REGISTRY)/e2e-test-echo + +# required to enable buildx +export DOCKER_CLI_EXPERIMENTAL=enabled + +# build with buildx +PLATFORMS?=linux/amd64 +OUTPUT= +PROGRESS=plain + +build: ensure-buildx + docker buildx build \ + --platform=${PLATFORMS} $(OUTPUT) \ + --progress=$(PROGRESS) \ + --pull \ + --build-arg BASE_IMAGE=k8s.gcr.io/ingress-nginx/nginx:v20210324-g8baef769d@sha256:fcfa3e9d1f8ec3141efedbf77cf659640f452a9c22165c78006ea462b84d06f6 \ + --build-arg LUAROCKS_VERSION=3.3.1 \ + --build-arg LUAROCKS_SHA=837481e408f7c06b59befe7ec194537c657687d624894bca7f79034302141a34 \ + -t $(IMAGE):$(TAG) rootfs + +# push the cross built image +push: OUTPUT=--push +push: build + +# enable buildx +ensure-buildx: +# this is required for cloudbuild +ifeq ("$(wildcard $(INIT_BUILDX))","") + @curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/hack/init-buildx.sh | bash +else + @exec $(INIT_BUILDX) +endif + @echo "done" + +.PHONY: build push ensure-buildx diff --git a/images/echo/cloudbuild.yaml b/images/echo/cloudbuild.yaml new file mode 100644 index 000000000..3e4d67d06 --- /dev/null +++ b/images/echo/cloudbuild.yaml @@ -0,0 +1,22 @@ +timeout: 1200s +options: + substitution_option: ALLOW_LOOSE +steps: + - name: gcr.io/k8s-testimages/gcb-docker-gcloud:v20201130-750d12f + entrypoint: bash + env: + - DOCKER_CLI_EXPERIMENTAL=enabled + - TAG=$_GIT_TAG + - BASE_REF=$_PULL_BASE_REF + - REGISTRY=gcr.io/k8s-staging-ingress-nginx + # default cloudbuild has HOME=/builder/home and docker buildx is in /root/.docker/cli-plugins/docker-buildx + # set the home to /root explicitly to if using docker buildx + - HOME=/root + args: + - -c + - | + gcloud auth configure-docker \ + && make push +substitutions: + _GIT_TAG: "12345" + _PULL_BASE_REF: "master" diff --git a/images/e2e-test-echo/rootfs/Dockerfile b/images/echo/rootfs/Dockerfile similarity index 77% rename from images/e2e-test-echo/rootfs/Dockerfile rename to images/echo/rootfs/Dockerfile index eaa919835..4c9b6ab55 100644 --- a/images/e2e-test-echo/rootfs/Dockerfile +++ b/images/echo/rootfs/Dockerfile @@ -2,7 +2,7 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -RUN apk update && apk upgrade && apk add -U --no-cache perl curl make unzip +RUN apk add -U perl curl make unzip ARG LUAROCKS_VERSION ARG LUAROCKS_SHA @@ -17,4 +17,4 @@ RUN wget -O /tmp/luarocks.tgz \ RUN luarocks install lua-resty-template -COPY nginx.conf /etc/nginx/nginx.conf \ No newline at end of file +COPY nginx.conf /etc/nginx/nginx.conf diff --git a/images/e2e-test-echo/rootfs/nginx.conf b/images/echo/rootfs/nginx.conf similarity index 100% rename from images/e2e-test-echo/rootfs/nginx.conf rename to images/echo/rootfs/nginx.conf diff --git a/images/ext-auth-example-authsvc/TAG b/images/ext-auth-example-authsvc/TAG deleted file mode 100644 index 56130fb3a..000000000 --- a/images/ext-auth-example-authsvc/TAG +++ /dev/null @@ -1 +0,0 @@ -v1.1.1 diff --git a/images/ext-auth-example-authsvc/rootfs/Dockerfile b/images/ext-auth-example-authsvc/rootfs/Dockerfile deleted file mode 100644 index ca142e663..000000000 --- a/images/ext-auth-example-authsvc/rootfs/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -ARG GOLANG_VERSION - -FROM golang:${GOLANG_VERSION}-alpine3.21 as builder -RUN mkdir /authsvc -WORKDIR /authsvc -COPY . ./ -RUN CGO_ENABLED=0 GOOS=linux go build -o authsvc authsvc.go - -FROM gcr.io/distroless/base-debian11 -COPY --from=builder /authsvc/authsvc / -EXPOSE 8080 -ENTRYPOINT ["/authsvc"] diff --git a/images/ext-auth-example-authsvc/rootfs/go.mod b/images/ext-auth-example-authsvc/rootfs/go.mod deleted file mode 100644 index 5f33a86ff..000000000 --- a/images/ext-auth-example-authsvc/rootfs/go.mod +++ /dev/null @@ -1,7 +0,0 @@ -module example.com/authsvc - -go 1.23.6 - -require k8s.io/apimachinery v0.32.2 - -require github.com/google/uuid v1.6.0 // indirect diff --git a/images/ext-auth-example-authsvc/rootfs/go.sum b/images/ext-auth-example-authsvc/rootfs/go.sum deleted file mode 100644 index 770c22f6a..000000000 --- a/images/ext-auth-example-authsvc/rootfs/go.sum +++ /dev/null @@ -1,4 +0,0 @@ -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -k8s.io/apimachinery v0.32.2 h1:yoQBR9ZGkA6Rgmhbp/yuT9/g+4lxtsGYwW6dR6BDPLQ= -k8s.io/apimachinery v0.32.2/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= diff --git a/images/Makefile b/images/fastcgi-helloserver/Makefile similarity index 51% rename from images/Makefile rename to images/fastcgi-helloserver/Makefile index 31560168d..e35e1fa79 100644 --- a/images/Makefile +++ b/images/fastcgi-helloserver/Makefile @@ -1,4 +1,4 @@ -# Copyright 2024 The Kubernetes Authors. +# Copyright 2020 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,30 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -.DEFAULT_GOAL:=build +# Docker image for e2e testing. # set default shell SHELL=/bin/bash -o pipefail -o errexit DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))) -INIT_BUILDX=$(DIR)/../hack/init-buildx.sh - - -BASE_IMAGE = $(shell cat $(DIR)/../NGINX_BASE) - -# The env below is called GO_VERSION and not GOLANG_VERSION because -# the gcb image we use to build already defines GOLANG_VERSION and is a -# really old version -GO_VERSION = $(shell cat $(DIR)/../GOLANG_VERSION) +INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh +TAG ?=v$(shell date +%m%d%Y)-$(shell git rev-parse --short HEAD) REGISTRY ?= local -NAME ?= - -IMAGE = $(REGISTRY)/$(NAME) -TAG ?= $(shell cat $(NAME)/TAG) - -EXTRAARGS ?= $(shell cat $(NAME)/EXTRAARGS) +IMAGE = $(REGISTRY)/e2e-test-fastcgi-helloserver # required to enable buildx export DOCKER_CLI_EXPERIMENTAL=enabled @@ -45,42 +33,25 @@ PLATFORMS?=linux/amd64,linux/arm,linux/arm64 OUTPUT= PROGRESS=plain - -precheck: -ifndef NAME - $(error NAME variable is required) -endif - -build: precheck ensure-buildx +build: ensure-buildx docker buildx build \ - --label=org.opencontainers.image.source=https://github.com/kubernetes/ingress-nginx \ - --label=org.opencontainers.image.licenses=Apache-2.0 \ - --label=org.opencontainers.image.description="Ingress NGINX $(NAME) image" \ - --build-arg BASE_IMAGE=$(BASE_IMAGE) \ - --build-arg GOLANG_VERSION=$(GO_VERSION) \ --platform=${PLATFORMS} $(OUTPUT) \ --progress=$(PROGRESS) \ - --pull $(EXTRAARGS) \ - -t $(IMAGE):$(TAG) $(NAME)/rootfs + --pull \ + -t $(IMAGE):$(TAG) rootfs # push the cross built image push: OUTPUT=--push push: build -test: precheck - cd $(NAME)/rootfs && go test ./... - -test-e2e: precheck - cd $(NAME) && ./hack/e2e.sh - # enable buildx ensure-buildx: # this is required for cloudbuild ifeq ("$(wildcard $(INIT_BUILDX))","") - @curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/hack/init-buildx.sh | bash + @curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/hack/init-buildx.sh | bash else @exec $(INIT_BUILDX) endif @echo "done" -.PHONY: build push ensure-buildx test test-e2e precheck +.PHONY: build push ensure-buildx diff --git a/images/fastcgi-helloserver/TAG b/images/fastcgi-helloserver/TAG deleted file mode 100644 index 56130fb3a..000000000 --- a/images/fastcgi-helloserver/TAG +++ /dev/null @@ -1 +0,0 @@ -v1.1.1 diff --git a/images/fastcgi-helloserver/cloudbuild.yaml b/images/fastcgi-helloserver/cloudbuild.yaml index c2c6135df..d80d2aeaf 100644 --- a/images/fastcgi-helloserver/cloudbuild.yaml +++ b/images/fastcgi-helloserver/cloudbuild.yaml @@ -1,11 +1,22 @@ +timeout: 600s options: - # Ignore Prow provided substitutions. substitution_option: ALLOW_LOOSE steps: - - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20250116-2a05ea7e3d - env: - - REGISTRY=us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx + - name: gcr.io/k8s-testimages/gcb-docker-gcloud:v20201130-750d12f entrypoint: bash + env: + - DOCKER_CLI_EXPERIMENTAL=enabled + - TAG=$_GIT_TAG + - BASE_REF=$_PULL_BASE_REF + - REGISTRY=gcr.io/k8s-staging-ingress-nginx + # default cloudbuild has HOME=/builder/home and docker buildx is in /root/.docker/cli-plugins/docker-buildx + # set the home to /root explicitly to if using docker buildx + - HOME=/root args: - - -c - - gcloud auth configure-docker && cd images && make NAME=fastcgi-helloserver push + - -c + - | + gcloud auth configure-docker \ + && make push +substitutions: + _GIT_TAG: "12345" + _PULL_BASE_REF: "master" diff --git a/images/fastcgi-helloserver/rootfs/Dockerfile b/images/fastcgi-helloserver/rootfs/Dockerfile index 7d8e95c47..f572410e7 100755 --- a/images/fastcgi-helloserver/rootfs/Dockerfile +++ b/images/fastcgi-helloserver/rootfs/Dockerfile @@ -11,9 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -ARG GOLANG_VERSION -FROM golang:${GOLANG_VERSION}-alpine3.21 as builder +FROM golang:1.14-alpine as builder WORKDIR /go/src/k8s.io/ingress-nginx/images/fastcgi diff --git a/images/fastcgi-helloserver/rootfs/main.go b/images/fastcgi-helloserver/rootfs/main.go index 710f90f2c..91db60c26 100644 --- a/images/fastcgi-helloserver/rootfs/main.go +++ b/images/fastcgi-helloserver/rootfs/main.go @@ -16,17 +16,15 @@ func hello(w http.ResponseWriter, r *http.Request) { } key := keys[0] - fmt.Fprintf(w, "Hello "+key+"!") + fmt.Fprintf(w, "Hello "+string(key)+"!") } func main() { http.HandleFunc("/hello", hello) - l, err := net.Listen("tcp", "0.0.0.0:9000") //nolint:gosec // Ignore the gosec error since it's a hello server + l, err := net.Listen("tcp", "0.0.0.0:9000") if err != nil { panic(err) } - if err := fcgi.Serve(l, nil); err != nil { - panic(err) - } + fcgi.Serve(l, nil) } diff --git a/images/go-grpc-greeter-server/TAG b/images/go-grpc-greeter-server/TAG deleted file mode 100644 index 56130fb3a..000000000 --- a/images/go-grpc-greeter-server/TAG +++ /dev/null @@ -1 +0,0 @@ -v1.1.1 diff --git a/images/go-grpc-greeter-server/rootfs/Dockerfile b/images/go-grpc-greeter-server/rootfs/Dockerfile deleted file mode 100644 index 39be2990f..000000000 --- a/images/go-grpc-greeter-server/rootfs/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -ARG GOLANG_VERSION - -FROM golang:${GOLANG_VERSION}-alpine3.21 as build - -WORKDIR /go/src/greeter-server - -COPY main.go . -RUN go mod init greeter-server && \ - go mod tidy && \ - go build -o /greeter-server main.go - -FROM gcr.io/distroless/base-debian10 - -COPY --from=build /greeter-server / - -EXPOSE 50051 - -CMD ["/greeter-server"] diff --git a/images/go-grpc-greeter-server/rootfs/main.go b/images/go-grpc-greeter-server/rootfs/main.go deleted file mode 100644 index d7fc36208..000000000 --- a/images/go-grpc-greeter-server/rootfs/main.go +++ /dev/null @@ -1,78 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Binary server is an example server. -package main - -import ( - "context" - "flag" - "fmt" - "log" - "net" - - "google.golang.org/grpc" - "google.golang.org/grpc/reflection" - - ecpb "google.golang.org/grpc/examples/features/proto/echo" - hwpb "google.golang.org/grpc/examples/helloworld/helloworld" -) - -var port = flag.Int("port", 50051, "the port to serve on") - -// hwServer is used to implement helloworld.GreeterServer. -type hwServer struct { - hwpb.UnimplementedGreeterServer -} - -// SayHello implements helloworld.GreeterServer -func (s *hwServer) SayHello(_ context.Context, in *hwpb.HelloRequest) (*hwpb.HelloReply, error) { - return &hwpb.HelloReply{Message: "Hello " + in.Name}, nil -} - -type ecServer struct { - ecpb.UnimplementedEchoServer -} - -func (s *ecServer) UnaryEcho(_ context.Context, req *ecpb.EchoRequest) (*ecpb.EchoResponse, error) { - return &ecpb.EchoResponse{Message: req.Message}, nil -} - -func main() { - flag.Parse() - lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) - if err != nil { - log.Fatalf("failed to listen: %v", err) - } - fmt.Printf("server listening at %v\n", lis.Addr()) - - s := grpc.NewServer() - - // Register Greeter on the server. - hwpb.RegisterGreeterServer(s, &hwServer{}) - - // Register RouteGuide on the same server. - ecpb.RegisterEchoServer(s, &ecServer{}) - - // Register reflection service on gRPC server. - reflection.Register(s) - - if err := s.Serve(lis); err != nil { - log.Fatalf("failed to serve: %v", err) - } -} diff --git a/images/grpc-fortune-teller/.gitignore b/images/grpc-fortune-teller/.gitignore new file mode 100644 index 000000000..ac51a054d --- /dev/null +++ b/images/grpc-fortune-teller/.gitignore @@ -0,0 +1 @@ +bazel-* diff --git a/images/grpc-fortune-teller/BUILD.bazel b/images/grpc-fortune-teller/BUILD.bazel new file mode 100644 index 000000000..b000f8347 --- /dev/null +++ b/images/grpc-fortune-teller/BUILD.bazel @@ -0,0 +1,15 @@ +load("@io_bazel_rules_go//go:def.bzl", "gazelle", "go_prefix") + +gazelle( + name = "gazelle", + args = [ + "-build_file_name", + "BUILD,BUILD.bazel", + ], + command = "fix", + prefix = "github.com/kubernetes/ingress-nginx/images/grpc-fortune-teller", +) + +go_prefix( + prefix = "github.com/kubernetes/ingress-nginx/images/grpc-fortune-teller", +) diff --git a/images/grpc-fortune-teller/Gopkg.lock b/images/grpc-fortune-teller/Gopkg.lock new file mode 100644 index 000000000..e25ff02dd --- /dev/null +++ b/images/grpc-fortune-teller/Gopkg.lock @@ -0,0 +1,104 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/golang/protobuf" + packages = [ + "proto", + "protoc-gen-go/descriptor", + "ptypes", + "ptypes/any", + "ptypes/duration", + "ptypes/timestamp" + ] + revision = "925541529c1fa6821df4e44ce2723319eb2be768" + version = "v1.0.0" + +[[projects]] + name = "github.com/vromero/gofortune" + packages = [ + "lib", + "lib/fortune" + ] + revision = "8a3c485287c0d3d3e4f8f0d9e0058f6cdd29740d" + version = "v0.0.1" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = [ + "context", + "http2", + "http2/hpack", + "idna", + "internal/timeseries", + "lex/httplex", + "trace" + ] + revision = "b3c676e531a6dc479fa1b35ac961c13f5e2b4d2e" + +[[projects]] + name = "golang.org/x/text" + packages = [ + "collate", + "collate/build", + "internal/colltab", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "language", + "secure/bidirule", + "transform", + "unicode/bidi", + "unicode/cldr", + "unicode/norm", + "unicode/rangetable" + ] + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" + +[[projects]] + branch = "master" + name = "google.golang.org/genproto" + packages = ["googleapis/rpc/status"] + revision = "35de2414665fc36f56b72d982c5af480d86de5ab" + +[[projects]] + name = "google.golang.org/grpc" + packages = [ + ".", + "balancer", + "balancer/base", + "balancer/roundrobin", + "codes", + "connectivity", + "credentials", + "encoding", + "encoding/proto", + "grpclb/grpc_lb_v1/messages", + "grpclog", + "internal", + "keepalive", + "metadata", + "naming", + "peer", + "reflection", + "reflection/grpc_reflection_v1alpha", + "resolver", + "resolver/dns", + "resolver/passthrough", + "stats", + "status", + "tap", + "transport" + ] + revision = "d89cded64628466c4ab532d1f0ba5c220459ebe8" + version = "v1.11.2" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "fcd36504ba120cfbbe161eac2ddd439c884fcef61ce318b5c0a544d27bc47043" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/images/grpc-fortune-teller/README.md b/images/grpc-fortune-teller/README.md new file mode 100644 index 000000000..5cc755ef6 --- /dev/null +++ b/images/grpc-fortune-teller/README.md @@ -0,0 +1,59 @@ + +# grpc-fortune-teller + +This is the grpc server application for the nginx-ingress grpc example. Bazel +0.12 is used for the building and container management. + +## Build + +Builds a statically compiled go binary cross-compiled for linux: + +``` +$ bazel build //app:fortune +Target //app:fortune up-to-date: + bazel-bin/app/linux_amd64_static_pure_stripped/fortune +``` + +> To build for your host system, comment out the `goos` and `goarch` attributes +> in the `go_binary` rule. + +## Run + +Builds a minimal docker image that wraps the go_binary, loads it into your local +docker image repository, and runs it: + +```sh +$ bazel run //app:image +Loaded image ID: sha256:aa597c897c873116fcbfccafecf5ab0f6f4178a05e4a00c8e79de91ac0d2e9e7 +Tagging aa597c897c873116fcbfccafecf5ab0f6f4178a05e4a00c8e79de91ac0d2e9e7 as bazel/app:image +2018/05/01 02:13:43 Restored /tmp/fortune-teller/usr/share/games/fortunes/fortunes.dat +2018/05/01 02:13:43 Restored /tmp/fortune-teller/usr/share/games/fortunes/literature +2018/05/01 02:13:43 Restored /tmp/fortune-teller/usr/share/games/fortunes/literature.dat +2018/05/01 02:13:43 Restored /tmp/fortune-teller/usr/share/games/fortunes/riddles +2018/05/01 02:13:43 Restored /tmp/fortune-teller/usr/share/games/fortunes/riddles.dat +2018/05/01 02:13:43 Restored /tmp/fortune-teller/usr/share/games/fortunes/fortunes +2018/05/01 02:13:43 Assets restored to /tmp/fortune-teller +2018/05/01 02:13:43 Listening for gRPC requests at 50051 +``` + +Or run it via docker: + +```sh +$ docker run bazel/app:image +``` + +Build image and push to the container registry specified in the `container_push` +rule: + +```sh +$ bazel run //app:push +``` + +## Invoke + +```sh +$ grpcurl -plaintext localhost:50051 build.stack.fortune.FortuneTeller/Predict +{ + "message": "Whenever the literary German dives into a sentence, that is the last\nyou are going to see of him until he emerges on the other side of his\nAtlantic with his verb in his mouth.\n\t\t-- Mark Twain \"A Connecticut Yankee in King Arthur's Court\"" +} +``` diff --git a/images/grpc-fortune-teller/WORKSPACE b/images/grpc-fortune-teller/WORKSPACE new file mode 100644 index 000000000..a6960d054 --- /dev/null +++ b/images/grpc-fortune-teller/WORKSPACE @@ -0,0 +1,62 @@ +workspace(name = "com_github_kubernetes_ingress_nginx_images_grpc_fortune_teller") + +##################################################################### +# RULES_GO +##################################################################### + +git_repository( + name = "io_bazel_rules_go", + remote = "https://github.com/bazelbuild/rules_go.git", + commit = "161c91485b007c6bf51c0e81808cf4ee2ded299d", +) + +http_archive( + name = "com_github_scele_rules_go_dep", + urls = ["https://github.com/scele/rules_go_dep/archive/49a5e4ca9f6a16c9b4c930a51ce3a537498bb4e1.tar.gz"], + strip_prefix = "rules_go_dep-49a5e4ca9f6a16c9b4c930a51ce3a537498bb4e1", + sha256 = "f170d3d6f55e216f1493f975cde6c489d7070da2a8a41fd4de9812d96f4fb38b", +) + +load("@io_bazel_rules_go//go:def.bzl", "go_rules_dependencies", "go_register_toolchains") +load("@com_github_scele_rules_go_dep//dep:dep.bzl", "dep_import") + +go_register_toolchains(go_version = "1.10.1") + +go_rules_dependencies() + +dep_import( + name = "godeps", + prefix = "github.com/kubernetes/ingress-nginx/images/grpc-fortune-teller", + gopkg_lock = "//:Gopkg.lock", +) + +load("@godeps//:Gopkg.bzl", "go_deps") + +go_deps() + +############################################################# +# RULES_DOCKER +############################################################# + +RULES_DOCKER_VERSION = "553d5506bb7325185950f91533b967da8f5bc536" + +http_archive( + name = "io_bazel_rules_docker", + url = "https://github.com/bazelbuild/rules_docker/archive/%s.zip" % RULES_DOCKER_VERSION, + strip_prefix = "rules_docker-" + RULES_DOCKER_VERSION, + sha256 = "e0b3d966f2a5c0fe921b6294df7c823afa63b4c439f0a7f3b9da3ed6534bab83", +) + +load( + "@io_bazel_rules_docker//container:container.bzl", + container_repositories = "repositories", +) + +container_repositories() + +load( + "@io_bazel_rules_docker//go:image.bzl", + go_image_repositories = "repositories", +) + +go_image_repositories() diff --git a/images/grpc-fortune-teller/app/BUILD.bazel b/images/grpc-fortune-teller/app/BUILD.bazel new file mode 100644 index 000000000..4fffd481e --- /dev/null +++ b/images/grpc-fortune-teller/app/BUILD.bazel @@ -0,0 +1,69 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_embed_data") +load("@io_bazel_rules_docker//go:image.bzl", "go_image") +load("@io_bazel_rules_docker//container:push.bzl", "container_push") + +# Concatenates the fortune databases to a single bundle. +# May need to adjust paths for your system (built on ubuntu 16.04). +# $ apt-get install fortune +genrule( + name = "tar", + outs = ["fortune.tar"], + cmd = " && ".join([ + "OUT=$$(pwd)/$@", + "tar -cvf $$OUT /usr/share/games/fortunes", + ]), +) + +# Generates a .go source file with the tarball content in +# the fortuneFiles variable. +go_embed_data( + name = "fortune_assets", + srcs = [ + ":tar", + ], + package = "main", + unpack = True, + var = "fortuneFiles", +) + +go_library( + name = "go_default_library", + srcs = [ + "main.go", + ":fortune_assets", # keep + ], + importpath = "github.com/kubernetes/ingress-nginx/images/grpc-fortune-teller/app", + visibility = ["//visibility:private"], + deps = [ + "//proto/fortune:go_default_library", + "@com_github_vromero_gofortune//lib/fortune:go_default_library", + "@org_golang_google_grpc//:go_default_library", + "@org_golang_google_grpc//codes:go_default_library", + "@org_golang_google_grpc//reflection:go_default_library", + ], +) + +go_binary( + name = "fortune", + embed = [":go_default_library"], + goarch = "amd64", + goos = "linux", + pure = "on", + static = "on", + visibility = ["//visibility:public"], +) + +go_image( + name = "image", + binary = ":fortune", + visibility = ["//visibility:public"], +) + +container_push( + name = "push", + format = "Docker", + image = ":image", + registry = "quay.io", + repository = "kubernetes-ingress-controller/grpc-fortune-teller", + tag = "0.1", +) diff --git a/images/grpc-fortune-teller/app/main.go b/images/grpc-fortune-teller/app/main.go new file mode 100644 index 000000000..d09799cbe --- /dev/null +++ b/images/grpc-fortune-teller/app/main.go @@ -0,0 +1,137 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + "io/ioutil" + "log" + "net" + "os" + "path" + + proto "github.com/kubernetes/ingress-nginx/images/grpc-fortune-teller/proto/fortune" + "github.com/vromero/gofortune/lib/fortune" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/reflection" +) + +const ( + grpcPort = 50051 +) + +func main() { + + baseDir := "/tmp/fortune-teller" + mustMkdirAll(baseDir) + + opts := []grpc.ServerOption{ + grpc.MaxConcurrentStreams(200), + } + + grpcServer := grpc.NewServer(opts...) + + fortuneTeller := &FortuneTeller{ + fs: createFortuneFilesystemNodeDescriptor(baseDir), + } + proto.RegisterFortuneTellerServer(grpcServer, fortuneTeller) + + reflection.Register(grpcServer) + + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", grpcPort)) + if err != nil { + log.Fatalf("Error while starting grpc server: %v\n", err) + } + + log.Printf("Listening for gRPC requests at %d\n", grpcPort) + grpcServer.Serve(lis) +} + +// FortuneTeller - struct that will implement the grpc service interface. +type FortuneTeller struct { + fs *fortune.FileSystemNodeDescriptor +} + +// Predict - implementation for the grpc unary request method. +func (f *FortuneTeller) Predict(ctx context.Context, r *proto.PredictionRequest) (*proto.PredictionResponse, error) { + _, data, err := fortune.GetRandomFortune(*f.fs) + if err != nil { + return nil, grpc.Errorf(codes.Internal, "Unable to render fortune: %v", err) + } + return &proto.PredictionResponse{ + Message: data, + }, nil +} + +func createFortuneFilesystemNodeDescriptor(baseDir string) *fortune.FileSystemNodeDescriptor { + + // Restore the packed fortune data + fortuneDir := path.Join(baseDir, "usr/share/games/fortunes") + + mustRestore(baseDir, fortuneFiles, nil) + + // init gofortune fs + fs, err := fortune.LoadPaths([]fortune.ProbabilityPath{ + {Path: fortuneDir}, + }) + if err != nil { + log.Fatalf("Unable to load fortune paths: %v", err) + } + + fortune.SetProbabilities(&fs, true) // consider all equal probabilities + return &fs +} + +// mustRestore - Restore assets. +func mustRestore(baseDir string, assets map[string][]byte, mappings map[string]string) { + // unpack variable is provided by the go_embed data and is a + // map[string][]byte such as {"/usr/share/games/fortune/literature.dat": + // bytes... } + for basename, bytes := range assets { + if mappings != nil { + replacement := mappings[basename] + if replacement != "" { + basename = replacement + } + } + filename := path.Join(baseDir, basename) + dirname := path.Dir(filename) + //log.Printf("file %s, dir %s, rel %d, abs %s, absdir: %s", file, dir, rel, abs, absdir) + if err := os.MkdirAll(dirname, os.ModePerm); err != nil { + log.Fatalf("Failed to create asset dir %s: %v", dirname, err) + } + + if err := ioutil.WriteFile(filename, bytes, os.ModePerm); err != nil { + log.Fatalf("Failed to write asset %s: %v", filename, err) + } + log.Printf("Restored %s", filename) + } + + log.Printf("Assets restored to %s", baseDir) +} + +// mustMkdirAll - make all dirs and panic if fail +func mustMkdirAll(dirs ...string) { + for _, dir := range dirs { + err := os.MkdirAll(dir, os.ModePerm) + if err != nil { + panic(fmt.Sprintf("Failed mkdir %s: %v", dir, err)) + } + } +} diff --git a/.gcloudignore b/images/grpc-fortune-teller/go.mod similarity index 100% rename from .gcloudignore rename to images/grpc-fortune-teller/go.mod diff --git a/images/grpc-fortune-teller/proto/fortune/BUILD.bazel b/images/grpc-fortune-teller/proto/fortune/BUILD.bazel new file mode 100644 index 000000000..b3008b826 --- /dev/null +++ b/images/grpc-fortune-teller/proto/fortune/BUILD.bazel @@ -0,0 +1,24 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") + +go_library( + name = "go_default_library", + srcs = ["doc.go"], + embed = [":build_stack_fortune_go_proto"], # keep + importpath = "github.com/kubernetes/ingress-nginx/images/grpc-fortune-teller/proto/fortune", + visibility = ["//visibility:public"], +) + +proto_library( + name = "build_stack_fortune_proto", + srcs = ["fortune.proto"], + visibility = ["//visibility:public"], +) + +go_proto_library( + name = "build_stack_fortune_go_proto", + compilers = ["@io_bazel_rules_go//proto:go_grpc"], + importpath = "github.com/kubernetes/ingress-nginx/images/grpc-fortune-teller/proto/fortune", + proto = ":build_stack_fortune_proto", + visibility = ["//visibility:public"], +) diff --git a/images/grpc-fortune-teller/proto/fortune/fortune.proto b/images/grpc-fortune-teller/proto/fortune/fortune.proto new file mode 100644 index 000000000..d71b4acb1 --- /dev/null +++ b/images/grpc-fortune-teller/proto/fortune/fortune.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package build.stack.fortune; + +message PredictionRequest { +} + +message PredictionResponse { + string message = 1; +} + +service FortuneTeller { + rpc Predict(PredictionRequest) returns (PredictionResponse); +} \ No newline at end of file diff --git a/images/httpbin/Makefile b/images/httpbin/Makefile new file mode 100644 index 000000000..d0c7d40c2 --- /dev/null +++ b/images/httpbin/Makefile @@ -0,0 +1,55 @@ +# Copyright 2020 The Kubernetes Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# set default shell +SHELL=/bin/bash -o pipefail -o errexit + +DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))) +INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh + +TAG ?=v$(shell date +%m%d%Y)-$(shell git rev-parse --short HEAD) +REGISTRY ?= local + +IMAGE = $(REGISTRY)/e2e-test-httpbin + +# required to enable buildx +export DOCKER_CLI_EXPERIMENTAL=enabled + +# build with buildx +PLATFORMS?=linux/amd64,linux/arm,linux/arm64,linux/s390x +OUTPUT= +PROGRESS=plain + +build: ensure-buildx + docker buildx build \ + --platform=${PLATFORMS} $(OUTPUT) \ + --progress=$(PROGRESS) \ + --pull \ + -t $(IMAGE):$(TAG) rootfs + +# push the cross built image +push: OUTPUT=--push +push: build + +# enable buildx +ensure-buildx: +# this is required for cloudbuild +ifeq ("$(wildcard $(INIT_BUILDX))","") + @curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/hack/init-buildx.sh | bash +else + @exec $(INIT_BUILDX) +endif + @echo "done" + +.PHONY: build push ensure-buildx diff --git a/images/httpbin/cloudbuild.yaml b/images/httpbin/cloudbuild.yaml new file mode 100644 index 000000000..02f28b8ca --- /dev/null +++ b/images/httpbin/cloudbuild.yaml @@ -0,0 +1,24 @@ +timeout: 1200s +options: + substitution_option: ALLOW_LOOSE + # job builds a multi-arch docker image for amd64,arm,arm64 and s390x. + machineType: N1_HIGHCPU_8 +steps: + - name: gcr.io/k8s-testimages/gcb-docker-gcloud:v20201130-750d12f + entrypoint: bash + env: + - DOCKER_CLI_EXPERIMENTAL=enabled + - TAG=$_GIT_TAG + - BASE_REF=$_PULL_BASE_REF + - REGISTRY=gcr.io/k8s-staging-ingress-nginx + # default cloudbuild has HOME=/builder/home and docker buildx is in /root/.docker/cli-plugins/docker-buildx + # set the home to /root explicitly to if using docker buildx + - HOME=/root + args: + - -c + - | + gcloud auth configure-docker \ + && make push +substitutions: + _GIT_TAG: "12345" + _PULL_BASE_REF: "master" diff --git a/images/httpbun/rootfs/Dockerfile b/images/httpbin/rootfs/Dockerfile similarity index 59% rename from images/httpbun/rootfs/Dockerfile rename to images/httpbin/rootfs/Dockerfile index ada137ee9..9ef9ccc2b 100644 --- a/images/httpbun/rootfs/Dockerfile +++ b/images/httpbin/rootfs/Dockerfile @@ -12,25 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GOLANG_VERSION - -FROM golang:${GOLANG_VERSION} AS builder +FROM alpine:3.13 ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 -ENV HTTPBUN_COMMIT=a6b387c438b664322734250e426b6966966c2fe4 -RUN git clone https://github.com/sharat87/httpbun.git -WORKDIR httpbun -RUN git checkout $HTTPBUN_COMMIT +RUN apk update \ + && apk add --no-cache \ + python3 python3-dev \ + musl-dev gcc g++ make \ + libffi libffi-dev libstdc++ \ + py3-gevent py3-gunicorn py3-wheel \ + py3-pip \ + && pip3 install httpbin \ + && apk del python3-dev musl-dev gcc g++ make libffi-dev -RUN CGO_ENABLED=0 GOOS=linux go build \ - -installsuffix cgo \ - -trimpath \ - -o bin/httpbun . - -FROM scratch -COPY --from=builder /go/httpbun/bin/httpbun httpbun -ENV HTTPBUN_BIND=0.0.0.0:80 EXPOSE 80 -ENTRYPOINT ["/httpbun"] \ No newline at end of file + +CMD ["gunicorn", "-b", "0.0.0.0:80", "httpbin:app", "-k", "gevent"] diff --git a/images/httpbun/TAG b/images/httpbun/TAG deleted file mode 100644 index 56130fb3a..000000000 --- a/images/httpbun/TAG +++ /dev/null @@ -1 +0,0 @@ -v1.1.1 diff --git a/images/httpbun/cloudbuild.yaml b/images/httpbun/cloudbuild.yaml deleted file mode 100644 index c56820d15..000000000 --- a/images/httpbun/cloudbuild.yaml +++ /dev/null @@ -1,11 +0,0 @@ -options: - # Ignore Prow provided substitutions. - substitution_option: ALLOW_LOOSE -steps: - - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20250116-2a05ea7e3d - env: - - REGISTRY=us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx - entrypoint: bash - args: - - -c - - gcloud auth configure-docker && cd images && make NAME=httpbun push diff --git a/images/kube-webhook-certgen/README.md b/images/kube-webhook-certgen/README.md deleted file mode 100755 index 47a8c74c7..000000000 --- a/images/kube-webhook-certgen/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# Kubernetes webhook certificate generator and patcher - -**This is a copy/fork of the project existing in [jet/kube-webhook-certgen](https://github.com/jet/kube-webhook-certgen/)** - -We moved it here so we can change / update the Kubernetes APIs, and we are really thankful to the original -creators. - -## Overview -Generates a CA and leaf certificate with a long (100y) expiration, then patches [Kubernetes Admission Webhooks](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) -by setting the `caBundle` field with the generated CA. -Can optionally patch the hooks `failurePolicy` setting - useful in cases where a single Helm chart needs to provision resources -and hooks at the same time as patching. - -The utility works in two parts, optimized to work better with the Helm provisioning process that leverages pre-install and post-install hooks to execute this as a Kubernetes job. - -## Security Considerations -This tool may not be adequate in all security environments. If a more complete solution is required, you may want to -seek alternatives such as [jetstack/cert-manager](https://github.com/jetstack/cert-manager) - -## Command line options -``` -Use this to create a ca and signed certificates and patch admission webhooks to allow for quick - installation and configuration of validating and admission webhooks. - -Usage: - kube-webhook-certgen [flags] - kube-webhook-certgen [command] - -Available Commands: - create Generate a ca and server cert+key and store the results in a secret 'secret-name' in 'namespace' - help Help about any command - patch Patch a validatingwebhookconfiguration and mutatingwebhookconfiguration 'webhook-name' by using the ca from 'secret-name' in 'namespace' - version Prints the CLI version information - -Flags: - -h, --help help for kube-webhook-certgen - --kubeconfig string Path to kubeconfig file: e.g. ~/.kube/kind-config-kind - --log-format string Log format: text|json (default "text") - --log-level string Log level: panic|fatal|error|warn|info|debug|trace (default "info") -``` - -### Create -``` -Generate a ca and server cert+key and store the results in a secret 'secret-name' in 'namespace' - -Usage: - kube-webhook-certgen create [flags] - -Flags: - --cert-name string Name of cert file in the secret (default "cert") - -h, --help help for create - --host string Comma-separated hostnames and IPs to generate a certificate for - --key-name string Name of key file in the secret (default "key") - --namespace string Namespace of the secret where certificate information will be written - --secret-name string Name of the secret where certificate information will be written - -Global Flags: - --kubeconfig string Path to kubeconfig file: e.g. ~/.kube/kind-config-kind - --log-format string Log format: text|json (default "json") - --log-level string Log level: panic|fatal|error|warn|info|debug|trace (default "info") -``` - -### Patch -``` -Patch a validatingwebhookconfiguration and mutatingwebhookconfiguration 'webhook-name' by using the ca from 'secret-name' in 'namespace' - -Usage: - kube-webhook-certgen patch [flags] - -Flags: - -h, --help help for patch - --namespace string Namespace of the secret where certificate information will be read from - --patch-failure-policy string If set, patch the webhooks with this failure policy. Valid options are Ignore or Fail - --patch-mutating If true, patch mutatingwebhookconfiguration (default true) - --patch-validating If true, patch validatingwebhookconfiguration (default true) - --secret-name string Name of the secret where certificate information will be read from - --webhook-name string Name of validatingwebhookconfiguration and mutatingwebhookconfiguration that will be updated - -Global Flags: - --kubeconfig string Path to kubeconfig file: e.g. ~/.kube/kind-config-kind - --log-format string Log format: text|json (default "text") - --log-level string Log level: panic|fatal|error|warn|info|debug|trace (default "info") -``` - -## Known Users -- [stable/prometheus-operator](https://github.com/helm/charts/tree/master/stable/prometheus-operator) helm chart diff --git a/images/kube-webhook-certgen/TAG b/images/kube-webhook-certgen/TAG deleted file mode 100644 index 53b5bbb12..000000000 --- a/images/kube-webhook-certgen/TAG +++ /dev/null @@ -1 +0,0 @@ -v1.5.1 diff --git a/images/kube-webhook-certgen/cloudbuild.yaml b/images/kube-webhook-certgen/cloudbuild.yaml deleted file mode 100644 index e4118ff88..000000000 --- a/images/kube-webhook-certgen/cloudbuild.yaml +++ /dev/null @@ -1,11 +0,0 @@ -options: - # Ignore Prow provided substitutions. - substitution_option: ALLOW_LOOSE -steps: - - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20250116-2a05ea7e3d - env: - - REGISTRY=us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx - entrypoint: bash - args: - - -c - - gcloud auth configure-docker && cd images && make NAME=kube-webhook-certgen push diff --git a/images/kube-webhook-certgen/hack/e2e.sh b/images/kube-webhook-certgen/hack/e2e.sh deleted file mode 100755 index 7ebf1fb1e..000000000 --- a/images/kube-webhook-certgen/hack/e2e.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -# Copyright 2021 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ -n "$DEBUG" ]; then - set -x -fi - -set -o errexit -set -o nounset -set -o pipefail - -pushd rootfs -go build -popd - -export KUBECONFIG=${KUBECONFIG:-~/.kube/config} - -function error { - trap - EXIT - echo $@ - exit 1 -} - -function cleanup { - kubectl delete --recursive -f hack/e2e.yaml --ignore-not-found -} - -trap cleanup EXIT - -cleanup - -kubectl apply --recursive -f hack/e2e.yaml - -./rootfs/kube-webhook-certgen create --kubeconfig ${KUBECONFIG} --namespace test --secret-name test --host test --cert-name test -./rootfs/kube-webhook-certgen patch --kubeconfig ${KUBECONFIG} --namespace test --secret-name test --webhook-name pod-policy.example.com -./rootfs/kube-webhook-certgen patch --kubeconfig ${KUBECONFIG} --namespace test --secret-name test --apiservice-name v1alpha1.pod-policy.example.com -./rootfs/kube-webhook-certgen patch --kubeconfig ${KUBECONFIG} --namespace test --secret-name test --webhook-name pod-policy.example.com --apiservice-name v1alpha1.pod-policy.example.com - -if [[ -z "$(kubectl get validatingwebhookconfiguration pod-policy.example.com -o jsonpath='{.webhooks[0].clientConfig.caBundle}')" ]]; then - error "ValidatingWebhookConfiguration pod-policy.example.com did not get CA injected" -fi - -if [[ -z "$(kubectl get mutatingwebhookconfiguration pod-policy.example.com -o jsonpath='{.webhooks[0].clientConfig.caBundle}')" ]]; then - error "MutatingWebhookConfiguration pod-policy.example.com did not get CA injected" -fi - -if [[ -z "$(kubectl get apiservice v1alpha1.pod-policy.example.com -o jsonpath='{.spec.caBundle}')" ]];then - error "APIService v1alpha1.pod-policy.example.com did not get CA injected" -fi diff --git a/images/kube-webhook-certgen/hack/e2e.yaml b/images/kube-webhook-certgen/hack/e2e.yaml deleted file mode 100644 index f6f116f5e..000000000 --- a/images/kube-webhook-certgen/hack/e2e.yaml +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: test ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: "pod-policy.example.com" -webhooks: -- name: "pod-policy.example.com" - rules: - - apiGroups: [""] - apiVersions: ["v1"] - operations: ["CREATE"] - resources: ["pods"] - scope: "Namespaced" - clientConfig: - service: - namespace: "example-namespace" - name: "example-service" - admissionReviewVersions: ["v1", "v1beta1"] - sideEffects: None - timeoutSeconds: 5 ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: "pod-policy.example.com" -webhooks: -- name: "pod-policy.example.com" - rules: - - apiGroups: [""] - apiVersions: ["v1"] - operations: ["CREATE"] - resources: ["pods"] - scope: "Namespaced" - clientConfig: - service: - namespace: "example-namespace" - name: "example-service" - admissionReviewVersions: ["v1", "v1beta1"] - sideEffects: None - timeoutSeconds: 5 ---- -apiVersion: apiregistration.k8s.io/v1 -kind: APIService -metadata: - name: v1alpha1.pod-policy.example.com -spec: - service: - name: "example-namespace" - namespace: "example-service" - group: pod-policy.example.com - version: v1alpha1 - groupPriorityMinimum: 100 - versionPriority: 100 diff --git a/images/kube-webhook-certgen/rootfs/.gitignore b/images/kube-webhook-certgen/rootfs/.gitignore deleted file mode 100644 index f82bfe10c..000000000 --- a/images/kube-webhook-certgen/rootfs/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/kube-webhook-certgen diff --git a/images/kube-webhook-certgen/rootfs/Dockerfile b/images/kube-webhook-certgen/rootfs/Dockerfile deleted file mode 100644 index 2348db825..000000000 --- a/images/kube-webhook-certgen/rootfs/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2021 The Kubernetes Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG GOLANG_VERSION - -FROM --platform=$BUILDPLATFORM golang:${GOLANG_VERSION} as builder - -ARG BUILDPLATFORM -ARG TARGETARCH - - -WORKDIR /workspace -COPY . . -RUN go mod tidy && CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -a -o kube-webhook-certgen main.go - -FROM --platform=$BUILDPLATFORM gcr.io/distroless/static:nonroot -ARG BUILDPLATFORM -ARG TARGETARCH - -LABEL org.opencontainers.image.source=https://github.com/kubernetes/ingress-nginx - -WORKDIR / -COPY --from=builder /workspace/kube-webhook-certgen /kube-webhook-certgen -USER 65532:65532 -ENTRYPOINT ["/kube-webhook-certgen"] diff --git a/images/kube-webhook-certgen/rootfs/LICENSE b/images/kube-webhook-certgen/rootfs/LICENSE deleted file mode 100644 index 7cd4c37e3..000000000 --- a/images/kube-webhook-certgen/rootfs/LICENSE +++ /dev/null @@ -1,203 +0,0 @@ -Copyright (c) 2019-present, Jet.com, Inc. ----------------------------------------------------- - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/images/kube-webhook-certgen/rootfs/README.md b/images/kube-webhook-certgen/rootfs/README.md deleted file mode 100644 index 9f5781660..000000000 --- a/images/kube-webhook-certgen/rootfs/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# Kubernetes webhook certificate generator and patcher - -**This is a copy/fork of the project existing in [jet/kube-webhook-certgen](https://github.com/jet/kube-webhook-certgen/)** - -We moved it here so we can change / update the Kubernetes APIs, and we are really thankful to the original -creators. - -## Overview -Generates a CA and leaf certificate with a long (100y) expiration, then patches [Kubernetes Admission Webhooks](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) -by setting the `caBundle` field with the generated CA. -Can optionally patch the hooks `failurePolicy` setting - useful in cases where a single Helm chart needs to provision resources -and hooks at the same time as patching. - -The utility works in two parts, optimized to work better with the Helm provisioning process that leverages pre-install and post-install hooks to execute this as a Kubernetes job. - -## Security Considerations -This tool may not be adequate in all security environments. If a more complete solution is required, you may want to -seek alternatives such as [jetstack/cert-manager](https://github.com/jetstack/cert-manager) - -## Command line options -``` -Use this to create a ca and signed certificates and patch admission webhooks to allow for quick - installation and configuration of validating and admission webhooks. - -Usage: - kube-webhook-certgen [flags] - kube-webhook-certgen [command] - -Available Commands: - create Generate a ca and server cert+key and store the results in a secret 'secret-name' in 'namespace' - help Help about any command - patch Patch a validatingwebhookconfiguration and mutatingwebhookconfiguration 'webhook-name' by using the ca from 'secret-name' in 'namespace' - version Prints the CLI version information - -Flags: - -h, --help help for kube-webhook-certgen - --kubeconfig string Path to kubeconfig file: e.g. ~/.kube/kind-config-kind - --log-format string Log format: text|json (default "text") - --log-level string Log level: panic|fatal|error|warn|info|debug|trace (default "info") -``` - -### Create -``` -Generate a ca and server cert+key and store the results in a secret 'secret-name' in 'namespace' - -Usage: - kube-webhook-certgen create [flags] - -Flags: - --cert-name string Name of cert file in the secret (default "cert") - -h, --help help for create - --host string Comma-separated hostnames and IPs to generate a certificate for - --key-name string Name of key file in the secret (default "key") - --namespace string Namespace of the secret where certificate information will be written - --secret-name string Name of the secret where certificate information will be written - -Global Flags: - --kubeconfig string Path to kubeconfig file: e.g. ~/.kube/kind-config-kind - --log-format string Log format: text|json (default "json") - --log-level string Log level: panic|fatal|error|warn|info|debug|trace (default "info") -``` - -### Patch -``` -Patch a validatingwebhookconfiguration and mutatingwebhookconfiguration 'webhook-name' by using the ca from 'secret-name' in 'namespace' - -Usage: - kube-webhook-certgen patch [flags] - -Flags: - -h, --help help for patch - --namespace string Namespace of the secret where certificate information will be read from - --patch-failure-policy string If set, patch the webhooks with this failure policy. Valid options are Ignore or Fail - --patch-mutating If true, patch mutatingwebhookconfiguration (default true) - --patch-validating If true, patch validatingwebhookconfiguration (default true) - --secret-name string Name of the secret where certificate information will be read from - --webhook-name string Name of validatingwebhookconfiguration and mutatingwebhookconfiguration that will be updated - -Global Flags: - --kubeconfig string Path to kubeconfig file: e.g. ~/.kube/kind-config-kind - --log-format string Log format: text|json (default "text") - --log-level string Log level: panic|fatal|error|warn|info|debug|trace (default "info") -``` - -## Known Users -- [stable/prometheus-operator](https://github.com/helm/charts/tree/master/stable/prometheus-operator) helm chart diff --git a/images/kube-webhook-certgen/rootfs/cmd/create.go b/images/kube-webhook-certgen/rootfs/cmd/create.go deleted file mode 100644 index e61d2c4a1..000000000 --- a/images/kube-webhook-certgen/rootfs/cmd/create.go +++ /dev/null @@ -1,47 +0,0 @@ -package cmd - -import ( - "context" - - "github.com/jet/kube-webhook-certgen/pkg/certs" - "github.com/jet/kube-webhook-certgen/pkg/k8s" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" -) - -var create = &cobra.Command{ - Use: "create", - Short: "Generate a ca and server cert+key and store the results in a secret 'secret-name' in 'namespace'", - Long: "Generate a ca and server cert+key and store the results in a secret 'secret-name' in 'namespace'", - PreRun: configureLogging, - Run: createCommand, -} - -func createCommand(cmd *cobra.Command, args []string) { - clientset, aggregatorClientset := newKubernetesClients(cfg.kubeconfig) - k := k8s.New(clientset, aggregatorClientset) - - ctx := context.TODO() - - ca := k.GetCaFromSecret(ctx, cfg.secretName, cfg.namespace) - if ca == nil { - log.Info("creating new secret") - newCa, newCert, newKey := certs.GenerateCerts(cfg.host) - ca = newCa - k.SaveCertsToSecret(ctx, cfg.secretName, cfg.namespace, cfg.certName, cfg.keyName, ca, newCert, newKey) - } else { - log.Info("secret already exists") - } -} - -func init() { - rootCmd.AddCommand(create) - create.Flags().StringVar(&cfg.host, "host", "", "Comma-separated hostnames and IPs to generate a certificate for") - create.Flags().StringVar(&cfg.secretName, "secret-name", "", "Name of the secret where certificate information will be written") - create.Flags().StringVar(&cfg.namespace, "namespace", "", "Namespace of the secret where certificate information will be written") - create.Flags().StringVar(&cfg.certName, "cert-name", "cert", "Name of cert file in the secret") - create.Flags().StringVar(&cfg.keyName, "key-name", "key", "Name of key file in the secret") - create.MarkFlagRequired("host") - create.MarkFlagRequired("secret-name") - create.MarkFlagRequired("namespace") -} diff --git a/images/kube-webhook-certgen/rootfs/cmd/patch.go b/images/kube-webhook-certgen/rootfs/cmd/patch.go deleted file mode 100644 index dd41103ca..000000000 --- a/images/kube-webhook-certgen/rootfs/cmd/patch.go +++ /dev/null @@ -1,121 +0,0 @@ -package cmd - -import ( - "context" - "errors" - "fmt" - - "github.com/jet/kube-webhook-certgen/pkg/k8s" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - admissionv1 "k8s.io/api/admissionregistration/v1" -) - -var patch = &cobra.Command{ - Use: "patch", - Short: "Patch a ValidatingWebhookConfiguration, MutatingWebhookConfiguration or APIService 'object-name' by using the ca from 'secret-name' in 'namespace'", - Long: "Patch a ValidatingWebhookConfiguration, MutatingWebhookConfiguration or APIService 'object-name' by using the ca from 'secret-name' in 'namespace'", - PreRun: configureLogging, - Run: patchCommand, -} - -type PatchConfig struct { - PatchMutating bool - PatchValidating bool - PatchFailurePolicy string - APIServiceName string - WebhookName string - - SecretName string - Namespace string - - Patcher Patcher -} - -type Patcher interface { - PatchObjects(ctx context.Context, options k8s.PatchOptions) error - GetCaFromSecret(ctx context.Context, secretName, namespace string) []byte -} - -func Patch(ctx context.Context, cfg *PatchConfig) error { - if cfg.Patcher == nil { - return fmt.Errorf("no patcher defined") - } - - if !cfg.PatchMutating && !cfg.PatchValidating && cfg.APIServiceName == "" { - return fmt.Errorf("patch-validating=false, patch-mutating=false. You must patch at least one kind of webhook, otherwise this command is a no-op") - } - - var failurePolicy admissionv1.FailurePolicyType - - switch cfg.PatchFailurePolicy { - case "": - break - case "Ignore": - case "Fail": - failurePolicy = admissionv1.FailurePolicyType(cfg.PatchFailurePolicy) - break - default: - return fmt.Errorf("patch-failure-policy %s is not valid", cfg.PatchFailurePolicy) - } - - ca := cfg.Patcher.GetCaFromSecret(ctx, cfg.SecretName, cfg.Namespace) - - if ca == nil { - return fmt.Errorf("no secret with '%s' in '%s'", cfg.SecretName, cfg.Namespace) - } - - options := k8s.PatchOptions{ - CABundle: ca, - FailurePolicyType: failurePolicy, - APIServiceName: cfg.APIServiceName, - } - - if cfg.PatchMutating { - options.MutatingWebhookConfigurationName = cfg.WebhookName - } - - if cfg.PatchValidating { - options.ValidatingWebhookConfigurationName = cfg.WebhookName - } - - return cfg.Patcher.PatchObjects(ctx, options) -} - -func patchCommand(_ *cobra.Command, _ []string) { - client, aggregationClient := newKubernetesClients(cfg.kubeconfig) - - config := &PatchConfig{ - SecretName: cfg.secretName, - Namespace: cfg.namespace, - PatchMutating: cfg.patchMutating, - PatchValidating: cfg.patchValidating, - PatchFailurePolicy: cfg.patchFailurePolicy, - APIServiceName: cfg.apiServiceName, - WebhookName: cfg.webhookName, - Patcher: k8s.New(client, aggregationClient), - } - - ctx := context.TODO() - - if err := Patch(ctx, config); err != nil { - if wrappedErr := errors.Unwrap(err); wrappedErr != nil { - log.WithField("err", wrappedErr).Fatal(err.Error()) - } - - log.Fatal(err.Error()) - } -} - -func init() { - rootCmd.AddCommand(patch) - patch.Flags().StringVar(&cfg.secretName, "secret-name", "", "Name of the secret where certificate information will be read from") - patch.Flags().StringVar(&cfg.namespace, "namespace", "", "Namespace of the secret where certificate information will be read from") - patch.Flags().StringVar(&cfg.webhookName, "webhook-name", "", "Name of ValidatingWebhookConfiguration and MutatingWebhookConfiguration that will be updated") - patch.Flags().StringVar(&cfg.apiServiceName, "apiservice-name", "", "Name of APIService that will be patched") - patch.Flags().BoolVar(&cfg.patchValidating, "patch-validating", true, "If true, patch ValidatingWebhookConfiguration") - patch.Flags().BoolVar(&cfg.patchMutating, "patch-mutating", true, "If true, patch MutatingWebhookConfiguration") - patch.Flags().StringVar(&cfg.patchFailurePolicy, "patch-failure-policy", "", "If set, patch the webhooks with this failure policy. Valid options are Ignore or Fail") - patch.MarkFlagRequired("secret-name") - patch.MarkFlagRequired("namespace") -} diff --git a/images/kube-webhook-certgen/rootfs/cmd/patch_test.go b/images/kube-webhook-certgen/rootfs/cmd/patch_test.go deleted file mode 100644 index 2bec3b871..000000000 --- a/images/kube-webhook-certgen/rootfs/cmd/patch_test.go +++ /dev/null @@ -1,254 +0,0 @@ -package cmd_test - -import ( - "context" - "fmt" - "reflect" - "testing" - - "github.com/jet/kube-webhook-certgen/cmd" - "github.com/jet/kube-webhook-certgen/pkg/k8s" -) - -func Test_Patch(t *testing.T) { - t.Parallel() - - ctx := context.TODO() - - t.Run("patches_APIService_object_when_requested", func(t *testing.T) { - t.Parallel() - - config := testPatchConfig() - config.APIServiceName = "bar" - - patcher := testPatcher() - patcher.patchObjects = func(_ context.Context, options k8s.PatchOptions) error { - if options.APIServiceName != config.APIServiceName { - return fmt.Errorf("unexpected APIService name %q, expected %q", options.APIServiceName, config.APIServiceName) - } - - return nil - } - config.Patcher = patcher - - if err := cmd.Patch(ctx, config); err != nil { - t.Fatalf("Unexpected patching error: %v", err) - } - }) - - t.Run("use_configured_webhook_name_for_patching", func(t *testing.T) { - t.Parallel() - - config := testPatchConfig() - config.WebhookName = "foo" - - patcher := testPatcher() - patcher.patchObjects = func(_ context.Context, options k8s.PatchOptions) error { - if options.ValidatingWebhookConfigurationName != config.WebhookName { - return fmt.Errorf("unexpected object name %q, expected %q", options.ValidatingWebhookConfigurationName, config.WebhookName) - } - - return nil - } - config.Patcher = patcher - - if err := cmd.Patch(ctx, config); err != nil { - t.Fatalf("Unexpected patching error: %v", err) - } - }) - - t.Run("patches_only_validating_webhook_when_requested", func(t *testing.T) { - t.Parallel() - - config := testPatchConfig() - config.PatchValidating = true - config.PatchMutating = false - - patcher := testPatcher() - patcher.patchObjects = func(_ context.Context, options k8s.PatchOptions) error { - if options.ValidatingWebhookConfigurationName == "" { - t.Error("expected validating webhook to be patched") - } - - if options.MutatingWebhookConfigurationName != "" { - t.Error("expected mutating webhook to not be patched") - } - - if options.APIServiceName != "" { - t.Error("expected APIService to not be patched") - } - - return nil - } - config.Patcher = patcher - - if err := cmd.Patch(ctx, config); err != nil { - t.Fatalf("Unexpected patching error: %v", err) - } - }) - - t.Run("patches_both_webhooks_when_requested", func(t *testing.T) { - t.Parallel() - - config := testPatchConfig() - config.PatchValidating = true - config.PatchMutating = true - - patcher := testPatcher() - patcher.patchObjects = func(_ context.Context, options k8s.PatchOptions) error { - if options.ValidatingWebhookConfigurationName == "" { - t.Error("expected validating webhook to be patched") - } - - if options.MutatingWebhookConfigurationName == "" { - t.Error("expected mutating webhook to be patched") - } - - if options.APIServiceName != "" { - t.Error("expected APIService to not be patched") - } - - return nil - } - config.Patcher = patcher - - if err := cmd.Patch(ctx, config); err != nil { - t.Fatalf("Unexpected patching error: %v", err) - } - }) - - t.Run("use_empty_policy_when_ignore_is_requested", func(t *testing.T) { - t.Parallel() - - config := testPatchConfig() - config.PatchFailurePolicy = "Ignore" - - patcher := testPatcher() - patcher.patchObjects = func(_ context.Context, options k8s.PatchOptions) error { - if options.FailurePolicyType != "" { - return fmt.Errorf("expected policy to be nil. got: %q", options.FailurePolicyType) - } - - return nil - } - config.Patcher = patcher - - if err := cmd.Patch(ctx, config); err != nil { - t.Fatalf("Unexpected patching error: %v", err) - } - }) - - t.Run("use_fail_policy_when_fail_is_requested", func(t *testing.T) { - t.Parallel() - - config := testPatchConfig() - config.PatchFailurePolicy = "Fail" - - patcher := testPatcher() - patcher.patchObjects = func(_ context.Context, options k8s.PatchOptions) error { - if options.FailurePolicyType == "" || options.FailurePolicyType != "Fail" { - return fmt.Errorf("unexpected policy: %q", options.FailurePolicyType) - } - - return nil - } - config.Patcher = patcher - - if err := cmd.Patch(ctx, config); err != nil { - t.Fatalf("Unexpected patching error: %v", err) - } - }) - - t.Run("use_obtained_ca_certificate_for_patching", func(t *testing.T) { - t.Parallel() - - expectedCA := []byte("foo") - - config := testPatchConfig() - - patcher := testPatcher() - patcher.patchObjects = func(_ context.Context, options k8s.PatchOptions) error { - if !reflect.DeepEqual(options.CABundle, expectedCA) { - return fmt.Errorf("unexpected CA, expected %q, got %q", string(expectedCA), string(options.CABundle)) - } - - return nil - } - patcher.getCaFromSecret = func(context.Context, string, string) []byte { - return expectedCA - } - config.Patcher = patcher - - if err := cmd.Patch(ctx, config); err != nil { - t.Fatalf("Unexpected patching error: %v", err) - } - }) - - t.Run("returns_error_when", func(t *testing.T) { - t.Parallel() - - for name, mutateF := range map[string]func(*cmd.PatchConfig){ - "no_patcher_is_defined": func(c *cmd.PatchConfig) { - c.Patcher = nil - }, - "no_webhooks_are_requested_for_patching": func(c *cmd.PatchConfig) { - c.PatchValidating = false - c.PatchMutating = false - c.APIServiceName = "" - }, - "unsupported_patch_failure_policy_is_defined": func(c *cmd.PatchConfig) { - c.PatchFailurePolicy = "foo" - }, - "ca_certificate_from_secret_is_empty": func(c *cmd.PatchConfig) { - patcher := testPatcher() - patcher.getCaFromSecret = func(_ context.Context, _, _ string) []byte { - return nil - } - c.Patcher = patcher - }, - } { - mutateF := mutateF - - t.Run(name, func(t *testing.T) { - t.Parallel() - - config := testPatchConfig() - mutateF(config) - - if err := cmd.Patch(ctx, config); err == nil { - t.Fatalf("Expected error while patching") - } - }) - } - }) -} - -type patcher struct { - patchObjects func(context.Context, k8s.PatchOptions) error - getCaFromSecret func(context.Context, string, string) []byte -} - -func (p *patcher) PatchObjects(ctx context.Context, options k8s.PatchOptions) error { - return p.patchObjects(ctx, options) -} - -func (p *patcher) GetCaFromSecret(ctx context.Context, secretName, namespace string) []byte { - return p.getCaFromSecret(ctx, secretName, namespace) -} - -func testPatcher() *patcher { - return &patcher{ - patchObjects: func(context.Context, k8s.PatchOptions) error { - return nil - }, - getCaFromSecret: func(context.Context, string, string) []byte { return []byte{} }, - } -} - -func testPatchConfig() *cmd.PatchConfig { - return &cmd.PatchConfig{ - PatchValidating: true, - WebhookName: "foo", - Patcher: testPatcher(), - } -} diff --git a/images/kube-webhook-certgen/rootfs/cmd/root.go b/images/kube-webhook-certgen/rootfs/cmd/root.go deleted file mode 100644 index 7db870d15..000000000 --- a/images/kube-webhook-certgen/rootfs/cmd/root.go +++ /dev/null @@ -1,103 +0,0 @@ -package cmd - -import ( - "os" - - "github.com/onrik/logrus/filename" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" -) - -var ( - rootCmd = &cobra.Command{ - Use: "kube-webhook-certgen", - Short: "Create certificates and patch them to admission hooks", - Long: `Use this to create a ca and signed certificates and patch admission webhooks to allow for quick - installation and configuration of validating and admission webhooks.`, - PreRun: configureLogging, - Run: rootCommand, - } - - cfg = struct { - logLevel string - logfmt string - secretName string - namespace string - certName string - keyName string - host string - apiServiceName string - webhookName string - patchValidating bool - patchMutating bool - patchFailurePolicy string - kubeconfig string - }{} -) - -// Execute is the main entry point for the program -func Execute() { - if err := rootCmd.Execute(); err != nil { - os.Exit(1) - } -} - -func init() { - filenameHook := filename.NewHook() - filenameHook.Field = "source" - log.AddHook(filenameHook) - log.SetOutput(os.Stdout) - log.SetLevel(log.TraceLevel) - rootCmd.Flags() - rootCmd.PersistentFlags().StringVar(&cfg.logLevel, "log-level", "info", "Log level: panic|fatal|error|warn|info|debug|trace") - rootCmd.PersistentFlags().StringVar(&cfg.logfmt, "log-format", "json", "Log format: text|json") - rootCmd.PersistentFlags().StringVar(&cfg.kubeconfig, "kubeconfig", "", "Path to kubeconfig file: e.g. ~/.kube/kind-config-kind") -} - -func configureLogging(_ *cobra.Command, _ []string) { - l, err := log.ParseLevel(cfg.logLevel) - if err != nil { - log.WithField("err", err).Fatal("Invalid error level") - } - log.SetLevel(l) - log.SetFormatter(getFormatter(cfg.logfmt)) -} - -func rootCommand(cmd *cobra.Command, _ []string) { - cmd.Help() - os.Exit(1) -} - -func getFormatter(logfmt string) log.Formatter { - switch logfmt { - case "json": - return &log.JSONFormatter{} - case "text": - return &log.TextFormatter{} - } - - log.Fatalf("invalid log format '%s'", logfmt) - return nil -} - -func newKubernetesClients(kubeconfig string) (kubernetes.Interface, clientset.Interface) { - config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) - if err != nil { - log.WithError(err).Fatal("error building kubernetes config") - } - - c, err := kubernetes.NewForConfig(config) - if err != nil { - log.WithError(err).Fatal("error creating kubernetes client") - } - - aggregatorClientset, err := clientset.NewForConfig(config) - if err != nil { - log.WithError(err).Fatal("error creating kubernetes aggregator client") - } - - return c, aggregatorClientset -} diff --git a/images/kube-webhook-certgen/rootfs/cmd/version.go b/images/kube-webhook-certgen/rootfs/cmd/version.go deleted file mode 100644 index 8ec051acc..000000000 --- a/images/kube-webhook-certgen/rootfs/cmd/version.go +++ /dev/null @@ -1,25 +0,0 @@ -package cmd - -import ( - "fmt" - "runtime" - - "github.com/jet/kube-webhook-certgen/core" - "github.com/spf13/cobra" -) - -var version = &cobra.Command{ - Use: "version", - Short: "Prints the CLI version information", - Run: versionCmdRun, -} - -func versionCmdRun(cmd *cobra.Command, args []string) { - fmt.Printf("%s\n", core.Version) - fmt.Printf("build %s\n", core.BuildTime) - fmt.Printf("%s\n", runtime.Version()) -} - -func init() { - rootCmd.AddCommand(version) -} diff --git a/images/kube-webhook-certgen/rootfs/core/version.go b/images/kube-webhook-certgen/rootfs/core/version.go deleted file mode 100644 index d20623567..000000000 --- a/images/kube-webhook-certgen/rootfs/core/version.go +++ /dev/null @@ -1,8 +0,0 @@ -package core - -var ( - // Version is the current Semantic Version - Version = "0.0.1-dev" - // BuildTime is the timestamp (ISO-8601) of the build - BuildTime = "0001-01-01T00:00:00Z" -) diff --git a/images/kube-webhook-certgen/rootfs/go.mod b/images/kube-webhook-certgen/rootfs/go.mod deleted file mode 100644 index 2b8567748..000000000 --- a/images/kube-webhook-certgen/rootfs/go.mod +++ /dev/null @@ -1,57 +0,0 @@ -module github.com/jet/kube-webhook-certgen - -go 1.23.6 - -require ( - github.com/onrik/logrus v0.11.0 - github.com/sirupsen/logrus v1.9.3 - github.com/spf13/cobra v1.9.1 - k8s.io/api v0.32.2 - k8s.io/apimachinery v0.32.2 - k8s.io/client-go v0.32.2 - k8s.io/kube-aggregator v0.32.2 -) - -require ( - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/emicklei/go-restful/v3 v3.11.3 // indirect - github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.20.4 // indirect - github.com/go-openapi/swag v0.23.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/ginkgo/v2 v2.22.2 // indirect - github.com/onsi/gomega v1.36.2 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/spf13/pflag v1.0.6 // indirect - github.com/x448/float16 v0.8.4 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/oauth2 v0.23.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/term v0.27.0 // indirect - golang.org/x/text v0.21.0 // indirect - golang.org/x/time v0.7.0 // indirect - google.golang.org/protobuf v1.36.1 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect - k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect - sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect -) diff --git a/images/kube-webhook-certgen/rootfs/go.sum b/images/kube-webhook-certgen/rootfs/go.sum deleted file mode 100644 index 1c660f020..000000000 --- a/images/kube-webhook-certgen/rootfs/go.sum +++ /dev/null @@ -1,163 +0,0 @@ -github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/go-restful/v3 v3.11.3 h1:yagOQz/38xJmcNeZJtrUcKjkHRltIaIFXKWeG1SkWGE= -github.com/emicklei/go-restful/v3 v3.11.3/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= -github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= -github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= -github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= -github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= -github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onrik/logrus v0.11.0 h1:pu+BCaWL36t0yQaj/2UHK2erf88dwssAKOT51mxPUVs= -github.com/onrik/logrus v0.11.0/go.mod h1:fO2vlZwIdti6PidD3gV5YKt9Lq5ptpnP293RAe1ITwk= -github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= -github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= -github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= -github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= -github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= -github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= -golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= -golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= -google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.32.2 h1:bZrMLEkgizC24G9eViHGOPbW+aRo9duEISRIJKfdJuw= -k8s.io/api v0.32.2/go.mod h1:hKlhk4x1sJyYnHENsrdCWw31FEmCijNGPJO5WzHiJ6Y= -k8s.io/apimachinery v0.32.2 h1:yoQBR9ZGkA6Rgmhbp/yuT9/g+4lxtsGYwW6dR6BDPLQ= -k8s.io/apimachinery v0.32.2/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= -k8s.io/client-go v0.32.2 h1:4dYCD4Nz+9RApM2b/3BtVvBHw54QjMFUl1OLcJG5yOA= -k8s.io/client-go v0.32.2/go.mod h1:fpZ4oJXclZ3r2nDOv+Ux3XcJutfrwjKTCHz2H3sww94= -k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= -k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-aggregator v0.32.2 h1:kg9pke+i2qRbJwX1UKwZV4fsCRvmbaCEFk38R4FqHmw= -k8s.io/kube-aggregator v0.32.2/go.mod h1:rRm+xY1yIFIt3zBc727nG5SBLYywywD87klfIAw+7+c= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/images/kube-webhook-certgen/rootfs/main.go b/images/kube-webhook-certgen/rootfs/main.go deleted file mode 100644 index e6594e777..000000000 --- a/images/kube-webhook-certgen/rootfs/main.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -import ( - "github.com/jet/kube-webhook-certgen/cmd" -) - -func main() { - cmd.Execute() -} diff --git a/images/kube-webhook-certgen/rootfs/pkg/certs/certs.go b/images/kube-webhook-certgen/rootfs/pkg/certs/certs.go deleted file mode 100644 index ea6f4b27c..000000000 --- a/images/kube-webhook-certgen/rootfs/pkg/certs/certs.go +++ /dev/null @@ -1,100 +0,0 @@ -package certs - -import ( - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "math/big" - "net" - "strings" - "time" - - log "github.com/sirupsen/logrus" -) - -// GenerateCerts venerates a ca with a leaf certificate and key and returns the ca, cert and key as PEM encoded slices -func GenerateCerts(host string) (ca []byte, cert []byte, key []byte) { - notBefore := time.Now().Add(time.Minute * -5) - notAfter := notBefore.Add(100 * 365 * 24 * time.Hour) - - serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) - serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) - if err != nil { - log.WithField("err", err).Fatal("failed to generate serial number") - } - rootKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - log.WithField("err", err).Fatal("failed scdsa.GenerateKey") - } - - rootTemplate := x509.Certificate{ - SerialNumber: serialNumber, - NotBefore: notBefore, - NotAfter: notAfter, - KeyUsage: x509.KeyUsageCertSign, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, - IsCA: true, - Subject: pkix.Name{Organization: []string{"nil1"}}, - } - - derBytes, err := x509.CreateCertificate(rand.Reader, &rootTemplate, &rootTemplate, &rootKey.PublicKey, rootKey) - if err != nil { - log.WithField("err", err).Fatal("failed createCertificate for Ca") - } - - ca = encodeCert(derBytes) - - leafKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - log.WithField("err", err).Fatal("failed createLeafKey for certificate") - } - - key = encodeKey(leafKey) - - serialNumber, err = rand.Int(rand.Reader, serialNumberLimit) - if err != nil { - log.WithField("err", err).Fatal("failed to generate serial number") - } - leafTemplate := x509.Certificate{ - SerialNumber: serialNumber, - NotBefore: notBefore, - NotAfter: notAfter, - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, - IsCA: false, - Subject: pkix.Name{Organization: []string{"nil2"}}, - } - hosts := strings.Split(host, ",") - for _, h := range hosts { - if ip := net.ParseIP(h); ip != nil { - leafTemplate.IPAddresses = append(leafTemplate.IPAddresses, ip) - } else { - leafTemplate.DNSNames = append(leafTemplate.DNSNames, h) - } - } - - derBytes, err = x509.CreateCertificate(rand.Reader, &leafTemplate, &rootTemplate, &leafKey.PublicKey, rootKey) - if err != nil { - log.WithField("err", err).Fatal("failed createLeaf certificate") - } - - cert = encodeCert(derBytes) - return ca, cert, key -} - -func encodeKey(key *ecdsa.PrivateKey) []byte { - b, err := x509.MarshalECPrivateKey(key) - if err != nil { - log.WithField("err", err).Fatal("unable to marshal ECDSA private key") - } - return pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: b}) -} - -func encodeCert(derBytes []byte) []byte { - return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) -} diff --git a/images/kube-webhook-certgen/rootfs/pkg/certs/certs_test.go b/images/kube-webhook-certgen/rootfs/pkg/certs/certs_test.go deleted file mode 100644 index fa8383cbb..000000000 --- a/images/kube-webhook-certgen/rootfs/pkg/certs/certs_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package certs - -import ( - "bytes" - "crypto/tls" - "crypto/x509" - "fmt" - "io" - "net/http" - "net/http/httptest" - "testing" -) - -func handler(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprintf(w, "Hello World") -} - -func TestCertificateCreation(t *testing.T) { - ca, cert, key := GenerateCerts("localhost") - - c, err := tls.X509KeyPair(cert, key) - if err != nil { - t.Fatal(err) - } - - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(ca) - - tr := &http.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: caCertPool, - ServerName: "localhost", - }, - } - - ts := httptest.NewUnstartedServer(http.HandlerFunc(handler)) - ts.TLS = &tls.Config{Certificates: []tls.Certificate{c}} - ts.StartTLS() - defer ts.Close() - - client := &http.Client{Transport: tr} - res, err := client.Get(ts.URL) - if err != nil { - t.Fatal(err) - } - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - t.Errorf("Response code was %v; want 200", res.StatusCode) - } - - body, err := io.ReadAll(res.Body) - if err != nil { - t.Fatal(err) - } - - expected := []byte("Hello World") - - if bytes.Compare(expected, body) != 0 { - t.Errorf("Response body was '%v'; want '%v'", expected, body) - } -} diff --git a/images/kube-webhook-certgen/rootfs/pkg/k8s/k8s.go b/images/kube-webhook-certgen/rootfs/pkg/k8s/k8s.go deleted file mode 100644 index 6018b4253..000000000 --- a/images/kube-webhook-certgen/rootfs/pkg/k8s/k8s.go +++ /dev/null @@ -1,259 +0,0 @@ -package k8s - -import ( - "context" - "fmt" - - log "github.com/sirupsen/logrus" - admissionv1 "k8s.io/api/admissionregistration/v1" - v1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" -) - -type k8s struct { - clientset kubernetes.Interface - aggregatorClientset clientset.Interface -} - -func New(clientset kubernetes.Interface, aggregatorClientset clientset.Interface) *k8s { - if clientset == nil { - log.Fatal("no kubernetes client given") - } - - if aggregatorClientset == nil { - log.Fatal("no kubernetes aggregator client given") - } - - return &k8s{ - clientset: clientset, - aggregatorClientset: aggregatorClientset, - } -} - -type PatchOptions struct { - ValidatingWebhookConfigurationName string - MutatingWebhookConfigurationName string - APIServiceName string - CABundle []byte - FailurePolicyType admissionv1.FailurePolicyType -} - -func (k8s *k8s) PatchObjects(ctx context.Context, options PatchOptions) error { - patchMutating := options.MutatingWebhookConfigurationName != "" - patchValidating := options.ValidatingWebhookConfigurationName != "" - patchAPIService := options.APIServiceName != "" - - if !patchMutating && !patchValidating && options.FailurePolicyType != "" { - return fmt.Errorf("failurePolicy specified, but no webhook will be patched") - } - - if patchMutating && patchValidating && - options.MutatingWebhookConfigurationName != options.ValidatingWebhookConfigurationName { - return fmt.Errorf("webhook names must be the same") - } - - if patchAPIService { - log.Infof("patching APIService %q", options.APIServiceName) - - if err := k8s.patchAPIService(ctx, options.APIServiceName, options.CABundle); err != nil { - // Intentionally don't wrap error here to preserve old behavior and be able to log both - // original error and a message. - return err - } - } - - webhookName := options.ValidatingWebhookConfigurationName - if webhookName == "" { - webhookName = options.MutatingWebhookConfigurationName - } - - if patchMutating || patchValidating { - return k8s.patchWebhookConfigurations(ctx, webhookName, options.CABundle, options.FailurePolicyType, patchMutating, patchValidating) - } - - return nil -} - -func (k8s *k8s) patchAPIService(ctx context.Context, objectName string, ca []byte) error { - log.Infof("patching APIService %q", objectName) - - c := k8s.aggregatorClientset.ApiregistrationV1().APIServices() - - apiService, err := c.Get(ctx, objectName, metav1.GetOptions{}) - if err != nil { - return &wrappedError{ - err: err, - message: fmt.Sprintf("failed getting APIService %q", objectName), - } - } - - apiService.Spec.CABundle = ca - apiService.Spec.InsecureSkipTLSVerify = false - - if _, err := c.Update(ctx, apiService, metav1.UpdateOptions{}); err != nil { - return &wrappedError{ - err: err, - message: fmt.Sprintf("failed patching APIService %q", objectName), - } - } - - log.Debug("patched APIService") - - return nil -} - -// patchWebhookConfigurations will patch validatingWebhook and mutatingWebhook clientConfig configurations with -// the provided ca data. If failurePolicy is provided, patch all webhooks with this value -func (k8s *k8s) patchWebhookConfigurations( - ctx context.Context, - configurationName string, - ca []byte, - failurePolicy admissionv1.FailurePolicyType, - patchMutating bool, - patchValidating bool, -) error { - log.Infof("patching webhook configurations '%s' mutating=%t, validating=%t, failurePolicy=%s", configurationName, patchMutating, patchValidating, failurePolicy) - - if patchValidating { - if err := k8s.patchValidating(ctx, configurationName, ca, failurePolicy); err != nil { - // Intentionally don't wrap error here to preserve old behavior and be able to log both original error and a message. - return err - } - } else { - log.Debug("validating hook patching not required") - } - - if patchMutating { - if err := k8s.patchMutating(ctx, configurationName, ca, failurePolicy); err != nil { - // Intentionally don't wrap error here to preserve old behavior and be able to log both original error and a message. - return err - } - } else { - log.Debug("mutating hook patching not required") - } - - log.Info("Patched hook(s)") - - return nil -} - -type wrappedError struct { - err error - message string -} - -func (err wrappedError) Error() string { - return err.message -} - -func (err wrappedError) Unwrap() error { - return err.err -} - -func (k8s *k8s) patchValidating(ctx context.Context, configurationName string, ca []byte, failurePolicy admissionv1.FailurePolicyType) error { - valHook, err := k8s.clientset. - AdmissionregistrationV1(). - ValidatingWebhookConfigurations(). - Get(ctx, configurationName, metav1.GetOptions{}) - if err != nil { - return &wrappedError{ - err: err, - message: "failed getting validating webhook", - } - } - - for i := range valHook.Webhooks { - h := &valHook.Webhooks[i] - h.ClientConfig.CABundle = ca - if failurePolicy != "" { - h.FailurePolicy = &failurePolicy - } - } - - if _, err = k8s.clientset.AdmissionregistrationV1(). - ValidatingWebhookConfigurations(). - Update(ctx, valHook, metav1.UpdateOptions{}); err != nil { - return &wrappedError{ - err: err, - message: "failed patching validating webhook", - } - } - log.Debug("patched validating hook") - - return nil -} - -func (k8s *k8s) patchMutating(ctx context.Context, configurationName string, ca []byte, failurePolicy admissionv1.FailurePolicyType) error { - mutHook, err := k8s.clientset. - AdmissionregistrationV1(). - MutatingWebhookConfigurations(). - Get(ctx, configurationName, metav1.GetOptions{}) - if err != nil { - return &wrappedError{ - err: err, - message: "failed getting mutating webhook", - } - } - - for i := range mutHook.Webhooks { - h := &mutHook.Webhooks[i] - h.ClientConfig.CABundle = ca - if failurePolicy != "" { - h.FailurePolicy = &failurePolicy - } - } - - if _, err = k8s.clientset.AdmissionregistrationV1(). - MutatingWebhookConfigurations(). - Update(ctx, mutHook, metav1.UpdateOptions{}); err != nil { - return &wrappedError{ - err: err, - message: "failed patching mutating webhook", - } - } - log.Debug("patched mutating hook") - - return nil -} - -// GetCaFromSecret will check for the presence of a secret. If it exists, will return the content of the -// "ca" from the secret, otherwise will return nil -func (k8s *k8s) GetCaFromSecret(ctx context.Context, secretName string, namespace string) []byte { - log.Debugf("getting secret '%s' in namespace '%s'", secretName, namespace) - secret, err := k8s.clientset.CoreV1().Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{}) - if err != nil { - if k8serrors.IsNotFound(err) { - log.WithField("err", err).Info("no secret found") - return nil - } - log.WithField("err", err).Fatal("error getting secret") - } - - data := secret.Data["ca"] - if data == nil { - log.Fatal("got secret, but it did not contain a 'ca' key") - } - log.Debug("got secret") - return data -} - -// SaveCertsToSecret saves the provided ca, cert and key into a secret in the specified namespace. -func (k8s *k8s) SaveCertsToSecret(ctx context.Context, secretName, namespace, certName, keyName string, ca, cert, key []byte) { - log.Debugf("saving to secret '%s' in namespace '%s'", secretName, namespace) - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - }, - Data: map[string][]byte{"ca": ca, certName: cert, keyName: key}, - } - - log.Debug("saving secret") - _, err := k8s.clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) - if err != nil { - log.WithField("err", err).Fatal("failed creating secret") - } - log.Debug("saved secret") -} diff --git a/images/kube-webhook-certgen/rootfs/pkg/k8s/k8s_test.go b/images/kube-webhook-certgen/rootfs/pkg/k8s/k8s_test.go deleted file mode 100644 index b326697b6..000000000 --- a/images/kube-webhook-certgen/rootfs/pkg/k8s/k8s_test.go +++ /dev/null @@ -1,359 +0,0 @@ -package k8s - -import ( - "bytes" - "context" - "crypto/rand" - "errors" - "testing" - "time" - - admissionv1 "k8s.io/api/admissionregistration/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes/fake" - apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" - aggregatorfake "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/fake" -) - -const ( - testWebhookName = "c7c95710-d8c3-4cc3-a2a8-8d2b46909c76" - testSecretName = "15906410-af2a-4f9b-8a2d-c08ffdd5e129" - testAPIServiceName = "37f6a2d1-b401-4275-833b-9ff5004f0301" - testNamespace = "7cad5f92-c0d5-4bc9-87a3-6f44d5a5619d" -) - -var ( - fail = admissionv1.Fail - ignore = admissionv1.Ignore -) - -func genSecretData() (ca, cert, key []byte) { - ca = make([]byte, 4) - cert = make([]byte, 4) - key = make([]byte, 4) - rand.Read(cert) - rand.Read(key) - return -} - -func newTestSimpleK8s(objects ...runtime.Object) *k8s { - return &k8s{ - clientset: fake.NewSimpleClientset(objects...), - aggregatorClientset: aggregatorfake.NewSimpleClientset(), - } -} - -func TestGetCaFromCertificate(t *testing.T) { - ca, cert, key := genSecretData() - - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: testSecretName, - Namespace: testNamespace, - }, - Data: map[string][]byte{"ca": ca, "cert": cert, "key": key}, - } - - k := newTestSimpleK8s(secret) - - retrievedCa := k.GetCaFromSecret(contextWithDeadline(t), testSecretName, testNamespace) - if !bytes.Equal(retrievedCa, ca) { - t.Error("Was not able to retrieve CA information that was saved") - } -} - -func TestSaveCertsToSecret(t *testing.T) { - k := newTestSimpleK8s() - - ca, cert, key := genSecretData() - - ctx := contextWithDeadline(t) - - k.SaveCertsToSecret(ctx, testSecretName, testNamespace, "cert", "key", ca, cert, key) - - secret, _ := k.clientset.CoreV1().Secrets(testNamespace).Get(ctx, testSecretName, metav1.GetOptions{}) - - if !bytes.Equal(secret.Data["cert"], cert) { - t.Error("'cert' saved data does not match retrieved") - } - - if !bytes.Equal(secret.Data["key"], key) { - t.Error("'key' saved data does not match retrieved") - } -} - -func TestSaveThenLoadSecret(t *testing.T) { - k := newTestSimpleK8s() - ca, cert, key := genSecretData() - ctx := contextWithDeadline(t) - k.SaveCertsToSecret(ctx, testSecretName, testNamespace, "cert", "key", ca, cert, key) - retrievedCert := k.GetCaFromSecret(ctx, testSecretName, testNamespace) - if !bytes.Equal(retrievedCert, ca) { - t.Error("Was not able to retrieve CA information that was saved") - } -} - -func TestPatchWebhookConfigurations(t *testing.T) { - ca, _, _ := genSecretData() - - k := newTestSimpleK8s( - &admissionv1.MutatingWebhookConfiguration{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: testWebhookName, - }, - Webhooks: []admissionv1.MutatingWebhook{{Name: "m1"}, {Name: "m2"}}, - }, - &admissionv1.ValidatingWebhookConfiguration{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: testWebhookName, - }, - Webhooks: []admissionv1.ValidatingWebhook{{Name: "v1"}, {Name: "v2"}}, - }, - ) - - ctx := contextWithDeadline(t) - - if err := k.patchWebhookConfigurations(ctx, testWebhookName, ca, fail, true, true); err != nil { - t.Fatalf("Unexpected error patching webhooks: %s: %v", err.Error(), errors.Unwrap(err)) - } - - whmut, err := k.clientset. - AdmissionregistrationV1(). - MutatingWebhookConfigurations(). - Get(ctx, testWebhookName, metav1.GetOptions{}) - if err != nil { - t.Error(err) - } - - whval, err := k.clientset. - AdmissionregistrationV1(). - MutatingWebhookConfigurations(). - Get(ctx, testWebhookName, metav1.GetOptions{}) - if err != nil { - t.Error(err) - } - - if !bytes.Equal(whmut.Webhooks[0].ClientConfig.CABundle, ca) { - t.Error("Ca retrieved from first mutating webhook configuration does not match") - } - if !bytes.Equal(whmut.Webhooks[1].ClientConfig.CABundle, ca) { - t.Error("Ca retrieved from second mutating webhook configuration does not match") - } - if !bytes.Equal(whval.Webhooks[0].ClientConfig.CABundle, ca) { - t.Error("Ca retrieved from first validating webhook configuration does not match") - } - if !bytes.Equal(whval.Webhooks[1].ClientConfig.CABundle, ca) { - t.Error("Ca retrieved from second validating webhook configuration does not match") - } - if whmut.Webhooks[0].FailurePolicy == nil { - t.Errorf("Expected first mutating webhook failure policy to be set to %s", fail) - } - if whmut.Webhooks[1].FailurePolicy == nil { - t.Errorf("Expected second mutating webhook failure policy to be set to %s", fail) - } - if whval.Webhooks[0].FailurePolicy == nil { - t.Errorf("Expected first validating webhook failure policy to be set to %s", fail) - } - if whval.Webhooks[1].FailurePolicy == nil { - t.Errorf("Expected second validating webhook failure policy to be set to %s", fail) - } -} - -func Test_Patching_objects(t *testing.T) { - t.Parallel() - - ctx := contextWithDeadline(t) - - t.Run("returns_error_when", func(t *testing.T) { - t.Parallel() - - t.Run("failure_policy_is_defined_but_no_webhooks_will_be_patched", func(t *testing.T) { - t.Parallel() - - k := testK8sWithUnpatchedObjects() - - o := PatchOptions{ - FailurePolicyType: admissionv1.Fail, - } - - if err := k.PatchObjects(ctx, o); err == nil { - t.Fatalf("Expected error while patching") - } - }) - - // This is to preserve old behavior and log format, it could be improved. - t.Run("different_non_empty_names_are_specified_for_validating_and_mutating_webhook", func(t *testing.T) { - t.Parallel() - - k := testK8sWithUnpatchedObjects() - - o := PatchOptions{ - ValidatingWebhookConfigurationName: "foo", - MutatingWebhookConfigurationName: "bar", - } - - if err := k.PatchObjects(ctx, o); err == nil { - t.Fatalf("Expected error while patching") - } - }) - - t.Run("patching_webhook_is_requested_and_it_does_not_exist", func(t *testing.T) { - t.Parallel() - - k := newTestSimpleK8s() - - o := PatchOptions{ - ValidatingWebhookConfigurationName: "foo", - } - - if err := k.PatchObjects(ctx, o); err == nil { - t.Fatalf("Expected error while patching") - } - }) - - t.Run("patching_APIService_is_requested_and_it_does_not_exist", func(t *testing.T) { - t.Parallel() - - k := newTestSimpleK8s() - - o := PatchOptions{ - APIServiceName: "foo", - } - - if err := k.PatchObjects(ctx, o); err == nil { - t.Fatalf("Expected error while patching") - } - }) - }) - - t.Run("when_patching_APIService_object", func(t *testing.T) { - t.Parallel() - - k := testK8sWithUnpatchedObjects() - - o := PatchOptions{ - APIServiceName: testAPIServiceName, - CABundle: []byte("foo"), - } - - if err := k.PatchObjects(ctx, o); err != nil { - t.Fatalf("Unexpected error while patching objects: %v", err) - } - - c := k.aggregatorClientset.ApiregistrationV1().APIServices() - apiService, err := c.Get(ctx, testAPIServiceName, metav1.GetOptions{}) - if err != nil { - t.Fatalf("Unexpected error while getting APIService object %q: %v", testAPIServiceName, err) - } - - // This is required when CABundle field is populated. - t.Run("sets_insecure_skip_tls_verity_to_false", func(t *testing.T) { - t.Parallel() - - if apiService.Spec.InsecureSkipTLSVerify { - t.Fatalf("Expected insecureSkipTLSVerify of APIService to be false") - } - }) - - t.Run("sets_ca_bundle_with_ca_certificate_from_created_secret", func(t *testing.T) { - t.Parallel() - - if len(apiService.Spec.CABundle) == 0 { - t.Fatalf("Expected CABundle of APIService to be not empty") - } - - if !bytes.Equal(o.CABundle, apiService.Spec.CABundle) { - t.Fatalf("CABundle content of APIService does not match requested bundle") - } - }) - }) - - t.Run("allows_patching_only_validating_webhook", func(t *testing.T) { - t.Parallel() - - k := testK8sWithUnpatchedObjects() - - o := PatchOptions{ - ValidatingWebhookConfigurationName: testWebhookName, - } - - if err := k.PatchObjects(ctx, o); err != nil { - t.Fatalf("Unexpected error patching objects: %v", err) - } - }) - - t.Run("allows_patching_only_mutating_webhook", func(t *testing.T) { - t.Parallel() - - k := testK8sWithUnpatchedObjects() - - o := PatchOptions{ - MutatingWebhookConfigurationName: testWebhookName, - } - - if err := k.PatchObjects(ctx, o); err != nil { - t.Fatalf("Unexpected error patching objects: %v", err) - } - }) -} - -const ( - // Arbitrary amount of time to let tests exit cleanly before main process terminates. - timeoutGracePeriod = 10 * time.Second -) - -// contextWithDeadline returns context with will timeout before t.Deadline(). -func contextWithDeadline(t *testing.T) context.Context { - t.Helper() - - deadline, ok := t.Deadline() - if !ok { - return context.Background() - } - - ctx, cancel := context.WithDeadline(context.Background(), deadline.Truncate(timeoutGracePeriod)) - - t.Cleanup(cancel) - - return ctx -} - -func testK8sWithUnpatchedObjects() *k8s { - ca, cert, key := genSecretData() - - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: testSecretName, - Namespace: testNamespace, - }, - Data: map[string][]byte{"ca": ca, "cert": cert, "key": key}, - } - - validatingWebhook := &admissionv1.ValidatingWebhookConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - Name: testWebhookName, - }, - Webhooks: []admissionv1.ValidatingWebhook{{Name: "v1"}, {Name: "v2"}}, - } - mutatingWebhook := &admissionv1.MutatingWebhookConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - Name: testWebhookName, - }, - Webhooks: []admissionv1.MutatingWebhook{{Name: "m1"}, {Name: "m2"}}, - } - - apiService := &apiregistrationv1.APIService{ - ObjectMeta: metav1.ObjectMeta{ - Name: testAPIServiceName, - }, - } - - return &k8s{ - clientset: fake.NewSimpleClientset(secret, validatingWebhook, mutatingWebhook), - aggregatorClientset: aggregatorfake.NewSimpleClientset(apiService), - } -} diff --git a/images/nginx/Makefile b/images/nginx/Makefile index 3cc14e5b3..11ffd2f06 100644 --- a/images/nginx/Makefile +++ b/images/nginx/Makefile @@ -1,4 +1,4 @@ -# Copyright 2024 The Kubernetes Authors. All rights reserved. +# Copyright 2017 The Kubernetes Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,10 +21,8 @@ DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))) INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh # 0.0.0 shouldn't clobber any released builds -SHORT_SHA ?=$(shell git rev-parse --short HEAD) -TAG ?=$(shell cat TAG) - -REGISTRY ?= us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx +TAG ?= 0.0 +REGISTRY ?= gcr.io/k8s-staging-ingress-nginx IMAGE = $(REGISTRY)/nginx @@ -32,7 +30,7 @@ IMAGE = $(REGISTRY)/nginx export DOCKER_CLI_EXPERIMENTAL=enabled # build with buildx -PLATFORMS?=linux/amd64,linux/arm,linux/arm64 +PLATFORMS?=linux/amd64,linux/arm,linux/arm64,linux/s390x OUTPUT= PROGRESS=plain build: ensure-buildx @@ -50,7 +48,7 @@ push: build ensure-buildx: # this is required for cloudbuild ifeq ("$(wildcard $(INIT_BUILDX))","") - @curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/hack/init-buildx.sh | bash + @curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/hack/init-buildx.sh | bash else @exec $(INIT_BUILDX) endif diff --git a/images/nginx/README.md b/images/nginx/README.md index 768077215..a5c649b3c 100644 --- a/images/nginx/README.md +++ b/images/nginx/README.md @@ -1,47 +1,28 @@ -NGINX base image +NGINX base image using [alpine](https://www.alpinelinux.org/) -### HTTP/3 Support +This custom image contains: -**HTTP/3 support is experimental and under development** +- [nginx-http-auth-digest](https://github.com/atomx/nginx-http-auth-digest) +- [ngx_http_substitutions_filter_module](https://github.com/yaoweibin/ngx_http_substitutions_filter_module) +- [nginx-opentracing](https://github.com/opentracing-contrib/nginx-opentracing) +- [opentracing-cpp](https://github.com/opentracing/opentracing-cpp) +- [zipkin-cpp-opentracing](https://github.com/rnburn/zipkin-cpp-opentracing) +- [dd-opentracing-cpp](https://github.com/DataDog/dd-opentracing-cpp) +- [ModSecurity-nginx](https://github.com/SpiderLabs/ModSecurity-nginx) (only supported in x86_64) +- [brotli](https://github.com/google/brotli) +- [geoip2](https://github.com/leev/ngx_http_geoip2_module) -[HTTP/3](https://datatracker.ietf.org/doc/html/rfc9114)\ -[QUIC](https://datatracker.ietf.org/doc/html/rfc9000) +**How to use this image:** +This image provides a default configuration file with no backend servers. -[According to the documentation, NGINX 1.25.0 or higher supports HTTP/3:](https://nginx.org/en/docs/quic.html) +_Using docker_ -> Support for QUIC and HTTP/3 protocols is available since 1.25.0. +```console +docker run -v /some/nginx.con:/etc/nginx/nginx.conf:ro k8s.gcr.io/ingress-nginx/nginx:v20210324-g8baef769d@sha256:fcfa3e9d1f8ec3141efedbf77cf659640f452a9c22165c78006ea462b84d06f6 +``` -But this requires adding a new flag during the build: +_Creating a replication controller_ -> When configuring nginx, it is possible to enable QUIC and HTTP/3 using the --with-http_v3_module configuration parameter. - -[We have added this flag](https://github.com/kubernetes/ingress-nginx/pull/11470), but it is not enough to use HTTP/3 in ingress-nginx, this is the first step. - -The next steps will be: - -1. **Waiting for OpenSSL 3.4.**\ - The main problem is, that we still use OpenSSL (3.x) and it does not support the important mechanism of TLS 1.3 - [early_data](https://datatracker.ietf.org/doc/html/rfc8446#section-2.3): - - > Otherwise, the OpenSSL compatibility layer will be used that does not support early data. - - [And although another part of the documentation says that the directive is supported with OpenSSL:](https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_early_data) - - > The directive is supported when using OpenSSL 1.1.1 or higher. - - But this is incomplete support, because OpenSSL does not support this feature, and [it has only client side support:](https://github.com/openssl/openssl) - - > ... the QUIC (currently client side only) version 1 protocol - - [And also there are some issues even with client side](https://github.com/openssl/openssl/discussions/23339) - - Due to this, we currently have incomplete HTTP/3 support, without important security and performance features.\ - But the good news is that [OpenSSL plans to add server-side support in 3.4](https://github.com/openssl/web/blob/master/roadmap.md): - - > Server-side QUIC support - - [Overview of SSL libraries(HAProxy Documentation)](https://github.com/haproxy/wiki/wiki/SSL-Libraries-Support-Status#tldr) - -2. **Adding [parameters](https://nginx.org/en/docs/http/ngx_http_v3_module.html) to the configmap to configure HTTP/3 and quic(enableHTTP3, enableHTTP/0.9, maxCurrentStream, and so on).** -3. **Adding options to the nginx config template(`listen 443 quic` to server blocks and `add_header Alt-Svc 'h3=":8443"; ma=86400';` to location blocks).** -4. **Opening the https port for UDP in the container(because QUIC uses UDP).** -5. **Adding tests.** +```console +kubectl create -f ./rc.yaml +``` diff --git a/images/nginx/TAG b/images/nginx/TAG deleted file mode 100644 index 46b105a30..000000000 --- a/images/nginx/TAG +++ /dev/null @@ -1 +0,0 @@ -v2.0.0 diff --git a/images/nginx/cloudbuild.yaml b/images/nginx/cloudbuild.yaml index 2563692d7..ba09d2acd 100644 --- a/images/nginx/cloudbuild.yaml +++ b/images/nginx/cloudbuild.yaml @@ -1,14 +1,24 @@ +timeout: 10800s options: - # Increase machine type for multi-arch builds. - machineType: E2_HIGHCPU_32 - # Ignore Prow provided substitutions. substitution_option: ALLOW_LOOSE + # job builds a multi-arch docker image for amd64,arm,arm64 and s390x. + machineType: N1_HIGHCPU_32 steps: - - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20250116-2a05ea7e3d - env: - - REGISTRY=us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx + - name: gcr.io/k8s-testimages/gcb-docker-gcloud:v20201130-750d12f entrypoint: bash + env: + - DOCKER_CLI_EXPERIMENTAL=enabled + - TAG=$_GIT_TAG + - BASE_REF=$_PULL_BASE_REF + - REGISTRY=gcr.io/k8s-staging-ingress-nginx + # default cloudbuild has HOME=/builder/home and docker buildx is in /root/.docker/cli-plugins/docker-buildx + # set the home to /root explicitly to if using docker buildx + - HOME=/root args: - - -c - - gcloud auth configure-docker && cd images/nginx && make push -timeout: 7200s + - -c + - | + gcloud auth configure-docker \ + && make push +substitutions: + _GIT_TAG: "12345" + _PULL_BASE_REF: "master" diff --git a/images/nginx/rc.yaml b/images/nginx/rc.yaml new file mode 100644 index 000000000..8a6bec91c --- /dev/null +++ b/images/nginx/rc.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +spec: + type: NodePort + ports: + - port: 80 + protocol: TCP + name: http + - port: 443 + protocol: TCP + name: https + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +--- +apiVersion: v1 +kind: ReplicationController +metadata: + name: nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +spec: + replicas: 1 + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + template: + metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + spec: + containers: + - name: nginx + image: k8s.gcr.io/ingress-nginx/nginx:v20210530-g6aab4c291@sha256:a7356029dd0c26cc3466bf7a27daec0f4df73aa14ca6c8b871a767022a812c0b + ports: + - containerPort: 80 + - containerPort: 443 diff --git a/images/nginx/rootfs/Dockerfile b/images/nginx/rootfs/Dockerfile index 834a9bcf3..e639687f1 100644 --- a/images/nginx/rootfs/Dockerfile +++ b/images/nginx/rootfs/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2024 The Kubernetes Authors. All rights reserved. +# Copyright 2015 The Kubernetes Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,17 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM alpine:3.21 as builder + + +FROM alpine:3.13 as builder COPY . / RUN apk update \ && apk upgrade \ - && apk add -U bash --no-cache \ + && apk add -U bash \ && /build.sh # Use a multi-stage build -FROM alpine:3.21 +FROM alpine:3.13 ENV PATH=$PATH:/usr/local/luajit/bin:/usr/local/nginx/sbin:/usr/local/nginx/bin @@ -29,46 +31,44 @@ ENV LUA_PATH="/usr/local/share/luajit-2.1.0-beta3/?.lua;/usr/local/share/lua/5.1 ENV LUA_CPATH="/usr/local/lib/lua/?/?.so;/usr/local/lib/lua/?.so;;" COPY --from=builder /usr/local /usr/local -COPY --from=builder /usr/lib/libopentelemetry* /usr/local/lib COPY --from=builder /opt /opt COPY --from=builder /etc/nginx /etc/nginx RUN apk update \ && apk upgrade \ && apk add -U --no-cache \ - bash \ - openssl \ - pcre \ - zlib \ - ca-certificates \ - patch \ - yajl \ - lmdb \ - libxml2 \ - libmaxminddb \ - yaml-cpp \ - dumb-init \ - tzdata \ - grpc-cpp \ - libprotobuf \ - abseil-cpp-crc-cpu-detect \ - abseil-cpp-vlog-config-internal \ + bash \ + openssl \ + pcre \ + zlib \ + geoip \ + curl ca-certificates \ + patch \ + yajl \ + lmdb \ + libxml2 \ + libmaxminddb \ + yaml-cpp \ + dumb-init \ + nano \ + tzdata \ && ln -s /usr/local/nginx/sbin/nginx /sbin/nginx \ + && addgroup -Sg 101 www-data \ && adduser -S -D -H -u 101 -h /usr/local/nginx \ - -s /sbin/nologin -G www-data -g www-data www-data \ + -s /sbin/nologin -G www-data -g www-data www-data \ && bash -eu -c ' \ writeDirs=( \ - /var/log/nginx \ - /var/lib/nginx/body \ - /var/lib/nginx/fastcgi \ - /var/lib/nginx/proxy \ - /var/lib/nginx/scgi \ - /var/lib/nginx/uwsgi \ - /var/log/audit \ + /var/log/nginx \ + /var/lib/nginx/body \ + /var/lib/nginx/fastcgi \ + /var/lib/nginx/proxy \ + /var/lib/nginx/scgi \ + /var/lib/nginx/uwsgi \ + /var/log/audit \ ); \ for dir in "${writeDirs[@]}"; do \ - mkdir -p ${dir}; \ - chown -R www-data:www-data ${dir}; \ + mkdir -p ${dir}; \ + chown -R www-data.www-data ${dir}; \ done' EXPOSE 80 443 diff --git a/images/nginx/rootfs/build.sh b/images/nginx/rootfs/build.sh index 297abf777..ebea37cf6 100755 --- a/images/nginx/rootfs/build.sh +++ b/images/nginx/rootfs/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2023 The Kubernetes Authors. +# Copyright 2015 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,91 +18,45 @@ set -o errexit set -o nounset set -o pipefail -export NGINX_VERSION=1.27.1 +export NGINX_VERSION=1.20.1 +export NDK_VERSION=0.3.1 +export SETMISC_VERSION=0.32 +export MORE_HEADERS_VERSION=0.33 +export NGINX_DIGEST_AUTH=cd8641886c873cf543255aeda20d23e4cd603d05 +export NGINX_SUBSTITUTIONS=bc58cb11844bc42735bbaef7085ea86ace46d05b +export NGINX_OPENTRACING_VERSION=0.11.0 +export OPENTRACING_CPP_VERSION=1.6.0 +export ZIPKIN_CPP_VERSION=f69593138ff84ca2f6bc115992e18ca3d35f344a +export YAML_CPP_VERSION=yaml-cpp-0.6.3 +export JAEGER_VERSION=0.7.0 +export MSGPACK_VERSION=3.2.1 +export DATADOG_CPP_VERSION=7b560e5c13324c0581476dad3bd8ac4ac5f64045 +export MODSECURITY_VERSION=22e53aba4e3ae8c7d59a3672d6727e49246afe96 +export MODSECURITY_LIB_VERSION=v3.0.4 +export OWASP_MODSECURITY_CRS_VERSION=v3.3.0 +export LUA_NGX_VERSION=138c1b96423aa26defe00fe64dd5760ef17e5ad8 +export LUA_STREAM_NGX_VERSION=0.0.9 +export LUA_UPSTREAM_VERSION=0.07 +export LUA_CJSON_VERSION=2.1.0.8 +export NGINX_INFLUXDB_VERSION=5b09391cb7b9a889687c0aa67964c06a2d933e8b +export GEOIP2_VERSION=3.3 +export NGINX_AJP_VERSION=bf6cd93f2098b59260de8d494f0f4b1f11a84627 -# Check for recent changes: https://github.com/vision5/ngx_devel_kit/compare/v0.3.3...master -export NDK_VERSION=v0.3.3 +export LUAJIT_VERSION=2.1-20201027 -# Check for recent changes: https://github.com/openresty/set-misc-nginx-module/compare/v0.33...master -export SETMISC_VERSION=v0.33 - -# Check for recent changes: https://github.com/openresty/headers-more-nginx-module/compare/v0.37...master -export MORE_HEADERS_VERSION=v0.37 - -# Check for recent changes: https://github.com/atomx/nginx-http-auth-digest/compare/v1.0.0...master -export NGINX_DIGEST_AUTH=v1.0.0 - -# Check for recent changes: https://github.com/SpiderLabs/ModSecurity-nginx/compare/v1.0.3...master -export MODSECURITY_VERSION=v1.0.3 - -# Check for recent changes: https://github.com/SpiderLabs/ModSecurity/compare/v3.0.13...v3/master -export MODSECURITY_LIB_VERSION=v3.0.13 - -# Check for recent changes: https://github.com/coreruleset/coreruleset/compare/v4.10.0...main -export OWASP_MODSECURITY_CRS_VERSION=v4.10.0 - -# Check for recent changes: https://github.com/openresty/lua-nginx-module/compare/v0.10.27...master -export LUA_NGX_VERSION=v0.10.27 - -# Check for recent changes: https://github.com/openresty/stream-lua-nginx-module/compare/v0.0.15...master -export LUA_STREAM_NGX_VERSION=v0.0.15 - -# Check for recent changes: https://github.com/openresty/lua-upstream-nginx-module/compare/v0.07...master -export LUA_UPSTREAM_VERSION=v0.07 - -# Check for recent changes: https://github.com/openresty/lua-cjson/compare/2.1.0.14...master -export LUA_CJSON_VERSION=2.1.0.14 - -# Check for recent changes: https://github.com/leev/ngx_http_geoip2_module/compare/445df24ef3781e488cee3dfe8a1e111997fc1dfe...master -export GEOIP2_VERSION=445df24ef3781e488cee3dfe8a1e111997fc1dfe - -# Check for recent changes: https://github.com/openresty/luajit2/compare/v2.1-20240815...v2.1-agentzh -export LUAJIT_VERSION=v2.1-20240815 - -# Check for recent changes: https://github.com/openresty/lua-resty-balancer/compare/v0.05...master -export LUA_RESTY_BALANCER=v0.05 - -# Check for recent changes: https://github.com/openresty/lua-resty-lrucache/compare/v0.15...master -export LUA_RESTY_CACHE=v0.15 - -# Check for recent changes: https://github.com/openresty/lua-resty-core/compare/v0.1.30...master -export LUA_RESTY_CORE=v0.1.30 - -# Check for recent changes: https://github.com/cloudflare/lua-resty-cookie/compare/f418d77082eaef48331302e84330488fdc810ef4...master -export LUA_RESTY_COOKIE_VERSION=f418d77082eaef48331302e84330488fdc810ef4 - -# Check for recent changes: https://github.com/openresty/lua-resty-dns/compare/v0.23...master -export LUA_RESTY_DNS=v0.23 - -# Check for recent changes: https://github.com/ledgetech/lua-resty-http/compare/v0.17.2...master -export LUA_RESTY_HTTP=v0.17.2 - -# Check for recent changes: https://github.com/openresty/lua-resty-lock/compare/v0.09...master -export LUA_RESTY_LOCK=v0.09 - -# Check for recent changes: https://github.com/openresty/lua-resty-upload/compare/v0.11...master -export LUA_RESTY_UPLOAD_VERSION=v0.11 - -# Check for recent changes: https://github.com/openresty/lua-resty-string/compare/v0.16...master -export LUA_RESTY_STRING_VERSION=v0.16 - -# Check for recent changes: https://github.com/openresty/lua-resty-memcached/compare/v0.17...master -export LUA_RESTY_MEMCACHED_VERSION=v0.17 - -# Check for recent changes: https://github.com/openresty/lua-resty-redis/compare/v0.31...master -export LUA_RESTY_REDIS_VERSION=v0.31 - -# Check for recent changes: https://github.com/api7/lua-resty-ipmatcher/compare/3e93c53eb8c9884efe939ef070486a0e507cc5be...master -export LUA_RESTY_IPMATCHER_VERSION=3e93c53eb8c9884efe939ef070486a0e507cc5be - -# Check for recent changes: https://github.com/microsoft/mimalloc/compare/v2.1.9...master -export MIMALOC_VERSION=v2.1.9 - -# Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp/compare/v1.18.0...main -export OPENTELEMETRY_CPP_VERSION=v1.18.0 - -# Check for recent changes: https://github.com/open-telemetry/opentelemetry-proto/compare/v1.5.0...main -export OPENTELEMETRY_PROTO_VERSION=v1.5.0 +export LUA_RESTY_BALANCER=af4508f7aa5560c7d810922c2515b557f9e5d51a +export LUA_RESTY_CACHE=0.10 +export LUA_RESTY_CORE=0.1.21 +export LUA_RESTY_COOKIE_VERSION=766ad8c15e498850ac77f5e0265f1d3f30dc4027 +export LUA_RESTY_DNS=0.21 +export LUA_RESTY_HTTP=0.15 +export LUA_RESTY_LOCK=0.08 +export LUA_RESTY_UPLOAD_VERSION=0.10 +export LUA_RESTY_STRING_VERSION=0.12 +export LUA_RESTY_MEMCACHED_VERSION=0.15 +export LUA_RESTY_REDIS_VERSION=0.29 +export LUA_RESTY_IPMATCHER_VERSION=1a0a1c58fd085b15eedee58de8b5f45c27aff7bc +export LUA_RESTY_GLOBAL_THROTTLE_VERSION=0.2.0 export BUILD_PATH=/tmp/build @@ -112,25 +66,17 @@ get_src() { hash="$1" url="$2" - dest="${3-}" - ARGS="" f=$(basename "$url") echo "Downloading $url" curl -sSL "$url" -o "$f" - # TODO: Reenable checksum verification but make it smarter - # echo "$hash $f" | sha256sum -c - || exit 10 - if [ ! -z "$dest" ]; then - mkdir ${BUILD_PATH}/${dest} - ARGS="-C ${BUILD_PATH}/${dest} --strip-components=1" - fi - tar xvzf "$f" $ARGS + echo "$hash $f" | sha256sum -c - || exit 10 + tar xzf "$f" rm -rf "$f" } # install required packages to build -# Dependencies from "ninja" and below are OTEL dependencies apk add \ bash \ gcc \ @@ -144,13 +90,13 @@ apk add \ linux-headers \ libxslt-dev \ gd-dev \ + geoip-dev \ perl-dev \ libedit-dev \ mercurial \ alpine-sdk \ findutils \ - curl \ - ca-certificates \ + curl ca-certificates \ patch \ libaio-dev \ openssl \ @@ -167,22 +113,7 @@ apk add \ unzip \ dos2unix \ yaml-cpp \ - coreutils \ - ninja \ - gtest-dev \ - git \ - build-base \ - pkgconfig \ - c-ares-dev \ - re2-dev \ - grpc-dev \ - protobuf-dev - -# apk add -X http://dl-cdn.alpinelinux.org/alpine/edge/testing opentelemetry-cpp-dev - -# There is some bug with some platforms and git, so force HTTP/1.1 -git config --global http.version HTTP/1.1 -git config --global http.postBuffer 157286400 + coreutils mkdir -p /etc/nginx @@ -190,135 +121,263 @@ mkdir --verbose -p "$BUILD_PATH" cd "$BUILD_PATH" # download, verify and extract the source files -get_src 66dc7081488811e9f925719e34d1b4504c2801c81dee2920e5452a86b11405ae \ +get_src e462e11533d5c30baa05df7652160ff5979591d291736cfa5edb9fd2edb48c49 \ "https://nginx.org/download/nginx-$NGINX_VERSION.tar.gz" -get_src aa961eafb8317e0eb8da37eb6e2c9ff42267edd18b56947384e719b85188f58b \ - "https://github.com/vision5/ngx_devel_kit/archive/$NDK_VERSION.tar.gz" "ngx_devel_kit" +get_src 0e971105e210d272a497567fa2e2c256f4e39b845a5ba80d373e26ba1abfbd85 \ + "https://github.com/simpl/ngx_devel_kit/archive/v$NDK_VERSION.tar.gz" -get_src abc123 \ - "https://github.com/open-telemetry/opentelemetry-cpp/archive/$OPENTELEMETRY_CPP_VERSION.tar.gz" "opentelemetry-cpp" +get_src f1ad2459c4ee6a61771aa84f77871f4bfe42943a4aa4c30c62ba3f981f52c201 \ + "https://github.com/openresty/set-misc-nginx-module/archive/v$SETMISC_VERSION.tar.gz" -get_src abc123 \ - "https://github.com/open-telemetry/opentelemetry-proto/archive/$OPENTELEMETRY_PROTO_VERSION.tar.gz" "opentelemetry-proto" +get_src a3dcbab117a9c103bc1ea5200fc00a7b7d2af97ff7fd525f16f8ac2632e30fbf \ + "https://github.com/openresty/headers-more-nginx-module/archive/v$MORE_HEADERS_VERSION.tar.gz" -get_src cd5e2cc834bcfa30149e7511f2b5a2183baf0b70dc091af717a89a64e44a2985 \ - "https://github.com/openresty/set-misc-nginx-module/archive/$SETMISC_VERSION.tar.gz" "set-misc-nginx-module" +get_src fe683831f832aae4737de1e1026a4454017c2d5f98cb88b08c5411dc380062f8 \ + "https://github.com/atomx/nginx-http-auth-digest/archive/$NGINX_DIGEST_AUTH.tar.gz" -get_src 0c0d2ced2ce895b3f45eb2b230cd90508ab2a773299f153de14a43e44c1209b3 \ - "https://github.com/openresty/headers-more-nginx-module/archive/$MORE_HEADERS_VERSION.tar.gz" "headers-more-nginx-module" +get_src 618551948ab14cac51d6e4ad00452312c7b09938f59ebff4f93875013be31f2d \ + "https://github.com/yaoweibin/ngx_http_substitutions_filter_module/archive/$NGINX_SUBSTITUTIONS.tar.gz" -get_src f09851e6309560a8ff3e901548405066c83f1f6ff88aa7171e0763bd9514762b \ - "https://github.com/atomx/nginx-http-auth-digest/archive/$NGINX_DIGEST_AUTH.tar.gz" "nginx-http-auth-digest" +get_src a0087c61e82651dbdeeef1ceec375ce103f0ce571a1c8b35863cff4e596434a6 \ + "https://github.com/opentracing-contrib/nginx-opentracing/archive/v$NGINX_OPENTRACING_VERSION.tar.gz" -get_src 32a42256616cc674dca24c8654397390adff15b888b77eb74e0687f023c8751b \ - "https://github.com/SpiderLabs/ModSecurity-nginx/archive/$MODSECURITY_VERSION.tar.gz" "ModSecurity-nginx" +get_src 5b170042da4d1c4c231df6594da120875429d5231e9baa5179822ee8d1054ac3 \ + "https://github.com/opentracing/opentracing-cpp/archive/v$OPENTRACING_CPP_VERSION.tar.gz" -get_src bc764db42830aeaf74755754b900253c233ad57498debe7a441cee2c6f4b07c2 \ - "https://github.com/openresty/lua-nginx-module/archive/$LUA_NGX_VERSION.tar.gz" "lua-nginx-module" +get_src 71de3d0658935db7ccea20e006b35e58ddc7e4c18878b9523f2addc2371e9270 \ + "https://github.com/rnburn/zipkin-cpp-opentracing/archive/$ZIPKIN_CPP_VERSION.tar.gz" -get_src 01b715754a8248cc7228e0c8f97f7488ae429d90208de0481394e35d24cef32f \ - "https://github.com/openresty/stream-lua-nginx-module/archive/$LUA_STREAM_NGX_VERSION.tar.gz" "stream-lua-nginx-module" +get_src 38f2ae43fceda683f652065e13a80b14a580ede476a4b44eb0ddd85665380360 \ + "https://github.com/SpiderLabs/ModSecurity-nginx/archive/$MODSECURITY_VERSION.tar.gz" -get_src a92c9ee6682567605ece55d4eed5d1d54446ba6fba748cff0a2482aea5713d5f \ - "https://github.com/openresty/lua-upstream-nginx-module/archive/$LUA_UPSTREAM_VERSION.tar.gz" "lua-upstream-nginx-module" +get_src 77ea1b90b3718aa0c324207cb29418f5bced2354c2e483a9523d98c3460af1ed \ + "https://github.com/jbeder/yaml-cpp/archive/$YAML_CPP_VERSION.tar.gz" -get_src 77bbcbb24c3c78f51560017288f3118d995fe71240aa379f5818ff6b166712ff \ - "https://github.com/openresty/luajit2/archive/$LUAJIT_VERSION.tar.gz" "luajit2" +get_src 3a3a03060bf5e3fef52c9a2de02e6035cb557f389453d8f3b0c1d3d570636994 \ + "https://github.com/jaegertracing/jaeger-client-cpp/archive/v$JAEGER_VERSION.tar.gz" -get_src b6c9c09fd43eb34a71e706ad780b2ead26549a9a9f59280fe558f5b7b980b7c6 \ - "https://github.com/leev/ngx_http_geoip2_module/archive/$GEOIP2_VERSION.tar.gz" "ngx_http_geoip2_module" +get_src 464f46744a6be778626d11452c4db3c2d09461080c6db42e358e21af19d542f6 \ + "https://github.com/msgpack/msgpack-c/archive/cpp-$MSGPACK_VERSION.tar.gz" -get_src deb4ab1ffb9f3d962c4b4a2c4bdff692b86a209e3835ae71ebdf3b97189e40a9 \ - "https://github.com/openresty/lua-resty-upload/archive/$LUA_RESTY_UPLOAD_VERSION.tar.gz" "lua-resty-upload" +get_src 7dc05df3d1824b02c6958ff37f9e682b73c1737dcfee93212ca3f6c5bfae08f3 \ + "https://github.com/openresty/lua-nginx-module/archive/$LUA_NGX_VERSION.tar.gz" -get_src bdbf271003d95aa91cab0a92f24dca129e99b33f79c13ebfcdbbcbb558129491 \ - "https://github.com/openresty/lua-resty-string/archive/$LUA_RESTY_STRING_VERSION.tar.gz" "lua-resty-string" +get_src 6fcf7054f412a19c23c1ac3c0663f42f40bccc907d98c5d1657ae5cab9973ee9 \ + "https://github.com/openresty/stream-lua-nginx-module/archive/v$LUA_STREAM_NGX_VERSION.tar.gz" -get_src 16d72ed133f0c6df376a327386c3ef4e9406cf51003a700737c3805770ade7c5 \ - "https://github.com/openresty/lua-resty-balancer/archive/$LUA_RESTY_BALANCER.tar.gz" "lua-resty-balancer" +get_src 2a69815e4ae01aa8b170941a8e1a10b6f6a9aab699dee485d58f021dd933829a \ + "https://github.com/openresty/lua-upstream-nginx-module/archive/v$LUA_UPSTREAM_VERSION.tar.gz" -get_src 39baab9e2b31cc48cecf896cea40ef6e80559054fd8a6e440cc804a858ea84d4 \ - "https://github.com/openresty/lua-resty-core/archive/$LUA_RESTY_CORE.tar.gz" "lua-resty-core" +get_src f74a0821b079ea1fd63dd8659064356fc3f421ff4b35c17877140d2b2841cc3b \ + "https://github.com/openresty/luajit2/archive/v$LUAJIT_VERSION.tar.gz" -get_src a77b9de160d81712f2f442e1de8b78a5a7ef0d08f13430ff619f79235db974d4 \ - "https://github.com/openresty/lua-cjson/archive/$LUA_CJSON_VERSION.tar.gz" "lua-cjson" +get_src 40cc298f22bc29621024b68503335dcce464e42bcf02246f5864d7f8f2f5c379 \ + "https://github.com/DataDog/dd-opentracing-cpp/archive/$DATADOG_CPP_VERSION.tar.gz" -get_src 5ed48c36231e2622b001308622d46a0077525ac2f751e8cc0c9905914254baa4 \ - "https://github.com/cloudflare/lua-resty-cookie/archive/$LUA_RESTY_COOKIE_VERSION.tar.gz" "lua-resty-cookie" +get_src 1af5a5632dc8b00ae103d51b7bf225de3a7f0df82f5c6a401996c080106e600e \ + "https://github.com/influxdata/nginx-influxdb-module/archive/$NGINX_INFLUXDB_VERSION.tar.gz" -get_src 573184006b98ccee2594b0d134fa4d05e5d2afd5141cbad315051ccf7e9b6403 \ - "https://github.com/openresty/lua-resty-lrucache/archive/$LUA_RESTY_CACHE.tar.gz" "lua-resty-lrucache" +get_src 41378438c833e313a18869d0c4a72704b4835c30acaf7fd68013ab6732ff78a7 \ + "https://github.com/leev/ngx_http_geoip2_module/archive/$GEOIP2_VERSION.tar.gz" -get_src b4ddcd47db347e9adf5c1e1491a6279a6ae2a3aff3155ef77ea0a65c998a69c1 \ - "https://github.com/openresty/lua-resty-lock/archive/$LUA_RESTY_LOCK.tar.gz" "lua-resty-lock" +get_src 5f629a50ba22347c441421091da70fdc2ac14586619934534e5a0f8a1390a950 \ + "https://github.com/yaoweibin/nginx_ajp_module/archive/$NGINX_AJP_VERSION.tar.gz" -get_src 70e9a01eb32ccade0d5116a25bcffde0445b94ad35035ce06b94ccd260ad1bf0 \ - "https://github.com/openresty/lua-resty-dns/archive/$LUA_RESTY_DNS.tar.gz" "lua-resty-dns" +get_src 5d16e623d17d4f42cc64ea9cfb69ca960d313e12f5d828f785dd227cc483fcbd \ + "https://github.com/openresty/lua-resty-upload/archive/v$LUA_RESTY_UPLOAD_VERSION.tar.gz" -get_src 9fcb6db95bc37b6fce77d3b3dc740d593f9d90dce0369b405eb04844d56ac43f \ - "https://github.com/ledgetech/lua-resty-http/archive/$LUA_RESTY_HTTP.tar.gz" "lua-resty-http" +get_src bfd8c4b6c90aa9dcbe047ac798593a41a3f21edcb71904d50d8ac0e8c77d1132 \ + "https://github.com/openresty/lua-resty-string/archive/v$LUA_RESTY_STRING_VERSION.tar.gz" -get_src 02733575c4aed15f6cab662378e4b071c0a4a4d07940c4ef19a7319e9be943d4 \ - "https://github.com/openresty/lua-resty-memcached/archive/$LUA_RESTY_MEMCACHED_VERSION.tar.gz" "lua-resty-memcached" +get_src a21ec0d78a5dc5856df2374890a8a58e51de866b3d5978aceb0109a094367630 \ + "https://github.com/openresty/lua-resty-balancer/archive/$LUA_RESTY_BALANCER.tar.gz" -get_src c15aed1a01c88a3a6387d9af67a957dff670357f5fdb4ee182beb44635eef3f1 \ - "https://github.com/openresty/lua-resty-redis/archive/$LUA_RESTY_REDIS_VERSION.tar.gz" "lua-resty-redis" +get_src a377fbce78ba10f3ed3a8b5173ea318f8cf8da9d2ab127bb1e1f263078bf7da0 \ + "https://github.com/openresty/lua-resty-core/archive/v$LUA_RESTY_CORE.tar.gz" -get_src efb767487ea3f6031577b9b224467ddbda2ad51a41c5867a47582d4ad85d609e \ - "https://github.com/api7/lua-resty-ipmatcher/archive/$LUA_RESTY_IPMATCHER_VERSION.tar.gz" "lua-resty-ipmatcher" +get_src bd6bee4ccc6cf3307ab6ca0eea693a921fab9b067ba40ae12a652636da588ff7 \ + "https://github.com/openresty/lua-cjson/archive/$LUA_CJSON_VERSION.tar.gz" -get_src d74f86ada2329016068bc5a243268f1f555edd620b6a7d6ce89295e7d6cf18da \ - "https://github.com/microsoft/mimalloc/archive/${MIMALOC_VERSION}.tar.gz" "mimalloc" +get_src f818b5cef0881e5987606f2acda0e491531a0cb0c126d8dca02e2343edf641ef \ + "https://github.com/cloudflare/lua-resty-cookie/archive/$LUA_RESTY_COOKIE_VERSION.tar.gz" + +get_src dae9fb572f04e7df0dabc228f21cdd8bbfa1ff88e682e983ef558585bc899de0 \ + "https://github.com/openresty/lua-resty-lrucache/archive/v$LUA_RESTY_CACHE.tar.gz" + +get_src 2b4683f9abe73e18ca00345c65010c9056777970907a311d6e1699f753141de2 \ + "https://github.com/openresty/lua-resty-lock/archive/v$LUA_RESTY_LOCK.tar.gz" + +get_src 4aca34f324d543754968359672dcf5f856234574ee4da360ce02c778d244572a \ + "https://github.com/openresty/lua-resty-dns/archive/v$LUA_RESTY_DNS.tar.gz" + +get_src 987d5754a366d3ccbf745d2765f82595dcff5b94ba6c755eeb6d310447996f32 \ + "https://github.com/ledgetech/lua-resty-http/archive/v$LUA_RESTY_HTTP.tar.gz" + +get_src 8257e8fbf78eb2cc2cf2fdca2fda3c2e755f7d3222e7d15cc322111a0f720f9c \ + "https://github.com/openresty/lua-resty-memcached/archive/v$LUA_RESTY_MEMCACHED_VERSION.tar.gz" + +get_src 3f602af507aacd1f7aaeddfe7b77627fcde095fe9f115cb9d6ad8de2a52520e1 \ + "https://github.com/openresty/lua-resty-redis/archive/v$LUA_RESTY_REDIS_VERSION.tar.gz" + +get_src d0eacda122ab36585936256cb222ea9147bc5ad1fc3f24fd3748475653dd27ad \ + "https://github.com/api7/lua-resty-ipmatcher/archive/$LUA_RESTY_IPMATCHER_VERSION.tar.gz" + +get_src 0fb790e394510e73fdba1492e576aaec0b8ee9ef08e3e821ce253a07719cf7ea \ + "https://github.com/ElvinEfendi/lua-resty-global-throttle/archive/v$LUA_RESTY_GLOBAL_THROTTLE_VERSION.tar.gz" # improve compilation times CORES=$(($(grep -c ^processor /proc/cpuinfo) - 1)) export MAKEFLAGS=-j${CORES} export CTEST_BUILD_FLAGS=${MAKEFLAGS} +export HUNTER_JOBS_NUMBER=${CORES} +export HUNTER_USE_CACHE_SERVERS=true # Install luajit from openresty fork export LUAJIT_LIB=/usr/local/lib export LUA_LIB_DIR="$LUAJIT_LIB/lua" export LUAJIT_INC=/usr/local/include/luajit-2.1 -cd "$BUILD_PATH/luajit2" +cd "$BUILD_PATH/luajit2-$LUAJIT_VERSION" make CCDEBUG=-g make install ln -s /usr/local/bin/luajit /usr/local/bin/lua ln -s "$LUAJIT_INC" /usr/local/include/lua -cd "$BUILD_PATH/opentelemetry-cpp" -export CXXFLAGS="-DBENCHMARK_HAS_NO_INLINE_ASSEMBLY" -cmake -B build -G Ninja -Wno-dev \ - -DOTELCPP_PROTO_PATH="${BUILD_PATH}/opentelemetry-proto/" \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_TESTING="OFF" \ - -DBUILD_W3CTRACECONTEXT_TEST="OFF" \ - -DCMAKE_BUILD_TYPE=None \ - -DWITH_ABSEIL=ON \ - -DWITH_STL=ON \ - -DWITH_EXAMPLES=OFF \ - -DWITH_ZPAGES=OFF \ - -DWITH_OTLP_GRPC=ON \ - -DWITH_OTLP_HTTP=ON \ - -DWITH_ZIPKIN=ON \ - -DWITH_PROMETHEUS=OFF \ - -DWITH_ASYNC_EXPORT_PREVIEW=OFF \ - -DWITH_METRICS_EXEMPLAR_PREVIEW=OFF - cmake --build build - cmake --install build +cd "$BUILD_PATH" # Git tuning git config --global --add core.compression -1 +# build opentracing lib +cd "$BUILD_PATH/opentracing-cpp-$OPENTRACING_CPP_VERSION" +mkdir .build +cd .build + +cmake -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TESTING=OFF \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_MOCKTRACER=OFF \ + -DBUILD_STATIC_LIBS=ON \ + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true \ + .. + +make +make install + +# build yaml-cpp +# TODO @timmysilv: remove this and jaeger sed calls once it is fixed in jaeger-client-cpp +cd "$BUILD_PATH/yaml-cpp-$YAML_CPP_VERSION" +mkdir .build +cd .build + +cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true \ + -DYAML_BUILD_SHARED_LIBS=ON \ + -DYAML_CPP_BUILD_TESTS=OFF \ + -DYAML_CPP_BUILD_TOOLS=OFF \ + .. + +make +make install + +# build jaeger lib +cd "$BUILD_PATH/jaeger-client-cpp-$JAEGER_VERSION" +sed -i 's/-Werror/-Wno-psabi/' CMakeLists.txt +# use the above built yaml-cpp instead until a new version of jaeger-client-cpp fixes the yaml-cpp issue +# tl;dr new hunter is needed for new yaml-cpp, but new hunter has a conflict with old Thrift and new Boost +sed -i 's/hunter_add_package(yaml-cpp)/#hunter_add_package(yaml-cpp)/' CMakeLists.txt +sed -i 's/yaml-cpp::yaml-cpp/yaml-cpp/' CMakeLists.txt + +cat < export.map +{ + global: + OpenTracingMakeTracerFactory; + local: *; +}; +EOF + +mkdir .build +cd .build + +cmake -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TESTING=OFF \ + -DJAEGERTRACING_BUILD_EXAMPLES=OFF \ + -DJAEGERTRACING_BUILD_CROSSDOCK=OFF \ + -DJAEGERTRACING_COVERAGE=OFF \ + -DJAEGERTRACING_PLUGIN=ON \ + -DHUNTER_CONFIGURATION_TYPES=Release \ + -DBUILD_SHARED_LIBS=OFF \ + -DJAEGERTRACING_WITH_YAML_CPP=ON \ + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true \ + .. + +make +make install + +export HUNTER_INSTALL_DIR=$(cat _3rdParty/Hunter/install-root-dir) \ + +mv libjaegertracing_plugin.so /usr/local/lib/libjaegertracing_plugin.so + + +# build zipkin lib +cd "$BUILD_PATH/zipkin-cpp-opentracing-$ZIPKIN_CPP_VERSION" + +cat < export.map +{ + global: + OpenTracingMakeTracerFactory; + local: *; +}; +EOF + +mkdir .build +cd .build + +cmake -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_PLUGIN=ON \ + -DBUILD_TESTING=OFF \ + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true \ + .. + +make +make install + +# build msgpack lib +cd "$BUILD_PATH/msgpack-c-cpp-$MSGPACK_VERSION" + +mkdir .build +cd .build +cmake -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=OFF \ + -DMSGPACK_BUILD_EXAMPLES=OFF \ + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true \ + .. + +make +make install + +# build datadog lib +cd "$BUILD_PATH/dd-opentracing-cpp-$DATADOG_CPP_VERSION" + +mkdir .build +cd .build + +cmake -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TESTING=OFF \ + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true \ + .. + +make +make install + # Get Brotli source and deps cd "$BUILD_PATH" -git clone --depth=100 https://github.com/google/ngx_brotli.git +git clone --depth=1 https://github.com/google/ngx_brotli.git cd ngx_brotli -git reset --hard a71f9312c2deb28875acc7bacfdd5695a111aa53 git submodule init git submodule update @@ -334,9 +393,8 @@ make install # build modsecurity library cd "$BUILD_PATH" -git clone -n https://github.com/SpiderLabs/ModSecurity +git clone --depth=1 -b $MODSECURITY_LIB_VERSION https://github.com/SpiderLabs/ModSecurity cd ModSecurity/ -git checkout $MODSECURITY_LIB_VERSION git submodule init git submodule update @@ -376,22 +434,25 @@ mv rules/REQUEST-900-EXCLUSION-RULES-BEFORE-CRS.conf.example rules/REQUEST-900-E mv rules/RESPONSE-999-EXCLUSION-RULES-AFTER-CRS.conf.example rules/RESPONSE-999-EXCLUSION-RULES-AFTER-CRS.conf cd .. -# OWASP CRS v4 rules +# OWASP CRS v3 rules echo " Include /etc/nginx/owasp-modsecurity-crs/crs-setup.conf Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-900-EXCLUSION-RULES-BEFORE-CRS.conf Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-901-INITIALIZATION.conf +Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-903.9001-DRUPAL-EXCLUSION-RULES.conf +Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-903.9002-WORDPRESS-EXCLUSION-RULES.conf Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-905-COMMON-EXCEPTIONS.conf +Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-910-IP-REPUTATION.conf Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-911-METHOD-ENFORCEMENT.conf +Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-912-DOS-PROTECTION.conf Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-913-SCANNER-DETECTION.conf Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-921-PROTOCOL-ATTACK.conf -Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-922-MULTIPART-ATTACK.conf Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-930-APPLICATION-ATTACK-LFI.conf Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-931-APPLICATION-ATTACK-RFI.conf Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-932-APPLICATION-ATTACK-RCE.conf Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-933-APPLICATION-ATTACK-PHP.conf -Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-934-APPLICATION-ATTACK-GENERIC.conf +Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-934-APPLICATION-ATTACK-NODEJS.conf Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-941-APPLICATION-ATTACK-XSS.conf Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-942-APPLICATION-ATTACK-SQLI.conf Include /etc/nginx/owasp-modsecurity-crs/rules/REQUEST-943-APPLICATION-ATTACK-SESSION-FIXATION.conf @@ -402,38 +463,18 @@ Include /etc/nginx/owasp-modsecurity-crs/rules/RESPONSE-951-DATA-LEAKAGES-SQL.co Include /etc/nginx/owasp-modsecurity-crs/rules/RESPONSE-952-DATA-LEAKAGES-JAVA.conf Include /etc/nginx/owasp-modsecurity-crs/rules/RESPONSE-953-DATA-LEAKAGES-PHP.conf Include /etc/nginx/owasp-modsecurity-crs/rules/RESPONSE-954-DATA-LEAKAGES-IIS.conf -Include /etc/nginx/owasp-modsecurity-crs/rules/RESPONSE-955-WEB-SHELLS.conf Include /etc/nginx/owasp-modsecurity-crs/rules/RESPONSE-959-BLOCKING-EVALUATION.conf Include /etc/nginx/owasp-modsecurity-crs/rules/RESPONSE-980-CORRELATION.conf Include /etc/nginx/owasp-modsecurity-crs/rules/RESPONSE-999-EXCLUSION-RULES-AFTER-CRS.conf " > /etc/nginx/owasp-modsecurity-crs/nginx-modsecurity.conf -# NGINX compiles a small test program to check if an added module works as expected. -# -# ModSecurity-nginx provides 'printf("hello");' as a test, but newer versions of GCC, -# as included in Alpine 3.21, do not allow implicit declaration of function 'printf': -# -# objs/autotest.c:7:5: error: implicit declaration of function 'printf' [-Wimplicit-function-declaration] -# -# For this reason we replace 'printf("hello");' by 'msc_init();', which is always available. -# -# This fix is taken from a PR, that has been proposed to the ModSecurity-nginx project: -# -# https://github.com/owasp-modsecurity/ModSecurity-nginx/pull/275 -# -sed -i "s/ngx_feature_test='printf(\"hello\");'/ngx_feature_test='msc_init();'/" $BUILD_PATH/ModSecurity-nginx/config - # build nginx cd "$BUILD_PATH/nginx-$NGINX_VERSION" # apply nginx patches for PATCH in `ls /patches`;do echo "Patch: $PATCH" - if [[ "$PATCH" == *.txt ]]; then - patch -p0 < /patches/$PATCH - else - patch -p1 < /patches/$PATCH - fi + patch -p1 < /patches/$PATCH done WITH_FLAGS="--with-debug \ @@ -444,10 +485,10 @@ WITH_FLAGS="--with-debug \ --with-http_realip_module \ --with-http_auth_request_module \ --with-http_addition_module \ + --with-http_geoip_module \ --with-http_gzip_static_module \ --with-http_sub_module \ --with-http_v2_module \ - --with-http_v3_module \ --with-stream \ --with-stream_ssl_module \ --with-stream_realip_module \ @@ -458,7 +499,7 @@ WITH_FLAGS="--with-debug \ # "Combining -flto with -g is currently experimental and expected to produce unexpected results." # https://gcc.gnu.org/onlinedocs/gcc/Optimize-Options.html -CC_OPT="-g -O2 -fPIE -fstack-protector-strong \ +CC_OPT="-g -Og -fPIE -fstack-protector-strong \ -Wformat \ -Werror=format-security \ -Wno-deprecated-declarations \ @@ -467,28 +508,33 @@ CC_OPT="-g -O2 -fPIE -fstack-protector-strong \ --param=ssp-buffer-size=4 \ -DTCP_FASTOPEN=23 \ -fPIC \ + -I$HUNTER_INSTALL_DIR/include \ -Wno-cast-function-type" -LD_OPT="-fPIE -fPIC -pie -Wl,-z,relro -Wl,-z,now" +LD_OPT="-fPIE -fPIC -pie -Wl,-z,relro -Wl,-z,now -L$HUNTER_INSTALL_DIR/lib" if [[ ${ARCH} != "aarch64" ]]; then WITH_FLAGS+=" --with-file-aio" fi if [[ ${ARCH} == "x86_64" ]]; then - CC_OPT+=' -m64 -mtune=generic' + CC_OPT+=' -m64 -mtune=native' fi WITH_MODULES=" \ - --add-module=$BUILD_PATH/ngx_devel_kit \ - --add-module=$BUILD_PATH/set-misc-nginx-module \ - --add-module=$BUILD_PATH/headers-more-nginx-module \ - --add-module=$BUILD_PATH/lua-nginx-module \ - --add-module=$BUILD_PATH/stream-lua-nginx-module \ - --add-module=$BUILD_PATH/lua-upstream-nginx-module \ - --add-dynamic-module=$BUILD_PATH/nginx-http-auth-digest \ - --add-dynamic-module=$BUILD_PATH/ModSecurity-nginx \ - --add-dynamic-module=$BUILD_PATH/ngx_http_geoip2_module \ + --add-module=$BUILD_PATH/ngx_devel_kit-$NDK_VERSION \ + --add-module=$BUILD_PATH/set-misc-nginx-module-$SETMISC_VERSION \ + --add-module=$BUILD_PATH/headers-more-nginx-module-$MORE_HEADERS_VERSION \ + --add-module=$BUILD_PATH/ngx_http_substitutions_filter_module-$NGINX_SUBSTITUTIONS \ + --add-module=$BUILD_PATH/lua-nginx-module-$LUA_NGX_VERSION \ + --add-module=$BUILD_PATH/stream-lua-nginx-module-$LUA_STREAM_NGX_VERSION \ + --add-module=$BUILD_PATH/lua-upstream-nginx-module-$LUA_UPSTREAM_VERSION \ + --add-module=$BUILD_PATH/nginx_ajp_module-${NGINX_AJP_VERSION} \ + --add-dynamic-module=$BUILD_PATH/nginx-http-auth-digest-$NGINX_DIGEST_AUTH \ + --add-dynamic-module=$BUILD_PATH/nginx-influxdb-module-$NGINX_INFLUXDB_VERSION \ + --add-dynamic-module=$BUILD_PATH/nginx-opentracing-$NGINX_OPENTRACING_VERSION/opentracing \ + --add-dynamic-module=$BUILD_PATH/ModSecurity-nginx-$MODSECURITY_VERSION \ + --add-dynamic-module=$BUILD_PATH/ngx_http_geoip2_module-${GEOIP2_VERSION} \ --add-dynamic-module=$BUILD_PATH/ngx_brotli" ./configure \ @@ -520,81 +566,60 @@ make make modules make install -# Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp-contrib/compare/8933841f0a7f8737f61404cf0a64acf6b079c8a5...main -export OPENTELEMETRY_CONTRIB_COMMIT=8933841f0a7f8737f61404cf0a64acf6b079c8a5 -cd "$BUILD_PATH" - -git clone https://github.com/open-telemetry/opentelemetry-cpp-contrib.git opentelemetry-cpp-contrib-${OPENTELEMETRY_CONTRIB_COMMIT} - -cd ${BUILD_PATH}/opentelemetry-cpp-contrib-${OPENTELEMETRY_CONTRIB_COMMIT} -git reset --hard ${OPENTELEMETRY_CONTRIB_COMMIT} - -export OTEL_TEMP_INSTALL=/tmp/otel -mkdir -p ${OTEL_TEMP_INSTALL} - -cd ${BUILD_PATH}/opentelemetry-cpp-contrib-${OPENTELEMETRY_CONTRIB_COMMIT}/instrumentation/nginx -mkdir -p build -cd build -cmake -DCMAKE_BUILD_TYPE=Release \ - -G Ninja \ - -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_INSTALL_PREFIX=${OTEL_TEMP_INSTALL} \ - -DBUILD_SHARED_LIBS=ON \ - -DNGINX_VERSION=${NGINX_VERSION} \ - .. -cmake --build . -j ${CORES} --target install - -mkdir -p /etc/nginx/modules -cp ${OTEL_TEMP_INSTALL}/otel_ngx_module.so /etc/nginx/modules/otel_ngx_module.so - - -cd "$BUILD_PATH/lua-resty-core" +cd "$BUILD_PATH/lua-resty-core-$LUA_RESTY_CORE" make install -cd "$BUILD_PATH/lua-resty-balancer" +cd "$BUILD_PATH/lua-resty-balancer-$LUA_RESTY_BALANCER" make all make install export LUA_INCLUDE_DIR=/usr/local/include/luajit-2.1 ln -s $LUA_INCLUDE_DIR /usr/include/lua5.1 -cd "$BUILD_PATH/lua-cjson" +cd "$BUILD_PATH/lua-cjson-$LUA_CJSON_VERSION" make all make install -cd "$BUILD_PATH/lua-resty-cookie" +cd "$BUILD_PATH/lua-resty-cookie-$LUA_RESTY_COOKIE_VERSION" make all make install -cd "$BUILD_PATH/lua-resty-lrucache" +cd "$BUILD_PATH/lua-resty-lrucache-$LUA_RESTY_CACHE" make install -cd "$BUILD_PATH/lua-resty-dns" +cd "$BUILD_PATH/lua-resty-dns-$LUA_RESTY_DNS" make install -cd "$BUILD_PATH/lua-resty-lock" +cd "$BUILD_PATH/lua-resty-lock-$LUA_RESTY_LOCK" make install # required for OCSP verification -cd "$BUILD_PATH/lua-resty-http" +cd "$BUILD_PATH/lua-resty-http-$LUA_RESTY_HTTP" make install -cd "$BUILD_PATH/lua-resty-upload" +cd "$BUILD_PATH/lua-resty-upload-$LUA_RESTY_UPLOAD_VERSION" make install -cd "$BUILD_PATH/lua-resty-string" +cd "$BUILD_PATH/lua-resty-string-$LUA_RESTY_STRING_VERSION" make install -cd "$BUILD_PATH/lua-resty-memcached" +cd "$BUILD_PATH/lua-resty-memcached-$LUA_RESTY_MEMCACHED_VERSION" make install -cd "$BUILD_PATH/lua-resty-redis" +cd "$BUILD_PATH/lua-resty-redis-$LUA_RESTY_REDIS_VERSION" make install -cd "$BUILD_PATH/lua-resty-ipmatcher" +cd "$BUILD_PATH/lua-resty-ipmatcher-$LUA_RESTY_IPMATCHER_VERSION" INST_LUADIR=/usr/local/lib/lua make install -cd "$BUILD_PATH/mimalloc" +cd "$BUILD_PATH/lua-resty-global-throttle-$LUA_RESTY_GLOBAL_THROTTLE_VERSION" +make install + +# mimalloc +cd "$BUILD_PATH" +git clone --depth=1 -b v1.6.7 https://github.com/microsoft/mimalloc +cd mimalloc + mkdir -p out/release cd out/release @@ -614,15 +639,16 @@ writeDirs=( \ /var/log/nginx \ ); +addgroup -Sg 101 www-data adduser -S -D -H -u 101 -h /usr/local/nginx -s /sbin/nologin -G www-data -g www-data www-data for dir in "${writeDirs[@]}"; do mkdir -p ${dir}; - chown -R www-data:www-data ${dir}; + chown -R www-data.www-data ${dir}; done rm -rf /etc/nginx/owasp-modsecurity-crs/.git -rm -rf /etc/nginx/owasp-modsecurity-crs/tests +rm -rf /etc/nginx/owasp-modsecurity-crs/util/regression-tests # remove .a files find /usr/local -name "*.a" -print | xargs /bin/rm diff --git a/images/nginx/rootfs/etc/nginx/geoip/GeoIP.dat b/images/nginx/rootfs/etc/nginx/geoip/GeoIP.dat new file mode 100644 index 000000000..be8b031f7 Binary files /dev/null and b/images/nginx/rootfs/etc/nginx/geoip/GeoIP.dat differ diff --git a/images/nginx/rootfs/etc/nginx/geoip/GeoIPASNum.dat b/images/nginx/rootfs/etc/nginx/geoip/GeoIPASNum.dat new file mode 100644 index 000000000..85c2cb329 Binary files /dev/null and b/images/nginx/rootfs/etc/nginx/geoip/GeoIPASNum.dat differ diff --git a/images/nginx/rootfs/etc/nginx/geoip/GeoLiteCity.dat b/images/nginx/rootfs/etc/nginx/geoip/GeoLiteCity.dat new file mode 100644 index 000000000..1adb8c3c4 Binary files /dev/null and b/images/nginx/rootfs/etc/nginx/geoip/GeoLiteCity.dat differ diff --git a/images/nginx/rootfs/patches/00_drop-alias-root.patch b/images/nginx/rootfs/patches/00_drop-alias-root.patch deleted file mode 100644 index a92e08bd0..000000000 --- a/images/nginx/rootfs/patches/00_drop-alias-root.patch +++ /dev/null @@ -1,144 +0,0 @@ -:100644 100644 c7463dcd 00000000 M src/http/ngx_http_core_module.c -diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c -index c7463dcd..e2e45931 100644 ---- a/src/http/ngx_http_core_module.c -+++ b/src/http/ngx_http_core_module.c -@@ -55,7 +55,6 @@ static char *ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, - void *conf); - static char *ngx_http_core_server_name(ngx_conf_t *cf, ngx_command_t *cmd, - void *conf); --static char *ngx_http_core_root(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); - static char *ngx_http_core_limit_except(ngx_conf_t *cf, ngx_command_t *cmd, - void *conf); - static char *ngx_http_core_set_aio(ngx_conf_t *cf, ngx_command_t *cmd, -@@ -323,21 +322,6 @@ static ngx_command_t ngx_http_core_commands[] = { - offsetof(ngx_http_core_loc_conf_t, default_type), - NULL }, - -- { ngx_string("root"), -- NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF -- |NGX_CONF_TAKE1, -- ngx_http_core_root, -- NGX_HTTP_LOC_CONF_OFFSET, -- 0, -- NULL }, -- -- { ngx_string("alias"), -- NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, -- ngx_http_core_root, -- NGX_HTTP_LOC_CONF_OFFSET, -- 0, -- NULL }, -- - { ngx_string("limit_except"), - NGX_HTTP_LOC_CONF|NGX_CONF_BLOCK|NGX_CONF_1MORE, - ngx_http_core_limit_except, -@@ -4312,108 +4296,6 @@ ngx_http_core_server_name(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) - } - - --static char * --ngx_http_core_root(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) --{ -- ngx_http_core_loc_conf_t *clcf = conf; -- -- ngx_str_t *value; -- ngx_int_t alias; -- ngx_uint_t n; -- ngx_http_script_compile_t sc; -- -- alias = (cmd->name.len == sizeof("alias") - 1) ? 1 : 0; -- -- if (clcf->root.data) { -- -- if ((clcf->alias != 0) == alias) { -- return "is duplicate"; -- } -- -- ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, -- "\"%V\" directive is duplicate, " -- "\"%s\" directive was specified earlier", -- &cmd->name, clcf->alias ? "alias" : "root"); -- -- return NGX_CONF_ERROR; -- } -- -- if (clcf->named && alias) { -- ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, -- "the \"alias\" directive cannot be used " -- "inside the named location"); -- -- return NGX_CONF_ERROR; -- } -- -- value = cf->args->elts; -- -- if (ngx_strstr(value[1].data, "$document_root") -- || ngx_strstr(value[1].data, "${document_root}")) -- { -- ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, -- "the $document_root variable cannot be used " -- "in the \"%V\" directive", -- &cmd->name); -- -- return NGX_CONF_ERROR; -- } -- -- if (ngx_strstr(value[1].data, "$realpath_root") -- || ngx_strstr(value[1].data, "${realpath_root}")) -- { -- ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, -- "the $realpath_root variable cannot be used " -- "in the \"%V\" directive", -- &cmd->name); -- -- return NGX_CONF_ERROR; -- } -- -- clcf->alias = alias ? clcf->name.len : 0; -- clcf->root = value[1]; -- -- if (!alias && clcf->root.len > 0 -- && clcf->root.data[clcf->root.len - 1] == '/') -- { -- clcf->root.len--; -- } -- -- if (clcf->root.data[0] != '$') { -- if (ngx_conf_full_name(cf->cycle, &clcf->root, 0) != NGX_OK) { -- return NGX_CONF_ERROR; -- } -- } -- -- n = ngx_http_script_variables_count(&clcf->root); -- -- ngx_memzero(&sc, sizeof(ngx_http_script_compile_t)); -- sc.variables = n; -- --#if (NGX_PCRE) -- if (alias && clcf->regex) { -- clcf->alias = NGX_MAX_SIZE_T_VALUE; -- n = 1; -- } --#endif -- -- if (n) { -- sc.cf = cf; -- sc.source = &clcf->root; -- sc.lengths = &clcf->root_lengths; -- sc.values = &clcf->root_values; -- sc.complete_lengths = 1; -- sc.complete_values = 1; -- -- if (ngx_http_script_compile(&sc) != NGX_OK) { -- return NGX_CONF_ERROR; -- } -- } -- -- return NGX_CONF_OK; --} -- -- - static ngx_http_method_name_t ngx_methods_names[] = { - { (u_char *) "GET", (uint32_t) ~NGX_HTTP_GET }, - { (u_char *) "HEAD", (uint32_t) ~NGX_HTTP_HEAD }, diff --git a/images/nginx/rootfs/patches/01_nginx-1.27.1-win32_max_err_str.patch b/images/nginx/rootfs/patches/01_nginx-1.27.1-win32_max_err_str.patch deleted file mode 100644 index 8c3ba2791..000000000 --- a/images/nginx/rootfs/patches/01_nginx-1.27.1-win32_max_err_str.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/src/os/win32/ngx_event_log.c b/src/os/win32/ngx_event_log.c -index e11ed1e8..dce8eddd 100644 ---- a/src/os/win32/ngx_event_log.c -+++ b/src/os/win32/ngx_event_log.c -@@ -8,7 +8,9 @@ - #include - - --#define NGX_MAX_ERROR_STR 2048 -+#ifndef NGX_MAX_ERROR_STR -+#define NGX_MAX_ERROR_STR 4096 -+#endif - - - void ngx_cdecl diff --git a/images/nginx/rootfs/patches/02_nginx-1.27.1-stream_balancer_export.patch b/images/nginx/rootfs/patches/02_nginx-1.27.1-stream_balancer_export.patch deleted file mode 100644 index f56bc5257..000000000 --- a/images/nginx/rootfs/patches/02_nginx-1.27.1-stream_balancer_export.patch +++ /dev/null @@ -1,53 +0,0 @@ -diff --git a/src/stream/ngx_stream_upstream_round_robin.c b/src/stream/ngx_stream_upstream_round_robin.c -index 526de3a..b531ce1 100644 ---- a/src/stream/ngx_stream_upstream_round_robin.c -+++ b/src/stream/ngx_stream_upstream_round_robin.c -@@ -21,10 +21,6 @@ static void ngx_stream_upstream_notify_round_robin_peer( - - #if (NGX_STREAM_SSL) - --static ngx_int_t ngx_stream_upstream_set_round_robin_peer_session( -- ngx_peer_connection_t *pc, void *data); --static void ngx_stream_upstream_save_round_robin_peer_session( -- ngx_peer_connection_t *pc, void *data); - static ngx_int_t ngx_stream_upstream_empty_set_session( - ngx_peer_connection_t *pc, void *data); - static void ngx_stream_upstream_empty_save_session(ngx_peer_connection_t *pc, -@@ -690,7 +686,7 @@ ngx_stream_upstream_notify_round_robin_peer(ngx_peer_connection_t *pc, - - #if (NGX_STREAM_SSL) - --static ngx_int_t -+ngx_int_t - ngx_stream_upstream_set_round_robin_peer_session(ngx_peer_connection_t *pc, - void *data) - { -@@ -756,7 +752,7 @@ ngx_stream_upstream_set_round_robin_peer_session(ngx_peer_connection_t *pc, - } - - --static void -+void - ngx_stream_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc, - void *data) - { -diff --git a/src/stream/ngx_stream_upstream_round_robin.h b/src/stream/ngx_stream_upstream_round_robin.h -index 35d9fce..75f3e31 100644 ---- a/src/stream/ngx_stream_upstream_round_robin.h -+++ b/src/stream/ngx_stream_upstream_round_robin.h -@@ -142,5 +142,15 @@ ngx_int_t ngx_stream_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, - void ngx_stream_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, - void *data, ngx_uint_t state); - -+#if (NGX_STREAM_SSL) -+ngx_int_t ngx_stream_upstream_set_round_robin_peer_session( -+ ngx_peer_connection_t *pc, void *data); -+void ngx_stream_upstream_save_round_robin_peer_session( -+ ngx_peer_connection_t *pc, void *data); -+#endif -+ -+ -+#define HAVE_NGX_STREAM_BALANCER_EXPORT_PATCH 1 -+ - - #endif /* _NGX_STREAM_UPSTREAM_ROUND_ROBIN_H_INCLUDED_ */ diff --git a/images/nginx/rootfs/patches/04_nginx-1.27.1-stream_proxy_timeout_fields.patch b/images/nginx/rootfs/patches/04_nginx-1.27.1-stream_proxy_timeout_fields.patch deleted file mode 100644 index e205abb8b..000000000 --- a/images/nginx/rootfs/patches/04_nginx-1.27.1-stream_proxy_timeout_fields.patch +++ /dev/null @@ -1,178 +0,0 @@ -diff -u -r -p -Naur nginx-1.27.1/src/stream/ngx_stream.h nginx-1.27.1-patched/src/stream/ngx_stream.h ---- nginx-1.27.1/src/stream/ngx_stream.h 2021-11-04 21:27:55.288708527 +0800 -+++ nginx-1.27.1-patched/src/stream/ngx_stream.h 2021-11-04 21:28:50.768035209 +0800 -@@ -254,6 +254,15 @@ typedef struct { - } ngx_stream_module_t; - - -+typedef struct { -+ ngx_msec_t connect_timeout; -+ ngx_msec_t timeout; -+} ngx_stream_proxy_ctx_t; -+ -+ -+#define NGX_STREAM_HAVE_PROXY_TIMEOUT_FIELDS_PATCH 1 -+ -+ - #define NGX_STREAM_MODULE 0x4d525453 /* "STRM" */ - - #define NGX_STREAM_MAIN_CONF 0x02000000 -@@ -307,6 +316,7 @@ void ngx_stream_finalize_session(ngx_str - extern ngx_module_t ngx_stream_module; - extern ngx_uint_t ngx_stream_max_module; - extern ngx_module_t ngx_stream_core_module; -+extern ngx_module_t ngx_stream_proxy_module; - - - typedef ngx_int_t (*ngx_stream_filter_pt)(ngx_stream_session_t *s, -diff -u -r -p -Naur nginx-1.27.1/src/stream/ngx_stream_proxy_module.c nginx-1.27.1-patched/src/stream/ngx_stream_proxy_module.c ---- nginx-1.27.1/src/stream/ngx_stream_proxy_module.c 2021-11-04 21:27:55.289708533 +0800 -+++ nginx-1.27.1-patched/src/stream/ngx_stream_proxy_module.c 2021-11-04 21:37:03.578936990 +0800 -@@ -400,6 +400,7 @@ ngx_stream_proxy_handler(ngx_stream_sess - ngx_stream_proxy_srv_conf_t *pscf; - ngx_stream_upstream_srv_conf_t *uscf, **uscfp; - ngx_stream_upstream_main_conf_t *umcf; -+ ngx_stream_proxy_ctx_t *pctx; - - c = s->connection; - -@@ -408,6 +409,17 @@ ngx_stream_proxy_handler(ngx_stream_sess - ngx_log_debug0(NGX_LOG_DEBUG_STREAM, c->log, 0, - "proxy connection handler"); - -+ pctx = ngx_palloc(c->pool, sizeof(ngx_stream_proxy_ctx_t)); -+ if (pctx == NULL) { -+ ngx_stream_proxy_finalize(s, NGX_STREAM_INTERNAL_SERVER_ERROR); -+ return; -+ } -+ -+ pctx->connect_timeout = pscf->connect_timeout; -+ pctx->timeout = pscf->timeout; -+ -+ ngx_stream_set_ctx(s, pctx, ngx_stream_proxy_module); -+ - u = ngx_pcalloc(c->pool, sizeof(ngx_stream_upstream_t)); - if (u == NULL) { - ngx_stream_proxy_finalize(s, NGX_STREAM_INTERNAL_SERVER_ERROR); -@@ -699,6 +711,7 @@ ngx_stream_proxy_connect(ngx_stream_sess - ngx_connection_t *c, *pc; - ngx_stream_upstream_t *u; - ngx_stream_proxy_srv_conf_t *pscf; -+ ngx_stream_proxy_ctx_t *ctx; - - c = s->connection; - -@@ -706,6 +719,8 @@ ngx_stream_proxy_connect(ngx_stream_sess - - pscf = ngx_stream_get_module_srv_conf(s, ngx_stream_proxy_module); - -+ ctx = ngx_stream_get_module_ctx(s, ngx_stream_proxy_module); -+ - u = s->upstream; - - u->connected = 0; -@@ -774,7 +789,7 @@ ngx_stream_proxy_connect(ngx_stream_sess - pc->read->handler = ngx_stream_proxy_connect_handler; - pc->write->handler = ngx_stream_proxy_connect_handler; - -- ngx_add_timer(pc->write, pscf->connect_timeout); -+ ngx_add_timer(pc->write, ctx->connect_timeout); - } - - -@@ -957,12 +957,14 @@ ngx_stream_proxy_init_upstream(ngx_stream_session_t *s) - static ngx_int_t - ngx_stream_proxy_send_proxy_protocol(ngx_stream_session_t *s) - { -- u_char *p; -- ssize_t n, size; -- ngx_connection_t *c, *pc; -- ngx_stream_upstream_t *u; -- ngx_stream_proxy_srv_conf_t *pscf; -- u_char buf[NGX_PROXY_PROTOCOL_V1_MAX_HEADER]; -+ u_char *p; -+ u_char buf[NGX_PROXY_PROTOCOL_V1_MAX_HEADER]; -+ ssize_t n, size; -+ ngx_connection_t *c, *pc; -+ ngx_stream_upstream_t *u; -+ ngx_stream_proxy_ctx_t *ctx; -+ -+ ctx = ngx_stream_get_module_ctx(s, ngx_stream_proxy_module); - - c = s->connection; - -@@ -976,9 +993,7 @@ ngx_stream_proxy_send_proxy_protocol(ngx - return NGX_ERROR; - } - -- pscf = ngx_stream_get_module_srv_conf(s, ngx_stream_proxy_module); -- -- ngx_add_timer(pc->write, pscf->timeout); -+ ngx_add_timer(pc->write, ctx->timeout); - - pc->write->handler = ngx_stream_proxy_connect_handler; - -@@ -1053,6 +1068,9 @@ ngx_stream_proxy_ssl_init_connection(ngx - ngx_connection_t *pc; - ngx_stream_upstream_t *u; - ngx_stream_proxy_srv_conf_t *pscf; -+ ngx_stream_proxy_ctx_t *ctx; -+ -+ ctx = ngx_stream_get_module_ctx(s, ngx_stream_proxy_module); - - u = s->upstream; - -@@ -1099,7 +1117,7 @@ ngx_stream_proxy_ssl_init_connection(ngx - if (rc == NGX_AGAIN) { - - if (!pc->write->timer_set) { -- ngx_add_timer(pc->write, pscf->connect_timeout); -+ ngx_add_timer(pc->write, ctx->connect_timeout); - } - - pc->ssl->handler = ngx_stream_proxy_ssl_handshake; -@@ -1408,6 +1426,7 @@ ngx_stream_proxy_process_connection(ngx_ - ngx_stream_session_t *s; - ngx_stream_upstream_t *u; - ngx_stream_proxy_srv_conf_t *pscf; -+ ngx_stream_proxy_ctx_t *ctx; - - c = ev->data; - s = c->data; -@@ -1419,6 +1438,8 @@ ngx_stream_proxy_process_connection(ngx_ - return; - } - -+ ctx = ngx_stream_get_module_ctx(s, ngx_stream_proxy_module); -+ - c = s->connection; - pc = u->peer.connection; - -@@ -1438,7 +1459,7 @@ ngx_stream_proxy_process_connection(ngx_ - } - - if (u->connected && !c->read->delayed && !pc->read->delayed) { -- ngx_add_timer(c->write, pscf->timeout); -+ ngx_add_timer(c->write, ctx->timeout); - } - - return; -@@ -1600,6 +1621,9 @@ ngx_stream_proxy_process(ngx_stream_sess - ngx_log_handler_pt handler; - ngx_stream_upstream_t *u; - ngx_stream_proxy_srv_conf_t *pscf; -+ ngx_stream_proxy_ctx_t *ctx; -+ -+ ctx = ngx_stream_get_module_ctx(s, ngx_stream_proxy_module); - - u = s->upstream; - -@@ -1807,7 +1831,7 @@ ngx_stream_proxy_process(ngx_stream_sess - } - - if (!c->read->delayed && !pc->read->delayed) { -- ngx_add_timer(c->write, pscf->timeout); -+ ngx_add_timer(c->write, ctx->timeout); - - } else if (c->write->timer_set) { - ngx_del_timer(c->write); diff --git a/images/nginx/rootfs/patches/07_nginx-1.27.1-daemon_destroy_pool.patch b/images/nginx/rootfs/patches/07_nginx-1.27.1-daemon_destroy_pool.patch deleted file mode 100644 index 5690b88f0..000000000 --- a/images/nginx/rootfs/patches/07_nginx-1.27.1-daemon_destroy_pool.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/src/os/unix/ngx_daemon.c b/src/os/unix/ngx_daemon.c -index ab672110..f259af31 100644 ---- a/src/os/unix/ngx_daemon.c -+++ b/src/os/unix/ngx_daemon.c -@@ -23,6 +23,8 @@ ngx_daemon(ngx_log_t *log) - break; - - default: -+ /* just to make it ASAN or Valgrind clean */ -+ ngx_destroy_pool(ngx_cycle->pool); - exit(0); - } diff --git a/images/nginx/rootfs/patches/11_nginx-1.27.1-privileged_agent_process.patch b/images/nginx/rootfs/patches/11_nginx-1.27.1-privileged_agent_process.patch deleted file mode 100644 index 164004eba..000000000 --- a/images/nginx/rootfs/patches/11_nginx-1.27.1-privileged_agent_process.patch +++ /dev/null @@ -1,203 +0,0 @@ -diff --git a/src/core/nginx.c b/src/core/nginx.c -index 60f8fe7..4bd244b 100644 ---- a/src/core/nginx.c -+++ b/src/core/nginx.c -@@ -981,6 +981,7 @@ ngx_core_module_create_conf(ngx_cycle_t *cycle) - - ccf->daemon = NGX_CONF_UNSET; - ccf->master = NGX_CONF_UNSET; -+ ccf->privileged_agent = NGX_CONF_UNSET; - ccf->timer_resolution = NGX_CONF_UNSET_MSEC; - - ccf->worker_processes = NGX_CONF_UNSET; -@@ -1009,6 +1010,7 @@ ngx_core_module_init_conf(ngx_cycle_t *cycle, void *conf) - - ngx_conf_init_value(ccf->daemon, 1); - ngx_conf_init_value(ccf->master, 1); -+ ngx_conf_init_value(ccf->privileged_agent, 0); - ngx_conf_init_msec_value(ccf->timer_resolution, 0); - - ngx_conf_init_value(ccf->worker_processes, 1); -diff --git a/src/core/ngx_cycle.h b/src/core/ngx_cycle.h -index c51b7ff..3261f90 100644 ---- a/src/core/ngx_cycle.h -+++ b/src/core/ngx_cycle.h -@@ -22,6 +22,9 @@ - #define NGX_DEBUG_POINTS_ABORT 2 - - -+#define HAVE_PRIVILEGED_PROCESS_PATCH 1 -+ -+ - typedef struct ngx_shm_zone_s ngx_shm_zone_t; - - typedef ngx_int_t (*ngx_shm_zone_init_pt) (ngx_shm_zone_t *zone, void *data); -@@ -81,6 +84,7 @@ struct ngx_cycle_s { - typedef struct { - ngx_flag_t daemon; - ngx_flag_t master; -+ ngx_flag_t privileged_agent; - - ngx_msec_t timer_resolution; - -diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c -index 7cee1c5..c4f70d6 100644 ---- a/src/os/unix/ngx_process_cycle.c -+++ b/src/os/unix/ngx_process_cycle.c -@@ -15,6 +15,8 @@ static void ngx_start_worker_processes(ngx_cycle_t *cycle, ngx_int_t n, - ngx_int_t type); - static void ngx_start_cache_manager_processes(ngx_cycle_t *cycle, - ngx_uint_t respawn); -+static void ngx_start_privileged_agent_processes(ngx_cycle_t *cycle, -+ ngx_uint_t respawn); - static void ngx_pass_open_channel(ngx_cycle_t *cycle); - static void ngx_signal_worker_processes(ngx_cycle_t *cycle, int signo); - static ngx_uint_t ngx_reap_children(ngx_cycle_t *cycle); -@@ -24,6 +26,7 @@ static void ngx_worker_process_init(ngx_cycle_t *cycle, ngx_int_t worker); - static void ngx_worker_process_exit(ngx_cycle_t *cycle); - static void ngx_channel_handler(ngx_event_t *ev); - static void ngx_cache_manager_process_cycle(ngx_cycle_t *cycle, void *data); -+static void ngx_privileged_agent_process_cycle(ngx_cycle_t *cycle, void *data); - static void ngx_cache_manager_process_handler(ngx_event_t *ev); - static void ngx_cache_loader_process_handler(ngx_event_t *ev); - -@@ -51,6 +54,8 @@ sig_atomic_t ngx_noaccept; - ngx_uint_t ngx_noaccepting; - ngx_uint_t ngx_restart; - -+ngx_uint_t ngx_is_privileged_agent; -+ - - static u_char master_process[] = "master process"; - -@@ -130,6 +135,7 @@ ngx_master_process_cycle(ngx_cycle_t *cycle) - ngx_start_worker_processes(cycle, ccf->worker_processes, - NGX_PROCESS_RESPAWN); - ngx_start_cache_manager_processes(cycle, 0); -+ ngx_start_privileged_agent_processes(cycle, 0); - - ngx_new_binary = 0; - delay = 0; -@@ -215,6 +221,7 @@ ngx_master_process_cycle(ngx_cycle_t *cycle) - ngx_start_worker_processes(cycle, ccf->worker_processes, - NGX_PROCESS_RESPAWN); - ngx_start_cache_manager_processes(cycle, 0); -+ ngx_start_privileged_agent_processes(cycle, 0); - ngx_noaccepting = 0; - - continue; -@@ -234,6 +241,7 @@ ngx_master_process_cycle(ngx_cycle_t *cycle) - ngx_start_worker_processes(cycle, ccf->worker_processes, - NGX_PROCESS_JUST_RESPAWN); - ngx_start_cache_manager_processes(cycle, 1); -+ ngx_start_privileged_agent_processes(cycle, 1); - - /* allow new processes to start */ - ngx_msleep(100); -@@ -248,6 +256,7 @@ ngx_master_process_cycle(ngx_cycle_t *cycle) - ngx_start_worker_processes(cycle, ccf->worker_processes, - NGX_PROCESS_RESPAWN); - ngx_start_cache_manager_processes(cycle, 0); -+ ngx_start_privileged_agent_processes(cycle, 0); - live = 1; - } - -@@ -393,6 +431,26 @@ ngx_start_cache_manager_processes(ngx_cycle_t *cycle, ngx_uint_t respawn) - - - static void -+ngx_start_privileged_agent_processes(ngx_cycle_t *cycle, ngx_uint_t respawn) -+{ -+ ngx_core_conf_t *ccf; -+ -+ ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, -+ ngx_core_module); -+ -+ if (!ccf->privileged_agent) { -+ return; -+ } -+ -+ ngx_spawn_process(cycle, ngx_privileged_agent_process_cycle, -+ "privileged agent process", "privileged agent process", -+ respawn ? NGX_PROCESS_JUST_RESPAWN : NGX_PROCESS_RESPAWN); -+ -+ ngx_pass_open_channel(cycle); -+} -+ -+ -+static void - ngx_pass_open_channel(ngx_cycle_t *cycle) - { - ngx_int_t i; -@@ -794,7 +860,10 @@ ngx_worker_process_init(ngx_cycle_t *cycle, ngx_int_t worker) - } - } - -- if (geteuid() == 0) { -+ /* -+ * privileged agent process has the same permission as master process -+ */ -+ if (!ngx_is_privileged_agent && geteuid() == 0) { - if (setgid(ccf->group) == -1) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, - "setgid(%d) failed", ccf->group); -@@ -1149,6 +1216,47 @@ ngx_cache_manager_process_cycle(ngx_cycle_t *cycle, void *data) - - - static void -+ngx_privileged_agent_process_cycle(ngx_cycle_t *cycle, void *data) -+{ -+ char *name = data; -+ -+ /* -+ * Set correct process type since closing listening Unix domain socket -+ * in a master process also removes the Unix domain socket file. -+ */ -+ ngx_process = NGX_PROCESS_HELPER; -+ ngx_is_privileged_agent = 1; -+ -+ ngx_close_listening_sockets(cycle); -+ -+ /* Set a moderate number of connections for a helper process. */ -+ cycle->connection_n = 512; -+ -+ ngx_worker_process_init(cycle, -1); -+ -+ ngx_use_accept_mutex = 0; -+ -+ ngx_setproctitle(name); -+ -+ for ( ;; ) { -+ -+ if (ngx_terminate || ngx_quit) { -+ ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "exiting"); -+ ngx_worker_process_exit(cycle); -+ } -+ -+ if (ngx_reopen) { -+ ngx_reopen = 0; -+ ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "reopening logs"); -+ ngx_reopen_files(cycle, -1); -+ } -+ -+ ngx_process_events_and_timers(cycle); -+ } -+} -+ -+ -+static void - ngx_cache_manager_process_handler(ngx_event_t *ev) - { - time_t next, n; -diff --git a/src/os/unix/ngx_process_cycle.h b/src/os/unix/ngx_process_cycle.h -index 69495d5..5149396 100644 ---- a/src/os/unix/ngx_process_cycle.h -+++ b/src/os/unix/ngx_process_cycle.h -@@ -45,6 +45,7 @@ extern ngx_pid_t ngx_new_binary; - extern ngx_uint_t ngx_inherited; - extern ngx_uint_t ngx_daemonized; - extern ngx_uint_t ngx_exiting; -+extern ngx_uint_t ngx_is_privileged_agent; - - extern sig_atomic_t ngx_reap; - extern sig_atomic_t ngx_sigio; diff --git a/images/nginx/rootfs/patches/12_nginx-1.27.1-privileged_agent_process_connections.patch b/images/nginx/rootfs/patches/12_nginx-1.27.1-privileged_agent_process_connections.patch deleted file mode 100644 index 5c38929cf..000000000 --- a/images/nginx/rootfs/patches/12_nginx-1.27.1-privileged_agent_process_connections.patch +++ /dev/null @@ -1,73 +0,0 @@ -diff --git a/src/core/nginx.c b/src/core/nginx.c -index 269ff84..48329bd 100644 ---- a/src/core/nginx.c -+++ b/src/core/nginx.c -@@ -1062,6 +1062,7 @@ ngx_core_module_create_conf(ngx_cycle_t *cycle) - ccf->daemon = NGX_CONF_UNSET; - ccf->master = NGX_CONF_UNSET; - ccf->privileged_agent = NGX_CONF_UNSET; -+ ccf->privileged_agent_connections = NGX_CONF_UNSET_UINT; - ccf->timer_resolution = NGX_CONF_UNSET_MSEC; - ccf->shutdown_timeout = NGX_CONF_UNSET_MSEC; - -@@ -1092,6 +1093,7 @@ ngx_core_module_init_conf(ngx_cycle_t *cycle, void *conf) - ngx_conf_init_value(ccf->daemon, 1); - ngx_conf_init_value(ccf->master, 1); - ngx_conf_init_value(ccf->privileged_agent, 0); -+ ngx_conf_init_uint_value(ccf->privileged_agent_connections, 512); - ngx_conf_init_msec_value(ccf->timer_resolution, 0); - ngx_conf_init_msec_value(ccf->shutdown_timeout, 0); - -diff --git a/src/core/ngx_cycle.h b/src/core/ngx_cycle.h -index 6a9583e..4469390 100644 ---- a/src/core/ngx_cycle.h -+++ b/src/core/ngx_cycle.h -@@ -93,6 +93,7 @@ typedef struct { - ngx_flag_t daemon; - ngx_flag_t master; - ngx_flag_t privileged_agent; -+ ngx_uint_t privileged_agent_connections; - - ngx_msec_t timer_resolution; - ngx_msec_t shutdown_timeout; -diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c -index df25f9d..bd259c1 100644 ---- a/src/os/unix/ngx_process_cycle.c -+++ b/src/os/unix/ngx_process_cycle.c -@@ -1179,6 +1179,7 @@ static void - ngx_privileged_agent_process_cycle(ngx_cycle_t *cycle, void *data) - { - char *name = data; -+ ngx_core_conf_t *ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module); - - /* - * Set correct process type since closing listening Unix domain socket -@@ -1190,7 +1191,7 @@ ngx_privileged_agent_process_cycle(ngx_cycle_t *cycle, void *data) - ngx_close_listening_sockets(cycle); - - /* Set a moderate number of connections for a helper process. */ -- cycle->connection_n = 512; -+ cycle->connection_n = ccf->privileged_agent_connections; - - ngx_worker_process_init(cycle, -1); - -diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c -index df25f9d..bd259c1 100644 ---- a/src/os/unix/ngx_process_cycle.c -+++ b/src/os/unix/ngx_process_cycle.c -@@ -442,6 +442,15 @@ - return; - } - -+ /* 0 is an illegal value and may result in a core dump later */ -+ if (ccf->privileged_agent_connections == 0) { -+ ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, -+ "%ui worker_connection is not enough, " -+ "privileged agent process cannot be spawned", -+ ccf->privileged_agent_connections); -+ return; -+ } -+ - ngx_spawn_process(cycle, ngx_privileged_agent_process_cycle, - "privileged agent process", "privileged agent process", - respawn ? NGX_PROCESS_JUST_RESPAWN : NGX_PROCESS_RESPAWN); diff --git a/images/nginx/rootfs/patches/13_nginx-1.27.1-privileged_agent_process_thread_pool.patch b/images/nginx/rootfs/patches/13_nginx-1.27.1-privileged_agent_process_thread_pool.patch deleted file mode 100644 index 829f21460..000000000 --- a/images/nginx/rootfs/patches/13_nginx-1.27.1-privileged_agent_process_thread_pool.patch +++ /dev/null @@ -1,12 +0,0 @@ ---- a/src/core/ngx_thread_pool.c -+++ b/src/core/ngx_thread_pool.c -@@ -587,7 +587,8 @@ - ngx_thread_pool_conf_t *tcf; - - if (ngx_process != NGX_PROCESS_WORKER -- && ngx_process != NGX_PROCESS_SINGLE) -+ && ngx_process != NGX_PROCESS_SINGLE -+ && !ngx_is_privileged_agent) - { - return NGX_OK; - } diff --git a/images/nginx/rootfs/patches/15_nginx-1.27.1-intercept_error_log.patch b/images/nginx/rootfs/patches/15_nginx-1.27.1-intercept_error_log.patch deleted file mode 100644 index 5de769517..000000000 --- a/images/nginx/rootfs/patches/15_nginx-1.27.1-intercept_error_log.patch +++ /dev/null @@ -1,60 +0,0 @@ -diff --git a/src/core/ngx_cycle.h b/src/core/ngx_cycle.h -index c51b7ff..4c335b9 100644 ---- a/src/core/ngx_cycle.h -+++ b/src/core/ngx_cycle.h -@@ -22,9 +22,14 @@ - #define NGX_DEBUG_POINTS_ABORT 2 - - -+#define HAVE_INTERCEPT_ERROR_LOG_PATCH -+ -+ - typedef struct ngx_shm_zone_s ngx_shm_zone_t; - - typedef ngx_int_t (*ngx_shm_zone_init_pt) (ngx_shm_zone_t *zone, void *data); -+typedef ngx_int_t (*ngx_log_intercept_pt) (ngx_log_t *log, ngx_uint_t level, -+ u_char *buf, size_t len); - - struct ngx_shm_zone_s { - void *data; -@@ -75,6 +80,10 @@ struct ngx_cycle_s { - ngx_str_t prefix; - ngx_str_t lock_file; - ngx_str_t hostname; -+ -+ ngx_log_intercept_pt intercept_error_log_handler; -+ void *intercept_error_log_data; -+ unsigned entered_logger; /* :1 */ - }; - - -diff --git a/src/core/ngx_log.c b/src/core/ngx_log.c -index 8e9408d..ed9b11b 100644 ---- a/src/core/ngx_log.c -+++ b/src/core/ngx_log.c -@@ -112,6 +112,8 @@ ngx_log_error_core(ngx_uint_t level, ngx_log_t *log, ngx_err_t err, - ngx_uint_t wrote_stderr, debug_connection; - u_char errstr[NGX_MAX_ERROR_STR]; - -+ ngx_log_intercept_pt log_intercept = NULL; -+ - last = errstr + NGX_MAX_ERROR_STR; - - p = ngx_cpymem(errstr, ngx_cached_err_log_time.data, -@@ -153,6 +155,16 @@ ngx_log_error_core(ngx_uint_t level, ngx_log_t *log, ngx_err_t err, - p = last - NGX_LINEFEED_SIZE; - } - -+ if (ngx_cycle) { -+ log_intercept = ngx_cycle->intercept_error_log_handler; -+ } -+ -+ if (log_intercept && !ngx_cycle->entered_logger) { -+ ngx_cycle->entered_logger = 1; -+ log_intercept(log, level, errstr, p - errstr); -+ ngx_cycle->entered_logger = 0; -+ } -+ - ngx_linefeed(p); - - wrote_stderr = 0; diff --git a/images/nginx/rootfs/patches/17_nginx-1.27.1-no_error_pages.patch b/images/nginx/rootfs/patches/17_nginx-1.27.1-no_error_pages.patch deleted file mode 100644 index 593fcefd6..000000000 --- a/images/nginx/rootfs/patches/17_nginx-1.27.1-no_error_pages.patch +++ /dev/null @@ -1,91 +0,0 @@ -diff -upr nginx-1.27.1/src/http/ngx_http_core_module.c nginx-1.27.1-patched/src/http/ngx_http_core_module.c ---- nginx-1.27.1/src/http/ngx_http_core_module.c 2017-08-31 18:14:41.000000000 -0700 -+++ nginx-1.27.1-patched/src/http/ngx_http_core_module.c 2017-08-31 18:21:31.638098196 -0700 -@@ -64,6 +64,8 @@ static char *ngx_http_core_directio(ngx_conf_t *cf, ngx_command_t *cmd, - void *conf); - static char *ngx_http_core_error_page(ngx_conf_t *cf, ngx_command_t *cmd, - void *conf); -+static char *ngx_http_core_no_error_pages(ngx_conf_t *cf, ngx_command_t *cmd, -+ void *conf); - static char *ngx_http_core_open_file_cache(ngx_conf_t *cf, ngx_command_t *cmd, - void *conf); - static char *ngx_http_core_error_log(ngx_conf_t *cf, ngx_command_t *cmd, -@@ -671,6 +673,14 @@ static ngx_command_t ngx_http_core_commands[] = { - 0, - NULL }, - -+ { ngx_string("no_error_pages"), -+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF -+ |NGX_CONF_NOARGS, -+ ngx_http_core_no_error_pages, -+ NGX_HTTP_LOC_CONF_OFFSET, -+ 0, -+ NULL }, -+ - { ngx_string("post_action"), - NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF - |NGX_CONF_TAKE1, -@@ -3564,7 +3574,6 @@ ngx_http_core_create_loc_conf(ngx_conf_t *cf) - * clcf->types = NULL; - * clcf->default_type = { 0, NULL }; - * clcf->error_log = NULL; -- * clcf->error_pages = NULL; - * clcf->client_body_path = NULL; - * clcf->regex = NULL; - * clcf->exact_match = 0; -@@ -3574,6 +3583,7 @@ ngx_http_core_create_loc_conf(ngx_conf_t *cf) - * clcf->keepalive_disable = 0; - */ - -+ clcf->error_pages = NGX_CONF_UNSET_PTR; - clcf->client_max_body_size = NGX_CONF_UNSET; - clcf->client_body_buffer_size = NGX_CONF_UNSET_SIZE; - clcf->client_body_timeout = NGX_CONF_UNSET_MSEC; -@@ -3776,9 +3786,7 @@ ngx_http_core_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) - } - } - -- if (conf->error_pages == NULL && prev->error_pages) { -- conf->error_pages = prev->error_pages; -- } -+ ngx_conf_merge_ptr_value(conf->error_pages, prev->error_pages, NULL); - - ngx_conf_merge_str_value(conf->default_type, - prev->default_type, "text/plain"); -@@ -4815,6 +4823,10 @@ ngx_http_core_error_page(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) - ngx_http_compile_complex_value_t ccv; - - if (clcf->error_pages == NULL) { -+ return "conflicts with \"no_error_pages\""; -+ } -+ -+ if (clcf->error_pages == NGX_CONF_UNSET_PTR) { - clcf->error_pages = ngx_array_create(cf->pool, 4, - sizeof(ngx_http_err_page_t)); - if (clcf->error_pages == NULL) { -@@ -4920,6 +4932,25 @@ ngx_http_core_error_page(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) - } - - -+static char * -+ngx_http_core_no_error_pages(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) -+{ -+ ngx_http_core_loc_conf_t *clcf = conf; -+ -+ if (clcf->error_pages == NULL) { -+ return "is duplicate"; -+ } -+ -+ if (clcf->error_pages != NGX_CONF_UNSET_PTR) { -+ return "conflicts with \"error_page\""; -+ } -+ -+ clcf->error_pages = NULL; -+ -+ return NGX_CONF_OK; -+} -+ -+ - static char * - ngx_http_core_open_file_cache(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) - { diff --git a/images/nginx/rootfs/patches/18_nginx-1.27.1-no_Werror.patch b/images/nginx/rootfs/patches/18_nginx-1.27.1-no_Werror.patch deleted file mode 100644 index d0aa7a31e..000000000 --- a/images/nginx/rootfs/patches/18_nginx-1.27.1-no_Werror.patch +++ /dev/null @@ -1,36 +0,0 @@ -diff -urp nginx-1.27.1/auto/cc/clang nginx-1.27.1-patched/auto/cc/clang ---- nginx-1.27.1/auto/cc/clang 2014-03-04 03:39:24.000000000 -0800 -+++ nginx-1.27.1-patched/auto/cc/clang 2014-03-13 20:54:26.241413360 -0700 -@@ -89,7 +89,7 @@ CFLAGS="$CFLAGS -Wconditional-uninitiali - CFLAGS="$CFLAGS -Wno-unused-parameter" - - # stop on warning --CFLAGS="$CFLAGS -Werror" -+#CFLAGS="$CFLAGS -Werror" - - # debug - CFLAGS="$CFLAGS -g" -diff -urp nginx-1.27.1/auto/cc/gcc nginx-1.27.1-patched/auto/cc/gcc ---- nginx-1.27.1/auto/cc/gcc 2014-03-04 03:39:24.000000000 -0800 -+++ nginx-1.27.1-patched/auto/cc/gcc 2014-03-13 20:54:13.301355329 -0700 -@@ -168,7 +168,7 @@ esac - - - # stop on warning --CFLAGS="$CFLAGS -Werror" -+#CFLAGS="$CFLAGS -Werror" - - # debug - CFLAGS="$CFLAGS -g" -diff -urp nginx-1.27.1/auto/cc/icc nginx-1.27.1-patched/auto/cc/icc ---- nginx-1.27.1/auto/cc/icc 2014-03-04 03:39:24.000000000 -0800 -+++ nginx-1.27.1-patched/auto/cc/icc 2014-03-13 20:54:13.301355329 -0700 -@@ -115,7 +115,7 @@ case "$NGX_ICC_VER" in - esac - - # stop on warning --CFLAGS="$CFLAGS -Werror" -+#CFLAGS="$CFLAGS -Werror" - - # debug - CFLAGS="$CFLAGS -g" diff --git a/images/nginx/rootfs/patches/19_nginx-1.27.1-log_escape_non_ascii.patch b/images/nginx/rootfs/patches/19_nginx-1.27.1-log_escape_non_ascii.patch deleted file mode 100644 index bea6e52ee..000000000 --- a/images/nginx/rootfs/patches/19_nginx-1.27.1-log_escape_non_ascii.patch +++ /dev/null @@ -1,117 +0,0 @@ -diff --git a/src/http/modules/ngx_http_log_module.c b/src/http/modules/ngx_http_log_module.c -index 917ed55f..b769dfd3 100644 ---- a/src/http/modules/ngx_http_log_module.c -+++ b/src/http/modules/ngx_http_log_module.c -@@ -79,6 +79,8 @@ typedef struct { - time_t open_file_cache_valid; - ngx_uint_t open_file_cache_min_uses; - -+ ngx_flag_t escape_non_ascii; -+ - ngx_uint_t off; /* unsigned off:1 */ - } ngx_http_log_loc_conf_t; - -@@ -131,7 +133,8 @@ static size_t ngx_http_log_variable_getlen(ngx_http_request_t *r, - uintptr_t data); - static u_char *ngx_http_log_variable(ngx_http_request_t *r, u_char *buf, - ngx_http_log_op_t *op); --static uintptr_t ngx_http_log_escape(u_char *dst, u_char *src, size_t size); -+static uintptr_t ngx_http_log_escape(ngx_http_log_loc_conf_t *lcf, u_char *dst, -+ u_char *src, size_t size); - static size_t ngx_http_log_json_variable_getlen(ngx_http_request_t *r, - uintptr_t data); - static u_char *ngx_http_log_json_variable(ngx_http_request_t *r, u_char *buf, -@@ -177,6 +180,13 @@ static ngx_command_t ngx_http_log_commands[] = { - 0, - NULL }, - -+ { ngx_string("log_escape_non_ascii"), -+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, -+ ngx_conf_set_flag_slot, -+ NGX_HTTP_LOC_CONF_OFFSET, -+ offsetof(ngx_http_log_loc_conf_t, escape_non_ascii), -+ NULL }, -+ - ngx_null_command - }; - -@@ -935,6 +945,7 @@ static size_t - ngx_http_log_variable_getlen(ngx_http_request_t *r, uintptr_t data) - { - uintptr_t len; -+ ngx_http_log_loc_conf_t *lcf; - ngx_http_variable_value_t *value; - - value = ngx_http_get_indexed_variable(r, data); -@@ -943,7 +954,9 @@ ngx_http_log_variable_getlen(ngx_http_request_t *r, uintptr_t data) - return 1; - } - -- len = ngx_http_log_escape(NULL, value->data, value->len); -+ lcf = ngx_http_get_module_loc_conf(r, ngx_http_log_module); -+ -+ len = ngx_http_log_escape(lcf, NULL, value->data, value->len); - - value->escape = len ? 1 : 0; - -@@ -954,6 +967,7 @@ ngx_http_log_variable_getlen(ngx_http_request_t *r, uintptr_t data) - static u_char * - ngx_http_log_variable(ngx_http_request_t *r, u_char *buf, ngx_http_log_op_t *op) - { -+ ngx_http_log_loc_conf_t *lcf; - ngx_http_variable_value_t *value; - - value = ngx_http_get_indexed_variable(r, op->data); -@@ -967,16 +981,18 @@ ngx_http_log_variable(ngx_http_request_t *r, u_char *buf, ngx_http_log_op_t *op) - return ngx_cpymem(buf, value->data, value->len); - - } else { -- return (u_char *) ngx_http_log_escape(buf, value->data, value->len); -+ lcf = ngx_http_get_module_loc_conf(r, ngx_http_log_module); -+ return (u_char *) ngx_http_log_escape(lcf, buf, value->data, value->len); - } - } - - - static uintptr_t --ngx_http_log_escape(u_char *dst, u_char *src, size_t size) -+ngx_http_log_escape(ngx_http_log_loc_conf_t *lcf, u_char *dst, u_char *src, -+ size_t size) - { -- ngx_uint_t n; -- static u_char hex[] = "0123456789ABCDEF"; -+ ngx_uint_t n; -+ static u_char hex[] = "0123456789ABCDEF"; - - static uint32_t escape[] = { - 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ -@@ -996,6 +1012,12 @@ ngx_http_log_escape(u_char *dst, u_char *src, size_t size) - 0xffffffff, /* 1111 1111 1111 1111 1111 1111 1111 1111 */ - }; - -+ if (lcf->escape_non_ascii) { -+ ngx_memset(&escape[4], 0xff, sizeof(uint32_t) * 4); -+ -+ } else { -+ ngx_memzero(&escape[4], sizeof(uint32_t) * 4); -+ } - - if (dst == NULL) { - -@@ -1120,6 +1142,7 @@ ngx_http_log_create_loc_conf(ngx_conf_t *cf) - } - - conf->open_file_cache = NGX_CONF_UNSET_PTR; -+ conf->escape_non_ascii = NGX_CONF_UNSET; - - return conf; - } -@@ -1135,6 +1158,8 @@ ngx_http_log_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) - ngx_http_log_fmt_t *fmt; - ngx_http_log_main_conf_t *lmcf; - -+ ngx_conf_merge_value(conf->escape_non_ascii, prev->escape_non_ascii, 1); -+ - if (conf->open_file_cache == NGX_CONF_UNSET_PTR) { - - conf->open_file_cache = prev->open_file_cache; diff --git a/images/nginx/rootfs/patches/23_nginx-1.27.1-pcre_conf_opt.patch b/images/nginx/rootfs/patches/23_nginx-1.27.1-pcre_conf_opt.patch deleted file mode 100644 index eb17e0642..000000000 --- a/images/nginx/rootfs/patches/23_nginx-1.27.1-pcre_conf_opt.patch +++ /dev/null @@ -1,26 +0,0 @@ -# HG changeset patch -# User Yichun Zhang -# Date 1386694955 28800 -# Node ID 9ba6b149669f1f02eeb4cdc0ebd364a949b5c469 -# Parent 30e806b8636af5fd3f03ec17df24801f390f7511 -Configure: added new option --with-pcre-conf-opt=OPTIONS. - -diff -r 30e806b8636a -r 9ba6b149669f auto/options ---- a/auto/options Mon Dec 09 10:16:44 2013 +0400 -+++ b/auto/options Tue Dec 10 09:02:35 2013 -0800 -@@ -286,6 +286,7 @@ - --with-pcre) USE_PCRE=YES ;; - --with-pcre=*) PCRE="$value" ;; - --with-pcre-opt=*) PCRE_OPT="$value" ;; -+ --with-pcre-conf-opt=*) PCRE_CONF_OPT="$value" ;; - --with-pcre-jit) PCRE_JIT=YES ;; - - --with-openssl=*) OPENSSL="$value" ;; -@@ -441,6 +442,7 @@ - --with-pcre force PCRE library usage - --with-pcre=DIR set path to PCRE library sources - --with-pcre-opt=OPTIONS set additional build options for PCRE -+ --with-pcre-conf-opt=OPTIONS set additional configure options for PCRE - --with-pcre-jit build PCRE with JIT compilation support - - --with-md5=DIR set path to md5 library sources diff --git a/images/nginx/rootfs/patches/24_nginx-1.27.1-always_enable_cc_feature_tests.patch b/images/nginx/rootfs/patches/24_nginx-1.27.1-always_enable_cc_feature_tests.patch deleted file mode 100644 index 9517e92c4..000000000 --- a/images/nginx/rootfs/patches/24_nginx-1.27.1-always_enable_cc_feature_tests.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- nginx-1.27.1/auto/cc/conf 2015-10-30 22:47:50.000000000 +0800 -+++ nginx-1.27.1-patched/auto/cc/conf 2015-11-02 12:23:05.385156987 +0800 -@@ -144,7 +144,7 @@ fi - CFLAGS="$CFLAGS $NGX_CC_OPT" - NGX_TEST_LD_OPT="$NGX_LD_OPT" - --if [ "$NGX_PLATFORM" != win32 ]; then -+if [ 1 ]; then - - if test -n "$NGX_LD_OPT"; then - ngx_feature=--with-ld-opt=\"$NGX_LD_OPT\" diff --git a/images/nginx/rootfs/patches/27_nginx-1.27.1-ssl_client_hello_cb_yield.patch b/images/nginx/rootfs/patches/27_nginx-1.27.1-ssl_client_hello_cb_yield.patch deleted file mode 100644 index 0e97be992..000000000 --- a/images/nginx/rootfs/patches/27_nginx-1.27.1-ssl_client_hello_cb_yield.patch +++ /dev/null @@ -1,38 +0,0 @@ -diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c -index 8ba30e58..2b2db95c 100644 ---- a/src/event/ngx_event_openssl.c -+++ b/src/event/ngx_event_openssl.c -@@ -1712,6 +1712,9 @@ ngx_ssl_handshake(ngx_connection_t *c) - if (sslerr == SSL_ERROR_WANT_X509_LOOKUP - # ifdef SSL_ERROR_PENDING_SESSION - || sslerr == SSL_ERROR_PENDING_SESSION -+# endif -+# ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB -+ || sslerr == SSL_ERROR_WANT_CLIENT_HELLO_CB - # endif - ) - { -@@ -1889,6 +1892,23 @@ ngx_ssl_try_early_data(ngx_connection_t *c) - } - #endif - -+#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB -+ if (sslerr == SSL_ERROR_WANT_CLIENT_HELLO_CB) { -+ c->read->handler = ngx_ssl_handshake_handler; -+ c->write->handler = ngx_ssl_handshake_handler; -+ -+ if (ngx_handle_read_event(c->read, 0) != NGX_OK) { -+ return NGX_ERROR; -+ } -+ -+ if (ngx_handle_write_event(c->write, 0) != NGX_OK) { -+ return NGX_ERROR; -+ } -+ -+ return NGX_AGAIN; -+ } -+#endif -+ - err = (sslerr == SSL_ERROR_SYSCALL) ? ngx_errno : 0; - - c->ssl->no_wait_shutdown = 1; diff --git a/images/nginx/rootfs/patches/29_nginx-1.27.1-safe_resolver_ipv6_option.patch b/images/nginx/rootfs/patches/29_nginx-1.27.1-safe_resolver_ipv6_option.patch deleted file mode 100644 index 6c54c6c4c..000000000 --- a/images/nginx/rootfs/patches/29_nginx-1.27.1-safe_resolver_ipv6_option.patch +++ /dev/null @@ -1,60 +0,0 @@ -# HG changeset patch -# User Thibault Charbonnier -# Date 1481847421 28800 -# Thu Dec 15 16:17:01 2016 -0800 -# Node ID 8bf038fe006fd8ae253d6b41fc6cf109a8912d3e -# Parent a3dc657f4e9530623683e6b85bd7492662e4dc47 -Resolver: ignore ipv6=off resolver option when no ipv6 support - -Makes the resolver directive more robust: we only error out when ipv6 -resolution is desired but not supported (ipv6=on). - -use case 1: some configurations are sometimes re-used between builds with and -without ipv6 support. This patch avoids the need to remove the "ipv6=off" flag. - -use case 2: currently, some tools rely on the --with-ipv6 configure option from -"nginx -V" to determine if ipv6 resolution should be disabled in some cases. -With this option disappearing in Nginx 1.11.5, this patch would allow such tools -to assume "ipv6=off" to be safe regardless of ipv6 support in the current -build. - -diff --git a/src/core/ngx_resolver.c b/src/core/ngx_resolver.c -index dade1846..5a3f0aa4 100644 ---- a/src/core/ngx_resolver.c -+++ b/src/core/ngx_resolver.c -@@ -425,7 +425,6 @@ ngx_resolver_create(ngx_conf_t *cf, ngx_str_t *names, ngx_uint_t n) - continue; - } - --#if (NGX_HAVE_INET6) - if (ngx_strncmp(names[i].data, "ipv4=", 5) == 0) { - - if (ngx_strcmp(&names[i].data[5], "on") == 0) { -@@ -446,10 +445,19 @@ ngx_resolver_create(ngx_conf_t *cf, ngx_str_t *names, ngx_uint_t n) - if (ngx_strncmp(names[i].data, "ipv6=", 5) == 0) { - - if (ngx_strcmp(&names[i].data[5], "on") == 0) { -+#if (NGX_HAVE_INET6) - r->ipv6 = 1; -+#else -+ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, -+ "no ipv6 support but \"%V\" in resolver", -+ &names[i]); -+ return NULL; -+#endif - - } else if (ngx_strcmp(&names[i].data[5], "off") == 0) { -+#if (NGX_HAVE_INET6) - r->ipv6 = 0; -+#endif - - } else { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, -@@ -459,7 +467,6 @@ ngx_resolver_create(ngx_conf_t *cf, ngx_str_t *names, ngx_uint_t n) - - continue; - } --#endif - - #if !(NGX_WIN32) - if (ngx_strncmp(names[i].data, "local=", 6) == 0) { diff --git a/images/nginx/rootfs/patches/32_nginx-1.27.1-proc_exit_handler.patch b/images/nginx/rootfs/patches/32_nginx-1.27.1-proc_exit_handler.patch deleted file mode 100644 index f050c09d8..000000000 --- a/images/nginx/rootfs/patches/32_nginx-1.27.1-proc_exit_handler.patch +++ /dev/null @@ -1,77 +0,0 @@ -diff --git a/src/core/ngx_cycle.c b/src/core/ngx_cycle.c -index c4e3c50..fa1408b 100644 ---- a/src/core/ngx_cycle.c -+++ b/src/core/ngx_cycle.c -@@ -264,6 +264,9 @@ ngx_init_cycle(ngx_cycle_t *old_cycle) - } - - -+#if !(NGX_WIN32) -+ ngx_proc_exit_top_handler = ngx_proc_exit_def_handler; -+#endif - conf.ctx = cycle->conf_ctx; - conf.cycle = cycle; - conf.pool = pool; -diff --git a/src/os/unix/ngx_process.c b/src/os/unix/ngx_process.c -index 12a8c68..874c9bf 100644 ---- a/src/os/unix/ngx_process.c -+++ b/src/os/unix/ngx_process.c -@@ -34,6 +34,7 @@ ngx_int_t ngx_process_slot; - ngx_socket_t ngx_channel; - ngx_int_t ngx_last_process; - ngx_process_t ngx_processes[NGX_MAX_PROCESSES]; -+ngx_proc_exit_pt ngx_proc_exit_top_handler; - - - ngx_signal_t signals[] = { -@@ -83,6 +84,13 @@ ngx_signal_t signals[] = { - }; - - -+void -+ngx_proc_exit_def_handler(ngx_pid_t pid) -+{ -+ /* do nothing */ -+} -+ -+ - ngx_pid_t - ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data, - char *name, ngx_int_t respawn) -@@ -564,6 +572,7 @@ ngx_process_get_status(void) - } - - ngx_unlock_mutexes(pid); -+ ngx_proc_exit_top_handler(pid); - } - } - -diff --git a/src/os/unix/ngx_process.h b/src/os/unix/ngx_process.h -index 3986639..0b55d98 100644 ---- a/src/os/unix/ngx_process.h -+++ b/src/os/unix/ngx_process.h -@@ -18,6 +18,8 @@ typedef pid_t ngx_pid_t; - #define NGX_INVALID_PID -1 - - typedef void (*ngx_spawn_proc_pt) (ngx_cycle_t *cycle, void *data); -+#define NGX_HAVE_PROC_EXIT 1 -+typedef void (*ngx_proc_exit_pt)(ngx_pid_t pid); - - typedef struct { - ngx_pid_t pid; -@@ -66,6 +68,7 @@ ngx_pid_t ngx_spawn_process(ngx_cycle_t *cycle, - ngx_pid_t ngx_execute(ngx_cycle_t *cycle, ngx_exec_ctx_t *ctx); - ngx_int_t ngx_init_signals(ngx_log_t *log); - void ngx_debug_point(void); -+void ngx_proc_exit_def_handler(ngx_pid_t pid); - - - #if (NGX_HAVE_SCHED_YIELD) -@@ -85,6 +88,7 @@ extern ngx_socket_t ngx_channel; - extern ngx_int_t ngx_process_slot; - extern ngx_int_t ngx_last_process; - extern ngx_process_t ngx_processes[NGX_MAX_PROCESSES]; -+extern ngx_proc_exit_pt ngx_proc_exit_top_handler; - - - #endif /* _NGX_PROCESS_H_INCLUDED_ */ diff --git a/images/nginx/rootfs/patches/09_nginx-1.27.1-balancer_status_code.patch b/images/nginx/rootfs/patches/nginx-1.19.3-balancer_status_code.patch similarity index 100% rename from images/nginx/rootfs/patches/09_nginx-1.27.1-balancer_status_code.patch rename to images/nginx/rootfs/patches/nginx-1.19.3-balancer_status_code.patch diff --git a/images/nginx/rootfs/patches/21_nginx-1.27.1-cache_manager_exit.patch b/images/nginx/rootfs/patches/nginx-1.19.3-cache_manager_exit.patch similarity index 96% rename from images/nginx/rootfs/patches/21_nginx-1.27.1-cache_manager_exit.patch rename to images/nginx/rootfs/patches/nginx-1.19.3-cache_manager_exit.patch index 91ee63a26..f1f81da2c 100644 --- a/images/nginx/rootfs/patches/21_nginx-1.27.1-cache_manager_exit.patch +++ b/images/nginx/rootfs/patches/nginx-1.19.3-cache_manager_exit.patch @@ -8,7 +8,7 @@ Cache: gracefully exit the cache manager process. diff -r dea321e5c021 -r f64218e1ac96 src/os/unix/ngx_process_cycle.c --- a/src/os/unix/ngx_process_cycle.c Thu Oct 31 18:23:49 2013 +0400 +++ b/src/os/unix/ngx_process_cycle.c Mon Nov 04 12:48:50 2013 -0800 -@@ -1134,7 +1134,7 @@ +@@ -1335,7 +1335,7 @@ if (ngx_terminate || ngx_quit) { ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "exiting"); diff --git a/images/nginx/rootfs/patches/10_nginx-1.27.1-delayed_posted_events.patch b/images/nginx/rootfs/patches/nginx-1.19.3-delayed_posted_events.patch similarity index 100% rename from images/nginx/rootfs/patches/10_nginx-1.27.1-delayed_posted_events.patch rename to images/nginx/rootfs/patches/nginx-1.19.3-delayed_posted_events.patch diff --git a/images/nginx/rootfs/patches/nginx-1.19.3-hash_overflow.patch b/images/nginx/rootfs/patches/nginx-1.19.3-hash_overflow.patch new file mode 100644 index 000000000..449d214ba --- /dev/null +++ b/images/nginx/rootfs/patches/nginx-1.19.3-hash_overflow.patch @@ -0,0 +1,20 @@ +# HG changeset patch +# User Yichun Zhang +# Date 1412276417 25200 +# Thu Oct 02 12:00:17 2014 -0700 +# Node ID 4032b992f23b054c1a2cfb0be879330d2c6708e5 +# Parent 1ff0f68d9376e3d184d65814a6372856bf65cfcd +Hash: buffer overflow might happen when exceeding the pre-configured limits. + +diff -r 1ff0f68d9376 -r 4032b992f23b src/core/ngx_hash.c +--- a/src/core/ngx_hash.c Tue Sep 30 15:50:28 2014 -0700 ++++ b/src/core/ngx_hash.c Thu Oct 02 12:00:17 2014 -0700 +@@ -312,6 +312,8 @@ ngx_hash_init(ngx_hash_init_t *hinit, ng + continue; + } + ++ size--; ++ + ngx_log_error(NGX_LOG_WARN, hinit->pool->log, 0, + "could not build optimal %s, you should increase " + "either %s_max_size: %i or %s_bucket_size: %i; " diff --git a/images/nginx/rootfs/patches/08_nginx-1.27.1-init_cycle_pool_release.patch b/images/nginx/rootfs/patches/nginx-1.19.3-init_cycle_pool_release.patch similarity index 62% rename from images/nginx/rootfs/patches/08_nginx-1.27.1-init_cycle_pool_release.patch rename to images/nginx/rootfs/patches/nginx-1.19.3-init_cycle_pool_release.patch index 4a26b92c5..b6f46674f 100644 --- a/images/nginx/rootfs/patches/08_nginx-1.27.1-init_cycle_pool_release.patch +++ b/images/nginx/rootfs/patches/nginx-1.19.3-init_cycle_pool_release.patch @@ -1,6 +1,6 @@ -diff -rup nginx-1.27.1/src/core/nginx.c nginx-1.27.1-patched/src/core/nginx.c ---- nginx-1.27.1/src/core/nginx.c 2017-12-17 00:00:38.136470108 -0800 -+++ nginx-1.27.1-patched/src/core/nginx.c 2017-12-16 23:59:51.680958322 -0800 +diff -rup nginx-1.19.3/src/core/nginx.c nginx-1.19.3-patched/src/core/nginx.c +--- nginx-1.19.3/src/core/nginx.c 2017-12-17 00:00:38.136470108 -0800 ++++ nginx-1.19.3-patched/src/core/nginx.c 2017-12-16 23:59:51.680958322 -0800 @@ -186,6 +186,7 @@ static u_char *ngx_prefix; static u_char *ngx_conf_file; static u_char *ngx_conf_params; @@ -18,9 +18,9 @@ diff -rup nginx-1.27.1/src/core/nginx.c nginx-1.27.1-patched/src/core/nginx.c if (ngx_save_argv(&init_cycle, argc, argv) != NGX_OK) { return 1; } -diff -rup nginx-1.27.1/src/core/ngx_core.h nginx-1.27.1-patched/src/core/ngx_core.h ---- nginx-1.27.1/src/core/ngx_core.h 2017-10-10 08:22:51.000000000 -0700 -+++ nginx-1.27.1-patched/src/core/ngx_core.h 2017-12-16 23:59:51.679958370 -0800 +diff -rup nginx-1.19.3/src/core/ngx_core.h nginx-1.19.3-patched/src/core/ngx_core.h +--- nginx-1.19.3/src/core/ngx_core.h 2017-10-10 08:22:51.000000000 -0700 ++++ nginx-1.19.3-patched/src/core/ngx_core.h 2017-12-16 23:59:51.679958370 -0800 @@ -108,4 +108,6 @@ void ngx_cpuinfo(void); #define NGX_DISABLE_SYMLINKS_NOTOWNER 2 #endif @@ -28,9 +28,9 @@ diff -rup nginx-1.27.1/src/core/ngx_core.h nginx-1.27.1-patched/src/core/ngx_cor +extern ngx_pool_t *saved_init_cycle_pool; + #endif /* _NGX_CORE_H_INCLUDED_ */ -diff -rup nginx-1.27.1/src/core/ngx_cycle.c nginx-1.27.1-patched/src/core/ngx_cycle.c ---- nginx-1.27.1/src/core/ngx_cycle.c 2017-10-10 08:22:51.000000000 -0700 -+++ nginx-1.27.1-patched/src/core/ngx_cycle.c 2017-12-16 23:59:51.678958419 -0800 +diff -rup nginx-1.19.3/src/core/ngx_cycle.c nginx-1.19.3-patched/src/core/ngx_cycle.c +--- nginx-1.19.3/src/core/ngx_cycle.c 2017-10-10 08:22:51.000000000 -0700 ++++ nginx-1.19.3-patched/src/core/ngx_cycle.c 2017-12-16 23:59:51.678958419 -0800 @@ -748,6 +748,10 @@ old_shm_zone_done: if (ngx_process == NGX_PROCESS_MASTER || ngx_is_init_cycle(old_cycle)) { @@ -42,10 +42,10 @@ diff -rup nginx-1.27.1/src/core/ngx_cycle.c nginx-1.27.1-patched/src/core/ngx_cy ngx_destroy_pool(old_cycle->pool); cycle->old_cycle = NULL; -diff -rup nginx-1.27.1/src/os/unix/ngx_process_cycle.c nginx-1.27.1-patched/src/os/unix/ngx_process_cycle.c ---- nginx-1.27.1/src/os/unix/ngx_process_cycle.c 2017-12-17 00:00:38.142469762 -0800 -+++ nginx-1.27.1-patched/src/os/unix/ngx_process_cycle.c 2017-12-16 23:59:51.691957791 -0800 -@@ -687,6 +692,11 @@ ngx_master_process_exit(ngx_cycle_t *cyc +diff -rup nginx-1.19.3/src/os/unix/ngx_process_cycle.c nginx-1.19.3-patched/src/os/unix/ngx_process_cycle.c +--- nginx-1.19.3/src/os/unix/ngx_process_cycle.c 2017-12-17 00:00:38.142469762 -0800 ++++ nginx-1.19.3-patched/src/os/unix/ngx_process_cycle.c 2017-12-16 23:59:51.691957791 -0800 +@@ -783,6 +783,11 @@ ngx_master_process_exit(ngx_cycle_t *cyc ngx_exit_cycle.files_n = ngx_cycle->files_n; ngx_cycle = &ngx_exit_cycle; diff --git a/images/nginx/rootfs/patches/22_nginx-1.27.1-larger_max_error_str.patch b/images/nginx/rootfs/patches/nginx-1.19.3-larger_max_error_str.patch similarity index 62% rename from images/nginx/rootfs/patches/22_nginx-1.27.1-larger_max_error_str.patch rename to images/nginx/rootfs/patches/nginx-1.19.3-larger_max_error_str.patch index b821297e6..c7013e05d 100644 --- a/images/nginx/rootfs/patches/22_nginx-1.27.1-larger_max_error_str.patch +++ b/images/nginx/rootfs/patches/nginx-1.19.3-larger_max_error_str.patch @@ -1,5 +1,5 @@ ---- nginx-1.27.1/src/core/ngx_log.h 2013-10-08 05:07:14.000000000 -0700 -+++ nginx-1.27.1-patched/src/core/ngx_log.h 2013-12-05 20:35:35.996236720 -0800 +--- nginx-1.19.3/src/core/ngx_log.h 2013-10-08 05:07:14.000000000 -0700 ++++ nginx-1.19.3-patched/src/core/ngx_log.h 2013-12-05 20:35:35.996236720 -0800 @@ -64,7 +64,9 @@ struct ngx_log_s { }; diff --git a/images/nginx/rootfs/patches/nginx-1.19.3-no_Werror.patch b/images/nginx/rootfs/patches/nginx-1.19.3-no_Werror.patch new file mode 100644 index 000000000..2cf7cd9c7 --- /dev/null +++ b/images/nginx/rootfs/patches/nginx-1.19.3-no_Werror.patch @@ -0,0 +1,36 @@ +diff -urp nginx-1.19.3/auto/cc/clang nginx-1.19.3-patched/auto/cc/clang +--- nginx-1.19.3/auto/cc/clang 2014-03-04 03:39:24.000000000 -0800 ++++ nginx-1.19.3-patched/auto/cc/clang 2014-03-13 20:54:26.241413360 -0700 +@@ -89,7 +89,7 @@ CFLAGS="$CFLAGS -Wconditional-uninitiali + CFLAGS="$CFLAGS -Wno-unused-parameter" + + # stop on warning +-CFLAGS="$CFLAGS -Werror" ++#CFLAGS="$CFLAGS -Werror" + + # debug + CFLAGS="$CFLAGS -g" +diff -urp nginx-1.19.3/auto/cc/gcc nginx-1.19.3-patched/auto/cc/gcc +--- nginx-1.19.3/auto/cc/gcc 2014-03-04 03:39:24.000000000 -0800 ++++ nginx-1.19.3-patched/auto/cc/gcc 2014-03-13 20:54:13.301355329 -0700 +@@ -168,7 +168,7 @@ esac + + + # stop on warning +-CFLAGS="$CFLAGS -Werror" ++#CFLAGS="$CFLAGS -Werror" + + # debug + CFLAGS="$CFLAGS -g" +diff -urp nginx-1.19.3/auto/cc/icc nginx-1.19.3-patched/auto/cc/icc +--- nginx-1.19.3/auto/cc/icc 2014-03-04 03:39:24.000000000 -0800 ++++ nginx-1.19.3-patched/auto/cc/icc 2014-03-13 20:54:13.301355329 -0700 +@@ -115,7 +115,7 @@ case "$NGX_ICC_VER" in + esac + + # stop on warning +-CFLAGS="$CFLAGS -Werror" ++#CFLAGS="$CFLAGS -Werror" + + # debug + CFLAGS="$CFLAGS -g" diff --git a/images/nginx/rootfs/patches/20_nginx-1.27.1-proxy_host_port_vars.patch b/images/nginx/rootfs/patches/nginx-1.19.3-proxy_host_port_vars.patch similarity index 87% rename from images/nginx/rootfs/patches/20_nginx-1.27.1-proxy_host_port_vars.patch rename to images/nginx/rootfs/patches/nginx-1.19.3-proxy_host_port_vars.patch index b81a299c8..63eaf0ca7 100644 --- a/images/nginx/rootfs/patches/20_nginx-1.27.1-proxy_host_port_vars.patch +++ b/images/nginx/rootfs/patches/nginx-1.19.3-proxy_host_port_vars.patch @@ -1,5 +1,5 @@ ---- nginx-1.27.1/src/http/modules/ngx_http_proxy_module.c 2017-07-16 14:02:51.000000000 +0800 -+++ nginx-1.27.1-patched/src/http/modules/ngx_http_proxy_module.c 2017-07-16 14:02:51.000000000 +0800 +--- nginx-1.19.3/src/http/modules/ngx_http_proxy_module.c 2017-07-16 14:02:51.000000000 +0800 ++++ nginx-1.19.3-patched/src/http/modules/ngx_http_proxy_module.c 2017-07-16 14:02:51.000000000 +0800 @@ -793,13 +793,13 @@ static ngx_keyval_t ngx_http_proxy_cach static ngx_http_variable_t ngx_http_proxy_vars[] = { diff --git a/images/nginx/rootfs/patches/06_nginx-1.27.1-resolver_conf_parsing.patch b/images/nginx/rootfs/patches/nginx-1.19.3-resolver_conf_parsing.patch similarity index 100% rename from images/nginx/rootfs/patches/06_nginx-1.27.1-resolver_conf_parsing.patch rename to images/nginx/rootfs/patches/nginx-1.19.3-resolver_conf_parsing.patch diff --git a/images/nginx/rootfs/patches/31_nginx-1.27.1-reuseport_close_unused_fds.patch b/images/nginx/rootfs/patches/nginx-1.19.3-reuseport_close_unused_fds.patch similarity index 100% rename from images/nginx/rootfs/patches/31_nginx-1.27.1-reuseport_close_unused_fds.patch rename to images/nginx/rootfs/patches/nginx-1.19.3-reuseport_close_unused_fds.patch diff --git a/images/nginx/rootfs/patches/14_nginx-1.27.1-single_process_graceful_exit.patch b/images/nginx/rootfs/patches/nginx-1.19.3-single_process_graceful_exit.patch similarity index 100% rename from images/nginx/rootfs/patches/14_nginx-1.27.1-single_process_graceful_exit.patch rename to images/nginx/rootfs/patches/nginx-1.19.3-single_process_graceful_exit.patch diff --git a/images/nginx/rootfs/patches/30_nginx-1.27.1-socket_cloexec.patch b/images/nginx/rootfs/patches/nginx-1.19.3-socket_cloexec.patch similarity index 98% rename from images/nginx/rootfs/patches/30_nginx-1.27.1-socket_cloexec.patch rename to images/nginx/rootfs/patches/nginx-1.19.3-socket_cloexec.patch index 8ffe4c167..985ce573b 100644 --- a/images/nginx/rootfs/patches/30_nginx-1.27.1-socket_cloexec.patch +++ b/images/nginx/rootfs/patches/nginx-1.19.3-socket_cloexec.patch @@ -151,7 +151,7 @@ diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c index c4376a5..48e8fa8 100644 --- a/src/os/unix/ngx_process_cycle.c +++ b/src/os/unix/ngx_process_cycle.c -@@ -960,6 +1029,9 @@ ngx_worker_process_exit(ngx_cycle_t *cycle) +@@ -1032,6 +1032,9 @@ ngx_worker_process_exit(ngx_cycle_t *cycle) for (i = 0; i < cycle->connection_n; i++) { if (c[i].fd != -1 && c[i].read diff --git a/images/nginx/rootfs/patches/25_nginx-1.27.1-ssl_cert_cb_yield.patch b/images/nginx/rootfs/patches/nginx-1.19.3-ssl_cert_cb_yield.patch similarity index 100% rename from images/nginx/rootfs/patches/25_nginx-1.27.1-ssl_cert_cb_yield.patch rename to images/nginx/rootfs/patches/nginx-1.19.3-ssl_cert_cb_yield.patch diff --git a/images/nginx/rootfs/patches/26_nginx-1.27.1-ssl_sess_cb_yield.patch b/images/nginx/rootfs/patches/nginx-1.19.3-ssl_sess_cb_yield.patch similarity index 100% rename from images/nginx/rootfs/patches/26_nginx-1.27.1-ssl_sess_cb_yield.patch rename to images/nginx/rootfs/patches/nginx-1.19.3-ssl_sess_cb_yield.patch diff --git a/images/nginx/rootfs/patches/03_nginx-1.27.1-stream_proxy_get_next_upstream_tries.patch b/images/nginx/rootfs/patches/nginx-1.19.3-stream_proxy_get_next_upstream_tries.patch similarity index 100% rename from images/nginx/rootfs/patches/03_nginx-1.27.1-stream_proxy_get_next_upstream_tries.patch rename to images/nginx/rootfs/patches/nginx-1.19.3-stream_proxy_get_next_upstream_tries.patch diff --git a/images/nginx/rootfs/patches/05_nginx-1.27.1-stream_ssl_preread_no_skip.patch b/images/nginx/rootfs/patches/nginx-1.19.3-stream_ssl_preread_no_skip.patch similarity index 100% rename from images/nginx/rootfs/patches/05_nginx-1.27.1-stream_ssl_preread_no_skip.patch rename to images/nginx/rootfs/patches/nginx-1.19.3-stream_ssl_preread_no_skip.patch diff --git a/images/nginx/rootfs/patches/16_nginx-1.27.1-upstream_pipelining.patch b/images/nginx/rootfs/patches/nginx-1.19.3-upstream_pipelining.patch similarity index 100% rename from images/nginx/rootfs/patches/16_nginx-1.27.1-upstream_pipelining.patch rename to images/nginx/rootfs/patches/nginx-1.19.3-upstream_pipelining.patch diff --git a/images/nginx/rootfs/patches/28_nginx-1.27.1-upstream_timeout_fields.patch b/images/nginx/rootfs/patches/nginx-1.19.3-upstream_timeout_fields.patch similarity index 100% rename from images/nginx/rootfs/patches/28_nginx-1.27.1-upstream_timeout_fields.patch rename to images/nginx/rootfs/patches/nginx-1.19.3-upstream_timeout_fields.patch diff --git a/images/test-runner/Makefile b/images/test-runner/Makefile index 8adbf0b41..752596c2d 100644 --- a/images/test-runner/Makefile +++ b/images/test-runner/Makefile @@ -18,71 +18,39 @@ SHELL=/bin/bash -o pipefail -o errexit DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))) INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh -SHORT_SHA ?=$(shell git rev-parse --short HEAD) -TAG ?=v$(shell date +%Y%m%d)-$(SHORT_SHA) - +TAG ?=v$(shell date +%m%d%Y)-$(shell git rev-parse --short HEAD) REGISTRY ?= local IMAGE = $(REGISTRY)/e2e-test-runner -NGINX_BASE_IMAGE ?= $(shell cat $(DIR)/../../NGINX_BASE) - -# The env below is called GO_VERSION and not GOLANG_VERSION because -# the gcb image we use to build already defines GOLANG_VERSION and is a -# really old version -GO_VERSION ?= $(shell cat $(DIR)/../../GOLANG_VERSION) +NGINX_BASE_IMAGE ?= k8s.gcr.io/ingress-nginx/nginx:v20210530-g6aab4c291@sha256:a7356029dd0c26cc3466bf7a27daec0f4df73aa14ca6c8b871a767022a812c0b # required to enable buildx export DOCKER_CLI_EXPERIMENTAL=enabled # build with buildx PLATFORMS?=linux/amd64,linux/arm64 -OUTPUT?= +OUTPUT= PROGRESS=plain -image: - echo "Building docker image ($(ARCH))..." - docker build \ - ${PLATFORM_FLAG} ${PLATFORM} \ - --no-cache \ - --pull \ - --push \ - --build-arg BASE_IMAGE=${NGINX_BASE_IMAGE} \ - --build-arg GOLANG_VERSION=${GO_VERSION} \ - --build-arg ETCD_VERSION=3.5.13-0 \ - --build-arg K8S_RELEASE=v1.32.2 \ - --build-arg RESTY_CLI_VERSION=0.27 \ - --build-arg RESTY_CLI_SHA=e5f4f3128af49ba5c4d039d0554e5ae91bbe05866f60eccfa96d3653274bff90 \ - --build-arg LUAROCKS_VERSION=3.8.0 \ - --build-arg LUAROCKS_SHA=ab6612ca9ab87c6984871d2712d05525775e8b50172701a0a1cabddf76de2be7 \ - --build-arg CHART_TESTING_VERSION=3.8.0 \ - --build-arg YAML_LINT_VERSION=1.33.0 \ - --build-arg YAMALE_VERSION=4.0.4 \ - --build-arg HELM_VERSION=3.14.4 \ - --build-arg GINKGO_VERSION=2.22.2 \ - --build-arg GOLINT_VERSION=latest \ - -t ${IMAGE}:${TAG} rootfs - build: ensure-buildx docker buildx build \ - --platform=${PLATFORMS} ${OUTPUT} \ - --progress=${PROGRESS} \ + --platform=${PLATFORMS} $(OUTPUT) \ + --progress=$(PROGRESS) \ --pull \ - --build-arg BASE_IMAGE=${NGINX_BASE_IMAGE} \ - --build-arg GOLANG_VERSION=${GO_VERSION} \ - --build-arg ETCD_VERSION=3.5.13-0 \ - --build-arg K8S_RELEASE=v1.32.2 \ + --build-arg BASE_IMAGE=$(NGINX_BASE_IMAGE) \ + --build-arg GOLANG_VERSION=1.15.6 \ + --build-arg ETCD_VERSION=3.4.3-0 \ + --build-arg K8S_RELEASE=v1.19.4 \ --build-arg RESTY_CLI_VERSION=0.27 \ --build-arg RESTY_CLI_SHA=e5f4f3128af49ba5c4d039d0554e5ae91bbe05866f60eccfa96d3653274bff90 \ - --build-arg LUAROCKS_VERSION=3.8.0 \ - --build-arg LUAROCKS_SHA=ab6612ca9ab87c6984871d2712d05525775e8b50172701a0a1cabddf76de2be7 \ - --build-arg CHART_TESTING_VERSION=3.8.0 \ - --build-arg YAML_LINT_VERSION=1.33.0 \ - --build-arg YAMALE_VERSION=4.0.4 \ - --build-arg HELM_VERSION=3.14.4 \ - --build-arg GINKGO_VERSION=2.22.2 \ - --build-arg GOLINT_VERSION=latest \ - -t ${IMAGE}:${TAG} rootfs + --build-arg LUAROCKS_VERSION=3.3.1 \ + --build-arg LUAROCKS_SHA=837481e408f7c06b59befe7ec194537c657687d624894bca7f79034302141a34 \ + --build-arg CHART_TESTING_VERSION=3.0.0 \ + --build-arg YAML_LINT_VERSION=1.13.0 \ + --build-arg YAMALE_VERSION=1.8.0 \ + --build-arg HELM_VERSION=v3.4.2 \ + -t $(IMAGE):$(TAG) rootfs # push the cross built image push: OUTPUT=--push @@ -92,7 +60,7 @@ push: build ensure-buildx: # this is required for cloudbuild ifeq ("$(wildcard $(INIT_BUILDX))","") - @curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/hack/init-buildx.sh | bash + @curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/hack/init-buildx.sh | bash else @exec $(INIT_BUILDX) endif diff --git a/images/test-runner/TAG b/images/test-runner/TAG deleted file mode 100644 index 46b105a30..000000000 --- a/images/test-runner/TAG +++ /dev/null @@ -1 +0,0 @@ -v2.0.0 diff --git a/images/test-runner/cloudbuild.yaml b/images/test-runner/cloudbuild.yaml index 93dce3ec9..3e4d67d06 100644 --- a/images/test-runner/cloudbuild.yaml +++ b/images/test-runner/cloudbuild.yaml @@ -1,11 +1,22 @@ +timeout: 1200s options: - # Ignore Prow provided substitutions. substitution_option: ALLOW_LOOSE steps: - - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20250116-2a05ea7e3d - env: - - REGISTRY=us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx + - name: gcr.io/k8s-testimages/gcb-docker-gcloud:v20201130-750d12f entrypoint: bash + env: + - DOCKER_CLI_EXPERIMENTAL=enabled + - TAG=$_GIT_TAG + - BASE_REF=$_PULL_BASE_REF + - REGISTRY=gcr.io/k8s-staging-ingress-nginx + # default cloudbuild has HOME=/builder/home and docker buildx is in /root/.docker/cli-plugins/docker-buildx + # set the home to /root explicitly to if using docker buildx + - HOME=/root args: - - -c - - gcloud auth configure-docker && cd images/test-runner && make push + - -c + - | + gcloud auth configure-docker \ + && make push +substitutions: + _GIT_TAG: "12345" + _PULL_BASE_REF: "master" diff --git a/images/test-runner/rootfs/Dockerfile b/images/test-runner/rootfs/Dockerfile index d871461bf..e58cf47ad 100644 --- a/images/test-runner/rootfs/Dockerfile +++ b/images/test-runner/rootfs/Dockerfile @@ -11,46 +11,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + ARG BASE_IMAGE ARG GOLANG_VERSION ARG ETCD_VERSION -FROM golang:${GOLANG_VERSION}-alpine3.21 as GO -FROM registry.k8s.io/etcd:${ETCD_VERSION} as etcd +FROM golang:${GOLANG_VERSION}-alpine as GO +FROM k8s.gcr.io/etcd:${ETCD_VERSION} as etcd FROM ${BASE_IMAGE} -ARG RESTY_CLI_VERSION -ARG RESTY_CLI_SHA -ARG LUAROCKS_VERSION -ARG LUAROCKS_SHA -ARG TARGETARCH -ARG K8S_RELEASE -ARG CHART_TESTING_VERSION -ARG HELM_VERSION -ARG YAMALE_VERSION -ARG YAML_LINT_VERSION -ARG GINKGO_VERSION -ARG GOLINT_VERSION - - -RUN set -eux; \ - if [ -e /etc/nsswitch.conf ]; then \ - grep '^hosts: files dns' /etc/nsswitch.conf; \ - else \ - echo 'hosts: files dns' > /etc/nsswitch.conf; \ - fi +RUN [ ! -e /etc/nsswitch.conf ] && echo 'hosts: files dns' > /etc/nsswitch.conf COPY --from=GO /usr/local/go /usr/local/go -ENV GOPATH /go -ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH -RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" - COPY --from=etcd /usr/local/bin/etcd /usr/local/bin/etcd -RUN echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories +RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories -RUN apk update && apk upgrade && apk add --no-cache \ +RUN apk add --no-cache \ bash \ ca-certificates \ wget \ @@ -64,15 +42,19 @@ RUN apk update && apk upgrade && apk add --no-cache \ py-pip \ unzip \ openssl \ - cfssl@testing \ - tzdata \ - libc6-compat \ - sqlite-dev + cfssl@testing -RUN go install -v github.com/onsi/ginkgo/v2/ginkgo@v${GINKGO_VERSION} \ - && go install golang.org/x/lint/golint@${GOLINT_VERSION} +ENV GOPATH /go +ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH -RUN wget -qO /tmp/resty_cli.tgz \ +RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" + +RUN go get github.com/onsi/ginkgo/ginkgo golang.org/x/lint/golint + +ARG RESTY_CLI_VERSION +ARG RESTY_CLI_SHA + +RUN wget -O /tmp/resty_cli.tgz \ https://github.com/openresty/resty-cli/archive/v${RESTY_CLI_VERSION}.tar.gz \ && echo "${RESTY_CLI_SHA} */tmp/resty_cli.tgz" | sha256sum -c - \ && tar -C /tmp -xzf /tmp/resty_cli.tgz \ @@ -80,7 +62,10 @@ RUN wget -qO /tmp/resty_cli.tgz \ && resty -V \ && rm -rf /tmp/* -RUN wget -qO /tmp/luarocks.tgz \ +ARG LUAROCKS_VERSION +ARG LUAROCKS_SHA + +RUN wget -O /tmp/luarocks.tgz \ https://github.com/luarocks/luarocks/archive/v${LUAROCKS_VERSION}.tar.gz \ && echo "${LUAROCKS_SHA} */tmp/luarocks.tgz" | sha256sum -c - \ && tar -C /tmp -xzf /tmp/luarocks.tgz \ @@ -91,51 +76,48 @@ RUN wget -qO /tmp/luarocks.tgz \ RUN luarocks install busted \ && luarocks install luacheck -RUN wget -qO /usr/local/bin/kubectl \ - https://dl.k8s.io/release/${K8S_RELEASE}/bin/linux/${TARGETARCH}/kubectl \ +ARG TARGETARCH + +ARG K8S_RELEASE + +RUN wget -O /usr/local/bin/kubectl \ + https://storage.googleapis.com/kubernetes-release/release/${K8S_RELEASE}/bin/linux/${TARGETARCH}/kubectl \ && chmod +x /usr/local/bin/kubectl -RUN wget -qO /usr/local/bin/kube-apiserver \ - https://dl.k8s.io/release/${K8S_RELEASE}/bin/linux/${TARGETARCH}/kube-apiserver \ +RUN wget -O /usr/local/bin/kube-apiserver \ + https://storage.googleapis.com/kubernetes-release/release/${K8S_RELEASE}/bin/linux/${TARGETARCH}/kube-apiserver \ && chmod +x /usr/local/bin/kube-apiserver -RUN wget -qO /tmp/ct-${CHART_TESTING_VERSION}-linux-${TARGETARCH}.tar.gz \ - https://github.com/helm/chart-testing/releases/download/v${CHART_TESTING_VERSION}/chart-testing_${CHART_TESTING_VERSION}_linux_${TARGETARCH}.tar.gz \ +ARG CHART_TESTING_VERSION + +RUN wget -O /tmp/ct-${CHART_TESTING_VERSION}-linux-amd64.tar.gz \ + https://github.com/helm/chart-testing/releases/download/v${CHART_TESTING_VERSION}/chart-testing_${CHART_TESTING_VERSION}_linux_amd64.tar.gz \ && mkdir -p /tmp/ct-download \ - && tar xzvf /tmp/ct-${CHART_TESTING_VERSION}-linux-${TARGETARCH}.tar.gz -C /tmp/ct-download \ - && rm /tmp/ct-${CHART_TESTING_VERSION}-linux-${TARGETARCH}.tar.gz \ + && tar xzvf /tmp/ct-${CHART_TESTING_VERSION}-linux-amd64.tar.gz -C /tmp/ct-download \ + && rm /tmp/ct-${CHART_TESTING_VERSION}-linux-amd64.tar.gz \ && cp /tmp/ct-download/ct /usr/local/bin \ && mkdir -p /etc/ct \ && cp -R /tmp/ct-download/etc/* /etc/ct \ && rm -rf /tmp/* -RUN wget -qO /usr/local/bin/lj-releng \ +RUN wget -O /usr/local/bin/lj-releng \ https://raw.githubusercontent.com/openresty/openresty-devel-utils/master/lj-releng \ && chmod +x /usr/local/bin/lj-releng +ARG HELM_VERSION -RUN wget -qO /tmp/helm.tgz \ - https://get.helm.sh/helm-v${HELM_VERSION}-linux-${TARGETARCH}.tar.gz \ +RUN wget -O /tmp/helm.tgz \ + https://get.helm.sh/helm-${HELM_VERSION}-linux-${TARGETARCH}.tar.gz \ && tar -C /tmp -xzf /tmp/helm.tgz \ && cp /tmp/linux*/helm /usr/local/bin \ && rm -rf /tmp/* # Install a YAML Linter -# Pip not working. Check PR https://github.com/kubernetes/ingress-nginx/pull/10874 -# RUN pip install --user "yamllint==$YAML_LINT_VERSION" -RUN apk update -U \ - && apk add yamllint \ - && yamllint --version +ARG YAML_LINT_VERSION +RUN pip install "yamllint==$YAML_LINT_VERSION" # Install Yamale YAML schema validator -# Commenting pip install yamale because broken cloudbuild https://github.com/kubernetes/ingress-nginx/pull/10885 -# RUN pip install --user "yamale==$YAMALE_VERSION" -RUN wget https://github.com/23andMe/Yamale/archive/master.zip \ - && unzip -d /tmp master.zip \ - && cd /tmp/Yamale-master \ - && python setup.py install \ - && yamale -V - -LABEL org.opencontainers.image.source=https://github.com/kubernetes/ingress-nginx +ARG YAMALE_VERSION +RUN pip install "yamale==$YAMALE_VERSION" WORKDIR $GOPATH diff --git a/internal/admission/controller/convert.go b/internal/admission/controller/convert.go new file mode 100644 index 000000000..004e5d057 --- /dev/null +++ b/internal/admission/controller/convert.go @@ -0,0 +1,90 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "unsafe" + + admissionv1 "k8s.io/api/admission/v1" + admissionv1beta1 "k8s.io/api/admission/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// these conversions are copied from https://github.com/kubernetes/kubernetes/blob/4db3a096ce8ac730b2280494422e1c4cf5fe875e/pkg/apis/admission/v1beta1/zz_generated.conversion.go +// to avoid copying in kubernetes/kubernetes +// they are sightly modified to remove complexity + +func convertV1beta1AdmissionReviewToAdmissionAdmissionReview(in *admissionv1beta1.AdmissionReview, out *admissionv1.AdmissionReview) { + if in.Request != nil { + if out.Request == nil { + out.Request = &admissionv1.AdmissionRequest{} + } + in, out := &in.Request, &out.Request + *out = new(admissionv1.AdmissionRequest) + convertV1beta1AdmissionRequestToAdmissionAdmissionRequest(*in, *out) + } else { + out.Request = nil + } + out.Response = (*admissionv1.AdmissionResponse)(unsafe.Pointer(in.Response)) // #nosec +} + +func convertV1beta1AdmissionRequestToAdmissionAdmissionRequest(in *admissionv1beta1.AdmissionRequest, out *admissionv1.AdmissionRequest) { + out.UID = types.UID(in.UID) + out.Kind = in.Kind + out.Resource = in.Resource + out.SubResource = in.SubResource + out.RequestKind = (*metav1.GroupVersionKind)(unsafe.Pointer(in.RequestKind)) // #nosec + out.RequestResource = (*metav1.GroupVersionResource)(unsafe.Pointer(in.RequestResource)) // #nosec + out.RequestSubResource = in.RequestSubResource + out.Name = in.Name + out.Namespace = in.Namespace + out.Operation = admissionv1.Operation(in.Operation) + out.Object = in.Object + out.OldObject = in.OldObject + out.Options = in.Options +} + +func convertAdmissionAdmissionReviewToV1beta1AdmissionReview(in *admissionv1.AdmissionReview, out *admissionv1beta1.AdmissionReview) { + if in.Request != nil { + if out.Request == nil { + out.Request = &admissionv1beta1.AdmissionRequest{} + } + in, out := &in.Request, &out.Request + *out = new(admissionv1beta1.AdmissionRequest) + convertAdmissionAdmissionRequestToV1beta1AdmissionRequest(*in, *out) + } else { + out.Request = nil + } + out.Response = (*admissionv1beta1.AdmissionResponse)(unsafe.Pointer(in.Response)) // #nosec +} + +func convertAdmissionAdmissionRequestToV1beta1AdmissionRequest(in *admissionv1.AdmissionRequest, out *admissionv1beta1.AdmissionRequest) { + out.UID = types.UID(in.UID) + out.Kind = in.Kind + out.Resource = in.Resource + out.SubResource = in.SubResource + out.RequestKind = (*metav1.GroupVersionKind)(unsafe.Pointer(in.RequestKind)) // #nosec + out.RequestResource = (*metav1.GroupVersionResource)(unsafe.Pointer(in.RequestResource)) // #nosec + out.RequestSubResource = in.RequestSubResource + out.Name = in.Name + out.Namespace = in.Namespace + out.Operation = admissionv1beta1.Operation(in.Operation) + out.Object = in.Object + out.OldObject = in.OldObject + out.Options = in.Options +} diff --git a/internal/admission/controller/main.go b/internal/admission/controller/main.go index 888818c89..7c59cdd9e 100644 --- a/internal/admission/controller/main.go +++ b/internal/admission/controller/main.go @@ -21,10 +21,12 @@ import ( "net/http" admissionv1 "k8s.io/api/admission/v1" - networking "k8s.io/api/networking/v1" + admissionv1beta1 "k8s.io/api/admission/v1beta1" + networking "k8s.io/api/networking/v1beta1" apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/klog/v2" ) @@ -33,7 +35,6 @@ import ( // contains invalid instructions type Checker interface { CheckIngress(ing *networking.Ingress) error - CheckWarning(ing *networking.Ingress) ([]string, error) } // IngressAdmission implements the AdmissionController interface @@ -42,19 +43,31 @@ type IngressAdmission struct { Checker Checker } -var ingressResource = metav1.GroupVersionKind{ - Group: networking.GroupName, - Version: "v1", - Kind: "Ingress", -} +var ( + ingressResource = metav1.GroupVersionKind{ + Group: networking.GroupName, + Version: "v1beta1", + Kind: "Ingress", + } +) // HandleAdmission populates the admission Response // with Allowed=false if the Object is an ingress that would prevent nginx to reload the configuration // with Allowed=true otherwise func (ia *IngressAdmission) HandleAdmission(obj runtime.Object) (runtime.Object, error) { + outputVersion := admissionv1.SchemeGroupVersion + review, isV1 := obj.(*admissionv1.AdmissionReview) + if !isV1 { - return nil, fmt.Errorf("request is not of type AdmissionReview v1 or v1beta1") + outputVersion = admissionv1beta1.SchemeGroupVersion + reviewv1beta1, isv1beta1 := obj.(*admissionv1beta1.AdmissionReview) + if !isv1beta1 { + return nil, fmt.Errorf("request is not of type AdmissionReview v1 or v1beta1") + } + + review = &admissionv1.AdmissionReview{} + convertV1beta1AdmissionReviewToAdmissionAdmissionReview(reviewv1beta1, review) } if !apiequality.Semantic.DeepEqual(review.Request.Kind, ingressResource) { @@ -70,6 +83,7 @@ func (ia *IngressAdmission) HandleAdmission(obj runtime.Object) (runtime.Object, codec := json.NewSerializerWithOptions(json.DefaultMetaFactory, scheme, scheme, json.SerializerOptions{ Pretty: true, }) + codec.Decode(review.Request.Object.Raw, nil, nil) _, _, err := codec.Decode(review.Request.Object.Raw, nil, &ingress) if err != nil { klog.ErrorS(err, "failed to decode ingress") @@ -80,20 +94,11 @@ func (ia *IngressAdmission) HandleAdmission(obj runtime.Object) (runtime.Object, } review.Response = status - return review, nil - } - - // Adds the warnings regardless of operation being allowed or not - warning, err := ia.Checker.CheckWarning(&ingress) - if err != nil { - klog.ErrorS(err, "failed to get ingress warnings") - } - if len(warning) > 0 { - status.Warnings = warning + return convertResponse(review, outputVersion), nil } if err := ia.Checker.CheckIngress(&ingress); err != nil { - klog.ErrorS(err, "invalid ingress configuration", "ingress", fmt.Sprintf("%v/%v", review.Request.Namespace, review.Request.Name)) + klog.ErrorS(err, "invalid ingress configuration", "ingress", fmt.Sprintf("%v/%v", review.Request.Name, review.Request.Namespace)) status.Allowed = false status.Result = &metav1.Status{ Status: metav1.StatusFailure, Code: http.StatusBadRequest, Reason: metav1.StatusReasonBadRequest, @@ -101,12 +106,24 @@ func (ia *IngressAdmission) HandleAdmission(obj runtime.Object) (runtime.Object, } review.Response = status - return review, nil + return convertResponse(review, outputVersion), nil } - klog.InfoS("successfully validated configuration, accepting", "ingress", fmt.Sprintf("%v/%v", review.Request.Namespace, review.Request.Name)) + klog.InfoS("successfully validated configuration, accepting", "ingress", fmt.Sprintf("%v/%v", review.Request.Name, review.Request.Namespace)) status.Allowed = true review.Response = status - return review, nil + return convertResponse(review, outputVersion), nil +} + +func convertResponse(review *admissionv1.AdmissionReview, outputVersion schema.GroupVersion) runtime.Object { + // reply v1 + if outputVersion.Version == admissionv1.SchemeGroupVersion.Version { + return review + } + + // reply v1beta1 + reviewv1beta1 := &admissionv1beta1.AdmissionReview{} + convertAdmissionAdmissionReviewToV1beta1AdmissionReview(review, reviewv1beta1) + return review } diff --git a/internal/admission/controller/main_test.go b/internal/admission/controller/main_test.go index 1d223cc9e..b745c14a2 100644 --- a/internal/admission/controller/main_test.go +++ b/internal/admission/controller/main_test.go @@ -21,7 +21,7 @@ import ( "testing" admissionv1 "k8s.io/api/admission/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/json" @@ -33,16 +33,11 @@ type failTestChecker struct { t *testing.T } -func (ftc failTestChecker) CheckIngress(_ *networking.Ingress) error { +func (ftc failTestChecker) CheckIngress(ing *networking.Ingress) error { ftc.t.Error("checker should not be called") return nil } -func (ftc failTestChecker) CheckWarning(_ *networking.Ingress) ([]string, error) { - ftc.t.Error("checker should not be called") - return nil, nil -} - type testChecker struct { t *testing.T err error @@ -55,19 +50,12 @@ func (tc testChecker) CheckIngress(ing *networking.Ingress) error { return tc.err } -func (tc testChecker) CheckWarning(ing *networking.Ingress) ([]string, error) { - if ing.ObjectMeta.Name != testIngressName { - tc.t.Errorf("CheckWarning should be called with %v ingress, but got %v", testIngressName, ing.ObjectMeta.Name) - } - return nil, tc.err -} - func TestHandleAdmission(t *testing.T) { adm := &IngressAdmission{ Checker: failTestChecker{t: t}, } - _, err := adm.HandleAdmission(&admissionv1.AdmissionReview{ + result, err := adm.HandleAdmission(&admissionv1.AdmissionReview{ Request: &admissionv1.AdmissionRequest{ Kind: v1.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}, }, @@ -76,14 +64,14 @@ func TestHandleAdmission(t *testing.T) { t.Fatalf("with a non ingress resource, the check should not pass") } - _, err = adm.HandleAdmission(nil) + result, err = adm.HandleAdmission(nil) if err == nil { t.Fatalf("with a nil AdmissionReview request, the check should not pass") } - result, err := adm.HandleAdmission(&admissionv1.AdmissionReview{ + result, err = adm.HandleAdmission(&admissionv1.AdmissionReview{ Request: &admissionv1.AdmissionRequest{ - Kind: v1.GroupVersionKind{Group: networking.GroupName, Version: "v1", Kind: "Ingress"}, + Kind: v1.GroupVersionKind{Group: networking.GroupName, Version: "v1beta1", Kind: "Ingress"}, Object: runtime.RawExtension{ Raw: []byte{0xff}, }, @@ -114,9 +102,7 @@ func TestHandleAdmission(t *testing.T) { err: fmt.Errorf("this is a test error"), } - if _, err := adm.HandleAdmission(review); err != nil { - t.Errorf("unexpected error: %v", err) - } + adm.HandleAdmission(review) if review.Response.Allowed { t.Fatalf("when the checker returns an error, the request should not be allowed") } @@ -126,9 +112,7 @@ func TestHandleAdmission(t *testing.T) { err: nil, } - if _, err := adm.HandleAdmission(review); err != nil { - t.Errorf("unexpected error: %v", err) - } + adm.HandleAdmission(review) if !review.Response.Allowed { t.Fatalf("when the checker returns no error, the request should be allowed") } diff --git a/internal/admission/controller/server.go b/internal/admission/controller/server.go index 74f55fd01..8c9235bdc 100644 --- a/internal/admission/controller/server.go +++ b/internal/admission/controller/server.go @@ -17,21 +17,23 @@ limitations under the License. package controller import ( - "io" + "io/ioutil" "net/http" admissionv1 "k8s.io/api/admission/v1" + admissionv1beta1 "k8s.io/api/admission/v1beta1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/klog/v2" ) -var scheme = runtime.NewScheme() +var ( + scheme = runtime.NewScheme() +) func init() { - if err := admissionv1.AddToScheme(scheme); err != nil { - klog.ErrorS(err, "Failed to add scheme") - } + admissionv1beta1.AddToScheme(scheme) + admissionv1.AddToScheme(scheme) } // AdmissionController checks if an object @@ -47,7 +49,7 @@ type AdmissionControllerServer struct { AdmissionController AdmissionController } -// NewAdmissionControllerServer instantiates an admission controller server with +// NewAdmissionControllerServer instanciates an admission controller server with // a default codec func NewAdmissionControllerServer(ac AdmissionController) *AdmissionControllerServer { return &AdmissionControllerServer{ @@ -59,7 +61,7 @@ func NewAdmissionControllerServer(ac AdmissionController) *AdmissionControllerSe func (acs *AdmissionControllerServer) ServeHTTP(w http.ResponseWriter, req *http.Request) { defer req.Body.Close() - data, err := io.ReadAll(req.Body) + data, err := ioutil.ReadAll(req.Body) if err != nil { klog.ErrorS(err, "Failed to read request body") w.WriteHeader(http.StatusBadRequest) diff --git a/pkg/util/file/file.go b/internal/file/file.go similarity index 94% rename from pkg/util/file/file.go rename to internal/file/file.go index 18205f3b0..30dc9868d 100644 --- a/pkg/util/file/file.go +++ b/internal/file/file.go @@ -19,7 +19,7 @@ package file import ( "crypto/sha1" // #nosec "encoding/hex" - "os" + "io/ioutil" "k8s.io/klog/v2" ) @@ -27,7 +27,7 @@ import ( // SHA1 returns the SHA1 of a file. func SHA1(filename string) string { hasher := sha1.New() // #nosec - s, err := os.ReadFile(filename) + s, err := ioutil.ReadFile(filename) if err != nil { klog.ErrorS(err, "Error reading file", "path", filename) return "" diff --git a/pkg/util/file/file_test.go b/internal/file/file_test.go similarity index 86% rename from pkg/util/file/file_test.go rename to internal/file/file_test.go index 9a43946b3..3256f4aa5 100644 --- a/pkg/util/file/file_test.go +++ b/internal/file/file_test.go @@ -17,7 +17,7 @@ limitations under the License. package file import ( - "os" + "io/ioutil" "testing" ) @@ -31,16 +31,12 @@ func TestSHA1(t *testing.T) { } for _, test := range tests { - f, err := os.CreateTemp("", "sha-test") + f, err := ioutil.TempFile("", "sha-test") if err != nil { t.Fatal(err) } - if _, err := f.Write(test.content); err != nil { - t.Error(err) - } - if err := f.Sync(); err != nil { - t.Error(err) - } + f.Write(test.content) + f.Sync() sha := SHA1(f.Name()) f.Close() diff --git a/pkg/util/file/filesystem.go b/internal/file/filesystem.go similarity index 95% rename from pkg/util/file/filesystem.go rename to internal/file/filesystem.go index ccb93ed06..7c0db9f12 100644 --- a/pkg/util/file/filesystem.go +++ b/internal/file/filesystem.go @@ -17,4 +17,4 @@ limitations under the License. package file // ReadWriteByUser defines linux permission to read and write files for the owner user -const ReadWriteByUser = 0o700 +const ReadWriteByUser = 0700 diff --git a/pkg/util/file/structure.go b/internal/file/structure.go similarity index 86% rename from pkg/util/file/structure.go rename to internal/file/structure.go index 7d4f26da9..485db79c8 100644 --- a/pkg/util/file/structure.go +++ b/internal/file/structure.go @@ -17,8 +17,9 @@ limitations under the License. package file import ( - "fmt" "os" + + "github.com/pkg/errors" ) const ( @@ -33,10 +34,12 @@ const ( DefaultSSLDirectory = "/etc/ingress-controller/ssl" ) -var directories = []string{ - DefaultSSLDirectory, - AuthDirectory, -} +var ( + directories = []string{ + DefaultSSLDirectory, + AuthDirectory, + } +) // CreateRequiredDirectories verifies if the required directories to // start the ingress controller exist and creates the missing ones. @@ -47,13 +50,13 @@ func CreateRequiredDirectories() error { if os.IsNotExist(err) { err = os.MkdirAll(directory, ReadWriteByUser) if err != nil { - return fmt.Errorf("creating directory %s: %w", directory, err) + return errors.Wrapf(err, "creating directory '%v'", directory) } continue } - return fmt.Errorf("checking directory %s: %w", directory, err) + return errors.Wrapf(err, "checking directory %v", directory) } } diff --git a/internal/ingress/annotations/alias/main.go b/internal/ingress/annotations/alias/main.go index 7a33e648e..6cbe4c6dc 100644 --- a/internal/ingress/annotations/alias/main.go +++ b/internal/ingress/annotations/alias/main.go @@ -20,51 +20,26 @@ import ( "sort" "strings" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - serverAliasAnnotation = "server-alias" -) - -var aliasAnnotation = parser.Annotation{ - Group: "alias", - Annotations: parser.AnnotationFields{ - serverAliasAnnotation: { - Validator: parser.ValidateArrayOfServerName, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskHigh, // High as this allows regex chars - Documentation: `this annotation can be used to define additional server - aliases for this Ingress`, - }, - }, -} - type alias struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new Alias annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return alias{ - r: r, - annotationConfig: aliasAnnotation, - } -} - -func (a alias) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations + return alias{r} } // Parse parses the annotations contained in the ingress rule // used to add an alias to the provided hosts func (a alias) Parse(ing *networking.Ingress) (interface{}, error) { - val, err := parser.GetStringAnnotation(serverAliasAnnotation, ing, a.annotationConfig.Annotations) + val, err := parser.GetStringAnnotation("server-alias", ing) if err != nil { return []string{}, err } @@ -72,7 +47,7 @@ func (a alias) Parse(ing *networking.Ingress) (interface{}, error) { aliases := sets.NewString() for _, alias := range strings.Split(val, ",") { alias = strings.TrimSpace(alias) - if alias == "" { + if len(alias) == 0 { continue } @@ -86,8 +61,3 @@ func (a alias) Parse(ing *networking.Ingress) (interface{}, error) { return l, nil } - -func (a alias) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, aliasAnnotation.Annotations) -} diff --git a/internal/ingress/annotations/alias/main_test.go b/internal/ingress/annotations/alias/main_test.go index f1d68c1f7..a482fc7c1 100644 --- a/internal/ingress/annotations/alias/main_test.go +++ b/internal/ingress/annotations/alias/main_test.go @@ -21,13 +21,13 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -var annotation = parser.GetAnnotationWithPrefix(serverAliasAnnotation) +var annotation = parser.GetAnnotationWithPrefix("server-alias") func TestParse(t *testing.T) { ap := NewParser(&resolver.Mock{}) @@ -36,20 +36,16 @@ func TestParse(t *testing.T) { } testCases := []struct { - annotations map[string]string - expected []string - skipValidation bool - wantErr bool + annotations map[string]string + expected []string }{ - {map[string]string{annotation: "a.com, b.com, , c.com"}, []string{"a.com", "b.com", "c.com"}, false, false}, - {map[string]string{annotation: "www.example.com"}, []string{"www.example.com"}, false, false}, - {map[string]string{annotation: "*.example.com,www.example.*"}, []string{"*.example.com", "www.example.*"}, false, false}, - {map[string]string{annotation: `~^www\d+\.example\.com$`}, []string{`~^www\d+\.example\.com$`}, false, false}, - {map[string]string{annotation: `www.xpto;lala`}, []string{}, false, true}, - {map[string]string{annotation: `www.xpto;lala`}, []string{"www.xpto;lala"}, true, false}, // When we skip validation no error should happen - {map[string]string{annotation: ""}, []string{}, false, true}, - {map[string]string{}, []string{}, false, true}, - {nil, []string{}, false, true}, + {map[string]string{annotation: "a.com, b.com, , c.com"}, []string{"a.com", "b.com", "c.com"}}, + {map[string]string{annotation: "www.example.com"}, []string{"www.example.com"}}, + {map[string]string{annotation: "*.example.com,www.example.*"}, []string{"*.example.com", "www.example.*"}}, + {map[string]string{annotation: `~^www\d+\.example\.com$`}, []string{`~^www\d+\.example\.com$`}}, + {map[string]string{annotation: ""}, []string{}}, + {map[string]string{}, []string{}}, + {nil, []string{}}, } ing := &networking.Ingress{ @@ -62,16 +58,7 @@ func TestParse(t *testing.T) { for _, testCase := range testCases { ing.SetAnnotations(testCase.annotations) - if testCase.skipValidation { - parser.EnableAnnotationValidation = false - } - t.Cleanup(func() { - parser.EnableAnnotationValidation = true - }) - result, err := ap.Parse(ing) - if (err != nil) != testCase.wantErr { - t.Errorf("ParseAliasAnnotation() annotation: %s, error = %v, wantErr %v", testCase.annotations, err, testCase.wantErr) - } + result, _ := ap.Parse(ing) if !reflect.DeepEqual(result, testCase.expected) { t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations) } diff --git a/internal/ingress/annotations/annotations.go b/internal/ingress/annotations/annotations.go index e10cc9be1..2c3b3c1cf 100644 --- a/internal/ingress/annotations/annotations.go +++ b/internal/ingress/annotations/annotations.go @@ -17,12 +17,16 @@ limitations under the License. package annotations import ( - "dario.cat/mergo" + "github.com/imdario/mergo" + "k8s.io/ingress-nginx/internal/ingress/annotations/canary" + "k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity" + "k8s.io/ingress-nginx/internal/ingress/annotations/proxyssl" + "k8s.io/ingress-nginx/internal/ingress/annotations/sslcipher" + "k8s.io/klog/v2" apiv1 "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog/v2" "k8s.io/ingress-nginx/internal/ingress/annotations/alias" "k8s.io/ingress-nginx/internal/ingress/annotations/auth" @@ -30,38 +34,33 @@ import ( "k8s.io/ingress-nginx/internal/ingress/annotations/authreqglobal" "k8s.io/ingress-nginx/internal/ingress/annotations/authtls" "k8s.io/ingress-nginx/internal/ingress/annotations/backendprotocol" - "k8s.io/ingress-nginx/internal/ingress/annotations/canary" "k8s.io/ingress-nginx/internal/ingress/annotations/clientbodybuffersize" "k8s.io/ingress-nginx/internal/ingress/annotations/connection" "k8s.io/ingress-nginx/internal/ingress/annotations/cors" - "k8s.io/ingress-nginx/internal/ingress/annotations/customheaders" "k8s.io/ingress-nginx/internal/ingress/annotations/customhttperrors" "k8s.io/ingress-nginx/internal/ingress/annotations/defaultbackend" - "k8s.io/ingress-nginx/internal/ingress/annotations/disableproxyintercepterrors" "k8s.io/ingress-nginx/internal/ingress/annotations/fastcgi" + "k8s.io/ingress-nginx/internal/ingress/annotations/globalratelimit" "k8s.io/ingress-nginx/internal/ingress/annotations/http2pushpreload" - "k8s.io/ingress-nginx/internal/ingress/annotations/ipallowlist" - "k8s.io/ingress-nginx/internal/ingress/annotations/ipdenylist" + "k8s.io/ingress-nginx/internal/ingress/annotations/influxdb" + "k8s.io/ingress-nginx/internal/ingress/annotations/ipwhitelist" "k8s.io/ingress-nginx/internal/ingress/annotations/loadbalancing" "k8s.io/ingress-nginx/internal/ingress/annotations/log" "k8s.io/ingress-nginx/internal/ingress/annotations/mirror" - "k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity" - "k8s.io/ingress-nginx/internal/ingress/annotations/opentelemetry" + "k8s.io/ingress-nginx/internal/ingress/annotations/opentracing" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/annotations/portinredirect" "k8s.io/ingress-nginx/internal/ingress/annotations/proxy" - "k8s.io/ingress-nginx/internal/ingress/annotations/proxyssl" "k8s.io/ingress-nginx/internal/ingress/annotations/ratelimit" "k8s.io/ingress-nginx/internal/ingress/annotations/redirect" "k8s.io/ingress-nginx/internal/ingress/annotations/rewrite" "k8s.io/ingress-nginx/internal/ingress/annotations/satisfy" + "k8s.io/ingress-nginx/internal/ingress/annotations/secureupstream" "k8s.io/ingress-nginx/internal/ingress/annotations/serversnippet" "k8s.io/ingress-nginx/internal/ingress/annotations/serviceupstream" "k8s.io/ingress-nginx/internal/ingress/annotations/sessionaffinity" "k8s.io/ingress-nginx/internal/ingress/annotations/snippet" - "k8s.io/ingress-nginx/internal/ingress/annotations/sslcipher" "k8s.io/ingress-nginx/internal/ingress/annotations/sslpassthrough" - "k8s.io/ingress-nginx/internal/ingress/annotations/streamsnippet" "k8s.io/ingress-nginx/internal/ingress/annotations/upstreamhashby" "k8s.io/ingress-nginx/internal/ingress/annotations/upstreamvhost" "k8s.io/ingress-nginx/internal/ingress/annotations/xforwardedprefix" @@ -75,47 +74,47 @@ const DeniedKeyName = "Denied" // Ingress defines the valid annotations present in one NGINX Ingress rule type Ingress struct { metav1.ObjectMeta - BackendProtocol string - Aliases []string - BasicDigestAuth auth.Config - Canary canary.Config - CertificateAuth authtls.Config - ClientBodyBufferSize string - CustomHeaders customheaders.Config - ConfigurationSnippet string - Connection connection.Config - CorsConfig cors.Config - CustomHTTPErrors []int - DisableProxyInterceptErrors bool - DefaultBackend *apiv1.Service - FastCGI fastcgi.Config - Denied *string - ExternalAuth authreq.Config - EnableGlobalAuth bool - HTTP2PushPreload bool - Opentelemetry opentelemetry.Config - Proxy proxy.Config - ProxySSL proxyssl.Config - RateLimit ratelimit.Config - Redirect redirect.Config - Rewrite rewrite.Config - Satisfy string - ServerSnippet string - ServiceUpstream bool - SessionAffinity sessionaffinity.Config - SSLPassthrough bool - UsePortInRedirects bool - UpstreamHashBy upstreamhashby.Config - LoadBalancing string - UpstreamVhost string - Denylist ipdenylist.SourceRange - XForwardedPrefix string - SSLCipher sslcipher.Config - Logs log.Config - ModSecurity modsecurity.Config - Mirror mirror.Config - StreamSnippet string - Allowlist ipallowlist.SourceRange + BackendProtocol string + Aliases []string + BasicDigestAuth auth.Config + Canary canary.Config + CertificateAuth authtls.Config + ClientBodyBufferSize string + ConfigurationSnippet string + Connection connection.Config + CorsConfig cors.Config + CustomHTTPErrors []int + DefaultBackend *apiv1.Service + //TODO: Change this back into an error when https://github.com/imdario/mergo/issues/100 is resolved + FastCGI fastcgi.Config + Denied *string + ExternalAuth authreq.Config + EnableGlobalAuth bool + HTTP2PushPreload bool + Opentracing opentracing.Config + Proxy proxy.Config + ProxySSL proxyssl.Config + RateLimit ratelimit.Config + GlobalRateLimit globalratelimit.Config + Redirect redirect.Config + Rewrite rewrite.Config + Satisfy string + SecureUpstream secureupstream.Config + ServerSnippet string + ServiceUpstream bool + SessionAffinity sessionaffinity.Config + SSLPassthrough bool + UsePortInRedirects bool + UpstreamHashBy upstreamhashby.Config + LoadBalancing string + UpstreamVhost string + Whitelist ipwhitelist.SourceRange + XForwardedPrefix string + SSLCipher sslcipher.Config + Logs log.Config + InfluxDB influxdb.Config + ModSecurity modsecurity.Config + Mirror mirror.Config } // Extractor defines the annotation parsers to be used in the extraction of annotations @@ -123,76 +122,64 @@ type Extractor struct { annotations map[string]parser.IngressAnnotation } -func NewAnnotationFactory(cfg resolver.Resolver) map[string]parser.IngressAnnotation { - return map[string]parser.IngressAnnotation{ - "Aliases": alias.NewParser(cfg), - "BasicDigestAuth": auth.NewParser(auth.AuthDirectory, cfg), - "Canary": canary.NewParser(cfg), - "CertificateAuth": authtls.NewParser(cfg), - "ClientBodyBufferSize": clientbodybuffersize.NewParser(cfg), - "CustomHeaders": customheaders.NewParser(cfg), - "ConfigurationSnippet": snippet.NewParser(cfg), - "Connection": connection.NewParser(cfg), - "CorsConfig": cors.NewParser(cfg), - "CustomHTTPErrors": customhttperrors.NewParser(cfg), - "DisableProxyInterceptErrors": disableproxyintercepterrors.NewParser(cfg), - "DefaultBackend": defaultbackend.NewParser(cfg), - "FastCGI": fastcgi.NewParser(cfg), - "ExternalAuth": authreq.NewParser(cfg), - "EnableGlobalAuth": authreqglobal.NewParser(cfg), - "HTTP2PushPreload": http2pushpreload.NewParser(cfg), - "Opentelemetry": opentelemetry.NewParser(cfg), - "Proxy": proxy.NewParser(cfg), - "ProxySSL": proxyssl.NewParser(cfg), - "RateLimit": ratelimit.NewParser(cfg), - "Redirect": redirect.NewParser(cfg), - "Rewrite": rewrite.NewParser(cfg), - "Satisfy": satisfy.NewParser(cfg), - "ServerSnippet": serversnippet.NewParser(cfg), - "ServiceUpstream": serviceupstream.NewParser(cfg), - "SessionAffinity": sessionaffinity.NewParser(cfg), - "SSLPassthrough": sslpassthrough.NewParser(cfg), - "UsePortInRedirects": portinredirect.NewParser(cfg), - "UpstreamHashBy": upstreamhashby.NewParser(cfg), - "LoadBalancing": loadbalancing.NewParser(cfg), - "UpstreamVhost": upstreamvhost.NewParser(cfg), - "Allowlist": ipallowlist.NewParser(cfg), - "Denylist": ipdenylist.NewParser(cfg), - "XForwardedPrefix": xforwardedprefix.NewParser(cfg), - "SSLCipher": sslcipher.NewParser(cfg), - "Logs": log.NewParser(cfg), - "BackendProtocol": backendprotocol.NewParser(cfg), - "ModSecurity": modsecurity.NewParser(cfg), - "Mirror": mirror.NewParser(cfg), - "StreamSnippet": streamsnippet.NewParser(cfg), - } -} - // NewAnnotationExtractor creates a new annotations extractor func NewAnnotationExtractor(cfg resolver.Resolver) Extractor { return Extractor{ - NewAnnotationFactory(cfg), + map[string]parser.IngressAnnotation{ + "Aliases": alias.NewParser(cfg), + "BasicDigestAuth": auth.NewParser(auth.AuthDirectory, cfg), + "Canary": canary.NewParser(cfg), + "CertificateAuth": authtls.NewParser(cfg), + "ClientBodyBufferSize": clientbodybuffersize.NewParser(cfg), + "ConfigurationSnippet": snippet.NewParser(cfg), + "Connection": connection.NewParser(cfg), + "CorsConfig": cors.NewParser(cfg), + "CustomHTTPErrors": customhttperrors.NewParser(cfg), + "DefaultBackend": defaultbackend.NewParser(cfg), + "FastCGI": fastcgi.NewParser(cfg), + "ExternalAuth": authreq.NewParser(cfg), + "EnableGlobalAuth": authreqglobal.NewParser(cfg), + "HTTP2PushPreload": http2pushpreload.NewParser(cfg), + "Opentracing": opentracing.NewParser(cfg), + "Proxy": proxy.NewParser(cfg), + "ProxySSL": proxyssl.NewParser(cfg), + "RateLimit": ratelimit.NewParser(cfg), + "GlobalRateLimit": globalratelimit.NewParser(cfg), + "Redirect": redirect.NewParser(cfg), + "Rewrite": rewrite.NewParser(cfg), + "Satisfy": satisfy.NewParser(cfg), + "SecureUpstream": secureupstream.NewParser(cfg), + "ServerSnippet": serversnippet.NewParser(cfg), + "ServiceUpstream": serviceupstream.NewParser(cfg), + "SessionAffinity": sessionaffinity.NewParser(cfg), + "SSLPassthrough": sslpassthrough.NewParser(cfg), + "UsePortInRedirects": portinredirect.NewParser(cfg), + "UpstreamHashBy": upstreamhashby.NewParser(cfg), + "LoadBalancing": loadbalancing.NewParser(cfg), + "UpstreamVhost": upstreamvhost.NewParser(cfg), + "Whitelist": ipwhitelist.NewParser(cfg), + "XForwardedPrefix": xforwardedprefix.NewParser(cfg), + "SSLCipher": sslcipher.NewParser(cfg), + "Logs": log.NewParser(cfg), + "InfluxDB": influxdb.NewParser(cfg), + "BackendProtocol": backendprotocol.NewParser(cfg), + "ModSecurity": modsecurity.NewParser(cfg), + "Mirror": mirror.NewParser(cfg), + }, } } // Extract extracts the annotations from an Ingress -func (e Extractor) Extract(ing *networking.Ingress) (*Ingress, error) { +func (e Extractor) Extract(ing *networking.Ingress) *Ingress { pia := &Ingress{ ObjectMeta: ing.ObjectMeta, } data := make(map[string]interface{}) for name, annotationParser := range e.annotations { - if err := annotationParser.Validate(ing.GetAnnotations()); err != nil { - return nil, errors.NewRiskyAnnotations(name) - } val, err := annotationParser.Parse(ing) klog.V(5).InfoS("Parsing Ingress annotation", "name", name, "ingress", klog.KObj(ing), "value", val) if err != nil { - if errors.IsValidationError(err) { - klog.ErrorS(err, "ingress contains invalid annotation value") - return nil, err - } if errors.IsMissingAnnotations(err) { continue } @@ -230,5 +217,5 @@ func (e Extractor) Extract(ing *networking.Ingress) (*Ingress, error) { klog.ErrorS(err, "unexpected error merging extracted annotations") } - return pia, nil + return pia } diff --git a/internal/ingress/annotations/annotations_test.go b/internal/ingress/annotations/annotations_test.go index 5df3cdc0e..1a428e729 100644 --- a/internal/ingress/annotations/annotations_test.go +++ b/internal/ingress/annotations/annotations_test.go @@ -20,8 +20,9 @@ import ( "testing" apiv1 "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/defaults" @@ -29,34 +30,29 @@ import ( ) var ( - annotationPassthrough = parser.GetAnnotationWithPrefix("ssl-passthrough") - annotationAffinityType = parser.GetAnnotationWithPrefix("affinity") - annotationAffinityMode = parser.GetAnnotationWithPrefix("affinity-mode") - annotationAffinityCanaryBehavior = parser.GetAnnotationWithPrefix("affinity-canary-behavior") - annotationCorsEnabled = parser.GetAnnotationWithPrefix("enable-cors") - annotationCorsAllowMethods = parser.GetAnnotationWithPrefix("cors-allow-methods") - annotationCorsAllowHeaders = parser.GetAnnotationWithPrefix("cors-allow-headers") - annotationCorsExposeHeaders = parser.GetAnnotationWithPrefix("cors-expose-headers") - annotationCorsAllowCredentials = parser.GetAnnotationWithPrefix("cors-allow-credentials") - defaultCorsMethods = "GET, PUT, POST, DELETE, PATCH, OPTIONS" - defaultCorsHeaders = "DNT,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization" - annotationAffinityCookieName = parser.GetAnnotationWithPrefix("session-cookie-name") - annotationUpstreamHashBy = parser.GetAnnotationWithPrefix("upstream-hash-by") - annotationCustomHTTPErrors = parser.GetAnnotationWithPrefix("custom-http-errors") - annotationCustomHeaders = parser.GetAnnotationWithPrefix("custom-headers") + annotationPassthrough = parser.GetAnnotationWithPrefix("ssl-passthrough") + annotationAffinityType = parser.GetAnnotationWithPrefix("affinity") + annotationAffinityMode = parser.GetAnnotationWithPrefix("affinity-mode") + annotationCorsEnabled = parser.GetAnnotationWithPrefix("enable-cors") + annotationCorsAllowMethods = parser.GetAnnotationWithPrefix("cors-allow-methods") + annotationCorsAllowHeaders = parser.GetAnnotationWithPrefix("cors-allow-headers") + annotationCorsExposeHeaders = parser.GetAnnotationWithPrefix("cors-expose-headers") + annotationCorsAllowCredentials = parser.GetAnnotationWithPrefix("cors-allow-credentials") + defaultCorsMethods = "GET, PUT, POST, DELETE, PATCH, OPTIONS" + defaultCorsHeaders = "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization" + annotationAffinityCookieName = parser.GetAnnotationWithPrefix("session-cookie-name") + annotationUpstreamHashBy = parser.GetAnnotationWithPrefix("upstream-hash-by") + annotationCustomHTTPErrors = parser.GetAnnotationWithPrefix("custom-http-errors") ) type mockCfg struct { resolver.Mock - MockSecrets map[string]*apiv1.Secret - MockServices map[string]*apiv1.Service - MockConfigMaps map[string]*apiv1.ConfigMap + MockSecrets map[string]*apiv1.Secret + MockServices map[string]*apiv1.Service } func (m mockCfg) GetDefaultBackend() defaults.Backend { - return defaults.Backend{ - AllowedResponseHeaders: []string{"Content-Type"}, - } + return defaults.Backend{} } func (m mockCfg) GetSecret(name string) (*apiv1.Secret, error) { @@ -67,16 +63,8 @@ func (m mockCfg) GetService(name string) (*apiv1.Service, error) { return m.MockServices[name], nil } -func (m mockCfg) GetConfigMap(name string) (*apiv1.ConfigMap, error) { - return m.MockConfigMaps[name], nil -} - func (m mockCfg) GetAuthCertificate(name string) (*resolver.AuthSSLCert, error) { - secret, err := m.GetSecret(name) - if err != nil { - return nil, err - } - if secret != nil { + if secret, _ := m.GetSecret(name); secret != nil { return &resolver.AuthSSLCert{ Secret: name, CAFileName: "/opt/ca.pem", @@ -88,12 +76,8 @@ func (m mockCfg) GetAuthCertificate(name string) (*resolver.AuthSSLCert, error) func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -102,13 +86,9 @@ func buildIngress() *networking.Ingress { Namespace: apiv1.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -133,7 +113,6 @@ func TestSSLPassthrough(t *testing.T) { ec := NewAnnotationExtractor(mockCfg{}) ing := buildIngress() - //nolint:goconst //already a constant fooAnns := []struct { annotations map[string]string er bool @@ -147,11 +126,8 @@ func TestSSLPassthrough(t *testing.T) { for _, foo := range fooAnns { ing.SetAnnotations(foo.annotations) - r, err := ec.Extract(ing) - if err != nil { - t.Errorf("Errors should be null: %v", err) - } - if r.SSLPassthrough != foo.er { + r := ec.Extract(ing).SSLPassthrough + if r != foo.er { t.Errorf("Returned %v but expected %v", r, foo.er) } } @@ -174,11 +150,8 @@ func TestUpstreamHashBy(t *testing.T) { for _, foo := range fooAnns { ing.SetAnnotations(foo.annotations) - r, err := ec.Extract(ing) - if err != nil { - t.Errorf("error should be null: %v", err) - } - if r.UpstreamHashBy.UpstreamHashBy != foo.er { + r := ec.Extract(ing).UpstreamHashBy.UpstreamHashBy + if r != foo.er { t.Errorf("Returned %v but expected %v", r, foo.er) } } @@ -189,42 +162,29 @@ func TestAffinitySession(t *testing.T) { ing := buildIngress() fooAnns := []struct { - annotations map[string]string - affinitytype string - affinitymode string - cookiename string - canarybehavior string + annotations map[string]string + affinitytype string + affinitymode string + name string }{ - {map[string]string{annotationAffinityType: "cookie", annotationAffinityMode: "balanced", annotationAffinityCookieName: "route", annotationAffinityCanaryBehavior: ""}, "cookie", "balanced", "route", ""}, - {map[string]string{annotationAffinityType: "cookie", annotationAffinityMode: "persistent", annotationAffinityCookieName: "route1", annotationAffinityCanaryBehavior: "sticky"}, "cookie", "persistent", "route1", "sticky"}, - {map[string]string{annotationAffinityType: "cookie", annotationAffinityMode: "balanced", annotationAffinityCookieName: "", annotationAffinityCanaryBehavior: "legacy"}, "cookie", "balanced", "INGRESSCOOKIE", "legacy"}, - {map[string]string{}, "", "", "", ""}, - {nil, "", "", "", ""}, + {map[string]string{annotationAffinityType: "cookie", annotationAffinityMode: "balanced", annotationAffinityCookieName: "route"}, "cookie", "balanced", "route"}, + {map[string]string{annotationAffinityType: "cookie", annotationAffinityMode: "persistent", annotationAffinityCookieName: "route1"}, "cookie", "persistent", "route1"}, + {map[string]string{annotationAffinityType: "cookie", annotationAffinityMode: "balanced", annotationAffinityCookieName: ""}, "cookie", "balanced", "INGRESSCOOKIE"}, + {map[string]string{}, "", "", ""}, + {nil, "", "", ""}, } for _, foo := range fooAnns { ing.SetAnnotations(foo.annotations) - rann, err := ec.Extract(ing) - if err != nil { - t.Errorf("error should be null: %v", err) - } - r := rann.SessionAffinity - t.Logf("Testing pass %v %v", foo.affinitytype, foo.cookiename) - - if r.Type != foo.affinitytype { - t.Errorf("Returned %v but expected %v for Type", r.Type, foo.affinitytype) - } + r := ec.Extract(ing).SessionAffinity + t.Logf("Testing pass %v %v", foo.affinitytype, foo.name) if r.Mode != foo.affinitymode { - t.Errorf("Returned %v but expected %v for Mode", r.Mode, foo.affinitymode) + t.Errorf("Returned %v but expected %v for Name", r.Mode, foo.affinitymode) } - if r.CanaryBehavior != foo.canarybehavior { - t.Errorf("Returned %v but expected %v for CanaryBehavior", r.CanaryBehavior, foo.canarybehavior) - } - - if r.Cookie.Name != foo.cookiename { - t.Errorf("Returned %v but expected %v for Cookie.Name", r.Cookie.Name, foo.cookiename) + if r.Cookie.Name != foo.name { + t.Errorf("Returned %v but expected %v for Name", r.Cookie.Name, foo.name) } } } @@ -238,24 +198,20 @@ func TestCors(t *testing.T) { corsenabled bool methods string headers string - origin []string + origin string credentials bool expose string }{ - {map[string]string{annotationCorsEnabled: "true"}, true, defaultCorsMethods, defaultCorsHeaders, []string{"*"}, true, ""}, - {map[string]string{annotationCorsEnabled: "true", annotationCorsAllowMethods: "POST, GET, OPTIONS", annotationCorsAllowHeaders: "$nginx_version", annotationCorsAllowCredentials: "false", annotationCorsExposeHeaders: "X-CustomResponseHeader"}, true, "POST, GET, OPTIONS", defaultCorsHeaders, []string{"*"}, false, "X-CustomResponseHeader"}, - {map[string]string{annotationCorsEnabled: "true", annotationCorsAllowCredentials: "false"}, true, defaultCorsMethods, defaultCorsHeaders, []string{"*"}, false, ""}, - {map[string]string{}, false, defaultCorsMethods, defaultCorsHeaders, []string{"*"}, true, ""}, - {nil, false, defaultCorsMethods, defaultCorsHeaders, []string{"*"}, true, ""}, + {map[string]string{annotationCorsEnabled: "true"}, true, defaultCorsMethods, defaultCorsHeaders, "*", true, ""}, + {map[string]string{annotationCorsEnabled: "true", annotationCorsAllowMethods: "POST, GET, OPTIONS", annotationCorsAllowHeaders: "$nginx_version", annotationCorsAllowCredentials: "false", annotationCorsExposeHeaders: "X-CustomResponseHeader"}, true, "POST, GET, OPTIONS", defaultCorsHeaders, "*", false, "X-CustomResponseHeader"}, + {map[string]string{annotationCorsEnabled: "true", annotationCorsAllowCredentials: "false"}, true, defaultCorsMethods, defaultCorsHeaders, "*", false, ""}, + {map[string]string{}, false, defaultCorsMethods, defaultCorsHeaders, "*", true, ""}, + {nil, false, defaultCorsMethods, defaultCorsHeaders, "*", true, ""}, } for _, foo := range fooAnns { ing.SetAnnotations(foo.annotations) - rann, err := ec.Extract(ing) - if err != nil { - t.Errorf("error should be null: %v", err) - } - r := rann.CorsConfig + r := ec.Extract(ing).CorsConfig t.Logf("Testing pass %v %v %v %v %v", foo.corsenabled, foo.methods, foo.headers, foo.origin, foo.credentials) if r.CorsEnabled != foo.corsenabled { @@ -270,22 +226,16 @@ func TestCors(t *testing.T) { t.Errorf("Returned %v but expected %v for Cors Methods", r.CorsAllowMethods, foo.methods) } - if len(r.CorsAllowOrigin) != len(foo.origin) { - t.Errorf("Lengths of Cors Origins are not equal. Expected %v - Actual %v", r.CorsAllowOrigin, foo.origin) - } - - for i, v := range r.CorsAllowOrigin { - if v != foo.origin[i] { - t.Errorf("Values of Cors Origins are not equal. Expected %v - Actual %v", r.CorsAllowOrigin, foo.origin) - } + if r.CorsAllowOrigin != foo.origin { + t.Errorf("Returned %v but expected %v for Cors Methods", r.CorsAllowOrigin, foo.origin) } if r.CorsAllowCredentials != foo.credentials { - t.Errorf("Returned %v but expected %v for Cors Credentials", r.CorsAllowCredentials, foo.credentials) + t.Errorf("Returned %v but expected %v for Cors Methods", r.CorsAllowCredentials, foo.credentials) } + } } - func TestCustomHTTPErrors(t *testing.T) { ec := NewAnnotationExtractor(mockCfg{}) ing := buildIngress() @@ -304,11 +254,7 @@ func TestCustomHTTPErrors(t *testing.T) { for _, foo := range fooAnns { ing.SetAnnotations(foo.annotations) - rann, err := ec.Extract(ing) - if err != nil { - t.Errorf("error should be null: %v", err) - } - r := rann.CustomHTTPErrors + r := ec.Extract(ing).CustomHTTPErrors // Check that expected codes were created for i := range foo.er { @@ -326,43 +272,47 @@ func TestCustomHTTPErrors(t *testing.T) { } } -func TestCustomResponseHeaders(t *testing.T) { - mockObj := mockCfg{} - mockObj.MockConfigMaps = map[string]*apiv1.ConfigMap{} - mockObj.MockConfigMaps["custom-headers"] = &apiv1.ConfigMap{Data: map[string]string{"Content-Type": "application/json"}} - mockObj.MockConfigMaps["empty-custom-headers"] = &apiv1.ConfigMap{Data: map[string]string{}} +/* +func TestMergeLocationAnnotations(t *testing.T) { + // initial parameters + keys := []string{"BasicDigestAuth", "CorsConfig", "ExternalAuth", "RateLimit", "Redirect", "Rewrite", "Whitelist", "Proxy", "UsePortInRedirects"} - ec := NewAnnotationExtractor(mockObj) - ing := buildIngress() - fooAnns := []struct { - annotations map[string]string - headers map[string]string - }{ - {map[string]string{annotationCustomHeaders: "custom-headers"}, map[string]string{"Content-Type": "application/json"}}, - {map[string]string{annotationCustomHeaders: "empty-custom-headers"}, map[string]string{}}, - {nil, map[string]string{}}, + loc := ingress.Location{} + annotations := &Ingress{ + BasicDigestAuth: &auth.Config{}, + CorsConfig: &cors.Config{}, + ExternalAuth: &authreq.Config{}, + RateLimit: &ratelimit.Config{}, + Redirect: &redirect.Config{}, + Rewrite: &rewrite.Config{}, + Whitelist: &ipwhitelist.SourceRange{}, + Proxy: &proxy.Config{}, + UsePortInRedirects: true, } - for _, foo := range fooAnns { - ing.SetAnnotations(foo.annotations) - rann, err := ec.Extract(ing) - if err != nil { - t.Errorf("error should be null: %v", err) - } - r := rann.CustomHeaders.Headers + // create test table + type fooMergeLocationAnnotationsStruct struct { + fName string + er interface{} + } + fooTests := []fooMergeLocationAnnotationsStruct{} + for name, value := range keys { + fva := fooMergeLocationAnnotationsStruct{name, value} + fooTests = append(fooTests, fva) + } - // Check that expected headers were created - for i := range foo.headers { - if r[i] != foo.headers[i] { - t.Errorf("Returned %v but expected %v", r, foo.headers) - } - } + // execute test + MergeWithLocation(&loc, annotations) - // Check that no unexpected headers were created - for i := range r { - if r[i] != foo.headers[i] { - t.Errorf("Returned %v but expected %v", r, foo.headers) - } + // check result + for _, foo := range fooTests { + fv := reflect.ValueOf(loc).FieldByName(foo.fName).Interface() + if !reflect.DeepEqual(fv, foo.er) { + t.Errorf("Returned %v but expected %v for the field %s", fv, foo.er, foo.fName) } } + if _, ok := annotations[DeniedKeyName]; ok { + t.Errorf("%s should be removed after mergeLocationAnnotations", DeniedKeyName) + } } +*/ diff --git a/internal/ingress/annotations/auth/main.go b/internal/ingress/annotations/auth/main.go index 7c7fde7b8..9edfc1751 100644 --- a/internal/ingress/annotations/auth/main.go +++ b/internal/ingress/annotations/auth/main.go @@ -18,70 +18,28 @@ package auth import ( "fmt" - "os" + "io/ioutil" "regexp" "strings" + "github.com/pkg/errors" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/client-go/tools/cache" + "k8s.io/ingress-nginx/internal/file" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" ing_errors "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" - "k8s.io/ingress-nginx/pkg/util/file" -) - -const ( - authSecretTypeAnnotation = "auth-secret-type" //#nosec G101 - authRealmAnnotation = "auth-realm" - authTypeAnnotation = "auth-type" - // This should be exported as it is imported by other packages - AuthSecretAnnotation = "auth-secret" //#nosec G101 ) var ( - authTypeRegex = regexp.MustCompile(`basic|digest`) - authSecretTypeRegex = regexp.MustCompile(`auth-file|auth-map`) - + authTypeRegex = regexp.MustCompile(`basic|digest`) // AuthDirectory default directory used to store files // to authenticate request AuthDirectory = "/etc/ingress-controller/auth" ) -var AuthSecretConfig = parser.AnnotationConfig{ - Validator: parser.ValidateRegex(parser.BasicCharsRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, // Medium as it allows a subset of chars - Documentation: `This annotation defines the name of the Secret that contains the usernames and passwords which are granted access to the paths defined in the Ingress rules. `, -} - -var authSecretAnnotations = parser.Annotation{ - Group: "authentication", - Annotations: parser.AnnotationFields{ - AuthSecretAnnotation: AuthSecretConfig, - authSecretTypeAnnotation: { - Validator: parser.ValidateRegex(authSecretTypeRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation what is the format of auth-secret value. Can be "auth-file" that defines the content of an htpasswd file, or "auth-map" where each key - is a user and each value is the password.`, - }, - authRealmAnnotation: { - Validator: parser.ValidateRegex(parser.CharsWithSpace, false), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, // Medium as it allows a subset of chars - Documentation: `This annotation defines the realm (message) that should be shown to user when authentication is requested.`, - }, - authTypeAnnotation: { - Validator: parser.ValidateRegex(authTypeRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation defines the basic authentication type. Should be "basic" or "digest"`, - }, - }, -} - const ( fileAuth = "auth-file" mapAuth = "auth-map" @@ -128,18 +86,13 @@ func (bd1 *Config) Equal(bd2 *Config) bool { } type auth struct { - r resolver.Resolver - authDirectory string - annotationConfig parser.Annotation + r resolver.Resolver + authDirectory string } // NewParser creates a new authentication annotation parser func NewParser(authDirectory string, r resolver.Resolver) parser.IngressAnnotation { - return auth{ - r: r, - authDirectory: authDirectory, - annotationConfig: authSecretAnnotations, - } + return auth{r, authDirectory} } // Parse parses the annotations contained in the ingress @@ -147,7 +100,7 @@ func NewParser(authDirectory string, r resolver.Resolver) parser.IngressAnnotati // and generated an htpasswd compatible file to be used as source // during the authentication process func (a auth) Parse(ing *networking.Ingress) (interface{}, error) { - at, err := parser.GetStringAnnotation(authTypeAnnotation, ing, a.annotationConfig.Annotations) + at, err := parser.GetStringAnnotation("auth-type", ing) if err != nil { return nil, err } @@ -157,51 +110,38 @@ func (a auth) Parse(ing *networking.Ingress) (interface{}, error) { } var secretType string - secretType, err = parser.GetStringAnnotation(authSecretTypeAnnotation, ing, a.annotationConfig.Annotations) + secretType, err = parser.GetStringAnnotation("auth-secret-type", ing) if err != nil { - if ing_errors.IsValidationError(err) { - return nil, err - } secretType = fileAuth } - s, err := parser.GetStringAnnotation(AuthSecretAnnotation, ing, a.annotationConfig.Annotations) + s, err := parser.GetStringAnnotation("auth-secret", ing) if err != nil { - return nil, ing_errors.LocationDeniedError{ - Reason: fmt.Errorf("error reading secret name from annotation: %w", err), + return nil, ing_errors.LocationDenied{ + Reason: errors.Wrap(err, "error reading secret name from annotation"), } } sns, sname, err := cache.SplitMetaNamespaceKey(s) if err != nil { - return nil, ing_errors.LocationDeniedError{ - Reason: fmt.Errorf("error reading secret name from annotation: %w", err), + return nil, ing_errors.LocationDenied{ + Reason: errors.Wrap(err, "error reading secret name from annotation"), } } if sns == "" { sns = ing.Namespace } - secCfg := a.r.GetSecurityConfiguration() - // We don't accept different namespaces for secrets. - if !secCfg.AllowCrossNamespaceResources && sns != ing.Namespace { - return nil, ing_errors.LocationDeniedError{ - Reason: fmt.Errorf("cross namespace usage of secrets is not allowed"), - } - } name := fmt.Sprintf("%v/%v", sns, sname) secret, err := a.r.GetSecret(name) if err != nil { - return nil, ing_errors.LocationDeniedError{ - Reason: fmt.Errorf("unexpected error reading secret %s: %w", name, err), + return nil, ing_errors.LocationDenied{ + Reason: errors.Wrapf(err, "unexpected error reading secret %v", name), } } - realm, err := parser.GetStringAnnotation(authRealmAnnotation, ing, a.annotationConfig.Annotations) - if ing_errors.IsValidationError(err) { - return nil, err - } + realm, _ := parser.GetStringAnnotation("auth-realm", ing) passFilename := fmt.Sprintf("%v/%v-%v-%v.passwd", a.authDirectory, ing.GetNamespace(), ing.UID, secret.UID) @@ -217,8 +157,8 @@ func (a auth) Parse(ing *networking.Ingress) (interface{}, error) { return nil, err } default: - return nil, ing_errors.LocationDeniedError{ - Reason: fmt.Errorf("invalid auth-secret-type in annotation, must be 'auth-file' or 'auth-map': %w", err), + return nil, ing_errors.LocationDenied{ + Reason: errors.Wrap(err, "invalid auth-secret-type in annotation, must be 'auth-file' or 'auth-map'"), } } @@ -238,15 +178,15 @@ func (a auth) Parse(ing *networking.Ingress) (interface{}, error) { func dumpSecretAuthFile(filename string, secret *api.Secret) error { val, ok := secret.Data["auth"] if !ok { - return ing_errors.LocationDeniedError{ - Reason: fmt.Errorf("the secret %s does not contain a key with value auth", secret.Name), + return ing_errors.LocationDenied{ + Reason: errors.Errorf("the secret %v does not contain a key with value auth", secret.Name), } } - err := os.WriteFile(filename, val, file.ReadWriteByUser) + err := ioutil.WriteFile(filename, val, file.ReadWriteByUser) if err != nil { - return ing_errors.LocationDeniedError{ - Reason: fmt.Errorf("unexpected error creating password file: %w", err), + return ing_errors.LocationDenied{ + Reason: errors.Wrap(err, "unexpected error creating password file"), } } @@ -258,25 +198,16 @@ func dumpSecretAuthMap(filename string, secret *api.Secret) error { for user, pass := range secret.Data { builder.WriteString(user) builder.WriteString(":") - builder.Write(pass) + builder.WriteString(string(pass)) builder.WriteString("\n") } - err := os.WriteFile(filename, []byte(builder.String()), file.ReadWriteByUser) + err := ioutil.WriteFile(filename, []byte(builder.String()), file.ReadWriteByUser) if err != nil { - return ing_errors.LocationDeniedError{ - Reason: fmt.Errorf("unexpected error creating password file: %w", err), + return ing_errors.LocationDenied{ + Reason: errors.Wrap(err, "unexpected error creating password file"), } } return nil } - -func (a auth) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a auth) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, authSecretAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/auth/main_test.go b/internal/ingress/annotations/auth/main_test.go index 868f31a05..b02741d01 100644 --- a/internal/ingress/annotations/auth/main_test.go +++ b/internal/ingress/annotations/auth/main_test.go @@ -17,38 +17,27 @@ limitations under the License. package auth import ( - "errors" "fmt" + "io/ioutil" "os" "testing" "time" + "github.com/pkg/errors" + api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/cache" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" ing_errors "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -//nolint:gosec // Ignore hardcoded credentials error in testing -const ( - authType = "basic" - authRealm = "-realm-" - defaultDemoSecret = "default/demo-secret" - othernsDemoSecret = "otherns/demo-secret" - demoSecret = "demo-secret" -) - func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -57,13 +46,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -89,19 +74,14 @@ type mockSecret struct { } func (m mockSecret) GetSecret(name string) (*api.Secret, error) { - if name != defaultDemoSecret && name != othernsDemoSecret { - return nil, fmt.Errorf("there is no secret with name %v", name) - } - - ns, _, err := cache.SplitMetaNamespaceKey(name) - if err != nil { - return nil, err + if name != "default/demo-secret" { + return nil, errors.Errorf("there is no secret with name %v", name) } return &api.Secret{ ObjectMeta: meta_v1.ObjectMeta{ - Namespace: ns, - Name: demoSecret, + Namespace: api.NamespaceDefault, + Name: "demo-secret", }, Data: map[string][]byte{"auth": []byte("foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0")}, }, nil @@ -121,91 +101,13 @@ func TestIngressAuthBadAuthType(t *testing.T) { ing := buildIngress() data := map[string]string{} - data[parser.GetAnnotationWithPrefix(authTypeAnnotation)] = "invalid" + data[parser.GetAnnotationWithPrefix("auth-type")] = "invalid" ing.SetAnnotations(data) _, dir, _ := dummySecretContent(t) defer os.RemoveAll(dir) - expected := ing_errors.NewValidationError("nginx.ingress.kubernetes.io/auth-type") - _, err := NewParser(dir, &mockSecret{}).Parse(ing) - if err.Error() != expected.Error() { - t.Errorf("expected '%v' but got '%v'", expected, err) - } -} - -func TestIngressInvalidRealm(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix(authTypeAnnotation)] = authType - data[parser.GetAnnotationWithPrefix(authRealmAnnotation)] = "something weird ; location trying to { break }" - data[parser.GetAnnotationWithPrefix(AuthSecretAnnotation)] = demoSecret - ing.SetAnnotations(data) - - _, dir, _ := dummySecretContent(t) - defer os.RemoveAll(dir) - - expected := ing_errors.NewValidationError("nginx.ingress.kubernetes.io/auth-realm") - _, err := NewParser(dir, &mockSecret{}).Parse(ing) - if err.Error() != expected.Error() { - t.Errorf("expected '%v' but got '%v'", expected, err) - } -} - -func TestIngressInvalidDifferentNamespace(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix(authTypeAnnotation)] = authType - data[parser.GetAnnotationWithPrefix(AuthSecretAnnotation)] = othernsDemoSecret - ing.SetAnnotations(data) - - _, dir, _ := dummySecretContent(t) - defer os.RemoveAll(dir) - - expected := ing_errors.LocationDeniedError{ - Reason: errors.New("cross namespace usage of secrets is not allowed"), - } - _, err := NewParser(dir, &mockSecret{}).Parse(ing) - if err.Error() != expected.Error() { - t.Errorf("expected '%v' but got '%v'", expected, err) - } -} - -func TestIngressInvalidDifferentNamespaceAllowed(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix(authTypeAnnotation)] = authType - data[parser.GetAnnotationWithPrefix(AuthSecretAnnotation)] = othernsDemoSecret - ing.SetAnnotations(data) - - _, dir, _ := dummySecretContent(t) - defer os.RemoveAll(dir) - - r := mockSecret{} - r.AllowCrossNamespace = true - _, err := NewParser(dir, r).Parse(ing) - if err != nil { - t.Errorf("not expecting an error") - } -} - -func TestIngressInvalidSecretName(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix(authTypeAnnotation)] = authType - data[parser.GetAnnotationWithPrefix(AuthSecretAnnotation)] = "demo-secret;xpto" - ing.SetAnnotations(data) - - _, dir, _ := dummySecretContent(t) - defer os.RemoveAll(dir) - - expected := ing_errors.LocationDeniedError{ - Reason: errors.New("error reading secret name from annotation: annotation nginx.ingress.kubernetes.io/auth-secret contains invalid value"), - } + expected := ing_errors.NewLocationDenied("invalid authentication type") _, err := NewParser(dir, &mockSecret{}).Parse(ing) if err.Error() != expected.Error() { t.Errorf("expected '%v' but got '%v'", expected, err) @@ -216,13 +118,13 @@ func TestInvalidIngressAuthNoSecret(t *testing.T) { ing := buildIngress() data := map[string]string{} - data[parser.GetAnnotationWithPrefix(authTypeAnnotation)] = authType + data[parser.GetAnnotationWithPrefix("auth-type")] = "basic" ing.SetAnnotations(data) _, dir, _ := dummySecretContent(t) defer os.RemoveAll(dir) - expected := ing_errors.LocationDeniedError{ + expected := ing_errors.LocationDenied{ Reason: errors.New("error reading secret name from annotation: ingress rule without annotations"), } _, err := NewParser(dir, &mockSecret{}).Parse(ing) @@ -235,9 +137,9 @@ func TestIngressAuth(t *testing.T) { ing := buildIngress() data := map[string]string{} - data[parser.GetAnnotationWithPrefix(authTypeAnnotation)] = authType - data[parser.GetAnnotationWithPrefix(AuthSecretAnnotation)] = demoSecret - data[parser.GetAnnotationWithPrefix(authRealmAnnotation)] = authRealm + data[parser.GetAnnotationWithPrefix("auth-type")] = "basic" + data[parser.GetAnnotationWithPrefix("auth-secret")] = "demo-secret" + data[parser.GetAnnotationWithPrefix("auth-realm")] = "-realm-" ing.SetAnnotations(data) _, dir, _ := dummySecretContent(t) @@ -251,10 +153,10 @@ func TestIngressAuth(t *testing.T) { if !ok { t.Errorf("expected a BasicDigest type") } - if auth.Type != authType { + if auth.Type != "basic" { t.Errorf("Expected basic as auth type but returned %s", auth.Type) } - if auth.Realm != authRealm { + if auth.Realm != "-realm-" { t.Errorf("Expected -realm- as realm but returned %s", auth.Realm) } if !auth.Secured { @@ -266,9 +168,9 @@ func TestIngressAuthWithoutSecret(t *testing.T) { ing := buildIngress() data := map[string]string{} - data[parser.GetAnnotationWithPrefix(authTypeAnnotation)] = authType - data[parser.GetAnnotationWithPrefix(AuthSecretAnnotation)] = "invalid-secret" - data[parser.GetAnnotationWithPrefix(authRealmAnnotation)] = authRealm + data[parser.GetAnnotationWithPrefix("auth-type")] = "basic" + data[parser.GetAnnotationWithPrefix("auth-secret")] = "invalid-secret" + data[parser.GetAnnotationWithPrefix("auth-realm")] = "-realm-" ing.SetAnnotations(data) _, dir, _ := dummySecretContent(t) @@ -284,10 +186,10 @@ func TestIngressAuthInvalidSecretKey(t *testing.T) { ing := buildIngress() data := map[string]string{} - data[parser.GetAnnotationWithPrefix(authTypeAnnotation)] = authType - data[parser.GetAnnotationWithPrefix(AuthSecretAnnotation)] = demoSecret - data[parser.GetAnnotationWithPrefix(authSecretTypeAnnotation)] = "invalid-type" - data[parser.GetAnnotationWithPrefix(authRealmAnnotation)] = authRealm + data[parser.GetAnnotationWithPrefix("auth-type")] = "basic" + data[parser.GetAnnotationWithPrefix("auth-secret")] = "demo-secret" + data[parser.GetAnnotationWithPrefix("auth-secret-type")] = "invalid-type" + data[parser.GetAnnotationWithPrefix("auth-realm")] = "-realm-" ing.SetAnnotations(data) _, dir, _ := dummySecretContent(t) @@ -299,21 +201,18 @@ func TestIngressAuthInvalidSecretKey(t *testing.T) { } } -func dummySecretContent(t *testing.T) (fileName, dir string, s *api.Secret) { - dir, err := os.MkdirTemp("", fmt.Sprintf("%v", time.Now().Unix())) +func dummySecretContent(t *testing.T) (string, string, *api.Secret) { + dir, err := ioutil.TempDir("", fmt.Sprintf("%v", time.Now().Unix())) if err != nil { t.Error(err) } - tmpfile, err := os.CreateTemp("", "example-") + tmpfile, err := ioutil.TempFile("", "example-") if err != nil { t.Error(err) } defer tmpfile.Close() - s, err = mockSecret{}.GetSecret(defaultDemoSecret) - if err != nil { - t.Errorf("unexpected error: %v", err) - } + s, _ := mockSecret{}.GetSecret("default/demo-secret") return tmpfile.Name(), dir, s } diff --git a/internal/ingress/annotations/authreq/main.go b/internal/ingress/annotations/authreq/main.go index ad38c36b1..afcd84a65 100644 --- a/internal/ingress/annotations/authreq/main.go +++ b/internal/ingress/annotations/authreq/main.go @@ -23,134 +23,14 @@ import ( "k8s.io/klog/v2" - networking "k8s.io/api/networking/v1" - "k8s.io/client-go/tools/cache" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" ing_errors "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" - "k8s.io/ingress-nginx/pkg/util/sets" + "k8s.io/ingress-nginx/internal/sets" ) -const ( - authReqURLAnnotation = "auth-url" - authReqMethodAnnotation = "auth-method" - authReqSigninAnnotation = "auth-signin" - authReqSigninRedirParamAnnotation = "auth-signin-redirect-param" - authReqSnippetAnnotation = "auth-snippet" - authReqCacheKeyAnnotation = "auth-cache-key" - authReqKeepaliveAnnotation = "auth-keepalive" - authReqKeepaliveShareVarsAnnotation = "auth-keepalive-share-vars" - authReqKeepaliveRequestsAnnotation = "auth-keepalive-requests" - authReqKeepaliveTimeout = "auth-keepalive-timeout" - authReqCacheDuration = "auth-cache-duration" - authReqResponseHeadersAnnotation = "auth-response-headers" - authReqProxySetHeadersAnnotation = "auth-proxy-set-headers" - authReqRequestRedirectAnnotation = "auth-request-redirect" - authReqAlwaysSetCookieAnnotation = "auth-always-set-cookie" - - // This should be exported as it is imported by other packages - AuthSecretAnnotation = "auth-secret" -) - -var authReqAnnotations = parser.Annotation{ - Group: "authentication", - Annotations: parser.AnnotationFields{ - authReqURLAnnotation: { - Validator: parser.ValidateRegex(parser.URLWithNginxVariableRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskHigh, - Documentation: `This annotation allows to indicate the URL where the HTTP request should be sent`, - }, - authReqMethodAnnotation: { - Validator: parser.ValidateRegex(methodsRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation allows to specify the HTTP method to use`, - }, - authReqSigninAnnotation: { - Validator: parser.ValidateRegex(parser.URLWithNginxVariableRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskHigh, - Documentation: `This annotation allows to specify the location of the error page`, - }, - authReqSigninRedirParamAnnotation: { - Validator: parser.ValidateRegex(parser.URLIsValidRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation allows to specify the URL parameter in the error page which should contain the original URL for a failed signin request`, - }, - authReqSnippetAnnotation: { - Validator: parser.ValidateNull, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskCritical, - Documentation: `This annotation allows to specify a custom snippet to use with external authentication`, - }, - authReqCacheKeyAnnotation: { - Validator: parser.ValidateRegex(parser.NGINXVariable, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation enables caching for auth requests.`, - }, - authReqKeepaliveAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation specifies the maximum number of keepalive connections to auth-url. Only takes effect when no variables are used in the host part of the URL`, - }, - authReqKeepaliveShareVarsAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation specifies whether to share Nginx variables among the current request and the auth request`, - }, - authReqKeepaliveRequestsAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation defines the maximum number of requests that can be served through one keepalive connection`, - }, - authReqKeepaliveTimeout: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation specifies a duration in seconds which an idle keepalive connection to an upstream server will stay open`, - }, - authReqCacheDuration: { - Validator: parser.ValidateRegex(parser.ExtendedCharsRegex, false), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation allows to specify a caching time for auth responses based on their response codes, e.g. 200 202 30m`, - }, - authReqResponseHeadersAnnotation: { - Validator: parser.ValidateRegex(parser.HeadersVariable, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation sets the headers to pass to backend once authentication request completes. They should be separated by comma.`, - }, - authReqProxySetHeadersAnnotation: { - Validator: parser.ValidateRegex(parser.BasicCharsRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation sets the name of a ConfigMap that specifies headers to pass to the authentication service. - Only ConfigMaps on the same namespace are allowed`, - }, - authReqRequestRedirectAnnotation: { - Validator: parser.ValidateRegex(parser.URLIsValidRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation allows to specify the X-Auth-Request-Redirect header value`, - }, - authReqAlwaysSetCookieAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation enables setting a cookie returned by auth request. - By default, the cookie will be set only if an upstream reports with the code 200, 201, 204, 206, 301, 302, 303, 304, 307, or 308`, - }, - }, -} - // Config returns external authentication configuration for an Ingress rule type Config struct { URL string `json:"url"` @@ -164,25 +44,12 @@ type Config struct { AuthSnippet string `json:"authSnippet"` AuthCacheKey string `json:"authCacheKey"` AuthCacheDuration []string `json:"authCacheDuration"` - KeepaliveConnections int `json:"keepaliveConnections"` - KeepaliveShareVars bool `json:"keepaliveShareVars"` - KeepaliveRequests int `json:"keepaliveRequests"` - KeepaliveTimeout int `json:"keepaliveTimeout"` ProxySetHeaders map[string]string `json:"proxySetHeaders,omitempty"` - AlwaysSetCookie bool `json:"alwaysSetCookie,omitempty"` } // DefaultCacheDuration is the fallback value if no cache duration is provided const DefaultCacheDuration = "200 202 401 5m" -// fallback values when no keepalive parameters are set -const ( - defaultKeepaliveConnections = 0 - defaultKeepaliveShareVars = false - defaultKeepaliveRequests = 1000 - defaultKeepaliveTimeout = 60 -) - // Equal tests for equality between two Config types func (e1 *Config) Equal(e2 *Config) bool { if e1 == e2 { @@ -223,66 +90,54 @@ func (e1 *Config) Equal(e2 *Config) bool { return false } - if e1.KeepaliveConnections != e2.KeepaliveConnections { - return false - } - - if e1.KeepaliveShareVars != e2.KeepaliveShareVars { - return false - } - - if e1.KeepaliveRequests != e2.KeepaliveRequests { - return false - } - - if e1.KeepaliveTimeout != e2.KeepaliveTimeout { - return false - } - - if e1.AlwaysSetCookie != e2.AlwaysSetCookie { - return false - } - return sets.StringElementsMatch(e1.AuthCacheDuration, e2.AuthCacheDuration) } var ( - methodsRegex = regexp.MustCompile("(GET|HEAD|POST|PUT|PATCH|DELETE|CONNECT|OPTIONS|TRACE)") + methods = []string{"GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "CONNECT", "OPTIONS", "TRACE"} headerRegexp = regexp.MustCompile(`^[a-zA-Z\d\-_]+$`) - statusCodeRegex = regexp.MustCompile(`^\d{3}$`) - durationRegex = regexp.MustCompile(`^\d+(ms|s|m|h|d|w|M|y)$`) // see http://nginx.org/en/docs/syntax.html + statusCodeRegex = regexp.MustCompile(`^[\d]{3}$`) + durationRegex = regexp.MustCompile(`^[\d]+(ms|s|m|h|d|w|M|y)$`) // see http://nginx.org/en/docs/syntax.html ) // ValidMethod checks is the provided string a valid HTTP method func ValidMethod(method string) bool { - return methodsRegex.MatchString(method) + if len(method) == 0 { + return false + } + + for _, m := range methods { + if method == m { + return true + } + } + return false } // ValidHeader checks is the provided string satisfies the header's name regex func ValidHeader(header string) bool { - return headerRegexp.MatchString(header) + return headerRegexp.Match([]byte(header)) } // ValidCacheDuration checks if the provided string is a valid cache duration // spec: [code ...] [time ...]; // with: code is an http status code -// -// time must match the time regex and may appear multiple times, e.g. `1h 30m` +// time must match the time regex and may appear multiple times, e.g. `1h 30m` func ValidCacheDuration(duration string) bool { elements := strings.Split(duration, " ") seenDuration := false for _, element := range elements { - if element == "" { + if len(element) == 0 { continue } - if statusCodeRegex.MatchString(element) { + if statusCodeRegex.Match([]byte(element)) { if seenDuration { return false // code after duration } continue } - if durationRegex.MatchString(element) { + if durationRegex.Match([]byte(element)) { seenDuration = true } } @@ -290,133 +145,67 @@ func ValidCacheDuration(duration string) bool { } type authReq struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new authentication request annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return authReq{ - r: r, - annotationConfig: authReqAnnotations, - } + return authReq{r} } // ParseAnnotations parses the annotations contained in the ingress // rule used to use an Config URL as source for authentication -// -//nolint:gocyclo // Ignore function complexity error func (a authReq) Parse(ing *networking.Ingress) (interface{}, error) { // Required Parameters - urlString, err := parser.GetStringAnnotation(authReqURLAnnotation, ing, a.annotationConfig.Annotations) + urlString, err := parser.GetStringAnnotation("auth-url", ing) if err != nil { return nil, err } authURL, err := parser.StringToURL(urlString) if err != nil { - return nil, ing_errors.LocationDeniedError{Reason: fmt.Errorf("could not parse auth-url annotation: %v", err)} + return nil, ing_errors.InvalidContent{Name: err.Error()} } - authMethod, err := parser.GetStringAnnotation(authReqMethodAnnotation, ing, a.annotationConfig.Annotations) - if err != nil { - if ing_errors.IsValidationError(err) { - return nil, ing_errors.NewLocationDenied("invalid HTTP method") - } + authMethod, _ := parser.GetStringAnnotation("auth-method", ing) + if len(authMethod) != 0 && !ValidMethod(authMethod) { + return nil, ing_errors.NewLocationDenied("invalid HTTP method") } // Optional Parameters - signIn, err := parser.GetStringAnnotation(authReqSigninAnnotation, ing, a.annotationConfig.Annotations) + signIn, err := parser.GetStringAnnotation("auth-signin", ing) if err != nil { - if ing_errors.IsValidationError(err) { - klog.Warningf("%s value is invalid: %s", authReqSigninAnnotation, err) - } klog.V(3).InfoS("auth-signin annotation is undefined and will not be set") } - signInRedirectParam, err := parser.GetStringAnnotation(authReqSigninRedirParamAnnotation, ing, a.annotationConfig.Annotations) + signInRedirectParam, err := parser.GetStringAnnotation("auth-signin-redirect-param", ing) if err != nil { - if ing_errors.IsValidationError(err) { - klog.Warningf("%s value is invalid: %s", authReqSigninRedirParamAnnotation, err) - } klog.V(3).Infof("auth-signin-redirect-param annotation is undefined and will not be set") } - authSnippet, err := parser.GetStringAnnotation(authReqSnippetAnnotation, ing, a.annotationConfig.Annotations) + authSnippet, err := parser.GetStringAnnotation("auth-snippet", ing) if err != nil { klog.V(3).InfoS("auth-snippet annotation is undefined and will not be set") } - authCacheKey, err := parser.GetStringAnnotation(authReqCacheKeyAnnotation, ing, a.annotationConfig.Annotations) + authCacheKey, err := parser.GetStringAnnotation("auth-cache-key", ing) if err != nil { - if ing_errors.IsValidationError(err) { - klog.Warningf("%s value is invalid: %s", authReqCacheKeyAnnotation, err) - } klog.V(3).InfoS("auth-cache-key annotation is undefined and will not be set") } - keepaliveConnections, err := parser.GetIntAnnotation(authReqKeepaliveAnnotation, ing, a.annotationConfig.Annotations) - if err != nil { - klog.V(3).InfoS("auth-keepalive annotation is undefined and will be set to its default value") - keepaliveConnections = defaultKeepaliveConnections - } - switch { - case keepaliveConnections < 0: - klog.Warningf("auth-keepalive annotation (%s) contains a negative value, setting auth-keepalive to 0", authURL.Host) - keepaliveConnections = 0 - case keepaliveConnections > 0: - // NOTE: upstream block cannot reference a variable in the server directive - if strings.IndexByte(authURL.Host, '$') != -1 { - klog.Warningf("auth-url annotation (%s) contains $ in the host:port part, setting auth-keepalive to 0", authURL.Host) - keepaliveConnections = 0 - } - } - - keepaliveShareVars, err := parser.GetBoolAnnotation(authReqKeepaliveShareVarsAnnotation, ing, a.annotationConfig.Annotations) - if err != nil { - klog.V(3).InfoS("auth-keepalive-share-vars annotation is undefined and will be set to its default value") - keepaliveShareVars = defaultKeepaliveShareVars - } - - keepaliveRequests, err := parser.GetIntAnnotation(authReqKeepaliveRequestsAnnotation, ing, a.annotationConfig.Annotations) - if err != nil { - klog.V(3).InfoS("auth-keepalive-requests annotation is undefined or invalid and will be set to its default value") - keepaliveRequests = defaultKeepaliveRequests - } - if keepaliveRequests <= 0 { - klog.Warningf("auth-keepalive-requests annotation (%s) should be greater than zero, setting auth-keepalive to 0", authURL.Host) - keepaliveConnections = 0 - } - - keepaliveTimeout, err := parser.GetIntAnnotation(authReqKeepaliveTimeout, ing, a.annotationConfig.Annotations) - if err != nil { - klog.V(3).InfoS("auth-keepalive-timeout annotation is undefined and will be set to its default value") - keepaliveTimeout = defaultKeepaliveTimeout - } - if keepaliveTimeout <= 0 { - klog.Warningf("auth-keepalive-timeout annotation (%s) should be greater than zero, setting auth-keepalive 0", authURL.Host) - keepaliveConnections = 0 - } - - durstr, err := parser.GetStringAnnotation(authReqCacheDuration, ing, a.annotationConfig.Annotations) - if err != nil && ing_errors.IsValidationError(err) { - return nil, fmt.Errorf("%s contains invalid value", authReqCacheDuration) - } + durstr, _ := parser.GetStringAnnotation("auth-cache-duration", ing) authCacheDuration, err := ParseStringToCacheDurations(durstr) if err != nil { return nil, err } responseHeaders := []string{} - hstr, err := parser.GetStringAnnotation(authReqResponseHeadersAnnotation, ing, a.annotationConfig.Annotations) - if err != nil && ing_errors.IsValidationError(err) { - return nil, ing_errors.NewLocationDenied("validation error") - } - if hstr != "" { + hstr, _ := parser.GetStringAnnotation("auth-response-headers", ing) + if len(hstr) != 0 { harr := strings.Split(hstr, ",") for _, header := range harr { header = strings.TrimSpace(header) - if header != "" { + if len(header) > 0 { if !ValidHeader(header) { return nil, ing_errors.NewLocationDenied("invalid headers list") } @@ -425,28 +214,9 @@ func (a authReq) Parse(ing *networking.Ingress) (interface{}, error) { } } - proxySetHeaderMap, err := parser.GetStringAnnotation(authReqProxySetHeadersAnnotation, ing, a.annotationConfig.Annotations) + proxySetHeaderMap, err := parser.GetStringAnnotation("auth-proxy-set-headers", ing) if err != nil { - klog.V(3).InfoS("auth-set-proxy-headers annotation is undefined and will not be set", "err", err) - } - - cns, _, err := cache.SplitMetaNamespaceKey(proxySetHeaderMap) - if err != nil { - return nil, ing_errors.LocationDeniedError{ - Reason: fmt.Errorf("error reading configmap name %s from annotation: %w", proxySetHeaderMap, err), - } - } - - if cns == "" { - cns = ing.Namespace - } - - secCfg := a.r.GetSecurityConfiguration() - // We don't accept different namespaces for secrets. - if !secCfg.AllowCrossNamespaceResources && cns != ing.Namespace { - return nil, ing_errors.LocationDeniedError{ - Reason: fmt.Errorf("cross namespace usage of secrets is not allowed"), - } + klog.V(3).InfoS("auth-set-proxy-headers annotation is undefined and will not be set") } var proxySetHeaders map[string]string @@ -466,15 +236,7 @@ func (a authReq) Parse(ing *networking.Ingress) (interface{}, error) { proxySetHeaders = proxySetHeadersMapContents.Data } - requestRedirect, err := parser.GetStringAnnotation(authReqRequestRedirectAnnotation, ing, a.annotationConfig.Annotations) - if err != nil && ing_errors.IsValidationError(err) { - return nil, fmt.Errorf("%s is invalid: %w", authReqRequestRedirectAnnotation, err) - } - - alwaysSetCookie, err := parser.GetBoolAnnotation(authReqAlwaysSetCookieAnnotation, ing, a.annotationConfig.Annotations) - if err != nil && ing_errors.IsValidationError(err) { - return nil, fmt.Errorf("%s is invalid: %w", authReqAlwaysSetCookieAnnotation, err) - } + requestRedirect, _ := parser.GetStringAnnotation("auth-request-redirect", ing) return &Config{ URL: urlString, @@ -487,12 +249,7 @@ func (a authReq) Parse(ing *networking.Ingress) (interface{}, error) { AuthSnippet: authSnippet, AuthCacheKey: authCacheKey, AuthCacheDuration: authCacheDuration, - KeepaliveConnections: keepaliveConnections, - KeepaliveShareVars: keepaliveShareVars, - KeepaliveRequests: keepaliveRequests, - KeepaliveTimeout: keepaliveTimeout, ProxySetHeaders: proxySetHeaders, - AlwaysSetCookie: alwaysSetCookie, }, nil } @@ -501,11 +258,11 @@ func (a authReq) Parse(ing *networking.Ingress) (interface{}, error) { // It will always return at least one duration (the default duration) func ParseStringToCacheDurations(input string) ([]string, error) { authCacheDuration := []string{} - if input != "" { + if len(input) != 0 { arr := strings.Split(input, ",") for _, duration := range arr { duration = strings.TrimSpace(duration) - if duration != "" { + if len(duration) > 0 { if !ValidCacheDuration(duration) { authCacheDuration = []string{DefaultCacheDuration} return authCacheDuration, ing_errors.NewLocationDenied(fmt.Sprintf("invalid cache duration: %s", duration)) @@ -520,12 +277,3 @@ func ParseStringToCacheDurations(input string) ([]string, error) { } return authCacheDuration, nil } - -func (a authReq) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a authReq) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, authReqAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/authreq/main_test.go b/internal/ingress/annotations/authreq/main_test.go index c261795c6..276779c1c 100644 --- a/internal/ingress/annotations/authreq/main_test.go +++ b/internal/ingress/annotations/authreq/main_test.go @@ -17,24 +17,23 @@ limitations under the License. package authreq import ( + "fmt" "reflect" "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" + + "k8s.io/apimachinery/pkg/util/intstr" ) func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -43,13 +42,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -70,13 +65,6 @@ func buildIngress() *networking.Ingress { } } -func boolToString(v bool) string { - if v { - return "true" - } - return "false" -} - func TestAnnotations(t *testing.T) { ing := buildIngress() @@ -92,31 +80,29 @@ func TestAnnotations(t *testing.T) { requestRedirect string authSnippet string authCacheKey string - authAlwaysSetCookie bool expErr bool }{ - {"empty", "", "", "", "", "", "", "", false, true}, - {"no scheme", "bar", "bar", "", "", "", "", "", false, true}, - {"invalid host", "http://", "http://", "", "", "", "", "", false, true}, - {"invalid host (multiple dots)", "http://foo..bar.com", "http://foo..bar.com", "", "", "", "", "", false, true}, - {"valid URL", "http://bar.foo.com/external-auth/auth?allowed_groups=snow-group,rain-group", "http://bar.foo.com/external-auth/start?rd=https://$host$escaped_request_uri", "", "", "", "", "", false, false}, - {"valid URL - send body", "http://foo.com/external-auth/auth?allowed_groups=snow-group,rain-group", "http://foo.com/external-auth/start?rd=https://$host$escaped_request_uri", "", "POST", "", "", "", false, false}, - {"valid URL - send body", "http://foo.com/external-auth/auth?allowed_groups=snow-group,rain-group", "http://foo.com/external-auth/start?rd=https://$host$escaped_request_uri", "", "GET", "", "", "", false, false}, - {"valid URL - request redirect", "http://foo.com/external-auth/auth?allowed_groups=snow-group,rain-group", "http://foo.com/external-auth/start?rd=https://$host$escaped_request_uri", "", "GET", "http://foo.com/redirect-me", "", "", false, false}, - {"auth snippet", "http://foo.com/external-auth/auth?allowed_groups=snow-group,rain-group", "http://foo.com/external-auth/start?rd=https://$host$escaped_request_uri", "", "", "", "proxy_set_header My-Custom-Header 42;", "", false, false}, - {"auth cache ", "http://foo.com/external-auth/auth?allowed_groups=snow-group,rain-group", "http://foo.com/external-auth/start?rd=https://$host$escaped_request_uri", "", "", "", "", "$foo$bar", false, false}, - {"redirect param", "http://bar.foo.com/external-auth/auth?allowed_groups=snow-group,rain-group", "http://bar.foo.com/external-auth/start?rd=https://$host$escaped_request_uri", "origUrl", "", "", "", "", true, false}, + {"empty", "", "", "", "", "", "", "", true}, + {"no scheme", "bar", "bar", "", "", "", "", "", true}, + {"invalid host", "http://", "http://", "", "", "", "", "", true}, + {"invalid host (multiple dots)", "http://foo..bar.com", "http://foo..bar.com", "", "", "", "", "", true}, + {"valid URL", "http://bar.foo.com/external-auth", "http://bar.foo.com/external-auth", "", "", "", "", "", false}, + {"valid URL - send body", "http://foo.com/external-auth", "http://foo.com/external-auth", "", "POST", "", "", "", false}, + {"valid URL - send body", "http://foo.com/external-auth", "http://foo.com/external-auth", "", "GET", "", "", "", false}, + {"valid URL - request redirect", "http://foo.com/external-auth", "http://foo.com/external-auth", "", "GET", "http://foo.com/redirect-me", "", "", false}, + {"auth snippet", "http://foo.com/external-auth", "http://foo.com/external-auth", "", "", "", "proxy_set_header My-Custom-Header 42;", "", false}, + {"auth cache ", "http://foo.com/external-auth", "http://foo.com/external-auth", "", "", "", "", "$foo$bar", false}, + {"redirect param", "http://bar.foo.com/external-auth", "http://bar.foo.com/external-auth", "origUrl", "", "", "", "", false}, } for _, test := range tests { data[parser.GetAnnotationWithPrefix("auth-url")] = test.url data[parser.GetAnnotationWithPrefix("auth-signin")] = test.signinURL data[parser.GetAnnotationWithPrefix("auth-signin-redirect-param")] = test.signinURLRedirectParam - data[parser.GetAnnotationWithPrefix("auth-method")] = test.method + data[parser.GetAnnotationWithPrefix("auth-method")] = fmt.Sprintf("%v", test.method) data[parser.GetAnnotationWithPrefix("auth-request-redirect")] = test.requestRedirect data[parser.GetAnnotationWithPrefix("auth-snippet")] = test.authSnippet data[parser.GetAnnotationWithPrefix("auth-cache-key")] = test.authCacheKey - data[parser.GetAnnotationWithPrefix("auth-always-set-cookie")] = boolToString(test.authAlwaysSetCookie) i, err := NewParser(&resolver.Mock{}).Parse(ing) if test.expErr { @@ -154,10 +140,6 @@ func TestAnnotations(t *testing.T) { if u.AuthCacheKey != test.authCacheKey { t.Errorf("%v: expected \"%v\" but \"%v\" was returned", test.title, test.authCacheKey, u.AuthCacheKey) } - - if u.AlwaysSetCookie != test.authAlwaysSetCookie { - t.Errorf("%v: expected \"%v\" but \"%v\" was returned", test.title, test.authAlwaysSetCookie, u.AlwaysSetCookie) - } } } @@ -191,13 +173,11 @@ func TestHeaderAnnotations(t *testing.T) { i, err := NewParser(&resolver.Mock{}).Parse(ing) if test.expErr { if err == nil { - t.Errorf("%v expected error but retuned nil", test.title) + t.Error("expected error but retuned nil") } continue } - if err != nil { - t.Errorf("no error was expected but %v happened in %s", err, test.title) - } + u, ok := i.(*Config) if !ok { t.Errorf("%v: expected an External type", test.title) @@ -257,79 +237,8 @@ func TestCacheDurationAnnotations(t *testing.T) { } } -func TestKeepaliveAnnotations(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - ing.SetAnnotations(data) - - tests := []struct { - title string - url string - keepaliveConnections string - keepaliveShareVars string - keepaliveRequests string - keepaliveTimeout string - expectedConnections int - expectedShareVars bool - expectedRequests int - expectedTimeout int - }{ - {"all set", "http://goog.url", "5", "false", "500", "50", 5, false, 500, 50}, - {"no annotation", "http://goog.url", "", "", "", "", defaultKeepaliveConnections, defaultKeepaliveShareVars, defaultKeepaliveRequests, defaultKeepaliveTimeout}, - {"default for connections", "http://goog.url", "x", "true", "500", "50", defaultKeepaliveConnections, true, 500, 50}, - {"default for requests", "http://goog.url", "5", "x", "dummy", "50", 5, defaultKeepaliveShareVars, defaultKeepaliveRequests, 50}, - {"default for invalid timeout", "http://goog.url", "5", "t", "500", "x", 5, true, 500, defaultKeepaliveTimeout}, - {"variable in host", "http://$host:5000/a/b", "5", "1", "", "", 0, true, defaultKeepaliveRequests, defaultKeepaliveTimeout}, - {"variable in path", "http://goog.url:5000/$path", "5", "t", "", "", 5, true, defaultKeepaliveRequests, defaultKeepaliveTimeout}, - {"negative connections", "http://goog.url", "-2", "f", "", "", 0, false, defaultKeepaliveRequests, defaultKeepaliveTimeout}, - {"negative requests", "http://goog.url", "5", "True", "-1", "", 0, true, -1, defaultKeepaliveTimeout}, - {"negative timeout", "http://goog.url", "5", "0", "", "-1", 0, false, defaultKeepaliveRequests, -1}, - {"negative request and timeout", "http://goog.url", "5", "False", "-2", "-3", 0, false, -2, -3}, - } - - for _, test := range tests { - data[parser.GetAnnotationWithPrefix("auth-url")] = test.url - data[parser.GetAnnotationWithPrefix("auth-keepalive")] = test.keepaliveConnections - data[parser.GetAnnotationWithPrefix("auth-keepalive-share-vars")] = test.keepaliveShareVars - data[parser.GetAnnotationWithPrefix("auth-keepalive-timeout")] = test.keepaliveTimeout - data[parser.GetAnnotationWithPrefix("auth-keepalive-requests")] = test.keepaliveRequests - - i, err := NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("%v: unexpected error: %v", test.title, err) - continue - } - - u, ok := i.(*Config) - if !ok { - t.Errorf("%v: expected an External type", test.title) - continue - } - - if u.URL != test.url { - t.Errorf("%v: expected \"%v\" but \"%v\" was returned", test.title, test.url, u.URL) - } - - if u.KeepaliveConnections != test.expectedConnections { - t.Errorf("%v: expected \"%v\" but \"%v\" was returned", test.title, test.expectedConnections, u.KeepaliveConnections) - } - - if u.KeepaliveShareVars != test.expectedShareVars { - t.Errorf("%v: expected \"%v\" but \"%v\" was returned", test.title, test.expectedShareVars, u.KeepaliveShareVars) - } - - if u.KeepaliveRequests != test.expectedRequests { - t.Errorf("%v: expected \"%v\" but \"%v\" was returned", test.title, test.expectedRequests, u.KeepaliveRequests) - } - - if u.KeepaliveTimeout != test.expectedTimeout { - t.Errorf("%v: expected \"%v\" but \"%v\" was returned", test.title, test.expectedTimeout, u.KeepaliveTimeout) - } - } -} - func TestParseStringToCacheDurations(t *testing.T) { + tests := []struct { title string duration string @@ -344,6 +253,7 @@ func TestParseStringToCacheDurations(t *testing.T) { } for _, test := range tests { + dur, err := ParseStringToCacheDurations(test.duration) if test.expErr { if err == nil { diff --git a/internal/ingress/annotations/authreqglobal/main.go b/internal/ingress/annotations/authreqglobal/main.go index e8a259047..170f6957d 100644 --- a/internal/ingress/annotations/authreqglobal/main.go +++ b/internal/ingress/annotations/authreqglobal/main.go @@ -17,57 +17,29 @@ limitations under the License. package authreqglobal import ( - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - enableGlobalAuthAnnotation = "enable-global-auth" -) - -var globalAuthAnnotations = parser.Annotation{ - Group: "authentication", - Annotations: parser.AnnotationFields{ - enableGlobalAuthAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `Defines if the global external authentication should be enabled.`, - }, - }, -} - type authReqGlobal struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new authentication request annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return authReqGlobal{ - r: r, - annotationConfig: globalAuthAnnotations, - } + return authReqGlobal{r} } // ParseAnnotations parses the annotations contained in the ingress // rule used to enable or disable global external authentication func (a authReqGlobal) Parse(ing *networking.Ingress) (interface{}, error) { - enableGlobalAuth, err := parser.GetBoolAnnotation(enableGlobalAuthAnnotation, ing, a.annotationConfig.Annotations) + + enableGlobalAuth, err := parser.GetBoolAnnotation("enable-global-auth", ing) if err != nil { enableGlobalAuth = true } return enableGlobalAuth, nil } - -func (a authReqGlobal) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a authReqGlobal) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, globalAuthAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/authreqglobal/main_test.go b/internal/ingress/annotations/authreqglobal/main_test.go index 734f97c0f..a4096f7da 100644 --- a/internal/ingress/annotations/authreqglobal/main_test.go +++ b/internal/ingress/annotations/authreqglobal/main_test.go @@ -20,20 +20,18 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" + + "k8s.io/apimachinery/pkg/util/intstr" ) func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -42,13 +40,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -77,10 +71,7 @@ func TestAnnotation(t *testing.T) { data[parser.GetAnnotationWithPrefix("enable-global-auth")] = "false" ing.SetAnnotations(data) - i, err := NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } + i, _ := NewParser(&resolver.Mock{}).Parse(ing) u, ok := i.(bool) if !ok { t.Errorf("expected a Config type") diff --git a/internal/ingress/annotations/authtls/main.go b/internal/ingress/annotations/authtls/main.go index de4d1cc1d..16e218d33 100644 --- a/internal/ingress/annotations/authtls/main.go +++ b/internal/ingress/annotations/authtls/main.go @@ -17,10 +17,10 @@ limitations under the License. package authtls import ( - "fmt" - "regexp" + "github.com/pkg/errors" + networking "k8s.io/api/networking/v1beta1" - networking "k8s.io/api/networking/v1" + "regexp" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" ing_errors "k8s.io/ingress-nginx/internal/ingress/errors" @@ -31,62 +31,12 @@ import ( const ( defaultAuthTLSDepth = 1 defaultAuthVerifyClient = "on" - - annotationAuthTLSSecret = "auth-tls-secret" //#nosec G101 - annotationAuthTLSVerifyClient = "auth-tls-verify-client" - annotationAuthTLSVerifyDepth = "auth-tls-verify-depth" - annotationAuthTLSErrorPage = "auth-tls-error-page" - annotationAuthTLSPassCertToUpstream = "auth-tls-pass-certificate-to-upstream" //#nosec G101 - annotationAuthTLSMatchCN = "auth-tls-match-cn" ) var ( - authVerifyClientRegex = regexp.MustCompile(`^(on|off|optional|optional_no_ca)$`) - redirectRegex = regexp.MustCompile(`^((https?://)?[A-Za-z0-9\-.]+(:\d+)?)?(/[A-Za-z0-9\-_.]+)*/?$`) + authVerifyClientRegex = regexp.MustCompile(`on|off|optional|optional_no_ca`) ) -var authTLSAnnotations = parser.Annotation{ - Group: "authentication", - Annotations: parser.AnnotationFields{ - annotationAuthTLSSecret: { - Validator: parser.ValidateRegex(parser.BasicCharsRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, // Medium as it allows a subset of chars - Documentation: `This annotation defines the secret that contains the certificate chain of allowed certs`, - }, - annotationAuthTLSVerifyClient: { - Validator: parser.ValidateRegex(authVerifyClientRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, // Medium as it allows a subset of chars - Documentation: `This annotation enables verification of client certificates. Can be "on", "off", "optional" or "optional_no_ca"`, - }, - annotationAuthTLSVerifyDepth: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation defines validation depth between the provided client certificate and the Certification Authority chain.`, - }, - annotationAuthTLSErrorPage: { - Validator: parser.ValidateRegex(redirectRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskHigh, - Documentation: `This annotation defines the URL/Page that user should be redirected in case of a Certificate Authentication Error`, - }, - annotationAuthTLSPassCertToUpstream: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation defines if the received certificates should be passed or not to the upstream server in the header "ssl-client-cert"`, - }, - annotationAuthTLSMatchCN: { - Validator: parser.CommonNameAnnotationValidator, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskHigh, - Documentation: `This annotation adds a sanity check for the CN of the client certificate that is sent over using a string / regex starting with "CN="`, - }, - }, -} - // Config contains the AuthSSLCert used for mutual authentication // and the configured ValidationDepth type Config struct { @@ -95,7 +45,6 @@ type Config struct { ValidationDepth int `json:"validationDepth"` ErrorPage string `json:"errorPage"` PassCertToUpstream bool `json:"passCertToUpstream"` - MatchCN string `json:"matchCN"` AuthTLSError string } @@ -122,24 +71,17 @@ func (assl1 *Config) Equal(assl2 *Config) bool { if assl1.PassCertToUpstream != assl2.PassCertToUpstream { return false } - if assl1.MatchCN != assl2.MatchCN { - return false - } return true } // NewParser creates a new TLS authentication annotation parser -func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return authTLS{ - r: r, - annotationConfig: authTLSAnnotations, - } +func NewParser(resolver resolver.Resolver) parser.IngressAnnotation { + return authTLS{resolver} } type authTLS struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // Parse parses the annotations contained in the ingress @@ -148,75 +90,42 @@ func (a authTLS) Parse(ing *networking.Ingress) (interface{}, error) { var err error config := &Config{} - tlsauthsecret, err := parser.GetStringAnnotation(annotationAuthTLSSecret, ing, a.annotationConfig.Annotations) + tlsauthsecret, err := parser.GetStringAnnotation("auth-tls-secret", ing) if err != nil { return &Config{}, err } - ns, _, err := k8s.ParseNameNS(tlsauthsecret) + _, _, err = k8s.ParseNameNS(tlsauthsecret) if err != nil { return &Config{}, ing_errors.NewLocationDenied(err.Error()) } - if ns == "" { - ns = ing.Namespace - } - secCfg := a.r.GetSecurityConfiguration() - // We don't accept different namespaces for secrets. - if !secCfg.AllowCrossNamespaceResources && ns != ing.Namespace { - return &Config{}, ing_errors.NewLocationDenied("cross namespace secrets are not supported") - } authCert, err := a.r.GetAuthCertificate(tlsauthsecret) if err != nil { - e := fmt.Errorf("error obtaining certificate: %w", err) - return &Config{}, ing_errors.LocationDeniedError{Reason: e} + e := errors.Wrap(err, "error obtaining certificate") + return &Config{}, ing_errors.LocationDenied{Reason: e} } config.AuthSSLCert = *authCert - config.VerifyClient, err = parser.GetStringAnnotation(annotationAuthTLSVerifyClient, ing, a.annotationConfig.Annotations) - // We can set a default value here in case of validation error + config.VerifyClient, err = parser.GetStringAnnotation("auth-tls-verify-client", ing) if err != nil || !authVerifyClientRegex.MatchString(config.VerifyClient) { config.VerifyClient = defaultAuthVerifyClient } - config.ValidationDepth, err = parser.GetIntAnnotation(annotationAuthTLSVerifyDepth, ing, a.annotationConfig.Annotations) - // We can set a default value here in case of validation error + config.ValidationDepth, err = parser.GetIntAnnotation("auth-tls-verify-depth", ing) if err != nil || config.ValidationDepth == 0 { config.ValidationDepth = defaultAuthTLSDepth } - config.ErrorPage, err = parser.GetStringAnnotation(annotationAuthTLSErrorPage, ing, a.annotationConfig.Annotations) + config.ErrorPage, err = parser.GetStringAnnotation("auth-tls-error-page", ing) if err != nil { - if ing_errors.IsValidationError(err) { - return &Config{}, err - } config.ErrorPage = "" } - config.PassCertToUpstream, err = parser.GetBoolAnnotation(annotationAuthTLSPassCertToUpstream, ing, a.annotationConfig.Annotations) + config.PassCertToUpstream, err = parser.GetBoolAnnotation("auth-tls-pass-certificate-to-upstream", ing) if err != nil { - if ing_errors.IsValidationError(err) { - return &Config{}, err - } config.PassCertToUpstream = false } - config.MatchCN, err = parser.GetStringAnnotation(annotationAuthTLSMatchCN, ing, a.annotationConfig.Annotations) - if err != nil { - if ing_errors.IsValidationError(err) { - return &Config{}, err - } - config.MatchCN = "" - } - return config, nil } - -func (a authTLS) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a authTLS) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, authTLSAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/authtls/main_test.go b/internal/ingress/annotations/authtls/main_test.go index 37342e513..d1a54ad70 100644 --- a/internal/ingress/annotations/authtls/main_test.go +++ b/internal/ingress/annotations/authtls/main_test.go @@ -20,26 +20,18 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - defaultDemoSecret = "default/demo-secret" - off = "off" -) - func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -48,13 +40,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -82,22 +70,27 @@ type mockSecret struct { // GetAuthCertificate from mockSecret mocks the GetAuthCertificate for authTLS func (m mockSecret) GetAuthCertificate(name string) (*resolver.AuthSSLCert, error) { - if name != defaultDemoSecret { + if name != "default/demo-secret" { return nil, errors.Errorf("there is no secret with name %v", name) } return &resolver.AuthSSLCert{ - Secret: defaultDemoSecret, + Secret: "default/demo-secret", CAFileName: "/ssl/ca.crt", CASHA: "abc", }, nil + } func TestAnnotations(t *testing.T) { ing := buildIngress() data := map[string]string{} - data[parser.GetAnnotationWithPrefix(annotationAuthTLSSecret)] = defaultDemoSecret + data[parser.GetAnnotationWithPrefix("auth-tls-secret")] = "default/demo-secret" + data[parser.GetAnnotationWithPrefix("auth-tls-verify-client")] = "off" + data[parser.GetAnnotationWithPrefix("auth-tls-verify-depth")] = "1" + data[parser.GetAnnotationWithPrefix("auth-tls-error-page")] = "ok.com/error" + data[parser.GetAnnotationWithPrefix("auth-tls-pass-certificate-to-upstream")] = "true" ing.SetAnnotations(data) @@ -112,7 +105,7 @@ func TestAnnotations(t *testing.T) { t.Errorf("expected *Config but got %v", u) } - secret, err := fakeSecret.GetAuthCertificate(defaultDemoSecret) + secret, err := fakeSecret.GetAuthCertificate("default/demo-secret") if err != nil { t.Errorf("unexpected error getting secret %v", err) } @@ -120,58 +113,18 @@ func TestAnnotations(t *testing.T) { if u.AuthSSLCert.Secret != secret.Secret { t.Errorf("expected %v but got %v", secret.Secret, u.AuthSSLCert.Secret) } - if u.VerifyClient != "on" { - t.Errorf("expected %v but got %v", "on", u.VerifyClient) + if u.VerifyClient != "off" { + t.Errorf("expected %v but got %v", "off", u.VerifyClient) } if u.ValidationDepth != 1 { t.Errorf("expected %v but got %v", 1, u.ValidationDepth) } - if u.ErrorPage != "" { - t.Errorf("expected %v but got %v", "", u.ErrorPage) - } - if u.PassCertToUpstream != false { - t.Errorf("expected %v but got %v", false, u.PassCertToUpstream) - } - if u.MatchCN != "" { - t.Errorf("expected empty string, but got %v", u.MatchCN) - } - - data[parser.GetAnnotationWithPrefix(annotationAuthTLSVerifyClient)] = off - data[parser.GetAnnotationWithPrefix(annotationAuthTLSVerifyDepth)] = "2" - data[parser.GetAnnotationWithPrefix(annotationAuthTLSErrorPage)] = "ok.com/error" - data[parser.GetAnnotationWithPrefix(annotationAuthTLSPassCertToUpstream)] = "true" - data[parser.GetAnnotationWithPrefix(annotationAuthTLSMatchCN)] = "CN=(hello-app|ok|goodbye)" - - ing.SetAnnotations(data) - - i, err = NewParser(fakeSecret).Parse(ing) - if err != nil { - t.Errorf("Unexpected error with ingress: %v", err) - } - - u, ok = i.(*Config) - if !ok { - t.Errorf("expected *Config but got %v", u) - } - - if u.AuthSSLCert.Secret != secret.Secret { - t.Errorf("expected %v but got %v", secret.Secret, u.AuthSSLCert.Secret) - } - if u.VerifyClient != off { - t.Errorf("expected %v but got %v", off, u.VerifyClient) - } - if u.ValidationDepth != 2 { - t.Errorf("expected %v but got %v", 2, u.ValidationDepth) - } if u.ErrorPage != "ok.com/error" { t.Errorf("expected %v but got %v", "ok.com/error", u.ErrorPage) } if u.PassCertToUpstream != true { t.Errorf("expected %v but got %v", true, u.PassCertToUpstream) } - if u.MatchCN != "CN=(hello-app|ok|goodbye)" { - t.Errorf("expected %v but got %v", "CN=(hello-app|ok|goodbye)", u.MatchCN) - } } func TestInvalidAnnotations(t *testing.T) { @@ -186,24 +139,15 @@ func TestInvalidAnnotations(t *testing.T) { } // Invalid NameSpace - data[parser.GetAnnotationWithPrefix(annotationAuthTLSSecret)] = "demo-secret" + data[parser.GetAnnotationWithPrefix("auth-tls-secret")] = "demo-secret" ing.SetAnnotations(data) _, err = NewParser(fakeSecret).Parse(ing) if err == nil { t.Errorf("Expected error with ingress but got nil") } - // Invalid Cross NameSpace - data[parser.GetAnnotationWithPrefix(annotationAuthTLSSecret)] = "nondefault/demo-secret" - ing.SetAnnotations(data) - _, err = NewParser(fakeSecret).Parse(ing) - expErr := errors.NewLocationDenied("cross namespace secrets are not supported") - if err.Error() != expErr.Error() { - t.Errorf("received error is different from cross namespace error: %s Expected %s", err, expErr) - } - // Invalid Auth Certificate - data[parser.GetAnnotationWithPrefix(annotationAuthTLSSecret)] = "default/invalid-demo-secret" + data[parser.GetAnnotationWithPrefix("auth-tls-secret")] = "default/invalid-demo-secret" ing.SetAnnotations(data) _, err = NewParser(fakeSecret).Parse(ing) if err == nil { @@ -211,38 +155,10 @@ func TestInvalidAnnotations(t *testing.T) { } // Invalid optional Annotations - data[parser.GetAnnotationWithPrefix(annotationAuthTLSSecret)] = "default/demo-secret" - - data[parser.GetAnnotationWithPrefix(annotationAuthTLSVerifyClient)] = "w00t" - ing.SetAnnotations(data) - _, err = NewParser(fakeSecret).Parse(ing) - if err != nil { - t.Errorf("Error should be nil and verify client should be defaulted") - } - - data[parser.GetAnnotationWithPrefix(annotationAuthTLSVerifyDepth)] = "abcd" - ing.SetAnnotations(data) - _, err = NewParser(fakeSecret).Parse(ing) - if err != nil { - t.Errorf("Error should be nil and verify depth should be defaulted") - } - - data[parser.GetAnnotationWithPrefix(annotationAuthTLSPassCertToUpstream)] = "nahh" - ing.SetAnnotations(data) - _, err = NewParser(fakeSecret).Parse(ing) - if err == nil { - t.Errorf("Expected error with ingress but got nil") - } - delete(data, parser.GetAnnotationWithPrefix(annotationAuthTLSPassCertToUpstream)) - - data[parser.GetAnnotationWithPrefix(annotationAuthTLSMatchCN)] = "" - ing.SetAnnotations(data) - _, err = NewParser(fakeSecret).Parse(ing) - if err == nil { - t.Errorf("Expected error with ingress CN but got nil") - } - delete(data, parser.GetAnnotationWithPrefix(annotationAuthTLSMatchCN)) - + data[parser.GetAnnotationWithPrefix("auth-tls-secret")] = "default/demo-secret" + data[parser.GetAnnotationWithPrefix("auth-tls-verify-client")] = "w00t" + data[parser.GetAnnotationWithPrefix("auth-tls-verify-depth")] = "abcd" + data[parser.GetAnnotationWithPrefix("auth-tls-pass-certificate-to-upstream")] = "nahh" ing.SetAnnotations(data) i, err := NewParser(fakeSecret).Parse(ing) @@ -263,24 +179,28 @@ func TestInvalidAnnotations(t *testing.T) { if u.PassCertToUpstream != false { t.Errorf("expected %v but got %v", false, u.PassCertToUpstream) } - if u.MatchCN != "" { - t.Errorf("expected empty string but got %v", u.MatchCN) - } + } func TestEquals(t *testing.T) { cfg1 := &Config{} cfg2 := &Config{} + // Same config + result := cfg1.Equal(cfg1) + if result != true { + t.Errorf("Expected true") + } + // compare nil - result := cfg1.Equal(nil) + result = cfg1.Equal(nil) if result != false { t.Errorf("Expected false") } // Different Certs sslCert1 := resolver.AuthSSLCert{ - Secret: defaultDemoSecret, + Secret: "default/demo-secret", CAFileName: "/ssl/ca.crt", CASHA: "abc", } @@ -299,7 +219,7 @@ func TestEquals(t *testing.T) { // Different Verify Client cfg1.VerifyClient = "on" - cfg2.VerifyClient = off + cfg2.VerifyClient = "off" result = cfg1.Equal(cfg2) if result != false { t.Errorf("Expected false") @@ -333,15 +253,6 @@ func TestEquals(t *testing.T) { } cfg2.PassCertToUpstream = true - // Different MatchCN - cfg1.MatchCN = "CN=(hello-app|goodbye)" - cfg2.MatchCN = "CN=(hello-app)" - result = cfg1.Equal(cfg2) - if result != false { - t.Errorf("Expected false") - } - cfg2.MatchCN = "CN=(hello-app|goodbye)" - // Equal Configs result = cfg1.Equal(cfg2) if result != true { diff --git a/internal/ingress/annotations/backendprotocol/main.go b/internal/ingress/annotations/backendprotocol/main.go index 62674c4e5..d6e7beb4c 100644 --- a/internal/ingress/annotations/backendprotocol/main.go +++ b/internal/ingress/annotations/backendprotocol/main.go @@ -17,70 +17,49 @@ limitations under the License. package backendprotocol import ( - networking "k8s.io/api/networking/v1" + "regexp" + "strings" + + networking "k8s.io/api/networking/v1beta1" "k8s.io/klog/v2" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -var validProtocols = []string{"auto_http", "http", "https", "grpc", "grpcs", "fcgi"} +// HTTP protocol +const HTTP = "HTTP" -const ( - http = "HTTP" - backendProtocolAnnotation = "backend-protocol" +var ( + validProtocols = regexp.MustCompile(`^(HTTP|HTTPS|AJP|GRPC|GRPCS|FCGI)$`) ) -var backendProtocolConfig = parser.Annotation{ - Group: "backend", - Annotations: parser.AnnotationFields{ - backendProtocolAnnotation: { - Validator: parser.ValidateOptions(validProtocols, false, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, // Low, as it allows just a set of options - Documentation: `this annotation can be used to define which protocol should - be used to communicate with backends`, - }, - }, -} - type backendProtocol struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new backend protocol annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return backendProtocol{ - r: r, - annotationConfig: backendProtocolConfig, - } -} - -func (a backendProtocol) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations + return backendProtocol{r} } // ParseAnnotations parses the annotations contained in the ingress // rule used to indicate the backend protocol. func (a backendProtocol) Parse(ing *networking.Ingress) (interface{}, error) { if ing.GetAnnotations() == nil { - return http, nil + return HTTP, nil } - proto, err := parser.GetStringAnnotation(backendProtocolAnnotation, ing, a.annotationConfig.Annotations) + proto, err := parser.GetStringAnnotation("backend-protocol", ing) if err != nil { - if errors.IsValidationError(err) { - klog.Warningf("validation error %s. Using HTTP as protocol", err) - } - return http, nil + return HTTP, nil + } + + proto = strings.TrimSpace(strings.ToUpper(proto)) + if !validProtocols.MatchString(proto) { + klog.Warningf("Protocol %v is not a valid value for the backend-protocol annotation. Using HTTP as protocol", proto) + return HTTP, nil } return proto, nil } - -func (a backendProtocol) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, backendProtocolConfig.Annotations) -} diff --git a/internal/ingress/annotations/backendprotocol/main_test.go b/internal/ingress/annotations/backendprotocol/main_test.go index cb1ec779f..4a1c1bf31 100644 --- a/internal/ingress/annotations/backendprotocol/main_test.go +++ b/internal/ingress/annotations/backendprotocol/main_test.go @@ -20,10 +20,12 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" + + "k8s.io/apimachinery/pkg/util/intstr" ) func buildIngress() *networking.Ingress { @@ -33,18 +35,13 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, }, } } - func TestParseInvalidAnnotations(t *testing.T) { ing := buildIngress() @@ -57,7 +54,7 @@ func TestParseInvalidAnnotations(t *testing.T) { if !ok { t.Errorf("expected a string type") } - if val != http { + if val != "HTTP" { t.Errorf("expected HTTPS but %v returned", val) } @@ -73,12 +70,12 @@ func TestParseInvalidAnnotations(t *testing.T) { if !ok { t.Errorf("expected a string type") } - if val != http { + if val != "HTTP" { t.Errorf("expected HTTPS but %v returned", val) } // Test invalid annotation set - data[parser.GetAnnotationWithPrefix(backendProtocolAnnotation)] = "INVALID" + data[parser.GetAnnotationWithPrefix("backend-protocol")] = "INVALID" ing.SetAnnotations(data) i, err = NewParser(&resolver.Mock{}).Parse(ing) @@ -89,7 +86,7 @@ func TestParseInvalidAnnotations(t *testing.T) { if !ok { t.Errorf("expected a string type") } - if val != http { + if val != "HTTP" { t.Errorf("expected HTTPS but %v returned", val) } } @@ -98,7 +95,7 @@ func TestParseAnnotations(t *testing.T) { ing := buildIngress() data := map[string]string{} - data[parser.GetAnnotationWithPrefix(backendProtocolAnnotation)] = " HTTPS " + data[parser.GetAnnotationWithPrefix("backend-protocol")] = "HTTPS" ing.SetAnnotations(data) i, err := NewParser(&resolver.Mock{}).Parse(ing) diff --git a/internal/ingress/annotations/canary/main.go b/internal/ingress/annotations/canary/main.go index ccfe14261..2cc88021b 100644 --- a/internal/ingress/annotations/canary/main.go +++ b/internal/ingress/annotations/canary/main.go @@ -17,90 +17,21 @@ limitations under the License. package canary import ( - networking "k8s.io/api/networking/v1" - "k8s.io/klog/v2" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - canaryAnnotation = "canary" - canaryWeightAnnotation = "canary-weight" - canaryWeightTotalAnnotation = "canary-weight-total" - canaryByHeaderAnnotation = "canary-by-header" - canaryByHeaderValueAnnotation = "canary-by-header-value" - canaryByHeaderPatternAnnotation = "canary-by-header-pattern" - canaryByCookieAnnotation = "canary-by-cookie" -) - -var CanaryAnnotations = parser.Annotation{ - Group: "canary", - Annotations: parser.AnnotationFields{ - canaryAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation enables the Ingress spec to act as an alternative service for requests to route to depending on the rules applied`, - }, - canaryWeightAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation defines the integer based (0 - ) percent of random requests that should be routed to the service specified in the canary Ingress`, - }, - canaryWeightTotalAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation The total weight of traffic. If unspecified, it defaults to 100`, - }, - canaryByHeaderAnnotation: { - Validator: parser.ValidateRegex(parser.BasicCharsRegex, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation defines the header that should be used for notifying the Ingress to route the request to the service specified in the Canary Ingress. - When the request header is set to 'always', it will be routed to the canary. When the header is set to 'never', it will never be routed to the canary. - For any other value, the header will be ignored and the request compared against the other canary rules by precedence`, - }, - canaryByHeaderValueAnnotation: { - Validator: parser.ValidateRegex(parser.BasicCharsRegex, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation defines the header value to match for notifying the Ingress to route the request to the service specified in the Canary Ingress. - When the request header is set to this value, it will be routed to the canary. For any other header value, the header will be ignored and the request compared against the other canary rules by precedence. - This annotation has to be used together with 'canary-by-header'. The annotation is an extension of the 'canary-by-header' to allow customizing the header value instead of using hardcoded values. - It doesn't have any effect if the 'canary-by-header' annotation is not defined`, - }, - canaryByHeaderPatternAnnotation: { - Validator: parser.ValidateRegex(parser.IsValidRegex, false), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation works the same way as canary-by-header-value except it does PCRE Regex matching. - Note that when 'canary-by-header-value' is set this annotation will be ignored. - When the given Regex causes error during request processing, the request will be considered as not matching.`, - }, - canaryByCookieAnnotation: { - Validator: parser.ValidateRegex(parser.BasicCharsRegex, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation defines the cookie that should be used for notifying the Ingress to route the request to the service specified in the Canary Ingress. - When the cookie is set to 'always', it will be routed to the canary. When the cookie is set to 'never', it will never be routed to the canary`, - }, - }, -} - type canary struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // Config returns the configuration rules for setting up the Canary type Config struct { Enabled bool Weight int - WeightTotal int Header string HeaderValue string HeaderPattern string @@ -109,10 +40,7 @@ type Config struct { // NewParser parses the ingress for canary related annotations func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return canary{ - r: r, - annotationConfig: CanaryAnnotations, - } + return canary{r} } // Parse parses the annotations contained in the ingress @@ -121,75 +49,40 @@ func (c canary) Parse(ing *networking.Ingress) (interface{}, error) { config := &Config{} var err error - config.Enabled, err = parser.GetBoolAnnotation(canaryAnnotation, ing, c.annotationConfig.Annotations) + config.Enabled, err = parser.GetBoolAnnotation("canary", ing) if err != nil { - if errors.IsValidationError(err) { - klog.Warningf("%s is invalid, defaulting to 'false'", canaryAnnotation) - } config.Enabled = false } - config.Weight, err = parser.GetIntAnnotation(canaryWeightAnnotation, ing, c.annotationConfig.Annotations) + config.Weight, err = parser.GetIntAnnotation("canary-weight", ing) if err != nil { - if errors.IsValidationError(err) { - klog.Warningf("%s is invalid, defaulting to '0'", canaryWeightAnnotation) - } config.Weight = 0 } - config.WeightTotal, err = parser.GetIntAnnotation(canaryWeightTotalAnnotation, ing, c.annotationConfig.Annotations) + config.Header, err = parser.GetStringAnnotation("canary-by-header", ing) if err != nil { - if errors.IsValidationError(err) { - klog.Warningf("%s is invalid, defaulting to '100'", canaryWeightTotalAnnotation) - } - config.WeightTotal = 100 - } - - config.Header, err = parser.GetStringAnnotation(canaryByHeaderAnnotation, ing, c.annotationConfig.Annotations) - if err != nil { - if errors.IsValidationError(err) { - klog.Warningf("%s is invalid, defaulting to ''", canaryByHeaderAnnotation) - } config.Header = "" } - config.HeaderValue, err = parser.GetStringAnnotation(canaryByHeaderValueAnnotation, ing, c.annotationConfig.Annotations) + config.HeaderValue, err = parser.GetStringAnnotation("canary-by-header-value", ing) if err != nil { - if errors.IsValidationError(err) { - klog.Warningf("%s is invalid, defaulting to ''", canaryByHeaderValueAnnotation) - } config.HeaderValue = "" } - config.HeaderPattern, err = parser.GetStringAnnotation(canaryByHeaderPatternAnnotation, ing, c.annotationConfig.Annotations) + config.HeaderPattern, err = parser.GetStringAnnotation("canary-by-header-pattern", ing) if err != nil { - if errors.IsValidationError(err) { - klog.Warningf("%s is invalid, defaulting to ''", canaryByHeaderPatternAnnotation) - } config.HeaderPattern = "" } - config.Cookie, err = parser.GetStringAnnotation(canaryByCookieAnnotation, ing, c.annotationConfig.Annotations) + config.Cookie, err = parser.GetStringAnnotation("canary-by-cookie", ing) if err != nil { - if errors.IsValidationError(err) { - klog.Warningf("%s is invalid, defaulting to ''", canaryByCookieAnnotation) - } config.Cookie = "" } - if !config.Enabled && (config.Weight > 0 || config.Header != "" || config.HeaderValue != "" || config.Cookie != "" || - config.HeaderPattern != "") { - return nil, errors.NewInvalidAnnotationConfiguration(canaryAnnotation, "configured but not enabled") + if !config.Enabled && (config.Weight > 0 || len(config.Header) > 0 || len(config.HeaderValue) > 0 || len(config.Cookie) > 0 || + len(config.HeaderPattern) > 0) { + return nil, errors.NewInvalidAnnotationConfiguration("canary", "configured but not enabled") } return config, nil } - -func (c canary) GetDocumentation() parser.AnnotationFields { - return c.annotationConfig.Annotations -} - -func (c canary) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(c.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, CanaryAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/canary/main_test.go b/internal/ingress/annotations/canary/main_test.go index 1787ddb85..f755fe865 100644 --- a/internal/ingress/annotations/canary/main_test.go +++ b/internal/ingress/annotations/canary/main_test.go @@ -17,25 +17,23 @@ limitations under the License. package canary import ( - "strconv" "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "strconv" + "k8s.io/ingress-nginx/internal/ingress/resolver" ) func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -44,13 +42,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -92,6 +86,7 @@ func TestCanaryInvalid(t *testing.T) { if val.Weight != 0 { t.Errorf("Expected %v but got %v", 0, val.Weight) } + } func TestAnnotations(t *testing.T) { @@ -131,9 +126,10 @@ func TestAnnotations(t *testing.T) { } continue - } - if err != nil { - t.Errorf("%v: expected nil but returned error %v", test.title, err) + } else { + if err != nil { + t.Errorf("%v: expected nil but returned error %v", test.title, err) + } } canaryConfig, ok := i.(*Config) diff --git a/internal/ingress/annotations/class/main.go b/internal/ingress/annotations/class/main.go new file mode 100644 index 000000000..b2fe32d63 --- /dev/null +++ b/internal/ingress/annotations/class/main.go @@ -0,0 +1,64 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package class + +import ( + networking "k8s.io/api/networking/v1beta1" + "k8s.io/ingress-nginx/internal/k8s" +) + +const ( + // IngressKey picks a specific "class" for the Ingress. + // The controller only processes Ingresses with this annotation either + // unset, or set to either the configured value or the empty string. + IngressKey = "kubernetes.io/ingress.class" +) + +var ( + // DefaultClass defines the default class used in the nginx ingress controller + DefaultClass = "nginx" + + // IngressClass sets the runtime ingress class to use + // An empty string means accept all ingresses without + // annotation and the ones configured with class nginx + IngressClass = "nginx" +) + +// IsValid returns true if the given Ingress specify the ingress.class +// annotation or IngressClassName resource for Kubernetes >= v1.18 +func IsValid(ing *networking.Ingress) bool { + // 1. with annotation or IngressClass + ingress, ok := ing.GetAnnotations()[IngressKey] + if !ok && ing.Spec.IngressClassName != nil { + ingress = *ing.Spec.IngressClassName + } + + // empty ingress and IngressClass equal default + if len(ingress) == 0 && IngressClass == DefaultClass { + return true + } + + // k8s > v1.18. + // Processing may be redundant because k8s.IngressClass is obtained by IngressClass + // 3. without annotation and IngressClass. Check IngressClass + if k8s.IngressClass != nil { + return ingress == k8s.IngressClass.Name + } + + // 4. with IngressClass + return ingress == IngressClass +} diff --git a/internal/ingress/annotations/class/main_test.go b/internal/ingress/annotations/class/main_test.go new file mode 100644 index 000000000..2d863f072 --- /dev/null +++ b/internal/ingress/annotations/class/main_test.go @@ -0,0 +1,103 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package class + +import ( + "testing" + + api "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1beta1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/ingress-nginx/internal/k8s" +) + +func TestIsValidClass(t *testing.T) { + dc := DefaultClass + ic := IngressClass + k8sic := k8s.IngressClass + v1Ready := k8s.IsIngressV1Ready + // restore original values after the tests + defer func() { + DefaultClass = dc + IngressClass = ic + k8s.IngressClass = k8sic + k8s.IsIngressV1Ready = v1Ready + }() + + tests := []struct { + ingress string + controller string + defClass string + annotation bool + ingressClassName bool + k8sClass *networking.IngressClass + v1Ready bool + isValid bool + }{ + {"", "", "nginx", true, false, nil, false, true}, + {"", "nginx", "nginx", true, false, nil, false, true}, + {"nginx", "nginx", "nginx", true, false, nil, false, true}, + {"custom", "custom", "nginx", true, false, nil, false, true}, + {"", "killer", "nginx", true, false, nil, false, false}, + {"custom", "nginx", "nginx", true, false, nil, false, false}, + {"nginx", "nginx", "nginx", false, true, nil, false, true}, + {"custom", "nginx", "nginx", false, true, nil, true, false}, + {"nginx", "nginx", "nginx", false, true, nil, true, true}, + {"", "custom", "nginx", false, false, + &networking.IngressClass{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "custom", + }, + }, + false, false}, + {"", "custom", "nginx", false, false, + &networking.IngressClass{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "custom", + }, + }, + true, false}, + } + + for _, test := range tests { + ing := &networking.Ingress{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "foo", + Namespace: api.NamespaceDefault, + }, + } + + data := map[string]string{} + ing.SetAnnotations(data) + if test.annotation { + ing.Annotations[IngressKey] = test.ingress + } + if test.ingressClassName { + ing.Spec.IngressClassName = &[]string{test.ingress}[0] + } + + IngressClass = test.controller + DefaultClass = test.defClass + k8s.IngressClass = test.k8sClass + k8s.IsIngressV1Ready = test.v1Ready + + b := IsValid(ing) + if b != test.isValid { + t.Errorf("test %v - expected %v but %v was returned", test, test.isValid, b) + } + } +} diff --git a/internal/ingress/annotations/clientbodybuffersize/main.go b/internal/ingress/annotations/clientbodybuffersize/main.go index c0fa79713..924ceecd1 100644 --- a/internal/ingress/annotations/clientbodybuffersize/main.go +++ b/internal/ingress/annotations/clientbodybuffersize/main.go @@ -17,55 +17,23 @@ limitations under the License. package clientbodybuffersize import ( - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - clientBodyBufferSizeAnnotation = "client-body-buffer-size" -) - -var clientBodyBufferSizeConfig = parser.Annotation{ - Group: "backend", - Annotations: parser.AnnotationFields{ - clientBodyBufferSizeAnnotation: { - Validator: parser.ValidateRegex(parser.SizeRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, // Low, as it allows just a set of options - Documentation: `Sets buffer size for reading client request body per location. - In case the request body is larger than the buffer, the whole body or only its part is written to a temporary file. - By default, buffer size is equal to two memory pages. This is 8K on x86, other 32-bit platforms, and x86-64. - It is usually 16K on other 64-bit platforms. This annotation is applied to each location provided in the ingress rule.`, - }, - }, -} - type clientBodyBufferSize struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new clientBodyBufferSize annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return clientBodyBufferSize{ - r: r, - annotationConfig: clientBodyBufferSizeConfig, - } -} - -func (cbbs clientBodyBufferSize) GetDocumentation() parser.AnnotationFields { - return cbbs.annotationConfig.Annotations + return clientBodyBufferSize{r} } // Parse parses the annotations contained in the ingress rule // used to add an client-body-buffer-size to the provided locations func (cbbs clientBodyBufferSize) Parse(ing *networking.Ingress) (interface{}, error) { - return parser.GetStringAnnotation(clientBodyBufferSizeAnnotation, ing, cbbs.annotationConfig.Annotations) -} - -func (cbbs clientBodyBufferSize) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(cbbs.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, clientBodyBufferSizeConfig.Annotations) + return parser.GetStringAnnotation("client-body-buffer-size", ing) } diff --git a/internal/ingress/annotations/clientbodybuffersize/main_test.go b/internal/ingress/annotations/clientbodybuffersize/main_test.go index 62257aeb9..56f64083c 100644 --- a/internal/ingress/annotations/clientbodybuffersize/main_test.go +++ b/internal/ingress/annotations/clientbodybuffersize/main_test.go @@ -20,7 +20,7 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" @@ -39,9 +39,6 @@ func TestParse(t *testing.T) { }{ {map[string]string{annotation: "8k"}, "8k"}, {map[string]string{annotation: "16k"}, "16k"}, - {map[string]string{annotation: "10000"}, "10000"}, - {map[string]string{annotation: "16R"}, ""}, - {map[string]string{annotation: "16kkk"}, ""}, {map[string]string{annotation: ""}, ""}, {map[string]string{}, ""}, {nil, ""}, @@ -57,7 +54,6 @@ func TestParse(t *testing.T) { for _, testCase := range testCases { ing.SetAnnotations(testCase.annotations) - //nolint:errcheck // Ignore the error since invalid cases will be checked with expected results result, _ := ap.Parse(ing) if result != testCase.expected { t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations) diff --git a/internal/ingress/annotations/connection/main.go b/internal/ingress/annotations/connection/main.go index b0242ac23..7d45fdc36 100644 --- a/internal/ingress/annotations/connection/main.go +++ b/internal/ingress/annotations/connection/main.go @@ -17,32 +17,12 @@ limitations under the License. package connection import ( - "regexp" - - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - connectionProxyHeaderAnnotation = "connection-proxy-header" -) - -var validConnectionHeaderValue = regexp.MustCompile(`^(close|keep-alive)$`) - -var connectionHeadersAnnotations = parser.Annotation{ - Group: "backend", - Annotations: parser.AnnotationFields{ - connectionProxyHeaderAnnotation: { - Validator: parser.ValidateRegex(validConnectionHeaderValue, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation allows setting a specific value for "proxy_set_header Connection" directive. Right now it is restricted to "close" or "keep-alive"`, - }, - }, -} - // Config returns the connection header configuration for an Ingress rule type Config struct { Header string `json:"header"` @@ -50,22 +30,18 @@ type Config struct { } type connection struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new port in redirect annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return connection{ - r: r, - annotationConfig: connectionHeadersAnnotations, - } + return connection{r} } // Parse parses the annotations contained in the ingress // rule used to indicate if the connection header should be overridden. func (a connection) Parse(ing *networking.Ingress) (interface{}, error) { - cp, err := parser.GetStringAnnotation(connectionProxyHeaderAnnotation, ing, a.annotationConfig.Annotations) + cp, err := parser.GetStringAnnotation("connection-proxy-header", ing) if err != nil { return &Config{ Enabled: false, @@ -94,12 +70,3 @@ func (r1 *Config) Equal(r2 *Config) bool { return true } - -func (a connection) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a connection) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, connectionHeadersAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/connection/main_test.go b/internal/ingress/annotations/connection/main_test.go index 1b9361f31..d86aeb16a 100644 --- a/internal/ingress/annotations/connection/main_test.go +++ b/internal/ingress/annotations/connection/main_test.go @@ -20,7 +20,7 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" @@ -37,12 +37,10 @@ func TestParse(t *testing.T) { testCases := []struct { annotations map[string]string expected *Config - expectErr bool }{ - {map[string]string{annotation: "keep-alive"}, &Config{Enabled: true, Header: "keep-alive"}, false}, - {map[string]string{annotation: "not-allowed-value"}, &Config{Enabled: false}, true}, - {map[string]string{}, &Config{Enabled: false}, true}, - {nil, &Config{Enabled: false}, true}, + {map[string]string{annotation: "keep-alive"}, &Config{Enabled: true, Header: "keep-alive"}}, + {map[string]string{}, &Config{Enabled: false}}, + {nil, &Config{Enabled: false}}, } ing := &networking.Ingress{ @@ -55,14 +53,9 @@ func TestParse(t *testing.T) { for _, testCase := range testCases { ing.SetAnnotations(testCase.annotations) - i, err := ap.Parse(ing) - if (err != nil) != testCase.expectErr { - t.Fatalf("expected error: %t got error: %t err value: %s. %+v", testCase.expectErr, err != nil, err, testCase.annotations) - } - p, ok := i.(*Config) - if !ok { - t.Fatalf("expected a Config type") - } + i, _ := ap.Parse(ing) + p, _ := i.(*Config) + if !p.Equal(testCase.expected) { t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, p, testCase.annotations) } diff --git a/internal/ingress/annotations/cors/main.go b/internal/ingress/annotations/cors/main.go index cef4fb1b2..2f1a0a37b 100644 --- a/internal/ingress/annotations/cors/main.go +++ b/internal/ingress/annotations/cors/main.go @@ -18,20 +18,17 @@ package cors import ( "regexp" - "strings" - networking "k8s.io/api/networking/v1" - "k8s.io/klog/v2" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" ) const ( // Default values defaultCorsMethods = "GET, PUT, POST, DELETE, PATCH, OPTIONS" - defaultCorsHeaders = "DNT,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization" + defaultCorsHeaders = "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization" defaultCorsMaxAge = 1728000 ) @@ -39,107 +36,36 @@ var ( // Regex are defined here to prevent information leak, if user tries to set anything not valid // that could cause the Response to contain some internal value/variable (like returning $pid, $upstream_addr, etc) // Origin must contain a http/s Origin (including or not the port) or the value '*' - // This Regex is composed of the following: - // * Sets a group that can be (https?://)?*?.something.com:port? OR null - // * Allows this to be repeated as much as possible, and separated by comma - // Otherwise it should be '*' - corsOriginRegexValidator = regexp.MustCompile(`^((((([a-z]+://)?(\*\.)?[A-Za-z0-9\-.]*(:\d+)?,?)|null)+)|\*)?$`) - // corsOriginRegex defines the regex for validation inside Parse - corsOriginRegex = regexp.MustCompile(`^([a-z]+://(\*\.)?[A-Za-z0-9\-.]*(:\d+)?|\*|null)?$`) + corsOriginRegex = regexp.MustCompile(`^(https?://[A-Za-z0-9\-\.]*(:[0-9]+)?|\*)?$`) // Method must contain valid methods list (PUT, GET, POST, BLA) // May contain or not spaces between each verb corsMethodsRegex = regexp.MustCompile(`^([A-Za-z]+,?\s?)+$`) + // Headers must contain valid values only (X-HEADER12, X-ABC) + // May contain or not spaces between each Header + corsHeadersRegex = regexp.MustCompile(`^([A-Za-z0-9\-\_]+,?\s?)+$`) // Expose Headers must contain valid values only (*, X-HEADER12, X-ABC) // May contain or not spaces between each Header corsExposeHeadersRegex = regexp.MustCompile(`^(([A-Za-z0-9\-\_]+|\*),?\s?)+$`) ) -const ( - corsEnableAnnotation = "enable-cors" - corsAllowOriginAnnotation = "cors-allow-origin" - corsAllowHeadersAnnotation = "cors-allow-headers" - corsAllowMethodsAnnotation = "cors-allow-methods" - corsAllowCredentialsAnnotation = "cors-allow-credentials" //#nosec G101 - corsExposeHeadersAnnotation = "cors-expose-headers" - corsMaxAgeAnnotation = "cors-max-age" -) - -var corsAnnotation = parser.Annotation{ - Group: "cors", - Annotations: parser.AnnotationFields{ - corsEnableAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation enables Cross-Origin Resource Sharing (CORS) in an Ingress rule`, - }, - corsAllowOriginAnnotation: { - Validator: parser.ValidateRegex(corsOriginRegexValidator, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation controls what's the accepted Origin for CORS. - This is a multi-valued field, separated by ','. It must follow this format: protocol://origin-site.com, protocol://origin-site.com:port, null, or *. - It also supports single level wildcard subdomains and follows this format: https://*.foo.bar, http://*.bar.foo:8080 or myprotocol://*.abc.bar.foo:9000 - Protocol can be any lowercase string, like http, https, or mycustomprotocol.`, - }, - corsAllowHeadersAnnotation: { - Validator: parser.ValidateRegex(parser.HeadersVariable, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation controls which headers are accepted. - This is a multi-valued field, separated by ',' and accepts letters, numbers, _ and -`, - }, - corsAllowMethodsAnnotation: { - Validator: parser.ValidateRegex(corsMethodsRegex, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation controls which methods are accepted. - This is a multi-valued field, separated by ',' and accepts only letters (upper and lower case)`, - }, - corsAllowCredentialsAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation controls if credentials can be passed during CORS operations.`, - }, - corsExposeHeadersAnnotation: { - Validator: parser.ValidateRegex(corsExposeHeadersRegex, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation controls which headers are exposed to response. - This is a multi-valued field, separated by ',' and accepts letters, numbers, _, - and *.`, - }, - corsMaxAgeAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation controls how long, in seconds, preflight requests can be cached.`, - }, - }, -} - type cors struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // Config contains the Cors configuration to be used in the Ingress type Config struct { - CorsEnabled bool `json:"corsEnabled"` - CorsAllowOrigin []string `json:"corsAllowOrigin"` - CorsAllowMethods string `json:"corsAllowMethods"` - CorsAllowHeaders string `json:"corsAllowHeaders"` - CorsAllowCredentials bool `json:"corsAllowCredentials"` - CorsExposeHeaders string `json:"corsExposeHeaders"` - CorsMaxAge int `json:"corsMaxAge"` + CorsEnabled bool `json:"corsEnabled"` + CorsAllowOrigin string `json:"corsAllowOrigin"` + CorsAllowMethods string `json:"corsAllowMethods"` + CorsAllowHeaders string `json:"corsAllowHeaders"` + CorsAllowCredentials bool `json:"corsAllowCredentials"` + CorsExposeHeaders string `json:"corsExposeHeaders"` + CorsMaxAge int `json:"corsMaxAge"` } // NewParser creates a new CORS annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return cors{ - r: r, - annotationConfig: corsAnnotation, - } + return cors{r} } // Equal tests for equality between two External types @@ -165,20 +91,13 @@ func (c1 *Config) Equal(c2 *Config) bool { if c1.CorsAllowMethods != c2.CorsAllowMethods { return false } + if c1.CorsAllowOrigin != c2.CorsAllowOrigin { + return false + } if c1.CorsEnabled != c2.CorsEnabled { return false } - if len(c1.CorsAllowOrigin) != len(c2.CorsAllowOrigin) { - return false - } - - for i, v := range c1.CorsAllowOrigin { - if v != c2.CorsAllowOrigin[i] { - return false - } - } - return true } @@ -188,84 +107,41 @@ func (c cors) Parse(ing *networking.Ingress) (interface{}, error) { var err error config := &Config{} - config.CorsEnabled, err = parser.GetBoolAnnotation(corsEnableAnnotation, ing, c.annotationConfig.Annotations) + config.CorsEnabled, err = parser.GetBoolAnnotation("enable-cors", ing) if err != nil { - if errors.IsValidationError(err) { - klog.Warningf("enable-cors is invalid, defaulting to 'false'") - } config.CorsEnabled = false } - config.CorsAllowOrigin = []string{} - unparsedOrigins, err := parser.GetStringAnnotation(corsAllowOriginAnnotation, ing, c.annotationConfig.Annotations) - if err == nil { - origins := strings.Split(unparsedOrigins, ",") - for _, origin := range origins { - origin = strings.TrimSpace(origin) - if origin == "" { - continue - } - - if origin == "*" { - config.CorsAllowOrigin = []string{"*"} - break - } - - if !corsOriginRegex.MatchString(origin) { - klog.Errorf("Error parsing cors-allow-origin parameters. Supplied incorrect origin: %s. Skipping.", origin) - continue - } - config.CorsAllowOrigin = append(config.CorsAllowOrigin, origin) - klog.Infof("Current config.corsAllowOrigin %v", config.CorsAllowOrigin) - } - } else { - if errors.IsValidationError(err) { - klog.Warningf("cors-allow-origin is invalid, defaulting to '*'") - } - config.CorsAllowOrigin = []string{"*"} + config.CorsAllowOrigin, err = parser.GetStringAnnotation("cors-allow-origin", ing) + if err != nil || !corsOriginRegex.MatchString(config.CorsAllowOrigin) { + config.CorsAllowOrigin = "*" } - config.CorsAllowHeaders, err = parser.GetStringAnnotation(corsAllowHeadersAnnotation, ing, c.annotationConfig.Annotations) - if err != nil || !parser.HeadersVariable.MatchString(config.CorsAllowHeaders) { + config.CorsAllowHeaders, err = parser.GetStringAnnotation("cors-allow-headers", ing) + if err != nil || !corsHeadersRegex.MatchString(config.CorsAllowHeaders) { config.CorsAllowHeaders = defaultCorsHeaders } - config.CorsAllowMethods, err = parser.GetStringAnnotation(corsAllowMethodsAnnotation, ing, c.annotationConfig.Annotations) + config.CorsAllowMethods, err = parser.GetStringAnnotation("cors-allow-methods", ing) if err != nil || !corsMethodsRegex.MatchString(config.CorsAllowMethods) { config.CorsAllowMethods = defaultCorsMethods } - config.CorsAllowCredentials, err = parser.GetBoolAnnotation(corsAllowCredentialsAnnotation, ing, c.annotationConfig.Annotations) + config.CorsAllowCredentials, err = parser.GetBoolAnnotation("cors-allow-credentials", ing) if err != nil { - if errors.IsValidationError(err) { - if errors.IsValidationError(err) { - klog.Warningf("cors-allow-credentials is invalid, defaulting to 'true'") - } - } config.CorsAllowCredentials = true } - config.CorsExposeHeaders, err = parser.GetStringAnnotation(corsExposeHeadersAnnotation, ing, c.annotationConfig.Annotations) + config.CorsExposeHeaders, err = parser.GetStringAnnotation("cors-expose-headers", ing) if err != nil || !corsExposeHeadersRegex.MatchString(config.CorsExposeHeaders) { config.CorsExposeHeaders = "" } - config.CorsMaxAge, err = parser.GetIntAnnotation(corsMaxAgeAnnotation, ing, c.annotationConfig.Annotations) + config.CorsMaxAge, err = parser.GetIntAnnotation("cors-max-age", ing) if err != nil { - if errors.IsValidationError(err) { - klog.Warningf("cors-max-age is invalid, defaulting to %d", defaultCorsMaxAge) - } config.CorsMaxAge = defaultCorsMaxAge } return config, nil -} -func (c cors) GetDocumentation() parser.AnnotationFields { - return c.annotationConfig.Annotations -} - -func (c cors) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(c.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, corsAnnotation.Annotations) } diff --git a/internal/ingress/annotations/cors/main_test.go b/internal/ingress/annotations/cors/main_test.go index 0b6b3671b..6f75ce6a7 100644 --- a/internal/ingress/annotations/cors/main_test.go +++ b/internal/ingress/annotations/cors/main_test.go @@ -17,26 +17,20 @@ limitations under the License. package cors import ( - "reflect" "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const enableAnnotation = "true" - func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -45,13 +39,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -78,13 +68,13 @@ func TestIngressCorsConfigValid(t *testing.T) { data := map[string]string{} // Valid - data[parser.GetAnnotationWithPrefix(corsEnableAnnotation)] = enableAnnotation - data[parser.GetAnnotationWithPrefix(corsAllowHeadersAnnotation)] = "DNT,X-CustomHeader, Keep-Alive,User-Agent" - data[parser.GetAnnotationWithPrefix(corsAllowCredentialsAnnotation)] = "false" - data[parser.GetAnnotationWithPrefix(corsAllowMethodsAnnotation)] = "GET, PATCH" - data[parser.GetAnnotationWithPrefix(corsAllowOriginAnnotation)] = "null, https://origin123.test.com:4443" - data[parser.GetAnnotationWithPrefix(corsExposeHeadersAnnotation)] = "*, X-CustomResponseHeader" - data[parser.GetAnnotationWithPrefix(corsMaxAgeAnnotation)] = "600" + data[parser.GetAnnotationWithPrefix("enable-cors")] = "true" + data[parser.GetAnnotationWithPrefix("cors-allow-headers")] = "DNT,X-CustomHeader, Keep-Alive,User-Agent" + data[parser.GetAnnotationWithPrefix("cors-allow-credentials")] = "false" + data[parser.GetAnnotationWithPrefix("cors-allow-methods")] = "GET, PATCH" + data[parser.GetAnnotationWithPrefix("cors-allow-origin")] = "https://origin123.test.com:4443" + data[parser.GetAnnotationWithPrefix("cors-expose-headers")] = "*, X-CustomResponseHeader" + data[parser.GetAnnotationWithPrefix("cors-max-age")] = "600" ing.SetAnnotations(data) corst, err := NewParser(&resolver.Mock{}).Parse(ing) @@ -98,31 +88,31 @@ func TestIngressCorsConfigValid(t *testing.T) { } if !nginxCors.CorsEnabled { - t.Errorf("expected %v but returned %v", data[parser.GetAnnotationWithPrefix(corsEnableAnnotation)], nginxCors.CorsEnabled) + t.Errorf("expected %v but returned %v", data[parser.GetAnnotationWithPrefix("enable-cors")], nginxCors.CorsEnabled) } if nginxCors.CorsAllowCredentials { - t.Errorf("expected %v but returned %v", data[parser.GetAnnotationWithPrefix(corsAllowCredentialsAnnotation)], nginxCors.CorsAllowCredentials) + t.Errorf("expected %v but returned %v", data[parser.GetAnnotationWithPrefix("cors-allow-credentials")], nginxCors.CorsAllowCredentials) } if nginxCors.CorsAllowHeaders != "DNT,X-CustomHeader, Keep-Alive,User-Agent" { - t.Errorf("expected %v but returned %v", data[parser.GetAnnotationWithPrefix(corsAllowHeadersAnnotation)], nginxCors.CorsAllowHeaders) + t.Errorf("expected %v but returned %v", data[parser.GetAnnotationWithPrefix("cors-allow-headers")], nginxCors.CorsAllowHeaders) } if nginxCors.CorsAllowMethods != "GET, PATCH" { - t.Errorf("expected %v but returned %v", data[parser.GetAnnotationWithPrefix(corsAllowMethodsAnnotation)], nginxCors.CorsAllowMethods) + t.Errorf("expected %v but returned %v", data[parser.GetAnnotationWithPrefix("cors-allow-methods")], nginxCors.CorsAllowMethods) } - if !reflect.DeepEqual(nginxCors.CorsAllowOrigin, []string{"null", "https://origin123.test.com:4443"}) { - t.Errorf("expected %v but returned %v", data[parser.GetAnnotationWithPrefix(corsAllowOriginAnnotation)], nginxCors.CorsAllowOrigin) + if nginxCors.CorsAllowOrigin != "https://origin123.test.com:4443" { + t.Errorf("expected %v but returned %v", data[parser.GetAnnotationWithPrefix("cors-allow-origin")], nginxCors.CorsAllowOrigin) } if nginxCors.CorsExposeHeaders != "*, X-CustomResponseHeader" { - t.Errorf("expected %v but returned %v", data[parser.GetAnnotationWithPrefix(corsExposeHeadersAnnotation)], nginxCors.CorsExposeHeaders) + t.Errorf("expected %v but returned %v", data[parser.GetAnnotationWithPrefix("cors-expose-headers")], nginxCors.CorsExposeHeaders) } if nginxCors.CorsMaxAge != 600 { - t.Errorf("expected %v but returned %v", data[parser.GetAnnotationWithPrefix(corsMaxAgeAnnotation)], nginxCors.CorsMaxAge) + t.Errorf("expected %v but returned %v", data[parser.GetAnnotationWithPrefix("cors-max-age")], nginxCors.CorsMaxAge) } } @@ -132,13 +122,13 @@ func TestIngressCorsConfigInvalid(t *testing.T) { data := map[string]string{} // Valid - data[parser.GetAnnotationWithPrefix(corsEnableAnnotation)] = "yes" - data[parser.GetAnnotationWithPrefix(corsAllowHeadersAnnotation)] = "@alright, #ingress" - data[parser.GetAnnotationWithPrefix(corsAllowCredentialsAnnotation)] = "no" - data[parser.GetAnnotationWithPrefix(corsAllowMethodsAnnotation)] = "GET, PATCH, $nginx" - data[parser.GetAnnotationWithPrefix(corsAllowOriginAnnotation)] = "origin123.test.com:4443" - data[parser.GetAnnotationWithPrefix(corsExposeHeadersAnnotation)] = "@alright, #ingress" - data[parser.GetAnnotationWithPrefix(corsMaxAgeAnnotation)] = "abcd" + data[parser.GetAnnotationWithPrefix("enable-cors")] = "yes" + data[parser.GetAnnotationWithPrefix("cors-allow-headers")] = "@alright, #ingress" + data[parser.GetAnnotationWithPrefix("cors-allow-credentials")] = "no" + data[parser.GetAnnotationWithPrefix("cors-allow-methods")] = "GET, PATCH, $nginx" + data[parser.GetAnnotationWithPrefix("cors-allow-origin")] = "origin123.test.com:4443" + data[parser.GetAnnotationWithPrefix("cors-expose-headers")] = "@alright, #ingress" + data[parser.GetAnnotationWithPrefix("cors-max-age")] = "abcd" ing.SetAnnotations(data) corst, err := NewParser(&resolver.Mock{}).Parse(ing) @@ -167,6 +157,10 @@ func TestIngressCorsConfigInvalid(t *testing.T) { t.Errorf("expected %v but returned %v", defaultCorsHeaders, nginxCors.CorsAllowMethods) } + if nginxCors.CorsAllowOrigin != "*" { + t.Errorf("expected %v but returned %v", "*", nginxCors.CorsAllowOrigin) + } + if nginxCors.CorsExposeHeaders != "" { t.Errorf("expected %v but returned %v", "", nginxCors.CorsExposeHeaders) } @@ -175,93 +169,3 @@ func TestIngressCorsConfigInvalid(t *testing.T) { t.Errorf("expected %v but returned %v", defaultCorsMaxAge, nginxCors.CorsMaxAge) } } - -func TestIngressCorsConfigAllowOriginWithTrailingComma(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix(corsEnableAnnotation)] = enableAnnotation - - // Include a trailing comma and an empty value between the commas. - data[parser.GetAnnotationWithPrefix(corsAllowOriginAnnotation)] = "https://origin123.test.com:4443, ,https://origin321.test.com:4443," - ing.SetAnnotations(data) - - corst, err := NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("error parsing annotations: %v", err) - } - - nginxCors, ok := corst.(*Config) - if !ok { - t.Errorf("expected a Config type but returned %t", corst) - } - - if !nginxCors.CorsEnabled { - t.Errorf("expected %v but returned %v", data[parser.GetAnnotationWithPrefix(corsEnableAnnotation)], nginxCors.CorsEnabled) - } - - expectedCorsAllowOrigins := []string{"https://origin123.test.com:4443", "https://origin321.test.com:4443"} - if !reflect.DeepEqual(nginxCors.CorsAllowOrigin, expectedCorsAllowOrigins) { - t.Errorf("expected %v but returned %v", expectedCorsAllowOrigins, nginxCors.CorsAllowOrigin) - } -} - -func TestIngressCorsConfigAllowOriginNull(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix(corsEnableAnnotation)] = enableAnnotation - - // Include a trailing comma and an empty value between the commas. - data[parser.GetAnnotationWithPrefix(corsAllowOriginAnnotation)] = "https://origin123.test.com:4443,null,https://origin321.test.com:4443" - ing.SetAnnotations(data) - - corst, err := NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("error parsing annotations: %v", err) - } - - nginxCors, ok := corst.(*Config) - if !ok { - t.Errorf("expected a Config type but returned %t", corst) - } - - if !nginxCors.CorsEnabled { - t.Errorf("expected %v but returned %v", data[parser.GetAnnotationWithPrefix(corsEnableAnnotation)], nginxCors.CorsEnabled) - } - - expectedCorsAllowOrigins := []string{"https://origin123.test.com:4443", "null", "https://origin321.test.com:4443"} - if !reflect.DeepEqual(nginxCors.CorsAllowOrigin, expectedCorsAllowOrigins) { - t.Errorf("expected %v but returned %v", expectedCorsAllowOrigins, nginxCors.CorsAllowOrigin) - } -} - -func TestIngressCorsConfigAllowOriginWithNonHttpProtocol(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix(corsEnableAnnotation)] = enableAnnotation - - // Include a trailing comma and an empty value between the commas. - data[parser.GetAnnotationWithPrefix(corsAllowOriginAnnotation)] = "test://localhost" - ing.SetAnnotations(data) - - corst, err := NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("error parsing annotations: %v", err) - } - - nginxCors, ok := corst.(*Config) - if !ok { - t.Errorf("expected a Config type but returned %t", corst) - } - - if !nginxCors.CorsEnabled { - t.Errorf("expected %v but returned %v", data[parser.GetAnnotationWithPrefix(corsEnableAnnotation)], nginxCors.CorsEnabled) - } - - expectedCorsAllowOrigins := []string{"test://localhost"} - if !reflect.DeepEqual(nginxCors.CorsAllowOrigin, expectedCorsAllowOrigins) { - t.Errorf("expected %v but returned %v", expectedCorsAllowOrigins, nginxCors.CorsAllowOrigin) - } -} diff --git a/internal/ingress/annotations/customheaders/main.go b/internal/ingress/annotations/customheaders/main.go deleted file mode 100644 index d48018d8c..000000000 --- a/internal/ingress/annotations/customheaders/main.go +++ /dev/null @@ -1,137 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package customheaders - -import ( - "fmt" - "reflect" - "regexp" - "slices" - - "k8s.io/klog/v2" - - networking "k8s.io/api/networking/v1" - - "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - ing_errors "k8s.io/ingress-nginx/internal/ingress/errors" - "k8s.io/ingress-nginx/internal/ingress/resolver" -) - -// Config returns the custom response headers for an Ingress rule -type Config struct { - Headers map[string]string `json:"headers,omitempty"` -} - -// Equal tests for equality between two Config types -func (c1 *Config) Equal(c2 *Config) bool { - if c1 == c2 { - return true - } - if c1 == nil || c2 == nil { - return false - } - - return reflect.DeepEqual(c1.Headers, c2.Headers) -} - -var ( - headerRegexp = regexp.MustCompile(`^[a-zA-Z\d\-_]+$`) - valueRegexp = regexp.MustCompile(`^[a-zA-Z\d_ :;.,\\/"'?!(){}\[\]@<>=\-+*#$&\x60|~^%]+$`) -) - -// ValidHeader checks is the provided string satisfies the header's name regex -func ValidHeader(header string) bool { - return headerRegexp.MatchString(header) -} - -// ValidValue checks is the provided string satisfies the value regex -func ValidValue(header string) bool { - return valueRegexp.MatchString(header) -} - -const ( - customHeadersConfigMapAnnotation = "custom-headers" -) - -var customHeadersAnnotation = parser.Annotation{ - Group: "backend", - Annotations: parser.AnnotationFields{ - customHeadersConfigMapAnnotation: { - Validator: parser.ValidateRegex(parser.BasicCharsRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation sets the name of a ConfigMap that specifies headers to pass to the client. - Only ConfigMaps on the same namespace are allowed`, - }, - }, -} - -type customHeaders struct { - r resolver.Resolver - annotationConfig parser.Annotation -} - -// NewParser creates a new custom response headers annotation parser -func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return customHeaders{r: r, annotationConfig: customHeadersAnnotation} -} - -func (a customHeaders) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -// Parse parses the annotations contained in the ingress to use -// custom response headers -func (a customHeaders) Parse(ing *networking.Ingress) (interface{}, error) { - clientHeadersConfigMapName, err := parser.GetStringAnnotation(customHeadersConfigMapAnnotation, ing, a.annotationConfig.Annotations) - if err != nil { - klog.V(3).InfoS("client-headers annotation is undefined and will not be set") - } - - var headers map[string]string - defBackend := a.r.GetDefaultBackend() - - if clientHeadersConfigMapName != "" { - clientHeadersMapContents, err := a.r.GetConfigMap(clientHeadersConfigMapName) - if err != nil { - return nil, ing_errors.NewLocationDenied(fmt.Sprintf("unable to find configMap %q", clientHeadersConfigMapName)) - } - - for header, value := range clientHeadersMapContents.Data { - if !ValidHeader(header) { - return nil, ing_errors.NewLocationDenied("invalid header name in configmap") - } - if !ValidValue(value) { - return nil, ing_errors.NewLocationDenied("invalid header value in configmap") - } - if !slices.Contains(defBackend.AllowedResponseHeaders, header) { - return nil, ing_errors.NewLocationDenied(fmt.Sprintf("header %s is not allowed, defined allowed headers inside global-allowed-response-headers %v", header, defBackend.AllowedResponseHeaders)) - } - } - - headers = clientHeadersMapContents.Data - } - - return &Config{ - Headers: headers, - }, nil -} - -func (a customHeaders) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, customHeadersAnnotation.Annotations) -} diff --git a/internal/ingress/annotations/customheaders/main_test.go b/internal/ingress/annotations/customheaders/main_test.go deleted file mode 100644 index 81c0b795a..000000000 --- a/internal/ingress/annotations/customheaders/main_test.go +++ /dev/null @@ -1,113 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package customheaders - -import ( - "reflect" - "testing" - - api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" - meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/defaults" - "k8s.io/ingress-nginx/internal/ingress/resolver" -) - -func buildIngress() *networking.Ingress { - return &networking.Ingress{ - ObjectMeta: meta_v1.ObjectMeta{ - Name: "foo", - Namespace: api.NamespaceDefault, - }, - Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, - }, - }, - } -} - -type mockBackend struct { - resolver.Mock -} - -// GetDefaultBackend returns the backend that must be used as default -func (m mockBackend) GetDefaultBackend() defaults.Backend { - return defaults.Backend{ - AllowedResponseHeaders: []string{"Content-Type", "Access-Control-Max-Age"}, - } -} - -func TestCustomHeadersParseInvalidAnnotations(t *testing.T) { - ing := buildIngress() - configMapResolver := mockBackend{} - configMapResolver.ConfigMaps = map[string]*api.ConfigMap{} - - _, err := NewParser(configMapResolver).Parse(ing) - if err != nil { - t.Errorf("expected error parsing ingress with custom-response-headers") - } - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix("custom-headers")] = "custom-headers-configmap" - ing.SetAnnotations(data) - i, err := NewParser(&resolver.Mock{}).Parse(ing) - if err == nil { - t.Errorf("expected error parsing ingress with custom-response-headers") - } - if i != nil { - t.Errorf("expected %v but got %v", nil, i) - } -} - -func TestCustomHeadersParseAnnotations(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix("custom-headers")] = "custom-headers-configmap" - ing.SetAnnotations(data) - - configMapResolver := mockBackend{} - configMapResolver.ConfigMaps = map[string]*api.ConfigMap{} - - configMapResolver.ConfigMaps["custom-headers-configmap"] = &api.ConfigMap{Data: map[string]string{"Content-Type": "application/json", "Access-Control-Max-Age": "600"}} - - i, err := NewParser(configMapResolver).Parse(ing) - if err != nil { - t.Errorf("unexpected error parsing ingress with custom-response-headers: %s", err) - } - val, ok := i.(*Config) - if !ok { - t.Errorf("expected a *Config type") - } - - expectedResponseHeaders := map[string]string{} - expectedResponseHeaders["Content-Type"] = "application/json" - expectedResponseHeaders["Access-Control-Max-Age"] = "600" - - c := &Config{expectedResponseHeaders} - - if !reflect.DeepEqual(c, val) { - t.Errorf("expected %v but got %v", c, val) - } -} diff --git a/internal/ingress/annotations/customhttperrors/main.go b/internal/ingress/annotations/customhttperrors/main.go index f3c72a22f..3c5fbf077 100644 --- a/internal/ingress/annotations/customhttperrors/main.go +++ b/internal/ingress/annotations/customhttperrors/main.go @@ -17,60 +17,34 @@ limitations under the License. package customhttperrors import ( - "regexp" "strconv" "strings" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - customHTTPErrorsAnnotation = "custom-http-errors" -) - -// We accept anything between 400 and 599, on a comma separated. -var arrayOfHTTPErrors = regexp.MustCompile(`^(?:[4,5]\d{2},?)*$`) - -var customHTTPErrorsAnnotations = parser.Annotation{ - Group: "backend", - Annotations: parser.AnnotationFields{ - customHTTPErrorsAnnotation: { - Validator: parser.ValidateRegex(arrayOfHTTPErrors, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `If a default backend annotation is specified on the ingress, the errors code specified on this annotation - will be routed to that annotation's default backend service. Otherwise they will be routed to the global default backend. - A comma-separated list of error codes is accepted (anything between 400 and 599, like 403, 503)`, - }, - }, -} - type customhttperrors struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new custom http errors annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return customhttperrors{ - r: r, - annotationConfig: customHTTPErrorsAnnotations, - } + return customhttperrors{r} } // Parse parses the annotations contained in the ingress to use // custom http errors func (e customhttperrors) Parse(ing *networking.Ingress) (interface{}, error) { - c, err := parser.GetStringAnnotation(customHTTPErrorsAnnotation, ing, e.annotationConfig.Annotations) + c, err := parser.GetStringAnnotation("custom-http-errors", ing) if err != nil { return nil, err } cSplit := strings.Split(c, ",") - codes := make([]int, 0, len(cSplit)) + var codes []int for _, i := range cSplit { num, err := strconv.Atoi(i) if err != nil { @@ -81,12 +55,3 @@ func (e customhttperrors) Parse(ing *networking.Ingress) (interface{}, error) { return codes, nil } - -func (e customhttperrors) GetDocumentation() parser.AnnotationFields { - return e.annotationConfig.Annotations -} - -func (e customhttperrors) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(e.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, customHTTPErrorsAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/customhttperrors/main_test.go b/internal/ingress/annotations/customhttperrors/main_test.go index 1f87247ed..3827d197f 100644 --- a/internal/ingress/annotations/customhttperrors/main_test.go +++ b/internal/ingress/annotations/customhttperrors/main_test.go @@ -22,10 +22,12 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" + + "k8s.io/apimachinery/pkg/util/intstr" ) func buildIngress() *networking.Ingress { @@ -35,13 +37,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, }, } diff --git a/internal/ingress/annotations/defaultbackend/main.go b/internal/ingress/annotations/defaultbackend/main.go index 9ae44f052..ff4f41ce3 100644 --- a/internal/ingress/annotations/defaultbackend/main.go +++ b/internal/ingress/annotations/defaultbackend/main.go @@ -19,64 +19,35 @@ package defaultbackend import ( "fmt" - networking "k8s.io/api/networking/v1" + "github.com/pkg/errors" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - defaultBackendAnnotation = "default-backend" -) - -var defaultBackendAnnotations = parser.Annotation{ - Group: "backend", - Annotations: parser.AnnotationFields{ - defaultBackendAnnotation: { - Validator: parser.ValidateServiceName, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This service will be used to handle the response when the configured service in the Ingress rule does not have any active endpoints. - It will also be used to handle the error responses if both this annotation and the custom-http-errors annotation are set.`, - }, - }, -} - type backend struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new default backend annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return backend{ - r: r, - annotationConfig: defaultBackendAnnotations, - } + return backend{r} } // Parse parses the annotations contained in the ingress to use // a custom default backend -func (b backend) Parse(ing *networking.Ingress) (interface{}, error) { - s, err := parser.GetStringAnnotation(defaultBackendAnnotation, ing, b.annotationConfig.Annotations) +func (db backend) Parse(ing *networking.Ingress) (interface{}, error) { + s, err := parser.GetStringAnnotation("default-backend", ing) if err != nil { return nil, err } name := fmt.Sprintf("%v/%v", ing.Namespace, s) - svc, err := b.r.GetService(name) + svc, err := db.r.GetService(name) if err != nil { - return nil, fmt.Errorf("unexpected error reading service %s: %w", name, err) + return nil, errors.Wrapf(err, "unexpected error reading service %v", name) } return svc, nil } - -func (b backend) GetDocumentation() parser.AnnotationFields { - return b.annotationConfig.Annotations -} - -func (b backend) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(b.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, defaultBackendAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/defaultbackend/main_test.go b/internal/ingress/annotations/defaultbackend/main_test.go index 214d07803..927860215 100644 --- a/internal/ingress/annotations/defaultbackend/main_test.go +++ b/internal/ingress/annotations/defaultbackend/main_test.go @@ -20,21 +20,19 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" + + "k8s.io/apimachinery/pkg/util/intstr" ) func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -43,13 +41,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -91,51 +85,21 @@ func (m mockService) GetService(name string) (*api.Service, error) { func TestAnnotations(t *testing.T) { ing := buildIngress() - tests := map[string]struct { - expectErr bool - serviceName string - }{ - "valid name": { - serviceName: "demo-service", - expectErr: false, - }, - "not in backend": { - serviceName: "demo1-service", - expectErr: true, - }, - "invalid dns name": { - serviceName: "demo-service.something.tld", - expectErr: true, - }, - "invalid name": { - serviceName: "something/xpto", - expectErr: true, - }, - "invalid characters": { - serviceName: "something;xpto", - expectErr: true, - }, + data := map[string]string{} + data[parser.GetAnnotationWithPrefix("default-backend")] = "demo-service" + ing.SetAnnotations(data) + + fakeService := &mockService{} + i, err := NewParser(fakeService).Parse(ing) + if err != nil { + t.Errorf("unexpected error %v", err) } - for _, test := range tests { - data := map[string]string{} - data[parser.GetAnnotationWithPrefix(defaultBackendAnnotation)] = test.serviceName - ing.SetAnnotations(data) - - fakeService := &mockService{} - i, err := NewParser(fakeService).Parse(ing) - if (err != nil) != test.expectErr { - t.Errorf("expected error: %t got error: %t err value: %s. %+v", test.expectErr, err != nil, err, i) - } - - if !test.expectErr { - svc, ok := i.(*api.Service) - if !ok { - t.Errorf("expected *api.Service but got %v", svc) - } - if svc.Name != test.serviceName { - t.Errorf("expected %v but got %v", test.serviceName, svc.Name) - } - } + svc, ok := i.(*api.Service) + if !ok { + t.Errorf("expected *api.Service but got %v", svc) + } + if svc.Name != "demo-service" { + t.Errorf("expected %v but got %v", "demo-service", svc.Name) } } diff --git a/internal/ingress/annotations/disableproxyintercepterrors/main.go b/internal/ingress/annotations/disableproxyintercepterrors/main.go deleted file mode 100644 index 650d29707..000000000 --- a/internal/ingress/annotations/disableproxyintercepterrors/main.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package disableproxyintercepterrors - -import ( - networking "k8s.io/api/networking/v1" - - "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/errors" - "k8s.io/ingress-nginx/internal/ingress/resolver" -) - -const ( - disableProxyInterceptErrorsAnnotation = "disable-proxy-intercept-errors" -) - -var disableProxyInterceptErrorsAnnotations = parser.Annotation{ - Group: "backend", - Annotations: parser.AnnotationFields{ - disableProxyInterceptErrorsAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation allows to disable NGINX proxy-intercept-errors when custom-http-errors are set. - If a default backend annotation is specified on the ingress, the errors will be routed to that annotation's default backend service (instead of the global default backend). - Different ingresses can specify different sets of errors codes and there are UseCases where NGINX shall not intercept all errors returned from upstream.`, - }, - }, -} - -type disableProxyInterceptErrors struct { - r resolver.Resolver - annotationConfig parser.Annotation -} - -func (pie disableProxyInterceptErrors) GetDocumentation() parser.AnnotationFields { - return pie.annotationConfig.Annotations -} - -func (pie disableProxyInterceptErrors) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(pie.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, disableProxyInterceptErrorsAnnotations.Annotations) -} - -// NewParser creates a new disableProxyInterceptErrors annotation parser -func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return disableProxyInterceptErrors{ - r: r, - annotationConfig: disableProxyInterceptErrorsAnnotations, - } -} - -func (pie disableProxyInterceptErrors) Parse(ing *networking.Ingress) (interface{}, error) { - val, err := parser.GetBoolAnnotation(disableProxyInterceptErrorsAnnotation, ing, pie.annotationConfig.Annotations) - - // A missing annotation is not a problem, just use the default - if err == errors.ErrMissingAnnotations { - return false, nil // default is false - } - - return val, nil -} diff --git a/internal/ingress/annotations/disableproxyintercepterrors/main_test.go b/internal/ingress/annotations/disableproxyintercepterrors/main_test.go deleted file mode 100644 index 1238bb4e0..000000000 --- a/internal/ingress/annotations/disableproxyintercepterrors/main_test.go +++ /dev/null @@ -1,64 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package disableproxyintercepterrors - -import ( - "testing" - - api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" - meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/resolver" -) - -func buildIngress() *networking.Ingress { - return &networking.Ingress{ - ObjectMeta: meta_v1.ObjectMeta{ - Name: "foo", - Namespace: api.NamespaceDefault, - }, - Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, - }, - }, - } -} - -func TestParseAnnotations(t *testing.T) { - ing := buildIngress() - - _, err := NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix("disable-proxy-intercept-errors")] = "true" - ing.SetAnnotations(data) - // test ingress using the annotation without a TLS section - _, err = NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error parsing ingress with disable-proxy-intercept-errors") - } -} diff --git a/internal/ingress/annotations/fastcgi/main.go b/internal/ingress/annotations/fastcgi/main.go index 9e7ab3c61..174a2716c 100644 --- a/internal/ingress/annotations/fastcgi/main.go +++ b/internal/ingress/annotations/fastcgi/main.go @@ -19,50 +19,18 @@ package fastcgi import ( "fmt" "reflect" - "regexp" - networking "k8s.io/api/networking/v1" + "github.com/pkg/errors" + networking "k8s.io/api/networking/v1beta1" "k8s.io/client-go/tools/cache" - "k8s.io/klog/v2" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" ing_errors "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - fastCGIIndexAnnotation = "fastcgi-index" - fastCGIParamsAnnotation = "fastcgi-params-configmap" //#nosec G101 -) - -// fast-cgi valid parameters is just a single file name (like index.php) -var ( - regexValidIndexAnnotationAndKey = regexp.MustCompile(`^[A-Za-z0-9.\-\_]+$`) - validFCGIValue = regexp.MustCompile(`^[A-Za-z0-9\-\_\$\{\}/.]*$`) -) - -var fastCGIAnnotations = parser.Annotation{ - Group: "fastcgi", - Annotations: parser.AnnotationFields{ - fastCGIIndexAnnotation: { - Validator: parser.ValidateRegex(regexValidIndexAnnotationAndKey, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation can be used to specify an index file`, - }, - fastCGIParamsAnnotation: { - Validator: parser.ValidateRegex(parser.BasicCharsRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation can be used to specify a ConfigMap containing the fastcgi parameters as a key/value. - Only ConfigMaps on the same namespace of ingress can be used. They key and value from ConfigMap are validated for unauthorized characters.`, - }, - }, -} - type fastcgi struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // Config describes the per location fastcgi config @@ -90,78 +58,50 @@ func (l1 *Config) Equal(l2 *Config) bool { // NewParser creates a new fastcgiConfig protocol annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return fastcgi{ - r: r, - annotationConfig: fastCGIAnnotations, - } + return fastcgi{r} } // ParseAnnotations parses the annotations contained in the ingress // rule used to indicate the fastcgiConfig. func (a fastcgi) Parse(ing *networking.Ingress) (interface{}, error) { + fcgiConfig := Config{} if ing.GetAnnotations() == nil { return fcgiConfig, nil } - index, err := parser.GetStringAnnotation(fastCGIIndexAnnotation, ing, a.annotationConfig.Annotations) + index, err := parser.GetStringAnnotation("fastcgi-index", ing) if err != nil { - if ing_errors.IsValidationError(err) { - return fcgiConfig, err - } index = "" } - fcgiConfig.Index = index - cm, err := parser.GetStringAnnotation(fastCGIParamsAnnotation, ing, a.annotationConfig.Annotations) + cm, err := parser.GetStringAnnotation("fastcgi-params-configmap", ing) if err != nil { - if ing_errors.IsValidationError(err) { - return fcgiConfig, err - } return fcgiConfig, nil } cmns, cmn, err := cache.SplitMetaNamespaceKey(cm) if err != nil { - return fcgiConfig, ing_errors.LocationDeniedError{ - Reason: fmt.Errorf("error reading configmap name from annotation: %w", err), + return fcgiConfig, ing_errors.LocationDenied{ + Reason: errors.Wrap(err, "error reading configmap name from annotation"), } } - secCfg := a.r.GetSecurityConfiguration() - // We don't accept different namespaces for secrets. - if cmns != "" && !secCfg.AllowCrossNamespaceResources && cmns != ing.Namespace { - return fcgiConfig, fmt.Errorf("different namespace is not supported on fast_cgi param configmap") + if cmns == "" { + cmns = ing.Namespace } - cm = fmt.Sprintf("%v/%v", ing.Namespace, cmn) + cm = fmt.Sprintf("%v/%v", cmns, cmn) cmap, err := a.r.GetConfigMap(cm) if err != nil { - return fcgiConfig, ing_errors.LocationDeniedError{ - Reason: fmt.Errorf("unexpected error reading configmap %s: %w", cm, err), + return fcgiConfig, ing_errors.LocationDenied{ + Reason: errors.Wrapf(err, "unexpected error reading configmap %v", cm), } } - for k, v := range cmap.Data { - if !regexValidIndexAnnotationAndKey.MatchString(k) || !validFCGIValue.MatchString(v) { - klog.ErrorS(fmt.Errorf("fcgi contains invalid key or value"), "fcgi annotation error", "configmap", cmap.Name, "namespace", cmap.Namespace, "key", k, "value", v) - return fcgiConfig, ing_errors.NewValidationError(fastCGIParamsAnnotation) - } - } - - fcgiConfig.Index = index fcgiConfig.Params = cmap.Data return fcgiConfig, nil } - -func (a fastcgi) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a fastcgi) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, fastCGIAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/fastcgi/main_test.go b/internal/ingress/annotations/fastcgi/main_test.go index f0bbc9d65..6802a41c2 100644 --- a/internal/ingress/annotations/fastcgi/main_test.go +++ b/internal/ingress/annotations/fastcgi/main_test.go @@ -17,16 +17,16 @@ limitations under the License. package fastcgi import ( - "fmt" - "reflect" "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/cache" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" + + "k8s.io/apimachinery/pkg/util/intstr" ) func buildIngress() *networking.Ingress { @@ -36,13 +36,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "fastcgi", + ServicePort: intstr.FromInt(80), }, }, } @@ -50,35 +46,19 @@ func buildIngress() *networking.Ingress { type mockConfigMap struct { resolver.Mock - extraConfigMap map[string]map[string]string } func (m mockConfigMap) GetConfigMap(name string) (*api.ConfigMap, error) { - if m.extraConfigMap == nil { - m.extraConfigMap = make(map[string]map[string]string) - } - cmdata, ok := m.extraConfigMap[name] - - if name != "default/demo-configmap" && name != "otherns/demo-configmap" && !ok { - return nil, fmt.Errorf("there is no configmap with name %v", name) - } - - cmns, cmn, err := cache.SplitMetaNamespaceKey(name) - if err != nil { - return nil, fmt.Errorf("invalid configmap name") - } - - data := map[string]string{"REDIRECT_STATUS": "200", "SERVER_NAME": "$server_name"} - if ok { - data = cmdata + if name != "default/demo-configmap" { + return nil, errors.Errorf("there is no configmap with name %v", name) } return &api.ConfigMap{ ObjectMeta: meta_v1.ObjectMeta{ - Namespace: cmns, - Name: cmn, + Namespace: api.NamespaceDefault, + Name: "demo-secret", }, - Data: data, + Data: map[string]string{"REDIRECT_STATUS": "200", "SERVER_NAME": "$server_name"}, }, nil } @@ -155,6 +135,7 @@ func TestParseFastCGIInvalidParamsConfigMapAnnotation(t *testing.T) { invalidConfigMapList := []string{"unknown/configMap", "unknown/config/map"} for _, configmap := range invalidConfigMapList { + data := map[string]string{} data[parser.GetAnnotationWithPrefix("fastcgi-params-configmap")] = configmap ing.SetAnnotations(data) @@ -227,20 +208,8 @@ func TestParseFastCGIParamsConfigMapAnnotationWithNS(t *testing.T) { } } -func TestParseFastCGIParamsConfigMapAnnotationWithDifferentNS(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix("fastcgi-params-configmap")] = "otherns/demo-configmap" - ing.SetAnnotations(data) - - _, err := NewParser(&mockConfigMap{}).Parse(ing) - if err == nil { - t.Errorf("Different namespace configmap should return an error") - } -} - func TestConfigEquality(t *testing.T) { + var nilConfig *Config config := Config{ @@ -292,121 +261,3 @@ func TestConfigEquality(t *testing.T) { t.Errorf("config4 should be equal to config") } } - -func Test_fastcgi_Parse(t *testing.T) { - tests := []struct { - name string - index string - configmapname string - configmap map[string]string - want interface{} - wantErr bool - }{ - { - name: "valid configuration", - index: "indexxpto-92123.php", - configmapname: "default/fcgiconfig", - configmap: map[string]string{ - "REQUEST_METHOD": "$request_method", - "SCRIPT_FILENAME": "$document_root$fastcgi_script_name", - }, - want: Config{ - Index: "indexxpto-92123.php", - Params: map[string]string{ - "REQUEST_METHOD": "$request_method", - "SCRIPT_FILENAME": "$document_root$fastcgi_script_name", - }, - }, - }, - { - name: "invalid index name", - index: "indexxpto-92123$xx.php", - configmapname: "default/fcgiconfig", - configmap: map[string]string{ - "REQUEST_METHOD": "$request_method", - "SCRIPT_FILENAME": "$document_root$fastcgi_script_name", - }, - want: Config{}, - wantErr: true, - }, - { - name: "invalid configmap namespace", - index: "indexxpto-92123.php", - configmapname: "otherns/fcgiconfig", - configmap: map[string]string{ - "REQUEST_METHOD": "$request_method", - "SCRIPT_FILENAME": "$document_root$fastcgi_script_name", - }, - want: Config{Index: "indexxpto-92123.php"}, - wantErr: true, - }, - { - name: "invalid configmap namespace name", - index: "indexxpto-92123.php", - configmapname: "otherns/fcgicon;{fig", - configmap: map[string]string{ - "REQUEST_METHOD": "$request_method", - "SCRIPT_FILENAME": "$document_root$fastcgi_script_name", - }, - want: Config{Index: "indexxpto-92123.php"}, - wantErr: true, - }, - { - name: "invalid configmap values key", - index: "indexxpto-92123.php", - configmapname: "default/fcgiconfig", - configmap: map[string]string{ - "REQUEST_METHOD$XPTO": "$request_method", - }, - want: Config{Index: "indexxpto-92123.php"}, - wantErr: true, - }, - { - name: "invalid configmap values val", - index: "indexxpto-92123.php", - configmapname: "default/fcgiconfig", - configmap: map[string]string{ - "REQUEST_METHOD_XPTO": "$request_method{test};a", - }, - want: Config{Index: "indexxpto-92123.php"}, - wantErr: true, - }, - { - name: "invalid configmap values val", - index: "indexxpto-92123.php", - configmapname: "default/fcgiconfig", - configmap: map[string]string{ - "SCRIPT_FILENAME": "/app/src/index.php", - }, - want: Config{Index: "indexxpto-92123.php", Params: map[string]string{ - "SCRIPT_FILENAME": "/app/src/index.php", - }}, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix("fastcgi-index")] = tt.index - data[parser.GetAnnotationWithPrefix("fastcgi-params-configmap")] = tt.configmapname - ing.SetAnnotations(data) - - m := &mockConfigMap{ - extraConfigMap: map[string]map[string]string{ - tt.configmapname: tt.configmap, - }, - } - - got, err := NewParser(m).Parse(ing) - if (err != nil) != tt.wantErr { - t.Errorf("fastcgi.Parse() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("fastcgi.Parse() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/internal/ingress/annotations/globalratelimit/main.go b/internal/ingress/annotations/globalratelimit/main.go new file mode 100644 index 000000000..e4b18bd66 --- /dev/null +++ b/internal/ingress/annotations/globalratelimit/main.go @@ -0,0 +1,111 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package globalratelimit + +import ( + "strings" + "time" + + "github.com/pkg/errors" + networking "k8s.io/api/networking/v1beta1" + + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + ing_errors "k8s.io/ingress-nginx/internal/ingress/errors" + "k8s.io/ingress-nginx/internal/ingress/resolver" + "k8s.io/ingress-nginx/internal/net" + "k8s.io/ingress-nginx/internal/sets" +) + +const defaultKey = "$remote_addr" + +// Config encapsulates all global rate limit attributes +type Config struct { + Namespace string `json:"namespace"` + Limit int `json:"limit"` + WindowSize int `json:"window-size"` + Key string `json:"key"` + IgnoredCIDRs []string `json:"ignored-cidrs"` +} + +// Equal tests for equality between two Config types +func (l *Config) Equal(r *Config) bool { + if l.Namespace != r.Namespace { + return false + } + if l.Limit != r.Limit { + return false + } + if l.WindowSize != r.WindowSize { + return false + } + if l.Key != r.Key { + return false + } + if len(l.IgnoredCIDRs) != len(r.IgnoredCIDRs) || !sets.StringElementsMatch(l.IgnoredCIDRs, r.IgnoredCIDRs) { + return false + } + + return true +} + +type globalratelimit struct { + r resolver.Resolver +} + +// NewParser creates a new globalratelimit annotation parser +func NewParser(r resolver.Resolver) parser.IngressAnnotation { + return globalratelimit{r} +} + +// Parse extracts globalratelimit annotations from the given ingress +// and returns them structured as Config type +func (a globalratelimit) Parse(ing *networking.Ingress) (interface{}, error) { + config := &Config{} + + limit, _ := parser.GetIntAnnotation("global-rate-limit", ing) + rawWindowSize, _ := parser.GetStringAnnotation("global-rate-limit-window", ing) + + if limit == 0 || len(rawWindowSize) == 0 { + return config, nil + } + + windowSize, err := time.ParseDuration(rawWindowSize) + if err != nil { + return config, ing_errors.LocationDenied{ + Reason: errors.Wrap(err, "failed to parse 'global-rate-limit-window' value"), + } + } + + key, _ := parser.GetStringAnnotation("global-rate-limit-key", ing) + if len(key) == 0 { + key = defaultKey + } + + rawIgnoredCIDRs, _ := parser.GetStringAnnotation("global-rate-limit-ignored-cidrs", ing) + ignoredCIDRs, err := net.ParseCIDRs(rawIgnoredCIDRs) + if err != nil { + return nil, err + } + + config.Namespace = strings.Replace(string(ing.UID), "-", "", -1) + config.Limit = limit + config.WindowSize = int(windowSize.Seconds()) + config.Key = key + config.IgnoredCIDRs = ignoredCIDRs + + return config, nil +} diff --git a/internal/ingress/annotations/globalratelimit/main_test.go b/internal/ingress/annotations/globalratelimit/main_test.go new file mode 100644 index 000000000..38da8f4a9 --- /dev/null +++ b/internal/ingress/annotations/globalratelimit/main_test.go @@ -0,0 +1,179 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package globalratelimit + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/pkg/errors" + api "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1beta1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + ing_errors "k8s.io/ingress-nginx/internal/ingress/errors" + "k8s.io/ingress-nginx/internal/ingress/resolver" +) + +const UID = "31285d47-b150-4dcf-bd6f-12c46d769f6e" +const expectedUID = "31285d47b1504dcfbd6f12c46d769f6e" + +func buildIngress() *networking.Ingress { + defaultBackend := networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), + } + + return &networking.Ingress{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "foo", + Namespace: api.NamespaceDefault, + UID: UID, + }, + Spec: networking.IngressSpec{ + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), + }, + Rules: []networking.IngressRule{ + { + Host: "foo.bar.com", + IngressRuleValue: networking.IngressRuleValue{ + HTTP: &networking.HTTPIngressRuleValue{ + Paths: []networking.HTTPIngressPath{ + { + Path: "/foo", + Backend: defaultBackend, + }, + }, + }, + }, + }, + }, + }, + } +} + +type mockBackend struct { + resolver.Mock +} + +func TestGlobalRateLimiting(t *testing.T) { + ing := buildIngress() + + annRateLimit := parser.GetAnnotationWithPrefix("global-rate-limit") + annRateLimitWindow := parser.GetAnnotationWithPrefix("global-rate-limit-window") + annRateLimitKey := parser.GetAnnotationWithPrefix("global-rate-limit-key") + annRateLimitIgnoredCIDRs := parser.GetAnnotationWithPrefix("global-rate-limit-ignored-cidrs") + + testCases := []struct { + title string + annotations map[string]string + expectedConfig *Config + expectedErr error + }{ + { + "no annotation", + nil, + &Config{}, + nil, + }, + { + "minimum required annotations", + map[string]string{ + annRateLimit: "100", + annRateLimitWindow: "2m", + }, + &Config{ + Namespace: expectedUID, + Limit: 100, + WindowSize: 120, + Key: "$remote_addr", + IgnoredCIDRs: make([]string, 0), + }, + nil, + }, + { + "global-rate-limit-key annotation", + map[string]string{ + annRateLimit: "100", + annRateLimitWindow: "2m", + annRateLimitKey: "$http_x_api_user", + }, + &Config{ + Namespace: expectedUID, + Limit: 100, + WindowSize: 120, + Key: "$http_x_api_user", + IgnoredCIDRs: make([]string, 0), + }, + nil, + }, + { + "global-rate-limit-ignored-cidrs annotation", + map[string]string{ + annRateLimit: "100", + annRateLimitWindow: "2m", + annRateLimitKey: "$http_x_api_user", + annRateLimitIgnoredCIDRs: "127.0.0.1, 200.200.24.0/24", + }, + &Config{ + Namespace: expectedUID, + Limit: 100, + WindowSize: 120, + Key: "$http_x_api_user", + IgnoredCIDRs: []string{"127.0.0.1", "200.200.24.0/24"}, + }, + nil, + }, + { + "incorrect duration for window", + map[string]string{ + annRateLimit: "100", + annRateLimitWindow: "2mb", + annRateLimitKey: "$http_x_api_user", + }, + &Config{}, + ing_errors.LocationDenied{ + Reason: errors.Wrap(fmt.Errorf(`time: unknown unit "mb" in duration "2mb"`), + "failed to parse 'global-rate-limit-window' value"), + }, + }, + } + + for _, testCase := range testCases { + ing.SetAnnotations(testCase.annotations) + + i, actualErr := NewParser(mockBackend{}).Parse(ing) + if (testCase.expectedErr == nil || actualErr == nil) && testCase.expectedErr != actualErr { + t.Errorf("expected error 'nil' but got '%v'", actualErr) + } else if testCase.expectedErr != nil && actualErr != nil && + testCase.expectedErr.Error() != actualErr.Error() { + t.Errorf("expected error '%v' but got '%v'", testCase.expectedErr, actualErr) + } + + actualConfig := i.(*Config) + if !testCase.expectedConfig.Equal(actualConfig) { + expectedJSON, _ := json.Marshal(testCase.expectedConfig) + actualJSON, _ := json.Marshal(actualConfig) + t.Errorf("%v: expected config '%s' but got '%s'", testCase.title, expectedJSON, actualJSON) + } + } +} diff --git a/internal/ingress/annotations/http2pushpreload/main.go b/internal/ingress/annotations/http2pushpreload/main.go index 123940d9d..c542f03cf 100644 --- a/internal/ingress/annotations/http2pushpreload/main.go +++ b/internal/ingress/annotations/http2pushpreload/main.go @@ -17,52 +17,23 @@ limitations under the License. package http2pushpreload import ( - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - http2PushPreloadAnnotation = "http2-push-preload" -) - -var http2PushPreloadAnnotations = parser.Annotation{ - Group: "http2", - Annotations: parser.AnnotationFields{ - http2PushPreloadAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `Enables automatic conversion of preload links specified in the “Link” response header fields into push requests`, - }, - }, -} - type http2PushPreload struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new http2PushPreload annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return http2PushPreload{ - r: r, - annotationConfig: http2PushPreloadAnnotations, - } + return http2PushPreload{r} } // Parse parses the annotations contained in the ingress rule // used to add http2 push preload to the server func (h2pp http2PushPreload) Parse(ing *networking.Ingress) (interface{}, error) { - return parser.GetBoolAnnotation(http2PushPreloadAnnotation, ing, h2pp.annotationConfig.Annotations) -} - -func (h2pp http2PushPreload) GetDocumentation() parser.AnnotationFields { - return h2pp.annotationConfig.Annotations -} - -func (h2pp http2PushPreload) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(h2pp.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, http2PushPreloadAnnotations.Annotations) + return parser.GetBoolAnnotation("http2-push-preload", ing) } diff --git a/internal/ingress/annotations/http2pushpreload/main_test.go b/internal/ingress/annotations/http2pushpreload/main_test.go index eb6e9111d..6b24ecfae 100644 --- a/internal/ingress/annotations/http2pushpreload/main_test.go +++ b/internal/ingress/annotations/http2pushpreload/main_test.go @@ -20,15 +20,14 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" ) func TestParse(t *testing.T) { - annotation := parser.GetAnnotationWithPrefix(http2PushPreloadAnnotation) + annotation := parser.GetAnnotationWithPrefix("http2-push-preload") ap := NewParser(&resolver.Mock{}) if ap == nil { t.Fatalf("expected a parser.IngressAnnotation but returned nil") @@ -37,14 +36,12 @@ func TestParse(t *testing.T) { testCases := []struct { annotations map[string]string expected bool - expectErr bool }{ - {map[string]string{annotation: "true"}, true, false}, - {map[string]string{annotation: "1"}, true, false}, - {map[string]string{annotation: "xpto"}, false, true}, - {map[string]string{annotation: ""}, false, false}, - {map[string]string{}, false, false}, - {nil, false, false}, + {map[string]string{annotation: "true"}, true}, + {map[string]string{annotation: "1"}, true}, + {map[string]string{annotation: ""}, false}, + {map[string]string{}, false}, + {nil, false}, } ing := &networking.Ingress{ @@ -57,10 +54,7 @@ func TestParse(t *testing.T) { for _, testCase := range testCases { ing.SetAnnotations(testCase.annotations) - result, err := ap.Parse(ing) - if ((err != nil) != testCase.expectErr) && !errors.IsInvalidContent(err) && !errors.IsMissingAnnotations(err) { - t.Fatalf("expected error: %t got error: %t err value: %s. %+v", testCase.expectErr, err != nil, err, testCase.annotations) - } + result, _ := ap.Parse(ing) if result != testCase.expected { t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations) } diff --git a/internal/ingress/annotations/influxdb/main.go b/internal/ingress/annotations/influxdb/main.go new file mode 100644 index 000000000..cec014b89 --- /dev/null +++ b/internal/ingress/annotations/influxdb/main.go @@ -0,0 +1,101 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package influxdb + +import ( + networking "k8s.io/api/networking/v1beta1" + + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "k8s.io/ingress-nginx/internal/ingress/resolver" +) + +type influxdb struct { + r resolver.Resolver +} + +// Config contains the IfluxDB configuration to be used in the Ingress +type Config struct { + InfluxDBEnabled bool `json:"influxDBEnabled"` + InfluxDBMeasurement string `json:"influxDBMeasurement"` + InfluxDBPort string `json:"influxDBPort"` + InfluxDBHost string `json:"influxDBHost"` + InfluxDBServerName string `json:"influxDBServerName"` +} + +// NewParser creates a new InfluxDB annotation parser +func NewParser(r resolver.Resolver) parser.IngressAnnotation { + return influxdb{r} +} + +// Parse parses the annotations to look for InfluxDB configurations +func (c influxdb) Parse(ing *networking.Ingress) (interface{}, error) { + var err error + config := &Config{} + + config.InfluxDBEnabled, err = parser.GetBoolAnnotation("enable-influxdb", ing) + if err != nil { + config.InfluxDBEnabled = false + } + + config.InfluxDBMeasurement, err = parser.GetStringAnnotation("influxdb-measurement", ing) + if err != nil { + config.InfluxDBMeasurement = "default" + } + + config.InfluxDBPort, err = parser.GetStringAnnotation("influxdb-port", ing) + if err != nil { + // This is not the default 8086 port but the port usually used to expose + // influxdb in UDP, the module uses UDP to talk to influx via the line protocol. + config.InfluxDBPort = "8089" + } + + config.InfluxDBHost, err = parser.GetStringAnnotation("influxdb-host", ing) + if err != nil { + config.InfluxDBHost = "127.0.0.1" + } + + config.InfluxDBServerName, err = parser.GetStringAnnotation("influxdb-server-name", ing) + if err != nil { + config.InfluxDBServerName = "nginx-ingress" + } + + return config, nil +} + +// Equal tests for equality between two Config types +func (e1 *Config) Equal(e2 *Config) bool { + if e1 == e2 { + return true + } + if e1 == nil || e2 == nil { + return false + } + if e1.InfluxDBEnabled != e2.InfluxDBEnabled { + return false + } + if e1.InfluxDBPort != e2.InfluxDBPort { + return false + } + if e1.InfluxDBHost != e2.InfluxDBHost { + return false + } + if e1.InfluxDBServerName != e2.InfluxDBServerName { + return false + } + + return true +} diff --git a/internal/ingress/annotations/influxdb/main_test.go b/internal/ingress/annotations/influxdb/main_test.go new file mode 100644 index 000000000..97ba14963 --- /dev/null +++ b/internal/ingress/annotations/influxdb/main_test.go @@ -0,0 +1,131 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package influxdb + +import ( + "testing" + + api "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1beta1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "k8s.io/ingress-nginx/internal/ingress/resolver" +) + +func buildIngress() *networking.Ingress { + defaultBackend := networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), + } + + return &networking.Ingress{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "foo", + Namespace: api.NamespaceDefault, + }, + Spec: networking.IngressSpec{ + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), + }, + Rules: []networking.IngressRule{ + { + Host: "foo.bar.com", + IngressRuleValue: networking.IngressRuleValue{ + HTTP: &networking.HTTPIngressRuleValue{ + Paths: []networking.HTTPIngressPath{ + { + Path: "/foo", + Backend: defaultBackend, + }, + }, + }, + }, + }, + }, + }, + } +} + +func TestIngressInvalidInfluxDB(t *testing.T) { + ing := buildIngress() + + influx, _ := NewParser(&resolver.Mock{}).Parse(ing) + nginxInflux, ok := influx.(*Config) + if !ok { + t.Errorf("expected a Config type") + } + + if nginxInflux.InfluxDBEnabled == true { + t.Errorf("expected influxdb enabled but returned %v", nginxInflux.InfluxDBEnabled) + } + + if nginxInflux.InfluxDBMeasurement != "default" { + t.Errorf("expected measurement name not found. Found %v", nginxInflux.InfluxDBMeasurement) + } + + if nginxInflux.InfluxDBPort != "8089" { + t.Errorf("expected port not found. Found %v", nginxInflux.InfluxDBPort) + } + + if nginxInflux.InfluxDBHost != "127.0.0.1" { + t.Errorf("expected host not found. Found %v", nginxInflux.InfluxDBHost) + } + + if nginxInflux.InfluxDBServerName != "nginx-ingress" { + t.Errorf("expected server name not found. Found %v", nginxInflux.InfluxDBServerName) + } +} + +func TestIngressInfluxDB(t *testing.T) { + ing := buildIngress() + + data := map[string]string{} + data[parser.GetAnnotationWithPrefix("enable-influxdb")] = "true" + data[parser.GetAnnotationWithPrefix("influxdb-measurement")] = "nginxmeasures" + data[parser.GetAnnotationWithPrefix("influxdb-port")] = "9091" + data[parser.GetAnnotationWithPrefix("influxdb-host")] = "10.99.0.13" + data[parser.GetAnnotationWithPrefix("influxdb-server-name")] = "nginx-test-1" + ing.SetAnnotations(data) + + influx, _ := NewParser(&resolver.Mock{}).Parse(ing) + nginxInflux, ok := influx.(*Config) + if !ok { + t.Errorf("expected a Config type") + } + + if !nginxInflux.InfluxDBEnabled { + t.Errorf("expected influxdb enabled but returned %v", nginxInflux.InfluxDBEnabled) + } + + if nginxInflux.InfluxDBMeasurement != "nginxmeasures" { + t.Errorf("expected measurement name not found. Found %v", nginxInflux.InfluxDBMeasurement) + } + + if nginxInflux.InfluxDBPort != "9091" { + t.Errorf("expected port not found. Found %v", nginxInflux.InfluxDBPort) + } + + if nginxInflux.InfluxDBHost != "10.99.0.13" { + t.Errorf("expected host not found. Found %v", nginxInflux.InfluxDBHost) + } + + if nginxInflux.InfluxDBServerName != "nginx-test-1" { + t.Errorf("expected server name not found. Found %v", nginxInflux.InfluxDBServerName) + } +} diff --git a/internal/ingress/annotations/ipallowlist/main.go b/internal/ingress/annotations/ipallowlist/main.go deleted file mode 100644 index 23a3bd9e7..000000000 --- a/internal/ingress/annotations/ipallowlist/main.go +++ /dev/null @@ -1,132 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipallowlist - -import ( - "fmt" - "sort" - "strings" - - networking "k8s.io/api/networking/v1" - "k8s.io/ingress-nginx/internal/net" - - "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - ing_errors "k8s.io/ingress-nginx/internal/ingress/errors" - "k8s.io/ingress-nginx/internal/ingress/resolver" - "k8s.io/ingress-nginx/pkg/util/sets" -) - -const ( - ipWhitelistAnnotation = "whitelist-source-range" - ipAllowlistAnnotation = "allowlist-source-range" -) - -var allowlistAnnotations = parser.Annotation{ - Group: "acl", - Annotations: parser.AnnotationFields{ - ipAllowlistAnnotation: { - Validator: parser.ValidateCIDRs, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, // Failure on parsing this may cause undesired access - Documentation: `This annotation allows setting a list of IPs and networks allowed to access this Location`, - AnnotationAliases: []string{ipWhitelistAnnotation}, - }, - }, -} - -// SourceRange returns the CIDR -type SourceRange struct { - CIDR []string `json:"cidr,omitempty"` -} - -// Equal tests for equality between two SourceRange types -func (sr1 *SourceRange) Equal(sr2 *SourceRange) bool { - if sr1 == sr2 { - return true - } - if sr1 == nil || sr2 == nil { - return false - } - - return sets.StringElementsMatch(sr1.CIDR, sr2.CIDR) -} - -type ipallowlist struct { - r resolver.Resolver - annotationConfig parser.Annotation -} - -// NewParser creates a new ipallowlist annotation parser -func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return ipallowlist{ - r: r, - annotationConfig: allowlistAnnotations, - } -} - -// ParseAnnotations parses the annotations contained in the ingress -// rule used to limit access to certain client addresses or networks. -// Multiple ranges can specified using commas as separator -// e.g. `18.0.0.0/8,56.0.0.0/8` -func (a ipallowlist) Parse(ing *networking.Ingress) (interface{}, error) { - defBackend := a.r.GetDefaultBackend() - - defaultAllowlistSourceRange := make([]string, len(defBackend.WhitelistSourceRange)) - copy(defaultAllowlistSourceRange, defBackend.WhitelistSourceRange) - sort.Strings(defaultAllowlistSourceRange) - - val, err := parser.GetStringAnnotation(ipAllowlistAnnotation, ing, a.annotationConfig.Annotations) - // A missing annotation is not a problem, just use the default - if err != nil { - if err == ing_errors.ErrMissingAnnotations { - return &SourceRange{CIDR: defaultAllowlistSourceRange}, nil - } - - return &SourceRange{CIDR: defaultAllowlistSourceRange}, ing_errors.LocationDeniedError{ - Reason: err, - } - } - - values := strings.Split(val, ",") - ipnets, ips, err := net.ParseIPNets(values...) - if err != nil && len(ips) == 0 { - return &SourceRange{CIDR: defaultAllowlistSourceRange}, ing_errors.LocationDeniedError{ - Reason: fmt.Errorf("the annotation does not contain a valid IP address or network: %w", err), - } - } - - cidrs := []string{} - for k := range ipnets { - cidrs = append(cidrs, k) - } - for k := range ips { - cidrs = append(cidrs, k) - } - - sort.Strings(cidrs) - - return &SourceRange{cidrs}, nil -} - -func (a ipallowlist) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a ipallowlist) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, allowlistAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/ipallowlist/main_test.go b/internal/ingress/annotations/ipallowlist/main_test.go deleted file mode 100644 index b16b25a5b..000000000 --- a/internal/ingress/annotations/ipallowlist/main_test.go +++ /dev/null @@ -1,262 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipallowlist - -import ( - "testing" - - api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" - meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/defaults" - "k8s.io/ingress-nginx/internal/ingress/resolver" -) - -func buildIngress() *networking.Ingress { - defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, - } - - return &networking.Ingress{ - ObjectMeta: meta_v1.ObjectMeta{ - Name: "foo", - Namespace: api.NamespaceDefault, - }, - Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, - }, - Rules: []networking.IngressRule{ - { - Host: "foo.bar.com", - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - Path: "/foo", - Backend: defaultBackend, - }, - }, - }, - }, - }, - }, - }, - } -} - -func TestParseAnnotations(t *testing.T) { - ing := buildIngress() - tests := map[string]struct { - net string - expectCidr []string - expectErr bool - errOut string - }{ - "test parse a valid net": { - net: "10.0.0.0/24", - expectCidr: []string{"10.0.0.0/24"}, - expectErr: false, - }, - "test parse a invalid net": { - net: "ww", - expectErr: true, - errOut: "annotation nginx.ingress.kubernetes.io/allowlist-source-range contains invalid value", - }, - "test parse a empty net": { - net: "", - expectErr: true, - errOut: "the annotation nginx.ingress.kubernetes.io/allowlist-source-range does not contain a valid value ()", - }, - "test parse multiple valid cidr": { - net: "2.2.2.2/32,1.1.1.1/32,3.3.3.0/24", - expectCidr: []string{"1.1.1.1/32", "2.2.2.2/32", "3.3.3.0/24"}, - expectErr: false, - }, - } - - for testName, test := range tests { - data := map[string]string{} - data[parser.GetAnnotationWithPrefix(ipAllowlistAnnotation)] = test.net - ing.SetAnnotations(data) - p := NewParser(&resolver.Mock{}) - i, err := p.Parse(ing) - if (err != nil) != test.expectErr { - t.Errorf("%s expected error: %t got error: %t err value: %s. %+v", testName, test.expectErr, err != nil, err, i) - } - if test.expectErr && err != nil { - if err.Error() != test.errOut { - t.Errorf("expected error %s but got %s", test.errOut, err) - } - } - if !test.expectErr { - sr, ok := i.(*SourceRange) - if !ok { - t.Errorf("%v:expected a SourceRange type", testName) - } - if !strsEquals(sr.CIDR, test.expectCidr) { - t.Errorf("%v:expected %v CIDR but %v returned", testName, test.expectCidr, sr.CIDR) - } - } - } -} - -type mockBackend struct { - resolver.Mock -} - -// GetDefaultBackend returns the backend that must be used as default -func (m mockBackend) GetDefaultBackend() defaults.Backend { - return defaults.Backend{ - WhitelistSourceRange: []string{"4.4.4.0/24", "1.2.3.4/32"}, - } -} - -// Test that when we have a allowlist set on the Backend that is used when we -// don't have the annotation -func TestParseAnnotationsWithDefaultConfig(t *testing.T) { - ing := buildIngress() - - mockBackend := mockBackend{} - - tests := map[string]struct { - net string - expectCidr []string - expectErr bool - errOut string - }{ - "test parse a valid net": { - net: "10.0.0.0/24", - expectCidr: []string{"10.0.0.0/24"}, - expectErr: false, - }, - "test parse a invalid net": { - net: "ww", - expectErr: true, - errOut: "annotation nginx.ingress.kubernetes.io/allowlist-source-range contains invalid value", - }, - "test parse a empty net": { - net: "", - expectErr: true, - errOut: "the annotation nginx.ingress.kubernetes.io/allowlist-source-range does not contain a valid value ()", - }, - "test parse multiple valid cidr": { - net: "2.2.2.2/32,1.1.1.1/32,3.3.3.0/24", - expectCidr: []string{"1.1.1.1/32", "2.2.2.2/32", "3.3.3.0/24"}, - expectErr: false, - }, - } - - for testName, test := range tests { - data := map[string]string{} - data[parser.GetAnnotationWithPrefix(ipAllowlistAnnotation)] = test.net - ing.SetAnnotations(data) - p := NewParser(mockBackend) - i, err := p.Parse(ing) - if (err != nil) != test.expectErr { - t.Errorf("expected error: %t got error: %t err value: %s. %+v", test.expectErr, err != nil, err, i) - } - if test.expectErr && err != nil { - if err.Error() != test.errOut { - t.Errorf("expected error %s but got %s", test.errOut, err) - } - } - if !test.expectErr { - sr, ok := i.(*SourceRange) - if !ok { - t.Errorf("%v:expected a SourceRange type", testName) - } - if !strsEquals(sr.CIDR, test.expectCidr) { - t.Errorf("%v:expected %v CIDR but %v returned", testName, test.expectCidr, sr.CIDR) - } - } - } -} - -// Test that when we have a whitelist set on the Backend that is used when we -// don't have the annotation -func TestLegacyAnnotation(t *testing.T) { - ing := buildIngress() - - mockBackend := mockBackend{} - - tests := map[string]struct { - net string - expectCidr []string - expectErr bool - errOut string - }{ - "test parse a valid net": { - net: "10.0.0.0/24", - expectCidr: []string{"10.0.0.0/24"}, - expectErr: false, - }, - "test parse multiple valid cidr": { - net: "2.2.2.2/32,1.1.1.1/32,3.3.3.0/24", - expectCidr: []string{"1.1.1.1/32", "2.2.2.2/32", "3.3.3.0/24"}, - expectErr: false, - }, - } - - for testName, test := range tests { - data := map[string]string{} - data[parser.GetAnnotationWithPrefix(ipWhitelistAnnotation)] = test.net - ing.SetAnnotations(data) - p := NewParser(mockBackend) - i, err := p.Parse(ing) - if (err != nil) != test.expectErr { - t.Errorf("expected error: %t got error: %t err value: %s. %+v", test.expectErr, err != nil, err, i) - } - if test.expectErr && err != nil { - if err.Error() != test.errOut { - t.Errorf("expected error %s but got %s", test.errOut, err) - } - } - if !test.expectErr { - sr, ok := i.(*SourceRange) - if !ok { - t.Errorf("%v:expected a SourceRange type", testName) - } - if !strsEquals(sr.CIDR, test.expectCidr) { - t.Errorf("%v:expected %v CIDR but %v returned", testName, test.expectCidr, sr.CIDR) - } - } - } -} - -func strsEquals(a, b []string) bool { - if len(a) != len(b) { - return false - } - for i, v := range a { - if v != b[i] { - return false - } - } - return true -} diff --git a/internal/ingress/annotations/ipdenylist/main.go b/internal/ingress/annotations/ipdenylist/main.go deleted file mode 100644 index e8f02fe50..000000000 --- a/internal/ingress/annotations/ipdenylist/main.go +++ /dev/null @@ -1,129 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipdenylist - -import ( - "fmt" - "sort" - "strings" - - networking "k8s.io/api/networking/v1" - "k8s.io/ingress-nginx/internal/net" - - "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - ing_errors "k8s.io/ingress-nginx/internal/ingress/errors" - "k8s.io/ingress-nginx/internal/ingress/resolver" - "k8s.io/ingress-nginx/pkg/util/sets" -) - -const ( - ipDenylistAnnotation = "denylist-source-range" -) - -var denylistAnnotations = parser.Annotation{ - Group: "acl", - Annotations: parser.AnnotationFields{ - ipDenylistAnnotation: { - Validator: parser.ValidateCIDRs, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, // Failure on parsing this may cause undesired access - Documentation: `This annotation allows setting a list of IPs and networks that should be blocked to access this Location`, - }, - }, -} - -// SourceRange returns the CIDR -type SourceRange struct { - CIDR []string `json:"cidr,omitempty"` -} - -// Equal tests for equality between two SourceRange types -func (sr1 *SourceRange) Equal(sr2 *SourceRange) bool { - if sr1 == sr2 { - return true - } - if sr1 == nil || sr2 == nil { - return false - } - - return sets.StringElementsMatch(sr1.CIDR, sr2.CIDR) -} - -type ipdenylist struct { - r resolver.Resolver - annotationConfig parser.Annotation -} - -// NewParser creates a new denylist annotation parser -func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return ipdenylist{ - r: r, - annotationConfig: denylistAnnotations, - } -} - -// ParseAnnotations parses the annotations contained in the ingress -// rule used to limit access to certain client addresses or networks. -// Multiple ranges can specified using commas as separator -// e.g. `18.0.0.0/8,56.0.0.0/8` -func (a ipdenylist) Parse(ing *networking.Ingress) (interface{}, error) { - defBackend := a.r.GetDefaultBackend() - - defaultDenylistSourceRange := make([]string, len(defBackend.DenylistSourceRange)) - copy(defaultDenylistSourceRange, defBackend.DenylistSourceRange) - sort.Strings(defaultDenylistSourceRange) - - val, err := parser.GetStringAnnotation(ipDenylistAnnotation, ing, a.annotationConfig.Annotations) - if err != nil { - if err == ing_errors.ErrMissingAnnotations { - return &SourceRange{CIDR: defaultDenylistSourceRange}, nil - } - - return &SourceRange{CIDR: defaultDenylistSourceRange}, ing_errors.LocationDeniedError{ - Reason: err, - } - } - - values := strings.Split(val, ",") - ipnets, ips, err := net.ParseIPNets(values...) - if err != nil && len(ips) == 0 { - return &SourceRange{CIDR: defaultDenylistSourceRange}, ing_errors.LocationDeniedError{ - Reason: fmt.Errorf("the annotation does not contain a valid IP address or network: %w", err), - } - } - - cidrs := []string{} - for k := range ipnets { - cidrs = append(cidrs, k) - } - for k := range ips { - cidrs = append(cidrs, k) - } - - sort.Strings(cidrs) - - return &SourceRange{cidrs}, nil -} - -func (a ipdenylist) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a ipdenylist) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, denylistAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/ipwhitelist/main.go b/internal/ingress/annotations/ipwhitelist/main.go new file mode 100644 index 000000000..42d424873 --- /dev/null +++ b/internal/ingress/annotations/ipwhitelist/main.go @@ -0,0 +1,93 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ipwhitelist + +import ( + "sort" + "strings" + + "github.com/pkg/errors" + + networking "k8s.io/api/networking/v1beta1" + "k8s.io/ingress-nginx/internal/net" + + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + ing_errors "k8s.io/ingress-nginx/internal/ingress/errors" + "k8s.io/ingress-nginx/internal/ingress/resolver" + "k8s.io/ingress-nginx/internal/sets" +) + +// SourceRange returns the CIDR +type SourceRange struct { + CIDR []string `json:"cidr,omitempty"` +} + +// Equal tests for equality between two SourceRange types +func (sr1 *SourceRange) Equal(sr2 *SourceRange) bool { + if sr1 == sr2 { + return true + } + if sr1 == nil || sr2 == nil { + return false + } + + return sets.StringElementsMatch(sr1.CIDR, sr2.CIDR) +} + +type ipwhitelist struct { + r resolver.Resolver +} + +// NewParser creates a new whitelist annotation parser +func NewParser(r resolver.Resolver) parser.IngressAnnotation { + return ipwhitelist{r} +} + +// ParseAnnotations parses the annotations contained in the ingress +// rule used to limit access to certain client addresses or networks. +// Multiple ranges can specified using commas as separator +// e.g. `18.0.0.0/8,56.0.0.0/8` +func (a ipwhitelist) Parse(ing *networking.Ingress) (interface{}, error) { + defBackend := a.r.GetDefaultBackend() + sort.Strings(defBackend.WhitelistSourceRange) + + val, err := parser.GetStringAnnotation("whitelist-source-range", ing) + // A missing annotation is not a problem, just use the default + if err == ing_errors.ErrMissingAnnotations { + return &SourceRange{CIDR: defBackend.WhitelistSourceRange}, nil + } + + values := strings.Split(val, ",") + ipnets, ips, err := net.ParseIPNets(values...) + if err != nil && len(ips) == 0 { + return &SourceRange{CIDR: defBackend.WhitelistSourceRange}, ing_errors.LocationDenied{ + Reason: errors.Wrap(err, "the annotation does not contain a valid IP address or network"), + } + } + + cidrs := []string{} + for k := range ipnets { + cidrs = append(cidrs, k) + } + for k := range ips { + cidrs = append(cidrs, k) + } + + sort.Strings(cidrs) + + return &SourceRange{cidrs}, nil +} diff --git a/internal/ingress/annotations/ipdenylist/main_test.go b/internal/ingress/annotations/ipwhitelist/main_test.go similarity index 68% rename from internal/ingress/annotations/ipdenylist/main_test.go rename to internal/ingress/annotations/ipwhitelist/main_test.go index ebd81179a..43aef7573 100644 --- a/internal/ingress/annotations/ipdenylist/main_test.go +++ b/internal/ingress/annotations/ipwhitelist/main_test.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The Kubernetes Authors. +Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,14 +14,15 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ipdenylist +package ipwhitelist import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/defaults" "k8s.io/ingress-nginx/internal/ingress/resolver" @@ -29,12 +30,8 @@ import ( func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -43,13 +40,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -86,17 +79,12 @@ func TestParseAnnotations(t *testing.T) { "test parse a invalid net": { net: "ww", expectErr: true, - errOut: "annotation nginx.ingress.kubernetes.io/denylist-source-range contains invalid value", + errOut: "the annotation does not contain a valid IP address or network: invalid CIDR address: ww", }, "test parse a empty net": { net: "", expectErr: true, - errOut: "the annotation nginx.ingress.kubernetes.io/denylist-source-range does not contain a valid value ()", - }, - "test parse a malicious escaped string": { - net: `10.0.0.0/8"rm /tmp",11.0.0.0/8`, - expectErr: true, - errOut: `annotation nginx.ingress.kubernetes.io/denylist-source-range contains invalid value`, + errOut: "the annotation does not contain a valid IP address or network: invalid CIDR address: ", }, "test parse multiple valid cidr": { net: "2.2.2.2/32,1.1.1.1/32,3.3.3.0/24", @@ -107,16 +95,16 @@ func TestParseAnnotations(t *testing.T) { for testName, test := range tests { data := map[string]string{} - data[parser.GetAnnotationWithPrefix(ipDenylistAnnotation)] = test.net + data[parser.GetAnnotationWithPrefix("whitelist-source-range")] = test.net ing.SetAnnotations(data) p := NewParser(&resolver.Mock{}) i, err := p.Parse(ing) - if (err != nil) != test.expectErr { - t.Errorf("expected error: %t got error: %t err value: %s. %+v", test.expectErr, err != nil, err, i) + if err != nil && !test.expectErr { + t.Errorf("%v:unexpected error: %v", testName, err) } - if test.expectErr && err != nil { + if test.expectErr { if err.Error() != test.errOut { - t.Errorf("expected error %s but got %s", test.errOut, err) + t.Errorf("%v:expected error: %v but %v return", testName, test.errOut, err.Error()) } } if !test.expectErr { @@ -138,11 +126,11 @@ type mockBackend struct { // GetDefaultBackend returns the backend that must be used as default func (m mockBackend) GetDefaultBackend() defaults.Backend { return defaults.Backend{ - DenylistSourceRange: []string{"4.4.4.0/24", "1.2.3.4/32"}, + WhitelistSourceRange: []string{"4.4.4.0/24", "1.2.3.4/32"}, } } -// Test that when we have a denylist set on the Backend that is used when we +// Test that when we have a whitelist set on the Backend that is used when we // don't have the annotation func TestParseAnnotationsWithDefaultConfig(t *testing.T) { ing := buildIngress() @@ -163,12 +151,12 @@ func TestParseAnnotationsWithDefaultConfig(t *testing.T) { "test parse a invalid net": { net: "ww", expectErr: true, - errOut: "annotation nginx.ingress.kubernetes.io/denylist-source-range contains invalid value", + errOut: "the annotation does not contain a valid IP address or network: invalid CIDR address: ww", }, "test parse a empty net": { net: "", expectErr: true, - errOut: "the annotation nginx.ingress.kubernetes.io/denylist-source-range does not contain a valid value ()", + errOut: "the annotation does not contain a valid IP address or network: invalid CIDR address: ", }, "test parse multiple valid cidr": { net: "2.2.2.2/32,1.1.1.1/32,3.3.3.0/24", @@ -179,16 +167,16 @@ func TestParseAnnotationsWithDefaultConfig(t *testing.T) { for testName, test := range tests { data := map[string]string{} - data[parser.GetAnnotationWithPrefix(ipDenylistAnnotation)] = test.net + data[parser.GetAnnotationWithPrefix("whitelist-source-range")] = test.net ing.SetAnnotations(data) p := NewParser(mockBackend) i, err := p.Parse(ing) - if (err != nil) != test.expectErr { - t.Errorf("expected error: %t got error: %t err value: %s. %+v", test.expectErr, err != nil, err, i) + if err != nil && !test.expectErr { + t.Errorf("%v:unexpected error: %v", testName, err) } - if test.expectErr && err != nil { + if test.expectErr { if err.Error() != test.errOut { - t.Errorf("expected error %s but got %s", test.errOut, err) + t.Errorf("%v:expected error: %v but %v return", testName, test.errOut, err.Error()) } } if !test.expectErr { diff --git a/internal/ingress/annotations/loadbalancing/main.go b/internal/ingress/annotations/loadbalancing/main.go index 0e5ca8bd8..ddae0ccbe 100644 --- a/internal/ingress/annotations/loadbalancing/main.go +++ b/internal/ingress/annotations/loadbalancing/main.go @@ -17,58 +17,24 @@ limitations under the License. package loadbalancing import ( - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -// LB Algorithms are defined in https://github.com/kubernetes/ingress-nginx/blob/d3e75b056f77be54e01bdb18675f1bb46caece31/rootfs/etc/nginx/lua/balancer.lua#L28 - -const ( - loadBalanceAlgorithmAnnotation = "load-balance" -) - -var loadBalanceAlgorithms = []string{"round_robin", "chash", "chashsubset", "sticky_balanced", "sticky_persistent", "ewma"} - -var loadBalanceAnnotations = parser.Annotation{ - Group: "backend", - Annotations: parser.AnnotationFields{ - loadBalanceAlgorithmAnnotation: { - Validator: parser.ValidateOptions(loadBalanceAlgorithms, true, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation allows setting the load balancing algorithm that should be used. If none is specified, defaults to - the default configured by Ingress admin, otherwise to round_robin`, - }, - }, -} - type loadbalancing struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } -// NewParser creates a new Load Balancer annotation parser +// NewParser creates a new CORS annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return loadbalancing{ - r: r, - annotationConfig: loadBalanceAnnotations, - } + return loadbalancing{r} } // Parse parses the annotations contained in the ingress rule // used to indicate if the location/s contains a fragment of // configuration to be included inside the paths of the rules func (a loadbalancing) Parse(ing *networking.Ingress) (interface{}, error) { - return parser.GetStringAnnotation(loadBalanceAlgorithmAnnotation, ing, a.annotationConfig.Annotations) -} - -func (a loadbalancing) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a loadbalancing) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, loadBalanceAnnotations.Annotations) + return parser.GetStringAnnotation("load-balance", ing) } diff --git a/internal/ingress/annotations/loadbalancing/main_test.go b/internal/ingress/annotations/loadbalancing/main_test.go index a5205e523..bbda79715 100644 --- a/internal/ingress/annotations/loadbalancing/main_test.go +++ b/internal/ingress/annotations/loadbalancing/main_test.go @@ -20,7 +20,7 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" @@ -38,8 +38,7 @@ func TestParse(t *testing.T) { annotations map[string]string expected string }{ - {map[string]string{annotation: "ewma"}, "ewma"}, - {map[string]string{annotation: "ip_hash"}, ""}, // This is invalid and should not return anything + {map[string]string{annotation: "ip_hash"}, "ip_hash"}, {map[string]string{}, ""}, {nil, ""}, } @@ -54,7 +53,6 @@ func TestParse(t *testing.T) { for _, testCase := range testCases { ing.SetAnnotations(testCase.annotations) - //nolint:errcheck // Ignore the error since invalid cases will be checked with expected results result, _ := ap.Parse(ing) if result != testCase.expected { t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations) diff --git a/internal/ingress/annotations/log/main.go b/internal/ingress/annotations/log/main.go index 82d50bac3..6cf99d9c8 100644 --- a/internal/ingress/annotations/log/main.go +++ b/internal/ingress/annotations/log/main.go @@ -17,38 +17,14 @@ limitations under the License. package log import ( - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - enableAccessLogAnnotation = "enable-access-log" - enableRewriteLogAnnotation = "enable-rewrite-log" -) - -var logAnnotations = parser.Annotation{ - Group: "log", - Annotations: parser.AnnotationFields{ - enableAccessLogAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This configuration setting allows you to control if this location should generate an access_log`, - }, - enableRewriteLogAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This configuration setting allows you to control if this location should generate logs from the rewrite feature usage`, - }, - }, -} - type log struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // Config contains the configuration to be used in the Ingress @@ -72,10 +48,7 @@ func (bd1 *Config) Equal(bd2 *Config) bool { // NewParser creates a new log annotations parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return log{ - r: r, - annotationConfig: logAnnotations, - } + return log{r} } // Parse parses the annotations contained in the ingress @@ -84,24 +57,15 @@ func (l log) Parse(ing *networking.Ingress) (interface{}, error) { var err error config := &Config{} - config.Access, err = parser.GetBoolAnnotation(enableAccessLogAnnotation, ing, l.annotationConfig.Annotations) + config.Access, err = parser.GetBoolAnnotation("enable-access-log", ing) if err != nil { config.Access = true } - config.Rewrite, err = parser.GetBoolAnnotation(enableRewriteLogAnnotation, ing, l.annotationConfig.Annotations) + config.Rewrite, err = parser.GetBoolAnnotation("enable-rewrite-log", ing) if err != nil { config.Rewrite = false } return config, nil } - -func (l log) GetDocumentation() parser.AnnotationFields { - return l.annotationConfig.Annotations -} - -func (l log) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(l.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, logAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/log/main_test.go b/internal/ingress/annotations/log/main_test.go index 950612b5b..068b1be16 100644 --- a/internal/ingress/annotations/log/main_test.go +++ b/internal/ingress/annotations/log/main_test.go @@ -20,20 +20,17 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -42,13 +39,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -73,13 +66,10 @@ func TestIngressAccessLogConfig(t *testing.T) { ing := buildIngress() data := map[string]string{} - data[parser.GetAnnotationWithPrefix(enableAccessLogAnnotation)] = "false" + data[parser.GetAnnotationWithPrefix("enable-access-log")] = "false" ing.SetAnnotations(data) - log, err := NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } + log, _ := NewParser(&resolver.Mock{}).Parse(ing) nginxLogs, ok := log.(*Config) if !ok { t.Errorf("expected a Config type") @@ -94,13 +84,10 @@ func TestIngressRewriteLogConfig(t *testing.T) { ing := buildIngress() data := map[string]string{} - data[parser.GetAnnotationWithPrefix(enableRewriteLogAnnotation)] = "true" + data[parser.GetAnnotationWithPrefix("enable-rewrite-log")] = "true" ing.SetAnnotations(data) - log, err := NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error parsing annotations %v", err) - } + log, _ := NewParser(&resolver.Mock{}).Parse(ing) nginxLogs, ok := log.(*Config) if !ok { t.Errorf("expected a Config type") @@ -110,24 +97,3 @@ func TestIngressRewriteLogConfig(t *testing.T) { t.Errorf("expected rewrite log to be enabled but it is disabled") } } - -func TestInvalidBoolConfig(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix(enableRewriteLogAnnotation)] = "blo" - ing.SetAnnotations(data) - - log, err := NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - nginxLogs, ok := log.(*Config) - if !ok { - t.Errorf("expected a Config type") - } - - if !nginxLogs.Access { - t.Errorf("expected access log to be enabled due to invalid config, but it is disabled") - } -} diff --git a/internal/ingress/annotations/mirror/main.go b/internal/ingress/annotations/mirror/main.go index d51bdd27c..b2591347e 100644 --- a/internal/ingress/annotations/mirror/main.go +++ b/internal/ingress/annotations/mirror/main.go @@ -18,54 +18,18 @@ package mirror import ( "fmt" - "regexp" - "strings" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" - "k8s.io/klog/v2" ) -const ( - mirrorRequestBodyAnnotation = "mirror-request-body" - mirrorTargetAnnotation = "mirror-target" - mirrorHostAnnotation = "mirror-host" -) - -var OnOffRegex = regexp.MustCompile(`^(on|off)$`) - -var mirrorAnnotation = parser.Annotation{ - Group: "mirror", - Annotations: parser.AnnotationFields{ - mirrorRequestBodyAnnotation: { - Validator: parser.ValidateRegex(OnOffRegex, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation defines if the request-body should be sent to the mirror backend. Can be 'on' or 'off'`, - }, - mirrorTargetAnnotation: { - Validator: parser.ValidateServerName, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskHigh, - Documentation: `This annotation enables a request to be mirrored to a mirror backend.`, - }, - mirrorHostAnnotation: { - Validator: parser.ValidateServerName, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskHigh, - Documentation: `This annotation defines if a specific Host header should be set for mirrored request.`, - }, - }, -} - // Config returns the mirror to use in a given location type Config struct { Source string `json:"source"` RequestBody string `json:"requestBody"` Target string `json:"target"` - Host string `json:"host"` } // Equal tests for equality between two Configuration types @@ -90,24 +54,16 @@ func (m1 *Config) Equal(m2 *Config) bool { return false } - if m1.Host != m2.Host { - return false - } - return true } type mirror struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new mirror configuration annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return mirror{ - r: r, - annotationConfig: mirrorAnnotation, - } + return mirror{r} } // ParseAnnotations parses the annotations contained in the ingress @@ -118,49 +74,16 @@ func (a mirror) Parse(ing *networking.Ingress) (interface{}, error) { } var err error - config.RequestBody, err = parser.GetStringAnnotation(mirrorRequestBodyAnnotation, ing, a.annotationConfig.Annotations) + config.RequestBody, err = parser.GetStringAnnotation("mirror-request-body", ing) if err != nil || config.RequestBody != "off" { - if errors.IsValidationError(err) { - klog.Warningf("annotation %s contains invalid value", mirrorRequestBodyAnnotation) - } config.RequestBody = "on" } - config.Target, err = parser.GetStringAnnotation(mirrorTargetAnnotation, ing, a.annotationConfig.Annotations) + config.Target, err = parser.GetStringAnnotation("mirror-target", ing) if err != nil { - if errors.IsValidationError(err) { - klog.Warningf("annotation %s contains invalid value, defaulting", mirrorTargetAnnotation) - } else { - config.Target = "" - config.Source = "" - } - } - - config.Host, err = parser.GetStringAnnotation(mirrorHostAnnotation, ing, a.annotationConfig.Annotations) - if err != nil { - if errors.IsValidationError(err) { - klog.Warningf("annotation %s contains invalid value, defaulting", mirrorHostAnnotation) - } - if config.Target != "" { - target := strings.Split(config.Target, "$") - - url, err := parser.StringToURL(target[0]) - if err != nil { - config.Host = "" - } else { - config.Host = url.Hostname() - } - } + config.Target = "" + config.Source = "" } return config, nil } - -func (a mirror) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a mirror) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, mirrorAnnotation.Annotations) -} diff --git a/internal/ingress/annotations/mirror/main_test.go b/internal/ingress/annotations/mirror/main_test.go index 805f1ef6d..1ecaef3b9 100644 --- a/internal/ingress/annotations/mirror/main_test.go +++ b/internal/ingress/annotations/mirror/main_test.go @@ -21,7 +21,7 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" @@ -30,7 +30,6 @@ import ( func TestParse(t *testing.T) { requestBody := parser.GetAnnotationWithPrefix("mirror-request-body") backendURL := parser.GetAnnotationWithPrefix("mirror-target") - host := parser.GetAnnotationWithPrefix("mirror-host") ap := NewParser(&resolver.Mock{}) if ap == nil { @@ -46,61 +45,11 @@ func TestParse(t *testing.T) { Source: ngxURI, RequestBody: "on", Target: "https://test.env.com/$request_uri", - Host: "test.env.com", - }}, - {map[string]string{backendURL: "https://test.env.com$request_uri"}, &Config{ - Source: ngxURI, - RequestBody: "on", - Target: "https://test.env.com$request_uri", - Host: "test.env.com", - }}, - {map[string]string{backendURL: "https://test.env.com:8080$request_uri"}, &Config{ - Source: ngxURI, - RequestBody: "on", - Target: "https://test.env.com:8080$request_uri", - Host: "test.env.com", - }}, - {map[string]string{backendURL: "https://test.env.com:8080/$request_uri"}, &Config{ - Source: ngxURI, - RequestBody: "on", - Target: "https://test.env.com:8080/$request_uri", - Host: "test.env.com", }}, {map[string]string{requestBody: "off"}, &Config{ Source: "", RequestBody: "off", Target: "", - Host: "", - }}, - {map[string]string{host: "test.env.com", backendURL: "http://some.test.env.com/$someparam"}, &Config{ - Source: ngxURI, - RequestBody: "on", - Target: "http://some.test.env.com/$someparam", - Host: "test.env.com", - }}, - {map[string]string{backendURL: "IamNotAURL"}, &Config{ - Source: ngxURI, - RequestBody: "on", - Target: "IamNotAURL", - Host: "", - }}, - {map[string]string{backendURL: "http://some.test.env.com:2121/$someparam=1&$someotherparam=2"}, &Config{ - Source: ngxURI, - RequestBody: "on", - Target: "http://some.test.env.com:2121/$someparam=1&$someotherparam=2", - Host: "some.test.env.com", - }}, - {map[string]string{backendURL: "http://some.test.env.com", host: "someInvalidParam.%^&*()_=!@#'\""}, &Config{ - Source: ngxURI, - RequestBody: "on", - Target: "http://some.test.env.com", - Host: "some.test.env.com", - }}, - {map[string]string{backendURL: "http://some.test.env.com", host: "_sbrubles-i\"@xpto:12345"}, &Config{ - Source: ngxURI, - RequestBody: "on", - Target: "http://some.test.env.com", - Host: "some.test.env.com", }}, } @@ -115,12 +64,9 @@ func TestParse(t *testing.T) { for _, testCase := range testCases { ing.SetAnnotations(testCase.annotations) - result, err := ap.Parse(ing) - if err != nil { - t.Errorf(err.Error()) - } + result, _ := ap.Parse(ing) if !reflect.DeepEqual(result, testCase.expected) { - t.Errorf("expected %+v but returned %+v, annotations: %s", testCase.expected, result, testCase.annotations) + t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations) } } } diff --git a/internal/ingress/annotations/modsecurity/main.go b/internal/ingress/annotations/modsecurity/main.go index 80ae78937..91a0a5a94 100644 --- a/internal/ingress/annotations/modsecurity/main.go +++ b/internal/ingress/annotations/modsecurity/main.go @@ -17,50 +17,11 @@ limitations under the License. package modsecurity import ( - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" - "k8s.io/klog/v2" ) -const ( - modsecEnableAnnotation = "enable-modsecurity" - modsecEnableOwaspCoreAnnotation = "enable-owasp-core-rules" - modesecTransactionIDAnnotation = "modsecurity-transaction-id" - modsecSnippetAnnotation = "modsecurity-snippet" -) - -var modsecurityAnnotation = parser.Annotation{ - Group: "modsecurity", - Annotations: parser.AnnotationFields{ - modsecEnableAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation enables ModSecurity`, - }, - modsecEnableOwaspCoreAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation enables the OWASP Core Rule Set`, - }, - modesecTransactionIDAnnotation: { - Validator: parser.ValidateRegex(parser.NGINXVariable, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskHigh, - Documentation: `This annotation enables passing an NGINX variable to ModSecurity.`, - }, - modsecSnippetAnnotation: { - Validator: parser.ValidateNull, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskCritical, - Documentation: `This annotation enables adding a specific snippet configuration for ModSecurity`, - }, - }, -} - // Config contains ModSecurity Configuration items type Config struct { Enable bool `json:"enable-modsecurity"` @@ -98,16 +59,12 @@ func (modsec1 *Config) Equal(modsec2 *Config) bool { } // NewParser creates a new ModSecurity annotation parser -func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return modSecurity{ - r: r, - annotationConfig: modsecurityAnnotation, - } +func NewParser(resolver resolver.Resolver) parser.IngressAnnotation { + return modSecurity{resolver} } type modSecurity struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // Parse parses the annotations contained in the ingress @@ -117,44 +74,26 @@ func (a modSecurity) Parse(ing *networking.Ingress) (interface{}, error) { config := &Config{} config.EnableSet = true - config.Enable, err = parser.GetBoolAnnotation(modsecEnableAnnotation, ing, a.annotationConfig.Annotations) + config.Enable, err = parser.GetBoolAnnotation("enable-modsecurity", ing) if err != nil { - if errors.IsInvalidContent(err) { - klog.Warningf("annotation %s contains invalid directive, defaulting to false", modsecEnableAnnotation) - } config.Enable = false config.EnableSet = false } - config.OWASPRules, err = parser.GetBoolAnnotation(modsecEnableOwaspCoreAnnotation, ing, a.annotationConfig.Annotations) + config.OWASPRules, err = parser.GetBoolAnnotation("enable-owasp-core-rules", ing) if err != nil { - if errors.IsInvalidContent(err) { - klog.Warningf("annotation %s contains invalid directive, defaulting to false", modsecEnableOwaspCoreAnnotation) - } config.OWASPRules = false } - config.TransactionID, err = parser.GetStringAnnotation(modesecTransactionIDAnnotation, ing, a.annotationConfig.Annotations) + config.TransactionID, err = parser.GetStringAnnotation("modsecurity-transaction-id", ing) if err != nil { - if errors.IsInvalidContent(err) { - klog.Warningf("annotation %s contains invalid directive, defaulting", modesecTransactionIDAnnotation) - } config.TransactionID = "" } - config.Snippet, err = parser.GetStringAnnotation("modsecurity-snippet", ing, a.annotationConfig.Annotations) + config.Snippet, err = parser.GetStringAnnotation("modsecurity-snippet", ing) if err != nil { config.Snippet = "" } return config, nil } - -func (a modSecurity) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a modSecurity) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, modsecurityAnnotation.Annotations) -} diff --git a/internal/ingress/annotations/modsecurity/main_test.go b/internal/ingress/annotations/modsecurity/main_test.go index 5cb05f201..34d92533d 100644 --- a/internal/ingress/annotations/modsecurity/main_test.go +++ b/internal/ingress/annotations/modsecurity/main_test.go @@ -20,7 +20,7 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" @@ -67,17 +67,11 @@ func TestParse(t *testing.T) { Spec: networking.IngressSpec{}, } - for i, testCase := range testCases { + for _, testCase := range testCases { ing.SetAnnotations(testCase.annotations) - result, err := ap.Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - config, ok := result.(*Config) - if !ok { - t.Errorf("unexpected type: %T", result) - } - if !config.Equal(&testCases[i].expected) { + result, _ := ap.Parse(ing) + config := result.(*Config) + if !config.Equal(&testCase.expected) { t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations) } } diff --git a/internal/ingress/annotations/opentelemetry/main.go b/internal/ingress/annotations/opentelemetry/main.go deleted file mode 100644 index ca9108548..000000000 --- a/internal/ingress/annotations/opentelemetry/main.go +++ /dev/null @@ -1,155 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package opentelemetry - -import ( - "regexp" - - networking "k8s.io/api/networking/v1" - - "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/errors" - "k8s.io/ingress-nginx/internal/ingress/resolver" -) - -const ( - enableOpenTelemetryAnnotation = "enable-opentelemetry" - otelTrustSpanAnnotation = "opentelemetry-trust-incoming-span" - otelOperationNameAnnotation = "opentelemetry-operation-name" -) - -var regexOperationName = regexp.MustCompile(`^[A-Za-z0-9_\-]*$`) - -var otelAnnotations = parser.Annotation{ - Group: "opentelemetry", - Annotations: parser.AnnotationFields{ - enableOpenTelemetryAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation defines if Open Telemetry collector should be enable for this location. OpenTelemetry should - already be configured by Ingress administrator`, - }, - otelTrustSpanAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation enables or disables using spans from incoming requests as parent for created ones`, - }, - otelOperationNameAnnotation: { - Validator: parser.ValidateRegex(regexOperationName, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation defines what operation name should be added to the span`, - }, - }, -} - -type opentelemetry struct { - r resolver.Resolver - annotationConfig parser.Annotation -} - -// Config contains the configuration to be used in the Ingress -type Config struct { - Enabled bool `json:"enabled"` - Set bool `json:"set"` - TrustEnabled bool `json:"trust-enabled"` - TrustSet bool `json:"trust-set"` - OperationName string `json:"operation-name"` -} - -// Equal tests for equality between two Config types -func (bd1 *Config) Equal(bd2 *Config) bool { - if bd1.Set != bd2.Set { - return false - } - - if bd1.Enabled != bd2.Enabled { - return false - } - - if bd1.TrustSet != bd2.TrustSet { - return false - } - - if bd1.TrustEnabled != bd2.TrustEnabled { - return false - } - - if bd1.OperationName != bd2.OperationName { - return false - } - - return true -} - -// NewParser creates a new serviceUpstream annotation parser -func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return opentelemetry{ - r: r, - annotationConfig: otelAnnotations, - } -} - -// Parse parses the annotations to look for opentelemetry configurations -func (c opentelemetry) Parse(ing *networking.Ingress) (interface{}, error) { - cfg := Config{} - enabled, err := parser.GetBoolAnnotation(enableOpenTelemetryAnnotation, ing, c.annotationConfig.Annotations) - if err != nil { - return &cfg, nil - } - cfg.Set = true - cfg.Enabled = enabled - if !enabled { - return &cfg, nil - } - - trustEnabled, err := parser.GetBoolAnnotation(otelTrustSpanAnnotation, ing, c.annotationConfig.Annotations) - if err != nil { - operationName, err := parser.GetStringAnnotation(otelOperationNameAnnotation, ing, c.annotationConfig.Annotations) - if err != nil { - if errors.IsValidationError(err) { - return nil, err - } - return &cfg, nil - } - cfg.OperationName = operationName - return &cfg, nil - } - - cfg.TrustSet = true - cfg.TrustEnabled = trustEnabled - operationName, err := parser.GetStringAnnotation(otelOperationNameAnnotation, ing, c.annotationConfig.Annotations) - if err != nil { - if errors.IsValidationError(err) { - return nil, err - } - return &cfg, nil - } - cfg.OperationName = operationName - return &cfg, nil -} - -func (c opentelemetry) GetDocumentation() parser.AnnotationFields { - return c.annotationConfig.Annotations -} - -func (c opentelemetry) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(c.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, otelAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/opentelemetry/main_test.go b/internal/ingress/annotations/opentelemetry/main_test.go deleted file mode 100644 index e5c1de0b3..000000000 --- a/internal/ingress/annotations/opentelemetry/main_test.go +++ /dev/null @@ -1,199 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package opentelemetry - -import ( - "testing" - - api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" - meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/resolver" -) - -const enableAnnotation = "true" - -func buildIngress() *networking.Ingress { - defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, - } - - return &networking.Ingress{ - ObjectMeta: meta_v1.ObjectMeta{ - Name: "foo", - Namespace: api.NamespaceDefault, - }, - Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, - }, - Rules: []networking.IngressRule{ - { - Host: "foo.bar.com", - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - Path: "/foo", - Backend: defaultBackend, - }, - }, - }, - }, - }, - }, - }, - } -} - -func TestIngressAnnotationOpentelemetrySetTrue(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix(enableOpenTelemetryAnnotation)] = enableAnnotation - ing.SetAnnotations(data) - - val, err := NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - openTelemetry, ok := val.(*Config) - if !ok { - t.Errorf("expected a Config type") - } - - if !openTelemetry.Enabled { - t.Errorf("expected annotation value to be true, got false") - } - - if !openTelemetry.Set { - t.Errorf("expected annotation value to be true, got false") - } - - if openTelemetry.TrustSet { - t.Errorf("expected annotation value to be false, got true") - } -} - -func TestIngressAnnotationOpentelemetrySetFalse(t *testing.T) { - ing := buildIngress() - - // Test with explicitly set to false - data := map[string]string{} - data[parser.GetAnnotationWithPrefix(enableOpenTelemetryAnnotation)] = "false" - ing.SetAnnotations(data) - - val, err := NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - openTelemetry, ok := val.(*Config) - if !ok { - t.Errorf("expected a Config type") - } - - if openTelemetry.Enabled { - t.Errorf("expected annotation value to be false, got true") - } - - if !openTelemetry.Set { - t.Errorf("expected annotation value to be true, got false") - } -} - -func TestIngressAnnotationOpentelemetryTrustSetTrue(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - opName := "foo-op" - data[parser.GetAnnotationWithPrefix(enableOpenTelemetryAnnotation)] = enableAnnotation - data[parser.GetAnnotationWithPrefix(otelTrustSpanAnnotation)] = enableAnnotation - data[parser.GetAnnotationWithPrefix(otelOperationNameAnnotation)] = opName - ing.SetAnnotations(data) - - val, err := NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Fatal(err) - } - openTelemetry, ok := val.(*Config) - if !ok { - t.Errorf("expected a Config type") - } - - if !openTelemetry.Enabled { - t.Errorf("expected annotation value to be true, got false") - } - - if !openTelemetry.Set { - t.Errorf("expected annotation value to be true, got false") - } - - if !openTelemetry.TrustEnabled { - t.Errorf("expected annotation value to be true, got false") - } - - if !openTelemetry.TrustSet { - t.Errorf("expected annotation value to be true, got false") - } - - if openTelemetry.OperationName != opName { - t.Errorf("expected annotation value to be %v, got %v", opName, openTelemetry.OperationName) - } -} - -func TestIngressAnnotationOpentelemetryWithBadOpName(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - opName := "fooxpto_123$la;" - data[parser.GetAnnotationWithPrefix(enableOpenTelemetryAnnotation)] = enableAnnotation - data[parser.GetAnnotationWithPrefix(otelOperationNameAnnotation)] = opName - ing.SetAnnotations(data) - - _, err := NewParser(&resolver.Mock{}).Parse(ing) - if err == nil { - t.Fatalf("This operation should return an error but no error was returned") - } -} - -func TestIngressAnnotationOpentelemetryUnset(t *testing.T) { - ing := buildIngress() - - // Test with no annotation specified - data := map[string]string{} - ing.SetAnnotations(data) - - val, err := NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - _, ok := val.(*Config) - if !ok { - t.Errorf("expected a Config type") - } -} diff --git a/internal/ingress/annotations/opentracing/main.go b/internal/ingress/annotations/opentracing/main.go new file mode 100644 index 000000000..875d695f7 --- /dev/null +++ b/internal/ingress/annotations/opentracing/main.go @@ -0,0 +1,61 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package opentracing + +import ( + networking "k8s.io/api/networking/v1beta1" + + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "k8s.io/ingress-nginx/internal/ingress/resolver" +) + +type opentracing struct { + r resolver.Resolver +} + +// Config contains the configuration to be used in the Ingress +type Config struct { + Enabled bool `json:"enabled"` + Set bool `json:"set"` +} + +// Equal tests for equality between two Config types +func (bd1 *Config) Equal(bd2 *Config) bool { + if bd1.Set != bd2.Set { + return false + } + + if bd1.Enabled != bd2.Enabled { + return false + } + + return true +} + +// NewParser creates a new serviceUpstream annotation parser +func NewParser(r resolver.Resolver) parser.IngressAnnotation { + return opentracing{r} +} + +func (s opentracing) Parse(ing *networking.Ingress) (interface{}, error) { + enabled, err := parser.GetBoolAnnotation("enable-opentracing", ing) + if err != nil { + return &Config{Set: false, Enabled: false}, nil + } + + return &Config{Set: true, Enabled: enabled}, nil +} diff --git a/internal/ingress/annotations/opentracing/main_test.go b/internal/ingress/annotations/opentracing/main_test.go new file mode 100644 index 000000000..f1e06b087 --- /dev/null +++ b/internal/ingress/annotations/opentracing/main_test.go @@ -0,0 +1,114 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package opentracing + +import ( + "testing" + + api "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1beta1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "k8s.io/ingress-nginx/internal/ingress/resolver" +) + +func buildIngress() *networking.Ingress { + defaultBackend := networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), + } + + return &networking.Ingress{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "foo", + Namespace: api.NamespaceDefault, + }, + Spec: networking.IngressSpec{ + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), + }, + Rules: []networking.IngressRule{ + { + Host: "foo.bar.com", + IngressRuleValue: networking.IngressRuleValue{ + HTTP: &networking.HTTPIngressRuleValue{ + Paths: []networking.HTTPIngressPath{ + { + Path: "/foo", + Backend: defaultBackend, + }, + }, + }, + }, + }, + }, + }, + } +} + +func TestIngressAnnotationOpentracingSetTrue(t *testing.T) { + ing := buildIngress() + + data := map[string]string{} + data[parser.GetAnnotationWithPrefix("enable-opentracing")] = "true" + ing.SetAnnotations(data) + + val, _ := NewParser(&resolver.Mock{}).Parse(ing) + openTracing, ok := val.(*Config) + if !ok { + t.Errorf("expected a Config type") + } + + if !openTracing.Enabled { + t.Errorf("expected annotation value to be true, got false") + } +} + +func TestIngressAnnotationOpentracingSetFalse(t *testing.T) { + ing := buildIngress() + + // Test with explicitly set to false + data := map[string]string{} + data[parser.GetAnnotationWithPrefix("enable-opentracing")] = "false" + ing.SetAnnotations(data) + + val, _ := NewParser(&resolver.Mock{}).Parse(ing) + openTracing, ok := val.(*Config) + if !ok { + t.Errorf("expected a Config type") + } + + if openTracing.Enabled { + t.Errorf("expected annotation value to be false, got true") + } +} + +func TestIngressAnnotationOpentracingUnset(t *testing.T) { + ing := buildIngress() + + // Test with no annotation specified + data := map[string]string{} + ing.SetAnnotations(data) + + val, _ := NewParser(&resolver.Mock{}).Parse(ing) + _, ok := val.(*Config) + if !ok { + t.Errorf("expected a Config type") + } +} diff --git a/internal/ingress/annotations/parser/main.go b/internal/ingress/annotations/parser/main.go index 3137afbfd..3fae804da 100644 --- a/internal/ingress/annotations/parser/main.go +++ b/internal/ingress/annotations/parser/main.go @@ -22,86 +22,27 @@ import ( "strconv" "strings" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/ingress-nginx/internal/ingress/errors" ) // DefaultAnnotationsPrefix defines the common prefix used in the nginx ingress controller -const ( - DefaultAnnotationsPrefix = "nginx.ingress.kubernetes.io" - DefaultEnableAnnotationValidation = true -) +const DefaultAnnotationsPrefix = "nginx.ingress.kubernetes.io" var ( // AnnotationsPrefix is the mutable attribute that the controller explicitly refers to AnnotationsPrefix = DefaultAnnotationsPrefix - // Enable is the mutable attribute for enabling or disabling the validation functions - EnableAnnotationValidation = DefaultEnableAnnotationValidation ) -// AnnotationGroup defines the group that this annotation may belong -// eg.: Security, Snippets, Rewrite, etc -type AnnotationGroup string - -// AnnotationScope defines which scope this annotation applies. May be to the whole -// ingress, per location, etc -type AnnotationScope string - -var ( - AnnotationScopeLocation AnnotationScope = "location" - AnnotationScopeIngress AnnotationScope = "ingress" -) - -// AnnotationRisk is a subset of risk that an annotation may represent. -// Based on the Risk, the admin will be able to allow or disallow users to set it -// on their ingress objects -type AnnotationRisk int - -type AnnotationFields map[string]AnnotationConfig - -// AnnotationConfig defines the configuration that a single annotation field -// has, with the Validator and the documentation of this field. -type AnnotationConfig struct { - // Validator defines a function to validate the annotation value - Validator AnnotationValidator - // Documentation defines a user facing documentation for this annotation. This - // field will be used to auto generate documentations - Documentation string - // Risk defines a risk of this annotation being exposed to the user. Annotations - // with bool fields, or to set timeout are usually low risk. Annotations that allows - // string input without a limited set of options may represent a high risk - Risk AnnotationRisk - - // Scope defines which scope this annotation applies, may be to location, to an Ingress object, etc - Scope AnnotationScope - - // AnnotationAliases defines other names this annotation may have. - AnnotationAliases []string -} - -// Annotation defines an annotation feature an Ingress may have. -// It should contain the internal resolver, and all the annotations -// with configs and Validators that should be used for each Annotation -type Annotation struct { - // Annotations contains all the annotations that belong to this feature - Annotations AnnotationFields - // Group defines which annotation group this feature belongs to - Group AnnotationGroup -} - // IngressAnnotation has a method to parse annotations located in Ingress type IngressAnnotation interface { Parse(ing *networking.Ingress) (interface{}, error) - GetDocumentation() AnnotationFields - Validate(anns map[string]string) error } type ingAnnotations map[string]string -// TODO: We already parse all of this on checkAnnotation and can just do a parse over the -// value func (a ingAnnotations) parseBool(name string) (bool, error) { val, ok := a[name] if ok { @@ -118,7 +59,7 @@ func (a ingAnnotations) parseString(name string) (string, error) { val, ok := a[name] if ok { s := normalizeString(val) - if s == "" { + if len(s) == 0 { return "", errors.NewInvalidAnnotationContent(name, val) } @@ -139,21 +80,21 @@ func (a ingAnnotations) parseInt(name string) (int, error) { return 0, errors.ErrMissingAnnotations } -func (a ingAnnotations) parseFloat32(name string) (float32, error) { - val, ok := a[name] - if ok { - i, err := strconv.ParseFloat(val, 32) - if err != nil { - return 0, errors.NewInvalidAnnotationContent(name, val) - } - return float32(i), nil +func checkAnnotation(name string, ing *networking.Ingress) error { + if ing == nil || len(ing.GetAnnotations()) == 0 { + return errors.ErrMissingAnnotations } - return 0, errors.ErrMissingAnnotations + if name == "" { + return errors.ErrInvalidAnnotationName + } + + return nil } // GetBoolAnnotation extracts a boolean from an Ingress annotation -func GetBoolAnnotation(name string, ing *networking.Ingress, fields AnnotationFields) (bool, error) { - v, err := checkAnnotation(name, ing, fields) +func GetBoolAnnotation(name string, ing *networking.Ingress) (bool, error) { + v := GetAnnotationWithPrefix(name) + err := checkAnnotation(v, ing) if err != nil { return false, err } @@ -161,8 +102,9 @@ func GetBoolAnnotation(name string, ing *networking.Ingress, fields AnnotationFi } // GetStringAnnotation extracts a string from an Ingress annotation -func GetStringAnnotation(name string, ing *networking.Ingress, fields AnnotationFields) (string, error) { - v, err := checkAnnotation(name, ing, fields) +func GetStringAnnotation(name string, ing *networking.Ingress) (string, error) { + v := GetAnnotationWithPrefix(name) + err := checkAnnotation(v, ing) if err != nil { return "", err } @@ -171,60 +113,20 @@ func GetStringAnnotation(name string, ing *networking.Ingress, fields Annotation } // GetIntAnnotation extracts an int from an Ingress annotation -func GetIntAnnotation(name string, ing *networking.Ingress, fields AnnotationFields) (int, error) { - v, err := checkAnnotation(name, ing, fields) +func GetIntAnnotation(name string, ing *networking.Ingress) (int, error) { + v := GetAnnotationWithPrefix(name) + err := checkAnnotation(v, ing) if err != nil { return 0, err } return ingAnnotations(ing.GetAnnotations()).parseInt(v) } -// GetFloatAnnotation extracts a float32 from an Ingress annotation -func GetFloatAnnotation(name string, ing *networking.Ingress, fields AnnotationFields) (float32, error) { - v, err := checkAnnotation(name, ing, fields) - if err != nil { - return 0, err - } - return ingAnnotations(ing.GetAnnotations()).parseFloat32(v) -} - // GetAnnotationWithPrefix returns the prefix of ingress annotations func GetAnnotationWithPrefix(suffix string) string { return fmt.Sprintf("%v/%v", AnnotationsPrefix, suffix) } -func TrimAnnotationPrefix(annotation string) string { - return strings.TrimPrefix(annotation, AnnotationsPrefix+"/") -} - -func StringRiskToRisk(risk string) AnnotationRisk { - switch strings.ToLower(risk) { - case "critical": - return AnnotationRiskCritical - case "high": - return AnnotationRiskHigh - case "medium": - return AnnotationRiskMedium - default: - return AnnotationRiskLow - } -} - -func (a AnnotationRisk) ToString() string { - switch a { - case AnnotationRiskCritical: - return "Critical" - case AnnotationRiskHigh: - return "High" - case AnnotationRiskMedium: - return "Medium" - case AnnotationRiskLow: - return "Low" - default: - return "Unknown" - } -} - func normalizeString(input string) string { trimmedContent := []string{} for _, line := range strings.Split(input, "\n") { @@ -263,14 +165,13 @@ func StringToURL(input string) (*url.URL, error) { return nil, fmt.Errorf("%v is not a valid URL: %v", input, err) } - switch { - case parsedURL.Scheme == "": + if parsedURL.Scheme == "" { return nil, fmt.Errorf("url scheme is empty") - case parsedURL.Host == "": + } else if parsedURL.Host == "" { return nil, fmt.Errorf("url host is empty") - case strings.Contains(parsedURL.Host, ".."): + } else if strings.Contains(parsedURL.Host, "..") { return nil, fmt.Errorf("invalid url host") - default: - return parsedURL, nil } + + return parsedURL, nil } diff --git a/internal/ingress/annotations/parser/main_test.go b/internal/ingress/annotations/parser/main_test.go index 0f7ecb459..218565183 100644 --- a/internal/ingress/annotations/parser/main_test.go +++ b/internal/ingress/annotations/parser/main_test.go @@ -21,7 +21,7 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -38,7 +38,7 @@ func buildIngress() *networking.Ingress { func TestGetBoolAnnotation(t *testing.T) { ing := buildIngress() - _, err := GetBoolAnnotation("", nil, nil) + _, err := GetBoolAnnotation("", nil) if err == nil { t.Errorf("expected error but retuned nil") } @@ -59,8 +59,8 @@ func TestGetBoolAnnotation(t *testing.T) { for _, test := range tests { data[GetAnnotationWithPrefix(test.field)] = test.value - ing.SetAnnotations(data) - u, err := GetBoolAnnotation(test.field, ing, nil) + + u, err := GetBoolAnnotation(test.field, ing) if test.expErr { if err == nil { t.Errorf("%v: expected error but retuned nil", test.name) @@ -68,7 +68,7 @@ func TestGetBoolAnnotation(t *testing.T) { continue } if u != test.exp { - t.Errorf("%v: expected \"%v\" but \"%v\" was returned, %+v", test.name, test.exp, u, ing) + t.Errorf("%v: expected \"%v\" but \"%v\" was returned", test.name, test.exp, u) } delete(data, test.field) @@ -78,7 +78,7 @@ func TestGetBoolAnnotation(t *testing.T) { func TestGetStringAnnotation(t *testing.T) { ing := buildIngress() - _, err := GetStringAnnotation("", nil, nil) + _, err := GetStringAnnotation("", nil) if err == nil { t.Errorf("expected error but none returned") } @@ -93,16 +93,14 @@ func TestGetStringAnnotation(t *testing.T) { {"valid - A", "string", "A ", "A", false}, {"valid - B", "string", " B", "B", false}, {"empty", "string", " ", "", true}, - { - "valid multiline", "string", ` + {"valid multiline", "string", ` rewrite (?i)/arcgis/rest/services/Utilities/Geometry/GeometryServer(.*)$ /arcgis/rest/services/Utilities/Geometry/GeometryServer$1 break; rewrite (?i)/arcgis/services/Utilities/Geometry/GeometryServer(.*)$ /arcgis/services/Utilities/Geometry/GeometryServer$1 break; `, ` rewrite (?i)/arcgis/rest/services/Utilities/Geometry/GeometryServer(.*)$ /arcgis/rest/services/Utilities/Geometry/GeometryServer$1 break; rewrite (?i)/arcgis/services/Utilities/Geometry/GeometryServer(.*)$ /arcgis/services/Utilities/Geometry/GeometryServer$1 break; `, - false, - }, + false}, } data := map[string]string{} @@ -111,60 +109,13 @@ rewrite (?i)/arcgis/services/Utilities/Geometry/GeometryServer(.*)$ /arcgis/serv for _, test := range tests { data[GetAnnotationWithPrefix(test.field)] = test.value - s, err := GetStringAnnotation(test.field, ing, nil) + s, err := GetStringAnnotation(test.field, ing) if test.expErr { if err == nil { t.Errorf("%v: expected error but none returned", test.name) } continue } - if !test.expErr { - if err != nil { - t.Errorf("%v: didn't expected error but error was returned: %v", test.name, err) - } - continue - } - if s != test.exp { - t.Errorf("%v: expected \"%v\" but \"%v\" was returned", test.name, test.exp, s) - } - - delete(data, test.field) - } -} - -func TestGetFloatAnnotation(t *testing.T) { - ing := buildIngress() - - _, err := GetFloatAnnotation("", nil, nil) - if err == nil { - t.Errorf("expected error but retuned nil") - } - - tests := []struct { - name string - field string - value string - exp float32 - expErr bool - }{ - {"valid - A", "string", "1.5", 1.5, false}, - {"valid - B", "string", "2", 2, false}, - {"valid - C", "string", "100.0", 100, false}, - } - - data := map[string]string{} - ing.SetAnnotations(data) - - for _, test := range tests { - data[GetAnnotationWithPrefix(test.field)] = test.value - - s, err := GetFloatAnnotation(test.field, ing, nil) - if test.expErr { - if err == nil { - t.Errorf("%v: expected error but retuned nil", test.name) - } - continue - } if s != test.exp { t.Errorf("%v: expected \"%v\" but \"%v\" was returned", test.name, test.exp, s) } @@ -176,7 +127,7 @@ func TestGetFloatAnnotation(t *testing.T) { func TestGetIntAnnotation(t *testing.T) { ing := buildIngress() - _, err := GetIntAnnotation("", nil, nil) + _, err := GetIntAnnotation("", nil) if err == nil { t.Errorf("expected error but retuned nil") } @@ -198,7 +149,7 @@ func TestGetIntAnnotation(t *testing.T) { for _, test := range tests { data[GetAnnotationWithPrefix(test.field)] = test.value - s, err := GetIntAnnotation(test.field, ing, nil) + s, err := GetIntAnnotation(test.field, ing) if test.expErr { if err == nil { t.Errorf("%v: expected error but retuned nil", test.name) @@ -215,10 +166,8 @@ func TestGetIntAnnotation(t *testing.T) { func TestStringToURL(t *testing.T) { validURL := "http://bar.foo.com/external-auth" - validParsedURL, err := url.Parse(validURL) - if err != nil { - t.Errorf("unexpected error: %v", err) - } + validParsedURL, _ := url.Parse(validURL) + tests := []struct { title string url string @@ -228,7 +177,6 @@ func TestStringToURL(t *testing.T) { }{ {"empty", "", "url scheme is empty", nil, true}, {"no scheme", "bar", "url scheme is empty", nil, true}, - {"invalid parse", "://lala.com", "://lala.com is not a valid URL: parse \"://lala.com\": missing protocol scheme", nil, true}, {"invalid host", "http://", "url host is empty", nil, true}, {"invalid host (multiple dots)", "http://foo..bar.com", "invalid url host", nil, true}, {"valid URL", validURL, "", validParsedURL, false}, diff --git a/internal/ingress/annotations/parser/validators.go b/internal/ingress/annotations/parser/validators.go deleted file mode 100644 index 3c724a311..000000000 --- a/internal/ingress/annotations/parser/validators.go +++ /dev/null @@ -1,261 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package parser - -import ( - "errors" - "fmt" - "regexp" - "strconv" - "strings" - "time" - - networking "k8s.io/api/networking/v1" - machineryvalidation "k8s.io/apimachinery/pkg/api/validation" - ing_errors "k8s.io/ingress-nginx/internal/ingress/errors" - "k8s.io/ingress-nginx/internal/net" - "k8s.io/klog/v2" -) - -type AnnotationValidator func(string) error - -const ( - AnnotationRiskLow AnnotationRisk = iota - AnnotationRiskMedium - AnnotationRiskHigh - AnnotationRiskCritical -) - -var ( - alphaNumericChars = `\-\.\_\~a-zA-Z0-9\/:` - extendedAlphaNumeric = alphaNumericChars + ", " - regexEnabledChars = regexp.QuoteMeta(`^$[](){}*+?|&=\`) - urlEnabledChars = regexp.QuoteMeta(`,:?&=`) -) - -// IsValidRegex checks if the tested string can be used as a regex, but without any weird character. -// It includes regex characters for paths that may contain regexes -// -//nolint:goconst //already a constant -var IsValidRegex = regexp.MustCompile("^[/" + alphaNumericChars + regexEnabledChars + "]*$") - -// SizeRegex validates sizes understood by NGINX, like 1000, 100k, 1000M -var SizeRegex = regexp.MustCompile(`^(?i)\d+[bkmg]?$`) - -// URLRegex is used to validate a URL but with only a specific set of characters: -// It is alphanumericChar + ":", "?", "&" -// A valid URL would be proto://something.com:port/something?arg=param -var ( - // URLIsValidRegex is used on full URLs, containing query strings (:, ? and &) - URLIsValidRegex = regexp.MustCompile("^[" + alphaNumericChars + urlEnabledChars + "]*$") - // BasicChars is alphanumeric and ".", "-", "_", "~" and ":", usually used on simple host:port/path composition. - // This combination can also be used on fields that may contain characters like / (as ns/name) - BasicCharsRegex = regexp.MustCompile("^[/" + alphaNumericChars + "]*$") - // ExtendedChars is alphanumeric and ".", "-", "_", "~" and ":" plus "," and spaces, usually used on simple host:port/path composition - ExtendedCharsRegex = regexp.MustCompile("^[/" + extendedAlphaNumeric + "]*$") - // CharsWithSpace is like basic chars, but includes the space character - CharsWithSpace = regexp.MustCompile("^[/" + alphaNumericChars + " ]*$") - // NGINXVariable allows entries with alphanumeric characters, -, _ and the special "$" - NGINXVariable = regexp.MustCompile(`^[A-Za-z0-9\-\_\$\{\}]*$`) - // RegexPathWithCapture allows entries that SHOULD start with "/" and may contain alphanumeric + capture - // character for regex based paths, like /something/$1/anything/$2 - RegexPathWithCapture = regexp.MustCompile(`^/?[` + alphaNumericChars + `\/\$]*$`) - // HeadersVariable defines a regex that allows headers separated by comma - HeadersVariable = regexp.MustCompile(`^[A-Za-z0-9-_, ]*$`) - // URLWithNginxVariableRegex defines a url that can contain nginx variables. - // It is a risky operation - URLWithNginxVariableRegex = regexp.MustCompile("^[" + extendedAlphaNumeric + urlEnabledChars + "$]*$") - // MaliciousRegex defines chars that are known to inject RCE - MaliciousRegex = regexp.MustCompile(`\r|\n`) -) - -// ValidateArrayOfServerName validates if all fields on a Server name annotation are -// regexes. They can be *.something*, ~^www\d+\.example\.com$ but not fancy character -func ValidateArrayOfServerName(value string) error { - for _, fqdn := range strings.Split(value, ",") { - if err := ValidateServerName(fqdn); err != nil { - return err - } - } - return nil -} - -// ValidateServerName validates if the passed value is an acceptable server name. The server name -// can contain regex characters, as those are accepted values on nginx configuration -func ValidateServerName(value string) error { - value = strings.TrimSpace(value) - if !IsValidRegex.MatchString(value) { - return fmt.Errorf("value %s is invalid server name", value) - } - return nil -} - -// ValidateRegex receives a regex as an argument and uses it to validate -// the value of the field. -// Annotation can define if the spaces should be trimmed before validating the value -func ValidateRegex(regex *regexp.Regexp, removeSpace bool) AnnotationValidator { - return func(s string) error { - if removeSpace { - s = strings.ReplaceAll(s, " ", "") - } - if !regex.MatchString(s) { - return fmt.Errorf("value %s is invalid", s) - } - if MaliciousRegex.MatchString(s) { - return fmt.Errorf("value %s contains malicious string", s) - } - - return nil - } -} - -// CommonNameAnnotationValidator checks whether the annotation value starts with -// 'CN=' and is followed by a valid regex. -func CommonNameAnnotationValidator(s string) error { - if !strings.HasPrefix(s, "CN=") { - return fmt.Errorf("value %s is not a valid Common Name annotation: missing prefix 'CN='", s) - } - - if _, err := regexp.Compile(s[3:]); err != nil { - return fmt.Errorf("value %s is not a valid regex: %w", s, err) - } - - return nil -} - -// ValidateOptions receives an array of valid options that can be the value of annotation. -// If no valid option is found, it will return an error -func ValidateOptions(options []string, caseSensitive, trimSpace bool) AnnotationValidator { - return func(s string) error { - if trimSpace { - s = strings.TrimSpace(s) - } - if !caseSensitive { - s = strings.ToLower(s) - } - for _, option := range options { - if s == option { - return nil - } - } - return fmt.Errorf("value does not match any valid option") - } -} - -// ValidateBool validates if the specified value is a bool -func ValidateBool(value string) error { - _, err := strconv.ParseBool(value) - return err -} - -// ValidateInt validates if the specified value is an integer -func ValidateInt(value string) error { - _, err := strconv.Atoi(value) - return err -} - -// ValidateCIDRs validates if the specified value is an array of IPs and CIDRs -func ValidateCIDRs(value string) error { - _, err := net.ParseCIDRs(value) - return err -} - -// ValidateDuration validates if the specified value is a valid time -func ValidateDuration(value string) error { - _, err := time.ParseDuration(value) - return err -} - -// ValidateNull always return null values and should not be widely used. -// It is used on the "snippet" annotations, as it is up to the admin to allow its -// usage, knowing it can be critical! -func ValidateNull(_ string) error { - return nil -} - -// ValidateServiceName validates if a provided service name is a valid string -func ValidateServiceName(value string) error { - errs := machineryvalidation.NameIsDNS1035Label(value, false) - if len(errs) != 0 { - return fmt.Errorf("annotation does not contain a valid service name: %+v", errs) - } - return nil -} - -// checkAnnotation will check each annotation for: -// 1 - Does it contain the internal validation and docs config? -// 2 - Does the ingress contains annotations? (validate null pointers) -// 3 - Does it contains a validator? Should it contain a validator (not containing is a bug!) -// 4 - Does the annotation contain aliases? So we should use if the alias is defined an the annotation not. -// 4 - Runs the validator on the value -// It will return the full annotation name if all is fine -func checkAnnotation(name string, ing *networking.Ingress, fields AnnotationFields) (string, error) { - var validateFunc AnnotationValidator - if fields != nil { - config, ok := fields[name] - if !ok { - return "", fmt.Errorf("annotation does not contain a valid internal configuration, this is an Ingress Controller issue! Please raise an issue on github.com/kubernetes/ingress-nginx") - } - validateFunc = config.Validator - } - - if ing == nil || len(ing.GetAnnotations()) == 0 { - return "", ing_errors.ErrMissingAnnotations - } - - annotationFullName := GetAnnotationWithPrefix(name) - if annotationFullName == "" { - return "", ing_errors.ErrInvalidAnnotationName - } - - annotationValue := ing.GetAnnotations()[annotationFullName] - if fields != nil { - if validateFunc == nil { - return "", fmt.Errorf("annotation does not contain a validator. This is an ingress-controller bug. Please open an issue") - } - if annotationValue == "" { - for _, annotationAlias := range fields[name].AnnotationAliases { - tempAnnotationFullName := GetAnnotationWithPrefix(annotationAlias) - if aliasVal := ing.GetAnnotations()[tempAnnotationFullName]; aliasVal != "" { - annotationValue = aliasVal - annotationFullName = tempAnnotationFullName - break - } - } - } - // We don't run validation against empty values - if EnableAnnotationValidation && annotationValue != "" { - if err := validateFunc(annotationValue); err != nil { - klog.Warningf("validation error on ingress %s/%s: annotation %s contains invalid value %s", ing.GetNamespace(), ing.GetName(), name, annotationValue) - return "", ing_errors.NewValidationError(annotationFullName) - } - } - } - - return annotationFullName, nil -} - -func CheckAnnotationRisk(annotations map[string]string, maxrisk AnnotationRisk, config AnnotationFields) error { - var err error - for annotation := range annotations { - annPure := TrimAnnotationPrefix(annotation) - if cfg, ok := config[annPure]; ok && cfg.Risk > maxrisk { - err = errors.Join(err, fmt.Errorf("annotation %s is too risky for environment", annotation)) - } - } - return err -} diff --git a/internal/ingress/annotations/parser/validators_test.go b/internal/ingress/annotations/parser/validators_test.go deleted file mode 100644 index 49923ba76..000000000 --- a/internal/ingress/annotations/parser/validators_test.go +++ /dev/null @@ -1,375 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package parser - -import ( - "fmt" - "testing" - - networking "k8s.io/api/networking/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestValidateArrayOfServerName(t *testing.T) { - tests := []struct { - name string - value string - wantErr bool - }{ - { - name: "should accept common name", - value: "something.com,anything.com", - wantErr: false, - }, - { - name: "should accept wildcard name", - value: "*.something.com,otherthing.com", - wantErr: false, - }, - { - name: "should allow names with spaces between array and some regexes", - value: `~^www\d+\.example\.com$,something.com`, - wantErr: false, - }, - { - name: "should allow names with regexes", - value: `http://some.test.env.com:2121/$someparam=1&$someotherparam=2`, - wantErr: false, - }, - { - name: "should allow names with wildcard in middle common name", - value: "*.so*mething.com,bla.com", - wantErr: false, - }, - { - name: "should allow comma separated query params", - value: "https://oauth.example/oauth2/auth?allowed_groups=gid1,gid2", - wantErr: false, - }, - { - name: "should deny names with weird characters", - value: "something.com,lolo;xpto.com,nothing.com", - wantErr: true, - }, - { - name: "should deny names with malicous chars", - value: "http://something.com/#;\nournewinjection", - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := ValidateArrayOfServerName(tt.value); (err != nil) != tt.wantErr { - t.Errorf("ValidateArrayOfServerName() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func Test_checkAnnotation(t *testing.T) { - type args struct { - name string - ing *networking.Ingress - fields AnnotationFields - } - tests := []struct { - name string - args args - want string - wantErr bool - }{ - { - name: "null ingress should error", - want: "", - args: args{ - name: "some-random-annotation", - }, - wantErr: true, - }, - { - name: "not having a validator for a specific annotation is a bug", - want: "", - args: args{ - name: "some-new-invalid-annotation", - ing: &networking.Ingress{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - GetAnnotationWithPrefix("some-new-invalid-annotation"): "xpto", - }, - }, - }, - fields: AnnotationFields{ - "otherannotation": AnnotationConfig{ - Validator: func(_ string) error { return nil }, - }, - }, - }, - wantErr: true, - }, - { - name: "annotationconfig found and no validation func defined on annotation is a bug", - want: "", - args: args{ - name: "some-new-invalid-annotation", - ing: &networking.Ingress{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - GetAnnotationWithPrefix("some-new-invalid-annotation"): "xpto", - }, - }, - }, - fields: AnnotationFields{ - "some-new-invalid-annotation": AnnotationConfig{}, - }, - }, - wantErr: true, - }, - { - name: "no annotation can turn into a null pointer and should fail", - want: "", - args: args{ - name: "some-new-invalid-annotation", - ing: &networking.Ingress{ - ObjectMeta: v1.ObjectMeta{}, - }, - fields: AnnotationFields{ - "some-new-invalid-annotation": AnnotationConfig{}, - }, - }, - wantErr: true, - }, - { - name: "no AnnotationField config should bypass validations", - want: GetAnnotationWithPrefix("some-valid-annotation"), - args: args{ - name: "some-valid-annotation", - ing: &networking.Ingress{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - GetAnnotationWithPrefix("some-valid-annotation"): "xpto", - }, - }, - }, - }, - wantErr: false, - }, - { - name: "annotation with invalid value should fail", - want: "", - args: args{ - name: "some-new-annotation", - ing: &networking.Ingress{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - GetAnnotationWithPrefix("some-new-annotation"): "xpto1", - }, - }, - }, - fields: AnnotationFields{ - "some-new-annotation": AnnotationConfig{ - Validator: func(value string) error { - if value != "xpto" { - return fmt.Errorf("this is an error") - } - return nil - }, - }, - }, - }, - wantErr: true, - }, - { - name: "annotation with valid value should pass", - want: GetAnnotationWithPrefix("some-other-annotation"), - args: args{ - name: "some-other-annotation", - ing: &networking.Ingress{ - ObjectMeta: v1.ObjectMeta{ - Annotations: map[string]string{ - GetAnnotationWithPrefix("some-other-annotation"): "xpto", - }, - }, - }, - fields: AnnotationFields{ - "some-other-annotation": AnnotationConfig{ - Validator: func(value string) error { - if value != "xpto" { - return fmt.Errorf("this is an error") - } - return nil - }, - }, - }, - }, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := checkAnnotation(tt.args.name, tt.args.ing, tt.args.fields) - if (err != nil) != tt.wantErr { - t.Errorf("checkAnnotation() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("checkAnnotation() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestCheckAnnotationRisk(t *testing.T) { - tests := []struct { - name string - annotations map[string]string - maxrisk AnnotationRisk - config AnnotationFields - wantErr bool - }{ - { - name: "high risk should not be accepted with maximum medium", - maxrisk: AnnotationRiskMedium, - annotations: map[string]string{ - "nginx.ingress.kubernetes.io/bla": "blo", - "nginx.ingress.kubernetes.io/bli": "bl3", - }, - config: AnnotationFields{ - "bla": { - Risk: AnnotationRiskHigh, - }, - "bli": { - Risk: AnnotationRiskMedium, - }, - }, - wantErr: true, - }, - { - name: "high risk should be accepted with maximum critical", - maxrisk: AnnotationRiskCritical, - annotations: map[string]string{ - "nginx.ingress.kubernetes.io/bla": "blo", - "nginx.ingress.kubernetes.io/bli": "bl3", - }, - config: AnnotationFields{ - "bla": { - Risk: AnnotationRiskHigh, - }, - "bli": { - Risk: AnnotationRiskMedium, - }, - }, - wantErr: false, - }, - { - name: "low risk should be accepted with maximum low", - maxrisk: AnnotationRiskLow, - annotations: map[string]string{ - "nginx.ingress.kubernetes.io/bla": "blo", - "nginx.ingress.kubernetes.io/bli": "bl3", - }, - config: AnnotationFields{ - "bla": { - Risk: AnnotationRiskLow, - }, - "bli": { - Risk: AnnotationRiskLow, - }, - }, - wantErr: false, - }, - { - name: "critical risk should be accepted with maximum critical", - maxrisk: AnnotationRiskCritical, - annotations: map[string]string{ - "nginx.ingress.kubernetes.io/bla": "blo", - "nginx.ingress.kubernetes.io/bli": "bl3", - }, - config: AnnotationFields{ - "bla": { - Risk: AnnotationRiskCritical, - }, - "bli": { - Risk: AnnotationRiskCritical, - }, - }, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := CheckAnnotationRisk(tt.annotations, tt.maxrisk, tt.config); (err != nil) != tt.wantErr { - t.Errorf("CheckAnnotationRisk() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestCommonNameAnnotationValidator(t *testing.T) { - tests := []struct { - name string - annotation string - wantErr bool - }{ - { - name: "correct example", - annotation: `CN=(my\.common\.name)`, - wantErr: false, - }, - { - name: "no CN= prefix", - annotation: `(my\.common\.name)`, - wantErr: true, - }, - { - name: "invalid prefix", - annotation: `CN(my\.common\.name)`, - wantErr: true, - }, - { - name: "invalid regex", - annotation: `CN=(my\.common\.name]`, - wantErr: true, - }, - { - name: "wildcard regex", - annotation: `CN=(my\..*\.name)`, - wantErr: false, - }, - { - name: "somewhat complex regex", - annotation: "CN=(my\\.app\\.dev|.*\\.bbb\\.aaaa\\.tld)", - wantErr: false, - }, - { - name: "another somewhat complex regex", - annotation: `CN=(my-app.*\.c\.defg\.net|other.app.com)`, - wantErr: false, - }, - { - name: "nested parenthesis regex", - annotation: `CN=(api-one\.(asdf)?qwer\.webpage\.organization\.org)`, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := CommonNameAnnotationValidator(tt.annotation); (err != nil) != tt.wantErr { - t.Errorf("CommonNameAnnotationValidator() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/internal/ingress/annotations/portinredirect/main.go b/internal/ingress/annotations/portinredirect/main.go index 7392ea3a6..bb5925c31 100644 --- a/internal/ingress/annotations/portinredirect/main.go +++ b/internal/ingress/annotations/portinredirect/main.go @@ -17,57 +17,28 @@ limitations under the License. package portinredirect import ( - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - portsInRedirectAnnotation = "use-port-in-redirects" -) - -var portsInRedirectAnnotations = parser.Annotation{ - Group: "redirect", - Annotations: parser.AnnotationFields{ - portsInRedirectAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, // Low, as it allows just a set of options - Documentation: `Enables or disables specifying the port in absolute redirects issued by nginx.`, - }, - }, -} - type portInRedirect struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new port in redirect annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return portInRedirect{ - r: r, - annotationConfig: portsInRedirectAnnotations, - } + return portInRedirect{r} } // Parse parses the annotations contained in the ingress // rule used to indicate if the redirects must func (a portInRedirect) Parse(ing *networking.Ingress) (interface{}, error) { - up, err := parser.GetBoolAnnotation(portsInRedirectAnnotation, ing, a.annotationConfig.Annotations) + up, err := parser.GetBoolAnnotation("use-port-in-redirects", ing) if err != nil { return a.r.GetDefaultBackend().UsePortInRedirects, nil } return up, nil } - -func (a portInRedirect) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a portInRedirect) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, portsInRedirectAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/portinredirect/main_test.go b/internal/ingress/annotations/portinredirect/main_test.go index f5806e01a..7087ddcd3 100644 --- a/internal/ingress/annotations/portinredirect/main_test.go +++ b/internal/ingress/annotations/portinredirect/main_test.go @@ -17,11 +17,13 @@ limitations under the License. package portinredirect import ( + "fmt" "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/defaults" @@ -30,12 +32,8 @@ import ( func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -44,13 +42,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -83,24 +77,23 @@ func (m mockBackend) GetDefaultBackend() defaults.Backend { func TestPortInRedirect(t *testing.T) { tests := []struct { title string - usePort string + usePort *bool def bool exp bool }{ - {"false - default false", "false", false, false}, - {"false - default true", "false", true, false}, - {"no annotation - default false", "", false, false}, - {"no annotation - default false", "not-a-bool", false, false}, - {"no annotation - default true", "", true, true}, - {"true - default true", "true", true, true}, + {"false - default false", newFalse(), false, false}, + {"false - default true", newFalse(), true, false}, + {"no annotation - default false", nil, false, false}, + {"no annotation - default true", nil, true, true}, + {"true - default true", newTrue(), true, true}, } for _, test := range tests { ing := buildIngress() data := map[string]string{} - if test.usePort != "" { - data[parser.GetAnnotationWithPrefix(portsInRedirectAnnotation)] = test.usePort + if test.usePort != nil { + data[parser.GetAnnotationWithPrefix("use-port-in-redirects")] = fmt.Sprintf("%v", *test.usePort) } ing.SetAnnotations(data) @@ -118,3 +111,13 @@ func TestPortInRedirect(t *testing.T) { } } } + +func newTrue() *bool { + b := true + return &b +} + +func newFalse() *bool { + b := false + return &b +} diff --git a/internal/ingress/annotations/proxy/main.go b/internal/ingress/annotations/proxy/main.go index aaa093eaf..f5c225258 100644 --- a/internal/ingress/annotations/proxy/main.go +++ b/internal/ingress/annotations/proxy/main.go @@ -17,155 +17,12 @@ limitations under the License. package proxy import ( - "regexp" - - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - proxyConnectTimeoutAnnotation = "proxy-connect-timeout" - proxySendTimeoutAnnotation = "proxy-send-timeout" - proxyReadTimeoutAnnotation = "proxy-read-timeout" - proxyBuffersNumberAnnotation = "proxy-buffers-number" - proxyBufferSizeAnnotation = "proxy-buffer-size" - proxyBusyBuffersSizeAnnotation = "proxy-busy-buffers-size" - proxyCookiePathAnnotation = "proxy-cookie-path" - proxyCookieDomainAnnotation = "proxy-cookie-domain" - proxyBodySizeAnnotation = "proxy-body-size" - proxyNextUpstreamAnnotation = "proxy-next-upstream" - proxyNextUpstreamTimeoutAnnotation = "proxy-next-upstream-timeout" - proxyNextUpstreamTriesAnnotation = "proxy-next-upstream-tries" - proxyRequestBufferingAnnotation = "proxy-request-buffering" - proxyRedirectFromAnnotation = "proxy-redirect-from" - proxyRedirectToAnnotation = "proxy-redirect-to" - proxyBufferingAnnotation = "proxy-buffering" - proxyHTTPVersionAnnotation = "proxy-http-version" - proxyMaxTempFileSizeAnnotation = "proxy-max-temp-file-size" //#nosec G101 -) - -var validUpstreamAnnotation = regexp.MustCompile(`^((error|timeout|invalid_header|http_500|http_502|http_503|http_504|http_403|http_404|http_429|non_idempotent|off)\s?)+$`) - -var proxyAnnotations = parser.Annotation{ - Group: "backend", - Annotations: parser.AnnotationFields{ - proxyConnectTimeoutAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation allows setting the timeout in seconds of the connect operation to the backend.`, - }, - proxySendTimeoutAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation allows setting the timeout in seconds of the send operation to the backend.`, - }, - proxyReadTimeoutAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation allows setting the timeout in seconds of the read operation to the backend.`, - }, - proxyBuffersNumberAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation sets the number of the buffers in proxy_buffers used for reading the first part of the response received from the proxied server. - By default proxy buffers number is set as 4`, - }, - proxyBufferSizeAnnotation: { - Validator: parser.ValidateRegex(parser.SizeRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation sets the size of the buffer proxy_buffer_size used for reading the first part of the response received from the proxied server. - By default proxy buffer size is set as "4k".`, - }, - proxyBusyBuffersSizeAnnotation: { - Validator: parser.ValidateRegex(parser.SizeRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation limits the total size of buffers that can be busy sending a response to the client while the response is not yet fully read. By default proxy busy buffers size is set as "8k".`, - }, - proxyCookiePathAnnotation: { - Validator: parser.ValidateRegex(parser.URLIsValidRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation sets a text that should be changed in the path attribute of the "Set-Cookie" header fields of a proxied server response.`, - }, - proxyCookieDomainAnnotation: { - Validator: parser.ValidateRegex(parser.BasicCharsRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation ets a text that should be changed in the domain attribute of the "Set-Cookie" header fields of a proxied server response.`, - }, - proxyBodySizeAnnotation: { - Validator: parser.ValidateRegex(parser.SizeRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation allows setting the maximum allowed size of a client request body.`, - }, - proxyNextUpstreamAnnotation: { - Validator: parser.ValidateRegex(validUpstreamAnnotation, false), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation defines when the next upstream should be used. - This annotation reflect the directive https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream - and only the allowed values on upstream are allowed here.`, - }, - proxyNextUpstreamTimeoutAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation limits the time during which a request can be passed to the next server`, - }, - proxyNextUpstreamTriesAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation limits the number of possible tries for passing a request to the next server`, - }, - proxyRequestBufferingAnnotation: { - Validator: parser.ValidateOptions([]string{"on", "off"}, true, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation enables or disables buffering of a client request body.`, - }, - proxyRedirectFromAnnotation: { - Validator: parser.ValidateRegex(parser.URLIsValidRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `The annotations proxy-redirect-from and proxy-redirect-to will set the first and second parameters of NGINX's proxy_redirect directive respectively`, - }, - proxyRedirectToAnnotation: { - Validator: parser.ValidateRegex(parser.URLIsValidRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `The annotations proxy-redirect-from and proxy-redirect-to will set the first and second parameters of NGINX's proxy_redirect directive respectively`, - }, - proxyBufferingAnnotation: { - Validator: parser.ValidateOptions([]string{"on", "off"}, true, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation enables or disables buffering of responses from the proxied server. It can be "on" or "off"`, - }, - proxyHTTPVersionAnnotation: { - Validator: parser.ValidateOptions([]string{"1.0", "1.1"}, true, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotations sets the HTTP protocol version for proxying. Can be "1.0" or "1.1".`, - }, - proxyMaxTempFileSizeAnnotation: { - Validator: parser.ValidateRegex(parser.SizeRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation defines the maximum size of a temporary file when buffering responses.`, - }, - }, -} - // Config returns the proxy timeout to use in the upstream server/s type Config struct { BodySize string `json:"bodySize"` @@ -174,7 +31,6 @@ type Config struct { ReadTimeout int `json:"readTimeout"` BuffersNumber int `json:"buffersNumber"` BufferSize string `json:"bufferSize"` - BusyBuffersSize string `json:"busyBuffersSize"` CookieDomain string `json:"cookieDomain"` CookiePath string `json:"cookiePath"` NextUpstream string `json:"nextUpstream"` @@ -214,9 +70,6 @@ func (l1 *Config) Equal(l2 *Config) bool { if l1.BufferSize != l2.BufferSize { return false } - if l1.BusyBuffersSize != l2.BusyBuffersSize { - return false - } if l1.CookieDomain != l2.CookieDomain { return false } @@ -256,16 +109,12 @@ func (l1 *Config) Equal(l2 *Config) bool { } type proxy struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new reverse proxy configuration annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return proxy{ - r: r, - annotationConfig: proxyAnnotations, - } + return proxy{r} } // ParseAnnotations parses the annotations contained in the ingress @@ -276,104 +125,90 @@ func (a proxy) Parse(ing *networking.Ingress) (interface{}, error) { var err error - config.ConnectTimeout, err = parser.GetIntAnnotation(proxyConnectTimeoutAnnotation, ing, a.annotationConfig.Annotations) + config.ConnectTimeout, err = parser.GetIntAnnotation("proxy-connect-timeout", ing) if err != nil { config.ConnectTimeout = defBackend.ProxyConnectTimeout } - config.SendTimeout, err = parser.GetIntAnnotation(proxySendTimeoutAnnotation, ing, a.annotationConfig.Annotations) + config.SendTimeout, err = parser.GetIntAnnotation("proxy-send-timeout", ing) if err != nil { config.SendTimeout = defBackend.ProxySendTimeout } - config.ReadTimeout, err = parser.GetIntAnnotation(proxyReadTimeoutAnnotation, ing, a.annotationConfig.Annotations) + config.ReadTimeout, err = parser.GetIntAnnotation("proxy-read-timeout", ing) if err != nil { config.ReadTimeout = defBackend.ProxyReadTimeout } - config.BuffersNumber, err = parser.GetIntAnnotation(proxyBuffersNumberAnnotation, ing, a.annotationConfig.Annotations) + config.BuffersNumber, err = parser.GetIntAnnotation("proxy-buffers-number", ing) if err != nil { config.BuffersNumber = defBackend.ProxyBuffersNumber } - config.BufferSize, err = parser.GetStringAnnotation(proxyBufferSizeAnnotation, ing, a.annotationConfig.Annotations) + config.BufferSize, err = parser.GetStringAnnotation("proxy-buffer-size", ing) if err != nil { config.BufferSize = defBackend.ProxyBufferSize } - config.BusyBuffersSize, err = parser.GetStringAnnotation(proxyBusyBuffersSizeAnnotation, ing, a.annotationConfig.Annotations) - if err != nil { - config.BusyBuffersSize = defBackend.ProxyBusyBuffersSize - } - - config.CookiePath, err = parser.GetStringAnnotation(proxyCookiePathAnnotation, ing, a.annotationConfig.Annotations) + config.CookiePath, err = parser.GetStringAnnotation("proxy-cookie-path", ing) if err != nil { config.CookiePath = defBackend.ProxyCookiePath } - config.CookieDomain, err = parser.GetStringAnnotation(proxyCookieDomainAnnotation, ing, a.annotationConfig.Annotations) + config.CookieDomain, err = parser.GetStringAnnotation("proxy-cookie-domain", ing) if err != nil { config.CookieDomain = defBackend.ProxyCookieDomain } - config.BodySize, err = parser.GetStringAnnotation(proxyBodySizeAnnotation, ing, a.annotationConfig.Annotations) + config.BodySize, err = parser.GetStringAnnotation("proxy-body-size", ing) if err != nil { config.BodySize = defBackend.ProxyBodySize } - config.NextUpstream, err = parser.GetStringAnnotation(proxyNextUpstreamAnnotation, ing, a.annotationConfig.Annotations) + config.NextUpstream, err = parser.GetStringAnnotation("proxy-next-upstream", ing) if err != nil { config.NextUpstream = defBackend.ProxyNextUpstream } - config.NextUpstreamTimeout, err = parser.GetIntAnnotation(proxyNextUpstreamTimeoutAnnotation, ing, a.annotationConfig.Annotations) + config.NextUpstreamTimeout, err = parser.GetIntAnnotation("proxy-next-upstream-timeout", ing) if err != nil { config.NextUpstreamTimeout = defBackend.ProxyNextUpstreamTimeout } - config.NextUpstreamTries, err = parser.GetIntAnnotation(proxyNextUpstreamTriesAnnotation, ing, a.annotationConfig.Annotations) + config.NextUpstreamTries, err = parser.GetIntAnnotation("proxy-next-upstream-tries", ing) if err != nil { config.NextUpstreamTries = defBackend.ProxyNextUpstreamTries } - config.RequestBuffering, err = parser.GetStringAnnotation(proxyRequestBufferingAnnotation, ing, a.annotationConfig.Annotations) + config.RequestBuffering, err = parser.GetStringAnnotation("proxy-request-buffering", ing) if err != nil { config.RequestBuffering = defBackend.ProxyRequestBuffering } - config.ProxyRedirectFrom, err = parser.GetStringAnnotation(proxyRedirectFromAnnotation, ing, a.annotationConfig.Annotations) + config.ProxyRedirectFrom, err = parser.GetStringAnnotation("proxy-redirect-from", ing) if err != nil { config.ProxyRedirectFrom = defBackend.ProxyRedirectFrom } - config.ProxyRedirectTo, err = parser.GetStringAnnotation(proxyRedirectToAnnotation, ing, a.annotationConfig.Annotations) + config.ProxyRedirectTo, err = parser.GetStringAnnotation("proxy-redirect-to", ing) if err != nil { config.ProxyRedirectTo = defBackend.ProxyRedirectTo } - config.ProxyBuffering, err = parser.GetStringAnnotation(proxyBufferingAnnotation, ing, a.annotationConfig.Annotations) + config.ProxyBuffering, err = parser.GetStringAnnotation("proxy-buffering", ing) if err != nil { config.ProxyBuffering = defBackend.ProxyBuffering } - config.ProxyHTTPVersion, err = parser.GetStringAnnotation(proxyHTTPVersionAnnotation, ing, a.annotationConfig.Annotations) + config.ProxyHTTPVersion, err = parser.GetStringAnnotation("proxy-http-version", ing) if err != nil { config.ProxyHTTPVersion = defBackend.ProxyHTTPVersion } - config.ProxyMaxTempFileSize, err = parser.GetStringAnnotation(proxyMaxTempFileSizeAnnotation, ing, a.annotationConfig.Annotations) + config.ProxyMaxTempFileSize, err = parser.GetStringAnnotation("proxy-max-temp-file-size", ing) if err != nil { config.ProxyMaxTempFileSize = defBackend.ProxyMaxTempFileSize } return config, nil } - -func (a proxy) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a proxy) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, proxyAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/proxy/main_test.go b/internal/ingress/annotations/proxy/main_test.go index b6ce07fb2..418db922e 100644 --- a/internal/ingress/annotations/proxy/main_test.go +++ b/internal/ingress/annotations/proxy/main_test.go @@ -20,28 +20,19 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/defaults" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - off = "off" - proxyHTTPVersion = "1.0" - proxyMaxTempFileSize = "128k" -) - func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -50,13 +41,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -88,13 +75,12 @@ func (m mockBackend) GetDefaultBackend() defaults.Backend { ProxyReadTimeout: 20, ProxyBuffersNumber: 4, ProxyBufferSize: "10k", - ProxyBusyBuffersSize: "15k", ProxyBodySize: "3k", ProxyNextUpstream: "error", ProxyNextUpstreamTimeout: 0, ProxyNextUpstreamTries: 3, ProxyRequestBuffering: "on", - ProxyBuffering: off, + ProxyBuffering: "off", ProxyHTTPVersion: "1.1", ProxyMaxTempFileSize: "1024m", } @@ -109,87 +95,14 @@ func TestProxy(t *testing.T) { data[parser.GetAnnotationWithPrefix("proxy-read-timeout")] = "3" data[parser.GetAnnotationWithPrefix("proxy-buffers-number")] = "8" data[parser.GetAnnotationWithPrefix("proxy-buffer-size")] = "1k" - data[parser.GetAnnotationWithPrefix("proxy-busy-buffers-size")] = "4k" data[parser.GetAnnotationWithPrefix("proxy-body-size")] = "2k" - data[parser.GetAnnotationWithPrefix("proxy-next-upstream")] = off - data[parser.GetAnnotationWithPrefix("proxy-next-upstream-timeout")] = "5" - data[parser.GetAnnotationWithPrefix("proxy-next-upstream-tries")] = "3" - data[parser.GetAnnotationWithPrefix("proxy-request-buffering")] = off - data[parser.GetAnnotationWithPrefix("proxy-buffering")] = "on" - data[parser.GetAnnotationWithPrefix("proxy-http-version")] = proxyHTTPVersion - data[parser.GetAnnotationWithPrefix("proxy-max-temp-file-size")] = proxyMaxTempFileSize - ing.SetAnnotations(data) - - i, err := NewParser(mockBackend{}).Parse(ing) - if err != nil { - t.Fatalf("unexpected error parsing a valid") - } - p, ok := i.(*Config) - if !ok { - t.Fatalf("expected a Config type") - } - if p.ConnectTimeout != 1 { - t.Errorf("expected 1 as connect-timeout but returned %v", p.ConnectTimeout) - } - if p.SendTimeout != 2 { - t.Errorf("expected 2 as send-timeout but returned %v", p.SendTimeout) - } - if p.ReadTimeout != 3 { - t.Errorf("expected 3 as read-timeout but returned %v", p.ReadTimeout) - } - if p.BuffersNumber != 8 { - t.Errorf("expected 8 as proxy-buffers-number but returned %v", p.BuffersNumber) - } - if p.BufferSize != "1k" { - t.Errorf("expected 1k as buffer-size but returned %v", p.BufferSize) - } - if p.BusyBuffersSize != "4k" { - t.Errorf("expected 4k as busy-buffers-size but returned %v", p.BusyBuffersSize) - } - if p.BodySize != "2k" { - t.Errorf("expected 2k as body-size but returned %v", p.BodySize) - } - if p.NextUpstream != off { - t.Errorf("expected off as next-upstream but returned %v", p.NextUpstream) - } - if p.NextUpstreamTimeout != 5 { - t.Errorf("expected 5 as next-upstream-timeout but returned %v", p.NextUpstreamTimeout) - } - if p.NextUpstreamTries != 3 { - t.Errorf("expected 3 as next-upstream-tries but returned %v", p.NextUpstreamTries) - } - if p.RequestBuffering != off { - t.Errorf("expected off as request-buffering but returned %v", p.RequestBuffering) - } - if p.ProxyBuffering != "on" { - t.Errorf("expected on as proxy-buffering but returned %v", p.ProxyBuffering) - } - if p.ProxyHTTPVersion != proxyHTTPVersion { - t.Errorf("expected 1.0 as proxy-http-version but returned %v", p.ProxyHTTPVersion) - } - if p.ProxyMaxTempFileSize != proxyMaxTempFileSize { - t.Errorf("expected 128k as proxy-max-temp-file-size but returned %v", p.ProxyMaxTempFileSize) - } -} - -func TestProxyComplex(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix("proxy-connect-timeout")] = "1" - data[parser.GetAnnotationWithPrefix("proxy-send-timeout")] = "2" - data[parser.GetAnnotationWithPrefix("proxy-read-timeout")] = "3" - data[parser.GetAnnotationWithPrefix("proxy-buffers-number")] = "8" - data[parser.GetAnnotationWithPrefix("proxy-buffer-size")] = "1k" - data[parser.GetAnnotationWithPrefix("proxy-busy-buffers-size")] = "4k" - data[parser.GetAnnotationWithPrefix("proxy-body-size")] = "2k" - data[parser.GetAnnotationWithPrefix("proxy-next-upstream")] = "error http_502" + data[parser.GetAnnotationWithPrefix("proxy-next-upstream")] = "off" data[parser.GetAnnotationWithPrefix("proxy-next-upstream-timeout")] = "5" data[parser.GetAnnotationWithPrefix("proxy-next-upstream-tries")] = "3" data[parser.GetAnnotationWithPrefix("proxy-request-buffering")] = "off" data[parser.GetAnnotationWithPrefix("proxy-buffering")] = "on" - data[parser.GetAnnotationWithPrefix("proxy-http-version")] = proxyHTTPVersion - data[parser.GetAnnotationWithPrefix("proxy-max-temp-file-size")] = proxyMaxTempFileSize + data[parser.GetAnnotationWithPrefix("proxy-http-version")] = "1.0" + data[parser.GetAnnotationWithPrefix("proxy-max-temp-file-size")] = "128k" ing.SetAnnotations(data) i, err := NewParser(mockBackend{}).Parse(ing) @@ -215,13 +128,10 @@ func TestProxyComplex(t *testing.T) { if p.BufferSize != "1k" { t.Errorf("expected 1k as buffer-size but returned %v", p.BufferSize) } - if p.BusyBuffersSize != "4k" { - t.Errorf("expected 4k as buffer-size but returned %v", p.BusyBuffersSize) - } if p.BodySize != "2k" { t.Errorf("expected 2k as body-size but returned %v", p.BodySize) } - if p.NextUpstream != "error http_502" { + if p.NextUpstream != "off" { t.Errorf("expected off as next-upstream but returned %v", p.NextUpstream) } if p.NextUpstreamTimeout != 5 { @@ -236,10 +146,10 @@ func TestProxyComplex(t *testing.T) { if p.ProxyBuffering != "on" { t.Errorf("expected on as proxy-buffering but returned %v", p.ProxyBuffering) } - if p.ProxyHTTPVersion != proxyHTTPVersion { + if p.ProxyHTTPVersion != "1.0" { t.Errorf("expected 1.0 as proxy-http-version but returned %v", p.ProxyHTTPVersion) } - if p.ProxyMaxTempFileSize != proxyMaxTempFileSize { + if p.ProxyMaxTempFileSize != "128k" { t.Errorf("expected 128k as proxy-max-temp-file-size but returned %v", p.ProxyMaxTempFileSize) } } @@ -273,9 +183,6 @@ func TestProxyWithNoAnnotation(t *testing.T) { if p.BufferSize != "10k" { t.Errorf("expected 10k as buffer-size but returned %v", p.BufferSize) } - if p.BusyBuffersSize != "15k" { - t.Errorf("expected 15k as buffer-size but returned %v", p.BusyBuffersSize) - } if p.BodySize != "3k" { t.Errorf("expected 3k as body-size but returned %v", p.BodySize) } diff --git a/internal/ingress/annotations/proxyssl/main.go b/internal/ingress/annotations/proxyssl/main.go index c40bc85e0..da3bbecc0 100644 --- a/internal/ingress/annotations/proxyssl/main.go +++ b/internal/ingress/annotations/proxyssl/main.go @@ -17,22 +17,21 @@ limitations under the License. package proxyssl import ( - "fmt" "regexp" "sort" "strings" - networking "k8s.io/api/networking/v1" + "github.com/pkg/errors" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" ing_errors "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" "k8s.io/ingress-nginx/internal/k8s" - "k8s.io/klog/v2" ) const ( defaultProxySSLCiphers = "DEFAULT" - defaultProxySSLProtocols = "TLSv1.2" + defaultProxySSLProtocols = "TLSv1 TLSv1.1 TLSv1.2" defaultProxySSLVerify = "off" defaultProxySSLVerifyDepth = 1 defaultProxySSLServerName = "off" @@ -40,73 +39,9 @@ const ( var ( proxySSLOnOffRegex = regexp.MustCompile(`^(on|off)$`) - proxySSLProtocolRegex = regexp.MustCompile(`^(TLSv1\.2|TLSv1\.3| )*$`) - proxySSLCiphersRegex = regexp.MustCompile(`^[A-Za-z0-9\+:\_\-!]*$`) + proxySSLProtocolRegex = regexp.MustCompile(`^(SSLv2|SSLv3|TLSv1|TLSv1\.1|TLSv1\.2|TLSv1\.3)$`) ) -const ( - proxySSLSecretAnnotation = "proxy-ssl-secret" - proxySSLCiphersAnnotation = "proxy-ssl-ciphers" - proxySSLProtocolsAnnotation = "proxy-ssl-protocols" - proxySSLNameAnnotation = "proxy-ssl-name" - proxySSLVerifyAnnotation = "proxy-ssl-verify" - proxySSLVerifyDepthAnnotation = "proxy-ssl-verify-depth" - proxySSLServerNameAnnotation = "proxy-ssl-server-name" -) - -var proxySSLAnnotation = parser.Annotation{ - Group: "proxy", - Annotations: parser.AnnotationFields{ - proxySSLSecretAnnotation: { - Validator: parser.ValidateRegex(parser.BasicCharsRegex, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation specifies a Secret with the certificate tls.crt, key tls.key in PEM format used for authentication to a proxied HTTPS server. - It should also contain trusted CA certificates ca.crt in PEM format used to verify the certificate of the proxied HTTPS server. - This annotation expects the Secret name in the form "namespace/secretName" - Just secrets on the same namespace of the ingress can be used.`, - }, - proxySSLCiphersAnnotation: { - Validator: parser.ValidateRegex(proxySSLCiphersRegex, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation Specifies the enabled ciphers for requests to a proxied HTTPS server. - The ciphers are specified in the format understood by the OpenSSL library.`, - }, - proxySSLProtocolsAnnotation: { - Validator: parser.ValidateRegex(proxySSLProtocolRegex, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation enables the specified protocols for requests to a proxied HTTPS server.`, - }, - proxySSLNameAnnotation: { - Validator: parser.ValidateServerName, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskHigh, - Documentation: `This annotation allows to set proxy_ssl_name. This allows overriding the server name used to verify the certificate of the proxied HTTPS server. - This value is also passed through SNI when a connection is established to the proxied HTTPS server.`, - }, - proxySSLVerifyAnnotation: { - Validator: parser.ValidateRegex(proxySSLOnOffRegex, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation enables or disables verification of the proxied HTTPS server certificate. (default: off)`, - }, - proxySSLVerifyDepthAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation Sets the verification depth in the proxied HTTPS server certificates chain. (default: 1).`, - }, - proxySSLServerNameAnnotation: { - Validator: parser.ValidateRegex(proxySSLOnOffRegex, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation enables passing of the server name through TLS Server Name Indication extension (SNI, RFC 6066) when establishing a connection with the proxied HTTPS server.`, - }, - }, -} - // Config contains the AuthSSLCert used for mutual authentication // and the configured VerifyDepth type Config struct { @@ -149,16 +84,12 @@ func (pssl1 *Config) Equal(pssl2 *Config) bool { } // NewParser creates a new TLS authentication annotation parser -func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return proxySSL{ - r: r, - annotationConfig: proxySSLAnnotation, - } +func NewParser(resolver resolver.Resolver) parser.IngressAnnotation { + return proxySSL{resolver} } type proxySSL struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } func sortProtocols(protocols string) string { @@ -189,78 +120,54 @@ func (p proxySSL) Parse(ing *networking.Ingress) (interface{}, error) { var err error config := &Config{} - proxysslsecret, err := parser.GetStringAnnotation(proxySSLSecretAnnotation, ing, p.annotationConfig.Annotations) + proxysslsecret, err := parser.GetStringAnnotation("proxy-ssl-secret", ing) if err != nil { return &Config{}, err } - ns, _, err := k8s.ParseNameNS(proxysslsecret) + _, _, err = k8s.ParseNameNS(proxysslsecret) if err != nil { return &Config{}, ing_errors.NewLocationDenied(err.Error()) } - secCfg := p.r.GetSecurityConfiguration() - // We don't accept different namespaces for secrets. - if !secCfg.AllowCrossNamespaceResources && ns != ing.Namespace { - return &Config{}, ing_errors.NewLocationDenied("cross namespace secrets are not supported") - } - proxyCert, err := p.r.GetAuthCertificate(proxysslsecret) if err != nil { - e := fmt.Errorf("error obtaining certificate: %w", err) - return &Config{}, ing_errors.LocationDeniedError{Reason: e} + e := errors.Wrap(err, "error obtaining certificate") + return &Config{}, ing_errors.LocationDenied{Reason: e} } config.AuthSSLCert = *proxyCert - config.Ciphers, err = parser.GetStringAnnotation(proxySSLCiphersAnnotation, ing, p.annotationConfig.Annotations) + config.Ciphers, err = parser.GetStringAnnotation("proxy-ssl-ciphers", ing) if err != nil { - if ing_errors.IsValidationError(err) { - klog.Warningf("invalid value passed to proxy-ssl-ciphers, defaulting to %s", defaultProxySSLCiphers) - } config.Ciphers = defaultProxySSLCiphers } - config.Protocols, err = parser.GetStringAnnotation(proxySSLProtocolsAnnotation, ing, p.annotationConfig.Annotations) + config.Protocols, err = parser.GetStringAnnotation("proxy-ssl-protocols", ing) if err != nil { - if ing_errors.IsValidationError(err) { - klog.Warningf("invalid value passed to proxy-ssl-protocols, defaulting to %s", defaultProxySSLProtocols) - } config.Protocols = defaultProxySSLProtocols } else { config.Protocols = sortProtocols(config.Protocols) } - config.ProxySSLName, err = parser.GetStringAnnotation(proxySSLNameAnnotation, ing, p.annotationConfig.Annotations) + config.ProxySSLName, err = parser.GetStringAnnotation("proxy-ssl-name", ing) if err != nil { - if ing_errors.IsValidationError(err) { - klog.Warningf("invalid value passed to proxy-ssl-name, defaulting to empty") - } config.ProxySSLName = "" } - config.Verify, err = parser.GetStringAnnotation(proxySSLVerifyAnnotation, ing, p.annotationConfig.Annotations) + config.Verify, err = parser.GetStringAnnotation("proxy-ssl-verify", ing) if err != nil || !proxySSLOnOffRegex.MatchString(config.Verify) { config.Verify = defaultProxySSLVerify } - config.VerifyDepth, err = parser.GetIntAnnotation(proxySSLVerifyDepthAnnotation, ing, p.annotationConfig.Annotations) + config.VerifyDepth, err = parser.GetIntAnnotation("proxy-ssl-verify-depth", ing) if err != nil || config.VerifyDepth == 0 { config.VerifyDepth = defaultProxySSLVerifyDepth } - config.ProxySSLServerName, err = parser.GetStringAnnotation(proxySSLServerNameAnnotation, ing, p.annotationConfig.Annotations) + config.ProxySSLServerName, err = parser.GetStringAnnotation("proxy-ssl-server-name", ing) if err != nil || !proxySSLOnOffRegex.MatchString(config.ProxySSLServerName) { config.ProxySSLServerName = defaultProxySSLServerName } return config, nil } - -func (p proxySSL) GetDocumentation() parser.AnnotationFields { - return p.annotationConfig.Annotations -} - -func (p proxySSL) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(p.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, proxySSLAnnotation.Annotations) -} diff --git a/internal/ingress/annotations/proxyssl/main_test.go b/internal/ingress/annotations/proxyssl/main_test.go index cfa31f1d1..a52fcb98a 100644 --- a/internal/ingress/annotations/proxyssl/main_test.go +++ b/internal/ingress/annotations/proxyssl/main_test.go @@ -20,29 +20,18 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - defaultDemoSecret = "default/demo-secret" - proxySslCiphers = "HIGH:-SHA" - off = "off" - sslServerName = "w00t" - defaultProtocol = "TLSv1.2 TLSv1.3" -) - func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -51,13 +40,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -85,27 +70,28 @@ type mockSecret struct { // GetAuthCertificate from mockSecret mocks the GetAuthCertificate for backend certificate authentication func (m mockSecret) GetAuthCertificate(name string) (*resolver.AuthSSLCert, error) { - if name != defaultDemoSecret { + if name != "default/demo-secret" { return nil, errors.Errorf("there is no secret with name %v", name) } return &resolver.AuthSSLCert{ - Secret: defaultDemoSecret, + Secret: "default/demo-secret", CAFileName: "/ssl/ca.crt", CASHA: "abc", }, nil + } func TestAnnotations(t *testing.T) { ing := buildIngress() data := map[string]string{} - data[parser.GetAnnotationWithPrefix(proxySSLSecretAnnotation)] = defaultDemoSecret - data[parser.GetAnnotationWithPrefix("proxy-ssl-ciphers")] = proxySslCiphers + data[parser.GetAnnotationWithPrefix("proxy-ssl-secret")] = "default/demo-secret" + data[parser.GetAnnotationWithPrefix("proxy-ssl-ciphers")] = "HIGH:-SHA" data[parser.GetAnnotationWithPrefix("proxy-ssl-name")] = "$host" - data[parser.GetAnnotationWithPrefix("proxy-ssl-protocols")] = "TLSv1.3 TLSv1.2" + data[parser.GetAnnotationWithPrefix("proxy-ssl-protocols")] = "TLSv1.3 SSLv2 TLSv1 TLSv1.2" data[parser.GetAnnotationWithPrefix("proxy-ssl-server-name")] = "on" - data[parser.GetAnnotationWithPrefix("proxy-ssl-session-reuse")] = off + data[parser.GetAnnotationWithPrefix("proxy-ssl-session-reuse")] = "off" data[parser.GetAnnotationWithPrefix("proxy-ssl-verify")] = "on" data[parser.GetAnnotationWithPrefix("proxy-ssl-verify-depth")] = "3" @@ -122,7 +108,7 @@ func TestAnnotations(t *testing.T) { t.Errorf("expected *Config but got %v", u) } - secret, err := fakeSecret.GetAuthCertificate(defaultDemoSecret) + secret, err := fakeSecret.GetAuthCertificate("default/demo-secret") if err != nil { t.Errorf("unexpected error getting secret %v", err) } @@ -130,11 +116,11 @@ func TestAnnotations(t *testing.T) { if u.AuthSSLCert.Secret != secret.Secret { t.Errorf("expected %v but got %v", secret.Secret, u.AuthSSLCert.Secret) } - if u.Ciphers != proxySslCiphers { - t.Errorf("expected %v but got %v", proxySslCiphers, u.Ciphers) + if u.Ciphers != "HIGH:-SHA" { + t.Errorf("expected %v but got %v", "HIGH:-SHA", u.Ciphers) } - if u.Protocols != defaultProtocol { - t.Errorf("expected %v but got %v", defaultProtocol, u.Protocols) + if u.Protocols != "SSLv2 TLSv1 TLSv1.2 TLSv1.3" { + t.Errorf("expected %v but got %v", "SSLv2 TLSv1 TLSv1.2 TLSv1.3", u.Protocols) } if u.Verify != "on" { t.Errorf("expected %v but got %v", "on", u.Verify) @@ -148,6 +134,7 @@ func TestAnnotations(t *testing.T) { if u.ProxySSLServerName != "on" { t.Errorf("expected %v but got %v", "on", u.ProxySSLServerName) } + } func TestInvalidAnnotations(t *testing.T) { @@ -178,11 +165,11 @@ func TestInvalidAnnotations(t *testing.T) { } // Invalid optional Annotations - data[parser.GetAnnotationWithPrefix("proxy-ssl-secret")] = defaultDemoSecret + data[parser.GetAnnotationWithPrefix("proxy-ssl-secret")] = "default/demo-secret" data[parser.GetAnnotationWithPrefix("proxy-ssl-protocols")] = "TLSv111 SSLv1" - data[parser.GetAnnotationWithPrefix("proxy-ssl-server-name")] = sslServerName - data[parser.GetAnnotationWithPrefix("proxy-ssl-session-reuse")] = sslServerName - data[parser.GetAnnotationWithPrefix("proxy-ssl-verify")] = sslServerName + data[parser.GetAnnotationWithPrefix("proxy-ssl-server-name")] = "w00t" + data[parser.GetAnnotationWithPrefix("proxy-ssl-session-reuse")] = "w00t" + data[parser.GetAnnotationWithPrefix("proxy-ssl-verify")] = "w00t" data[parser.GetAnnotationWithPrefix("proxy-ssl-verify-depth")] = "abcd" ing.SetAnnotations(data) @@ -213,15 +200,21 @@ func TestEquals(t *testing.T) { cfg1 := &Config{} cfg2 := &Config{} + // Same config + result := cfg1.Equal(cfg1) + if result != true { + t.Errorf("Expected true") + } + // compare nil - result := cfg1.Equal(nil) + result = cfg1.Equal(nil) if result != false { t.Errorf("Expected false") } // Different Certs sslCert1 := resolver.AuthSSLCert{ - Secret: defaultDemoSecret, + Secret: "default/demo-secret", CAFileName: "/ssl/ca.crt", CASHA: "abc", } @@ -240,7 +233,7 @@ func TestEquals(t *testing.T) { // Different Ciphers cfg1.Ciphers = "DEFAULT" - cfg2.Ciphers = proxySslCiphers + cfg2.Ciphers = "HIGH:-SHA" result = cfg1.Equal(cfg2) if result != false { t.Errorf("Expected false") @@ -248,22 +241,22 @@ func TestEquals(t *testing.T) { cfg2.Ciphers = "DEFAULT" // Different Protocols - cfg1.Protocols = defaultProtocol + cfg1.Protocols = "SSLv2 TLSv1 TLSv1.2 TLSv1.3" cfg2.Protocols = "SSLv3 TLSv1 TLSv1.2 TLSv1.3" result = cfg1.Equal(cfg2) if result != false { t.Errorf("Expected false") } - cfg2.Protocols = defaultProtocol + cfg2.Protocols = "SSLv2 TLSv1 TLSv1.2 TLSv1.3" // Different Verify - cfg1.Verify = off + cfg1.Verify = "off" cfg2.Verify = "on" result = cfg1.Equal(cfg2) if result != false { t.Errorf("Expected false") } - cfg2.Verify = off + cfg2.Verify = "off" // Different VerifyDepth cfg1.VerifyDepth = 1 @@ -275,13 +268,13 @@ func TestEquals(t *testing.T) { cfg2.VerifyDepth = 1 // Different ProxySSLServerName - cfg1.ProxySSLServerName = off + cfg1.ProxySSLServerName = "off" cfg2.ProxySSLServerName = "on" result = cfg1.Equal(cfg2) if result != false { t.Errorf("Expected false") } - cfg2.ProxySSLServerName = off + cfg2.ProxySSLServerName = "off" // Equal Configs result = cfg1.Equal(cfg2) diff --git a/internal/ingress/annotations/ratelimit/main.go b/internal/ingress/annotations/ratelimit/main.go index e79c698bf..7b7d6f4db 100644 --- a/internal/ingress/annotations/ratelimit/main.go +++ b/internal/ingress/annotations/ratelimit/main.go @@ -21,13 +21,12 @@ import ( "fmt" "strings" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" "k8s.io/ingress-nginx/internal/net" - "k8s.io/ingress-nginx/pkg/util/sets" + "k8s.io/ingress-nginx/internal/sets" ) const ( @@ -59,7 +58,7 @@ type Config struct { ID string `json:"id"` - Allowlist []string `json:"allowlist"` + Whitelist []string `json:"whitelist"` } // Equal tests for equality between two RateLimit types @@ -91,11 +90,11 @@ func (rt1 *Config) Equal(rt2 *Config) bool { if rt1.Name != rt2.Name { return false } - if len(rt1.Allowlist) != len(rt2.Allowlist) { + if len(rt1.Whitelist) != len(rt2.Whitelist) { return false } - return sets.StringElementsMatch(rt1.Allowlist, rt2.Allowlist) + return sets.StringElementsMatch(rt1.Whitelist, rt2.Whitelist) } // Zone returns information about the NGINX rate limit (limit_req_zone) @@ -132,119 +131,41 @@ func (z1 *Zone) Equal(z2 *Zone) bool { return true } -const ( - limitRateAnnotation = "limit-rate" - limitRateAfterAnnotation = "limit-rate-after" - limitRateRPMAnnotation = "limit-rpm" - limitRateRPSAnnotation = "limit-rps" - limitRateConnectionsAnnotation = "limit-connections" - limitRateBurstMultiplierAnnotation = "limit-burst-multiplier" - limitWhitelistAnnotation = "limit-whitelist" // This annotation is an alias for limit-allowlist - limitAllowlistAnnotation = "limit-allowlist" -) - -var rateLimitAnnotations = parser.Annotation{ - Group: "rate-limit", - Annotations: parser.AnnotationFields{ - limitRateAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, // Low, as it allows just a set of options - Documentation: `Limits the rate of response transmission to a client. The rate is specified in bytes per second. - The zero value disables rate limiting. The limit is set per a request, and so if a client simultaneously opens two connections, the overall rate will be twice as much as the specified limit. - References: https://nginx.org/en/docs/http/ngx_http_core_module.html#limit_rate`, - }, - limitRateAfterAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, // Low, as it allows just a set of options - Documentation: `Sets the initial amount after which the further transmission of a response to a client will be rate limited.`, - }, - limitRateRPMAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, // Low, as it allows just a set of options - Documentation: `Requests per minute that will be allowed.`, - }, - limitRateRPSAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, // Low, as it allows just a set of options - Documentation: `Requests per second that will be allowed.`, - }, - limitRateConnectionsAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, // Low, as it allows just a set of options - Documentation: `Number of connections that will be allowed`, - }, - limitRateBurstMultiplierAnnotation: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, // Low, as it allows just a set of options - Documentation: `Burst multiplier for a limit-rate enabled location.`, - }, - limitAllowlistAnnotation: { - Validator: parser.ValidateCIDRs, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, // Low, as it allows just a set of options - Documentation: `List of CIDR/IP addresses that will not be rate-limited.`, - AnnotationAliases: []string{limitWhitelistAnnotation}, - }, - }, -} - type ratelimit struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new ratelimit annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return ratelimit{ - r: r, - annotationConfig: rateLimitAnnotations, - } + return ratelimit{r} } // ParseAnnotations parses the annotations contained in the ingress // rule used to rewrite the defined paths func (a ratelimit) Parse(ing *networking.Ingress) (interface{}, error) { defBackend := a.r.GetDefaultBackend() - lr, err := parser.GetIntAnnotation(limitRateAnnotation, ing, a.annotationConfig.Annotations) + lr, err := parser.GetIntAnnotation("limit-rate", ing) if err != nil { lr = defBackend.LimitRate } - lra, err := parser.GetIntAnnotation(limitRateAfterAnnotation, ing, a.annotationConfig.Annotations) + lra, err := parser.GetIntAnnotation("limit-rate-after", ing) if err != nil { lra = defBackend.LimitRateAfter } - rpm, err := parser.GetIntAnnotation(limitRateRPMAnnotation, ing, a.annotationConfig.Annotations) - if err != nil && errors.IsValidationError(err) { - return nil, err - } - rps, err := parser.GetIntAnnotation(limitRateRPSAnnotation, ing, a.annotationConfig.Annotations) - if err != nil && errors.IsValidationError(err) { - return nil, err - } - conn, err := parser.GetIntAnnotation(limitRateConnectionsAnnotation, ing, a.annotationConfig.Annotations) - if err != nil && errors.IsValidationError(err) { - return nil, err - } - burstMultiplier, err := parser.GetIntAnnotation(limitRateBurstMultiplierAnnotation, ing, a.annotationConfig.Annotations) + rpm, _ := parser.GetIntAnnotation("limit-rpm", ing) + rps, _ := parser.GetIntAnnotation("limit-rps", ing) + conn, _ := parser.GetIntAnnotation("limit-connections", ing) + burstMultiplier, err := parser.GetIntAnnotation("limit-burst-multiplier", ing) if err != nil { burstMultiplier = defBurst } - val, err := parser.GetStringAnnotation(limitAllowlistAnnotation, ing, a.annotationConfig.Annotations) - if err != nil && errors.IsValidationError(err) { - return nil, err - } + val, _ := parser.GetStringAnnotation("limit-whitelist", ing) - cidrs, errCidr := net.ParseCIDRs(val) - if errCidr != nil { - return nil, errCidr + cidrs, err := net.ParseCIDRs(val) + if err != nil { + return nil, err } if rpm == 0 && rps == 0 && conn == 0 { @@ -282,20 +203,11 @@ func (a ratelimit) Parse(ing *networking.Ingress) (interface{}, error) { LimitRateAfter: lra, Name: zoneName, ID: encode(zoneName), - Allowlist: cidrs, + Whitelist: cidrs, }, nil } func encode(s string) string { str := base64.URLEncoding.EncodeToString([]byte(s)) - return strings.ReplaceAll(str, "=", "") -} - -func (a ratelimit) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a ratelimit) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, rateLimitAnnotations.Annotations) + return strings.Replace(str, "=", "", -1) } diff --git a/internal/ingress/annotations/ratelimit/main_test.go b/internal/ingress/annotations/ratelimit/main_test.go index d3a2cc0e9..7ffbac3ff 100644 --- a/internal/ingress/annotations/ratelimit/main_test.go +++ b/internal/ingress/annotations/ratelimit/main_test.go @@ -20,23 +20,19 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/defaults" - "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" ) func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -45,13 +41,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -86,8 +78,8 @@ func (m mockBackend) GetDefaultBackend() defaults.Backend { func TestWithoutAnnotations(t *testing.T) { ing := buildIngress() _, err := NewParser(mockBackend{}).Parse(ing) - if err != nil && !errors.IsMissingAnnotations(err) { - t.Errorf("unexpected error with ingress without annotations: %s", err) + if err != nil { + t.Error("unexpected error with ingress without annotations") } } @@ -95,22 +87,22 @@ func TestRateLimiting(t *testing.T) { ing := buildIngress() data := map[string]string{} - data[parser.GetAnnotationWithPrefix(limitRateConnectionsAnnotation)] = "0" - data[parser.GetAnnotationWithPrefix(limitRateRPSAnnotation)] = "0" - data[parser.GetAnnotationWithPrefix(limitRateRPMAnnotation)] = "0" + data[parser.GetAnnotationWithPrefix("limit-connections")] = "0" + data[parser.GetAnnotationWithPrefix("limit-rps")] = "0" + data[parser.GetAnnotationWithPrefix("limit-rpm")] = "0" ing.SetAnnotations(data) _, err := NewParser(mockBackend{}).Parse(ing) if err != nil { - t.Errorf("unexpected error with invalid limits (0): %s", err) + t.Errorf("unexpected error with invalid limits (0)") } data = map[string]string{} - data[parser.GetAnnotationWithPrefix(limitRateConnectionsAnnotation)] = "5" - data[parser.GetAnnotationWithPrefix(limitRateRPSAnnotation)] = "100" - data[parser.GetAnnotationWithPrefix(limitRateRPMAnnotation)] = "10" - data[parser.GetAnnotationWithPrefix(limitRateAfterAnnotation)] = "100" - data[parser.GetAnnotationWithPrefix(limitRateAnnotation)] = "10" + data[parser.GetAnnotationWithPrefix("limit-connections")] = "5" + data[parser.GetAnnotationWithPrefix("limit-rps")] = "100" + data[parser.GetAnnotationWithPrefix("limit-rpm")] = "10" + data[parser.GetAnnotationWithPrefix("limit-rate-after")] = "100" + data[parser.GetAnnotationWithPrefix("limit-rate")] = "10" ing.SetAnnotations(data) @@ -148,12 +140,12 @@ func TestRateLimiting(t *testing.T) { } data = map[string]string{} - data[parser.GetAnnotationWithPrefix(limitRateConnectionsAnnotation)] = "5" - data[parser.GetAnnotationWithPrefix(limitRateRPSAnnotation)] = "100" - data[parser.GetAnnotationWithPrefix(limitRateRPMAnnotation)] = "10" - data[parser.GetAnnotationWithPrefix(limitRateAfterAnnotation)] = "100" - data[parser.GetAnnotationWithPrefix(limitRateAnnotation)] = "10" - data[parser.GetAnnotationWithPrefix(limitRateBurstMultiplierAnnotation)] = "3" + data[parser.GetAnnotationWithPrefix("limit-connections")] = "5" + data[parser.GetAnnotationWithPrefix("limit-rps")] = "100" + data[parser.GetAnnotationWithPrefix("limit-rpm")] = "10" + data[parser.GetAnnotationWithPrefix("limit-rate-after")] = "100" + data[parser.GetAnnotationWithPrefix("limit-rate")] = "10" + data[parser.GetAnnotationWithPrefix("limit-burst-multiplier")] = "3" ing.SetAnnotations(data) @@ -190,61 +182,3 @@ func TestRateLimiting(t *testing.T) { t.Errorf("expected 10 in limit by limitrate but %v was returned", rateLimit.LimitRate) } } - -func TestAnnotationCIDR(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix(limitRateConnectionsAnnotation)] = "5" - data[parser.GetAnnotationWithPrefix(limitAllowlistAnnotation)] = "192.168.0.5, 192.168.50.32/24" - ing.SetAnnotations(data) - - i, err := NewParser(mockBackend{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - - rateLimit, ok := i.(*Config) - if !ok { - t.Errorf("expected a RateLimit type") - } - if len(rateLimit.Allowlist) != 2 { - t.Errorf("expected 2 cidrs in limit by ip but %v was returned", len(rateLimit.Allowlist)) - } - - data = map[string]string{} - data[parser.GetAnnotationWithPrefix(limitRateConnectionsAnnotation)] = "5" - data[parser.GetAnnotationWithPrefix(limitWhitelistAnnotation)] = "192.168.0.5, 192.168.50.32/24, 10.10.10.1" - ing.SetAnnotations(data) - - i, err = NewParser(mockBackend{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - rateLimit, ok = i.(*Config) - if !ok { - t.Errorf("expected a RateLimit type") - } - if len(rateLimit.Allowlist) != 3 { - t.Errorf("expected 3 cidrs in limit by ip but %v was returned", len(rateLimit.Allowlist)) - } - - // Parent annotation surpasses any alias - data = map[string]string{} - data[parser.GetAnnotationWithPrefix(limitRateConnectionsAnnotation)] = "5" - data[parser.GetAnnotationWithPrefix(limitWhitelistAnnotation)] = "192.168.0.5, 192.168.50.32/24, 10.10.10.1" - data[parser.GetAnnotationWithPrefix(limitAllowlistAnnotation)] = "192.168.0.9" - ing.SetAnnotations(data) - - i, err = NewParser(mockBackend{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - rateLimit, ok = i.(*Config) - if !ok { - t.Errorf("expected a RateLimit type") - } - if len(rateLimit.Allowlist) != 1 { - t.Errorf("expected 1 cidrs in limit by ip but %v was returned", len(rateLimit.Allowlist)) - } -} diff --git a/internal/ingress/annotations/redirect/redirect.go b/internal/ingress/annotations/redirect/redirect.go index edc3d279c..02ee1d522 100644 --- a/internal/ingress/annotations/redirect/redirect.go +++ b/internal/ingress/annotations/redirect/redirect.go @@ -21,90 +21,29 @@ import ( "net/url" "strings" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - defaultPermanentRedirectCode = http.StatusMovedPermanently - defaultTemporalRedirectCode = http.StatusFound -) +const defaultPermanentRedirectCode = http.StatusMovedPermanently // Config returns the redirect configuration for an Ingress rule type Config struct { URL string `json:"url"` Code int `json:"code"` FromToWWW bool `json:"fromToWWW"` - Relative bool `json:"relative"` -} - -const ( - fromToWWWRedirAnnotation = "from-to-www-redirect" - temporalRedirectAnnotation = "temporal-redirect" - temporalRedirectAnnotationCode = "temporal-redirect-code" - permanentRedirectAnnotation = "permanent-redirect" - permanentRedirectAnnotationCode = "permanent-redirect-code" - relativeRedirectsAnnotation = "relative-redirects" -) - -var redirectAnnotations = parser.Annotation{ - Group: "redirect", - Annotations: parser.AnnotationFields{ - fromToWWWRedirAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, // Low, as it allows just a set of options - Documentation: `In some scenarios, it is required to redirect from www.domain.com to domain.com or vice versa, which way the redirect is performed depends on the configured host value in the Ingress object.`, - }, - temporalRedirectAnnotation: { - Validator: parser.ValidateRegex(parser.URLIsValidRegex, false), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, // Medium, as it allows arbitrary URLs that needs to be validated - Documentation: `This annotation allows you to return a temporal redirect (Return Code 302) instead of sending data to the upstream. - For example setting this annotation to https://www.google.com would redirect everything to Google with a Return Code of 302 (Moved Temporarily).`, - }, - temporalRedirectAnnotationCode: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, // Low, as it allows just a set of options - Documentation: `This annotation allows you to modify the status code used for temporal redirects.`, - }, - permanentRedirectAnnotation: { - Validator: parser.ValidateRegex(parser.URLIsValidRegex, false), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, // Medium, as it allows arbitrary URLs that needs to be validated - Documentation: `This annotation allows to return a permanent redirect (Return Code 301) instead of sending data to the upstream. - For example setting this annotation https://www.google.com would redirect everything to Google with a code 301`, - }, - permanentRedirectAnnotationCode: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, // Low, as it allows just a set of options - Documentation: `This annotation allows you to modify the status code used for permanent redirects.`, - }, - relativeRedirectsAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `If enabled, redirects issued by nginx will be relative. See https://nginx.org/en/docs/http/ngx_http_core_module.html#absolute_redirect`, - }, - }, } type redirect struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new redirect annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return redirect{ - r: r, - annotationConfig: redirectAnnotations, - } + return redirect{r} } // Parse parses the annotations contained in the ingress @@ -112,49 +51,31 @@ func NewParser(r resolver.Resolver) parser.IngressAnnotation { // If the Ingress contains both annotations the execution order is // temporal and then permanent func (r redirect) Parse(ing *networking.Ingress) (interface{}, error) { - r3w, err := parser.GetBoolAnnotation(fromToWWWRedirAnnotation, ing, r.annotationConfig.Annotations) - if err != nil && !errors.IsMissingAnnotations(err) { - return nil, err - } + r3w, _ := parser.GetBoolAnnotation("from-to-www-redirect", ing) - rr, err := parser.GetBoolAnnotation(relativeRedirectsAnnotation, ing, r.annotationConfig.Annotations) - if err != nil && !errors.IsMissingAnnotations(err) { - return nil, err - } - - tr, err := parser.GetStringAnnotation(temporalRedirectAnnotation, ing, r.annotationConfig.Annotations) + tr, err := parser.GetStringAnnotation("temporal-redirect", ing) if err != nil && !errors.IsMissingAnnotations(err) { return nil, err } if tr != "" { - trc, err := parser.GetIntAnnotation(temporalRedirectAnnotationCode, ing, r.annotationConfig.Annotations) - if err != nil && !errors.IsMissingAnnotations(err) { - return nil, err - } - - if trc < http.StatusMultipleChoices || trc > http.StatusTemporaryRedirect { - trc = defaultTemporalRedirectCode - } - if err := isValidURL(tr); err != nil { return nil, err } return &Config{ URL: tr, - Code: trc, + Code: http.StatusFound, FromToWWW: r3w, - Relative: rr, }, nil } - pr, err := parser.GetStringAnnotation(permanentRedirectAnnotation, ing, r.annotationConfig.Annotations) + pr, err := parser.GetStringAnnotation("permanent-redirect", ing) if err != nil && !errors.IsMissingAnnotations(err) { return nil, err } - prc, err := parser.GetIntAnnotation(permanentRedirectAnnotationCode, ing, r.annotationConfig.Annotations) + prc, err := parser.GetIntAnnotation("permanent-redirect-code", ing) if err != nil && !errors.IsMissingAnnotations(err) { return nil, err } @@ -168,13 +89,6 @@ func (r redirect) Parse(ing *networking.Ingress) (interface{}, error) { URL: pr, Code: prc, FromToWWW: r3w, - Relative: rr, - }, nil - } - - if rr { - return &Config{ - Relative: rr, }, nil } @@ -198,9 +112,6 @@ func (r1 *Config) Equal(r2 *Config) bool { if r1.FromToWWW != r2.FromToWWW { return false } - if r1.Relative != r2.Relative { - return false - } return true } @@ -216,12 +127,3 @@ func isValidURL(s string) error { return nil } - -func (r redirect) GetDocumentation() parser.AnnotationFields { - return r.annotationConfig.Annotations -} - -func (r redirect) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(r.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, redirectAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/redirect/redirect_test.go b/internal/ingress/annotations/redirect/redirect_test.go index f4734ae5b..b9bda6688 100644 --- a/internal/ingress/annotations/redirect/redirect_test.go +++ b/internal/ingress/annotations/redirect/redirect_test.go @@ -23,7 +23,7 @@ import ( "strconv" "testing" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/errors" @@ -43,7 +43,7 @@ func TestPermanentRedirectWithDefaultCode(t *testing.T) { ing := new(networking.Ingress) data := make(map[string]string, 1) - data[parser.GetAnnotationWithPrefix(permanentRedirectAnnotation)] = defRedirectURL + data[parser.GetAnnotationWithPrefix("permanent-redirect")] = defRedirectURL ing.SetAnnotations(data) i, err := rp.Parse(ing) @@ -81,8 +81,8 @@ func TestPermanentRedirectWithCustomCode(t *testing.T) { ing := new(networking.Ingress) data := make(map[string]string, 2) - data[parser.GetAnnotationWithPrefix(permanentRedirectAnnotation)] = defRedirectURL - data[parser.GetAnnotationWithPrefix(permanentRedirectAnnotationCode)] = strconv.Itoa(tc.input) + data[parser.GetAnnotationWithPrefix("permanent-redirect")] = defRedirectURL + data[parser.GetAnnotationWithPrefix("permanent-redirect-code")] = strconv.Itoa(tc.input) ing.SetAnnotations(data) i, err := rp.Parse(ing) @@ -103,7 +103,7 @@ func TestPermanentRedirectWithCustomCode(t *testing.T) { } } -func TestTemporalRedirectWithDefaultCode(t *testing.T) { +func TestTemporalRedirect(t *testing.T) { rp := NewParser(resolver.Mock{}) if rp == nil { t.Fatalf("Expected a parser.IngressAnnotation but returned nil") @@ -112,8 +112,8 @@ func TestTemporalRedirectWithDefaultCode(t *testing.T) { ing := new(networking.Ingress) data := make(map[string]string, 1) - data[parser.GetAnnotationWithPrefix(fromToWWWRedirAnnotation)] = "true" - data[parser.GetAnnotationWithPrefix(temporalRedirectAnnotation)] = defRedirectURL + data[parser.GetAnnotationWithPrefix("from-to-www-redirect")] = "true" + data[parser.GetAnnotationWithPrefix("temporal-redirect")] = defRedirectURL ing.SetAnnotations(data) i, err := rp.Parse(ing) @@ -128,53 +128,15 @@ func TestTemporalRedirectWithDefaultCode(t *testing.T) { t.Errorf("Expected %v as redirect but returned %s", defRedirectURL, redirect.URL) } if redirect.Code != http.StatusFound { - t.Errorf("Expected %v as redirect to have a code %d but had %d", defRedirectURL, http.StatusFound, redirect.Code) + t.Errorf("Expected %v as redirect to have a code %d but had %d", defRedirectURL, defaultPermanentRedirectCode, redirect.Code) } -} - -func TestTemporalRedirectWithCustomCode(t *testing.T) { - rp := NewParser(resolver.Mock{}) - if rp == nil { - t.Fatalf("Expected a parser.IngressAnnotation but returned nil") - } - - testCases := map[string]struct { - input int - expectOutput int - }{ - "valid code": {http.StatusTemporaryRedirect, http.StatusTemporaryRedirect}, - "invalid code": {http.StatusTeapot, http.StatusFound}, - } - - for n, tc := range testCases { - t.Run(n, func(t *testing.T) { - ing := new(networking.Ingress) - - data := make(map[string]string, 2) - data[parser.GetAnnotationWithPrefix(fromToWWWRedirAnnotation)] = "true" - data[parser.GetAnnotationWithPrefix(temporalRedirectAnnotation)] = defRedirectURL - data[parser.GetAnnotationWithPrefix(temporalRedirectAnnotationCode)] = strconv.Itoa(tc.input) - ing.SetAnnotations(data) - - i, err := rp.Parse(ing) - if err != nil { - t.Errorf("Unexpected error with ingress: %v", err) - } - redirect, ok := i.(*Config) - if !ok { - t.Errorf("Expected a Redirect type") - } - if redirect.URL != defRedirectURL { - t.Errorf("Expected %v as redirect but returned %s", defRedirectURL, redirect.URL) - } - if redirect.Code != tc.expectOutput { - t.Errorf("Expected %v as redirect to have a code %d but had %d", defRedirectURL, tc.expectOutput, redirect.Code) - } - }) + if redirect.FromToWWW != true { + t.Errorf("Expected %v as redirect to have from-to-www as %v but got %v", defRedirectURL, true, redirect.FromToWWW) } } func TestIsValidURL(t *testing.T) { + invalid := "ok.com" urlParse, err := url.Parse(invalid) if err != nil { @@ -193,22 +155,3 @@ func TestIsValidURL(t *testing.T) { t.Errorf("expected nil but got %v", err) } } - -func TestParseAnnotations(t *testing.T) { - ing := new(networking.Ingress) - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix(relativeRedirectsAnnotation)] = "true" - ing.SetAnnotations(data) - - _, err := NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - - // test ingress using the annotation without a TLS section - _, err = NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error parsing ingress with relative-redirects") - } -} diff --git a/internal/ingress/annotations/rewrite/main.go b/internal/ingress/annotations/rewrite/main.go index d78a004b9..902f00f4c 100644 --- a/internal/ingress/annotations/rewrite/main.go +++ b/internal/ingress/annotations/rewrite/main.go @@ -19,7 +19,7 @@ package rewrite import ( "net/url" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/klog/v2" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" @@ -27,59 +27,6 @@ import ( "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - rewriteTargetAnnotation = "rewrite-target" - sslRedirectAnnotation = "ssl-redirect" - preserveTrailingSlashAnnotation = "preserve-trailing-slash" - forceSSLRedirectAnnotation = "force-ssl-redirect" - useRegexAnnotation = "use-regex" - appRootAnnotation = "app-root" -) - -var rewriteAnnotations = parser.Annotation{ - Group: "rewrite", - Annotations: parser.AnnotationFields{ - rewriteTargetAnnotation: { - Validator: parser.ValidateRegex(parser.RegexPathWithCapture, false), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation allows to specify the target URI where the traffic must be redirected. It can contain regular characters and captured - groups specified as '$1', '$2', etc.`, - }, - sslRedirectAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation defines if the location section is only accessible via SSL`, - }, - preserveTrailingSlashAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation defines if the trailing slash should be preserved in the URI with 'ssl-redirect'`, - }, - forceSSLRedirectAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation forces the redirection to HTTPS even if the Ingress is not TLS Enabled`, - }, - useRegexAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation defines if the paths defined on an Ingress use regular expressions. To use regex on path - the pathType should also be defined as 'ImplementationSpecific'.`, - }, - appRootAnnotation: { - Validator: parser.ValidateRegex(parser.RegexPathWithCapture, false), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation defines the Application Root that the Controller must redirect if it's in / context`, - }, - }, -} - // Config describes the per location redirect config type Config struct { // Target URI where the traffic must be redirected @@ -124,16 +71,12 @@ func (r1 *Config) Equal(r2 *Config) bool { } type rewrite struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new rewrite annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return rewrite{ - r: r, - annotationConfig: rewriteAnnotations, - } + return rewrite{r} } // ParseAnnotations parses the annotations contained in the ingress @@ -142,45 +85,24 @@ func (a rewrite) Parse(ing *networking.Ingress) (interface{}, error) { var err error config := &Config{} - config.Target, err = parser.GetStringAnnotation(rewriteTargetAnnotation, ing, a.annotationConfig.Annotations) + config.Target, _ = parser.GetStringAnnotation("rewrite-target", ing) + config.SSLRedirect, err = parser.GetBoolAnnotation("ssl-redirect", ing) if err != nil { - if errors.IsValidationError(err) { - klog.Warningf("%sis invalid, defaulting to empty", rewriteTargetAnnotation) - } - config.Target = "" - } - config.SSLRedirect, err = parser.GetBoolAnnotation(sslRedirectAnnotation, ing, a.annotationConfig.Annotations) - if err != nil { - if errors.IsValidationError(err) { - klog.Warningf("%s is invalid, defaulting to '%t'", sslRedirectAnnotation, a.r.GetDefaultBackend().SSLRedirect) - } config.SSLRedirect = a.r.GetDefaultBackend().SSLRedirect } - config.PreserveTrailingSlash, err = parser.GetBoolAnnotation(preserveTrailingSlashAnnotation, ing, a.annotationConfig.Annotations) + config.PreserveTrailingSlash, err = parser.GetBoolAnnotation("preserve-trailing-slash", ing) if err != nil { - if errors.IsValidationError(err) { - klog.Warningf("%s is invalid, defaulting to '%t'", preserveTrailingSlashAnnotation, a.r.GetDefaultBackend().PreserveTrailingSlash) - } config.PreserveTrailingSlash = a.r.GetDefaultBackend().PreserveTrailingSlash } - config.ForceSSLRedirect, err = parser.GetBoolAnnotation(forceSSLRedirectAnnotation, ing, a.annotationConfig.Annotations) + config.ForceSSLRedirect, err = parser.GetBoolAnnotation("force-ssl-redirect", ing) if err != nil { - if errors.IsValidationError(err) { - klog.Warningf("%s is invalid, defaulting to '%t'", forceSSLRedirectAnnotation, a.r.GetDefaultBackend().ForceSSLRedirect) - } config.ForceSSLRedirect = a.r.GetDefaultBackend().ForceSSLRedirect } - config.UseRegex, err = parser.GetBoolAnnotation(useRegexAnnotation, ing, a.annotationConfig.Annotations) - if err != nil { - if errors.IsValidationError(err) { - klog.Warningf("%s is invalid, defaulting to 'false'", useRegexAnnotation) - } - config.UseRegex = false - } + config.UseRegex, _ = parser.GetBoolAnnotation("use-regex", ing) - config.AppRoot, err = parser.GetStringAnnotation(appRootAnnotation, ing, a.annotationConfig.Annotations) + config.AppRoot, err = parser.GetStringAnnotation("app-root", ing) if err != nil { if !errors.IsMissingAnnotations(err) && !errors.IsInvalidContent(err) { klog.Warningf("Annotation app-root contains an invalid value: %v", err) @@ -204,12 +126,3 @@ func (a rewrite) Parse(ing *networking.Ingress) (interface{}, error) { return config, nil } - -func (a rewrite) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a rewrite) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, rewriteAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/rewrite/main_test.go b/internal/ingress/annotations/rewrite/main_test.go index b68b901b4..beece494a 100644 --- a/internal/ingress/annotations/rewrite/main_test.go +++ b/internal/ingress/annotations/rewrite/main_test.go @@ -20,8 +20,9 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/defaults" @@ -34,12 +35,8 @@ const ( func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -49,13 +46,9 @@ func buildIngress() *networking.Ingress { Annotations: map[string]string{}, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -120,10 +113,7 @@ func TestSSLRedirect(t *testing.T) { data[parser.GetAnnotationWithPrefix("rewrite-target")] = defRoute ing.SetAnnotations(data) - i, err := NewParser(mockBackend{redirect: true}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } + i, _ := NewParser(mockBackend{redirect: true}).Parse(ing) redirect, ok := i.(*Config) if !ok { t.Errorf("expected a Redirect type") @@ -132,43 +122,10 @@ func TestSSLRedirect(t *testing.T) { t.Errorf("Expected true but returned false") } - data[parser.GetAnnotationWithPrefix("rewrite-target")] = "/xpto/$1/abc/$2" - ing.SetAnnotations(data) - - i, err = NewParser(mockBackend{redirect: true}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - redirect, ok = i.(*Config) - if !ok { - t.Errorf("expected a Redirect type") - } - if redirect.Target != "/xpto/$1/abc/$2" { - t.Errorf("Expected /xpto/$1/abc/$2 but returned %s", redirect.Target) - } - - data[parser.GetAnnotationWithPrefix("rewrite-target")] = "/xpto/xas{445}" - ing.SetAnnotations(data) - - i, err = NewParser(mockBackend{redirect: true}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - redirect, ok = i.(*Config) - if !ok { - t.Errorf("expected a Redirect type") - } - if redirect.Target != "" { - t.Errorf("Expected empty rewrite target but returned %s", redirect.Target) - } - data[parser.GetAnnotationWithPrefix("ssl-redirect")] = "false" ing.SetAnnotations(data) - i, err = NewParser(mockBackend{redirect: false}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } + i, _ = NewParser(mockBackend{redirect: false}).Parse(ing) redirect, ok = i.(*Config) if !ok { t.Errorf("expected a Redirect type") @@ -185,10 +142,7 @@ func TestForceSSLRedirect(t *testing.T) { data[parser.GetAnnotationWithPrefix("rewrite-target")] = defRoute ing.SetAnnotations(data) - i, err := NewParser(mockBackend{redirect: true}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } + i, _ := NewParser(mockBackend{redirect: true}).Parse(ing) redirect, ok := i.(*Config) if !ok { t.Errorf("expected a Redirect type") @@ -200,10 +154,7 @@ func TestForceSSLRedirect(t *testing.T) { data[parser.GetAnnotationWithPrefix("force-ssl-redirect")] = "true" ing.SetAnnotations(data) - i, err = NewParser(mockBackend{redirect: false}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } + i, _ = NewParser(mockBackend{redirect: false}).Parse(ing) redirect, ok = i.(*Config) if !ok { t.Errorf("expected a Redirect type") @@ -212,7 +163,6 @@ func TestForceSSLRedirect(t *testing.T) { t.Errorf("Expected true but returned false") } } - func TestAppRoot(t *testing.T) { ap := NewParser(mockBackend{redirect: true}) @@ -260,10 +210,7 @@ func TestUseRegex(t *testing.T) { data[parser.GetAnnotationWithPrefix("use-regex")] = "true" ing.SetAnnotations(data) - i, err := NewParser(mockBackend{redirect: true}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } + i, _ := NewParser(mockBackend{redirect: true}).Parse(ing) redirect, ok := i.(*Config) if !ok { t.Errorf("expected a App Context") diff --git a/internal/ingress/annotations/satisfy/main.go b/internal/ingress/annotations/satisfy/main.go index 13a4532c1..a064bdf96 100644 --- a/internal/ingress/annotations/satisfy/main.go +++ b/internal/ingress/annotations/satisfy/main.go @@ -17,46 +17,24 @@ limitations under the License. package satisfy import ( - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - satisfyAnnotation = "satisfy" -) - -var satisfyAnnotations = parser.Annotation{ - Group: "authentication", - Annotations: parser.AnnotationFields{ - satisfyAnnotation: { - Validator: parser.ValidateOptions([]string{"any", "all"}, true, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `By default, a request would need to satisfy all authentication requirements in order to be allowed. - By using this annotation, requests that satisfy either any or all authentication requirements are allowed, based on the configuration value. - Valid options are "all" and "any"`, - }, - }, -} - type satisfy struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new SATISFY annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return satisfy{ - r: r, - annotationConfig: satisfyAnnotations, - } + return satisfy{r} } // Parse parses annotation contained in the ingress func (s satisfy) Parse(ing *networking.Ingress) (interface{}, error) { - satisfy, err := parser.GetStringAnnotation(satisfyAnnotation, ing, s.annotationConfig.Annotations) + satisfy, err := parser.GetStringAnnotation("satisfy", ing) if err != nil || (satisfy != "any" && satisfy != "all") { satisfy = "" @@ -64,12 +42,3 @@ func (s satisfy) Parse(ing *networking.Ingress) (interface{}, error) { return satisfy, nil } - -func (s satisfy) GetDocumentation() parser.AnnotationFields { - return s.annotationConfig.Annotations -} - -func (s satisfy) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(s.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, satisfyAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/satisfy/main_test.go b/internal/ingress/annotations/satisfy/main_test.go index c8d5782f9..a3475316a 100644 --- a/internal/ingress/annotations/satisfy/main_test.go +++ b/internal/ingress/annotations/satisfy/main_test.go @@ -20,21 +20,18 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -43,13 +40,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -83,7 +76,7 @@ func TestSatisfyParser(t *testing.T) { annotations := map[string]string{} for input, expected := range data { - annotations[parser.GetAnnotationWithPrefix(satisfyAnnotation)] = input + annotations[parser.GetAnnotationWithPrefix("satisfy")] = input ing.SetAnnotations(annotations) satisfyt, err := NewParser(&resolver.Mock{}).Parse(ing) diff --git a/internal/ingress/annotations/secureupstream/main.go b/internal/ingress/annotations/secureupstream/main.go new file mode 100644 index 000000000..631804464 --- /dev/null +++ b/internal/ingress/annotations/secureupstream/main.go @@ -0,0 +1,48 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package secureupstream + +import ( + networking "k8s.io/api/networking/v1beta1" + "k8s.io/klog/v2" + + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "k8s.io/ingress-nginx/internal/ingress/resolver" +) + +// Config describes SSL backend configuration +type Config struct { + CACert resolver.AuthSSLCert `json:"caCert"` +} + +type su struct { + r resolver.Resolver +} + +// NewParser creates a new secure upstream annotation parser +func NewParser(r resolver.Resolver) parser.IngressAnnotation { + return su{r} +} + +// Parse parses the annotations contained in the ingress +// rule used to indicate if the upstream servers should use SSL +func (a su) Parse(ing *networking.Ingress) (secure interface{}, err error) { + if ca, _ := parser.GetStringAnnotation("secure-verify-ca-secret", ing); ca != "" { + klog.Warningf("NOTE! secure-verify-ca-secret is not supported anymore. Please use proxy-ssl-secret instead") + } + return +} diff --git a/internal/ingress/annotations/secureupstream/main_test.go b/internal/ingress/annotations/secureupstream/main_test.go new file mode 100644 index 000000000..508d54a84 --- /dev/null +++ b/internal/ingress/annotations/secureupstream/main_test.go @@ -0,0 +1,155 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package secureupstream + +import ( + "fmt" + "testing" + + api "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1beta1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "k8s.io/ingress-nginx/internal/ingress/resolver" +) + +func buildIngress() *networking.Ingress { + defaultBackend := networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), + } + + return &networking.Ingress{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "foo", + Namespace: api.NamespaceDefault, + }, + Spec: networking.IngressSpec{ + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), + }, + Rules: []networking.IngressRule{ + { + Host: "foo.bar.com", + IngressRuleValue: networking.IngressRuleValue{ + HTTP: &networking.HTTPIngressRuleValue{ + Paths: []networking.HTTPIngressPath{ + { + Path: "/foo", + Backend: defaultBackend, + }, + }, + }, + }, + }, + }, + }, + } +} + +type mockCfg struct { + resolver.Mock + certs map[string]resolver.AuthSSLCert +} + +func (cfg mockCfg) GetAuthCertificate(secret string) (*resolver.AuthSSLCert, error) { + if cert, ok := cfg.certs[secret]; ok { + return &cert, nil + } + return nil, fmt.Errorf("secret not found: %v", secret) +} + +func TestNoCA(t *testing.T) { + ing := buildIngress() + data := map[string]string{} + data[parser.GetAnnotationWithPrefix("backend-protocol")] = "HTTPS" + ing.SetAnnotations(data) + + _, err := NewParser(mockCfg{ + certs: map[string]resolver.AuthSSLCert{ + "default/secure-verify-ca": {}, + }, + }).Parse(ing) + if err != nil { + t.Errorf("Unexpected error on ingress: %v", err) + } +} + +func TestAnnotations(t *testing.T) { + ing := buildIngress() + data := map[string]string{} + data[parser.GetAnnotationWithPrefix("backend-protocol")] = "HTTPS" + data[parser.GetAnnotationWithPrefix("secure-verify-ca-secret")] = "secure-verify-ca" + ing.SetAnnotations(data) + + _, err := NewParser(mockCfg{ + certs: map[string]resolver.AuthSSLCert{ + "default/secure-verify-ca": {}, + }, + }).Parse(ing) + if err != nil { + t.Errorf("Unexpected error on ingress: %v", err) + } +} + +func TestSecretNotFound(t *testing.T) { + ing := buildIngress() + data := map[string]string{} + data[parser.GetAnnotationWithPrefix("backend-protocol")] = "HTTPS" + data[parser.GetAnnotationWithPrefix("secure-verify-ca-secret")] = "secure-verify-ca" + ing.SetAnnotations(data) + _, err := NewParser(mockCfg{}).Parse(ing) + if err != nil { + t.Error("Expected secret not found error on ingress") + } +} + +func TestSecretOnNonSecure(t *testing.T) { + ing := buildIngress() + data := map[string]string{} + data[parser.GetAnnotationWithPrefix("backend-protocol")] = "HTTP" + data[parser.GetAnnotationWithPrefix("secure-verify-ca-secret")] = "secure-verify-ca" + ing.SetAnnotations(data) + _, err := NewParser(mockCfg{ + certs: map[string]resolver.AuthSSLCert{ + "default/secure-verify-ca": {}, + }, + }).Parse(ing) + if err != nil { + t.Error("Expected CA secret on non secure backend error on ingress") + } +} + +func TestUnsupportedAnnotation(t *testing.T) { + ing := buildIngress() + data := map[string]string{} + data[parser.GetAnnotationWithPrefix("backend-protocol")] = "HTTPS" + data[parser.GetAnnotationWithPrefix("secure-verify-ca-secret")] = "secure-verify-ca" + ing.SetAnnotations(data) + + _, err := NewParser(mockCfg{ + certs: map[string]resolver.AuthSSLCert{ + "default/secure-verify-ca": {}, + }, + }).Parse(ing) + if err != nil { + t.Errorf("Unexpected error on ingress: %v", err) + } +} diff --git a/internal/ingress/annotations/serversnippet/main.go b/internal/ingress/annotations/serversnippet/main.go index bce764e69..33a672650 100644 --- a/internal/ingress/annotations/serversnippet/main.go +++ b/internal/ingress/annotations/serversnippet/main.go @@ -17,53 +17,24 @@ limitations under the License. package serversnippet import ( - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - serverSnippetAnnotation = "server-snippet" -) - -var serverSnippetAnnotations = parser.Annotation{ - Group: "snippets", - Annotations: parser.AnnotationFields{ - serverSnippetAnnotation: { - Validator: parser.ValidateNull, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskCritical, // Critical, this annotation is not validated at all and allows arbitrary configurations - Documentation: `This annotation allows setting a custom NGINX configuration on a server block. This annotation does not contain any validation and it's usage is not recommended!`, - }, - }, -} - type serverSnippet struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new server snippet annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return serverSnippet{ - r: r, - annotationConfig: serverSnippetAnnotations, - } + return serverSnippet{r} } // Parse parses the annotations contained in the ingress rule // used to indicate if the location/s contains a fragment of // configuration to be included inside the paths of the rules func (a serverSnippet) Parse(ing *networking.Ingress) (interface{}, error) { - return parser.GetStringAnnotation(serverSnippetAnnotation, ing, a.annotationConfig.Annotations) -} - -func (a serverSnippet) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a serverSnippet) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, serverSnippetAnnotations.Annotations) + return parser.GetStringAnnotation("server-snippet", ing) } diff --git a/internal/ingress/annotations/serversnippet/main_test.go b/internal/ingress/annotations/serversnippet/main_test.go index 72e757dc1..066334f69 100644 --- a/internal/ingress/annotations/serversnippet/main_test.go +++ b/internal/ingress/annotations/serversnippet/main_test.go @@ -20,14 +20,14 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) func TestParse(t *testing.T) { - annotation := parser.GetAnnotationWithPrefix(serverSnippetAnnotation) + annotation := parser.GetAnnotationWithPrefix("server-snippet") ap := NewParser(&resolver.Mock{}) if ap == nil { @@ -54,7 +54,6 @@ func TestParse(t *testing.T) { for _, testCase := range testCases { ing.SetAnnotations(testCase.annotations) - //nolint:errcheck // Ignore the error since invalid cases will be checked with expected results result, _ := ap.Parse(ing) if result != testCase.expected { t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations) diff --git a/internal/ingress/annotations/serviceupstream/main.go b/internal/ingress/annotations/serviceupstream/main.go index 7819d87d8..ff90f8160 100644 --- a/internal/ingress/annotations/serviceupstream/main.go +++ b/internal/ingress/annotations/serviceupstream/main.go @@ -17,59 +17,21 @@ limitations under the License. package serviceupstream import ( - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - serviceUpstreamAnnotation = "service-upstream" -) - -var serviceUpstreamAnnotations = parser.Annotation{ - Group: "backend", - Annotations: parser.AnnotationFields{ - serviceUpstreamAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, // Critical, this annotation is not validated at all and allows arbitrary configurations - Documentation: `This annotation makes NGINX use Service's Cluster IP and Port instead of Endpoints as the backend endpoints`, - }, - }, -} - type serviceUpstream struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new serviceUpstream annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return serviceUpstream{ - r: r, - annotationConfig: serviceUpstreamAnnotations, - } + return serviceUpstream{r} } func (s serviceUpstream) Parse(ing *networking.Ingress) (interface{}, error) { - defBackend := s.r.GetDefaultBackend() - - val, err := parser.GetBoolAnnotation(serviceUpstreamAnnotation, ing, s.annotationConfig.Annotations) - // A missing annotation is not a problem, just use the default - if err == errors.ErrMissingAnnotations { - return defBackend.ServiceUpstream, nil - } - - return val, nil -} - -func (s serviceUpstream) GetDocumentation() parser.AnnotationFields { - return s.annotationConfig.Annotations -} - -func (s serviceUpstream) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(s.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, serviceUpstreamAnnotations.Annotations) + return parser.GetBoolAnnotation("service-upstream", ing) } diff --git a/internal/ingress/annotations/serviceupstream/main_test.go b/internal/ingress/annotations/serviceupstream/main_test.go index e6216ee5a..c7f44598e 100644 --- a/internal/ingress/annotations/serviceupstream/main_test.go +++ b/internal/ingress/annotations/serviceupstream/main_test.go @@ -20,21 +20,17 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/defaults" "k8s.io/ingress-nginx/internal/ingress/resolver" ) func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -43,13 +39,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -74,13 +66,10 @@ func TestIngressAnnotationServiceUpstreamEnabled(t *testing.T) { ing := buildIngress() data := map[string]string{} - data[parser.GetAnnotationWithPrefix(serviceUpstreamAnnotation)] = "true" + data[parser.GetAnnotationWithPrefix("service-upstream")] = "true" ing.SetAnnotations(data) - val, err := NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } + val, _ := NewParser(&resolver.Mock{}).Parse(ing) enabled, ok := val.(bool) if !ok { t.Errorf("expected a bool type") @@ -96,13 +85,10 @@ func TestIngressAnnotationServiceUpstreamSetFalse(t *testing.T) { // Test with explicitly set to false data := map[string]string{} - data[parser.GetAnnotationWithPrefix(serviceUpstreamAnnotation)] = "false" + data[parser.GetAnnotationWithPrefix("service-upstream")] = "false" ing.SetAnnotations(data) - val, err := NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } + val, _ := NewParser(&resolver.Mock{}).Parse(ing) enabled, ok := val.(bool) if !ok { t.Errorf("expected a bool type") @@ -116,10 +102,7 @@ func TestIngressAnnotationServiceUpstreamSetFalse(t *testing.T) { data = map[string]string{} ing.SetAnnotations(data) - val, err = NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } + val, _ = NewParser(&resolver.Mock{}).Parse(ing) enabled, ok = val.(bool) if !ok { t.Errorf("expected a bool type") @@ -129,58 +112,3 @@ func TestIngressAnnotationServiceUpstreamSetFalse(t *testing.T) { t.Errorf("expected annotation value to be false, got true") } } - -type mockBackend struct { - resolver.Mock -} - -// GetDefaultBackend returns the backend that must be used as default -func (m mockBackend) GetDefaultBackend() defaults.Backend { - return defaults.Backend{ - ServiceUpstream: true, - } -} - -// Test that when we have a default configuration set on the Backend that is used -// when we don't have the annotation -func TestParseAnnotationsWithDefaultConfig(t *testing.T) { - ing := buildIngress() - - val, err := NewParser(mockBackend{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - enabled, ok := val.(bool) - - if !ok { - t.Errorf("expected a bool type") - } - - if !enabled { - t.Errorf("expected annotation value to be true, got false") - } -} - -// Test that the annotation will disable the service upstream when enabled -// in the default configuration -func TestParseAnnotationsOverridesDefaultConfig(t *testing.T) { - ing := buildIngress() - - data := map[string]string{} - data[parser.GetAnnotationWithPrefix(serviceUpstreamAnnotation)] = "false" - ing.SetAnnotations(data) - - val, err := NewParser(mockBackend{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - enabled, ok := val.(bool) - - if !ok { - t.Errorf("expected a bool type") - } - - if enabled { - t.Errorf("expected annotation value to be false, got true") - } -} diff --git a/internal/ingress/annotations/sessionaffinity/main.go b/internal/ingress/annotations/sessionaffinity/main.go index bee4a2094..c5b340ec7 100644 --- a/internal/ingress/annotations/sessionaffinity/main.go +++ b/internal/ingress/annotations/sessionaffinity/main.go @@ -19,7 +19,7 @@ package sessionaffinity import ( "regexp" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/klog/v2" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" @@ -27,20 +27,14 @@ import ( ) const ( - annotationAffinityType = "affinity" - annotationAffinityMode = "affinity-mode" - annotationAffinityCanaryBehavior = "affinity-canary-behavior" - + annotationAffinityType = "affinity" + annotationAffinityMode = "affinity-mode" // If a cookie with this name exists, // its value is used as an index into the list of available backends. annotationAffinityCookieName = "session-cookie-name" defaultAffinityCookieName = "INGRESSCOOKIE" - // This is used to force the Secure flag on the cookie even if the - // incoming request is not secured. (https://github.com/kubernetes/ingress-nginx/issues/6812) - annotationAffinityCookieSecure = "session-cookie-secure" - // This is used to control the cookie expires, its value is a number of seconds until the // cookie expires annotationAffinityCookieExpires = "session-cookie-expires" @@ -52,9 +46,6 @@ const ( // This is used to control the cookie path when use-regex is set to true annotationAffinityCookiePath = "session-cookie-path" - // This is used to control the cookie Domain - annotationAffinityCookieDomain = "session-cookie-domain" - // This is used to control the SameSite attribute of the cookie annotationAffinityCookieSameSite = "session-cookie-samesite" @@ -63,95 +54,11 @@ const ( // This is used to control the cookie change after request failure annotationAffinityCookieChangeOnFailure = "session-cookie-change-on-failure" - - cookieAffinity = "cookie" ) -var sessionAffinityAnnotations = parser.Annotation{ - Group: "affinity", - Annotations: parser.AnnotationFields{ - annotationAffinityType: { - Validator: parser.ValidateOptions([]string{cookieAffinity}, true, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation enables and sets the affinity type in all Upstreams of an Ingress. This way, a request will always be directed to the same upstream server. The only affinity type available for NGINX is cookie`, - }, - annotationAffinityMode: { - Validator: parser.ValidateOptions([]string{"balanced", "persistent"}, true, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation defines the stickiness of a session. - Setting this to balanced (default) will redistribute some sessions if a deployment gets scaled up, therefore rebalancing the load on the servers. - Setting this to persistent will not rebalance sessions to new servers, therefore providing maximum stickiness.`, - }, - annotationAffinityCanaryBehavior: { - Validator: parser.ValidateOptions([]string{"sticky", "legacy"}, true, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation defines the behavior of canaries when session affinity is enabled. - Setting this to sticky (default) will ensure that users that were served by canaries, will continue to be served by canaries. - Setting this to legacy will restore original canary behavior, when session affinity was ignored.`, - }, - annotationAffinityCookieName: { - Validator: parser.ValidateRegex(parser.BasicCharsRegex, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation allows to specify the name of the cookie that will be used to route the requests`, - }, - annotationAffinityCookieSecure: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation set the cookie as secure regardless the protocol of the incoming request`, - }, - annotationAffinityCookieExpires: { - Validator: parser.ValidateRegex(affinityCookieExpiresRegex, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation is a legacy version of "session-cookie-max-age" for compatibility with older browsers, generates an "Expires" cookie directive by adding the seconds to the current date`, - }, - annotationAffinityCookieMaxAge: { - Validator: parser.ValidateRegex(affinityCookieExpiresRegex, false), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation sets the time until the cookie expires`, - }, - annotationAffinityCookiePath: { - Validator: parser.ValidateRegex(parser.URLIsValidRegex, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation defines the Path that will be set on the cookie (required if your Ingress paths use regular expressions)`, - }, - annotationAffinityCookieDomain: { - Validator: parser.ValidateRegex(parser.BasicCharsRegex, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation defines the Domain attribute of the sticky cookie.`, - }, - annotationAffinityCookieSameSite: { - Validator: parser.ValidateOptions([]string{"none", "lax", "strict"}, false, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation is used to apply a SameSite attribute to the sticky cookie. - Browser accepted values are None, Lax, and Strict`, - }, - annotationAffinityCookieConditionalSameSiteNone: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation is used to omit SameSite=None from browsers with SameSite attribute incompatibilities`, - }, - annotationAffinityCookieChangeOnFailure: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation, when set to false will send request to upstream pointed by sticky cookie even if previous attempt failed. - When set to true and previous attempt failed, sticky cookie will be changed to point to another upstream.`, - }, - }, -} - -var affinityCookieExpiresRegex = regexp.MustCompile(`(^0|-?[1-9]\d*$)`) +var ( + affinityCookieExpiresRegex = regexp.MustCompile(`(^0|-?[1-9]\d*$)`) +) // Config describes the per ingress session affinity config type Config struct { @@ -159,8 +66,6 @@ type Config struct { Type string `json:"type"` // The affinity mode, i.e. how sticky a session is Mode string `json:"mode"` - // Affinity behavior for canaries (sticky or legacy) - CanaryBehavior string `json:"canaryBehavior"` Cookie } @@ -174,23 +79,14 @@ type Cookie struct { MaxAge string `json:"maxage"` // The path that a cookie will be set on Path string `json:"path"` - // The domain that a cookie will be set on - Domain string `json:"domain"` // Flag that allows cookie regeneration on request failure ChangeOnFailure bool `json:"changeonfailure"` - // Secure flag to be set - Secure bool `json:"secure"` // SameSite attribute value SameSite string `json:"samesite"` // Flag that conditionally applies SameSite=None attribute on cookie if user agent accepts it. ConditionalSameSiteNone bool `json:"conditional-samesite-none"` } -type affinity struct { - r resolver.Resolver - annotationConfig parser.Annotation -} - // cookieAffinityParse gets the annotation values related to Cookie Affinity // It also sets default values when no value or incorrect value is found func (a affinity) cookieAffinityParse(ing *networking.Ingress) *Cookie { @@ -198,50 +94,40 @@ func (a affinity) cookieAffinityParse(ing *networking.Ingress) *Cookie { cookie := &Cookie{} - cookie.Name, err = parser.GetStringAnnotation(annotationAffinityCookieName, ing, a.annotationConfig.Annotations) + cookie.Name, err = parser.GetStringAnnotation(annotationAffinityCookieName, ing) if err != nil { klog.V(3).InfoS("Invalid or no annotation value found. Ignoring", "ingress", klog.KObj(ing), "annotation", annotationAffinityCookieName, "default", defaultAffinityCookieName) cookie.Name = defaultAffinityCookieName } - cookie.Expires, err = parser.GetStringAnnotation(annotationAffinityCookieExpires, ing, a.annotationConfig.Annotations) + cookie.Expires, err = parser.GetStringAnnotation(annotationAffinityCookieExpires, ing) if err != nil || !affinityCookieExpiresRegex.MatchString(cookie.Expires) { klog.V(3).InfoS("Invalid or no annotation value found. Ignoring", "ingress", klog.KObj(ing), "annotation", annotationAffinityCookieExpires) cookie.Expires = "" } - cookie.MaxAge, err = parser.GetStringAnnotation(annotationAffinityCookieMaxAge, ing, a.annotationConfig.Annotations) + cookie.MaxAge, err = parser.GetStringAnnotation(annotationAffinityCookieMaxAge, ing) if err != nil || !affinityCookieExpiresRegex.MatchString(cookie.MaxAge) { klog.V(3).InfoS("Invalid or no annotation value found. Ignoring", "ingress", klog.KObj(ing), "annotation", annotationAffinityCookieMaxAge) cookie.MaxAge = "" } - cookie.Path, err = parser.GetStringAnnotation(annotationAffinityCookiePath, ing, a.annotationConfig.Annotations) + cookie.Path, err = parser.GetStringAnnotation(annotationAffinityCookiePath, ing) if err != nil { klog.V(3).InfoS("Invalid or no annotation value found. Ignoring", "ingress", klog.KObj(ing), "annotation", annotationAffinityCookiePath) } - cookie.Domain, err = parser.GetStringAnnotation(annotationAffinityCookieDomain, ing, a.annotationConfig.Annotations) - if err != nil { - klog.V(3).InfoS("Invalid or no annotation value found. Ignoring", "ingress", klog.KObj(ing), "annotation", annotationAffinityCookieDomain) - } - - cookie.SameSite, err = parser.GetStringAnnotation(annotationAffinityCookieSameSite, ing, a.annotationConfig.Annotations) + cookie.SameSite, err = parser.GetStringAnnotation(annotationAffinityCookieSameSite, ing) if err != nil { klog.V(3).InfoS("Invalid or no annotation value found. Ignoring", "ingress", klog.KObj(ing), "annotation", annotationAffinityCookieSameSite) } - cookie.Secure, err = parser.GetBoolAnnotation(annotationAffinityCookieSecure, ing, a.annotationConfig.Annotations) - if err != nil { - klog.V(3).InfoS("Invalid or no annotation value found. Ignoring", "ingress", klog.KObj(ing), "annotation", annotationAffinityCookieSecure) - } - - cookie.ConditionalSameSiteNone, err = parser.GetBoolAnnotation(annotationAffinityCookieConditionalSameSiteNone, ing, a.annotationConfig.Annotations) + cookie.ConditionalSameSiteNone, err = parser.GetBoolAnnotation(annotationAffinityCookieConditionalSameSiteNone, ing) if err != nil { klog.V(3).InfoS("Invalid or no annotation value found. Ignoring", "ingress", klog.KObj(ing), "annotation", annotationAffinityCookieConditionalSameSiteNone) } - cookie.ChangeOnFailure, err = parser.GetBoolAnnotation(annotationAffinityCookieChangeOnFailure, ing, a.annotationConfig.Annotations) + cookie.ChangeOnFailure, err = parser.GetBoolAnnotation(annotationAffinityCookieChangeOnFailure, ing) if err != nil { klog.V(3).InfoS("Invalid or no annotation value found. Ignoring", "ingress", klog.KObj(ing), "annotation", annotationAffinityCookieChangeOnFailure) } @@ -251,10 +137,11 @@ func (a affinity) cookieAffinityParse(ing *networking.Ingress) *Cookie { // NewParser creates a new Affinity annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return affinity{ - r: r, - annotationConfig: sessionAffinityAnnotations, - } + return affinity{r} +} + +type affinity struct { + r resolver.Resolver } // ParseAnnotations parses the annotations contained in the ingress @@ -262,42 +149,28 @@ func NewParser(r resolver.Resolver) parser.IngressAnnotation { func (a affinity) Parse(ing *networking.Ingress) (interface{}, error) { cookie := &Cookie{} // Check the type of affinity that will be used - at, err := parser.GetStringAnnotation(annotationAffinityType, ing, a.annotationConfig.Annotations) + at, err := parser.GetStringAnnotation(annotationAffinityType, ing) if err != nil { at = "" } // Check the affinity mode that will be used - am, err := parser.GetStringAnnotation(annotationAffinityMode, ing, a.annotationConfig.Annotations) + am, err := parser.GetStringAnnotation(annotationAffinityMode, ing) if err != nil { am = "" } - cb, err := parser.GetStringAnnotation(annotationAffinityCanaryBehavior, ing, a.annotationConfig.Annotations) - if err != nil { - cb = "" - } - switch at { - case cookieAffinity: + case "cookie": cookie = a.cookieAffinityParse(ing) default: klog.V(3).InfoS("No default affinity found", "ingress", ing.Name) + } return &Config{ - Type: at, - Mode: am, - CanaryBehavior: cb, - Cookie: *cookie, + Type: at, + Mode: am, + Cookie: *cookie, }, nil } - -func (a affinity) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a affinity) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, sessionAffinityAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/sessionaffinity/main_test.go b/internal/ingress/annotations/sessionaffinity/main_test.go index 4b7ea5e61..51d92ffb0 100644 --- a/internal/ingress/annotations/sessionaffinity/main_test.go +++ b/internal/ingress/annotations/sessionaffinity/main_test.go @@ -20,20 +20,17 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) func buildIngress() *networking.Ingress { defaultBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), } return &networking.Ingress{ @@ -42,13 +39,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -71,6 +64,7 @@ func buildIngress() *networking.Ingress { func TestIngressAffinityCookieConfig(t *testing.T) { ing := buildIngress() + data := map[string]string{} data[parser.GetAnnotationWithPrefix(annotationAffinityType)] = "cookie" data[parser.GetAnnotationWithPrefix(annotationAffinityMode)] = "balanced" @@ -78,17 +72,10 @@ func TestIngressAffinityCookieConfig(t *testing.T) { data[parser.GetAnnotationWithPrefix(annotationAffinityCookieExpires)] = "4500" data[parser.GetAnnotationWithPrefix(annotationAffinityCookieMaxAge)] = "3000" data[parser.GetAnnotationWithPrefix(annotationAffinityCookiePath)] = "/foo" - data[parser.GetAnnotationWithPrefix(annotationAffinityCookieDomain)] = "foo.bar" - data[parser.GetAnnotationWithPrefix(annotationAffinityCookieSameSite)] = "Strict" data[parser.GetAnnotationWithPrefix(annotationAffinityCookieChangeOnFailure)] = "true" - data[parser.GetAnnotationWithPrefix(annotationAffinityCookieSecure)] = "true" ing.SetAnnotations(data) - affin, err := NewParser(&resolver.Mock{}).Parse(ing) - if err != nil { - t.Errorf("unexpected error parsing annotations: %v", err) - } - + affin, _ := NewParser(&resolver.Mock{}).Parse(ing) nginxAffinity, ok := affin.(*Config) if !ok { t.Errorf("expected a Config type") @@ -118,19 +105,7 @@ func TestIngressAffinityCookieConfig(t *testing.T) { t.Errorf("expected /foo as session-cookie-path but returned %v", nginxAffinity.Cookie.Path) } - if nginxAffinity.Cookie.Domain != "foo.bar" { - t.Errorf("expected foo.bar as session-cookie-domain but returned %v", nginxAffinity.Cookie.Domain) - } - - if nginxAffinity.Cookie.SameSite != "Strict" { - t.Errorf("expected Strict as session-cookie-same-site but returned %v", nginxAffinity.Cookie.SameSite) - } - if !nginxAffinity.Cookie.ChangeOnFailure { t.Errorf("expected change of failure parameter set to true but returned %v", nginxAffinity.Cookie.ChangeOnFailure) } - - if !nginxAffinity.Cookie.Secure { - t.Errorf("expected secure parameter set to true but returned %v", nginxAffinity.Cookie.Secure) - } } diff --git a/internal/ingress/annotations/snippet/main.go b/internal/ingress/annotations/snippet/main.go index c2df84cdd..9a3878603 100644 --- a/internal/ingress/annotations/snippet/main.go +++ b/internal/ingress/annotations/snippet/main.go @@ -17,53 +17,24 @@ limitations under the License. package snippet import ( - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - configurationSnippetAnnotation = "configuration-snippet" -) - -var configurationSnippetAnnotations = parser.Annotation{ - Group: "snippets", - Annotations: parser.AnnotationFields{ - configurationSnippetAnnotation: { - Validator: parser.ValidateNull, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskCritical, // Critical, this annotation is not validated at all and allows arbitrary configurations - Documentation: `This annotation allows setting a custom NGINX configuration on a location block. This annotation does not contain any validation and it's usage is not recommended!`, - }, - }, -} - type snippet struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new CORS annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return snippet{ - r: r, - annotationConfig: configurationSnippetAnnotations, - } + return snippet{r} } // Parse parses the annotations contained in the ingress rule // used to indicate if the location/s contains a fragment of // configuration to be included inside the paths of the rules func (a snippet) Parse(ing *networking.Ingress) (interface{}, error) { - return parser.GetStringAnnotation(configurationSnippetAnnotation, ing, a.annotationConfig.Annotations) -} - -func (a snippet) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a snippet) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, configurationSnippetAnnotations.Annotations) + return parser.GetStringAnnotation("configuration-snippet", ing) } diff --git a/internal/ingress/annotations/snippet/main_test.go b/internal/ingress/annotations/snippet/main_test.go index b29b2950d..0abeaed8a 100644 --- a/internal/ingress/annotations/snippet/main_test.go +++ b/internal/ingress/annotations/snippet/main_test.go @@ -20,14 +20,14 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) func TestParse(t *testing.T) { - annotation := parser.GetAnnotationWithPrefix(configurationSnippetAnnotation) + annotation := parser.GetAnnotationWithPrefix("configuration-snippet") ap := NewParser(&resolver.Mock{}) if ap == nil { @@ -54,7 +54,6 @@ func TestParse(t *testing.T) { for _, testCase := range testCases { ing.SetAnnotations(testCase.annotations) - //nolint:errcheck // Ignore the error since invalid cases will be checked with expected results result, _ := ap.Parse(ing) if result != testCase.expected { t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations) diff --git a/internal/ingress/annotations/sslcipher/main.go b/internal/ingress/annotations/sslcipher/main.go index 3cfd903bd..d100a0da4 100644 --- a/internal/ingress/annotations/sslcipher/main.go +++ b/internal/ingress/annotations/sslcipher/main.go @@ -17,46 +17,14 @@ limitations under the License. package sslcipher import ( - "regexp" - - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - sslPreferServerCipherAnnotation = "ssl-prefer-server-ciphers" - sslCipherAnnotation = "ssl-ciphers" -) - -// Should cover something like "ALL:!aNULL:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP" -// (?:@STRENGTH) is included twice so it can appear before or after @SECLEVEL=n -var regexValidSSLCipher = regexp.MustCompile(`^(?:(?:[A-Za-z0-9!:+\-])*(?:@STRENGTH)*(?:@SECLEVEL=[0-5])*(?:@STRENGTH)*)*$`) - -var sslCipherAnnotations = parser.Annotation{ - Group: "backend", - Annotations: parser.AnnotationFields{ - sslPreferServerCipherAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `The following annotation will set the ssl_prefer_server_ciphers directive at the server level. - This configuration specifies that server ciphers should be preferred over client ciphers when using the TLS protocols.`, - }, - sslCipherAnnotation: { - Validator: parser.ValidateRegex(regexValidSSLCipher, true), - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, - Documentation: `Using this annotation will set the ssl_ciphers directive at the server level. This configuration is active for all the paths in the host.`, - }, - }, -} - type sslCipher struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // Config contains the ssl-ciphers & ssl-prefer-server-ciphers configuration @@ -67,10 +35,7 @@ type Config struct { // NewParser creates a new sslCipher annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return sslCipher{ - r: r, - annotationConfig: sslCipherAnnotations, - } + return sslCipher{r} } // Parse parses the annotations contained in the ingress rule @@ -80,7 +45,7 @@ func (sc sslCipher) Parse(ing *networking.Ingress) (interface{}, error) { var err error var sslPreferServerCiphers bool - sslPreferServerCiphers, err = parser.GetBoolAnnotation(sslPreferServerCipherAnnotation, ing, sc.annotationConfig.Annotations) + sslPreferServerCiphers, err = parser.GetBoolAnnotation("ssl-prefer-server-ciphers", ing) if err != nil { config.SSLPreferServerCiphers = "" } else { @@ -91,19 +56,7 @@ func (sc sslCipher) Parse(ing *networking.Ingress) (interface{}, error) { } } - config.SSLCiphers, err = parser.GetStringAnnotation(sslCipherAnnotation, ing, sc.annotationConfig.Annotations) - if err != nil && !errors.IsInvalidContent(err) && !errors.IsMissingAnnotations(err) { - return config, err - } + config.SSLCiphers, _ = parser.GetStringAnnotation("ssl-ciphers", ing) return config, nil } - -func (sc sslCipher) GetDocumentation() parser.AnnotationFields { - return sc.annotationConfig.Annotations -} - -func (sc sslCipher) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(sc.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, sslCipherAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/sslcipher/main_test.go b/internal/ingress/annotations/sslcipher/main_test.go index f1675d4c7..8110697dc 100644 --- a/internal/ingress/annotations/sslcipher/main_test.go +++ b/internal/ingress/annotations/sslcipher/main_test.go @@ -21,7 +21,7 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" @@ -33,31 +33,22 @@ func TestParse(t *testing.T) { t.Fatalf("expected a parser.IngressAnnotation but returned nil") } - annotationSSLCiphers := parser.GetAnnotationWithPrefix(sslCipherAnnotation) - annotationSSLPreferServerCiphers := parser.GetAnnotationWithPrefix(sslPreferServerCipherAnnotation) + annotationSSLCiphers := parser.GetAnnotationWithPrefix("ssl-ciphers") + annotationSSLPreferServerCiphers := parser.GetAnnotationWithPrefix("ssl-prefer-server-ciphers") testCases := []struct { annotations map[string]string expected Config - expectErr bool }{ - {map[string]string{annotationSSLCiphers: "ALL:!aNULL:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP"}, Config{"ALL:!aNULL:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP", ""}, false}, - {map[string]string{annotationSSLCiphers: "ALL:!aNULL:!EXPORT56@SECLEVEL=2:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP"}, Config{"ALL:!aNULL:!EXPORT56@SECLEVEL=2:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP", ""}, false}, - {map[string]string{annotationSSLCiphers: "ALL:!aNULL:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP@STRENGTH"}, Config{"ALL:!aNULL:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP@STRENGTH", ""}, false}, - {map[string]string{annotationSSLCiphers: "ALL:!aNULL:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP@STRENGTH@SECLEVEL=3"}, Config{"ALL:!aNULL:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP@STRENGTH@SECLEVEL=3", ""}, false}, - {map[string]string{annotationSSLCiphers: "ALL:!aNULL:!EXPORT56:RC4+RSA@STRENGTH:+HIGH@SECLEVEL=5:+MEDIUM:+LOW:+SSLv2:+EXP"}, Config{"ALL:!aNULL:!EXPORT56:RC4+RSA@STRENGTH:+HIGH@SECLEVEL=5:+MEDIUM:+LOW:+SSLv2:+EXP", ""}, false}, - { - map[string]string{annotationSSLCiphers: "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256"}, - Config{"ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256", ""}, - false, - }, - {map[string]string{annotationSSLCiphers: ""}, Config{"", ""}, false}, - {map[string]string{annotationSSLPreferServerCiphers: "true"}, Config{"", "on"}, false}, - {map[string]string{annotationSSLPreferServerCiphers: "false"}, Config{"", "off"}, false}, - {map[string]string{annotationSSLCiphers: "ALL:!aNULL:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP", annotationSSLPreferServerCiphers: "true"}, Config{"ALL:!aNULL:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP", "on"}, false}, - {map[string]string{annotationSSLCiphers: "ALL:SOMETHING:;locationXPTO"}, Config{"", ""}, true}, - {map[string]string{}, Config{"", ""}, false}, - {nil, Config{"", ""}, false}, + {map[string]string{annotationSSLCiphers: "ALL:!aNULL:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP"}, Config{"ALL:!aNULL:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP", ""}}, + {map[string]string{annotationSSLCiphers: "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256"}, + Config{"ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256", ""}}, + {map[string]string{annotationSSLCiphers: ""}, Config{"", ""}}, + {map[string]string{annotationSSLPreferServerCiphers: "true"}, Config{"", "on"}}, + {map[string]string{annotationSSLPreferServerCiphers: "false"}, Config{"", "off"}}, + {map[string]string{annotationSSLCiphers: "ALL:!aNULL:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP", annotationSSLPreferServerCiphers: "true"}, Config{"ALL:!aNULL:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP", "on"}}, + {map[string]string{}, Config{"", ""}}, + {nil, Config{"", ""}}, } ing := &networking.Ingress{ @@ -68,13 +59,10 @@ func TestParse(t *testing.T) { Spec: networking.IngressSpec{}, } - for i, testCase := range testCases { + for _, testCase := range testCases { ing.SetAnnotations(testCase.annotations) - result, err := ap.Parse(ing) - if (err != nil) != testCase.expectErr { - t.Fatalf("expected error: %t got error: %t err value: %s. %+v", testCase.expectErr, err != nil, err, testCase.annotations) - } - if !reflect.DeepEqual(result, &testCases[i].expected) { + result, _ := ap.Parse(ing) + if !reflect.DeepEqual(result, &testCase.expected) { t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations) } } diff --git a/internal/ingress/annotations/sslpassthrough/main.go b/internal/ingress/annotations/sslpassthrough/main.go index c06db8715..20ff1a010 100644 --- a/internal/ingress/annotations/sslpassthrough/main.go +++ b/internal/ingress/annotations/sslpassthrough/main.go @@ -17,40 +17,20 @@ limitations under the License. package sslpassthrough import ( - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" ing_errors "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - sslPassthroughAnnotation = "ssl-passthrough" -) - -var sslPassthroughAnnotations = parser.Annotation{ - Group: "", // TBD - Annotations: parser.AnnotationFields{ - sslPassthroughAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskLow, // Low, as it allows regexes but on a very limited set - Documentation: `This annotation instructs the controller to send TLS connections directly to the backend instead of letting NGINX decrypt the communication.`, - }, - }, -} - type sslpt struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new SSL passthrough annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return sslpt{ - r: r, - annotationConfig: sslPassthroughAnnotations, - } + return sslpt{r} } // ParseAnnotations parses the annotations contained in the ingress @@ -60,14 +40,5 @@ func (a sslpt) Parse(ing *networking.Ingress) (interface{}, error) { return false, ing_errors.ErrMissingAnnotations } - return parser.GetBoolAnnotation(sslPassthroughAnnotation, ing, a.annotationConfig.Annotations) -} - -func (a sslpt) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a sslpt) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, sslPassthroughAnnotations.Annotations) + return parser.GetBoolAnnotation("ssl-passthrough", ing) } diff --git a/internal/ingress/annotations/sslpassthrough/main_test.go b/internal/ingress/annotations/sslpassthrough/main_test.go index b712fda19..d5e54b2e2 100644 --- a/internal/ingress/annotations/sslpassthrough/main_test.go +++ b/internal/ingress/annotations/sslpassthrough/main_test.go @@ -20,10 +20,12 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" + + "k8s.io/apimachinery/pkg/util/intstr" ) func buildIngress() *networking.Ingress { @@ -33,13 +35,9 @@ func buildIngress() *networking.Ingress { Namespace: api.NamespaceDefault, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "default-backend", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), }, }, } @@ -54,7 +52,7 @@ func TestParseAnnotations(t *testing.T) { } data := map[string]string{} - data[parser.GetAnnotationWithPrefix(sslPassthroughAnnotation)] = "true" + data[parser.GetAnnotationWithPrefix("ssl-passthrough")] = "true" ing.SetAnnotations(data) // test ingress using the annotation without a TLS section _, err = NewParser(&resolver.Mock{}).Parse(ing) diff --git a/internal/ingress/annotations/streamsnippet/main.go b/internal/ingress/annotations/streamsnippet/main.go deleted file mode 100644 index 7743d3fee..000000000 --- a/internal/ingress/annotations/streamsnippet/main.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package streamsnippet - -import ( - networking "k8s.io/api/networking/v1" - - "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/resolver" -) - -const ( - streamSnippetAnnotation = "stream-snippet" -) - -var streamSnippetAnnotations = parser.Annotation{ - Group: "snippets", - Annotations: parser.AnnotationFields{ - streamSnippetAnnotation: { - Validator: parser.ValidateNull, - Scope: parser.AnnotationScopeIngress, - Risk: parser.AnnotationRiskCritical, // Critical, this annotation is not validated at all and allows arbitrary configurations - Documentation: `This annotation allows setting a custom NGINX configuration on a stream block. This annotation does not contain any validation and it's usage is not recommended!`, - }, - }, -} - -type streamSnippet struct { - r resolver.Resolver - annotationConfig parser.Annotation -} - -// NewParser creates a new server snippet annotation parser -func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return streamSnippet{ - r: r, - annotationConfig: streamSnippetAnnotations, - } -} - -// Parse parses the annotations contained in the ingress rule -// used to indicate if the location/s contains a fragment of -// configuration to be included inside the paths of the rules -func (a streamSnippet) Parse(ing *networking.Ingress) (interface{}, error) { - return parser.GetStringAnnotation("stream-snippet", ing, a.annotationConfig.Annotations) -} - -func (a streamSnippet) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a streamSnippet) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, streamSnippetAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/streamsnippet/main_test.go b/internal/ingress/annotations/streamsnippet/main_test.go deleted file mode 100644 index e8a1ff8d0..000000000 --- a/internal/ingress/annotations/streamsnippet/main_test.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package streamsnippet - -import ( - "testing" - - api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" - meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/resolver" -) - -func TestParse(t *testing.T) { - annotation := parser.GetAnnotationWithPrefix(streamSnippetAnnotation) - - ap := NewParser(&resolver.Mock{}) - if ap == nil { - t.Fatalf("expected a parser.IngressAnnotation but returned nil") - } - - testCases := []struct { - annotations map[string]string - expected string - }{ - { - map[string]string{annotation: "server { listen: 8000; proxy_pass 127.0.0.1:80}"}, - "server { listen: 8000; proxy_pass 127.0.0.1:80}", - }, - {map[string]string{annotation: "false"}, "false"}, - {map[string]string{}, ""}, - {nil, ""}, - } - - ing := &networking.Ingress{ - ObjectMeta: meta_v1.ObjectMeta{ - Name: "foo", - Namespace: api.NamespaceDefault, - }, - Spec: networking.IngressSpec{}, - } - - for _, testCase := range testCases { - ing.SetAnnotations(testCase.annotations) - //nolint:errcheck // Ignore the error since invalid cases will be checked with expected results - result, _ := ap.Parse(ing) - if result != testCase.expected { - t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations) - } - } -} diff --git a/internal/ingress/annotations/upstreamhashby/main.go b/internal/ingress/annotations/upstreamhashby/main.go index 25cc88b8e..bb202f1b0 100644 --- a/internal/ingress/annotations/upstreamhashby/main.go +++ b/internal/ingress/annotations/upstreamhashby/main.go @@ -17,54 +17,14 @@ limitations under the License. package upstreamhashby import ( - "regexp" - - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - upstreamHashByAnnotation = "upstream-hash-by" - upstreamHashBySubsetAnnotation = "upstream-hash-by-subset" - upstreamHashBySubsetSize = "upstream-hash-by-subset-size" -) - -var ( - specialChars = regexp.QuoteMeta("_${}") - hashByRegex = regexp.MustCompilePOSIX(`^[A-Za-z0-9\-` + specialChars + `]*$`) -) - -var upstreamHashByAnnotations = parser.Annotation{ - Group: "backend", - Annotations: parser.AnnotationFields{ - upstreamHashByAnnotation: { - Validator: parser.ValidateRegex(hashByRegex, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskHigh, // High, this annotation allows accessing NGINX variables - Documentation: `This annotation defines the nginx variable, text value or any combination thereof to use for consistent hashing. - For example: nginx.ingress.kubernetes.io/upstream-hash-by: "$request_uri" or nginx.ingress.kubernetes.io/upstream-hash-by: "$request_uri$host" or nginx.ingress.kubernetes.io/upstream-hash-by: "${request_uri}-text-value" to consistently hash upstream requests by the current request URI.`, - }, - upstreamHashBySubsetAnnotation: { - Validator: parser.ValidateBool, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation maps requests to subset of nodes instead of a single one.`, - }, - upstreamHashBySubsetSize: { - Validator: parser.ValidateInt, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, - Documentation: `This annotation determines the size of each subset (default 3)`, - }, - }, -} - type upstreamhashby struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // Config contains the Consistent hash configuration to be used in the Ingress @@ -76,26 +36,14 @@ type Config struct { // NewParser creates a new UpstreamHashBy annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return upstreamhashby{ - r: r, - annotationConfig: upstreamHashByAnnotations, - } + return upstreamhashby{r} } // Parse parses the annotations contained in the ingress rule func (a upstreamhashby) Parse(ing *networking.Ingress) (interface{}, error) { - upstreamHashBy, err := parser.GetStringAnnotation(upstreamHashByAnnotation, ing, a.annotationConfig.Annotations) - if err != nil && !errors.IsMissingAnnotations(err) { - return nil, err - } - upstreamHashBySubset, err := parser.GetBoolAnnotation(upstreamHashBySubsetAnnotation, ing, a.annotationConfig.Annotations) - if err != nil && !errors.IsMissingAnnotations(err) { - return nil, err - } - upstreamHashbySubsetSize, err := parser.GetIntAnnotation(upstreamHashBySubsetSize, ing, a.annotationConfig.Annotations) - if err != nil && !errors.IsMissingAnnotations(err) { - return nil, err - } + upstreamHashBy, _ := parser.GetStringAnnotation("upstream-hash-by", ing) + upstreamHashBySubset, _ := parser.GetBoolAnnotation("upstream-hash-by-subset", ing) + upstreamHashbySubsetSize, _ := parser.GetIntAnnotation("upstream-hash-by-subset-size", ing) if upstreamHashbySubsetSize == 0 { upstreamHashbySubsetSize = 3 @@ -103,12 +51,3 @@ func (a upstreamhashby) Parse(ing *networking.Ingress) (interface{}, error) { return &Config{upstreamHashBy, upstreamHashBySubset, upstreamHashbySubsetSize}, nil } - -func (a upstreamhashby) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a upstreamhashby) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, upstreamHashByAnnotations.Annotations) -} diff --git a/internal/ingress/annotations/upstreamhashby/main_test.go b/internal/ingress/annotations/upstreamhashby/main_test.go index bdbd9c350..5a71be56f 100644 --- a/internal/ingress/annotations/upstreamhashby/main_test.go +++ b/internal/ingress/annotations/upstreamhashby/main_test.go @@ -20,14 +20,14 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) func TestParse(t *testing.T) { - annotation := parser.GetAnnotationWithPrefix(upstreamHashByAnnotation) + annotation := parser.GetAnnotationWithPrefix("upstream-hash-by") ap := NewParser(&resolver.Mock{}) if ap == nil { @@ -37,15 +37,12 @@ func TestParse(t *testing.T) { testCases := []struct { annotations map[string]string expected string - expectErr bool }{ - {map[string]string{annotation: "$request_URI"}, "$request_URI", false}, - {map[string]string{annotation: "$request_uri$scheme"}, "$request_uri$scheme", false}, - {map[string]string{annotation: "xpto;[]"}, "", true}, - {map[string]string{annotation: "lalal${scheme_test}"}, "lalal${scheme_test}", false}, - {map[string]string{annotation: "false"}, "false", false}, - {map[string]string{}, "", false}, - {nil, "", false}, + {map[string]string{annotation: "$request_uri"}, "$request_uri"}, + {map[string]string{annotation: "$request_uri$scheme"}, "$request_uri$scheme"}, + {map[string]string{annotation: "false"}, "false"}, + {map[string]string{}, ""}, + {nil, ""}, } ing := &networking.Ingress{ @@ -58,19 +55,14 @@ func TestParse(t *testing.T) { for _, testCase := range testCases { ing.SetAnnotations(testCase.annotations) - result, err := ap.Parse(ing) - if (err != nil) != testCase.expectErr { - t.Fatalf("expected error: %t got error: %t err value: %s. %+v", testCase.expectErr, err != nil, err, testCase.annotations) + result, _ := ap.Parse(ing) + uc, ok := result.(*Config) + if !ok { + t.Fatalf("expected a Config type") } - if !testCase.expectErr { - uc, ok := result.(*Config) - if !ok { - t.Fatalf("expected a Config type") - } - if uc.UpstreamHashBy != testCase.expected { - t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations) - } + if uc.UpstreamHashBy != testCase.expected { + t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations) } } } diff --git a/internal/ingress/annotations/upstreamvhost/main.go b/internal/ingress/annotations/upstreamvhost/main.go index 052ca2344..bf761a70f 100644 --- a/internal/ingress/annotations/upstreamvhost/main.go +++ b/internal/ingress/annotations/upstreamvhost/main.go @@ -17,54 +17,24 @@ limitations under the License. package upstreamvhost import ( - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - upstreamVhostAnnotation = "upstream-vhost" -) - -var upstreamVhostAnnotations = parser.Annotation{ - Group: "backend", - Annotations: parser.AnnotationFields{ - upstreamVhostAnnotation: { - Validator: parser.ValidateServerName, - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskLow, // Low, as it allows regexes but on a very limited set - Documentation: `This configuration setting allows you to control the value for host in the following statement: proxy_set_header Host $host, which forms part of the location block. - This is useful if you need to call the upstream server by something other than $host`, - }, - }, -} - type upstreamVhost struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new upstream VHost annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return upstreamVhost{ - r: r, - annotationConfig: upstreamVhostAnnotations, - } + return upstreamVhost{r} } // Parse parses the annotations contained in the ingress rule // used to indicate if the location/s contains a fragment of // configuration to be included inside the paths of the rules func (a upstreamVhost) Parse(ing *networking.Ingress) (interface{}, error) { - return parser.GetStringAnnotation(upstreamVhostAnnotation, ing, a.annotationConfig.Annotations) -} - -func (a upstreamVhost) GetDocumentation() parser.AnnotationFields { - return a.annotationConfig.Annotations -} - -func (a upstreamVhost) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(a.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, upstreamVhostAnnotations.Annotations) + return parser.GetStringAnnotation("upstream-vhost", ing) } diff --git a/internal/ingress/annotations/upstreamvhost/main_test.go b/internal/ingress/annotations/upstreamvhost/main_test.go index 87324b181..1506c4f7f 100644 --- a/internal/ingress/annotations/upstreamvhost/main_test.go +++ b/internal/ingress/annotations/upstreamvhost/main_test.go @@ -20,7 +20,7 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" @@ -36,7 +36,7 @@ func TestParse(t *testing.T) { } data := map[string]string{} - data[parser.GetAnnotationWithPrefix(upstreamVhostAnnotation)] = "ok.com" + data[parser.GetAnnotationWithPrefix("upstream-vhost")] = "ok.com" ing.SetAnnotations(data) diff --git a/internal/ingress/annotations/xforwardedprefix/main.go b/internal/ingress/annotations/xforwardedprefix/main.go index 54bb7199a..2071b6411 100644 --- a/internal/ingress/annotations/xforwardedprefix/main.go +++ b/internal/ingress/annotations/xforwardedprefix/main.go @@ -17,53 +17,23 @@ limitations under the License. package xforwardedprefix import ( - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) -const ( - xForwardedForPrefixAnnotation = "x-forwarded-prefix" -) - -var xForwardedForAnnotations = parser.Annotation{ - Group: "backend", - Annotations: parser.AnnotationFields{ - xForwardedForPrefixAnnotation: { - Validator: parser.ValidateRegex(parser.RegexPathWithCapture, true), - Scope: parser.AnnotationScopeLocation, - Risk: parser.AnnotationRiskMedium, - Documentation: `This annotation can be used to add the non-standard X-Forwarded-Prefix header to the upstream request with a string value. It can - contain regular characters and captured groups specified as '$1', '$2', etc.`, - }, - }, -} - type xforwardedprefix struct { - r resolver.Resolver - annotationConfig parser.Annotation + r resolver.Resolver } // NewParser creates a new xforwardedprefix annotation parser func NewParser(r resolver.Resolver) parser.IngressAnnotation { - return xforwardedprefix{ - r: r, - annotationConfig: xForwardedForAnnotations, - } + return xforwardedprefix{r} } // Parse parses the annotations contained in the ingress rule // used to add an x-forwarded-prefix header to the request -func (x xforwardedprefix) Parse(ing *networking.Ingress) (interface{}, error) { - return parser.GetStringAnnotation(xForwardedForPrefixAnnotation, ing, x.annotationConfig.Annotations) -} - -func (x xforwardedprefix) GetDocumentation() parser.AnnotationFields { - return x.annotationConfig.Annotations -} - -func (x xforwardedprefix) Validate(anns map[string]string) error { - maxrisk := parser.StringRiskToRisk(x.r.GetSecurityConfiguration().AnnotationsRiskLevel) - return parser.CheckAnnotationRisk(anns, maxrisk, xForwardedForAnnotations.Annotations) +func (cbbs xforwardedprefix) Parse(ing *networking.Ingress) (interface{}, error) { + return parser.GetStringAnnotation("x-forwarded-prefix", ing) } diff --git a/internal/ingress/annotations/xforwardedprefix/main_test.go b/internal/ingress/annotations/xforwardedprefix/main_test.go index 977e7d372..c94df3ab2 100644 --- a/internal/ingress/annotations/xforwardedprefix/main_test.go +++ b/internal/ingress/annotations/xforwardedprefix/main_test.go @@ -20,14 +20,14 @@ import ( "testing" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) func TestParse(t *testing.T) { - annotation := parser.GetAnnotationWithPrefix(xForwardedForPrefixAnnotation) + annotation := parser.GetAnnotationWithPrefix("x-forwarded-prefix") ap := NewParser(&resolver.Mock{}) if ap == nil { t.Fatalf("expected a parser.IngressAnnotation but returned nil") @@ -40,7 +40,6 @@ func TestParse(t *testing.T) { {map[string]string{annotation: "true"}, "true"}, {map[string]string{annotation: "1"}, "1"}, {map[string]string{annotation: ""}, ""}, - {map[string]string{annotation: "/$1"}, "/$1"}, {map[string]string{}, ""}, {nil, ""}, } @@ -55,7 +54,6 @@ func TestParse(t *testing.T) { for _, testCase := range testCases { ing.SetAnnotations(testCase.annotations) - //nolint:errcheck // Ignore the error since invalid cases will be checked with expected results result, _ := ap.Parse(ing) if result != testCase.expected { t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations) diff --git a/internal/ingress/controller/certificate.go b/internal/ingress/controller/certificate.go index 76aafec3c..e8707c716 100644 --- a/internal/ingress/controller/certificate.go +++ b/internal/ingress/controller/certificate.go @@ -100,7 +100,7 @@ func matchHostnames(pattern, host string) bool { host = strings.TrimSuffix(host, ".") pattern = strings.TrimSuffix(pattern, ".") - if pattern == "" || host == "" { + if len(pattern) == 0 || len(host) == 0 { return false } diff --git a/internal/ingress/controller/checker.go b/internal/ingress/controller/checker.go index d1bf19ddf..81c6e12be 100644 --- a/internal/ingress/controller/checker.go +++ b/internal/ingress/controller/checker.go @@ -18,18 +18,19 @@ package controller import ( "fmt" + "io/ioutil" "net/http" - "os" "strconv" "strings" "github.com/ncabatoff/process-exporter/proc" + "github.com/pkg/errors" "k8s.io/ingress-nginx/internal/nginx" ) // Name returns the healthcheck name -func (n *NGINXController) Name() string { +func (n NGINXController) Name() string { return "nginx-ingress-controller" } @@ -42,27 +43,27 @@ func (n *NGINXController) Check(_ *http.Request) error { // check the nginx master process is running fs, err := proc.NewFS("/proc", false) if err != nil { - return fmt.Errorf("reading /proc directory: %w", err) + return errors.Wrap(err, "reading /proc directory") } - f, err := os.ReadFile(nginx.PID) + f, err := ioutil.ReadFile(nginx.PID) if err != nil { - return fmt.Errorf("reading %v: %w", nginx.PID, err) + return errors.Wrapf(err, "reading %v", nginx.PID) } pid, err := strconv.Atoi(strings.TrimRight(string(f), "\r\n")) if err != nil { - return fmt.Errorf("reading NGINX PID from file %v: %w", nginx.PID, err) + return errors.Wrapf(err, "reading NGINX PID from file %v", nginx.PID) } _, err = fs.Proc(pid) if err != nil { - return fmt.Errorf("checking for NGINX process with PID %v: %w", pid, err) + return errors.Wrapf(err, "checking for NGINX process with PID %v", pid) } statusCode, _, err := nginx.NewGetStatusRequest("/is-dynamic-lb-initialized") if err != nil { - return fmt.Errorf("checking if the dynamic load balancer started: %w", err) + return errors.Wrapf(err, "checking if the dynamic load balancer started") } if statusCode != 200 { diff --git a/internal/ingress/controller/checker_test.go b/internal/ingress/controller/checker_test.go index a8a0b67ad..3f02f4016 100644 --- a/internal/ingress/controller/checker_test.go +++ b/internal/ingress/controller/checker_test.go @@ -26,13 +26,13 @@ import ( "k8s.io/apiserver/pkg/server/healthz" + "k8s.io/ingress-nginx/internal/file" ngx_config "k8s.io/ingress-nginx/internal/ingress/controller/config" "k8s.io/ingress-nginx/internal/nginx" - "k8s.io/ingress-nginx/pkg/util/file" ) func TestNginxCheck(t *testing.T) { - tests := []struct { + var tests = []struct { healthzPath string }{ {"/healthz"}, @@ -42,6 +42,7 @@ func TestNginxCheck(t *testing.T) { for _, tt := range tests { testName := fmt.Sprintf("health path: %s", tt.healthzPath) t.Run(testName, func(t *testing.T) { + mux := http.NewServeMux() listener, err := tryListen("tcp", fmt.Sprintf(":%v", nginx.StatusPort)) @@ -49,11 +50,11 @@ func TestNginxCheck(t *testing.T) { t.Fatalf("creating tcp listener: %s", err) } defer listener.Close() - //nolint:gosec // Ignore not configured ReadHeaderTimeout in testing + server := &httptest.Server{ Listener: listener, Config: &http.Server{ - Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) fmt.Fprintf(w, "ok") }), @@ -75,10 +76,7 @@ func TestNginxCheck(t *testing.T) { }) // create pid file - if err := os.MkdirAll("/tmp/nginx", file.ReadWriteByUser); err != nil { - t.Errorf("unexpected error creating pid file: %v", err) - } - + os.MkdirAll("/tmp", file.ReadWriteByUser) pidFile, err := os.Create(nginx.PID) if err != nil { t.Fatalf("unexpected error: %v", err) @@ -92,23 +90,14 @@ func TestNginxCheck(t *testing.T) { // start dummy process to use the PID cmd := exec.Command("sleep", "3600") - if err := cmd.Start(); err != nil { - t.Errorf("unexpected error: %v", err) - } + cmd.Start() pid := cmd.Process.Pid - defer func() { - if err := cmd.Process.Kill(); err != nil { - t.Errorf("unexpected error killing the process: %v", err) - } - }() + defer cmd.Process.Kill() go func() { - cmd.Wait() //nolint:errcheck // Ignore the error + cmd.Wait() }() - if _, err := fmt.Fprintf(pidFile, "%v", pid); err != nil { - t.Errorf("unexpected error writing the pid file: %v", err) - } - + pidFile.Write([]byte(fmt.Sprintf("%v", pid))) pidFile.Close() healthz.InstallPathHandler(mux, tt.healthzPath, n) @@ -120,7 +109,7 @@ func TestNginxCheck(t *testing.T) { }) // pollute pid file - pidFile.WriteString("999999") //nolint:errcheck // Ignore the error + pidFile.Write([]byte(fmt.Sprint("999999"))) pidFile.Close() t.Run("bad pid", func(t *testing.T) { @@ -133,7 +122,7 @@ func TestNginxCheck(t *testing.T) { } func callHealthz(expErr bool, healthzPath string, mux *http.ServeMux) error { - req, err := http.NewRequest(http.MethodGet, healthzPath, http.NoBody) + req, err := http.NewRequest("GET", healthzPath, nil) if err != nil { return fmt.Errorf("healthz error: %v", err) } diff --git a/internal/ingress/controller/config/config.go b/internal/ingress/controller/config/config.go index beac1405d..f8e79e66e 100644 --- a/internal/ingress/controller/config/config.go +++ b/internal/ingress/controller/config/config.go @@ -20,16 +20,19 @@ import ( "strconv" "time" - apiv1 "k8s.io/api/core/v1" "k8s.io/klog/v2" + apiv1 "k8s.io/api/core/v1" + + "k8s.io/ingress-nginx/internal/ingress" "k8s.io/ingress-nginx/internal/ingress/defaults" - "k8s.io/ingress-nginx/pkg/apis/ingress" - "k8s.io/ingress-nginx/pkg/util/runtime" + "k8s.io/ingress-nginx/internal/runtime" ) -// EnableSSLChainCompletion Autocomplete SSL certificate chains with missing intermediate CA certificates. -var EnableSSLChainCompletion = false +var ( + // EnableSSLChainCompletion Autocomplete SSL certificate chains with missing intermediate CA certificates. + EnableSSLChainCompletion = false +) const ( // http://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size @@ -45,7 +48,7 @@ const ( // that tell browsers that it should only be communicated with using HTTPS, instead of using HTTP. // https://developer.mozilla.org/en-US/docs/Web/Security/HTTP_strict_transport_security // max-age is the time, in seconds, that the browser should remember that this site is only to be accessed using HTTPS. - hstsMaxAge = "31536000" + hstsMaxAge = "15724800" gzipTypes = "application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/javascript text/plain text/x-component" @@ -88,27 +91,7 @@ const ( // Configuration represents the content of nginx.conf file type Configuration struct { - defaults.Backend `json:",squash"` //nolint:staticcheck // Ignore unknown JSON option "squash" error - - // AllowSnippetAnnotations enable users to add their own snippets via ingress annotation. - // If disabled, only snippets added via ConfigMap are added to ingress. - AllowSnippetAnnotations bool `json:"allow-snippet-annotations"` - - // AllowCrossNamespaceResources enables users to consume cross namespace resource on annotations - // Case disabled, attempts to use secrets or configmaps from a namespace different from Ingress will - // be denied - // This value will default to `false` on future releases - AllowCrossNamespaceResources bool `json:"allow-cross-namespace-resources"` - - // AnnotationsRiskLevel represents the risk accepted on an annotation. If the risk is, for instance `Medium`, annotations - // with risk High and Critical will not be accepted. - // Default Risk is Critical by default, but this may be changed in future releases - AnnotationsRiskLevel string `json:"annotations-risk-level"` - - // AnnotationValueWordBlocklist defines words that should not be part of an user annotation value - // (can be used to run arbitrary code or configs, for example) and that should be dropped. - // This list should be separated by "," character - AnnotationValueWordBlocklist string `json:"annotation-value-word-blocklist"` + defaults.Backend `json:",squash"` // Sets the name of the configmap that contains the headers to pass to the client AddHeaders string `json:"add-headers,omitempty"` @@ -119,7 +102,7 @@ type Configuration struct { // By default this is disabled AllowBackendServerHeader bool `json:"allow-backend-server-header"` - // AccessLogParams sets additional params for access_log + // AccessLogParams sets additionals params for access_log // http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log // By default it's empty AccessLogParams string `json:"access-log-params,omitempty"` @@ -128,19 +111,15 @@ type Configuration struct { // By default this is disabled EnableAccessLogForDefaultBackend bool `json:"enable-access-log-for-default-backend"` - // EnableAuthAccessLog enable auth access log - // By default this is disabled - EnableAuthAccessLog bool `json:"enable-auth-access-log"` - // AccessLogPath sets the path of the access logs for both http and stream contexts if enabled // http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log // http://nginx.org/en/docs/stream/ngx_stream_log_module.html#access_log // By default access logs go to /var/log/nginx/access.log AccessLogPath string `json:"access-log-path,omitempty"` - // HTTPAccessLogPath sets the path of the access logs for http context globally if enabled + // HttpAccessLogPath sets the path of the access logs for http context globally if enabled // http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log - HTTPAccessLogPath string `json:"http-access-log-path,omitempty"` + HttpAccessLogPath string `json:"http-access-log-path,omitempty"` // StreamAccessLogPath sets the path of the access logs for stream context globally if enabled // http://nginx.org/en/docs/stream/ngx_stream_log_module.html#access_log @@ -227,19 +206,16 @@ type Configuration struct { // https://nginx.org/en/docs/http/ngx_http_v2_module.html#http2_max_field_size // HTTP2MaxFieldSize Limits the maximum size of an HPACK-compressed request header field - // Deprecated: HTTP2MaxFieldSize is deprecated. HTTP2MaxFieldSize string `json:"http2-max-field-size,omitempty"` // https://nginx.org/en/docs/http/ngx_http_v2_module.html#http2_max_header_size // HTTP2MaxHeaderSize Limits the maximum size of the entire request header list after HPACK decompression - // Deprecated: HTTP2MaxHeaderSize is deprecated. HTTP2MaxHeaderSize string `json:"http2-max-header-size,omitempty"` // http://nginx.org/en/docs/http/ngx_http_v2_module.html#http2_max_requests // HTTP2MaxRequests Sets the maximum number of requests (including push requests) that can be served // through one HTTP/2 connection, after which the next client request will lead to connection closing // and the need of establishing a new connection. - // Deprecated: HTTP2MaxRequests is deprecated. HTTP2MaxRequests int `json:"http2-max-requests,omitempty"` // http://nginx.org/en/docs/http/ngx_http_v2_module.html#http2_max_concurrent_streams @@ -278,10 +254,6 @@ type Configuration struct { // Default: 4 8k LargeClientHeaderBuffers string `json:"large-client-header-buffers"` - // Disable all escaping - // http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format - LogFormatEscapeNone bool `json:"log-format-escape-none,omitempty"` - // Enable json escaping // http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format LogFormatEscapeJSON bool `json:"log-format-escape-json,omitempty"` @@ -318,6 +290,11 @@ type Configuration struct { NginxStatusIpv4Whitelist []string `json:"nginx-status-ipv4-whitelist,omitempty"` NginxStatusIpv6Whitelist []string `json:"nginx-status-ipv6-whitelist,omitempty"` + // Plugins configures plugins to use placed in the directory /etc/nginx/lua/plugins. + // Every plugin has to have main.lua in the root. Every plugin has to bundle all of its dependencies. + // The execution order follows the definition. + Plugins []string `json:"plugins,omitempty"` + // If UseProxyProtocol is enabled ProxyRealIPCIDR defines the default the IP/network address // of your external load balancer ProxyRealIPCIDR []string `json:"proxy-real-ip-cidr,omitempty"` @@ -402,11 +379,6 @@ type Configuration struct { // https://www.igvita.com/2013/12/16/optimizing-nginx-tls-time-to-first-byte/ SSLBufferSize string `json:"ssl-buffer-size,omitempty"` - // https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_reject_handshake - // If enabled, SSL handshakes to an invalid virtualhost will be rejected - // Default: false - SSLRejectHandshake bool `json:"ssl-reject-handshake"` - // Enables or disables the use of the PROXY protocol to receive client connection // (real IP address) information passed through proxy servers and load balancers // such as HAproxy and Amazon Elastic Load Balancer (ELB). @@ -418,22 +390,18 @@ type Configuration struct { // Example '60s' ProxyProtocolHeaderTimeout time.Duration `json:"proxy-protocol-header-timeout,omitempty"` - // Enables or disables the directive aio_write that writes files asynchronously - // https://nginx.org/en/docs/http/ngx_http_core_module.html#aio_write - EnableAioWrite bool `json:"enable-aio-write,omitempty"` - // Enables or disables the use of the nginx module that compresses responses using the "gzip" method // http://nginx.org/en/docs/http/ngx_http_gzip_module.html UseGzip bool `json:"use-gzip,omitempty"` + // Enables or disables the use of the nginx geoip module that creates variables with values depending on the client IP + // http://nginx.org/en/docs/http/ngx_http_geoip_module.html + UseGeoIP bool `json:"use-geoip,omitempty"` + // UseGeoIP2 enables the geoip2 module for NGINX // By default this is disabled UseGeoIP2 bool `json:"use-geoip2,omitempty"` - // GeoIP2AutoReloadMinutes enables autoreload on geoip2 setting the interval in minutes - // By default this is disabled using 0 - GeoIP2AutoReloadMinutes int `json:"geoip2-autoreload-in-minutes,omitempty"` - // Enables or disables the use of the NGINX Brotli Module for compression // https://github.com/google/ngx_brotli EnableBrotli bool `json:"enable-brotli,omitempty"` @@ -441,9 +409,6 @@ type Configuration struct { // Brotli Compression Level that will be used BrotliLevel int `json:"brotli-level,omitempty"` - // Minimum length of responses, in bytes, that will be eligible for brotli compression - BrotliMinLength int `json:"brotli-min-length,omitempty"` - // MIME Types that will be compressed on-the-fly using Brotli module BrotliTypes string `json:"brotli-types,omitempty"` @@ -452,11 +417,6 @@ type Configuration struct { // Default: true UseHTTP2 bool `json:"use-http2,omitempty"` - // Disables gzipping of responses for requests with "User-Agent" header fields matching any of - // the specified regular expressions. - // http://nginx.org/en/docs/http/ngx_http_gzip_module.html#gzip_disable - GzipDisable string `json:"gzip-disable,omitempty"` - // gzip Compression Level that will be used GzipLevel int `json:"gzip-level,omitempty"` @@ -472,13 +432,6 @@ type Configuration struct { // http://nginx.org/en/docs/ngx_core_module.html#worker_processes WorkerProcesses string `json:"worker-processes,omitempty"` - // Defines whether multiple concurrent reloads of worker processes should occur. - // Set this to false to prevent more than n x 2 workers to exist at any time, to avoid potential OOM situations and high CPU load - // With this setting on false, configuration changes in the queue will be re-queued with an exponential backoff, until the number of worker process is the expected value. - // By default new worker processes are spawned every time there's a change that cannot be applied dynamically with no upper limit to the number of running workers - // http://nginx.org/en/docs/ngx_core_module.html#worker_processes - WorkerSerialReloads bool `json:"enable-serial-reloads,omitempty"` - // Defines a timeout for a graceful shutdown of worker processes // http://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout WorkerShutdownTimeout string `json:"worker-shutdown-timeout,omitempty"` @@ -498,10 +451,6 @@ type Configuration struct { // http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive UpstreamKeepaliveConnections int `json:"upstream-keepalive-connections,omitempty"` - // Sets the maximum time during which requests can be processed through one keepalive connection - // https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_time - UpstreamKeepaliveTime string `json:"upstream-keepalive-time,omitempty"` - // Sets a timeout during which an idle keepalive connection to an upstream server will stay open. // http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_timeout UpstreamKeepaliveTimeout int `json:"upstream-keepalive-timeout,omitempty"` @@ -545,14 +494,6 @@ type Configuration struct { // http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_http_version ProxyHTTPVersion string `json:"proxy-http-version"` - // Disables NGINX proxy-intercept-errors when error_page/custom-http-errors are set - // https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_intercept_errors - DisableProxyInterceptErrors bool `json:"disable-proxy-intercept-errors,omitempty"` - - // Disable absolute redirects and enables relative redirects. - // https://nginx.org/en/docs/http/ngx_http_core_module.html#absolute_redirect - RelativeRedirects bool `json:"relative-redirects"` - // Sets the ipv4 addresses on which the server will accept requests. BindAddressIpv4 []string `json:"bind-address-ipv4,omitempty"` @@ -563,7 +504,7 @@ type Configuration struct { UseForwardedHeaders bool `json:"use-forwarded-headers"` // Sets whether to enable the real ip module - EnableRealIP bool `json:"enable-real-ip"` + EnableRealIp bool `json:"enable-real-ip"` // Sets the header field for identifying the originating IP address of a client // Default is X-Forwarded-For @@ -581,56 +522,111 @@ type Configuration struct { // Default: true ProxyAddOriginalURIHeader bool `json:"proxy-add-original-uri-header"` - // EnableOpentelemetry enables the nginx Opentelemetry extension + // EnableOpentracing enables the nginx Opentracing extension + // https://github.com/opentracing-contrib/nginx-opentracing // By default this is disabled - EnableOpentelemetry bool `json:"enable-opentelemetry"` + EnableOpentracing bool `json:"enable-opentracing"` - // OpentelemetryConfig sets the opentelemetry config file - // Default: /etc/nginx/opentelemetry.toml - OpentelemetryConfig string `json:"opentelemetry-config"` + // OpentracingOperationName specifies a custom name for the server span + OpentracingOperationName string `json:"opentracing-operation-name"` - // OpentelemetryOperationName specifies a custom name for the server span - OpentelemetryOperationName string `json:"opentelemetry-operation-name"` + // OpentracingOperationName specifies a custom name for the location span + OpentracingLocationOperationName string `json:"opentracing-location-operation-name"` - // OpentelemetryTrustIncomingSpan sets whether or not to trust incoming trace spans - // If false, incoming span headers will be rejected - // Default: true - OpentelemetryTrustIncomingSpan bool `json:"opentelemetry-trust-incoming-span"` + // ZipkinCollectorHost specifies the host to use when uploading traces + ZipkinCollectorHost string `json:"zipkin-collector-host"` - // OtlpCollectorHost specifies the host to use when uploading traces - OtlpCollectorHost string `json:"otlp-collector-host"` + // ZipkinCollectorPort specifies the port to use when uploading traces + // Default: 9411 + ZipkinCollectorPort int `json:"zipkin-collector-port"` - // OtlpCollectorPort specifies the port to use when uploading traces - // Default: 4317 - OtlpCollectorPort string `json:"otlp-collector-port"` - - // OtelServiceName specifies the service name to use for any traces created + // ZipkinServiceName specifies the service name to use for any traces created // Default: nginx - OtelServiceName string `json:"otel-service-name"` + ZipkinServiceName string `json:"zipkin-service-name"` - // OtelSampler specifies the sampler to use for any traces created - // Default: AlwaysOn - OtelSampler string `json:"otel-sampler"` + // ZipkinSampleRate specifies sampling rate for traces + // Default: 1.0 + ZipkinSampleRate float32 `json:"zipkin-sample-rate"` - // OtelSamplerRatio specifies the sampler ratio to use for any traces created - // Default: 0.01 - OtelSamplerRatio float32 `json:"otel-sampler-ratio"` + // JaegerCollectorHost specifies the host to use when uploading traces + JaegerCollectorHost string `json:"jaeger-collector-host"` - // OtelSamplerParentBased specifies the parent based sampler to be used for any traces created + // JaegerCollectorPort specifies the port to use when uploading traces + // Default: 6831 + JaegerCollectorPort int `json:"jaeger-collector-port"` + + // JaegerEndpoint specifies the enpoint to use when uploading traces to a collector over TCP + JaegerEndpoint string `json:"jaeger-endpoint"` + + // JaegerServiceName specifies the service name to use for any traces created + // Default: nginx + JaegerServiceName string `json:"jaeger-service-name"` + + // JaegerPropagationFormat specifies the traceparent/tracestate propagation format + JaegerPropagationFormat string `json:"jaeger-propagation-format"` + + // JaegerSamplerType specifies the sampler to be used when sampling traces. + // The available samplers are: const, probabilistic, ratelimiting, remote + // Default: const + JaegerSamplerType string `json:"jaeger-sampler-type"` + + // JaegerSamplerParam specifies the argument to be passed to the sampler constructor + // Default: 1 + JaegerSamplerParam string `json:"jaeger-sampler-param"` + + // JaegerSamplerHost specifies the host used for remote sampling consultation + // Default: http://127.0.0.1 + JaegerSamplerHost string `json:"jaeger-sampler-host"` + + // JaegerSamplerHost specifies the host used for remote sampling consultation + // Default: 5778 + JaegerSamplerPort int `json:"jaeger-sampler-port"` + + // JaegerTraceContextHeaderName specifies the header name used for passing trace context + // Default: uber-trace-id + JaegerTraceContextHeaderName string `json:"jaeger-trace-context-header-name"` + + // JaegerDebugHeader specifies the header name used for force sampling + // Default: jaeger-debug-id + JaegerDebugHeader string `json:"jaeger-debug-header"` + + // JaegerBaggageHeader specifies the header name used to submit baggage if there is no root span + // Default: jaeger-baggage + JaegerBaggageHeader string `json:"jaeger-baggage-header"` + + // TraceBaggageHeaderPrefix specifies the header prefix used to propagate baggage + // Default: uberctx- + JaegerTraceBaggageHeaderPrefix string `json:"jaeger-tracer-baggage-header-prefix"` + + // DatadogCollectorHost specifies the datadog agent host to use when uploading traces + DatadogCollectorHost string `json:"datadog-collector-host"` + + // DatadogCollectorPort specifies the port to use when uploading traces + // Default: 8126 + DatadogCollectorPort int `json:"datadog-collector-port"` + + // DatadogEnvironment specifies the environment this trace belongs to. + // Default: prod + DatadogEnvironment string `json:"datadog-environment"` + + // DatadogServiceName specifies the service name to use for any traces created + // Default: nginx + DatadogServiceName string `json:"datadog-service-name"` + + // DatadogOperationNameOverride overrides the operation naem to use for any traces crated + // Default: nginx.handle + DatadogOperationNameOverride string `json:"datadog-operation-name-override"` + + // DatadogPrioritySampling specifies to use client-side sampling + // If true disables client-side sampling (thus ignoring sample_rate) and enables distributed + // priority sampling, where traces are sampled based on a combination of user-assigned // Default: true - OtelSamplerParentBased bool `json:"otel-sampler-parent-based"` + DatadogPrioritySampling bool `json:"datadog-priority-sampling"` - // MaxQueueSize specifies the max queue size for uploading traces - // Default: 2048 - OtelMaxQueueSize int32 `json:"otel-max-queuesize"` - - // ScheduleDelayMillis specifies the max delay between uploading traces - // Default: 5000 - OtelScheduleDelayMillis int32 `json:"otel-schedule-delay-millis"` - - // MaxExportBatchSize specifies the max export batch size to used when uploading traces - // Default: 512 - OtelMaxExportBatchSize int32 `json:"otel-max-export-batch-size"` + // DatadogSampleRate specifies sample rate for any traces created. + // This is effective only when datadog-priority-sampling is false + // Default: 1.0 + DatadogSampleRate float32 `json:"datadog-sample-rate"` // MainSnippet adds custom configuration to the main section of the nginx configuration MainSnippet string `json:"main-snippet"` @@ -641,9 +637,6 @@ type Configuration struct { // ServerSnippet adds custom configuration to all the servers in the nginx configuration ServerSnippet string `json:"server-snippet"` - // StreamSnippet adds custom configuration to the stream section of the nginx configuration - StreamSnippet string `json:"stream-snippet"` - // LocationSnippet adds custom configuration to all the locations in the nginx configuration LocationSnippet string `json:"location-snippet"` @@ -713,7 +706,7 @@ type Configuration struct { DefaultSSLCertificate *ingress.SSLCert `json:"-"` // ProxySSLLocationOnly controls whether the proxy-ssl parameters defined in the - // proxy-ssl-* annotations are applied on location level only in the nginx.conf file + // proxy-ssl-* annotations are applied on on location level only in the nginx.conf file // Default is that those are applied on server level, too ProxySSLLocationOnly bool `json:"proxy-ssl-location-only"` @@ -722,21 +715,30 @@ type Configuration struct { // Default: text/html DefaultType string `json:"default-type"` - // DebugConnections Enables debugging log for selected client connections - // http://nginx.org/en/docs/ngx_core_module.html#debug_connection - // Default: "" - DebugConnections []string `json:"debug-connections"` + // GlobalRateLimitMemcachedHost configures memcached host. + GlobalRateLimitMemcachedHost string `json:"global-rate-limit-memcached-host"` - // StrictValidatePathType enable the strict validation of Ingress Paths - // It enforces that pathType of type Exact or Prefix should start with / and contain only - // alphanumeric chars, "-", "_", "/".In case of additional characters, - // like used on Rewrite configurations the user should use pathType as ImplementationSpecific - StrictValidatePathType bool `json:"strict-validate-path-type"` + // GlobalRateLimitMemcachedPort configures memcached port. + GlobalRateLimitMemcachedPort int `json:"global-rate-limit-memcached-port"` - // GRPCBufferSizeKb Sets the size of the buffer used for reading the response received - // from the gRPC server. The response is passed to the client synchronously, - // as soon as it is received. - GRPCBufferSizeKb int `json:"grpc-buffer-size-kb"` + // GlobalRateLimitMemcachedConnectTimeout configures timeout when connecting to memcached. + // The unit is millisecond. + GlobalRateLimitMemcachedConnectTimeout int `json:"global-rate-limit-memcached-connect-timeout"` + + // GlobalRateLimitMemcachedMaxIdleTimeout configured how long connections + // should be kept alive in idle state. The unit is millisecond. + GlobalRateLimitMemcachedMaxIdleTimeout int `json:"global-rate-limit-memcached-max-idle-timeout"` + + // GlobalRateLimitMemcachedPoolSize configures how many connections + // should be kept alive in the pool. + // Note that this is per NGINX worker. Make sure your memcached server can + // handle `MemcachedPoolSize * * ` + // simultaneous connections. + GlobalRateLimitMemcachedPoolSize int `json:"global-rate-limit-memcached-pool-size"` + + // GlobalRateLimitStatucCode determines the HTTP status code to return + // when limit is exceeding during global rate limiting. + GlobalRateLimitStatucCode int `json:"global-rate-limit-status-code"` } // NewDefault returns the default nginx configuration @@ -747,29 +749,24 @@ func NewDefault() Configuration { defNginxStatusIpv4Whitelist := make([]string, 0) defNginxStatusIpv6Whitelist := make([]string, 0) defResponseHeaders := make([]string, 0) + defIPCIDR = append(defIPCIDR, "0.0.0.0/0") defNginxStatusIpv4Whitelist = append(defNginxStatusIpv4Whitelist, "127.0.0.1") defNginxStatusIpv6Whitelist = append(defNginxStatusIpv6Whitelist, "::1") defProxyDeadlineDuration := time.Duration(5) * time.Second - defGlobalExternalAuth := GlobalExternalAuth{"", "", "", "", "", append(defResponseHeaders, ""), "", "", "", []string{}, map[string]string{}, false} + defGlobalExternalAuth := GlobalExternalAuth{"", "", "", "", "", append(defResponseHeaders, ""), "", "", "", []string{}, map[string]string{}} cfg := Configuration{ - AllowSnippetAnnotations: false, - AllowCrossNamespaceResources: false, AllowBackendServerHeader: false, - AnnotationValueWordBlocklist: "", - AnnotationsRiskLevel: "High", AccessLogPath: "/var/log/nginx/access.log", AccessLogParams: "", EnableAccessLogForDefaultBackend: false, - EnableAuthAccessLog: false, WorkerCPUAffinity: "", ErrorLogPath: "/var/log/nginx/error.log", BlockCIDRs: defBlockEntity, BlockUserAgents: defBlockEntity, BlockReferers: defBlockEntity, BrotliLevel: 4, - BrotliMinLength: 20, BrotliTypes: brotliTypes, ClientHeaderBufferSize: "1k", ClientHeaderTimeout: 60, @@ -778,14 +775,14 @@ func NewDefault() Configuration { EnableUnderscoresInHeaders: false, ErrorLogLevel: errorLevel, UseForwardedHeaders: false, - EnableRealIP: false, + EnableRealIp: false, ForwardedForHeader: "X-Forwarded-For", ComputeFullForwardedFor: false, ProxyAddOriginalURIHeader: false, GenerateRequestID: true, - HTTP2MaxFieldSize: "", - HTTP2MaxHeaderSize: "", - HTTP2MaxRequests: 0, + HTTP2MaxFieldSize: "4k", + HTTP2MaxHeaderSize: "16k", + HTTP2MaxRequests: 1000, HTTP2MaxConcurrentStreams: 128, HTTPRedirectCode: 308, HSTS: true, @@ -797,7 +794,7 @@ func NewDefault() Configuration { GzipMinLength: 256, GzipTypes: gzipTypes, KeepAlive: 75, - KeepAliveRequests: 1000, + KeepAliveRequests: 100, LargeClientHeaderBuffers: "4 8k", LogFormatEscapeJSON: false, LogFormatStream: logFormatStream, @@ -821,88 +818,84 @@ func NewDefault() Configuration { SSLECDHCurve: "auto", SSLProtocols: sslProtocols, SSLEarlyData: sslEarlyData, - SSLRejectHandshake: false, SSLSessionCache: true, SSLSessionCacheSize: sslSessionCacheSize, SSLSessionTickets: false, SSLSessionTimeout: sslSessionTimeout, EnableBrotli: false, - EnableAioWrite: true, UseGzip: false, + UseGeoIP: true, UseGeoIP2: false, - GeoIP2AutoReloadMinutes: 0, WorkerProcesses: strconv.Itoa(runtime.NumCPU()), - WorkerSerialReloads: false, WorkerShutdownTimeout: "240s", VariablesHashBucketSize: 256, VariablesHashMaxSize: 2048, UseHTTP2: true, - DisableProxyInterceptErrors: false, - RelativeRedirects: false, ProxyStreamTimeout: "600s", ProxyStreamNextUpstream: true, ProxyStreamNextUpstreamTimeout: "600s", ProxyStreamNextUpstreamTries: 3, Backend: defaults.Backend{ - ProxyBodySize: bodySize, - ProxyConnectTimeout: 5, - ProxyReadTimeout: 60, - ProxySendTimeout: 60, - ProxyBuffersNumber: 4, - ProxyBufferSize: "4k", - ProxyBusyBuffersSize: "8k", - ProxyCookieDomain: "off", - ProxyCookiePath: "off", - ProxyNextUpstream: "error timeout", - ProxyNextUpstreamTimeout: 0, - ProxyNextUpstreamTries: 3, - ProxyRequestBuffering: "on", - ProxyRedirectFrom: "off", - ProxyRedirectTo: "off", - PreserveTrailingSlash: false, - SSLRedirect: true, - CustomHTTPErrors: []int{}, - DisableProxyInterceptErrors: false, - RelativeRedirects: false, - DenylistSourceRange: []string{}, - WhitelistSourceRange: []string{}, - SkipAccessLogURLs: []string{}, - LimitRate: 0, - LimitRateAfter: 0, - ProxyBuffering: "off", - ProxyHTTPVersion: "1.1", - ProxyMaxTempFileSize: "1024m", - ServiceUpstream: false, - AllowedResponseHeaders: []string{}, + ProxyBodySize: bodySize, + ProxyConnectTimeout: 5, + ProxyReadTimeout: 60, + ProxySendTimeout: 60, + ProxyBuffersNumber: 4, + ProxyBufferSize: "4k", + ProxyCookieDomain: "off", + ProxyCookiePath: "off", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: 0, + ProxyNextUpstreamTries: 3, + ProxyRequestBuffering: "on", + ProxyRedirectFrom: "off", + ProxyRedirectTo: "off", + PreserveTrailingSlash: false, + SSLRedirect: true, + CustomHTTPErrors: []int{}, + WhitelistSourceRange: []string{}, + SkipAccessLogURLs: []string{}, + LimitRate: 0, + LimitRateAfter: 0, + ProxyBuffering: "off", + ProxyHTTPVersion: "1.1", + ProxyMaxTempFileSize: "1024m", }, - UpstreamKeepaliveConnections: 320, - UpstreamKeepaliveTime: "1h", - UpstreamKeepaliveTimeout: 60, - UpstreamKeepaliveRequests: 10000, - LimitConnZoneVariable: defaultLimitConnZoneVariable, - BindAddressIpv4: defBindAddress, - BindAddressIpv6: defBindAddress, - OpentelemetryTrustIncomingSpan: true, - OpentelemetryConfig: "/etc/ingress-controller/telemetry/opentelemetry.toml", - OtlpCollectorPort: "4317", - OtelServiceName: "nginx", - OtelSampler: "AlwaysOn", - OtelSamplerRatio: 0.01, - OtelSamplerParentBased: true, - OtelScheduleDelayMillis: 5000, - OtelMaxExportBatchSize: 512, - OtelMaxQueueSize: 2048, - LimitReqStatusCode: 503, - LimitConnStatusCode: 503, - SyslogPort: 514, - NoTLSRedirectLocations: "/.well-known/acme-challenge", - NoAuthLocations: "/.well-known/acme-challenge", - GlobalExternalAuth: defGlobalExternalAuth, - ProxySSLLocationOnly: false, - DefaultType: "text/html", - DebugConnections: []string{}, - StrictValidatePathType: true, - GRPCBufferSizeKb: 0, + UpstreamKeepaliveConnections: 320, + UpstreamKeepaliveTimeout: 60, + UpstreamKeepaliveRequests: 10000, + LimitConnZoneVariable: defaultLimitConnZoneVariable, + BindAddressIpv4: defBindAddress, + BindAddressIpv6: defBindAddress, + ZipkinCollectorPort: 9411, + ZipkinServiceName: "nginx", + ZipkinSampleRate: 1.0, + JaegerCollectorPort: 6831, + JaegerPropagationFormat: "jaeger", + JaegerServiceName: "nginx", + JaegerSamplerType: "const", + JaegerSamplerParam: "1", + JaegerSamplerPort: 5778, + JaegerSamplerHost: "http://127.0.0.1", + DatadogServiceName: "nginx", + DatadogEnvironment: "prod", + DatadogCollectorPort: 8126, + DatadogOperationNameOverride: "nginx.handle", + DatadogSampleRate: 1.0, + DatadogPrioritySampling: true, + LimitReqStatusCode: 503, + LimitConnStatusCode: 503, + SyslogPort: 514, + NoTLSRedirectLocations: "/.well-known/acme-challenge", + NoAuthLocations: "/.well-known/acme-challenge", + GlobalExternalAuth: defGlobalExternalAuth, + ProxySSLLocationOnly: false, + DefaultType: "text/html", + GlobalRateLimitMemcachedPort: 11211, + GlobalRateLimitMemcachedConnectTimeout: 50, + GlobalRateLimitMemcachedMaxIdleTimeout: 10000, + GlobalRateLimitMemcachedPoolSize: 50, + GlobalRateLimitStatucCode: 429, } if klog.V(5).Enabled() { @@ -914,41 +907,41 @@ func NewDefault() Configuration { // TemplateConfig contains the nginx configuration to render the file nginx.conf type TemplateConfig struct { - ProxySetHeaders map[string]string `json:"ProxySetHeaders"` - AddHeaders map[string]string `json:"AddHeaders"` - BacklogSize int `json:"BacklogSize"` - Backends []*ingress.Backend `json:"Backends"` - PassthroughBackends []*ingress.SSLPassthroughBackend `json:"PassthroughBackends"` - Servers []*ingress.Server `json:"Servers"` - TCPBackends []ingress.L4Service `json:"TCPBackends"` - UDPBackends []ingress.L4Service `json:"UDPBackends"` - HealthzURI string `json:"HealthzURI"` - Cfg Configuration `json:"Cfg"` - IsIPV6Enabled bool `json:"IsIPV6Enabled"` - IsSSLPassthroughEnabled bool `json:"IsSSLPassthroughEnabled"` - NginxStatusIpv4Whitelist []string `json:"NginxStatusIpv4Whitelist"` - NginxStatusIpv6Whitelist []string `json:"NginxStatusIpv6Whitelist"` - RedirectServers interface{} `json:"RedirectServers"` - ListenPorts *ListenPorts `json:"ListenPorts"` - PublishService *apiv1.Service `json:"PublishService"` - EnableMetrics bool `json:"EnableMetrics"` - MaxmindEditionFiles *[]string `json:"MaxmindEditionFiles"` - MonitorMaxBatchSize int `json:"MonitorMaxBatchSize"` - PID string `json:"PID"` - StatusPath string `json:"StatusPath"` - StatusPort int `json:"StatusPort"` - StreamPort int `json:"StreamPort"` - StreamSnippets []string `json:"StreamSnippets"` + ProxySetHeaders map[string]string + AddHeaders map[string]string + BacklogSize int + Backends []*ingress.Backend + PassthroughBackends []*ingress.SSLPassthroughBackend + Servers []*ingress.Server + TCPBackends []ingress.L4Service + UDPBackends []ingress.L4Service + HealthzURI string + Cfg Configuration + IsIPV6Enabled bool + IsSSLPassthroughEnabled bool + NginxStatusIpv4Whitelist []string + NginxStatusIpv6Whitelist []string + RedirectServers interface{} + ListenPorts *ListenPorts + PublishService *apiv1.Service + EnableMetrics bool + MaxmindEditionFiles []string + MonitorMaxBatchSize int + + PID string + StatusPath string + StatusPort int + StreamPort int } // ListenPorts describe the ports required to run the // NGINX Ingress controller type ListenPorts struct { - HTTP int `json:"HTTP"` - HTTPS int `json:"HTTPS"` - Health int `json:"Health"` - Default int `json:"Default"` - SSLProxy int `json:"SSLProxy"` + HTTP int + HTTPS int + Health int + Default int + SSLProxy int } // GlobalExternalAuth describe external authentication configuration for the @@ -966,5 +959,4 @@ type GlobalExternalAuth struct { AuthCacheKey string `json:"authCacheKey"` AuthCacheDuration []string `json:"authCacheDuration"` ProxySetHeaders map[string]string `json:"proxySetHeaders,omitempty"` - AlwaysSetCookie bool `json:"alwaysSetCookie,omitempty"` } diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go index 652a80e49..f17f73ef3 100644 --- a/internal/ingress/controller/controller.go +++ b/internal/ingress/controller/controller.go @@ -23,40 +23,32 @@ import ( "strings" "time" - "github.com/mitchellh/hashstructure/v2" + "github.com/mitchellh/hashstructure" apiv1 "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" apiequality "k8s.io/apimachinery/pkg/api/equality" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + "k8s.io/ingress-nginx/internal/ingress" "k8s.io/ingress-nginx/internal/ingress/annotations" - "k8s.io/ingress-nginx/internal/ingress/annotations/canary" + "k8s.io/ingress-nginx/internal/ingress/annotations/class" "k8s.io/ingress-nginx/internal/ingress/annotations/log" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/annotations/proxy" ngx_config "k8s.io/ingress-nginx/internal/ingress/controller/config" - "k8s.io/ingress-nginx/internal/ingress/controller/ingressclass" "k8s.io/ingress-nginx/internal/ingress/controller/store" "k8s.io/ingress-nginx/internal/ingress/errors" - "k8s.io/ingress-nginx/internal/ingress/inspector" - "k8s.io/ingress-nginx/internal/ingress/metric/collectors" "k8s.io/ingress-nginx/internal/k8s" "k8s.io/ingress-nginx/internal/nginx" - "k8s.io/ingress-nginx/pkg/apis/ingress" - utilingress "k8s.io/ingress-nginx/pkg/util/ingress" "k8s.io/klog/v2" ) const ( - defUpstreamName = "upstream-default-backend" - defServerName = "_" - rootLocation = "/" - emptyZone = "" - orphanMetricLabelNoService = "no-service" - orphanMetricLabelNoEndpoint = "no-endpoint" + defUpstreamName = "upstream-default-backend" + defServerName = "_" + rootLocation = "/" ) // Configuration contains all the settings required by an Ingress controller @@ -75,8 +67,6 @@ type Configuration struct { Namespace string - WatchNamespaceSelector labels.Selector - // +optional TCPConfigMapName string // +optional @@ -91,28 +81,16 @@ type Configuration struct { UpdateStatus bool UseNodeInternalIP bool ElectionID string - ElectionTTL time.Duration UpdateStatusOnShutdown bool - HealthCheckHost string - ListenPorts *ngx_config.ListenPorts - - DisableServiceExternalName bool + ListenPorts *ngx_config.ListenPorts EnableSSLPassthrough bool - DisableLeaderElection bool - EnableProfiling bool - EnableMetrics bool - MetricsPerHost bool - MetricsPerUndefinedHost bool - MetricsBuckets *collectors.HistogramBuckets - MetricsBucketFactor float64 - MetricsMaxBuckets uint32 - ReportStatusClasses bool - ExcludeSocketMetrics []string + EnableMetrics bool + MetricsPerHost bool FakeCertificate *ingress.SSLCert @@ -120,47 +98,20 @@ type Configuration struct { DisableCatchAll bool - IngressClassConfiguration *ingressclass.Configuration - ValidationWebhook string ValidationWebhookCertPath string ValidationWebhookKeyPath string - DisableFullValidationTest bool GlobalExternalAuth *ngx_config.GlobalExternalAuth - MaxmindEditionFiles *[]string + MaxmindEditionFiles []string MonitorMaxBatchSize int - PostShutdownGracePeriod int - ShutdownGracePeriod int - - InternalLoggerAddress string - IsChroot bool - DeepInspector bool - - DynamicConfigurationRetries int - - DisableSyncEvents bool - - EnableTopologyAwareRouting bool -} - -func getIngressPodZone(svc *apiv1.Service) string { - svcKey := k8s.MetaNamespaceKey(svc) - if svcZoneAnnotation, ok := svc.ObjectMeta.GetAnnotations()[apiv1.AnnotationTopologyMode]; ok { - if strings.EqualFold(svcZoneAnnotation, "auto") { - if foundZone, ok := k8s.IngressNodeDetails.GetLabels()[apiv1.LabelTopologyZone]; ok { - klog.V(3).Infof("Svc has topology aware annotation enabled, try to use zone %q where controller pod is running for Service %q ", foundZone, svcKey) - return foundZone - } - } - } - return emptyZone + ShutdownGracePeriod int } // GetPublishService returns the Service used to set the load-balancer status of Ingresses. -func (n *NGINXController) GetPublishService() *apiv1.Service { +func (n NGINXController) GetPublishService() *apiv1.Service { s, err := n.store.GetService(n.cfg.PublishService) if err != nil { return nil @@ -183,7 +134,6 @@ func (n *NGINXController) syncIngress(interface{}) error { hosts, servers, pcfg := n.getConfiguration(ings) n.metricCollector.SetSSLExpireTime(servers) - n.metricCollector.SetSSLInfo(servers) if n.runningConfig.Equal(pcfg) { klog.V(3).Infof("No configuration change detected, skipping backend reload") @@ -192,19 +142,16 @@ func (n *NGINXController) syncIngress(interface{}) error { n.metricCollector.SetHosts(hosts) - if !utilingress.IsDynamicConfigurationEnough(pcfg, n.runningConfig) { + if !n.IsDynamicConfigurationEnough(pcfg) { klog.InfoS("Configuration changes detected, backend reload required") - hash, err := hashstructure.Hash(pcfg, hashstructure.FormatV1, &hashstructure.HashOptions{ + hash, _ := hashstructure.Hash(pcfg, &hashstructure.HashOptions{ TagName: "json", }) - if err != nil { - klog.Errorf("unexpected error hashing configuration: %v", err) - } pcfg.ConfigurationChecksum = fmt.Sprintf("%v", hash) - err = n.OnUpdate(*pcfg) + err := n.OnUpdate(*pcfg) if err != nil { n.metricCollector.IncReloadErrorCount() n.metricCollector.ConfigSuccess(hash, false) @@ -229,24 +176,19 @@ func (n *NGINXController) syncIngress(interface{}) error { } retry := wait.Backoff{ - Steps: 1 + n.cfg.DynamicConfigurationRetries, - Duration: time.Second, - Factor: 1.3, + Steps: 15, + Duration: 1 * time.Second, + Factor: 0.8, Jitter: 0.1, } - retriesRemaining := retry.Steps err := wait.ExponentialBackoff(retry, func() (bool, error) { err := n.configureDynamically(pcfg) if err == nil { klog.V(2).Infof("Dynamic reconfiguration succeeded.") return true, nil } - retriesRemaining-- - if retriesRemaining > 0 { - klog.Warningf("Dynamic reconfiguration failed (retrying; %d retries left): %v", retriesRemaining, err) - return false, nil - } + klog.Warningf("Dynamic reconfiguration failed: %v", err) return false, err }) @@ -255,85 +197,25 @@ func (n *NGINXController) syncIngress(interface{}) error { return err } - ri := utilingress.GetRemovedIngresses(n.runningConfig, pcfg) - rc := utilingress.GetRemovedCertificateSerialNumbers(n.runningConfig, pcfg) - n.metricCollector.RemoveMetrics(ri, rc) + ri := getRemovedIngresses(n.runningConfig, pcfg) + re := getRemovedHosts(n.runningConfig, pcfg) + n.metricCollector.RemoveMetrics(ri, re) n.runningConfig = pcfg return nil } -// GetWarnings returns a list of warnings an Ingress gets when being created. -// The warnings are going to be used in an admission webhook, and they represent -// a list of messages that users need to be aware (like deprecation notices) -// when creating a new ingress object -func (n *NGINXController) CheckWarning(ing *networking.Ingress) ([]string, error) { - warnings := make([]string, 0) - - deprecatedAnnotations := sets.NewString() - deprecatedAnnotations.Insert( - "enable-influxdb", - "influxdb-measurement", - "influxdb-port", - "influxdb-host", - "influxdb-server-name", - "secure-verify-ca-secret", - ) - - // Skip checks if the ingress is marked as deleted - if !ing.DeletionTimestamp.IsZero() { - return warnings, nil - } - - anns := ing.GetAnnotations() - for k := range anns { - trimmedkey := strings.TrimPrefix(k, parser.AnnotationsPrefix+"/") - if deprecatedAnnotations.Has(trimmedkey) { - warnings = append(warnings, fmt.Sprintf("annotation %s is deprecated", k)) - } - } - - // Add each validation as a single warning - // rikatz: I know this is somehow a duplicated code from CheckIngress, but my goal was to deliver fast warning on this behavior. We - // can and should, tho, simplify this in the near future - if err := inspector.ValidatePathType(ing); err != nil { - if errs, is := err.(interface{ Unwrap() []error }); is { - for _, errW := range errs.Unwrap() { - warnings = append(warnings, errW.Error()) - } - } else { - warnings = append(warnings, err.Error()) - } - } - - return warnings, nil -} - // CheckIngress returns an error in case the provided ingress, when added // to the current configuration, generates an invalid configuration func (n *NGINXController) CheckIngress(ing *networking.Ingress) error { - startCheck := time.Now().UnixNano() / 1000000 - if ing == nil { // no ingress to add, no state change return nil } - // Skip checks if the ingress is marked as deleted - if !ing.DeletionTimestamp.IsZero() { - return nil - } - - if n.cfg.DeepInspector { - if err := inspector.DeepInspect(ing); err != nil { - return fmt.Errorf("invalid object: %w", err) - } - } - - // Do not attempt to validate an ingress that's not meant to be controlled by the current instance of the controller. - if ingressClass, err := n.store.GetIngressClass(ing, n.cfg.IngressClassConfiguration); ingressClass == "" { - klog.Warningf("ignoring ingress %v in %v based on annotation %v: %v", ing.Name, ing.ObjectMeta.Namespace, ingressClass, err) + if !class.IsValid(ing) { + klog.Warningf("ignoring ingress %v in %v based on annotation %v", ing.Name, ing.ObjectMeta.Namespace, class.IngressKey) return nil } @@ -342,48 +224,31 @@ func (n *NGINXController) CheckIngress(ing *networking.Ingress) error { return nil } - if n.cfg.DisableCatchAll && ing.Spec.DefaultBackend != nil { - return fmt.Errorf("this deployment is trying to create a catch-all ingress while DisableCatchAll flag is set to true. Remove '.spec.defaultBackend' or set DisableCatchAll flag to false") - } - startRender := time.Now().UnixNano() / 1000000 - cfg := n.store.GetBackendConfiguration() - cfg.Resolver = n.resolver - - // Adds the pathType Validation - if cfg.StrictValidatePathType { - if err := inspector.ValidatePathType(ing); err != nil { - return fmt.Errorf("ingress contains invalid paths: %w", err) - } + if n.cfg.DisableCatchAll && ing.Spec.Backend != nil { + return fmt.Errorf("This deployment is trying to create a catch-all ingress while DisableCatchAll flag is set to true. Remove '.spec.backend' or set DisableCatchAll flag to false.") } - var arrayBadWords []string - - if cfg.AnnotationValueWordBlocklist != "" { - arrayBadWords = strings.Split(strings.TrimSpace(cfg.AnnotationValueWordBlocklist), ",") - } - - for key, value := range ing.ObjectMeta.GetAnnotations() { - if parser.AnnotationsPrefix != parser.DefaultAnnotationsPrefix { + if parser.AnnotationsPrefix != parser.DefaultAnnotationsPrefix { + for key := range ing.ObjectMeta.GetAnnotations() { if strings.HasPrefix(key, fmt.Sprintf("%s/", parser.DefaultAnnotationsPrefix)) { - return fmt.Errorf("this deployment has a custom annotation prefix defined. Use '%s' instead of '%s'", parser.AnnotationsPrefix, parser.DefaultAnnotationsPrefix) + return fmt.Errorf("This deployment has a custom annotation prefix defined. Use '%s' instead of '%s'", parser.AnnotationsPrefix, parser.DefaultAnnotationsPrefix) } } - - if strings.HasPrefix(key, fmt.Sprintf("%s/", parser.AnnotationsPrefix)) && len(arrayBadWords) != 0 { - for _, forbiddenvalue := range arrayBadWords { - if strings.Contains(value, strings.TrimSpace(forbiddenvalue)) { - return fmt.Errorf("%s annotation contains invalid word %s", key, forbiddenvalue) - } - } - } - - if !cfg.AllowSnippetAnnotations && strings.HasSuffix(key, "-snippet") { - return fmt.Errorf("%s annotation cannot be used. Snippet directives are disabled by the Ingress administrator", key) - } } k8s.SetDefaultNGINXPathType(ing) + cfg := n.store.GetBackendConfiguration() + cfg.Resolver = n.resolver + + if len(cfg.GlobalRateLimitMemcachedHost) == 0 { + for key := range ing.ObjectMeta.GetAnnotations() { + if strings.HasPrefix(key, fmt.Sprintf("%s/%s", parser.AnnotationsPrefix, "global-rate-limit")) { + return fmt.Errorf("'global-rate-limit*' annotations require 'global-rate-limit-memcached-host' settings configured in the global configmap") + } + } + } + allIngresses := n.store.ListIngresses() filter := func(toCheck *ingress.Ingress) bool { @@ -391,28 +256,18 @@ func (n *NGINXController) CheckIngress(ing *networking.Ingress) error { toCheck.ObjectMeta.Name == ing.ObjectMeta.Name } ings := store.FilterIngresses(allIngresses, filter) - parsed, err := annotations.NewAnnotationExtractor(n.store).Extract(ing) - if err != nil { - n.metricCollector.IncCheckErrorCount(ing.ObjectMeta.Namespace, ing.Name) - return err - } ings = append(ings, &ingress.Ingress{ Ingress: *ing, - ParsedAnnotations: parsed, + ParsedAnnotations: annotations.NewAnnotationExtractor(n.store).Extract(ing), }) - startTest := time.Now().UnixNano() / 1000000 + _, servers, pcfg := n.getConfiguration(ings) - err = checkOverlap(ing, servers) + err := checkOverlap(ing, allIngresses, servers) if err != nil { n.metricCollector.IncCheckErrorCount(ing.ObjectMeta.Namespace, ing.Name) return err } - testedSize := len(ings) - if n.cfg.DisableFullValidationTest { - _, _, pcfg = n.getConfiguration(ings[len(ings)-1:]) - testedSize = 1 - } content, err := n.generateTemplate(cfg, *pcfg) if err != nil { @@ -425,16 +280,8 @@ func (n *NGINXController) CheckIngress(ing *networking.Ingress) error { n.metricCollector.IncCheckErrorCount(ing.ObjectMeta.Namespace, ing.Name) return err } + n.metricCollector.IncCheckCount(ing.ObjectMeta.Namespace, ing.Name) - endCheck := time.Now().UnixNano() / 1000000 - n.metricCollector.SetAdmissionMetrics( - float64(testedSize), - float64(endCheck-startTest)/1000, - float64(len(ings)), - float64(startTest-startRender)/1000, - float64(len(content)), - float64(endCheck-startCheck)/1000, - ) return nil } @@ -454,7 +301,7 @@ func (n *NGINXController) getStreamServices(configmapName string, proto apiv1.Pr return []ingress.L4Service{} } - svcs := make([]ingress.L4Service, 0, len(configmap.Data)) + var svcs []ingress.L4Service var svcProxyProtocol ingress.ProxyProtocol rp := []int{ @@ -491,10 +338,10 @@ func (n *NGINXController) getStreamServices(configmapName string, proto apiv1.Pr svcProxyProtocol.Encode = false // Proxy Protocol is only compatible with TCP Services if len(nsSvcPort) >= 3 && proto == apiv1.ProtocolTCP { - if len(nsSvcPort) >= 3 && strings.EqualFold(nsSvcPort[2], "PROXY") { + if len(nsSvcPort) >= 3 && strings.ToUpper(nsSvcPort[2]) == "PROXY" { svcProxyProtocol.Decode = true } - if len(nsSvcPort) == 4 && strings.EqualFold(nsSvcPort[3], "PROXY") { + if len(nsSvcPort) == 4 && strings.ToUpper(nsSvcPort[3]) == "PROXY" { svcProxyProtocol.Encode = true } } @@ -511,13 +358,6 @@ func (n *NGINXController) getStreamServices(configmapName string, proto apiv1.Pr var endps []ingress.Endpoint /* #nosec */ targetPort, err := strconv.Atoi(svcPort) // #nosec - var zone string - if n.cfg.EnableTopologyAwareRouting { - zone = getIngressPodZone(svc) - } else { - zone = emptyZone - } - if err != nil { // not a port number, fall back to using port name klog.V(3).Infof("Searching Endpoints with %v port name %q for Service %q", proto, svcPort, nsName) @@ -525,7 +365,7 @@ func (n *NGINXController) getStreamServices(configmapName string, proto apiv1.Pr sp := svc.Spec.Ports[i] if sp.Name == svcPort { if sp.Protocol == proto { - endps = getEndpointsFromSlices(svc, &sp, proto, zone, n.store.GetServiceEndpointsSlices) + endps = getEndpoints(svc, &sp, proto, n.store.GetServiceEndpoints) break } } @@ -534,10 +374,9 @@ func (n *NGINXController) getStreamServices(configmapName string, proto apiv1.Pr klog.V(3).Infof("Searching Endpoints with %v port number %d for Service %q", proto, targetPort, nsName) for i := range svc.Spec.Ports { sp := svc.Spec.Ports[i] - //nolint:gosec // Ignore G109 error if sp.Port == int32(targetPort) { if sp.Protocol == proto { - endps = getEndpointsFromSlices(svc, &sp, proto, zone, n.store.GetServiceEndpointsSlices) + endps = getEndpoints(svc, &sp, proto, n.store.GetServiceEndpoints) break } } @@ -577,7 +416,7 @@ func (n *NGINXController) getDefaultUpstream() *ingress.Backend { } svcKey := n.cfg.DefaultService - if svcKey == "" { + if len(svcKey) == 0 { upstream.Endpoints = append(upstream.Endpoints, n.DefaultEndpoint()) return upstream } @@ -588,13 +427,8 @@ func (n *NGINXController) getDefaultUpstream() *ingress.Backend { upstream.Endpoints = append(upstream.Endpoints, n.DefaultEndpoint()) return upstream } - var zone string - if n.cfg.EnableTopologyAwareRouting { - zone = getIngressPodZone(svc) - } else { - zone = emptyZone - } - endps := getEndpointsFromSlices(svc, &svc.Spec.Ports[0], apiv1.ProtocolTCP, zone, n.store.GetServiceEndpointsSlices) + + endps := getEndpoints(svc, &svc.Spec.Ports[0], apiv1.ProtocolTCP, n.store.GetServiceEndpoints) if len(endps) == 0 { klog.Warningf("Service %q does not have any active Endpoint", svcKey) endps = []ingress.Endpoint{n.DefaultEndpoint()} @@ -606,11 +440,11 @@ func (n *NGINXController) getDefaultUpstream() *ingress.Backend { } // getConfiguration returns the configuration matching the standard kubernetes ingress -func (n *NGINXController) getConfiguration(ingresses []*ingress.Ingress) (sets.Set[string], []*ingress.Server, *ingress.Configuration) { +func (n *NGINXController) getConfiguration(ingresses []*ingress.Ingress) (sets.String, []*ingress.Server, *ingress.Configuration) { upstreams, servers := n.getBackendServers(ingresses) var passUpstreams []*ingress.SSLPassthroughBackend - hosts := sets.New[string]() + hosts := sets.NewString() for _, server := range servers { // If a location is defined by a prefix string that ends with the slash character, and requests are processed by one of @@ -664,43 +498,12 @@ func (n *NGINXController) getConfiguration(ingresses []*ingress.Ingress) (sets.S PassthroughBackends: passUpstreams, BackendConfigChecksum: n.store.GetBackendConfiguration().Checksum, DefaultSSLCertificate: n.getDefaultSSLCertificate(), - StreamSnippets: n.getStreamSnippets(ingresses), - } -} - -func dropSnippetDirectives(anns *annotations.Ingress, ingKey string) { - if anns != nil { - if anns.ConfigurationSnippet != "" { - klog.V(3).Infof("Ingress %q tried to use configuration-snippet and the annotation is disabled by the admin. Removing the annotation", ingKey) - anns.ConfigurationSnippet = "" - } - if anns.ServerSnippet != "" { - klog.V(3).Infof("Ingress %q tried to use server-snippet and the annotation is disabled by the admin. Removing the annotation", ingKey) - anns.ServerSnippet = "" - } - - if anns.ModSecurity.Snippet != "" { - klog.V(3).Infof("Ingress %q tried to use modsecurity-snippet and the annotation is disabled by the admin. Removing the annotation", ingKey) - anns.ModSecurity.Snippet = "" - } - - if anns.ExternalAuth.AuthSnippet != "" { - klog.V(3).Infof("Ingress %q tried to use auth-snippet and the annotation is disabled by the admin. Removing the annotation", ingKey) - anns.ExternalAuth.AuthSnippet = "" - } - - if anns.StreamSnippet != "" { - klog.V(3).Infof("Ingress %q tried to use stream-snippet and the annotation is disabled by the admin. Removing the annotation", ingKey) - anns.StreamSnippet = "" - } } } // getBackendServers returns a list of Upstream and Server to be used by the // backend. An upstream can be used in multiple servers if the namespace, // service name and port are the same. -// -//nolint:gocyclo // Ignore function complexity error func (n *NGINXController) getBackendServers(ingresses []*ingress.Ingress) ([]*ingress.Backend, []*ingress.Server) { du := n.getDefaultUpstream() upstreams := n.createUpstreams(ingresses, du) @@ -712,10 +515,6 @@ func (n *NGINXController) getBackendServers(ingresses []*ingress.Ingress) ([]*in ingKey := k8s.MetaNamespaceKey(ing) anns := ing.ParsedAnnotations - if !n.store.GetBackendConfiguration().AllowSnippetAnnotations { - dropSnippetDirectives(anns, ingKey) - } - for _, rule := range ing.Spec.Rules { host := rule.Host if host == "" { @@ -767,13 +566,7 @@ func (n *NGINXController) getBackendServers(ingresses []*ingress.Ingress) ([]*in } for _, path := range rule.HTTP.Paths { - if path.Backend.Service == nil { - // skip non-service backends - klog.V(3).Infof("Ingress %q and path %q does not contain a service backend, using default backend", ingKey, path.Path) - continue - } - - upsName := upstreamName(ing.Namespace, path.Backend.Service) + upsName := upstreamName(ing.Namespace, path.Backend.ServiceName, path.Backend.ServicePort) ups := upstreams[upsName] @@ -863,9 +656,7 @@ func (n *NGINXController) getBackendServers(ingresses []*ingress.Ingress) ([]*in ups.SessionAffinity.CookieSessionAffinity.Name = anns.SessionAffinity.Cookie.Name ups.SessionAffinity.CookieSessionAffinity.Expires = anns.SessionAffinity.Cookie.Expires ups.SessionAffinity.CookieSessionAffinity.MaxAge = anns.SessionAffinity.Cookie.MaxAge - ups.SessionAffinity.CookieSessionAffinity.Secure = anns.SessionAffinity.Cookie.Secure ups.SessionAffinity.CookieSessionAffinity.Path = cookiePath - ups.SessionAffinity.CookieSessionAffinity.Domain = anns.SessionAffinity.Cookie.Domain ups.SessionAffinity.CookieSessionAffinity.SameSite = anns.SessionAffinity.Cookie.SameSite ups.SessionAffinity.CookieSessionAffinity.ConditionalSameSiteNone = anns.SessionAffinity.Cookie.ConditionalSameSiteNone ups.SessionAffinity.CookieSessionAffinity.ChangeOnFailure = anns.SessionAffinity.Cookie.ChangeOnFailure @@ -923,16 +714,10 @@ func (n *NGINXController) getBackendServers(ingresses []*ingress.Ingress) ([]*in } sp := location.DefaultBackend.Spec.Ports[0] - var zone string - if n.cfg.EnableTopologyAwareRouting { - zone = getIngressPodZone(location.DefaultBackend) - } else { - zone = emptyZone - } - endps := getEndpointsFromSlices(location.DefaultBackend, &sp, apiv1.ProtocolTCP, zone, n.store.GetServiceEndpointsSlices) + endps := getEndpoints(location.DefaultBackend, &sp, apiv1.ProtocolTCP, n.store.GetServiceEndpoints) // custom backend is valid only if contains at least one endpoint if len(endps) > 0 { - name := fmt.Sprintf("custom-default-backend-%v-%v", location.DefaultBackend.GetNamespace(), location.DefaultBackend.GetName()) + name := fmt.Sprintf("custom-default-backend-%v", location.DefaultBackend.GetName()) klog.V(3).Infof("Creating \"%v\" upstream based on default backend annotation", name) nb := upstream.DeepCopy() @@ -996,16 +781,11 @@ func (n *NGINXController) createUpstreams(data []*ingress.Ingress, du *ingress.B upstreams[defUpstreamName] = du for _, ing := range data { - ingKey := k8s.MetaNamespaceKey(ing) anns := ing.ParsedAnnotations - if !n.store.GetBackendConfiguration().AllowSnippetAnnotations { - dropSnippetDirectives(anns, ingKey) - } - var defBackend string - if ing.Spec.DefaultBackend != nil && ing.Spec.DefaultBackend.Service != nil { - defBackend = upstreamName(ing.Namespace, ing.Spec.DefaultBackend.Service) + if ing.Spec.Backend != nil { + defBackend = upstreamName(ing.Namespace, ing.Spec.Backend.ServiceName, ing.Spec.Backend.ServicePort) klog.V(3).Infof("Creating upstream %q", defBackend) upstreams[defBackend] = newUpstream(defBackend) @@ -1019,11 +799,11 @@ func (n *NGINXController) createUpstreams(data []*ingress.Ingress, du *ingress.B upstreams[defBackend].LoadBalancing = n.store.GetBackendConfiguration().LoadBalancing } - svcKey := fmt.Sprintf("%v/%v", ing.Namespace, ing.Spec.DefaultBackend.Service.Name) + svcKey := fmt.Sprintf("%v/%v", ing.Namespace, ing.Spec.Backend.ServiceName) // add the service ClusterIP as a single Endpoint instead of individual Endpoints if anns.ServiceUpstream { - endpoint, err := n.getServiceClusterEndpoint(svcKey, ing.Spec.DefaultBackend) + endpoint, err := n.getServiceClusterEndpoint(svcKey, ing.Spec.Backend) if err != nil { klog.Errorf("Failed to determine a suitable ClusterIP Endpoint for Service %q: %v", svcKey, err) } else { @@ -1034,12 +814,17 @@ func (n *NGINXController) createUpstreams(data []*ingress.Ingress, du *ingress.B // configure traffic shaping for canary if anns.Canary.Enabled { upstreams[defBackend].NoServer = true - upstreams[defBackend].TrafficShapingPolicy = newTrafficShapingPolicy(&anns.Canary) + upstreams[defBackend].TrafficShapingPolicy = ingress.TrafficShapingPolicy{ + Weight: anns.Canary.Weight, + Header: anns.Canary.Header, + HeaderValue: anns.Canary.HeaderValue, + HeaderPattern: anns.Canary.HeaderPattern, + Cookie: anns.Canary.Cookie, + } } if len(upstreams[defBackend].Endpoints) == 0 { - _, port := upstreamServiceNameAndPort(ing.Spec.DefaultBackend.Service) - endps, err := n.serviceEndpoints(svcKey, port.String()) + endps, err := n.serviceEndpoints(svcKey, ing.Spec.Backend.ServicePort.String()) upstreams[defBackend].Endpoints = append(upstreams[defBackend].Endpoints, endps...) if err != nil { klog.Warningf("Error creating upstream %q: %v", defBackend, err) @@ -1058,22 +843,16 @@ func (n *NGINXController) createUpstreams(data []*ingress.Ingress, du *ingress.B continue } - for i, path := range rule.HTTP.Paths { - if path.Backend.Service == nil { - // skip non-service backends - klog.V(3).Infof("Ingress %q and path %q does not contain a service backend, using default backend", ingKey, path.Path) - continue - } + for _, path := range rule.HTTP.Paths { + name := upstreamName(ing.Namespace, path.Backend.ServiceName, path.Backend.ServicePort) - name := upstreamName(ing.Namespace, path.Backend.Service) - svcName, svcPort := upstreamServiceNameAndPort(path.Backend.Service) if _, ok := upstreams[name]; ok { continue } klog.V(3).Infof("Creating upstream %q", name) upstreams[name] = newUpstream(name) - upstreams[name].Port = svcPort + upstreams[name].Port = path.Backend.ServicePort upstreams[name].UpstreamHashBy.UpstreamHashBy = anns.UpstreamHashBy.UpstreamHashBy upstreams[name].UpstreamHashBy.UpstreamHashBySubset = anns.UpstreamHashBy.UpstreamHashBySubset @@ -1084,11 +863,11 @@ func (n *NGINXController) createUpstreams(data []*ingress.Ingress, du *ingress.B upstreams[name].LoadBalancing = n.store.GetBackendConfiguration().LoadBalancing } - svcKey := fmt.Sprintf("%v/%v", ing.Namespace, svcName) + svcKey := fmt.Sprintf("%v/%v", ing.Namespace, path.Backend.ServiceName) // add the service ClusterIP as a single Endpoint instead of individual Endpoints if anns.ServiceUpstream { - endpoint, err := n.getServiceClusterEndpoint(svcKey, &rule.HTTP.Paths[i].Backend) + endpoint, err := n.getServiceClusterEndpoint(svcKey, &path.Backend) if err != nil { klog.Errorf("Failed to determine a suitable ClusterIP Endpoint for Service %q: %v", svcKey, err) } else { @@ -1099,24 +878,21 @@ func (n *NGINXController) createUpstreams(data []*ingress.Ingress, du *ingress.B // configure traffic shaping for canary if anns.Canary.Enabled { upstreams[name].NoServer = true - upstreams[name].TrafficShapingPolicy = newTrafficShapingPolicy(&anns.Canary) + upstreams[name].TrafficShapingPolicy = ingress.TrafficShapingPolicy{ + Weight: anns.Canary.Weight, + Header: anns.Canary.Header, + HeaderValue: anns.Canary.HeaderValue, + HeaderPattern: anns.Canary.HeaderPattern, + Cookie: anns.Canary.Cookie, + } } if len(upstreams[name].Endpoints) == 0 { - _, port := upstreamServiceNameAndPort(path.Backend.Service) - endp, err := n.serviceEndpoints(svcKey, port.String()) + endp, err := n.serviceEndpoints(svcKey, path.Backend.ServicePort.String()) if err != nil { klog.Warningf("Error obtaining Endpoints for Service %q: %v", svcKey, err) - n.metricCollector.IncOrphanIngress(ing.Namespace, ing.Name, orphanMetricLabelNoService) continue } - n.metricCollector.DecOrphanIngress(ing.Namespace, ing.Name, orphanMetricLabelNoService) - - if len(endp) == 0 { - n.metricCollector.IncOrphanIngress(ing.Namespace, ing.Name, orphanMetricLabelNoEndpoint) - } else { - n.metricCollector.DecOrphanIngress(ing.Namespace, ing.Name, orphanMetricLabelNoEndpoint) - } upstreams[name].Endpoints = endp } @@ -1150,23 +926,20 @@ func (n *NGINXController) getServiceClusterEndpoint(svcKey string, backend *netw // if the Service port is referenced by name in the Ingress, lookup the // actual port in the service spec - if backend.Service != nil { - _, svcportintorstr := upstreamServiceNameAndPort(backend.Service) - if svcportintorstr.Type == intstr.String { - var port int32 = -1 - for _, svcPort := range svc.Spec.Ports { - if svcPort.Name == svcportintorstr.String() { - port = svcPort.Port - break - } + if backend.ServicePort.Type == intstr.String { + var port int32 = -1 + for _, svcPort := range svc.Spec.Ports { + if svcPort.Name == backend.ServicePort.String() { + port = svcPort.Port + break } - if port == -1 { - return endpoint, fmt.Errorf("service %q does not have a port named %q", svc.Name, svcportintorstr.String()) - } - endpoint.Port = fmt.Sprintf("%d", port) - } else { - endpoint.Port = svcportintorstr.String() } + if port == -1 { + return endpoint, fmt.Errorf("service %q does not have a port named %q", svc.Name, backend.ServicePort) + } + endpoint.Port = fmt.Sprintf("%d", port) + } else { + endpoint.Port = backend.ServicePort.String() } return endpoint, err @@ -1180,21 +953,13 @@ func (n *NGINXController) serviceEndpoints(svcKey, backendPort string) ([]ingres if err != nil { return upstreams, err } - var zone string - if n.cfg.EnableTopologyAwareRouting { - zone = getIngressPodZone(svc) - } else { - zone = emptyZone - } + klog.V(3).Infof("Obtaining ports information for Service %q", svcKey) + // Ingress with an ExternalName Service and no port defined for that Service if svc.Spec.Type == apiv1.ServiceTypeExternalName { - if n.cfg.DisableServiceExternalName { - klog.Warningf("Service %q of type ExternalName not allowed due to Ingress configuration.", svcKey) - return upstreams, nil - } servicePort := externalNamePorts(backendPort, svc) - endps := getEndpointsFromSlices(svc, servicePort, apiv1.ProtocolTCP, zone, n.store.GetServiceEndpointsSlices) + endps := getEndpoints(svc, servicePort, apiv1.ProtocolTCP, n.store.GetServiceEndpoints) if len(endps) == 0 { klog.Warningf("Service %q does not have any active Endpoint.", svcKey) return upstreams, nil @@ -1210,7 +975,8 @@ func (n *NGINXController) serviceEndpoints(svcKey, backendPort string) ([]ingres if strconv.Itoa(int(servicePort.Port)) == backendPort || servicePort.TargetPort.String() == backendPort || servicePort.Name == backendPort { - endps := getEndpointsFromSlices(svc, &servicePort, apiv1.ProtocolTCP, zone, n.store.GetServiceEndpointsSlices) + + endps := getEndpoints(svc, &servicePort, apiv1.ProtocolTCP, n.store.GetServiceEndpoints) if len(endps) == 0 { klog.Warningf("Service %q does not have any active Endpoint.", svcKey) } @@ -1242,8 +1008,8 @@ func (n *NGINXController) getDefaultSSLCertificate() *ingress.SSLCert { // one root location, which uses a default backend if left unspecified. func (n *NGINXController) createServers(data []*ingress.Ingress, upstreams map[string]*ingress.Backend, - du *ingress.Backend, -) map[string]*ingress.Server { + du *ingress.Backend) map[string]*ingress.Server { + servers := make(map[string]*ingress.Server, len(data)) allAliases := make(map[string][]string, len(data)) @@ -1255,7 +1021,6 @@ func (n *NGINXController) createServers(data []*ingress.Ingress, ReadTimeout: bdef.ProxyReadTimeout, BuffersNumber: bdef.ProxyBuffersNumber, BufferSize: bdef.ProxyBufferSize, - BusyBuffersSize: bdef.ProxyBusyBuffersSize, CookieDomain: bdef.ProxyCookieDomain, CookiePath: bdef.ProxyCookiePath, NextUpstream: bdef.ProxyNextUpstream, @@ -1286,18 +1051,13 @@ func (n *NGINXController) createServers(data []*ingress.Ingress, Rewrite: false, }, }, - }, - } + }} // initialize all other servers for _, ing := range data { ingKey := k8s.MetaNamespaceKey(ing) anns := ing.ParsedAnnotations - if !n.store.GetBackendConfiguration().AllowSnippetAnnotations { - dropSnippetDirectives(anns, ingKey) - } - // default upstream name un := du.Name @@ -1306,22 +1066,24 @@ func (n *NGINXController) createServers(data []*ingress.Ingress, continue } - if ing.Spec.DefaultBackend != nil && ing.Spec.DefaultBackend.Service != nil { - defUpstream := upstreamName(ing.Namespace, ing.Spec.DefaultBackend.Service) + if ing.Spec.Backend != nil { + defUpstream := upstreamName(ing.Namespace, ing.Spec.Backend.ServiceName, ing.Spec.Backend.ServicePort) if backendUpstream, ok := upstreams[defUpstream]; ok { // use backend specified in Ingress as the default backend for all its rules un = backendUpstream.Name + // special "catch all" case, Ingress with a backend but no rule defLoc := servers[defServerName].Locations[0] + defLoc.Backend = backendUpstream.Name + defLoc.Service = backendUpstream.Service + defLoc.Ingress = ing + if defLoc.IsDefBackend && len(ing.Spec.Rules) == 0 { klog.V(2).Infof("Ingress %q defines a backend but no rule. Using it to configure the catch-all server %q", ingKey, defServerName) defLoc.IsDefBackend = false - // special "catch all" case, Ingress with a backend but no rule - defLoc.Backend = backendUpstream.Name - defLoc.Service = backendUpstream.Service - defLoc.Ingress = ing + // TODO: Redirect and rewrite can affect the catch all behavior, skip for now originalRedirect := defLoc.Redirect originalRewrite := defLoc.Rewrite @@ -1372,10 +1134,6 @@ func (n *NGINXController) createServers(data []*ingress.Ingress, ingKey := k8s.MetaNamespaceKey(ing) anns := ing.ParsedAnnotations - if !n.store.GetBackendConfiguration().AllowSnippetAnnotations { - dropSnippetDirectives(anns, ingKey) - } - if anns.Canary.Enabled { klog.V(2).Infof("Ingress %v is marked as Canary, ignoring", ingKey) continue @@ -1464,10 +1222,7 @@ func (n *NGINXController) createServers(data []*ingress.Ingress, servers[host].SSLCert = cert - now := time.Now() - if cert.ExpireTime.Before(now) { - klog.Warningf("SSL certificate for server %q expired (%v)", host, cert.ExpireTime) - } else if cert.ExpireTime.Before(now.Add(240 * time.Hour)) { + if cert.ExpireTime.Before(time.Now().Add(240 * time.Hour)) { klog.Warningf("SSL certificate for server %q is about to expire (%v)", host, cert.ExpireTime) } } @@ -1504,31 +1259,30 @@ func (n *NGINXController) createServers(data []*ingress.Ingress, func locationApplyAnnotations(loc *ingress.Location, anns *annotations.Ingress) { loc.BasicDigestAuth = anns.BasicDigestAuth loc.ClientBodyBufferSize = anns.ClientBodyBufferSize - loc.CustomHeaders = anns.CustomHeaders loc.ConfigurationSnippet = anns.ConfigurationSnippet loc.CorsConfig = anns.CorsConfig loc.ExternalAuth = anns.ExternalAuth loc.EnableGlobalAuth = anns.EnableGlobalAuth loc.HTTP2PushPreload = anns.HTTP2PushPreload - loc.Opentelemetry = anns.Opentelemetry + loc.Opentracing = anns.Opentracing loc.Proxy = anns.Proxy loc.ProxySSL = anns.ProxySSL loc.RateLimit = anns.RateLimit + loc.GlobalRateLimit = anns.GlobalRateLimit loc.Redirect = anns.Redirect loc.Rewrite = anns.Rewrite loc.UpstreamVhost = anns.UpstreamVhost - loc.Denylist = anns.Denylist - loc.Allowlist = anns.Allowlist + loc.Whitelist = anns.Whitelist loc.Denied = anns.Denied loc.XForwardedPrefix = anns.XForwardedPrefix loc.UsePortInRedirects = anns.UsePortInRedirects loc.Connection = anns.Connection loc.Logs = anns.Logs + loc.InfluxDB = anns.InfluxDB loc.DefaultBackend = anns.DefaultBackend loc.BackendProtocol = anns.BackendProtocol loc.FastCGI = anns.FastCGI loc.CustomHTTPErrors = anns.CustomHTTPErrors - loc.DisableProxyInterceptErrors = anns.DisableProxyInterceptErrors loc.ModSecurity = anns.ModSecurity loc.Satisfy = anns.Satisfy loc.Mirror = anns.Mirror @@ -1537,7 +1291,7 @@ func locationApplyAnnotations(loc *ingress.Location, anns *annotations.Ingress) } // OK to merge canary ingresses iff there exists one or more ingresses to potentially merge into -func nonCanaryIngressExists(ingresses, canaryIngresses []*ingress.Ingress) bool { +func nonCanaryIngressExists(ingresses []*ingress.Ingress, canaryIngresses []*ingress.Ingress) bool { return len(ingresses)-len(canaryIngresses) > 0 } @@ -1545,12 +1299,12 @@ func nonCanaryIngressExists(ingresses, canaryIngresses []*ingress.Ingress) bool // 1) names of backends do not match and canary doesn't merge into itself // 2) primary name is not the default upstream // 3) the primary has a server -func canMergeBackend(primary, alternative *ingress.Backend) bool { +func canMergeBackend(primary *ingress.Backend, alternative *ingress.Backend) bool { return alternative != nil && primary.Name != alternative.Name && primary.Name != defUpstreamName && !primary.NoServer } // Performs the merge action and checks to ensure that one two alternative backends do not merge into each other -func mergeAlternativeBackend(ing *ingress.Ingress, priUps, altUps *ingress.Backend) bool { +func mergeAlternativeBackend(priUps *ingress.Backend, altUps *ingress.Backend) bool { if priUps.NoServer { klog.Warningf("unable to merge alternative backend %v into primary backend %v because %v is a primary backend", altUps.Name, priUps.Name, priUps.Name) @@ -1564,11 +1318,8 @@ func mergeAlternativeBackend(ing *ingress.Ingress, priUps, altUps *ingress.Backe } } - if ing.ParsedAnnotations != nil && ing.ParsedAnnotations.SessionAffinity.CanaryBehavior != "legacy" { - priUps.SessionAffinity.DeepCopyInto(&altUps.SessionAffinity) - } - - priUps.AlternativeBackends = append(priUps.AlternativeBackends, altUps.Name) + priUps.AlternativeBackends = + append(priUps.AlternativeBackends, altUps.Name) return true } @@ -1578,26 +1329,23 @@ func mergeAlternativeBackend(ing *ingress.Ingress, priUps, altUps *ingress.Backe // to a backend's alternative list. // If no match is found, then the serverless backend is deleted. func mergeAlternativeBackends(ing *ingress.Ingress, upstreams map[string]*ingress.Backend, - servers map[string]*ingress.Server, -) { + servers map[string]*ingress.Server) { + // merge catch-all alternative backends - if ing.Spec.DefaultBackend != nil { - upsName := upstreamName(ing.Namespace, ing.Spec.DefaultBackend.Service) + if ing.Spec.Backend != nil { + upsName := upstreamName(ing.Namespace, ing.Spec.Backend.ServiceName, ing.Spec.Backend.ServicePort) altUps := upstreams[upsName] if altUps == nil { klog.Warningf("alternative backend %s has already been removed", upsName) } else { + merged := false altEqualsPri := false for _, loc := range servers[defServerName].Locations { - priUps, ok := upstreams[loc.Backend] - if !ok { - klog.Warningf("cannot find primary backend %s for location %s%s", loc.Backend, servers[defServerName].Hostname, loc.Path) - continue - } + priUps := upstreams[loc.Backend] altEqualsPri = altUps.Name == priUps.Name if altEqualsPri { klog.Warningf("alternative upstream %s in Ingress %s/%s is primary upstream in Other Ingress for location %s%s!", @@ -1609,7 +1357,7 @@ func mergeAlternativeBackends(ing *ingress.Ingress, upstreams map[string]*ingres klog.V(2).Infof("matching backend %v found for alternative backend %v", priUps.Name, altUps.Name) - merged = mergeAlternativeBackend(ing, priUps, altUps) + merged = mergeAlternativeBackend(priUps, altUps) } } @@ -1627,13 +1375,7 @@ func mergeAlternativeBackends(ing *ingress.Ingress, upstreams map[string]*ingres } for _, path := range rule.HTTP.Paths { - if path.Backend.Service == nil { - // skip non-service backends - klog.V(3).Infof("Ingress %q and path %q does not contain a service backend, using default backend", k8s.MetaNamespaceKey(ing), path.Path) - continue - } - - upsName := upstreamName(ing.Namespace, path.Backend.Service) + upsName := upstreamName(ing.Namespace, path.Backend.ServiceName, path.Backend.ServicePort) altUps := upstreams[upsName] @@ -1656,11 +1398,7 @@ func mergeAlternativeBackends(ing *ingress.Ingress, upstreams map[string]*ingres // find matching paths for _, loc := range server.Locations { - priUps, ok := upstreams[loc.Backend] - if !ok { - klog.Warningf("cannot find primary backend %s for location %s%s", loc.Backend, server.Hostname, loc.Path) - continue - } + priUps := upstreams[loc.Backend] altEqualsPri = altUps.Name == priUps.Name if altEqualsPri { klog.Warningf("alternative upstream %s in Ingress %s/%s is primary upstream in Other Ingress for location %s%s!", @@ -1672,7 +1410,7 @@ func mergeAlternativeBackends(ing *ingress.Ingress, upstreams map[string]*ingres klog.V(2).Infof("matching backend %v found for alternative backend %v", priUps.Name, altUps.Name) - merged = mergeAlternativeBackend(ing, priUps, altUps) + merged = mergeAlternativeBackend(priUps, altUps) } } @@ -1687,8 +1425,8 @@ func mergeAlternativeBackends(ing *ingress.Ingress, upstreams map[string]*ingres // extractTLSSecretName returns the name of the Secret containing a SSL // certificate for the given host name, or an empty string. func extractTLSSecretName(host string, ing *ingress.Ingress, - getLocalSSLCert func(string) (*ingress.SSLCert, error), -) string { + getLocalSSLCert func(string) (*ingress.SSLCert, error)) string { + if ing == nil { return "" } @@ -1705,6 +1443,7 @@ func extractTLSSecretName(host string, ing *ingress.Ingress, // no TLS host matching host name, try each TLS host for matching SAN or CN for _, tls := range ing.Spec.TLS { + if tls.SecretName == "" { // There's no secretName specified, so it will never be available continue @@ -1718,7 +1457,7 @@ func extractTLSSecretName(host string, ing *ingress.Ingress, continue } - if cert == nil || cert.Certificate == nil { + if cert == nil { // for tests continue } @@ -1733,6 +1472,60 @@ func extractTLSSecretName(host string, ing *ingress.Ingress, return "" } +// getRemovedHosts returns a list of the hostnames +// that are not associated anymore to the NGINX configuration. +func getRemovedHosts(rucfg, newcfg *ingress.Configuration) []string { + old := sets.NewString() + new := sets.NewString() + + for _, s := range rucfg.Servers { + if !old.Has(s.Hostname) { + old.Insert(s.Hostname) + } + } + + for _, s := range newcfg.Servers { + if !new.Has(s.Hostname) { + new.Insert(s.Hostname) + } + } + + return old.Difference(new).List() +} + +func getRemovedIngresses(rucfg, newcfg *ingress.Configuration) []string { + oldIngresses := sets.NewString() + newIngresses := sets.NewString() + + for _, server := range rucfg.Servers { + for _, location := range server.Locations { + if location.Ingress == nil { + continue + } + + ingKey := k8s.MetaNamespaceKey(location.Ingress) + if !oldIngresses.Has(ingKey) { + oldIngresses.Insert(ingKey) + } + } + } + + for _, server := range newcfg.Servers { + for _, location := range server.Locations { + if location.Ingress == nil { + continue + } + + ingKey := k8s.MetaNamespaceKey(location.Ingress) + if !newIngresses.Has(ingKey) { + newIngresses.Insert(ingKey) + } + } + } + + return oldIngresses.Difference(newIngresses).List() +} + // checks conditions for whether or not an upstream should be created for a custom default backend func shouldCreateUpstreamForLocationDefaultBackend(upstream *ingress.Backend, location *ingress.Location) bool { return (upstream.Name == location.Backend) && @@ -1763,7 +1556,6 @@ func externalNamePorts(name string, svc *apiv1.Service) *apiv1.ServicePort { } for _, svcPort := range svc.Spec.Ports { - //nolint:gosec // Ignore G109 error if svcPort.Port != int32(port) { continue } @@ -1782,14 +1574,13 @@ func externalNamePorts(name string, svc *apiv1.Service) *apiv1.ServicePort { // ExternalName without port return &apiv1.ServicePort{ - Protocol: "TCP", - //nolint:gosec // Ignore G109 error + Protocol: "TCP", Port: int32(port), TargetPort: intstr.FromInt(port), } } -func checkOverlap(ing *networking.Ingress, servers []*ingress.Server) error { +func checkOverlap(ing *networking.Ingress, ingresses []*ingress.Ingress, servers []*ingress.Server) error { for _, rule := range ing.Spec.Rules { if rule.HTTP == nil { continue @@ -1800,12 +1591,6 @@ func checkOverlap(ing *networking.Ingress, servers []*ingress.Server) error { } for _, path := range rule.HTTP.Paths { - if path.Backend.Service == nil { - // skip non-service backends - klog.V(3).Infof("Ingress %q and path %q does not contain a service backend, using default backend", k8s.MetaNamespaceKey(ing), path.Path) - continue - } - if path.Path == "" { path.Path = rootLocation } @@ -1818,22 +1603,27 @@ func checkOverlap(ing *networking.Ingress, servers []*ingress.Server) error { } // same ingress + skipValidation := false for _, existing := range existingIngresses { if existing.ObjectMeta.Namespace == ing.ObjectMeta.Namespace && existing.ObjectMeta.Name == ing.ObjectMeta.Name { return nil } } + if skipValidation { + continue + } + // path overlap. Check if one of the ingresses has a canary annotation - isCanaryEnabled, annotationErr := parser.GetBoolAnnotation("canary", ing, canary.CanaryAnnotations.Annotations) + isCanaryEnabled, annotationErr := parser.GetBoolAnnotation("canary", ing) for _, existing := range existingIngresses { - isExistingCanaryEnabled, existingAnnotationErr := parser.GetBoolAnnotation("canary", existing, canary.CanaryAnnotations.Annotations) + isExistingCanaryEnabled, existingAnnotationErr := parser.GetBoolAnnotation("canary", existing) if isCanaryEnabled && isExistingCanaryEnabled { return fmt.Errorf(`host "%s" and path "%s" is already defined in ingress %s/%s`, rule.Host, path.Path, existing.Namespace, existing.Name) } - if annotationErr == errors.ErrMissingAnnotations && existingAnnotationErr == errors.ErrMissingAnnotations { + if annotationErr == errors.ErrMissingAnnotations && existingAnnotationErr == existingAnnotationErr { return fmt.Errorf(`host "%s" and path "%s" is already defined in ingress %s/%s`, rule.Host, path.Path, existing.Namespace, existing.Name) } } @@ -1854,41 +1644,14 @@ func ingressForHostPath(hostname, path string, servers []*ingress.Server) []*net continue } - for i, location := range server.Locations { + for _, location := range server.Locations { if location.Path != path { continue } - if location.IsDefBackend { - continue - } - - ingresses = append(ingresses, &server.Locations[i].Ingress.Ingress) + ingresses = append(ingresses, &location.Ingress.Ingress) } } return ingresses } - -func (n *NGINXController) getStreamSnippets(ingresses []*ingress.Ingress) []string { - snippets := make([]string, 0, len(ingresses)) - for _, i := range ingresses { - if i.ParsedAnnotations.StreamSnippet == "" { - continue - } - snippets = append(snippets, i.ParsedAnnotations.StreamSnippet) - } - return snippets -} - -// newTrafficShapingPolicy creates new ingress.TrafficShapingPolicy instance using canary configuration -func newTrafficShapingPolicy(cfg *canary.Config) ingress.TrafficShapingPolicy { - return ingress.TrafficShapingPolicy{ - Weight: cfg.Weight, - WeightTotal: cfg.WeightTotal, - Header: cfg.Header, - HeaderValue: cfg.HeaderValue, - HeaderPattern: cfg.HeaderPattern, - Cookie: cfg.Cookie, - } -} diff --git a/internal/ingress/controller/controller_test.go b/internal/ingress/controller/controller_test.go index 9d3fea470..0aa522b7f 100644 --- a/internal/ingress/controller/controller_test.go +++ b/internal/ingress/controller/controller_test.go @@ -21,9 +21,8 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/asn1" - "encoding/base64" "fmt" - "io" + "io/ioutil" "os" "os/exec" "path/filepath" @@ -33,83 +32,61 @@ import ( "github.com/eapache/channels" corev1 "k8s.io/api/core/v1" - - discoveryv1 "k8s.io/api/discovery/v1" - networking "k8s.io/api/networking/v1" + v1 "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes/fake" - "k8s.io/ingress-nginx/pkg/apis/ingress" - + "k8s.io/ingress-nginx/internal/file" + "k8s.io/ingress-nginx/internal/ingress" "k8s.io/ingress-nginx/internal/ingress/annotations" "k8s.io/ingress-nginx/internal/ingress/annotations/canary" - "k8s.io/ingress-nginx/internal/ingress/annotations/ipallowlist" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/annotations/proxyssl" - "k8s.io/ingress-nginx/internal/ingress/annotations/sessionaffinity" + "k8s.io/ingress-nginx/internal/ingress/controller/config" ngx_config "k8s.io/ingress-nginx/internal/ingress/controller/config" - "k8s.io/ingress-nginx/internal/ingress/controller/ingressclass" "k8s.io/ingress-nginx/internal/ingress/controller/store" "k8s.io/ingress-nginx/internal/ingress/defaults" "k8s.io/ingress-nginx/internal/ingress/metric" "k8s.io/ingress-nginx/internal/ingress/resolver" "k8s.io/ingress-nginx/internal/k8s" "k8s.io/ingress-nginx/internal/net/ssl" - - "k8s.io/ingress-nginx/pkg/util/file" -) - -const ( - exampleBackend = "example-http-svc-1-80" - TRUE = "true" ) type fakeIngressStore struct { - ingresses []*ingress.Ingress - configuration ngx_config.Configuration + ingresses []*ingress.Ingress } -func (fakeIngressStore) GetIngressClass(_ *networking.Ingress, _ *ingressclass.Configuration) (string, error) { - return "nginx", nil +func (fakeIngressStore) GetBackendConfiguration() ngx_config.Configuration { + return ngx_config.Configuration{} } -func (fis *fakeIngressStore) GetBackendConfiguration() ngx_config.Configuration { - return fis.configuration -} - -func (fis *fakeIngressStore) GetSecurityConfiguration() defaults.SecurityConfiguration { - return defaults.SecurityConfiguration{ - AnnotationsRiskLevel: fis.configuration.AnnotationsRiskLevel, - AllowCrossNamespaceResources: fis.configuration.AllowCrossNamespaceResources, - } -} - -func (fakeIngressStore) GetConfigMap(_ string) (*corev1.ConfigMap, error) { +func (fakeIngressStore) GetConfigMap(key string) (*corev1.ConfigMap, error) { return nil, fmt.Errorf("test error") } -func (fakeIngressStore) GetSecret(_ string) (*corev1.Secret, error) { +func (fakeIngressStore) GetSecret(key string) (*corev1.Secret, error) { return nil, fmt.Errorf("test error") } -func (fakeIngressStore) GetService(_ string) (*corev1.Service, error) { +func (fakeIngressStore) GetService(key string) (*corev1.Service, error) { return nil, fmt.Errorf("test error") } -func (fakeIngressStore) GetServiceEndpointsSlices(_ string) ([]*discoveryv1.EndpointSlice, error) { +func (fakeIngressStore) GetServiceEndpoints(key string) (*corev1.Endpoints, error) { return nil, fmt.Errorf("test error") } -func (fis *fakeIngressStore) ListIngresses() []*ingress.Ingress { +func (fis fakeIngressStore) ListIngresses() []*ingress.Ingress { return fis.ingresses } -func (fis *fakeIngressStore) FilterIngresses(ingresses []*ingress.Ingress, _ store.IngressFilterFunc) []*ingress.Ingress { +func (fis fakeIngressStore) FilterIngresses(ingresses []*ingress.Ingress, filterFunc store.IngressFilterFunc) []*ingress.Ingress { return ingresses } -func (fakeIngressStore) GetLocalSSLCert(_ string) (*ingress.SSLCert, error) { +func (fakeIngressStore) GetLocalSSLCert(name string) (*ingress.SSLCert, error) { return nil, fmt.Errorf("test error") } @@ -125,7 +102,7 @@ func (fakeIngressStore) GetDefaultBackend() defaults.Backend { return defaults.Backend{} } -func (fakeIngressStore) Run(_ chan struct{}) {} +func (fakeIngressStore) Run(stopCh chan struct{}) {} type testNginxTestCommand struct { t *testing.T @@ -134,7 +111,7 @@ type testNginxTestCommand struct { err error } -func (ntc testNginxTestCommand) ExecCommand(_ ...string) *exec.Cmd { +func (ntc testNginxTestCommand) ExecCommand(args ...string) *exec.Cmd { return nil } @@ -145,7 +122,7 @@ func (ntc testNginxTestCommand) Test(cfg string) ([]byte, error) { return nil, err } defer fd.Close() - bytes, err := io.ReadAll(fd) + bytes, err := ioutil.ReadAll(fd) if err != nil { ntc.t.Errorf("could not read generated nginx configuration: %v", err.Error()) } @@ -157,7 +134,7 @@ func (ntc testNginxTestCommand) Test(cfg string) ([]byte, error) { type fakeTemplate struct{} -func (fakeTemplate) Write(conf *ngx_config.TemplateConfig) ([]byte, error) { +func (fakeTemplate) Write(conf config.TemplateConfig) ([]byte, error) { r := []byte{} for _, s := range conf.Servers { if len(r) > 0 { @@ -170,7 +147,7 @@ func (fakeTemplate) Write(conf *ngx_config.TemplateConfig) ([]byte, error) { func TestCheckIngress(t *testing.T) { defer func() { - err := filepath.Walk(os.TempDir(), func(path string, info os.FileInfo, _ error) error { + filepath.Walk(os.TempDir(), func(path string, info os.FileInfo, err error) error { if info.IsDir() && os.TempDir() != path { return filepath.SkipDir } @@ -179,9 +156,6 @@ func TestCheckIngress(t *testing.T) { } return nil }) - if err != nil { - t.Errorf("unexpected error: %v", err) - } }() err := file.CreateRequiredDirectories() @@ -191,17 +165,13 @@ func TestCheckIngress(t *testing.T) { // Ensure no panic with wrong arguments var nginx *NGINXController - if err := nginx.CheckIngress(nil); err != nil { - t.Errorf("unexpected error: %v", err) - } + nginx.CheckIngress(nil) nginx = newNGINXController(t) - if err := nginx.CheckIngress(nil); err != nil { - t.Errorf("unexpected error: %v", err) - } + nginx.CheckIngress(nil) nginx.metricCollector = metric.DummyCollector{} nginx.t = fakeTemplate{} - nginx.store = &fakeIngressStore{ + nginx.store = fakeIngressStore{ ingresses: []*ingress.Ingress{}, } @@ -219,6 +189,18 @@ func TestCheckIngress(t *testing.T) { }, }, } + + t.Run("When the ingress class differs from nginx", func(t *testing.T) { + ing.ObjectMeta.Annotations["kubernetes.io/ingress.class"] = "different" + nginx.command = testNginxTestCommand{ + t: t, + err: fmt.Errorf("test error"), + } + if nginx.CheckIngress(ing) != nil { + t.Errorf("with a different ingress class, no error should be returned") + } + }) + t.Run("when the class is the nginx one", func(t *testing.T) { ing.ObjectMeta.Annotations["kubernetes.io/ingress.class"] = "nginx" nginx.command = testNginxTestCommand{ @@ -231,7 +213,7 @@ func TestCheckIngress(t *testing.T) { } t.Run("When the hostname is updated", func(t *testing.T) { - nginx.store = &fakeIngressStore{ + nginx.store = fakeIngressStore{ ingresses: []*ingress.Ingress{ { Ingress: *ing, @@ -263,9 +245,6 @@ func TestCheckIngress(t *testing.T) { }) t.Run("When the default annotation prefix is used despite an override", func(t *testing.T) { - defer func() { - parser.AnnotationsPrefix = "nginx.ingress.kubernetes.io" - }() parser.AnnotationsPrefix = "ingress.kubernetes.io" ing.ObjectMeta.Annotations["nginx.ingress.kubernetes.io/backend-protocol"] = "GRPC" nginx.command = testNginxTestCommand{ @@ -277,46 +256,8 @@ func TestCheckIngress(t *testing.T) { } }) - t.Run("When snippets are disabled and user tries to use snippet annotation", func(t *testing.T) { - nginx.store = &fakeIngressStore{ - ingresses: []*ingress.Ingress{}, - configuration: ngx_config.Configuration{ - AllowSnippetAnnotations: false, - }, - } - nginx.command = testNginxTestCommand{ - t: t, - err: nil, - } - ing.ObjectMeta.Annotations["nginx.ingress.kubernetes.io/server-snippet"] = "bla" - if err := nginx.CheckIngress(ing); err == nil { - t.Errorf("with a snippet annotation, ingresses using the default should be rejected") - } - }) - - t.Run("When invalid directives are used in annotation values", func(t *testing.T) { - nginx.store = &fakeIngressStore{ - ingresses: []*ingress.Ingress{}, - configuration: ngx_config.Configuration{ - AnnotationValueWordBlocklist: "invalid_directive, another_directive", - }, - } - nginx.command = testNginxTestCommand{ - t: t, - err: nil, - } - ing.ObjectMeta.Annotations["nginx.ingress.kubernetes.io/custom-headers"] = "invalid_directive" - if err := nginx.CheckIngress(ing); err == nil { - t.Errorf("with an invalid value in annotation the ingress should be rejected") - } - ing.ObjectMeta.Annotations["nginx.ingress.kubernetes.io/custom-headers"] = "another_directive" - if err := nginx.CheckIngress(ing); err == nil { - t.Errorf("with an invalid value in annotation the ingress should be rejected") - } - }) - t.Run("When a new catch-all ingress is being created despite catch-alls being disabled ", func(t *testing.T) { - backendBefore := ing.Spec.DefaultBackend + backendBefore := ing.Spec.Backend disableCatchAllBefore := nginx.cfg.DisableCatchAll nginx.command = testNginxTestCommand{ @@ -325,12 +266,10 @@ func TestCheckIngress(t *testing.T) { } nginx.cfg.DisableCatchAll = true - ing.Spec.DefaultBackend = &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ing.Spec.Backend = &networking.IngressBackend{ + ServiceName: "http-svc", + ServicePort: intstr.IntOrString{ + IntVal: 80, }, } @@ -339,14 +278,11 @@ func TestCheckIngress(t *testing.T) { } // reset backend and catch-all flag - ing.Spec.DefaultBackend = backendBefore + ing.Spec.Backend = backendBefore nginx.cfg.DisableCatchAll = disableCatchAllBefore }) t.Run("When the ingress is in a different namespace than the watched one", func(t *testing.T) { - defer func() { - nginx.cfg.Namespace = "test-namespace" - }() nginx.command = testNginxTestCommand{ t: t, err: fmt.Errorf("test error"), @@ -358,124 +294,8 @@ func TestCheckIngress(t *testing.T) { } }) }) - - t.Run("When the ingress is marked as deleted", func(t *testing.T) { - ing.DeletionTimestamp = &metav1.Time{ - Time: time.Now(), - } - - if nginx.CheckIngress(ing) != nil { - t.Errorf("when the ingress is marked as deleted, no error should be returned") - } - }) } -func TestCheckWarning(t *testing.T) { - // Ensure no panic with wrong arguments - nginx := &NGINXController{} - - nginx.t = fakeTemplate{} - nginx.store = &fakeIngressStore{ - ingresses: []*ingress.Ingress{}, - } - - ing := &networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ingress-warning", - Namespace: "user-namespace", - Annotations: map[string]string{}, - }, - Spec: networking.IngressSpec{ - Rules: []networking.IngressRule{ - { - Host: "example.com", - }, - }, - }, - } - t.Run("when a deprecated annotation is used a warning should be returned", func(t *testing.T) { - ing.ObjectMeta.Annotations[parser.GetAnnotationWithPrefix("enable-influxdb")] = TRUE - defer func() { - ing.ObjectMeta.Annotations = map[string]string{} - }() - - warnings, err := nginx.CheckWarning(ing) - if err != nil { - t.Errorf("no error should be returned, but %s was returned", err) - } - if len(warnings) != 1 { - t.Errorf("expected 1 warning to occur but %d occurred", len(warnings)) - } else { - t.Logf("got warning %s correctly", warnings[0]) - } - }) - - t.Run("When an invalid pathType is used, a warning should be returned", func(t *testing.T) { - rules := ing.Spec.DeepCopy().Rules - ing.Spec.Rules = []networking.IngressRule{ - { - Host: "example.com", - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - Path: "/xpto{$2}", - PathType: &pathTypePrefix, - }, - { - Path: "/ok", - PathType: &pathTypeExact, - }, - }, - }, - }, - }, - } - defer func() { - ing.Spec.Rules = rules - }() - - warnings, err := nginx.CheckWarning(ing) - if err != nil { - t.Errorf("no error should be returned, but %s was returned", err) - } - if len(warnings) != 1 { - t.Errorf("expected 1 warning to occur but %d occurred", len(warnings)) - } else { - t.Logf("got warnings %v correctly", warnings) - } - - t.Run("adding invalid annotations increases the warning count", func(t *testing.T) { - ing.ObjectMeta.Annotations[parser.GetAnnotationWithPrefix("enable-influxdb")] = TRUE - ing.ObjectMeta.Annotations[parser.GetAnnotationWithPrefix("secure-verify-ca-secret")] = TRUE - ing.ObjectMeta.Annotations[parser.GetAnnotationWithPrefix("influxdb-host")] = "blabla" - defer func() { - ing.ObjectMeta.Annotations = map[string]string{} - }() - warnings, err := nginx.CheckWarning(ing) - if err != nil { - t.Errorf("no error should be returned, but %s was returned", err) - } - if len(warnings) != 4 { - t.Errorf("expected 4 warning to occur but %d occurred", len(warnings)) - } else { - t.Logf("got warnings %v correctly", warnings) - } - }) - }) - - t.Run("When the ingress is marked as deleted", func(t *testing.T) { - ing.DeletionTimestamp = &metav1.Time{ - Time: time.Now(), - } - - if warnings, err := nginx.CheckWarning(ing); err != nil || len(warnings) != 0 { - t.Errorf("when the ingress is marked as deleted, no warning should be returned") - } - }) -} - -//nolint:dupl // Ignore dupl errors for similar test case func TestMergeAlternativeBackends(t *testing.T) { testCases := map[string]struct { ingress *ingress.Ingress @@ -501,11 +321,10 @@ func TestMergeAlternativeBackends(t *testing.T) { Path: "/", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-canary", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "http-svc-canary", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, }, }, }, @@ -586,11 +405,10 @@ func TestMergeAlternativeBackends(t *testing.T) { Path: "/", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "foo-http-svc-canary", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "foo-http-svc-canary", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, }, }, }, @@ -607,11 +425,10 @@ func TestMergeAlternativeBackends(t *testing.T) { Path: "/", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-canary", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "http-svc-canary", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, }, }, }, @@ -726,11 +543,10 @@ func TestMergeAlternativeBackends(t *testing.T) { Path: "/", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-canary", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "http-svc-canary", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, }, }, }, @@ -762,12 +578,10 @@ func TestMergeAlternativeBackends(t *testing.T) { Namespace: "example", }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-canary", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + Backend: &networking.IngressBackend{ + ServiceName: "http-svc-canary", + ServicePort: intstr.IntOrString{ + IntVal: 80, }, }, }, @@ -832,12 +646,10 @@ func TestMergeAlternativeBackends(t *testing.T) { Namespace: "example", }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-canary", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + Backend: &networking.IngressBackend{ + ServiceName: "http-svc-canary", + ServicePort: intstr.IntOrString{ + IntVal: 80, }, }, }, @@ -898,11 +710,9 @@ func TestMergeAlternativeBackends(t *testing.T) { Path: "/", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-canary", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "http-svc-canary", + ServicePort: intstr.IntOrString{ + IntVal: 80, }, }, }, @@ -966,400 +776,6 @@ func TestMergeAlternativeBackends(t *testing.T) { }, }, }, - "alternative backend gets SessionAffinitySettings configured when CanaryBehavior is 'sticky'": { - &ingress.Ingress{ - Ingress: networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "example", - }, - Spec: networking.IngressSpec{ - Rules: []networking.IngressRule{ - { - Host: "example.com", - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - Path: "/", - PathType: &pathTypePrefix, - Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-canary", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - ParsedAnnotations: &annotations.Ingress{ - SessionAffinity: sessionaffinity.Config{ - CanaryBehavior: "sticky", - }, - }, - }, - map[string]*ingress.Backend{ - "example-http-svc-80": { - Name: "example-http-svc-80", - NoServer: false, - SessionAffinity: ingress.SessionAffinityConfig{ - AffinityType: "cookie", - AffinityMode: "balanced", - CookieSessionAffinity: ingress.CookieSessionAffinity{ - Name: "test", - }, - }, - }, - "example-http-svc-canary-80": { - Name: "example-http-svc-canary-80", - NoServer: true, - TrafficShapingPolicy: ingress.TrafficShapingPolicy{ - Weight: 20, - }, - }, - }, - map[string]*ingress.Server{ - "example.com": { - Hostname: "example.com", - Locations: []*ingress.Location{ - { - Path: "/", - PathType: &pathTypePrefix, - Backend: "example-http-svc-80", - }, - }, - }, - }, - map[string]*ingress.Backend{ - "example-http-svc-80": { - Name: "example-http-svc-80", - NoServer: false, - AlternativeBackends: []string{"example-http-svc-canary-80"}, - SessionAffinity: ingress.SessionAffinityConfig{ - AffinityType: "cookie", - AffinityMode: "balanced", - CookieSessionAffinity: ingress.CookieSessionAffinity{ - Name: "test", - }, - }, - }, - "example-http-svc-canary-80": { - Name: "example-http-svc-canary-80", - NoServer: true, - TrafficShapingPolicy: ingress.TrafficShapingPolicy{ - Weight: 20, - }, - SessionAffinity: ingress.SessionAffinityConfig{ - AffinityType: "cookie", - AffinityMode: "balanced", - CookieSessionAffinity: ingress.CookieSessionAffinity{ - Name: "test", - }, - }, - }, - }, - map[string]*ingress.Server{ - "example.com": { - Hostname: "example.com", - Locations: []*ingress.Location{ - { - Path: "/", - PathType: &pathTypePrefix, - Backend: "example-http-svc-80", - }, - }, - }, - }, - }, - "alternative backend gets SessionAffinitySettings configured when CanaryBehavior is not 'legacy'": { - &ingress.Ingress{ - Ingress: networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "example", - }, - Spec: networking.IngressSpec{ - Rules: []networking.IngressRule{ - { - Host: "example.com", - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - Path: "/", - PathType: &pathTypePrefix, - Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-canary", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - ParsedAnnotations: &annotations.Ingress{ - SessionAffinity: sessionaffinity.Config{ - CanaryBehavior: "", // In fact any value but 'legacy' would do the trick. - }, - }, - }, - map[string]*ingress.Backend{ - "example-http-svc-80": { - Name: "example-http-svc-80", - NoServer: false, - SessionAffinity: ingress.SessionAffinityConfig{ - AffinityType: "cookie", - AffinityMode: "balanced", - CookieSessionAffinity: ingress.CookieSessionAffinity{ - Name: "test", - }, - }, - }, - "example-http-svc-canary-80": { - Name: "example-http-svc-canary-80", - NoServer: true, - TrafficShapingPolicy: ingress.TrafficShapingPolicy{ - Weight: 20, - }, - }, - }, - map[string]*ingress.Server{ - "example.com": { - Hostname: "example.com", - Locations: []*ingress.Location{ - { - Path: "/", - PathType: &pathTypePrefix, - Backend: "example-http-svc-80", - }, - }, - }, - }, - map[string]*ingress.Backend{ - "example-http-svc-80": { - Name: "example-http-svc-80", - NoServer: false, - AlternativeBackends: []string{"example-http-svc-canary-80"}, - SessionAffinity: ingress.SessionAffinityConfig{ - AffinityType: "cookie", - AffinityMode: "balanced", - CookieSessionAffinity: ingress.CookieSessionAffinity{ - Name: "test", - }, - }, - }, - "example-http-svc-canary-80": { - Name: "example-http-svc-canary-80", - NoServer: true, - TrafficShapingPolicy: ingress.TrafficShapingPolicy{ - Weight: 20, - }, - SessionAffinity: ingress.SessionAffinityConfig{ - AffinityType: "cookie", - AffinityMode: "balanced", - CookieSessionAffinity: ingress.CookieSessionAffinity{ - Name: "test", - }, - }, - }, - }, - map[string]*ingress.Server{ - "example.com": { - Hostname: "example.com", - Locations: []*ingress.Location{ - { - Path: "/", - PathType: &pathTypePrefix, - Backend: "example-http-svc-80", - }, - }, - }, - }, - }, - "alternative backend doesn't get SessionAffinitySettings configured when CanaryBehavior is 'legacy'": { - &ingress.Ingress{ - Ingress: networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "example", - }, - Spec: networking.IngressSpec{ - Rules: []networking.IngressRule{ - { - Host: "example.com", - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - Path: "/", - PathType: &pathTypePrefix, - Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-canary", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - ParsedAnnotations: &annotations.Ingress{ - SessionAffinity: sessionaffinity.Config{ - CanaryBehavior: "legacy", - }, - }, - }, - map[string]*ingress.Backend{ - "example-http-svc-80": { - Name: "example-http-svc-80", - NoServer: false, - SessionAffinity: ingress.SessionAffinityConfig{ - AffinityType: "cookie", - AffinityMode: "balanced", - CookieSessionAffinity: ingress.CookieSessionAffinity{ - Name: "test", - }, - }, - }, - "example-http-svc-canary-80": { - Name: "example-http-svc-canary-80", - NoServer: true, - TrafficShapingPolicy: ingress.TrafficShapingPolicy{ - Weight: 20, - }, - }, - }, - map[string]*ingress.Server{ - "example.com": { - Hostname: "example.com", - Locations: []*ingress.Location{ - { - Path: "/", - PathType: &pathTypePrefix, - Backend: "example-http-svc-80", - }, - }, - }, - }, - map[string]*ingress.Backend{ - "example-http-svc-80": { - Name: "example-http-svc-80", - NoServer: false, - AlternativeBackends: []string{"example-http-svc-canary-80"}, - SessionAffinity: ingress.SessionAffinityConfig{ - AffinityType: "cookie", - AffinityMode: "balanced", - CookieSessionAffinity: ingress.CookieSessionAffinity{ - Name: "test", - }, - }, - }, - "example-http-svc-canary-80": { - Name: "example-http-svc-canary-80", - NoServer: true, - TrafficShapingPolicy: ingress.TrafficShapingPolicy{ - Weight: 20, - }, - }, - }, - map[string]*ingress.Server{ - "example.com": { - Hostname: "example.com", - Locations: []*ingress.Location{ - { - Path: "/", - PathType: &pathTypePrefix, - Backend: "example-http-svc-80", - }, - }, - }, - }, - }, - "alternative backend does not merge for missing upstream": { - &ingress.Ingress{ - Ingress: networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "example", - }, - Spec: networking.IngressSpec{ - Rules: []networking.IngressRule{ - { - Host: "example.com", - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - Path: "/", - PathType: &pathTypePrefix, - Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-canary", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - map[string]*ingress.Backend{ - "example-http-svc-canary-80": { - Name: "example-http-svc-canary-80", - NoServer: true, - TrafficShapingPolicy: ingress.TrafficShapingPolicy{ - Weight: 20, - }, - }, - }, - map[string]*ingress.Server{ - "example.com": { - Hostname: "example.com", - Locations: []*ingress.Location{ - { - Path: "/", - PathType: &pathTypePrefix, - Backend: "example-http-svc-80", - }, - }, - }, - }, - map[string]*ingress.Backend{}, - map[string]*ingress.Server{ - "example.com": { - Hostname: "example.com", - Locations: []*ingress.Location{ - { - Path: "/", - PathType: &pathTypePrefix, - Backend: "example-http-svc-80", - }, - }, - }, - }, - }, } for title, tc := range testCases { @@ -1375,7 +791,7 @@ func TestMergeAlternativeBackends(t *testing.T) { if !actualUpstream.Equal(expUpstream) { t.Logf("actual upstream %s alternative backends: %s", actualUpstream.Name, actualUpstream.AlternativeBackends) t.Logf("expected upstream %s alternative backends: %s", expUpstream.Name, expUpstream.AlternativeBackends) - t.Errorf("upstream %s was not equal to what was expected", actualUpstream.Name) + t.Errorf("upstream %s was not equal to what was expected: ", upsName) } } @@ -1548,55 +964,6 @@ func TestExtractTLSSecretName(t *testing.T) { }, "demo", }, - "ingress tls, hosts, bad format cert, host not in tls Hosts": { - "foo1.bar", - &ingress.Ingress{ - Ingress: networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - }, - Spec: networking.IngressSpec{ - TLS: []networking.IngressTLS{ - { - Hosts: []string{"foo.bar"}, - SecretName: "demo", - }, - }, - Rules: []networking.IngressRule{ - { - Host: "foo.bar", - }, - { - Host: "foo1.bar", - }, - }, - }, - }, - }, - func(string) (*ingress.SSLCert, error) { - secretData := map[string]string{ - "ca.crt": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJd01EVXhOVEEzTXpJMU5sb1hEVE13TURVeE16QTNNekkxTmxvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTUpGClVDcXBxY09mb2pPeUduTGRsRDlZSG5VVVFiTEQzZkxyN0FzM0RNTk9FNVkyTDJ2TUFhRXQxYkRvYWNqUDFMRlkKcy9ieFBVRmFmVEZ5cmY0SU1iN1FHUlZMYW84aVhmU3p4TlcwanE2dTc0OGZHN3E3c3QvNWR2c28yekpxcHNrcQo2QzNIS3liajMxMTBPNlh2N1I2VDlqdkxzT0M2Vm5VK3BtVHo4RzZ0YVdUeTFBdktQdU5Cc1puUWVRWis3Nk5hClRqSVpsaGlIMnZTUStSMFFzenBoU2tDQmVYbmdkaFloSDYyRVJvZDVZaDNiV1E3T2U0UjFKbHpNUjN3VzFSVGcKZE83aXoxSXdJT3drdlN6T1RlaElqU0pISHVWV3pJOXVqOTBkRTJUQmladzNheUVxdnhrbUxFT3Y1SFRONzRvbwpJQ216WEZzK1kxOFRCb2ZNRXNNQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFKajArK3lCVXBIekYxdWMxSmcyczhGRGNRRHIKdFBuanFROWE0QkwwN2JZdWZyeUJqdUlpaDJReWVKcFB3cTh0RklFa3ZuWjI4VDY4bDFLajRmMnRIU0Y0MG14WAoxVGlkNWxMc1ZjZytja1B2YWVRL1MrUmxnZDNCT1hjY3FFUWR0dithanhqOTdCZXZSelQ1SWd6UFVna3VtSU5wCkxrNS9kSWdxYTIrbmorVUpxdm9TWEhtZG4rMEdvNFJRMXMyZlBJUDhhRFIyL1paQThSTE1rSTJ2R1FVVUJ3RHMKeVkzVy9oWmRWeUhpWEcvRkJKRHNZU1cyZjFrZ1AzRzlyNjdnZG1WT05JckNCdHBSWkU3NllTRjFqNUtocFlUNgp3UDFpSVNDOUc0ZmRCeGxkaXJqRWM3VU9PQTlNQ0JzYXZ4R1IreCtBTEVnTnhOUlVZdnEvZWl0OGtRVT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=", - "namespace": "ZGVtbw==", - "token": "ZXlKaGJHY2lPaUpTVXpJMU5pSXNJbXRwWkNJNklqaExObWcyVnpWM01ERm9Ua1ZpVFVwTlYwbDRPV3RMVkhaTE1XRnpOa010VjI5WE55MTRaRzR6VFVVaWZRLmV5SnBjM01pT2lKcmRXSmxjbTVsZEdWekwzTmxjblpwWTJWaFkyTnZkVzUwSWl3aWEzVmlaWEp1WlhSbGN5NXBieTl6WlhKMmFXTmxZV05qYjNWdWRDOXVZVzFsYzNCaFkyVWlPaUprWlcxdklpd2lhM1ZpWlhKdVpYUmxjeTVwYnk5elpYSjJhV05sWVdOamIzVnVkQzl6WldOeVpYUXVibUZ0WlNJNkltUmxabUYxYkhRdGRHOXJaVzR0Wkc0MmVHSWlMQ0pyZFdKbGNtNWxkR1Z6TG1sdkwzTmxjblpwWTJWaFkyTnZkVzUwTDNObGNuWnBZMlV0WVdOamIzVnVkQzV1WVcxbElqb2laR1ZtWVhWc2RDSXNJbXQxWW1WeWJtVjBaWE11YVc4dmMyVnlkbWxqWldGalkyOTFiblF2YzJWeWRtbGpaUzFoWTJOdmRXNTBMblZwWkNJNkltVm1OR0kxWW1NMExUTXdPREV0TkRFNU15MWlZakl6TFRoaE5qRmhNV0ptTWpRNFlTSXNJbk4xWWlJNkluTjVjM1JsYlRwelpYSjJhV05sWVdOamIzVnVkRHBrWlcxdk9tUmxabUYxYkhRaWZRLnEzaGFxVVFDN2Z6a1V3UldKazM0RjRsamktbWs5cWdPcDJHSFlSZ1JrWUk0WW8xclhoSURCSnUzWkFPdjhMN3doZkgzcmo4ZjFnNFpMSFBkd3JKT2lZdWlvXzVXdDZPSXZtbXFaU2VncnRmV1MwUFZXYzJ1d0xweDJpSElTbUlHd21uQ1hYQzNRX05RNFRlQnZxWEMyUHR4REFwM19QM3QyZnRKN0w2Z1kzTkcyZUsyQTVFZG82azQtR2wzN0Zaam51NmRzc0FocVZaeld0NE9ZS3hTWWtpN003dnh5ZWtJQ091UmJ6SW5DNmhldEhtbHhyaF9ObWplMHhfY2M4V3ZkUnJYbFlpRWxnYXZCY1FtMTJ2YkxBQWlzWkFrT2Y1T3VvaEhLUmpEOGlMS1pRMXdKRHNnRmYzd1BFWGxTWkg2QkVZdS1TU0laSDNKYWVWU3llWjExdw==", - } - ca, err := base64.StdEncoding.DecodeString(secretData["ca.crt"]) - if err != nil { - t.Fatalf("unexpected error decoding ca.crt: %v", err) - } - cert, err := ssl.CreateCACert(ca) - if err != nil { - t.Fatalf("unexpected error creating SSL Cert: %v", err) - } - err = ssl.ConfigureCACert("demo", ca, cert) - if err != nil { - t.Fatalf("error configuring CA certificate: %v", err) - } - cert.Name = "default-token-dn6xb" - cert.Namespace = "demo" - return cert, nil - }, - "", - }, } for title, tc := range testCases { @@ -1609,12 +976,12 @@ func TestExtractTLSSecretName(t *testing.T) { } } -//nolint:gocyclo // Ignore function complexity error func TestGetBackendServers(t *testing.T) { + testCases := []struct { Ingresses []*ingress.Ingress Validate func(ingresses []*ingress.Ingress, upstreams []*ingress.Backend, servers []*ingress.Server) - SetConfigMap func(namespace string) *corev1.ConfigMap + SetConfigMap func(namespace string) *v1.ConfigMap }{ { Ingresses: []*ingress.Ingress{ @@ -1624,12 +991,10 @@ func TestGetBackendServers(t *testing.T) { Namespace: "example", }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-canary", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + Backend: &networking.IngressBackend{ + ServiceName: "http-svc-canary", + ServicePort: intstr.IntOrString{ + IntVal: 80, }, }, }, @@ -1641,7 +1006,7 @@ func TestGetBackendServers(t *testing.T) { }, }, }, - Validate: func(_ []*ingress.Ingress, _ []*ingress.Backend, servers []*ingress.Server) { + Validate: func(ingresses []*ingress.Ingress, upstreams []*ingress.Backend, servers []*ingress.Server) { if len(servers) != 1 { t.Errorf("servers count should be 1, got %d", len(servers)) return @@ -1669,12 +1034,10 @@ func TestGetBackendServers(t *testing.T) { Namespace: "example", }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-canary", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + Backend: &networking.IngressBackend{ + ServiceName: "http-svc-canary", + ServicePort: intstr.IntOrString{ + IntVal: 80, }, }, }, @@ -1691,12 +1054,10 @@ func TestGetBackendServers(t *testing.T) { Namespace: "example", }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + Backend: &networking.IngressBackend{ + ServiceName: "http-svc", + ServicePort: intstr.IntOrString{ + IntVal: 80, }, }, }, @@ -1708,7 +1069,7 @@ func TestGetBackendServers(t *testing.T) { }, }, }, - Validate: func(_ []*ingress.Ingress, _ []*ingress.Backend, servers []*ingress.Server) { + Validate: func(ingresses []*ingress.Ingress, upstreams []*ingress.Backend, servers []*ingress.Server) { if len(servers) != 1 { t.Errorf("servers count should be 1, got %d", len(servers)) return @@ -1746,11 +1107,10 @@ func TestGetBackendServers(t *testing.T) { Path: "/", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-canary", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "http-svc-canary", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, }, }, }, @@ -1768,7 +1128,7 @@ func TestGetBackendServers(t *testing.T) { }, }, }, - Validate: func(_ []*ingress.Ingress, _ []*ingress.Backend, servers []*ingress.Server) { + Validate: func(ingresses []*ingress.Ingress, upstreams []*ingress.Backend, servers []*ingress.Server) { if len(servers) != 1 { t.Errorf("servers count should be 1, got %d", len(servers)) return @@ -1807,11 +1167,10 @@ func TestGetBackendServers(t *testing.T) { Path: "/", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "http-svc", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, }, }, }, @@ -1845,11 +1204,10 @@ func TestGetBackendServers(t *testing.T) { Path: "/", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-canary", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "http-svc-canary", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, }, }, }, @@ -1867,7 +1225,7 @@ func TestGetBackendServers(t *testing.T) { }, }, }, - Validate: func(_ []*ingress.Ingress, _ []*ingress.Backend, servers []*ingress.Server) { + Validate: func(ingresses []*ingress.Ingress, upstreams []*ingress.Backend, servers []*ingress.Server) { if len(servers) != 2 { t.Errorf("servers count should be 2, got %d", len(servers)) return @@ -1915,11 +1273,10 @@ func TestGetBackendServers(t *testing.T) { Path: "/a", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-1", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "http-svc-1", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, }, }, }, @@ -1953,11 +1310,10 @@ func TestGetBackendServers(t *testing.T) { Path: "/a", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-2", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "http-svc-2", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, }, }, }, @@ -1991,11 +1347,10 @@ func TestGetBackendServers(t *testing.T) { Path: "/b", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-2", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "http-svc-2", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, }, }, }, @@ -2029,11 +1384,10 @@ func TestGetBackendServers(t *testing.T) { Path: "/b", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-1", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "http-svc-1", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, }, }, }, @@ -2067,11 +1421,10 @@ func TestGetBackendServers(t *testing.T) { Path: "/c", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-1", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "http-svc-1", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, }, }, }, @@ -2105,11 +1458,10 @@ func TestGetBackendServers(t *testing.T) { Path: "/c", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc-2", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "http-svc-2", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, }, }, }, @@ -2127,7 +1479,7 @@ func TestGetBackendServers(t *testing.T) { }, }, }, - Validate: func(_ []*ingress.Ingress, upstreams []*ingress.Backend, servers []*ingress.Server) { + Validate: func(ingresses []*ingress.Ingress, upstreams []*ingress.Backend, servers []*ingress.Server) { if len(servers) != 2 { t.Errorf("servers count should be 2, got %d", len(servers)) return @@ -2150,7 +1502,7 @@ func TestGetBackendServers(t *testing.T) { t.Errorf("server hostname should be 'example.com', got '%s'", s.Hostname) } - if s.Locations[0].Backend != exampleBackend || s.Locations[1].Backend != exampleBackend || s.Locations[2].Backend != exampleBackend { + if s.Locations[0].Backend != "example-http-svc-1-80" || s.Locations[1].Backend != "example-http-svc-1-80" || s.Locations[2].Backend != "example-http-svc-1-80" { t.Errorf("all location backend should be 'example-http-svc-1-80'") } @@ -2159,7 +1511,7 @@ func TestGetBackendServers(t *testing.T) { return } - if upstreams[0].Name != exampleBackend { + if upstreams[0].Name != "example-http-svc-1-80" { t.Errorf("example-http-svc-1-80 should be first upstream, got %s", upstreams[0].Name) return } @@ -2173,7 +1525,6 @@ func TestGetBackendServers(t *testing.T) { SetConfigMap: testConfigMap, }, { - //nolint:dupl // Ignore dupl errors for similar test case Ingresses: []*ingress.Ingress{ { Ingress: networking.Ingress{ @@ -2192,11 +1543,10 @@ func TestGetBackendServers(t *testing.T) { Path: "/path1", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "path1-svc", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "path1-svc", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, }, }, }, @@ -2233,11 +1583,10 @@ func TestGetBackendServers(t *testing.T) { Path: "/path2", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "path2-svc", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "path2-svc", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, }, }, }, @@ -2258,7 +1607,7 @@ func TestGetBackendServers(t *testing.T) { }, }, }, - Validate: func(ingresses []*ingress.Ingress, _ []*ingress.Backend, servers []*ingress.Server) { + Validate: func(ingresses []*ingress.Ingress, upstreams []*ingress.Backend, servers []*ingress.Server) { if len(servers) != 2 { t.Errorf("servers count should be 2, got %d", len(servers)) return @@ -2281,7 +1630,6 @@ func TestGetBackendServers(t *testing.T) { SetConfigMap: testConfigMap, }, { - //nolint:dupl // Ignore dupl errors for similar test case Ingresses: []*ingress.Ingress{ { Ingress: networking.Ingress{ @@ -2300,11 +1648,10 @@ func TestGetBackendServers(t *testing.T) { Path: "/path1", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "path1-svc", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "path1-svc", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, }, }, }, @@ -2341,11 +1688,10 @@ func TestGetBackendServers(t *testing.T) { Path: "/path2", PathType: &pathTypePrefix, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "path2-svc", - Port: networking.ServiceBackendPort{ - Number: 80, - }, + ServiceName: "path2-svc", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, }, }, }, @@ -2366,7 +1712,7 @@ func TestGetBackendServers(t *testing.T) { }, }, }, - Validate: func(ingresses []*ingress.Ingress, _ []*ingress.Backend, servers []*ingress.Server) { + Validate: func(ingresses []*ingress.Ingress, upstreams []*ingress.Backend, servers []*ingress.Server) { if len(servers) != 2 { t.Errorf("servers count should be 2, got %d", len(servers)) return @@ -2386,152 +1732,14 @@ func TestGetBackendServers(t *testing.T) { t.Errorf("location cafilename should be '%s', got '%s'", ingresses[1].ParsedAnnotations.ProxySSL.CAFileName, s.Locations[0].ProxySSL.CAFileName) } }, - SetConfigMap: func(ns string) *corev1.ConfigMap { - return &corev1.ConfigMap{ + SetConfigMap: func(ns string) *v1.ConfigMap { + return &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "config", SelfLink: fmt.Sprintf("/api/v1/namespaces/%s/configmaps/config", ns), }, Data: map[string]string{ - "proxy-ssl-location-only": TRUE, - }, - } - }, - }, - { - Ingresses: []*ingress.Ingress{ - { - Ingress: networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "proxy-ssl-1", - Namespace: "proxyssl", - }, - Spec: networking.IngressSpec{ - Rules: []networking.IngressRule{ - { - Host: "example.com", - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - Path: "/path1", - PathType: &pathTypePrefix, - Backend: networking.IngressBackend{}, - }, - }, - }, - }, - }, - }, - }, - }, - ParsedAnnotations: &annotations.Ingress{ - ProxySSL: proxyssl.Config{ - AuthSSLCert: resolver.AuthSSLCert{ - CAFileName: "cafile1.crt", - Secret: "secret1", - }, - }, - }, - }, - }, - Validate: func(_ []*ingress.Ingress, _ []*ingress.Backend, servers []*ingress.Server) { - if len(servers) != 2 { - t.Errorf("servers count should be 1, got %d", len(servers)) - return - } - - s := servers[1] - - if s.Locations[0].Backend != "upstream-default-backend" { - t.Errorf("backend should be upstream-default-backend, got '%s'", s.Locations[0].Backend) - } - }, - SetConfigMap: func(ns string) *corev1.ConfigMap { - return &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "config", - SelfLink: fmt.Sprintf("/api/v1/namespaces/%s/configmaps/config", ns), - }, - Data: map[string]string{ - "proxy-ssl-location-only": TRUE, - }, - } - }, - }, - { - Ingresses: []*ingress.Ingress{ - { - Ingress: networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "not-allowed-snippet", - Namespace: "default", - Annotations: map[string]string{ - "nginx.ingress.kubernetes.io/server-snippet": "bla", - "nginx.ingress.kubernetes.io/configuration-snippet": "blo", - "nginx.ingress.kubernetes.io/whitelist-source-range": "10.0.0.0/24", - }, - }, - Spec: networking.IngressSpec{ - Rules: []networking.IngressRule{ - { - Host: "example.com", - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - Path: "/path1", - PathType: &pathTypePrefix, - Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "path1-svc", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - ParsedAnnotations: &annotations.Ingress{ - Allowlist: ipallowlist.SourceRange{CIDR: []string{"10.0.0.0/24"}}, - ServerSnippet: "bla", - ConfigurationSnippet: "blo", - }, - }, - }, - Validate: func(_ []*ingress.Ingress, _ []*ingress.Backend, servers []*ingress.Server) { - if len(servers) != 2 { - t.Errorf("servers count should be 2, got %d", len(servers)) - return - } - s := servers[1] - - if s.ServerSnippet != "" { - t.Errorf("server snippet should be empty, got '%s'", s.ServerSnippet) - } - - if s.Locations[0].ConfigurationSnippet != "" { - t.Errorf("config snippet should be empty, got '%s'", s.Locations[0].ConfigurationSnippet) - } - - if len(s.Locations[0].Allowlist.CIDR) != 1 || s.Locations[0].Allowlist.CIDR[0] != "10.0.0.0/24" { - t.Errorf("allow list was incorrectly dropped, len should be 1 and contain 10.0.0.0/24") - } - }, - SetConfigMap: func(ns string) *corev1.ConfigMap { - return &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "config", - SelfLink: fmt.Sprintf("/api/v1/namespaces/%s/configmaps/config", ns), - }, - Data: map[string]string{ - "allow-snippet-annotations": "false", + "proxy-ssl-location-only": "true", }, } }, @@ -2545,8 +1753,8 @@ func TestGetBackendServers(t *testing.T) { } } -func testConfigMap(ns string) *corev1.ConfigMap { - return &corev1.ConfigMap{ +func testConfigMap(ns string) *v1.ConfigMap { + return &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "config", SelfLink: fmt.Sprintf("/api/v1/namespaces/%s/configmaps/config", ns), @@ -2555,11 +1763,11 @@ func testConfigMap(ns string) *corev1.ConfigMap { } func newNGINXController(t *testing.T) *NGINXController { - ns := corev1.NamespaceDefault + ns := v1.NamespaceDefault clientSet := fake.NewSimpleClientset() - configMap := &corev1.ConfigMap{ + configMap := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "config", SelfLink: fmt.Sprintf("/api/v1/namespaces/%s/configmaps/config", ns), @@ -2583,7 +1791,6 @@ func newNGINXController(t *testing.T) *NGINXController { storer := store.New( ns, - labels.Nothing(), fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -2591,14 +1798,7 @@ func newNGINXController(t *testing.T) *NGINXController { 10*time.Minute, clientSet, channels.NewRingChannel(10), - false, - true, - &ingressclass.Configuration{ - Controller: "k8s.io/ingress-nginx", - AnnotationValue: "nginx", - }, - false, - ) + false) sslCert := ssl.GetFakeSSLCert() config := &Configuration{ @@ -2626,8 +1826,8 @@ func fakeX509Cert(dnsNames []string) *x509.Certificate { } } -func newDynamicNginxController(t *testing.T, setConfigMap func(string) *corev1.ConfigMap) *NGINXController { - ns := corev1.NamespaceDefault +func newDynamicNginxController(t *testing.T, setConfigMap func(string) *v1.ConfigMap) *NGINXController { + ns := v1.NamespaceDefault clientSet := fake.NewSimpleClientset() configMap := setConfigMap(ns) @@ -2649,7 +1849,6 @@ func newDynamicNginxController(t *testing.T, setConfigMap func(string) *corev1.C storer := store.New( ns, - labels.Nothing(), fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -2657,12 +1856,6 @@ func newDynamicNginxController(t *testing.T, setConfigMap func(string) *corev1.C 10*time.Minute, clientSet, channels.NewRingChannel(10), - false, - true, - &ingressclass.Configuration{ - Controller: "k8s.io/ingress-nginx", - AnnotationValue: "nginx", - }, false) sslCert := ssl.GetFakeSSLCert() @@ -2674,9 +1867,8 @@ func newDynamicNginxController(t *testing.T, setConfigMap func(string) *corev1.C } return &NGINXController{ - store: storer, - cfg: config, - command: NewNginxCommand(), - metricCollector: metric.DummyCollector{}, + store: storer, + cfg: config, + command: NewNginxCommand(), } } diff --git a/internal/ingress/controller/endpoints.go b/internal/ingress/controller/endpoints.go new file mode 100644 index 000000000..bdddcb0a0 --- /dev/null +++ b/internal/ingress/controller/endpoints.go @@ -0,0 +1,121 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "fmt" + "net" + "reflect" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/klog/v2" + + corev1 "k8s.io/api/core/v1" + + "k8s.io/ingress-nginx/internal/ingress" + "k8s.io/ingress-nginx/internal/k8s" +) + +// getEndpoints returns a list of Endpoint structs for a given service/target port combination. +func getEndpoints(s *corev1.Service, port *corev1.ServicePort, proto corev1.Protocol, + getServiceEndpoints func(string) (*corev1.Endpoints, error)) []ingress.Endpoint { + + upsServers := []ingress.Endpoint{} + + if s == nil || port == nil { + return upsServers + } + + // using a map avoids duplicated upstream servers when the service + // contains multiple port definitions sharing the same targetport + processedUpstreamServers := make(map[string]struct{}) + + svcKey := k8s.MetaNamespaceKey(s) + + // ExternalName services + if s.Spec.Type == corev1.ServiceTypeExternalName { + if ip := net.ParseIP(s.Spec.ExternalName); s.Spec.ExternalName == "localhost" || + (ip != nil && ip.IsLoopback()) { + klog.Errorf("Invalid attempt to use localhost name %s in %q", s.Spec.ExternalName, svcKey) + return upsServers + } + + klog.V(3).Infof("Ingress using Service %q of type ExternalName.", svcKey) + targetPort := port.TargetPort.IntValue() + // if the externalName is not an IP address we need to validate is a valid FQDN + if net.ParseIP(s.Spec.ExternalName) == nil { + externalName := strings.TrimSuffix(s.Spec.ExternalName, ".") + if errs := validation.IsDNS1123Subdomain(externalName); len(errs) > 0 { + klog.Errorf("Invalid DNS name %s: %v", s.Spec.ExternalName, errs) + return upsServers + } + } + + return append(upsServers, ingress.Endpoint{ + Address: s.Spec.ExternalName, + Port: fmt.Sprintf("%v", targetPort), + }) + } + + klog.V(3).Infof("Getting Endpoints for Service %q and port %v", svcKey, port.String()) + ep, err := getServiceEndpoints(svcKey) + if err != nil { + klog.Warningf("Error obtaining Endpoints for Service %q: %v", svcKey, err) + return upsServers + } + + for _, ss := range ep.Subsets { + for _, epPort := range ss.Ports { + + if !reflect.DeepEqual(epPort.Protocol, proto) { + continue + } + + var targetPort int32 + + if port.Name == "" { + // port.Name is optional if there is only one port + targetPort = epPort.Port + } else if port.Name == epPort.Name { + targetPort = epPort.Port + } + + if targetPort <= 0 { + continue + } + + for _, epAddress := range ss.Addresses { + ep := net.JoinHostPort(epAddress.IP, strconv.Itoa(int(targetPort))) + if _, exists := processedUpstreamServers[ep]; exists { + continue + } + ups := ingress.Endpoint{ + Address: epAddress.IP, + Port: fmt.Sprintf("%v", targetPort), + Target: epAddress.TargetRef, + } + upsServers = append(upsServers, ups) + processedUpstreamServers[ep] = struct{}{} + } + } + } + + klog.V(3).Infof("Endpoints found for Service %q: %v", svcKey, upsServers) + return upsServers +} diff --git a/internal/ingress/controller/endpoints_test.go b/internal/ingress/controller/endpoints_test.go new file mode 100644 index 000000000..20d53c526 --- /dev/null +++ b/internal/ingress/controller/endpoints_test.go @@ -0,0 +1,471 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "fmt" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/ingress-nginx/internal/ingress" +) + +func TestGetEndpoints(t *testing.T) { + tests := []struct { + name string + svc *corev1.Service + port *corev1.ServicePort + proto corev1.Protocol + fn func(string) (*corev1.Endpoints, error) + result []ingress.Endpoint + }{ + { + "no service should return 0 endpoint", + nil, + nil, + corev1.ProtocolTCP, + func(string) (*corev1.Endpoints, error) { + return nil, nil + }, + []ingress.Endpoint{}, + }, + { + "no service port should return 0 endpoint", + &corev1.Service{}, + nil, + corev1.ProtocolTCP, + func(string) (*corev1.Endpoints, error) { + return nil, nil + }, + []ingress.Endpoint{}, + }, + { + "a service without endpoint should return 0 endpoint", + &corev1.Service{}, + &corev1.ServicePort{Name: "default"}, + corev1.ProtocolTCP, + func(string) (*corev1.Endpoints, error) { + return &corev1.Endpoints{}, nil + }, + []ingress.Endpoint{}, + }, + { + "a service type ServiceTypeExternalName service with an invalid port should return 0 endpoint", + &corev1.Service{ + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeExternalName, + }, + }, + &corev1.ServicePort{Name: "default"}, + corev1.ProtocolTCP, + func(string) (*corev1.Endpoints, error) { + return &corev1.Endpoints{}, nil + }, + []ingress.Endpoint{}, + }, + { + "a service type ServiceTypeExternalName service with localhost in name should return 0 endpoint", + &corev1.Service{ + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeExternalName, + ExternalName: "localhost", + Ports: []corev1.ServicePort{ + { + Name: "default", + TargetPort: intstr.FromInt(443), + }, + }, + }, + }, + &corev1.ServicePort{ + Name: "default", + TargetPort: intstr.FromInt(80), + }, + corev1.ProtocolTCP, + func(string) (*corev1.Endpoints, error) { + return &corev1.Endpoints{}, nil + }, + []ingress.Endpoint{}, + }, + { + "a service type ServiceTypeExternalName service with 127.0.0.1 in name should return 0 endpoint", + &corev1.Service{ + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeExternalName, + ExternalName: "127.0.0.1", + Ports: []corev1.ServicePort{ + { + Name: "default", + TargetPort: intstr.FromInt(443), + }, + }, + }, + }, + &corev1.ServicePort{ + Name: "default", + TargetPort: intstr.FromInt(80), + }, + corev1.ProtocolTCP, + func(string) (*corev1.Endpoints, error) { + return &corev1.Endpoints{}, nil + }, + []ingress.Endpoint{}, + }, + { + "a service type ServiceTypeExternalName with a valid port should return one endpoint", + &corev1.Service{ + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeExternalName, + ExternalName: "www.google.com", + Ports: []corev1.ServicePort{ + { + Name: "default", + TargetPort: intstr.FromInt(443), + }, + }, + }, + }, + &corev1.ServicePort{ + Name: "default", + TargetPort: intstr.FromInt(80), + }, + corev1.ProtocolTCP, + func(string) (*corev1.Endpoints, error) { + return &corev1.Endpoints{}, nil + }, + []ingress.Endpoint{ + { + Address: "www.google.com", + Port: "443", + }, + }, + }, + { + "a service type ServiceTypeExternalName with an trailing dot ExternalName value should return one endpoints", + &corev1.Service{ + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeExternalName, + ExternalName: "www.google.com.", + Ports: []corev1.ServicePort{ + { + Name: "default", + TargetPort: intstr.FromInt(80), + }, + }, + }, + }, + &corev1.ServicePort{ + Name: "default", + TargetPort: intstr.FromInt(80), + }, + corev1.ProtocolTCP, + func(string) (*corev1.Endpoints, error) { + return &corev1.Endpoints{}, nil + }, + []ingress.Endpoint{ + { + Address: "www.google.com", + Port: "443", + }, + }, + }, + { + "a service type ServiceTypeExternalName with an invalid ExternalName value should no return endpoints", + &corev1.Service{ + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeExternalName, + ExternalName: "1#invalid.hostname", + Ports: []corev1.ServicePort{ + { + Name: "default", + TargetPort: intstr.FromInt(80), + }, + }, + }, + }, + &corev1.ServicePort{ + Name: "default", + TargetPort: intstr.FromInt(80), + }, + corev1.ProtocolTCP, + func(string) (*corev1.Endpoints, error) { + return &corev1.Endpoints{}, nil + }, + []ingress.Endpoint{}, + }, + { + "should return no endpoint when there is an error searching for endpoints", + &corev1.Service{ + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "1.1.1.1", + Ports: []corev1.ServicePort{ + { + Name: "default", + TargetPort: intstr.FromInt(80), + }, + }, + }, + }, + &corev1.ServicePort{ + Name: "default", + TargetPort: intstr.FromInt(80), + }, + corev1.ProtocolTCP, + func(string) (*corev1.Endpoints, error) { + return nil, fmt.Errorf("unexpected error") + }, + []ingress.Endpoint{}, + }, + { + "should return no endpoint when the protocol does not match", + &corev1.Service{ + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "1.1.1.1", + Ports: []corev1.ServicePort{ + { + Name: "default", + TargetPort: intstr.FromInt(80), + }, + }, + }, + }, + &corev1.ServicePort{ + Name: "default", + TargetPort: intstr.FromInt(80), + }, + corev1.ProtocolTCP, + func(string) (*corev1.Endpoints, error) { + nodeName := "dummy" + return &corev1.Endpoints{ + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{ + { + IP: "1.1.1.1", + NodeName: &nodeName, + }, + }, + Ports: []corev1.EndpointPort{ + { + Protocol: corev1.ProtocolUDP, + }, + }, + }, + }, + }, nil + }, + []ingress.Endpoint{}, + }, + { + "should return no endpoint when there is no ready Addresses", + &corev1.Service{ + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "1.1.1.1", + Ports: []corev1.ServicePort{ + { + Name: "default", + TargetPort: intstr.FromInt(80), + }, + }, + }, + }, + &corev1.ServicePort{ + Name: "default", + TargetPort: intstr.FromInt(80), + }, + corev1.ProtocolTCP, + func(string) (*corev1.Endpoints, error) { + nodeName := "dummy" + return &corev1.Endpoints{ + Subsets: []corev1.EndpointSubset{ + { + NotReadyAddresses: []corev1.EndpointAddress{ + { + IP: "1.1.1.1", + NodeName: &nodeName, + }, + }, + Ports: []corev1.EndpointPort{ + { + Protocol: corev1.ProtocolUDP, + }, + }, + }, + }, + }, nil + }, + []ingress.Endpoint{}, + }, + { + "should return no endpoint when the name of the port name do not match any port in the endpoint Subsets", + &corev1.Service{ + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "1.1.1.1", + Ports: []corev1.ServicePort{ + { + Name: "default", + TargetPort: intstr.FromInt(80), + }, + }, + }, + }, + &corev1.ServicePort{ + Name: "default", + TargetPort: intstr.FromInt(80), + }, + corev1.ProtocolTCP, + func(string) (*corev1.Endpoints, error) { + nodeName := "dummy" + return &corev1.Endpoints{ + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{ + { + IP: "1.1.1.1", + NodeName: &nodeName, + }, + }, + Ports: []corev1.EndpointPort{ + { + Protocol: corev1.ProtocolTCP, + Port: int32(80), + Name: "another-name", + }, + }, + }, + }, + }, nil + }, + []ingress.Endpoint{}, + }, + { + "should return one endpoint when the name of the port name match a port in the endpoint Subsets", + &corev1.Service{ + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "1.1.1.1", + Ports: []corev1.ServicePort{ + { + Name: "default", + TargetPort: intstr.FromInt(80), + }, + }, + }, + }, + &corev1.ServicePort{ + Name: "default", + TargetPort: intstr.FromInt(80), + }, + corev1.ProtocolTCP, + func(string) (*corev1.Endpoints, error) { + nodeName := "dummy" + return &corev1.Endpoints{ + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{ + { + IP: "1.1.1.1", + NodeName: &nodeName, + }, + }, + Ports: []corev1.EndpointPort{ + { + Protocol: corev1.ProtocolTCP, + Port: int32(80), + Name: "default", + }, + }, + }, + }, + }, nil + }, + []ingress.Endpoint{ + { + Address: "1.1.1.1", + Port: "80", + }, + }, + }, + { + "should return one endpoint when the name of the port name match more than one port in the endpoint Subsets", + &corev1.Service{ + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "1.1.1.1", + Ports: []corev1.ServicePort{ + { + Name: "default", + TargetPort: intstr.FromString("port-1"), + }, + }, + }, + }, + &corev1.ServicePort{ + Name: "port-1", + TargetPort: intstr.FromString("port-1"), + }, + corev1.ProtocolTCP, + func(string) (*corev1.Endpoints, error) { + nodeName := "dummy" + return &corev1.Endpoints{ + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{ + { + IP: "1.1.1.1", + NodeName: &nodeName, + }, + }, + Ports: []corev1.EndpointPort{ + { + Name: "port-1", + Protocol: corev1.ProtocolTCP, + Port: 80, + }, + { + Name: "port-1", + Protocol: corev1.ProtocolTCP, + Port: 80, + }, + }, + }, + }, + }, nil + }, + []ingress.Endpoint{ + { + Address: "1.1.1.1", + Port: "80", + }, + }, + }, + } + + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + result := getEndpoints(testCase.svc, testCase.port, testCase.proto, testCase.fn) + if len(testCase.result) != len(result) { + t.Errorf("Expected %d Endpoints but got %d", len(testCase.result), len(result)) + } + }) + } +} diff --git a/internal/ingress/controller/endpointslices.go b/internal/ingress/controller/endpointslices.go deleted file mode 100644 index ed46e2c85..000000000 --- a/internal/ingress/controller/endpointslices.go +++ /dev/null @@ -1,168 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "fmt" - "net" - "reflect" - "strconv" - "strings" - - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/apimachinery/pkg/util/validation" - "k8s.io/klog/v2" - - corev1 "k8s.io/api/core/v1" - discoveryv1 "k8s.io/api/discovery/v1" - - "k8s.io/ingress-nginx/internal/k8s" - "k8s.io/ingress-nginx/pkg/apis/ingress" -) - -// getEndpointsFromSlices returns a list of Endpoint structs for a given service/target port combination. -func getEndpointsFromSlices(s *corev1.Service, port *corev1.ServicePort, proto corev1.Protocol, zoneForHints string, - getServiceEndpointsSlices func(string) ([]*discoveryv1.EndpointSlice, error), -) []ingress.Endpoint { - upsServers := []ingress.Endpoint{} - - if s == nil || port == nil { - return upsServers - } - - // using a map avoids duplicated upstream servers when the service - // contains multiple port definitions sharing the same targetport - processedUpstreamServers := make(map[string]struct{}) - - svcKey := k8s.MetaNamespaceKey(s) - var useTopologyHints bool - - // ExternalName services - if s.Spec.Type == corev1.ServiceTypeExternalName { - if ip := net.ParseIP(s.Spec.ExternalName); s.Spec.ExternalName == "localhost" || - (ip != nil && ip.IsLoopback()) { - klog.Errorf("Invalid attempt to use localhost name %s in %q", s.Spec.ExternalName, svcKey) - return upsServers - } - - klog.V(3).Infof("Ingress using Service %q of type ExternalName.", svcKey) - targetPort := port.TargetPort.IntValue() - // if the externalName is not an IP address we need to validate is a valid FQDN - if net.ParseIP(s.Spec.ExternalName) == nil { - externalName := strings.TrimSuffix(s.Spec.ExternalName, ".") - if errs := validation.IsDNS1123Subdomain(externalName); len(errs) > 0 { - klog.Errorf("Invalid DNS name %s: %v", s.Spec.ExternalName, errs) - return upsServers - } - } - - return append(upsServers, ingress.Endpoint{ - Address: s.Spec.ExternalName, - Port: fmt.Sprintf("%v", targetPort), - }) - } - - klog.V(3).Infof("Getting Endpoints from endpointSlices for Service %q and port %v", svcKey, port.String()) - epss, err := getServiceEndpointsSlices(svcKey) - if err != nil { - klog.Warningf("Error obtaining Endpoints for Service %q: %v", svcKey, err) - return upsServers - } - // loop over all endpointSlices generated for service - for _, eps := range epss { - var ports []int32 - if len(eps.Ports) == 0 && port.TargetPort.Type == intstr.Int { - // When ports is empty, it indicates that there are no defined ports, using svc targePort if it's a number - klog.V(3).Infof("No ports found on endpointSlice, using service TargetPort %v for Service %q", port.String(), svcKey) - ports = append(ports, port.TargetPort.IntVal) - } else { - for _, epPort := range eps.Ports { - if !reflect.DeepEqual(*epPort.Protocol, proto) { - continue - } - var targetPort int32 - if port.Name == "" { - // port.Name is optional if there is only one port - targetPort = *epPort.Port - } else if port.Name == *epPort.Name { - targetPort = *epPort.Port - } - if targetPort == 0 && port.TargetPort.Type == intstr.Int { - // use service target port if it's a number and no port name matched - // https://github.com/kubernetes/ingress-nginx/issues/7390 - targetPort = port.TargetPort.IntVal - } - if targetPort == 0 { - continue - } - ports = append(ports, targetPort) - } - } - useTopologyHints = false - if zoneForHints != emptyZone { - useTopologyHints = true - // check if all endpointslices have zone hints - for _, ep := range eps.Endpoints { - if ep.Hints == nil || len(ep.Hints.ForZones) == 0 { - useTopologyHints = false - break - } - } - if useTopologyHints { - klog.V(3).Infof("All endpoint slices has zone hint, using zone %q for Service %q", zoneForHints, svcKey) - } - } - - for _, ep := range eps.Endpoints { - if (ep.Conditions.Ready != nil) && !(*ep.Conditions.Ready) { - continue - } - epHasZone := false - if useTopologyHints { - for _, epzone := range ep.Hints.ForZones { - if epzone.Name == zoneForHints { - epHasZone = true - break - } - } - } - - if useTopologyHints && !epHasZone { - continue - } - - for _, epPort := range ports { - for _, epAddress := range ep.Addresses { - hostPort := net.JoinHostPort(epAddress, strconv.Itoa(int(epPort))) - if _, exists := processedUpstreamServers[hostPort]; exists { - continue - } - ups := ingress.Endpoint{ - Address: epAddress, - Port: fmt.Sprintf("%v", epPort), - Target: ep.TargetRef, - } - upsServers = append(upsServers, ups) - processedUpstreamServers[hostPort] = struct{}{} - } - } - } - } - - klog.V(3).Infof("Endpoints found for Service %q: %v", svcKey, upsServers) - return upsServers -} diff --git a/internal/ingress/controller/endpointslices_test.go b/internal/ingress/controller/endpointslices_test.go deleted file mode 100644 index 69ef3ef1b..000000000 --- a/internal/ingress/controller/endpointslices_test.go +++ /dev/null @@ -1,929 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "fmt" - "testing" - - corev1 "k8s.io/api/core/v1" - discoveryv1 "k8s.io/api/discovery/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/ingress-nginx/pkg/apis/ingress" -) - -//nolint:dupl // Ignore dupl errors for similar test case -func TestGetEndpointsFromSlices(t *testing.T) { - tests := []struct { - name string - svc *corev1.Service - port *corev1.ServicePort - proto corev1.Protocol - zone string - fn func(string) ([]*discoveryv1.EndpointSlice, error) - result []ingress.Endpoint - }{ - { - "no service should return 0 endpoint", - nil, - nil, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return nil, nil - }, - []ingress.Endpoint{}, - }, - { - "no service port should return 0 endpoint", - &corev1.Service{}, - nil, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return nil, nil - }, - []ingress.Endpoint{}, - }, - { - "a service without endpoint should return 0 endpoint", - &corev1.Service{}, - &corev1.ServicePort{Name: "default"}, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{}, nil - }, - []ingress.Endpoint{}, - }, - { - "a service type ServiceTypeExternalName service with an invalid port should return 0 endpoint", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeExternalName, - }, - }, - &corev1.ServicePort{Name: "default"}, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{}, nil - }, - []ingress.Endpoint{}, - }, - { - "a service type ServiceTypeExternalName service with localhost in name should return 0 endpoint", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeExternalName, - ExternalName: "localhost", - Ports: []corev1.ServicePort{ - { - Name: "default", - TargetPort: intstr.FromInt(443), - }, - }, - }, - }, - &corev1.ServicePort{ - Name: "default", - TargetPort: intstr.FromInt(80), - }, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{}, nil - }, - []ingress.Endpoint{}, - }, - { - "a service type ServiceTypeExternalName service with 127.0.0.1 in name should return 0 endpoint", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeExternalName, - ExternalName: "127.0.0.1", - Ports: []corev1.ServicePort{ - { - Name: "default", - TargetPort: intstr.FromInt(443), - }, - }, - }, - }, - &corev1.ServicePort{ - Name: "default", - TargetPort: intstr.FromInt(80), - }, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{}, nil - }, - []ingress.Endpoint{}, - }, - { - "a service type ServiceTypeExternalName with a valid port should return one endpoint", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeExternalName, - ExternalName: "www.google.com", - Ports: []corev1.ServicePort{ - { - Name: "default", - TargetPort: intstr.FromInt(443), - }, - }, - }, - }, - &corev1.ServicePort{ - Name: "default", - TargetPort: intstr.FromInt(80), - }, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{}, nil - }, - []ingress.Endpoint{ - { - Address: "www.google.com", - Port: "443", - }, - }, - }, - { - "a service type ServiceTypeExternalName with an trailing dot ExternalName value should return one endpoints", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeExternalName, - ExternalName: "www.google.com.", - Ports: []corev1.ServicePort{ - { - Name: "default", - TargetPort: intstr.FromInt(80), - }, - }, - }, - }, - &corev1.ServicePort{ - Name: "default", - TargetPort: intstr.FromInt(80), - }, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{}, nil - }, - []ingress.Endpoint{ - { - Address: "www.google.com", - Port: "443", - }, - }, - }, - { - "a service type ServiceTypeExternalName with an invalid ExternalName value should no return endpoints", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeExternalName, - ExternalName: "1#invalid.hostname", - Ports: []corev1.ServicePort{ - { - Name: "default", - TargetPort: intstr.FromInt(80), - }, - }, - }, - }, - &corev1.ServicePort{ - Name: "default", - TargetPort: intstr.FromInt(80), - }, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{}, nil - }, - []ingress.Endpoint{}, - }, - { - "should return no endpoint when there is an error searching for endpoints", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "1.1.1.1", - Ports: []corev1.ServicePort{ - { - Name: "default", - TargetPort: intstr.FromInt(80), - }, - }, - }, - }, - &corev1.ServicePort{ - Name: "default", - TargetPort: intstr.FromInt(80), - }, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return nil, fmt.Errorf("unexpected error") - }, - []ingress.Endpoint{}, - }, - { - "should return no endpoint when the protocol does not match", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "1.1.1.1", - Ports: []corev1.ServicePort{ - { - Name: "default", - TargetPort: intstr.FromInt(80), - }, - }, - }, - }, - &corev1.ServicePort{ - Name: "default", - TargetPort: intstr.FromInt(80), - }, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{discoveryv1.LabelServiceName: "default"}, - }, - Endpoints: []discoveryv1.Endpoint{ - { - Addresses: []string{"1.1.1.1"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - }, - }, - Ports: []discoveryv1.EndpointPort{ - { - Name: &[]string{""}[0], - Port: &[]int32{80}[0], - Protocol: &[]corev1.Protocol{corev1.ProtocolUDP}[0], - }, - }, - }}, nil - }, - []ingress.Endpoint{}, - }, - { - "should return no endpoint when there is no ready Addresses", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "1.1.1.1", - Ports: []corev1.ServicePort{ - { - Name: "default", - TargetPort: intstr.FromInt(80), - }, - }, - }, - }, - &corev1.ServicePort{ - Name: "default", - TargetPort: intstr.FromInt(80), - }, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{discoveryv1.LabelServiceName: "default"}, - }, - Endpoints: []discoveryv1.Endpoint{ - { - Addresses: []string{"1.1.1.1"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{false}[0], - }, - }, - }, - Ports: []discoveryv1.EndpointPort{ - { - Name: &[]string{""}[0], - Port: &[]int32{80}[0], - Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0], - }, - }, - }}, nil - }, - []ingress.Endpoint{}, - }, - { - "should return no endpoint when the name of the port name do not match any port in the endpointPort and TargetPort is string", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "1.1.1.1", - Ports: []corev1.ServicePort{ - { - Name: "default", - TargetPort: intstr.FromString("port-1"), - }, - }, - }, - }, - &corev1.ServicePort{ - Name: "default", - TargetPort: intstr.FromString("port-1"), - }, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{discoveryv1.LabelServiceName: "default"}, - }, - Endpoints: []discoveryv1.Endpoint{ - { - Addresses: []string{"1.1.1.1"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - }, - }, - Ports: []discoveryv1.EndpointPort{ - { - Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0], - Port: &[]int32{80}[0], - Name: &[]string{"another-name"}[0], - }, - }, - }}, nil - }, - []ingress.Endpoint{}, - }, - { - "should return one endpoint when the name of the port name do not match any port in the endpointPort and TargetPort is int", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "1.1.1.1", - Ports: []corev1.ServicePort{ - { - Name: "default", - TargetPort: intstr.FromInt(80), - }, - }, - }, - }, - &corev1.ServicePort{ - Name: "default", - TargetPort: intstr.FromInt(80), - }, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{discoveryv1.LabelServiceName: "default"}, - }, - Endpoints: []discoveryv1.Endpoint{ - { - Addresses: []string{"1.1.1.1"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - }, - }, - Ports: []discoveryv1.EndpointPort{ - { - Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0], - Port: &[]int32{80}[0], - Name: &[]string{"another-name"}[0], - }, - }, - }}, nil - }, - []ingress.Endpoint{ - { - Address: "1.1.1.1", - Port: "80", - }, - }, - }, - { - "should return one endpoint when the name of the port name match a port in the endpointPort", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "1.1.1.1", - Ports: []corev1.ServicePort{ - { - Name: "default", - TargetPort: intstr.FromInt(80), - }, - }, - }, - }, - &corev1.ServicePort{ - Name: "default", - TargetPort: intstr.FromInt(80), - }, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{discoveryv1.LabelServiceName: "default"}, - }, - Endpoints: []discoveryv1.Endpoint{ - { - Addresses: []string{"1.1.1.1"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - }, - }, - Ports: []discoveryv1.EndpointPort{ - { - Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0], - Port: &[]int32{80}[0], - Name: &[]string{"default"}[0], - }, - }, - }}, nil - }, - []ingress.Endpoint{ - { - Address: "1.1.1.1", - Port: "80", - }, - }, - }, - { - "should return two endpoints when the name of the port name match a port in the endpointPort", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "1.1.1.1", - Ports: []corev1.ServicePort{ - { - Name: "default", - TargetPort: intstr.FromInt(80), - }, - }, - }, - }, - &corev1.ServicePort{ - Name: "default", - TargetPort: intstr.FromString("port-1"), - }, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{ - { - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{discoveryv1.LabelServiceName: "default"}, - }, - Endpoints: []discoveryv1.Endpoint{ - { - Addresses: []string{"1.1.1.1"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - }, - }, - Ports: []discoveryv1.EndpointPort{ - { - Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0], - Port: &[]int32{80}[0], - Name: &[]string{"default"}[0], - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{discoveryv1.LabelServiceName: "default"}, - }, - Endpoints: []discoveryv1.Endpoint{ - { - Addresses: []string{"2.2.2.2"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - }, - }, - Ports: []discoveryv1.EndpointPort{ - { - Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0], - Port: &[]int32{80}[0], - Name: &[]string{"default"}[0], - }, - }, - }, - }, nil - }, - []ingress.Endpoint{ - { - Address: "1.1.1.1", - Port: "80", - }, - { - Address: "2.2.2.2", - Port: "80", - }, - }, - }, - { - "should return one endpoints when the name of the port name match a port in the endpointPort", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "1.1.1.1", - Ports: []corev1.ServicePort{ - { - Name: "default", - TargetPort: intstr.FromInt(80), - }, - }, - }, - }, - &corev1.ServicePort{ - Name: "port-1", - TargetPort: intstr.FromString("port-1"), - }, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{ - { - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{discoveryv1.LabelServiceName: "default"}, - }, - Endpoints: []discoveryv1.Endpoint{ - { - Addresses: []string{"1.1.1.1"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - }, - }, - Ports: []discoveryv1.EndpointPort{ - { - Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0], - Port: &[]int32{80}[0], - Name: &[]string{"port-1"}[0], - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{discoveryv1.LabelServiceName: "default"}, - }, - Endpoints: []discoveryv1.Endpoint{ - { - Addresses: []string{"2.2.2.2"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - }, - }, - Ports: []discoveryv1.EndpointPort{ - { - Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0], - Port: &[]int32{80}[0], - Name: &[]string{"another-name"}[0], - }, - }, - }, - }, nil - }, - []ingress.Endpoint{ - { - Address: "1.1.1.1", - Port: "80", - }, - }, - }, - { - "should return one endpoint when the name of the port name match more than one port in the endpointPort", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "1.1.1.1", - Ports: []corev1.ServicePort{ - { - Name: "default", - TargetPort: intstr.FromString("port-1"), - }, - }, - }, - }, - &corev1.ServicePort{ - Name: "port-1", - TargetPort: intstr.FromString("port-1"), - }, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{discoveryv1.LabelServiceName: "default"}, - }, - Endpoints: []discoveryv1.Endpoint{ - { - Addresses: []string{"1.1.1.1"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - }, - }, - Ports: []discoveryv1.EndpointPort{ - { - Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0], - Port: &[]int32{80}[0], - Name: &[]string{"port-1"}[0], - }, - { - Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0], - Port: &[]int32{80}[0], - Name: &[]string{"port-2"}[0], - }, - }, - }}, nil - }, - []ingress.Endpoint{ - { - Address: "1.1.1.1", - Port: "80", - }, - }, - }, - { - "should return one endpoint which belongs to zone", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "1.1.1.1", - Ports: []corev1.ServicePort{ - { - Name: "default", - TargetPort: intstr.FromString("port-1"), - }, - }, - }, - }, - &corev1.ServicePort{ - Name: "port-1", - TargetPort: intstr.FromString("port-1"), - }, - corev1.ProtocolTCP, - "eu-west-1b", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{discoveryv1.LabelServiceName: "default"}, - }, - Endpoints: []discoveryv1.Endpoint{ - { - Addresses: []string{"1.1.1.1"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - Hints: &[]discoveryv1.EndpointHints{{ - ForZones: []discoveryv1.ForZone{{ - Name: "eu-west-1b", - }}, - }}[0], - }, - { - Addresses: []string{"1.1.1.2"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - Hints: &[]discoveryv1.EndpointHints{{ - ForZones: []discoveryv1.ForZone{{ - Name: "eu-west-1a", - }}, - }}[0], - }, - { - Addresses: []string{"1.1.1.3"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - Hints: &[]discoveryv1.EndpointHints{{ - ForZones: []discoveryv1.ForZone{{ - Name: "eu-west-1c", - }}, - }}[0], - }, - }, - Ports: []discoveryv1.EndpointPort{ - { - Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0], - Port: &[]int32{80}[0], - Name: &[]string{"port-1"}[0], - }, - }, - }}, nil - }, - []ingress.Endpoint{ - { - Address: "1.1.1.1", - Port: "80", - }, - }, - }, - { - "should return all endpoints because one is missing zone hint", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "1.1.1.1", - Ports: []corev1.ServicePort{ - { - Name: "default", - TargetPort: intstr.FromString("port-1"), - }, - }, - }, - }, - &corev1.ServicePort{ - Name: "port-1", - TargetPort: intstr.FromString("port-1"), - }, - corev1.ProtocolTCP, - "eu-west-1b", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{discoveryv1.LabelServiceName: "default"}, - }, - Endpoints: []discoveryv1.Endpoint{ - { - Addresses: []string{"1.1.1.1"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - Hints: &[]discoveryv1.EndpointHints{{ - ForZones: []discoveryv1.ForZone{{ - Name: "eu-west-1b", - }}, - }}[0], - }, - { - Addresses: []string{"1.1.1.2"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - Hints: &[]discoveryv1.EndpointHints{{ - ForZones: []discoveryv1.ForZone{{ - Name: "eu-west-1b", - }}, - }}[0], - }, - { - Addresses: []string{"1.1.1.3"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - Hints: &[]discoveryv1.EndpointHints{{}}[0], - }, - }, - Ports: []discoveryv1.EndpointPort{ - { - Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0], - Port: &[]int32{80}[0], - Name: &[]string{"port-1"}[0], - }, - }, - }}, nil - }, - []ingress.Endpoint{ - { - Address: "1.1.1.1", - Port: "80", - }, - { - Address: "1.1.1.2", - Port: "80", - }, - { - Address: "1.1.1.3", - Port: "80", - }, - }, - }, - { - "should return all endpoints because no zone from controller node", - &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "1.1.1.1", - Ports: []corev1.ServicePort{ - { - Name: "default", - TargetPort: intstr.FromString("port-1"), - }, - }, - }, - }, - &corev1.ServicePort{ - Name: "port-1", - TargetPort: intstr.FromString("port-1"), - }, - corev1.ProtocolTCP, - "", - func(string) ([]*discoveryv1.EndpointSlice, error) { - return []*discoveryv1.EndpointSlice{{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{discoveryv1.LabelServiceName: "default"}, - }, - Endpoints: []discoveryv1.Endpoint{ - { - Addresses: []string{"1.1.1.1"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - Hints: &[]discoveryv1.EndpointHints{{ - ForZones: []discoveryv1.ForZone{{ - Name: "eu-west-1a", - }}, - }}[0], - }, - { - Addresses: []string{"1.1.1.2"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - Hints: &[]discoveryv1.EndpointHints{{ - ForZones: []discoveryv1.ForZone{{ - Name: "eu-west-1b", - }}, - }}[0], - }, - { - Addresses: []string{"1.1.1.3"}, - Conditions: discoveryv1.EndpointConditions{ - Ready: &[]bool{true}[0], - }, - Hints: &[]discoveryv1.EndpointHints{{ - ForZones: []discoveryv1.ForZone{{ - Name: "eu-west-1c", - }}, - }}[0], - }, - }, - Ports: []discoveryv1.EndpointPort{ - { - Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0], - Port: &[]int32{80}[0], - Name: &[]string{"port-1"}[0], - }, - }, - }}, nil - }, - []ingress.Endpoint{ - { - Address: "1.1.1.1", - Port: "80", - }, - { - Address: "1.1.1.2", - Port: "80", - }, - { - Address: "1.1.1.3", - Port: "80", - }, - }, - }, - } - - for _, testCase := range tests { - t.Run(testCase.name, func(t *testing.T) { - result := getEndpointsFromSlices(testCase.svc, testCase.port, testCase.proto, testCase.zone, testCase.fn) - if len(testCase.result) != len(result) { - t.Errorf("Expected %d Endpoints but got %d", len(testCase.result), len(result)) - } - }) - } -} diff --git a/internal/ingress/controller/ingressclass/ingressclass.go b/internal/ingress/controller/ingressclass/ingressclass.go deleted file mode 100644 index 90c6a4d67..000000000 --- a/internal/ingress/controller/ingressclass/ingressclass.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ingressclass - -const ( - // IngressKey picks a specific "class" for the Ingress. - // The controller only processes Ingresses with this annotation either - // unset, or set to either the configured value or the empty string. - IngressKey = "kubernetes.io/ingress.class" - - // DefaultControllerName defines the default controller name for Ingress NGINX - DefaultControllerName = "k8s.io/ingress-nginx" - - // DefaultAnnotationValue defines the default annotation value for the ingress-nginx controller - DefaultAnnotationValue = "nginx" -) - -// Configuration defines the various aspects of IngressClass parsing -// and how the controller should behave in each case -type Configuration struct { - // Controller defines the controller value this daemon watch to. - // Defaults to "k8s.io/ingress-nginx" defined in flags - Controller string - // AnnotationValue defines the annotation value this Controller watch to, in case of the - // ingressSpecName is not found but the annotation is. - // The Annotation is deprecated and should not be used in future releases - AnnotationValue string - // WatchWithoutClass defines if Controller should watch to Ingress Objects that does - // not contain an IngressClass configuration - WatchWithoutClass bool - // IgnoreIngressClass defines if Controller should ignore the IngressClass Object if no permissions are - // granted on IngressClass - IgnoreIngressClass bool - // IngressClassByName defines if the Controller should watch for Ingress Classes by - // .metadata.name together with .spec.Controller - IngressClassByName bool -} diff --git a/internal/ingress/controller/location.go b/internal/ingress/controller/location.go index 221c089c2..e6b6c44bd 100644 --- a/internal/ingress/controller/location.go +++ b/internal/ingress/controller/location.go @@ -20,8 +20,10 @@ import ( "fmt" "strings" - networking "k8s.io/api/networking/v1" - "k8s.io/ingress-nginx/pkg/apis/ingress" + "github.com/mitchellh/copystructure" + networking "k8s.io/api/networking/v1beta1" + "k8s.io/ingress-nginx/internal/ingress" + "k8s.io/klog/v2" ) var ( @@ -71,14 +73,18 @@ func updateServerLocations(locations []*ingress.Location) []*ingress.Location { continue } - var el ingress.Location = *location + // copy location before any change + el, err := copystructure.Copy(location) + if err != nil { + klog.ErrorS(err, "copying location") + } // normalize path. Must end in / location.Path = normalizePrefixPath(location.Path) newLocations = append(newLocations, location) // add exact location - exactLocation := &el + exactLocation := el.(*ingress.Location) exactLocation.PathType = &pathTypeExact newLocations = append(newLocations, exactLocation) @@ -100,7 +106,7 @@ func normalizePrefixPath(path string) string { } func needsRewrite(location *ingress.Location) bool { - if location.Rewrite.Target != "" && location.Rewrite.Target != location.Path { + if len(location.Rewrite.Target) > 0 && location.Rewrite.Target != location.Path { return true } diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index 20fad5afb..664d36b75 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -22,7 +22,7 @@ import ( "encoding/json" "errors" "fmt" - "io/fs" + "io/ioutil" "net" "net/http" "os" @@ -35,19 +35,22 @@ import ( "syscall" "text/template" "time" - "unicode" proxyproto "github.com/armon/go-proxyproto" "github.com/eapache/channels" apiv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes/scheme" v1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/flowcontrol" - "k8s.io/ingress-nginx/pkg/tcpproxy" + "k8s.io/klog/v2" adm_controller "k8s.io/ingress-nginx/internal/admission/controller" + "k8s.io/ingress-nginx/internal/file" + "k8s.io/ingress-nginx/internal/ingress" + "k8s.io/ingress-nginx/internal/ingress/annotations/class" ngx_config "k8s.io/ingress-nginx/internal/ingress/controller/config" "k8s.io/ingress-nginx/internal/ingress/controller/process" "k8s.io/ingress-nginx/internal/ingress/controller/store" @@ -59,12 +62,7 @@ import ( "k8s.io/ingress-nginx/internal/net/ssl" "k8s.io/ingress-nginx/internal/nginx" "k8s.io/ingress-nginx/internal/task" - "k8s.io/ingress-nginx/pkg/apis/ingress" - - "k8s.io/ingress-nginx/pkg/util/file" - utilingress "k8s.io/ingress-nginx/pkg/util/ingress" - - klog "k8s.io/klog/v2" + "k8s.io/ingress-nginx/internal/watch" ) const ( @@ -88,10 +86,9 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro n := &NGINXController{ isIPV6Enabled: ing_net.IsIPv6Enabled(), - resolver: h, - cfg: config, - syncRateLimiter: flowcontrol.NewTokenBucketRateLimiter(config.SyncRateLimit, 1), - workersReloading: false, + resolver: h, + cfg: config, + syncRateLimiter: flowcontrol.NewTokenBucketRateLimiter(config.SyncRateLimit, 1), recorder: eventBroadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{ Component: "nginx-ingress-controller", @@ -106,7 +103,7 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro runningConfig: new(ingress.Configuration), - Proxy: &tcpproxy.TCPProxy{}, + Proxy: &TCPProxy{}, metricCollector: mc, @@ -115,11 +112,9 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro if n.cfg.ValidationWebhook != "" { n.validationWebhookServer = &http.Server{ - Addr: config.ValidationWebhook, - // G112 (CWE-400): Potential Slowloris Attack - ReadHeaderTimeout: 10 * time.Second, - Handler: adm_controller.NewAdmissionControllerServer(&adm_controller.IngressAdmission{Checker: n}), - TLSConfig: ssl.NewTLSListener(n.cfg.ValidationWebhookCertPath, n.cfg.ValidationWebhookKeyPath).TLSConfig(), + Addr: config.ValidationWebhook, + Handler: adm_controller.NewAdmissionControllerServer(&adm_controller.IngressAdmission{Checker: n}), + TLSConfig: ssl.NewTLSListener(n.cfg.ValidationWebhookCertPath, n.cfg.ValidationWebhookKeyPath).TLSConfig(), // disable http/2 // https://github.com/kubernetes/kubernetes/issues/80313 // https://github.com/kubernetes/ingress-nginx/issues/6323#issuecomment-737239159 @@ -129,7 +124,6 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro n.store = store.New( config.Namespace, - config.WatchNamespaceSelector, config.ConfigMapName, config.TCPConfigMapName, config.UDPConfigMapName, @@ -137,10 +131,7 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro config.ResyncPeriod, config.Client, n.updateCh, - config.DisableCatchAll, - config.DeepInspector, - config.IngressClassConfiguration, - config.DisableSyncEvents) + config.DisableCatchAll) n.syncQueue = task.NewTaskQueue(n.syncIngress) @@ -177,17 +168,13 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro n.t = ngxTpl - _, err = file.NewFileWatcher(nginx.TemplatePath, onTemplateChange) + _, err = watch.NewFileWatcher(nginx.TemplatePath, onTemplateChange) if err != nil { klog.Fatalf("Error creating file watcher for %v: %v", nginx.TemplatePath, err) } filesToWatch := []string{} - - if err := os.Mkdir("/etc/ingress-controller/geoip/", 0o755); err != nil && !os.IsExist(err) { - klog.Fatalf("Error creating geoip dir: %v", err) - } - err = filepath.WalkDir("/etc/ingress-controller/geoip/", func(path string, info fs.DirEntry, err error) error { + err = filepath.Walk("/etc/nginx/geoip/", func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -199,16 +186,14 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro filesToWatch = append(filesToWatch, path) return nil }) + if err != nil { klog.Fatalf("Error creating file watchers: %v", err) } for _, f := range filesToWatch { - // This redeclaration is necessary for the closure to get the correct value for the iteration in go versions <1.22 - // See https://go.dev/blog/loopvar-preview - f := f - _, err = file.NewFileWatcher(f, func() { - klog.InfoS("File change detected. Reloading NGINX", "path", f) + _, err = watch.NewFileWatcher(f, func() { + klog.InfoS("File changed detected. Reloading NGINX", "path", f) n.syncQueue.EnqueueTask(task.GetDummyObject("file-change")) }) if err != nil { @@ -231,8 +216,6 @@ type NGINXController struct { syncRateLimiter flowcontrol.RateLimiter - workersReloading bool - // stopLock is used to enforce that only a single call to Stop send at // a given time. We allow stopping through an HTTP endpoint and // allowing concurrent stoppers leads to stack traces. @@ -247,7 +230,7 @@ type NGINXController struct { // runningConfig contains the running configuration in the Backend runningConfig *ingress.Configuration - t ngx_template.Writer + t ngx_template.TemplateWriter resolver []net.IP @@ -255,7 +238,7 @@ type NGINXController struct { isShuttingDown bool - Proxy *tcpproxy.TCPProxy + Proxy *TCPProxy store store.Storer @@ -274,33 +257,29 @@ func (n *NGINXController) Start() { // we need to use the defined ingress class to allow multiple leaders // in order to update information about ingress status - // TODO: For now, as the the IngressClass logics has changed, is up to the - // cluster admin to create different Leader Election IDs. - // Should revisit this in a future - - if !n.cfg.DisableLeaderElection { - electionID := n.cfg.ElectionID - setupLeaderElection(&leaderElectionConfig{ - Client: n.cfg.Client, - ElectionID: electionID, - ElectionTTL: n.cfg.ElectionTTL, - OnStartedLeading: func(stopCh chan struct{}) { - if n.syncStatus != nil { - go n.syncStatus.Run(stopCh) - } - - n.metricCollector.OnStartedLeading(electionID) - // manually update SSL expiration metrics - // (to not wait for a reload) - n.metricCollector.SetSSLExpireTime(n.runningConfig.Servers) - n.metricCollector.SetSSLInfo(n.runningConfig.Servers) - }, - OnStoppedLeading: func() { - n.metricCollector.OnStoppedLeading(electionID) - }, - }) + electionID := fmt.Sprintf("%v-%v", n.cfg.ElectionID, class.DefaultClass) + if class.IngressClass != "" { + electionID = fmt.Sprintf("%v-%v", n.cfg.ElectionID, class.IngressClass) } + setupLeaderElection(&leaderElectionConfig{ + Client: n.cfg.Client, + ElectionID: electionID, + OnStartedLeading: func(stopCh chan struct{}) { + if n.syncStatus != nil { + go n.syncStatus.Run(stopCh) + } + + n.metricCollector.OnStartedLeading(electionID) + // manually update SSL expiration metrics + // (to not wait for a reload) + n.metricCollector.SetSSLExpireTime(n.runningConfig.Servers) + }, + OnStoppedLeading: func() { + n.metricCollector.OnStoppedLeading(electionID) + }, + }) + cmd := n.command.ExecCommand() // put NGINX in another process group to prevent it @@ -443,7 +422,7 @@ func (n *NGINXController) start(cmd *exec.Cmd) { } // DefaultEndpoint returns the default endpoint to be use as default server that returns 404. -func (n *NGINXController) DefaultEndpoint() ingress.Endpoint { +func (n NGINXController) DefaultEndpoint() ingress.Endpoint { return ingress.Endpoint{ Address: "127.0.0.1", Port: fmt.Sprintf("%v", n.cfg.ListenPorts.Default), @@ -452,11 +431,10 @@ func (n *NGINXController) DefaultEndpoint() ingress.Endpoint { } // generateTemplate returns the nginx configuration file content -// -//nolint:gocritic // the cfg shouldn't be changed, and shouldn't be mutated by other processes while being rendered. -func (n *NGINXController) generateTemplate(cfg ngx_config.Configuration, ingressCfg ingress.Configuration) ([]byte, error) { +func (n NGINXController) generateTemplate(cfg ngx_config.Configuration, ingressCfg ingress.Configuration) ([]byte, error) { + if n.cfg.EnableSSLPassthrough { - servers := []*tcpproxy.TCPServer{} + servers := []*TCPServer{} for _, pb := range ingressCfg.PassthroughBackends { svc := pb.Service if svc == nil { @@ -473,7 +451,6 @@ func (n *NGINXController) generateTemplate(cfg ngx_config.Configuration, ingress } } else { for _, sp := range svc.Spec.Ports { - //nolint:gosec // Ignore G109 error if sp.Port == int32(port) { port = int(sp.Port) break @@ -482,7 +459,7 @@ func (n *NGINXController) generateTemplate(cfg ngx_config.Configuration, ingress } // TODO: Allow PassthroughBackends to specify they support proxy-protocol - servers = append(servers, &tcpproxy.TCPServer{ + servers = append(servers, &TCPServer{ Hostname: pb.Hostname, IP: svc.Spec.ClusterIP, Port: port, @@ -535,7 +512,12 @@ func (n *NGINXController) generateTemplate(cfg ngx_config.Configuration, ingress if cfg.MaxWorkerOpenFiles == 0 { // the limit of open files is per worker process // and we leave some room to avoid consuming all the FDs available - maxOpenFiles := rlimitMaxNumFiles() - 1024 + wp, err := strconv.Atoi(cfg.WorkerProcesses) + klog.V(3).InfoS("Worker processes", "count", wp) + if err != nil { + wp = 1 + } + maxOpenFiles := (rlimitMaxNumFiles() / wp) - 1024 klog.V(3).InfoS("Maximum number of open file descriptors", "value", maxOpenFiles) if maxOpenFiles < 1024 { // this means the value of RLIMIT_NOFILE is too low. @@ -579,7 +561,7 @@ func (n *NGINXController) generateTemplate(cfg ngx_config.Configuration, ingress if err != nil { klog.Warningf("Error reading Secret %q from local store: %v", secretName, err) } else { - nsSecName := strings.ReplaceAll(secretName, "/", "-") + nsSecName := strings.Replace(secretName, "/", "-", -1) dh, ok := secret.Data["dhparam.pem"] if ok { pemFileName, err := ssl.AddOrUpdateDHParam(nsSecName, dh) @@ -596,16 +578,7 @@ func (n *NGINXController) generateTemplate(cfg ngx_config.Configuration, ingress cfg.DefaultSSLCertificate = n.getDefaultSSLCertificate() - if n.cfg.IsChroot { - if cfg.AccessLogPath == "/var/log/nginx/access.log" { - cfg.AccessLogPath = fmt.Sprintf("syslog:server=%s", n.cfg.InternalLoggerAddress) - } - if cfg.ErrorLogPath == "/var/log/nginx/error.log" { - cfg.ErrorLogPath = fmt.Sprintf("syslog:server=%s", n.cfg.InternalLoggerAddress) - } - } - - tc := &ngx_config.TemplateConfig{ + tc := ngx_config.TemplateConfig{ ProxySetHeaders: setHeaders, AddHeaders: addHeaders, BacklogSize: sysctlSomaxconn(), @@ -618,9 +591,10 @@ func (n *NGINXController) generateTemplate(cfg ngx_config.Configuration, ingress IsIPV6Enabled: n.isIPV6Enabled && !cfg.DisableIpv6, NginxStatusIpv4Whitelist: cfg.NginxStatusIpv4Whitelist, NginxStatusIpv6Whitelist: cfg.NginxStatusIpv6Whitelist, - RedirectServers: utilingress.BuildRedirects(ingressCfg.Servers), + RedirectServers: buildRedirects(ingressCfg.Servers), IsSSLPassthroughEnabled: n.cfg.EnableSSLPassthrough, ListenPorts: n.cfg.ListenPorts, + PublishService: n.GetPublishService(), EnableMetrics: n.cfg.EnableMetrics, MaxmindEditionFiles: n.cfg.MaxmindEditionFiles, HealthzURI: nginx.HealthPath, @@ -629,7 +603,6 @@ func (n *NGINXController) generateTemplate(cfg ngx_config.Configuration, ingress StatusPath: nginx.StatusPath, StatusPort: nginx.StatusPort, StreamPort: nginx.StreamPort, - StreamSnippets: append(ingressCfg.StreamSnippets, cfg.StreamSnippet), } tc.Cfg.Checksum = ingressCfg.ConfigurationChecksum @@ -639,16 +612,16 @@ func (n *NGINXController) generateTemplate(cfg ngx_config.Configuration, ingress // testTemplate checks if the NGINX configuration inside the byte array is valid // running the command "nginx -t" using a temporal file. -func (n *NGINXController) testTemplate(cfg []byte) error { +func (n NGINXController) testTemplate(cfg []byte) error { if len(cfg) == 0 { return fmt.Errorf("invalid NGINX configuration (empty)") } - tmpfile, err := os.CreateTemp(filepath.Join(os.TempDir(), "nginx"), tempNginxPattern) + tmpfile, err := ioutil.TempFile("", tempNginxPattern) if err != nil { return err } defer tmpfile.Close() - err = os.WriteFile(tmpfile.Name(), cfg, file.ReadWriteByUser) + err = ioutil.WriteFile(tmpfile.Name(), cfg, file.ReadWriteByUser) if err != nil { return err } @@ -673,27 +646,16 @@ Error: %v // changes were detected. The received backend Configuration is merged with the // configuration ConfigMap before generating the final configuration file. // Returns nil in case the backend was successfully reloaded. -// -//nolint:gocritic // the cfg shouldn't be changed, and shouldn't be mutated by other processes while being rendered. func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error { cfg := n.store.GetBackendConfiguration() cfg.Resolver = n.resolver - workerSerialReloads := cfg.WorkerSerialReloads - if workerSerialReloads && n.workersReloading { - return errors.New("worker reload already in progress, requeuing reload") - } - content, err := n.generateTemplate(cfg, ingressCfg) if err != nil { return err } - err = n.createLuaConfig(&cfg) - if err != nil { - return err - } - err = createOpentelemetryCfg(&cfg) + err = createOpentracingCfg(cfg) if err != nil { return err } @@ -704,28 +666,22 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error { } if klog.V(2).Enabled() { - src, err := os.ReadFile(cfgPath) - if err != nil { - return err - } + src, _ := ioutil.ReadFile(cfgPath) if !bytes.Equal(src, content) { - tmpfile, err := os.CreateTemp("", "new-nginx-cfg") + tmpfile, err := ioutil.TempFile("", "new-nginx-cfg") if err != nil { return err } defer tmpfile.Close() - err = os.WriteFile(tmpfile.Name(), content, file.ReadWriteByUser) + err = ioutil.WriteFile(tmpfile.Name(), content, file.ReadWriteByUser) if err != nil { return err } - //nolint:gosec //Ignore G204 error + diffOutput, err := exec.Command("diff", "-I", "'# Configuration.*'", "-u", cfgPath, tmpfile.Name()).CombinedOutput() if err != nil { if exitError, ok := err.(*exec.ExitError); ok { - ws, ok := exitError.Sys().(syscall.WaitStatus) - if !ok { - klog.Errorf("unexpected type: %T", exitError.Sys()) - } + ws := exitError.Sys().(syscall.WaitStatus) if ws.ExitStatus() == 2 { klog.Warningf("Failed to executing diff command: %v", err) } @@ -740,7 +696,7 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error { } } - err = os.WriteFile(cfgPath, content, file.ReadWriteByUser) + err = ioutil.WriteFile(cfgPath, content, file.ReadWriteByUser) if err != nil { return err } @@ -750,41 +706,9 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error { return fmt.Errorf("%v\n%v", err, string(o)) } - // Reload status checking runs in a separate goroutine to avoid blocking the sync queue - if workerSerialReloads { - go n.awaitWorkersReload() - } - return nil } -// awaitWorkersReload checks if the number of workers has returned to the expected count -func (n *NGINXController) awaitWorkersReload() { - n.workersReloading = true - defer func() { n.workersReloading = false }() - - expectedWorkers := n.store.GetBackendConfiguration().WorkerProcesses - var numWorkers string - klog.V(3).Infof("waiting for worker count to be equal to %s", expectedWorkers) - for numWorkers != expectedWorkers { - time.Sleep(time.Second) - o, err := exec.Command("/bin/sh", "-c", "pgrep worker | wc -l").Output() - if err != nil { - klog.ErrorS(err, numWorkers) - return - } - // cleanup any non-printable chars from shell output - numWorkers = strings.Map(func(r rune) rune { - if unicode.IsPrint(r) { - return r - } - return -1 - }, string(o)) - - klog.V(3).Infof("Currently running nginx worker processes: %s, expected %s", numWorkers, expectedWorkers) - } -} - // nginxHashBucketSize computes the correct NGINX hash_bucket_size for a hash // with the given longest key. func nginxHashBucketSize(longestString int) int { @@ -816,8 +740,8 @@ func (n *NGINXController) setupSSLProxy() { proxyPort := n.cfg.ListenPorts.SSLProxy klog.InfoS("Starting TLS proxy for SSL Passthrough") - n.Proxy = &tcpproxy.TCPProxy{ - Default: &tcpproxy.TCPServer{ + n.Proxy = &TCPProxy{ + Default: &TCPServer{ Hostname: "localhost", IP: "127.0.0.1", Port: proxyPort, @@ -857,6 +781,63 @@ func (n *NGINXController) setupSSLProxy() { }() } +// Helper function to clear Certificates from the ingress configuration since they should be ignored when +// checking if the new configuration changes can be applied dynamically if dynamic certificates is on +func clearCertificates(config *ingress.Configuration) { + var clearedServers []*ingress.Server + for _, server := range config.Servers { + copyOfServer := *server + copyOfServer.SSLCert = nil + clearedServers = append(clearedServers, ©OfServer) + } + config.Servers = clearedServers +} + +// Helper function to clear endpoints from the ingress configuration since they should be ignored when +// checking if the new configuration changes can be applied dynamically. +func clearL4serviceEndpoints(config *ingress.Configuration) { + var clearedTCPL4Services []ingress.L4Service + var clearedUDPL4Services []ingress.L4Service + for _, service := range config.TCPEndpoints { + copyofService := ingress.L4Service{ + Port: service.Port, + Backend: service.Backend, + Endpoints: []ingress.Endpoint{}, + Service: nil, + } + clearedTCPL4Services = append(clearedTCPL4Services, copyofService) + } + for _, service := range config.UDPEndpoints { + copyofService := ingress.L4Service{ + Port: service.Port, + Backend: service.Backend, + Endpoints: []ingress.Endpoint{}, + Service: nil, + } + clearedUDPL4Services = append(clearedUDPL4Services, copyofService) + } + config.TCPEndpoints = clearedTCPL4Services + config.UDPEndpoints = clearedUDPL4Services +} + +// IsDynamicConfigurationEnough returns whether a Configuration can be +// dynamically applied, without reloading the backend. +func (n *NGINXController) IsDynamicConfigurationEnough(pcfg *ingress.Configuration) bool { + copyOfRunningConfig := *n.runningConfig + copyOfPcfg := *pcfg + + copyOfRunningConfig.Backends = []*ingress.Backend{} + copyOfPcfg.Backends = []*ingress.Backend{} + + clearL4serviceEndpoints(©OfRunningConfig) + clearL4serviceEndpoints(©OfPcfg) + + clearCertificates(©OfRunningConfig) + clearCertificates(©OfPcfg) + + return copyOfRunningConfig.Equal(©OfPcfg) +} + // configureDynamically encodes new Backends in JSON format and POSTs the // payload to an internal HTTP endpoint handled by Lua. func (n *NGINXController) configureDynamically(pcfg *ingress.Configuration) error { @@ -887,10 +868,9 @@ func (n *NGINXController) configureDynamically(pcfg *ingress.Configuration) erro return nil } -func updateStreamConfiguration(tcpEndpoints, udpEndpoints []ingress.L4Service) error { +func updateStreamConfiguration(TCPEndpoints []ingress.L4Service, UDPEndpoints []ingress.L4Service) error { streams := make([]ingress.Backend, 0) - for i := range tcpEndpoints { - ep := &tcpEndpoints[i] + for _, ep := range TCPEndpoints { var service *apiv1.Service if ep.Service != nil { service = &apiv1.Service{Spec: ep.Service.Spec} @@ -904,8 +884,7 @@ func updateStreamConfiguration(tcpEndpoints, udpEndpoints []ingress.L4Service) e Service: service, }) } - for i := range udpEndpoints { - ep := &udpEndpoints[i] + for _, ep := range UDPEndpoints { var service *apiv1.Service if ep.Service != nil { service = &apiv1.Service{Spec: ep.Service.Spec} @@ -1028,7 +1007,7 @@ func configureCertificates(rawServers []*ingress.Server) error { } } - redirects := utilingress.BuildRedirects(rawServers) + redirects := buildRedirects(rawServers) for _, redirect := range redirects { configure(redirect.From, redirect.SSLCert) } @@ -1045,84 +1024,90 @@ func configureCertificates(rawServers []*ingress.Server) error { return nil } -const otelTmpl = ` -exporter = "otlp" -processor = "batch" +const zipkinTmpl = `{ + "service_name": "{{ .ZipkinServiceName }}", + "collector_host": "{{ .ZipkinCollectorHost }}", + "collector_port": {{ .ZipkinCollectorPort }}, + "sample_rate": {{ .ZipkinSampleRate }} +}` -[exporters.otlp] -# Alternatively the OTEL_EXPORTER_OTLP_ENDPOINT environment variable can also be used. -host = "{{ .OtlpCollectorHost }}" -port = {{ .OtlpCollectorPort }} +const jaegerTmpl = `{ + "service_name": "{{ .JaegerServiceName }}", + "propagation_format": "{{ .JaegerPropagationFormat }}", + "sampler": { + "type": "{{ .JaegerSamplerType }}", + "param": {{ .JaegerSamplerParam }}, + "samplingServerURL": "{{ .JaegerSamplerHost }}:{{ .JaegerSamplerPort }}/sampling" + }, + "reporter": { + "endpoint": "{{ .JaegerEndpoint }}", + "localAgentHostPort": "{{ .JaegerCollectorHost }}:{{ .JaegerCollectorPort }}" + }, + "headers": { + "TraceContextHeaderName": "{{ .JaegerTraceContextHeaderName }}", + "jaegerDebugHeader": "{{ .JaegerDebugHeader }}", + "jaegerBaggageHeader": "{{ .JaegerBaggageHeader }}", + "traceBaggageHeaderPrefix": "{{ .JaegerTraceBaggageHeaderPrefix }}" + } +}` -[processors.batch] -max_queue_size = {{ .OtelMaxQueueSize }} -schedule_delay_millis = {{ .OtelScheduleDelayMillis }} -max_export_batch_size = {{ .OtelMaxExportBatchSize }} +const datadogTmpl = `{ + "service": "{{ .DatadogServiceName }}", + "agent_host": "{{ .DatadogCollectorHost }}", + "agent_port": {{ .DatadogCollectorPort }}, + "environment": "{{ .DatadogEnvironment }}", + "operation_name_override": "{{ .DatadogOperationNameOverride }}", + "sample_rate": {{ .DatadogSampleRate }}, + "dd.priority.sampling": {{ .DatadogPrioritySampling }} +}` -[service] -name = "{{ .OtelServiceName }}" # Opentelemetry resource name +func createOpentracingCfg(cfg ngx_config.Configuration) error { + var tmpl *template.Template + var err error -[sampler] -name = "{{ .OtelSampler }}" # Also: AlwaysOff, TraceIdRatioBased -ratio = {{ .OtelSamplerRatio }} -parent_based = {{ .OtelSamplerParentBased }} -` - -func createOpentelemetryCfg(cfg *ngx_config.Configuration) error { - tmpl, err := template.New("otel").Parse(otelTmpl) - if err != nil { - return err + if cfg.ZipkinCollectorHost != "" { + tmpl, err = template.New("zipkin").Parse(zipkinTmpl) + if err != nil { + return err + } + } else if cfg.JaegerCollectorHost != "" || cfg.JaegerEndpoint != "" { + tmpl, err = template.New("jaeger").Parse(jaegerTmpl) + if err != nil { + return err + } + } else if cfg.DatadogCollectorHost != "" { + tmpl, err = template.New("datadog").Parse(datadogTmpl) + if err != nil { + return err + } + } else { + tmpl, _ = template.New("empty").Parse("{}") } + tmplBuf := bytes.NewBuffer(make([]byte, 0)) err = tmpl.Execute(tmplBuf, cfg) if err != nil { return err } - return os.WriteFile(cfg.OpentelemetryConfig, tmplBuf.Bytes(), file.ReadWriteByUser) -} + // Expand possible environment variables before writing the configuration to file. + expanded := os.ExpandEnv(tmplBuf.String()) -func (n *NGINXController) createLuaConfig(cfg *ngx_config.Configuration) error { - luaconfigs := &ngx_template.LuaConfig{ - EnableMetrics: n.cfg.EnableMetrics, - ListenPorts: ngx_template.LuaListenPorts{ - HTTPSPort: strconv.Itoa(n.cfg.ListenPorts.HTTPS), - StatusPort: strconv.Itoa(nginx.StatusPort), - SSLProxyPort: strconv.Itoa(n.cfg.ListenPorts.SSLProxy), - }, - UseProxyProtocol: cfg.UseProxyProtocol, - UseForwardedHeaders: cfg.UseForwardedHeaders, - IsSSLPassthroughEnabled: n.cfg.EnableSSLPassthrough, - HTTPRedirectCode: cfg.HTTPRedirectCode, - EnableOCSP: cfg.EnableOCSP, - MonitorBatchMaxSize: n.cfg.MonitorMaxBatchSize, - HSTS: cfg.HSTS, - HSTSMaxAge: cfg.HSTSMaxAge, - HSTSIncludeSubdomains: cfg.HSTSIncludeSubdomains, - HSTSPreload: cfg.HSTSPreload, - } - jsonCfg, err := json.Marshal(luaconfigs) - if err != nil { - return err - } - return os.WriteFile(luaCfgPath, jsonCfg, file.ReadWriteByUser) + return ioutil.WriteFile("/etc/nginx/opentracing.json", []byte(expanded), file.ReadWriteByUser) } func cleanTempNginxCfg() error { var files []string - err := filepath.Walk(filepath.Join(os.TempDir(), "nginx"), func(path string, info os.FileInfo, err error) error { + err := filepath.Walk(os.TempDir(), func(path string, info os.FileInfo, err error) error { if err != nil { return err } - if info.IsDir() && path != filepath.Join(os.TempDir(), "nginx") { + if info.IsDir() && os.TempDir() != path { return filepath.SkipDir } - dur, err := time.ParseDuration("-5m") - if err != nil { - return err - } + dur, _ := time.ParseDuration("-5m") fiveMinutesAgo := time.Now().Add(dur) if strings.HasPrefix(info.Name(), tempNginxPattern) && info.ModTime().Before(fiveMinutesAgo) { files = append(files, path) @@ -1134,7 +1119,7 @@ func cleanTempNginxCfg() error { } for _, file := range files { - err = os.Remove(file) + err := os.Remove(file) if err != nil { return err } @@ -1142,3 +1127,65 @@ func cleanTempNginxCfg() error { return nil } + +type redirect struct { + From string + To string + SSLCert *ingress.SSLCert +} + +func buildRedirects(servers []*ingress.Server) []*redirect { + names := sets.String{} + redirectServers := make([]*redirect, 0) + + for _, srv := range servers { + if !srv.RedirectFromToWWW { + continue + } + + to := srv.Hostname + + var from string + if strings.HasPrefix(to, "www.") { + from = strings.TrimPrefix(to, "www.") + } else { + from = fmt.Sprintf("www.%v", to) + } + + if names.Has(to) { + continue + } + + klog.V(3).InfoS("Creating redirect", "from", from, "to", to) + found := false + for _, esrv := range servers { + if esrv.Hostname == from { + found = true + break + } + } + + if found { + klog.Warningf("Already exists an Ingress with %q hostname. Skipping creation of redirection from %q to %q.", from, from, to) + continue + } + + r := &redirect{ + From: from, + To: to, + } + + if srv.SSLCert != nil { + if ssl.IsValidHostname(from, srv.SSLCert.CN) { + r.SSLCert = srv.SSLCert + } else { + klog.Warningf("the server %v has SSL configured but the SSL certificate does not contains a CN for %v. Redirects will not work for HTTPS to HTTPS", from, to) + } + } + + redirectServers = append(redirectServers, r) + names.Insert(to) + } + + return redirectServers +} diff --git a/internal/ingress/controller/nginx_test.go b/internal/ingress/controller/nginx_test.go index fc0a5b6d2..fb8632029 100644 --- a/internal/ingress/controller/nginx_test.go +++ b/internal/ingress/controller/nginx_test.go @@ -19,6 +19,7 @@ package controller import ( "fmt" "io" + "io/ioutil" "net" "net/http" "net/http/httptest" @@ -32,10 +33,122 @@ import ( apiv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/ingress-nginx/internal/ingress" "k8s.io/ingress-nginx/internal/nginx" - "k8s.io/ingress-nginx/pkg/apis/ingress" ) +func TestIsDynamicConfigurationEnough(t *testing.T) { + backends := []*ingress.Backend{{ + Name: "fakenamespace-myapp-80", + Endpoints: []ingress.Endpoint{ + { + Address: "10.0.0.1", + Port: "8080", + }, + { + Address: "10.0.0.2", + Port: "8080", + }, + }, + }} + + servers := []*ingress.Server{{ + Hostname: "myapp.fake", + Locations: []*ingress.Location{ + { + Path: "/", + Backend: "fakenamespace-myapp-80", + }, + }, + SSLCert: &ingress.SSLCert{ + PemCertKey: "fake-certificate", + }, + }} + + commonConfig := &ingress.Configuration{ + Backends: backends, + Servers: servers, + } + + n := &NGINXController{ + runningConfig: &ingress.Configuration{ + Backends: backends, + Servers: servers, + }, + cfg: &Configuration{}, + } + + newConfig := commonConfig + if !n.IsDynamicConfigurationEnough(newConfig) { + t.Errorf("When new config is same as the running config it should be deemed as dynamically configurable") + } + + newConfig = &ingress.Configuration{ + Backends: []*ingress.Backend{{Name: "another-backend-8081"}}, + Servers: []*ingress.Server{{Hostname: "myapp1.fake"}}, + } + if n.IsDynamicConfigurationEnough(newConfig) { + t.Errorf("Expected to not be dynamically configurable when there's more than just backends change") + } + + newConfig = &ingress.Configuration{ + Backends: []*ingress.Backend{{Name: "a-backend-8080"}}, + Servers: servers, + } + + if !n.IsDynamicConfigurationEnough(newConfig) { + t.Errorf("Expected to be dynamically configurable when only backends change") + } + + newServers := []*ingress.Server{{ + Hostname: "myapp1.fake", + Locations: []*ingress.Location{ + { + Path: "/", + Backend: "fakenamespace-myapp-80", + }, + }, + SSLCert: &ingress.SSLCert{ + PemCertKey: "fake-certificate", + }, + }} + + newConfig = &ingress.Configuration{ + Backends: backends, + Servers: newServers, + } + if n.IsDynamicConfigurationEnough(newConfig) { + t.Errorf("Expected to not be dynamically configurable when dynamic certificates is enabled and a non-certificate field in servers is updated") + } + + newServers[0].Hostname = "myapp.fake" + newServers[0].SSLCert.PemCertKey = "new-fake-certificate" + + newConfig = &ingress.Configuration{ + Backends: backends, + Servers: newServers, + } + if !n.IsDynamicConfigurationEnough(newConfig) { + t.Errorf("Expected to be dynamically configurable when only SSLCert changes") + } + + newConfig = &ingress.Configuration{ + Backends: []*ingress.Backend{{Name: "a-backend-8080"}}, + Servers: newServers, + } + if !n.IsDynamicConfigurationEnough(newConfig) { + t.Errorf("Expected to be dynamically configurable when backend and SSLCert changes") + } + + if !n.runningConfig.Equal(commonConfig) { + t.Errorf("Expected running config to not change") + } + + if !newConfig.Equal(&ingress.Configuration{Backends: []*ingress.Backend{{Name: "a-backend-8080"}}, Servers: newServers}) { + t.Errorf("Expected new config to not change") + } +} + func TestConfigureDynamically(t *testing.T) { listener, err := tryListen("tcp", fmt.Sprintf(":%v", nginx.StatusPort)) if err != nil { @@ -58,16 +171,15 @@ func TestConfigureDynamically(t *testing.T) { server := &httptest.Server{ Listener: listener, - //nolint:gosec // Ignore not configured ReadHeaderTimeout in testing Config: &http.Server{ Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusCreated) - if r.Method != http.MethodPost { + if r.Method != "POST" { t.Errorf("expected a 'POST' request, got '%s'", r.Method) } - b, err := io.ReadAll(r.Body) + b, err := ioutil.ReadAll(r.Body) if err != nil && err != io.EOF { t.Fatal(err) } @@ -77,17 +189,23 @@ func TestConfigureDynamically(t *testing.T) { switch r.URL.Path { case "/configuration/backends": - if strings.Contains(body, "target") { - t.Errorf("unexpected target reference in JSON content: %v", body) - } + { + if strings.Contains(body, "target") { + t.Errorf("unexpected target reference in JSON content: %v", body) + } - if !strings.Contains(body, "service") { - t.Errorf("service reference should be present in JSON content: %v", body) + if !strings.Contains(body, "service") { + t.Errorf("service reference should be present in JSON content: %v", body) + } } case "/configuration/general": + { + } case "/configuration/servers": - if !strings.Contains(body, `{"certificates":{},"servers":{"myapp.fake":"-1"}}`) { - t.Errorf("should be present in JSON content: %v", body) + { + if !strings.Contains(body, `{"certificates":{},"servers":{"myapp.fake":"-1"}}`) { + t.Errorf("should be present in JSON content: %v", body) + } } default: t.Errorf("unknown request to %s", r.URL.Path) @@ -213,16 +331,15 @@ func TestConfigureCertificates(t *testing.T) { server := &httptest.Server{ Listener: listener, - //nolint:gosec // Ignore not configured ReadHeaderTimeout in testing Config: &http.Server{ Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusCreated) - if r.Method != http.MethodPost { + if r.Method != "POST" { t.Errorf("expected a 'POST' request, got '%s'", r.Method) } - b, err := io.ReadAll(r.Body) + b, err := ioutil.ReadAll(r.Body) if err != nil && err != io.EOF { t.Fatal(err) } @@ -361,11 +478,10 @@ func TestCleanTempNginxCfg(t *testing.T) { t.Fatal(err) } - tmpfile, err := os.CreateTemp(filepath.Join(os.TempDir(), "nginx"), tempNginxPattern) + tmpfile, err := ioutil.TempFile("", tempNginxPattern) if err != nil { t.Fatal(err) } - expectedDeletedFile := tmpfile.Name() defer tmpfile.Close() dur, err := time.ParseDuration("-10m") @@ -379,11 +495,10 @@ func TestCleanTempNginxCfg(t *testing.T) { t.Fatal(err) } - tmpfile, err = os.CreateTemp(filepath.Join(os.TempDir(), "nginx"), tempNginxPattern) + tmpfile, err = ioutil.TempFile("", tempNginxPattern) if err != nil { t.Fatal(err) } - expectedFile := tmpfile.Name() defer tmpfile.Close() err = cleanTempNginxCfg() @@ -393,8 +508,8 @@ func TestCleanTempNginxCfg(t *testing.T) { var files []string - err = filepath.Walk(filepath.Join(os.TempDir(), "nginx"), func(path string, info os.FileInfo, _ error) error { - if info.IsDir() && filepath.Join(os.TempDir(), "nginx") != path { + err = filepath.Walk(os.TempDir(), func(path string, info os.FileInfo, err error) error { + if info.IsDir() && os.TempDir() != path { return filepath.SkipDir } @@ -407,22 +522,11 @@ func TestCleanTempNginxCfg(t *testing.T) { t.Fatal(err) } - // some other files can be created by other tests - var found bool - for _, file := range files { - if file == expectedDeletedFile { - t.Errorf("file %s should be deleted", file) - } - if file == expectedFile { - found = true - } - } - if !found { - t.Errorf("file %s should not be deleted", expectedFile) + if len(files) != 1 { + t.Errorf("expected one file but %d were found", len(files)) } } -//nolint:unparam // Ignore `network` always receives `"tcp"` error func tryListen(network, address string) (l net.Listener, err error) { condFunc := func() (bool, error) { l, err = net.Listen(network, address) diff --git a/internal/ingress/controller/process/nginx.go b/internal/ingress/controller/process/nginx.go index fdf501616..702f60da8 100644 --- a/internal/ingress/controller/process/nginx.go +++ b/internal/ingress/controller/process/nginx.go @@ -20,7 +20,7 @@ import ( "os/exec" "syscall" - klog "k8s.io/klog/v2" + "k8s.io/klog/v2" ) // IsRespawnIfRequired checks if error type is exec.ExitError or not @@ -30,10 +30,7 @@ func IsRespawnIfRequired(err error) bool { return false } - waitStatus, ok := exitError.Sys().(syscall.WaitStatus) - if !ok { - return false - } + waitStatus := exitError.Sys().(syscall.WaitStatus) klog.Warningf(` ------------------------------------------------------------------------------- NGINX master process died (%v): %v diff --git a/internal/ingress/controller/status.go b/internal/ingress/controller/status.go index 5a169e1c3..7b90594a9 100644 --- a/internal/ingress/controller/status.go +++ b/internal/ingress/controller/status.go @@ -36,8 +36,7 @@ import ( type leaderElectionConfig struct { Client clientset.Interface - ElectionID string - ElectionTTL time.Duration + ElectionID string OnStartedLeading func(chan struct{}) OnStoppedLeading func() @@ -51,7 +50,7 @@ func setupLeaderElection(config *leaderElectionConfig) { var cancelContext context.CancelFunc - newLeaderCtx := func(ctx context.Context) context.CancelFunc { + var newLeaderCtx = func(ctx context.Context) context.CancelFunc { // allow to cancel the context in case we stop being the leader leaderCtx, cancel := context.WithCancel(ctx) go elector.Run(leaderCtx) @@ -60,7 +59,7 @@ func setupLeaderElection(config *leaderElectionConfig) { var stopCh chan struct{} callbacks := leaderelection.LeaderCallbacks{ - OnStartedLeading: func(_ context.Context) { + OnStartedLeading: func(ctx context.Context) { klog.V(2).InfoS("I am the new leader") stopCh = make(chan struct{}) @@ -87,32 +86,29 @@ func setupLeaderElection(config *leaderElectionConfig) { } broadcaster := record.NewBroadcaster() - hostname, err := os.Hostname() - if err != nil { - klog.Errorf("unexpected error getting hostname: %v", err) - } + hostname, _ := os.Hostname() + recorder := broadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{ Component: "ingress-leader-elector", Host: hostname, }) - objectMeta := metav1.ObjectMeta{Namespace: k8s.IngressPodDetails.Namespace, Name: config.ElectionID} - resourceLockConfig := resourcelock.ResourceLockConfig{ - Identity: k8s.IngressPodDetails.Name, - EventRecorder: recorder, + lock := resourcelock.ConfigMapLock{ + ConfigMapMeta: metav1.ObjectMeta{Namespace: k8s.IngressPodDetails.Namespace, Name: config.ElectionID}, + Client: config.Client.CoreV1(), + LockConfig: resourcelock.ResourceLockConfig{ + Identity: k8s.IngressPodDetails.Name, + EventRecorder: recorder, + }, } - lock := &resourcelock.LeaseLock{ - LeaseMeta: objectMeta, - Client: config.Client.CoordinationV1(), - LockConfig: resourceLockConfig, - } + ttl := 30 * time.Second - elector, err = leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ - Lock: lock, - LeaseDuration: config.ElectionTTL, - RenewDeadline: config.ElectionTTL / 2, - RetryPeriod: config.ElectionTTL / 4, + elector, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ + Lock: &lock, + LeaseDuration: ttl, + RenewDeadline: ttl / 2, + RetryPeriod: ttl / 4, Callbacks: callbacks, }) diff --git a/internal/ingress/controller/store/backend_ssl.go b/internal/ingress/controller/store/backend_ssl.go index 81b508cd2..fe9138c6f 100644 --- a/internal/ingress/controller/store/backend_ssl.go +++ b/internal/ingress/controller/store/backend_ssl.go @@ -20,17 +20,16 @@ import ( "fmt" "strings" + "k8s.io/klog/v2" + + "github.com/pkg/errors" apiv1 "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/ingress-nginx/pkg/apis/ingress" - - klog "k8s.io/klog/v2" - + "k8s.io/ingress-nginx/internal/file" + "k8s.io/ingress-nginx/internal/ingress" "k8s.io/ingress-nginx/internal/net/ssl" - - "k8s.io/ingress-nginx/pkg/util/file" ) // syncSecret synchronizes the content of a TLS Secret (certificate(s), secret @@ -88,11 +87,10 @@ func (s *k8sStore) getPemCertificate(secretName string) (*ingress.SSLCert, error auth := secret.Data["auth"] // namespace/secretName -> namespace-secretName - nsSecName := strings.ReplaceAll(secretName, "/", "-") + nsSecName := strings.Replace(secretName, "/", "-", -1) var sslCert *ingress.SSLCert - switch { - case okcert && okkey: + if okcert && okkey { if cert == nil { return nil, fmt.Errorf("key 'tls.crt' missing from Secret %q", secretName) } @@ -145,7 +143,7 @@ func (s *k8sStore) getPemCertificate(secretName string) (*ingress.SSLCert, error } klog.V(3).InfoS(msg) - case len(ca) > 0: + } else if len(ca) > 0 { sslCert, err = ssl.CreateCACert(ca) if err != nil { return nil, fmt.Errorf("unexpected error creating SSL Cert: %v", err) @@ -156,8 +154,6 @@ func (s *k8sStore) getPemCertificate(secretName string) (*ingress.SSLCert, error return nil, fmt.Errorf("error configuring CA certificate: %v", err) } - sslCert.CASHA = file.SHA1(sslCert.CAFileName) - if len(crl) > 0 { err = ssl.ConfigureCRL(nsSecName, crl, sslCert) if err != nil { @@ -167,7 +163,7 @@ func (s *k8sStore) getPemCertificate(secretName string) (*ingress.SSLCert, error // makes this secret in 'syncSecret' to be used for Certificate Authentication // this does not enable Certificate Authentication klog.V(3).InfoS("Configuring Secret for TLS authentication", "secret", secretName) - default: + } else { if auth != nil { return nil, ErrSecretForAuth } @@ -182,7 +178,7 @@ func (s *k8sStore) getPemCertificate(secretName string) (*ingress.SSLCert, error if secretName == s.defaultSSLCertificate { path, err := ssl.StoreSSLCertOnDisk(nsSecName, sslCert) if err != nil { - return nil, fmt.Errorf("storing default SSL Certificate: %w", err) + return nil, errors.Wrap(err, "storing default SSL Certificate") } sslCert.PemFileName = path diff --git a/internal/ingress/controller/store/namespace.go b/internal/ingress/controller/store/endpoint.go similarity index 67% rename from internal/ingress/controller/store/namespace.go rename to internal/ingress/controller/store/endpoint.go index b29eb0326..ff6fbd715 100644 --- a/internal/ingress/controller/store/namespace.go +++ b/internal/ingress/controller/store/endpoint.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Kubernetes Authors. +Copyright 2015 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,19 +21,19 @@ import ( "k8s.io/client-go/tools/cache" ) -// NamespaceLister makes a Store that lists Namespaces. -type NamespaceLister struct { +// EndpointLister makes a Store that lists Endpoints. +type EndpointLister struct { cache.Store } -// ByKey returns the Namespace matching key in the local Namespace Store. -func (cml *NamespaceLister) ByKey(key string) (*apiv1.Namespace, error) { - s, exists, err := cml.GetByKey(key) +// ByKey returns the Endpoints of the Service matching key in the local Endpoint Store. +func (s *EndpointLister) ByKey(key string) (*apiv1.Endpoints, error) { + eps, exists, err := s.GetByKey(key) if err != nil { return nil, err } if !exists { return nil, NotExistsError(key) } - return s.(*apiv1.Namespace), nil + return eps.(*apiv1.Endpoints), nil } diff --git a/internal/ingress/controller/store/endpointslice.go b/internal/ingress/controller/store/endpointslice.go deleted file mode 100644 index 61bc63ae7..000000000 --- a/internal/ingress/controller/store/endpointslice.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package store - -import ( - "fmt" - "strings" - - discoveryv1 "k8s.io/api/discovery/v1" - apiNames "k8s.io/apiserver/pkg/storage/names" - "k8s.io/client-go/tools/cache" -) - -// EndpointSliceLister makes a Store that lists Endpoints. -type EndpointSliceLister struct { - cache.Store -} - -// MatchByKey returns the EndpointsSlices of the Service matching key in the local Endpoint Store. -func (s *EndpointSliceLister) MatchByKey(key string) ([]*discoveryv1.EndpointSlice, error) { - var eps []*discoveryv1.EndpointSlice - keyNsLen := strings.Index(key, "/") - if keyNsLen < -1 { - keyNsLen = 0 - } else { - // count '/' char - keyNsLen++ - } - // filter endpointSlices owned by svc - for _, listKey := range s.ListKeys() { - if len(key) < (apiNames.MaxGeneratedNameLength+keyNsLen) && !strings.HasPrefix(listKey, key) { - continue - } - // generated endpointslices names has truncated svc name as prefix when svc name is too long, we compare only non truncated part - // https://github.com/kubernetes/ingress-nginx/issues/9240 - if len(key) >= (apiNames.MaxGeneratedNameLength+keyNsLen) && !strings.HasPrefix(listKey, key[:apiNames.MaxGeneratedNameLength+keyNsLen-1]) { - continue - } - epss, exists, err := s.GetByKey(listKey) - if exists && err == nil { - // check for svc owner label - if svcName, ok := epss.(*discoveryv1.EndpointSlice).ObjectMeta.GetLabels()[discoveryv1.LabelServiceName]; ok { - namespace := epss.(*discoveryv1.EndpointSlice).ObjectMeta.GetNamespace() - if key == fmt.Sprintf("%s/%s", namespace, svcName) { - eps = append(eps, epss.(*discoveryv1.EndpointSlice)) - } - } - } - } - if len(eps) == 0 { - return nil, NotExistsError(key) - } - return eps, nil -} diff --git a/internal/ingress/controller/store/endpointslice_test.go b/internal/ingress/controller/store/endpointslice_test.go deleted file mode 100644 index 0bdb3aa33..000000000 --- a/internal/ingress/controller/store/endpointslice_test.go +++ /dev/null @@ -1,142 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package store - -import ( - "fmt" - "testing" - - discoveryv1 "k8s.io/api/discovery/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/cache" -) - -func newEndpointSliceLister(t *testing.T) *EndpointSliceLister { - t.Helper() - - return &EndpointSliceLister{Store: cache.NewStore(cache.MetaNamespaceKeyFunc)} -} - -func TestEndpointSliceLister(t *testing.T) { - t.Run("the key does not exist", func(t *testing.T) { - el := newEndpointSliceLister(t) - - key := "namespace/svcname" - _, err := el.MatchByKey(key) - - if err == nil { - t.Error("expected an error but nothing has been returned") - } - - if _, ok := err.(NotExistsError); !ok { - t.Errorf("expected NotExistsError, got %v", err) - } - }) - t.Run("the key exists", func(t *testing.T) { - el := newEndpointSliceLister(t) - - key := "namespace/svcname" - endpointSlice := &discoveryv1.EndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "namespace", - Name: "anothername-foo", - Labels: map[string]string{ - discoveryv1.LabelServiceName: "svcname", - }, - }, - } - if err := el.Add(endpointSlice); err != nil { - t.Errorf("unexpected error %v", err) - } - endpointSlice = &discoveryv1.EndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "namespace", - Name: "svcname-bar", - Labels: map[string]string{ - discoveryv1.LabelServiceName: "othersvc", - }, - }, - } - if err := el.Add(endpointSlice); err != nil { - t.Errorf("unexpected error %v", err) - } - endpointSlice = &discoveryv1.EndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "namespace", - Name: "svcname-buz", - Labels: map[string]string{ - discoveryv1.LabelServiceName: "svcname", - }, - }, - } - if err := el.Add(endpointSlice); err != nil { - t.Errorf("unexpected error %v", err) - } - eps, err := el.MatchByKey(key) - if err != nil { - t.Errorf("unexpected error %v", err) - } - if err == nil && len(eps) != 1 { - t.Errorf("expected one slice %v, error, got %d slices", endpointSlice, len(eps)) - } - if len(eps) > 0 && eps[0].GetName() != endpointSlice.GetName() { - t.Errorf("expected %v, error, got %v", endpointSlice.GetName(), eps[0].GetName()) - } - }) - t.Run("svc long name", func(t *testing.T) { - el := newEndpointSliceLister(t) - ns := "namespace" - ns2 := "another-ns" - svcName := "test-backend-http-test-http-test-http-test-http-test-http-truncated" - svcName2 := "another-long-svc-name-for-test-inhttp-test-http-test-http-truncated" - key := fmt.Sprintf("%s/%s", ns, svcName) - endpointSlice := &discoveryv1.EndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns, - Name: "test-backend-http-test-http-test-http-test-http-test-http-bar88", - Labels: map[string]string{ - discoveryv1.LabelServiceName: svcName, - }, - }, - } - if err := el.Add(endpointSlice); err != nil { - t.Errorf("unexpected error %v", err) - } - endpointSlice2 := &discoveryv1.EndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns2, - Name: "another-long-svc-name-for-test-inhttp-test-http-test-http-bar88", - Labels: map[string]string{ - discoveryv1.LabelServiceName: svcName2, - }, - }, - } - if err := el.Add(endpointSlice2); err != nil { - t.Errorf("unexpected error %v", err) - } - eps, err := el.MatchByKey(key) - if err != nil { - t.Errorf("unexpected error %v", err) - } - if len(eps) != 1 { - t.Errorf("expected one slice %v, error, got %d slices", endpointSlice, len(eps)) - } - if len(eps) == 1 && eps[0].Labels[discoveryv1.LabelServiceName] != svcName { - t.Errorf("expected slice %v, error, got %v slices", endpointSlice, eps[0]) - } - }) -} diff --git a/internal/ingress/controller/store/ingress.go b/internal/ingress/controller/store/ingress.go index d05b2624b..dafc48924 100644 --- a/internal/ingress/controller/store/ingress.go +++ b/internal/ingress/controller/store/ingress.go @@ -17,9 +17,9 @@ limitations under the License. package store import ( - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/client-go/tools/cache" - "k8s.io/ingress-nginx/pkg/apis/ingress" + "k8s.io/ingress-nginx/internal/ingress" ) // IngressLister makes a Store that lists Ingress. diff --git a/internal/ingress/controller/store/ingress_annotation.go b/internal/ingress/controller/store/ingress_annotation.go index b70ceb1c6..c9c596d1e 100644 --- a/internal/ingress/controller/store/ingress_annotation.go +++ b/internal/ingress/controller/store/ingress_annotation.go @@ -18,7 +18,7 @@ package store import ( "k8s.io/client-go/tools/cache" - "k8s.io/ingress-nginx/pkg/apis/ingress" + "k8s.io/ingress-nginx/internal/ingress" ) // IngressWithAnnotationsLister makes a Store that lists Ingress rules with annotations already parsed diff --git a/internal/ingress/controller/store/ingressclass.go b/internal/ingress/controller/store/ingressclass.go deleted file mode 100644 index da613d035..000000000 --- a/internal/ingress/controller/store/ingressclass.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package store - -import ( - networking "k8s.io/api/networking/v1" - "k8s.io/client-go/tools/cache" -) - -// IngressClassLister makes a Store that lists IngressClass. -type IngressClassLister struct { - cache.Store -} - -// ByKey returns the Ingress matching key in the local Ingress Store. -func (il IngressClassLister) ByKey(key string) (*networking.IngressClass, error) { - i, exists, err := il.GetByKey(key) - if err != nil { - return nil, err - } - if !exists { - return nil, NotExistsError(key) - } - return i.(*networking.IngressClass), nil -} diff --git a/internal/ingress/controller/store/local_secret.go b/internal/ingress/controller/store/local_secret.go index 8b26e7a53..ee3ced398 100644 --- a/internal/ingress/controller/store/local_secret.go +++ b/internal/ingress/controller/store/local_secret.go @@ -21,7 +21,7 @@ import ( "k8s.io/client-go/tools/cache" - "k8s.io/ingress-nginx/pkg/apis/ingress" + "k8s.io/ingress-nginx/internal/ingress" ) // SSLCertTracker holds a store of referenced Secrets in Ingress rules diff --git a/internal/ingress/controller/store/objectref.go b/internal/ingress/controller/store/objectref.go index 89ea47251..9ef13bf07 100644 --- a/internal/ingress/controller/store/objectref.go +++ b/internal/ingress/controller/store/objectref.go @@ -37,13 +37,13 @@ type ObjectRefMap interface { type objectRefMap struct { sync.Mutex - v map[string]sets.Set[string] + v map[string]sets.String } // NewObjectRefMap returns a new ObjectRefMap. func NewObjectRefMap() ObjectRefMap { return &objectRefMap{ - v: make(map[string]sets.Set[string]), + v: make(map[string]sets.String), } } @@ -54,7 +54,7 @@ func (o *objectRefMap) Insert(consumer string, ref ...string) { for _, r := range ref { if _, ok := o.v[r]; !ok { - o.v[r] = sets.New[string](consumer) + o.v[r] = sets.NewString(consumer) continue } o.v[r].Insert(consumer) @@ -112,7 +112,7 @@ func (o *objectRefMap) Reference(ref string) []string { if !ok { return make([]string, 0) } - return consumers.UnsortedList() + return consumers.List() } // ReferencedBy returns all objects referenced by the given object. diff --git a/internal/ingress/controller/store/store.go b/internal/ingress/controller/store/store.go index d4bd6136f..0088c3b01 100644 --- a/internal/ingress/controller/store/store.go +++ b/internal/ingress/controller/store/store.go @@ -20,44 +20,41 @@ import ( "context" "encoding/base64" "fmt" - "os" + "io/ioutil" "reflect" "sort" - "strings" "sync" "time" "github.com/eapache/channels" corev1 "k8s.io/api/core/v1" - discoveryv1 "k8s.io/api/discovery/v1" - networkingv1 "k8s.io/api/networking/v1" + networkingv1beta1 "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" clientcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" - "k8s.io/ingress-nginx/internal/ingress/inspector" - - "k8s.io/ingress-nginx/internal/nginx" - "k8s.io/ingress-nginx/pkg/util/file" - klog "k8s.io/klog/v2" + "k8s.io/klog/v2" + "k8s.io/utils/pointer" + "k8s.io/ingress-nginx/internal/file" + "k8s.io/ingress-nginx/internal/ingress" "k8s.io/ingress-nginx/internal/ingress/annotations" + "k8s.io/ingress-nginx/internal/ingress/annotations/class" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" ngx_config "k8s.io/ingress-nginx/internal/ingress/controller/config" - "k8s.io/ingress-nginx/internal/ingress/controller/ingressclass" ngx_template "k8s.io/ingress-nginx/internal/ingress/controller/template" "k8s.io/ingress-nginx/internal/ingress/defaults" "k8s.io/ingress-nginx/internal/ingress/errors" "k8s.io/ingress-nginx/internal/ingress/resolver" "k8s.io/ingress-nginx/internal/k8s" - "k8s.io/ingress-nginx/pkg/apis/ingress" + "k8s.io/ingress-nginx/internal/nginx" ) // IngressFilterFunc decides if an Ingress should be omitted or not @@ -69,9 +66,6 @@ type Storer interface { // GetBackendConfiguration returns the nginx configuration stored in a configmap GetBackendConfiguration() ngx_config.Configuration - // GetSecurityConfiguration returns the configuration options from Ingress - GetSecurityConfiguration() defaults.SecurityConfiguration - // GetConfigMap returns the ConfigMap matching key. GetConfigMap(key string) (*corev1.ConfigMap, error) @@ -81,8 +75,8 @@ type Storer interface { // GetService returns the Service matching key. GetService(key string) (*corev1.Service, error) - // GetServiceEndpointsSlices returns the EndpointSlices of a Service matching key. - GetServiceEndpointsSlices(key string) ([]*discoveryv1.EndpointSlice, error) + // GetServiceEndpoints returns the Endpoints of a Service matching key. + GetServiceEndpoints(key string) (*corev1.Endpoints, error) // ListIngresses returns a list of all Ingresses in the store. ListIngresses() []*ingress.Ingress @@ -103,9 +97,6 @@ type Storer interface { // Run initiates the synchronization of the controllers Run(stopCh chan struct{}) - - // GetIngressClass validates given ingress against ingress class configuration and returns the ingress class. - GetIngressClass(ing *networkingv1.Ingress, icConfig *ingressclass.Configuration) (string, error) } // EventType type of event associated with an informer @@ -130,24 +121,20 @@ type Event struct { // Informer defines the required SharedIndexInformers that interact with the API server. type Informer struct { - Ingress cache.SharedIndexInformer - IngressClass cache.SharedIndexInformer - EndpointSlice cache.SharedIndexInformer - Service cache.SharedIndexInformer - Secret cache.SharedIndexInformer - ConfigMap cache.SharedIndexInformer - Namespace cache.SharedIndexInformer + Ingress cache.SharedIndexInformer + Endpoint cache.SharedIndexInformer + Service cache.SharedIndexInformer + Secret cache.SharedIndexInformer + ConfigMap cache.SharedIndexInformer } // Lister contains object listers (stores). type Lister struct { Ingress IngressLister - IngressClass IngressClassLister Service ServiceLister - EndpointSlice EndpointSliceLister + Endpoint EndpointLister Secret SecretLister ConfigMap ConfigMapLister - Namespace NamespaceLister IngressWithAnnotation IngressWithAnnotationsLister } @@ -162,34 +149,20 @@ func (e NotExistsError) Error() string { // Run initiates the synchronization of the informers against the API server. func (i *Informer) Run(stopCh chan struct{}) { go i.Secret.Run(stopCh) - go i.EndpointSlice.Run(stopCh) - if i.IngressClass != nil { - go i.IngressClass.Run(stopCh) - } + go i.Endpoint.Run(stopCh) go i.Service.Run(stopCh) go i.ConfigMap.Run(stopCh) // wait for all involved caches to be synced before processing items // from the queue if !cache.WaitForCacheSync(stopCh, + i.Endpoint.HasSynced, i.Service.HasSynced, i.Secret.HasSynced, i.ConfigMap.HasSynced, ) { runtime.HandleError(fmt.Errorf("timed out waiting for caches to sync")) } - if i.IngressClass != nil && !cache.WaitForCacheSync(stopCh, i.IngressClass.HasSynced) { - runtime.HandleError(fmt.Errorf("timed out waiting for ingress classcaches to sync")) - } - - // when limit controller scope to one namespace, skip sync namespaces at cluster scope - if i.Namespace != nil { - go i.Namespace.Run(stopCh) - - if !cache.WaitForCacheSync(stopCh, i.Namespace.HasSynced) { - runtime.HandleError(fmt.Errorf("timed out waiting for caches to sync")) - } - } // in big clusters, deltas can keep arriving even after HasSynced // functions have returned 'true' @@ -240,25 +213,16 @@ type k8sStore struct { backendConfigMu *sync.RWMutex defaultSSLCertificate string - - recorder record.EventRecorder } -// New creates a new object store to be used in the ingress controller. -// -//nolint:gocyclo // Ignore function complexity error. +// New creates a new object store to be used in the ingress controller func New( - namespace string, - namespaceSelector labels.Selector, - configmap, tcp, udp, defaultSSLCertificate string, + namespace, configmap, tcp, udp, defaultSSLCertificate string, resyncPeriod time.Duration, client clientset.Interface, updateCh *channels.RingChannel, - disableCatchAll bool, - deepInspector bool, - icConfig *ingressclass.Configuration, - disableSyncEvents bool, -) Storer { + disableCatchAll bool) Storer { + store := &k8sStore{ informers: &Informer{}, listers: &Lister{}, @@ -273,15 +237,12 @@ func New( eventBroadcaster := record.NewBroadcaster() eventBroadcaster.StartLogging(klog.Infof) - if !disableSyncEvents { - eventBroadcaster.StartRecordingToSink(&clientcorev1.EventSinkImpl{ - Interface: client.CoreV1().Events(namespace), - }) - } + eventBroadcaster.StartRecordingToSink(&clientcorev1.EventSinkImpl{ + Interface: client.CoreV1().Events(namespace), + }) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{ Component: "nginx-ingress-controller", }) - store.recorder = recorder // k8sStore fulfills resolver.Resolver interface store.annotations = annotations.NewAnnotationExtractor(store) @@ -295,7 +256,7 @@ func New( // the memory consumption of nginx-ingress-controller explode. // In order to avoid that we filter out labels OWNER=TILLER. labelsTweakListOptionsFunc := func(options *metav1.ListOptions) { - if options.LabelSelector != "" { + if len(options.LabelSelector) > 0 { options.LabelSelector += ",OWNER!=TILLER" } else { options.LabelSelector = "OWNER!=TILLER" @@ -332,16 +293,11 @@ func New( informers.WithTweakListOptions(secretsTweakListOptionsFunc), ) - store.informers.Ingress = infFactory.Networking().V1().Ingresses().Informer() + store.informers.Ingress = infFactory.Networking().V1beta1().Ingresses().Informer() store.listers.Ingress.Store = store.informers.Ingress.GetStore() - if !icConfig.IgnoreIngressClass { - store.informers.IngressClass = infFactory.Networking().V1().IngressClasses().Informer() - store.listers.IngressClass.Store = cache.NewStore(cache.MetaNamespaceKeyFunc) - } - - store.informers.EndpointSlice = infFactory.Discovery().V1().EndpointSlices().Informer() - store.listers.EndpointSlice.Store = store.informers.EndpointSlice.GetStore() + store.informers.Endpoint = infFactory.Core().V1().Endpoints().Informer() + store.listers.Endpoint.Store = store.informers.Endpoint.GetStore() store.informers.Secret = infFactorySecrets.Core().V1().Secrets().Informer() store.listers.Secret.Store = store.informers.Secret.GetStore() @@ -352,35 +308,6 @@ func New( store.informers.Service = infFactory.Core().V1().Services().Informer() store.listers.Service.Store = store.informers.Service.GetStore() - // avoid caching namespaces at cluster scope when watching single namespace - if namespaceSelector != nil && !namespaceSelector.Empty() { - // cache informers factory for namespaces - infFactoryNamespaces := informers.NewSharedInformerFactoryWithOptions(client, resyncPeriod, - informers.WithTweakListOptions(labelsTweakListOptionsFunc), - ) - - store.informers.Namespace = infFactoryNamespaces.Core().V1().Namespaces().Informer() - store.listers.Namespace.Store = store.informers.Namespace.GetStore() - } - - watchedNamespace := func(namespace string) bool { - if namespaceSelector == nil || namespaceSelector.Empty() { - return true - } - - item, ok, err := store.listers.Namespace.GetByKey(namespace) - if !ok { - klog.Errorf("Namespace %s not existed: %v.", namespace, err) - return false - } - ns, ok := item.(*corev1.Namespace) - if !ok { - return false - } - - return namespaceSelector.Matches(labels.Set(ns.Labels)) - } - ingDeleteHandler := func(obj interface{}) { ing, ok := toIngress(obj) if !ok { @@ -390,20 +317,14 @@ func New( klog.ErrorS(nil, "Error obtaining object from tombstone", "key", obj) return } - ing, ok = tombstone.Obj.(*networkingv1.Ingress) + ing, ok = tombstone.Obj.(*networkingv1beta1.Ingress) if !ok { klog.Errorf("Tombstone contained object that is not an Ingress: %#v", obj) return } } - if !watchedNamespace(ing.Namespace) { - return - } - - _, err := store.GetIngressClass(ing, icConfig) - if err != nil { - klog.InfoS("Ignoring ingress because of error while validating ingress class", "ingress", klog.KObj(ing), "error", err) + if !class.IsValid(ing) { return } @@ -412,10 +333,7 @@ func New( return } - if err := store.listers.IngressWithAnnotation.Delete(ing); err != nil { - klog.ErrorS(err, "Error while deleting ingress from store", "ingress", klog.KObj(ing)) - return - } + store.listers.IngressWithAnnotation.Delete(ing) key := k8s.MetaNamespaceKey(ing) store.secretIngressMap.Delete(key) @@ -429,25 +347,12 @@ func New( ingEventHandler := cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { ing, _ := toIngress(obj) - - if !watchedNamespace(ing.Namespace) { + if !class.IsValid(ing) { + ingressClass, _ := parser.GetStringAnnotation(class.IngressKey, ing) + klog.InfoS("Ignoring ingress", "ingress", klog.KObj(ing), "kubernetes.io/ingress.class", ingressClass, "ingressClassName", pointer.StringPtrDerefOr(ing.Spec.IngressClassName, "")) return } - ic, err := store.GetIngressClass(ing, icConfig) - if err != nil { - klog.InfoS("Ignoring ingress because of error while validating ingress class", "ingress", klog.KObj(ing), "error", err) - return - } - - klog.InfoS("Found valid IngressClass", "ingress", klog.KObj(ing), "ingressclass", ic) - - if deepInspector { - if err := inspector.DeepInspect(ing); err != nil { - klog.ErrorS(err, "received invalid ingress", "ingress", klog.KObj(ing)) - return - } - } if hasCatchAllIngressRule(ing.Spec) && disableCatchAll { klog.InfoS("Ignoring add for catch-all ingress because of --disable-catch-all", "ingress", klog.KObj(ing)) return @@ -469,30 +374,21 @@ func New( oldIng, _ := toIngress(old) curIng, _ := toIngress(cur) - if !watchedNamespace(oldIng.Namespace) { - return - } - - var errOld, errCur error - var classCur string - if !icConfig.IgnoreIngressClass { - _, errOld = store.GetIngressClass(oldIng, icConfig) - classCur, errCur = store.GetIngressClass(curIng, icConfig) - } - switch { - case errOld != nil && errCur == nil: + validOld := class.IsValid(oldIng) + validCur := class.IsValid(curIng) + if !validOld && validCur { if hasCatchAllIngressRule(curIng.Spec) && disableCatchAll { klog.InfoS("ignoring update for catch-all ingress because of --disable-catch-all", "ingress", klog.KObj(curIng)) return } - klog.InfoS("creating ingress", "ingress", klog.KObj(curIng), "ingressclass", classCur) + klog.InfoS("creating ingress", "ingress", klog.KObj(curIng), "class", class.IngressKey) recorder.Eventf(curIng, corev1.EventTypeNormal, "Sync", "Scheduled for sync") - case errOld == nil && errCur != nil: - klog.InfoS("removing ingress because of unknown ingressclass", "ingress", klog.KObj(curIng)) + } else if validOld && !validCur { + klog.InfoS("removing ingress", "ingress", klog.KObj(curIng), "class", class.IngressKey) ingDeleteHandler(old) return - case errCur == nil && !reflect.DeepEqual(old, cur): + } else if validCur && !reflect.DeepEqual(old, cur) { if hasCatchAllIngressRule(curIng.Spec) && disableCatchAll { klog.InfoS("ignoring update for catch-all ingress and delete old one because of --disable-catch-all", "ingress", klog.KObj(curIng)) ingDeleteHandler(old) @@ -500,18 +396,11 @@ func New( } recorder.Eventf(curIng, corev1.EventTypeNormal, "Sync", "Scheduled for sync") - default: + } else { klog.V(3).InfoS("No changes on ingress. Skipping update", "ingress", klog.KObj(curIng)) return } - if deepInspector { - if err := inspector.DeepInspect(curIng); err != nil { - klog.ErrorS(err, "received invalid ingress", "ingress", klog.KObj(curIng)) - return - } - } - store.syncIngress(curIng) store.updateSecretIngressMap(curIng) store.syncSecrets(curIng) @@ -523,86 +412,9 @@ func New( }, } - ingressClassEventHandler := cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - ingressclass, ok := obj.(*networkingv1.IngressClass) - if !ok { - klog.Errorf("unexpected type: %T", obj) - } - foundClassByName := false - if icConfig.IngressClassByName && ingressclass.Name == icConfig.AnnotationValue { - klog.InfoS("adding ingressclass as ingress-class-by-name is configured", "ingressclass", klog.KObj(ingressclass)) - foundClassByName = true - } - if !foundClassByName && ingressclass.Spec.Controller != icConfig.Controller { - klog.InfoS("ignoring ingressclass as the spec.controller is not the same of this ingress", "ingressclass", klog.KObj(ingressclass)) - return - } - err := store.listers.IngressClass.Add(ingressclass) - if err != nil { - klog.InfoS("error adding ingressclass to store", "ingressclass", klog.KObj(ingressclass), "error", err) - return - } - - updateCh.In() <- Event{ - Type: CreateEvent, - Obj: obj, - } - }, - DeleteFunc: func(obj interface{}) { - ingressclass, ok := obj.(*networkingv1.IngressClass) - if !ok { - klog.Errorf("unexpected type: %T", obj) - } - if ingressclass.Spec.Controller != icConfig.Controller { - klog.InfoS("ignoring ingressclass as the spec.controller is not the same of this ingress", "ingressclass", klog.KObj(ingressclass)) - return - } - err := store.listers.IngressClass.Delete(ingressclass) - if err != nil { - klog.InfoS("error removing ingressclass from store", "ingressclass", klog.KObj(ingressclass), "error", err) - return - } - updateCh.In() <- Event{ - Type: DeleteEvent, - Obj: obj, - } - }, - UpdateFunc: func(old, cur interface{}) { - oic, ok := old.(*networkingv1.IngressClass) - if !ok { - klog.Errorf("unexpected type: %T", old) - } - cic, ok := cur.(*networkingv1.IngressClass) - if !ok { - klog.Errorf("unexpected type: %T", cur) - } - if cic.Spec.Controller != icConfig.Controller { - klog.InfoS("ignoring ingressclass as the spec.controller is not the same of this ingress", "ingressclass", klog.KObj(cic)) - return - } - // TODO: In a future we might be interested in parse parameters and use as - // current IngressClass for this case, crossing with configmap - if !reflect.DeepEqual(cic.Spec.Parameters, oic.Spec.Parameters) { - err := store.listers.IngressClass.Update(cic) - if err != nil { - klog.InfoS("error updating ingressclass in store", "ingressclass", klog.KObj(cic), "error", err) - return - } - updateCh.In() <- Event{ - Type: UpdateEvent, - Obj: cur, - } - } - }, - } - secrEventHandler := cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - sec, ok := obj.(*corev1.Secret) - if !ok { - klog.Errorf("unexpected type: %T", obj) - } + sec := obj.(*corev1.Secret) key := k8s.MetaNamespaceKey(sec) if store.defaultSSLCertificate == key { @@ -629,16 +441,9 @@ func New( }, UpdateFunc: func(old, cur interface{}) { if !reflect.DeepEqual(old, cur) { - sec, ok := cur.(*corev1.Secret) - if !ok { - klog.Errorf("unexpected type: %T", cur) - } + sec := cur.(*corev1.Secret) key := k8s.MetaNamespaceKey(sec) - if !watchedNamespace(sec.Namespace) { - return - } - if store.defaultSSLCertificate == key { store.syncSecret(store.defaultSSLCertificate) } @@ -677,10 +482,6 @@ func New( } } - if !watchedNamespace(sec.Namespace) { - return - } - store.sslStore.Delete(k8s.MetaNamespaceKey(sec)) key := k8s.MetaNamespaceKey(sec) @@ -705,7 +506,7 @@ func New( }, } - epsEventHandler := cache.ResourceEventHandlerFuncs{ + epEventHandler := cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { updateCh.In() <- Event{ Type: CreateEvent, @@ -719,15 +520,9 @@ func New( } }, UpdateFunc: func(old, cur interface{}) { - oeps, ok := old.(*discoveryv1.EndpointSlice) - if !ok { - klog.Errorf("unexpected type: %T", old) - } - ceps, ok := cur.(*discoveryv1.EndpointSlice) - if !ok { - klog.Errorf("unexpected type: %T", cur) - } - if !reflect.DeepEqual(ceps.Endpoints, oeps.Endpoints) { + oep := old.(*corev1.Endpoints) + cep := cur.(*corev1.Endpoints) + if !reflect.DeepEqual(cep.Subsets, oep.Subsets) { updateCh.In() <- Event{ Type: UpdateEvent, Obj: cur, @@ -736,6 +531,7 @@ func New( }, } + // TODO: add e2e test to verify that changes to one or more configmap trigger an update changeTriggerUpdate := func(name string) bool { return name == configmap || name == tcp || name == udp } @@ -780,10 +576,7 @@ func New( cmEventHandler := cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - cfgMap, ok := obj.(*corev1.ConfigMap) - if !ok { - klog.Errorf("unexpected type: %T", obj) - } + cfgMap := obj.(*corev1.ConfigMap) key := k8s.MetaNamespaceKey(cfgMap) handleCfgMapEvent(key, cfgMap, "CREATE") }, @@ -792,49 +585,16 @@ func New( return } - cfgMap, ok := cur.(*corev1.ConfigMap) - if !ok { - klog.Errorf("unexpected type: %T", cur) - } + cfgMap := cur.(*corev1.ConfigMap) key := k8s.MetaNamespaceKey(cfgMap) handleCfgMapEvent(key, cfgMap, "UPDATE") }, } serviceHandler := cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - svc, ok := obj.(*corev1.Service) - if !ok { - klog.Errorf("unexpected type: %T", obj) - } - if svc.Spec.Type == corev1.ServiceTypeExternalName { - updateCh.In() <- Event{ - Type: CreateEvent, - Obj: obj, - } - } - }, - DeleteFunc: func(obj interface{}) { - svc, ok := obj.(*corev1.Service) - if !ok { - klog.Errorf("unexpected type: %T", obj) - } - if svc.Spec.Type == corev1.ServiceTypeExternalName { - updateCh.In() <- Event{ - Type: DeleteEvent, - Obj: obj, - } - } - }, UpdateFunc: func(old, cur interface{}) { - oldSvc, ok := old.(*corev1.Service) - if !ok { - klog.Errorf("unexpected type: %T", old) - } - curSvc, ok := cur.(*corev1.Service) - if !ok { - klog.Errorf("unexpected type: %T", cur) - } + oldSvc := old.(*corev1.Service) + curSvc := cur.(*corev1.Service) if reflect.DeepEqual(oldSvc, curSvc) { return @@ -847,32 +607,14 @@ func New( }, } - if _, err := store.informers.Ingress.AddEventHandler(ingEventHandler); err != nil { - klog.Errorf("Error adding ingress event handler: %v", err) - } - if !icConfig.IgnoreIngressClass { - if _, err := store.informers.IngressClass.AddEventHandler(ingressClassEventHandler); err != nil { - klog.Errorf("Error adding ingress class event handler: %v", err) - } - } - if _, err := store.informers.EndpointSlice.AddEventHandler(epsEventHandler); err != nil { - klog.Errorf("Error adding endpoint slice event handler: %v", err) - } - if _, err := store.informers.Secret.AddEventHandler(secrEventHandler); err != nil { - klog.Errorf("Error adding secret event handler: %v", err) - } - if _, err := store.informers.ConfigMap.AddEventHandler(cmEventHandler); err != nil { - klog.Errorf("Error adding configmap event handler: %v", err) - } - if _, err := store.informers.Service.AddEventHandler(serviceHandler); err != nil { - klog.Errorf("Error adding service event handler: %v", err) - } + store.informers.Ingress.AddEventHandler(ingEventHandler) + store.informers.Endpoint.AddEventHandler(epEventHandler) + store.informers.Secret.AddEventHandler(secrEventHandler) + store.informers.ConfigMap.AddEventHandler(cmEventHandler) + store.informers.Service.AddEventHandler(serviceHandler) // do not wait for informers to read the configmap configuration - ns, name, err := k8s.ParseNameNS(configmap) - if err != nil { - klog.Errorf("unexpected error parsing name and ns: %v", err) - } + ns, name, _ := k8s.ParseNameNS(configmap) cm, err := client.CoreV1().ConfigMaps(ns).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { klog.Warningf("Unexpected error reading configuration configmap: %v", err) @@ -884,41 +626,18 @@ func New( // hasCatchAllIngressRule returns whether or not an ingress produces a // catch-all server, and so should be ignored when --disable-catch-all is set -func hasCatchAllIngressRule(spec networkingv1.IngressSpec) bool { - return spec.DefaultBackend != nil -} - -func checkBadAnnotationValue(annotationMap map[string]string, badwords string) error { - arraybadWords := strings.Split(strings.TrimSpace(badwords), ",") - - for annotation, value := range annotationMap { - if strings.HasPrefix(annotation, fmt.Sprintf("%s/", parser.AnnotationsPrefix)) { - for _, forbiddenvalue := range arraybadWords { - if strings.Contains(value, forbiddenvalue) { - return fmt.Errorf("%s annotation contains invalid word %s", annotation, forbiddenvalue) - } - } - } - } - return nil +func hasCatchAllIngressRule(spec networkingv1beta1.IngressSpec) bool { + return spec.Backend != nil } // syncIngress parses ingress annotations converting the value of the // annotation to a go struct -func (s *k8sStore) syncIngress(ing *networkingv1.Ingress) { +func (s *k8sStore) syncIngress(ing *networkingv1beta1.Ingress) { key := k8s.MetaNamespaceKey(ing) klog.V(3).Infof("updating annotations information for ingress %v", key) - copyIng := &networkingv1.Ingress{} + copyIng := &networkingv1beta1.Ingress{} ing.ObjectMeta.DeepCopyInto(©Ing.ObjectMeta) - - if s.backendConfig.AnnotationValueWordBlocklist != "" { - if err := checkBadAnnotationValue(copyIng.Annotations, s.backendConfig.AnnotationValueWordBlocklist); err != nil { - klog.Warningf("skipping ingress %s: %s", key, err) - return - } - } - ing.Spec.DeepCopyInto(©Ing.Spec) ing.Status.DeepCopyInto(©Ing.Status) @@ -936,17 +655,9 @@ func (s *k8sStore) syncIngress(ing *networkingv1.Ingress) { k8s.SetDefaultNGINXPathType(copyIng) - parsed, err := s.annotations.Extract(ing) - if err != nil { - klog.Error(err) - return - } - if parsed.Denied != nil { - s.recorder.Eventf(ing, corev1.EventTypeWarning, "AnnotationParsingFailed", fmt.Sprintf("Error parsing annotations: %v", *parsed.Denied)) - } - err = s.listers.IngressWithAnnotation.Update(&ingress.Ingress{ + err := s.listers.IngressWithAnnotation.Update(&ingress.Ingress{ Ingress: *copyIng, - ParsedAnnotations: parsed, + ParsedAnnotations: s.annotations.Extract(ing), }) if err != nil { klog.Error(err) @@ -955,7 +666,7 @@ func (s *k8sStore) syncIngress(ing *networkingv1.Ingress) { // updateSecretIngressMap takes an Ingress and updates all Secret objects it // references in secretIngressMap. -func (s *k8sStore) updateSecretIngressMap(ing *networkingv1.Ingress) { +func (s *k8sStore) updateSecretIngressMap(ing *networkingv1beta1.Ingress) { key := k8s.MetaNamespaceKey(ing) klog.V(3).Infof("updating references to secrets for ingress %v", key) @@ -982,10 +693,8 @@ func (s *k8sStore) updateSecretIngressMap(ing *networkingv1.Ingress) { "proxy-ssl-secret", "secure-verify-ca-secret", } - - secConfig := s.GetSecurityConfiguration().AllowCrossNamespaceResources for _, ann := range secretAnnotations { - secrKey, err := objectRefAnnotationNsKey(ann, ing, secConfig) + secrKey, err := objectRefAnnotationNsKey(ann, ing) if err != nil && !errors.IsMissingAnnotations(err) { klog.Errorf("error reading secret reference in annotation %q: %s", ann, err) continue @@ -1001,9 +710,8 @@ func (s *k8sStore) updateSecretIngressMap(ing *networkingv1.Ingress) { // objectRefAnnotationNsKey returns an object reference formatted as a // 'namespace/name' key from the given annotation name. -func objectRefAnnotationNsKey(ann string, ing *networkingv1.Ingress, allowCrossNamespace bool) (string, error) { - // We pass nil fields, as this is an internal process and we don't need to validate it. - annValue, err := parser.GetStringAnnotation(ann, ing, nil) +func objectRefAnnotationNsKey(ann string, ing *networkingv1beta1.Ingress) (string, error) { + annValue, err := parser.GetStringAnnotation(ann, ing) if err != nil { return "", err } @@ -1016,15 +724,12 @@ func objectRefAnnotationNsKey(ann string, ing *networkingv1.Ingress, allowCrossN if secrNs == "" { return fmt.Sprintf("%v/%v", ing.Namespace, secrName), nil } - if !allowCrossNamespace && secrNs != ing.Namespace { - return "", fmt.Errorf("cross namespace secret is not supported") - } return annValue, nil } // syncSecrets synchronizes data from all Secrets referenced by the given // Ingress with the local store and file system. -func (s *k8sStore) syncSecrets(ing *networkingv1.Ingress) { +func (s *k8sStore) syncSecrets(ing *networkingv1beta1.Ingress) { key := k8s.MetaNamespaceKey(ing) for _, secrKey := range s.secretIngressMap.ReferencedBy(key) { s.syncSecret(secrKey) @@ -1053,34 +758,8 @@ func (s *k8sStore) GetService(key string) (*corev1.Service, error) { return s.listers.Service.ByKey(key) } -func (s *k8sStore) GetIngressClass(ing *networkingv1.Ingress, icConfig *ingressclass.Configuration) (string, error) { - // First we try ingressClassName - if !icConfig.IgnoreIngressClass && ing.Spec.IngressClassName != nil { - iclass, err := s.listers.IngressClass.ByKey(*ing.Spec.IngressClassName) - if err != nil { - return "", err - } - return iclass.Name, nil - } - - // Then we try annotation - if class, ok := ing.GetAnnotations()[ingressclass.IngressKey]; ok { - if class != icConfig.AnnotationValue { - return "", fmt.Errorf("ingress class annotation is not equal to the expected by Ingress Controller") - } - return class, nil - } - - // Then we accept if the WithoutClass is enabled - if icConfig.WatchWithoutClass { - // Reserving "_" as a "wildcard" name - return "_", nil - } - return "", fmt.Errorf("ingress does not contain a valid IngressClass") -} - // getIngress returns the Ingress matching key. -func (s *k8sStore) getIngress(key string) (*networkingv1.Ingress, error) { +func (s *k8sStore) getIngress(key string) (*networkingv1beta1.Ingress, error) { ing, err := s.listers.IngressWithAnnotation.ByKey(key) if err != nil { return nil, err @@ -1109,10 +788,7 @@ func (s *k8sStore) ListIngresses() []*ingress.Ingress { // filter ingress rules ingresses := make([]*ingress.Ingress, 0) for _, item := range s.listers.IngressWithAnnotation.List() { - ing, ok := item.(*ingress.Ingress) - if !ok { - klog.Errorf("unexpected type: %T", item) - } + ing := item.(*ingress.Ingress) ingresses = append(ingresses, ing) } @@ -1131,8 +807,9 @@ func (s *k8sStore) GetConfigMap(key string) (*corev1.ConfigMap, error) { return s.listers.ConfigMap.ByKey(key) } -func (s *k8sStore) GetServiceEndpointsSlices(key string) ([]*discoveryv1.EndpointSlice, error) { - return s.listers.EndpointSlice.MatchByKey(key) +// GetServiceEndpoints returns the Endpoints of a Service matching key. +func (s *k8sStore) GetServiceEndpoints(key string) (*corev1.Endpoints, error) { + return s.listers.Endpoint.ByKey(key) } // GetAuthCertificate is used by the auth-tls annotations to get a cert from a secret @@ -1174,7 +851,7 @@ func (s *k8sStore) writeSSLSessionTicketKey(cmap *corev1.ConfigMap, fileName str return } - err = os.WriteFile(fileName, decodedTicket, file.ReadWriteByUser) + err = ioutil.WriteFile(fileName, decodedTicket, file.ReadWriteByUser) if err != nil { klog.Errorf("unexpected error writing ssl-session-ticket-key to %s: %v", fileName, err) return @@ -1196,17 +873,6 @@ func (s *k8sStore) GetBackendConfiguration() ngx_config.Configuration { return s.backendConfig } -func (s *k8sStore) GetSecurityConfiguration() defaults.SecurityConfiguration { - s.backendConfigMu.RLock() - defer s.backendConfigMu.RUnlock() - - secConfig := defaults.SecurityConfiguration{ - AllowCrossNamespaceResources: s.backendConfig.AllowCrossNamespaceResources, - AnnotationsRiskLevel: s.backendConfig.AnnotationsRiskLevel, - } - return secConfig -} - func (s *k8sStore) setConfig(cmap *corev1.ConfigMap) { s.backendConfigMu.Lock() defer s.backendConfigMu.Unlock() @@ -1221,7 +887,7 @@ func (s *k8sStore) setConfig(cmap *corev1.ConfigMap) { s.backendConfig.UseGeoIP2 = false } - s.writeSSLSessionTicketKey(cmap, "/etc/ingress-controller/tickets.key") + s.writeSSLSessionTicketKey(cmap, "/etc/nginx/tickets.key") } // Run initiates the synchronization of the informers and the initial @@ -1234,11 +900,11 @@ func (s *k8sStore) Run(stopCh chan struct{}) { var runtimeScheme = k8sruntime.NewScheme() func init() { - runtime.Must(networkingv1.AddToScheme(runtimeScheme)) + utilruntime.Must(networkingv1beta1.AddToScheme(runtimeScheme)) } -func toIngress(obj interface{}) (*networkingv1.Ingress, bool) { - if ing, ok := obj.(*networkingv1.Ingress); ok { +func toIngress(obj interface{}) (*networkingv1beta1.Ingress, bool) { + if ing, ok := obj.(*networkingv1beta1.Ingress); ok { k8s.SetDefaultNGINXPathType(ing) return ing, true } diff --git a/internal/ingress/controller/store/store_test.go b/internal/ingress/controller/store/store_test.go index 9c719af3b..bcc8f67d6 100644 --- a/internal/ingress/controller/store/store_test.go +++ b/internal/ingress/controller/store/store_test.go @@ -20,6 +20,7 @@ import ( "context" "encoding/base64" "fmt" + "io/ioutil" "os" "sync" "sync/atomic" @@ -28,62 +29,23 @@ import ( "github.com/eapache/channels" v1 "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/envtest" + "k8s.io/ingress-nginx/internal/ingress" + "k8s.io/ingress-nginx/internal/ingress/annotations/class" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" - "k8s.io/ingress-nginx/internal/ingress/controller/ingressclass" - "k8s.io/ingress-nginx/pkg/apis/ingress" "k8s.io/ingress-nginx/test/e2e/framework" ) -var pathPrefix networking.PathType = networking.PathTypePrefix - -var DefaultClassConfig = &ingressclass.Configuration{ - Controller: ingressclass.DefaultControllerName, - AnnotationValue: ingressclass.DefaultAnnotationValue, - WatchWithoutClass: false, -} - -var commonIngressSpec = networking.IngressSpec{ - Rules: []networking.IngressRule{ - { - Host: "dummy", - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - Path: "/", - PathType: &pathPrefix, - Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, - }, - }, - }, - }, - }, - }, - }, -} - -const updateDummyHost = "update-dummy" - -//nolint:gocyclo // Ignore function complexity error func TestStore(t *testing.T) { - // TODO: move env definition to docker image? - t.Setenv("KUBEBUILDER_ASSETS", "/usr/local/bin") - - pathPrefix = networking.PathTypePrefix + //TODO: move env definition to docker image? + os.Setenv("KUBEBUILDER_ASSETS", "/usr/local/bin") te := &envtest.Environment{} cfg, err := te.Start() @@ -91,12 +53,7 @@ func TestStore(t *testing.T) { t.Fatalf("error: %v", err) } - emptySelector, err := labels.Parse("") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - defer te.Stop() //nolint:errcheck // Ignore the error + defer te.Stop() clientSet, err := kubernetes.NewForConfig(cfg) if err != nil { @@ -119,7 +76,6 @@ func TestStore(t *testing.T) { storer := New( ns, - emptySelector, fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -127,9 +83,6 @@ func TestStore(t *testing.T) { 10*time.Minute, clientSet, updateCh, - false, - true, - DefaultClassConfig, false) storer.Run(stopCh) @@ -156,123 +109,13 @@ func TestStore(t *testing.T) { t.Errorf("expected an error but none returned") } if svc != nil { - t.Errorf("expected an Ingress but none returned") - } - }) - - t.Run("should return no event for add, update and delete of ingress as the existing ingressclass is not the expected", func(t *testing.T) { - ns := createNamespace(clientSet, t) - defer deleteNamespace(ns, clientSet, t) - - createConfigMap(clientSet, ns, t) - - stopCh := make(chan struct{}) - updateCh := channels.NewRingChannel(1024) - - var add uint64 - var upd uint64 - var del uint64 - - go func(ch *channels.RingChannel) { - for { - evt, ok := <-ch.Out() - if !ok { - return - } - - e, ok := evt.(Event) - if !ok { - return - } - - if e.Obj == nil { - continue - } - if _, ok := e.Obj.(*networking.Ingress); !ok { - continue - } - - switch e.Type { - case CreateEvent: - atomic.AddUint64(&add, 1) - case UpdateEvent: - atomic.AddUint64(&upd, 1) - case DeleteEvent: - atomic.AddUint64(&del, 1) - } - } - }(updateCh) - - storer := New( - ns, - emptySelector, - fmt.Sprintf("%v/config", ns), - fmt.Sprintf("%v/tcp", ns), - fmt.Sprintf("%v/udp", ns), - "", - 10*time.Minute, - clientSet, - updateCh, - false, - true, - DefaultClassConfig, - false) - - storer.Run(stopCh) - ic := createIngressClass(clientSet, t, "not-k8s.io/not-ingress-nginx") - defer deleteIngressClass(ic, clientSet, t) - validSpec := commonIngressSpec - validSpec.IngressClassName = &ic - ing := ensureIngress(&networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "dummy-no-class", - Namespace: ns, - }, - Spec: validSpec, - }, clientSet, t) - - err := framework.WaitForIngressInNamespace(clientSet, ns, ing.Name) - if err != nil { - t.Errorf("error waiting for secret: %v", err) - } - time.Sleep(1 * time.Second) - - ni := ing.DeepCopy() - ni.Spec.Rules[0].Host = updateDummyHost - _ = ensureIngress(ni, clientSet, t) - if err != nil { - t.Errorf("error creating ingress: %v", err) - } - // Secret takes a bit to update - time.Sleep(3 * time.Second) - - err = clientSet.NetworkingV1().Ingresses(ni.Namespace).Delete(context.TODO(), ni.Name, metav1.DeleteOptions{}) - if err != nil { - t.Errorf("error deleting ingress: %v", err) - } - - err = framework.WaitForNoIngressInNamespace(clientSet, ni.Namespace, ni.Name) - if err != nil { - t.Errorf("error waiting for secret: %v", err) - } - time.Sleep(1 * time.Second) - - if atomic.LoadUint64(&add) != 0 { - t.Errorf("expected 0 event of type Create but %v occurred", add) - } - if atomic.LoadUint64(&upd) != 0 { - t.Errorf("expected 0 event of type Update but %v occurred", upd) - } - if atomic.LoadUint64(&del) != 0 { - t.Errorf("expected 0 event of type Delete but %v occurred", del) + t.Errorf("expected an Ingres but none returned") } }) t.Run("should return one event for add, update and delete of ingress", func(t *testing.T) { ns := createNamespace(clientSet, t) defer deleteNamespace(ns, clientSet, t) - ic := createIngressClass(clientSet, t, ingressclass.DefaultControllerName) - defer deleteIngressClass(ic, clientSet, t) createConfigMap(clientSet, ns, t) stopCh := make(chan struct{}) @@ -289,10 +132,7 @@ func TestStore(t *testing.T) { return } - e, ok := evt.(Event) - if !ok { - return - } + e := evt.(Event) if e.Obj == nil { continue } @@ -313,7 +153,6 @@ func TestStore(t *testing.T) { storer := New( ns, - emptySelector, fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -321,40 +160,77 @@ func TestStore(t *testing.T) { 10*time.Minute, clientSet, updateCh, - false, - true, - DefaultClassConfig, false) storer.Run(stopCh) - validSpec := commonIngressSpec - validSpec.IngressClassName = &ic + ing := ensureIngress(&networking.Ingress{ ObjectMeta: metav1.ObjectMeta{ - Name: "dummy-class", + Name: "dummy", Namespace: ns, }, - Spec: validSpec, + Spec: networking.IngressSpec{ + Rules: []networking.IngressRule{ + { + Host: "dummy", + IngressRuleValue: networking.IngressRuleValue{ + HTTP: &networking.HTTPIngressRuleValue{ + Paths: []networking.HTTPIngressPath{ + { + Path: "/", + Backend: networking.IngressBackend{ + ServiceName: "http-svc", + ServicePort: intstr.FromInt(80), + }, + }, + }, + }, + }, + }, + }, + }, }, clientSet, t) err := framework.WaitForIngressInNamespace(clientSet, ns, ing.Name) if err != nil { - t.Errorf("error waiting for ingress: %v", err) + t.Errorf("error waiting for secret: %v", err) } time.Sleep(1 * time.Second) - // create an invalid ingress (no ingress class and no watchWithoutClass config) + // create an invalid ingress (different class) invalidIngress := ensureIngress(&networking.Ingress{ ObjectMeta: metav1.ObjectMeta{ - Name: "no-class", + Name: "custom-class", Namespace: ns, + Annotations: map[string]string{ + class.IngressKey: "something", + }, + }, + Spec: networking.IngressSpec{ + Rules: []networking.IngressRule{ + { + Host: "dummy", + IngressRuleValue: networking.IngressRuleValue{ + HTTP: &networking.HTTPIngressRuleValue{ + Paths: []networking.HTTPIngressPath{ + { + Path: "/", + Backend: networking.IngressBackend{ + ServiceName: "http-svc", + ServicePort: intstr.FromInt(80), + }, + }, + }, + }, + }, + }, + }, }, - Spec: commonIngressSpec, }, clientSet, t) defer deleteIngress(invalidIngress, clientSet, t) ni := ing.DeepCopy() - ni.Spec.Rules[0].Host = updateDummyHost + ni.Spec.Rules[0].Host = "update-dummy" _ = ensureIngress(ni, clientSet, t) if err != nil { t.Errorf("error creating ingress: %v", err) @@ -362,9 +238,9 @@ func TestStore(t *testing.T) { // Secret takes a bit to update time.Sleep(3 * time.Second) - err = clientSet.NetworkingV1().Ingresses(ni.Namespace).Delete(context.TODO(), ni.Name, metav1.DeleteOptions{}) + err = clientSet.NetworkingV1beta1().Ingresses(ni.Namespace).Delete(context.TODO(), ni.Name, metav1.DeleteOptions{}) if err != nil { - t.Errorf("error deleting ingress: %v", err) + t.Errorf("error creating ingress: %v", err) } err = framework.WaitForNoIngressInNamespace(clientSet, ni.Namespace, ni.Name) @@ -384,7 +260,7 @@ func TestStore(t *testing.T) { } }) - t.Run("should return two events for add and delete and one for update of ingress and watch-without-class", func(t *testing.T) { + t.Run("should not receive updates for ingress with invalid class", func(t *testing.T) { ns := createNamespace(clientSet, t) defer deleteNamespace(ns, clientSet, t) createConfigMap(clientSet, ns, t) @@ -403,254 +279,7 @@ func TestStore(t *testing.T) { return } - e, ok := evt.(Event) - if !ok { - return - } - if e.Obj == nil { - continue - } - if _, ok := e.Obj.(*networking.Ingress); !ok { - continue - } - - switch e.Type { - case CreateEvent: - atomic.AddUint64(&add, 1) - case UpdateEvent: - atomic.AddUint64(&upd, 1) - case DeleteEvent: - atomic.AddUint64(&del, 1) - } - } - }(updateCh) - - ingressClassconfig := &ingressclass.Configuration{ - Controller: ingressclass.DefaultControllerName, - AnnotationValue: ingressclass.DefaultAnnotationValue, - WatchWithoutClass: true, - } - - storer := New( - ns, - emptySelector, - fmt.Sprintf("%v/config", ns), - fmt.Sprintf("%v/tcp", ns), - fmt.Sprintf("%v/udp", ns), - "", - 10*time.Minute, - clientSet, - updateCh, - false, - true, - ingressClassconfig, - false) - - storer.Run(stopCh) - - validIngress1 := ensureIngress(&networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ing1", - Namespace: ns, - }, - Spec: commonIngressSpec, - }, clientSet, t) - err := framework.WaitForIngressInNamespace(clientSet, ns, validIngress1.Name) - if err != nil { - t.Errorf("error waiting for ingress: %v", err) - } - - otherIngress := commonIngressSpec - otherIngress.Rules[0].Host = "other-ingress" - validIngress2 := ensureIngress(&networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ing2", - Namespace: ns, - }, - Spec: otherIngress, - }, clientSet, t) - err = framework.WaitForIngressInNamespace(clientSet, ns, validIngress2.Name) - if err != nil { - t.Errorf("error waiting for ingress: %v", err) - } - - time.Sleep(1 * time.Second) - - validIngressUpdated := validIngress1.DeepCopy() - validIngressUpdated.Spec.Rules[0].Host = updateDummyHost - _ = ensureIngress(validIngressUpdated, clientSet, t) - if err != nil { - t.Errorf("error updating ingress: %v", err) - } - // Secret takes a bit to update - time.Sleep(3 * time.Second) - - err = clientSet.NetworkingV1().Ingresses(validIngressUpdated.Namespace).Delete(context.TODO(), validIngressUpdated.Name, metav1.DeleteOptions{}) - if err != nil { - t.Errorf("error deleting ingress: %v", err) - } - err = clientSet.NetworkingV1().Ingresses(validIngress2.Namespace).Delete(context.TODO(), validIngress2.Name, metav1.DeleteOptions{}) - if err != nil { - t.Errorf("error deleting ingress: %v", err) - } - - err = framework.WaitForNoIngressInNamespace(clientSet, validIngressUpdated.Namespace, validIngressUpdated.Name) - if err != nil { - t.Errorf("error waiting for ingress deletion: %v", err) - } - err = framework.WaitForNoIngressInNamespace(clientSet, validIngress2.Namespace, validIngress2.Name) - if err != nil { - t.Errorf("error waiting for ingress deletion: %v", err) - } - time.Sleep(1 * time.Second) - - if atomic.LoadUint64(&add) != 2 { - t.Errorf("expected 0 event of type Create but %v occurred", add) - } - if atomic.LoadUint64(&upd) != 1 { - t.Errorf("expected 0 event of type Update but %v occurred", upd) - } - if atomic.LoadUint64(&del) != 2 { - t.Errorf("expected 0 event of type Delete but %v occurred", del) - } - }) - - t.Run("should return two events for add and delete and one for update of ingress and watch-ingress-by-name", func(t *testing.T) { - ns := createNamespace(clientSet, t) - defer deleteNamespace(ns, clientSet, t) - ic := createIngressClass(clientSet, t, "not-k8s.io/by-name") - defer deleteIngressClass(ic, clientSet, t) - - createConfigMap(clientSet, ns, t) - - stopCh := make(chan struct{}) - updateCh := channels.NewRingChannel(1024) - - var add uint64 - var upd uint64 - var del uint64 - - go func(ch *channels.RingChannel) { - for { - evt, ok := <-ch.Out() - if !ok { - return - } - - e, ok := evt.(Event) - if !ok { - return - } - if e.Obj == nil { - continue - } - if _, ok := e.Obj.(*networking.Ingress); !ok { - continue - } - - switch e.Type { - case CreateEvent: - atomic.AddUint64(&add, 1) - case UpdateEvent: - atomic.AddUint64(&upd, 1) - case DeleteEvent: - atomic.AddUint64(&del, 1) - } - } - }(updateCh) - - ingressClassconfig := &ingressclass.Configuration{ - Controller: ingressclass.DefaultControllerName, - AnnotationValue: ic, - IngressClassByName: true, - } - - storer := New( - ns, - emptySelector, - fmt.Sprintf("%v/config", ns), - fmt.Sprintf("%v/tcp", ns), - fmt.Sprintf("%v/udp", ns), - "", - 10*time.Minute, - clientSet, - updateCh, - false, - true, - ingressClassconfig, - false) - - storer.Run(stopCh) - validSpec := commonIngressSpec - validSpec.IngressClassName = &ic - ing := ensureIngress(&networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ingclass-by-name", - Namespace: ns, - }, - Spec: validSpec, - }, clientSet, t) - - err := framework.WaitForIngressInNamespace(clientSet, ns, ing.Name) - if err != nil { - t.Errorf("error waiting for ingress: %v", err) - } - time.Sleep(1 * time.Second) - - ingressUpdated := ing.DeepCopy() - ingressUpdated.Spec.Rules[0].Host = updateDummyHost - _ = ensureIngress(ingressUpdated, clientSet, t) - if err != nil { - t.Errorf("error updating ingress: %v", err) - } - // Secret takes a bit to update - time.Sleep(3 * time.Second) - - err = clientSet.NetworkingV1().Ingresses(ingressUpdated.Namespace).Delete(context.TODO(), ingressUpdated.Name, metav1.DeleteOptions{}) - if err != nil { - t.Errorf("error deleting ingress: %v", err) - } - - err = framework.WaitForNoIngressInNamespace(clientSet, ingressUpdated.Namespace, ingressUpdated.Name) - if err != nil { - t.Errorf("error waiting for ingress deletion: %v", err) - } - - if atomic.LoadUint64(&add) != 1 { - t.Errorf("expected 1 event of type Create but %v occurred", add) - } - if atomic.LoadUint64(&upd) != 1 { - t.Errorf("expected 1 event of type Update but %v occurred", upd) - } - if atomic.LoadUint64(&del) != 1 { - t.Errorf("expected 1 event of type Delete but %v occurred", del) - } - }) - - t.Run("should not receive updates for ingress with invalid class annotation", func(t *testing.T) { - ns := createNamespace(clientSet, t) - defer deleteNamespace(ns, clientSet, t) - createConfigMap(clientSet, ns, t) - - stopCh := make(chan struct{}) - updateCh := channels.NewRingChannel(1024) - - var add uint64 - var upd uint64 - var del uint64 - - // TODO: This repeats a lot, transform in a local function - go func(ch *channels.RingChannel) { - for { - evt, ok := <-ch.Out() - if !ok { - return - } - - e, ok := evt.(Event) - if !ok { - return - } + e := evt.(Event) if e.Obj == nil { continue } @@ -671,7 +300,6 @@ func TestStore(t *testing.T) { storer := New( ns, - emptySelector, fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -679,9 +307,6 @@ func TestStore(t *testing.T) { 10*time.Minute, clientSet, updateCh, - false, - true, - DefaultClassConfig, false) storer.Run(stopCh) @@ -692,107 +317,29 @@ func TestStore(t *testing.T) { Name: "custom-class", Namespace: ns, Annotations: map[string]string{ - ingressclass.IngressKey: "something", + class.IngressKey: "something", }, }, - Spec: commonIngressSpec, - }, clientSet, t) - err := framework.WaitForIngressInNamespace(clientSet, ns, invalidIngress.Name) - if err != nil { - t.Errorf("error waiting for ingress: %v", err) - } - time.Sleep(1 * time.Second) - - invalidIngressUpdated := invalidIngress.DeepCopy() - invalidIngressUpdated.Spec.Rules[0].Host = updateDummyHost - _ = ensureIngress(invalidIngressUpdated, clientSet, t) - if err != nil { - t.Errorf("error creating ingress: %v", err) - } - // Secret takes a bit to update - time.Sleep(3 * time.Second) - - if atomic.LoadUint64(&add) != 0 { - t.Errorf("expected 0 event of type Create but %v occurred", add) - } - if atomic.LoadUint64(&upd) != 0 { - t.Errorf("expected 0 event of type Update but %v occurred", upd) - } - if atomic.LoadUint64(&del) != 0 { - t.Errorf("expected 0 event of type Delete but %v occurred", del) - } - }) - - t.Run("should not receive updates for ingress with invalid class specification", func(t *testing.T) { - ns := createNamespace(clientSet, t) - defer deleteNamespace(ns, clientSet, t) - ic := createIngressClass(clientSet, t, ingressclass.DefaultControllerName) - defer deleteIngressClass(ic, clientSet, t) - - createConfigMap(clientSet, ns, t) - - stopCh := make(chan struct{}) - updateCh := channels.NewRingChannel(1024) - - var add uint64 - var upd uint64 - var del uint64 - - go func(ch *channels.RingChannel) { - for { - evt, ok := <-ch.Out() - if !ok { - return - } - - e, ok := evt.(Event) - if !ok { - return - } - if e.Obj == nil { - continue - } - if _, ok := e.Obj.(*networking.Ingress); !ok { - continue - } - - switch e.Type { - case CreateEvent: - atomic.AddUint64(&add, 1) - case UpdateEvent: - atomic.AddUint64(&upd, 1) - case DeleteEvent: - atomic.AddUint64(&del, 1) - } - } - }(updateCh) - - storer := New( - ns, - emptySelector, - fmt.Sprintf("%v/config", ns), - fmt.Sprintf("%v/tcp", ns), - fmt.Sprintf("%v/udp", ns), - "", - 10*time.Minute, - clientSet, - updateCh, - false, - true, - DefaultClassConfig, - false) - - storer.Run(stopCh) - invalidSpec := commonIngressSpec - invalidClassName := "blo123" - invalidSpec.IngressClassName = &invalidClassName - // create an invalid ingress (different class) - invalidIngress := ensureIngress(&networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "custom-class", - Namespace: ns, + Spec: networking.IngressSpec{ + Rules: []networking.IngressRule{ + { + Host: "dummy", + IngressRuleValue: networking.IngressRuleValue{ + HTTP: &networking.HTTPIngressRuleValue{ + Paths: []networking.HTTPIngressPath{ + { + Path: "/", + Backend: networking.IngressBackend{ + ServiceName: "http-svc", + ServicePort: intstr.FromInt(80), + }, + }, + }, + }, + }, + }, + }, }, - Spec: invalidSpec, }, clientSet, t) err := framework.WaitForIngressInNamespace(clientSet, ns, invalidIngress.Name) if err != nil { @@ -801,7 +348,7 @@ func TestStore(t *testing.T) { time.Sleep(1 * time.Second) invalidIngressUpdated := invalidIngress.DeepCopy() - invalidIngressUpdated.Spec.Rules[0].Host = updateDummyHost + invalidIngressUpdated.Spec.Rules[0].Host = "update-dummy" _ = ensureIngress(invalidIngressUpdated, clientSet, t) if err != nil { t.Errorf("error creating ingress: %v", err) @@ -839,10 +386,7 @@ func TestStore(t *testing.T) { return } - e, ok := evt.(Event) - if !ok { - return - } + e := evt.(Event) if e.Obj == nil { continue } @@ -859,7 +403,6 @@ func TestStore(t *testing.T) { storer := New( ns, - emptySelector, fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -867,9 +410,6 @@ func TestStore(t *testing.T) { 10*time.Minute, clientSet, updateCh, - false, - true, - DefaultClassConfig, false) storer.Run(stopCh) @@ -916,8 +456,6 @@ func TestStore(t *testing.T) { t.Run("should receive events from secret referenced from ingress", func(t *testing.T) { ns := createNamespace(clientSet, t) defer deleteNamespace(ns, clientSet, t) - ic := createIngressClass(clientSet, t, ingressclass.DefaultControllerName) - defer deleteIngressClass(ic, clientSet, t) createConfigMap(clientSet, ns, t) stopCh := make(chan struct{}) @@ -934,18 +472,10 @@ func TestStore(t *testing.T) { return } - e, ok := evt.(Event) - if !ok { - return - } + e := evt.(Event) if e.Obj == nil { continue } - - // We should skip IngressClass events - if _, ok := e.Obj.(*networking.IngressClass); ok { - continue - } switch e.Type { case CreateEvent: atomic.AddUint64(&add, 1) @@ -959,7 +489,6 @@ func TestStore(t *testing.T) { storer := New( ns, - emptySelector, fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -967,9 +496,6 @@ func TestStore(t *testing.T) { 10*time.Minute, clientSet, updateCh, - false, - true, - DefaultClassConfig, false) storer.Run(stopCh) @@ -983,19 +509,14 @@ func TestStore(t *testing.T) { Namespace: ns, }, Spec: networking.IngressSpec{ - IngressClassName: &ic, TLS: []networking.IngressTLS{ { SecretName: secretName, }, }, - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "http-svc", + ServicePort: intstr.FromInt(80), }, }, }, clientSet, t) @@ -1037,13 +558,12 @@ func TestStore(t *testing.T) { if atomic.LoadUint64(&del) != 1 { t.Errorf("expected 1 events of type Delete but %v occurred", del) } + }) t.Run("should create an ingress with a secret which does not exist", func(t *testing.T) { ns := createNamespace(clientSet, t) defer deleteNamespace(ns, clientSet, t) - ic := createIngressClass(clientSet, t, ingressclass.DefaultControllerName) - defer deleteIngressClass(ic, clientSet, t) createConfigMap(clientSet, ns, t) stopCh := make(chan struct{}) @@ -1060,19 +580,10 @@ func TestStore(t *testing.T) { return } - e, ok := evt.(Event) - if !ok { - return - } + e := evt.(Event) if e.Obj == nil { continue } - - // We should skip IngressClass objects here - if _, ok := e.Obj.(*networking.IngressClass); ok { - continue - } - switch e.Type { case CreateEvent: atomic.AddUint64(&add, 1) @@ -1086,7 +597,6 @@ func TestStore(t *testing.T) { storer := New( ns, - emptySelector, fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -1094,9 +604,6 @@ func TestStore(t *testing.T) { 10*time.Minute, clientSet, updateCh, - false, - true, - DefaultClassConfig, false) storer.Run(stopCh) @@ -1110,7 +617,6 @@ func TestStore(t *testing.T) { Namespace: ns, }, Spec: networking.IngressSpec{ - IngressClassName: &ic, TLS: []networking.IngressTLS{ { Hosts: secretHosts, @@ -1124,15 +630,10 @@ func TestStore(t *testing.T) { HTTP: &networking.HTTPIngressRuleValue{ Paths: []networking.HTTPIngressPath{ { - Path: "/", - PathType: &pathPrefix, + Path: "/", Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "http-svc", + ServicePort: intstr.FromInt(80), }, }, }, @@ -1171,110 +672,7 @@ func TestStore(t *testing.T) { } }) - t.Run("should not receive events whose namespace doesn't match watch namespace selector", func(t *testing.T) { - ns := createNamespace(clientSet, t) - defer deleteNamespace(ns, clientSet, t) - createConfigMap(clientSet, ns, t) - - stopCh := make(chan struct{}) - updateCh := channels.NewRingChannel(1024) - - var add uint64 - var upd uint64 - var del uint64 - - go func(ch *channels.RingChannel) { - for { - evt, ok := <-ch.Out() - if !ok { - return - } - - e, ok := evt.(Event) - if !ok { - return - } - if e.Obj == nil { - continue - } - switch e.Type { - case CreateEvent: - atomic.AddUint64(&add, 1) - case UpdateEvent: - atomic.AddUint64(&upd, 1) - case DeleteEvent: - atomic.AddUint64(&del, 1) - } - } - }(updateCh) - - namespaceSelector, err := labels.Parse("foo=bar") - if err != nil { - t.Errorf("unexpected error: %v", err) - } - storer := New( - ns, - namespaceSelector, - fmt.Sprintf("%v/config", ns), - fmt.Sprintf("%v/tcp", ns), - fmt.Sprintf("%v/udp", ns), - "", - 10*time.Minute, - clientSet, - updateCh, - false, - true, - DefaultClassConfig, - false) - - storer.Run(stopCh) - - ing := ensureIngress(&networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "dummy", - Namespace: ns, - }, - Spec: networking.IngressSpec{ - Rules: []networking.IngressRule{ - { - Host: "dummy", - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - Path: "/", - PathType: &pathPrefix, - Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "http-svc", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, clientSet, t) - defer deleteIngress(ing, clientSet, t) - - time.Sleep(1 * time.Second) - - if atomic.LoadUint64(&add) != 0 { - t.Errorf("expected 0 events of type Create but %v occurred", add) - } - if atomic.LoadUint64(&upd) != 0 { - t.Errorf("expected 0 events of type Update but %v occurred", upd) - } - if atomic.LoadUint64(&del) != 0 { - t.Errorf("expected 0 events of type Delete but %v occurred", del) - } - }) - // test add ingress with secret it doesn't exist and then add secret + // test add ingress with secret it doesn't exists and then add secret // check secret is generated on fs // check ocsp // check invalid secret (missing crt) @@ -1308,34 +706,7 @@ func deleteNamespace(ns string, clientSet kubernetes.Interface, t *testing.T) { } } -func createIngressClass(clientSet kubernetes.Interface, t *testing.T, controller string) string { - t.Helper() - class := &networking.IngressClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("ingress-nginx-%v", time.Now().Unix()), - // Namespace: "xpto" // TODO: We don't support namespaced ingress-class yet - }, - Spec: networking.IngressClassSpec{ - Controller: controller, - }, - } - ic, err := clientSet.NetworkingV1().IngressClasses().Create(context.TODO(), class, metav1.CreateOptions{}) - if err != nil { - t.Errorf("error creating ingress class: %v", err) - } - return ic.Name -} - -func deleteIngressClass(ic string, clientSet kubernetes.Interface, t *testing.T) { - t.Helper() - - err := clientSet.NetworkingV1().IngressClasses().Delete(context.TODO(), ic, metav1.DeleteOptions{}) - if err != nil { - t.Errorf("error deleting the ingress class: %v", err) - } -} - -func createConfigMap(clientSet kubernetes.Interface, ns string, t *testing.T) { +func createConfigMap(clientSet kubernetes.Interface, ns string, t *testing.T) string { t.Helper() configMap := &v1.ConfigMap{ @@ -1344,51 +715,54 @@ func createConfigMap(clientSet kubernetes.Interface, ns string, t *testing.T) { }, } - _, err := clientSet.CoreV1().ConfigMaps(ns).Create(context.TODO(), configMap, metav1.CreateOptions{}) + cm, err := clientSet.CoreV1().ConfigMaps(ns).Create(context.TODO(), configMap, metav1.CreateOptions{}) if err != nil { t.Errorf("error creating the configuration map: %v", err) } + + return cm.Name } -func ensureIngress(ing *networking.Ingress, clientSet kubernetes.Interface, t *testing.T) *networking.Ingress { +func ensureIngress(ingress *networking.Ingress, clientSet kubernetes.Interface, t *testing.T) *networking.Ingress { t.Helper() - newIngress, err := clientSet.NetworkingV1().Ingresses(ing.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{}) + ing, err := clientSet.NetworkingV1beta1().Ingresses(ingress.Namespace).Update(context.TODO(), ingress, metav1.UpdateOptions{}) + if err != nil { if k8sErrors.IsNotFound(err) { - t.Logf("Ingress %v not found, creating", ing) + t.Logf("Ingress %v not found, creating", ingress) - newIngress, err = clientSet.NetworkingV1().Ingresses(ing.Namespace).Create(context.TODO(), ing, metav1.CreateOptions{}) + ing, err = clientSet.NetworkingV1beta1().Ingresses(ingress.Namespace).Create(context.TODO(), ingress, metav1.CreateOptions{}) if err != nil { - t.Fatalf("error creating ingress %+v: %v", ing, err) + t.Fatalf("error creating ingress %+v: %v", ingress, err) } - t.Logf("Ingress %+v created", ing) - return newIngress + t.Logf("Ingress %+v created", ingress) + return ing } - t.Fatalf("error updating ingress %+v: %v", ing, err) + t.Fatalf("error updating ingress %+v: %v", ingress, err) } - return newIngress + return ing } -func deleteIngress(ing *networking.Ingress, clientSet kubernetes.Interface, t *testing.T) { +func deleteIngress(ingress *networking.Ingress, clientSet kubernetes.Interface, t *testing.T) { t.Helper() - err := clientSet.NetworkingV1().Ingresses(ing.Namespace).Delete(context.TODO(), ing.Name, metav1.DeleteOptions{}) + err := clientSet.NetworkingV1beta1().Ingresses(ingress.Namespace).Delete(context.TODO(), ingress.Name, metav1.DeleteOptions{}) + if err != nil { - t.Errorf("failed to delete ingress %+v: %v", ing, err) + t.Errorf("failed to delete ingress %+v: %v", ingress, err) } - t.Logf("Ingress %+v deleted", ing) + t.Logf("Ingress %+v deleted", ingress) } // newStore creates a new mock object store for tests which do not require the // use of Informers. -func newStore() *k8sStore { +func newStore(t *testing.T) *k8sStore { return &k8sStore{ listers: &Lister{ // add more listers if needed - IngressClass: IngressClassLister{cache.NewStore(cache.MetaNamespaceKeyFunc)}, Ingress: IngressLister{cache.NewStore(cache.MetaNamespaceKeyFunc)}, IngressWithAnnotation: IngressWithAnnotationsLister{cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc)}, }, @@ -1401,7 +775,7 @@ func newStore() *k8sStore { } func TestUpdateSecretIngressMap(t *testing.T) { - s := newStore() + s := newStore(t) ingTpl := &networking.Ingress{ ObjectMeta: metav1.ObjectMeta{ @@ -1409,18 +783,14 @@ func TestUpdateSecretIngressMap(t *testing.T) { Namespace: "testns", }, } - if err := s.listers.Ingress.Add(ingTpl); err != nil { - t.Errorf("error adding the Ingress template: %v", err) - } + s.listers.Ingress.Add(ingTpl) t.Run("with TLS secret", func(t *testing.T) { ing := ingTpl.DeepCopy() ing.Spec = networking.IngressSpec{ TLS: []networking.IngressTLS{{SecretName: "tls"}}, } - if err := s.listers.Ingress.Update(ing); err != nil { - t.Errorf("error updating the Ingress: %v", err) - } + s.listers.Ingress.Update(ing) s.updateSecretIngressMap(ing) if l := s.secretIngressMap.Len(); !(l == 1 && s.secretIngressMap.Has("testns/tls")) { @@ -1433,9 +803,7 @@ func TestUpdateSecretIngressMap(t *testing.T) { ing.ObjectMeta.SetAnnotations(map[string]string{ parser.GetAnnotationWithPrefix("auth-secret"): "auth", }) - if err := s.listers.Ingress.Update(ing); err != nil { - t.Errorf("error updating the Ingress: %v", err) - } + s.listers.Ingress.Update(ing) s.updateSecretIngressMap(ing) if l := s.secretIngressMap.Len(); !(l == 1 && s.secretIngressMap.Has("testns/auth")) { @@ -1446,41 +814,22 @@ func TestUpdateSecretIngressMap(t *testing.T) { t.Run("with annotation in namespace/name format", func(t *testing.T) { ing := ingTpl.DeepCopy() ing.ObjectMeta.SetAnnotations(map[string]string{ - parser.GetAnnotationWithPrefix("auth-secret"): "testns/auth", + parser.GetAnnotationWithPrefix("auth-secret"): "otherns/auth", }) - if err := s.listers.Ingress.Update(ing); err != nil { - t.Errorf("error updating the Ingress: %v", err) - } + s.listers.Ingress.Update(ing) s.updateSecretIngressMap(ing) - if l := s.secretIngressMap.Len(); !(l == 1 && s.secretIngressMap.Has("testns/auth")) { + if l := s.secretIngressMap.Len(); !(l == 1 && s.secretIngressMap.Has("otherns/auth")) { t.Errorf("Expected \"otherns/auth\" to be the only referenced Secret (got %d)", l) } }) - t.Run("with annotation in namespace/name format should not be supported", func(t *testing.T) { - ing := ingTpl.DeepCopy() - ing.ObjectMeta.SetAnnotations(map[string]string{ - parser.GetAnnotationWithPrefix("auth-secret"): "anotherns/auth", - }) - if err := s.listers.Ingress.Update(ing); err != nil { - t.Errorf("error updating the Ingress: %v", err) - } - s.updateSecretIngressMap(ing) - - if l := s.secretIngressMap.Len(); l != 0 { - t.Errorf("Expected \"otherns/auth\" to be denied as it contains a different namespace (got %d)", l) - } - }) - t.Run("with annotation in invalid format", func(t *testing.T) { ing := ingTpl.DeepCopy() ing.ObjectMeta.SetAnnotations(map[string]string{ parser.GetAnnotationWithPrefix("auth-secret"): "ns/name/garbage", }) - if err := s.listers.Ingress.Update(ing); err != nil { - t.Errorf("error updating the Ingress: %v", err) - } + s.listers.Ingress.Update(ing) s.updateSecretIngressMap(ing) if l := s.secretIngressMap.Len(); l != 0 { @@ -1490,33 +839,27 @@ func TestUpdateSecretIngressMap(t *testing.T) { } func TestListIngresses(t *testing.T) { - s := newStore() - invalidIngressClass := "something" - validIngressClass := ingressclass.DefaultControllerName + s := newStore(t) ingressToIgnore := &ingress.Ingress{ Ingress: networking.Ingress{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-2", - Namespace: "testns", + Name: "test-2", + Namespace: "testns", + Annotations: map[string]string{ + class.IngressKey: "something", + }, CreationTimestamp: metav1.NewTime(time.Now()), }, Spec: networking.IngressSpec{ - IngressClassName: &invalidIngressClass, - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "demo", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: "demo", + ServicePort: intstr.FromInt(80), }, }, }, } - if err := s.listers.IngressWithAnnotation.Add(ingressToIgnore); err != nil { - t.Errorf("error adding the Ingress: %v", err) - } + s.listers.IngressWithAnnotation.Add(ingressToIgnore) ingressWithoutPath := &ingress.Ingress{ Ingress: networking.Ingress{ @@ -1526,7 +869,6 @@ func TestListIngresses(t *testing.T) { CreationTimestamp: metav1.NewTime(time.Now()), }, Spec: networking.IngressSpec{ - IngressClassName: &validIngressClass, Rules: []networking.IngressRule{ { Host: "foo.bar", @@ -1535,12 +877,8 @@ func TestListIngresses(t *testing.T) { Paths: []networking.HTTPIngressPath{ { Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "demo", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "demo", + ServicePort: intstr.FromInt(80), }, }, }, @@ -1551,16 +889,15 @@ func TestListIngresses(t *testing.T) { }, }, } - if err := s.listers.IngressWithAnnotation.Add(ingressWithoutPath); err != nil { - t.Errorf("error adding the Ingress: %v", err) - } - ingressWithNginxClassAnnotation := &ingress.Ingress{ + s.listers.IngressWithAnnotation.Add(ingressWithoutPath) + + ingressWithNginxClass := &ingress.Ingress{ Ingress: networking.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: "test-4", Namespace: "testns", Annotations: map[string]string{ - ingressclass.IngressKey: ingressclass.DefaultAnnotationValue, + class.IngressKey: "nginx", }, CreationTimestamp: metav1.NewTime(time.Now()), }, @@ -1572,15 +909,10 @@ func TestListIngresses(t *testing.T) { HTTP: &networking.HTTPIngressRuleValue{ Paths: []networking.HTTPIngressPath{ { - Path: "/demo", - PathType: &pathPrefix, + Path: "/demo", Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "demo", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "demo", + ServicePort: intstr.FromInt(80), }, }, }, @@ -1591,9 +923,8 @@ func TestListIngresses(t *testing.T) { }, }, } - if err := s.listers.IngressWithAnnotation.Add(ingressWithNginxClassAnnotation); err != nil { - t.Errorf("error adding the Ingress: %v", err) - } + s.listers.IngressWithAnnotation.Add(ingressWithNginxClass) + ingresses := s.ListIngresses() if s := len(ingresses); s != 3 { @@ -1620,7 +951,7 @@ func TestWriteSSLSessionTicketKey(t *testing.T) { } for _, test := range tests { - s := newStore() + s := newStore(t) cmap := &v1.ConfigMap{ Data: map[string]string{ @@ -1628,14 +959,14 @@ func TestWriteSSLSessionTicketKey(t *testing.T) { }, } - f, err := os.CreateTemp("", "ssl-session-ticket-test") + f, err := ioutil.TempFile("", "ssl-session-ticket-test") if err != nil { t.Fatal(err) } s.writeSSLSessionTicketKey(cmap, f.Name()) - content, err := os.ReadFile(f.Name()) + content, err := ioutil.ReadFile(f.Name()) if err != nil { t.Fatal(err) } diff --git a/pkg/tcpproxy/tcp.go b/internal/ingress/controller/tcp.go similarity index 78% rename from pkg/tcpproxy/tcp.go rename to internal/ingress/controller/tcp.go index fba4d21be..a97c46a7e 100644 --- a/pkg/tcpproxy/tcp.go +++ b/internal/ingress/controller/tcp.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package tcpproxy +package controller import ( "fmt" @@ -59,17 +59,16 @@ func (p *TCPProxy) Get(host string) *TCPServer { // and open a connection to the passthrough server. func (p *TCPProxy) Handle(conn net.Conn) { defer conn.Close() - // See: https://www.ibm.com/docs/en/ztpf/1.1.0.15?topic=sessions-ssl-record-format - data := make([]byte, 16384) + data := make([]byte, 4096) length, err := conn.Read(data) if err != nil { - klog.V(4).ErrorS(err, "Error reading data from the connection") + klog.V(4).ErrorS(err, "Error reading the first 4k of the connection") return } proxy := p.Default - hostname, err := parser.GetHostname(data) + hostname, err := parser.GetHostname(data[:]) if err == nil { klog.V(4).InfoS("TLS Client Hello", "host", hostname) proxy = p.Get(hostname) @@ -81,24 +80,16 @@ func (p *TCPProxy) Handle(conn net.Conn) { } hostPort := net.JoinHostPort(proxy.IP, fmt.Sprintf("%v", proxy.Port)) - klog.V(4).InfoS("passing to", "hostport", hostPort) clientConn, err := net.Dial("tcp", hostPort) if err != nil { - klog.V(4).ErrorS(err, "error dialing proxy", "ip", proxy.IP, "port", proxy.Port, "hostname", proxy.Hostname) return } defer clientConn.Close() if proxy.ProxyProtocol { // write out the Proxy Protocol header - localAddr, ok := conn.LocalAddr().(*net.TCPAddr) - if !ok { - klog.Errorf("unexpected type: %T", conn.LocalAddr()) - } - remoteAddr, ok := conn.RemoteAddr().(*net.TCPAddr) - if !ok { - klog.Errorf("unexpected type: %T", conn.RemoteAddr()) - } + localAddr := conn.LocalAddr().(*net.TCPAddr) + remoteAddr := conn.RemoteAddr().(*net.TCPAddr) protocol := "UNKNOWN" if remoteAddr.IP.To4() != nil { protocol = "TCP4" @@ -107,7 +98,7 @@ func (p *TCPProxy) Handle(conn net.Conn) { } proxyProtocolHeader := fmt.Sprintf("PROXY %s %s %s %d %d\r\n", protocol, remoteAddr.IP.String(), localAddr.IP.String(), remoteAddr.Port, localAddr.Port) klog.V(4).InfoS("Writing Proxy Protocol", "header", proxyProtocolHeader) - _, err = fmt.Fprint(clientConn, proxyProtocolHeader) + _, err = fmt.Fprintf(clientConn, proxyProtocolHeader) } if err != nil { klog.ErrorS(err, "Error writing Proxy Protocol header") @@ -125,7 +116,6 @@ func (p *TCPProxy) Handle(conn net.Conn) { func pipe(client, server net.Conn) { doCopy := func(s, c net.Conn, cancel chan<- bool) { - //nolint:errcheck // No need to catch these errors io.Copy(s, c) cancel <- true } @@ -135,5 +125,8 @@ func pipe(client, server net.Conn) { go doCopy(server, client, cancel) go doCopy(client, server, cancel) - <-cancel + select { + case <-cancel: + return + } } diff --git a/internal/ingress/controller/template/configmap.go b/internal/ingress/controller/template/configmap.go index febf20be0..93a6b430d 100644 --- a/internal/ingress/controller/template/configmap.go +++ b/internal/ingress/controller/template/configmap.go @@ -19,30 +19,27 @@ package template import ( "fmt" "net" - "regexp" "strconv" "strings" "time" "k8s.io/klog/v2" - "github.com/mitchellh/hashstructure/v2" + "github.com/mitchellh/hashstructure" "github.com/mitchellh/mapstructure" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/ingress-nginx/internal/ingress/annotations/authreq" - "k8s.io/ingress-nginx/internal/ingress/annotations/customheaders" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/controller/config" ing_net "k8s.io/ingress-nginx/internal/net" - "k8s.io/ingress-nginx/pkg/util/runtime" + "k8s.io/ingress-nginx/internal/runtime" ) const ( customHTTPErrors = "custom-http-errors" skipAccessLogUrls = "skip-access-log-urls" whitelistSourceRange = "whitelist-source-range" - denylistSourceRange = "denylist-source-range" proxyRealIPCIDR = "proxy-real-ip-cidr" bindAddress = "bind-address" httpRedirectCode = "http-redirect-code" @@ -55,7 +52,6 @@ const ( nginxStatusIpv6Whitelist = "nginx-status-ipv6-whitelist" proxyHeaderTimeout = "proxy-protocol-header-timeout" workerProcesses = "worker-processes" - globalAllowedResponseHeaders = "global-allowed-response-headers" globalAuthURL = "global-auth-url" globalAuthMethod = "global-auth-method" globalAuthSignin = "global-auth-signin" @@ -65,35 +61,31 @@ const ( globalAuthSnippet = "global-auth-snippet" globalAuthCacheKey = "global-auth-cache-key" globalAuthCacheDuration = "global-auth-cache-duration" - globalAuthAlwaysSetCookie = "global-auth-always-set-cookie" luaSharedDictsKey = "lua-shared-dicts" - debugConnections = "debug-connections" - workerSerialReloads = "enable-serial-reloads" + plugins = "plugins" ) var ( validRedirectCodes = sets.NewInt([]int{301, 302, 307, 308}...) - dictSizeRegex = regexp.MustCompile(`^(\d+)([kKmM])?$`) defaultLuaSharedDicts = map[string]int{ - "configuration_data": 20480, - "certificate_data": 20480, - "balancer_ewma": 10240, - "balancer_ewma_last_touched_at": 10240, - "balancer_ewma_locks": 1024, - "certificate_servers": 5120, - "ocsp_response_cache": 5120, // keep this same as certificate_servers + "configuration_data": 20, + "certificate_data": 20, + "balancer_ewma": 10, + "balancer_ewma_last_touched_at": 10, + "balancer_ewma_locks": 1, + "certificate_servers": 5, + "ocsp_response_cache": 5, // keep this same as certificate_servers + "global_throttle_cache": 10, } defaultGlobalAuthRedirectParam = "rd" ) const ( - maxAllowedLuaDictSize = 204800 + maxAllowedLuaDictSize = 200 maxNumberOfLuaDicts = 100 ) // ReadConfig obtains the configuration defined by the user merged with the defaults. -// -//nolint:gocyclo // Ignore function complexity error func ReadConfig(src map[string]string) config.Configuration { conf := map[string]string{} // we need to copy the configmap data because the content is altered @@ -104,7 +96,6 @@ func ReadConfig(src map[string]string) config.Configuration { to := config.NewDefault() errors := make([]int, 0) skipUrls := make([]string, 0) - denyList := make([]string, 0) whiteList := make([]string, 0) proxyList := make([]string, 0) hideHeadersList := make([]string, 0) @@ -116,33 +107,28 @@ func ReadConfig(src map[string]string) config.Configuration { blockUserAgentList := make([]string, 0) blockRefererList := make([]string, 0) responseHeaders := make([]string, 0) - allowedResponseHeaders := make([]string, 0) luaSharedDicts := make(map[string]int) - debugConnectionsList := make([]string, 0) - // parse lua shared dict values + //parse lua shared dict values if val, ok := conf[luaSharedDictsKey]; ok { delete(conf, luaSharedDictsKey) lsd := splitAndTrimSpace(val, ",") for _, v := range lsd { - results := strings.SplitN(strings.ReplaceAll(v, " ", ""), ":", 2) - if len(results) != 2 { - klog.Errorf("Ignoring poorly formatted Lua dictionary %v", v) - continue - } + v = strings.Replace(v, " ", "", -1) + results := strings.SplitN(v, ":", 2) dictName := results[0] - size := dictStrToKb(results[1]) - if size < 0 { - klog.Errorf("Ignoring poorly formatted value %v for Lua dictionary %v", results[1], dictName) + size, err := strconv.Atoi(results[1]) + if err != nil { + klog.Errorf("Ignoring non integer value %v for Lua dictionary %v: %v.", results[1], dictName, err) continue } if size > maxAllowedLuaDictSize { - klog.Errorf("Ignoring %v for Lua dictionary %v: maximum size is %vk.", results[1], dictName, maxAllowedLuaDictSize) + klog.Errorf("Ignoring %v for Lua dictionary %v: maximum size is %v.", size, dictName, maxAllowedLuaDictSize) continue } if len(luaSharedDicts)+1 > maxNumberOfLuaDicts { klog.Errorf("Ignoring %v for Lua dictionary %v: can not configure more than %v dictionaries.", - results[1], dictName, maxNumberOfLuaDicts) + size, dictName, maxNumberOfLuaDicts) continue } @@ -178,11 +164,6 @@ func ReadConfig(src map[string]string) config.Configuration { skipUrls = splitAndTrimSpace(val, ",") } - if val, ok := conf[denylistSourceRange]; ok { - delete(conf, denylistSourceRange) - denyList = append(denyList, splitAndTrimSpace(val, ",")...) - } - if val, ok := conf[whitelistSourceRange]; ok { delete(conf, whitelistSourceRange) whiteList = append(whiteList, splitAndTrimSpace(val, ",")...) @@ -203,7 +184,7 @@ func ReadConfig(src map[string]string) config.Configuration { if ing_net.IsIPV6(ns) { bindAddressIpv6List = append(bindAddressIpv6List, fmt.Sprintf("[%v]", ns)) } else { - bindAddressIpv4List = append(bindAddressIpv4List, ns.String()) + bindAddressIpv4List = append(bindAddressIpv4List, fmt.Sprintf("%v", ns)) } } else { klog.Warningf("%v is not a valid textual representation of an IP address", i) @@ -253,27 +234,11 @@ func ReadConfig(src map[string]string) config.Configuration { } } - // Verify that the configured global external authorization response headers are valid. if not, set the default value - if val, ok := conf[globalAllowedResponseHeaders]; ok { - delete(conf, globalAllowedResponseHeaders) - - if val != "" { - harr := splitAndTrimSpace(val, ",") - for _, header := range harr { - if !customheaders.ValidHeader(header) { - klog.Warningf("Global allowed response headers denied - %s.", header) - } else { - allowedResponseHeaders = append(allowedResponseHeaders, header) - } - } - } - } - // Verify that the configured global external authorization method is a valid HTTP method. if not, set the default value if val, ok := conf[globalAuthMethod]; ok { delete(conf, globalAuthMethod) - if val != "" && !authreq.ValidMethod(val) { + if len(val) != 0 && !authreq.ValidMethod(val) { klog.Warningf("Global auth location denied - %v.", "invalid HTTP method") } else { to.GlobalExternalAuth.Method = val @@ -284,10 +249,7 @@ func ReadConfig(src map[string]string) config.Configuration { if val, ok := conf[globalAuthSignin]; ok { delete(conf, globalAuthSignin) - signinURL, err := parser.StringToURL(val) - if err != nil { - klog.Errorf("string to URL conversion failed: %v", err) - } + signinURL, _ := parser.StringToURL(val) if signinURL == nil { klog.Warningf("Global auth location denied - %v.", "global-auth-signin setting is undefined and will not be set") } else { @@ -300,10 +262,7 @@ func ReadConfig(src map[string]string) config.Configuration { delete(conf, globalAuthSigninRedirectParam) redirectParam := strings.TrimSpace(val) - dummySigninURL, err := parser.StringToURL(fmt.Sprintf("%s?%s=dummy", to.GlobalExternalAuth.SigninURL, redirectParam)) - if err != nil { - klog.Errorf("string to URL conversion failed: %v", err) - } + dummySigninURL, _ := parser.StringToURL(fmt.Sprintf("%s?%s=dummy", to.GlobalExternalAuth.SigninURL, redirectParam)) if dummySigninURL == nil { klog.Warningf("Global auth redirect parameter denied - %v.", "global-auth-signin-redirect-param setting is invalid and will not be set") } else { @@ -315,7 +274,7 @@ func ReadConfig(src map[string]string) config.Configuration { if val, ok := conf[globalAuthResponseHeaders]; ok { delete(conf, globalAuthResponseHeaders) - if val != "" { + if len(val) != 0 { harr := splitAndTrimSpace(val, ",") for _, header := range harr { if !authreq.ValidHeader(header) { @@ -354,16 +313,6 @@ func ReadConfig(src map[string]string) config.Configuration { to.GlobalExternalAuth.AuthCacheDuration = cacheDurations } - if val, ok := conf[globalAuthAlwaysSetCookie]; ok { - delete(conf, globalAuthAlwaysSetCookie) - - alwaysSetCookie, err := strconv.ParseBool(val) - if err != nil { - klog.Warningf("Global auth location denied - %s", fmt.Errorf("cannot convert %s to bool: %v", globalAuthAlwaysSetCookie, err)) - } - to.GlobalExternalAuth.AlwaysSetCookie = alwaysSetCookie - } - // Verify that the configured timeout is parsable as a duration. if not, set the default value if val, ok := conf[proxyHeaderTimeout]; ok { delete(conf, proxyHeaderTimeout) @@ -406,38 +355,13 @@ func ReadConfig(src map[string]string) config.Configuration { delete(conf, workerProcesses) } - if val, ok := conf[workerSerialReloads]; ok { - boolVal, err := strconv.ParseBool(val) - if err != nil { - to.WorkerSerialReloads = false - klog.Warningf("failed to parse enable-serial-reloads setting, valid values are true or false, found %s", val) - } else { - to.WorkerSerialReloads = boolVal - } - delete(conf, workerSerialReloads) - } - - if val, ok := conf[debugConnections]; ok { - delete(conf, debugConnections) - for _, i := range splitAndTrimSpace(val, ",") { - validIP := net.ParseIP(i) - if validIP != nil { - debugConnectionsList = append(debugConnectionsList, i) - } else { - _, _, err := net.ParseCIDR(i) - if err == nil { - debugConnectionsList = append(debugConnectionsList, i) - } else { - klog.Warningf("%v is not a valid IP or CIDR address", i) - } - } - } - to.DebugConnections = debugConnectionsList + if val, ok := conf[plugins]; ok { + to.Plugins = splitAndTrimSpace(val, ",") + delete(conf, plugins) } to.CustomHTTPErrors = filterErrors(errors) to.SkipAccessLogURLs = skipUrls - to.DenylistSourceRange = denyList to.WhitelistSourceRange = whiteList to.ProxyRealIPCIDR = proxyList to.BindAddressIpv4 = bindAddressIpv4List @@ -449,16 +373,15 @@ func ReadConfig(src map[string]string) config.Configuration { to.ProxyStreamResponses = streamResponses to.DisableIpv6DNS = !ing_net.IsIPv6Enabled() to.LuaSharedDicts = luaSharedDicts - to.Backend.AllowedResponseHeaders = allowedResponseHeaders - decoderConfig := &mapstructure.DecoderConfig{ + config := &mapstructure.DecoderConfig{ Metadata: nil, WeaklyTypedInput: true, Result: &to, TagName: "json", } - decoder, err := mapstructure.NewDecoder(decoderConfig) + decoder, err := mapstructure.NewDecoder(config) if err != nil { klog.Warningf("unexpected error merging defaults: %v", err) } @@ -467,7 +390,7 @@ func ReadConfig(src map[string]string) config.Configuration { klog.Warningf("unexpected error merging defaults: %v", err) } - hash, err := hashstructure.Hash(to, hashstructure.FormatV1, &hashstructure.HashOptions{ + hash, err := hashstructure.Hash(to, &hashstructure.HashOptions{ TagName: "json", }) if err != nil { @@ -492,7 +415,6 @@ func filterErrors(codes []int) []int { return fa } -//nolint:unparam // Ignore `sep` always receives `,` error func splitAndTrimSpace(s, sep string) []string { f := func(c rune) bool { return strings.EqualFold(string(c), sep) @@ -505,25 +427,3 @@ func splitAndTrimSpace(s, sep string) []string { return values } - -func dictStrToKb(sizeStr string) int { - sizeMatch := dictSizeRegex.FindStringSubmatch(sizeStr) - if sizeMatch == nil { - return -1 - } - size, err := strconv.Atoi(sizeMatch[1]) // validated already with regex - if err != nil { - klog.Errorf("unexpected error converting size string %s to int: %v", sizeStr, err) - } - if sizeMatch[2] == "" || strings.EqualFold(sizeMatch[2], "m") { - size *= 1024 - } - return size -} - -func dictKbToStr(size int) string { - if size%1024 == 0 { - return fmt.Sprintf("%dM", size/1024) - } - return fmt.Sprintf("%dK", size) -} diff --git a/internal/ingress/controller/template/configmap_test.go b/internal/ingress/controller/template/configmap_test.go index 6c7468303..65bb485ea 100644 --- a/internal/ingress/controller/template/configmap_test.go +++ b/internal/ingress/controller/template/configmap_test.go @@ -23,7 +23,7 @@ import ( "time" "github.com/kylelemons/godebug/pretty" - "github.com/mitchellh/hashstructure/v2" + "github.com/mitchellh/hashstructure" "k8s.io/ingress-nginx/internal/ingress/annotations/authreq" "k8s.io/ingress-nginx/internal/ingress/controller/config" @@ -64,7 +64,6 @@ func TestMergeConfigMapToStruct(t *testing.T) { "access-log-path": "/var/log/test/access.log", "error-log-path": "/var/log/test/error.log", "use-gzip": "false", - "gzip-disable": "msie6", "gzip-level": "9", "gzip-min-length": "1024", "gzip-types": "text/html", @@ -76,7 +75,6 @@ func TestMergeConfigMapToStruct(t *testing.T) { "proxy-add-original-uri-header": "false", "disable-ipv6-dns": "true", "default-type": "text/plain", - "debug-connections": "127.0.0.1,1.1.1.1/24,::1", } def := config.NewDefault() def.CustomHTTPErrors = []int{300, 400} @@ -88,7 +86,6 @@ func TestMergeConfigMapToStruct(t *testing.T) { def.ProxyReadTimeout = 1 def.ProxySendTimeout = 2 def.UseProxyProtocol = true - def.GzipDisable = "msie6" def.GzipLevel = 9 def.GzipMinLength = 1024 def.GzipTypes = "text/html" @@ -102,9 +99,8 @@ func TestMergeConfigMapToStruct(t *testing.T) { def.LuaSharedDicts = defaultLuaSharedDicts def.DisableIpv6DNS = true def.DefaultType = "text/plain" - def.DebugConnections = []string{"127.0.0.1", "1.1.1.1/24", "::1"} - hash, err := hashstructure.Hash(def, hashstructure.FormatV1, &hashstructure.HashOptions{ + hash, err := hashstructure.Hash(def, &hashstructure.HashOptions{ TagName: "json", }) if err != nil { @@ -134,7 +130,7 @@ func TestMergeConfigMapToStruct(t *testing.T) { def.LuaSharedDicts = defaultLuaSharedDicts def.DisableIpv6DNS = true - hash, err = hashstructure.Hash(def, hashstructure.FormatV1, &hashstructure.HashOptions{ + hash, err = hashstructure.Hash(def, &hashstructure.HashOptions{ TagName: "json", }) if err != nil { @@ -151,11 +147,10 @@ func TestMergeConfigMapToStruct(t *testing.T) { def = config.NewDefault() def.LuaSharedDicts = defaultLuaSharedDicts - def.DenylistSourceRange = []string{"2.2.2.2/32"} def.WhitelistSourceRange = []string{"1.1.1.1/32"} def.DisableIpv6DNS = true - hash, err = hashstructure.Hash(def, hashstructure.FormatV1, &hashstructure.HashOptions{ + hash, err = hashstructure.Hash(def, &hashstructure.HashOptions{ TagName: "json", }) if err != nil { @@ -164,7 +159,6 @@ func TestMergeConfigMapToStruct(t *testing.T) { def.Checksum = fmt.Sprintf("%v", hash) to = ReadConfig(map[string]string{ - "denylist-source-range": "2.2.2.2/32", "whitelist-source-range": "1.1.1.1/32", "disable-ipv6-dns": "true", }) @@ -235,34 +229,6 @@ func TestGlobalExternalAuthSigninParsing(t *testing.T) { } } -func TestGlobalExternalAlwaysSetCookie(t *testing.T) { - testCases := map[string]struct { - alwaysSetCookie string - result bool - }{ - "true": { - alwaysSetCookie: "true", - result: true, - }, - "false": { - alwaysSetCookie: "false", - }, - "set empty": { - alwaysSetCookie: "", - }, - "error": { - alwaysSetCookie: "error string", - }, - } - - for n, tc := range testCases { - cfg := ReadConfig(map[string]string{"global-auth-always-set-cookie": tc.alwaysSetCookie}) - if cfg.GlobalExternalAuth.AlwaysSetCookie != tc.result { - t.Errorf("Testing %v. Expected \"%v\" but \"%v\" was returned", n, tc.result, cfg.GlobalExternalAuth.AlwaysSetCookie) - } - } -} - func TestGlobalExternalAuthSigninRedirectParamParsing(t *testing.T) { testCases := map[string]struct { param string @@ -378,37 +344,27 @@ func TestLuaSharedDictsParsing(t *testing.T) { { name: "configuration_data only", entry: map[string]string{"lua-shared-dicts": "configuration_data:5"}, - expect: map[string]int{"configuration_data": 5120}, + expect: map[string]int{"configuration_data": 5}, }, { name: "certificate_data only", entry: map[string]string{"lua-shared-dicts": "certificate_data: 4"}, - expect: map[string]int{"certificate_data": 4096}, + expect: map[string]int{"certificate_data": 4}, }, { name: "custom dicts", entry: map[string]string{"lua-shared-dicts": "configuration_data: 10, my_random_dict:15 , another_example:2"}, - expect: map[string]int{"configuration_data": 10240, "my_random_dict": 15360, "another_example": 2048}, - }, - { - name: "invalid format", - entry: map[string]string{"lua-shared-dicts": "mydict: 10, invalid_dict 100"}, - expect: map[string]int{"mydict": 10240}, + expect: map[string]int{"configuration_data": 10, "my_random_dict": 15, "another_example": 2}, }, { name: "invalid size value should be ignored", - entry: map[string]string{"lua-shared-dicts": "mydict: 10, invalid_dict: 1a, bad_mb_dict:10mb"}, - expect: map[string]int{"mydict": 10240}, + entry: map[string]string{"lua-shared-dicts": "mydict: 10, invalid_dict: 1a"}, + expect: map[string]int{"mydict": 10}, }, { name: "dictionary size can not be larger than 200", - entry: map[string]string{"lua-shared-dicts": "mydict: 10, invalid_dict: 201, invalid_kb: 204801k"}, - expect: map[string]int{"mydict": 10240}, - }, - { - name: "specified units are interpreted properly", - entry: map[string]string{"lua-shared-dicts": "kb_dict_a: 512k, mb_dict_a: 30m, kb_dict_b:16K, mb_dict_b:4M"}, - expect: map[string]int{"kb_dict_a": 512, "mb_dict_a": 30720, "kb_dict_b": 16, "mb_dict_b": 4096}, + entry: map[string]string{"lua-shared-dicts": "mydict: 10, invalid_dict: 201"}, + expect: map[string]int{"mydict": 10}, }, } @@ -462,76 +418,3 @@ func TestSplitAndTrimSpace(t *testing.T) { } } } - -func TestDictStrToKb(t *testing.T) { - testCases := []struct { - name string - input string - expect int - }{ - { - name: "unitless int size converted to kb", - input: "50", - expect: 51200, - }, - { - name: "lowercase k accepted", - input: "512k", - expect: 512, - }, - { - name: "uppercase K accepted", - input: "512K", - expect: 512, - }, - { - name: "lowercase m accepted", - input: "10m", - expect: 10240, - }, - { - name: "uppercase M accepted", - input: "10M", - expect: 10240, - }, - { - name: "trailing characters fail", - input: "50kb", - expect: -1, - }, - { - name: "leading characters fail", - input: " 50k", - expect: -1, - }, - } - for _, tc := range testCases { - if size := dictStrToKb(tc.input); size != tc.expect { - t.Errorf("Testing %v. Expected \"%v\" but \"%v\" was returned", tc.name, tc.expect, size) - } - } -} - -func TestDictKbToStr(t *testing.T) { - testCases := []struct { - name string - input int - expect string - }{ - { - name: "mod 1024 reports as M", - input: 5120, - expect: "5M", - }, - { - name: "non-mod 1024 reports as K", - input: 5001, - expect: "5001K", - }, - } - for _, tc := range testCases { - if sizeStr := dictKbToStr(tc.input); sizeStr != tc.expect { - t.Errorf("Testing %v. Expected \"%v\" but \"%v\" was returned", tc.name, tc.expect, sizeStr) - } - } -} diff --git a/internal/ingress/controller/template/template.go b/internal/ingress/controller/template/template.go index ed052e4ec..98e737a63 100644 --- a/internal/ingress/controller/template/template.go +++ b/internal/ingress/controller/template/template.go @@ -18,74 +18,61 @@ package template import ( "bytes" - "crypto/rand" "crypto/sha1" // #nosec "encoding/base64" "encoding/hex" "encoding/json" "fmt" - "io" - "math/big" + "io/ioutil" + "math/rand" // #nosec "net" "net/url" "os" + "os/exec" "reflect" "regexp" "sort" - "strconv" "strings" text_template "text/template" + "time" - networkingv1 "k8s.io/api/networking/v1" + "github.com/pkg/errors" + + networkingv1beta1 "k8s.io/api/networking/v1beta1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" - "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "k8s.io/ingress-nginx/internal/ingress" + "k8s.io/ingress-nginx/internal/ingress/annotations/influxdb" "k8s.io/ingress-nginx/internal/ingress/annotations/ratelimit" "k8s.io/ingress-nginx/internal/ingress/controller/config" ing_net "k8s.io/ingress-nginx/internal/net" - "k8s.io/ingress-nginx/pkg/apis/ingress" ) const ( - slash = "/" - nonIdempotent = "non_idempotent" - defBufferSize = 65535 - writeIndentOnEmptyLines = true // backward-compatibility - httpProtocol = "HTTP" - autoHTTPProtocol = "AUTO_HTTP" - httpsProtocol = "HTTPS" - grpcProtocol = "GRPC" - grpcsProtocol = "GRPCS" - fcgiProtocol = "FCGI" + slash = "/" + nonIdempotent = "non_idempotent" + defBufferSize = 65535 ) -const ( - stateCode = iota - stateComment -) - -// Writer is the interface to render a template -type Writer interface { - // Write renders the template. - // NOTE: Implementors must ensure that the content of the returned slice is not modified by the implementation - // after the return of this function. - Write(conf *config.TemplateConfig) ([]byte, error) +// TemplateWriter is the interface to render a template +type TemplateWriter interface { + Write(conf config.TemplateConfig) ([]byte, error) } -// Template ingress template +// Template ... type Template struct { tmpl *text_template.Template - + //fw watch.FileWatcher bp *BufferPool } -// NewTemplate returns a new Template instance or an -// error if the specified template file contains errors +//NewTemplate returns a new Template instance or an +//error if the specified template file contains errors func NewTemplate(file string) (*Template, error) { - data, err := os.ReadFile(file) + data, err := ioutil.ReadFile(file) if err != nil { - return nil, fmt.Errorf("unexpected error reading template %s: %w", file, err) + return nil, errors.Wrapf(err, "unexpected error reading template %v", file) } tmpl, err := text_template.New("nginx.tmpl").Funcs(funcMap).Parse(string(data)) @@ -99,139 +86,9 @@ func NewTemplate(file string) (*Template, error) { }, nil } -// 1. Removes carriage return symbol (\r) -// 2. Collapses multiple empty lines to single one -// 3. Re-indent -// (ATW: always returns nil) -func cleanConf(in, out *bytes.Buffer) error { - depth := 0 - lineStarted := false - emptyLineWritten := false - state := stateCode - for { - c, err := in.ReadByte() - if err != nil { - if err == io.EOF { - return nil - } - return err // unreachable - } - - needOutput := false - nextDepth := depth - nextLineStarted := lineStarted - - switch state { - case stateCode: - switch c { - case '{': - needOutput = true - nextDepth = depth + 1 - nextLineStarted = true - case '}': - needOutput = true - depth-- - nextDepth = depth - nextLineStarted = true - case ' ', '\t': - needOutput = lineStarted - case '\r': - rest := in.Bytes() - if len(rest) > 0 { - if rest[0] != '\n' { - c = ' ' - needOutput = lineStarted - } - } - case '\n': - needOutput = !(!lineStarted && emptyLineWritten) - nextLineStarted = false - case '#': - needOutput = true - nextLineStarted = true - state = stateComment - default: - needOutput = true - nextLineStarted = true - } - case stateComment: - switch c { - case '\r': - rest := in.Bytes() - if len(rest) > 0 { - if rest[0] != '\n' { - c = ' ' - needOutput = lineStarted - } - } - case '\n': - needOutput = true - nextLineStarted = false - state = stateCode - default: - needOutput = true - } - } - - if needOutput { - if !lineStarted && (writeIndentOnEmptyLines || c != '\n') { - for i := 0; i < depth; i++ { - err = out.WriteByte('\t') // always nil - if err != nil { - return err - } - } - } - emptyLineWritten = !lineStarted - err = out.WriteByte(c) // always nil - if err != nil { - return err - } - } - - depth = nextDepth - lineStarted = nextLineStarted - } -} - -/* LuaConfig defines the structure that will be written as a config for lua scripts -The json format should follow what's expected by lua: - use_forwarded_headers = %t, - use_proxy_protocol = %t, - is_ssl_passthrough_enabled = %t, - http_redirect_code = %v, - listen_ports = { ssl_proxy = "%v", https = "%v" }, - - hsts = %t, - hsts_max_age = %v, - hsts_include_subdomains = %t, - hsts_preload = %t, -*/ - -type LuaConfig struct { - EnableMetrics bool `json:"enable_metrics"` - ListenPorts LuaListenPorts `json:"listen_ports"` - UseForwardedHeaders bool `json:"use_forwarded_headers"` - UseProxyProtocol bool `json:"use_proxy_protocol"` - IsSSLPassthroughEnabled bool `json:"is_ssl_passthrough_enabled"` - HTTPRedirectCode int `json:"http_redirect_code"` - EnableOCSP bool `json:"enable_ocsp"` - MonitorBatchMaxSize int `json:"monitor_batch_max_size"` - HSTS bool `json:"hsts"` - HSTSMaxAge string `json:"hsts_max_age"` - HSTSIncludeSubdomains bool `json:"hsts_include_subdomains"` - HSTSPreload bool `json:"hsts_preload"` -} - -type LuaListenPorts struct { - HTTPSPort string `json:"https"` - StatusPort string `json:"status_port"` - SSLProxyPort string `json:"ssl_proxy"` -} - // Write populates a buffer using a template with NGINX configuration // and the servers and upstreams created by Ingress rules -func (t *Template) Write(conf *config.TemplateConfig) ([]byte, error) { +func (t *Template) Write(conf config.TemplateConfig) ([]byte, error) { tmplBuf := t.bp.Get() defer t.bp.Put(tmplBuf) @@ -239,113 +96,109 @@ func (t *Template) Write(conf *config.TemplateConfig) ([]byte, error) { defer t.bp.Put(outCmdBuf) if klog.V(3).Enabled() { - b, err := json.Marshal(*conf) + b, err := json.Marshal(conf) if err != nil { klog.Errorf("unexpected error: %v", err) } klog.InfoS("NGINX", "configuration", string(b)) } - err := t.tmpl.Execute(tmplBuf, *conf) + err := t.tmpl.Execute(tmplBuf, conf) if err != nil { return nil, err } // squeezes multiple adjacent empty lines to be single // spaced this is to avoid the use of regular expressions - err = cleanConf(tmplBuf, outCmdBuf) - if err != nil { - return nil, err + cmd := exec.Command("/ingress-controller/clean-nginx-conf.sh") + cmd.Stdin = tmplBuf + cmd.Stdout = outCmdBuf + if err := cmd.Run(); err != nil { + klog.Warningf("unexpected error cleaning template: %v", err) + return tmplBuf.Bytes(), nil } - // make a copy to ensure that we are no longer modifying the content of the buffer - out := outCmdBuf.Bytes() - res := make([]byte, len(out)) - copy(res, out) - - return res, nil + return outCmdBuf.Bytes(), nil } -var funcMap = text_template.FuncMap{ - "empty": func(input interface{}) bool { - check, ok := input.(string) - if ok { - return check == "" - } - return true - }, - "escapeLiteralDollar": escapeLiteralDollar, - "buildLuaSharedDictionaries": buildLuaSharedDictionaries, - "luaConfigurationRequestBodySize": luaConfigurationRequestBodySize, - "buildLocation": buildLocation, - "buildAuthLocation": buildAuthLocation, - "shouldApplyGlobalAuth": shouldApplyGlobalAuth, - "buildAuthResponseHeaders": buildAuthResponseHeaders, - "buildAuthUpstreamLuaHeaders": buildAuthUpstreamLuaHeaders, - "buildAuthProxySetHeaders": buildAuthProxySetHeaders, - "buildAuthUpstreamName": buildAuthUpstreamName, - "shouldApplyAuthUpstream": shouldApplyAuthUpstream, - "extractHostPort": extractHostPort, - "changeHostPort": changeHostPort, - "buildProxyPass": buildProxyPass, - "filterRateLimits": filterRateLimits, - "buildRateLimitZones": buildRateLimitZones, - "buildRateLimit": buildRateLimit, - "locationConfigForLua": locationConfigForLua, - "buildResolvers": buildResolvers, - "buildUpstreamName": buildUpstreamName, - "isLocationInLocationList": isLocationInLocationList, - "isLocationAllowed": isLocationAllowed, - "buildDenyVariable": buildDenyVariable, - "getenv": os.Getenv, - "contains": strings.Contains, - "split": strings.Split, - "hasPrefix": strings.HasPrefix, - "hasSuffix": strings.HasSuffix, - "trimSpace": strings.TrimSpace, - "toUpper": strings.ToUpper, - "toLower": strings.ToLower, - "formatIP": formatIP, - "quote": quote, - "buildNextUpstream": buildNextUpstream, - "getIngressInformation": getIngressInformation, - "serverConfig": func(all config.TemplateConfig, server *ingress.Server) interface{} { - return struct{ First, Second interface{} }{all, server} - }, - "isValidByteSize": isValidByteSize, - "buildForwardedFor": buildForwardedFor, - "buildAuthSignURL": buildAuthSignURL, - "buildAuthSignURLLocation": buildAuthSignURLLocation, - "buildOpentelemetry": buildOpentelemetry, - "proxySetHeader": proxySetHeader, - "enforceRegexModifier": enforceRegexModifier, - "buildCustomErrorDeps": buildCustomErrorDeps, - "buildCustomErrorLocationsPerServer": buildCustomErrorLocationsPerServer, - "shouldLoadModSecurityModule": shouldLoadModSecurityModule, - "buildHTTPListener": buildHTTPListener, - "buildHTTPSListener": buildHTTPSListener, - "buildOpentelemetryForLocation": buildOpentelemetryForLocation, - "shouldLoadOpentelemetryModule": shouldLoadOpentelemetryModule, - "buildModSecurityForLocation": buildModSecurityForLocation, - "buildMirrorLocations": buildMirrorLocations, - "shouldLoadAuthDigestModule": shouldLoadAuthDigestModule, - "buildServerName": buildServerName, - "buildCorsOriginRegex": buildCorsOriginRegex, -} +var ( + funcMap = text_template.FuncMap{ + "empty": func(input interface{}) bool { + check, ok := input.(string) + if ok { + return len(check) == 0 + } + return true + }, + "escapeLiteralDollar": escapeLiteralDollar, + "buildLuaSharedDictionaries": buildLuaSharedDictionaries, + "luaConfigurationRequestBodySize": luaConfigurationRequestBodySize, + "buildLocation": buildLocation, + "buildAuthLocation": buildAuthLocation, + "shouldApplyGlobalAuth": shouldApplyGlobalAuth, + "buildAuthResponseHeaders": buildAuthResponseHeaders, + "buildAuthProxySetHeaders": buildAuthProxySetHeaders, + "buildProxyPass": buildProxyPass, + "filterRateLimits": filterRateLimits, + "buildRateLimitZones": buildRateLimitZones, + "buildRateLimit": buildRateLimit, + "configForLua": configForLua, + "locationConfigForLua": locationConfigForLua, + "buildResolvers": buildResolvers, + "buildUpstreamName": buildUpstreamName, + "isLocationInLocationList": isLocationInLocationList, + "isLocationAllowed": isLocationAllowed, + "buildDenyVariable": buildDenyVariable, + "getenv": os.Getenv, + "contains": strings.Contains, + "split": strings.Split, + "hasPrefix": strings.HasPrefix, + "hasSuffix": strings.HasSuffix, + "trimSpace": strings.TrimSpace, + "toUpper": strings.ToUpper, + "toLower": strings.ToLower, + "formatIP": formatIP, + "quote": quote, + "buildNextUpstream": buildNextUpstream, + "getIngressInformation": getIngressInformation, + "serverConfig": func(all config.TemplateConfig, server *ingress.Server) interface{} { + return struct{ First, Second interface{} }{all, server} + }, + "isValidByteSize": isValidByteSize, + "buildForwardedFor": buildForwardedFor, + "buildAuthSignURL": buildAuthSignURL, + "buildAuthSignURLLocation": buildAuthSignURLLocation, + "buildOpentracing": buildOpentracing, + "proxySetHeader": proxySetHeader, + "buildInfluxDB": buildInfluxDB, + "enforceRegexModifier": enforceRegexModifier, + "buildCustomErrorDeps": buildCustomErrorDeps, + "buildCustomErrorLocationsPerServer": buildCustomErrorLocationsPerServer, + "shouldLoadModSecurityModule": shouldLoadModSecurityModule, + "buildHTTPListener": buildHTTPListener, + "buildHTTPSListener": buildHTTPSListener, + "buildOpentracingForLocation": buildOpentracingForLocation, + "shouldLoadOpentracingModule": shouldLoadOpentracingModule, + "buildModSecurityForLocation": buildModSecurityForLocation, + "buildMirrorLocations": buildMirrorLocations, + "shouldLoadAuthDigestModule": shouldLoadAuthDigestModule, + "shouldLoadInfluxDBModule": shouldLoadInfluxDBModule, + "buildServerName": buildServerName, + } +) // escapeLiteralDollar will replace the $ character with ${literal_dollar} // which is made to work via the following configuration in the http section of // the template: -// -// geo $literal_dollar { -// default "$"; -// } +// geo $literal_dollar { +// default "$"; +// } func escapeLiteralDollar(input interface{}) string { inputStr, ok := input.(string) if !ok { return "" } - return strings.ReplaceAll(inputStr, `$`, `${literal_dollar}`) + return strings.Replace(inputStr, `$`, `${literal_dollar}`, -1) } // formatIP will wrap IPv6 addresses in [] and return IPv4 addresses @@ -377,7 +230,9 @@ func quote(input interface{}) string { return fmt.Sprintf("%q", inputStr) } -func buildLuaSharedDictionaries(c, s interface{}) string { +func buildLuaSharedDictionaries(c interface{}, s interface{}) string { + var out []string + cfg, ok := c.(config.Configuration) if !ok { klog.Errorf("expected a 'config.Configuration' type but %T was returned", c) @@ -390,10 +245,8 @@ func buildLuaSharedDictionaries(c, s interface{}) string { return "" } - out := make([]string, 0, len(cfg.LuaSharedDicts)) for name, size := range cfg.LuaSharedDicts { - sizeStr := dictKbToStr(size) - out = append(out, fmt.Sprintf("lua_shared_dict %s %s", name, sizeStr)) + out = append(out, fmt.Sprintf("lua_shared_dict %s %dM", name, size)) } sort.Strings(out) @@ -405,20 +258,68 @@ func luaConfigurationRequestBodySize(c interface{}) string { cfg, ok := c.(config.Configuration) if !ok { klog.Errorf("expected a 'config.Configuration' type but %T was returned", c) - return "100M" // just a default number + return "100" // just a default number } size := cfg.LuaSharedDicts["configuration_data"] if size < cfg.LuaSharedDicts["certificate_data"] { size = cfg.LuaSharedDicts["certificate_data"] } - size += 1024 + size = size + 1 - return dictKbToStr(size) + return fmt.Sprintf("%d", size) +} + +// configForLua returns some general configuration as Lua table represented as string +func configForLua(input interface{}) string { + all, ok := input.(config.TemplateConfig) + if !ok { + klog.Errorf("expected a 'config.TemplateConfig' type but %T was given", input) + return "{}" + } + + return fmt.Sprintf(`{ + use_forwarded_headers = %t, + use_proxy_protocol = %t, + is_ssl_passthrough_enabled = %t, + http_redirect_code = %v, + listen_ports = { ssl_proxy = "%v", https = "%v" }, + + hsts = %t, + hsts_max_age = %v, + hsts_include_subdomains = %t, + hsts_preload = %t, + + global_throttle = { + memcached = { + host = "%v", port = %d, connect_timeout = %d, max_idle_timeout = %d, pool_size = %d, + }, + status_code = %d, + } + }`, + all.Cfg.UseForwardedHeaders, + all.Cfg.UseProxyProtocol, + all.IsSSLPassthroughEnabled, + all.Cfg.HTTPRedirectCode, + all.ListenPorts.SSLProxy, + all.ListenPorts.HTTPS, + + all.Cfg.HSTS, + all.Cfg.HSTSMaxAge, + all.Cfg.HSTSIncludeSubdomains, + all.Cfg.HSTSPreload, + + all.Cfg.GlobalRateLimitMemcachedHost, + all.Cfg.GlobalRateLimitMemcachedPort, + all.Cfg.GlobalRateLimitMemcachedConnectTimeout, + all.Cfg.GlobalRateLimitMemcachedMaxIdleTimeout, + all.Cfg.GlobalRateLimitMemcachedPoolSize, + all.Cfg.GlobalRateLimitStatucCode, + ) } // locationConfigForLua formats some location specific configuration into Lua table represented as string -func locationConfigForLua(l, a interface{}) string { +func locationConfigForLua(l interface{}, a interface{}) string { location, ok := l.(*ingress.Location) if !ok { klog.Errorf("expected an '*ingress.Location' type but %T was given", l) @@ -431,31 +332,35 @@ func locationConfigForLua(l, a interface{}) string { return "{}" } - /* Lua expects the following vars - force_ssl_redirect = string_to_bool(ngx.var.force_ssl_redirect), - ssl_redirect = string_to_bool(ngx.var.ssl_redirect), - force_no_ssl_redirect = string_to_bool(ngx.var.force_no_ssl_redirect), - preserve_trailing_slash = string_to_bool(ngx.var.preserve_trailing_slash), - use_port_in_redirects = string_to_bool(ngx.var.use_port_in_redirects), - */ + ignoredCIDRs, err := convertGoSliceIntoLuaTable(location.GlobalRateLimit.IgnoredCIDRs, false) + if err != nil { + klog.Errorf("failed to convert %v into Lua table: %q", location.GlobalRateLimit.IgnoredCIDRs, err) + ignoredCIDRs = "{}" + } - return fmt.Sprintf(` - set $force_ssl_redirect "%t"; - set $ssl_redirect "%t"; - set $force_no_ssl_redirect "%t"; - set $preserve_trailing_slash "%t"; - set $use_port_in_redirects "%t"; - `, + return fmt.Sprintf(`{ + force_ssl_redirect = %t, + ssl_redirect = %t, + force_no_ssl_redirect = %t, + preserve_trailing_slash = %t, + use_port_in_redirects = %t, + global_throttle = { namespace = "%v", limit = %d, window_size = %d, key = %v, ignored_cidrs = %v }, + }`, location.Rewrite.ForceSSLRedirect, location.Rewrite.SSLRedirect, isLocationInLocationList(l, all.Cfg.NoTLSRedirectLocations), location.Rewrite.PreserveTrailingSlash, location.UsePortInRedirects, + location.GlobalRateLimit.Namespace, + location.GlobalRateLimit.Limit, + location.GlobalRateLimit.WindowSize, + parseComplexNginxVarIntoLuaTable(location.GlobalRateLimit.Key), + ignoredCIDRs, ) } // buildResolvers returns the resolvers reading the /etc/resolv.conf file -func buildResolvers(res, disableIpv6 interface{}) string { +func buildResolvers(res interface{}, disableIpv6 interface{}) string { // NGINX need IPV6 addresses to be surrounded by brackets nss, ok := res.([]net.IP) if !ok { @@ -480,7 +385,7 @@ func buildResolvers(res, disableIpv6 interface{}) string { } r = append(r, fmt.Sprintf("[%v]", ns)) } else { - r = append(r, ns.String()) + r = append(r, fmt.Sprintf("%v", ns)) } } r = append(r, "valid=30s") @@ -493,7 +398,7 @@ func buildResolvers(res, disableIpv6 interface{}) string { } func needsRewrite(location *ingress.Location) bool { - if location.Rewrite.Target != "" && location.Rewrite.Target != location.Path { + if len(location.Rewrite.Target) > 0 && location.Rewrite.Target != location.Path { return true } return false @@ -530,7 +435,7 @@ func buildLocation(input interface{}, enforceRegex bool) string { return fmt.Sprintf(`~* "^%s"`, path) } - if location.PathType != nil && *location.PathType == networkingv1.PathTypeExact { + if location.PathType != nil && *location.PathType == networkingv1beta1.PathTypeExact { return fmt.Sprintf(`= %s`, path) } @@ -550,7 +455,7 @@ func buildAuthLocation(input interface{}, globalExternalAuthURL string) string { str := base64.URLEncoding.EncodeToString([]byte(location.Path)) // removes "=" after encoding - str = strings.ReplaceAll(str, "=", "") + str = strings.Replace(str, "=", "", -1) pathType := "default" if location.PathType != nil { @@ -575,14 +480,7 @@ func shouldApplyGlobalAuth(input interface{}, globalExternalAuthURL string) bool return false } -// buildAuthResponseHeaders sets HTTP response headers when `auth-url` is used. -// Based on `auth-keepalive` value we use auth_request_set Nginx directives, or -// we use Lua and Nginx variables instead. -// -// NOTE: Unfortunately auth_request module ignores the keepalive directive (see: -// https://trac.nginx.org/nginx/ticket/1579), that is why we mimic the same -// functionality with access_by_lua_block. -func buildAuthResponseHeaders(proxySetHeader string, headers []string, lua bool) []string { +func buildAuthResponseHeaders(headers []string) []string { res := []string{} if len(headers) == 0 { @@ -590,26 +488,14 @@ func buildAuthResponseHeaders(proxySetHeader string, headers []string, lua bool) } for i, h := range headers { - if lua { - res = append(res, fmt.Sprintf("set $authHeader%d '';", i)) - } else { - hvar := strings.ToLower(h) - hvar = strings.NewReplacer("-", "_").Replace(hvar) - res = append(res, fmt.Sprintf("auth_request_set $authHeader%v $upstream_http_%v;", i, hvar)) - } - res = append(res, fmt.Sprintf("%s '%v' $authHeader%v;", proxySetHeader, h, i)) + hvar := strings.ToLower(h) + hvar = strings.NewReplacer("-", "_").Replace(hvar) + res = append(res, fmt.Sprintf("auth_request_set $authHeader%v $upstream_http_%v;", i, hvar)) + res = append(res, fmt.Sprintf("proxy_set_header '%v' $authHeader%v;", h, i)) } return res } -func buildAuthUpstreamLuaHeaders(headers []string) string { - if len(headers) == 0 { - return "" - } - - return strings.Join(headers, ",") -} - func buildAuthProxySetHeaders(headers map[string]string) []string { res := []string{} @@ -624,81 +510,11 @@ func buildAuthProxySetHeaders(headers map[string]string) []string { return res } -func buildAuthUpstreamName(input interface{}, host string) string { - authPath := buildAuthLocation(input, "") - if authPath == "" || host == "" { - return "" - } - - return fmt.Sprintf("%s-%s", host, authPath[2:]) -} - -// shouldApplyAuthUpstream returns true only in case when ExternalAuth.URL and -// ExternalAuth.KeepaliveConnections are all set -func shouldApplyAuthUpstream(l, c interface{}) bool { - location, ok := l.(*ingress.Location) - if !ok { - klog.Errorf("expected an '*ingress.Location' type but %T was returned", l) - return false - } - - cfg, ok := c.(config.Configuration) - if !ok { - klog.Errorf("expected a 'config.Configuration' type but %T was returned", c) - return false - } - - if location.ExternalAuth.URL == "" || location.ExternalAuth.KeepaliveConnections == 0 { - return false - } - - // Unfortunately, `auth_request` module ignores keepalive in upstream block: https://trac.nginx.org/nginx/ticket/1579 - // The workaround is to use `ngx.location.capture` Lua subrequests but it is not supported with HTTP/2 - if cfg.UseHTTP2 { - klog.Warning("Upstream keepalive is not supported with HTTP/2") - return false - } - - return true -} - -// extractHostPort will extract the host:port part from the URL specified by url -func extractHostPort(newURL string) string { - if newURL == "" { - return "" - } - - authURL, err := parser.StringToURL(newURL) - if err != nil { - klog.Errorf("expected a valid URL but %s was returned", newURL) - return "" - } - - return authURL.Host -} - -// changeHostPort will change the host:port part of the url to value -func changeHostPort(newURL, value string) string { - if newURL == "" { - return "" - } - - authURL, err := parser.StringToURL(newURL) - if err != nil { - klog.Errorf("expected a valid URL but %s was returned", newURL) - return "" - } - - authURL.Host = value - - return authURL.String() -} - // buildProxyPass produces the proxy pass string, if the ingress has redirects // (specified through the nginx.ingress.kubernetes.io/rewrite-target annotation) // If the annotation nginx.ingress.kubernetes.io/add-base-url:"true" is specified it will // add a base tag in the head of the response from the service -func buildProxyPass(_ string, b, loc interface{}) string { +func buildProxyPass(host string, b interface{}, loc interface{}) string { backends, ok := b.([]*ingress.Backend) if !ok { klog.Errorf("expected an '[]*ingress.Backend' type but %T was returned", b) @@ -716,18 +532,19 @@ func buildProxyPass(_ string, b, loc interface{}) string { proxyPass := "proxy_pass" - switch strings.ToUpper(location.BackendProtocol) { - case autoHTTPProtocol: - proto = "$scheme://" - case httpsProtocol: + switch location.BackendProtocol { + case "HTTPS": proto = "https://" - case grpcProtocol: + case "GRPC": proto = "grpc://" proxyPass = "grpc_pass" - case grpcsProtocol: + case "GRPCS": proto = "grpcs://" proxyPass = "grpc_pass" - case fcgiProtocol: + case "AJP": + proto = "" + proxyPass = "ajp_pass" + case "FCGI": proto = "" proxyPass = "fastcgi_pass" } @@ -739,7 +556,7 @@ func buildProxyPass(_ string, b, loc interface{}) string { if backend.SSLPassthrough { proto = "https://" - if location.BackendProtocol == grpcsProtocol { + if location.BackendProtocol == "GRPCS" { proto = "grpcs://" } } @@ -762,11 +579,11 @@ func buildProxyPass(_ string, b, loc interface{}) string { return defProxyPass } - if location.Rewrite.Target != "" { + if len(location.Rewrite.Target) > 0 { var xForwardedPrefix string - if location.XForwardedPrefix != "" { - xForwardedPrefix = fmt.Sprintf("%s X-Forwarded-Prefix %q;\n", proxySetHeader(location), location.XForwardedPrefix) + if len(location.XForwardedPrefix) > 0 { + xForwardedPrefix = fmt.Sprintf("proxy_set_header X-Forwarded-Prefix \"%s\";\n", location.XForwardedPrefix) } return fmt.Sprintf(` @@ -780,7 +597,7 @@ rewrite "(?i)%s" %s break; func filterRateLimits(input interface{}) []ratelimit.Config { ratelimits := []ratelimit.Config{} - found := sets.Set[string]{} + found := sets.String{} servers, ok := input.([]*ingress.Server) if !ok { @@ -803,12 +620,12 @@ func filterRateLimits(input interface{}) []ratelimit.Config { // for connection limit by IP address, one for limiting requests per minute, and // one for limiting requests per second. func buildRateLimitZones(input interface{}) []string { - zones := sets.Set[string]{} + zones := sets.String{} servers, ok := input.([]*ingress.Server) if !ok { klog.Errorf("expected a '[]*ingress.Server' type but %T was returned", input) - return zones.UnsortedList() + return zones.List() } for _, server := range servers { @@ -847,7 +664,7 @@ func buildRateLimitZones(input interface{}) []string { } } - return zones.UnsortedList() + return zones.List() } // buildRateLimit produces an array of limit_req to be used inside the Path of @@ -926,7 +743,9 @@ func isLocationAllowed(input interface{}) bool { return loc.Denied == nil } -var denyPathSlugMap = map[string]string{} +var ( + denyPathSlugMap = map[string]string{} +) // buildDenyVariable returns a nginx variable for a location in a // server to be used in the whitelist check @@ -966,11 +785,7 @@ func buildNextUpstream(i, r interface{}) string { return "" } - retryNonIdempotent, ok := r.(bool) - if !ok { - klog.Errorf("expected a 'bool' type but %T was returned", i) - return "" - } + retryNonIdempotent := r.(bool) parts := strings.Split(nextUpstream, " ") @@ -995,10 +810,8 @@ func buildNextUpstream(i, r interface{}) string { // refer to http://nginx.org/en/docs/syntax.html // Nginx differentiates between size and offset // offset directives support gigabytes in addition -var ( - nginxSizeRegex = regexp.MustCompile(`^\d+[kKmM]?$`) - nginxOffsetRegex = regexp.MustCompile(`^\d+[kKmMgG]?$`) -) +var nginxSizeRegex = regexp.MustCompile("^[0-9]+[kKmM]{0,1}$") +var nginxOffsetRegex = regexp.MustCompile("^[0-9]+[kKmMgG]{0,1}$") // isValidByteSize validates size units valid in nginx // http://nginx.org/en/docs/syntax.html @@ -1086,12 +899,10 @@ func getIngressInformation(i, h, p interface{}) *ingressInformation { info.Path = "/" } - if ing.Spec.DefaultBackend != nil && ing.Spec.DefaultBackend.Service != nil { - info.Service = ing.Spec.DefaultBackend.Service.Name - if ing.Spec.DefaultBackend.Service.Port.Number > 0 { - info.ServicePort = strconv.Itoa(int(ing.Spec.DefaultBackend.Service.Port.Number)) - } else { - info.ServicePort = ing.Spec.DefaultBackend.Service.Port.Name + if ing.Spec.Backend != nil { + info.Service = ing.Spec.Backend.ServiceName + if ing.Spec.Backend.ServicePort.String() != "0" { + info.ServicePort = ing.Spec.Backend.ServicePort.String() } } @@ -1118,20 +929,14 @@ func getIngressInformation(i, h, p interface{}) *ingressInformation { continue } - if rPath.Backend.Service == nil { - continue - } - - if info.Service != "" && rPath.Backend.Service.Name == "" { + if info.Service != "" && rPath.Backend.ServiceName == "" { // empty rule. Only contains a Path and PathType return info } - info.Service = rPath.Backend.Service.Name - if rPath.Backend.Service.Port.Number > 0 { - info.ServicePort = strconv.Itoa(int(rPath.Backend.Service.Port.Number)) - } else { - info.ServicePort = rPath.Backend.Service.Port.Name + info.Service = rPath.Backend.ServiceName + if rPath.Backend.ServicePort.String() != "0" { + info.ServicePort = rPath.Backend.ServicePort.String() } return info @@ -1148,17 +953,13 @@ func buildForwardedFor(input interface{}) string { return "" } - ffh := strings.ReplaceAll(s, "-", "_") + ffh := strings.Replace(s, "-", "_", -1) ffh = strings.ToLower(ffh) return fmt.Sprintf("$http_%v", ffh) } func buildAuthSignURL(authSignURL, authRedirectParam string) string { - u, err := url.Parse(authSignURL) - if err != nil { - klog.Errorf("error parsing authSignURL: %v", err) - return "" - } + u, _ := url.Parse(authSignURL) q := u.Query() if authRedirectParam == "" { authRedirectParam = defaultGlobalAuthRedirectParam @@ -1183,21 +984,20 @@ func buildAuthSignURLLocation(location, authSignURL string) string { var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") +func init() { + rand.Seed(time.Now().UnixNano()) +} + func randomString() string { b := make([]rune, 32) for i := range b { - idx, err := rand.Int(rand.Reader, big.NewInt(int64(len(letters)))) - if err != nil { - klog.Errorf("unexpected error generating random index: %v", err) - return "" - } - b[i] = letters[idx.Int64()] + b[i] = letters[rand.Intn(len(letters))] // #nosec } return string(b) } -func buildOpentelemetry(c, s interface{}) string { +func buildOpentracing(c interface{}, s interface{}) string { cfg, ok := c.(config.Configuration) if !ok { klog.Errorf("expected a 'config.Configuration' type but %T was returned", c) @@ -1210,20 +1010,55 @@ func buildOpentelemetry(c, s interface{}) string { return "" } - if !shouldLoadOpentelemetryModule(cfg, servers) { + if !shouldLoadOpentracingModule(cfg, servers) { return "" } buf := bytes.NewBufferString("") + if cfg.DatadogCollectorHost != "" { + buf.WriteString("opentracing_load_tracer /usr/local/lib64/libdd_opentracing.so /etc/nginx/opentracing.json;") + } else if cfg.ZipkinCollectorHost != "" { + buf.WriteString("opentracing_load_tracer /usr/local/lib/libzipkin_opentracing_plugin.so /etc/nginx/opentracing.json;") + } else if cfg.JaegerCollectorHost != "" || cfg.JaegerEndpoint != "" { + buf.WriteString("opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.so /etc/nginx/opentracing.json;") + } + buf.WriteString("\r\n") - if cfg.OpentelemetryOperationName != "" { - fmt.Fprintf(buf, "opentelemetry_operation_name \"%s\";\n", cfg.OpentelemetryOperationName) + if cfg.OpentracingOperationName != "" { + buf.WriteString(fmt.Sprintf("opentracing_operation_name \"%s\";\n", cfg.OpentracingOperationName)) } + if cfg.OpentracingLocationOperationName != "" { + buf.WriteString(fmt.Sprintf("opentracing_location_operation_name \"%s\";\n", cfg.OpentracingLocationOperationName)) + } + return buf.String() } +// buildInfluxDB produces the single line configuration +// needed by the InfluxDB module to send request's metrics +// for the current resource +func buildInfluxDB(input interface{}) string { + cfg, ok := input.(influxdb.Config) + if !ok { + klog.Errorf("expected an 'influxdb.Config' type but %T was returned", input) + return "" + } + + if !cfg.InfluxDBEnabled { + return "" + } + + return fmt.Sprintf( + "influxdb server_name=%s host=%s port=%s measurement=%s enabled=true;", + cfg.InfluxDBServerName, + cfg.InfluxDBHost, + cfg.InfluxDBPort, + cfg.InfluxDBMeasurement, + ) +} + func proxySetHeader(loc interface{}) string { location, ok := loc.(*ingress.Location) if !ok { @@ -1231,7 +1066,7 @@ func proxySetHeader(loc interface{}) string { return "proxy_set_header" } - if location.BackendProtocol == grpcProtocol || location.BackendProtocol == grpcsProtocol { + if location.BackendProtocol == "GRPC" || location.BackendProtocol == "GRPCS" { return "grpc_set_header" } @@ -1240,17 +1075,15 @@ func proxySetHeader(loc interface{}) string { // buildCustomErrorDeps is a utility function returning a struct wrapper with // the data required to build the 'CUSTOM_ERRORS' template -func buildCustomErrorDeps(upstreamName string, errorCodes []int, enableMetrics, modsecurityEnabled bool) interface{} { +func buildCustomErrorDeps(upstreamName string, errorCodes []int, enableMetrics bool) interface{} { return struct { - UpstreamName string - ErrorCodes []int - EnableMetrics bool - ModsecurityEnabled bool + UpstreamName string + ErrorCodes []int + EnableMetrics bool }{ - UpstreamName: upstreamName, - ErrorCodes: errorCodes, - EnableMetrics: enableMetrics, - ModsecurityEnabled: modsecurityEnabled, + UpstreamName: upstreamName, + ErrorCodes: errorCodes, + EnableMetrics: enableMetrics, } } @@ -1310,17 +1143,22 @@ func buildCustomErrorLocationsPerServer(input interface{}) []errorLocation { return errorLocations } -func opentelemetryPropagateContext(location *ingress.Location) string { +func opentracingPropagateContext(location *ingress.Location) string { if location == nil { return "" } - return "opentelemetry_propagate;" + + if location.BackendProtocol == "GRPC" || location.BackendProtocol == "GRPCS" { + return "opentracing_grpc_propagate_context;" + } + + return "opentracing_propagate_context;" } // shouldLoadModSecurityModule determines whether or not the ModSecurity module needs to be loaded. // First, it checks if `enable-modsecurity` is set in the ConfigMap. If it is not, it iterates over all locations to // check if ModSecurity is enabled by the annotation `nginx.ingress.kubernetes.io/enable-modsecurity`. -func shouldLoadModSecurityModule(c, s interface{}) bool { +func shouldLoadModSecurityModule(c interface{}, s interface{}) bool { cfg, ok := c.(config.Configuration) if !ok { klog.Errorf("expected a 'config.Configuration' type but %T was returned", c) @@ -1351,7 +1189,7 @@ func shouldLoadModSecurityModule(c, s interface{}) bool { return false } -func buildHTTPListener(t, s interface{}) string { +func buildHTTPListener(t interface{}, s interface{}) string { var out []string tc, ok := t.(config.TemplateConfig) @@ -1371,9 +1209,9 @@ func buildHTTPListener(t, s interface{}) string { addrV4 = tc.Cfg.BindAddressIpv4 } - co := commonListenOptions(&tc, hostname) + co := commonListenOptions(tc, hostname) - out = append(out, httpListener(addrV4, co, &tc)...) + out = append(out, httpListener(addrV4, co, tc)...) if !tc.IsIPV6Enabled { return strings.Join(out, "\n") @@ -1384,12 +1222,12 @@ func buildHTTPListener(t, s interface{}) string { addrV6 = tc.Cfg.BindAddressIpv6 } - out = append(out, httpListener(addrV6, co, &tc)...) + out = append(out, httpListener(addrV6, co, tc)...) return strings.Join(out, "\n") } -func buildHTTPSListener(t, s interface{}) string { +func buildHTTPSListener(t interface{}, s interface{}) string { var out []string tc, ok := t.(config.TemplateConfig) @@ -1404,14 +1242,14 @@ func buildHTTPSListener(t, s interface{}) string { return "" } - co := commonListenOptions(&tc, hostname) + co := commonListenOptions(tc, hostname) addrV4 := []string{""} if len(tc.Cfg.BindAddressIpv4) > 0 { addrV4 = tc.Cfg.BindAddressIpv4 } - out = append(out, httpsListener(addrV4, co, &tc)...) + out = append(out, httpsListener(addrV4, co, tc)...) if !tc.IsIPV6Enabled { return strings.Join(out, "\n") @@ -1422,12 +1260,12 @@ func buildHTTPSListener(t, s interface{}) string { addrV6 = tc.Cfg.BindAddressIpv6 } - out = append(out, httpsListener(addrV6, co, &tc)...) + out = append(out, httpsListener(addrV6, co, tc)...) return strings.Join(out, "\n") } -func commonListenOptions(template *config.TemplateConfig, hostname string) string { +func commonListenOptions(template config.TemplateConfig, hostname string) string { var out []string if template.Cfg.UseProxyProtocol { @@ -1451,7 +1289,7 @@ func commonListenOptions(template *config.TemplateConfig, hostname string) strin return strings.Join(out, " ") } -func httpListener(addresses []string, co string, tc *config.TemplateConfig) []string { +func httpListener(addresses []string, co string, tc config.TemplateConfig) []string { out := make([]string, 0) for _, address := range addresses { lo := []string{"listen"} @@ -1462,14 +1300,15 @@ func httpListener(addresses []string, co string, tc *config.TemplateConfig) []st lo = append(lo, fmt.Sprintf("%v:%v", address, tc.ListenPorts.HTTP)) } - lo = append(lo, co, ";") + lo = append(lo, co) + lo = append(lo, ";") out = append(out, strings.Join(lo, " ")) } return out } -func httpsListener(addresses []string, co string, tc *config.TemplateConfig) []string { +func httpsListener(addresses []string, co string, tc config.TemplateConfig) []string { out := make([]string, 0) for _, address := range addresses { lo := []string{"listen"} @@ -1492,47 +1331,53 @@ func httpsListener(addresses []string, co string, tc *config.TemplateConfig) []s } } - lo = append(lo, co, "ssl;") + lo = append(lo, co) + lo = append(lo, "ssl") + if tc.Cfg.UseHTTP2 { + lo = append(lo, "http2") + } + + lo = append(lo, ";") out = append(out, strings.Join(lo, " ")) } return out } -func buildOpentelemetryForLocation(isOTEnabled, isOTTrustSet bool, location *ingress.Location) string { - isOTEnabledInLoc := location.Opentelemetry.Enabled - isOTSetInLoc := location.Opentelemetry.Set +func buildOpentracingForLocation(isOTEnabled bool, location *ingress.Location) string { + isOTEnabledInLoc := location.Opentracing.Enabled + isOTSetInLoc := location.Opentracing.Set if isOTEnabled { if isOTSetInLoc && !isOTEnabledInLoc { - return "opentelemetry off;" + return "opentracing off;" } - } else if !isOTSetInLoc || !isOTEnabledInLoc { - return "" + + opc := opentracingPropagateContext(location) + if opc != "" { + opc = fmt.Sprintf("opentracing on;\n%v", opc) + } + + return opc } - opc := opentelemetryPropagateContext(location) - if opc != "" { - opc = fmt.Sprintf("opentelemetry on;\n%v", opc) + if isOTSetInLoc && isOTEnabledInLoc { + opc := opentracingPropagateContext(location) + if opc != "" { + opc = fmt.Sprintf("opentracing on;\n%v", opc) + } + + return opc } - if location.Opentelemetry.OperationName != "" { - opc += "\nopentelemetry_operation_name " + location.Opentelemetry.OperationName + ";" - } - - if (!isOTTrustSet && !location.Opentelemetry.TrustSet) || - (location.Opentelemetry.TrustSet && !location.Opentelemetry.TrustEnabled) { - opc += "\nopentelemetry_trust_incoming_spans off;" - } else { - opc += "\nopentelemetry_trust_incoming_spans on;" - } - return opc + return "" } -// shouldLoadOpentelemetryModule determines whether or not the Opentelemetry module needs to be loaded. -// It checks if `enable-opentelemetry` is set in the ConfigMap. -func shouldLoadOpentelemetryModule(c, s interface{}) bool { +// shouldLoadOpentracingModule determines whether or not the Opentracing module needs to be loaded. +// First, it checks if `enable-opentracing` is set in the ConfigMap. If it is not, it iterates over all locations to +// check if Opentracing is enabled by the annotation `nginx.ingress.kubernetes.io/enable-opentracing`. +func shouldLoadOpentracingModule(c interface{}, s interface{}) bool { cfg, ok := c.(config.Configuration) if !ok { klog.Errorf("expected a 'config.Configuration' type but %T was returned", c) @@ -1545,21 +1390,21 @@ func shouldLoadOpentelemetryModule(c, s interface{}) bool { return false } - if cfg.EnableOpentelemetry { + if cfg.EnableOpentracing { return true } for _, server := range servers { for _, location := range server.Locations { - if location.Opentelemetry.Enabled { + if location.Opentracing.Enabled { return true } } } + return false } -//nolint:gocritic // Ignore passing cfg by pointer error func buildModSecurityForLocation(cfg config.Configuration, location *ingress.Location) string { isMSEnabledInLoc := location.ModSecurity.Enable isMSEnableSetInLoc := location.ModSecurity.EnableSet @@ -1592,7 +1437,7 @@ func buildModSecurityForLocation(cfg config.Configuration, location *ingress.Loc `, location.ModSecurity.TransactionID)) } - if !isMSEnabled && location.ModSecurity.Snippet == "" { + if !isMSEnabled { buffer.WriteString(`modsecurity_rules_file /etc/nginx/modsecurity/modsecurity.conf; `) } @@ -1608,10 +1453,10 @@ func buildModSecurityForLocation(cfg config.Configuration, location *ingress.Loc func buildMirrorLocations(locs []*ingress.Location) string { var buffer bytes.Buffer - mapped := sets.Set[string]{} + mapped := sets.String{} for _, loc := range locs { - if loc.Mirror.Source == "" || loc.Mirror.Target == "" || loc.Mirror.Host == "" { + if loc.Mirror.Source == "" || loc.Mirror.Target == "" { continue } @@ -1622,11 +1467,10 @@ func buildMirrorLocations(locs []*ingress.Location) string { mapped.Insert(loc.Mirror.Source) buffer.WriteString(fmt.Sprintf(`location = %v { internal; -proxy_set_header Host "%v"; -proxy_pass "%v"; +proxy_pass %v; } -`, loc.Mirror.Source, loc.Mirror.Host, loc.Mirror.Target)) +`, loc.Mirror.Source, loc.Mirror.Target)) } return buffer.String() @@ -1655,6 +1499,25 @@ func shouldLoadAuthDigestModule(s interface{}) bool { return false } +// shouldLoadInfluxDBModule determines whether or not the ngx_http_auth_digest_module module needs to be loaded. +func shouldLoadInfluxDBModule(s interface{}) bool { + servers, ok := s.([]*ingress.Server) + if !ok { + klog.Errorf("expected an '[]*ingress.Server' type but %T was returned", s) + return false + } + + for _, server := range servers { + for _, location := range server.Locations { + if location.InfluxDB.InfluxDBEnabled { + return true + } + } + } + + return false +} + // buildServerName ensures wildcard hostnames are valid func buildServerName(hostname string) string { if !strings.HasPrefix(hostname, "*") { @@ -1667,28 +1530,50 @@ func buildServerName(hostname string) string { return `~^(?[\w-]+)\.` + strings.Join(parts, "\\.") + `$` } -func buildOriginRegex(origin string) string { - origin = regexp.QuoteMeta(origin) - origin = strings.Replace(origin, "\\*", `[A-Za-z0-9\-]+`, 1) - return fmt.Sprintf("(%s)", origin) -} - -func buildCorsOriginRegex(corsOrigins []string) string { - if len(corsOrigins) == 1 && corsOrigins[0] == "*" { - return "set $http_origin *;\nset $cors 'true';" +// parseComplexNGINXVar parses things like "$my${complex}ngx\$var" into +// [["$var", "complex", "my", "ngx"]]. In other words, 2nd and 3rd elements +// in the result are actual NGINX variable names, whereas first and 4th elements +// are string literals. +func parseComplexNginxVarIntoLuaTable(ngxVar string) string { + r := regexp.MustCompile(`(\\\$[0-9a-zA-Z_]+)|\$\{([0-9a-zA-Z_]+)\}|\$([0-9a-zA-Z_]+)|(\$|[^$\\]+)`) + matches := r.FindAllStringSubmatch(ngxVar, -1) + components := make([][]string, len(matches)) + for i, match := range matches { + components[i] = match[1:] } - originsRegex := "if ($http_origin ~* (" - for i, origin := range corsOrigins { - originTrimmed := strings.TrimSpace(origin) - if originTrimmed != "" { - builtOrigin := buildOriginRegex(originTrimmed) - originsRegex += builtOrigin - if i != len(corsOrigins)-1 { - originsRegex += "|" - } + luaTable, err := convertGoSliceIntoLuaTable(components, true) + if err != nil { + klog.Errorf("unexpected error: %v", err) + luaTable = "{}" + } + return luaTable +} + +func convertGoSliceIntoLuaTable(goSliceInterface interface{}, emptyStringAsNil bool) (string, error) { + goSlice := reflect.ValueOf(goSliceInterface) + kind := goSlice.Kind() + + switch kind { + case reflect.String: + if emptyStringAsNil && len(goSlice.Interface().(string)) == 0 { + return "nil", nil } + return fmt.Sprintf(`"%v"`, goSlice.Interface()), nil + case reflect.Int, reflect.Bool: + return fmt.Sprintf(`%v`, goSlice.Interface()), nil + case reflect.Slice, reflect.Array: + luaTable := "{ " + for i := 0; i < goSlice.Len(); i++ { + luaEl, err := convertGoSliceIntoLuaTable(goSlice.Index(i).Interface(), emptyStringAsNil) + if err != nil { + return "", err + } + luaTable = luaTable + luaEl + ", " + } + luaTable += "}" + return luaTable, nil + default: + return "", fmt.Errorf("could not process type: %s", kind) } - originsRegex += ")$ ) { set $cors 'true'; }" - return originsRegex } diff --git a/internal/ingress/controller/template/template_test.go b/internal/ingress/controller/template/template_test.go index 6553f5daf..1af988197 100644 --- a/internal/ingress/controller/template/template_test.go +++ b/internal/ingress/controller/template/template_test.go @@ -17,9 +17,9 @@ limitations under the License. package template import ( - "bytes" "encoding/base64" "fmt" + "io/ioutil" "net" "os" "path" @@ -29,51 +29,48 @@ import ( "testing" jsoniter "github.com/json-iterator/go" - "github.com/pmezard/go-difflib/difflib" apiv1 "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/ingress-nginx/internal/ingress" "k8s.io/ingress-nginx/internal/ingress/annotations/authreq" + "k8s.io/ingress-nginx/internal/ingress/annotations/influxdb" "k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity" - "k8s.io/ingress-nginx/internal/ingress/annotations/opentelemetry" + "k8s.io/ingress-nginx/internal/ingress/annotations/opentracing" "k8s.io/ingress-nginx/internal/ingress/annotations/ratelimit" "k8s.io/ingress-nginx/internal/ingress/annotations/rewrite" "k8s.io/ingress-nginx/internal/ingress/controller/config" "k8s.io/ingress-nginx/internal/nginx" - "k8s.io/ingress-nginx/pkg/apis/ingress" ) func init() { // the default value of nginx.TemplatePath assumes the template exists in // the root filesystem and not in the rootfs directory - absPath, err := filepath.Abs(filepath.Join("..", "..", "..", "..", "rootfs", nginx.TemplatePath)) + path, err := filepath.Abs(filepath.Join("../../../../rootfs/", nginx.TemplatePath)) if err == nil { - nginx.TemplatePath = absPath + nginx.TemplatePath = path } } var ( - pathPrefix networking.PathType = networking.PathTypePrefix - // TODO: add tests for SSLPassthrough tmplFuncTestcases = map[string]struct { - Path string - Target string - Location string - ProxyPass string - AutoHTTPProxyPass string - Sticky bool - XForwardedPrefix string - SecureBackend bool - enforceRegex bool + Path string + Target string + Location string + ProxyPass string + Sticky bool + XForwardedPrefix string + SecureBackend bool + enforceRegex bool }{ "when secure backend enabled": { "/", "/", "/", "proxy_pass https://upstream_balancer;", - "proxy_pass https://upstream_balancer;", false, "", true, @@ -84,7 +81,6 @@ var ( "/", "/", "proxy_pass https://upstream_balancer;", - "proxy_pass https://upstream_balancer;", false, "", true, @@ -95,7 +91,6 @@ var ( "/", "/", "proxy_pass https://upstream_balancer;", - "proxy_pass https://upstream_balancer;", true, "", true, @@ -106,7 +101,6 @@ var ( "/", "/", "proxy_pass http://upstream_balancer;", - "proxy_pass $scheme://upstream_balancer;", false, "", false, @@ -117,7 +111,6 @@ var ( "/", "/", "proxy_pass http://upstream_balancer;", - "proxy_pass $scheme://upstream_balancer;", false, "", false, @@ -130,9 +123,6 @@ var ( ` rewrite "(?i)/" /jenkins break; proxy_pass http://upstream_balancer;`, - ` -rewrite "(?i)/" /jenkins break; -proxy_pass $scheme://upstream_balancer;`, false, "", false, @@ -145,9 +135,6 @@ proxy_pass $scheme://upstream_balancer;`, ` rewrite "(?i)/" /something break; proxy_pass http://upstream_balancer;`, - ` -rewrite "(?i)/" /something break; -proxy_pass $scheme://upstream_balancer;`, true, "", false, @@ -160,9 +147,6 @@ proxy_pass $scheme://upstream_balancer;`, ` rewrite "(?i)/" /something break; proxy_pass http://upstream_balancer;`, - ` -rewrite "(?i)/" /something break; -proxy_pass $scheme://upstream_balancer;`, true, "", false, @@ -176,10 +160,6 @@ proxy_pass $scheme://upstream_balancer;`, rewrite "(?i)/there" /something break; proxy_set_header X-Forwarded-Prefix "/there"; proxy_pass http://upstream_balancer;`, - ` -rewrite "(?i)/there" /something break; -proxy_set_header X-Forwarded-Prefix "/there"; -proxy_pass $scheme://upstream_balancer;`, true, "/there", false, @@ -190,7 +170,6 @@ proxy_pass $scheme://upstream_balancer;`, "/something", `~* "^/something"`, "proxy_pass http://upstream_balancer;", - "proxy_pass $scheme://upstream_balancer;", false, "", false, @@ -199,20 +178,6 @@ proxy_pass $scheme://upstream_balancer;`, } ) -const ( - defaultBackend = "upstream-name" - defaultHost = "example.com" - fooAuthHost = "foo.com/auth" -) - -func getTestDataDir() (string, error) { - pwd, err := os.Getwd() - if err != nil { - return "", err - } - return path.Join(pwd, "../../../../test/data"), nil -} - func TestBuildLuaSharedDictionaries(t *testing.T) { invalidType := &ingress.Ingress{} expected := "" @@ -220,7 +185,7 @@ func TestBuildLuaSharedDictionaries(t *testing.T) { // config lua dict cfg := config.Configuration{ LuaSharedDicts: map[string]int{ - "configuration_data": 10240, "certificate_data": 20480, + "configuration_data": 10, "certificate_data": 20, }, } actual := buildLuaSharedDictionaries(cfg, invalidType) @@ -232,11 +197,11 @@ func TestBuildLuaSharedDictionaries(t *testing.T) { servers := []*ingress.Server{ { Hostname: "foo.bar", - Locations: []*ingress.Location{{Path: "/", PathType: &pathPrefix}}, + Locations: []*ingress.Location{{Path: "/"}}, }, { Hostname: "another.host", - Locations: []*ingress.Location{{Path: "/", PathType: &pathPrefix}}, + Locations: []*ingress.Location{{Path: "/"}}, }, } // returns value from config @@ -261,13 +226,13 @@ func TestBuildLuaSharedDictionaries(t *testing.T) { func TestLuaConfigurationRequestBodySize(t *testing.T) { cfg := config.Configuration{ LuaSharedDicts: map[string]int{ - "configuration_data": 10240, "certificate_data": 20480, + "configuration_data": 10, "certificate_data": 20, }, } size := luaConfigurationRequestBodySize(cfg) - if size != "21M" { - t.Errorf("expected the size to be 21M but got: %v", size) + if size != "21" { + t.Errorf("expected the size to be 20 but got: %v", size) } } @@ -318,9 +283,8 @@ func TestBuildLocation(t *testing.T) { for k, tc := range tmplFuncTestcases { loc := &ingress.Location{ - Path: tc.Path, - PathType: &pathPrefix, - Rewrite: rewrite.Config{Target: tc.Target}, + Path: tc.Path, + Rewrite: rewrite.Config{Target: tc.Target}, } newLoc := buildLocation(loc, tc.enforceRegex) @@ -331,17 +295,19 @@ func TestBuildLocation(t *testing.T) { } func TestBuildProxyPass(t *testing.T) { + defaultBackend := "upstream-name" + defaultHost := "example.com" + for k, tc := range tmplFuncTestcases { loc := &ingress.Location{ Path: tc.Path, - PathType: &pathPrefix, Rewrite: rewrite.Config{Target: tc.Target}, Backend: defaultBackend, XForwardedPrefix: tc.XForwardedPrefix, } if tc.SecureBackend { - loc.BackendProtocol = httpsProtocol + loc.BackendProtocol = "HTTPS" } backend := &ingress.Backend{ @@ -368,45 +334,6 @@ func TestBuildProxyPass(t *testing.T) { } } -func TestBuildProxyPassAutoHttp(t *testing.T) { - for k, tc := range tmplFuncTestcases { - loc := &ingress.Location{ - Path: tc.Path, - Rewrite: rewrite.Config{Target: tc.Target}, - Backend: defaultBackend, - XForwardedPrefix: tc.XForwardedPrefix, - } - - if tc.SecureBackend { - loc.BackendProtocol = httpsProtocol - } else { - loc.BackendProtocol = autoHTTPProtocol - } - - backend := &ingress.Backend{ - Name: defaultBackend, - } - - if tc.Sticky { - backend.SessionAffinity = ingress.SessionAffinityConfig{ - AffinityType: "cookie", - CookieSessionAffinity: ingress.CookieSessionAffinity{ - Locations: map[string][]string{ - defaultHost: {tc.Path}, - }, - }, - } - } - - backends := []*ingress.Backend{backend} - - pp := buildProxyPass(defaultHost, backends, loc) - if !strings.EqualFold(tc.AutoHTTPProxyPass, pp) { - t.Errorf("%s: expected \n'%v'\nbut returned \n'%v'", k, tc.ProxyPass, pp) - } - } -} - func TestBuildAuthLocation(t *testing.T) { invalidType := &ingress.Ingress{} expected := "" @@ -416,7 +343,7 @@ func TestBuildAuthLocation(t *testing.T) { t.Errorf("Expected '%v' but returned '%v'", expected, actual) } - authURL := fooAuthHost + authURL := "foo.com/auth" globalAuthURL := "foo.com/global-auth" loc := &ingress.Location{ @@ -427,7 +354,7 @@ func TestBuildAuthLocation(t *testing.T) { EnableGlobalAuth: true, } - encodedAuthURL := strings.ReplaceAll(base64.URLEncoding.EncodeToString([]byte(loc.Path)), "=", "") + encodedAuthURL := strings.Replace(base64.URLEncoding.EncodeToString([]byte(loc.Path)), "=", "", -1) externalAuthPath := fmt.Sprintf("/_external-auth-%v-default", encodedAuthURL) testCases := []struct { @@ -459,7 +386,8 @@ func TestBuildAuthLocation(t *testing.T) { } func TestShouldApplyGlobalAuth(t *testing.T) { - authURL := fooAuthHost + + authURL := "foo.com/auth" globalAuthURL := "foo.com/global-auth" loc := &ingress.Location{ @@ -500,46 +428,14 @@ func TestShouldApplyGlobalAuth(t *testing.T) { func TestBuildAuthResponseHeaders(t *testing.T) { externalAuthResponseHeaders := []string{"h1", "H-With-Caps-And-Dashes"} - tests := []struct { - headers []string - expected []string - lua bool - }{ - { - headers: externalAuthResponseHeaders, - lua: false, - expected: []string{ - "auth_request_set $authHeader0 $upstream_http_h1;", - "proxy_set_header 'h1' $authHeader0;", - "auth_request_set $authHeader1 $upstream_http_h_with_caps_and_dashes;", - "proxy_set_header 'H-With-Caps-And-Dashes' $authHeader1;", - }, - }, - { - headers: externalAuthResponseHeaders, - lua: true, - expected: []string{ - "set $authHeader0 '';", - "proxy_set_header 'h1' $authHeader0;", - "set $authHeader1 '';", - "proxy_set_header 'H-With-Caps-And-Dashes' $authHeader1;", - }, - }, + expected := []string{ + "auth_request_set $authHeader0 $upstream_http_h1;", + "proxy_set_header 'h1' $authHeader0;", + "auth_request_set $authHeader1 $upstream_http_h_with_caps_and_dashes;", + "proxy_set_header 'H-With-Caps-And-Dashes' $authHeader1;", } - for _, test := range tests { - got := buildAuthResponseHeaders(proxySetHeader(nil), test.headers, test.lua) - if !reflect.DeepEqual(test.expected, got) { - t.Errorf("Expected \n'%v'\nbut returned \n'%v'", test.expected, got) - } - } -} - -func TestBuildAuthResponseLua(t *testing.T) { - externalAuthResponseHeaders := []string{"h1", "H-With-Caps-And-Dashes"} - expected := "h1,H-With-Caps-And-Dashes" - - headers := buildAuthUpstreamLuaHeaders(externalAuthResponseHeaders) + headers := buildAuthResponseHeaders(externalAuthResponseHeaders) if !reflect.DeepEqual(expected, headers) { t.Errorf("Expected \n'%v'\nbut returned \n'%v'", expected, headers) @@ -563,150 +459,14 @@ func TestBuildAuthProxySetHeaders(t *testing.T) { } } -func TestBuildAuthUpstreamName(t *testing.T) { - invalidType := &ingress.Ingress{} - expected := "" - actual := buildAuthUpstreamName(invalidType, "") - - if !reflect.DeepEqual(expected, actual) { - t.Errorf("Expected '%v' but returned '%v'", expected, actual) - } - - loc := &ingress.Location{ - ExternalAuth: authreq.Config{ - URL: fooAuthHost, - }, - Path: "/cat", - } - - encodedAuthURL := strings.ReplaceAll(base64.URLEncoding.EncodeToString([]byte(loc.Path)), "=", "") - externalAuthPath := fmt.Sprintf("external-auth-%v-default", encodedAuthURL) - - testCases := []struct { - title string - host string - expected string - }{ - {"valid host", "auth.my.site", fmt.Sprintf("%s-%s", "auth.my.site", externalAuthPath)}, - {"valid host", "your.auth.site", fmt.Sprintf("%s-%s", "your.auth.site", externalAuthPath)}, - {"empty host", "", ""}, - } - - for _, testCase := range testCases { - str := buildAuthUpstreamName(loc, testCase.host) - if str != testCase.expected { - t.Errorf("%v: expected '%v' but returned '%v'", testCase.title, testCase.expected, str) - } - } -} - -func TestShouldApplyAuthUpstream(t *testing.T) { - authURL := fooAuthHost - - loc := &ingress.Location{ - ExternalAuth: authreq.Config{ - URL: authURL, - KeepaliveConnections: 0, - }, - Path: "/cat", - } - - cfg := config.Configuration{ - UseHTTP2: false, - } - - testCases := []struct { - title string - authURL string - keepaliveConnections int - expected bool - }{ - {"authURL, no keepalive", authURL, 0, false}, - {"authURL, keepalive", authURL, 10, true}, - {"empty, no keepalive", "", 0, false}, - {"empty, keepalive", "", 10, false}, - } - - for _, testCase := range testCases { - loc.ExternalAuth.URL = testCase.authURL - loc.ExternalAuth.KeepaliveConnections = testCase.keepaliveConnections - - result := shouldApplyAuthUpstream(loc, cfg) - if result != testCase.expected { - t.Errorf("%v: expected '%v' but returned '%v'", testCase.title, testCase.expected, result) - } - } - - // keepalive is not supported with UseHTTP2 - cfg.UseHTTP2 = true - for _, testCase := range testCases { - loc.ExternalAuth.URL = testCase.authURL - loc.ExternalAuth.KeepaliveConnections = testCase.keepaliveConnections - - result := shouldApplyAuthUpstream(loc, cfg) - if result != false { - t.Errorf("%v: expected '%v' but returned '%v'", testCase.title, false, result) - } - } -} - -func TestExtractHostPort(t *testing.T) { - testCases := []struct { - title string - url string - expected string - }{ - {"full URL", "https://my.auth.site:5000/path", "my.auth.site:5000"}, - {"URL with no port", "http://my.auth.site/path", "my.auth.site"}, - {"URL with no path", "https://my.auth.site:5000", "my.auth.site:5000"}, - {"URL no port and path", "http://my.auth.site", "my.auth.site"}, - {"missing method", "my.auth.site/path", ""}, - {"all empty", "", ""}, - } - - for _, testCase := range testCases { - result := extractHostPort(testCase.url) - if result != testCase.expected { - t.Errorf("%v: expected '%v' but returned '%v'", testCase.title, testCase.expected, result) - } - } -} - -func TestChangeHostPort(t *testing.T) { - testCases := []struct { - title string - url string - host string - expected string - }{ - {"full URL", "https://my.auth.site:5000/path", "your.domain", "https://your.domain/path"}, - {"URL with no port", "http://my.auth.site/path", "your.domain", "http://your.domain/path"}, - {"URL with no path", "http://my.auth.site:5000", "your.domain:8888", "http://your.domain:8888"}, - {"URL with no port and path", "https://my.auth.site", "your.domain:8888", "https://your.domain:8888"}, - {"invalid host", "my.auth.site/path", "", ""}, - {"missing method", "my.auth.site/path", "your.domain", ""}, - {"all empty", "", "", ""}, - } - - for _, testCase := range testCases { - result := changeHostPort(testCase.url, testCase.host) - if result != testCase.expected { - t.Errorf("%v: expected '%v' but returned '%v'", testCase.title, testCase.expected, result) - } - } -} - func TestTemplateWithData(t *testing.T) { - pwd, err := os.Getwd() - if err != nil { - t.Errorf("unexpected error: %v", err) - } + pwd, _ := os.Getwd() f, err := os.Open(path.Join(pwd, "../../../../test/data/config.json")) if err != nil { t.Errorf("unexpected error reading json file: %v", err) } defer f.Close() - data, err := os.ReadFile(f.Name()) + data, err := ioutil.ReadFile(f.Name()) if err != nil { t.Error("unexpected error reading json file: ", err) } @@ -725,7 +485,7 @@ func TestTemplateWithData(t *testing.T) { dat.Cfg.DefaultSSLCertificate = &ingress.SSLCert{} - rt, err := ngxTpl.Write(&dat) + rt, err := ngxTpl.Write(dat) if err != nil { t.Errorf("invalid NGINX template: %v", err) } @@ -744,16 +504,13 @@ func TestTemplateWithData(t *testing.T) { } func BenchmarkTemplateWithData(b *testing.B) { - pwd, err := os.Getwd() - if err != nil { - b.Errorf("unexpected error: %v", err) - } + pwd, _ := os.Getwd() f, err := os.Open(path.Join(pwd, "../../../../test/data/config.json")) if err != nil { b.Errorf("unexpected error reading json file: %v", err) } defer f.Close() - data, err := os.ReadFile(f.Name()) + data, err := ioutil.ReadFile(f.Name()) if err != nil { b.Error("unexpected error reading json file: ", err) } @@ -768,9 +525,7 @@ func BenchmarkTemplateWithData(b *testing.B) { } for i := 0; i < b.N; i++ { - if _, err := ngxTpl.Write(&dat); err != nil { - b.Errorf("unexpected error writing template: %v", err) - } + ngxTpl.Write(dat) } } @@ -1067,17 +822,19 @@ func TestBuildUpstreamName(t *testing.T) { t.Errorf("Expected '%v' but returned '%v'", expected, actual) } + defaultBackend := "upstream-name" + defaultHost := "example.com" + for k, tc := range tmplFuncTestcases { loc := &ingress.Location{ Path: tc.Path, - PathType: &pathPrefix, Rewrite: rewrite.Config{Target: tc.Target}, Backend: defaultBackend, XForwardedPrefix: tc.XForwardedPrefix, } if tc.SecureBackend { - loc.BackendProtocol = httpsProtocol + loc.BackendProtocol = "HTTPS" } backend := &ingress.Backend{ @@ -1130,19 +887,19 @@ func TestEscapeLiteralDollar(t *testing.T) { } } -func TestOpentelemetryPropagateContext(t *testing.T) { +func TestOpentracingPropagateContext(t *testing.T) { tests := map[*ingress.Location]string{ - {BackendProtocol: httpProtocol}: "opentelemetry_propagate;", - {BackendProtocol: httpsProtocol}: "opentelemetry_propagate;", - {BackendProtocol: autoHTTPProtocol}: "opentelemetry_propagate;", - {BackendProtocol: grpcProtocol}: "opentelemetry_propagate;", - {BackendProtocol: grpcsProtocol}: "opentelemetry_propagate;", - {BackendProtocol: fcgiProtocol}: "opentelemetry_propagate;", - nil: "", + {BackendProtocol: "HTTP"}: "opentracing_propagate_context;", + {BackendProtocol: "HTTPS"}: "opentracing_propagate_context;", + {BackendProtocol: "GRPC"}: "opentracing_grpc_propagate_context;", + {BackendProtocol: "GRPCS"}: "opentracing_grpc_propagate_context;", + {BackendProtocol: "AJP"}: "opentracing_propagate_context;", + {BackendProtocol: "FCGI"}: "opentracing_propagate_context;", + nil: "", } for loc, expectedDirective := range tests { - actualDirective := opentelemetryPropagateContext(loc) + actualDirective := opentracingPropagateContext(loc) if actualDirective != expectedDirective { t.Errorf("Expected %v but returned %v", expectedDirective, actualDirective) } @@ -1150,6 +907,7 @@ func TestOpentelemetryPropagateContext(t *testing.T) { } func TestGetIngressInformation(t *testing.T) { + testcases := map[string]struct { Ingress interface{} Host string @@ -1168,76 +926,6 @@ func TestGetIngressInformation(t *testing.T) { 10, &ingressInformation{}, }, - "valid ingress definition with name validIng in namespace default using a service with name a-svc port number 8080": { - &ingress.Ingress{ - Ingress: networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "validIng", - Namespace: apiv1.NamespaceDefault, - Annotations: map[string]string{ - "ingress.annotation": "ok", - }, - }, - Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "a-svc", - Port: networking.ServiceBackendPort{ - Number: 8080, - }, - }, - }, - }, - }, - }, - "host1", - "", - &ingressInformation{ - Namespace: "default", - Rule: "validIng", - Path: "/", - Annotations: map[string]string{ - "ingress.annotation": "ok", - }, - Service: "a-svc", - ServicePort: "8080", - }, - }, - "valid ingress definition with name validIng in namespace default using a service with name a-svc port name b-svc": { - &ingress.Ingress{ - Ingress: networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "validIng", - Namespace: apiv1.NamespaceDefault, - Annotations: map[string]string{ - "ingress.annotation": "ok", - }, - }, - Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "a-svc", - Port: networking.ServiceBackendPort{ - Name: "b-svc", - }, - }, - }, - }, - }, - }, - "host1", - "", - &ingressInformation{ - Namespace: "default", - Rule: "validIng", - Path: "/", - Annotations: map[string]string{ - "ingress.annotation": "ok", - }, - Service: "a-svc", - ServicePort: "b-svc", - }, - }, "valid ingress definition with name validIng in namespace default": { &ingress.Ingress{ Ingress: networking.Ingress{ @@ -1249,10 +937,8 @@ func TestGetIngressInformation(t *testing.T) { }, }, Spec: networking.IngressSpec{ - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "a-svc", - }, + Backend: &networking.IngressBackend{ + ServiceName: "a-svc", }, }, }, @@ -1262,7 +948,6 @@ func TestGetIngressInformation(t *testing.T) { &ingressInformation{ Namespace: "default", Rule: "validIng", - Path: "/", Annotations: map[string]string{ "ingress.annotation": "ok", }, @@ -1287,15 +972,10 @@ func TestGetIngressInformation(t *testing.T) { HTTP: &networking.HTTPIngressRuleValue{ Paths: []networking.HTTPIngressPath{ { - Path: "/ok", - PathType: &pathPrefix, + Path: "/ok", Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "b-svc", - Port: networking.ServiceBackendPort{ - Number: 80, - }, - }, + ServiceName: "b-svc", + ServicePort: intstr.FromInt(80), }, }, }, @@ -1319,156 +999,6 @@ func TestGetIngressInformation(t *testing.T) { ServicePort: "80", }, }, - "valid ingress definition with name demo in namespace something and path /ok using a service with name b-svc port name b-svc-80": { - &ingress.Ingress{ - Ingress: networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "demo", - Namespace: "something", - Annotations: map[string]string{ - "ingress.annotation": "ok", - }, - }, - Spec: networking.IngressSpec{ - Rules: []networking.IngressRule{ - { - Host: "foo.bar", - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - Path: "/ok", - PathType: &pathPrefix, - Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "b-svc", - Port: networking.ServiceBackendPort{ - Name: "b-svc-80", - }, - }, - }, - }, - }, - }, - }, - }, - {}, - }, - }, - }, - }, - "foo.bar", - "/ok", - &ingressInformation{ - Namespace: "something", - Rule: "demo", - Annotations: map[string]string{ - "ingress.annotation": "ok", - }, - Service: "b-svc", - ServicePort: "b-svc-80", - }, - }, - "valid ingress definition with name demo in namespace something and path /ok with a nil backend service": { - &ingress.Ingress{ - Ingress: networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "demo", - Namespace: "something", - Annotations: map[string]string{ - "ingress.annotation": "ok", - }, - }, - Spec: networking.IngressSpec{ - Rules: []networking.IngressRule{ - { - Host: "foo.bar", - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - Path: "/ok", - PathType: &pathPrefix, - Backend: networking.IngressBackend{ - Service: nil, - }, - }, - }, - }, - }, - }, - {}, - }, - }, - }, - }, - "foo.bar", - "/ok", - &ingressInformation{ - Namespace: "something", - Rule: "demo", - Annotations: map[string]string{ - "ingress.annotation": "ok", - }, - }, - }, - "valid ingress definition with name demo in namespace something and path /ok with both a nil service and a valid one": { - &ingress.Ingress{ - Ingress: networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "demo", - Namespace: "something", - Annotations: map[string]string{ - "ingress.annotation": "ok", - }, - }, - Spec: networking.IngressSpec{ - Rules: []networking.IngressRule{ - { - Host: "foo.bar", - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - Path: "/ok", - PathType: &pathPrefix, - Backend: networking.IngressBackend{ - Service: nil, - }, - }, - { - Path: "/oksvc", - PathType: &pathPrefix, - Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "b-svc", - Port: networking.ServiceBackendPort{ - Name: "b-svc-80", - }, - }, - }, - }, - }, - }, - }, - }, - {}, - }, - }, - }, - }, - "foo.bar", - "/oksvc", - &ingressInformation{ - Namespace: "something", - Rule: "demo", - Annotations: map[string]string{ - "ingress.annotation": "ok", - }, - Service: "b-svc", - ServicePort: "b-svc-80", - }, - }, } for title, testCase := range testcases { @@ -1580,72 +1110,128 @@ func TestBuildCustomErrorLocationsPerServer(t *testing.T) { } func TestProxySetHeader(t *testing.T) { - tests := []struct { - name string - loc interface{} - expected string - }{ - { - name: "nil", - loc: nil, - expected: "proxy_set_header", - }, - { - name: "invalid type", - loc: &ingress.Ingress{}, - expected: "proxy_set_header", - }, - { - name: "http backend", - loc: &ingress.Location{}, - expected: "proxy_set_header", - }, - { - name: "gRPC backend", - loc: &ingress.Location{ - BackendProtocol: grpcProtocol, - }, - expected: "grpc_set_header", - }, + invalidType := &ingress.Ingress{} + expected := "proxy_set_header" + actual := proxySetHeader(invalidType) + + if expected != actual { + t.Errorf("Expected '%v' but returned '%v'", expected, actual) } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := proxySetHeader(tt.loc); got != tt.expected { - t.Errorf("proxySetHeader() = %v, expected %v", got, tt.expected) - } - }) + + grpcBackend := &ingress.Location{ + BackendProtocol: "GRPC", + } + + expected = "grpc_set_header" + actual = proxySetHeader(grpcBackend) + + if expected != actual { + t.Errorf("Expected '%v' but returned '%v'", expected, actual) } } -func TestBuildOpenTelemetry(t *testing.T) { +func TestBuildInfluxDB(t *testing.T) { invalidType := &ingress.Ingress{} expected := "" - actual := buildOpentelemetry(invalidType, []*ingress.Server{}) + actual := buildInfluxDB(invalidType) + + if expected != actual { + t.Errorf("Expected '%v' but returned '%v'", expected, actual) + } + + cfg := influxdb.Config{ + InfluxDBEnabled: true, + InfluxDBServerName: "ok.com", + InfluxDBHost: "host.com", + InfluxDBPort: "5252", + InfluxDBMeasurement: "ok", + } + expected = "influxdb server_name=ok.com host=host.com port=5252 measurement=ok enabled=true;" + actual = buildInfluxDB(cfg) + + if expected != actual { + t.Errorf("Expected '%v' but returned '%v'", expected, actual) + } +} + +func TestBuildOpenTracing(t *testing.T) { + invalidType := &ingress.Ingress{} + expected := "" + actual := buildOpentracing(invalidType, []*ingress.Server{}) if expected != actual { t.Errorf("Expected '%v' but returned '%v'", expected, actual) } cfgNoHost := config.Configuration{ - EnableOpentelemetry: true, + EnableOpentracing: true, } expected = "\r\n" - actual = buildOpentelemetry(cfgNoHost, []*ingress.Server{}) + actual = buildOpentracing(cfgNoHost, []*ingress.Server{}) if expected != actual { t.Errorf("Expected '%v' but returned '%v'", expected, actual) } - cfgOpenTelemetry := config.Configuration{ - EnableOpentelemetry: true, - OpentelemetryOperationName: "my-operation-name", + cfgJaeger := config.Configuration{ + EnableOpentracing: true, + JaegerCollectorHost: "jaeger-host.com", } - expected = "\r\n" - expected += "opentelemetry_operation_name \"my-operation-name\";\n" - actual = buildOpentelemetry(cfgOpenTelemetry, []*ingress.Server{}) + expected = "opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.so /etc/nginx/opentracing.json;\r\n" + actual = buildOpentracing(cfgJaeger, []*ingress.Server{}) + if expected != actual { t.Errorf("Expected '%v' but returned '%v'", expected, actual) } + + cfgZipkin := config.Configuration{ + EnableOpentracing: true, + ZipkinCollectorHost: "zipkin-host.com", + } + expected = "opentracing_load_tracer /usr/local/lib/libzipkin_opentracing_plugin.so /etc/nginx/opentracing.json;\r\n" + actual = buildOpentracing(cfgZipkin, []*ingress.Server{}) + + if expected != actual { + t.Errorf("Expected '%v' but returned '%v'", expected, actual) + } + + cfgDatadog := config.Configuration{ + EnableOpentracing: true, + DatadogCollectorHost: "datadog-host.com", + } + expected = "opentracing_load_tracer /usr/local/lib64/libdd_opentracing.so /etc/nginx/opentracing.json;\r\n" + actual = buildOpentracing(cfgDatadog, []*ingress.Server{}) + + if expected != actual { + t.Errorf("Expected '%v' but returned '%v'", expected, actual) + } + + cfgJaegerEndpoint := config.Configuration{ + EnableOpentracing: true, + JaegerEndpoint: "http://jaeger-collector.com:14268/api/traces", + } + expected = "opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.so /etc/nginx/opentracing.json;\r\n" + actual = buildOpentracing(cfgJaegerEndpoint, []*ingress.Server{}) + + if expected != actual { + t.Errorf("Expected '%v' but returned '%v'", expected, actual) + } + + cfgOpenTracing := config.Configuration{ + EnableOpentracing: true, + DatadogCollectorHost: "datadog-host.com", + OpentracingOperationName: "my-operation-name", + OpentracingLocationOperationName: "my-location-operation-name", + } + expected = "opentracing_load_tracer /usr/local/lib64/libdd_opentracing.so /etc/nginx/opentracing.json;\r\n" + expected += "opentracing_operation_name \"my-operation-name\";\n" + expected += "opentracing_location_operation_name \"my-location-operation-name\";\n" + actual = buildOpentracing(cfgOpenTracing, []*ingress.Server{}) + + if expected != actual { + t.Errorf("Expected '%v' but returned '%v'", expected, actual) + } + } func TestEnforceRegexModifier(t *testing.T) { @@ -1663,8 +1249,7 @@ func TestEnforceRegexModifier(t *testing.T) { Target: "/alright", UseRegex: true, }, - Path: "/ok", - PathType: &pathPrefix, + Path: "/ok", }, } expected = true @@ -1675,7 +1260,6 @@ func TestEnforceRegexModifier(t *testing.T) { } } -//nolint:dupl // Ignore dupl errors for similar test case func TestShouldLoadModSecurityModule(t *testing.T) { // ### Invalid argument type tests ### // The first tests should return false. @@ -1729,47 +1313,34 @@ func TestShouldLoadModSecurityModule(t *testing.T) { } } -func TestOpentelemetryForLocation(t *testing.T) { +func TestOpentracingForLocation(t *testing.T) { trueVal := true - falseVal := false - loadOT := `opentelemetry on; -opentelemetry_propagate; -opentelemetry_trust_incoming_spans on;` - loadOTUntrustedSpan := `opentelemetry on; -opentelemetry_propagate; -opentelemetry_trust_incoming_spans off;` + loadOT := `opentracing on; +opentracing_propagate_context;` testCases := []struct { - description string - globalOT bool - isSetInLoc bool - isOTInLoc *bool - globalTrust bool - isTrustSetInLoc bool - isTrustInLoc *bool - expected string + description string + globalOT bool + isSetInLoc bool + isOTInLoc *bool + expected string }{ - {"globally enabled, without annotation", true, false, nil, true, false, nil, loadOT}, - {"globally enabled and enabled in location", true, true, &trueVal, true, false, nil, loadOT}, - {"globally disabled and not enabled in location", false, false, nil, true, false, nil, ""}, - {"globally disabled but enabled in location", false, true, &trueVal, true, false, nil, loadOT}, - {"globally trusted, not trusted in location", true, false, nil, true, true, &falseVal, loadOTUntrustedSpan}, - {"not globally trusted, trust set in location", true, false, nil, false, true, &trueVal, loadOT}, - {"not globally trusted, trust not set in location", true, false, nil, false, false, nil, loadOTUntrustedSpan}, + {"globally enabled, without annotation", true, false, nil, loadOT}, + {"globally enabled and enabled in location", true, true, &trueVal, loadOT}, + {"globally disabled and not enabled in location", false, false, nil, ""}, + {"globally disabled but enabled in location", false, true, &trueVal, loadOT}, + {"globally disabled, enabled in location but false", false, true, &trueVal, loadOT}, } for _, testCase := range testCases { il := &ingress.Location{ - Opentelemetry: opentelemetry.Config{Set: testCase.isSetInLoc, TrustSet: testCase.isTrustSetInLoc}, + Opentracing: opentracing.Config{Set: testCase.isSetInLoc}, } - if il.Opentelemetry.Set { - il.Opentelemetry.Enabled = *testCase.isOTInLoc - } - if il.Opentelemetry.TrustSet { - il.Opentelemetry.TrustEnabled = *testCase.isTrustInLoc + if il.Opentracing.Set { + il.Opentracing.Enabled = *testCase.isOTInLoc } - actual := buildOpentelemetryForLocation(testCase.globalOT, testCase.globalTrust, il) + actual := buildOpentracingForLocation(testCase.globalOT, il) if testCase.expected != actual { t.Errorf("%v: expected '%v' but returned '%v'", testCase.description, testCase.expected, actual) @@ -1777,25 +1348,24 @@ opentelemetry_trust_incoming_spans off;` } } -//nolint:dupl // Ignore dupl errors for similar test case -func TestShouldLoadOpentelemetryModule(t *testing.T) { +func TestShouldLoadOpentracingModule(t *testing.T) { // ### Invalid argument type tests ### // The first tests should return false. expected := false invalidType := &ingress.Ingress{} - actual := shouldLoadOpentelemetryModule(config.Configuration{}, invalidType) + actual := shouldLoadOpentracingModule(config.Configuration{}, invalidType) if expected != actual { t.Errorf("Expected '%v' but returned '%v'", expected, actual) } - actual = shouldLoadOpentelemetryModule(invalidType, []*ingress.Server{}) + actual = shouldLoadOpentracingModule(invalidType, []*ingress.Server{}) if expected != actual { t.Errorf("Expected '%v' but returned '%v'", expected, actual) } // ### Functional tests ### - actual = shouldLoadOpentelemetryModule(config.Configuration{}, []*ingress.Server{}) + actual = shouldLoadOpentracingModule(config.Configuration{}, []*ingress.Server{}) if expected != actual { t.Errorf("Expected '%v' but returned '%v'", expected, actual) } @@ -1803,8 +1373,8 @@ func TestShouldLoadOpentelemetryModule(t *testing.T) { // All further tests should return true. expected = true - configuration := config.Configuration{EnableOpentelemetry: true} - actual = shouldLoadOpentelemetryModule(configuration, []*ingress.Server{}) + configuration := config.Configuration{EnableOpentracing: true} + actual = shouldLoadOpentracingModule(configuration, []*ingress.Server{}) if expected != actual { t.Errorf("Expected '%v' but returned '%v'", expected, actual) } @@ -1813,19 +1383,19 @@ func TestShouldLoadOpentelemetryModule(t *testing.T) { { Locations: []*ingress.Location{ { - Opentelemetry: opentelemetry.Config{ + Opentracing: opentracing.Config{ Enabled: true, }, }, }, }, } - actual = shouldLoadOpentelemetryModule(config.Configuration{}, servers) + actual = shouldLoadOpentracingModule(config.Configuration{}, servers) if expected != actual { t.Errorf("Expected '%v' but returned '%v'", expected, actual) } - actual = shouldLoadOpentelemetryModule(configuration, servers) + actual = shouldLoadOpentracingModule(configuration, servers) if expected != actual { t.Errorf("Expected '%v' but returned '%v'", expected, actual) } @@ -1877,8 +1447,8 @@ func TestModSecurityForLocation(t *testing.T) { {"configmap enabled, configmap OWASP enabled, annotation enabled, OWASP disabled", true, true, true, true, false, "", "", ""}, {"configmap disabled, annotation enabled, OWASP disabled", false, false, true, true, false, "", "", fmt.Sprintf("%v%v", loadModule, modSecCfg)}, {"configmap disabled, annotation disabled, OWASP disabled", false, false, false, true, false, "", "", ""}, - {"configmap disabled, annotation enabled, OWASP disabled", false, false, true, true, false, testRule, "", fmt.Sprintf("%v%v", loadModule, modsecRule)}, - {"configmap disabled, annotation enabled, OWASP enabled", false, false, true, true, false, testRule, "", fmt.Sprintf("%v%v", loadModule, modsecRule)}, + {"configmap disabled, annotation enabled, OWASP disabled", false, false, true, true, false, testRule, "", fmt.Sprintf("%v%v%v", loadModule, modsecRule, modSecCfg)}, + {"configmap disabled, annotation enabled, OWASP enabled", false, false, true, true, false, testRule, "", fmt.Sprintf("%v%v%v", loadModule, modsecRule, modSecCfg)}, } for _, testCase := range testCases { @@ -1904,6 +1474,7 @@ func TestModSecurityForLocation(t *testing.T) { } func TestBuildServerName(t *testing.T) { + testCases := []struct { title string hostname string @@ -1923,33 +1494,85 @@ func TestBuildServerName(t *testing.T) { } } -func TestCleanConf(t *testing.T) { - testDataDir, err := getTestDataDir() - if err != nil { - t.Error("unexpected error reading conf file: ", err) - } - actual := &bytes.Buffer{} - { - data, err := os.ReadFile(testDataDir + "/cleanConf.src.conf") - if err != nil { - t.Error("unexpected error reading conf file: ", err) - } - in := bytes.NewBuffer(data) - err = cleanConf(in, actual) - if err != nil { - t.Error("cleanConf failed: ", err) - } +func TestParseComplexNginxVarIntoLuaTable(t *testing.T) { + testCases := []struct { + ngxVar string + expectedLuaTable string + }{ + {"foo", `{ { nil, nil, nil, "foo", }, }`}, + {"$foo", `{ { nil, nil, "foo", nil, }, }`}, + {"${foo}", `{ { nil, "foo", nil, nil, }, }`}, + {"\\$foo", `{ { "\$foo", nil, nil, nil, }, }`}, + { + "foo\\$bar$baz${daz}xiyar$pomidor", + `{ { nil, nil, nil, "foo", }, { "\$bar", nil, nil, nil, }, { nil, nil, "baz", nil, }, ` + + `{ nil, "daz", nil, nil, }, { nil, nil, nil, "xiyar", }, { nil, nil, "pomidor", nil, }, }`, + }, } - expected, err := os.ReadFile(testDataDir + "/cleanConf.expected.conf") - if err != nil { - t.Error("unexpected error reading conf file: ", err) - } - if !bytes.Equal(expected, actual.Bytes()) { - diff, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{A: strings.SplitAfter(string(expected), "\n"), B: strings.SplitAfter(actual.String(), "\n"), Context: 3}) - if err != nil { - t.Error("failed to get diff for cleanConf", err) + for _, testCase := range testCases { + actualLuaTable := parseComplexNginxVarIntoLuaTable(testCase.ngxVar) + if actualLuaTable != testCase.expectedLuaTable { + t.Errorf("expected %v but returned %v", testCase.expectedLuaTable, actualLuaTable) + } + } +} + +func TestConvertGoSliceIntoLuaTablet(t *testing.T) { + testCases := []struct { + title string + goSlice interface{} + emptyStringAsNil bool + expectedLuaTable string + expectedErr error + }{ + { + "flat string slice", + []string{"one", "two", "three"}, + false, + `{ "one", "two", "three", }`, + nil, + }, + { + "nested string slice", + [][]string{{"one", "", "three"}, {"foo", "bar"}}, + false, + `{ { "one", "", "three", }, { "foo", "bar", }, }`, + nil, + }, + { + "converts empty string to nil when enabled", + [][]string{{"one", "", "three"}, {"foo", "bar"}}, + true, + `{ { "one", nil, "three", }, { "foo", "bar", }, }`, + nil, + }, + { + "boolean slice", + []bool{true, true, false}, + false, + `{ true, true, false, }`, + nil, + }, + { + "integer slice", + []int{4, 3, 6}, + false, + `{ 4, 3, 6, }`, + nil, + }, + } + + for _, testCase := range testCases { + actualLuaTable, err := convertGoSliceIntoLuaTable(testCase.goSlice, testCase.emptyStringAsNil) + if testCase.expectedErr != nil && err != nil && testCase.expectedErr.Error() != err.Error() { + t.Errorf("expected error '%v' but returned '%v'", testCase.expectedErr, err) + } + if testCase.expectedErr == nil && err != nil { + t.Errorf("expected error to be nil but returned '%v'", err) + } + if testCase.expectedLuaTable != actualLuaTable { + t.Errorf("%v: expected '%v' but returned '%v'", testCase.title, testCase.expectedLuaTable, actualLuaTable) } - t.Errorf("cleanConf result don't match with expected: %s", diff) } } diff --git a/internal/ingress/controller/util.go b/internal/ingress/controller/util.go index 975fb822a..5946cb47c 100644 --- a/internal/ingress/controller/util.go +++ b/internal/ingress/controller/util.go @@ -18,6 +18,7 @@ package controller import ( "fmt" + "io/ioutil" "os" "os/exec" "path" @@ -26,10 +27,9 @@ import ( "syscall" api "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/ingress-nginx/pkg/apis/ingress" - klog "k8s.io/klog/v2" + "k8s.io/ingress-nginx/internal/ingress" + "k8s.io/klog/v2" ) // newUpstream creates an upstream without servers. @@ -47,30 +47,8 @@ func newUpstream(name string) *ingress.Backend { } // upstreamName returns a formatted upstream name based on namespace, service, and port -func upstreamName(namespace string, service *networking.IngressServiceBackend) string { - if service != nil { - if service.Port.Number > 0 { - return fmt.Sprintf("%s-%s-%d", namespace, service.Name, service.Port.Number) - } - if service.Port.Name != "" { - return fmt.Sprintf("%s-%s-%s", namespace, service.Name, service.Port.Name) - } - } - return fmt.Sprintf("%s-INVALID", namespace) -} - -// upstreamServiceNameAndPort verifies if service is not nil, and then return the -// correct serviceName and Port -func upstreamServiceNameAndPort(service *networking.IngressServiceBackend) (string, intstr.IntOrString) { - if service != nil { - if service.Port.Number > 0 { - return service.Name, intstr.FromInt(int(service.Port.Number)) - } - if service.Port.Name != "" { - return service.Name, intstr.FromString(service.Port.Name) - } - } - return "", intstr.IntOrString{} +func upstreamName(namespace string, service string, port intstr.IntOrString) string { + return fmt.Sprintf("%v-%v-%v", namespace, service, port.String()) } // sysctlSomaxconn returns the maximum number of connections that can be queued @@ -98,9 +76,8 @@ func rlimitMaxNumFiles() int { } const ( - defBinary = "/usr/bin/nginx" - cfgPath = "/etc/nginx/nginx.conf" - luaCfgPath = "/etc/nginx/lua/cfg.json" + defBinary = "/usr/local/nginx/sbin/nginx" + cfgPath = "/etc/nginx/nginx.conf" ) // NginxExecTester defines the interface to execute @@ -130,25 +107,23 @@ func NewNginxCommand() NginxCommand { return command } -// ExecCommand instantiates an exec.Cmd object to call nginx program +// ExecCommand instanciates an exec.Cmd object to call nginx program func (nc NginxCommand) ExecCommand(args ...string) *exec.Cmd { cmdArgs := []string{} cmdArgs = append(cmdArgs, "-c", cfgPath) cmdArgs = append(cmdArgs, args...) - //nolint:gosec // Ignore G204 error return exec.Command(nc.Binary, cmdArgs...) } // Test checks if config file is a syntax valid nginx configuration func (nc NginxCommand) Test(cfg string) ([]byte, error) { - //nolint:gosec // Ignore G204 error return exec.Command(nc.Binary, "-c", cfg, "-t").CombinedOutput() } // getSysctl returns the value for the specified sysctl setting func getSysctl(sysctl string) (int, error) { - data, err := os.ReadFile(path.Join("/proc/sys", sysctl)) + data, err := ioutil.ReadFile(path.Join("/proc/sys", sysctl)) if err != nil { return -1, err } diff --git a/internal/ingress/defaults/main.go b/internal/ingress/defaults/main.go index bec1b08e2..03926baa0 100644 --- a/internal/ingress/defaults/main.go +++ b/internal/ingress/defaults/main.go @@ -34,13 +34,6 @@ type Backend struct { // toggles whether or not to remove trailing slashes during TLS redirects PreserveTrailingSlash bool `json:"preserve-trailing-slash"` - // allows usage of CustomHTTPErrors without intercepting service errors - // e.g. custom 404 and 503 when service-a does not exist or is not available - // but service-a can return 404 and 503 error codes without intercept - // http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_intercept_errors - // By default this is false - DisableProxyInterceptErrors bool `json:"disable-proxy-intercept-errors"` - // http://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size // Sets the maximum allowed size of the client request body ProxyBodySize string `json:"proxy-body-size"` @@ -69,11 +62,6 @@ type Backend struct { // http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) ProxyBufferSize string `json:"proxy-buffer-size"` - // Limits the total size of buffers that can be busy sending a response to the client while - // the response is not yet fully read. - // http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_busy_buffers_size - ProxyBusyBuffersSize string `json:"proxy-busy-buffers-size"` - // Sets a text that should be changed in the path attribute of the “Set-Cookie” header fields of // a proxied server response. // http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cookie_path @@ -112,7 +100,7 @@ type Backend struct { // Name server/s used to resolve names of upstream servers into IP addresses. // The file /etc/resolv.conf is used as DNS resolution configuration. - Resolver []net.IP `json:"Resolver"` + Resolver []net.IP // SkipAccessLogURLs sets a list of URLs that should not appear in the NGINX access log // This is useful with urls like `/health` or `health-check` that make "complex" reading the logs @@ -130,11 +118,6 @@ type Backend struct { // Default: false UsePortInRedirects bool `json:"use-port-in-redirects"` - // Enables or disables relative redirects. By default nginx uses absolute redirects. - // http://nginx.org/en/docs/http/ngx_http_core_module.html#absolute_redirect - // Default: false - RelativeRedirects bool `json:"relative-redirects"` - // Enable stickiness by client-server mapping based on a NGINX variable, text or a combination of both. // A consistent hashing method will be used which ensures only a few keys would be remapped to different // servers on upstream group changes @@ -156,10 +139,6 @@ type Backend struct { // http://nginx.org/en/docs/http/ngx_http_access_module.html WhitelistSourceRange []string `json:"whitelist-source-range"` - // DenylistSourceRange allows limiting access to certain client addresses - // http://nginx.org/en/docs/http/ngx_http_access_module.html - DenylistSourceRange []string `json:"denylist-source-range"` - // Limits the rate of response transmission to a client. // The rate is specified in bytes per second. The zero value disables rate limiting. // The limit is set per a request, and so if a client simultaneously opens two connections, @@ -182,23 +161,4 @@ type Backend struct { // Sets the maximum temp file size when proxy-buffers capacity is exceeded. // http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_max_temp_file_size ProxyMaxTempFileSize string `json:"proxy-max-temp-file-size"` - - // By default, the NGINX ingress controller uses a list of all endpoints (Pod IP/port) in the NGINX upstream configuration. - // It disables that behavior and instead uses a single upstream in NGINX, the service's Cluster IP and port. - ServiceUpstream bool `json:"service-upstream"` - - // AllowedResponseHeaders allows to define allow response headers for custom header annotation - AllowedResponseHeaders []string `json:"global-allowed-response-headers"` -} - -type SecurityConfiguration struct { - // AllowCrossNamespaceResources enables users to consume cross namespace resource on annotations - // Case disabled, attempts to use secrets or configmaps from a namespace different from Ingress will - // be denied - // This valid will default to `false` on future releases - AllowCrossNamespaceResources bool `json:"allow-cross-namespace-resources"` - - // AnnotationsRiskLevel represents the risk accepted on an annotation. If the risk is, for instance `Medium`, annotations - // with risk High and Critical will not be accepted - AnnotationsRiskLevel string `json:"annotations-risk-level"` } diff --git a/internal/ingress/errors/errors.go b/internal/ingress/errors/errors.go index 62be8b77f..11ca0f3cf 100644 --- a/internal/ingress/errors/errors.go +++ b/internal/ingress/errors/errors.go @@ -17,8 +17,9 @@ limitations under the License. package errors import ( - "errors" "fmt" + + "github.com/pkg/errors" ) var ( @@ -33,57 +34,57 @@ var ( // NewInvalidAnnotationConfiguration returns a new InvalidConfiguration error for use when // annotations are not correctly configured -func NewInvalidAnnotationConfiguration(name, reason string) error { - return InvalidConfigurationError{ +func NewInvalidAnnotationConfiguration(name string, reason string) error { + return InvalidConfiguration{ Name: fmt.Sprintf("the annotation %v does not contain a valid configuration: %v", name, reason), } } // NewInvalidAnnotationContent returns a new InvalidContent error func NewInvalidAnnotationContent(name string, val interface{}) error { - return InvalidContentError{ + return InvalidContent{ Name: fmt.Sprintf("the annotation %v does not contain a valid value (%v)", name, val), } } // NewLocationDenied returns a new LocationDenied error func NewLocationDenied(reason string) error { - return LocationDeniedError{ - Reason: fmt.Errorf("location denied, reason: %v", reason), + return LocationDenied{ + Reason: errors.Errorf("Location denied, reason: %v", reason), } } -// InvalidConfigurationError -type InvalidConfigurationError struct { +// InvalidConfiguration Error +type InvalidConfiguration struct { Name string } -func (e InvalidConfigurationError) Error() string { +func (e InvalidConfiguration) Error() string { return e.Name } -// InvalidContentError -type InvalidContentError struct { +// InvalidContent error +type InvalidContent struct { Name string } -func (e InvalidContentError) Error() string { +func (e InvalidContent) Error() string { return e.Name } -// LocationDeniedError -type LocationDeniedError struct { +// LocationDenied error +type LocationDenied struct { Reason error } -func (e LocationDeniedError) Error() string { +func (e LocationDenied) Error() string { return e.Reason.Error() } // IsLocationDenied checks if the err is an error which // indicates a location should return HTTP code 503 func IsLocationDenied(e error) bool { - _, ok := e.(LocationDeniedError) + _, ok := e.(LocationDenied) return ok } @@ -96,7 +97,7 @@ func IsMissingAnnotations(e error) bool { // IsInvalidContent checks if the err is an error which // indicates an annotations value is not valid func IsInvalidContent(e error) bool { - _, ok := e.(InvalidContentError) + _, ok := e.(InvalidContent) return ok } @@ -108,49 +109,5 @@ func New(m string) error { // Errorf formats according to a format specifier and returns the string // as a value that satisfies error. func Errorf(format string, args ...interface{}) error { - return fmt.Errorf(format, args...) -} - -type ValidationError struct { - Reason error -} - -type RiskyAnnotationError struct { - Reason error -} - -func (e ValidationError) Error() string { - return e.Reason.Error() -} - -// NewValidationError returns a new LocationDenied error -func NewValidationError(annotation string) error { - return ValidationError{ - Reason: fmt.Errorf("annotation %s contains invalid value", annotation), - } -} - -// IsValidationError checks if the err is an error which -// indicates that some annotation value is invalid -func IsValidationError(e error) bool { - _, ok := e.(ValidationError) - return ok -} - -// NewRiskyAnnotations returns a new LocationDenied error -func NewRiskyAnnotations(name string) error { - return RiskyAnnotationError{ - Reason: fmt.Errorf("annotation group %s contains risky annotation based on ingress configuration", name), - } -} - -// IsRiskyAnnotationError checks if the err is an error which -// indicates that some annotation value is invalid -func IsRiskyAnnotationError(e error) bool { - _, ok := e.(ValidationError) - return ok -} - -func (e RiskyAnnotationError) Error() string { - return e.Reason.Error() + return errors.Errorf(format, args...) } diff --git a/internal/ingress/inspector/ingress.go b/internal/ingress/inspector/ingress.go deleted file mode 100644 index 12db946ea..000000000 --- a/internal/ingress/inspector/ingress.go +++ /dev/null @@ -1,62 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package inspector - -import ( - "fmt" - - networking "k8s.io/api/networking/v1" -) - -// InspectIngress is used to do the deep inspection of an ingress object, walking through all -// of the spec fields and checking for matching strings and configurations that may represent -// an attempt to escape configs -func InspectIngress(ingress *networking.Ingress) error { - for _, rule := range ingress.Spec.Rules { - if rule.Host != "" { - if err := CheckRegex(rule.Host); err != nil { - return fmt.Errorf("invalid host in ingress %s/%s: %s", ingress.Namespace, ingress.Name, err) - } - } - if rule.HTTP != nil { - if err := inspectIngressRule(rule.HTTP); err != nil { - return fmt.Errorf("invalid rule in ingress %s/%s: %s", ingress.Namespace, ingress.Name, err) - } - } - } - - for _, tls := range ingress.Spec.TLS { - if err := CheckRegex(tls.SecretName); err != nil { - return fmt.Errorf("invalid secret in ingress %s/%s: %s", ingress.Namespace, ingress.Name, err) - } - for _, host := range tls.Hosts { - if err := CheckRegex(host); err != nil { - return fmt.Errorf("invalid host in ingress tls config %s/%s: %s", ingress.Namespace, ingress.Name, err) - } - } - } - return nil -} - -func inspectIngressRule(httprule *networking.HTTPIngressRuleValue) error { - for _, path := range httprule.Paths { - if err := CheckRegex(path.Path); err != nil { - return fmt.Errorf("invalid http path: %s", err) - } - } - return nil -} diff --git a/internal/ingress/inspector/ingress_test.go b/internal/ingress/inspector/ingress_test.go deleted file mode 100644 index 52ad8d431..000000000 --- a/internal/ingress/inspector/ingress_test.go +++ /dev/null @@ -1,98 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package inspector - -import ( - "testing" - - networking "k8s.io/api/networking/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func makeSimpleIngress(hostname string, paths ...string) *networking.Ingress { - newIngress := networking.Ingress{ - ObjectMeta: v1.ObjectMeta{ - Name: "test1", - Namespace: "default", - }, - Spec: networking.IngressSpec{ - Rules: []networking.IngressRule{ - { - Host: hostname, - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{}, - }, - }, - }, - }, - }, - } - - prefix := networking.PathTypePrefix - for _, path := range paths { - newPath := networking.HTTPIngressPath{ - Path: path, - PathType: &prefix, - } - newIngress.Spec.Rules[0].IngressRuleValue.HTTP.Paths = append(newIngress.Spec.Rules[0].IngressRuleValue.HTTP.Paths, newPath) - } - return &newIngress -} - -func TestInspectIngress(t *testing.T) { - tests := []struct { - name string - hostname string - path []string - wantErr bool - }{ - { - name: "invalid-path-etc", - hostname: "invalid.etc.com", - path: []string{ - "/var/run/secrets", - "/mypage", - }, - wantErr: true, - }, - { - name: "invalid-path-etc", - hostname: "invalid.etc.com", - path: []string{ - "/etc/nginx", - }, - wantErr: true, - }, - { - name: "invalid-path-etc", - hostname: "invalid.etc.com", - path: []string{ - "/mypage", - }, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ingress := makeSimpleIngress(tt.hostname, tt.path...) - if err := InspectIngress(ingress); (err != nil) != tt.wantErr { - t.Errorf("InspectIngress() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/internal/ingress/inspector/inspector.go b/internal/ingress/inspector/inspector.go deleted file mode 100644 index b41e18d9e..000000000 --- a/internal/ingress/inspector/inspector.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package inspector - -import ( - "errors" - "fmt" - - corev1 "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" - "k8s.io/klog/v2" -) - -// DeepInspect is the function called by admissionwebhook and store syncer to check -// if an object contains invalid configurations that may represent a security risk, -// and returning an error in this case -func DeepInspect(obj interface{}) error { - switch obj := obj.(type) { - case *networking.Ingress: - return InspectIngress(obj) - case *corev1.Service: - return InspectService(obj) - default: - klog.Warningf("received invalid object to inspect: %T", obj) - return nil - } -} - -var implSpecific = networking.PathTypeImplementationSpecific - -func ValidatePathType(ing *networking.Ingress) error { - if ing == nil { - return fmt.Errorf("received null ingress") - } - var err error - for _, rule := range ing.Spec.Rules { - if rule.HTTP != nil { - for _, path := range rule.HTTP.Paths { - if path.Path == "" { - continue - } - if path.PathType == nil || *path.PathType != implSpecific { - if isValid := validPathType.MatchString(path.Path); !isValid { - err = errors.Join(err, fmt.Errorf("path %s cannot be used with pathType %s", path.Path, string(*path.PathType))) - } - } - } - } - } - return err -} diff --git a/internal/ingress/inspector/inspector_test.go b/internal/ingress/inspector/inspector_test.go deleted file mode 100644 index 36b029cff..000000000 --- a/internal/ingress/inspector/inspector_test.go +++ /dev/null @@ -1,191 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package inspector - -import ( - "errors" - "fmt" - "testing" - - networking "k8s.io/api/networking/v1" -) - -var ( - exact = networking.PathTypeExact - prefix = networking.PathTypePrefix -) - -var ( - validIngress = &networking.Ingress{ - Spec: networking.IngressSpec{ - Rules: []networking.IngressRule{ - { - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - Path: "/test", - }, - { - PathType: &prefix, - Path: "/xpto/ab0/x_ss-9", - }, - { - PathType: &exact, - Path: "/bla/", - }, - }, - }, - }, - }, - }, - }, - } - - emptyIngress = &networking.Ingress{ - Spec: networking.IngressSpec{ - Rules: []networking.IngressRule{ - { - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - PathType: &exact, - }, - }, - }, - }, - }, - }, - }, - } - - invalidIngress = &networking.Ingress{ - Spec: networking.IngressSpec{ - Rules: []networking.IngressRule{ - { - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - PathType: &exact, - Path: "/foo.+", - }, - { - PathType: &exact, - Path: "xpto/lala", - }, - { - PathType: &exact, - Path: "/xpto/lala", - }, - { - PathType: &prefix, - Path: "/foo/bar/[a-z]{3}", - }, - { - PathType: &prefix, - Path: "/lala/xp\ntest", - }, - }, - }, - }, - }, - }, - }, - } - - validImplSpecific = &networking.Ingress{ - Spec: networking.IngressSpec{ - Rules: []networking.IngressRule{ - { - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - PathType: &implSpecific, - Path: "/foo.+", - }, - { - PathType: &implSpecific, - Path: "xpto/lala", - }, - }, - }, - }, - }, - }, - }, - } -) - -var aErr = func(s, pathType string) error { - return fmt.Errorf("path %s cannot be used with pathType %s", s, pathType) -} - -func TestValidatePathType(t *testing.T) { - tests := []struct { - name string - ing *networking.Ingress - wantErr bool - err error - }{ - { - name: "nil should return an error", - ing: nil, - wantErr: true, - err: fmt.Errorf("received null ingress"), - }, - { - name: "valid should not return an error", - ing: validIngress, - wantErr: false, - }, - { - name: "empty should not return an error", - ing: emptyIngress, - wantErr: false, - }, - { - name: "empty should not return an error", - ing: validImplSpecific, - wantErr: false, - }, - { - name: "invalid should return multiple errors", - ing: invalidIngress, - wantErr: true, - err: errors.Join( - aErr("/foo.+", "Exact"), - aErr("xpto/lala", "Exact"), - aErr("/foo/bar/[a-z]{3}", "Prefix"), - aErr("/lala/xp\ntest", "Prefix"), - ), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := ValidatePathType(tt.ing) - if (err != nil) != tt.wantErr { - t.Errorf("ValidatePathType() error = %v, wantErr %v", err, tt.wantErr) - } - if (err != nil && tt.err != nil) && tt.err.Error() != err.Error() { - t.Errorf("received invalid error: want = %v, expected %v", tt.err, err) - } - }) - } -} diff --git a/internal/ingress/inspector/rules.go b/internal/ingress/inspector/rules.go deleted file mode 100644 index 8388efdd5..000000000 --- a/internal/ingress/inspector/rules.go +++ /dev/null @@ -1,61 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package inspector - -import ( - "fmt" - "regexp" -) - -var ( - invalidAliasDirective = regexp.MustCompile(`(?s)\s*alias\s*.*;`) - invalidRootDirective = regexp.MustCompile(`(?s)\s*root\s*.*;`) - invalidEtcDir = regexp.MustCompile(`/etc/(passwd|shadow|group|nginx|ingress-controller)`) - invalidSecretsDir = regexp.MustCompile(`/var/run/secrets`) - invalidByLuaDirective = regexp.MustCompile(`.*_by_lua.*`) - - // validPathType enforces alphanumeric, -, _ and / characters. - // The field (?i) turns this regex case insensitive - // The remaining regex says that the string must start with a "/" (^/) - // the group [[:alnum:]\_\-\/]* says that any amount of characters (A-Za-z0-9), _, - and / - // are accepted until the end of the line - // Nothing else is accepted. - validPathType = regexp.MustCompile(`(?i)^/[[:alnum:]\_\-/]*$`) - - invalidRegex = []regexp.Regexp{} -) - -func init() { - invalidRegex = []regexp.Regexp{ - *invalidAliasDirective, - *invalidRootDirective, - *invalidEtcDir, - *invalidSecretsDir, - *invalidByLuaDirective, - } -} - -// CheckRegex receives a value/configuration and validates if it matches with one of the -// forbidden regexes. -func CheckRegex(value string) error { - for i := range invalidRegex { - if invalidRegex[i].MatchString(value) { - return fmt.Errorf("invalid value found: %s", value) - } - } - return nil -} diff --git a/internal/ingress/inspector/rules_test.go b/internal/ingress/inspector/rules_test.go deleted file mode 100644 index 3945a3bd4..000000000 --- a/internal/ingress/inspector/rules_test.go +++ /dev/null @@ -1,70 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package inspector - -import "testing" - -func TestCheckRegex(t *testing.T) { - tests := []struct { - name string - value string - wantErr bool - }{ - { - name: "must refuse invalid root", - wantErr: true, - value: " root blabla/lala ;", - }, - { - name: "must refuse invalid alias", - wantErr: true, - value: " alias blabla/lala ;", - }, - { - name: "must refuse invalid alias with line break", - wantErr: true, - value: "alias #\n/lalala/1/;", - }, - { - name: "must refuse invalid attempt to call /etc", - wantErr: true, - value: "location /etc/nginx/lalala", - }, - { - name: "must refuse invalid attempt to call k8s secret", - wantErr: true, - value: "ssl_cert /var/run/secrets/kubernetes.io/lalala; xpto", - }, - { - name: "must refuse invalid attempt to call lua directives", - wantErr: true, - value: "set_by_lua lala", - }, - { - name: "must pass with valid configuration", - wantErr: false, - value: "/test/mypage1", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := CheckRegex(tt.value); (err != nil) != tt.wantErr { - t.Errorf("CheckRegex() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/internal/ingress/inspector/service.go b/internal/ingress/inspector/service.go deleted file mode 100644 index 8be08490b..000000000 --- a/internal/ingress/inspector/service.go +++ /dev/null @@ -1,26 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package inspector - -import ( - corev1 "k8s.io/api/core/v1" -) - -// InspectService will be used to inspect service objects for possible invalid configurations -func InspectService(_ *corev1.Service) error { - return nil -} diff --git a/internal/ingress/metric/collectors/admission.go b/internal/ingress/metric/collectors/admission.go deleted file mode 100644 index 456f56963..000000000 --- a/internal/ingress/metric/collectors/admission.go +++ /dev/null @@ -1,157 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package collectors - -import ( - "fmt" - - "github.com/prometheus/client_golang/prometheus" - "k8s.io/klog/v2" -) - -// AdmissionCollector stores prometheus metrics of the admission webhook -type AdmissionCollector struct { - prometheus.Collector - - testedIngressLength prometheus.Gauge - testedIngressTime prometheus.Gauge - - renderingIngressLength prometheus.Gauge - renderingIngressTime prometheus.Gauge - - admissionTime prometheus.Gauge - - testedConfigurationSize prometheus.Gauge - - constLabels prometheus.Labels - labels prometheus.Labels -} - -// NewAdmissionCollector creates a new AdmissionCollector instance for the admission collector -func NewAdmissionCollector(pod, namespace, class string) *AdmissionCollector { - constLabels := prometheus.Labels{ - "controller_namespace": namespace, - "controller_class": class, - "controller_pod": pod, - } - - am := &AdmissionCollector{ - constLabels: constLabels, - - labels: prometheus.Labels{ - "namespace": namespace, - "class": class, - }, - - testedIngressLength: prometheus.NewGauge( - prometheus.GaugeOpts{ - Name: "admission_tested_ingresses", - Help: "The length of ingresses processed by the admission controller", - Namespace: PrometheusNamespace, - ConstLabels: constLabels, - }), - testedIngressTime: prometheus.NewGauge( - prometheus.GaugeOpts{ - Name: "admission_tested_duration", - Help: "The processing duration of the admission controller tests (float seconds)", - Namespace: PrometheusNamespace, - ConstLabels: constLabels, - }), - renderingIngressLength: prometheus.NewGauge( - prometheus.GaugeOpts{ - Name: "admission_render_ingresses", - Help: "The length of ingresses rendered by the admission controller", - Namespace: PrometheusNamespace, - ConstLabels: constLabels, - }), - renderingIngressTime: prometheus.NewGauge( - prometheus.GaugeOpts{ - Name: "admission_render_duration", - Help: "The processing duration of ingresses rendering by the admission controller (float seconds)", - Namespace: PrometheusNamespace, - ConstLabels: constLabels, - }), - testedConfigurationSize: prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: PrometheusNamespace, - Name: "admission_config_size", - Help: "The size of the tested configuration", - ConstLabels: constLabels, - }), - admissionTime: prometheus.NewGauge( - prometheus.GaugeOpts{ - Name: "admission_roundtrip_duration", - Help: "The complete duration of the admission controller at the time to process a new event (float seconds)", - Namespace: PrometheusNamespace, - ConstLabels: constLabels, - }), - } - return am -} - -// Describe implements prometheus.Collector -func (am *AdmissionCollector) Describe(ch chan<- *prometheus.Desc) { - am.testedIngressLength.Describe(ch) - am.testedIngressTime.Describe(ch) - am.renderingIngressLength.Describe(ch) - am.renderingIngressTime.Describe(ch) - am.testedConfigurationSize.Describe(ch) - am.admissionTime.Describe(ch) -} - -// Collect implements the prometheus.Collector interface. -func (am *AdmissionCollector) Collect(ch chan<- prometheus.Metric) { - am.testedIngressLength.Collect(ch) - am.testedIngressTime.Collect(ch) - am.renderingIngressLength.Collect(ch) - am.renderingIngressTime.Collect(ch) - am.testedConfigurationSize.Collect(ch) - am.admissionTime.Collect(ch) -} - -// ByteFormat formats humanReadable bytes -func ByteFormat(bytes int64) string { - const unit = 1000 - if bytes < unit { - return fmt.Sprintf("%d B", bytes) - } - div, exp := int64(unit), 0 - for n := bytes / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f%cB", - float64(bytes)/float64(div), "kMGTPE"[exp]) -} - -// SetAdmissionMetrics sets the values for AdmissionMetrics that can be called externally -func (am *AdmissionCollector) SetAdmissionMetrics(testedIngressLength, testedIngressTime, renderingIngressLength, renderingIngressTime, testedConfigurationSize, admissionTime float64) { - am.testedIngressLength.Set(testedIngressLength) - am.testedIngressTime.Set(testedIngressTime) - am.renderingIngressLength.Set(renderingIngressLength) - am.renderingIngressTime.Set(renderingIngressTime) - am.testedConfigurationSize.Set(testedConfigurationSize) - am.admissionTime.Set(admissionTime) - klog.Infof("processed ingress via admission controller {testedIngressLength:%v testedIngressTime:%vs renderingIngressLength:%v renderingIngressTime:%vs admissionTime:%vs testedConfigurationSize:%v}", - testedIngressLength, - testedIngressTime, - renderingIngressLength, - renderingIngressTime, - admissionTime, - ByteFormat(int64(testedConfigurationSize)), - ) -} diff --git a/internal/ingress/metric/collectors/admission_test.go b/internal/ingress/metric/collectors/admission_test.go deleted file mode 100644 index a23384a88..000000000 --- a/internal/ingress/metric/collectors/admission_test.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package collectors - -import ( - "testing" - - "github.com/prometheus/client_golang/prometheus" -) - -func TestAdmissionCounters(t *testing.T) { - const ( - metadataFirst = ` - # HELP nginx_ingress_controller_admission_config_size The size of the tested configuration - # TYPE nginx_ingress_controller_admission_config_size gauge - # HELP nginx_ingress_controller_admission_roundtrip_duration The complete duration of the admission controller at the time to process a new event (float seconds) - # TYPE nginx_ingress_controller_admission_roundtrip_duration gauge - ` - metadataSecond = ` - # HELP nginx_ingress_controller_admission_render_ingresses The length of ingresses rendered by the admission controller - # TYPE nginx_ingress_controller_admission_render_ingresses gauge - # HELP nginx_ingress_controller_admission_tested_duration The processing duration of the admission controller tests (float seconds) - # TYPE nginx_ingress_controller_admission_tested_duration gauge - ` - metadataThird = ` - # HELP nginx_ingress_controller_admission_config_size The size of the tested configuration - # TYPE nginx_ingress_controller_admission_config_size gauge - # HELP nginx_ingress_controller_admission_render_duration The processing duration of ingresses rendering by the admission controller (float seconds) - # TYPE nginx_ingress_controller_admission_render_duration gauge - # HELP nginx_ingress_controller_admission_render_ingresses The length of ingresses rendered by the admission controller - # TYPE nginx_ingress_controller_admission_render_ingresses gauge - # HELP nginx_ingress_controller_admission_roundtrip_duration The complete duration of the admission controller at the time to process a new event (float seconds) - # TYPE nginx_ingress_controller_admission_roundtrip_duration gauge - # HELP nginx_ingress_controller_admission_tested_ingresses The length of ingresses processed by the admission controller - # TYPE nginx_ingress_controller_admission_tested_ingresses gauge - # HELP nginx_ingress_controller_admission_tested_duration The processing duration of the admission controller tests (float seconds) - # TYPE nginx_ingress_controller_admission_tested_duration gauge - ` - ) - cases := []struct { - name string - test func(*AdmissionCollector) - metrics []string - want string - }{ - { - name: "should return 0 as values on a fresh initiated collector", - test: func(_ *AdmissionCollector) { - }, - want: metadataFirst + ` - nginx_ingress_controller_admission_config_size{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 0 - nginx_ingress_controller_admission_roundtrip_duration{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 0 - `, - metrics: []string{"nginx_ingress_controller_admission_config_size", "nginx_ingress_controller_admission_roundtrip_duration"}, - }, - { - name: "set admission metrics to 1 in all fields and validate next set", - test: func(am *AdmissionCollector) { - am.SetAdmissionMetrics(1, 1, 1, 1, 1, 1) - }, - want: metadataSecond + ` - nginx_ingress_controller_admission_render_ingresses{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 1 - nginx_ingress_controller_admission_tested_duration{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 1 - `, - metrics: []string{"nginx_ingress_controller_admission_render_ingresses", "nginx_ingress_controller_admission_tested_duration"}, - }, - { - name: "set admission metrics to 5 in all fields and validate all sets", - test: func(am *AdmissionCollector) { - am.SetAdmissionMetrics(5, 5, 5, 5, 5, 5) - }, - want: metadataThird + ` - nginx_ingress_controller_admission_config_size{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5 - nginx_ingress_controller_admission_render_duration{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5 - nginx_ingress_controller_admission_render_ingresses{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5 - nginx_ingress_controller_admission_roundtrip_duration{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5 - nginx_ingress_controller_admission_tested_ingresses{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5 - nginx_ingress_controller_admission_tested_duration{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5 - `, - metrics: []string{ - "nginx_ingress_controller_admission_config_size", - "nginx_ingress_controller_admission_render_duration", - "nginx_ingress_controller_admission_render_ingresses", - "nginx_ingress_controller_admission_roundtrip_duration", - "nginx_ingress_controller_admission_tested_ingresses", - "nginx_ingress_controller_admission_tested_duration", - }, - }, - } - - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - am := NewAdmissionCollector("pod", "default", "nginx") - reg := prometheus.NewPedanticRegistry() - if err := reg.Register(am); err != nil { - t.Errorf("registering collector failed: %s", err) - } - - c.test(am) - - if err := GatherAndCompare(am, c.want, c.metrics, reg); err != nil { - t.Errorf("unexpected collecting result:\n%s", err) - } - - reg.Unregister(am) - }) - } -} diff --git a/internal/ingress/metric/collectors/controller.go b/internal/ingress/metric/collectors/controller.go index 0df04c1d0..af746b4ba 100644 --- a/internal/ingress/metric/collectors/controller.go +++ b/internal/ingress/metric/collectors/controller.go @@ -22,17 +22,14 @@ import ( "github.com/prometheus/client_golang/prometheus" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/ingress-nginx/pkg/apis/ingress" - "k8s.io/ingress-nginx/version" + "k8s.io/ingress-nginx/internal/ingress" "k8s.io/klog/v2" ) var ( operation = []string{"controller_namespace", "controller_class", "controller_pod"} ingressOperation = []string{"controller_namespace", "controller_class", "controller_pod", "namespace", "ingress"} - sslLabelHost = []string{"namespace", "class", "host", "secret_name", "identifier"} - sslInfoLabels = []string{"namespace", "class", "host", "secret_name", "identifier", "issuer_organization", "issuer_common_name", "serial_number", "public_key_algorithm"} - orphanityLabels = []string{"controller_namespace", "controller_class", "controller_pod", "namespace", "ingress", "type"} + sslLabelHost = []string{"namespace", "class", "host"} ) // Controller defines base metrics about the ingress controller @@ -48,15 +45,11 @@ type Controller struct { checkIngressOperation *prometheus.CounterVec checkIngressOperationErrors *prometheus.CounterVec sslExpireTime *prometheus.GaugeVec - sslInfo *prometheus.GaugeVec - OrphanIngress *prometheus.GaugeVec constLabels prometheus.Labels labels prometheus.Labels leaderElection *prometheus.GaugeVec - - buildInfo prometheus.Collector } // NewController creates a new prometheus collector for the @@ -76,23 +69,6 @@ func NewController(pod, namespace, class string) *Controller { "class": class, }, - buildInfo: prometheus.NewGaugeFunc( - prometheus.GaugeOpts{ - Namespace: PrometheusNamespace, - Name: "build_info", - Help: "A metric with a constant '1' labeled with information about the build.", - ConstLabels: prometheus.Labels{ - "controller_namespace": namespace, - "controller_class": class, - "controller_pod": pod, - "release": version.RELEASE, - "build": version.COMMIT, - "repository": version.REPO, - }, - }, - func() float64 { return 1 }, - ), - configHash: prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: PrometheusNamespace, @@ -156,14 +132,6 @@ func NewController(pod, namespace, class string) *Controller { }, sslLabelHost, ), - sslInfo: prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: PrometheusNamespace, - Name: "ssl_certificate_info", - Help: `Hold all labels associated to a certificate`, - }, - sslInfoLabels, - ), leaderElection: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: PrometheusNamespace, @@ -173,15 +141,6 @@ func NewController(pod, namespace, class string) *Controller { }, []string{"name"}, ), - OrphanIngress: prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: PrometheusNamespace, - Name: "orphan_ingress", - Help: `Gauge reporting status of ingress orphanity, 1 indicates orphaned ingress. - 'namespace' is the string used to identify namespace of ingress, 'ingress' for ingress name and 'type' for 'no-service' or 'no-endpoint' of orphanity`, - }, - orphanityLabels, - ), } return cm @@ -225,26 +184,6 @@ func (cm *Controller) IncCheckErrorCount(namespace, name string) { cm.checkIngressOperationErrors.MustCurryWith(cm.constLabels).With(labels).Inc() } -// IncOrphanIngress sets the orphaned ingress gauge to one -func (cm *Controller) IncOrphanIngress(namespace, name, orphanityType string) { - labels := prometheus.Labels{ - "namespace": namespace, - "ingress": name, - "type": orphanityType, - } - cm.OrphanIngress.MustCurryWith(cm.constLabels).With(labels).Set(1.0) -} - -// DecOrphanIngress sets the orphaned ingress gauge to zero (all services has their endpoints) -func (cm *Controller) DecOrphanIngress(namespace, name, orphanityType string) { - labels := prometheus.Labels{ - "namespace": namespace, - "ingress": name, - "type": orphanityType, - } - cm.OrphanIngress.MustCurryWith(cm.constLabels).With(labels).Set(0.0) -} - // ConfigSuccess set a boolean flag according to the output of the controller configuration reload func (cm *Controller) ConfigSuccess(hash uint64, success bool) { if success { @@ -261,7 +200,7 @@ func (cm *Controller) ConfigSuccess(hash uint64, success bool) { } // Describe implements prometheus.Collector -func (cm *Controller) Describe(ch chan<- *prometheus.Desc) { +func (cm Controller) Describe(ch chan<- *prometheus.Desc) { cm.configHash.Describe(ch) cm.configSuccess.Describe(ch) cm.configSuccessTime.Describe(ch) @@ -270,14 +209,11 @@ func (cm *Controller) Describe(ch chan<- *prometheus.Desc) { cm.checkIngressOperation.Describe(ch) cm.checkIngressOperationErrors.Describe(ch) cm.sslExpireTime.Describe(ch) - cm.sslInfo.Describe(ch) cm.leaderElection.Describe(ch) - cm.buildInfo.Describe(ch) - cm.OrphanIngress.Describe(ch) } // Collect implements the prometheus.Collector interface. -func (cm *Controller) Collect(ch chan<- prometheus.Metric) { +func (cm Controller) Collect(ch chan<- prometheus.Metric) { cm.configHash.Collect(ch) cm.configSuccess.Collect(ch) cm.configSuccessTime.Collect(ch) @@ -286,120 +222,42 @@ func (cm *Controller) Collect(ch chan<- prometheus.Metric) { cm.checkIngressOperation.Collect(ch) cm.checkIngressOperationErrors.Collect(ch) cm.sslExpireTime.Collect(ch) - cm.sslInfo.Collect(ch) cm.leaderElection.Collect(ch) - cm.buildInfo.Collect(ch) - cm.OrphanIngress.Collect(ch) } // SetSSLExpireTime sets the expiration time of SSL Certificates func (cm *Controller) SetSSLExpireTime(servers []*ingress.Server) { for _, s := range servers { - if !(s.Hostname != "" && s.SSLCert != nil && s.SSLCert.ExpireTime.Unix() > 0) { - continue - } + if s.Hostname != "" && s.SSLCert != nil && s.SSLCert.ExpireTime.Unix() > 0 { + labels := make(prometheus.Labels, len(cm.labels)+1) + for k, v := range cm.labels { + labels[k] = v + } + labels["host"] = s.Hostname - labels := make(prometheus.Labels, len(cm.labels)+1) - for k, v := range cm.labels { - labels[k] = v + cm.sslExpireTime.With(labels).Set(float64(s.SSLCert.ExpireTime.Unix())) } - labels["host"] = s.Hostname - labels["secret_name"] = s.SSLCert.Name - labels["namespace"] = s.SSLCert.Namespace - labels["identifier"] = s.SSLCert.Identifier() - - cm.sslExpireTime.With(labels).Set(float64(s.SSLCert.ExpireTime.Unix())) } } -// SetSSLInfo creates a metric with all certificate information -func (cm *Controller) SetSSLInfo(servers []*ingress.Server) { - for _, s := range servers { - if s.SSLCert == nil || s.SSLCert.Certificate == nil || s.SSLCert.Certificate.SerialNumber == nil { - continue - } - - labels := make(prometheus.Labels, len(cm.labels)+1) - for k, v := range cm.labels { - labels[k] = v - } - labels["identifier"] = s.SSLCert.Identifier() - labels["host"] = s.Hostname - labels["secret_name"] = s.SSLCert.Name - labels["namespace"] = s.SSLCert.Namespace - labels["issuer_common_name"] = s.SSLCert.Certificate.Issuer.CommonName - labels["issuer_organization"] = "" - if len(s.SSLCert.Certificate.Issuer.Organization) > 0 { - labels["issuer_organization"] = s.SSLCert.Certificate.Issuer.Organization[0] - } - labels["serial_number"] = s.SSLCert.Certificate.SerialNumber.String() - labels["public_key_algorithm"] = s.SSLCert.Certificate.PublicKeyAlgorithm.String() - - cm.sslInfo.With(labels).Set(1) - } +// RemoveMetrics removes metrics for hostnames not available anymore +func (cm *Controller) RemoveMetrics(hosts []string, registry prometheus.Gatherer) { + cm.removeSSLExpireMetrics(true, hosts, registry) } -// RemoveMetrics removes metrics for certificates not available anymore by identifier -func (cm *Controller) RemoveMetrics(certificates []string, registry prometheus.Gatherer) { - cm.removeSSLExpireMetrics(true, certificates, registry) - cm.removeCertificatesMetrics(true, certificates, registry) -} - -// RemoveAllSSLMetrics removes metrics for expiration of SSL Certificates -func (cm *Controller) RemoveAllSSLMetrics(registry prometheus.Gatherer) { +// RemoveAllSSLExpireMetrics removes metrics for expiration of SSL Certificates +func (cm *Controller) RemoveAllSSLExpireMetrics(registry prometheus.Gatherer) { cm.removeSSLExpireMetrics(false, []string{}, registry) - cm.removeCertificatesMetrics(false, []string{}, registry) } -func (cm *Controller) removeCertificatesMetrics(onlyDefinedHosts bool, certificates []string, registry prometheus.Gatherer) { - mfs, err := registry.Gather() - if err != nil { - klog.Errorf("Error gathering metrics: %v", err) - return - } - - toRemove := sets.NewString(certificates...) - - for _, mf := range mfs { - metricName := mf.GetName() - if fmt.Sprintf("%v_ssl_certificate_info", PrometheusNamespace) != metricName { - continue - } - - for _, m := range mf.GetMetric() { - labels := make(map[string]string, len(m.GetLabel())) - for _, labelPair := range m.GetLabel() { - labels[*labelPair.Name] = *labelPair.Value - } - - // remove labels that are constant - deleteConstants(labels) - - identifier, ok := labels["identifier"] - if !ok { - continue - } - if onlyDefinedHosts && !toRemove.Has(identifier) { - continue - } - - klog.V(2).Infof("Removing prometheus metric from gauge %v for identifier %v", metricName, identifier) - removed := cm.sslInfo.Delete(labels) - if !removed { - klog.V(2).Infof("metric %v for identifier %v with labels not removed: %v", metricName, identifier, labels) - } - } - } -} - -func (cm *Controller) removeSSLExpireMetrics(onlyDefinedCerts bool, certificates []string, registry prometheus.Gatherer) { +func (cm *Controller) removeSSLExpireMetrics(onlyDefinedHosts bool, hosts []string, registry prometheus.Gatherer) { mfs, err := registry.Gather() if err != nil { klog.ErrorS(err, "Error gathering metrics") return } - toRemove := sets.NewString(certificates...) + toRemove := sets.NewString(hosts...) for _, mf := range mfs { metricName := mf.GetName() @@ -416,24 +274,19 @@ func (cm *Controller) removeSSLExpireMetrics(onlyDefinedCerts bool, certificates // remove labels that are constant deleteConstants(labels) - identifier, ok := labels["identifier"] - if !ok { - continue - } - host, ok := labels["host"] if !ok { continue } - if onlyDefinedCerts && !toRemove.Has(identifier) { + if onlyDefinedHosts && !toRemove.Has(host) { continue } - klog.V(2).InfoS("Removing prometheus metric", "gauge", metricName, "host", host, "identifier", identifier) + klog.V(2).InfoS("Removing prometheus metric", "gauge", metricName, "host", host) removed := cm.sslExpireTime.Delete(labels) if !removed { - klog.V(2).InfoS("metric removed", "metric", metricName, "host", host, "identifier", identifier, "labels", labels) + klog.V(2).InfoS("metric removed", "metric", metricName, "host", host, "labels", labels) } } } diff --git a/internal/ingress/metric/collectors/controller_test.go b/internal/ingress/metric/collectors/controller_test.go index a77293c20..e10dafdfe 100644 --- a/internal/ingress/metric/collectors/controller_test.go +++ b/internal/ingress/metric/collectors/controller_test.go @@ -17,14 +17,11 @@ limitations under the License. package collectors import ( - "crypto/x509" - "crypto/x509/pkix" - "math/big" "testing" "time" "github.com/prometheus/client_golang/prometheus" - "k8s.io/ingress-nginx/pkg/apis/ingress" + "k8s.io/ingress-nginx/internal/ingress" ) func TestControllerCounters(t *testing.T) { @@ -42,7 +39,7 @@ func TestControllerCounters(t *testing.T) { }{ { name: "should return not increment in metrics if no operations are invoked", - test: func(_ *Controller) { + test: func(cm *Controller) { }, want: metadata + ` nginx_ingress_controller_config_last_reload_successful{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 0 @@ -76,29 +73,15 @@ func TestControllerCounters(t *testing.T) { { name: "should set SSL certificates metrics", test: func(cm *Controller) { - t1, err := time.Parse( + t1, _ := time.Parse( time.RFC3339, "2012-11-01T22:08:41+00:00") - if err != nil { - t.Errorf("unexpected error: %v", err) - } servers := []*ingress.Server{ { Hostname: "demo", SSLCert: &ingress.SSLCert{ ExpireTime: t1, - Name: "secret-name", - Namespace: "secret-namespace", - Certificate: &x509.Certificate{ - PublicKeyAlgorithm: x509.ECDSA, - Issuer: pkix.Name{ - CommonName: "certificate issuer", - SerialNumber: "abcd1234", - Organization: []string{"issuer org"}, - }, - SerialNumber: big.NewInt(100), - }, }, }, { @@ -113,107 +96,10 @@ func TestControllerCounters(t *testing.T) { want: ` # HELP nginx_ingress_controller_ssl_expire_time_seconds Number of seconds since 1970 to the SSL Certificate expire.\n An example to check if this certificate will expire in 10 days is: "nginx_ingress_controller_ssl_expire_time_seconds < (time() + (10 * 24 * 3600))" # TYPE nginx_ingress_controller_ssl_expire_time_seconds gauge - nginx_ingress_controller_ssl_expire_time_seconds{class="nginx",host="demo",identifier="abcd1234-100",namespace="secret-namespace",secret_name="secret-name"} 1.351807721e+09 + nginx_ingress_controller_ssl_expire_time_seconds{class="nginx",host="demo",namespace="default"} 1.351807721e+09 `, metrics: []string{"nginx_ingress_controller_ssl_expire_time_seconds"}, }, - { - name: "should set SSL certificates infos metrics", - test: func(cm *Controller) { - servers := []*ingress.Server{ - { - Hostname: "demo", - SSLCert: &ingress.SSLCert{ - Name: "secret-name", - Namespace: "ingress-namespace", - Certificate: &x509.Certificate{ - PublicKeyAlgorithm: x509.ECDSA, - Issuer: pkix.Name{ - CommonName: "certificate issuer", - SerialNumber: "abcd1234", - Organization: []string{"issuer org"}, - }, - SerialNumber: big.NewInt(100), - }, - }, - }, - { - Hostname: "invalid", - SSLCert: &ingress.SSLCert{ - ExpireTime: time.Unix(0, 0), - }, - }, - } - cm.SetSSLInfo(servers) - }, - want: ` - # HELP nginx_ingress_controller_ssl_certificate_info Hold all labels associated to a certificate - # TYPE nginx_ingress_controller_ssl_certificate_info gauge - nginx_ingress_controller_ssl_certificate_info{class="nginx",host="demo",identifier="abcd1234-100",issuer_common_name="certificate issuer",issuer_organization="issuer org",namespace="ingress-namespace",public_key_algorithm="ECDSA",secret_name="secret-name",serial_number="100"} 1 - `, - metrics: []string{"nginx_ingress_controller_ssl_certificate_info"}, - }, - { - name: "should ignore certificates without serial number", - test: func(cm *Controller) { - servers := []*ingress.Server{ - { - Hostname: "demo", - SSLCert: &ingress.SSLCert{ - Name: "secret-name", - Namespace: "ingress-namespace", - Certificate: &x509.Certificate{ - PublicKeyAlgorithm: x509.ECDSA, - Issuer: pkix.Name{ - CommonName: "certificate issuer", - SerialNumber: "abcd1234", - }, - }, - }, - }, - } - cm.SetSSLInfo(servers) - }, - want: ``, - metrics: []string{"nginx_ingress_controller_ssl_certificate_info"}, - }, - { - name: "should ignore certificates with nil x509 pointer", - test: func(cm *Controller) { - servers := []*ingress.Server{ - { - Hostname: "demo", - SSLCert: &ingress.SSLCert{ - Name: "secret-name", - Namespace: "ingress-namespace", - Certificate: &x509.Certificate{ - PublicKeyAlgorithm: x509.ECDSA, - Issuer: pkix.Name{ - CommonName: "certificate issuer", - SerialNumber: "abcd1234", - }, - }, - }, - }, - } - cm.SetSSLInfo(servers) - }, - want: ``, - metrics: []string{"nginx_ingress_controller_ssl_certificate_info"}, - }, - { - name: "should ignore servers without certificates", - test: func(cm *Controller) { - servers := []*ingress.Server{ - { - Hostname: "demo", - }, - } - cm.SetSSLInfo(servers) - }, - want: ``, - metrics: []string{"nginx_ingress_controller_ssl_certificate_info"}, - }, } for _, c := range cases { @@ -242,25 +128,15 @@ func TestRemoveMetrics(t *testing.T) { t.Errorf("registering collector failed: %s", err) } - t1, err := time.Parse( + t1, _ := time.Parse( time.RFC3339, "2012-11-01T22:08:41+00:00") - if err != nil { - t.Errorf("Unexpected error: %v", err) - } servers := []*ingress.Server{ { Hostname: "demo", SSLCert: &ingress.SSLCert{ ExpireTime: t1, - Certificate: &x509.Certificate{ - Issuer: pkix.Name{ - CommonName: "certificate issuer", - SerialNumber: "abcd1234", - }, - SerialNumber: big.NewInt(100), - }, }, }, { @@ -271,65 +147,12 @@ func TestRemoveMetrics(t *testing.T) { }, } cm.SetSSLExpireTime(servers) - cm.SetSSLInfo(servers) - cm.RemoveMetrics([]string{"abcd1234-100"}, reg) + cm.RemoveMetrics([]string{"demo"}, reg) if err := GatherAndCompare(cm, "", []string{"nginx_ingress_controller_ssl_expire_time_seconds"}, reg); err != nil { t.Errorf("unexpected collecting result:\n%s", err) } - if err := GatherAndCompare(cm, "", []string{"nginx_ingress_controller_ssl_certificate_info"}, reg); err != nil { - t.Errorf("unexpected collecting result:\n%s", err) - } - - reg.Unregister(cm) -} - -func TestRemoveAllSSLMetrics(t *testing.T) { - cm := NewController("pod", "default", "nginx") - reg := prometheus.NewPedanticRegistry() - if err := reg.Register(cm); err != nil { - t.Errorf("registering collector failed: %s", err) - } - - t1, err := time.Parse( - time.RFC3339, - "2012-11-01T22:08:41+00:00") - if err != nil { - t.Errorf("unexpected error: %v", err) - } - servers := []*ingress.Server{ - { - Hostname: "demo", - SSLCert: &ingress.SSLCert{ - ExpireTime: t1, - Certificate: &x509.Certificate{ - Issuer: pkix.Name{ - CommonName: "certificate issuer", - SerialNumber: "abcd1234", - }, - SerialNumber: big.NewInt(100), - }, - }, - }, - { - Hostname: "invalid", - SSLCert: &ingress.SSLCert{ - ExpireTime: time.Unix(0, 0), - }, - }, - } - cm.SetSSLExpireTime(servers) - cm.SetSSLInfo(servers) - - cm.RemoveAllSSLMetrics(reg) - - if err := GatherAndCompare(cm, "", []string{"nginx_ingress_controller_ssl_expire_time_seconds"}, reg); err != nil { - t.Errorf("unexpected collecting result:\n%s", err) - } - if err := GatherAndCompare(cm, "", []string{"nginx_ingress_controller_ssl_certificate_info"}, reg); err != nil { - t.Errorf("unexpected collecting result:\n%s", err) - } reg.Unregister(cm) } diff --git a/internal/ingress/metric/collectors/main.go b/internal/ingress/metric/collectors/main.go index b2479929b..2c57ad774 100644 --- a/internal/ingress/metric/collectors/main.go +++ b/internal/ingress/metric/collectors/main.go @@ -17,4 +17,4 @@ limitations under the License. package collectors // PrometheusNamespace default metric namespace -var PrometheusNamespace = "nginx_ingress_controller" //#nosec G101 +var PrometheusNamespace = "nginx_ingress_controller" diff --git a/internal/ingress/metric/collectors/nginx_status.go b/internal/ingress/metric/collectors/nginx_status.go index f3afdc334..5aaa787de 100644 --- a/internal/ingress/metric/collectors/nginx_status.go +++ b/internal/ingress/metric/collectors/nginx_status.go @@ -75,6 +75,7 @@ type NGINXStatusCollector interface { // NewNGINXStatus returns a new prometheus collector the default nginx status module func NewNGINXStatus(podName, namespace, ingressClass string) (NGINXStatusCollector, error) { + p := nginxStatusCollector{ scrapeChan: make(chan scrapeRequest), } diff --git a/internal/ingress/metric/collectors/nginx_status_test.go b/internal/ingress/metric/collectors/nginx_status_test.go index ec535745d..4dc67c425 100644 --- a/internal/ingress/metric/collectors/nginx_status_test.go +++ b/internal/ingress/metric/collectors/nginx_status_test.go @@ -106,7 +106,7 @@ func TestStatusCollector(t *testing.T) { server := &httptest.Server{ Listener: listener, - Config: &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { //nolint:gosec // Ignore the gosec error in testing + Config: &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) if r.URL.Path == "/nginx_status" { diff --git a/internal/ingress/metric/collectors/process.go b/internal/ingress/metric/collectors/process.go index 85e8066b5..23f533876 100644 --- a/internal/ingress/metric/collectors/process.go +++ b/internal/ingress/metric/collectors/process.go @@ -53,8 +53,6 @@ type BinaryNameMatcher struct { // MatchAndName returns false if the match failed, otherwise // true and the resulting name. -// -//nolint:gocritic // nacl param cannot be a pointer since it's to implement common.MatchNamer interface func (em BinaryNameMatcher) MatchAndName(nacl common.ProcAttributes) (bool, string) { if len(nacl.Cmdline) == 0 { return false, "" @@ -96,10 +94,8 @@ type NGINXProcessCollector interface { Stop() } -var ( - name = "nginx" - binary = "/usr/bin/nginx" -) +var name = "nginx" +var binary = "/usr/bin/nginx" // NewNGINXProcess returns a new prometheus collector for the nginx process func NewNGINXProcess(pod, namespace, ingressClass string) (NGINXProcessCollector, error) { @@ -110,9 +106,9 @@ func NewNGINXProcess(pod, namespace, ingressClass string) (NGINXProcessCollector nm := newBinaryNameMatcher(name, binary) - p := &namedProcess{ + p := namedProcess{ scrapeChan: make(chan scrapeRequest), - Grouper: proc.NewGrouper(nm, true, false, false, 0, false, false), + Grouper: proc.NewGrouper(nm, true, false, false, false), fs: fs, } @@ -168,7 +164,7 @@ func NewNGINXProcess(pod, namespace, ingressClass string) (NGINXProcessCollector } // Describe implements prometheus.Collector. -func (p *namedProcess) Describe(ch chan<- *prometheus.Desc) { +func (p namedProcess) Describe(ch chan<- *prometheus.Desc) { ch <- p.data.cpuSecs ch <- p.data.numProcs ch <- p.data.readBytes @@ -179,13 +175,13 @@ func (p *namedProcess) Describe(ch chan<- *prometheus.Desc) { } // Collect implements prometheus.Collector. -func (p *namedProcess) Collect(ch chan<- prometheus.Metric) { +func (p namedProcess) Collect(ch chan<- prometheus.Metric) { req := scrapeRequest{results: ch, done: make(chan struct{})} p.scrapeChan <- req <-req.done } -func (p *namedProcess) Start() { +func (p namedProcess) Start() { for req := range p.scrapeChan { ch := req.results p.scrape(ch) @@ -193,19 +189,18 @@ func (p *namedProcess) Start() { } } -func (p *namedProcess) Stop() { +func (p namedProcess) Stop() { close(p.scrapeChan) } -func (p *namedProcess) scrape(ch chan<- prometheus.Metric) { +func (p namedProcess) scrape(ch chan<- prometheus.Metric) { _, groups, err := p.Update(p.fs.AllProcs()) if err != nil { klog.Warningf("unexpected error obtaining nginx process info: %v", err) return } - for i := range groups { - gcounts := groups[i] + for _, gcounts := range groups { ch <- prometheus.MustNewConstMetric(p.data.numProcs, prometheus.GaugeValue, float64(gcounts.Procs)) ch <- prometheus.MustNewConstMetric(p.data.memResidentbytes, diff --git a/internal/ingress/metric/collectors/process_test.go b/internal/ingress/metric/collectors/process_test.go index 588cbafef..45170572b 100644 --- a/internal/ingress/metric/collectors/process_test.go +++ b/internal/ingress/metric/collectors/process_test.go @@ -48,11 +48,8 @@ func TestProcessCollector(t *testing.T) { done := make(chan struct{}) go func() { - cmd.Wait() //nolint:errcheck // Ignore the error - status, ok := cmd.ProcessState.Sys().(syscall.WaitStatus) - if !ok { - t.Errorf("unexpected type: %T", cmd.ProcessState.Sys()) - } + cmd.Wait() + status := cmd.ProcessState.Sys().(syscall.WaitStatus) if status.Signaled() { t.Logf("Signal: %v", status.Signal()) } else { @@ -72,7 +69,7 @@ func TestProcessCollector(t *testing.T) { defer func() { cm.Stop() - cmd.Process.Kill() //nolint:errcheck // Ignore the error + cmd.Process.Kill() <-done close(done) }() diff --git a/internal/ingress/metric/collectors/socket.go b/internal/ingress/metric/collectors/socket.go index 0bdd816ae..08c3865e3 100644 --- a/internal/ingress/metric/collectors/socket.go +++ b/internal/ingress/metric/collectors/socket.go @@ -19,9 +19,9 @@ package collectors import ( "fmt" "io" + "io/ioutil" "net" "os" - "strings" "syscall" jsoniter "github.com/json-iterator/go" @@ -30,6 +30,13 @@ import ( "k8s.io/klog/v2" ) +type upstream struct { + Latency float64 `json:"upstreamLatency"` + ResponseLength float64 `json:"upstreamResponseLength"` + ResponseTime float64 `json:"upstreamResponseTime"` + //Status string `json:"upstreamStatus"` +} + type socketData struct { Host string `json:"host"` Status string `json:"status"` @@ -41,69 +48,61 @@ type socketData struct { RequestLength float64 `json:"requestLength"` RequestTime float64 `json:"requestTime"` - Latency float64 `json:"upstreamLatency"` - HeaderTime float64 `json:"upstreamHeaderTime"` - ResponseTime float64 `json:"upstreamResponseTime"` - Namespace string `json:"namespace"` - Ingress string `json:"ingress"` - Service string `json:"service"` - Canary string `json:"canary"` - Path string `json:"path"` -} + upstream -// HistogramBuckets allow customizing prometheus histogram buckets values -type HistogramBuckets struct { - TimeBuckets []float64 - LengthBuckets []float64 - SizeBuckets []float64 + Namespace string `json:"namespace"` + Ingress string `json:"ingress"` + Service string `json:"service"` + Path string `json:"path"` } -type metricMapping map[string]prometheus.Collector - // SocketCollector stores prometheus metrics and ingress meta-data type SocketCollector struct { prometheus.Collector - connectTime *prometheus.HistogramVec - headerTime *prometheus.HistogramVec - requestTime *prometheus.HistogramVec - responseTime *prometheus.HistogramVec + requestTime *prometheus.HistogramVec + requestLength *prometheus.HistogramVec - requestLength *prometheus.HistogramVec + responseTime *prometheus.HistogramVec responseLength *prometheus.HistogramVec - bytesSent *prometheus.HistogramVec // TODO: DEPRECATED, remove + + upstreamLatency *prometheus.SummaryVec + + bytesSent *prometheus.HistogramVec requests *prometheus.CounterVec listener net.Listener - metricMapping metricMapping + metricMapping map[string]interface{} - hosts sets.Set[string] + hosts sets.String - metricsPerHost bool - metricsPerUndefinedHost bool - reportStatusClasses bool + metricsPerHost bool } -var requestTags = []string{ - "status", +var ( + requestTags = []string{ + "status", - "method", - "path", + "method", + "path", - "namespace", - "ingress", - "service", - "canary", -} + "namespace", + "ingress", + "service", + } +) + +// DefObjectives was removed in https://github.com/prometheus/client_golang/pull/262 +// updating the library to latest version changed the output of the metrics +var defObjectives = map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001} // NewSocketCollector creates a new SocketCollector instance using // the ingress watch namespace and class used by the controller -func NewSocketCollector(pod, namespace, class string, metricsPerHost, metricsPerUndefinedHost, reportStatusClasses bool, buckets HistogramBuckets, bucketFactor float64, maxBuckets uint32, excludeMetrics []string) (*SocketCollector, error) { - socket := "/tmp/nginx/prometheus-nginx.socket" +func NewSocketCollector(pod, namespace, class string, metricsPerHost bool) (*SocketCollector, error) { + socket := "/tmp/prometheus-nginx.socket" // unix sockets must be unlink()ed before being used - //nolint:errcheck // Ignore unlink error _ = syscall.Unlink(socket) listener, err := net.Listen("unix", socket) @@ -111,7 +110,7 @@ func NewSocketCollector(pod, namespace, class string, metricsPerHost, metricsPer return nil, err } - err = os.Chmod(socket, 0o777) // #nosec + err = os.Chmod(socket, 0777) // #nosec if err != nil { return nil, err } @@ -127,174 +126,98 @@ func NewSocketCollector(pod, namespace, class string, metricsPerHost, metricsPer requestTags = append(requestTags, "host") } - em := make(map[string]struct{}, len(excludeMetrics)) - for _, m := range excludeMetrics { - // remove potential nginx_ingress_controller prefix from the metric name - // TBD: how to handle fully qualified histogram metrics e.g. _buckets and _sum. Should we just remove the suffix and remove the histogram metric or ignore it? - em[strings.TrimPrefix(m, "nginx_ingress_controller_")] = struct{}{} - } - - // create metric mapping with only the metrics that are not excluded - mm := make(metricMapping) - sc := &SocketCollector{ listener: listener, - metricsPerHost: metricsPerHost, - metricsPerUndefinedHost: metricsPerUndefinedHost, - reportStatusClasses: reportStatusClasses, + metricsPerHost: metricsPerHost, - connectTime: histogramMetric( - &prometheus.HistogramOpts{ - Name: "connect_duration_seconds", - Help: "The time spent on establishing a connection with the upstream server", - Namespace: PrometheusNamespace, - ConstLabels: constLabels, - Buckets: buckets.TimeBuckets, - NativeHistogramBucketFactor: bucketFactor, - NativeHistogramMaxBucketNumber: maxBuckets, + responseTime: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "response_duration_seconds", + Help: "The time spent on receiving the response from the upstream server", + Namespace: PrometheusNamespace, + ConstLabels: constLabels, }, requestTags, - em, - mm, ), - - headerTime: histogramMetric( - &prometheus.HistogramOpts{ - Name: "header_duration_seconds", - Help: "The time spent on receiving first header from the upstream server", - Namespace: PrometheusNamespace, - ConstLabels: constLabels, - Buckets: buckets.TimeBuckets, - NativeHistogramBucketFactor: bucketFactor, - NativeHistogramMaxBucketNumber: maxBuckets, + responseLength: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "response_size", + Help: "The response length (including request line, header, and request body)", + Namespace: PrometheusNamespace, + ConstLabels: constLabels, }, requestTags, - em, - mm, - ), - responseTime: histogramMetric( - &prometheus.HistogramOpts{ - Name: "response_duration_seconds", - Help: "The time spent on receiving the response from the upstream server", - Namespace: PrometheusNamespace, - ConstLabels: constLabels, - Buckets: buckets.TimeBuckets, - NativeHistogramBucketFactor: bucketFactor, - NativeHistogramMaxBucketNumber: maxBuckets, - }, - requestTags, - em, - mm, ), - requestTime: histogramMetric( - &prometheus.HistogramOpts{ - Name: "request_duration_seconds", - Help: "The request processing time in milliseconds", - Namespace: PrometheusNamespace, - ConstLabels: constLabels, - Buckets: buckets.TimeBuckets, - NativeHistogramBucketFactor: bucketFactor, - NativeHistogramMaxBucketNumber: maxBuckets, + requestTime: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "request_duration_seconds", + Help: "The request processing time in milliseconds", + Namespace: PrometheusNamespace, + ConstLabels: constLabels, }, requestTags, - em, - mm, ), - - responseLength: histogramMetric( - &prometheus.HistogramOpts{ - Name: "response_size", - Help: "The response length (including request line, header, and request body)", - Namespace: PrometheusNamespace, - ConstLabels: constLabels, - Buckets: buckets.LengthBuckets, - NativeHistogramBucketFactor: bucketFactor, - NativeHistogramMaxBucketNumber: maxBuckets, + requestLength: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "request_size", + Help: "The request length (including request line, header, and request body)", + Namespace: PrometheusNamespace, + Buckets: prometheus.LinearBuckets(10, 10, 10), // 10 buckets, each 10 bytes wide. + ConstLabels: constLabels, }, requestTags, - em, - mm, ), - requestLength: histogramMetric( - &prometheus.HistogramOpts{ - Name: "request_size", - Help: "The request length (including request line, header, and request body)", - Namespace: PrometheusNamespace, - ConstLabels: constLabels, - Buckets: buckets.LengthBuckets, - NativeHistogramBucketFactor: bucketFactor, - NativeHistogramMaxBucketNumber: maxBuckets, - }, - requestTags, - em, - mm, - ), - - requests: counterMetric( - &prometheus.CounterOpts{ + requests: prometheus.NewCounterVec( + prometheus.CounterOpts{ Name: "requests", - Help: "The total number of client requests", + Help: "The total number of client requests.", Namespace: PrometheusNamespace, ConstLabels: constLabels, }, - requestTags, - em, - mm, + []string{"ingress", "namespace", "status", "service"}, ), - bytesSent: histogramMetric( - &prometheus.HistogramOpts{ + bytesSent: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ Name: "bytes_sent", - Help: "DEPRECATED The number of bytes sent to a client", + Help: "The number of bytes sent to a client", Namespace: PrometheusNamespace, - Buckets: buckets.SizeBuckets, + Buckets: prometheus.ExponentialBuckets(10, 10, 7), // 7 buckets, exponential factor of 10. ConstLabels: constLabels, }, requestTags, - em, - mm, + ), + + upstreamLatency: prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Name: "ingress_upstream_latency_seconds", + Help: "Upstream service latency per Ingress", + Namespace: PrometheusNamespace, + ConstLabels: constLabels, + Objectives: defObjectives, + }, + []string{"ingress", "namespace", "service"}, ), } - sc.metricMapping = mm + sc.metricMapping = map[string]interface{}{ + prometheus.BuildFQName(PrometheusNamespace, "", "request_duration_seconds"): sc.requestTime, + prometheus.BuildFQName(PrometheusNamespace, "", "request_size"): sc.requestLength, + + prometheus.BuildFQName(PrometheusNamespace, "", "response_duration_seconds"): sc.responseTime, + prometheus.BuildFQName(PrometheusNamespace, "", "response_size"): sc.responseLength, + + prometheus.BuildFQName(PrometheusNamespace, "", "bytes_sent"): sc.bytesSent, + + prometheus.BuildFQName(PrometheusNamespace, "", "ingress_upstream_latency_seconds"): sc.upstreamLatency, + } + return sc, nil } -func containsMetric(excludeMetrics map[string]struct{}, name string) bool { - if _, ok := excludeMetrics[name]; ok { - klog.V(3).InfoS("Skipping metric", "metric", name) - return true - } - return false -} - -func counterMetric(opts *prometheus.CounterOpts, requestTags []string, excludeMetrics map[string]struct{}, metricMapping metricMapping) *prometheus.CounterVec { - if containsMetric(excludeMetrics, opts.Name) { - return nil - } - m := prometheus.NewCounterVec( - *opts, - requestTags, - ) - metricMapping[prometheus.BuildFQName(PrometheusNamespace, "", opts.Name)] = m - return m -} - -func histogramMetric(opts *prometheus.HistogramOpts, requestTags []string, excludeMetrics map[string]struct{}, metricMapping metricMapping) *prometheus.HistogramVec { - if containsMetric(excludeMetrics, opts.Name) { - return nil - } - m := prometheus.NewHistogramVec( - *opts, - requestTags, - ) - metricMapping[prometheus.BuildFQName(PrometheusNamespace, "", opts.Name)] = m - return m -} - func (sc *SocketCollector) handleMessage(msg []byte) { klog.V(5).InfoS("Metric", "message", string(msg)) @@ -306,17 +229,12 @@ func (sc *SocketCollector) handleMessage(msg []byte) { return } - for i := range statsBatch { - stats := &statsBatch[i] - if sc.metricsPerHost && !sc.hosts.Has(stats.Host) && !sc.metricsPerUndefinedHost { - klog.V(3).InfoS("Skipping metric for host not explicitly defined in an ingress", "host", stats.Host) + for _, stats := range statsBatch { + if sc.metricsPerHost && !sc.hosts.Has(stats.Host) { + klog.V(3).InfoS("Skipping metric for host not being served", "host", stats.Host) continue } - if sc.reportStatusClasses && stats.Status != "" { - stats.Status = fmt.Sprintf("%cxx", stats.Status[0]) - } - // Note these must match the order in requestTags at the top requestLabels := prometheus.Labels{ "status": stats.Status, @@ -325,7 +243,9 @@ func (sc *SocketCollector) handleMessage(msg []byte) { "namespace": stats.Namespace, "ingress": stats.Ingress, "service": stats.Service, - "canary": stats.Canary, + } + if sc.metricsPerHost { + requestLabels["host"] = stats.Host } collectorLabels := prometheus.Labels{ @@ -333,45 +253,31 @@ func (sc *SocketCollector) handleMessage(msg []byte) { "ingress": stats.Ingress, "status": stats.Status, "service": stats.Service, - "canary": stats.Canary, - "method": stats.Method, - "path": stats.Path, - } - if sc.metricsPerHost { - requestLabels["host"] = stats.Host - collectorLabels["host"] = stats.Host } - if sc.requests != nil { - requestsMetric, err := sc.requests.GetMetricWith(collectorLabels) - if err != nil { - klog.ErrorS(err, "Error fetching requests metric") - } else { - requestsMetric.Inc() - } + latencyLabels := prometheus.Labels{ + "namespace": stats.Namespace, + "ingress": stats.Ingress, + "service": stats.Service, + } + + requestsMetric, err := sc.requests.GetMetricWith(collectorLabels) + if err != nil { + klog.ErrorS(err, "Error fetching requests metric") + } else { + requestsMetric.Inc() } if stats.Latency != -1 { - if sc.connectTime != nil { - connectTimeMetric, err := sc.connectTime.GetMetricWith(requestLabels) - if err != nil { - klog.ErrorS(err, "Error fetching connect time metric") - } else { - connectTimeMetric.Observe(stats.Latency) - } - } - } - - if stats.HeaderTime != -1 && sc.headerTime != nil { - headerTimeMetric, err := sc.headerTime.GetMetricWith(requestLabels) + latencyMetric, err := sc.upstreamLatency.GetMetricWith(latencyLabels) if err != nil { - klog.ErrorS(err, "Error fetching header time metric") + klog.ErrorS(err, "Error fetching latency metric") } else { - headerTimeMetric.Observe(stats.HeaderTime) + latencyMetric.Observe(stats.Latency) } } - if stats.RequestTime != -1 && sc.requestTime != nil { + if stats.RequestTime != -1 { requestTimeMetric, err := sc.requestTime.GetMetricWith(requestLabels) if err != nil { klog.ErrorS(err, "Error fetching request duration metric") @@ -380,7 +286,7 @@ func (sc *SocketCollector) handleMessage(msg []byte) { } } - if stats.RequestLength != -1 && sc.requestLength != nil { + if stats.RequestLength != -1 { requestLengthMetric, err := sc.requestLength.GetMetricWith(requestLabels) if err != nil { klog.ErrorS(err, "Error fetching request length metric") @@ -389,7 +295,7 @@ func (sc *SocketCollector) handleMessage(msg []byte) { } } - if stats.ResponseTime != -1 && sc.responseTime != nil { + if stats.ResponseTime != -1 { responseTimeMetric, err := sc.responseTime.GetMetricWith(requestLabels) if err != nil { klog.ErrorS(err, "Error fetching upstream response time metric") @@ -399,22 +305,18 @@ func (sc *SocketCollector) handleMessage(msg []byte) { } if stats.ResponseLength != -1 { - if sc.bytesSent != nil { - bytesSentMetric, err := sc.bytesSent.GetMetricWith(requestLabels) - if err != nil { - klog.ErrorS(err, "Error fetching bytes sent metric") - } else { - bytesSentMetric.Observe(stats.ResponseLength) - } + bytesSentMetric, err := sc.bytesSent.GetMetricWith(requestLabels) + if err != nil { + klog.ErrorS(err, "Error fetching bytes sent metric") + } else { + bytesSentMetric.Observe(stats.ResponseLength) } - if sc.responseLength != nil { - responseSizeMetric, err := sc.responseLength.GetMetricWith(requestLabels) - if err != nil { - klog.ErrorS(err, "Error fetching bytes sent metric") - } else { - responseSizeMetric.Observe(stats.ResponseLength) - } + responseSizeMetric, err := sc.responseLength.GetMetricWith(requestLabels) + if err != nil { + klog.ErrorS(err, "Error fetching bytes sent metric") + } else { + responseSizeMetric.Observe(stats.ResponseLength) } } } @@ -497,40 +399,50 @@ func (sc *SocketCollector) RemoveMetrics(ingresses []string, registry prometheus klog.V(2).InfoS("metric not removed", "name", metricName, "ingress", ingKey, "labels", labels) } } - - if c, ok := metric.(*prometheus.CounterVec); ok { - if removed := c.Delete(labels); !removed { - klog.V(2).InfoS("metric not removed", "name", metricName, "ingress", ingKey, "labels", labels) - } - } } } } // Describe implements prometheus.Collector -func (sc *SocketCollector) Describe(ch chan<- *prometheus.Desc) { - for _, metric := range sc.metricMapping { - metric.Describe(ch) - } +func (sc SocketCollector) Describe(ch chan<- *prometheus.Desc) { + sc.requestTime.Describe(ch) + sc.requestLength.Describe(ch) + + sc.requests.Describe(ch) + + sc.upstreamLatency.Describe(ch) + + sc.responseTime.Describe(ch) + sc.responseLength.Describe(ch) + + sc.bytesSent.Describe(ch) } // Collect implements the prometheus.Collector interface. -func (sc *SocketCollector) Collect(ch chan<- prometheus.Metric) { - for _, metric := range sc.metricMapping { - metric.Collect(ch) - } +func (sc SocketCollector) Collect(ch chan<- prometheus.Metric) { + sc.requestTime.Collect(ch) + sc.requestLength.Collect(ch) + + sc.requests.Collect(ch) + + sc.upstreamLatency.Collect(ch) + + sc.responseTime.Collect(ch) + sc.responseLength.Collect(ch) + + sc.bytesSent.Collect(ch) } // SetHosts sets the hostnames that are being served by the ingress controller // This set of hostnames is used to filter the metrics to be exposed -func (sc *SocketCollector) SetHosts(hosts sets.Set[string]) { +func (sc *SocketCollector) SetHosts(hosts sets.String) { sc.hosts = hosts } // handleMessages process the content received in a network connection func handleMessages(conn io.ReadCloser, fn func([]byte)) { defer conn.Close() - data, err := io.ReadAll(conn) + data, err := ioutil.ReadAll(conn) if err != nil { return } diff --git a/internal/ingress/metric/collectors/socket_test.go b/internal/ingress/metric/collectors/socket_test.go index 3a1f29f35..b9b3c6848 100644 --- a/internal/ingress/metric/collectors/socket_test.go +++ b/internal/ingress/metric/collectors/socket_test.go @@ -30,7 +30,7 @@ import ( func TestNewUDPLogListener(t *testing.T) { var count uint64 - fn := func(message []byte) { //nolint:unparam,revive // Unused `message` param is required by the handleMessages function + fn := func(message []byte) { atomic.AddUint64(&count, 1) } @@ -57,13 +57,8 @@ func TestNewUDPLogListener(t *testing.T) { } }() - conn, err := net.Dial("unix", tmpFile) - if err != nil { - t.Errorf("unexpected error connecting to unix socket: %v", err) - } - if _, err := conn.Write([]byte("message")); err != nil { - t.Errorf("unexpected error writing to unix socket: %v", err) - } + conn, _ := net.Dial("unix", tmpFile) + conn.Write([]byte("message")) conn.Close() time.Sleep(1 * time.Millisecond) @@ -73,29 +68,13 @@ func TestNewUDPLogListener(t *testing.T) { } func TestCollector(t *testing.T) { - buckets := struct { - TimeBuckets []float64 - LengthBuckets []float64 - SizeBuckets []float64 - }{ - prometheus.DefBuckets, - prometheus.LinearBuckets(10, 10, 10), - prometheus.ExponentialBuckets(10, 10, 7), - } - - bucketFactor := 1.1 - maxBuckets := uint32(100) - cases := []struct { - name string - data []string - metrics []string - metricsPerUndefinedHost bool - useStatusClasses bool - excludeMetrics []string - wantBefore string - removeIngresses []string - wantAfter string + name string + data []string + metrics []string + wantBefore string + removeIngresses []string + wantAfter string }{ { name: "invalid metric object should not increase prometheus metrics", @@ -107,16 +86,13 @@ func TestCollector(t *testing.T) { "path":"/admin", "requestLength":300.0, "requestTime":60.0, - "upstreamLatency":1.0, - "upstreamHeaderTime":5.0, "upstreamName":"test-upstream", "upstreamIP":"1.1.1.1:8080", "upstreamResponseTime":200, "upstreamStatus":"220", "namespace":"test-app-production", "ingress":"web-yml", - "service":"test-app", - "canary":"" + "service":"test-app" }`}, metrics: []string{"nginx_ingress_controller_response_duration_seconds"}, wantBefore: ` @@ -133,146 +109,32 @@ func TestCollector(t *testing.T) { "path":"/admin", "requestLength":300.0, "requestTime":60.0, - "upstreamLatency":1.0, - "upstreamHeaderTime":5.0, "upstreamName":"test-upstream", "upstreamIP":"1.1.1.1:8080", "upstreamResponseTime":200, "upstreamStatus":"220", "namespace":"test-app-production", "ingress":"web-yml", - "service":"test-app", - "canary":"" - }]`}, - metrics: []string{ - "nginx_ingress_controller_connect_duration_seconds", - "nginx_ingress_controller_header_duration_seconds", - "nginx_ingress_controller_response_duration_seconds", - }, - wantBefore: ` - # HELP nginx_ingress_controller_connect_duration_seconds The time spent on establishing a connection with the upstream server - # TYPE nginx_ingress_controller_connect_duration_seconds histogram - nginx_ingress_controller_connect_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.005"} 0 - nginx_ingress_controller_connect_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.01"} 0 - nginx_ingress_controller_connect_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.025"} 0 - nginx_ingress_controller_connect_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.05"} 0 - nginx_ingress_controller_connect_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.1"} 0 - nginx_ingress_controller_connect_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.25"} 0 - nginx_ingress_controller_connect_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.5"} 0 - nginx_ingress_controller_connect_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="1"} 1 - nginx_ingress_controller_connect_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="2.5"} 1 - nginx_ingress_controller_connect_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="5"} 1 - nginx_ingress_controller_connect_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="10"} 1 - nginx_ingress_controller_connect_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="+Inf"} 1 - nginx_ingress_controller_connect_duration_seconds_sum{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 1 - nginx_ingress_controller_connect_duration_seconds_count{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 1 - # HELP nginx_ingress_controller_header_duration_seconds The time spent on receiving first header from the upstream server - # TYPE nginx_ingress_controller_header_duration_seconds histogram - nginx_ingress_controller_header_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.005"} 0 - nginx_ingress_controller_header_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.01"} 0 - nginx_ingress_controller_header_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.025"} 0 - nginx_ingress_controller_header_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.05"} 0 - nginx_ingress_controller_header_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.1"} 0 - nginx_ingress_controller_header_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.25"} 0 - nginx_ingress_controller_header_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.5"} 0 - nginx_ingress_controller_header_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="1"} 0 - nginx_ingress_controller_header_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="2.5"} 0 - nginx_ingress_controller_header_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="5"} 1 - nginx_ingress_controller_header_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="10"} 1 - nginx_ingress_controller_header_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="+Inf"} 1 - nginx_ingress_controller_header_duration_seconds_sum{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 5 - nginx_ingress_controller_header_duration_seconds_count{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 1 - # HELP nginx_ingress_controller_response_duration_seconds The time spent on receiving the response from the upstream server - # TYPE nginx_ingress_controller_response_duration_seconds histogram - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.005"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.01"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.025"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.05"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.1"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.25"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="1"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="2.5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="10"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="+Inf"} 1 - nginx_ingress_controller_response_duration_seconds_sum{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 200 - nginx_ingress_controller_response_duration_seconds_count{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 1 - `, - removeIngresses: []string{"test-app-production/web-yml"}, - wantAfter: ` - `, - }, - { - name: "valid metric object should update requests metrics", - data: []string{`[{ - "host":"testshop.com", - "status":"200", - "bytesSent":150.0, - "method":"GET", - "path":"/admin", - "requestLength":300.0, - "requestTime":60.0, - "upstreamLatency":1.0, - "upstreamHeaderTime":5.0, - "upstreamName":"test-upstream", - "upstreamIP":"1.1.1.1:8080", - "upstreamResponseTime":200, - "upstreamStatus":"220", - "namespace":"test-app-production", - "ingress":"web-yml", - "service":"test-app", - "canary":"" - }]`}, - metrics: []string{"nginx_ingress_controller_requests"}, - wantBefore: ` - # HELP nginx_ingress_controller_requests The total number of client requests - # TYPE nginx_ingress_controller_requests counter - nginx_ingress_controller_requests{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 1 - `, - removeIngresses: []string{"test-app-production/web-yml"}, - wantAfter: ` - `, - }, - { - name: "valid metric object with canary information should update prometheus metrics", - data: []string{`[{ - "host":"testshop.com", - "status":"200", - "bytesSent":150.0, - "method":"GET", - "path":"/admin", - "requestLength":300.0, - "requestTime":60.0, - "upstreamLatency":1.0, - "upstreamHeaderTime":5.0, - "upstreamName":"test-upstream", - "upstreamIP":"1.1.1.1:8080", - "upstreamResponseTime":200, - "upstreamStatus":"220", - "namespace":"test-app-production", - "ingress":"web-yml", - "service":"test-app", - "canary":"test-app-production-test-app-canary-80" + "service":"test-app" }]`}, metrics: []string{"nginx_ingress_controller_response_duration_seconds"}, wantBefore: ` # HELP nginx_ingress_controller_response_duration_seconds The time spent on receiving the response from the upstream server # TYPE nginx_ingress_controller_response_duration_seconds histogram - nginx_ingress_controller_response_duration_seconds_bucket{canary="test-app-production-test-app-canary-80",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.005"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="test-app-production-test-app-canary-80",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.01"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="test-app-production-test-app-canary-80",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.025"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="test-app-production-test-app-canary-80",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.05"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="test-app-production-test-app-canary-80",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.1"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="test-app-production-test-app-canary-80",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.25"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="test-app-production-test-app-canary-80",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="test-app-production-test-app-canary-80",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="1"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="test-app-production-test-app-canary-80",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="2.5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="test-app-production-test-app-canary-80",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="test-app-production-test-app-canary-80",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="10"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="test-app-production-test-app-canary-80",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="+Inf"} 1 - nginx_ingress_controller_response_duration_seconds_sum{canary="test-app-production-test-app-canary-80",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 200 - nginx_ingress_controller_response_duration_seconds_count{canary="test-app-production-test-app-canary-80",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 1 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.005"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.01"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.025"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.05"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.1"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.25"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="1"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="2.5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="10"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="+Inf"} 1 + nginx_ingress_controller_response_duration_seconds_sum{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 200 + nginx_ingress_controller_response_duration_seconds_count{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 1 `, removeIngresses: []string{"test-app-production/web-yml"}, wantAfter: ` @@ -289,16 +151,13 @@ func TestCollector(t *testing.T) { "path":"/admin", "requestLength":300.0, "requestTime":60.0, - "upstreamLatency":1.0, - "upstreamHeaderTime":5.0, "upstreamName":"test-upstream", "upstreamIP":"1.1.1.1:8080", "upstreamResponseTime":200, "upstreamStatus":"220", "namespace":"test-app-production", "ingress":"web-yml", - "service":"test-app", - "canary":"" + "service":"test-app" }]`, `[{ "host":"testshop.com", "status":"200", @@ -307,16 +166,13 @@ func TestCollector(t *testing.T) { "path":"/admin", "requestLength":300.0, "requestTime":60.0, - "upstreamLatency":1.0, - "upstreamHeaderTime":5.0, "upstreamName":"test-upstream", "upstreamIP":"1.1.1.1:8080", "upstreamResponseTime":200, "upstreamStatus":"220", "namespace":"test-app-qa", "ingress":"web-yml-qa", - "service":"test-app-qa", - "canary":"" + "service":"test-app-qa" }]`, `[{ "host":"testshop.com", "status":"200", @@ -325,49 +181,46 @@ func TestCollector(t *testing.T) { "path":"/admin", "requestLength":300.0, "requestTime":60.0, - "upstreamLatency":1.0, - "upstreamHeaderTime":5.0, "upstreamName":"test-upstream", "upstreamIP":"1.1.1.1:8080", "upstreamResponseTime":200, "upstreamStatus":"220", "namespace":"test-app-qa", "ingress":"web-yml-qa", - "service":"test-app-qa", - "canary":"" + "service":"test-app-qa" }]`}, metrics: []string{"nginx_ingress_controller_response_duration_seconds"}, wantBefore: ` # HELP nginx_ingress_controller_response_duration_seconds The time spent on receiving the response from the upstream server # TYPE nginx_ingress_controller_response_duration_seconds histogram - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.005"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.01"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.025"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.05"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.1"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.25"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="1"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="2.5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="10"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="+Inf"} 1 - nginx_ingress_controller_response_duration_seconds_sum{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 200 - nginx_ingress_controller_response_duration_seconds_count{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 1 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.005"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.01"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.025"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.05"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.1"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.25"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="1"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="2.5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="10"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="+Inf"} 2 - nginx_ingress_controller_response_duration_seconds_sum{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200"} 400 - nginx_ingress_controller_response_duration_seconds_count{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200"} 2 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.005"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.01"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.025"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.05"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.1"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.25"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="1"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="2.5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="10"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="+Inf"} 1 + nginx_ingress_controller_response_duration_seconds_sum{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 200 + nginx_ingress_controller_response_duration_seconds_count{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 1 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.005"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.01"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.025"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.05"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.1"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.25"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="0.5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="1"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="2.5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="10"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200",le="+Inf"} 2 + nginx_ingress_controller_response_duration_seconds_sum{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200"} 400 + nginx_ingress_controller_response_duration_seconds_count{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml-qa",method="GET",namespace="test-app-qa",path="/admin",service="test-app-qa",status="200"} 2 `, }, @@ -382,16 +235,13 @@ func TestCollector(t *testing.T) { "path":"/admin", "requestLength":300.0, "requestTime":60.0, - "upstreamLatency":1.0, - "upstreamHeaderTime":5.0, "upstreamName":"test-upstream", "upstreamIP":"1.1.1.1:8080", "upstreamResponseTime":200, "upstreamStatus":"220", "namespace":"test-app-production", "ingress":"web-yml", - "service":"test-app", - "canary":"" + "service":"test-app" }, { "host":"testshop.com", @@ -401,260 +251,44 @@ func TestCollector(t *testing.T) { "path":"/admin", "requestLength":300.0, "requestTime":60.0, - "upstreamLatency":1.0, - "upstreamHeaderTime":5.0, "upstreamName":"test-upstream", "upstreamIP":"1.1.1.1:8080", "upstreamResponseTime":100, "upstreamStatus":"220", "namespace":"test-app-production", "ingress":"web-yml", - "service":"test-app", - "canary":"" + "service":"test-app" }]`}, metrics: []string{"nginx_ingress_controller_response_duration_seconds"}, wantBefore: ` # HELP nginx_ingress_controller_response_duration_seconds The time spent on receiving the response from the upstream server # TYPE nginx_ingress_controller_response_duration_seconds histogram - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.005"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.01"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.025"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.05"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.1"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.25"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="1"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="2.5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="10"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="+Inf"} 2 - nginx_ingress_controller_response_duration_seconds_sum{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 300 - nginx_ingress_controller_response_duration_seconds_count{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 2 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.005"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.01"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.025"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.05"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.1"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.25"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="0.5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="1"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="2.5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="5"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="10"} 0 + nginx_ingress_controller_response_duration_seconds_bucket{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200",le="+Inf"} 2 + nginx_ingress_controller_response_duration_seconds_sum{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 300 + nginx_ingress_controller_response_duration_seconds_count{controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="200"} 2 `, removeIngresses: []string{"test-app-production/web-yml"}, wantAfter: ` `, }, - { - name: "valid metric object with status classes should update prometheus metrics", - data: []string{`[{ - "host":"testshop.com", - "status":"200", - "bytesSent":150.0, - "method":"GET", - "path":"/admin", - "requestLength":300.0, - "requestTime":60.0, - "upstreamLatency":1.0, - "upstreamHeaderTime":5.0, - "upstreamName":"test-upstream", - "upstreamIP":"1.1.1.1:8080", - "upstreamResponseTime":200, - "upstreamStatus":"220", - "namespace":"test-app-production", - "ingress":"web-yml", - "service":"test-app", - "canary":"" - }]`}, - metrics: []string{"nginx_ingress_controller_response_duration_seconds"}, - useStatusClasses: true, - wantBefore: ` - # HELP nginx_ingress_controller_response_duration_seconds The time spent on receiving the response from the upstream server - # TYPE nginx_ingress_controller_response_duration_seconds histogram - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.005"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.01"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.025"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.05"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.1"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.25"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="1"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="2.5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="10"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="+Inf"} 1 - nginx_ingress_controller_response_duration_seconds_sum{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx"} 200 - nginx_ingress_controller_response_duration_seconds_count{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx"} 1 - `, - wantAfter: ` - `, - }, - { - name: "basic exclude metrics test", - data: []string{`[{ - "host":"testshop.com", - "status":"200", - "bytesSent":150.0, - "method":"GET", - "path":"/admin", - "requestLength":300.0, - "requestTime":60.0, - "upstreamLatency":1.0, - "upstreamHeaderTime":5.0, - "upstreamName":"test-upstream", - "upstreamIP":"1.1.1.1:8080", - "upstreamResponseTime":200, - "upstreamStatus":"220", - "namespace":"test-app-production", - "ingress":"web-yml", - "service":"test-app", - "canary":"" - }]`}, - excludeMetrics: []string{"nginx_ingress_controller_connect_duration_seconds"}, - metrics: []string{"nginx_ingress_controller_connect_duration_seconds", "nginx_ingress_controller_response_duration_seconds"}, - useStatusClasses: true, - wantBefore: ` - # HELP nginx_ingress_controller_response_duration_seconds The time spent on receiving the response from the upstream server - # TYPE nginx_ingress_controller_response_duration_seconds histogram - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.005"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.01"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.025"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.05"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.1"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.25"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="1"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="2.5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="10"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="+Inf"} 1 - nginx_ingress_controller_response_duration_seconds_sum{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx"} 200 - nginx_ingress_controller_response_duration_seconds_count{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx"} 1 - `, - }, - { - name: "remove metrics with the short metric name", - data: []string{`[{ - "host":"testshop.com", - "status":"200", - "bytesSent":150.0, - "method":"GET", - "path":"/admin", - "requestLength":300.0, - "requestTime":60.0, - "upstreamLatency":1.0, - "upstreamHeaderTime":5.0, - "upstreamName":"test-upstream", - "upstreamIP":"1.1.1.1:8080", - "upstreamResponseTime":200, - "upstreamStatus":"220", - "namespace":"test-app-production", - "ingress":"web-yml", - "service":"test-app", - "canary":"" - }]`}, - excludeMetrics: []string{"response_duration_seconds"}, - metrics: []string{"nginx_ingress_controller_response_duration_seconds"}, - useStatusClasses: true, - wantBefore: ` - `, - }, - { - name: "exclude metrics make sure to only remove exactly matched metrics", - data: []string{`[{ - "host":"testshop.com", - "status":"200", - "bytesSent":150.0, - "method":"GET", - "path":"/admin", - "requestLength":300.0, - "requestTime":60.0, - "upstreamLatency":1.0, - "upstreamHeaderTime":5.0, - "upstreamName":"test-upstream", - "upstreamIP":"1.1.1.1:8080", - "upstreamResponseTime":200, - "upstreamStatus":"220", - "namespace":"test-app-production", - "ingress":"web-yml", - "service":"test-app", - "canary":"" - }]`}, - excludeMetrics: []string{"response_duration_seconds2", "test.*", "nginx_ingress_.*", "response_duration_secon"}, - metrics: []string{"nginx_ingress_controller_response_duration_seconds"}, - useStatusClasses: true, - wantBefore: ` - # HELP nginx_ingress_controller_response_duration_seconds The time spent on receiving the response from the upstream server - # TYPE nginx_ingress_controller_response_duration_seconds histogram - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.005"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.01"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.025"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.05"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.1"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.25"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="1"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="2.5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="5"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="10"} 0 - nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="+Inf"} 1 - nginx_ingress_controller_response_duration_seconds_sum{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx"} 200 - nginx_ingress_controller_response_duration_seconds_count{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx"} 1 - `, - }, - { - name: "metrics with a host should not be dropped when the host is not in the hosts slice but metricsPerUndefinedHost is true", - data: []string{`[{ - "host":"wildcard.testshop.com", - "status":"200", - "bytesSent":150.0, - "method":"GET", - "path":"/admin", - "requestLength":300.0, - "requestTime":60.0, - "upstreamLatency":1.0, - "upstreamHeaderTime":5.0, - "upstreamName":"test-upstream", - "upstreamIP":"1.1.1.1:8080", - "upstreamResponseTime":200, - "upstreamStatus":"220", - "namespace":"test-app-production", - "ingress":"web-yml", - "service":"test-app", - "canary":"" - }]`}, - excludeMetrics: []string{"response_duration_seconds2", "test.*", "nginx_ingress_.*", "response_duration_secon"}, - metrics: []string{"nginx_ingress_controller_requests"}, - metricsPerUndefinedHost: true, - useStatusClasses: true, - wantBefore: ` - # HELP nginx_ingress_controller_requests The total number of client requests - # TYPE nginx_ingress_controller_requests counter - nginx_ingress_controller_requests{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="wildcard.testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx"} 1 - `, - }, - { - name: "metrics with a host should be dropped when the host is not in the hosts slice", - data: []string{`[{ - "host":"wildcard.testshop.com", - "status":"200", - "bytesSent":150.0, - "method":"GET", - "path":"/admin", - "requestLength":300.0, - "requestTime":60.0, - "upstreamLatency":1.0, - "upstreamHeaderTime":5.0, - "upstreamName":"test-upstream", - "upstreamIP":"1.1.1.1:8080", - "upstreamResponseTime":200, - "upstreamStatus":"220", - "namespace":"test-app-production", - "ingress":"web-yml", - "service":"test-app", - "canary":"" - }]`}, - excludeMetrics: []string{"response_duration_seconds2", "test.*", "nginx_ingress_.*", "response_duration_secon"}, - metrics: []string{"nginx_ingress_controller_requests"}, - useStatusClasses: true, - }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { registry := prometheus.NewPedanticRegistry() - sc, err := NewSocketCollector("pod", "default", "ingress", true, c.metricsPerUndefinedHost, c.useStatusClasses, buckets, bucketFactor, maxBuckets, c.excludeMetrics) + sc, err := NewSocketCollector("pod", "default", "ingress", true) if err != nil { t.Errorf("%v: unexpected error creating new SocketCollector: %v", c.name, err) } @@ -663,7 +297,7 @@ func TestCollector(t *testing.T) { t.Errorf("registering collector failed: %s", err) } - sc.SetHosts(sets.New[string]("testshop.com")) + sc.SetHosts(sets.NewString("testshop.com")) for _, d := range c.data { sc.handleMessage([]byte(d)) diff --git a/internal/ingress/metric/collectors/testutils.go b/internal/ingress/metric/collectors/testutils.go index 3fc0fc754..8c5c27b62 100644 --- a/internal/ingress/metric/collectors/testutils.go +++ b/internal/ingress/metric/collectors/testutils.go @@ -31,7 +31,7 @@ import ( // GatherAndCompare retrieves all metrics exposed by a collector and compares it // to an expected output in the Prometheus text exposition format. // metricNames allows only comparing the given metrics. All are compared if it's nil. -func GatherAndCompare(_ prometheus.Collector, expected string, metricNames []string, reg prometheus.Gatherer) error { +func GatherAndCompare(c prometheus.Collector, expected string, metricNames []string, reg prometheus.Gatherer) error { expected = removeUnusedWhitespace(expected) metrics, err := reg.Gather() @@ -50,7 +50,7 @@ func GatherAndCompare(_ prometheus.Collector, expected string, metricNames []str if !reflect.DeepEqual(metrics, normalizeMetricFamilies(expectedMetrics)) { // Encode the gathered output to the readable text format for comparison. var buf1 bytes.Buffer - enc := expfmt.NewEncoder(&buf1, expfmt.NewFormat(expfmt.TypeTextPlain)) + enc := expfmt.NewEncoder(&buf1, expfmt.FmtText) for _, mf := range metrics { if err := enc.Encode(mf); err != nil { return fmt.Errorf("encoding result failed: %s", err) @@ -59,7 +59,7 @@ func GatherAndCompare(_ prometheus.Collector, expected string, metricNames []str // Encode normalized expected metrics again to generate them in the same ordering // the registry does to spot differences more easily. var buf2 bytes.Buffer - enc = expfmt.NewEncoder(&buf2, expfmt.NewFormat(expfmt.TypeTextPlain)) + enc = expfmt.NewEncoder(&buf2, expfmt.FmtText) for _, mf := range normalizeMetricFamilies(expectedMetrics) { if err := enc.Encode(mf); err != nil { return fmt.Errorf("encoding result failed: %s", err) @@ -77,7 +77,9 @@ metric output does not match expectation; want: got: -'%s'`, buf2.String(), buf1.String()) +'%s' + +`, buf2.String(), buf1.String()) } return nil } @@ -109,7 +111,7 @@ func removeUnusedWhitespace(s string) string { for _, l := range lines { trimmedLine = strings.TrimSpace(l) - if trimmedLine != "" { + if len(trimmedLine) > 0 { trimmedLines = append(trimmedLines, trimmedLine) } } diff --git a/internal/ingress/metric/dummy.go b/internal/ingress/metric/dummy.go index 7b1485280..59a9144e0 100644 --- a/internal/ingress/metric/dummy.go +++ b/internal/ingress/metric/dummy.go @@ -18,7 +18,7 @@ package metric import ( "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/ingress-nginx/pkg/apis/ingress" + "k8s.io/ingress-nginx/internal/ingress" ) // NewDummyCollector returns a dummy metric collector @@ -29,50 +29,38 @@ func NewDummyCollector() Collector { // DummyCollector dummy implementation for mocks in tests type DummyCollector struct{} -// ConfigSuccess dummy implementation +// ConfigSuccess ... func (dc DummyCollector) ConfigSuccess(uint64, bool) {} -// SetAdmissionMetrics dummy implementation -func (dc DummyCollector) SetAdmissionMetrics(float64, float64, float64, float64, float64, float64) {} - -// IncReloadCount dummy implementation +// IncReloadCount ... func (dc DummyCollector) IncReloadCount() {} -// IncReloadErrorCount dummy implementation +// IncReloadErrorCount ... func (dc DummyCollector) IncReloadErrorCount() {} -// IncOrphanIngress dummy implementation -func (dc DummyCollector) IncOrphanIngress(string, string, string) {} - -// DecOrphanIngress dummy implementation -func (dc DummyCollector) DecOrphanIngress(string, string, string) {} - -// IncCheckCount dummy implementation +// IncCheckCount ... func (dc DummyCollector) IncCheckCount(string, string) {} -// IncCheckErrorCount dummy implementation +// IncCheckErrorCount ... func (dc DummyCollector) IncCheckErrorCount(string, string) {} -// RemoveMetrics dummy implementation -func (dc DummyCollector) RemoveMetrics(_, _ []string) {} +// RemoveMetrics ... +func (dc DummyCollector) RemoveMetrics(ingresses, endpoints []string) {} -// Start dummy implementation -func (dc DummyCollector) Start(_ string) {} +// Start ... +func (dc DummyCollector) Start() {} -// Stop dummy implementation -func (dc DummyCollector) Stop(_ string) {} +// Stop ... +func (dc DummyCollector) Stop() {} -// SetSSLInfo dummy implementation -func (dc DummyCollector) SetSSLInfo([]*ingress.Server) {} - -// SetSSLExpireTime dummy implementation +// SetSSLExpireTime ... func (dc DummyCollector) SetSSLExpireTime([]*ingress.Server) {} -// SetHosts dummy implementation -func (dc DummyCollector) SetHosts(_ sets.Set[string]) {} +// SetHosts ... +func (dc DummyCollector) SetHosts(hosts sets.String) {} // OnStartedLeading indicates the pod is not the current leader -func (dc DummyCollector) OnStartedLeading(_ string) {} +func (dc DummyCollector) OnStartedLeading(electionID string) {} // OnStoppedLeading indicates the pod is not the current leader -func (dc DummyCollector) OnStoppedLeading(_ string) {} +func (dc DummyCollector) OnStoppedLeading(electionID string) {} diff --git a/internal/ingress/metric/main.go b/internal/ingress/metric/main.go index 9ed401d19..d85e1c979 100644 --- a/internal/ingress/metric/main.go +++ b/internal/ingress/metric/main.go @@ -25,8 +25,9 @@ import ( "k8s.io/klog/v2" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/ingress-nginx/internal/ingress" + "k8s.io/ingress-nginx/internal/ingress/annotations/class" "k8s.io/ingress-nginx/internal/ingress/metric/collectors" - "k8s.io/ingress-nginx/pkg/apis/ingress" ) // Collector defines the interface for a metric collector @@ -36,34 +37,28 @@ type Collector interface { IncReloadCount() IncReloadErrorCount() - SetAdmissionMetrics(float64, float64, float64, float64, float64, float64) - OnStartedLeading(string) OnStoppedLeading(string) IncCheckCount(string, string) IncCheckErrorCount(string, string) - IncOrphanIngress(string, string, string) - DecOrphanIngress(string, string, string) - RemoveMetrics(ingresses, certificates []string) + RemoveMetrics(ingresses, endpoints []string) SetSSLExpireTime([]*ingress.Server) - SetSSLInfo(servers []*ingress.Server) // SetHosts sets the hostnames that are being served by the ingress controller - SetHosts(set sets.Set[string]) + SetHosts(sets.String) - Start(string) - Stop(string) + Start() + Stop() } type collector struct { nginxStatus collectors.NGINXStatusCollector nginxProcess collectors.NGINXProcessCollector - ingressController *collectors.Controller - admissionController *collectors.AdmissionCollector + ingressController *collectors.Controller socket *collectors.SocketCollector @@ -71,7 +66,7 @@ type collector struct { } // NewCollector creates a new metric collector the for ingress controller -func NewCollector(metricsPerHost, metricsPerUndefinedHost, reportStatusClasses bool, registry *prometheus.Registry, ingressclass string, buckets collectors.HistogramBuckets, bucketFactor float64, maxBuckets uint32, excludedSocketMetrics []string) (Collector, error) { +func NewCollector(metricsPerHost bool, registry *prometheus.Registry) (Collector, error) { podNamespace := os.Getenv("POD_NAMESPACE") if podNamespace == "" { podNamespace = "default" @@ -79,31 +74,28 @@ func NewCollector(metricsPerHost, metricsPerUndefinedHost, reportStatusClasses b podName := os.Getenv("POD_NAME") - nc, err := collectors.NewNGINXStatus(podName, podNamespace, ingressclass) + nc, err := collectors.NewNGINXStatus(podName, podNamespace, class.IngressClass) if err != nil { return nil, err } - pc, err := collectors.NewNGINXProcess(podName, podNamespace, ingressclass) + pc, err := collectors.NewNGINXProcess(podName, podNamespace, class.IngressClass) if err != nil { return nil, err } - s, err := collectors.NewSocketCollector(podName, podNamespace, ingressclass, metricsPerHost, metricsPerUndefinedHost, reportStatusClasses, buckets, bucketFactor, maxBuckets, excludedSocketMetrics) + s, err := collectors.NewSocketCollector(podName, podNamespace, class.IngressClass, metricsPerHost) if err != nil { return nil, err } - ic := collectors.NewController(podName, podNamespace, ingressclass) - - am := collectors.NewAdmissionCollector(podName, podNamespace, ingressclass) + ic := collectors.NewController(podName, podNamespace, class.IngressClass) return Collector(&collector{ nginxStatus: nc, nginxProcess: pc, - admissionController: am, - ingressController: ic, + ingressController: ic, socket: s, @@ -115,11 +107,11 @@ func (c *collector) ConfigSuccess(hash uint64, success bool) { c.ingressController.ConfigSuccess(hash, success) } -func (c *collector) IncCheckCount(namespace, name string) { +func (c *collector) IncCheckCount(namespace string, name string) { c.ingressController.IncCheckCount(namespace, name) } -func (c *collector) IncCheckErrorCount(namespace, name string) { +func (c *collector) IncCheckErrorCount(namespace string, name string) { c.ingressController.IncCheckErrorCount(namespace, name) } @@ -131,17 +123,14 @@ func (c *collector) IncReloadErrorCount() { c.ingressController.IncReloadErrorCount() } -func (c *collector) RemoveMetrics(ingresses, certificates []string) { +func (c *collector) RemoveMetrics(ingresses, hosts []string) { c.socket.RemoveMetrics(ingresses, c.registry) - c.ingressController.RemoveMetrics(certificates, c.registry) + c.ingressController.RemoveMetrics(hosts, c.registry) } -func (c *collector) Start(admissionStatus string) { +func (c *collector) Start() { c.registry.MustRegister(c.nginxStatus) c.registry.MustRegister(c.nginxProcess) - if admissionStatus != "" { - c.registry.MustRegister(c.admissionController) - } c.registry.MustRegister(c.ingressController) c.registry.MustRegister(c.socket) @@ -155,12 +144,9 @@ func (c *collector) Start(admissionStatus string) { go c.socket.Start() } -func (c *collector) Stop(admissionStatus string) { +func (c *collector) Stop() { c.registry.Unregister(c.nginxStatus) c.registry.Unregister(c.nginxProcess) - if admissionStatus != "" { - c.registry.Unregister(c.admissionController) - } c.registry.Unregister(c.ingressController) c.registry.Unregister(c.socket) @@ -178,34 +164,10 @@ func (c *collector) SetSSLExpireTime(servers []*ingress.Server) { c.ingressController.SetSSLExpireTime(servers) } -func (c *collector) SetSSLInfo(servers []*ingress.Server) { - klog.V(2).Infof("Updating ssl certificate info metrics") - c.ingressController.SetSSLInfo(servers) -} - -func (c *collector) IncOrphanIngress(namespace, name, orphanityType string) { - c.ingressController.IncOrphanIngress(namespace, name, orphanityType) -} - -func (c *collector) DecOrphanIngress(namespace, name, orphanityType string) { - c.ingressController.DecOrphanIngress(namespace, name, orphanityType) -} - -func (c *collector) SetHosts(hosts sets.Set[string]) { +func (c *collector) SetHosts(hosts sets.String) { c.socket.SetHosts(hosts) } -func (c *collector) SetAdmissionMetrics(testedIngressLength, testedIngressTime, renderingIngressLength, renderingIngressTime, testedConfigurationSize, admissionTime float64) { - c.admissionController.SetAdmissionMetrics( - testedIngressLength, - testedIngressTime, - renderingIngressLength, - renderingIngressTime, - testedConfigurationSize, - admissionTime, - ) -} - // OnStartedLeading indicates the pod was elected as the leader func (c *collector) OnStartedLeading(electionID string) { setLeader(true) @@ -216,10 +178,12 @@ func (c *collector) OnStartedLeading(electionID string) { func (c *collector) OnStoppedLeading(electionID string) { setLeader(false) c.ingressController.OnStoppedLeading(electionID) - c.ingressController.RemoveAllSSLMetrics(c.registry) + c.ingressController.RemoveAllSSLExpireMetrics(c.registry) } -var currentLeader uint32 +var ( + currentLeader uint32 +) func setLeader(leader bool) { var i uint32 diff --git a/internal/ingress/resolver/main.go b/internal/ingress/resolver/main.go index 259f44e49..e05a2aaae 100644 --- a/internal/ingress/resolver/main.go +++ b/internal/ingress/resolver/main.go @@ -26,13 +26,10 @@ type Resolver interface { // GetDefaultBackend returns the backend that must be used as default GetDefaultBackend() defaults.Backend - // GetSecurityConfiguration returns the configuration options from Ingress - GetSecurityConfiguration() defaults.SecurityConfiguration - - // GetConfigMap searches for configmap containing the namespace and name using the character / + // GetConfigMap searches for configmap containing the namespace and name usting the character / GetConfigMap(string) (*apiv1.ConfigMap, error) - // GetSecret searches for secrets containing the namespace and name using the character / + // GetSecret searches for secrets containing the namespace and name using a the character / GetSecret(string) (*apiv1.Secret, error) // GetAuthCertificate resolves a given secret name into an SSL certificate and CRL. @@ -42,7 +39,7 @@ type Resolver interface { // ca.crl: contains the revocation list used for authentication GetAuthCertificate(string) (*AuthSSLCert, error) - // GetService searches for services containing the namespace and name using the character / + // GetService searches for services containing the namespace and name using a the character / GetService(string) (*apiv1.Service, error) } diff --git a/internal/ingress/resolver/mock.go b/internal/ingress/resolver/mock.go index 3abfe7eda..556262b42 100644 --- a/internal/ingress/resolver/mock.go +++ b/internal/ingress/resolver/mock.go @@ -26,9 +26,7 @@ import ( // Mock implements the Resolver interface type Mock struct { - ConfigMaps map[string]*apiv1.ConfigMap - AnnotationsRiskLevel string - AllowCrossNamespace bool + ConfigMaps map[string]*apiv1.ConfigMap } // GetDefaultBackend returns the backend that must be used as default @@ -36,36 +34,24 @@ func (m Mock) GetDefaultBackend() defaults.Backend { return defaults.Backend{} } -func (m Mock) GetSecurityConfiguration() defaults.SecurityConfiguration { - defRisk := m.AnnotationsRiskLevel - if defRisk == "" { - defRisk = "High" - } - return defaults.SecurityConfiguration{ - AnnotationsRiskLevel: defRisk, - AllowCrossNamespaceResources: m.AllowCrossNamespace, - } -} - -// GetSecret searches for secrets containing the namespace and name using the character / +// GetSecret searches for secrets contenating the namespace and name using a the character / func (m Mock) GetSecret(string) (*apiv1.Secret, error) { return nil, nil } // GetAuthCertificate resolves a given secret name into an SSL certificate. // The secret must contain 3 keys named: -// -// ca.crt: contains the certificate chain used for authentication +// ca.crt: contains the certificate chain used for authentication func (m Mock) GetAuthCertificate(string) (*AuthSSLCert, error) { return nil, nil } -// GetService searches for services containing the namespace and name using the character / +// GetService searches for services contenating the namespace and name using a the character / func (m Mock) GetService(string) (*apiv1.Service, error) { return nil, nil } -// GetConfigMap searches for configMaps containing the namespace and name using the character / +// GetConfigMap searches for configMaps contenating the namespace and name using a the character / func (m Mock) GetConfigMap(name string) (*apiv1.ConfigMap, error) { if v, ok := m.ConfigMaps[name]; ok { return v, nil diff --git a/pkg/apis/ingress/sslcert.go b/internal/ingress/sslcert.go similarity index 83% rename from pkg/apis/ingress/sslcert.go rename to internal/ingress/sslcert.go index b340b2d9a..9265b07ac 100644 --- a/pkg/apis/ingress/sslcert.go +++ b/internal/ingress/sslcert.go @@ -18,7 +18,6 @@ package ingress import ( "crypto/x509" - "fmt" "time" "k8s.io/apimachinery/pkg/runtime/schema" @@ -66,22 +65,12 @@ type SSLCert struct { } // GetObjectKind implements the ObjectKind interface as a noop -func (s *SSLCert) GetObjectKind() schema.ObjectKind { +func (s SSLCert) GetObjectKind() schema.ObjectKind { return schema.EmptyObjectKind } -// Identifier returns a the couple issuer / serial number if they both exist, an empty string otherwise -func (s *SSLCert) Identifier() string { - if s.Certificate != nil { - if s.Certificate.SerialNumber != nil { - return fmt.Sprintf("%s-%s", s.Certificate.Issuer.SerialNumber, s.Certificate.SerialNumber.String()) - } - } - return "" -} - // HashInclude defines if a field should be used or not to calculate the hash -func (s *SSLCert) HashInclude(field string, _ interface{}) (bool, error) { +func (s SSLCert) HashInclude(field string, v interface{}) (bool, error) { switch field { case "PemSHA", "CASHA", "ExpireTime": return true, nil diff --git a/internal/ingress/status/status.go b/internal/ingress/status/status.go index ef01cdd24..506ae398c 100644 --- a/internal/ingress/status/status.go +++ b/internal/ingress/status/status.go @@ -25,26 +25,27 @@ import ( "strings" "time" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" pool "gopkg.in/go-playground/pool.v3" apiv1 "k8s.io/api/core/v1" - v1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + "k8s.io/ingress-nginx/internal/ingress" "k8s.io/ingress-nginx/internal/k8s" "k8s.io/ingress-nginx/internal/task" - "k8s.io/ingress-nginx/pkg/apis/ingress" ) // UpdateInterval defines the time interval, in seconds, in // which the status should check if an update is required. var UpdateInterval = 60 -// Syncer is an interface that implements syncer +// Syncer ... type Syncer interface { Run(chan struct{}) @@ -56,7 +57,7 @@ type ingressLister interface { ListIngresses() []*ingress.Ingress } -// Config is a structure that implements Client interfaces +// Config ... type Config struct { Client clientset.Interface @@ -87,7 +88,7 @@ type statusSync struct { } // Start starts the loop to keep the status in sync -func (s *statusSync) Run(stopCh chan struct{}) { +func (s statusSync) Run(stopCh chan struct{}) { go s.syncQueue.Run(time.Second, stopCh) // trigger initial sync @@ -95,19 +96,15 @@ func (s *statusSync) Run(stopCh chan struct{}) { // when this instance is the leader we need to enqueue // an item to trigger the update of the Ingress status. - //nolint:staticcheck // TODO: will replace it since wait.PollUntil is deprecated - err := wait.PollUntil(time.Duration(UpdateInterval)*time.Second, func() (bool, error) { + wait.PollUntil(time.Duration(UpdateInterval)*time.Second, func() (bool, error) { s.syncQueue.EnqueueTask(task.GetDummyObject("sync status")) return false, nil }, stopCh) - if err != nil { - klog.ErrorS(err, "error running poll") - } } // Shutdown stops the sync. In case the instance is the leader it will remove the current IP // if there is no other instances running. -func (s *statusSync) Shutdown() { +func (s statusSync) Shutdown() { go s.syncQueue.Shutdown() if !s.UpdateStatusOnShutdown { @@ -133,10 +130,10 @@ func (s *statusSync) Shutdown() { } klog.InfoS("removing value from ingress status", "address", addrs) - s.updateStatus([]v1.IngressLoadBalancerIngress{}) + s.updateStatus([]apiv1.LoadBalancerIngress{}) } -func (s *statusSync) sync(_ interface{}) error { +func (s *statusSync) sync(key interface{}) error { if s.syncQueue.IsShuttingDown() { klog.V(2).InfoS("skipping Ingress status update (shutting down in progress)") return nil @@ -146,18 +143,18 @@ func (s *statusSync) sync(_ interface{}) error { if err != nil { return err } - s.updateStatus(standardizeLoadBalancerIngresses(addrs)) + s.updateStatus(sliceToStatus(addrs)) return nil } -func (s *statusSync) keyfunc(input interface{}) (interface{}, error) { +func (s statusSync) keyfunc(input interface{}) (interface{}, error) { return input, nil } // NewStatusSyncer returns a new Syncer instance func NewStatusSyncer(config Config) Syncer { - st := &statusSync{ + st := statusSync{ Config: config, } st.syncQueue = task.NewCustomTaskQueue(st.sync, st.keyfunc) @@ -165,25 +162,13 @@ func NewStatusSyncer(config Config) Syncer { return st } -func nameOrIPToLoadBalancerIngress(nameOrIP string) v1.IngressLoadBalancerIngress { - if net.ParseIP(nameOrIP) != nil { - return v1.IngressLoadBalancerIngress{IP: nameOrIP} - } - - return v1.IngressLoadBalancerIngress{Hostname: nameOrIP} -} - // runningAddresses returns a list of IP addresses and/or FQDN where the // ingress controller is currently running -func (s *statusSync) runningAddresses() ([]v1.IngressLoadBalancerIngress, error) { +func (s *statusSync) runningAddresses() ([]string, error) { if s.PublishStatusAddress != "" { re := regexp.MustCompile(`,\s*`) multipleAddrs := re.Split(s.PublishStatusAddress, -1) - addrs := make([]v1.IngressLoadBalancerIngress, len(multipleAddrs)) - for i, addr := range multipleAddrs { - addrs[i] = nameOrIPToLoadBalancerIngress(addr) - } - return addrs, nil + return multipleAddrs, nil } if s.PublishService != "" { @@ -198,7 +183,7 @@ func (s *statusSync) runningAddresses() ([]v1.IngressLoadBalancerIngress, error) return nil, err } - addrs := make([]v1.IngressLoadBalancerIngress, 0) + addrs := make([]string, 0) for i := range pods.Items { pod := pods.Items[i] // only Running pods are valid @@ -221,8 +206,8 @@ func (s *statusSync) runningAddresses() ([]v1.IngressLoadBalancerIngress, error) } name := k8s.GetNodeIPOrName(s.Client, pod.Spec.NodeName, s.UseNodeInternalIP) - if !stringInIngresses(name, addrs) { - addrs = append(addrs, nameOrIPToLoadBalancerIngress(name)) + if !stringInSlice(name, addrs) { + addrs = append(addrs, name) } } @@ -230,20 +215,8 @@ func (s *statusSync) runningAddresses() ([]v1.IngressLoadBalancerIngress, error) } func (s *statusSync) isRunningMultiplePods() bool { - // As a standard, app.kubernetes.io are "reserved well-known" labels. - // In our case, we add those labels as identifiers of the Ingress - // deployment in this namespace, so we can select it as a set of Ingress instances. - // As those labels are also generated as part of a HELM deployment, we can be "safe" they - // cover 95% of the cases - podLabel := make(map[string]string) - for k, v := range k8s.IngressPodDetails.Labels { - if k != "pod-template-hash" && k != "controller-revision-hash" && k != "pod-template-generation" { - podLabel[k] = v - } - } - pods, err := s.Client.CoreV1().Pods(k8s.IngressPodDetails.Namespace).List(context.TODO(), metav1.ListOptions{ - LabelSelector: labels.SelectorFromSet(podLabel).String(), + LabelSelector: labels.SelectorFromSet(k8s.IngressPodDetails.Labels).String(), }) if err != nil { return false @@ -252,9 +225,17 @@ func (s *statusSync) isRunningMultiplePods() bool { return len(pods.Items) > 1 } -// standardizeLoadBalancerIngresses sorts the list of loadbalancer by -// IP -func standardizeLoadBalancerIngresses(lbi []v1.IngressLoadBalancerIngress) []v1.IngressLoadBalancerIngress { +// sliceToStatus converts a slice of IP and/or hostnames to LoadBalancerIngress +func sliceToStatus(endpoints []string) []apiv1.LoadBalancerIngress { + lbi := []apiv1.LoadBalancerIngress{} + for _, ep := range endpoints { + if net.ParseIP(ep) == nil { + lbi = append(lbi, apiv1.LoadBalancerIngress{Hostname: ep}) + } else { + lbi = append(lbi, apiv1.LoadBalancerIngress{IP: ep}) + } + } + sort.SliceStable(lbi, func(a, b int) bool { return lbi[a].IP < lbi[b].IP }) @@ -263,7 +244,7 @@ func standardizeLoadBalancerIngresses(lbi []v1.IngressLoadBalancerIngress) []v1. } // updateStatus changes the status information of Ingress rules -func (s *statusSync) updateStatus(newIngressPoint []v1.IngressLoadBalancerIngress) { +func (s *statusSync) updateStatus(newIngressPoint []apiv1.LoadBalancerIngress) { ings := s.IngressLister.ListIngresses() p := pool.NewLimited(10) @@ -287,18 +268,17 @@ func (s *statusSync) updateStatus(newIngressPoint []v1.IngressLoadBalancerIngres batch.WaitAll() } -func runUpdate(ing *ingress.Ingress, status []v1.IngressLoadBalancerIngress, - client clientset.Interface, -) pool.WorkFunc { +func runUpdate(ing *ingress.Ingress, status []apiv1.LoadBalancerIngress, + client clientset.Interface) pool.WorkFunc { return func(wu pool.WorkUnit) (interface{}, error) { if wu.IsCancelled() { return nil, nil } - ingClient := client.NetworkingV1().Ingresses(ing.Namespace) + ingClient := client.NetworkingV1beta1().Ingresses(ing.Namespace) currIng, err := ingClient.Get(context.TODO(), ing.Name, metav1.GetOptions{}) if err != nil { - return nil, fmt.Errorf("unexpected error searching Ingress %s/%s: %w", ing.Namespace, ing.Name, err) + return nil, errors.Wrap(err, fmt.Sprintf("unexpected error searching Ingress %v/%v", ing.Namespace, ing.Name)) } klog.InfoS("updating Ingress status", "namespace", currIng.Namespace, "ingress", currIng.Name, "currentValue", currIng.Status.LoadBalancer.Ingress, "newValue", status) @@ -312,7 +292,7 @@ func runUpdate(ing *ingress.Ingress, status []v1.IngressLoadBalancerIngress, } } -func lessLoadBalancerIngress(addrs []v1.IngressLoadBalancerIngress) func(int, int) bool { +func lessLoadBalancerIngress(addrs []apiv1.LoadBalancerIngress) func(int, int) bool { return func(a, b int) bool { switch strings.Compare(addrs[a].Hostname, addrs[b].Hostname) { case -1: @@ -324,7 +304,7 @@ func lessLoadBalancerIngress(addrs []v1.IngressLoadBalancerIngress) func(int, in } } -func ingressSliceEqual(lhs, rhs []v1.IngressLoadBalancerIngress) bool { +func ingressSliceEqual(lhs, rhs []apiv1.LoadBalancerIngress) bool { if len(lhs) != len(rhs) { return false } @@ -341,11 +321,8 @@ func ingressSliceEqual(lhs, rhs []v1.IngressLoadBalancerIngress) bool { return true } -func statusAddressFromService(service string, kubeClient clientset.Interface) ([]v1.IngressLoadBalancerIngress, error) { - ns, name, err := k8s.ParseNameNS(service) - if err != nil { - return nil, err - } +func statusAddressFromService(service string, kubeClient clientset.Interface) ([]string, error) { + ns, name, _ := k8s.ParseNameNS(service) svc, err := kubeClient.CoreV1().Services(ns).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { return nil, err @@ -353,50 +330,39 @@ func statusAddressFromService(service string, kubeClient clientset.Interface) ([ switch svc.Spec.Type { case apiv1.ServiceTypeExternalName: - return []v1.IngressLoadBalancerIngress{{ - Hostname: svc.Spec.ExternalName, - }}, nil + return []string{svc.Spec.ExternalName}, nil case apiv1.ServiceTypeClusterIP: - return []v1.IngressLoadBalancerIngress{{ - IP: svc.Spec.ClusterIP, - }}, nil + return []string{svc.Spec.ClusterIP}, nil case apiv1.ServiceTypeNodePort: - if svc.Spec.ExternalIPs == nil { - return []v1.IngressLoadBalancerIngress{{ - IP: svc.Spec.ClusterIP, - }}, nil + addresses := sets.NewString() + if svc.Spec.ExternalIPs != nil { + addresses.Insert(svc.Spec.ExternalIPs...) + } else { + addresses.Insert(svc.Spec.ClusterIP) } - addrs := make([]v1.IngressLoadBalancerIngress, 0, len(svc.Spec.ExternalIPs)) - for _, ip := range svc.Spec.ExternalIPs { - addrs = append(addrs, v1.IngressLoadBalancerIngress{IP: ip}) - } - return addrs, nil + return addresses.List(), nil case apiv1.ServiceTypeLoadBalancer: - addrs := make([]v1.IngressLoadBalancerIngress, 0, len(svc.Status.LoadBalancer.Ingress)) - for i, ingress := range svc.Status.LoadBalancer.Ingress { - addrs = append(addrs, v1.IngressLoadBalancerIngress{}) - if ingress.Hostname != "" { - addrs[i].Hostname = ingress.Hostname - } - if ingress.IP != "" { - addrs[i].IP = ingress.IP + addresses := sets.NewString() + for _, ip := range svc.Status.LoadBalancer.Ingress { + if ip.IP == "" { + addresses.Insert(ip.Hostname) + } else { + addresses.Insert(ip.IP) } } - for _, ip := range svc.Spec.ExternalIPs { - if !stringInIngresses(ip, addrs) { - addrs = append(addrs, v1.IngressLoadBalancerIngress{IP: ip}) - } - } - return addrs, nil + + addresses.Insert(svc.Spec.ExternalIPs...) + + return addresses.List(), nil } return nil, fmt.Errorf("unable to extract IP address/es from service %v", service) } -// stringInIngresses returns true if s is in list -func stringInIngresses(s string, list []v1.IngressLoadBalancerIngress) bool { +// stringInSlice returns true if s is in list +func stringInSlice(s string, list []string) bool { for _, v := range list { - if v.IP == s || v.Hostname == s { + if v == s { return true } } diff --git a/internal/ingress/status/status_test.go b/internal/ingress/status/status_test.go index 01419708b..9ad0c29cf 100644 --- a/internal/ingress/status/status_test.go +++ b/internal/ingress/status/status_test.go @@ -18,25 +18,24 @@ package status import ( "context" + "os" "reflect" "testing" "time" apiv1 "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" testclient "k8s.io/client-go/kubernetes/fake" - "k8s.io/ingress-nginx/internal/ingress/controller/ingressclass" + "k8s.io/ingress-nginx/internal/ingress" + "k8s.io/ingress-nginx/internal/ingress/annotations/class" "k8s.io/ingress-nginx/internal/k8s" "k8s.io/ingress-nginx/internal/task" - "k8s.io/ingress-nginx/pkg/apis/ingress" ) -const localhost = "127.0.0.1" - -func buildLoadBalancerIngressByIP() []networking.IngressLoadBalancerIngress { - return []networking.IngressLoadBalancerIngress{ +func buildLoadBalancerIngressByIP() []apiv1.LoadBalancerIngress { + return []apiv1.LoadBalancerIngress{ { IP: "10.0.0.1", Hostname: "foo1", @@ -124,9 +123,17 @@ func buildSimpleClientSet() *testclient.Clientset { }, }}, &apiv1.ServiceList{Items: []apiv1.Service{ - // This is commented out as the ServiceStatus.LoadBalancer field expects a LoadBalancerStatus object - // which is incompatible with the current Ingress struct which expects a IngressLoadBalancerStatus object - // TODO: update this service when the ServiceStatus struct gets updated + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: apiv1.NamespaceDefault, + }, + Status: apiv1.ServiceStatus{ + LoadBalancer: apiv1.LoadBalancerStatus{ + Ingress: buildLoadBalancerIngressByIP(), + }, + }, + }, { ObjectMeta: metav1.ObjectMeta{ Name: "foo_non_exist", @@ -175,8 +182,7 @@ func buildSimpleClientSet() *testclient.Clientset { Name: "ingress-controller-leader", Namespace: apiv1.NamespaceDefault, }, - }, - }}, + }}}, &networking.IngressList{Items: buildExtensionsIngresses()}, ) } @@ -193,8 +199,8 @@ func buildExtensionsIngresses() []networking.Ingress { Namespace: apiv1.NamespaceDefault, }, Status: networking.IngressStatus{ - LoadBalancer: networking.IngressLoadBalancerStatus{ - Ingress: []networking.IngressLoadBalancerIngress{ + LoadBalancer: apiv1.LoadBalancerStatus{ + Ingress: []apiv1.LoadBalancerIngress{ { IP: "10.0.0.1", Hostname: "foo1", @@ -208,12 +214,12 @@ func buildExtensionsIngresses() []networking.Ingress { Name: "foo_ingress_different_class", Namespace: metav1.NamespaceDefault, Annotations: map[string]string{ - ingressclass.IngressKey: "no-nginx", + class.IngressKey: "no-nginx", }, }, Status: networking.IngressStatus{ - LoadBalancer: networking.IngressLoadBalancerStatus{ - Ingress: []networking.IngressLoadBalancerIngress{ + LoadBalancer: apiv1.LoadBalancerStatus{ + Ingress: []apiv1.LoadBalancerIngress{ { IP: "0.0.0.0", Hostname: "foo.bar.com", @@ -228,41 +234,38 @@ func buildExtensionsIngresses() []networking.Ingress { Namespace: apiv1.NamespaceDefault, }, Status: networking.IngressStatus{ - LoadBalancer: networking.IngressLoadBalancerStatus{ - Ingress: []networking.IngressLoadBalancerIngress{}, + LoadBalancer: apiv1.LoadBalancerStatus{ + Ingress: []apiv1.LoadBalancerIngress{}, }, }, }, } } -type testIngressLister struct{} +type testIngressLister struct { +} func (til *testIngressLister) ListIngresses() []*ingress.Ingress { var ingresses []*ingress.Ingress - ingresses = append(ingresses, - &ingress.Ingress{ - Ingress: networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo_ingress_non_01", - Namespace: apiv1.NamespaceDefault, + ingresses = append(ingresses, &ingress.Ingress{ + Ingress: networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo_ingress_non_01", + Namespace: apiv1.NamespaceDefault, + }}}) + + ingresses = append(ingresses, &ingress.Ingress{ + Ingress: networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo_ingress_1", + Namespace: apiv1.NamespaceDefault, + }, + Status: networking.IngressStatus{ + LoadBalancer: apiv1.LoadBalancerStatus{ + Ingress: buildLoadBalancerIngressByIP(), }, }, - }, - &ingress.Ingress{ - Ingress: networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo_ingress_1", - Namespace: apiv1.NamespaceDefault, - }, - Status: networking.IngressStatus{ - LoadBalancer: networking.IngressLoadBalancerStatus{ - Ingress: buildLoadBalancerIngressByIP(), - }, - }, - }, - }, - ) + }}) return ingresses } @@ -284,8 +287,8 @@ func buildStatusSync() statusSync { func TestStatusActions(t *testing.T) { // make sure election can be created - t.Setenv("POD_NAME", "foo1") - t.Setenv("POD_NAMESPACE", apiv1.NamespaceDefault) + os.Setenv("POD_NAME", "foo1") + os.Setenv("POD_NAMESPACE", apiv1.NamespaceDefault) c := Config{ Client: buildSimpleClientSet(), PublishService: "", @@ -309,10 +312,7 @@ func TestStatusActions(t *testing.T) { t.Fatalf("expected a valid Sync") } - fk, ok := fkSync.(*statusSync) - if !ok { - t.Errorf("unexpected type: %T", fkSync) - } + fk := fkSync.(statusSync) // start it and wait for the election and syn actions stopCh := make(chan struct{}) @@ -322,15 +322,13 @@ func TestStatusActions(t *testing.T) { // wait for the election time.Sleep(100 * time.Millisecond) // execute sync - if err := fk.sync("just-test"); err != nil { - t.Errorf("unexpected error: %v", err) - } + fk.sync("just-test") // PublishService is empty, so the running address is: ["11.0.0.2"] // after updated, the ingress's ip should only be "11.0.0.2" - newIPs := []networking.IngressLoadBalancerIngress{{ + newIPs := []apiv1.LoadBalancerIngress{{ IP: "11.0.0.2", }} - fooIngress1, err1 := fk.Client.NetworkingV1().Ingresses(apiv1.NamespaceDefault).Get(context.TODO(), "foo_ingress_1", metav1.GetOptions{}) + fooIngress1, err1 := fk.Client.NetworkingV1beta1().Ingresses(apiv1.NamespaceDefault).Get(context.TODO(), "foo_ingress_1", metav1.GetOptions{}) if err1 != nil { t.Fatalf("unexpected error") } @@ -344,8 +342,8 @@ func TestStatusActions(t *testing.T) { // execute shutdown fk.Shutdown() // ingress should be empty - var newIPs2 []networking.IngressLoadBalancerIngress - fooIngress2, err2 := fk.Client.NetworkingV1().Ingresses(apiv1.NamespaceDefault).Get(context.TODO(), "foo_ingress_1", metav1.GetOptions{}) + newIPs2 := []apiv1.LoadBalancerIngress{} + fooIngress2, err2 := fk.Client.NetworkingV1beta1().Ingresses(apiv1.NamespaceDefault).Get(context.TODO(), "foo_ingress_1", metav1.GetOptions{}) if err2 != nil { t.Fatalf("unexpected error") } @@ -354,7 +352,7 @@ func TestStatusActions(t *testing.T) { t.Fatalf("returned %v but expected %v", fooIngress2CurIPs, newIPs2) } - oic, err := fk.Client.NetworkingV1().Ingresses(metav1.NamespaceDefault).Get(context.TODO(), "foo_ingress_different_class", metav1.GetOptions{}) + oic, err := fk.Client.NetworkingV1beta1().Ingresses(metav1.NamespaceDefault).Get(context.TODO(), "foo_ingress_different_class", metav1.GetOptions{}) if err != nil { t.Fatalf("unexpected error") } @@ -363,7 +361,7 @@ func TestStatusActions(t *testing.T) { } } -func TestCallback(_ *testing.T) { +func TestCallback(t *testing.T) { buildStatusSync() } @@ -372,6 +370,7 @@ func TestKeyfunc(t *testing.T) { i := "foo_base_pod" r, err := fk.keyfunc(i) + if err != nil { t.Fatalf("unexpected error") } @@ -383,177 +382,151 @@ func TestKeyfunc(t *testing.T) { func TestRunningAddressesWithPublishService(t *testing.T) { testCases := map[string]struct { fakeClient *testclient.Clientset - expected []networking.IngressLoadBalancerIngress + expected []string errExpected bool }{ "service type ClusterIP": { testclient.NewSimpleClientset( - &apiv1.PodList{ - Items: []apiv1.Pod{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: apiv1.NamespaceDefault, - }, - Spec: apiv1.PodSpec{ - NodeName: "foo_node", - }, - Status: apiv1.PodStatus{ - Phase: apiv1.PodRunning, - }, + &apiv1.PodList{Items: []apiv1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: apiv1.NamespaceDefault, + }, + Spec: apiv1.PodSpec{ + NodeName: "foo_node", + }, + Status: apiv1.PodStatus{ + Phase: apiv1.PodRunning, }, }, }, - &apiv1.ServiceList{ - Items: []apiv1.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: apiv1.NamespaceDefault, - }, - Spec: apiv1.ServiceSpec{ - Type: apiv1.ServiceTypeClusterIP, - ClusterIP: "1.1.1.1", - }, + }, + &apiv1.ServiceList{Items: []apiv1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: apiv1.NamespaceDefault, + }, + Spec: apiv1.ServiceSpec{ + Type: apiv1.ServiceTypeClusterIP, + ClusterIP: "1.1.1.1", }, }, }, + }, ), - []networking.IngressLoadBalancerIngress{ - {IP: "1.1.1.1"}, - }, + []string{"1.1.1.1"}, false, }, "service type NodePort": { testclient.NewSimpleClientset( - &apiv1.ServiceList{ - Items: []apiv1.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: apiv1.NamespaceDefault, - }, - Spec: apiv1.ServiceSpec{ - Type: apiv1.ServiceTypeNodePort, - ClusterIP: "1.1.1.1", - }, + &apiv1.ServiceList{Items: []apiv1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: apiv1.NamespaceDefault, + }, + Spec: apiv1.ServiceSpec{ + Type: apiv1.ServiceTypeNodePort, + ClusterIP: "1.1.1.1", }, }, }, + }, ), - []networking.IngressLoadBalancerIngress{ - {IP: "1.1.1.1"}, - }, + []string{"1.1.1.1"}, false, }, "service type ExternalName": { testclient.NewSimpleClientset( - &apiv1.ServiceList{ - Items: []apiv1.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: apiv1.NamespaceDefault, - }, - Spec: apiv1.ServiceSpec{ - Type: apiv1.ServiceTypeExternalName, - ExternalName: "foo.bar", - }, + &apiv1.ServiceList{Items: []apiv1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: apiv1.NamespaceDefault, + }, + Spec: apiv1.ServiceSpec{ + Type: apiv1.ServiceTypeExternalName, + ExternalName: "foo.bar", }, }, }, + }, ), - []networking.IngressLoadBalancerIngress{ - {Hostname: "foo.bar"}, - }, + []string{"foo.bar"}, false, }, "service type LoadBalancer": { testclient.NewSimpleClientset( - &apiv1.ServiceList{ - Items: []apiv1.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: apiv1.NamespaceDefault, - }, - Spec: apiv1.ServiceSpec{ - Type: apiv1.ServiceTypeLoadBalancer, - }, - Status: apiv1.ServiceStatus{ - LoadBalancer: apiv1.LoadBalancerStatus{ - Ingress: []apiv1.LoadBalancerIngress{ - { - IP: "10.0.0.1", - }, - { - IP: "", - Hostname: "foo", - }, - { - IP: "10.0.0.2", - Hostname: "10-0-0-2.cloudprovider.example.net", - }, + &apiv1.ServiceList{Items: []apiv1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: apiv1.NamespaceDefault, + }, + Spec: apiv1.ServiceSpec{ + Type: apiv1.ServiceTypeLoadBalancer, + }, + Status: apiv1.ServiceStatus{ + LoadBalancer: apiv1.LoadBalancerStatus{ + Ingress: []apiv1.LoadBalancerIngress{ + { + IP: "10.0.0.1", + }, + { + IP: "", + Hostname: "foo", }, }, }, }, }, }, - ), - []networking.IngressLoadBalancerIngress{ - {IP: "10.0.0.1"}, - {Hostname: "foo"}, - { - IP: "10.0.0.2", - Hostname: "10-0-0-2.cloudprovider.example.net", }, - }, + ), + []string{"10.0.0.1", "foo"}, false, }, "service type LoadBalancer with same externalIP and ingress IP": { testclient.NewSimpleClientset( - &apiv1.ServiceList{ - Items: []apiv1.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: apiv1.NamespaceDefault, - }, - Spec: apiv1.ServiceSpec{ - Type: apiv1.ServiceTypeLoadBalancer, - ExternalIPs: []string{"10.0.0.1"}, - }, - Status: apiv1.ServiceStatus{ - LoadBalancer: apiv1.LoadBalancerStatus{ - Ingress: []apiv1.LoadBalancerIngress{ - { - IP: "10.0.0.1", - }, + &apiv1.ServiceList{Items: []apiv1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: apiv1.NamespaceDefault, + }, + Spec: apiv1.ServiceSpec{ + Type: apiv1.ServiceTypeLoadBalancer, + ExternalIPs: []string{"10.0.0.1"}, + }, + Status: apiv1.ServiceStatus{ + LoadBalancer: apiv1.LoadBalancerStatus{ + Ingress: []apiv1.LoadBalancerIngress{ + { + IP: "10.0.0.1", }, }, }, }, }, }, + }, ), - []networking.IngressLoadBalancerIngress{ - {IP: "10.0.0.1"}, - }, + []string{"10.0.0.1"}, false, }, "invalid service type": { testclient.NewSimpleClientset( - &apiv1.ServiceList{ - Items: []apiv1.Service{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: apiv1.NamespaceDefault, - }, + &apiv1.ServiceList{Items: []apiv1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: apiv1.NamespaceDefault, }, }, }, + }, ), nil, true, @@ -562,6 +535,7 @@ func TestRunningAddressesWithPublishService(t *testing.T) { for title, tc := range testCases { t.Run(title, func(t *testing.T) { + fk := buildStatusSync() fk.Config.Client = tc.fakeClient @@ -575,7 +549,7 @@ func TestRunningAddressesWithPublishService(t *testing.T) { } if ra == nil { - t.Fatalf("returned nil but expected valid []networking.IngressLoadBalancerIngress") + t.Fatalf("returned nil but expected valid []string") } if !reflect.DeepEqual(tc.expected, ra) { @@ -589,42 +563,35 @@ func TestRunningAddressesWithPods(t *testing.T) { fk := buildStatusSync() fk.PublishService = "" - r, err := fk.runningAddresses() - if err != nil { - t.Errorf("unexpected error: %v", err) - } - + r, _ := fk.runningAddresses() if r == nil { - t.Fatalf("returned nil but expected valid []networking.IngressLoadBalancerIngress") + t.Fatalf("returned nil but expected valid []string") } rl := len(r) if len(r) != 1 { t.Fatalf("returned %v but expected %v", rl, 1) } rv := r[0] - if rv.IP != "11.0.0.2" { - t.Errorf("returned %v but expected %v", rv, networking.IngressLoadBalancerIngress{IP: "11.0.0.2"}) + if rv != "11.0.0.2" { + t.Errorf("returned %v but expected %v", rv, "11.0.0.2") } } func TestRunningAddressesWithPublishStatusAddress(t *testing.T) { fk := buildStatusSync() - fk.PublishStatusAddress = localhost + fk.PublishStatusAddress = "127.0.0.1" - ra, err := fk.runningAddresses() - if err != nil { - t.Errorf("unexpected error: %v", err) - } + ra, _ := fk.runningAddresses() if ra == nil { - t.Fatalf("returned nil but expected valid []networking.IngressLoadBalancerIngress") + t.Fatalf("returned nil but expected valid []string") } rl := len(ra) if len(ra) != 1 { t.Errorf("returned %v but expected %v", rl, 1) } rv := ra[0] - if rv.IP != localhost { - t.Errorf("returned %v but expected %v", rv, networking.IngressLoadBalancerIngress{IP: localhost}) + if rv != "127.0.0.1" { + t.Errorf("returned %v but expected %v", rv, "127.0.0.1") } } @@ -632,12 +599,9 @@ func TestRunningAddressesWithPublishStatusAddresses(t *testing.T) { fk := buildStatusSync() fk.PublishStatusAddress = "127.0.0.1,1.1.1.1" - ra, err := fk.runningAddresses() - if err != nil { - t.Errorf("unexpected error: %v", err) - } + ra, _ := fk.runningAddresses() if ra == nil { - t.Fatalf("returned nil but expected valid []networking.IngressLoadBalancerIngress") + t.Fatalf("returned nil but expected valid []string") } rl := len(ra) if len(ra) != 2 { @@ -645,11 +609,11 @@ func TestRunningAddressesWithPublishStatusAddresses(t *testing.T) { } rv := ra[0] rv2 := ra[1] - if rv.IP != localhost { - t.Errorf("returned %v but expected %v", rv, networking.IngressLoadBalancerIngress{IP: localhost}) + if rv != "127.0.0.1" { + t.Errorf("returned %v but expected %v", rv, "127.0.0.1") } - if rv2.IP != "1.1.1.1" { - t.Errorf("returned %v but expected %v", rv2, networking.IngressLoadBalancerIngress{IP: "1.1.1.1"}) + if rv2 != "1.1.1.1" { + t.Errorf("returned %v but expected %v", rv2, "1.1.1.1") } } @@ -657,12 +621,9 @@ func TestRunningAddressesWithPublishStatusAddressesAndSpaces(t *testing.T) { fk := buildStatusSync() fk.PublishStatusAddress = "127.0.0.1, 1.1.1.1" - ra, err := fk.runningAddresses() - if err != nil { - t.Errorf("unexpected error: %v", err) - } + ra, _ := fk.runningAddresses() if ra == nil { - t.Fatalf("returned nil but expected valid []networking.IngressLoadBalancerIngresst") + t.Fatalf("returned nil but expected valid []string") } rl := len(ra) if len(ra) != 2 { @@ -670,25 +631,25 @@ func TestRunningAddressesWithPublishStatusAddressesAndSpaces(t *testing.T) { } rv := ra[0] rv2 := ra[1] - if rv.IP != localhost { - t.Errorf("returned %v but expected %v", rv, networking.IngressLoadBalancerIngress{IP: localhost}) + if rv != "127.0.0.1" { + t.Errorf("returned %v but expected %v", rv, "127.0.0.1") } - if rv2.IP != "1.1.1.1" { - t.Errorf("returned %v but expected %v", rv2, networking.IngressLoadBalancerIngress{IP: "1.1.1.1"}) + if rv2 != "1.1.1.1" { + t.Errorf("returned %v but expected %v", rv2, "1.1.1.1") } } -func TestStandardizeLoadBalancerIngresses(t *testing.T) { - fkEndpoints := []networking.IngressLoadBalancerIngress{ - {IP: "2001:db8::68"}, - {IP: "10.0.0.1"}, - {Hostname: "opensource-k8s-ingress"}, +func TestSliceToStatus(t *testing.T) { + fkEndpoints := []string{ + "10.0.0.1", + "2001:db8::68", + "opensource-k8s-ingress", } - r := standardizeLoadBalancerIngresses(fkEndpoints) + r := sliceToStatus(fkEndpoints) if r == nil { - t.Fatalf("returned nil but expected a valid []networking.IngressLoadBalancerIngress") + t.Fatalf("returned nil but expected a valid []apiv1.LoadBalancerIngress") } rl := len(r) if rl != 3 { @@ -696,21 +657,21 @@ func TestStandardizeLoadBalancerIngresses(t *testing.T) { } re1 := r[0] if re1.Hostname != "opensource-k8s-ingress" { - t.Fatalf("returned %v but expected %v", re1, networking.IngressLoadBalancerIngress{Hostname: "opensource-k8s-ingress"}) + t.Fatalf("returned %v but expected %v", re1, apiv1.LoadBalancerIngress{Hostname: "opensource-k8s-ingress"}) } re2 := r[1] if re2.IP != "10.0.0.1" { - t.Fatalf("returned %v but expected %v", re2, networking.IngressLoadBalancerIngress{IP: "10.0.0.1"}) + t.Fatalf("returned %v but expected %v", re2, apiv1.LoadBalancerIngress{IP: "10.0.0.1"}) } re3 := r[2] if re3.IP != "2001:db8::68" { - t.Fatalf("returned %v but expected %v", re3, networking.IngressLoadBalancerIngress{IP: "2001:db8::68"}) + t.Fatalf("returned %v but expected %v", re3, apiv1.LoadBalancerIngress{IP: "2001:db8::68"}) } } func TestIngressSliceEqual(t *testing.T) { fk1 := buildLoadBalancerIngressByIP() - fk2 := append(buildLoadBalancerIngressByIP(), networking.IngressLoadBalancerIngress{ + fk2 := append(buildLoadBalancerIngressByIP(), apiv1.LoadBalancerIngress{ IP: "10.0.0.5", Hostname: "foo5", }) @@ -720,8 +681,8 @@ func TestIngressSliceEqual(t *testing.T) { fk4[2].IP = "11.0.0.3" fooTests := []struct { - lhs []networking.IngressLoadBalancerIngress - rhs []networking.IngressLoadBalancerIngress + lhs []apiv1.LoadBalancerIngress + rhs []apiv1.LoadBalancerIngress er bool }{ {fk1, fk1, true}, @@ -730,7 +691,7 @@ func TestIngressSliceEqual(t *testing.T) { {fk4, fk1, false}, {fk1, nil, false}, {nil, nil, true}, - {[]networking.IngressLoadBalancerIngress{}, []networking.IngressLoadBalancerIngress{}, true}, + {[]apiv1.LoadBalancerIngress{}, []apiv1.LoadBalancerIngress{}, true}, } for _, fooTest := range fooTests { diff --git a/pkg/apis/ingress/types.go b/internal/ingress/types.go similarity index 86% rename from pkg/apis/ingress/types.go rename to internal/ingress/types.go index ccdd49fe9..84d964b7a 100644 --- a/pkg/apis/ingress/types.go +++ b/internal/ingress/types.go @@ -18,7 +18,7 @@ package ingress import ( apiv1 "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/internal/ingress/annotations" @@ -27,14 +27,14 @@ import ( "k8s.io/ingress-nginx/internal/ingress/annotations/authtls" "k8s.io/ingress-nginx/internal/ingress/annotations/connection" "k8s.io/ingress-nginx/internal/ingress/annotations/cors" - "k8s.io/ingress-nginx/internal/ingress/annotations/customheaders" "k8s.io/ingress-nginx/internal/ingress/annotations/fastcgi" - "k8s.io/ingress-nginx/internal/ingress/annotations/ipallowlist" - "k8s.io/ingress-nginx/internal/ingress/annotations/ipdenylist" + "k8s.io/ingress-nginx/internal/ingress/annotations/globalratelimit" + "k8s.io/ingress-nginx/internal/ingress/annotations/influxdb" + "k8s.io/ingress-nginx/internal/ingress/annotations/ipwhitelist" "k8s.io/ingress-nginx/internal/ingress/annotations/log" "k8s.io/ingress-nginx/internal/ingress/annotations/mirror" "k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity" - "k8s.io/ingress-nginx/internal/ingress/annotations/opentelemetry" + "k8s.io/ingress-nginx/internal/ingress/annotations/opentracing" "k8s.io/ingress-nginx/internal/ingress/annotations/proxy" "k8s.io/ingress-nginx/internal/ingress/annotations/proxyssl" "k8s.io/ingress-nginx/internal/ingress/annotations/ratelimit" @@ -42,8 +42,13 @@ import ( "k8s.io/ingress-nginx/internal/ingress/annotations/rewrite" ) -// TODO: The API shouldn't be importing structs from annotation code. Instead we probably want a conversion from internal -// to API object, or see how much effort is to move the structs of annotations to become an "API'ish" as well +var ( + // DefaultSSLDirectory defines the location where the SSL certificates will be generated + // This directory contains all the SSL certificates that are specified in Ingress rules. + // The name of each file is -.pem. The content is the concatenated + // certificate and key. + DefaultSSLDirectory = "/ingress-controller/ssl" +) // Configuration holds the definition of all the parts required to describe all // ingresses reachable by the ingress controller (using a filter by namespace) @@ -71,8 +76,6 @@ type Configuration struct { ConfigurationChecksum string `json:"configurationChecksum,omitempty"` DefaultSSLCertificate *SSLCert `json:"-"` - - StreamSnippets []string `json:"StreamSnippets"` } // Backend describes one or more remote server/s (endpoints) associated with a service @@ -108,15 +111,10 @@ type Backend struct { // alternative backend // +k8s:deepcopy-gen=true type TrafficShapingPolicy struct { - // Weight (0-) of traffic to redirect to the backend. - // e.g. defaults to 100, weight 20 means 20% of traffic will be - // redirected to the backend and 80% will remain with the other backend. If - // is set to 1000, weight 2 means 0.2% of traffic will be - // redirected to the backend and 99.8% will remain with the other backend. - // 0 weight will not send any traffic to this backend + // Weight (0-100) of traffic to redirect to the backend. + // e.g. Weight 20 means 20% of traffic will be redirected to the backend and 80% will remain + // with the other backend. 0 weight will not send any traffic to this backend Weight int `json:"weight"` - // The total weight of traffic (>= 100). If unspecified, it defaults to 100. - WeightTotal int `json:"weightTotal"` // Header on which to redirect requests to this backend Header string `json:"header"` // HeaderValue on which to redirect requests to this backend @@ -128,7 +126,7 @@ type TrafficShapingPolicy struct { } // HashInclude defines if a field should be used or not to calculate the hash -func (b *Backend) HashInclude(field string, _ interface{}) (bool, error) { +func (s Backend) HashInclude(field string, v interface{}) (bool, error) { switch field { case "Endpoints": return false, nil @@ -157,9 +155,7 @@ type CookieSessionAffinity struct { Expires string `json:"expires,omitempty"` MaxAge string `json:"maxage,omitempty"` Locations map[string][]string `json:"locations,omitempty"` - Secure bool `json:"secure,omitempty"` Path string `json:"path,omitempty"` - Domain string `json:"domain,omitempty"` SameSite string `json:"samesite,omitempty"` ConditionalSameSiteNone bool `json:"conditional_samesite_none,omitempty"` ChangeOnFailure bool `json:"change_on_failure,omitempty"` @@ -198,10 +194,10 @@ type Server struct { Aliases []string `json:"aliases,omitempty"` // RedirectFromToWWW returns if a redirect to/from prefix www is required RedirectFromToWWW bool `json:"redirectFromToWWW,omitempty"` - // CertificateAuth indicates this server requires mutual authentication + // CertificateAuth indicates the this server requires mutual authentication // +optional CertificateAuth authtls.Config `json:"certificateAuth"` - // ProxySSL indicates this server uses client certificate to access backends + // ProxySSL indicates the this server uses client certificate to access backends // +optional ProxySSL proxyssl.Config `json:"proxySSL"` // ServerSnippet returns the snippet of server @@ -210,7 +206,7 @@ type Server struct { // SSLCiphers returns list of ciphers to be enabled SSLCiphers string `json:"sslCiphers,omitempty"` // SSLPreferServerCiphers indicates that server ciphers should be preferred - // over client ciphers when using the TLS protocols. + // over client ciphers when using the SSLv3 and TLS protocols. SSLPreferServerCiphers string `json:"sslPreferServerCiphers,omitempty"` // AuthTLSError contains the reason why the access to a server should be denied AuthTLSError string `json:"authTLSError,omitempty"` @@ -219,11 +215,10 @@ type Server struct { // Location describes an URI inside a server. // Also contains additional information about annotations in the Ingress. // -// In some cases when more than one annotation is defined a particular order in the execution +// In some cases when more than one annotations is defined a particular order in the execution // is required. // The chain in the execution order of annotations should be: -// - Denylist -// - Allowlist +// - Whitelist // - RateLimit // - BasicDigestAuth // - ExternalAuth @@ -263,8 +258,7 @@ type Location struct { BasicDigestAuth auth.Config `json:"basicDigestAuth,omitempty"` // Denied returns an error when this location cannot not be allowed // Requesting a denied location should return HTTP code 403. - Denied *string `json:"denied,omitempty"` - CustomHeaders customheaders.Config `json:"customHeaders,omitempty"` + Denied *string `json:"denied,omitempty"` // CorsConfig returns the Cors Configuration for the ingress rule // +optional CorsConfig cors.Config `json:"corsConfig,omitempty"` @@ -284,20 +278,20 @@ type Location struct { // The Redirect annotation precedes RateLimit // +optional RateLimit ratelimit.Config `json:"rateLimit,omitempty"` + // GlobalRateLimit similar to RateLimit + // but this is applied globally across multiple replicas. + // +optional + GlobalRateLimit globalratelimit.Config `json:"globalRateLimit,omitempty"` // Redirect describes a temporal o permanent redirection this location. // +optional Redirect redirect.Config `json:"redirect,omitempty"` // Rewrite describes the redirection this location. // +optional Rewrite rewrite.Config `json:"rewrite,omitempty"` - // Denylist indicates only connections from certain client + // Whitelist indicates only connections from certain client // addresses or networks are allowed. // +optional - Denylist ipdenylist.SourceRange `json:"denylist,omitempty"` - // Allowlist indicates only connections from certain client - // addresses or networks are allowed. - // +optional - Allowlist ipallowlist.SourceRange `json:"allowlist,omitempty"` + Whitelist ipwhitelist.SourceRange `json:"whitelist,omitempty"` // Proxy contains information about timeouts and buffer sizes // to be used in connections against endpoints // +optional @@ -333,6 +327,9 @@ type Location struct { // Logs allows to enable or disable the nginx logs // By default access logs are enabled and rewrite logs are disabled Logs log.Config `json:"logs,omitempty"` + // InfluxDB allows to monitor the incoming request by sending them to an influxdb database + // +optional + InfluxDB influxdb.Config `json:"influxDB,omitempty"` // BackendProtocol indicates which protocol should be used to communicate with the service // By default this is HTTP BackendProtocol string `json:"backend-protocol"` @@ -342,11 +339,6 @@ type Location struct { // CustomHTTPErrors specifies the error codes that should be intercepted. // +optional CustomHTTPErrors []int `json:"custom-http-errors"` - // ProxyInterceptErrors disables error interception when using CustomHTTPErrors - // e.g. custom 404 and 503 when service-a does not exist or is not available - // but service-a can return 404 and 503 error codes without intercept - // +optional - DisableProxyInterceptErrors bool `json:"disable-proxy-intercept-errors"` // ModSecurity allows to enable and configure modsecurity // +optional ModSecurity modsecurity.Config `json:"modsecurity"` @@ -355,9 +347,9 @@ type Location struct { // Mirror allows you to mirror traffic to a "test" backend // +optional Mirror mirror.Config `json:"mirror,omitempty"` - // Opentelemetry allows the global opentelemetry setting to be overridden for a location + // Opentracing allows the global opentracing setting to be overridden for a location // +optional - Opentelemetry opentelemetry.Config `json:"opentelemetry"` + Opentracing opentracing.Config `json:"opentracing"` } // SSLPassthroughBackend describes a SSL upstream server configured @@ -408,4 +400,5 @@ type Ingress struct { } // GeneralConfig holds the definition of lua general configuration data -type GeneralConfig struct{} +type GeneralConfig struct { +} diff --git a/pkg/apis/ingress/types_equals.go b/internal/ingress/types_equals.go similarity index 84% rename from pkg/apis/ingress/types_equals.go rename to internal/ingress/types_equals.go index 6fba3bd45..87af2bcfc 100644 --- a/pkg/apis/ingress/types_equals.go +++ b/internal/ingress/types_equals.go @@ -17,7 +17,7 @@ limitations under the License. package ingress import ( - "k8s.io/ingress-nginx/pkg/util/sets" + "k8s.io/ingress-nginx/internal/sets" ) // Equal tests for equality between two Configuration types @@ -76,62 +76,66 @@ func (c1 *Configuration) Equal(c2 *Configuration) bool { } } - return c1.BackendConfigChecksum == c2.BackendConfigChecksum + if c1.BackendConfigChecksum != c2.BackendConfigChecksum { + return false + } + + return true } // Equal tests for equality between two Backend types -func (b *Backend) Equal(newB *Backend) bool { - if b == newB { +func (b1 *Backend) Equal(b2 *Backend) bool { + if b1 == b2 { return true } - if b == nil || newB == nil { + if b1 == nil || b2 == nil { return false } - if b.Name != newB.Name { + if b1.Name != b2.Name { return false } - if b.NoServer != newB.NoServer { + if b1.NoServer != b2.NoServer { return false } - if b.Service != newB.Service { - if b.Service == nil || newB.Service == nil { + if b1.Service != b2.Service { + if b1.Service == nil || b2.Service == nil { return false } - if b.Service.GetNamespace() != newB.Service.GetNamespace() { + if b1.Service.GetNamespace() != b2.Service.GetNamespace() { return false } - if b.Service.GetName() != newB.Service.GetName() { + if b1.Service.GetName() != b2.Service.GetName() { return false } } - if b.Port != newB.Port { + if b1.Port != b2.Port { return false } - if b.SSLPassthrough != newB.SSLPassthrough { + if b1.SSLPassthrough != b2.SSLPassthrough { return false } - if !(&b.SessionAffinity).Equal(&newB.SessionAffinity) { + if !(&b1.SessionAffinity).Equal(&b2.SessionAffinity) { return false } - if b.UpstreamHashBy != newB.UpstreamHashBy { + if b1.UpstreamHashBy != b2.UpstreamHashBy { return false } - if b.LoadBalancing != newB.LoadBalancing { + if b1.LoadBalancing != b2.LoadBalancing { return false } - match := compareEndpoints(b.Endpoints, newB.Endpoints) + match := compareEndpoints(b1.Endpoints, b2.Endpoints) if !match { return false } - if !b.TrafficShapingPolicy.Equal(&newB.TrafficShapingPolicy) { + if !b1.TrafficShapingPolicy.Equal(b2.TrafficShapingPolicy) { return false } - return sets.StringElementsMatch(b.AlternativeBackends, newB.AlternativeBackends) + return sets.StringElementsMatch(b1.AlternativeBackends, b2.AlternativeBackends) } // Equal tests for equality between two SessionAffinityConfig types @@ -169,9 +173,6 @@ func (csa1 *CookieSessionAffinity) Equal(csa2 *CookieSessionAffinity) bool { if csa1.Path != csa2.Path { return false } - if csa1.Domain != csa2.Domain { - return false - } if csa1.Expires != csa2.Expires { return false } @@ -181,9 +182,6 @@ func (csa1 *CookieSessionAffinity) Equal(csa2 *CookieSessionAffinity) bool { if csa1.SameSite != csa2.SameSite { return false } - if csa1.Secure != csa2.Secure { - return false - } if csa1.ConditionalSameSiteNone != csa2.ConditionalSameSiteNone { return false } @@ -191,7 +189,7 @@ func (csa1 *CookieSessionAffinity) Equal(csa2 *CookieSessionAffinity) bool { return true } -// Equal checks the equality between UpstreamByConfig types +//Equal checks the equality between UpstreamByConfig types func (u1 *UpstreamHashByConfig) Equal(u2 *UpstreamHashByConfig) bool { if u1 == u2 { return true @@ -243,7 +241,7 @@ func (e1 *Endpoint) Equal(e2 *Endpoint) bool { } // Equal checks for equality between two TrafficShapingPolicies -func (tsp1 *TrafficShapingPolicy) Equal(tsp2 *TrafficShapingPolicy) bool { +func (tsp1 TrafficShapingPolicy) Equal(tsp2 TrafficShapingPolicy) bool { if tsp1.Weight != tsp2.Weight { return false } @@ -316,9 +314,6 @@ func (s1 *Server) Equal(s2 *Server) bool { if s1.AuthTLSError != s2.AuthTLSError { return false } - if !(&s1.ProxySSL).Equal(&s2.ProxySSL) { - return false - } if len(s1.Locations) != len(s2.Locations) { return false @@ -335,8 +330,6 @@ func (s1 *Server) Equal(s2 *Server) bool { } // Equal tests for equality between two Location types -// -//nolint:gocyclo // Ignore function complexity error func (l1 *Location) Equal(l2 *Location) bool { if l1 == l2 { return true @@ -390,24 +383,21 @@ func (l1 *Location) Equal(l2 *Location) bool { if !(&l1.RateLimit).Equal(&l2.RateLimit) { return false } + if !(&l1.GlobalRateLimit).Equal(&l2.GlobalRateLimit) { + return false + } if !(&l1.Redirect).Equal(&l2.Redirect) { return false } if !(&l1.Rewrite).Equal(&l2.Rewrite) { return false } - if !(&l1.Denylist).Equal(&l2.Denylist) { - return false - } - if !(&l1.Allowlist).Equal(&l2.Allowlist) { + if !(&l1.Whitelist).Equal(&l2.Whitelist) { return false } if !(&l1.Proxy).Equal(&l2.Proxy) { return false } - if !(&l1.ProxySSL).Equal(&l2.ProxySSL) { - return false - } if l1.UsePortInRedirects != l2.UsePortInRedirects { return false } @@ -430,6 +420,10 @@ func (l1 *Location) Equal(l2 *Location) bool { return false } + if !(&l1.InfluxDB).Equal(&l2.InfluxDB) { + return false + } + if l1.BackendProtocol != l2.BackendProtocol { return false } @@ -455,7 +449,7 @@ func (l1 *Location) Equal(l2 *Location) bool { return false } - if !l1.Opentelemetry.Equal(&l2.Opentelemetry) { + if !l1.Opentracing.Equal(&l2.Opentracing) { return false } @@ -463,14 +457,6 @@ func (l1 *Location) Equal(l2 *Location) bool { return false } - if l1.DisableProxyInterceptErrors != l2.DisableProxyInterceptErrors { - return false - } - - if !l1.CustomHeaders.Equal(&l2.CustomHeaders) { - return false - } - return true } @@ -553,39 +539,33 @@ func (l4b1 *L4Backend) Equal(l4b2 *L4Backend) bool { } // Equal tests for equality between two SSLCert types -func (s *SSLCert) Equal(newS *SSLCert) bool { - if s == newS { +func (s1 *SSLCert) Equal(s2 *SSLCert) bool { + if s1 == s2 { return true } - if s == nil || newS == nil { + if s1 == nil || s2 == nil { return false } - if s.CASHA != newS.CASHA { + if s1.CASHA != s2.CASHA { return false } - if s.CRLSHA != newS.CRLSHA { + if s1.CRLSHA != s2.CRLSHA { return false } - if s.PemSHA != newS.PemSHA { + if s1.PemSHA != s2.PemSHA { return false } - if s.CAFileName != newS.CAFileName { + if !s1.ExpireTime.Equal(s2.ExpireTime) { return false } - if s.CRLFileName != newS.CRLFileName { + if s1.PemCertKey != s2.PemCertKey { return false } - if !s.ExpireTime.Equal(newS.ExpireTime) { - return false - } - if s.PemCertKey != newS.PemCertKey { - return false - } - if s.UID != newS.UID { + if s1.UID != s2.UID { return false } - return sets.StringElementsMatch(s.CN, newS.CN) + return sets.StringElementsMatch(s1.CN, s2.CN) } var compareEndpointsFunc = func(e1, e2 interface{}) bool { diff --git a/pkg/apis/ingress/types_equals_test.go b/internal/ingress/types_equals_test.go similarity index 92% rename from pkg/apis/ingress/types_equals_test.go rename to internal/ingress/types_equals_test.go index 53643f912..8119d26c7 100644 --- a/pkg/apis/ingress/types_equals_test.go +++ b/internal/ingress/types_equals_test.go @@ -25,29 +25,19 @@ import ( ) func TestEqualConfiguration(t *testing.T) { - ap, err := filepath.Abs("../../../test/manifests/configuration-a.json") - if err != nil { - t.Errorf("unexpected error: %v", err) - } + ap, _ := filepath.Abs("../../test/manifests/configuration-a.json") a, err := readJSON(ap) if err != nil { t.Errorf("unexpected error reading JSON file: %v", err) } - bp, err := filepath.Abs("../../../test/manifests/configuration-b.json") - if err != nil { - t.Errorf("unexpected error: %v", err) - } - + bp, _ := filepath.Abs("../../test/manifests/configuration-b.json") b, err := readJSON(bp) if err != nil { t.Errorf("unexpected error reading JSON file: %v", err) } - cp, err := filepath.Abs("../../../test/manifests/configuration-c.json") - if err != nil { - t.Errorf("unexpected error: %v", err) - } + cp, _ := filepath.Abs("../../test/manifests/configuration-c.json") c, err := readJSON(cp) if err != nil { t.Errorf("unexpected error reading JSON file: %v", err) @@ -94,18 +84,15 @@ func TestL4ServiceElementsMatch(t *testing.T) { {[]L4Service{{Port: 80}}, []L4Service{{Port: 80}}, true}, { []L4Service{ - {Port: 80, Endpoints: []Endpoint{{Address: "1.1.1.1"}}}, - }, + {Port: 80, Endpoints: []Endpoint{{Address: "1.1.1.1"}}}}, []L4Service{{Port: 80}}, false, }, { []L4Service{ - {Port: 80, Endpoints: []Endpoint{{Address: "1.1.1.1"}, {Address: "1.1.1.2"}}}, - }, + {Port: 80, Endpoints: []Endpoint{{Address: "1.1.1.1"}, {Address: "1.1.1.2"}}}}, []L4Service{ - {Port: 80, Endpoints: []Endpoint{{Address: "1.1.1.2"}, {Address: "1.1.1.1"}}}, - }, + {Port: 80, Endpoints: []Endpoint{{Address: "1.1.1.2"}, {Address: "1.1.1.1"}}}}, true, }, { diff --git a/pkg/apis/ingress/zz_generated.deepcopy.go b/internal/ingress/zz_generated.deepcopy.go similarity index 99% rename from pkg/apis/ingress/zz_generated.deepcopy.go rename to internal/ingress/zz_generated.deepcopy.go index 410173e26..5d49fb05e 100644 --- a/pkg/apis/ingress/zz_generated.deepcopy.go +++ b/internal/ingress/zz_generated.deepcopy.go @@ -1,4 +1,3 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/internal/k8s/main.go b/internal/k8s/main.go index 5e93e560d..364d7c8ec 100644 --- a/internal/k8s/main.go +++ b/internal/k8s/main.go @@ -25,7 +25,7 @@ import ( "k8s.io/klog/v2" apiv1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" + networkingv1beta1 "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/version" clientset "k8s.io/client-go/kubernetes" @@ -33,7 +33,7 @@ import ( ) // ParseNameNS parses a string searching a namespace and name -func ParseNameNS(input string) (ns, name string, err error) { +func ParseNameNS(input string) (string, string, error) { nsName := strings.Split(input, "/") if len(nsName) != 2 { return "", "", fmt.Errorf("invalid format (namespace/name) found in '%v'", input) @@ -78,8 +78,6 @@ func GetNodeIPOrName(kubeClient clientset.Interface, name string, useInternalIP var ( // IngressPodDetails hold information about the ingress-nginx pod IngressPodDetails *PodInfo - // IngressNodeDetails hold information about the node running ingress-nginx pod - IngressNodeDetails *NodeInfo ) // PodInfo contains runtime information about the pod running the Ingres controller @@ -89,12 +87,6 @@ type PodInfo struct { metav1.ObjectMeta } -// NodeInfo contains runtime information about the node pod running the Ingres controller, eg. zone where pod is running -type NodeInfo struct { - metav1.TypeMeta - metav1.ObjectMeta -} - // GetIngressPod load the ingress-nginx pod func GetIngressPod(kubeClient clientset.Interface) error { podName := os.Getenv("POD_NAME") @@ -116,18 +108,6 @@ func GetIngressPod(kubeClient clientset.Interface) error { pod.ObjectMeta.DeepCopyInto(&IngressPodDetails.ObjectMeta) IngressPodDetails.SetLabels(pod.GetLabels()) - IngressNodeDetails = &NodeInfo{ - TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Node"}, - } - // Try to get node info/labels to determine topology zone where pod is running - node, err := kubeClient.CoreV1().Nodes().Get(context.TODO(), pod.Spec.NodeName, metav1.GetOptions{}) - if err != nil { - klog.Warningf("Unable to get NODE information: %v", err) - } else { - node.ObjectMeta.DeepCopyInto(&IngressNodeDetails.ObjectMeta) - IngressNodeDetails.SetLabels(node.GetLabels()) - } - return nil } @@ -141,37 +121,46 @@ func MetaNamespaceKey(obj interface{}) string { return key } +// IsIngressV1Beta1Ready indicates if the running Kubernetes version is at least v1.18.0 +var IsIngressV1Beta1Ready bool + +// IsIngressV1Ready indicates if the running Kubernetes version is at least v1.19.0 +var IsIngressV1Ready bool + +// IngressClass indicates the class of the Ingress to use as filter +var IngressClass *networkingv1beta1.IngressClass + // IngressNGINXController defines the valid value of IngressClass // Controller field for ingress-nginx const IngressNGINXController = "k8s.io/ingress-nginx" -// NetworkingIngressAvailable checks if the package "k8s.io/api/networking/v1" -// is available or not and if Ingress V1 is supported (k8s >= v1.19.0) -func NetworkingIngressAvailable(client clientset.Interface) bool { - version119, err := version.ParseGeneric("v1.19.0") - if err != nil { - return false - } +// NetworkingIngressAvailable checks if the package "k8s.io/api/networking/v1beta1" +// is available or not and if Ingress V1 is supported (k8s >= v1.18.0) +func NetworkingIngressAvailable(client clientset.Interface) (bool, bool, bool) { + // check kubernetes version to use new ingress package or not + version114, _ := version.ParseGeneric("v1.14.0") + version118, _ := version.ParseGeneric("v1.18.0") + version119, _ := version.ParseGeneric("v1.19.0") serverVersion, err := client.Discovery().ServerVersion() if err != nil { - return false + return false, false, false } runningVersion, err := version.ParseGeneric(serverVersion.String()) if err != nil { klog.ErrorS(err, "unexpected error parsing running Kubernetes version") - return false + return false, false, false } - return runningVersion.AtLeast(version119) + return runningVersion.AtLeast(version114), runningVersion.AtLeast(version118), runningVersion.AtLeast(version119) } // default path type is Prefix to not break existing definitions -var defaultPathType = networkingv1.PathTypePrefix +var defaultPathType = networkingv1beta1.PathTypePrefix // SetDefaultNGINXPathType sets a default PathType when is not defined. -func SetDefaultNGINXPathType(ing *networkingv1.Ingress) { +func SetDefaultNGINXPathType(ing *networkingv1beta1.Ingress) { for _, rule := range ing.Spec.Rules { if rule.IngressRuleValue.HTTP == nil { continue @@ -183,7 +172,7 @@ func SetDefaultNGINXPathType(ing *networkingv1.Ingress) { p.PathType = &defaultPathType } - if *p.PathType == networkingv1.PathTypeImplementationSpecific { + if *p.PathType == networkingv1beta1.PathTypeImplementationSpecific { p.PathType = &defaultPathType } } diff --git a/internal/k8s/main_test.go b/internal/k8s/main_test.go index 1721c1fb2..f3f8b652e 100644 --- a/internal/k8s/main_test.go +++ b/internal/k8s/main_test.go @@ -17,6 +17,7 @@ limitations under the License. package k8s import ( + "os" "testing" apiv1 "k8s.io/api/core/v1" @@ -202,8 +203,7 @@ func TestGetNodeIP(t *testing.T) { }, }, }}}), - "demo", "10.0.0.2", true, - }, + "demo", "10.0.0.2", true}, } for _, fk := range fKNodes { @@ -216,32 +216,32 @@ func TestGetNodeIP(t *testing.T) { func TestGetIngressPod(t *testing.T) { // POD_NAME & POD_NAMESPACE not exist - t.Setenv("POD_NAME", "") - t.Setenv("POD_NAMESPACE", "") + os.Setenv("POD_NAME", "") + os.Setenv("POD_NAMESPACE", "") err := GetIngressPod(testclient.NewSimpleClientset()) if err == nil { t.Errorf("expected an error but returned nil") } // POD_NAME not exist - t.Setenv("POD_NAME", "") - t.Setenv("POD_NAMESPACE", apiv1.NamespaceDefault) + os.Setenv("POD_NAME", "") + os.Setenv("POD_NAMESPACE", apiv1.NamespaceDefault) err = GetIngressPod(testclient.NewSimpleClientset()) if err == nil { t.Errorf("expected an error but returned nil") } // POD_NAMESPACE not exist - t.Setenv("POD_NAME", "testpod") - t.Setenv("POD_NAMESPACE", "") + os.Setenv("POD_NAME", "testpod") + os.Setenv("POD_NAMESPACE", "") err = GetIngressPod(testclient.NewSimpleClientset()) if err == nil { t.Errorf("expected an error but returned nil") } // POD not exist - t.Setenv("POD_NAME", "testpod") - t.Setenv("POD_NAMESPACE", apiv1.NamespaceDefault) + os.Setenv("POD_NAME", "testpod") + os.Setenv("POD_NAMESPACE", apiv1.NamespaceDefault) err = GetIngressPod(testclient.NewSimpleClientset()) if err == nil { t.Errorf("expected an error but returned nil") diff --git a/internal/k8s/zz_generated.deepcopy.go b/internal/k8s/zz_generated.deepcopy.go index 29f1163bc..0261457fe 100644 --- a/internal/k8s/zz_generated.deepcopy.go +++ b/internal/k8s/zz_generated.deepcopy.go @@ -1,4 +1,3 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/internal/net/dns/dns.go b/internal/net/dns/dns.go index 6b250e8cb..b300e1aae 100644 --- a/internal/net/dns/dns.go +++ b/internal/net/dns/dns.go @@ -17,8 +17,8 @@ limitations under the License. package dns import ( + "io/ioutil" "net" - "os" "strings" "k8s.io/klog/v2" @@ -29,7 +29,7 @@ var defResolvConf = "/etc/resolv.conf" // GetSystemNameServers returns the list of nameservers located in the file /etc/resolv.conf func GetSystemNameServers() ([]net.IP, error) { var nameservers []net.IP - file, err := os.ReadFile(defResolvConf) + file, err := ioutil.ReadFile(defResolvConf) if err != nil { return nameservers, err } @@ -38,7 +38,7 @@ func GetSystemNameServers() ([]net.IP, error) { lines := strings.Split(string(file), "\n") for l := range lines { trimmed := strings.TrimSpace(lines[l]) - if trimmed == "" || trimmed[0] == '#' || trimmed[0] == ';' { + if len(trimmed) == 0 || trimmed[0] == '#' || trimmed[0] == ';' { continue } fields := strings.Fields(trimmed) diff --git a/internal/net/dns/dns_test.go b/internal/net/dns/dns_test.go index 7e030e17a..bd2243ae7 100644 --- a/internal/net/dns/dns_test.go +++ b/internal/net/dns/dns_test.go @@ -17,11 +17,12 @@ limitations under the License. package dns import ( + "io/ioutil" "net" "os" "testing" - "k8s.io/ingress-nginx/pkg/util/file" + "k8s.io/ingress-nginx/internal/file" ) func TestGetDNSServers(t *testing.T) { @@ -33,23 +34,20 @@ func TestGetDNSServers(t *testing.T) { t.Error("expected at least 1 nameserver in /etc/resolv.conf") } - f, err := os.CreateTemp("", "fw") + f, err := ioutil.TempFile("", "fw") if err != nil { t.Fatalf("unexpected error: %v", err) } defer f.Close() defer os.Remove(f.Name()) - err = os.WriteFile(f.Name(), []byte(` + ioutil.WriteFile(f.Name(), []byte(` # comment ; comment nameserver 2001:4860:4860::8844 nameserver 2001:4860:4860::8888 nameserver 8.8.8.8 `), file.ReadWriteByUser) - if err != nil { - t.Errorf("unexpected error: %v", err) - } defResolvConf = f.Name() s, err = GetSystemNameServers() diff --git a/internal/net/ipnet_test.go b/internal/net/ipnet_test.go index 8e460aae9..95e6b9c32 100644 --- a/internal/net/ipnet_test.go +++ b/internal/net/ipnet_test.go @@ -36,17 +36,13 @@ func TestNewIPSet(t *testing.T) { } func TestParseCIDRs(t *testing.T) { - cidr, err := ParseCIDRs("invalid.com") - if err == nil { - t.Errorf("expected error but got nil") - } - + cidr, _ := ParseCIDRs("invalid.com") if cidr != nil { t.Errorf("expected %v but got %v", nil, cidr) } expected := []string{"192.0.0.1", "192.0.1.0/24"} - cidr, err = ParseCIDRs("192.0.0.1, 192.0.1.0/24") + cidr, err := ParseCIDRs("192.0.0.1, 192.0.1.0/24") if err != nil { t.Errorf("unexpected error %v", err) } diff --git a/internal/net/net.go b/internal/net/net.go index b0952d9cc..b7fefb8b3 100644 --- a/internal/net/net.go +++ b/internal/net/net.go @@ -29,13 +29,12 @@ func IsIPV6(ip _net.IP) bool { // IsPortAvailable checks if a TCP port is available or not func IsPortAvailable(p int) bool { - ln, err := _net.Listen("tcp", fmt.Sprintf(":%v", p)) - defer func() { - if ln != nil { - ln.Close() - } - }() - return err == nil + conn, err := _net.Dial("tcp", fmt.Sprintf(":%v", p)) + if err != nil { + return true + } + defer conn.Close() + return false } // IsIPv6Enabled checks if IPV6 is enabled or not and we have @@ -52,10 +51,7 @@ func IsIPv6Enabled() bool { } for _, addr := range addrs { - ip, _, err := _net.ParseCIDR(addr.String()) - if err != nil { - return false - } + ip, _, _ := _net.ParseCIDR(addr.String()) if IsIPV6(ip) { return true } diff --git a/internal/net/net_test.go b/internal/net/net_test.go index f7a4f52e1..f2f37a838 100644 --- a/internal/net/net_test.go +++ b/internal/net/net_test.go @@ -47,7 +47,7 @@ func TestIsPortAvailable(t *testing.T) { t.Fatal("expected port 0 to be available (random port) but returned false") } - ln, err := net.Listen("tcp", ":0") //nolint:gosec // Ignore the gosec error in testing + ln, err := net.Listen("tcp", ":0") if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/internal/net/ssl/ssl.go b/internal/net/ssl/ssl.go index 0592303bc..a329b8303 100644 --- a/internal/net/ssl/ssl.go +++ b/internal/net/ssl/ssl.go @@ -29,6 +29,7 @@ import ( "encoding/pem" "errors" "fmt" + "io/ioutil" "math/big" "net" "os" @@ -39,28 +40,28 @@ import ( "github.com/zakjan/cert-chain-resolver/certUtil" "k8s.io/apimachinery/pkg/util/sets" - - "k8s.io/ingress-nginx/pkg/apis/ingress" - + "k8s.io/ingress-nginx/internal/file" + "k8s.io/ingress-nginx/internal/ingress" ngx_config "k8s.io/ingress-nginx/internal/ingress/controller/config" - "k8s.io/ingress-nginx/pkg/util/file" - - klog "k8s.io/klog/v2" + "k8s.io/ingress-nginx/internal/watch" + "k8s.io/klog/v2" ) // FakeSSLCertificateUID defines the default UID to use for the fake SSL // certificate generated by the ingress controller var FakeSSLCertificateUID = "00000000-0000-0000-0000-000000000000" -var oidExtensionSubjectAltName = asn1.ObjectIdentifier{2, 5, 29, 17} +var ( + oidExtensionSubjectAltName = asn1.ObjectIdentifier{2, 5, 29, 17} +) const ( - fakeCertificateName = "default-fake-certificate" //#nosec G101 + fakeCertificateName = "default-fake-certificate" ) // getPemFileName returns absolute file path and file name of pem cert related to given fullSecretName -func getPemFileName(fullSecretName string) (filePath, pemName string) { - pemName = fmt.Sprintf("%v.pem", fullSecretName) +func getPemFileName(fullSecretName string) (string, string) { + pemName := fmt.Sprintf("%v.pem", fullSecretName) return fmt.Sprintf("%v/%v", file.DefaultSSLDirectory, pemName), pemName } @@ -79,7 +80,7 @@ func CreateSSLCert(cert, key []byte, uid string) (*ingress.SSLCert, error) { } } - pemCertBuffer.WriteString("\n") + pemCertBuffer.Write([]byte("\n")) pemCertBuffer.Write(key) pemBlock, _ := pem.Decode(pemCertBuffer.Bytes()) @@ -180,7 +181,7 @@ func CheckCACert(caBytes []byte) ([]*x509.Certificate, error) { func StoreSSLCertOnDisk(name string, sslCert *ingress.SSLCert) (string, error) { pemFileName, _ := getPemFileName(name) - err := os.WriteFile(pemFileName, []byte(sslCert.PemCertKey), file.ReadWriteByUser) + err := ioutil.WriteFile(pemFileName, []byte(sslCert.PemCertKey), file.ReadWriteByUser) if err != nil { return "", fmt.Errorf("could not create PEM certificate file %v: %v", pemFileName, err) } @@ -190,15 +191,15 @@ func StoreSSLCertOnDisk(name string, sslCert *ingress.SSLCert) (string, error) { // ConfigureCACertWithCertAndKey appends ca into existing PEM file consisting of cert and key // and sets relevant fields in sslCert object -func ConfigureCACertWithCertAndKey(_ string, ca []byte, sslCert *ingress.SSLCert) error { +func ConfigureCACertWithCertAndKey(name string, ca []byte, sslCert *ingress.SSLCert) error { var buffer bytes.Buffer - _, err := buffer.WriteString(sslCert.PemCertKey) + _, err := buffer.Write([]byte(sslCert.PemCertKey)) if err != nil { return fmt.Errorf("could not append newline to cert file %v: %v", sslCert.CAFileName, err) } - _, err = buffer.WriteString("\n") + _, err = buffer.Write([]byte("\n")) if err != nil { return fmt.Errorf("could not append newline to cert file %v: %v", sslCert.CAFileName, err) } @@ -208,12 +209,12 @@ func ConfigureCACertWithCertAndKey(_ string, ca []byte, sslCert *ingress.SSLCert return fmt.Errorf("could not write ca data to cert file %v: %v", sslCert.CAFileName, err) } - //nolint:gosec // Not change permission to avoid possible issues - return os.WriteFile(sslCert.CAFileName, buffer.Bytes(), 0o644) + return ioutil.WriteFile(sslCert.CAFileName, buffer.Bytes(), 0644) } // ConfigureCRL creates a CRL file and append it into the SSLCert func ConfigureCRL(name string, crl []byte, sslCert *ingress.SSLCert) error { + crlName := fmt.Sprintf("crl-%v.pem", name) crlFileName := fmt.Sprintf("%v/%v", file.DefaultSSLDirectory, crlName) @@ -226,13 +227,12 @@ func ConfigureCRL(name string, crl []byte, sslCert *ingress.SSLCert) error { return fmt.Errorf("CRL file %v contains invalid data, and must be created only with PEM formatted certificates", name) } - _, err := x509.ParseRevocationList(pemCRLBlock.Bytes) + _, err := x509.ParseCRL(pemCRLBlock.Bytes) if err != nil { - return err + return fmt.Errorf(err.Error()) } - //nolint:gosec // Not change permission to avoid possible issues - err = os.WriteFile(crlFileName, crl, 0o644) + err = ioutil.WriteFile(crlFileName, crl, 0644) if err != nil { return fmt.Errorf("could not write CRL file %v: %v", crlFileName, err) } @@ -241,6 +241,7 @@ func ConfigureCRL(name string, crl []byte, sslCert *ingress.SSLCert) error { sslCert.CRLSHA = file.SHA1(crlFileName) return nil + } // ConfigureCACert is similar to ConfigureCACertWithCertAndKey but it creates a separate file @@ -249,8 +250,7 @@ func ConfigureCACert(name string, ca []byte, sslCert *ingress.SSLCert) error { caName := fmt.Sprintf("ca-%v.pem", name) fileName := fmt.Sprintf("%v/%v", file.DefaultSSLDirectory, caName) - //nolint:gosec // Not change permission to avoid possible issues - err := os.WriteFile(fileName, ca, 0o644) + err := ioutil.WriteFile(fileName, ca, 0644) if err != nil { return fmt.Errorf("could not write CA file %v: %v", fileName, err) } @@ -292,14 +292,14 @@ func parseSANExtension(value []byte) (dnsNames, emailAddresses []string, ipAddre var seq asn1.RawValue var rest []byte if rest, err = asn1.Unmarshal(value, &seq); err != nil { - return dnsNames, emailAddresses, ipAddresses, err + return } else if len(rest) != 0 { err = errors.New("x509: trailing data after X.509 extension") - return dnsNames, emailAddresses, ipAddresses, err + return } if !seq.IsCompound || seq.Tag != 16 || seq.Class != 0 { err = asn1.StructuralError{Msg: "bad SAN sequence"} - return dnsNames, emailAddresses, ipAddresses, err + return } rest = seq.Bytes @@ -307,7 +307,7 @@ func parseSANExtension(value []byte) (dnsNames, emailAddresses []string, ipAddre var v asn1.RawValue rest, err = asn1.Unmarshal(rest, &v) if err != nil { - return dnsNames, emailAddresses, ipAddresses, err + return } switch v.Tag { case 1: @@ -320,19 +320,19 @@ func parseSANExtension(value []byte) (dnsNames, emailAddresses []string, ipAddre ipAddresses = append(ipAddresses, v.Bytes) default: err = errors.New("x509: certificate contained IP address of length " + strconv.Itoa(len(v.Bytes))) - return dnsNames, emailAddresses, ipAddresses, err + return } } } - return dnsNames, emailAddresses, ipAddresses, err + return } // AddOrUpdateDHParam creates a dh parameters file with the specified name func AddOrUpdateDHParam(name string, dh []byte) (string, error) { pemFileName, pemName := getPemFileName(name) - tempPemFile, err := os.CreateTemp(file.DefaultSSLDirectory, pemName) + tempPemFile, err := ioutil.TempFile(file.DefaultSSLDirectory, pemName) klog.V(3).InfoS("Creating temporal file for DH", "path", tempPemFile.Name(), "name", pemName) if err != nil { @@ -351,7 +351,7 @@ func AddOrUpdateDHParam(name string, dh []byte) (string, error) { defer os.Remove(tempPemFile.Name()) - pemCerts, err := os.ReadFile(tempPemFile.Name()) + pemCerts, err := ioutil.ReadFile(tempPemFile.Name()) if err != nil { return "", err } @@ -395,11 +395,12 @@ func GetFakeSSLCert() *ingress.SSLCert { return sslCert } -func getFakeHostSSLCert(host string) (cert, key []byte) { +func getFakeHostSSLCert(host string) ([]byte, []byte) { var priv interface{} var err error priv, err = rsa.GenerateKey(rand.Reader, 2048) + if err != nil { klog.Fatalf("failed to generate fake private key: %v", err) } @@ -410,6 +411,7 @@ func getFakeHostSSLCert(host string) (cert, key []byte) { serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { klog.Fatalf("failed to generate fake serial number: %v", err) } @@ -433,16 +435,16 @@ func getFakeHostSSLCert(host string) (cert, key []byte) { klog.Fatalf("Failed to create fake certificate: %v", err) } - cert = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + cert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) - key = pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv.(*rsa.PrivateKey))}) + key := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv.(*rsa.PrivateKey))}) return cert, key } // fullChainCert checks if a certificate file contains issues in the intermediate CA chain // Returns a new certificate with the intermediate certificates. -// If the certificate does not contain issues with the chain it returns an empty byte array +// If the certificate does not contains issues with the chain it return an empty byte array func fullChainCert(in []byte) ([]byte, error) { cert, err := certUtil.DecodeCertificate(in) if err != nil { @@ -505,14 +507,9 @@ func NewTLSListener(certificate, key string) *TLSListener { l.load() - _, err := file.NewFileWatcher(certificate, l.load) - if err != nil { - klog.Errorf("unexpected error: %v", err) - } - _, err = file.NewFileWatcher(key, l.load) - if err != nil { - klog.Errorf("unexpected error: %v", err) - } + _, _ = watch.NewFileWatcher(certificate, l.load) + _, _ = watch.NewFileWatcher(key, l.load) + return &l } @@ -523,7 +520,7 @@ func (tl *TLSListener) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, e return tl.certificate, tl.err } -// TLSConfig instantiates a TLS configuration, always providing an up to date certificate +// TLSConfig instanciates a TLS configuration, always providing an up to date certificate func (tl *TLSListener) TLSConfig() *tls.Config { return &tls.Config{ GetCertificate: tl.GetCertificate, @@ -533,12 +530,12 @@ func (tl *TLSListener) TLSConfig() *tls.Config { func (tl *TLSListener) load() { klog.InfoS("loading tls certificate", "path", tl.certificatePath, "key", tl.keyPath) - certBytes, err := os.ReadFile(tl.certificatePath) + certBytes, err := ioutil.ReadFile(tl.certificatePath) if err != nil { tl.certificate = nil tl.err = err } - keyBytes, err := os.ReadFile(tl.keyPath) + keyBytes, err := ioutil.ReadFile(tl.keyPath) if err != nil { tl.certificate = nil tl.err = err diff --git a/internal/net/ssl/ssl_test.go b/internal/net/ssl/ssl_test.go index 47fa07c7c..d161b57ff 100644 --- a/internal/net/ssl/ssl_test.go +++ b/internal/net/ssl/ssl_test.go @@ -20,6 +20,7 @@ import ( "bytes" "crypto" "crypto/rand" + cryptorand "crypto/rand" "crypto/rsa" "crypto/tls" "crypto/x509" @@ -27,22 +28,22 @@ import ( "encoding/pem" "errors" "fmt" + "io/ioutil" "math" "math/big" "net/http" "net/http/httptest" - "os" "strings" "sync" "testing" "time" certutil "k8s.io/client-go/util/cert" - "k8s.io/ingress-nginx/pkg/util/file" + "k8s.io/ingress-nginx/internal/file" ) // generateRSACerts generates a self signed certificate using a self generated ca -func generateRSACerts(host string) (newCert, newCa *keyPair, err error) { +func generateRSACerts(host string) (*keyPair, *keyPair, error) { ca, err := newCA("self-sign-ca") if err != nil { return nil, nil, err @@ -57,7 +58,7 @@ func generateRSACerts(host string) (newCert, newCa *keyPair, err error) { CommonName: host, Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, } - cert, err := newSignedCert(&config, key, ca.Cert, ca.Key) + cert, err := newSignedCert(config, key, ca.Cert, ca.Key) if err != nil { return nil, nil, fmt.Errorf("unable to sign the server certificate: %v", err) } @@ -139,11 +140,11 @@ func TestCACert(t *testing.T) { func TestGetFakeSSLCert(t *testing.T) { sslCert := GetFakeSSLCert() - if sslCert.PemCertKey == "" { + if len(sslCert.PemCertKey) == 0 { t.Fatalf("expected PemCertKey to not be empty") } - if sslCert.PemFileName == "" { + if len(sslCert.PemFileName) == 0 { t.Fatalf("expected PemFileName to not be empty") } @@ -195,7 +196,7 @@ func TestConfigureCRL(t *testing.T) { // Demo CRL from https://csrc.nist.gov/projects/pki-testing/sample-certificates-and-crls // Converted to PEM to be tested // SHA: ef21f9c97ec2ef84ba3b2ab007c858a6f760d813 - crl := []byte(`-----BEGIN X509 CRL----- + var crl = []byte(`-----BEGIN X509 CRL----- MIIBYDCBygIBATANBgkqhkiG9w0BAQUFADBDMRMwEQYKCZImiZPyLGQBGRYDY29t MRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTETMBEGA1UEAxMKRXhhbXBsZSBDQRcN MDUwMjA1MTIwMDAwWhcNMDUwMjA2MTIwMDAwWjAiMCACARIXDTA0MTExOTE1NTcw @@ -237,7 +238,6 @@ fUNCdMGmr8FVF6IzTNYGmCuk/C4= t.Fatalf("the expected CRL SHA wasn't found") } } - func TestCreateSSLCert(t *testing.T) { cert, _, err := generateRSACerts("echoheaders") if err != nil { @@ -336,16 +336,16 @@ const ( // newPrivateKey creates an RSA private key func newPrivateKey() (*rsa.PrivateKey, error) { - return rsa.GenerateKey(rand.Reader, rsaKeySize) + return rsa.GenerateKey(cryptorand.Reader, rsaKeySize) } // newSignedCert creates a signed certificate using the given CA certificate and key -func newSignedCert(cfg *certutil.Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) { +func newSignedCert(cfg certutil.Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) { serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64)) if err != nil { return nil, err } - if cfg.CommonName == "" { + if len(cfg.CommonName) == 0 { return nil, errors.New("must specify a CommonName") } if len(cfg.Usages) == 0 { @@ -365,7 +365,7 @@ func newSignedCert(cfg *certutil.Config, key crypto.Signer, caCert *x509.Certifi KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, ExtKeyUsage: cfg.Usages, } - certDERBytes, err := x509.CreateCertificate(rand.Reader, &certTmpl, caCert, key.Public(), caKey) + certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey) if err != nil { return nil, err } @@ -390,27 +390,23 @@ func encodeCertPEM(cert *x509.Certificate) []byte { return pem.EncodeToMemory(&block) } -func newFakeCertificate(t *testing.T) (sslCert []byte, certFileName, keyFileName string) { +func newFakeCertificate(t *testing.T) ([]byte, string, string) { cert, key := getFakeHostSSLCert("localhost") - certFile, err := os.CreateTemp("", "crt-") + certFile, err := ioutil.TempFile("", "crt-") if err != nil { t.Errorf("failed to write test key: %v", err) } - if _, err := certFile.Write(cert); err != nil { - t.Errorf("failed to write cert: %v", err) - } + certFile.Write(cert) defer certFile.Close() - keyFile, err := os.CreateTemp("", "key-") + keyFile, err := ioutil.TempFile("", "key-") if err != nil { t.Errorf("failed to write test key: %v", err) } - if _, err := keyFile.Write(key); err != nil { - t.Errorf("failed to write key: %v", err) - } + keyFile.Write(key) defer keyFile.Close() return cert, certFile.Name(), keyFile.Name() @@ -424,9 +420,10 @@ func dialTestServer(port string, rootCertificates ...[]byte) error { return fmt.Errorf("failed to add root certificate") } } - resp, err := tls.Dial("tcp", "localhost:"+port, &tls.Config{ //nolint:gosec // Ignore the gosec error in testing + resp, err := tls.Dial("tcp", "localhost:"+port, &tls.Config{ RootCAs: roots, }) + if err != nil { return err } @@ -446,7 +443,7 @@ func TestTLSKeyReloader(t *testing.T) { } watcher.load() - s := httptest.NewUnstartedServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {})) + s := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) s.Config.TLSConfig = watcher.TLSConfig() s.Listener = tls.NewListener(s.Listener, s.Config.TLSConfig) go s.Start() @@ -473,14 +470,15 @@ func TestTLSKeyReloader(t *testing.T) { } }) - /*TODO: fix - // simulate watch.NewFileWatcher to call the load function - watcher.load() - t.Run("when the certificate is reloaded", func(t *testing.T) { - if err := dialTestServer(port, cert); err != nil { - t.Errorf("TLS dial should succeed, got error: %v", err) - } - }) + //TODO: fix + /* + // simulate watch.NewFileWatcher to call the load function + watcher.load() + t.Run("when the certificate is reloaded", func(t *testing.T) { + if err := dialTestServer(port, cert); err != nil { + t.Errorf("TLS dial should succeed, got error: %v", err) + } + }) */ }) } diff --git a/internal/nginx/main.go b/internal/nginx/main.go index fc586e9e8..bc88391f6 100644 --- a/internal/nginx/main.go +++ b/internal/nginx/main.go @@ -20,7 +20,7 @@ import ( "bytes" "encoding/json" "fmt" - "io" + "io/ioutil" "net/http" "os" "os/exec" @@ -28,7 +28,7 @@ import ( "time" ps "github.com/mitchellh/go-ps" - klog "k8s.io/klog/v2" + "k8s.io/klog/v2" ) // TODO: Check https://github.com/kubernetes/kubernetes/blob/master/pkg/master/ports/ports.go for ports already being used @@ -36,14 +36,11 @@ import ( // ProfilerPort port used by the ingress controller to expose the Go Profiler when it is enabled. var ProfilerPort = 10245 -// ProfilerAddress IP address used by the ingress controller to expose the Go Profiler when it is enabled. -var ProfilerAddress = "127.0.0.1" - // TemplatePath path of the NGINX template var TemplatePath = "/etc/nginx/template/nginx.tmpl" // PID defines the location of the pid file used by NGINX -var PID = "/tmp/nginx/nginx.pid" +var PID = "/tmp/nginx.pid" // StatusPort port used by NGINX for the status server var StatusPort = 10246 @@ -62,7 +59,7 @@ var StatusPath = "/nginx_status" var StreamPort = 10247 // NewGetStatusRequest creates a new GET request to the internal NGINX status server -func NewGetStatusRequest(path string) (statusCode int, data []byte, err error) { +func NewGetStatusRequest(path string) (int, []byte, error) { url := fmt.Sprintf("http://127.0.0.1:%v%v", StatusPort, path) client := http.Client{} @@ -72,7 +69,7 @@ func NewGetStatusRequest(path string) (statusCode int, data []byte, err error) { } defer res.Body.Close() - data, err = io.ReadAll(res.Body) + data, err := ioutil.ReadAll(res.Body) if err != nil { return 0, nil, err } @@ -81,7 +78,7 @@ func NewGetStatusRequest(path string) (statusCode int, data []byte, err error) { } // NewPostStatusRequest creates a new POST request to the internal NGINX status server -func NewPostStatusRequest(path, contentType string, data interface{}) (statusCode int, body []byte, err error) { +func NewPostStatusRequest(path, contentType string, data interface{}) (int, []byte, error) { url := fmt.Sprintf("http://127.0.0.1:%v%v", StatusPort, path) buf, err := json.Marshal(data) @@ -96,7 +93,7 @@ func NewPostStatusRequest(path, contentType string, data interface{}) (statusCod } defer res.Body.Close() - body, err = io.ReadAll(res.Body) + body, err := ioutil.ReadAll(res.Body) if err != nil { return 0, nil, err } @@ -105,7 +102,7 @@ func NewPostStatusRequest(path, contentType string, data interface{}) (statusCod } // GetServerBlock takes an nginx.conf file and a host and tries to find the server block for that host -func GetServerBlock(conf, host string) (string, error) { +func GetServerBlock(conf string, host string) (string, error) { startMsg := fmt.Sprintf("## start server %v\n", host) endMsg := fmt.Sprintf("## end server %v", host) @@ -113,7 +110,7 @@ func GetServerBlock(conf, host string) (string, error) { if blockStart < 0 { return "", fmt.Errorf("host %v was not found in the controller's nginx.conf", host) } - blockStart += len(startMsg) + blockStart = blockStart + len(startMsg) blockEnd := strings.Index(conf, endMsg) if blockEnd < 0 { @@ -136,7 +133,7 @@ func readFileToString(path string) (string, error) { } defer f.Close() - contents, err := io.ReadAll(f) + contents, err := ioutil.ReadAll(f) if err != nil { return "", err } @@ -163,10 +160,7 @@ func Version() string { // IsRunning returns true if a process with the name 'nginx' is found func IsRunning() bool { - processes, err := ps.Processes() - if err != nil { - klog.ErrorS(err, "unexpected error obtaining process list") - } + processes, _ := ps.Processes() for _, p := range processes { if p.Executable() == "nginx" { return true diff --git a/internal/nginx/maxmind.go b/internal/nginx/maxmind.go index 3bd895b5d..0f3c37016 100644 --- a/internal/nginx/maxmind.go +++ b/internal/nginx/maxmind.go @@ -21,17 +21,10 @@ import ( "compress/gzip" "fmt" "io" - "net" "net/http" - "net/url" "os" "path" "strings" - "syscall" - "time" - - "k8s.io/apimachinery/pkg/util/wait" - klog "k8s.io/klog/v2" ) // MaxmindLicenseKey maxmind license key to download databases @@ -46,17 +39,8 @@ var MaxmindEditionFiles []string // MaxmindMirror maxmind database mirror url (http://geoip.local) var MaxmindMirror = "" -// MaxmindRetriesCount number of attempts to download the GeoIP DB -var MaxmindRetriesCount = 1 - -// MaxmindRetriesTimeout maxmind download retries timeout in seconds, 0 - do not retry to download if something went wrong -var MaxmindRetriesTimeout = time.Second * 0 - -// minimumRetriesCount minimum value of the MaxmindRetriesCount parameter. If MaxmindRetriesCount less than minimumRetriesCount, it will be set to minimumRetriesCount -const minimumRetriesCount = 1 - const ( - geoIPPath = "/etc/ingress-controller/geoip" + geoIPPath = "/etc/nginx/geoip" dbExtension = ".mmdb" maxmindURL = "https://download.maxmind.com/app/geoip_download?license_key=%v&edition_id=%v&suffix=tar.gz" @@ -64,83 +48,39 @@ const ( // GeoLite2DBExists checks if the required databases for // the GeoIP2 NGINX module are present in the filesystem -// and indexes the discovered databases for iteration in -// the config. func GeoLite2DBExists() bool { - files := []string{} for _, dbName := range strings.Split(MaxmindEditionIDs, ",") { - filename := dbName + dbExtension - if !fileExists(path.Join(geoIPPath, filename)) { - klog.Error(filename, " not found") + if !fileExists(path.Join(geoIPPath, dbName+dbExtension)) { return false } - files = append(files, filename) } - MaxmindEditionFiles = files return true } // DownloadGeoLite2DB downloads the required databases by the // GeoIP2 NGINX module using a license key from MaxMind. -func DownloadGeoLite2DB(attempts int, period time.Duration) error { - if attempts < minimumRetriesCount { - attempts = minimumRetriesCount - } - - defaultRetry := wait.Backoff{ - Steps: attempts, - Duration: period, - Factor: 1.5, - Jitter: 0.1, - } - if period == time.Duration(0) { - defaultRetry.Steps = minimumRetriesCount - } - - var lastErr error - retries := 0 - - lastErr = wait.ExponentialBackoff(defaultRetry, func() (bool, error) { - var dlError error - for _, dbName := range strings.Split(MaxmindEditionIDs, ",") { - dlError = downloadDatabase(dbName) - if dlError != nil { - break - } +func DownloadGeoLite2DB() error { + for _, dbName := range strings.Split(MaxmindEditionIDs, ",") { + err := downloadDatabase(dbName) + if err != nil { + return err } - - lastErr = dlError - if dlError == nil { - return true, nil - } - - if e, ok := dlError.(*url.Error); ok { - if e, ok := e.Err.(*net.OpError); ok { - if e, ok := e.Err.(*os.SyscallError); ok { - if e.Err == syscall.ECONNREFUSED { - retries++ - klog.InfoS("download failed on attempt " + fmt.Sprint(retries)) - return false, nil - } - } - } - } - return true, nil - }) - return lastErr + MaxmindEditionFiles = append(MaxmindEditionFiles, dbName+dbExtension) + } + return nil } func createURL(mirror, licenseKey, dbName string) string { - if mirror != "" { + if len(mirror) > 0 { return fmt.Sprintf("%s/%s.tar.gz", mirror, dbName) } return fmt.Sprintf(maxmindURL, licenseKey, dbName) } func downloadDatabase(dbName string) error { - newURL := createURL(MaxmindMirror, MaxmindLicenseKey, dbName) - req, err := http.NewRequest(http.MethodGet, newURL, http.NoBody) + url := createURL(MaxmindMirror, MaxmindLicenseKey, dbName) + req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return err } @@ -165,7 +105,7 @@ func downloadDatabase(dbName string) error { mmdbFile := dbName + dbExtension tarReader := tar.NewReader(archive) - for { + for true { header, err := tarReader.Next() if err == io.EOF { break @@ -175,23 +115,24 @@ func downloadDatabase(dbName string) error { return err } - if header.Typeflag == tar.TypeReg { + switch header.Typeflag { + case tar.TypeReg: if !strings.HasSuffix(header.Name, mmdbFile) { continue } - return func() error { - outFile, err := os.Create(path.Join(geoIPPath, mmdbFile)) - if err != nil { - return err - } - defer outFile.Close() + outFile, err := os.Create(path.Join(geoIPPath, mmdbFile)) + if err != nil { + return err + } - if _, err := io.CopyN(outFile, tarReader, header.Size); err != nil { - return err - } - return nil - }() + defer outFile.Close() + + if _, err := io.CopyN(outFile, tarReader, header.Size); err != nil { + return err + } + + return nil } } @@ -222,7 +163,7 @@ func ValidateGeoLite2DBEditions() error { return nil } -func _fileExists(filePath string) bool { +func fileExists(filePath string) bool { info, err := os.Stat(filePath) if os.IsNotExist(err) { return false @@ -230,5 +171,3 @@ func _fileExists(filePath string) bool { return !info.IsDir() } - -var fileExists = _fileExists diff --git a/internal/nginx/maxmind_test.go b/internal/nginx/maxmind_test.go deleted file mode 100644 index ed78c32a1..000000000 --- a/internal/nginx/maxmind_test.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package nginx - -import ( - "reflect" - "testing" -) - -func resetForTesting() { - fileExists = _fileExists - MaxmindLicenseKey = "" - MaxmindEditionIDs = "" - MaxmindEditionFiles = []string{} - MaxmindMirror = "" -} - -func TestGeoLite2DBExists(t *testing.T) { - tests := []struct { - name string - setup func() - want bool - wantFiles []string - }{ - { - name: "empty", - wantFiles: []string{}, - }, - { - name: "existing files", - setup: func() { - MaxmindEditionIDs = "GeoLite2-City,GeoLite2-ASN" - fileExists = func(string) bool { - return true - } - }, - want: true, - wantFiles: []string{"GeoLite2-City.mmdb", "GeoLite2-ASN.mmdb"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - resetForTesting() - // mimics assignment in flags.go - config := &MaxmindEditionFiles - - if tt.setup != nil { - tt.setup() - } - if got := GeoLite2DBExists(); got != tt.want { - t.Errorf("GeoLite2DBExists() = %v, want %v", got, tt.want) - } - if !reflect.DeepEqual(MaxmindEditionFiles, tt.wantFiles) { - t.Errorf("nginx.MaxmindEditionFiles = %v, want %v", MaxmindEditionFiles, tt.wantFiles) - } - if !reflect.DeepEqual(*config, tt.wantFiles) { - t.Errorf("config.MaxmindEditionFiles = %v, want %v", *config, tt.wantFiles) - } - }) - } -} diff --git a/internal/runtime/cpu_linux.go b/internal/runtime/cpu_linux.go new file mode 100644 index 000000000..41b969df1 --- /dev/null +++ b/internal/runtime/cpu_linux.go @@ -0,0 +1,66 @@ +// +build linux + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runtime + +import ( + "io/ioutil" + "math" + "path/filepath" + "runtime" + "strconv" + "strings" + + libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups" +) + +// NumCPU returns the number of logical CPUs usable by the current process. +// If CPU cgroups limits are configured, use cfs_quota_us / cfs_period_us +// as formula +// https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt +func NumCPU() int { + cpus := runtime.NumCPU() + + cgroupPath, err := libcontainercgroups.FindCgroupMountpoint("", "cpu") + if err != nil { + return cpus + } + + cpuQuota := readCgroupFileToInt64(cgroupPath, "cpu.cfs_quota_us") + cpuPeriod := readCgroupFileToInt64(cgroupPath, "cpu.cfs_period_us") + + if cpuQuota == -1 || cpuPeriod == -1 { + return cpus + } + + return int(math.Ceil(float64(cpuQuota) / float64(cpuPeriod))) +} + +func readCgroupFileToInt64(cgroupPath, cgroupFile string) int64 { + contents, err := ioutil.ReadFile(filepath.Join(cgroupPath, cgroupFile)) + if err != nil { + return -1 + } + + strValue := strings.TrimSpace(string(contents)) + if value, err := strconv.ParseInt(strValue, 10, 64); err == nil { + return value + } + + return -1 +} diff --git a/pkg/util/runtime/cpu_notlinux.go b/internal/runtime/cpu_notlinux.go similarity index 87% rename from pkg/util/runtime/cpu_notlinux.go rename to internal/runtime/cpu_notlinux.go index 97c72cd5a..86a649e62 100644 --- a/pkg/util/runtime/cpu_notlinux.go +++ b/internal/runtime/cpu_notlinux.go @@ -1,4 +1,3 @@ -//go:build !linux // +build !linux /* @@ -23,7 +22,7 @@ import ( "runtime" ) -// NumCPU returns the number of logical CPUs usable by the current process. +// NumCPU ... func NumCPU() int { return runtime.NumCPU() } diff --git a/pkg/util/sets/match.go b/internal/sets/match.go similarity index 100% rename from pkg/util/sets/match.go rename to internal/sets/match.go diff --git a/pkg/util/sets/match_test.go b/internal/sets/match_test.go similarity index 78% rename from pkg/util/sets/match_test.go rename to internal/sets/match_test.go index a65c5a05f..e2366d2c7 100644 --- a/pkg/util/sets/match_test.go +++ b/internal/sets/match_test.go @@ -20,20 +20,23 @@ import ( "testing" ) -var testCasesElementMatch = []struct { - listA []string - listB []string - expected bool -}{ - {nil, nil, true}, - {[]string{"1"}, nil, false}, - {[]string{"1"}, []string{"1"}, true}, - {[]string{"1", "2", "1"}, []string{"1", "1", "2"}, true}, - {[]string{"1", "3", "1"}, []string{"1", "1", "2"}, false}, - {[]string{"1", "1"}, []string{"1", "2"}, false}, -} +var ( + testCasesElementMatch = []struct { + listA []string + listB []string + expected bool + }{ + {nil, nil, true}, + {[]string{"1"}, nil, false}, + {[]string{"1"}, []string{"1"}, true}, + {[]string{"1", "2", "1"}, []string{"1", "1", "2"}, true}, + {[]string{"1", "3", "1"}, []string{"1", "1", "2"}, false}, + {[]string{"1", "1"}, []string{"1", "2"}, false}, + } +) func TestElementsMatch(t *testing.T) { + for _, testCase := range testCasesElementMatch { result := StringElementsMatch(testCase.listA, testCase.listB) if result != testCase.expected { diff --git a/internal/task/queue.go b/internal/task/queue.go index 8753bed34..3c6788409 100644 --- a/internal/task/queue.go +++ b/internal/task/queue.go @@ -28,7 +28,9 @@ import ( "k8s.io/client-go/util/workqueue" ) -var keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc +var ( + keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc +) // Queue manages a time work queue through an independent worker that invokes the // given sync function for every work item inserted. @@ -36,7 +38,7 @@ var keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc // which timestamp is older than the last successful get operation. type Queue struct { // queue is the work queue the worker polls - queue workqueue.TypedRateLimitingInterface[any] + queue workqueue.RateLimitingInterface // sync is called for each item in the queue sync func(interface{}) error // workerDone is closed when the worker exits @@ -115,11 +117,8 @@ func (t *Queue) worker() { } ts := time.Now().UnixNano() - item, ok := key.(Element) - if !ok { - klog.ErrorS(nil, "invalid item type", "key", key) - } - if item.Timestamp != 0 && t.lastSync > item.Timestamp { + item := key.(Element) + if t.lastSync > item.Timestamp { klog.V(3).InfoS("skipping sync", "key", item.Key, "last", t.lastSync, "now", item.Timestamp) t.queue.Forget(key) t.queue.Done(key) @@ -131,7 +130,7 @@ func (t *Queue) worker() { klog.ErrorS(err, "requeuing", "key", item.Key) t.queue.AddRateLimited(Element{ Key: item.Key, - Timestamp: 0, + Timestamp: time.Now().UnixNano(), }) } else { t.queue.Forget(key) @@ -169,10 +168,10 @@ func NewTaskQueue(syncFn func(interface{}) error) *Queue { return NewCustomTaskQueue(syncFn, nil) } -// NewCustomTaskQueue creates a new custom task queue with the given sync function. +// NewCustomTaskQueue ... func NewCustomTaskQueue(syncFn func(interface{}) error, fn func(interface{}) (interface{}, error)) *Queue { q := &Queue{ - queue: workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[any]()), + queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), sync: syncFn, workerDone: make(chan bool), fn: fn, diff --git a/pkg/util/file/file_watcher.go b/internal/watch/file_watcher.go similarity index 88% rename from pkg/util/file/file_watcher.go rename to internal/watch/file_watcher.go index 3899e41f8..b393045b2 100644 --- a/pkg/util/file/file_watcher.go +++ b/internal/watch/file_watcher.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package file +package watch import ( "log" @@ -26,8 +26,8 @@ import ( "github.com/fsnotify/fsnotify" ) -// Watcher is an interface we use to watch changes in files -type Watcher interface { +// FileWatcher is an interface we use to watch changes in files +type FileWatcher interface { Close() error } @@ -40,7 +40,7 @@ type OSFileWatcher struct { } // NewFileWatcher creates a new FileWatcher -func NewFileWatcher(file string, onEvent func()) (Watcher, error) { +func NewFileWatcher(file string, onEvent func()) (FileWatcher, error) { fw := OSFileWatcher{ file: file, onEvent: onEvent, @@ -72,8 +72,8 @@ func (f *OSFileWatcher) watch() error { for { select { case event := <-watcher.Events: - if event.Has(fsnotify.Create) || - event.Has(fsnotify.Write) { + if event.Op&fsnotify.Create == fsnotify.Create || + event.Op&fsnotify.Write == fsnotify.Write { if finfo, err := os.Lstat(event.Name); err != nil { log.Printf("can not lstat file: %v\n", err) } else if finfo.Mode()&os.ModeSymlink != 0 { diff --git a/pkg/util/file/file_watcher_test.go b/internal/watch/file_watcher_test.go similarity index 92% rename from pkg/util/file/file_watcher_test.go rename to internal/watch/file_watcher_test.go index dd2755812..7b39c4e30 100644 --- a/pkg/util/file/file_watcher_test.go +++ b/internal/watch/file_watcher_test.go @@ -14,14 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */ -package file +package watch import ( + "io/ioutil" "os" "path" "path/filepath" "testing" "time" + + "k8s.io/ingress-nginx/internal/file" ) func prepareTimeout() chan bool { @@ -34,7 +37,7 @@ func prepareTimeout() chan bool { } func TestFileWatcher(t *testing.T) { - f, err := os.CreateTemp("", "fw") + f, err := ioutil.TempFile("", "fw") if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -59,9 +62,7 @@ func TestFileWatcher(t *testing.T) { t.Fatalf("expected no events before writing a file") case <-timeoutChan: } - if err := os.WriteFile(f.Name(), []byte{}, ReadWriteByUser); err != nil { - t.Errorf("unexpected error: %v", err) - } + ioutil.WriteFile(f.Name(), []byte{}, file.ReadWriteByUser) select { case <-events: case <-timeoutChan: @@ -70,7 +71,7 @@ func TestFileWatcher(t *testing.T) { } func TestFileWatcherWithNestedSymlink(t *testing.T) { - target1, err := os.CreateTemp("", "t1") + target1, err := ioutil.TempFile("", "t1") if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -114,7 +115,7 @@ func TestFileWatcherWithNestedSymlink(t *testing.T) { } defer fw.Close() - target2, err := os.CreateTemp("", "t2") + target2, err := ioutil.TempFile("", "t2") if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/magefiles/go.mod b/magefiles/go.mod deleted file mode 100644 index f11ab740e..000000000 --- a/magefiles/go.mod +++ /dev/null @@ -1,38 +0,0 @@ -module k8s.io/ingress-nginx/magefiles - -go 1.23.6 - -require ( - github.com/blang/semver/v4 v4.0.0 - github.com/google/go-github/v48 v48.2.0 - github.com/helm/helm v2.17.0+incompatible - github.com/magefile/mage v1.15.0 - github.com/vmware-labs/yaml-jsonpath v0.3.2 - golang.org/x/oauth2 v0.18.0 - gopkg.in/yaml.v3 v3.0.1 -) - -require ( - github.com/BurntSushi/toml v1.3.2 // indirect - github.com/Masterminds/semver v1.5.0 // indirect - github.com/cyphar/filepath-securejoin v0.2.4 // indirect - github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/ghodss/yaml v1.0.0 // indirect - github.com/gobwas/glob v0.2.3 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/go-querystring v1.1.0 // indirect - github.com/kr/text v0.2.0 // indirect - github.com/onsi/ginkgo v1.16.5 // indirect - github.com/onsi/gomega v1.30.0 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect - github.com/sergi/go-diff v1.3.1 // indirect - github.com/stretchr/testify v1.9.0 // indirect - golang.org/x/crypto v0.31.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/protobuf v1.33.0 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/apimachinery v0.29.3 // indirect - k8s.io/helm v2.17.0+incompatible // indirect -) diff --git a/magefiles/go.sum b/magefiles/go.sum deleted file mode 100644 index 8684d4701..000000000 --- a/magefiles/go.sum +++ /dev/null @@ -1,174 +0,0 @@ -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= -github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= -github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960 h1:aRd8M7HJVZOqn/vhOzrGcQH0lNAMkqMn+pXUYkatmcA= -github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960/go.mod h1:9HQzr9D/0PGwMEbC3d5AB7oi67+h4TsQqItC1GVYG58= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-github/v48 v48.2.0 h1:68puzySE6WqUY9KWmpOsDEQfDZsso98rT6pZcz9HqcE= -github.com/google/go-github/v48 v48.2.0/go.mod h1:dDlehKBDo850ZPvCTK0sEqTCVWcrGl2LcDiajkYi89Y= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= -github.com/helm/helm v2.17.0+incompatible h1:0iy95yMXrfWpwaoOA9XRP+cTvitTrq+LcJV9DvR5n1Y= -github.com/helm/helm v2.17.0+incompatible/go.mod h1:ahXhuvluW4YnSL6W6hDVetZsVK8Pv4BP8OwKli7aMqo= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= -github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= -github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/vmware-labs/yaml-jsonpath v0.3.2 h1:/5QKeCBGdsInyDCyVNLbXyilb61MXGi9NP674f9Hobk= -github.com/vmware-labs/yaml-jsonpath v0.3.2/go.mod h1:U6whw1z03QyqgWdgXxvVnQ90zN1BWz5V+51Ewf8k+rQ= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= -k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= -k8s.io/helm v2.17.0+incompatible h1:Bpn6o1wKLYqKM3+Osh8e+1/K2g/GsQJ4F4yNF2+deao= -k8s.io/helm v2.17.0+incompatible/go.mod h1:LZzlS4LQBHfciFOurYBFkCMTaZ0D1l+p0teMg7TSULI= diff --git a/magefiles/mage.go b/magefiles/mage.go deleted file mode 100644 index 1dfaa4dbd..000000000 --- a/magefiles/mage.go +++ /dev/null @@ -1,24 +0,0 @@ -//go:build mage - -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - //mage:import - _ "k8s.io/ingress-nginx/magefiles/steps" -) diff --git a/magefiles/steps/helm.go b/magefiles/steps/helm.go deleted file mode 100644 index 73c9b0b3b..000000000 --- a/magefiles/steps/helm.go +++ /dev/null @@ -1,189 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package steps - -import ( - "bytes" - "os" - - semver "github.com/blang/semver/v4" - "github.com/helm/helm/pkg/chartutil" - "github.com/magefile/mage/mg" - "github.com/magefile/mage/sh" - yamlpath "github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath" - "gopkg.in/yaml.v3" - - utils "k8s.io/ingress-nginx/magefiles/utils" -) - -const ( - HelmChartPath = "charts/ingress-nginx/Chart.yaml" - HelmChartValues = "charts/ingress-nginx/values.yaml" -) - -type Helm mg.Namespace - -// UpdateVersion Update Helm Version of the Chart -func (Helm) UpdateVersion(version string) { - updateVersion(version) -} - -func currentChartVersion() string { - chart, err := chartutil.LoadChartfile(HelmChartPath) - utils.CheckIfError(err, "HELM Could not Load Chart") - return chart.Version -} - -func updateVersion(version string) { - utils.Info("HELM Reading File %v", HelmChartPath) - - chart, err := chartutil.LoadChartfile(HelmChartPath) - utils.CheckIfError(err, "HELM Could not Load Chart") - - // Get the current tag - // appVersionV, err := getIngressNGINXVersion() - // utils.CheckIfError(err, "HELM Issue Retrieving the Current Ingress Nginx Version") - - // remove the v from TAG - appVersion := version - - utils.Info("HELM Ingress-Nginx App Version: %s Chart AppVersion: %s", appVersion, chart.AppVersion) - if appVersion == chart.AppVersion { - utils.Warning("HELM Ingress NGINX Version didnt change Ingress-Nginx App Version: %s Chart AppVersion: %s", appVersion, chart.AppVersion) - return - } - - controllerSemVer, err := semver.Parse(version) - utils.CheckIfError(err, "error parsing semver of new app") - isPreRelease := len(controllerSemVer.Pre) > 0 - oldControllerSemVer, err := semver.Parse(chart.AppVersion) - utils.CheckIfError(err, "error parsing semver of old chart") - isBreakingChange := controllerSemVer.Major > oldControllerSemVer.Major || controllerSemVer.Minor > oldControllerSemVer.Minor - - // Update the helm chart - chart.AppVersion = appVersion - cTag, err := semver.Make(chart.Version) - utils.CheckIfError(err, "HELM Creating Chart Version: %v", err) - - incrFunc := cTag.IncrementPatch - if isBreakingChange { - cTag.Patch = 0 - incrFunc = cTag.IncrementMinor - - } - - if isPreRelease { - chart.Annotations["artifacthub.io/prerelease"] = "true" - cTag.Pre = controllerSemVer.Pre - } - - if err = incrFunc(); err != nil { - utils.ErrorF("HELM Incrementing Chart Version: %v", err) - os.Exit(1) - } - chart.Version = cTag.String() - utils.Debug("HELM Updated Chart Version: %v", chart.Version) - - err = chartutil.SaveChartfile(HelmChartPath, chart) - utils.CheckIfError(err, "HELM Saving new Chart") -} - -func updateChartReleaseNotes(releaseNotes []string) { - utils.Info("HELM Updating chart release notes") - chart, err := chartutil.LoadChartfile(HelmChartPath) - utils.CheckIfError(err, "HELM Failed to load chart manifest: %s", HelmChartPath) - - releaseNotesBytes, err := yaml.Marshal(releaseNotes) - utils.CheckIfError(err, "HELM Failed to marshal release notes") - - releaseNotesString := string(releaseNotesBytes) - utils.Info("HELM Chart release notes:\n%s", releaseNotesString) - chart.Annotations["artifacthub.io/changes"] = releaseNotesString - - utils.Info("HELM Saving chart release notes") - err = chartutil.SaveChartfile(HelmChartPath, chart) - utils.CheckIfError(err, "HELM Failed to save chart manifest: %s", HelmChartPath) -} - -// UpdateChartValue Updates the Helm ChartValue -func (Helm) UpdateChartValue(key, value string) { - updateChartValue(key, value) -} - -func updateChartValue(key, value string) { - utils.Info("HELM Updating Chart %s %s:%s", HelmChartValues, key, value) - - // read current values.yaml - data, err := os.ReadFile(HelmChartValues) - utils.CheckIfError(err, "HELM Could not Load Helm Chart Values files %s", HelmChartValues) - - // var valuesStruct IngressChartValue - var n yaml.Node - utils.CheckIfError(yaml.Unmarshal(data, &n), "HELM Could not Unmarshal %s", HelmChartValues) - - // update value - // keyParse := parsePath(key) - p, err := yamlpath.NewPath(key) - utils.CheckIfError(err, "HELM cannot create path") - - q, err := p.Find(&n) - utils.CheckIfError(err, "HELM unexpected error finding path") - - for _, i := range q { - utils.Info("HELM Found %s at %s", i.Value, key) - i.Value = value - utils.Info("HELM Updated %s at %s", i.Value, key) - } - - //// write to file - var b bytes.Buffer - yamlEncoder := yaml.NewEncoder(&b) - yamlEncoder.SetIndent(2) - err = yamlEncoder.Encode(&n) - utils.CheckIfError(err, "HELM Could not Marshal new Values file") - err = os.WriteFile(HelmChartValues, b.Bytes(), 0o644) - utils.CheckIfError(err, "HELM Could not write new Values file to %s", HelmChartValues) - - utils.Info("HELM Ingress Nginx Helm Chart update %s %s", key, value) -} - -func (Helm) Helmdocs() error { - return runHelmDocs() -} - -func runHelmDocs() error { - err := installHelmDocs() - if err != nil { - return err - } - err = sh.RunV("helm-docs", "--chart-search-root", "${PWD}/charts") - if err != nil { - return err - } - return nil -} - -func installHelmDocs() error { - utils.Info("HELM Install HelmDocs") - g0 := sh.RunCmd("go") - - err := g0("install", "github.com/norwoodj/helm-docs/cmd/helm-docs@latest") - if err != nil { - return err - } - return nil -} diff --git a/magefiles/steps/release.go b/magefiles/steps/release.go deleted file mode 100644 index b6d46336a..000000000 --- a/magefiles/steps/release.go +++ /dev/null @@ -1,395 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package steps - -import ( - "context" - "errors" - "fmt" - "os" - "regexp" - "strings" - - "github.com/google/go-github/v48/github" - "github.com/magefile/mage/mg" - "github.com/magefile/mage/sh" - "golang.org/x/oauth2" - "gopkg.in/yaml.v3" - - utils "k8s.io/ingress-nginx/magefiles/utils" -) - -type Release mg.Namespace - -var ( - INGRESS_ORG = "kubernetes" // the owner so we can test from forks - INGRESS_REPO = "ingress-nginx" // the repo to pull from - RELEASE_BRANCH = "main" // we only release from main - GITHUB_TOKEN string // the Google/gogithub lib needs an PAT to access the GitHub API - K8S_IO_ORG = "kubernetes" // the owner or organization for the k8s.io repo - K8S_IO_REPO = "k8s.io" // the repo that holds the images yaml for production promotion - INGRESS_REGISTRY = "registry.k8s.io" // Container registry for storage Ingress-nginx images - KUSTOMIZE_INSTALL_VERSION = "sigs.k8s.io/kustomize/kustomize/v4@v4.5.4" // static deploys needs kustomize to generate the template -) - -// ingress-nginx releases start with a TAG then a cloudbuild, then a promotion through a PR, this the location of that PR -var ( - IMAGES_YAML = "https://raw.githubusercontent.com/kubernetes/k8s.io/main/registry.k8s.io/images/k8s-staging-ingress-nginx/images.yaml" - ctx = context.Background() // Context used for GitHub Client -) - -const ( - INDEX_DOCS = "docs/deploy/index.md" // index.md has a version of the controller and needs to updated - CHANGELOG = "Changelog.md" // Name of the changelog -) - -// init will set the GitHub token from the committers/releasers env var -func init() { - GITHUB_TOKEN = os.Getenv("GITHUB_TOKEN") -} - -// Release Create a new release of ingress nginx controller -func (Release) NewRelease(version string) { - newRelease(version, "") -} - -func (Release) NewReleaseFromOld(version, oldversion string) { - newRelease(version, oldversion) -} - -func (Release) E2EDocs() { - e2edocs, err := utils.GenerateE2EDocs() - utils.CheckIfError(err, "error on template") - err = os.WriteFile("docs/e2e-tests.md", []byte(e2edocs), 0644) - utils.CheckIfError(err, "Could not write new e2e test file ") -} - -func newRelease(version, oldversion string) { - // newRelease := Release{} - - // update ingress-nginx version - // This is the step that kicks all the release process - // it is already done, so it kicks off the gcloud build of the controller images - // mg.Deps(mg.F(Tag.BumpNginx, version)) - - tag, err := getIngressNGINXVersion() - utils.CheckIfError(err, "RELEASE Retrieving the current Ingress Nginx Version") - - utils.Info("RELEASE Checking Current Version %s to New Version %s", tag, version) - // if the version were upgrading does not match the TAG file, lets update the TAG file - if tag[1:] != version { - utils.Warning("RELEASE Ingress Nginx TAG %s and new version %s do not match", tag, version) - mg.Deps(mg.F(Tag.BumpNginx, fmt.Sprintf("v%s", version))) - } - - // update git controller tag controller-v$version - mg.Deps(mg.F(Tag.NewControllerTag, version)) - - // make release notes - releaseNotes, err := makeReleaseNotes(version, oldversion) - utils.CheckIfError(err, "RELEASE Creating Release Notes for version %s", version) - utils.Info("RELEASE Release Notes %s completed", releaseNotes.Version) - - // update chart values.yaml new controller tag and image digest - releaseNotes.PreviousHelmChartVersion = currentChartVersion() - - // controller tag - updateChartValue("controller.image.tag", fmt.Sprintf("v%s", releaseNotes.Version)) - utils.Debug("releaseNotes.ControllerImages[0].Name %s", releaseNotes.ControllerImages[0].Name) - utils.Debug("releaseNotes.ControllerImages[1].Name %s", releaseNotes.ControllerImages[1].Name) - // controller digest - if releaseNotes.ControllerImages[0].Name == "ingress-nginx/controller" { - utils.Debug("Updating Chart Value %s with %s", "controller.image.digest", releaseNotes.ControllerImages[0].Digest) - updateChartValue("controller.image.digest", releaseNotes.ControllerImages[0].Digest) - } - // controller chroot digest - if releaseNotes.ControllerImages[1].Name == "ingress-nginx/controller-chroot" { - utils.Debug("Updating Chart Value %s with %s", "controller.image.digestChroot", releaseNotes.ControllerImages[1].Digest) - updateChartValue("controller.image.digestChroot", releaseNotes.ControllerImages[1].Digest) - } - - // update helm chart app version - mg.Deps(mg.F(Helm.UpdateVersion, version)) - - releaseNotes.NewHelmChartVersion = currentChartVersion() - - // update helm chart release notes - updateChartReleaseNotes(releaseNotes.HelmUpdates) - - // Run helm docs update - utils.CheckIfError(runHelmDocs(), "Error Updating Helm Docs ") - - releaseNotes.HelmTemplate() - - // update static manifest - utils.CheckIfError(updateStaticManifest(), "Error Updating Static manifests") - - ////update e2e docs - mg.Deps(mg.F(Release.E2EDocs)) - - // update documentation with ingress-nginx version - utils.CheckIfError(updateIndexMD(releaseNotes.PreviousControllerVersion, releaseNotes.NewControllerVersion), "Error Updating %s", INDEX_DOCS) - - // keeping these manual for now - // git commit TODO - // make Pull Request TODO - // make release TODO - // mg.Deps(mg.F(Release.CreateRelease, version)) -} - -// the index.md doc needs the controller version updated -func updateIndexMD(old, new string) error { - utils.Info("Updating Deploy docs with new version") - data, err := os.ReadFile(INDEX_DOCS) - utils.CheckIfError(err, "Could not read INDEX_DOCS file %s", INDEX_DOCS) - datString := string(data) - datString = strings.Replace(datString, old, new, -1) - err = os.WriteFile(INDEX_DOCS, []byte(datString), 0644) - if err != nil { - utils.ErrorF("Could not write new %s %s", INDEX_DOCS, err) - return err - } - return nil -} - -// The static deploy scripts use kustomize to generate them, this function ensures kustomize is installed -func installKustomize() error { - utils.Info("Install Kustomize") - g0 := sh.RunCmd("go") - // somewhere in your main code - err := g0("install", KUSTOMIZE_INSTALL_VERSION) - if err != nil { - return err - } - return nil -} - -func updateStaticManifest() error { - utils.CheckIfError(installKustomize(), "error installing kustomize") - // hack/generate-deploy-scripts.sh - err := sh.RunV("./hack/generate-deploy-scripts.sh") - if err != nil { - return err - } - return nil -} - -//// CreateRelease Creates a new GitHub Release -//func (Release) CreateRelease(name string) { -// releaser, err := gh_release.NewReleaser(INGRESS_ORG, INGRESS_REPO, GITHUB_TOKEN) -// utils.CheckIfError(err, "GitHub Release Client error") -// newRelease, err := releaser.Create(fmt.Sprintf("controller-%s", name)) -// utils.CheckIfError(err, "Create release error") -// utils.Info("New Release: Tag %v, ID: %v", newRelease.TagName, newRelease.ID) -//} - -// Returns a GitHub client ready for use -func githubClient() *github.Client { - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: GITHUB_TOKEN}, - ) - oauthClient := oauth2.NewClient(ctx, ts) - return github.NewClient(oauthClient) -} - -// LatestCommitLogs Retrieves the commit log between the latest two controller versions. -func (Release) LatestCommitLogs() { - commitLog := commitsBetweenTags("", "") - for i, s := range commitLog { - utils.Info("#%v Version %v", i, s) - } -} - -func commitsBetweenTags(newversion, oldversion string) []string { - var newTag, oldTag string - tags := getAllControllerTags() - newTag, oldTag = tags[0], tags[1] - if newversion != "" { - newTag = newversion - } - if oldversion != "" { - oldTag = oldversion - } - - utils.Info("Getting Commits between %v and %v", newTag, oldTag) - commitLog, err := git("log", "--full-history", "--pretty", "--oneline", fmt.Sprintf("%v..%v", oldTag, newTag)) - - if commitLog == "" { - utils.Warning("All Controller Tags is empty") - } - utils.CheckIfError(err, "Retrieving Commit log") - return strings.Split(commitLog, "\n") -} - -// Generate Release Notes -func (Release) ReleaseNotes(newVersion string) error { - notes, err := makeReleaseNotes(newVersion, "") - utils.CheckIfError(err, "Creating Release Notes for version %s", newVersion) - utils.Info("Release Notes %s completed", notes.Version) - return nil -} - -func makeReleaseNotes(newVersion, oldVersion string) (*utils.ReleaseNote, error) { - newReleaseNotes := utils.ReleaseNote{} - - newReleaseNotes.Version = newVersion - allControllerTags := getAllControllerTags() - - // new version - newReleaseNotes.NewControllerVersion = allControllerTags[0] - newControllerVersion := fmt.Sprintf("controller-v%s", newVersion) - - // the newControllerVersion should match the latest tag - if newControllerVersion != allControllerTags[0] { - return nil, fmt.Errorf("generating release new version %s didnt match the current latest tag %s", newControllerVersion, allControllerTags[0]) - } - // previous version - newReleaseNotes.PreviousControllerVersion = allControllerTags[1] - if oldVersion != "" { - newReleaseNotes.PreviousControllerVersion = oldVersion - } - - utils.Info("New Version: %s Old Version: %s", newReleaseNotes.NewControllerVersion, newReleaseNotes.PreviousControllerVersion) - - commits := commitsBetweenTags(newReleaseNotes.NewControllerVersion, newReleaseNotes.PreviousControllerVersion) - - // dependency_updates - // all_updates - var allUpdates []string - var depUpdates []string - var helmUpdates []string - prRegex := regexp.MustCompile(`\(#\d+\)`) - depBot := regexp.MustCompile(`^(\w){1,10} Bump `) - helmRegex := regexp.MustCompile("helm|chart") - for i, s := range commits { - // matches on PR - if prRegex.Match([]byte(s)) { - // matches a dependant bot update - if depBot.Match([]byte(s)) { // - utils.Debug("#%v DEPENDABOT %v", i, s) - u := strings.SplitN(s, " ", 2) - depUpdates = append(depUpdates, u[1]) - } else { // add it to the all updates slice - utils.Debug("#%v ALL UPDATES %v", i, s) - u := strings.SplitN(s, " ", 2) - allUpdates = append(allUpdates, u[1]) - - // helm chart updates - if helmRegex.Match([]byte(s)) { - u := strings.SplitN(s, " ", 2) - helmUpdates = append(helmUpdates, u[1]) - } - } - } - } - helmUpdates = append(helmUpdates, fmt.Sprintf("Update Ingress-Nginx version %s", newReleaseNotes.NewControllerVersion)) - - newReleaseNotes.Updates = allUpdates - newReleaseNotes.DepUpdates = depUpdates - newReleaseNotes.HelmUpdates = helmUpdates - - // controller_image_digests - imagesYaml, err := utils.DownloadFile(IMAGES_YAML) - if err != nil { - utils.ErrorF("Could not download file %s : %s", IMAGES_YAML, err) - return nil, err - } - utils.Debug("%s", imagesYaml) - - data := utils.ImageYamls{} - - err = yaml.Unmarshal([]byte(imagesYaml), &data) - if err != nil { - utils.ErrorF("Could not unmarshal images yaml %s", err) - return nil, err - } - - // controller - controllerDigest := utils.FindImageDigest(data, "controller", newVersion) - if len(controllerDigest) == 0 { - utils.ErrorF("Controller Digest could not be found") - return nil, errors.New("controller digest could not be found") - } - - controllerChrootDigest := utils.FindImageDigest(data, "controller-chroot", newVersion) - if len(controllerChrootDigest) == 0 { - utils.ErrorF("Controller Chroot Digest could not be found") - return nil, errors.New("controller chroot digest could not be found") - } - - utils.Debug("Latest Controller Digest %v", controllerDigest) - utils.Debug("Latest Controller Chroot Digest %v", controllerChrootDigest) - c1 := utils.ControllerImage{ - Digest: controllerDigest, - Registry: INGRESS_REGISTRY, - Name: "ingress-nginx/controller", - Tag: fmt.Sprintf("v%s", newReleaseNotes.Version), - } - - c2 := utils.ControllerImage{ - Digest: controllerChrootDigest, - Registry: INGRESS_REGISTRY, - Name: "ingress-nginx/controller-chroot", - Tag: fmt.Sprintf("v%s", newReleaseNotes.Version), - } - - newReleaseNotes.ControllerImages = append(newReleaseNotes.ControllerImages, c1) - newReleaseNotes.ControllerImages = append(newReleaseNotes.ControllerImages, c2) - utils.Debug("New Release Controller Images %s %s", newReleaseNotes.ControllerImages[0].Digest, newReleaseNotes.ControllerImages[1].Digest) - - if utils.DEBUG { - newReleaseNotes.PrintRelease() - } - - // write it all out to the changelog file - newReleaseNotes.Template() - - return &newReleaseNotes, nil -} - -// Latest returns latest Github Release -func (Release) Latest() error { - r, _, err := latestRelease() - if err != nil { - utils.ErrorF("Latest Release error %s", err) - return err - } - utils.Info("Latest Release %v", r.String()) - return nil -} - -func (Release) ReleaseByTag(tag string) error { - r, _, err := releaseByTag(tag) - if err != nil { - utils.ErrorF("Release retrieve tag %s error %s", tag, err) - return err - } - - utils.Info("Latest Release %v", r.String()) - - return nil -} - -func releaseByTag(tag string) (*github.RepositoryRelease, *github.Response, error) { - ghClient := githubClient() - return ghClient.Repositories.GetReleaseByTag(ctx, INGRESS_ORG, INGRESS_REPO, tag) -} - -func latestRelease() (*github.RepositoryRelease, *github.Response, error) { - ghClient := githubClient() - return ghClient.Repositories.GetLatestRelease(ctx, INGRESS_ORG, INGRESS_REPO) -} diff --git a/magefiles/steps/tags.go b/magefiles/steps/tags.go deleted file mode 100644 index e3c811f8c..000000000 --- a/magefiles/steps/tags.go +++ /dev/null @@ -1,147 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package steps - -import ( - "fmt" - "os" - "strings" - - semver "github.com/blang/semver/v4" - "github.com/magefile/mage/mg" - "github.com/magefile/mage/sh" - - utils "k8s.io/ingress-nginx/magefiles/utils" -) - -type Tag mg.Namespace - -var git = sh.OutCmd("git") - -// Nginx returns the ingress-nginx current version -func (Tag) Nginx() { - tag, err := getIngressNGINXVersion() - utils.CheckIfError(err, "") - fmt.Printf("%v", tag) -} - -func getIngressNGINXVersion() (string, error) { - dat, err := os.ReadFile("TAG") - utils.CheckIfError(err, "Could not read TAG file") - datString := string(dat) - // remove newline - datString = strings.Replace(datString, "\n", "", -1) - return datString, nil -} - -func checkSemVer(currentVersion, newVersion string) bool { - utils.Info("Checking Sem Ver between current %s and new %s", currentVersion, newVersion) - cVersion, err := semver.Make(currentVersion[1:]) - if err != nil { - utils.ErrorF("TAG Error Current Tag %v Making Semver : %v", currentVersion[1:], err) - return false - } - nVersion, err := semver.Make(newVersion) - if err != nil { - utils.ErrorF("TAG %v Error Making Semver %v \n", newVersion, err) - return false - } - - err = nVersion.Validate() - if err != nil { - utils.ErrorF("TAG %v not a valid Semver %v \n", newVersion, err) - return false - } - - //The result will be - //0 if newVersion == currentVersion - //-1 if newVersion < currentVersion - //+1 if newVersion > currentVersion. - utils.Info("TAG Comparing Old %s to New %s", cVersion.String(), nVersion.String()) - comp := nVersion.Compare(cVersion) - if comp <= 0 { - utils.Warning("SemVer:%v is not an update\n", newVersion) - return false - } - return true -} - -// BumpNginx will update the nginx TAG -func (Tag) BumpNginx(newTag string) { - utils.Info("TAG BumpNginx version %v", newTag) - currentTag, err := getIngressNGINXVersion() - utils.CheckIfError(err, "Getting Ingress-nginx Version") - bump(currentTag, newTag) -} - -func bump(currentTag, newTag string) { - // check if semver is valid - if !checkSemVer(currentTag, newTag) { - utils.ErrorF("ERROR: Semver is not valid %v", newTag) - os.Exit(1) - } - - utils.Info("Updating Tag %v to %v", currentTag, newTag) - err := os.WriteFile("TAG", []byte(newTag), 0o666) - utils.CheckIfError(err, "Error Writing New Tag File") -} - -// Git Returns the latest git tag -func (Tag) Git() { - tag, err := getGitTag() - utils.CheckIfError(err, "Retrieving Git Tag") - utils.Info("Git tag: %v", tag) -} - -func getGitTag() (string, error) { - return git("describe", "--tags", "--match", "controller-v*", "--abbrev=0") -} - -// ControllerTag Creates a new Git Tag for the ingress controller -func (Tag) NewControllerTag(version string) { - utils.Info("Create Ingress Nginx Controller Tag v%s", version) - tag, err := controllerTag(version) - utils.CheckIfError(err, "Creating git tag") - utils.Debug("Git Tag: %s", tag) -} - -func controllerTag(version string) (string, error) { - return git("tag", "-a", "-m", fmt.Sprintf("-m \"Automated Controller release %v\"", version), fmt.Sprintf("controller-v%s", version)) -} - -func (Tag) AllControllerTags() { - tags := getAllControllerTags() - for i, s := range tags { - utils.Info("#%v Version %v", i, s) - } -} - -func getAllControllerTags() []string { - allControllerTags, err := git("tag", "-l", "--sort=-v:refname", "controller-v*") - utils.CheckIfError(err, "Retrieving git tags") - if !sh.CmdRan(err) { - utils.Warning("Issue Running Command") - } - if allControllerTags == "" { - utils.Warning("All Controller Tags is empty") - } - utils.Debug("Controller Tags: %v", allControllerTags) - - temp := strings.Split(allControllerTags, "\n") - utils.Debug("There are %v controller tags", len(temp)) - return temp -} diff --git a/magefiles/utils/common.go b/magefiles/utils/common.go deleted file mode 100644 index 19cad7bba..000000000 --- a/magefiles/utils/common.go +++ /dev/null @@ -1,113 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import ( - "errors" - "fmt" - "io" - "net" - "net/http" - "os" - "strings" - "time" -) - -var DEBUG bool - -func init() { - DEBUG = false - debugENV := os.Getenv("MAGE_DEBUG") - if debugENV == "true" { - DEBUG = true - } -} - -// CheckArgs should be used to ensure the right command line arguments are -// passed before executing an example. -func CheckArgs(arg ...string) { - if len(os.Args) < len(arg)+1 { - ErrorF("Usage: %s %s", os.Args[0], strings.Join(arg, " ")) - os.Exit(1) - } -} - -// CheckIfError should be used to naively panics if an error is not nil. -func CheckIfError(err error, format string, args ...interface{}) { - if err == nil { - return - } - - fmt.Printf("\x1b[31;1m%s ERROR %s %s\x1b[0m\n", timeStamp(), fmt.Sprintf(format, args...), err) - os.Exit(1) -} - -// Info should be used to describe the example commands that are about to run. -func Info(format string, args ...interface{}) { - fmt.Printf("\x1b[34;1m%s INFO: %s\x1b[0m\n", timeStamp(), fmt.Sprintf(format, args...)) -} - -func timeStamp() string { - t := time.Now() - return t.Format(time.RFC3339) -} - -// Warning should be used to display a warning -func Warning(format string, args ...interface{}) { - fmt.Printf("\x1b[36;1m%s WARNING: %s\x1b[0m\n", timeStamp(), fmt.Sprintf(format, args...)) -} - -// Info should be used to describe the example commands that are about to run. -func Debug(format string, args ...interface{}) { - if DEBUG { - fmt.Printf("\x1b[34;1m%s DEBUG: %s\x1b[0m\n", timeStamp(), fmt.Sprintf(format, args...)) - } -} - -// Info should be used to describe the example commands that are about to run. -func ErrorF(format string, args ...interface{}) { - fmt.Printf("\x1b[31;1m%s ERROR: %s\x1b[0m\n", timeStamp(), fmt.Sprintf(format, args...)) -} - -func DownloadFile(url string) (string, error) { - client := &http.Client{ - Transport: &http.Transport{ - DialContext: (&net.Dialer{ - Timeout: 5 * time.Second, - KeepAlive: 5 * time.Second, - }).DialContext, - TLSHandshakeTimeout: 5 * time.Second, - ResponseHeaderTimeout: 5 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - MaxIdleConnsPerHost: -1, - }, - } - resp, err := client.Get(url) - if err != nil { - return "", nil - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return "", errors.New(fmt.Sprintf("could not retrieve file, response from server %d for file %s", resp.StatusCode, url)) - } - bodyBytes, err := io.ReadAll(resp.Body) - if err != nil { - return "", nil - } - return string(bodyBytes), nil -} diff --git a/magefiles/utils/controller.go b/magefiles/utils/controller.go deleted file mode 100644 index c8e250a36..000000000 --- a/magefiles/utils/controller.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import ( - "fmt" - - "github.com/google/go-github/v48/github" -) - -// ControllerImage - struct with info about controllers -type ControllerImage struct { - Tag string - Digest string - Registry string - Name string -} - -// IngressRelease All the information about an ingress-nginx release that gets updated -type IngressRelease struct { - ControllerVersion string - ControllerImage ControllerImage - ReleaseNote ReleaseNote - Release *github.RepositoryRelease -} - -// IMAGES_YAML returns this data structure -type ImageYamls []ImageElement - -// ImageElement - a specific image and it's data structure the dmap is a list of shas and container versions -type ImageElement struct { - Name string `json:"name"` - Dmap map[string][]string `json:"dmap"` -} - -func (i ControllerImage) print() string { - return fmt.Sprintf("%s/%s:%s@%s", i.Registry, i.Name, i.Tag, i.Digest) -} - -func FindImageDigest(yaml ImageYamls, image, version string) string { - version = fmt.Sprintf("v%s", version) - Info("Searching Digest for %s:%s", image, version) - for i := range yaml { - if yaml[i].Name == image { - for k, v := range yaml[i].Dmap { - if v[0] == version { - return k - } - } - return "" - } - } - return "" -} diff --git a/magefiles/utils/e2edocs.go b/magefiles/utils/e2edocs.go deleted file mode 100644 index 2f5626db6..000000000 --- a/magefiles/utils/e2edocs.go +++ /dev/null @@ -1,134 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import ( - "bufio" - "bytes" - "embed" - "fmt" - "io/fs" - "os" - "path/filepath" - "regexp" - "slices" - "strings" - "text/template" -) - -//go:embed templates/e2edocs.tpl -var tplContent embed.FS - -var skipFiles = []string{ - "test/e2e/framework/framework.go", - "test/e2e/e2e.go", - "test/e2e/e2e_test.go", -} - -const ( - testDir = "test/e2e" - describe = "Describe" - URL = "https://github.com/kubernetes/ingress-nginx/tree/main/" -) - -var betweenquotes = regexp.MustCompile(`("|\')(?P.*)("|\')`) - -type E2ETemplate struct { - URL string - Tests []string -} - -func getDescription(linetext, path, url string, lineN int, isDescription bool) string { - var descriptionLine string - prefix := "-" - if isDescription { - prefix = "###" - } - - matches := betweenquotes.FindStringSubmatch(linetext) - contentIndex := betweenquotes.SubexpIndex("TestDescription") - if len(matches) < 2 || contentIndex == -1 { - return "" - } - - fileName := fmt.Sprintf("%s/%s", url, path) - descriptionLine = fmt.Sprintf("%s [%s](%s#L%d)", prefix, matches[contentIndex], fileName, lineN) - - return descriptionLine -} - -func containsGinkgoTest(line string) bool { - if !strings.Contains(line, describe) && !strings.Contains(line, "It") { - return false - } - return strings.Contains(line, "func() {") -} - -func (t *E2ETemplate) walkE2eDir(path string, d fs.DirEntry, errAggregated error) error { - if errAggregated != nil { - return errAggregated - } - // Remove ignored files or non .go files - if d.IsDir() || slices.Contains(skipFiles, path) || !strings.HasSuffix(path, ".go") { - return nil - } - - content, err := os.ReadFile(path) - if err != nil { - return err - } - - fileScanner := bufio.NewScanner(bytes.NewReader(content)) - - fileScanner.Split(bufio.ScanLines) - - tests := make([]string, 0) - lineN := 0 - for fileScanner.Scan() { - lineN = lineN + 1 - if !containsGinkgoTest(fileScanner.Text()) { - continue - } - - line := getDescription(fileScanner.Text(), path, t.URL, lineN, strings.Contains(fileScanner.Text(), describe)) - if line != "" { - tests = append(tests, line) - } - } - t.Tests = append(t.Tests, tests...) - return nil -} - -func GenerateE2EDocs() (string, error) { - e2etpl := &E2ETemplate{URL: URL} - err := filepath.WalkDir(testDir, e2etpl.walkE2eDir) - if err != nil { - return "", err - } - - tmpl, err := template.New("e2edocs.tpl").ParseFS(tplContent, "templates/e2edocs.tpl") - if err != nil { - return "", fmt.Errorf("error parsing the template file: %s", err) - } - - tplBuff := new(bytes.Buffer) - err = tmpl.Execute(tplBuff, e2etpl) - if err != nil { - return "", err - } - return tplBuff.String(), nil -} diff --git a/magefiles/utils/helm.go b/magefiles/utils/helm.go deleted file mode 100644 index cb8acae57..000000000 --- a/magefiles/utils/helm.go +++ /dev/null @@ -1,392 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -type IngressChartValue struct { - CommonLabels struct{} `yaml:"commonLabels"` - Controller struct { - Name string `yaml:"name"` - Image struct { - Chroot bool `yaml:"chroot"` - Registry string `yaml:"registry"` - Image string `yaml:"image"` - Tag string `yaml:"tag"` - Digest string `yaml:"digest"` - DigestChroot string `yaml:"digestChroot"` - PullPolicy string `yaml:"pullPolicy"` - RunAsUser int `yaml:"runAsUser"` - AllowPrivilegeEscalation bool `yaml:"allowPrivilegeEscalation"` - } `yaml:"image"` - ContainerName string `yaml:"containerName"` - ContainerPort struct { - HTTP int `yaml:"http"` - HTTPS int `yaml:"https"` - } `yaml:"containerPort"` - Config struct{} `yaml:"config"` - ConfigAnnotations struct{} `yaml:"configAnnotations"` - ProxySetHeaders struct{} `yaml:"proxySetHeaders"` - AddHeaders struct{} `yaml:"addHeaders"` - DNSConfig struct{} `yaml:"dnsConfig"` - Hostname struct{} `yaml:"hostname"` - DNSPolicy string `yaml:"dnsPolicy"` - ReportNodeInternalIP bool `yaml:"reportNodeInternalIp"` - WatchIngressWithoutClass bool `yaml:"watchIngressWithoutClass"` - IngressClassByName bool `yaml:"ingressClassByName"` - AllowSnippetAnnotations bool `yaml:"allowSnippetAnnotations"` - HostNetwork bool `yaml:"hostNetwork"` - HostPort struct { - Enabled bool `yaml:"enabled"` - Ports struct { - HTTP int `yaml:"http"` - HTTPS int `yaml:"https"` - } `yaml:"ports"` - } `yaml:"hostPort"` - ElectionID string `yaml:"electionID"` - IngressClassResource struct { - Name string `yaml:"name"` - Enabled bool `yaml:"enabled"` - Default bool `yaml:"default"` - ControllerValue string `yaml:"controllerValue"` - Parameters struct{} `yaml:"parameters"` - } `yaml:"ingressClassResource"` - IngressClass string `yaml:"ingressClass"` - PodLabels struct{} `yaml:"podLabels"` - PodSecurityContext struct{} `yaml:"podSecurityContext"` - Sysctls struct{} `yaml:"sysctls"` - PublishService struct { - Enabled bool `yaml:"enabled"` - PathOverride string `yaml:"pathOverride"` - } `yaml:"publishService"` - Scope struct { - Enabled bool `yaml:"enabled"` - Namespace string `yaml:"namespace"` - NamespaceSelector string `yaml:"namespaceSelector"` - } `yaml:"scope"` - ConfigMapNamespace string `yaml:"configMapNamespace"` - TCP struct { - ConfigMapNamespace string `yaml:"configMapNamespace"` - Annotations struct{} `yaml:"annotations"` - } `yaml:"tcp"` - UDP struct { - ConfigMapNamespace string `yaml:"configMapNamespace"` - Annotations struct{} `yaml:"annotations"` - } `yaml:"udp"` - MaxmindLicenseKey string `yaml:"maxmindLicenseKey"` - ExtraArgs struct{} `yaml:"extraArgs"` - ExtraEnvs []interface{} `yaml:"extraEnvs"` - Kind string `yaml:"kind"` - Annotations struct{} `yaml:"annotations"` - Labels struct{} `yaml:"labels"` - UpdateStrategy struct{} `yaml:"updateStrategy"` - MinReadySeconds int `yaml:"minReadySeconds"` - Tolerations []interface{} `yaml:"tolerations"` - Affinity struct{} `yaml:"affinity"` - TopologySpreadConstraints []interface{} `yaml:"topologySpreadConstraints"` - TerminationGracePeriodSeconds int `yaml:"terminationGracePeriodSeconds"` - NodeSelector struct { - KubernetesIoOs string `yaml:"kubernetes.io/os"` - } `yaml:"nodeSelector"` - LivenessProbe struct { - HTTPGet struct { - Path string `yaml:"path"` - Port int `yaml:"port"` - Scheme string `yaml:"scheme"` - } `yaml:"httpGet"` - InitialDelaySeconds int `yaml:"initialDelaySeconds"` - PeriodSeconds int `yaml:"periodSeconds"` - TimeoutSeconds int `yaml:"timeoutSeconds"` - SuccessThreshold int `yaml:"successThreshold"` - FailureThreshold int `yaml:"failureThreshold"` - } `yaml:"livenessProbe"` - ReadinessProbe struct { - HTTPGet struct { - Path string `yaml:"path"` - Port int `yaml:"port"` - Scheme string `yaml:"scheme"` - } `yaml:"httpGet"` - InitialDelaySeconds int `yaml:"initialDelaySeconds"` - PeriodSeconds int `yaml:"periodSeconds"` - TimeoutSeconds int `yaml:"timeoutSeconds"` - SuccessThreshold int `yaml:"successThreshold"` - FailureThreshold int `yaml:"failureThreshold"` - } `yaml:"readinessProbe"` - HealthCheckPath string `yaml:"healthCheckPath"` - HealthCheckHost string `yaml:"healthCheckHost"` - PodAnnotations struct{} `yaml:"podAnnotations"` - ReplicaCount int `yaml:"replicaCount"` - MinAvailable int `yaml:"minAvailable"` - Resources struct { - Requests struct { - CPU string `yaml:"cpu"` - Memory string `yaml:"memory"` - } `yaml:"requests"` - } `yaml:"resources"` - Autoscaling struct { - APIVersion string `yaml:"apiVersion"` - Enabled bool `yaml:"enabled"` - Annotations struct{} `yaml:"annotations"` - MinReplicas int `yaml:"minReplicas"` - MaxReplicas int `yaml:"maxReplicas"` - TargetCPUUtilizationPercentage int `yaml:"targetCPUUtilizationPercentage"` - TargetMemoryUtilizationPercentage int `yaml:"targetMemoryUtilizationPercentage"` - Behavior struct{} `yaml:"behavior"` - } `yaml:"autoscaling"` - AutoscalingTemplate []interface{} `yaml:"autoscalingTemplate"` - Keda struct { - APIVersion string `yaml:"apiVersion"` - Enabled bool `yaml:"enabled"` - MinReplicas int `yaml:"minReplicas"` - MaxReplicas int `yaml:"maxReplicas"` - PollingInterval int `yaml:"pollingInterval"` - CooldownPeriod int `yaml:"cooldownPeriod"` - RestoreToOriginalReplicaCount bool `yaml:"restoreToOriginalReplicaCount"` - ScaledObject struct { - Annotations struct{} `yaml:"annotations"` - } `yaml:"scaledObject"` - Triggers []interface{} `yaml:"triggers"` - Behavior struct{} `yaml:"behavior"` - } `yaml:"keda"` - EnableMimalloc bool `yaml:"enableMimalloc"` - CustomTemplate struct { - ConfigMapName string `yaml:"configMapName"` - ConfigMapKey string `yaml:"configMapKey"` - } `yaml:"customTemplate"` - Service struct { - Enabled bool `yaml:"enabled"` - AppProtocol bool `yaml:"appProtocol"` - Annotations struct{} `yaml:"annotations"` - Labels struct{} `yaml:"labels"` - ExternalIPs []interface{} `yaml:"externalIPs"` - LoadBalancerIP string `yaml:"loadBalancerIP"` - LoadBalancerSourceRanges []interface{} `yaml:"loadBalancerSourceRanges"` - EnableHTTP bool `yaml:"enableHttp"` - EnableHTTPS bool `yaml:"enableHttps"` - IPFamilyPolicy string `yaml:"ipFamilyPolicy"` - IPFamilies []string `yaml:"ipFamilies"` - Ports struct { - HTTP int `yaml:"http"` - HTTPS int `yaml:"https"` - } `yaml:"ports"` - TargetPorts struct { - HTTP string `yaml:"http"` - HTTPS string `yaml:"https"` - } `yaml:"targetPorts"` - Type string `yaml:"type"` - NodePorts struct { - HTTP string `yaml:"http"` - HTTPS string `yaml:"https"` - TCP struct{} `yaml:"tcp"` - UDP struct{} `yaml:"udp"` - } `yaml:"nodePorts"` - External struct { - Enabled bool `yaml:"enabled"` - } `yaml:"external"` - Internal struct { - Enabled bool `yaml:"enabled"` - Annotations struct{} `yaml:"annotations"` - LoadBalancerSourceRanges []interface{} `yaml:"loadBalancerSourceRanges"` - } `yaml:"internal"` - } `yaml:"service"` - ShareProcessNamespace bool `yaml:"shareProcessNamespace"` - ExtraContainers []interface{} `yaml:"extraContainers"` - ExtraVolumeMounts []interface{} `yaml:"extraVolumeMounts"` - ExtraVolumes []interface{} `yaml:"extraVolumes"` - ExtraInitContainers []interface{} `yaml:"extraInitContainers"` - ExtraModules []interface{} `yaml:"extraModules"` - AdmissionWebhooks struct { - Annotations struct{} `yaml:"annotations"` - Enabled bool `yaml:"enabled"` - ExtraEnvs []interface{} `yaml:"extraEnvs"` - FailurePolicy string `yaml:"failurePolicy"` - Port int `yaml:"port"` - Certificate string `yaml:"certificate"` - Key string `yaml:"key"` - NamespaceSelector struct{} `yaml:"namespaceSelector"` - ObjectSelector struct{} `yaml:"objectSelector"` - Labels struct{} `yaml:"labels"` - NetworkPolicyEnabled bool `yaml:"networkPolicyEnabled"` - Service struct { - Annotations struct{} `yaml:"annotations"` - ExternalIPs []interface{} `yaml:"externalIPs"` - LoadBalancerSourceRanges []interface{} `yaml:"loadBalancerSourceRanges"` - ServicePort int `yaml:"servicePort"` - Type string `yaml:"type"` - } `yaml:"service"` - CreateSecretJob struct { - SecurityContext struct { - AllowPrivilegeEscalation bool `yaml:"allowPrivilegeEscalation"` - } `yaml:"securityContext"` - Resources struct{} `yaml:"resources"` - } `yaml:"createSecretJob"` - PatchWebhookJob struct { - SecurityContext struct { - AllowPrivilegeEscalation bool `yaml:"allowPrivilegeEscalation"` - } `yaml:"securityContext"` - Resources struct{} `yaml:"resources"` - } `yaml:"patchWebhookJob"` - Patch struct { - Enabled bool `yaml:"enabled"` - Image struct { - Registry string `yaml:"registry"` - Image string `yaml:"image"` - Tag string `yaml:"tag"` - Digest string `yaml:"digest"` - PullPolicy string `yaml:"pullPolicy"` - } `yaml:"image"` - PriorityClassName string `yaml:"priorityClassName"` - PodAnnotations struct{} `yaml:"podAnnotations"` - NodeSelector struct { - KubernetesIoOs string `yaml:"kubernetes.io/os"` - } `yaml:"nodeSelector"` - Tolerations []interface{} `yaml:"tolerations"` - Labels struct{} `yaml:"labels"` - SecurityContext struct { - RunAsNonRoot bool `yaml:"runAsNonRoot"` - RunAsUser int `yaml:"runAsUser"` - FsGroup int `yaml:"fsGroup"` - } `yaml:"securityContext"` - } `yaml:"patch"` - CertManager struct { - Enabled bool `yaml:"enabled"` - RootCert struct { - Duration string `yaml:"duration"` - } `yaml:"rootCert"` - AdmissionCert struct { - Duration string `yaml:"duration"` - } `yaml:"admissionCert"` - } `yaml:"certManager"` - } `yaml:"admissionWebhooks"` - Metrics struct { - Port int `yaml:"port"` - PortName string `yaml:"portName"` - Enabled bool `yaml:"enabled"` - Service struct { - Annotations struct{} `yaml:"annotations"` - ExternalIPs []interface{} `yaml:"externalIPs"` - LoadBalancerSourceRanges []interface{} `yaml:"loadBalancerSourceRanges"` - ServicePort int `yaml:"servicePort"` - Type string `yaml:"type"` - } `yaml:"service"` - ServiceMonitor struct { - Enabled bool `yaml:"enabled"` - AdditionalLabels struct{} `yaml:"additionalLabels"` - Namespace string `yaml:"namespace"` - NamespaceSelector struct{} `yaml:"namespaceSelector"` - ScrapeInterval string `yaml:"scrapeInterval"` - TargetLabels []interface{} `yaml:"targetLabels"` - Relabelings []interface{} `yaml:"relabelings"` - MetricRelabelings []interface{} `yaml:"metricRelabelings"` - } `yaml:"serviceMonitor"` - PrometheusRule struct { - Enabled bool `yaml:"enabled"` - AdditionalLabels struct{} `yaml:"additionalLabels"` - Rules []interface{} `yaml:"rules"` - } `yaml:"prometheusRule"` - } `yaml:"metrics"` - Lifecycle struct { - PreStop struct { - Exec struct { - Command []string `yaml:"command"` - } `yaml:"exec"` - } `yaml:"preStop"` - } `yaml:"lifecycle"` - PriorityClassName string `yaml:"priorityClassName"` - } `yaml:"controller"` - RevisionHistoryLimit int `yaml:"revisionHistoryLimit"` - DefaultBackend struct { - Enabled bool `yaml:"enabled"` - Name string `yaml:"name"` - Image struct { - Registry string `yaml:"registry"` - Image string `yaml:"image"` - Tag string `yaml:"tag"` - PullPolicy string `yaml:"pullPolicy"` - RunAsUser int `yaml:"runAsUser"` - RunAsNonRoot bool `yaml:"runAsNonRoot"` - ReadOnlyRootFilesystem bool `yaml:"readOnlyRootFilesystem"` - AllowPrivilegeEscalation bool `yaml:"allowPrivilegeEscalation"` - } `yaml:"image"` - ExtraArgs struct{} `yaml:"extraArgs"` - ServiceAccount struct { - Create bool `yaml:"create"` - Name string `yaml:"name"` - AutomountServiceAccountToken bool `yaml:"automountServiceAccountToken"` - } `yaml:"serviceAccount"` - ExtraEnvs []interface{} `yaml:"extraEnvs"` - Port int `yaml:"port"` - LivenessProbe struct { - FailureThreshold int `yaml:"failureThreshold"` - InitialDelaySeconds int `yaml:"initialDelaySeconds"` - PeriodSeconds int `yaml:"periodSeconds"` - SuccessThreshold int `yaml:"successThreshold"` - TimeoutSeconds int `yaml:"timeoutSeconds"` - } `yaml:"livenessProbe"` - ReadinessProbe struct { - FailureThreshold int `yaml:"failureThreshold"` - InitialDelaySeconds int `yaml:"initialDelaySeconds"` - PeriodSeconds int `yaml:"periodSeconds"` - SuccessThreshold int `yaml:"successThreshold"` - TimeoutSeconds int `yaml:"timeoutSeconds"` - } `yaml:"readinessProbe"` - Tolerations []interface{} `yaml:"tolerations"` - Affinity struct{} `yaml:"affinity"` - PodSecurityContext struct{} `yaml:"podSecurityContext"` - ContainerSecurityContext struct{} `yaml:"containerSecurityContext"` - PodLabels struct{} `yaml:"podLabels"` - NodeSelector struct { - KubernetesIoOs string `yaml:"kubernetes.io/os"` - } `yaml:"nodeSelector"` - PodAnnotations struct{} `yaml:"podAnnotations"` - ReplicaCount int `yaml:"replicaCount"` - MinAvailable int `yaml:"minAvailable"` - Resources struct{} `yaml:"resources"` - ExtraVolumeMounts []interface{} `yaml:"extraVolumeMounts"` - ExtraVolumes []interface{} `yaml:"extraVolumes"` - Autoscaling struct { - Annotations struct{} `yaml:"annotations"` - Enabled bool `yaml:"enabled"` - MinReplicas int `yaml:"minReplicas"` - MaxReplicas int `yaml:"maxReplicas"` - TargetCPUUtilizationPercentage int `yaml:"targetCPUUtilizationPercentage"` - TargetMemoryUtilizationPercentage int `yaml:"targetMemoryUtilizationPercentage"` - } `yaml:"autoscaling"` - Service struct { - Annotations struct{} `yaml:"annotations"` - ExternalIPs []interface{} `yaml:"externalIPs"` - LoadBalancerSourceRanges []interface{} `yaml:"loadBalancerSourceRanges"` - ServicePort int `yaml:"servicePort"` - Type string `yaml:"type"` - } `yaml:"service"` - PriorityClassName string `yaml:"priorityClassName"` - Labels struct{} `yaml:"labels"` - } `yaml:"defaultBackend"` - Rbac struct { - Create bool `yaml:"create"` - Scope bool `yaml:"scope"` - } `yaml:"rbac"` - ServiceAccount struct { - Create bool `yaml:"create"` - Name string `yaml:"name"` - AutomountServiceAccountToken bool `yaml:"automountServiceAccountToken"` - Annotations struct{} `yaml:"annotations"` - } `yaml:"serviceAccount"` - ImagePullSecrets []interface{} `yaml:"imagePullSecrets"` - TCP struct{} `yaml:"tcp"` - UDP struct{} `yaml:"udp"` - PortNamePrefix string `yaml:"portNamePrefix"` - DhParam interface{} `yaml:"dhParam"` -} diff --git a/magefiles/utils/releasenote.go b/magefiles/utils/releasenote.go deleted file mode 100644 index 341595444..000000000 --- a/magefiles/utils/releasenote.go +++ /dev/null @@ -1,91 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import ( - "fmt" - "os" - "text/template" -) - -// ReleaseNote - All the pieces of information/documents that get updated during a release -type ReleaseNote struct { - Version string - NewControllerVersion string - PreviousControllerVersion string - ControllerImages []ControllerImage - DepUpdates []string - Updates []string - HelmUpdates []string - NewHelmChartVersion string - PreviousHelmChartVersion string -} - -func (r ReleaseNote) Template() { - // Files are provided as a slice of strings. - changelogTemplate, err := os.ReadFile("changelog/controller.md.gotmpl") - if err != nil { - ErrorF("Could not read changelog template file %s", err) - } - Debug("ChangeLog Templates %s", string(changelogTemplate)) - t := template.Must(template.New("changelog").Parse(string(changelogTemplate))) - // create a new file - file, err := os.Create(fmt.Sprintf("changelog/controller-%s.md", r.Version)) - if err != nil { - ErrorF("Could not create changelog file %s", err) - } - defer file.Close() - - err = t.Execute(file, r) - if err != nil { - ErrorF("executing template: %s", err) - } -} - -func (r ReleaseNote) HelmTemplate() { - // Files are provided as a slice of strings. - changelogTemplate, err := os.ReadFile("charts/ingress-nginx/changelog/helm-chart.md.gotmpl") - if err != nil { - ErrorF("Could not read changelog template file %s", err) - } - Debug("ChangeLog Templates %s", string(changelogTemplate)) - t := template.Must(template.New("changelog").Parse(string(changelogTemplate))) - // create a new file - file, err := os.Create(fmt.Sprintf("charts/ingress-nginx/changelog/helm-chart-%s.md", r.NewHelmChartVersion)) - if err != nil { - ErrorF("Could not create changelog file %s", err) - } - defer file.Close() - - err = t.Execute(file, r) - if err != nil { - ErrorF("executing template: %s", err) - } -} - -func (r ReleaseNote) PrintRelease() { - Info("Release Version: %v", r.NewControllerVersion) - Info("Previous Version: %v", r.PreviousControllerVersion) - Info("Controller Image: %v", r.ControllerImages[0].print()) - Info("Controller Chroot Image: %v", r.ControllerImages[1].print()) - for i := range r.Updates { - Info("Update #%v - %v", i, r.Updates[i]) - } - for j := range r.DepUpdates { - Info("Dependabot Update #%v - %v", j, r.DepUpdates[j]) - } -} diff --git a/magefiles/utils/templates/e2edocs.tpl b/magefiles/utils/templates/e2edocs.tpl deleted file mode 100644 index 8757c10bd..000000000 --- a/magefiles/utils/templates/e2edocs.tpl +++ /dev/null @@ -1,10 +0,0 @@ - - -# e2e test suite for [Ingress NGINX Controller]({{.URL}}) - -{{ range $test := .Tests }} -{{ $test }} -{{- end }} \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index b59546c9c..e15e015dc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,8 +1,7 @@ -site_name: Ingress-Nginx Controller +site_name: NGINX Ingress Controller repo_name: "kubernetes/ingress-nginx" repo_url: https://github.com/kubernetes/ingress-nginx site_url: https://kubernetes.github.io/ingress-nginx -edit_uri: edit/main/docs/ # Extensions markdown_extensions: @@ -50,7 +49,6 @@ theme: - navigation.tabs.sticky - navigation.instant - navigation.sections - - content.code.copy palette: primary: "teal" @@ -66,6 +64,7 @@ plugins: extra_css: [extra.css] +google_analytics: ["UA-118407822-1", "kubernetes.github.io"] nav: - Welcome: @@ -79,12 +78,11 @@ nav: - Role Based Access Control (RBAC): "deploy/rbac.md" - Upgrade: "deploy/upgrade.md" - Hardening guide: "deploy/hardening-guide.md" - - User Guide: + - User guide: - NGINX Configuration: - Introduction: "user-guide/nginx-configuration/index.md" - Basic usage: "user-guide/basic-usage.md" - Annotations: "user-guide/nginx-configuration/annotations.md" - - Annotations Risks: "user-guide/nginx-configuration/annotations-risk.md" - ConfigMap: "user-guide/nginx-configuration/configmap.md" - Custom NGINX template: "user-guide/nginx-configuration/custom-template.md" - Log format: "user-guide/nginx-configuration/log-format.md" @@ -101,7 +99,7 @@ nav: - TLS/HTTPS: "user-guide/tls.md" - Third party addons: - ModSecurity Web Application Firewall: "user-guide/third-party-addons/modsecurity.md" - - OpenTelemetry: "user-guide/third-party-addons/opentelemetry.md" + - OpenTracing: "user-guide/third-party-addons/opentracing.md" - Examples: - Introduction: "examples/index.md" - Prerequisites: "examples/PREREQUISITES.md" @@ -125,9 +123,7 @@ nav: - Rewrite: "examples/rewrite/README.md" - Static IPs: "examples/static-ip/README.md" - TLS termination: "examples/tls-termination/README.md" - - Open Policy Agent rules: "examples/openpolicyagent/README.md" - - Canary Deployments: "examples/canary/README.md" + - Pod Security Policy (PSP): "examples/psp/README.md" - Developer Guide: - Getting Started: "developer-guide/getting-started.md" - Code Overview: "developer-guide/code-overview.md" - - FAQ: "faq.md" diff --git a/netlify.toml b/netlify.toml deleted file mode 100644 index dc4b0d1ca..000000000 --- a/netlify.toml +++ /dev/null @@ -1,11 +0,0 @@ -# netlify configuration -[build] -publish = "site" -command = "make build-docs" -ignore = "git diff --quiet $CACHED_COMMIT_REF $COMMIT_REF ./docs" -# available here https://github.com/netlify/build-image/blob/focal/included_software.md#languages -environment = { PYTHON_VERSION = "3.8" } - -[context.deploy-preview] - publish = "site/" - command = "make build-docs" diff --git a/pkg/flags/flags_test.go b/pkg/flags/flags_test.go deleted file mode 100644 index fdf153021..000000000 --- a/pkg/flags/flags_test.go +++ /dev/null @@ -1,240 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package flags - -import ( - "os" - "testing" - "time" -) - -func TestNoMandatoryFlag(t *testing.T) { - _, _, err := ParseFlags() - if err != nil { - t.Fatalf("Expected no error but got: %s", err) - } -} - -func TestDefaults(t *testing.T) { - ResetForTesting(func() { t.Fatal("Parsing failed") }) - - oldArgs := os.Args - defer func() { os.Args = oldArgs }() - os.Args = []string{ - "cmd", - "--default-backend-service", "namespace/test", - "--http-port", "0", - "--https-port", "0", - } - - showVersion, conf, err := ParseFlags() - if err != nil { - t.Fatalf("Unexpected error parsing default flags: %v", err) - } - - if showVersion { - t.Fatal("Expected flag \"show-version\" to be false") - } - - if conf == nil { - t.Fatal("Expected a controller Configuration") - } -} - -func TestSetupSSLProxy(_ *testing.T) { - // TODO TestSetupSSLProxy -} - -func TestFlagConflict(t *testing.T) { - ResetForTesting(func() { t.Fatal("Parsing failed") }) - - oldArgs := os.Args - defer func() { os.Args = oldArgs }() - os.Args = []string{"cmd", "--publish-service", "namespace/test", "--http-port", "0", "--https-port", "0", "--publish-status-address", "1.1.1.1"} - - _, _, err := ParseFlags() - if err == nil { - t.Fatalf("Expected an error parsing flags but none returned") - } -} - -func TestMaxmindEdition(t *testing.T) { - ResetForTesting(func() { t.Fatal("Parsing failed") }) - - oldArgs := os.Args - defer func() { os.Args = oldArgs }() - os.Args = []string{"cmd", "--publish-service", "namespace/test", "--http-port", "0", "--https-port", "0", "--maxmind-license-key", "0000000", "--maxmind-edition-ids", "GeoLite2-City, TestCheck"} - - _, _, err := ParseFlags() - if err == nil { - t.Fatalf("Expected an error parsing flags but none returned") - } -} - -func TestMaxmindMirror(t *testing.T) { - ResetForTesting(func() { t.Fatal("Parsing failed") }) - - oldArgs := os.Args - defer func() { os.Args = oldArgs }() - os.Args = []string{"cmd", "--publish-service", "namespace/test", "--http-port", "0", "--https-port", "0", "--maxmind-mirror", "http://geoip.local", "--maxmind-license-key", "0000000", "--maxmind-edition-ids", "GeoLite2-City, TestCheck"} - - _, _, err := ParseFlags() - if err == nil { - t.Fatalf("Expected an error parsing flags but none returned") - } -} - -func TestMaxmindRetryDownload(t *testing.T) { - ResetForTesting(func() { t.Fatal("Parsing failed") }) - - oldArgs := os.Args - defer func() { os.Args = oldArgs }() - os.Args = []string{"cmd", "--publish-service", "namespace/test", "--http-port", "0", "--https-port", "0", "--maxmind-mirror", "http://127.0.0.1", "--maxmind-license-key", "0000000", "--maxmind-edition-ids", "GeoLite2-City", "--maxmind-retries-timeout", "1s", "--maxmind-retries-count", "3"} - - _, _, err := ParseFlags() - if err == nil { - t.Fatalf("Expected an error parsing flags but none returned") - } -} - -func TestDisableLeaderElectionFlag(t *testing.T) { - ResetForTesting(func() { t.Fatal("Parsing failed") }) - - oldArgs := os.Args - defer func() { os.Args = oldArgs }() - os.Args = []string{"cmd", "--disable-leader-election", "--http-port", "80", "--https-port", "443"} - - _, conf, err := ParseFlags() - if err != nil { - t.Fatalf("Unexpected error parsing default flags: %v", err) - } - - if !conf.DisableLeaderElection { - t.Fatalf("Expected --disable-leader-election and conf.DisableLeaderElection as true, but found: %v", conf.DisableLeaderElection) - } -} - -func TestIfLeaderElectionDisabledFlagIsFalse(t *testing.T) { - ResetForTesting(func() { t.Fatal("Parsing failed") }) - - oldArgs := os.Args - defer func() { os.Args = oldArgs }() - os.Args = []string{"cmd", "--http-port", "80", "--https-port", "443"} - - _, conf, err := ParseFlags() - if err != nil { - t.Fatalf("Unexpected error parsing default flags: %v", err) - } - - if conf.DisableLeaderElection { - t.Fatalf("Expected --disable-leader-election and conf.DisableLeaderElection as false, but found: %v", conf.DisableLeaderElection) - } -} - -func TestLeaderElectionTTLDefaultValue(t *testing.T) { - ResetForTesting(func() { t.Fatal("Parsing failed") }) - - oldArgs := os.Args - defer func() { os.Args = oldArgs }() - os.Args = []string{"cmd", "--http-port", "80", "--https-port", "443"} - - _, conf, err := ParseFlags() - if err != nil { - t.Fatalf("Unexpected error parsing default flags: %v", err) - } - - if conf.ElectionTTL != 30*time.Second { - t.Fatalf("Expected --election-ttl and conf.ElectionTTL as 30s, but found: %v", conf.ElectionTTL) - } -} - -func TestLeaderElectionTTLParseValueInSeconds(t *testing.T) { - ResetForTesting(func() { t.Fatal("Parsing failed") }) - - oldArgs := os.Args - defer func() { os.Args = oldArgs }() - os.Args = []string{"cmd", "--http-port", "80", "--https-port", "443", "--election-ttl", "10s"} - - _, conf, err := ParseFlags() - if err != nil { - t.Fatalf("Unexpected error parsing default flags: %v", err) - } - - if conf.ElectionTTL != 10*time.Second { - t.Fatalf("Expected --election-ttl and conf.ElectionTTL as 10s, but found: %v", conf.ElectionTTL) - } -} - -func TestLeaderElectionTTLParseValueInMinutes(t *testing.T) { - ResetForTesting(func() { t.Fatal("Parsing failed") }) - - oldArgs := os.Args - defer func() { os.Args = oldArgs }() - os.Args = []string{"cmd", "--http-port", "80", "--https-port", "443", "--election-ttl", "10m"} - - _, conf, err := ParseFlags() - if err != nil { - t.Fatalf("Unexpected error parsing default flags: %v", err) - } - - if conf.ElectionTTL != 10*time.Minute { - t.Fatalf("Expected --election-ttl and conf.ElectionTTL as 10m, but found: %v", conf.ElectionTTL) - } -} - -func TestLeaderElectionTTLParseValueInHours(t *testing.T) { - ResetForTesting(func() { t.Fatal("Parsing failed") }) - - oldArgs := os.Args - defer func() { os.Args = oldArgs }() - os.Args = []string{"cmd", "--http-port", "80", "--https-port", "443", "--election-ttl", "1h"} - - _, conf, err := ParseFlags() - if err != nil { - t.Fatalf("Unexpected error parsing default flags: %v", err) - } - - if conf.ElectionTTL != 1*time.Hour { - t.Fatalf("Expected --election-ttl and conf.ElectionTTL as 1h, but found: %v", conf.ElectionTTL) - } -} - -func TestMetricsPerUndefinedHost(t *testing.T) { - ResetForTesting(func() { t.Fatal("Parsing failed") }) - - oldArgs := os.Args - defer func() { os.Args = oldArgs }() - os.Args = []string{"cmd", "--metrics-per-undefined-host=true"} - - _, _, err := ParseFlags() - if err != nil { - t.Fatalf("Expected no error but got: %s", err) - } -} - -func TestMetricsPerUndefinedHostWithMetricsPerHostFalse(t *testing.T) { - ResetForTesting(func() { t.Fatal("Parsing failed") }) - - oldArgs := os.Args - defer func() { os.Args = oldArgs }() - os.Args = []string{"cmd", "--metrics-per-host=false", "--metrics-per-undefined-host=true"} - - _, _, err := ParseFlags() - if err == nil { - t.Fatalf("Expected an error parsing flags but none returned") - } -} diff --git a/pkg/metrics/handler.go b/pkg/metrics/handler.go deleted file mode 100644 index 73c7d328f..000000000 --- a/pkg/metrics/handler.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metrics - -import ( - "fmt" - "net/http" - "net/http/pprof" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - "k8s.io/apiserver/pkg/server/healthz" - klog "k8s.io/klog/v2" -) - -func RegisterHealthz(healthPath string, mux *http.ServeMux, checks ...healthz.HealthChecker) { - healthCheck := []healthz.HealthChecker{healthz.PingHealthz} - if len(checks) > 0 { - healthCheck = append(healthCheck, checks...) - } - // expose health check endpoint (/healthz) - healthz.InstallPathHandler(mux, - healthPath, - healthCheck..., - ) -} - -func RegisterMetrics(reg *prometheus.Registry, mux *http.ServeMux) { - mux.Handle( - "/metrics", - promhttp.InstrumentMetricHandler( - reg, - promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), - ), - ) -} - -func RegisterProfiler(host string, port int) { - mux := http.NewServeMux() - - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/heap", pprof.Index) - mux.HandleFunc("/debug/pprof/mutex", pprof.Index) - mux.HandleFunc("/debug/pprof/goroutine", pprof.Index) - mux.HandleFunc("/debug/pprof/threadcreate", pprof.Index) - mux.HandleFunc("/debug/pprof/block", pprof.Index) - mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - - server := &http.Server{ - Addr: fmt.Sprintf("%s:%d", host, port), - // G112 (CWE-400): Potential Slowloris Attack - ReadHeaderTimeout: 10 * time.Second, - Handler: mux, - } - klog.Fatal(server.ListenAndServe()) -} - -func StartHTTPServer(host string, port int, mux *http.ServeMux) { - server := &http.Server{ - Addr: fmt.Sprintf("%s:%v", host, port), - Handler: mux, - ReadTimeout: 10 * time.Second, - ReadHeaderTimeout: 10 * time.Second, - WriteTimeout: 300 * time.Second, - IdleTimeout: 120 * time.Second, - } - klog.Fatal(server.ListenAndServe()) -} diff --git a/pkg/util/ingress/ingress.go b/pkg/util/ingress/ingress.go deleted file mode 100644 index 881d5a001..000000000 --- a/pkg/util/ingress/ingress.go +++ /dev/null @@ -1,233 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ingress - -import ( - "fmt" - "strings" - - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/ingress-nginx/internal/k8s" - "k8s.io/ingress-nginx/internal/net/ssl" - "k8s.io/ingress-nginx/pkg/apis/ingress" - "k8s.io/klog/v2" -) - -func GetRemovedHosts(rucfg, newcfg *ingress.Configuration) []string { - oldSet := sets.NewString() - newSet := sets.NewString() - - for _, s := range rucfg.Servers { - if !oldSet.Has(s.Hostname) { - oldSet.Insert(s.Hostname) - } - } - - for _, s := range newcfg.Servers { - if !newSet.Has(s.Hostname) { - newSet.Insert(s.Hostname) - } - } - - return oldSet.Difference(newSet).List() -} - -// GetRemovedCertificateSerialNumbers extracts the difference of certificates between two configurations -func GetRemovedCertificateSerialNumbers(rucfg, newcfg *ingress.Configuration) []string { - oldCertificates := sets.NewString() - newCertificates := sets.NewString() - - for _, server := range rucfg.Servers { - if server.SSLCert == nil { - continue - } - identifier := server.SSLCert.Identifier() - if identifier != "" { - if !oldCertificates.Has(identifier) { - oldCertificates.Insert(identifier) - } - } - } - - for _, server := range newcfg.Servers { - if server.SSLCert == nil { - continue - } - identifier := server.SSLCert.Identifier() - if identifier != "" { - if !newCertificates.Has(identifier) { - newCertificates.Insert(identifier) - } - } - } - - return oldCertificates.Difference(newCertificates).List() -} - -// GetRemovedIngresses extracts the difference of ingresses between two configurations -func GetRemovedIngresses(rucfg, newcfg *ingress.Configuration) []string { - oldIngresses := sets.NewString() - newIngresses := sets.NewString() - - for _, server := range rucfg.Servers { - for _, location := range server.Locations { - if location.Ingress == nil { - continue - } - - ingKey := k8s.MetaNamespaceKey(location.Ingress) - if !oldIngresses.Has(ingKey) { - oldIngresses.Insert(ingKey) - } - } - } - - for _, server := range newcfg.Servers { - for _, location := range server.Locations { - if location.Ingress == nil { - continue - } - - ingKey := k8s.MetaNamespaceKey(location.Ingress) - if !newIngresses.Has(ingKey) { - newIngresses.Insert(ingKey) - } - } - } - - return oldIngresses.Difference(newIngresses).List() -} - -// IsDynamicConfigurationEnough returns whether a Configuration can be -// dynamically applied, without reloading the backend. -func IsDynamicConfigurationEnough(newcfg, oldcfg *ingress.Configuration) bool { - copyOfRunningConfig := *oldcfg - copyOfPcfg := *newcfg - - copyOfRunningConfig.Backends = []*ingress.Backend{} - copyOfPcfg.Backends = []*ingress.Backend{} - - clearL4serviceEndpoints(©OfRunningConfig) - clearL4serviceEndpoints(©OfPcfg) - - clearCertificates(©OfRunningConfig) - clearCertificates(©OfPcfg) - - return copyOfRunningConfig.Equal(©OfPcfg) -} - -// clearL4serviceEndpoints is a helper function to clear endpoints from the ingress configuration since they should be ignored when -// checking if the new configuration changes can be applied dynamically. -func clearL4serviceEndpoints(config *ingress.Configuration) { - clearedTCPL4Services := make([]ingress.L4Service, 0, len(config.TCPEndpoints)) - clearedUDPL4Services := make([]ingress.L4Service, 0, len(config.UDPEndpoints)) - for i := range config.TCPEndpoints { - copyofService := ingress.L4Service{ - Port: config.TCPEndpoints[i].Port, - Backend: config.TCPEndpoints[i].Backend, - Endpoints: []ingress.Endpoint{}, - Service: nil, - } - clearedTCPL4Services = append(clearedTCPL4Services, copyofService) - } - for i := range config.UDPEndpoints { - copyofService := ingress.L4Service{ - Port: config.UDPEndpoints[i].Port, - Backend: config.UDPEndpoints[i].Backend, - Endpoints: []ingress.Endpoint{}, - Service: nil, - } - clearedUDPL4Services = append(clearedUDPL4Services, copyofService) - } - config.TCPEndpoints = clearedTCPL4Services - config.UDPEndpoints = clearedUDPL4Services -} - -// clearCertificates is a helper function to clear Certificates from the ingress configuration since they should be ignored when -// checking if the new configuration changes can be applied dynamically if dynamic certificates is on -func clearCertificates(config *ingress.Configuration) { - clearedServers := make([]*ingress.Server, 0, len(config.Servers)) - for _, server := range config.Servers { - copyOfServer := *server - copyOfServer.SSLCert = nil - clearedServers = append(clearedServers, ©OfServer) - } - config.Servers = clearedServers -} - -type Redirect struct { - From string - To string - SSLCert *ingress.SSLCert -} - -// BuildRedirects build the redirects of servers based on configurations and certificates -func BuildRedirects(servers []*ingress.Server) []*Redirect { - names := sets.Set[string]{} - redirectServers := make([]*Redirect, 0) - - for _, srv := range servers { - if !srv.RedirectFromToWWW { - continue - } - - to := srv.Hostname - - var from string - if strings.HasPrefix(to, "www.") { - from = strings.TrimPrefix(to, "www.") - } else { - from = fmt.Sprintf("www.%v", to) - } - - if names.Has(to) { - continue - } - - klog.V(3).InfoS("Creating redirect", "from", from, "to", to) - found := false - for _, esrv := range servers { - if esrv.Hostname == from { - found = true - break - } - } - - if found { - klog.Warningf("Already exists an Ingress with %q hostname. Skipping creation of redirection from %q to %q.", from, from, to) - continue - } - - r := &Redirect{ - From: from, - To: to, - } - - if srv.SSLCert != nil { - if ssl.IsValidHostname(from, srv.SSLCert.CN) { - r.SSLCert = srv.SSLCert - } else { - klog.Warningf("the server %v has SSL configured but the SSL certificate does not contains a CN for %v. Redirects will not work for HTTPS to HTTPS", from, to) - } - } - - redirectServers = append(redirectServers, r) - names.Insert(to) - } - - return redirectServers -} diff --git a/pkg/util/ingress/ingress_test.go b/pkg/util/ingress/ingress_test.go deleted file mode 100644 index 24597fb6e..000000000 --- a/pkg/util/ingress/ingress_test.go +++ /dev/null @@ -1,132 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ingress - -import ( - "testing" - - "k8s.io/ingress-nginx/pkg/apis/ingress" -) - -func TestIsDynamicConfigurationEnough(t *testing.T) { - backends := []*ingress.Backend{{ - Name: "fakenamespace-myapp-80", - Endpoints: []ingress.Endpoint{ - { - Address: "10.0.0.1", - Port: "8080", - }, - { - Address: "10.0.0.2", - Port: "8080", - }, - }, - }} - - servers := []*ingress.Server{{ - Hostname: "myapp.fake", - Locations: []*ingress.Location{ - { - Path: "/", - Backend: "fakenamespace-myapp-80", - }, - }, - SSLCert: &ingress.SSLCert{ - PemCertKey: "fake-certificate", - }, - }} - - commonConfig := &ingress.Configuration{ - Backends: backends, - Servers: servers, - } - - runningConfig := &ingress.Configuration{ - Backends: backends, - Servers: servers, - } - - newConfig := commonConfig - if !IsDynamicConfigurationEnough(newConfig, runningConfig) { - t.Errorf("When new config is same as the running config it should be deemed as dynamically configurable") - } - - newConfig = &ingress.Configuration{ - Backends: []*ingress.Backend{{Name: "another-backend-8081"}}, - Servers: []*ingress.Server{{Hostname: "myapp1.fake"}}, - } - if IsDynamicConfigurationEnough(newConfig, runningConfig) { - t.Errorf("Expected to not be dynamically configurable when there's more than just backends change") - } - - newConfig = &ingress.Configuration{ - Backends: []*ingress.Backend{{Name: "a-backend-8080"}}, - Servers: servers, - } - - if !IsDynamicConfigurationEnough(newConfig, runningConfig) { - t.Errorf("Expected to be dynamically configurable when only backends change") - } - - newServers := []*ingress.Server{{ - Hostname: "myapp1.fake", - Locations: []*ingress.Location{ - { - Path: "/", - Backend: "fakenamespace-myapp-80", - }, - }, - SSLCert: &ingress.SSLCert{ - PemCertKey: "fake-certificate", - }, - }} - - newConfig = &ingress.Configuration{ - Backends: backends, - Servers: newServers, - } - if IsDynamicConfigurationEnough(newConfig, runningConfig) { - t.Errorf("Expected to not be dynamically configurable when dynamic certificates is enabled and a non-certificate field in servers is updated") - } - - newServers[0].Hostname = "myapp.fake" - newServers[0].SSLCert.PemCertKey = "new-fake-certificate" - - newConfig = &ingress.Configuration{ - Backends: backends, - Servers: newServers, - } - if !IsDynamicConfigurationEnough(newConfig, runningConfig) { - t.Errorf("Expected to be dynamically configurable when only SSLCert changes") - } - - newConfig = &ingress.Configuration{ - Backends: []*ingress.Backend{{Name: "a-backend-8080"}}, - Servers: newServers, - } - if !IsDynamicConfigurationEnough(newConfig, runningConfig) { - t.Errorf("Expected to be dynamically configurable when backend and SSLCert changes") - } - - if !runningConfig.Equal(commonConfig) { - t.Errorf("Expected running config to not change") - } - - if !newConfig.Equal(&ingress.Configuration{Backends: []*ingress.Backend{{Name: "a-backend-8080"}}, Servers: newServers}) { - t.Errorf("Expected new config to not change") - } -} diff --git a/pkg/util/process/controller.go b/pkg/util/process/controller.go deleted file mode 100644 index a73e81934..000000000 --- a/pkg/util/process/controller.go +++ /dev/null @@ -1,24 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package process - -// Controller defines a common interface for a process to be controlled, -// like the configurer, the webhook or the proper ingress controller -type Controller interface { - Start() - Stop() error -} diff --git a/pkg/util/process/sigterm.go b/pkg/util/process/sigterm.go deleted file mode 100644 index 1c0d729c1..000000000 --- a/pkg/util/process/sigterm.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package process - -import ( - "os" - "os/signal" - "syscall" - "time" - - klog "k8s.io/klog/v2" -) - -type exiter func(code int) - -// HandleSigterm receives a ProcessController interface and deals with -// the graceful shutdown -func HandleSigterm(ngx Controller, delay int, exit exiter) { - signalChan := make(chan os.Signal, 1) - signal.Notify(signalChan, syscall.SIGTERM) - <-signalChan - klog.InfoS("Received SIGTERM, shutting down") - - exitCode := 0 - if err := ngx.Stop(); err != nil { - klog.Warningf("Error during shutdown: %v", err) - exitCode = 1 - } - - klog.Infof("Handled quit, delaying controller exit for %d seconds", delay) - time.Sleep(time.Duration(delay) * time.Second) - - klog.InfoS("Exiting", "code", exitCode) - exit(exitCode) -} diff --git a/pkg/util/process/sigterm_test.go b/pkg/util/process/sigterm_test.go deleted file mode 100644 index 08c8275c3..000000000 --- a/pkg/util/process/sigterm_test.go +++ /dev/null @@ -1,84 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package process - -import ( - "fmt" - "syscall" - "testing" - "time" -) - -type FakeProcess struct { - shouldError bool - exitCode int -} - -func (f *FakeProcess) Start() { -} - -func (f *FakeProcess) Stop() error { - if f.shouldError { - return fmt.Errorf("error") - } - return nil -} - -func (f *FakeProcess) exiterFunc(code int) { - f.exitCode = code -} - -func sendDelayedSignal(delay time.Duration) error { - time.Sleep(delay) - return syscall.Kill(syscall.Getpid(), syscall.SIGTERM) -} - -func TestHandleSigterm(t *testing.T) { - tests := []struct { - name string - shouldError bool - delay int - }{ - { - name: "should exit without error", - shouldError: false, - }, - { - name: "should exit with error", - shouldError: true, - delay: 2, - }, - } - for _, tt := range tests { - process := &FakeProcess{shouldError: tt.shouldError} - t.Run(tt.name, func(t *testing.T) { - go func() { - err := sendDelayedSignal(2 * time.Second) // Send a signal after 2 seconds - if err != nil { - t.Errorf("error sending delayed signal: %v", err) - } - }() - HandleSigterm(process, tt.delay, process.exiterFunc) - }) - if tt.shouldError && process.exitCode != 1 { - t.Errorf("wrong return, should be 1 and returned %d", process.exitCode) - } - if !tt.shouldError && process.exitCode != 0 { - t.Errorf("wrong return, should be 0 and returned %d", process.exitCode) - } - } -} diff --git a/pkg/util/runtime/cpu_linux.go b/pkg/util/runtime/cpu_linux.go deleted file mode 100644 index 7db609053..000000000 --- a/pkg/util/runtime/cpu_linux.go +++ /dev/null @@ -1,145 +0,0 @@ -//go:build linux -// +build linux - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package runtime - -import ( - "math" - "os" - "path/filepath" - "runtime" - "strconv" - "strings" - - libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups" -) - -// NumCPU returns the number of logical CPUs usable by the current process. -// If CPU cgroups limits are configured, use cfs_quota_us / cfs_period_us -// as formula -// -// https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt -func NumCPU() int { - return NumCPUWithCustomPath("") -} - -func NumCPUWithCustomPath(path string) int { - cpus := runtime.NumCPU() - - cgroupVersionCheckPath := path - - if cgroupVersionCheckPath == "" { - cgroupVersionCheckPath = "/sys/fs/cgroup/" - } - - cgroupVersion := GetCgroupVersion(cgroupVersionCheckPath) - cpuQuota := int64(-1) - cpuPeriod := int64(-1) - - if cgroupVersion == 1 { - cgroupPath := "" - if path == "" { - cgroupPathRd, err := libcontainercgroups.FindCgroupMountpoint("", "cpu") - if err != nil { - return cpus - } - cgroupPath = cgroupPathRd - } else { - cgroupPath = path - } - cpuQuota = readCgroupFileToInt64(cgroupPath, "cpu.cfs_quota_us") - cpuPeriod = readCgroupFileToInt64(cgroupPath, "cpu.cfs_period_us") - } else if cgroupVersion == 2 { - cgroupPath := "/sys/fs/cgroup/" - if path != "" { - cgroupPath = path - } - cpuQuota, cpuPeriod = readCgroup2FileToInt64Tuple(cgroupPath, "cpu.max") - } - - if cpuQuota == -1 || cpuPeriod == -1 { - return cpus - } - - return int(math.Ceil(float64(cpuQuota) / float64(cpuPeriod))) -} - -func GetCgroupVersion(cgroupPath string) int64 { - // /sys/fs/cgroup/cgroup.controllers will not exist with cgroupsv1 - if _, err := os.Stat(filepath.Join(cgroupPath, "cgroup.controllers")); err == nil { - return 2 - } - - return 1 -} - -func readCgroup2StringToInt64Tuple(cgroupString string) (quota, period int64) { - // file contents looks like: $MAX $PERIOD - // $MAX can have value "max" indicating no limit - // it is possible for $PERIOD to be unset - - values := strings.Fields(cgroupString) - - if values[0] == "max" { - return -1, -1 - } - - cpuQuota, err := strconv.ParseInt(values[0], 10, 64) - if err != nil { - return -1, -1 - } - - if len(values) == 1 { - return cpuQuota, 100000 - } - - cpuPeriod, err := strconv.ParseInt(values[1], 10, 64) - if err != nil { - return -1, -1 - } - - return cpuQuota, cpuPeriod -} - -func readCgroup2FileToInt64Tuple(cgroupPath, cgroupFile string) (quota, period int64) { - contents, err := os.ReadFile(filepath.Join(cgroupPath, cgroupFile)) - if err != nil { - return -1, -1 - } - - return readCgroup2StringToInt64Tuple(string(contents)) -} - -func readCgroupStringToInt64(contents string) int64 { - strValue := strings.TrimSpace(contents) - if value, err := strconv.ParseInt(strValue, 10, 64); err == nil { - return value - } - - return -1 -} - -func readCgroupFileToInt64(cgroupPath, cgroupFile string) int64 { - contents, err := os.ReadFile(filepath.Join(cgroupPath, cgroupFile)) - if err != nil { - return -1 - } - - return readCgroupStringToInt64(string(contents)) -} diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 9b7753b5d..49abbdac2 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -40,6 +40,7 @@ RUN apk update \ && rm -rf /var/cache/apk/* COPY --chown=www-data:www-data etc /etc +COPY --chown=www-data:www-data ingress-controller /ingress-controller COPY --chown=www-data:www-data bin/${TARGETARCH}/dbg / COPY --chown=www-data:www-data bin/${TARGETARCH}/nginx-ingress-controller / @@ -49,33 +50,23 @@ COPY --chown=www-data:www-data bin/${TARGETARCH}/wait-shutdown / # with volumes (custom templates) RUN bash -xeu -c ' \ writeDirs=( \ + /etc/ingress-controller \ /etc/ingress-controller/ssl \ /etc/ingress-controller/auth \ - /etc/ingress-controller/geoip \ - /etc/ingress-controller/telemetry \ /var/log \ /var/log/nginx \ - /tmp/nginx \ ); \ for dir in "${writeDirs[@]}"; do \ mkdir -p ${dir}; \ - chown -R www-data:www-data ${dir}; \ - done' \ - # LD_LIBRARY_PATH does not work so below is needed for opentelemetry/other modules - # Put libs of newer modules under `/modules_mount//lib` and add that path below - # Could get complicated arch specific paths become a need - && echo "/lib:/usr/lib:/usr/local/lib:/modules_mount/etc/nginx/modules/otel" > /etc/ld-musl-x86_64.path - + chown -R www-data.www-data ${dir}; \ + done' RUN apk add --no-cache libcap \ && setcap cap_net_bind_service=+ep /nginx-ingress-controller \ && setcap -v cap_net_bind_service=+ep /nginx-ingress-controller \ && setcap cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \ && setcap -v cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \ - && setcap cap_net_bind_service=+ep /usr/bin/dumb-init \ - && setcap -v cap_net_bind_service=+ep /usr/bin/dumb-init \ - && apk del libcap \ - && ln -sf /usr/local/nginx/sbin/nginx /usr/bin/nginx + && apk del libcap USER www-data @@ -84,4 +75,5 @@ RUN ln -sf /dev/stdout /var/log/nginx/access.log \ && ln -sf /dev/stderr /var/log/nginx/error.log ENTRYPOINT ["/usr/bin/dumb-init", "--"] + CMD ["/nginx-ingress-controller"] diff --git a/rootfs/Dockerfile-chroot b/rootfs/Dockerfile-chroot deleted file mode 100644 index 481b311e2..000000000 --- a/rootfs/Dockerfile-chroot +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright 2022 The Kubernetes Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License - -ARG BASE_IMAGE - -FROM ${BASE_IMAGE} as chroot - -# This intermediary image will be used only to copy all the required files to the chroot -# TODO: Simplify in a future to a single Dockerfile -COPY chroot.sh /chroot.sh -RUN apk update \ - && apk upgrade \ - && /chroot.sh - -FROM alpine:3.21 - -ARG TARGETARCH -ARG VERSION -ARG COMMIT_SHA -ARG BUILD_ID=UNSET - -LABEL org.opencontainers.image.title="NGINX Ingress Controller for Kubernetes" -LABEL org.opencontainers.image.documentation="https://kubernetes.github.io/ingress-nginx/" -LABEL org.opencontainers.image.source="https://github.com/kubernetes/ingress-nginx" -LABEL org.opencontainers.image.vendor="The Kubernetes Authors" -LABEL org.opencontainers.image.licenses="Apache-2.0" -LABEL org.opencontainers.image.version="${VERSION}" -LABEL org.opencontainers.image.revision="${COMMIT_SHA}" - -LABEL build_id="${BUILD_ID}" - -# This will be injected in the chroot. Don't change :) -ENV LUA_PATH="/usr/local/share/luajit-2.1.0-beta3/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/lib/lua/?.lua;;" -ENV LUA_CPATH="/usr/local/lib/lua/?/?.so;/usr/local/lib/lua/?.so;;" -ENV PATH=$PATH:/usr/local/luajit/bin:/usr/local/nginx/sbin:/usr/local/nginx/bin - -RUN apk update \ - && apk upgrade \ - && apk add -U --no-cache \ - bash \ - openssl \ - ca-certificates \ - dumb-init \ - tzdata \ - diffutils \ - util-linux \ - && ln -s /usr/local/nginx/sbin/nginx /sbin/nginx \ - && adduser -S -D -H -u 101 -h /usr/local/nginx \ - -s /sbin/nologin -G www-data -g www-data www-data - -COPY --from=chroot /chroot /chroot - -COPY --chown=www-data:www-data etc /chroot/etc - -COPY --chown=www-data:www-data bin/${TARGETARCH}/dbg / -COPY --chown=www-data:www-data bin/${TARGETARCH}/nginx-ingress-controller / -COPY --chown=www-data:www-data bin/${TARGETARCH}/wait-shutdown / -COPY --chown=www-data:www-data nginx-chroot-wrapper.sh /usr/bin/nginx - -WORKDIR /chroot/etc/nginx - -# Fix permission during the build to avoid issues at runtime -# with volumes (custom templates) -RUN bash -xeu -c ' \ - writeDirs=( \ - /var/log \ - ); \ - for dir in "${writeDirs[@]}"; do \ - mkdir -p ${dir}; \ - chown -R www-data:www-data ${dir}; \ - done' \ - # LD_LIBRARY_PATH does not work so below is needed for opentelemetry/other modules - # Put libs of newer modules under `/modules_mount//lib` and add that path below - # Could get complicated arch specific paths become a need - && echo "/lib:/usr/lib:/usr/local/lib:/modules_mount/etc/nginx/modules/otel" > /chroot/etc/ld-musl-x86_64.path - -RUN apk add --no-cache libcap \ - && setcap cap_sys_chroot,cap_net_bind_service=+ep /nginx-ingress-controller \ - && setcap -v cap_sys_chroot,cap_net_bind_service=+ep /nginx-ingress-controller \ - && setcap cap_sys_chroot,cap_net_bind_service=+ep /usr/bin/unshare \ - && setcap -v cap_sys_chroot,cap_net_bind_service=+ep /usr/bin/unshare \ - && setcap cap_net_bind_service=+ep /chroot/usr/local/nginx/sbin/nginx \ - && setcap -v cap_net_bind_service=+ep /chroot/usr/local/nginx/sbin/nginx \ - && setcap cap_sys_chroot,cap_net_bind_service=+ep /usr/bin/dumb-init \ - && setcap -v cap_sys_chroot,cap_net_bind_service=+ep /usr/bin/dumb-init \ - && apk del libcap - -RUN ln -sf /chroot/etc/nginx /etc/nginx \ - && ln -sf /chroot/tmp/nginx /tmp/nginx \ - && ln -sf /chroot/etc/ingress-controller /etc/ingress-controller \ - && ln -sf /chroot/var/log/nginx /var/log/nginx \ - && touch /chroot/var/log/nginx/access.log \ - && chown www-data:www-data /chroot/var/log/nginx/access.log \ - && echo "" > /chroot/etc/resolv.conf \ - && chown -R www-data:www-data /chroot/var/log/nginx /chroot/etc/resolv.conf \ - && mknod -m 0666 /chroot/dev/null c 1 3 \ - && mknod -m 0666 /chroot/dev/random c 1 8 \ - && mknod -m 0666 /chroot/dev/urandom c 1 9 \ - && mknod -m 0666 /chroot/dev/full c 1 7 \ - && mknod -m 0666 /chroot/dev/ptmx c 5 2 \ - && mknod -m 0666 /chroot/dev/zero c 1 5 \ - && mknod -m 0666 /chroot/dev/tty c 5 0 - -RUN mkdir -p /chroot/modules_mount \ - && mkdir -p modules_mount \ - && ln -s /modules_mount /chroot/modules_mount - -USER www-data - -EXPOSE 80 443 -ENTRYPOINT ["/usr/bin/dumb-init", "--"] - -CMD ["/nginx-ingress-controller"] - diff --git a/rootfs/chroot.sh b/rootfs/chroot.sh deleted file mode 100755 index 4875ae535..000000000 --- a/rootfs/chroot.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -# Copyright 2022 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -x -writeDirs=( \ - /chroot/etc/nginx \ - /chroot/usr/local/ \ - /chroot/etc/ingress-controller \ - /chroot/etc/ingress-controller/ssl \ - /chroot/etc/ingress-controller/auth \ - /chroot/etc/ingress-controller/telemetry \ - /chroot/etc/ingress-controller/geoip \ - /chroot/opt/modsecurity/var/log \ - /chroot/opt/modsecurity/var/upload \ - /chroot/opt/modsecurity/var/audit \ - /chroot/var/log/audit \ - /chroot/var/lib/nginx \ - /chroot/var/log/nginx \ - /chroot/var/lib/nginx/body \ - /chroot/var/lib/nginx/fastcgi \ - /chroot/var/lib/nginx/proxy \ - /chroot/var/lib/nginx/scgi \ - /chroot/var/lib/nginx/uwsgi \ - /chroot/tmp/nginx -); - -for dir in "${writeDirs[@]}"; do - mkdir -p ${dir}; - chown -R www-data:www-data ${dir}; -done - -mkdir -p /chroot/lib /chroot/proc /chroot/usr /chroot/bin /chroot/dev /chroot/run -cp /etc/passwd /etc/group /etc/hosts /chroot/etc/ -cp -a /usr/* /chroot/usr/ -cp -a /etc/nginx/* /chroot/etc/nginx/ -cp -a /etc/ingress-controller/* /chroot/etc/ingress-controller/ -cp /lib/ld-musl-* /chroot/lib/ -cp /usr/lib/libcrypto* /usr/lib/libssl* /usr/lib/libz* /chroot/usr/lib/ diff --git a/rootfs/etc/nginx/lua/OWNERS b/rootfs/etc/nginx/lua/OWNERS deleted file mode 100644 index efb131316..000000000 --- a/rootfs/etc/nginx/lua/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs: https://www.kubernetes.dev/docs/guide/owners - -labels: -- area/lua diff --git a/rootfs/etc/nginx/lua/balancer.lua b/rootfs/etc/nginx/lua/balancer.lua index 00104c89d..afcfebb67 100644 --- a/rootfs/etc/nginx/lua/balancer.lua +++ b/rootfs/etc/nginx/lua/balancer.lua @@ -23,6 +23,7 @@ local ngx = ngx -- it will take + BACKENDS_SYNC_INTERVAL local BACKENDS_SYNC_INTERVAL = 1 +local BACKENDS_FORCE_SYNC_INTERVAL = 30 local DEFAULT_LB_ALG = "round_robin" local IMPLEMENTATIONS = { @@ -47,7 +48,7 @@ local function get_implementation(backend) if backend["sessionAffinityConfig"] and backend["sessionAffinityConfig"]["name"] == "cookie" then - if backend["sessionAffinityConfig"]["mode"] == "persistent" then + if backend["sessionAffinityConfig"]["mode"] == 'persistent' then name = "sticky_persistent" else name = "sticky_balanced" @@ -64,7 +65,7 @@ local function get_implementation(backend) local implementation = IMPLEMENTATIONS[name] if not implementation then - ngx.log(ngx.WARN, backend["load-balance"], " is not supported, ", + ngx.log(ngx.WARN, backend["load-balance"], "is not supported, ", "falling back to ", DEFAULT_LB_ALG) implementation = IMPLEMENTATIONS[DEFAULT_LB_ALG] end @@ -145,7 +146,10 @@ end local function sync_backends() local raw_backends_last_synced_at = configuration.get_raw_backends_last_synced_at() - if raw_backends_last_synced_at <= backends_last_synced_at then + ngx.update_time() + local current_timestamp = ngx.time() + if current_timestamp - backends_last_synced_at < BACKENDS_FORCE_SYNC_INTERVAL + and raw_backends_last_synced_at <= backends_last_synced_at then return end @@ -182,11 +186,6 @@ local function sync_backends() end local function route_to_alternative_balancer(balancer) - if balancer.is_affinitized(balancer) then - -- If request is already affinitized to a primary balancer, keep the primary balancer. - return false - end - if not balancer.alternative_backends then return false end @@ -205,13 +204,6 @@ local function route_to_alternative_balancer(balancer) return false end - if alternative_balancer.is_affinitized(alternative_balancer) then - -- If request is affinitized to an alternative balancer, instruct caller to - -- switch to alternative. - return true - end - - -- Use traffic shaping policy, if request didn't have affinity set. local traffic_shaping_policy = alternative_balancer.traffic_shaping_policy if not traffic_shaping_policy then ngx.log(ngx.ERR, "traffic shaping policy is not set for balancer ", @@ -255,21 +247,13 @@ local function route_to_alternative_balancer(balancer) end end - local weightTotal = 100 - if traffic_shaping_policy.weightTotal ~= nil and traffic_shaping_policy.weightTotal > 100 then - weightTotal = traffic_shaping_policy.weightTotal - end - if math.random(weightTotal) <= traffic_shaping_policy.weight then + if math.random(100) <= traffic_shaping_policy.weight then return true end return false end -local function get_balancer_by_upstream_name(upstream_name) - return balancers[upstream_name] -end - local function get_balancer() if ngx.ctx.balancer then return ngx.ctx.balancer @@ -279,7 +263,7 @@ local function get_balancer() local balancer = balancers[backend_name] if not balancer then - return nil + return end if route_to_alternative_balancer(balancer) then @@ -368,7 +352,6 @@ setmetatable(_M, {__index = { sync_backend = sync_backend, route_to_alternative_balancer = route_to_alternative_balancer, get_balancer = get_balancer, - get_balancer_by_upstream_name = get_balancer_by_upstream_name, }}) return _M diff --git a/rootfs/etc/nginx/lua/balancer/chashsubset.lua b/rootfs/etc/nginx/lua/balancer/chashsubset.lua index f004521bd..28c2354a1 100644 --- a/rootfs/etc/nginx/lua/balancer/chashsubset.lua +++ b/rootfs/etc/nginx/lua/balancer/chashsubset.lua @@ -25,6 +25,7 @@ local function build_subset_map(backend) local set_count = math.ceil(#endpoints/subset_size) local node_count = set_count * subset_size + -- if we don't have enough endpoints, we reuse endpoints in the last set to -- keep the same number on all of them. local j = 1 @@ -60,19 +61,13 @@ function _M.new(self, backend) instance = resty_chash:new(subset_map), hash_by = complex_val, subsets = subsets, - current_endpoints = backend.endpoints, - traffic_shaping_policy = backend.trafficShapingPolicy, - alternative_backends = backend.alternativeBackends, + current_endpoints = backend.endpoints } setmetatable(o, self) self.__index = self return o end -function _M.is_affinitized() - return false -end - function _M.balance(self) local key = util.generate_var_value(self.hash_by) local subset_id = self.instance:find(key) diff --git a/rootfs/etc/nginx/lua/balancer/ewma.lua b/rootfs/etc/nginx/lua/balancer/ewma.lua index 681866dc1..ae65ccc73 100644 --- a/rootfs/etc/nginx/lua/balancer/ewma.lua +++ b/rootfs/etc/nginx/lua/balancer/ewma.lua @@ -170,10 +170,6 @@ local function calculate_slow_start_ewma(self) return total_ewma / endpoints_count end -function _M.is_affinitized() - return false -end - function _M.balance(self) local peers = self.peers local endpoint, ewma_score = peers[1], -1 diff --git a/rootfs/etc/nginx/lua/balancer/resty.lua b/rootfs/etc/nginx/lua/balancer/resty.lua index 12b24be14..c1065ff19 100644 --- a/rootfs/etc/nginx/lua/balancer/resty.lua +++ b/rootfs/etc/nginx/lua/balancer/resty.lua @@ -14,10 +14,6 @@ function _M.new(self, o) return o end -function _M.is_affinitized() - return false -end - function _M.sync(self, backend) self.traffic_shaping_policy = backend.trafficShapingPolicy self.alternative_backends = backend.alternativeBackends diff --git a/rootfs/etc/nginx/lua/balancer/sticky.lua b/rootfs/etc/nginx/lua/balancer/sticky.lua index 9d0a54116..45ea9beaf 100644 --- a/rootfs/etc/nginx/lua/balancer/sticky.lua +++ b/rootfs/etc/nginx/lua/balancer/sticky.lua @@ -13,7 +13,6 @@ local setmetatable = setmetatable local _M = balancer_resty:new() local DEFAULT_COOKIE_NAME = "route" -local COOKIE_VALUE_DELIMITER = "|" function _M.cookie_name(self) return self.cookie_session_affinity.name or DEFAULT_COOKIE_NAME @@ -23,8 +22,7 @@ function _M.new(self) local o = { alternative_backends = nil, cookie_session_affinity = nil, - traffic_shaping_policy = nil, - backend_key = nil + traffic_shaping_policy = nil } setmetatable(o, self) @@ -33,37 +31,13 @@ function _M.new(self) return o end -function _M.get_cookie_parsed(self) +function _M.get_cookie(self) local cookie, err = ck:new() if not cookie then ngx.log(ngx.ERR, err) end - local result = { - upstream_key = nil, - backend_key = nil - } - - local raw_value = cookie:get(self:cookie_name()) - if not raw_value then - return result - end - - local parsed_value, len = split.split_string(raw_value, COOKIE_VALUE_DELIMITER) - if len == 0 then - return result - end - - result.upstream_key = parsed_value[1] - if len > 1 then - result.backend_key = parsed_value[2] - end - - return result -end - -function _M.get_cookie(self) - return self:get_cookie_parsed().upstream_key + return cookie:get(self:cookie_name()) end function _M.set_cookie(self, value) @@ -87,18 +61,13 @@ function _M.set_cookie(self, value) end end - local cookie_secure = self.cookie_session_affinity.secure - if cookie_secure == nil then - cookie_secure = ngx.var.https == "on" - end - local cookie_data = { key = self:cookie_name(), - value = value .. COOKIE_VALUE_DELIMITER .. self.backend_key, + value = value, path = cookie_path, httponly = true, samesite = cookie_samesite, - secure = cookie_secure, + secure = ngx.var.https == "on", } if self.cookie_session_affinity.expires and self.cookie_session_affinity.expires ~= "" then @@ -110,10 +79,6 @@ function _M.set_cookie(self, value) cookie_data.max_age = tonumber(self.cookie_session_affinity.maxage) end - if self.cookie_session_affinity.domain and self.cookie_session_affinity.domain ~= "" then - cookie_data.domain = self.cookie_session_affinity.domain - end - local ok ok, err = cookie:set(cookie_data) if not ok then @@ -121,10 +86,6 @@ function _M.set_cookie(self, value) end end -function _M.is_affinitized(self) - return self:get_cookie_parsed().backend_key == self.backend_key -end - function _M.get_last_failure() return ngx_balancer.get_last_failure() end @@ -205,7 +166,6 @@ function _M.sync(self, backend) self.traffic_shaping_policy = backend.trafficShapingPolicy self.alternative_backends = backend.alternativeBackends self.cookie_session_affinity = backend.sessionAffinityConfig.cookieSessionAffinity - self.backend_key = ngx.md5(ngx.md5(backend.name) .. backend.name) end return _M diff --git a/rootfs/etc/nginx/lua/certificate.lua b/rootfs/etc/nginx/lua/certificate.lua index 1b3d3b21f..630462fab 100644 --- a/rootfs/etc/nginx/lua/certificate.lua +++ b/rootfs/etc/nginx/lua/certificate.lua @@ -103,7 +103,6 @@ local function do_ocsp_request(url, ocsp_request) ["Host"] = parsed_url[2], }, body = ocsp_request, - ssl_server_name = parsed_url[2], }) if not http_response then return nil, err @@ -121,14 +120,10 @@ end -- While this has no functional implications, it generates extra load on OCSP servers. local function fetch_and_cache_ocsp_response(uid, der_cert) local url, err = ocsp.get_ocsp_responder_from_der_chain(der_cert) - if not url and err then + if not url then ngx.log(ngx.ERR, "could not extract OCSP responder URL: ", err) return end - if not url and not err then - ngx.log(ngx.DEBUG, "no OCSP responder URL returned") - return - end local request request, err = ocsp.create_ocsp_request(der_cert) diff --git a/rootfs/etc/nginx/lua/global_throttle.lua b/rootfs/etc/nginx/lua/global_throttle.lua new file mode 100644 index 000000000..bea8cfd17 --- /dev/null +++ b/rootfs/etc/nginx/lua/global_throttle.lua @@ -0,0 +1,131 @@ +local resty_global_throttle = require("resty.global_throttle") +local resty_ipmatcher = require("resty.ipmatcher") +local util = require("util") + +local ngx = ngx +local ngx_exit = ngx.exit +local ngx_log = ngx.log +local ngx_ERR = ngx.ERR +local ngx_INFO = ngx.INFO + +local _M = {} + +local DECISION_CACHE = ngx.shared.global_throttle_cache + +-- it does not make sense to cache decision for too little time +-- the benefit of caching likely is negated if we cache for too little time +-- Lua Shared Dict's time resolution for expiry is 0.001. +local CACHE_THRESHOLD = 0.001 + +local DEFAULT_RAW_KEY = "remote_addr" + +local function should_ignore_request(ignored_cidrs) + if not ignored_cidrs or #ignored_cidrs == 0 then + return false + end + + local ignored_cidrs_matcher, err = resty_ipmatcher.new(ignored_cidrs) + if not ignored_cidrs_matcher then + ngx_log(ngx_ERR, "failed to initialize resty-ipmatcher: ", err) + return false + end + + local is_ignored + is_ignored, err = ignored_cidrs_matcher:match(ngx.var.remote_addr) + if err then + ngx_log(ngx_ERR, "failed to match ip: '", + ngx.var.remote_addr, "': ", err) + return false + end + + return is_ignored +end + +local function is_enabled(config, location_config) + if config.memcached.host == "" or config.memcached.port == 0 then + return false + end + if location_config.limit == 0 or + location_config.window_size == 0 then + return false + end + + if should_ignore_request(location_config.ignored_cidrs) then + return false + end + + return true +end + +local function get_namespaced_key_value(namespace, key_value) + return namespace .. key_value +end + +function _M.throttle(config, location_config) + if not is_enabled(config, location_config) then + return + end + + local key_value = util.generate_var_value(location_config.key) + if not key_value or key_value == "" then + key_value = ngx.var[DEFAULT_RAW_KEY] + end + + local namespaced_key_value = + get_namespaced_key_value(location_config.namespace, key_value) + + local is_limit_exceeding = DECISION_CACHE:get(namespaced_key_value) + if is_limit_exceeding then + ngx.var.global_rate_limit_exceeding = "c" + return ngx_exit(config.status_code) + end + + local my_throttle, err = resty_global_throttle.new( + location_config.namespace, + location_config.limit, + location_config.window_size, + { + provider = "memcached", + host = config.memcached.host, + port = config.memcached.port, + connect_timeout = config.memcached.connect_timeout, + max_idle_timeout = config.memcached.max_idle_timeout, + pool_size = config.memcached.pool_size, + } + ) + if err then + ngx.log(ngx.ERR, "faled to initialize resty_global_throttle: ", err) + -- fail open + return + end + + local desired_delay, estimated_final_count + estimated_final_count, desired_delay, err = my_throttle:process(key_value) + if err then + ngx.log(ngx.ERR, "error while processing key: ", err) + -- fail open + return + end + + if desired_delay then + if desired_delay > CACHE_THRESHOLD then + local ok + ok, err = + DECISION_CACHE:safe_add(namespaced_key_value, true, desired_delay) + if not ok then + if err ~= "exists" then + ngx_log(ngx_ERR, "failed to cache decision: ", err) + end + end + end + + ngx.var.global_rate_limit_exceeding = "y" + ngx_log(ngx_INFO, "limit is exceeding for ", + location_config.namespace, "/", key_value, + " with estimated_final_count: ", estimated_final_count) + + return ngx_exit(config.status_code) + end +end + +return _M diff --git a/rootfs/etc/nginx/lua/lua_ingress.lua b/rootfs/etc/nginx/lua/lua_ingress.lua index a513928cf..49e0f5b05 100644 --- a/rootfs/etc/nginx/lua/lua_ingress.lua +++ b/rootfs/etc/nginx/lua/lua_ingress.lua @@ -1,8 +1,8 @@ local ngx_re_split = require("ngx.re").split -local string_to_bool = require("util").string_to_bool local certificate_configured_for_current_request = require("certificate").configured_for_current_request +local global_throttle = require("global_throttle") local ngx = ngx local io = io @@ -109,16 +109,7 @@ end -- rewrite gets called in every location context. -- This is where we do variable assignments to be used in subsequent -- phases or redirection -function _M.rewrite() - - local location_config = { - force_ssl_redirect = string_to_bool(ngx.var.force_ssl_redirect), - ssl_redirect = string_to_bool(ngx.var.ssl_redirect), - force_no_ssl_redirect = string_to_bool(ngx.var.force_no_ssl_redirect), - preserve_trailing_slash = string_to_bool(ngx.var.preserve_trailing_slash), - use_port_in_redirects = string_to_bool(ngx.var.use_port_in_redirects), - } - +function _M.rewrite(location_config) ngx.var.pass_access_scheme = ngx.var.scheme ngx.var.best_http_host = ngx.var.http_host or ngx.var.host @@ -173,6 +164,7 @@ function _M.rewrite() return ngx_redirect(uri, config.http_redirect_code) end + global_throttle.throttle(config.global_throttle, location_config.global_throttle) end function _M.header() diff --git a/rootfs/etc/nginx/lua/monitor.lua b/rootfs/etc/nginx/lua/monitor.lua index a42f26fbf..3e7d6b34c 100644 --- a/rootfs/etc/nginx/lua/monitor.lua +++ b/rootfs/etc/nginx/lua/monitor.lua @@ -26,7 +26,7 @@ local _M = {} local function send(payload) local s = assert(socket()) - assert(s:connect("unix:/tmp/nginx/prometheus-nginx.socket")) + assert(s:connect("unix:/tmp/prometheus-nginx.socket")) assert(s:send(payload)) assert(s:close()) end @@ -37,7 +37,6 @@ local function metrics() namespace = ngx.var.namespace or "-", ingress = ngx.var.ingress_name or "-", service = ngx.var.service_name or "-", - canary = ngx.var.proxy_alternative_upstream_name or "-", path = ngx.var.location_path or "-", method = ngx.var.request_method or "-", @@ -47,7 +46,6 @@ local function metrics() responseLength = tonumber(ngx.var.bytes_sent) or -1, upstreamLatency = tonumber(ngx.var.upstream_connect_time) or -1, - upstreamHeaderTime = tonumber(ngx.var.upstream_header_time) or -1, upstreamResponseTime = tonumber(ngx.var.upstream_response_time) or -1, upstreamResponseLength = tonumber(ngx.var.upstream_response_length) or -1, --upstreamStatus = ngx.var.upstream_status or "-", diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_balancer.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_balancer.lua deleted file mode 100644 index 977d3e964..000000000 --- a/rootfs/etc/nginx/lua/nginx/ngx_conf_balancer.lua +++ /dev/null @@ -1,2 +0,0 @@ -local balancer = require("balancer") -balancer.balance() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_balancer_tcp_udp.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_balancer_tcp_udp.lua deleted file mode 100644 index 0442df7d8..000000000 --- a/rootfs/etc/nginx/lua/nginx/ngx_conf_balancer_tcp_udp.lua +++ /dev/null @@ -1,2 +0,0 @@ -local tcp_udp_balancer = require("tcp_udp_balancer") -tcp_udp_balancer.balance() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_certificate.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_certificate.lua deleted file mode 100644 index d33d2171b..000000000 --- a/rootfs/etc/nginx/lua/nginx/ngx_conf_certificate.lua +++ /dev/null @@ -1,2 +0,0 @@ -local certificate = require("certificate") -certificate.call() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_configuration.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_configuration.lua deleted file mode 100644 index 7864f40ef..000000000 --- a/rootfs/etc/nginx/lua/nginx/ngx_conf_configuration.lua +++ /dev/null @@ -1,2 +0,0 @@ -local configuration = require("configuration") -configuration.call() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_content_tcp_udp.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_content_tcp_udp.lua deleted file mode 100644 index ed81e7ff3..000000000 --- a/rootfs/etc/nginx/lua/nginx/ngx_conf_content_tcp_udp.lua +++ /dev/null @@ -1,2 +0,0 @@ -local tcp_udp_configuration = require("tcp_udp_configuration") -tcp_udp_configuration.call() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_external_auth.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_external_auth.lua deleted file mode 100644 index 6c68cf07c..000000000 --- a/rootfs/etc/nginx/lua/nginx/ngx_conf_external_auth.lua +++ /dev/null @@ -1,30 +0,0 @@ -local auth_path = ngx.var.auth_path -local auth_keepalive_share_vars = ngx.var.auth_keepalive_share_vars -local auth_response_headers = ngx.var.auth_response_headers -local ngx_re_split = require("ngx.re").split -local ipairs = ipairs -local ngx_log = ngx.log -local ngx_ERR = ngx.ERR - -local res = ngx.location.capture(auth_path, { - method = ngx.HTTP_GET, body = '', - share_all_vars = auth_keepalive_share_vars }) - -if res.status == ngx.HTTP_OK then - local header_parts, err = ngx_re_split(auth_response_headers, ",") - if err then - ngx_log(ngx_ERR, err) - return - end - ngx.var.auth_cookie = res.header['Set-Cookie'] - for i, header_name in ipairs(header_parts) do - local varname = "authHeader" .. tostring(i) - ngx.var[varname] = res.header[header_name] - end - return -end - -if res.status == ngx.HTTP_UNAUTHORIZED or res.status == ngx.HTTP_FORBIDDEN then - ngx.exit(res.status) -end -ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_init_tcp_udp.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_init_tcp_udp.lua deleted file mode 100644 index 926ab7a03..000000000 --- a/rootfs/etc/nginx/lua/nginx/ngx_conf_init_tcp_udp.lua +++ /dev/null @@ -1,2 +0,0 @@ -local tcp_udp_balancer = require("tcp_udp_balancer") -tcp_udp_balancer.init_worker() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_is_dynamic_lb_initialized.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_is_dynamic_lb_initialized.lua deleted file mode 100644 index ade3114b1..000000000 --- a/rootfs/etc/nginx/lua/nginx/ngx_conf_is_dynamic_lb_initialized.lua +++ /dev/null @@ -1,9 +0,0 @@ -local configuration = require("configuration") -local backend_data = configuration.get_backends_data() -if not backend_data then - ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) - return -end - -ngx.say("OK") -ngx.exit(ngx.HTTP_OK) \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_log.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_log.lua deleted file mode 100644 index 8f3d57be6..000000000 --- a/rootfs/etc/nginx/lua/nginx/ngx_conf_log.lua +++ /dev/null @@ -1,2 +0,0 @@ -local monitor = require("monitor") -monitor.call() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_log_block.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_log_block.lua deleted file mode 100644 index 72f6a6430..000000000 --- a/rootfs/etc/nginx/lua/nginx/ngx_conf_log_block.lua +++ /dev/null @@ -1,11 +0,0 @@ -local balancer = require("balancer") -local monitor = require("monitor") - -local luaconfig = ngx.shared.luaconfig -local enablemetrics = luaconfig:get("enablemetrics") - -balancer.log() - -if enablemetrics then - monitor.call() -end \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_rewrite_auth.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_rewrite_auth.lua deleted file mode 100644 index 0d16faba0..000000000 --- a/rootfs/etc/nginx/lua/nginx/ngx_conf_rewrite_auth.lua +++ /dev/null @@ -1 +0,0 @@ -ngx.var.cache_key = ngx.encode_base64(ngx.sha1_bin(ngx.var.tmp_cache_key)) \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_srv_hdr_filter.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_srv_hdr_filter.lua deleted file mode 100644 index 311a9b433..000000000 --- a/rootfs/etc/nginx/lua/nginx/ngx_conf_srv_hdr_filter.lua +++ /dev/null @@ -1,2 +0,0 @@ -local lua_ingress = require("lua_ingress") -lua_ingress.header() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_rewrite.lua b/rootfs/etc/nginx/lua/nginx/ngx_rewrite.lua deleted file mode 100644 index 66fdd6d55..000000000 --- a/rootfs/etc/nginx/lua/nginx/ngx_rewrite.lua +++ /dev/null @@ -1,5 +0,0 @@ -local lua_ingress = require("lua_ingress") -local balancer = require("balancer") - -lua_ingress.rewrite() -balancer.rewrite() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_srv_redirect.lua b/rootfs/etc/nginx/lua/nginx/ngx_srv_redirect.lua deleted file mode 100644 index 4b9445d00..000000000 --- a/rootfs/etc/nginx/lua/nginx/ngx_srv_redirect.lua +++ /dev/null @@ -1,24 +0,0 @@ -local request_uri = ngx.var.request_uri -local redirect_to = ngx.arg[1] - -local luaconfig = ngx.shared.luaconfig -local use_forwarded_headers = luaconfig:get("use_forwarded_headers") - -if string.sub(request_uri, -1) == "/" then - request_uri = string.sub(request_uri, 1, -2) -end - -local redirectScheme = ngx.var.scheme -local redirectPort = ngx.var.server_port - -if use_forwarded_headers then - if ngx.var.http_x_forwarded_proto then - redirectScheme = ngx.var.http_x_forwarded_proto - end - if ngx.var.http_x_forwarded_port then - redirectPort = ngx.var.http_x_forwarded_port - end -end - -return string.format("%s://%s:%s%s", redirectScheme, - redirect_to, redirectPort, request_uri) \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/ngx_conf_init.lua b/rootfs/etc/nginx/lua/ngx_conf_init.lua deleted file mode 100644 index 9789386ac..000000000 --- a/rootfs/etc/nginx/lua/ngx_conf_init.lua +++ /dev/null @@ -1,53 +0,0 @@ -local cjson = require("cjson.safe") - -collectgarbage("collect") -local f = io.open("/etc/nginx/lua/cfg.json", "r") -local content = f:read("*a") -f:close() -local configfile = cjson.decode(content) - -local luaconfig = ngx.shared.luaconfig -luaconfig:set("enablemetrics", configfile.enable_metrics) -luaconfig:set("use_forwarded_headers", configfile.use_forwarded_headers) --- init modules -local ok, res -ok, res = pcall(require, "lua_ingress") -if not ok then - error("require failed: " .. tostring(res)) -else - lua_ingress = res - lua_ingress.set_config(configfile) -end -ok, res = pcall(require, "configuration") -if not ok then - error("require failed: " .. tostring(res)) -else - configuration = res - if not configfile.listen_ports.status_port then - error("required status port not found") - end - configuration.prohibited_localhost_port = configfile.listen_ports.status_port -end -ok, res = pcall(require, "balancer") -if not ok then - error("require failed: " .. tostring(res)) -else - balancer = res -end -if configfile.enable_metrics then - ok, res = pcall(require, "monitor") - if not ok then - error("require failed: " .. tostring(res)) - else - monitor = res - end -end -ok, res = pcall(require, "certificate") -if not ok then - error("require failed: " .. tostring(res)) -else - certificate = res - if configfile.enable_ocsp then - certificate.is_ocsp_stapling_enabled = configfile.enable_ocsp - end -end \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/ngx_conf_init_stream.lua b/rootfs/etc/nginx/lua/ngx_conf_init_stream.lua deleted file mode 100644 index a78062d0a..000000000 --- a/rootfs/etc/nginx/lua/ngx_conf_init_stream.lua +++ /dev/null @@ -1,30 +0,0 @@ -local cjson = require("cjson.safe") -collectgarbage("collect") -local f = io.open("/etc/nginx/lua/cfg.json", "r") -local content = f:read("*a") -f:close() -local configfile = cjson.decode(content) --- init modules -local ok, res -ok, res = pcall(require, "configuration") -if not ok then - error("require failed: " .. tostring(res)) -else - configuration = res -end -ok, res = pcall(require, "tcp_udp_configuration") -if not ok then - error("require failed: " .. tostring(res)) -else - tcp_udp_configuration = res - if not configfile.listen_ports.status_port then - error("required status port not found") - end - tcp_udp_configuration.prohibited_localhost_port = configfile.listen_ports.status_port -end -ok, res = pcall(require, "tcp_udp_balancer") -if not ok then - error("require failed: " .. tostring(res)) -else - tcp_udp_balancer = res -end diff --git a/rootfs/etc/nginx/lua/ngx_conf_init_worker.lua b/rootfs/etc/nginx/lua/ngx_conf_init_worker.lua deleted file mode 100644 index cba866136..000000000 --- a/rootfs/etc/nginx/lua/ngx_conf_init_worker.lua +++ /dev/null @@ -1,15 +0,0 @@ -local cjson = require("cjson.safe") - -local f = io.open("/etc/nginx/lua/cfg.json", "r") -local content = f:read("*a") -f:close() -local configfile = cjson.decode(content) - -local lua_ingress = require("lua_ingress") -local balancer = require("balancer") -local monitor = require("monitor") -lua_ingress.init_worker() -balancer.init_worker() -if configfile.enable_metrics and configfile.monitor_batch_max_size then - monitor.init_worker(configfile.monitor_batch_max_size) -end \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/plugins.lua b/rootfs/etc/nginx/lua/plugins.lua new file mode 100644 index 000000000..0c1fd899b --- /dev/null +++ b/rootfs/etc/nginx/lua/plugins.lua @@ -0,0 +1,59 @@ +local require = require +local ngx = ngx +local pairs = pairs +local ipairs = ipairs +local string_format = string.format +local ngx_log = ngx.log +local INFO = ngx.INFO +local ERR = ngx.ERR +local pcall = pcall + +local _M = {} +local MAX_NUMBER_OF_PLUGINS = 20 +local plugins = {} + +local function load_plugin(name) + local path = string_format("plugins.%s.main", name) + + local ok, plugin = pcall(require, path) + if not ok then + ngx_log(ERR, string_format("error loading plugin \"%s\": %s", path, plugin)) + return + end + + plugins[name] = plugin +end + +function _M.init(names) + local count = 0 + for _, name in ipairs(names) do + if count >= MAX_NUMBER_OF_PLUGINS then + ngx_log(ERR, "the total number of plugins exceed the maximum number: ", MAX_NUMBER_OF_PLUGINS) + break + end + load_plugin(name) + count = count + 1 -- ignore loading failure, just count the total + end +end + +function _M.run() + local phase = ngx.get_phase() + + for name, plugin in pairs(plugins) do + if plugin[phase] then + ngx_log(INFO, string_format("running plugin \"%s\" in phase \"%s\"", name, phase)) + + -- TODO: consider sandboxing this, should we? + -- probably yes, at least prohibit plugin from accessing env vars etc + -- but since the plugins are going to be installed by ingress-nginx + -- operator they can be assumed to be safe also + local ok, err = pcall(plugin[phase]) + if not ok then + ngx_log(ERR, string_format("error while running plugin \"%s\" in phase \"%s\": %s", + name, phase, err)) + end + end + end +end + +return _M diff --git a/rootfs/etc/nginx/lua/plugins/README.md b/rootfs/etc/nginx/lua/plugins/README.md new file mode 100644 index 000000000..0967c8d94 --- /dev/null +++ b/rootfs/etc/nginx/lua/plugins/README.md @@ -0,0 +1,36 @@ +# Custom Lua plugins + +ingress-nginx uses [https://github.com/openresty/lua-nginx-module](https://github.com/openresty/lua-nginx-module) to run custom Lua code +within Nginx workers. It is recommended to familiarize yourself with that ecosystem before deploying your custom Lua based ingress-nginx plugin. + +### Writing a plugin + +Every ingress-nginx Lua plugin is expected to have `main.lua` file and all of its dependencies. +`main.lua` is the entry point of the plugin. The plugin manager uses convention over configuration +strategy and automatically runs functions defined in `main.lua` in the corresponding Nginx phase based on their name. + +Nginx has different [request processing phases](http://nginx.org/en/docs/dev/development_guide.html#http_phases). +By defining functions with the following names, you can run your custom Lua code in the corresponding Nginx phase: + + - `init_worker`: useful for initializing some data per Nginx worker process + - `rewrite`: useful for modifying request, changing headers, redirection, dropping request, doing authentication etc + - `header_filter`: this is called when backend response header is received, it is useful for modifying response headers + - `body_filter`: this is called when response body is received, it is useful for logging response body + - `log`: this is called when request processing is completed and a response is delivered to the client + +Check this [`hello_world`](https://github.com/kubernetes/ingress-nginx/tree/master/rootfs/etc/nginx/lua/plugins/hello_world) plugin as a simple example or refer to [OpenID Connect integration](https://github.com/ElvinEfendi/ingress-nginx-openidc/tree/master/rootfs/etc/nginx/lua/plugins/openidc) for more advanced usage. + +Do not forget to write tests for your plugin. + +### Installing a plugin + +There are two options: + + - mount your plugin into `/etc/nginx/lua/plugins/` in the ingress-nginx pod + - build your own ingress-nginx image like it is done in the [example](https://github.com/ElvinEfendi/ingress-nginx-openidc/tree/master/rootfs/etc/nginx/lua/plugins/openidc) and install your plugin during image build + +Mounting is the quickest option. + +### Enabling plugins + +Once your plugin is ready you need to use [`plugins` configuration setting](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#plugins) to activate it. Let's say you want to active `hello_world` and `open_idc` plugins, then you set `plugins` setting to `"hello_world, open_idc"`. _Note_ that the plugins will be executed in the given order. diff --git a/rootfs/etc/nginx/lua/plugins/hello_world/main.lua b/rootfs/etc/nginx/lua/plugins/hello_world/main.lua new file mode 100644 index 000000000..03316c3ee --- /dev/null +++ b/rootfs/etc/nginx/lua/plugins/hello_world/main.lua @@ -0,0 +1,13 @@ +local ngx = ngx + +local _M = {} + +function _M.rewrite() + local ua = ngx.var.http_user_agent + + if ua == "hello" then + ngx.req.set_header("x-hello-world", "1") + end +end + +return _M diff --git a/rootfs/etc/nginx/lua/plugins/hello_world/test/main_test.lua b/rootfs/etc/nginx/lua/plugins/hello_world/test/main_test.lua new file mode 100644 index 000000000..5eda52259 --- /dev/null +++ b/rootfs/etc/nginx/lua/plugins/hello_world/test/main_test.lua @@ -0,0 +1,24 @@ + +local main = require("plugins.hello_world.main") + +-- The unit tests are run within a timer phase in a headless Nginx process. +-- Since `set_header` and `ngx.var.http_` API are disabled in this phase we have to stub it +-- to avoid `API disabled in the current context` error. + +describe("main", function() + describe("rewrite", function() + it("sets x-hello-world header to 1 when user agent is hello", function() + ngx.var = { http_user_agent = "hello" } + stub(ngx.req, "set_header") + main.rewrite() + assert.stub(ngx.req.set_header).was_called_with("x-hello-world", "1") + end) + + it("does not set x-hello-world header to 1 when user agent is not hello", function() + ngx.var = { http_user_agent = "not-hello" } + stub(ngx.req, "set_header") + main.rewrite() + assert.stub(ngx.req.set_header).was_not_called_with("x-hello-world", "1") + end) + end) +end) diff --git a/rootfs/etc/nginx/lua/tcp_udp_configuration.lua b/rootfs/etc/nginx/lua/tcp_udp_configuration.lua index f191454fa..85864b45b 100644 --- a/rootfs/etc/nginx/lua/tcp_udp_configuration.lua +++ b/rootfs/etc/nginx/lua/tcp_udp_configuration.lua @@ -1,6 +1,5 @@ local ngx = ngx local tostring = tostring -local cjson = require("cjson.safe") -- this is the Lua representation of TCP/UDP Configuration local tcp_udp_configuration_data = ngx.shared.tcp_udp_configuration_data @@ -38,14 +37,6 @@ function _M.call() return end - local _, backends_err = cjson.decode(backends) - - if backends_err then - ngx.log(ngx.ERR, "could not parse backends data: ", backends_err) - return - end - - local success, err_conf = tcp_udp_configuration_data:set("backends", backends) if not success then ngx.log(ngx.ERR, "dynamic-configuration: error updating configuration: " .. tostring(err_conf)) diff --git a/rootfs/etc/nginx/lua/test/balancer/chashsubset_test.lua b/rootfs/etc/nginx/lua/test/balancer/chashsubset_test.lua index 59cb1e588..bbf55838c 100644 --- a/rootfs/etc/nginx/lua/test/balancer/chashsubset_test.lua +++ b/rootfs/etc/nginx/lua/test/balancer/chashsubset_test.lua @@ -86,21 +86,5 @@ describe("Balancer chash subset", function() assert.are.equal(#endpoints, 3) end end) - - it("set alternative backends", function() - local backend = get_test_backend(7) - backend.trafficShapingPolicy = { - weight = 0, - header = "", - headerValue = "", - cookie = "" - } - backend.alternativeBackends = { - "my-dummy-canary-backend" - } - local instance = balancer_chashsubset:new(backend) - assert.not_equal(instance.traffic_shaping_policy, nil) - assert.not_equal(instance.alternative_backends, nil) - end) end) end) diff --git a/rootfs/etc/nginx/lua/test/balancer/sticky_test.lua b/rootfs/etc/nginx/lua/test/balancer/sticky_test.lua index 70723143b..44e103c1c 100644 --- a/rootfs/etc/nginx/lua/test/balancer/sticky_test.lua +++ b/rootfs/etc/nginx/lua/test/balancer/sticky_test.lua @@ -5,33 +5,22 @@ local util = require("util") local original_ngx = ngx -local function reset_sticky_balancer() - package.loaded["balancer.sticky"] = nil - package.loaded["balancer.sticky_balanced"] = nil - package.loaded["balancer.sticky_persistent"] = nil - - sticky_balanced = require("balancer.sticky_balanced") - sticky_persistent = require("balancer.sticky_persistent") -end - -local function mock_ngx(mock, after_mock_set) +function mock_ngx(mock) local _ngx = mock - setmetatable(_ngx, { __index = ngx }) + setmetatable(_ngx, {__index = _G.ngx}) _G.ngx = _ngx - - if after_mock_set then - after_mock_set() - end - - -- Balancer module caches ngx module, must be reset after mocks were configured. - reset_sticky_balancer() end local function reset_ngx() _G.ngx = original_ngx +end - -- Ensure balancer cache is reset. - _G.ngx.ctx.balancer = nil +local function reset_sticky_balancer() + package.loaded["balancer.sticky"] = nil + package.loaded["balancer.sticky_balanced"] = nil + package.loaded["balancer.sticky_persistent"] = nil + sticky_balanced = require("balancer.sticky_balanced") + sticky_persistent = require("balancer.sticky_persistent") end function get_mocked_cookie_new() @@ -66,6 +55,7 @@ end describe("Sticky", function() before_each(function() mock_ngx({ var = { location_path = "/", host = "test.com" } }) + reset_sticky_balancer() end) after_each(function() @@ -75,44 +65,29 @@ describe("Sticky", function() local test_backend = get_test_backend() local test_backend_endpoint= test_backend.endpoints[1].address .. ":" .. test_backend.endpoints[1].port - local legacy_cookie_value = test_backend_endpoint - local function create_current_cookie_value(backend_key) - return test_backend_endpoint .. "|" .. backend_key - end - describe("new(backend)", function() - describe("when backend specifies cookie name", function() - local function test_with(sticky_balancer_type) - local sticky_balancer_instance = sticky_balancer_type:new(test_backend) + context("when backend specifies cookie name", function() + local function test(sticky) + local sticky_balancer_instance = sticky:new(test_backend) local test_backend_cookie_name = test_backend.sessionAffinityConfig.cookieSessionAffinity.name assert.equal(sticky_balancer_instance:cookie_name(), test_backend_cookie_name) end - it("returns an instance containing the corresponding cookie name", function() test_with(sticky_balanced) end) - it("returns an instance containing the corresponding cookie name", function() test_with(sticky_persistent) end) + it("returns an instance containing the corresponding cookie name", function() test(sticky_balanced) end) + it("returns an instance containing the corresponding cookie name", function() test(sticky_persistent) end) end) - describe("when backend does not specify cookie name", function() - local function test_with(sticky_balancer_type) + context("when backend does not specify cookie name", function() + local function test(sticky) local temp_backend = util.deepcopy(test_backend) temp_backend.sessionAffinityConfig.cookieSessionAffinity.name = nil - local sticky_balancer_instance = sticky_balancer_type:new(temp_backend) + local sticky_balancer_instance = sticky:new(temp_backend) local default_cookie_name = "route" assert.equal(sticky_balancer_instance:cookie_name(), default_cookie_name) end - it("returns an instance with 'route' as cookie name", function() test_with(sticky_balanced) end) - it("returns an instance with 'route' as cookie name", function() test_with(sticky_persistent) end) - end) - - describe("backend_key", function() - local function test_with(sticky_balancer_type) - local sticky_balancer_instance = sticky_balancer_type:new(test_backend) - assert.is_truthy(sticky_balancer_instance.backend_key) - end - - it("calculates at construction time", function() test_with(sticky_balanced) end) - it("calculates at construction time", function() test_with(sticky_persistent) end) + it("returns an instance with 'route' as cookie name", function() test(sticky_balanced) end) + it("returns an instance with 'route' as cookie name", function() test(sticky_persistent) end) end) end) @@ -120,25 +95,28 @@ describe("Sticky", function() local mocked_cookie_new = cookie.new before_each(function() - reset_sticky_balancer() + package.loaded["balancer.sticky_balanced"] = nil + package.loaded["balancer.sticky_persistent"] = nil + sticky_balanced = require("balancer.sticky_balanced") + sticky_persistent = require("balancer.sticky_persistent") end) after_each(function() cookie.new = mocked_cookie_new end) - describe("when client doesn't have a cookie set and location is in cookie_locations", function() + context("when client doesn't have a cookie set and location is in cookie_locations", function() - local function test_pick_endpoint_with(sticky_balancer_type) - local sticky_balancer_instance = sticky_balancer_type:new(test_backend) + local function test_pick_endpoint(sticky) + local sticky_balancer_instance = sticky:new(test_backend) local peer = sticky_balancer_instance:balance() assert.equal(test_backend_endpoint, peer) end - it("picks an endpoint for the client", function() test_pick_endpoint_with(sticky_balanced) end) - it("picks an endpoint for the client", function() test_pick_endpoint_with(sticky_persistent) end) + it("picks an endpoint for the client", function() test_pick_endpoint(sticky_balanced) end) + it("picks an endpoint for the client", function() test_pick_endpoint(sticky_persistent) end) - local function test_set_cookie_with(sticky_balancer_type) + local function test_set_cookie(sticky) local s = {} cookie.new = function(self) local cookie_instance = { @@ -159,15 +137,15 @@ describe("Sticky", function() local b = get_test_backend() b.sessionAffinityConfig.cookieSessionAffinity.locations = {} b.sessionAffinityConfig.cookieSessionAffinity.locations["test.com"] = {"/"} - local sticky_balancer_instance = sticky_balancer_type:new(b) + local sticky_balancer_instance = sticky:new(b) assert.has_no.errors(function() sticky_balancer_instance:balance() end) assert.spy(s).was_called() end - it("sets a cookie on the client", function() test_set_cookie_with(sticky_balanced) end) - it("sets a cookie on the client", function() test_set_cookie_with(sticky_persistent) end) + it("sets a cookie on the client", function() test_set_cookie(sticky_balanced) end) + it("sets a cookie on the client", function() test_set_cookie(sticky_persistent) end) - local function test_set_ssl_cookie_with(sticky_balancer_type) + local function test_set_ssl_cookie(sticky) ngx.var.https = "on" local s = {} cookie.new = function(self) @@ -189,17 +167,21 @@ describe("Sticky", function() local b = get_test_backend() b.sessionAffinityConfig.cookieSessionAffinity.locations = {} b.sessionAffinityConfig.cookieSessionAffinity.locations["test.com"] = {"/"} - local sticky_balancer_instance = sticky_balancer_type:new(b) + local sticky_balancer_instance = sticky:new(b) assert.has_no.errors(function() sticky_balancer_instance:balance() end) assert.spy(s).was_called() end - it("sets a secure cookie on the client when being in ssl mode", function() test_set_ssl_cookie_with(sticky_balanced) end) - it("sets a secure cookie on the client when being in ssl mode", function() test_set_ssl_cookie_with(sticky_persistent) end) + it("sets a secure cookie on the client when being in ssl mode", function() + test_set_ssl_cookie(sticky_balanced) + end) + it("sets a secure cookie on the client when being in ssl mode", function() + test_set_ssl_cookie(sticky_persistent) + end) end) - describe("when client doesn't have a cookie set and cookie_locations contains a matching wildcard location", function() - + context("when client doesn't have a cookie set and cookie_locations contains a matching wildcard location", + function() before_each(function () ngx.var.host = "dev.test.com" end) @@ -207,7 +189,7 @@ describe("Sticky", function() ngx.var.host = "test.com" end) - local function test_with(sticky_balancer_type) + local function test(sticky) local s = {} cookie.new = function(self) local cookie_instance = { @@ -229,27 +211,27 @@ describe("Sticky", function() local b = get_test_backend() b.sessionAffinityConfig.cookieSessionAffinity.locations = {} b.sessionAffinityConfig.cookieSessionAffinity.locations["*.test.com"] = {"/"} - local sticky_balancer_instance = sticky_balancer_type:new(b) + local sticky_balancer_instance = sticky:new(b) assert.has_no.errors(function() sticky_balancer_instance:balance() end) assert.spy(s).was_called() end - it("sets a cookie on the client", function() test_with(sticky_balanced) end) - it("sets a cookie on the client", function() test_with(sticky_persistent) end) + it("sets a cookie on the client", function() test(sticky_balanced) end) + it("sets a cookie on the client", function() test(sticky_persistent) end) end) - describe("when client doesn't have a cookie set and location not in cookie_locations", function() + context("when client doesn't have a cookie set and location not in cookie_locations", function() - local function test_pick_endpoint_with(sticky_balancer_type) - local sticky_balancer_instance = sticky_balancer_type:new(test_backend) + local function test_pick_endpoint(sticky) + local sticky_balancer_instance = sticky:new(test_backend) local peer = sticky_balancer_instance:balance() assert.equal(peer, test_backend_endpoint) end - it("picks an endpoint for the client", function() test_pick_endpoint_with(sticky_balanced) end) - it("picks an endpoint for the client", function() test_pick_endpoint_with(sticky_persistent) end) + it("picks an endpoint for the client", function() test_pick_endpoint(sticky_balanced) end) + it("picks an endpoint for the client", function() test_pick_endpoint(sticky_persistent) end) - local function test_no_cookie_with(sticky_balancer_type) + local function test_no_cookie(sticky) local s = {} cookie.new = function(self) local cookie_instance = { @@ -266,34 +248,34 @@ describe("Sticky", function() s = spy.on(cookie_instance, "set") return cookie_instance, false end - local sticky_balancer_instance = sticky_balancer_type:new(get_test_backend()) + local sticky_balancer_instance = sticky:new(get_test_backend()) assert.has_no.errors(function() sticky_balancer_instance:balance() end) assert.spy(s).was_not_called() end - it("does not set a cookie on the client", function() test_no_cookie_with(sticky_balanced) end) - it("does not set a cookie on the client", function() test_no_cookie_with(sticky_persistent) end) + it("does not set a cookie on the client", function() test_no_cookie(sticky_balanced) end) + it("does not set a cookie on the client", function() test_no_cookie(sticky_persistent) end) end) - describe("when client has a cookie set", function() + context("when client has a cookie set", function() - local function test_no_cookie_with(sticky_balancer_type) + local function test_no_cookie(sticky) local s = {} cookie.new = function(self) local return_obj = { set = function(v) return false, nil end, - get = function(k) return legacy_cookie_value end, + get = function(k) return test_backend_endpoint end, } s = spy.on(return_obj, "set") return return_obj, false end - local sticky_balancer_instance = sticky_balancer_type:new(test_backend) + local sticky_balancer_instance = sticky:new(test_backend) assert.has_no.errors(function() sticky_balancer_instance:balance() end) assert.spy(s).was_not_called() end - it("does not set a cookie", function() test_no_cookie_with(sticky_balanced) end) - it("does not set a cookie", function() test_no_cookie_with(sticky_persistent) end) + it("does not set a cookie", function() test_no_cookie(sticky_balanced) end) + it("does not set a cookie", function() test_no_cookie(sticky_persistent) end) local function test_correct_endpoint(sticky) local sticky_balancer_instance = sticky:new(test_backend) @@ -330,16 +312,17 @@ describe("Sticky", function() before_each(function() mock_ngx({ var = { location_path = "/", host = "test.com" } }) + reset_sticky_balancer() end) after_each(function() reset_ngx() end) - describe("when request to upstream fails", function() + context("when request to upstream fails", function() - local function test_with(sticky_balancer_type, change_on_failure) - local sticky_balancer_instance = sticky_balancer_type:new(get_several_test_backends(change_on_failure)) + local function test(sticky, change_on_failure) + local sticky_balancer_instance = sticky:new(get_several_test_backends(change_on_failure)) local old_upstream = sticky_balancer_instance:balance() assert.is.Not.Nil(old_upstream) @@ -357,7 +340,7 @@ describe("Sticky", function() for _ = 1, 100 do local new_upstream = sticky_balancer_instance:balance() if change_on_failure == false then - -- upstream should be the same in spite of error, if change_on_failure option is false + -- upstream should be the same inspite of error, if change_on_failure option is false assert.equal(new_upstream, old_upstream) else -- upstream should change after error, if change_on_failure option is true @@ -366,21 +349,29 @@ describe("Sticky", function() end end - it("changes upstream when change_on_failure option is true", function() test_with(sticky_balanced, true) end) - it("changes upstream when change_on_failure option is true", function() test_with(sticky_persistent, true) end) - - it("changes upstream when change_on_failure option is false", function() test_with(sticky_balanced, false) end) - it("changes upstream when change_on_failure option is false", function() test_with(sticky_persistent, false) end) + it("changes upstream when change_on_failure option is true", function() + test(sticky_balanced, true) + end) + it("changes upstream when change_on_failure option is true", function() + test(sticky_balanced, false) + end) + it("changes upstream when change_on_failure option is true", function() + test(sticky_persistent, true) + end) + it("changes upstream when change_on_failure option is true", function() + test(sticky_persistent, false) + end) end) end) - describe("when client doesn't have a cookie set and no host header, matching default server '_'", function() + context("when client doesn't have a cookie set and no host header, matching default server '_'", + function() before_each(function () ngx.var.host = "not-default-server" ngx.var.server_name = "_" end) - local function test_with(sticky_balancer_type) + local function test(sticky) local s = {} cookie.new = function(self) local cookie_instance = { @@ -402,27 +393,30 @@ describe("Sticky", function() local b = get_test_backend() b.sessionAffinityConfig.cookieSessionAffinity.locations = {} b.sessionAffinityConfig.cookieSessionAffinity.locations["_"] = {"/"} - local sticky_balancer_instance = sticky_balancer_type:new(b) + local sticky_balancer_instance = sticky:new(b) assert.has_no.errors(function() sticky_balancer_instance:balance() end) assert.spy(s).was_called() end - it("sets a cookie on the client", function() test_with(sticky_balanced) end) - it("sets a cookie on the client", function() test_with(sticky_persistent) end) + it("sets a cookie on the client", function() test(sticky_balanced) end) + it("sets a cookie on the client", function() test(sticky_persistent) end) end) describe("SameSite settings", function() local mocked_cookie_new = cookie.new before_each(function() - reset_sticky_balancer() + package.loaded["balancer.sticky_balanced"] = nil + package.loaded["balancer.sticky_persistent"] = nil + sticky_balanced = require("balancer.sticky_balanced") + sticky_persistent = require("balancer.sticky_persistent") end) after_each(function() cookie.new = mocked_cookie_new end) - local function test_set_cookie_with(sticky_balancer_type, samesite, conditional_samesite_none, expected_path, expected_samesite, secure, expected_secure) + local function test_set_cookie(sticky, samesite, conditional_samesite_none, expected_path, expected_samesite) local s = {} cookie.new = function(self) local cookie_instance = { @@ -432,7 +426,7 @@ describe("Sticky", function() assert.equal(payload.samesite, expected_samesite) assert.equal(payload.domain, nil) assert.equal(payload.httponly, true) - assert.equal(payload.secure, expected_secure) + assert.equal(payload.secure, false) return true, nil end, get = function(k) return false end, @@ -445,183 +439,34 @@ describe("Sticky", function() b.sessionAffinityConfig.cookieSessionAffinity.locations["test.com"] = {"/"} b.sessionAffinityConfig.cookieSessionAffinity.samesite = samesite b.sessionAffinityConfig.cookieSessionAffinity.conditional_samesite_none = conditional_samesite_none - b.sessionAffinityConfig.cookieSessionAffinity.secure = secure - local sticky_balancer_instance = sticky_balancer_type:new(b) + local sticky_balancer_instance = sticky:new(b) assert.has_no.errors(function() sticky_balancer_instance:balance() end) assert.spy(s).was_called() end - it("returns a secure cookie with SameSite=Strict when user specifies samesite strict and secure=true", function() - test_set_cookie_with(sticky_balanced, "Lax", false, "/", "Lax", true, true) - end) - it("returns a cookie with SameSite=Strict when user specifies samesite strict and conditional samesite none", function() - test_set_cookie_with(sticky_balanced, "Strict", true, "/", "Strict", nil, false) - end) - it("returns a cookie with SameSite=Lax when user specifies samesite lax", function() - test_set_cookie_with(sticky_balanced, "Lax", false, "/", "Lax", nil, false) - end) - it("returns a cookie with SameSite=Lax when user specifies samesite lax and conditional samesite none", function() - test_set_cookie_with(sticky_balanced, "Lax", true, "/", "Lax", nil, false) - end) - it("returns a cookie with SameSite=None when user specifies samesite None", function() - test_set_cookie_with(sticky_balanced, "None", false, "/", "None", nil, false) - end) - it("returns a cookie with SameSite=None when user specifies samesite None and conditional samesite none with supported user agent", function() - mock_ngx({ var = { location_path = "/", host = "test.com" , http_user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.2704.103 Safari/537.36"} }) - test_set_cookie_with(sticky_balanced, "None", true, "/", "None", nil, false) - end) - it("returns a cookie without SameSite=None when user specifies samesite None and conditional samesite none with unsupported user agent", function() - mock_ngx({ var = { location_path = "/", host = "test.com" , http_user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"} }) - test_set_cookie_with(sticky_balanced, "None", true, "/", nil, nil, false) - end) - - it("returns a secure cookie with SameSite=Strict when user specifies samesite strict and secure=true", function() - test_set_cookie_with(sticky_persistent, "Lax", false, "/", "Lax", true, true) - end) it("returns a cookie with SameSite=Strict when user specifies samesite strict", function() - test_set_cookie_with(sticky_persistent, "Strict", false, "/", "Strict", nil, false) + test_set_cookie(sticky_balanced, "Strict", false, "/", "Strict") end) it("returns a cookie with SameSite=Strict when user specifies samesite strict and conditional samesite none", function() - test_set_cookie_with(sticky_persistent, "Strict", true, "/", "Strict", nil, false) + test_set_cookie(sticky_balanced, "Strict", true, "/", "Strict") end) it("returns a cookie with SameSite=Lax when user specifies samesite lax", function() - test_set_cookie_with(sticky_persistent, "Lax", false, "/", "Lax", nil, false) + test_set_cookie(sticky_balanced, "Lax", false, "/", "Lax") end) it("returns a cookie with SameSite=Lax when user specifies samesite lax and conditional samesite none", function() - test_set_cookie_with(sticky_persistent, "Lax", true, "/", "Lax", nil, false) + test_set_cookie(sticky_balanced, "Lax", true, "/", "Lax") end) it("returns a cookie with SameSite=None when user specifies samesite None", function() - test_set_cookie_with(sticky_persistent, "None", false, "/", "None", nil, false) + test_set_cookie(sticky_balanced, "None", false, "/", "None") end) it("returns a cookie with SameSite=None when user specifies samesite None and conditional samesite none with supported user agent", function() mock_ngx({ var = { location_path = "/", host = "test.com" , http_user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.2704.103 Safari/537.36"} }) - test_set_cookie_with(sticky_persistent, "None", true, "/", "None", nil, false) + test_set_cookie(sticky_balanced, "None", true, "/", "None") end) it("returns a cookie without SameSite=None when user specifies samesite None and conditional samesite none with unsupported user agent", function() mock_ngx({ var = { location_path = "/", host = "test.com" , http_user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"} }) - test_set_cookie_with(sticky_persistent, "None", true, "/", nil, nil, false) + reset_sticky_balancer() + test_set_cookie(sticky_balanced, "None", true, "/", nil) end) end) - - describe("get_cookie()", function() - - describe("legacy cookie value", function() - local function test_with(sticky_balancer_type) - local sticky_balancer_instance = sticky_balancer_type:new(test_backend) - - cookie.new = function(self) - local return_obj = { - set = function(v) return false, nil end, - get = function(k) return legacy_cookie_value end, - } - return return_obj, false - end - - assert.equal(test_backend_endpoint, sticky_balancer_instance.get_cookie(sticky_balancer_instance)) - end - - it("retrieves upstream key value", function() test_with(sticky_balanced) end) - it("retrieves upstream key value", function() test_with(sticky_persistent) end) - end) - - describe("current cookie value", function() - local function test_with(sticky_balancer_type) - local sticky_balancer_instance = sticky_balancer_type:new(test_backend) - - cookie.new = function(self) - local return_obj = { - set = function(v) return false, nil end, - get = function(k) return create_current_cookie_value(sticky_balancer_instance.backend_key) end, - } - return return_obj, false - end - - assert.equal(test_backend_endpoint, sticky_balancer_instance.get_cookie(sticky_balancer_instance)) - end - - it("retrieves upstream key value", function() test_with(sticky_balanced) end) - it("retrieves upstream key value", function() test_with(sticky_persistent) end) - end) - - end) - - describe("get_cookie_parsed()", function() - - describe("legacy cookie value", function() - local function test_with(sticky_balancer_type) - local sticky_balancer_instance = sticky_balancer_type:new(test_backend) - - cookie.new = function(self) - local return_obj = { - set = function(v) return false, nil end, - get = function(k) return legacy_cookie_value end, - } - return return_obj, false - end - - local parsed_cookie = sticky_balancer_instance.get_cookie_parsed(sticky_balancer_instance) - - assert.is_truthy(parsed_cookie) - assert.equal(test_backend_endpoint, parsed_cookie.upstream_key) - assert.is_falsy(parsed_cookie.backend_key) - end - - it("retrieves upstream key value", function() test_with(sticky_balanced) end) - it("retrieves upstream key value", function() test_with(sticky_persistent) end) - end) - - describe("current cookie value", function() - local function test_with(sticky_balancer_type) - local sticky_balancer_instance = sticky_balancer_type:new(test_backend) - - cookie.new = function(self) - local return_obj = { - set = function(v) return false, nil end, - get = function(k) return create_current_cookie_value(sticky_balancer_instance.backend_key) end, - } - return return_obj, false - end - - local parsed_cookie = sticky_balancer_instance.get_cookie_parsed(sticky_balancer_instance) - - assert.is_truthy(parsed_cookie) - assert.equal(test_backend_endpoint, parsed_cookie.upstream_key) - assert.equal(sticky_balancer_instance.backend_key, parsed_cookie.backend_key) - end - - it("retrieves all supported values", function() test_with(sticky_balanced) end) - it("retrieves all supported values", function() test_with(sticky_persistent) end) - end) - - end) - - describe("set_cookie()", function() - - local function test_with(sticky_balancer_type) - local sticky_balancer_instance = sticky_balancer_type:new(test_backend) - - local cookieSetSpy = {} - cookie.new = function(self) - local return_obj = { - set = function(self, payload) - assert.equal(create_current_cookie_value(sticky_balancer_instance.backend_key), payload.value) - - return true, nil - end, - get = function(k) return nil end, - } - cookieSetSpy = spy.on(return_obj, "set") - - return return_obj, false - end - - sticky_balancer_instance.set_cookie(sticky_balancer_instance, test_backend_endpoint) - - assert.spy(cookieSetSpy).was_called() - end - - it("constructs correct cookie value", function() test_with(sticky_balanced) end) - it("constructs correct cookie value", function() test_with(sticky_persistent) end) - - end) end) diff --git a/rootfs/etc/nginx/lua/test/balancer_test.lua b/rootfs/etc/nginx/lua/test/balancer_test.lua index 2d42ad330..1834b6f60 100644 --- a/rootfs/etc/nginx/lua/test/balancer_test.lua +++ b/rootfs/etc/nginx/lua/test/balancer_test.lua @@ -6,9 +6,12 @@ local original_ngx = ngx local function reset_ngx() _G.ngx = original_ngx +end - -- Ensure balancer cache is reset. - _G.ngx.ctx.balancer = nil +local function mock_ngx(mock) + local _ngx = mock + setmetatable(_ngx, { __index = ngx }) + _G.ngx = _ngx end local function reset_balancer() @@ -16,19 +19,6 @@ local function reset_balancer() balancer = require("balancer") end -local function mock_ngx(mock, after_mock_set) - local _ngx = mock - setmetatable(_ngx, { __index = ngx }) - _G.ngx = _ngx - - if after_mock_set then - after_mock_set() - end - - -- Balancer module caches ngx module, must be reset after mocks were configured. - reset_balancer() -end - local function reset_expected_implementations() expected_implementations = { ["access-router-production-web-80"] = package.loaded["balancer.round_robin"], @@ -36,8 +26,7 @@ local function reset_expected_implementations() ["my-dummy-app-2"] = package.loaded["balancer.chash"], ["my-dummy-app-3"] = package.loaded["balancer.sticky_persistent"], ["my-dummy-app-4"] = package.loaded["balancer.ewma"], - ["my-dummy-app-5"] = package.loaded["balancer.sticky_balanced"], - ["my-dummy-app-6"] = package.loaded["balancer.chashsubset"] + ["my-dummy-app-5"] = package.loaded["balancer.sticky_balanced"] } end @@ -59,35 +48,20 @@ local function reset_backends() cookie = "" }, }, + { name = "my-dummy-app-1", ["load-balance"] = "round_robin", }, { - name = "my-dummy-app-1", - ["load-balance"] = "round_robin", - }, - { - name = "my-dummy-app-2", - ["load-balance"] = "round_robin", -- upstreamHashByConfig will take priority. + name = "my-dummy-app-2", ["load-balance"] = "chash", upstreamHashByConfig = { ["upstream-hash-by"] = "$request_uri", }, }, { - name = "my-dummy-app-3", - ["load-balance"] = "ewma", -- sessionAffinityConfig will take priority. - sessionAffinityConfig = { name = "cookie", mode = "persistent", cookieSessionAffinity = { name = "route" } } + name = "my-dummy-app-3", ["load-balance"] = "ewma", + sessionAffinityConfig = { name = "cookie", mode = 'persistent', cookieSessionAffinity = { name = "route" } } }, + { name = "my-dummy-app-4", ["load-balance"] = "ewma", }, { - name = "my-dummy-app-4", - ["load-balance"] = "ewma", - }, - { - name = "my-dummy-app-5", - ["load-balance"] = "ewma", -- sessionAffinityConfig will take priority. - upstreamHashByConfig = { ["upstream-hash-by"] = "$request_uri", }, + name = "my-dummy-app-5", ["load-balance"] = "ewma", ["upstream-hash-by"] = "$request_uri", sessionAffinityConfig = { name = "cookie", cookieSessionAffinity = { name = "route" } } }, - { - name = "my-dummy-app-6", - ["load-balance"] = "ewma", -- upstreamHashByConfig will take priority. - upstreamHashByConfig = { ["upstream-hash-by"] = "$request_uri", ["upstream-hash-by-subset"] = "true", } - }, } end @@ -103,7 +77,7 @@ describe("Balancer", function() end) describe("get_implementation()", function() - it("uses heuristics to select correct load balancer implementation for a given backend", function() + it("returns correct implementation for given backend", function() for _, backend in pairs(backends) do local expected_implementation = expected_implementations[backend.name] local implementation = balancer.get_implementation(backend) @@ -115,8 +89,8 @@ describe("Balancer", function() describe("get_balancer()", function() it("always returns the same balancer for given request context", function() local backend = { - name = "my-dummy-app-100", ["load-balance"] = "ewma", - alternativeBackends = { "my-dummy-canary-app-100" }, + name = "my-dummy-app-6", ["load-balance"] = "ewma", + alternativeBackends = { "my-dummy-canary-app-6" }, endpoints = { { address = "10.184.7.40", port = "8080", maxFails = 0, failTimeout = 0 } }, trafficShapingPolicy = { weight = 0, @@ -126,8 +100,8 @@ describe("Balancer", function() }, } local canary_backend = { - name = "my-dummy-canary-app-100", ["load-balance"] = "ewma", - alternativeBackends = { "my-dummy-canary-app-100" }, + name = "my-dummy-canary-app-6", ["load-balance"] = "ewma", + alternativeBackends = { "my-dummy-canary-app-6" }, endpoints = { { address = "11.184.7.40", port = "8080", maxFails = 0, failTimeout = 0 } }, trafficShapingPolicy = { weight = 5, @@ -138,6 +112,7 @@ describe("Balancer", function() } mock_ngx({ var = { proxy_upstream_name = backend.name } }) + reset_balancer() balancer.sync_backend(backend) balancer.sync_backend(canary_backend) @@ -151,237 +126,172 @@ describe("Balancer", function() end) describe("route_to_alternative_balancer()", function() - local backend, _primaryBalancer + local backend, _balancer before_each(function() backend = backends[1] - _primaryBalancer = { + _balancer = { alternative_backends = { backend.name, } } mock_ngx({ var = { request_uri = "/" } }) + reset_balancer() end) - -- Not affinitized request must follow traffic shaping policies. - describe("not affinitized", function() - - before_each(function() - _primaryBalancer.is_affinitized = function (_) - return false - end - end) - - it("returns false when no trafficShapingPolicy is set", function() - balancer.sync_backend(backend) - assert.equal(false, balancer.route_to_alternative_balancer(_primaryBalancer)) - end) - - it("returns false when no alternative backends is set", function() - backend.trafficShapingPolicy.weight = 100 - balancer.sync_backend(backend) - _primaryBalancer.alternative_backends = nil - assert.equal(false, balancer.route_to_alternative_balancer(_primaryBalancer)) - end) - - it("returns false when alternative backends name does not match", function() - backend.trafficShapingPolicy.weight = 100 - balancer.sync_backend(backend) - _primaryBalancer.alternative_backends[1] = "nonExistingBackend" - assert.equal(false, balancer.route_to_alternative_balancer(_primaryBalancer)) - end) - - describe("canary by weight", function() - it("returns true when weight is 100", function() - backend.trafficShapingPolicy.weight = 100 - balancer.sync_backend(backend) - assert.equal(true, balancer.route_to_alternative_balancer(_primaryBalancer)) - end) - - it("returns false when weight is 0", function() - backend.trafficShapingPolicy.weight = 0 - balancer.sync_backend(backend) - assert.equal(false, balancer.route_to_alternative_balancer(_primaryBalancer)) - end) - - it("returns true when weight is 1000 and weight total is 1000", function() - backend.trafficShapingPolicy.weight = 1000 - backend.trafficShapingPolicy.weightTotal = 1000 - balancer.sync_backend(backend) - assert.equal(true, balancer.route_to_alternative_balancer(_primaryBalancer)) - end) - - it("returns false when weight is 0 and weight total is 1000", function() - backend.trafficShapingPolicy.weight = 1000 - backend.trafficShapingPolicy.weightTotal = 1000 - balancer.sync_backend(backend) - assert.equal(true, balancer.route_to_alternative_balancer(_primaryBalancer)) - end) - end) - - describe("canary by cookie", function() - it("returns correct result for given cookies", function() - local test_patterns = { - { - case_title = "cookie_value is 'always'", - request_cookie_name = "canaryCookie", - request_cookie_value = "always", - expected_result = true, - }, - { - case_title = "cookie_value is 'never'", - request_cookie_name = "canaryCookie", - request_cookie_value = "never", - expected_result = false, - }, - { - case_title = "cookie_value is undefined", - request_cookie_name = "canaryCookie", - request_cookie_value = "foo", - expected_result = false, - }, - { - case_title = "cookie_name is undefined", - request_cookie_name = "foo", - request_cookie_value = "always", - expected_result = false - }, - } - for _, test_pattern in pairs(test_patterns) do - mock_ngx({ var = { - ["cookie_" .. test_pattern.request_cookie_name] = test_pattern.request_cookie_value, - request_uri = "/" - }}) - backend.trafficShapingPolicy.cookie = "canaryCookie" - balancer.sync_backend(backend) - assert.message("\nTest data pattern: " .. test_pattern.case_title) - .equal(test_pattern.expected_result, balancer.route_to_alternative_balancer(_primaryBalancer)) - reset_ngx() - end - end) - end) - - describe("canary by header", function() - it("returns correct result for given headers", function() - local test_patterns = { - -- with no header value setting - { - case_title = "no custom header value and header value is 'always'", - header_name = "canaryHeader", - header_value = "", - request_header_name = "canaryHeader", - request_header_value = "always", - expected_result = true, - }, - { - case_title = "no custom header value and header value is 'never'", - header_name = "canaryHeader", - header_value = "", - request_header_name = "canaryHeader", - request_header_value = "never", - expected_result = false, - }, - { - case_title = "no custom header value and header value is undefined", - header_name = "canaryHeader", - header_value = "", - request_header_name = "canaryHeader", - request_header_value = "foo", - expected_result = false, - }, - { - case_title = "no custom header value and header name is undefined", - header_name = "canaryHeader", - header_value = "", - request_header_name = "foo", - request_header_value = "always", - expected_result = false, - }, - -- with header value setting - { - case_title = "custom header value is set and header value is 'always'", - header_name = "canaryHeader", - header_value = "foo", - request_header_name = "canaryHeader", - request_header_value = "always", - expected_result = false, - }, - { - case_title = "custom header value is set and header value match custom header value", - header_name = "canaryHeader", - header_value = "foo", - request_header_name = "canaryHeader", - request_header_value = "foo", - expected_result = true, - }, - { - case_title = "custom header value is set and header name is undefined", - header_name = "canaryHeader", - header_value = "foo", - request_header_name = "bar", - request_header_value = "foo", - expected_result = false - }, - } - - for _, test_pattern in pairs(test_patterns) do - mock_ngx({ var = { - ["http_" .. test_pattern.request_header_name] = test_pattern.request_header_value, - request_uri = "/" - }}) - backend.trafficShapingPolicy.header = test_pattern.header_name - backend.trafficShapingPolicy.headerValue = test_pattern.header_value - balancer.sync_backend(backend) - assert.message("\nTest data pattern: " .. test_pattern.case_title) - .equal(test_pattern.expected_result, balancer.route_to_alternative_balancer(_primaryBalancer)) - reset_ngx() - end - end) - end) - + it("returns false when no trafficShapingPolicy is set", function() + balancer.sync_backend(backend) + assert.equal(false, balancer.route_to_alternative_balancer(_balancer)) end) - -- Affinitized request prefers backend it is affinitized to. - describe("affinitized", function() + it("returns false when no alternative backends is set", function() + backend.trafficShapingPolicy.weight = 100 + balancer.sync_backend(backend) + _balancer.alternative_backends = nil + assert.equal(false, balancer.route_to_alternative_balancer(_balancer)) + end) - before_each(function() - mock_ngx({ var = { request_uri = "/", proxy_upstream_name = backend.name } }) + it("returns false when alternative backends name does not match", function() + backend.trafficShapingPolicy.weight = 100 + balancer.sync_backend(backend) + _balancer.alternative_backends[1] = "nonExistingBackend" + assert.equal(false, balancer.route_to_alternative_balancer(_balancer)) + end) + + context("canary by weight", function() + it("returns true when weight is 100", function() + backend.trafficShapingPolicy.weight = 100 balancer.sync_backend(backend) + assert.equal(true, balancer.route_to_alternative_balancer(_balancer)) end) - it("returns false if request is affinitized to primary backend", function() - _primaryBalancer.is_affinitized = function (_) - return true - end - - local alternativeBalancer = balancer.get_balancer_by_upstream_name(backend.name) - - local primarySpy = spy.on(_primaryBalancer, "is_affinitized") - local alternativeSpy = spy.on(alternativeBalancer, "is_affinitized") - - assert.is_false(balancer.route_to_alternative_balancer(_primaryBalancer)) - assert.spy(_primaryBalancer.is_affinitized).was_called() - assert.spy(alternativeBalancer.is_affinitized).was_not_called() + it("returns false when weight is 0", function() + backend.trafficShapingPolicy.weight = 0 + balancer.sync_backend(backend) + assert.equal(false, balancer.route_to_alternative_balancer(_balancer)) end) + end) - it("returns true if request is affinitized to alternative backend", function() - _primaryBalancer.is_affinitized = function (_) - return false + context("canary by cookie", function() + it("returns correct result for given cookies", function() + local test_patterns = { + { + case_title = "cookie_value is 'always'", + request_cookie_name = "canaryCookie", + request_cookie_value = "always", + expected_result = true, + }, + { + case_title = "cookie_value is 'never'", + request_cookie_name = "canaryCookie", + request_cookie_value = "never", + expected_result = false, + }, + { + case_title = "cookie_value is undefined", + request_cookie_name = "canaryCookie", + request_cookie_value = "foo", + expected_result = false, + }, + { + case_title = "cookie_name is undefined", + request_cookie_name = "foo", + request_cookie_value = "always", + expected_result = false + }, + } + for _, test_pattern in pairs(test_patterns) do + mock_ngx({ var = { + ["cookie_" .. test_pattern.request_cookie_name] = test_pattern.request_cookie_value, + request_uri = "/" + }}) + reset_balancer() + backend.trafficShapingPolicy.cookie = "canaryCookie" + balancer.sync_backend(backend) + assert.message("\nTest data pattern: " .. test_pattern.case_title) + .equal(test_pattern.expected_result, balancer.route_to_alternative_balancer(_balancer)) + reset_ngx() end - - local alternativeBalancer = balancer.get_balancer_by_upstream_name(backend.name) - alternativeBalancer.is_affinitized = function (_) - return true - end - - local primarySpy = spy.on(_primaryBalancer, "is_affinitized") - local alternativeSpy = spy.on(alternativeBalancer, "is_affinitized") - - assert.is_true(balancer.route_to_alternative_balancer(_primaryBalancer)) - assert.spy(_primaryBalancer.is_affinitized).was_called() - assert.spy(alternativeBalancer.is_affinitized).was_called() end) + end) + context("canary by header", function() + it("returns correct result for given headers", function() + local test_patterns = { + -- with no header value setting + { + case_title = "no custom header value and header value is 'always'", + header_name = "canaryHeader", + header_value = "", + request_header_name = "canaryHeader", + request_header_value = "always", + expected_result = true, + }, + { + case_title = "no custom header value and header value is 'never'", + header_name = "canaryHeader", + header_value = "", + request_header_name = "canaryHeader", + request_header_value = "never", + expected_result = false, + }, + { + case_title = "no custom header value and header value is undefined", + header_name = "canaryHeader", + header_value = "", + request_header_name = "canaryHeader", + request_header_value = "foo", + expected_result = false, + }, + { + case_title = "no custom header value and header name is undefined", + header_name = "canaryHeader", + header_value = "", + request_header_name = "foo", + request_header_value = "always", + expected_result = false, + }, + -- with header value setting + { + case_title = "custom header value is set and header value is 'always'", + header_name = "canaryHeader", + header_value = "foo", + request_header_name = "canaryHeader", + request_header_value = "always", + expected_result = false, + }, + { + case_title = "custom header value is set and header value match custom header value", + header_name = "canaryHeader", + header_value = "foo", + request_header_name = "canaryHeader", + request_header_value = "foo", + expected_result = true, + }, + { + case_title = "custom header value is set and header name is undefined", + header_name = "canaryHeader", + header_value = "foo", + request_header_name = "bar", + request_header_value = "foo", + expected_result = false + }, + } + + for _, test_pattern in pairs(test_patterns) do + mock_ngx({ var = { + ["http_" .. test_pattern.request_header_name] = test_pattern.request_header_value, + request_uri = "/" + }}) + reset_balancer() + backend.trafficShapingPolicy.header = test_pattern.header_name + backend.trafficShapingPolicy.headerValue = test_pattern.header_value + balancer.sync_backend(backend) + assert.message("\nTest data pattern: " .. test_pattern.case_title) + .equal(test_pattern.expected_result, balancer.route_to_alternative_balancer(_balancer)) + reset_ngx() + end + end) end) end) @@ -522,13 +432,10 @@ describe("Balancer", function() }, } } - - mock_ngx({ var = { proxy_upstream_name = "access-router-production-web-80" }, ctx = { } }, function() - ngx.shared.configuration_data:set("backends", cjson.encode(backends)) - end) - + mock_ngx({ var = { proxy_upstream_name = "access-router-production-web-80" }, ctx = { } }) + ngx.shared.configuration_data:set("backends", cjson.encode(backends)) + reset_balancer() balancer.init_worker() - assert.not_equal(balancer.get_balancer(), nil) end) diff --git a/rootfs/etc/nginx/lua/test/global_throttle_test.lua b/rootfs/etc/nginx/lua/test/global_throttle_test.lua new file mode 100644 index 000000000..b8db740ad --- /dev/null +++ b/rootfs/etc/nginx/lua/test/global_throttle_test.lua @@ -0,0 +1,258 @@ +local util = require("util") + +local function assert_request_rejected(config, location_config, opts) + stub(ngx, "exit") + + local global_throttle = require_without_cache("global_throttle") + assert.has_no.errors(function() + global_throttle.throttle(config, location_config) + end) + + assert.stub(ngx.exit).was_called_with(config.status_code) + if opts.with_cache then + assert.are.same("c", ngx.var.global_rate_limit_exceeding) + else + assert.are.same("y", ngx.var.global_rate_limit_exceeding) + end +end + +local function assert_request_not_rejected(config, location_config) + stub(ngx, "exit") + local cache_safe_add_spy = spy.on(ngx.shared.global_throttle_cache, "safe_add") + + local global_throttle = require_without_cache("global_throttle") + assert.has_no.errors(function() + global_throttle.throttle(config, location_config) + end) + + assert.stub(ngx.exit).was_not_called() + assert.is_nil(ngx.var.global_rate_limit_exceeding) + assert.spy(cache_safe_add_spy).was_not_called() +end + +local function assert_short_circuits(f) + local cache_get_spy = spy.on(ngx.shared.global_throttle_cache, "get") + + local resty_global_throttle = require_without_cache("resty.global_throttle") + local resty_global_throttle_new_spy = spy.on(resty_global_throttle, "new") + + local global_throttle = require_without_cache("global_throttle") + + f(global_throttle) + + assert.spy(resty_global_throttle_new_spy).was_not_called() + assert.spy(cache_get_spy).was_not_called() +end + +local function assert_fails_open(config, location_config, ...) + stub(ngx, "exit") + stub(ngx, "log") + + local global_throttle = require_without_cache("global_throttle") + + assert.has_no.errors(function() + global_throttle.throttle(config, location_config) + end) + + assert.stub(ngx.exit).was_not_called() + assert.stub(ngx.log).was_called_with(ngx.ERR, ...) + assert.is_nil(ngx.var.global_rate_limit_exceeding) +end + +local function stub_resty_global_throttle_process(ret1, ret2, ret3, f) + local resty_global_throttle = require_without_cache("resty.global_throttle") + local resty_global_throttle_mock = { + process = function(self, key) return ret1, ret2, ret3 end + } + stub(resty_global_throttle, "new", resty_global_throttle_mock) + + f() + + assert.stub(resty_global_throttle.new).was_called() +end + +local function cache_rejection_decision(namespace, key_value, desired_delay) + local namespaced_key_value = namespace .. key_value + local ok, err = ngx.shared.global_throttle_cache:safe_add(namespaced_key_value, true, desired_delay) + assert.is_nil(err) + assert.is_true(ok) + assert.is_true(ngx.shared.global_throttle_cache:get(namespaced_key_value)) +end + +describe("global_throttle", function() + local snapshot + + local NAMESPACE = "31285d47b1504dcfbd6f12c46d769f6e" + local LOCATION_CONFIG = { + namespace = NAMESPACE, + limit = 10, + window_size = 60, + key = {}, + ignored_cidrs = {}, + } + local CONFIG = { + memcached = { + host = "memc.default.svc.cluster.local", port = 11211, + connect_timeout = 50, max_idle_timeout = 10000, pool_size = 50, + }, + status_code = 429, + } + + before_each(function() + snapshot = assert:snapshot() + + ngx.var = { remote_addr = "127.0.0.1", global_rate_limit_exceeding = nil } + end) + + after_each(function() + snapshot:revert() + + ngx.shared.global_throttle_cache:flush_all() + reset_ngx() + end) + + it("short circuits when memcached is not configured", function() + assert_short_circuits(function(global_throttle) + assert.has_no.errors(function() + global_throttle.throttle({ memcached = { host = "", port = 0 } }, LOCATION_CONFIG) + end) + end) + end) + + it("short circuits when limit or window_size is not configured", function() + assert_short_circuits(function(global_throttle) + local location_config_copy = util.deepcopy(LOCATION_CONFIG) + location_config_copy.limit = 0 + assert.has_no.errors(function() + global_throttle.throttle(CONFIG, location_config_copy) + end) + end) + + assert_short_circuits(function(global_throttle) + local location_config_copy = util.deepcopy(LOCATION_CONFIG) + location_config_copy.window_size = 0 + assert.has_no.errors(function() + global_throttle.throttle(CONFIG, location_config_copy) + end) + end) + end) + + it("short circuits when remote_addr is in ignored_cidrs", function() + local global_throttle = require_without_cache("global_throttle") + local location_config = util.deepcopy(LOCATION_CONFIG) + location_config.ignored_cidrs = { ngx.var.remote_addr } + assert_short_circuits(function(global_throttle) + assert.has_no.errors(function() + global_throttle.throttle(CONFIG, location_config) + end) + end) + end) + + it("rejects when exceeding limit has already been cached", function() + local key_value = "foo" + local location_config = util.deepcopy(LOCATION_CONFIG) + location_config.key = { { nil, nil, nil, key_value } } + cache_rejection_decision(NAMESPACE, key_value, 0.5) + + assert_request_rejected(CONFIG, location_config, { with_cache = true }) + end) + + describe("when resty_global_throttle fails", function() + it("fails open in case of initialization error", function() + local too_long_namespace = "" + for i=1,36,1 do + too_long_namespace = too_long_namespace .. "a" + end + + local location_config = util.deepcopy(LOCATION_CONFIG) + location_config.namespace = too_long_namespace + + assert_fails_open(CONFIG, location_config, "faled to initialize resty_global_throttle: ", "'namespace' can be at most 35 characters") + end) + + it("fails open in case of key processing error", function() + stub_resty_global_throttle_process(nil, nil, "failed to process", function() + assert_fails_open(CONFIG, LOCATION_CONFIG, "error while processing key: ", "failed to process") + end) + end) + end) + + it("initializes resty_global_throttle with the right parameters", function() + local resty_global_throttle = require_without_cache("resty.global_throttle") + local resty_global_throttle_original_new = resty_global_throttle.new + resty_global_throttle.new = function(namespace, limit, window_size, store_opts) + local o, err = resty_global_throttle_original_new(namespace, limit, window_size, store_opts) + if not o then + return nil, err + end + o.process = function(self, key) return 1, nil, nil end + + local expected = LOCATION_CONFIG + assert.are.same(expected.namespace, namespace) + assert.are.same(expected.limit, limit) + assert.are.same(expected.window_size, window_size) + + assert.are.same("memcached", store_opts.provider) + assert.are.same(CONFIG.memcached.host, store_opts.host) + assert.are.same(CONFIG.memcached.port, store_opts.port) + assert.are.same(CONFIG.memcached.connect_timeout, store_opts.connect_timeout) + assert.are.same(CONFIG.memcached.max_idle_timeout, store_opts.max_idle_timeout) + assert.are.same(CONFIG.memcached.pool_size, store_opts.pool_size) + + return o, nil + end + local resty_global_throttle_new_spy = spy.on(resty_global_throttle, "new") + + local global_throttle = require_without_cache("global_throttle") + + assert.has_no.errors(function() + global_throttle.throttle(CONFIG, LOCATION_CONFIG) + end) + + assert.spy(resty_global_throttle_new_spy).was_called() + end) + + it("rejects request and caches decision when limit is exceeding after processing a key", function() + local desired_delay = 0.015 + + stub_resty_global_throttle_process(LOCATION_CONFIG.limit + 1, desired_delay, nil, function() + assert_request_rejected(CONFIG, LOCATION_CONFIG, { with_cache = false }) + + local cache_key = LOCATION_CONFIG.namespace .. ngx.var.remote_addr + assert.is_true(ngx.shared.global_throttle_cache:get(cache_key)) + + -- we assume it won't take more than this after caching + -- until we execute the assertion below + local delta = 0.001 + local ttl = ngx.shared.global_throttle_cache:ttl(cache_key) + assert.is_true(ttl > desired_delay - delta) + assert.is_true(ttl <= desired_delay) + end) + end) + + it("rejects request and skip caching of decision when limit is exceeding after processing a key but desired delay is lower than the threshold", function() + local desired_delay = 0.0009 + + stub_resty_global_throttle_process(LOCATION_CONFIG.limit, desired_delay, nil, function() + assert_request_rejected(CONFIG, LOCATION_CONFIG, { with_cache = false }) + + local cache_key = LOCATION_CONFIG.namespace .. ngx.var.remote_addr + assert.is_nil(ngx.shared.global_throttle_cache:get(cache_key)) + end) + end) + + it("allows the request when limit is not exceeding after processing a key", function() + stub_resty_global_throttle_process(LOCATION_CONFIG.limit - 3, nil, nil, + function() + assert_request_not_rejected(CONFIG, LOCATION_CONFIG) + end + ) + end) + + it("rejects with custom status code", function() + cache_rejection_decision(NAMESPACE, ngx.var.remote_addr, 0.3) + local config = util.deepcopy(CONFIG) + config.status_code = 503 + assert_request_rejected(config, LOCATION_CONFIG, { with_cache = true }) + end) +end) diff --git a/rootfs/etc/nginx/lua/test/monitor_test.lua b/rootfs/etc/nginx/lua/test/monitor_test.lua index 99e4697f0..78efed939 100644 --- a/rootfs/etc/nginx/lua/test/monitor_test.lua +++ b/rootfs/etc/nginx/lua/test/monitor_test.lua @@ -82,7 +82,6 @@ describe("Monitor", function() namespace = "default", ingress_name = "example", service_name = "http-svc", - proxy_alternative_upstream_name = "default-http-svc-canary-80", location_path = "/", request_method = "GET", @@ -93,8 +92,7 @@ describe("Monitor", function() upstream_addr = "10.10.0.1", upstream_connect_time = "0.01", - upstream_header_time = "0.02", - upstream_response_time = "0.03", + upstream_response_time = "0.02", upstream_response_length = "456", upstream_status = "200", } @@ -116,7 +114,6 @@ describe("Monitor", function() namespace = "default", ingress = "example", service = "http-svc", - canary = "default-http-svc-canary-80", path = "/", method = "GET", @@ -126,8 +123,7 @@ describe("Monitor", function() responseLength = 512, upstreamLatency = 0.01, - upstreamHeaderTime = 0.02, - upstreamResponseTime = 0.03, + upstreamResponseTime = 0.02, upstreamResponseLength = 456, }, { @@ -135,7 +131,6 @@ describe("Monitor", function() namespace = "default", ingress = "example", service = "http-svc", - canary = "default-http-svc-canary-80", path = "/", method = "POST", @@ -145,13 +140,12 @@ describe("Monitor", function() responseLength = 512, upstreamLatency = 0.01, - upstreamHeaderTime = 0.02, - upstreamResponseTime = 0.03, + upstreamResponseTime = 0.02, upstreamResponseLength = 456, }, }) - assert.stub(tcp_mock.connect).was_called_with(tcp_mock, "unix:/tmp/nginx/prometheus-nginx.socket") + assert.stub(tcp_mock.connect).was_called_with(tcp_mock, "unix:/tmp/prometheus-nginx.socket") assert.stub(tcp_mock.send).was_called_with(tcp_mock, expected_payload) assert.stub(tcp_mock.close).was_called_with(tcp_mock) end) diff --git a/rootfs/etc/nginx/lua/test/util/split.lua b/rootfs/etc/nginx/lua/test/util/split.lua new file mode 100644 index 000000000..3d3a6d7e9 --- /dev/null +++ b/rootfs/etc/nginx/lua/test/util/split.lua @@ -0,0 +1,15 @@ +local split = require("util.split") + + +describe("split", function() + it("get_last_value", function() + for _, case in ipairs({ + {"127.0.0.1:26157 : 127.0.0.1:26158", "127.0.0.1:26158"}, + {"127.0.0.1:26157, 127.0.0.1:26158", "127.0.0.1:26158"}, + {"127.0.0.1:26158", "127.0.0.1:26158"}, + }) do + local last = split.get_last_value(case[1]) + assert.equal(case[2], last) + end + end) +end) diff --git a/rootfs/etc/nginx/lua/test/util/split_test.lua b/rootfs/etc/nginx/lua/test/util/split_test.lua deleted file mode 100644 index d81a92c2d..000000000 --- a/rootfs/etc/nginx/lua/test/util/split_test.lua +++ /dev/null @@ -1,57 +0,0 @@ -local split = require("util.split") - -describe("split", function() - - describe("get_last_value", function() - it("splits value of an upstream variable and returns last value", function() - for _, case in ipairs({{"127.0.0.1:26157 : 127.0.0.1:26158", "127.0.0.1:26158"}, - {"127.0.0.1:26157, 127.0.0.1:26158", "127.0.0.1:26158"}, - {"127.0.0.1:26158", "127.0.0.1:26158"}}) do - local last = split.get_last_value(case[1]) - assert.equal(case[2], last) - end - end) - end) - - describe("split_string", function() - - it("returns empty array if input string is empty", function() - local splits, len = split.split_string("", ",") - assert.equal(0, len) - assert.is.truthy(splits) - end) - - it("returns empty array if input string is nil", function() - local splits, len = split.split_string(nil, ",") - assert.equal(0, len) - assert.is.truthy(splits) - end) - - it("returns empty array if delimiter is empty", function() - local splits, len = split.split_string("1,2", "") - assert.equal(0, len) - assert.is.truthy(splits) - end) - - it("returns empty array delimiter is nil", function() - local splits, len = split.split_string("1,2", nil) - assert.equal(0, len) - assert.is.truthy(splits) - end) - - it("returns array of 1 value if input string is not a list", function() - local splits, len = split.split_string("123", ",") - assert.equal(1, len) - assert.equal("123", splits[1]) - end) - - it("returns array of values extracted from the input string", function() - local splits, len = split.split_string("1,2,3", ",") - assert.equal(3, len) - assert.equal("1", splits[1]) - assert.equal("2", splits[2]) - assert.equal("3", splits[3]) - end) - - end) -end) diff --git a/rootfs/etc/nginx/lua/util.lua b/rootfs/etc/nginx/lua/util.lua index 1e4cd7c01..7389f3226 100644 --- a/rootfs/etc/nginx/lua/util.lua +++ b/rootfs/etc/nginx/lua/util.lua @@ -146,10 +146,6 @@ function _M.is_blank(str) return str == nil or string_len(str) == 0 end -function _M.string_to_bool(str) - return str == "true" -end - -- this implementation is taken from: -- https://github.com/luafun/luafun/blob/master/fun.lua#L33 -- SHA: 04c99f9c393e54a604adde4b25b794f48104e0d0 diff --git a/rootfs/etc/nginx/lua/util/split.lua b/rootfs/etc/nginx/lua/util/split.lua index 63edf0900..d5400ab57 100644 --- a/rootfs/etc/nginx/lua/util/split.lua +++ b/rootfs/etc/nginx/lua/util/split.lua @@ -65,19 +65,4 @@ function _M.split_upstream_addr(addrs_str) return host_and_ports end --- Splits string by delimiter. Returns array of parsed values and the length of the array. -function _M.split_string(what, delim) - local result = {} - local idx = 0 - - if what and delim and delim ~= "" then - for chunk in what:gmatch("([^" .. delim .. "]+)") do - idx = idx + 1 - result[idx] = chunk - end - end - - return result, idx -end - return _M diff --git a/rootfs/etc/nginx/nginx.conf b/rootfs/etc/nginx/nginx.conf index 7f60b846d..6f8e86b90 100644 --- a/rootfs/etc/nginx/nginx.conf +++ b/rootfs/etc/nginx/nginx.conf @@ -1,8 +1,6 @@ # A very simple nginx configuration file that forces nginx to start. -pid /tmp/nginx/nginx.pid; - -error_log stderr; +pid /tmp/nginx.pid; events {} http {} -daemon off; +daemon off; \ No newline at end of file diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl old mode 100644 new mode 100755 index 6b8e750b0..a18922895 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -21,6 +21,10 @@ load_module /etc/nginx/modules/ngx_http_brotli_filter_module.so; load_module /etc/nginx/modules/ngx_http_brotli_static_module.so; {{ end }} +{{ if (shouldLoadInfluxDBModule $servers) }} +load_module /etc/nginx/modules/ngx_http_influxdb_module.so; +{{ end }} + {{ if (shouldLoadAuthDigestModule $servers) }} load_module /etc/nginx/modules/ngx_http_auth_digest_module.so; {{ end }} @@ -29,8 +33,8 @@ load_module /etc/nginx/modules/ngx_http_auth_digest_module.so; load_module /etc/nginx/modules/ngx_http_modsecurity_module.so; {{ end }} -{{ if (shouldLoadOpentelemetryModule $cfg $servers) }} -load_module /etc/nginx/modules/otel_ngx_module.so; +{{ if (shouldLoadOpentracingModule $cfg $servers) }} +load_module /etc/nginx/modules/ngx_http_opentracing_module.so; {{ end }} daemon off; @@ -54,29 +58,82 @@ events { multi_accept {{ if $cfg.EnableMultiAccept }}on{{ else }}off{{ end }}; worker_connections {{ $cfg.MaxWorkerConnections }}; use epoll; - {{ range $index , $v := $cfg.DebugConnections }} - debug_connection {{ $v }}; - {{ end }} } http { - {{ if (shouldLoadOpentelemetryModule $cfg $servers) }} - opentelemetry_config {{ $cfg.OpentelemetryConfig }}; - {{ end }} - lua_package_path "/etc/nginx/lua/?.lua;;"; {{ buildLuaSharedDictionaries $cfg $servers }} - lua_shared_dict luaconfig 5m; + init_by_lua_block { + collectgarbage("collect") - init_by_lua_file /etc/nginx/lua/ngx_conf_init.lua; + -- init modules + local ok, res - init_worker_by_lua_file /etc/nginx/lua/ngx_conf_init_worker.lua; + ok, res = pcall(require, "lua_ingress") + if not ok then + error("require failed: " .. tostring(res)) + else + lua_ingress = res + lua_ingress.set_config({{ configForLua $all }}) + end + + ok, res = pcall(require, "configuration") + if not ok then + error("require failed: " .. tostring(res)) + else + configuration = res + configuration.prohibited_localhost_port = '{{ .StatusPort }}' + end + + ok, res = pcall(require, "balancer") + if not ok then + error("require failed: " .. tostring(res)) + else + balancer = res + end + + {{ if $all.EnableMetrics }} + ok, res = pcall(require, "monitor") + if not ok then + error("require failed: " .. tostring(res)) + else + monitor = res + end + {{ end }} + + ok, res = pcall(require, "certificate") + if not ok then + error("require failed: " .. tostring(res)) + else + certificate = res + certificate.is_ocsp_stapling_enabled = {{ $cfg.EnableOCSP }} + end + + ok, res = pcall(require, "plugins") + if not ok then + error("require failed: " .. tostring(res)) + else + plugins = res + end + -- load all plugins that'll be used here + plugins.init({ {{ range $idx, $plugin := $cfg.Plugins }}{{ if $idx }},{{ end }}{{ $plugin | quote }}{{ end }} }) + } + + init_worker_by_lua_block { + lua_ingress.init_worker() + balancer.init_worker() + {{ if $all.EnableMetrics }} + monitor.init_worker({{ $all.MonitorMaxBatchSize }}) + {{ end }} + + plugins.run() + } {{/* Enable the real_ip module only if we use either X-Forwarded headers or Proxy Protocol. */}} {{/* we use the value of the real IP for the geo_ip module */}} - {{ if or (or $cfg.UseForwardedHeaders $cfg.UseProxyProtocol) $cfg.EnableRealIP }} + {{ if or (or $cfg.UseForwardedHeaders $cfg.UseProxyProtocol) $cfg.EnableRealIp }} {{ if $cfg.UseProxyProtocol }} real_ip_header proxy_protocol; {{ else }} @@ -92,62 +149,53 @@ http { {{ if $all.Cfg.EnableModsecurity }} modsecurity on; - {{ if (not (empty $all.Cfg.ModsecuritySnippet)) }} - modsecurity_rules ' - {{ $all.Cfg.ModsecuritySnippet }} - '; - {{ else }} modsecurity_rules_file /etc/nginx/modsecurity/modsecurity.conf; - {{ end }} {{ if $all.Cfg.EnableOWASPCoreRules }} modsecurity_rules_file /etc/nginx/owasp-modsecurity-crs/nginx-modsecurity.conf; + {{ else if (not (empty $all.Cfg.ModsecuritySnippet)) }} + modsecurity_rules ' + {{ $all.Cfg.ModsecuritySnippet }} + '; {{ end }} {{ end }} + {{ if $cfg.UseGeoIP }} + {{/* databases used to determine the country depending on the client IP address */}} + {{/* http://nginx.org/en/docs/http/ngx_http_geoip_module.html */}} + {{/* this is require to calculate traffic for individual country using GeoIP in the status page */}} + geoip_country /etc/nginx/geoip/GeoIP.dat; + geoip_city /etc/nginx/geoip/GeoLiteCity.dat; + geoip_org /etc/nginx/geoip/GeoIPASNum.dat; + geoip_proxy_recursive on; + {{ end }} + {{ if $cfg.UseGeoIP2 }} # https://github.com/leev/ngx_http_geoip2_module#example-usage {{ range $index, $file := $all.MaxmindEditionFiles }} {{ if eq $file "GeoLite2-Country.mmdb" }} - geoip2 /etc/ingress-controller/geoip/GeoLite2-Country.mmdb { - {{ if (gt $cfg.GeoIP2AutoReloadMinutes 0) }} - auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; - {{ end }} + geoip2 /etc/nginx/geoip/GeoLite2-Country.mmdb { $geoip2_country_code source=$remote_addr country iso_code; $geoip2_country_name source=$remote_addr country names en; - $geoip2_country_geoname_id source=$remote_addr country geoname_id; - $geoip2_continent_code source=$remote_addr continent code; $geoip2_continent_name source=$remote_addr continent names en; - $geoip2_continent_geoname_id source=$remote_addr continent geoname_id; } {{ end }} {{ if eq $file "GeoIP2-Country.mmdb" }} - geoip2 /etc/ingress-controller/geoip/GeoIP2-Country.mmdb { - {{ if (gt $cfg.GeoIP2AutoReloadMinutes 0) }} - auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; - {{ end }} + geoip2 /etc/nginx/geoip/GeoIP2-Country.mmdb { $geoip2_country_code source=$remote_addr country iso_code; $geoip2_country_name source=$remote_addr country names en; - $geoip2_country_geoname_id source=$remote_addr country geoname_id; - $geoip2_continent_code source=$remote_addr continent code; $geoip2_continent_name source=$remote_addr continent names en; - $geoip2_continent_geoname_id source=$remote_addr continent geoname_id; } {{ end }} {{ if eq $file "GeoLite2-City.mmdb" }} - geoip2 /etc/ingress-controller/geoip/GeoLite2-City.mmdb { - {{ if (gt $cfg.GeoIP2AutoReloadMinutes 0) }} - auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; - {{ end }} + geoip2 /etc/nginx/geoip/GeoLite2-City.mmdb { $geoip2_city_country_code source=$remote_addr country iso_code; $geoip2_city_country_name source=$remote_addr country names en; - $geoip2_city_country_geoname_id source=$remote_addr country geoname_id; $geoip2_city source=$remote_addr city names en; - $geoip2_city_geoname_id source=$remote_addr city geoname_id; $geoip2_postal_code source=$remote_addr postal code; $geoip2_dma_code source=$remote_addr location metro_code; $geoip2_latitude source=$remote_addr location latitude; @@ -155,25 +203,16 @@ http { $geoip2_time_zone source=$remote_addr location time_zone; $geoip2_region_code source=$remote_addr subdivisions 0 iso_code; $geoip2_region_name source=$remote_addr subdivisions 0 names en; - $geoip2_region_geoname_id source=$remote_addr subdivisions 0 geoname_id; $geoip2_subregion_code source=$remote_addr subdivisions 1 iso_code; $geoip2_subregion_name source=$remote_addr subdivisions 1 names en; - $geoip2_subregion_geoname_id source=$remote_addr subdivisions 1 geoname_id; - $geoip2_city_continent_code source=$remote_addr continent code; - $geoip2_city_continent_name source=$remote_addr continent names en; } {{ end }} {{ if eq $file "GeoIP2-City.mmdb" }} - geoip2 /etc/ingress-controller/geoip/GeoIP2-City.mmdb { - {{ if (gt $cfg.GeoIP2AutoReloadMinutes 0) }} - auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; - {{ end }} + geoip2 /etc/nginx/geoip/GeoIP2-City.mmdb { $geoip2_city_country_code source=$remote_addr country iso_code; $geoip2_city_country_name source=$remote_addr country names en; - $geoip2_city_country_geoname_id source=$remote_addr country geoname_id; $geoip2_city source=$remote_addr city names en; - $geoip2_city_geoname_id source=$remote_addr city geoname_id; $geoip2_postal_code source=$remote_addr postal code; $geoip2_dma_code source=$remote_addr location metro_code; $geoip2_latitude source=$remote_addr location latitude; @@ -181,63 +220,43 @@ http { $geoip2_time_zone source=$remote_addr location time_zone; $geoip2_region_code source=$remote_addr subdivisions 0 iso_code; $geoip2_region_name source=$remote_addr subdivisions 0 names en; - $geoip2_region_geoname_id source=$remote_addr subdivisions 0 geoname_id; $geoip2_subregion_code source=$remote_addr subdivisions 1 iso_code; $geoip2_subregion_name source=$remote_addr subdivisions 1 names en; - $geoip2_subregion_geoname_id source=$remote_addr subdivisions 1 geoname_id; - $geoip2_city_continent_code source=$remote_addr continent code; - $geoip2_city_continent_name source=$remote_addr continent names en; } {{ end }} {{ if eq $file "GeoLite2-ASN.mmdb" }} - geoip2 /etc/ingress-controller/geoip/GeoLite2-ASN.mmdb { - {{ if (gt $cfg.GeoIP2AutoReloadMinutes 0) }} - auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; - {{ end }} + geoip2 /etc/nginx/geoip/GeoLite2-ASN.mmdb { $geoip2_asn source=$remote_addr autonomous_system_number; $geoip2_org source=$remote_addr autonomous_system_organization; } {{ end }} {{ if eq $file "GeoIP2-ASN.mmdb" }} - geoip2 /etc/ingress-controller/geoip/GeoIP2-ASN.mmdb { - {{ if (gt $cfg.GeoIP2AutoReloadMinutes 0) }} - auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; - {{ end }} + geoip2 /etc/nginx/geoip/GeoIP2-ASN.mmdb { $geoip2_asn source=$remote_addr autonomous_system_number; $geoip2_org source=$remote_addr autonomous_system_organization; } {{ end }} {{ if eq $file "GeoIP2-ISP.mmdb" }} - geoip2 /etc/ingress-controller/geoip/GeoIP2-ISP.mmdb { - {{ if (gt $cfg.GeoIP2AutoReloadMinutes 0) }} - auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; - {{ end }} - $geoip2_isp source=$remote_addr isp; - $geoip2_isp_org source=$remote_addr organization; - $geoip2_asn source=$remote_addr default=0 autonomous_system_number; + geoip2 /etc/nginx/geoip/GeoIP2-ISP.mmdb { + $geoip2_isp isp; + $geoip2_isp_org organization; } {{ end }} {{ if eq $file "GeoIP2-Connection-Type.mmdb" }} - geoip2 /etc/ingress-controller/geoip/GeoIP2-Connection-Type.mmdb { + geoip2 /etc/nginx/geoip/GeoIP2-Connection-Type.mmdb { $geoip2_connection_type connection_type; } {{ end }} {{ if eq $file "GeoIP2-Anonymous-IP.mmdb" }} - geoip2 /etc/ingress-controller/geoip/GeoIP2-Anonymous-IP.mmdb { - {{ if (gt $cfg.GeoIP2AutoReloadMinutes 0) }} - auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; - {{ end }} + geoip2 /etc/nginx/geoip/GeoIP2-Anonymous-IP.mmdb { $geoip2_is_anon source=$remote_addr is_anonymous; - $geoip2_is_anonymous source=$remote_addr default=0 is_anonymous; - $geoip2_is_anonymous_vpn source=$remote_addr default=0 is_anonymous_vpn; - $geoip2_is_hosting_provider source=$remote_addr default=0 is_hosting_provider; - $geoip2_is_public_proxy source=$remote_addr default=0 is_public_proxy; - $geoip2_is_tor_exit_node source=$remote_addr default=0 is_tor_exit_node; + $geoip2_is_hosting_provider source=$remote_addr is_hosting_provider; + $geoip2_is_public_proxy source=$remote_addr is_public_proxy; } {{ end }} @@ -246,10 +265,7 @@ http { {{ end }} aio threads; - - {{ if $cfg.EnableAioWrite }} aio_write on; - {{ end }} tcp_nopush on; tcp_nodelay on; @@ -261,9 +277,10 @@ http { keepalive_timeout {{ $cfg.KeepAlive }}s; keepalive_requests {{ $cfg.KeepAliveRequests }}; - client_body_temp_path /tmp/nginx/client-body; - fastcgi_temp_path /tmp/nginx/fastcgi-temp; - proxy_temp_path /tmp/nginx/proxy-temp; + client_body_temp_path /tmp/client-body; + fastcgi_temp_path /tmp/fastcgi-temp; + proxy_temp_path /tmp/proxy-temp; + ajp_temp_path /tmp/ajp-temp; client_header_buffer_size {{ $cfg.ClientHeaderBufferSize }}; client_header_timeout {{ $cfg.ClientHeaderTimeout }}s; @@ -271,19 +288,9 @@ http { client_body_buffer_size {{ $cfg.ClientBodyBufferSize }}; client_body_timeout {{ $cfg.ClientBodyTimeout }}s; - {{ if gt $cfg.GRPCBufferSizeKb 0 }} - grpc_buffer_size {{ $cfg.GRPCBufferSizeKb }}k; - {{ end }} - - {{ if and (ne $cfg.HTTP2MaxHeaderSize "") (ne $cfg.HTTP2MaxFieldSize "") }} http2_max_field_size {{ $cfg.HTTP2MaxFieldSize }}; http2_max_header_size {{ $cfg.HTTP2MaxHeaderSize }}; - {{ end }} - - {{ if (gt $cfg.HTTP2MaxRequests 0) }} http2_max_requests {{ $cfg.HTTP2MaxRequests }}; - {{ end }} - http2_max_concurrent_streams {{ $cfg.HTTP2MaxConcurrentStreams }}; types_hash_max_size 2048; @@ -303,7 +310,7 @@ http { limit_req_status {{ $cfg.LimitReqStatusCode }}; limit_conn_status {{ $cfg.LimitConnStatusCode }}; - {{ buildOpentelemetry $cfg $servers }} + {{ buildOpentracing $cfg $servers }} include /etc/nginx/mime.types; default_type {{ $cfg.DefaultType }}; @@ -311,16 +318,12 @@ http { {{ if $cfg.EnableBrotli }} brotli on; brotli_comp_level {{ $cfg.BrotliLevel }}; - brotli_min_length {{ $cfg.BrotliMinLength }}; brotli_types {{ $cfg.BrotliTypes }}; {{ end }} {{ if $cfg.UseGzip }} gzip on; gzip_comp_level {{ $cfg.GzipLevel }}; - {{- if $cfg.GzipDisable }} - gzip_disable "{{ $cfg.GzipDisable }}"; - {{- end }} gzip_http_version 1.1; gzip_min_length {{ $cfg.GzipMinLength}}; gzip_types {{ $cfg.GzipTypes }}; @@ -346,7 +349,7 @@ http { # $ingress_name # $service_name # $service_port - log_format upstreaminfo {{ if $cfg.LogFormatEscapeNone }}escape=none {{ else if $cfg.LogFormatEscapeJSON }}escape=json {{ end }}'{{ $cfg.LogFormatUpstream }}'; + log_format upstreaminfo {{ if $cfg.LogFormatEscapeJSON }}escape=json {{ end }}'{{ $cfg.LogFormatUpstream }}'; {{/* map urls that should not appear in access.log */}} {{/* http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log */}} @@ -362,7 +365,7 @@ http { {{ if $cfg.EnableSyslog }} access_log syslog:server={{ $cfg.SyslogHost }}:{{ $cfg.SyslogPort }} upstreaminfo if=$loggable; {{ else }} - access_log {{ or $cfg.HTTPAccessLogPath $cfg.AccessLogPath }} upstreaminfo {{ $cfg.AccessLogParams }} if=$loggable; + access_log {{ or $cfg.HttpAccessLogPath $cfg.AccessLogPath }} upstreaminfo {{ $cfg.AccessLogParams }} if=$loggable; {{ end }} {{ end }} @@ -424,7 +427,7 @@ http { # turn on session caching to drastically improve performance {{ if $cfg.SSLSessionCache }} - ssl_session_cache shared:SSL:{{ $cfg.SSLSessionCacheSize }}; + ssl_session_cache builtin:1000 shared:SSL:{{ $cfg.SSLSessionCacheSize }}; ssl_session_timeout {{ $cfg.SSLSessionTimeout }}; {{ end }} @@ -432,7 +435,7 @@ http { ssl_session_tickets {{ if $cfg.SSLSessionTickets }}on{{ else }}off{{ end }}; {{ if not (empty $cfg.SSLSessionTicketKey ) }} - ssl_session_ticket_key /etc/ingress-controller/tickets.key; + ssl_session_ticket_key /etc/nginx/tickets.key; {{ end }} # slightly reduce the time-to-first-byte @@ -455,14 +458,10 @@ http { ssl_certificate {{ $cfg.DefaultSSLCertificate.PemFileName }}; ssl_certificate_key {{ $cfg.DefaultSSLCertificate.PemFileName }}; - {{ if and $cfg.CustomHTTPErrors (not $cfg.DisableProxyInterceptErrors) }} + {{ if gt (len $cfg.CustomHTTPErrors) 0 }} proxy_intercept_errors on; {{ end }} - {{ if $cfg.RelativeRedirects }} - absolute_redirect off; - {{ end }} - {{ range $errCode := $cfg.CustomHTTPErrors }} error_page {{ $errCode }} = @custom_upstream-default-backend_{{ $errCode }};{{ end }} @@ -494,11 +493,13 @@ http { server 0.0.0.1; # placeholder - balancer_by_lua_file /etc/nginx/lua/nginx/ngx_conf_balancer.lua; + balancer_by_lua_block { + balancer.balance() + } {{ if (gt $cfg.UpstreamKeepaliveConnections 0) }} keepalive {{ $cfg.UpstreamKeepaliveConnections }}; - keepalive_time {{ $cfg.UpstreamKeepaliveTime }}; + keepalive_timeout {{ $cfg.UpstreamKeepaliveTimeout }}s; keepalive_requests {{ $cfg.UpstreamKeepaliveRequests }}; {{ end }} @@ -506,14 +507,14 @@ http { {{ range $rl := (filterRateLimits $servers ) }} # Ratelimit {{ $rl.Name }} - geo $remote_addr $allowlist_{{ $rl.ID }} { + geo $remote_addr $whitelist_{{ $rl.ID }} { default 0; - {{ range $ip := $rl.Allowlist }} + {{ range $ip := $rl.Whitelist }} {{ $ip }} 1;{{ end }} } # Ratelimit {{ $rl.Name }} - map $allowlist_{{ $rl.ID }} $limit_{{ $rl.ID }} { + map $whitelist_{{ $rl.ID }} $limit_{{ $rl.ID }} { 0 {{ $cfg.LimitConnZoneVariable }}; 1 ""; } @@ -526,7 +527,7 @@ http { {{ end }} # Cache for internal auth checks - proxy_cache_path /tmp/nginx/nginx-cache-auth levels=1:2 keys_zone=auth_cache:10m max_size=128m inactive=30m use_temp_path=off; + proxy_cache_path /tmp/nginx-cache-auth levels=1:2 keys_zone=auth_cache:10m max_size=128m inactive=30m use_temp_path=off; # Global filters {{ range $ip := $cfg.BlockCIDRs }}deny {{ trimSpace $ip }}; @@ -559,7 +560,9 @@ http { {{ buildHTTPListener $all $redirect.From }} {{ buildHTTPSListener $all $redirect.From }} - ssl_certificate_by_lua_file /etc/nginx/lua/nginx/ngx_conf_certificate.lua; + ssl_certificate_by_lua_block { + certificate.call() + } {{ if gt (len $cfg.BlockUserAgents) 0 }} if ($block_ua) { @@ -572,7 +575,19 @@ http { } {{ end }} - set_by_lua_file $redirect_to /etc/nginx/lua/nginx/ngx_srv_redirect.lua {{ $redirect.To }}; + set_by_lua_block $redirect_to { + local request_uri = ngx.var.request_uri + if string.sub(request_uri, -1) == "/" then + request_uri = string.sub(request_uri, 1, -2) + end + + {{ if ne $all.ListenPorts.HTTPS 443 }} + {{ $redirect_port := (printf ":%v" $all.ListenPorts.HTTPS) }} + return string.format("%s://%s%s%s", ngx.var.scheme, "{{ $redirect.To }}", "{{ $redirect_port }}", request_uri) + {{ else }} + return string.format("%s://%s%s", ngx.var.scheme, "{{ $redirect.To }}", request_uri) + {{ end }} + } return {{ $all.Cfg.HTTPRedirectCode }} $redirect_to; } @@ -580,33 +595,11 @@ http { {{ end }} {{ range $server := $servers }} - {{ range $location := $server.Locations }} - {{ $applyGlobalAuth := shouldApplyGlobalAuth $location $all.Cfg.GlobalExternalAuth.URL }} - {{ $applyAuthUpstream := shouldApplyAuthUpstream $location $all.Cfg }} - {{ if and (eq $applyAuthUpstream true) (eq $applyGlobalAuth false) }} - ## start auth upstream {{ $server.Hostname }}{{ $location.Path }} - upstream {{ buildAuthUpstreamName $location $server.Hostname }} { - {{- $externalAuth := $location.ExternalAuth }} - server {{ extractHostPort $externalAuth.URL }}; - keepalive {{ $externalAuth.KeepaliveConnections }}; - keepalive_requests {{ $externalAuth.KeepaliveRequests }}; - keepalive_timeout {{ $externalAuth.KeepaliveTimeout }}s; - } - ## end auth upstream {{ $server.Hostname }}{{ $location.Path }} - {{ end }} - {{ end }} - {{ end }} - - {{ range $server := $servers }} ## start server {{ $server.Hostname }} server { server_name {{ buildServerName $server.Hostname }} {{range $server.Aliases }}{{ . }} {{ end }}; - {{ if $cfg.UseHTTP2 }} - http2 on; - {{ end }} - {{ if gt (len $cfg.BlockUserAgents) 0 }} if ($block_ua) { return 403; @@ -625,7 +618,7 @@ http { {{ $cfg.ServerSnippet }} {{ end }} - {{ template "CUSTOM_ERRORS" (buildCustomErrorDeps "upstream-default-backend" $cfg.CustomHTTPErrors $all.EnableMetrics $cfg.EnableModsecurity) }} + {{ template "CUSTOM_ERRORS" (buildCustomErrorDeps "upstream-default-backend" $cfg.CustomHTTPErrors $all.EnableMetrics) }} } ## end server {{ $server.Hostname }} @@ -646,11 +639,6 @@ http { # default server, used for NGINX healthcheck and access to nginx stats server { - # Ensure that modsecurity will not run on an internal location as this is not accessible from outside - {{ if $all.Cfg.EnableModsecurity }} - modsecurity off; - {{ end }} - listen 127.0.0.1:{{ .StatusPort }}; set $proxy_upstream_name "internal"; @@ -659,15 +647,26 @@ http { access_log off; - {{ if $cfg.EnableOpentelemetry }} - opentelemetry off; + {{ if $cfg.EnableOpentracing }} + opentracing off; {{ end }} + location {{ $healthzURI }} { return 200; } location /is-dynamic-lb-initialized { - content_by_lua_file /etc/nginx/lua/nginx/ngx_conf_is_dynamic_lb_initialized.lua; + content_by_lua_block { + local configuration = require("configuration") + local backend_data = configuration.get_backends_data() + if not backend_data then + ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + return + end + + ngx.say("OK") + ngx.exit(ngx.HTTP_OK) + } } location {{ .StatusPath }} { @@ -675,15 +674,19 @@ http { } location /configuration { - client_max_body_size {{ luaConfigurationRequestBodySize $cfg }}; - client_body_buffer_size {{ luaConfigurationRequestBodySize $cfg }}; + client_max_body_size {{ luaConfigurationRequestBodySize $cfg }}m; + client_body_buffer_size {{ luaConfigurationRequestBodySize $cfg }}m; proxy_buffering off; - content_by_lua_file /etc/nginx/lua/nginx/ngx_conf_configuration.lua; + content_by_lua_block { + configuration.call() + } } location / { - return 404; + content_by_lua_block { + ngx.exit(ngx.HTTP_NOT_FOUND) + } } } } @@ -692,12 +695,40 @@ stream { lua_package_path "/etc/nginx/lua/?.lua;/etc/nginx/lua/vendor/?.lua;;"; lua_shared_dict tcp_udp_configuration_data 5M; - - {{ buildResolvers $cfg.Resolver $cfg.DisableIpv6DNS }} - init_by_lua_file /etc/nginx/lua/ngx_conf_init_stream.lua; + init_by_lua_block { + collectgarbage("collect") - init_worker_by_lua_file /etc/nginx/lua/nginx/ngx_conf_init_tcp_udp.lua; + -- init modules + local ok, res + + ok, res = pcall(require, "configuration") + if not ok then + error("require failed: " .. tostring(res)) + else + configuration = res + end + + ok, res = pcall(require, "tcp_udp_configuration") + if not ok then + error("require failed: " .. tostring(res)) + else + tcp_udp_configuration = res + tcp_udp_configuration.prohibited_localhost_port = '{{ .StatusPort }}' + + end + + ok, res = pcall(require, "tcp_udp_balancer") + if not ok then + error("require failed: " .. tostring(res)) + else + tcp_udp_balancer = res + end + } + + init_worker_by_lua_block { + tcp_udp_balancer.init_worker() + } lua_add_variable $proxy_upstream_name; @@ -709,9 +740,9 @@ stream { access_log {{ or $cfg.StreamAccessLogPath $cfg.AccessLogPath }} log_stream {{ $cfg.AccessLogParams }}; {{ end }} - error_log {{ $cfg.ErrorLogPath }} {{ $cfg.ErrorLogLevel }}; - {{ if $cfg.EnableRealIP }} + + {{ if $cfg.EnableRealIp }} {{ range $trusted_ip := $cfg.ProxyRealIPCIDR }} set_real_ip_from {{ $trusted_ip }}; {{ end }} @@ -719,7 +750,10 @@ stream { upstream upstream_balancer { server 0.0.0.1:1234; # placeholder - balancer_by_lua_file /etc/nginx/lua/nginx/ngx_conf_balancer_tcp_udp.lua; + + balancer_by_lua_block { + tcp_udp_balancer.balance() + } } server { @@ -727,7 +761,9 @@ stream { access_log off; - content_by_lua_file /etc/nginx/lua/nginx/ngx_conf_content_tcp_udp.lua; + content_by_lua_block { + tcp_udp_configuration.call() + } } # TCP services @@ -788,27 +824,16 @@ stream { proxy_pass upstream_balancer; } {{ end }} - - # Stream Snippets - {{ range $snippet := .StreamSnippets }} - {{ $snippet }} - {{ end }} } {{/* definition of templates to avoid repetitions */}} {{ define "CUSTOM_ERRORS" }} {{ $enableMetrics := .EnableMetrics }} - {{ $modsecurityEnabled := .ModsecurityEnabled }} {{ $upstreamName := .UpstreamName }} {{ range $errCode := .ErrorCodes }} location @custom_{{ $upstreamName }}_{{ $errCode }} { internal; - # Ensure that modsecurity will not run on custom error pages or they might be blocked - {{ if $modsecurityEnabled }} - modsecurity off; - {{ end }} - proxy_intercept_errors off; proxy_set_header X-Code {{ $errCode }}; @@ -819,7 +844,6 @@ stream { proxy_set_header X-Service-Name $service_name; proxy_set_header X-Service-Port $service_port; proxy_set_header X-Request-ID $req_id; - proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header Host $best_http_host; set $proxy_upstream_name {{ $upstreamName | quote }}; @@ -827,9 +851,11 @@ stream { rewrite (.*) / break; proxy_pass http://upstream_balancer; - {{ if $enableMetrics }} - log_by_lua_file /etc/nginx/lua/nginx/ngx_conf_log.lua; - {{ end }} + log_by_lua_block { + {{ if $enableMetrics }} + monitor.call() + {{ end }} + } } {{ end }} {{ end }} @@ -838,24 +864,8 @@ stream { {{ define "CORS" }} {{ $cors := .CorsConfig }} # Cors Preflight methods needs additional options and different Return Code - {{ if $cors.CorsAllowOrigin }} - {{ buildCorsOriginRegex $cors.CorsAllowOrigin }} - {{ end }} if ($request_method = 'OPTIONS') { - set $cors ${cors}options; - } - - if ($cors = "true") { - more_set_headers 'Access-Control-Allow-Origin: $http_origin'; - {{ if $cors.CorsAllowCredentials }} more_set_headers 'Access-Control-Allow-Credentials: {{ $cors.CorsAllowCredentials }}'; {{ end }} - more_set_headers 'Access-Control-Allow-Methods: {{ $cors.CorsAllowMethods }}'; - more_set_headers 'Access-Control-Allow-Headers: {{ $cors.CorsAllowHeaders }}'; - {{ if not (empty $cors.CorsExposeHeaders) }} more_set_headers 'Access-Control-Expose-Headers: {{ $cors.CorsExposeHeaders }}'; {{ end }} - more_set_headers 'Access-Control-Max-Age: {{ $cors.CorsMaxAge }}'; - } - - if ($cors = "trueoptions") { - more_set_headers 'Access-Control-Allow-Origin: $http_origin'; + more_set_headers 'Access-Control-Allow-Origin: {{ $cors.CorsAllowOrigin }}'; {{ if $cors.CorsAllowCredentials }} more_set_headers 'Access-Control-Allow-Credentials: {{ $cors.CorsAllowCredentials }}'; {{ end }} more_set_headers 'Access-Control-Allow-Methods: {{ $cors.CorsAllowMethods }}'; more_set_headers 'Access-Control-Allow-Headers: {{ $cors.CorsAllowHeaders }}'; @@ -865,6 +875,11 @@ stream { more_set_headers 'Content-Length: 0'; return 204; } + + more_set_headers 'Access-Control-Allow-Origin: {{ $cors.CorsAllowOrigin }}'; + {{ if $cors.CorsAllowCredentials }} more_set_headers 'Access-Control-Allow-Credentials: {{ $cors.CorsAllowCredentials }}'; {{ end }} + {{ if not (empty $cors.CorsExposeHeaders) }} more_set_headers 'Access-Control-Expose-Headers: {{ $cors.CorsExposeHeaders }}'; {{ end }} + {{ end }} {{/* definition of server-template to avoid repetitions with server-alias */}} @@ -877,19 +892,9 @@ stream { set $proxy_upstream_name "-"; - {{ if not ( empty $server.CertificateAuth.MatchCN ) }} - {{ if gt (len $server.CertificateAuth.MatchCN) 0 }} - if ( $ssl_client_s_dn !~ {{ $server.CertificateAuth.MatchCN }} ) { - return 403 "client certificate unauthorized"; + ssl_certificate_by_lua_block { + certificate.call() } - {{ end }} - {{ end }} - - {{ if eq $server.Hostname "_" }} - ssl_reject_handshake {{ if $all.Cfg.SSLRejectHandshake }}on{{ else }}off{{ end }}; - {{ end }} - - ssl_certificate_by_lua_file /etc/nginx/lua/nginx/ngx_conf_certificate.lua; {{ if not (empty $server.AuthTLSError) }} # {{ $server.AuthTLSError }} @@ -944,7 +949,7 @@ stream { {{ end }} {{ range $errorLocation := (buildCustomErrorLocationsPerServer $server) }} - {{ template "CUSTOM_ERRORS" (buildCustomErrorDeps $errorLocation.UpstreamName $errorLocation.Codes $all.EnableMetrics $all.Cfg.EnableModsecurity) }} + {{ template "CUSTOM_ERRORS" (buildCustomErrorDeps $errorLocation.UpstreamName $errorLocation.Codes $all.EnableMetrics) }} {{ end }} {{ buildMirrorLocations $server.Locations }} @@ -955,7 +960,6 @@ stream { {{ $proxySetHeader := proxySetHeader $location }} {{ $authPath := buildAuthLocation $location $all.Cfg.GlobalExternalAuth.URL }} {{ $applyGlobalAuth := shouldApplyGlobalAuth $location $all.Cfg.GlobalExternalAuth.URL }} - {{ $applyAuthUpstream := shouldApplyAuthUpstream $location $all.Cfg }} {{ $externalAuth := $location.ExternalAuth }} {{ if eq $applyGlobalAuth true }} @@ -972,25 +976,18 @@ stream { location = {{ $authPath }} { internal; - {{ if (or $all.Cfg.EnableOpentelemetry $location.Opentelemetry.Enabled) }} - opentelemetry on; - opentelemetry_propagate; - {{ end }} - - {{ if not $all.Cfg.EnableAuthAccessLog }} - access_log off; - {{ end }} - - # Ensure that modsecurity will not run on an internal location as this is not accessible from outside - {{ if $all.Cfg.EnableModsecurity }} - modsecurity off; + {{ if (or $all.Cfg.EnableOpentracing $location.Opentracing.Enabled) }} + opentracing on; + opentracing_propagate_context; {{ end }} {{ if $externalAuth.AuthCacheKey }} set $tmp_cache_key '{{ $server.Hostname }}{{ $authPath }}{{ $externalAuth.AuthCacheKey }}'; set $cache_key ''; - rewrite_by_lua_file /etc/nginx/lua/nginx/ngx_conf_rewrite_auth.lua; + rewrite_by_lua_block { + ngx.var.cache_key = ngx.encode_base64(ngx.sha1_bin(ngx.var.tmp_cache_key)) + } proxy_cache auth_cache; @@ -1041,8 +1038,8 @@ stream { {{ end }} proxy_buffer_size {{ $location.Proxy.BufferSize }}; proxy_buffers {{ $location.Proxy.BuffersNumber }} {{ $location.Proxy.BufferSize }}; - proxy_busy_buffers_size {{ $location.Proxy.BusyBuffersSize }}; proxy_request_buffering {{ $location.Proxy.RequestBuffering }}; + proxy_http_version {{ $location.Proxy.ProxyHTTPVersion }}; proxy_ssl_server_name on; proxy_pass_request_headers on; @@ -1071,19 +1068,7 @@ stream { {{ $externalAuth.AuthSnippet }} {{ end }} - {{ if and (eq $applyAuthUpstream true) (eq $applyGlobalAuth false) }} - {{ $authUpstreamName := buildAuthUpstreamName $location $server.Hostname }} - # The target is an upstream with HTTP keepalive, that is why the - # Connection header is cleared and the HTTP version is set to 1.1 as - # the Nginx documentation suggests: - # http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive - proxy_http_version 1.1; - proxy_set_header Connection ""; - set $target {{ changeHostPort $externalAuth.URL $authUpstreamName }}; - {{ else }} - proxy_http_version {{ $location.Proxy.ProxyHTTPVersion }}; set $target {{ $externalAuth.URL }}; - {{ end }} proxy_pass $target; } {{ end }} @@ -1095,15 +1080,6 @@ stream { add_header Set-Cookie $auth_cookie; - {{ if $location.CorsConfig.CorsEnabled }} - {{ template "CORS" $location }} - {{ end }} - - # Ensure that modsecurity will not run on an internal location as this is not accessible from outside - {{ if $all.Cfg.EnableModsecurity }} - modsecurity off; - {{ end }} - return 302 {{ buildAuthSignURL $externalAuth.SigninURL $externalAuth.SigninURLRedirectParam }}; } {{ end }} @@ -1116,21 +1092,44 @@ stream { set $service_name {{ $ing.Service | quote }}; set $service_port {{ $ing.ServicePort | quote }}; set $location_path {{ $ing.Path | escapeLiteralDollar | quote }}; + set $global_rate_limit_exceeding n; - {{ buildOpentelemetryForLocation $all.Cfg.EnableOpentelemetry $all.Cfg.OpentelemetryTrustIncomingSpan $location }} + {{ buildOpentracingForLocation $all.Cfg.EnableOpentracing $location }} {{ if $location.Mirror.Source }} mirror {{ $location.Mirror.Source }}; mirror_request_body {{ $location.Mirror.RequestBody }}; {{ end }} - {{ locationConfigForLua $location $all }} + rewrite_by_lua_block { + lua_ingress.rewrite({{ locationConfigForLua $location $all }}) + balancer.rewrite() + plugins.run() + } - rewrite_by_lua_file /etc/nginx/lua/nginx/ngx_rewrite.lua; + # be careful with `access_by_lua_block` and `satisfy any` directives as satisfy any + # will always succeed when there's `access_by_lua_block` that does not have any lua code doing `ngx.exit(ngx.DECLINED)` + # other authentication method such as basic auth or external auth useless - all requests will be allowed. + #access_by_lua_block { + #} - header_filter_by_lua_file /etc/nginx/lua/nginx/ngx_conf_srv_hdr_filter.lua; + header_filter_by_lua_block { + lua_ingress.header() + plugins.run() + } - log_by_lua_file /etc/nginx/lua/nginx/ngx_conf_log_block.lua; + body_filter_by_lua_block { + plugins.run() + } + + log_by_lua_block { + balancer.log() + {{ if $all.EnableMetrics }} + monitor.call() + {{ end }} + + plugins.run() + } {{ if not $location.Logs.Access }} access_log off; @@ -1165,48 +1164,22 @@ stream { {{ buildModSecurityForLocation $all.Cfg $location }} {{ if isLocationAllowed $location }} - {{ if gt (len $location.Denylist.CIDR) 0 }} - {{ range $ip := $location.Denylist.CIDR }} - deny {{ $ip }};{{ end }} - {{ end }} - {{ if gt (len $location.Allowlist.CIDR) 0 }} - {{ range $ip := $location.Allowlist.CIDR }} + {{ if gt (len $location.Whitelist.CIDR) 0 }} + {{ range $ip := $location.Whitelist.CIDR }} allow {{ $ip }};{{ end }} deny all; {{ end }} - {{ if $location.CorsConfig.CorsEnabled }} - {{ template "CORS" $location }} - {{ end }} - {{ if not (isLocationInLocationList $location $all.Cfg.NoAuthLocations) }} {{ if $authPath }} # this location requires authentication - {{ if and (eq $applyAuthUpstream true) (eq $applyGlobalAuth false) }} - set $auth_cookie ''; - add_header Set-Cookie $auth_cookie; - {{- range $line := buildAuthResponseHeaders $proxySetHeader $externalAuth.ResponseHeaders true }} - {{ $line }} - {{- end }} - # `auth_request` module does not support HTTP keepalives in upstream block: - # https://trac.nginx.org/nginx/ticket/1579 - set $auth_path '{{ $authPath }}'; - set $auth_keepalive_share_vars {{ $externalAuth.KeepaliveShareVars }}; - set $auth_response_headers '{{ buildAuthUpstreamLuaHeaders $externalAuth.ResponseHeaders }}'; - access_by_lua_file /etc/nginx/lua/nginx/ngx_conf_external_auth.lua; - {{ else }} auth_request {{ $authPath }}; auth_request_set $auth_cookie $upstream_http_set_cookie; - {{ if $externalAuth.AlwaysSetCookie }} - add_header Set-Cookie $auth_cookie always; - {{ else }} add_header Set-Cookie $auth_cookie; - {{ end }} - {{- range $line := buildAuthResponseHeaders $proxySetHeader $externalAuth.ResponseHeaders false }} + {{- range $line := buildAuthResponseHeaders $externalAuth.ResponseHeaders }} {{ $line }} {{- end }} {{ end }} - {{ end }} {{ if $externalAuth.SigninURL }} set_escape_uri $escaped_request_uri $request_uri; @@ -1221,7 +1194,7 @@ stream { auth_digest {{ $location.BasicDigestAuth.Realm | quote }}; auth_digest_user_file {{ $location.BasicDigestAuth.File }}; {{ end }} - {{ $proxySetHeader }} Authorization ""; + proxy_set_header Authorization ""; {{ end }} {{ end }} @@ -1230,6 +1203,12 @@ stream { {{ range $limit := $limits }} {{ $limit }}{{ end }} + {{ if $location.CorsConfig.CorsEnabled }} + {{ template "CORS" $location }} + {{ end }} + + {{ buildInfluxDB $location.InfluxDB }} + {{ if isValidByteSize $location.Proxy.BodySize true }} client_max_body_size {{ $location.Proxy.BodySize }}; {{ end }} @@ -1238,11 +1217,13 @@ stream { {{ end }} {{/* By default use vhost as Host to upstream, but allow overrides */}} + {{ if not (eq $proxySetHeader "grpc_set_header") }} {{ if not (empty $location.UpstreamVhost) }} {{ $proxySetHeader }} Host {{ $location.UpstreamVhost | quote }}; {{ else }} {{ $proxySetHeader }} Host $best_http_host; {{ end }} + {{ end }} # Pass the extracted client certificate to the backend {{ if not (empty $server.CertificateAuth.CAFileName) }} @@ -1297,7 +1278,6 @@ stream { proxy_buffering {{ $location.Proxy.ProxyBuffering }}; proxy_buffer_size {{ $location.Proxy.BufferSize }}; proxy_buffers {{ $location.Proxy.BuffersNumber }} {{ $location.Proxy.BufferSize }}; - proxy_busy_buffers_size {{ $location.Proxy.BusyBuffersSize }}; {{ if isValidByteSize $location.Proxy.ProxyMaxTempFileSize true }} proxy_max_temp_file_size {{ $location.Proxy.ProxyMaxTempFileSize }}; {{ end }} @@ -1312,13 +1292,6 @@ stream { proxy_next_upstream_timeout {{ $location.Proxy.NextUpstreamTimeout }}; proxy_next_upstream_tries {{ $location.Proxy.NextUpstreamTries }}; - {{ if or (eq $location.BackendProtocol "GRPC") (eq $location.BackendProtocol "GRPCS") }} - # Grpc settings - grpc_connect_timeout {{ $location.Proxy.ConnectTimeout }}s; - grpc_send_timeout {{ $location.Proxy.SendTimeout }}s; - grpc_read_timeout {{ $location.Proxy.ReadTimeout }}s; - {{ end }} - {{/* Add any additional configuration defined */}} {{ $location.ConfigurationSnippet }} @@ -1327,13 +1300,6 @@ stream { {{ $all.Cfg.LocationSnippet }} {{ end }} - {{ if $location.CustomHeaders }} - # Custom Response Headers - {{ range $k, $v := $location.CustomHeaders.Headers }} - more_set_headers {{ printf "%s: %s" $k $v | escapeLiteralDollar | quote }}; - {{ end }} - {{ end }} - {{/* if we are sending the request to a custom default backend, we add the required headers */}} {{ if (hasPrefix $location.Backend "custom-default-backend-") }} proxy_set_header X-Code 503; @@ -1349,12 +1315,8 @@ stream { satisfy {{ $location.Satisfy }}; {{ end }} - {{ if $location.Redirect.Relative }} - absolute_redirect off; - {{ end }} - {{/* if a location-specific error override is set, add the proxy_intercept here */}} - {{ if and $location.CustomHTTPErrors (not $location.DisableProxyInterceptErrors) }} + {{ if $location.CustomHTTPErrors }} # Custom error pages per ingress proxy_intercept_errors on; {{ end }} @@ -1413,9 +1375,8 @@ stream { {{ if eq $server.Hostname "_" }} # health checks in cloud providers require the use of port {{ $all.ListenPorts.HTTP }} location {{ $all.HealthzURI }} { - - {{ if $all.Cfg.EnableOpentelemetry }} - opentelemetry off; + {{ if $all.Cfg.EnableOpentracing }} + opentracing off; {{ end }} access_log off; @@ -1425,9 +1386,8 @@ stream { # this is required to avoid error if nginx is being monitored # with an external software (like sysdig) location /nginx_status { - - {{ if $all.Cfg.EnableOpentelemetry }} - opentelemetry off; + {{ if $all.Cfg.EnableOpentracing }} + opentracing off; {{ end }} {{ range $v := $all.NginxStatusIpv4Whitelist }} diff --git a/.ct.yaml b/rootfs/ingress-controller/clean-nginx-conf.sh old mode 100644 new mode 100755 similarity index 56% rename from .ct.yaml rename to rootfs/ingress-controller/clean-nginx-conf.sh index 229030219..07900981e --- a/.ct.yaml +++ b/rootfs/ingress-controller/clean-nginx-conf.sh @@ -1,4 +1,6 @@ -# Copyright 2024 The Kubernetes Authors. +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,16 +13,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ---- -remote: origin -target-branch: main -validate-maintainers: false -check-version-increment: false +# This script removes consecutive empty lines in nginx.conf +# Using sed is more simple than using a go regex -chart-repos: - - ingress-nginx=https://kubernetes.github.io/ingress-nginx -helm-extra-args: --timeout 800s +# Sed commands: +# 1. remove the return carrier character/s +# 2. remove empty lines +# 3. replace multiple empty lines -chart-dirs: - - charts +SCRIPT_ROOT=$(dirname ${BASH_SOURCE}) + +sed -e 's/\r//g' | sed -e 's/^ *$/\'$'\n/g' | sed -e '/^$/{N;/^\n$/D;}' | ${SCRIPT_ROOT}/indent.sh diff --git a/rootfs/nginx-chroot-wrapper.sh b/rootfs/ingress-controller/indent.sh similarity index 67% rename from rootfs/nginx-chroot-wrapper.sh rename to rootfs/ingress-controller/indent.sh index 0c8bdf3b7..83c0f0005 100755 --- a/rootfs/nginx-chroot-wrapper.sh +++ b/rootfs/ingress-controller/indent.sh @@ -1,6 +1,6 @@ -#!/bin/bash +#!/usr/bin/awk -f -# Copyright 2022 The Kubernetes Authors. +# Copyright 2017 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,5 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -cat /etc/resolv.conf > /chroot/etc/resolv.conf -unshare -S 101 -R /chroot nginx "$@" +# Credits to https://evasive.ru/f29bd7ebacf24a50c582f973a55eee28.html + +{sub(/^[ \t]+/,"");idx=0} +/\{/{ctx++;idx=1} +/\}/{ctx--} +{id="";for(i=idx;i> in /Makefile from inside /test/e2e-image/Makefile" make -C $(DIR)/../../ e2e-test-binary - echo "..done building e2e-test-binary from /test/e2e-image/Makefile" cp $(DIR)/../e2e/e2e.test . cp $(DIR)/../e2e/wait-for-nginx.sh . @@ -16,7 +12,6 @@ image: cp -R $(DIR)/../../test/e2e/settings/ocsp/* . docker build \ - --build-arg E2E_BASE_IMAGE=$(E2E_BASE_IMAGE) \ --tag nginx-ingress-controller:e2e . clean: @@ -24,4 +19,4 @@ clean: docker rmi -f nginx-ingress-controller:e2e || true -.PHONY: image clean +.PHONY: image clean \ No newline at end of file diff --git a/test/e2e-image/e2e.sh b/test/e2e-image/e2e.sh index f8ecd5337..240803c63 100755 --- a/test/e2e-image/e2e.sh +++ b/test/e2e-image/e2e.sh @@ -14,39 +14,42 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -eu +set -e NC='\e[0m' BGREEN='\e[32m' +SLOW_E2E_THRESHOLD=${SLOW_E2E_THRESHOLD:-5} +FOCUS=${FOCUS:-.*} E2E_NODES=${E2E_NODES:-5} E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS:-""} -reportFile="report-e2e-test-suite.xml" ginkgo_args=( - "--fail-fast" - "--flake-attempts=2" - "--junit-report=${reportFile}" - "--nodes=${E2E_NODES}" - "--poll-progress-after=180s" - "--randomize-all" - "--show-node-events" - "--succinct" - "--timeout=75m" + "-randomizeAllSpecs" + "-flakeAttempts=2" + "-progress" + "-slowSpecThreshold=${SLOW_E2E_THRESHOLD}" + "-succinct" + "-timeout=45m" # Suite timeout should be lower than Prow job timeout to avoid abrupt termination ) -if [ -n "${FOCUS}" ]; then - ginkgo_args+=("--focus=${FOCUS}") +echo -e "${BGREEN}Running e2e test suite (FOCUS=${FOCUS})...${NC}" +ginkgo "${ginkgo_args[@]}" \ + -focus="${FOCUS}" \ + -skip="\[Serial\]|\[MemoryLeak\]" \ + -nodes="${E2E_NODES}" \ + /e2e.test + +echo -e "${BGREEN}Running e2e test suite with tests that require serial execution...${NC}" +ginkgo "${ginkgo_args[@]}" \ + -focus="\[Serial\]" \ + -skip="\[MemoryLeak\]" \ + /e2e.test + +if [[ ${E2E_CHECK_LEAKS} != "" ]]; then + echo -e "${BGREEN}Running e2e test suite with tests that check for memory leaks...${NC}" + ginkgo "${ginkgo_args[@]}" \ + -focus="\[MemoryLeak\]" \ + -skip="\[Serial\]" \ + /e2e.test fi - -if [ -z "${E2E_CHECK_LEAKS}" ]; then - ginkgo_args+=("--skip=\[Memory Leak\]") -fi - -echo -e "${BGREEN}Running e2e test suite...${NC}" -(set -x; ginkgo "${ginkgo_args[@]}" /e2e.test) - -# Create configMap out of a compressed report file for extraction later -gzip -k ${reportFile} -kubectl create cm ${reportFile}.gz --from-file ${reportFile}.gz -kubectl label cm ${reportFile}.gz junitreport=true diff --git a/test/e2e-image/namespace-overlays/admission/values.yaml b/test/e2e-image/namespace-overlays/admission/values.yaml index d423217db..b88e8a02e 100644 --- a/test/e2e-image/namespace-overlays/admission/values.yaml +++ b/test/e2e-image/namespace-overlays/admission/values.yaml @@ -3,10 +3,8 @@ fullnameOverride: nginx-ingress controller: image: repository: ingress-controller/controller - chroot: true tag: 1.0.0-dev digest: - digestChroot: containerPort: http: "1080" https: "1443" diff --git a/test/e2e-image/namespace-overlays/custom-health-check-path/values.yaml b/test/e2e-image/namespace-overlays/custom-health-check-path/values.yaml index 4c6bef855..57bd1d105 100644 --- a/test/e2e-image/namespace-overlays/custom-health-check-path/values.yaml +++ b/test/e2e-image/namespace-overlays/custom-health-check-path/values.yaml @@ -3,10 +3,8 @@ fullnameOverride: nginx-ingress controller: image: repository: ingress-controller/controller - chroot: true tag: 1.0.0-dev digest: - digestChroot: extraArgs: healthz-port: "9090" # e2e tests do not require information about ingress status diff --git a/test/e2e-image/namespace-overlays/disableleaderelection/values.yaml b/test/e2e-image/namespace-overlays/disableleaderelection/values.yaml deleted file mode 100644 index f312a9fb4..000000000 --- a/test/e2e-image/namespace-overlays/disableleaderelection/values.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# TODO: remove the need to use fullnameOverride -fullnameOverride: nginx-ingress -controller: - image: - repository: ingress-controller/controller - chroot: true - tag: 1.0.0-dev - digest: - digestChroot: - scope: - # Necessary to allow the ingress controller to get the topology information from the nodes - enabled: false - config: - worker-processes: "1" - readinessProbe: - initialDelaySeconds: 3 - periodSeconds: 1 - livenessProbe: - initialDelaySeconds: 3 - periodSeconds: 1 - service: - type: NodePort - extraArgs: - # e2e tests do not require information about ingress status - update-status: "false" - terminationGracePeriodSeconds: 1 - admissionWebhooks: - enabled: false - - disableLeaderElection: true - -rbac: - create: true - scope: false diff --git a/test/e2e-image/namespace-overlays/forwarded-port-headers/values.yaml b/test/e2e-image/namespace-overlays/forwarded-port-headers/values.yaml index 68b4a074c..c79154d28 100644 --- a/test/e2e-image/namespace-overlays/forwarded-port-headers/values.yaml +++ b/test/e2e-image/namespace-overlays/forwarded-port-headers/values.yaml @@ -3,10 +3,8 @@ fullnameOverride: nginx-ingress controller: image: repository: ingress-controller/controller - chroot: true tag: 1.0.0-dev digest: - digestChroot: containerPort: http: "1080" https: "1443" @@ -16,9 +14,7 @@ controller: https-port: "1443" # e2e tests do not require information about ingress status update-status: "false" - ingressClassResource: - # We will create and remove each IC/ClusterRole/ClusterRoleBinding per test so there's no conflict - enabled: false + scope: enabled: true diff --git a/test/e2e-image/namespace-overlays/namespace-selector/values.yaml b/test/e2e-image/namespace-overlays/namespace-selector/values.yaml deleted file mode 100644 index 2c8957f66..000000000 --- a/test/e2e-image/namespace-overlays/namespace-selector/values.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# TODO: remove the need to use fullnameOverride -fullnameOverride: nginx-ingress -controller: - image: - repository: ingress-controller/controller - chroot: true - tag: 1.0.0-dev - digest: - digestChroot: - containerPort: - http: "1080" - https: "1443" - - extraArgs: - http-port: "1080" - https-port: "1443" - # e2e tests do not require information about ingress status - update-status: "false" - ingressClassResource: - # We will create and remove each IC/ClusterRole/ClusterRoleBinding per test so there's no conflict - enabled: false - scope: - enabled: false - namespaceSelector: "foo=bar" - - config: - worker-processes: "1" - service: - type: NodePort - admissionWebhooks: - enabled: false - -defaultBackend: - enabled: false - -rbac: - create: true - scope: false diff --git a/test/e2e-image/namespace-overlays/topology/values.yaml b/test/e2e-image/namespace-overlays/topology/values.yaml deleted file mode 100644 index 5e27f7902..000000000 --- a/test/e2e-image/namespace-overlays/topology/values.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# TODO: remove the need to use fullnameOverride -fullnameOverride: nginx-ingress -controller: - image: - repository: ingress-controller/controller - chroot: true - tag: 1.0.0-dev - digest: - digestChroot: - scope: - # Necessary to allow the ingress controller to get the topology information from the nodes - enabled: false - config: - worker-processes: "1" - readinessProbe: - initialDelaySeconds: 3 - periodSeconds: 1 - livenessProbe: - initialDelaySeconds: 3 - periodSeconds: 1 - service: - type: NodePort - extraArgs: - # e2e tests do not require information about ingress status - update-status: "false" - terminationGracePeriodSeconds: 1 - admissionWebhooks: - enabled: false - - enableTopologyAwareRouting: true - -rbac: - create: true - scope: false diff --git a/test/e2e-image/namespace-overlays/validations/values.yaml b/test/e2e-image/namespace-overlays/validations/values.yaml deleted file mode 100644 index d423217db..000000000 --- a/test/e2e-image/namespace-overlays/validations/values.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# TODO: remove the need to use fullnameOverride -fullnameOverride: nginx-ingress -controller: - image: - repository: ingress-controller/controller - chroot: true - tag: 1.0.0-dev - digest: - digestChroot: - containerPort: - http: "1080" - https: "1443" - - extraArgs: - http-port: "1080" - https-port: "1443" - # e2e tests do not require information about ingress status - update-status: "false" - - scope: - enabled: true - - config: - worker-processes: "1" - service: - type: NodePort - - admissionWebhooks: - enabled: true - certificate: "/usr/local/certificates/cert" - key: "/usr/local/certificates/key" - -defaultBackend: - enabled: false - -rbac: - create: true - scope: true diff --git a/test/e2e/HTTPBUN_IMAGE b/test/e2e/HTTPBUN_IMAGE deleted file mode 100644 index 491b333c7..000000000 --- a/test/e2e/HTTPBUN_IMAGE +++ /dev/null @@ -1 +0,0 @@ -registry.k8s.io/ingress-nginx/httpbun:v1.1.1@sha256:4569515d9b74470c915566a010792e7202b6769443fb1f3bb1b1e87376028634 diff --git a/test/e2e/admission/admission.go b/test/e2e/admission/admission.go index 873e6719d..b6cf638fe 100644 --- a/test/e2e/admission/admission.go +++ b/test/e2e/admission/admission.go @@ -24,19 +24,15 @@ import ( "os/exec" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/test/e2e/framework" - - networking "k8s.io/api/networking/v1" ) -const admissionTestHost = "admission-test" - -var _ = framework.IngressNginxDescribeSerial("[Admission] admission controller", func() { +var _ = framework.IngressNginxDescribe("[Serial] admission controller", func() { f := framework.NewDefaultFramework("admission") ginkgo.BeforeEach(func() { @@ -44,11 +40,43 @@ var _ = framework.IngressNginxDescribeSerial("[Admission] admission controller", f.NewSlowEchoDeployment() }) + ginkgo.AfterEach(func() { + err := uninstallChart(f) + assert.Nil(ginkgo.GinkgoT(), err, "uninstalling helm chart") + }) + + ginkgo.It("reject ingress with global-rate-limit annotations when memcached is not configured", func() { + host := "admission-test" + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/global-rate-limit": "100", + "nginx.ingress.kubernetes.io/global-rate-limit-window": "1m", + } + ing := framework.NewSingleIngress("first-ingress", "/", host, f.Namespace, framework.EchoService, 80, annotations) + + ginkgo.By("rejects ingress when memcached is not configured") + + _, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), ing, metav1.CreateOptions{}) + assert.NotNil(ginkgo.GinkgoT(), err, "creating ingress with global throttle annotations when memcached is not configured") + + ginkgo.By("accepts ingress when memcached is not configured") + + f.UpdateNginxConfigMapData("global-rate-limit-memcached-host", "memc.default.svc.cluster.local") + + _, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), ing, metav1.CreateOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "creating ingress with global throttle annotations when memcached is configured") + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, fmt.Sprintf("server_name %v", host)) + }) + }) + ginkgo.It("should not allow overlaps of host and paths without canary annotations", func() { - host := admissionTestHost + host := "admission-test" firstIngress := framework.NewSingleIngress("first-ingress", "/", host, f.Namespace, framework.EchoService, 80, nil) - _, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Create(context.TODO(), firstIngress, metav1.CreateOptions{}) + _, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), firstIngress, metav1.CreateOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "creating ingress") f.WaitForNginxServer(host, @@ -57,15 +85,15 @@ var _ = framework.IngressNginxDescribeSerial("[Admission] admission controller", }) secondIngress := framework.NewSingleIngress("second-ingress", "/", host, f.Namespace, framework.EchoService, 80, nil) - _, err = f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Create(context.TODO(), secondIngress, metav1.CreateOptions{}) + _, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), secondIngress, metav1.CreateOptions{}) assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress with the same host and path should return an error") }) ginkgo.It("should allow overlaps of host and paths with canary annotation", func() { - host := admissionTestHost + host := "admission-test" firstIngress := framework.NewSingleIngress("first-ingress", "/", host, f.Namespace, framework.EchoService, 80, nil) - _, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Create(context.TODO(), firstIngress, metav1.CreateOptions{}) + _, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), firstIngress, metav1.CreateOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "creating ingress") f.WaitForNginxServer(host, @@ -78,106 +106,23 @@ var _ = framework.IngressNginxDescribeSerial("[Admission] admission controller", "nginx.ingress.kubernetes.io/canary-by-header": "CanaryByHeader", } secondIngress := framework.NewSingleIngress("second-ingress", "/", host, f.Namespace, framework.SlowEchoService, 80, canaryAnnotations) - _, err = f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Create(context.TODO(), secondIngress, metav1.CreateOptions{}) + _, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), secondIngress, metav1.CreateOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "creating an ingress with the same host and path should not return an error using a canary annotation") }) - ginkgo.It("should block ingress with invalid path", func() { - host := "invalid-path" - - firstIngress := framework.NewSingleIngress("valid-path", "/mypage", host, f.Namespace, framework.EchoService, 80, nil) - _, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Create(context.TODO(), firstIngress, metav1.CreateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "creating ingress") - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %v", host)) - }) - - secondIngress := framework.NewSingleIngress("second-ingress", "/etc/nginx", host, f.Namespace, framework.EchoService, 80, nil) - _, err = f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Create(context.TODO(), secondIngress, metav1.CreateOptions{}) - assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress with invalid path should return an error") - }) - ginkgo.It("should return an error if there is an error validating the ingress definition", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - - host := admissionTestHost + host := "admission-test" annotations := map[string]string{ "nginx.ingress.kubernetes.io/configuration-snippet": "something invalid", } firstIngress := framework.NewSingleIngress("first-ingress", "/", host, f.Namespace, framework.EchoService, 80, annotations) - _, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Create(context.TODO(), firstIngress, metav1.CreateOptions{}) + _, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), firstIngress, metav1.CreateOptions{}) assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress with invalid configuration should return an error") }) - ginkgo.It("should return an error if there is an invalid value in some annotation", func() { - host := admissionTestHost - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/connection-proxy-header": "a;}", - } - - f.UpdateNginxConfigMapData("annotation-value-word-blocklist", "}") - - firstIngress := framework.NewSingleIngress("first-ingress", "/", host, f.Namespace, framework.EchoService, 80, annotations) - _, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Create(context.TODO(), firstIngress, metav1.CreateOptions{}) - assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress with invalid annotation value should return an error") - }) - - ginkgo.It("should return an error if there is a forbidden value in some annotation", func() { - host := admissionTestHost - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/connection-proxy-header": "set_by_lua", - } - - f.UpdateNginxConfigMapData("annotation-value-word-blocklist", "set_by_lua") - - firstIngress := framework.NewSingleIngress("first-ingress", "/", host, f.Namespace, framework.EchoService, 80, annotations) - _, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Create(context.TODO(), firstIngress, metav1.CreateOptions{}) - assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress with invalid annotation value should return an error") - }) - - ginkgo.It("should return an error if there is an invalid path and wrong pathType is set", func() { - host := "path-validation" - var ( - exactPathType = networking.PathTypeExact - prefixPathType = networking.PathTypePrefix - implSpecific = networking.PathTypeImplementationSpecific - ) - - f.UpdateNginxConfigMapData("strict-validate-path-type", "true") - - invalidPath := framework.NewSingleIngress("first-ingress", "/foo/bar/[a-z]{3}", host, f.Namespace, framework.EchoService, 80, nil) - invalidPath.Spec.Rules[0].IngressRuleValue.HTTP.Paths[0].PathType = &exactPathType - - _, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Create(context.TODO(), invalidPath, metav1.CreateOptions{}) - assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress with invalid path value should return an error") - - invalidPath.Spec.Rules[0].IngressRuleValue.HTTP.Paths[0].PathType = &prefixPathType - _, err = f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Create(context.TODO(), invalidPath, metav1.CreateOptions{}) - assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress with invalid path value should return an error") - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/use-regex": "true", - "nginx.ingress.kubernetes.io/rewrite-target": "/new/backend", - } - pathSpecific := framework.NewSingleIngress("pathspec-ingress", "/foo/bar/[a-z]{3}", host, f.Namespace, framework.EchoService, 80, annotations) - pathSpecific.Spec.Rules[0].IngressRuleValue.HTTP.Paths[0].PathType = &implSpecific - _, err = f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Create(context.TODO(), pathSpecific, metav1.CreateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "creating an ingress with arbitrary path and implSpecific value should not return an error") - - validPath := framework.NewSingleIngress("second-ingress", "/bloblo", host, f.Namespace, framework.EchoService, 80, nil) - _, err = f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Create(context.TODO(), validPath, metav1.CreateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "creating an ingress with valid path should not return an error") - }) - - ginkgo.It("should not return an error if the Ingress V1 definition is valid with Ingress Class", func() { - out, err := createIngress(f.Namespace, validV1Ingress) - assert.Equal(ginkgo.GinkgoT(), "ingress.networking.k8s.io/extensions created\n", out) + ginkgo.It("should not return an error the ingress definition uses the deprecated extensions package", func() { + err := createIngress(f.Namespace, validIngress) assert.Nil(ginkgo.GinkgoT(), err, "creating an ingress using kubectl") f.WaitForNginxConfiguration(func(cfg string) bool { @@ -191,53 +136,104 @@ var _ = framework.IngressNginxDescribeSerial("[Admission] admission controller", Status(http.StatusOK) }) - ginkgo.It("should not return an error if the Ingress V1 definition is valid with IngressClass annotation", func() { - out, err := createIngress(f.Namespace, validV1IngressAnnotation) - assert.Equal(ginkgo.GinkgoT(), "ingress.networking.k8s.io/extensions-class created\n", out) - assert.Nil(ginkgo.GinkgoT(), err, "creating an ingress using kubectl") - - f.WaitForNginxConfiguration(func(cfg string) bool { - return strings.Contains(cfg, "extensions-class") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", "extensions-class"). - Expect(). - Status(http.StatusOK) - }) - - ginkgo.It("should return an error if the Ingress V1 definition contains invalid annotations", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - - out, err := createIngress(f.Namespace, invalidV1Ingress) - assert.Empty(ginkgo.GinkgoT(), out) + ginkgo.It("should return an error if the ingress definition uses the deprecated extensions package and invalid annotations", func() { + err := createIngress(f.Namespace, invalidIngress) assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress using kubectl") - _, err = f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Get(context.TODO(), "extensions-invalid", metav1.GetOptions{}) + _, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), "extensions", metav1.GetOptions{}) if !apierrors.IsNotFound(err) { assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress with invalid configuration should return an error") } }) - ginkgo.It("should not return an error for an invalid Ingress when it has unknown class", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - out, err := createIngress(f.Namespace, invalidV1IngressWithOtherClass) - assert.Equal(ginkgo.GinkgoT(), "ingress.networking.k8s.io/extensions-invalid-other created\n", out) - assert.Nil(ginkgo.GinkgoT(), err, "creating an invalid ingress with unknown class using kubectl") + ginkgo.It("should not return an error if the Ingress V1 definition is valid", func() { + if !f.IsIngressV1Ready { + ginkgo.Skip("Test requires Kubernetes v1.19 or higher") + } + + err := createIngress(f.Namespace, validV1Ingress) + assert.Nil(ginkgo.GinkgoT(), err, "creating an ingress using kubectl") + + f.WaitForNginxConfiguration(func(cfg string) bool { + return strings.Contains(cfg, "extensions") + }) + + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", "extensions"). + Expect(). + Status(http.StatusOK) + }) + + ginkgo.It("should return an error if the Ingress V1 definition contains invalid annotations", func() { + if !f.IsIngressV1Ready { + ginkgo.Skip("Test requires Kubernetes v1.19 or higher") + } + + err := createIngress(f.Namespace, invalidV1Ingress) + assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress using kubectl") + + _, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), "extensions", metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress with invalid configuration should return an error") + } }) }) +func uninstallChart(f *framework.Framework) error { + cmd := exec.Command("helm", "uninstall", "--namespace", f.Namespace, "nginx-ingress") + _, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("unexpected error uninstalling ingress-nginx release: %v", err) + } + + return nil +} + const ( + validIngress = ` +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: extensions +spec: + rules: + - host: extensions + http: + paths: + - path: / + backend: + serviceName: echo + servicePort: 80 +--- +` + + invalidIngress = ` +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: extensions + annotations: + nginx.ingress.kubernetes.io/configuration-snippet: | + invalid directive +spec: + rules: + - host: extensions + http: + paths: + - path: / + backend: + serviceName: echo + servicePort: 80 +--- +` + validV1Ingress = ` apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: extensions spec: - ingressClassName: nginx rules: - host: extensions http: @@ -250,28 +246,6 @@ spec: port: number: 80 ---- -` - validV1IngressAnnotation = ` -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: extensions-class - annotations: - kubernetes.io/ingress.class: nginx -spec: - rules: - - host: extensions-class - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: echo - port: - number: 80 - --- ` @@ -279,37 +253,13 @@ spec: apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: extensions-invalid + name: extensions annotations: nginx.ingress.kubernetes.io/configuration-snippet: | invalid directive spec: - ingressClassName: nginx rules: - - host: extensions-invalid - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: echo - port: - number: 80 ---- -` - invalidV1IngressWithOtherClass = ` -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: extensions-invalid-other - annotations: - nginx.ingress.kubernetes.io/configuration-snippet: | - invalid directive -spec: - ingressClassName: nginx-other - rules: - - host: extensions-invalid + - host: extensions http: paths: - path: / @@ -323,22 +273,20 @@ spec: ` ) -func createIngress(namespace, ingressDefinition string) (string, error) { +func createIngress(namespace, ingressDefinition string) error { var ( - execOut bytes.Buffer execErr bytes.Buffer ) - //nolint:gosec // Ignore G204 error + cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v --warnings-as-errors=false apply --namespace %s -f -", framework.KubectlPath, namespace)) cmd.Stdin = strings.NewReader(ingressDefinition) - cmd.Stdout = &execOut cmd.Stderr = &execErr err := cmd.Run() if err != nil { stderr := strings.TrimSpace(execErr.String()) - return "", fmt.Errorf("kubectl error: %v\n%v", err, stderr) + return fmt.Errorf("Kubectl error: %v\n%v", err, stderr) } - return execOut.String(), nil + return nil } diff --git a/test/e2e/annotations/affinity.go b/test/e2e/annotations/affinity.go index b64581ef6..a17df1cc6 100644 --- a/test/e2e/annotations/affinity.go +++ b/test/e2e/annotations/affinity.go @@ -18,40 +18,32 @@ package annotations import ( "context" - "crypto/tls" "fmt" "net/http" "strings" "time" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/test/e2e/framework" ) -const ( - affinityAnnotation = "cookie" - cookieName = "SERVERID" - enableAnnotation = "true" - disableAnnotation = "false" - defaultHost = "foo.com" -) - var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { f := framework.NewDefaultFramework("affinity") ginkgo.BeforeEach(func() { - f.NewEchoDeployment(framework.WithDeploymentReplicas(2)) + f.NewEchoDeploymentWithReplicas(2) }) ginkgo.It("should set sticky cookie SERVERID", func() { host := "sticky.foo.com" annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/affinity"] = affinityAnnotation - annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = cookieName + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID" ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -72,8 +64,8 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { ginkgo.It("should change cookie name on ingress definition change", func() { host := "change.foo.com" annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/affinity"] = affinityAnnotation - annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = cookieName + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID" ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -88,11 +80,11 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { WithHeader("Host", host). Expect(). Status(http.StatusOK). - Header("Set-Cookie").Contains(cookieName) + Header("Set-Cookie").Contains("SERVERID") ing.ObjectMeta.Annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "OTHERCOOKIENAME" - _, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{}) + _, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "updating ingress") framework.Sleep() @@ -107,8 +99,8 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { ginkgo.It("should set the path to /something on the generated cookie", func() { host := "path.foo.com" annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/affinity"] = affinityAnnotation - annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = cookieName + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID" ing := framework.NewSingleIngress(host, "/something", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -127,11 +119,10 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { }) ginkgo.It("does not set the path to / on the generated cookie if there's more than one rule referring to the same backend", func() { - pathtype := networking.PathTypePrefix host := "morethanonerule.foo.com" annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/affinity"] = affinityAnnotation - annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = cookieName + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID" f.EnsureIngress(&networking.Ingress{ ObjectMeta: metav1.ObjectMeta{ @@ -140,7 +131,6 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { Annotations: annotations, }, Spec: networking.IngressSpec{ - IngressClassName: &f.IngressClass, Rules: []networking.IngressRule{ { Host: host, @@ -148,27 +138,17 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { HTTP: &networking.HTTPIngressRuleValue{ Paths: []networking.HTTPIngressPath{ { - Path: "/something", - PathType: &pathtype, + Path: "/something", Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: framework.EchoService, - Port: networking.ServiceBackendPort{ - Number: int32(80), - }, - }, + ServiceName: framework.EchoService, + ServicePort: intstr.FromInt(80), }, }, { - Path: "/somewhereelse", - PathType: &pathtype, + Path: "/somewhereelse", Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: framework.EchoService, - Port: networking.ServiceBackendPort{ - Number: int32(80), - }, - }, + ServiceName: framework.EchoService, + ServicePort: intstr.FromInt(80), }, }, }, @@ -202,7 +182,7 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { ginkgo.It("should set cookie with expires", func() { host := "cookieexpires.foo.com" annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/affinity"] = affinityAnnotation + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "ExpiresCookie" annotations["nginx.ingress.kubernetes.io/session-cookie-expires"] = "172800" annotations["nginx.ingress.kubernetes.io/session-cookie-max-age"] = "259200" @@ -219,8 +199,7 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { assert.Nil(ginkgo.GinkgoT(), err, "loading GMT location") assert.NotNil(ginkgo.GinkgoT(), local, "expected a location but none returned") - duration, err := time.ParseDuration("48h") - assert.Nil(ginkgo.GinkgoT(), err, "parsing duration") + duration, _ := time.ParseDuration("48h") expected := time.Now().In(local).Add(duration).Format("Mon, 02-Jan-06 15:04") f.HTTPTestClient(). @@ -231,57 +210,12 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { Header("Set-Cookie").Contains(fmt.Sprintf("Expires=%s", expected)).Contains("Max-Age=259200") }) - ginkgo.It("should set cookie with domain", func() { - host := "cookiedomain.foo.com" - annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/affinity"] = affinityAnnotation - annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "DomainCookie" - annotations["nginx.ingress.kubernetes.io/session-cookie-domain"] = "foo.bar" - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK). - Header("Set-Cookie").Contains("Domain=foo.bar") - }) - - ginkgo.It("should not set cookie without domain annotation", func() { - host := "cookienodomain.foo.com" - annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/affinity"] = affinityAnnotation - annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "NoDomainCookie" - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK). - Header("Set-Cookie").NotContains("; Domain") - }) - ginkgo.It("should work with use-regex annotation and session-cookie-path", func() { host := "useregex.foo.com" annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/affinity"] = affinityAnnotation - annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = cookieName - annotations["nginx.ingress.kubernetes.io/use-regex"] = enableAnnotation + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID" + annotations["nginx.ingress.kubernetes.io/use-regex"] = "true" annotations["nginx.ingress.kubernetes.io/session-cookie-path"] = "/foo/bar" ing := framework.NewSingleIngress(host, "/foo/.*", host, f.Namespace, framework.EchoService, 80, annotations) @@ -303,9 +237,9 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { ginkgo.It("should warn user when use-regex is true and session-cookie-path is not set", func() { host := "useregexwarn.foo.com" annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/affinity"] = affinityAnnotation - annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = cookieName - annotations["nginx.ingress.kubernetes.io/use-regex"] = enableAnnotation + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID" + annotations["nginx.ingress.kubernetes.io/use-regex"] = "true" ing := framework.NewSingleIngress(host, "/foo/.*", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -330,7 +264,7 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { host := "separate.foo.com" annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/affinity"] = affinityAnnotation + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" ing1 := framework.NewSingleIngress("ingress1", "/foo/bar", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing1) @@ -360,8 +294,8 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { ginkgo.It("should set sticky cookie without host", func() { annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/affinity"] = affinityAnnotation - annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = cookieName + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID" ing := framework.NewSingleIngress("default-no-host", "/", "", f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -379,12 +313,12 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { }) ginkgo.It("should work with server-alias annotation", func() { - host := defaultHost + host := "foo.com" alias1 := "a1.foo.com" alias2 := "a2.foo.com" annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/affinity"] = affinityAnnotation - annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = cookieName + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID" annotations["nginx.ingress.kubernetes.io/server-alias"] = fmt.Sprintf("%s,%s", alias1, alias2) ing := framework.NewSingleIngress(host, "/bar", host, f.Namespace, framework.EchoService, 80, annotations) @@ -392,7 +326,7 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { f.WaitForNginxServer(host, func(server string) bool { - // server alias sort by sort.Strings(), see: internal/ingress/annotations/alias/main.go:60 + //server alias sort by sort.Strings(), see: internal/ingress/annotations/alias/main.go:60 return strings.Contains(server, fmt.Sprintf("server_name %s %s %s ;", host, alias1, alias2)) }) @@ -418,74 +352,4 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { Header("Set-Cookie").Contains("SERVERID=") }) - ginkgo.It("should set secure in cookie with provided true annotation on http", func() { - host := defaultHost - annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/affinity"] = affinityAnnotation - annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = cookieName - annotations["nginx.ingress.kubernetes.io/session-cookie-secure"] = enableAnnotation - - ing := framework.NewSingleIngress(host, "/bar", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) - }) - - f.HTTPTestClient(). - GET("/bar"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK). - Header("Set-Cookie").Contains("; Secure") - }) - - ginkgo.It("should not set secure in cookie with provided false annotation on http", func() { - host := defaultHost - annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/affinity"] = affinityAnnotation - annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = cookieName - annotations["nginx.ingress.kubernetes.io/session-cookie-secure"] = disableAnnotation - - ing := framework.NewSingleIngress(host, "/bar", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) - }) - - f.HTTPTestClient(). - GET("/bar"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK). - Header("Set-Cookie").NotContains("; Secure") - }) - - ginkgo.It("should set secure in cookie with provided false annotation on https", func() { - host := defaultHost - annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/affinity"] = affinityAnnotation - annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = cookieName - annotations["nginx.ingress.kubernetes.io/session-cookie-secure"] = disableAnnotation - - f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, framework.EchoService, 80, annotations)) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) && - strings.Contains(server, "listen 443") - }) - - //nolint:gosec // Ignore the gosec error in testing - f.HTTPTestClientWithTLSConfig(&tls.Config{ServerName: host, InsecureSkipVerify: true}). - GET("/"). - WithURL(f.GetURL(framework.HTTPS)). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK). - Header("Set-Cookie").Contains("; Secure") - }) }) diff --git a/test/e2e/annotations/affinitymode.go b/test/e2e/annotations/affinitymode.go index e6253b6ff..3b533906e 100644 --- a/test/e2e/annotations/affinitymode.go +++ b/test/e2e/annotations/affinitymode.go @@ -22,32 +22,27 @@ import ( "reflect" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" ) -const sslRedirectValue = "false" - var _ = framework.DescribeAnnotation("affinitymode", func() { f := framework.NewDefaultFramework("affinity") ginkgo.It("Balanced affinity mode should balance", func() { deploymentName := "affinitybalanceecho" replicas := 5 - f.NewEchoDeployment( - framework.WithDeploymentName(deploymentName), - framework.WithDeploymentReplicas(replicas), - ) + f.NewEchoDeploymentWithNameAndReplicas(deploymentName, replicas) host := "affinity-mode-balance.com" annotations := make(map[string]string) annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" - annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "hello-cookie" + annotations["ginx.ingress.kubernetes.io/session-cookie-name"] = "hello-cookie" annotations["nginx.ingress.kubernetes.io/session-cookie-expires"] = "172800" annotations["nginx.ingress.kubernetes.io/session-cookie-max-age"] = "172800" - annotations["nginx.ingress.kubernetes.io/ssl-redirect"] = sslRedirectValue + annotations["nginx.ingress.kubernetes.io/ssl-redirect"] = "false" annotations["nginx.ingress.kubernetes.io/affinity-mode"] = "balanced" annotations["nginx.ingress.kubernetes.io/session-cookie-hash"] = "sha1" @@ -69,18 +64,15 @@ var _ = framework.DescribeAnnotation("affinitymode", func() { ginkgo.It("Check persistent affinity mode", func() { deploymentName := "affinitypersistentecho" replicas := 5 - f.NewEchoDeployment( - framework.WithDeploymentName(deploymentName), - framework.WithDeploymentReplicas(replicas), - ) + f.NewEchoDeploymentWithNameAndReplicas(deploymentName, replicas) host := "affinity-mode-persistent.com" annotations := make(map[string]string) annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" - annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "hello-cookie" + annotations["ginx.ingress.kubernetes.io/session-cookie-name"] = "hello-cookie" annotations["nginx.ingress.kubernetes.io/session-cookie-expires"] = "172800" annotations["nginx.ingress.kubernetes.io/session-cookie-max-age"] = "172800" - annotations["nginx.ingress.kubernetes.io/ssl-redirect"] = sslRedirectValue + annotations["nginx.ingress.kubernetes.io/ssl-redirect"] = "false" annotations["nginx.ingress.kubernetes.io/affinity-mode"] = "persistent" annotations["nginx.ingress.kubernetes.io/session-cookie-hash"] = "sha1" @@ -108,7 +100,7 @@ var _ = framework.DescribeAnnotation("affinitymode", func() { // Send new requests and add new backends. Check which backend responded to the sent request cookies := getCookiesFromHeader(response.Header("Set-Cookie").Raw()) for sendRequestNumber := 0; sendRequestNumber < 10; sendRequestNumber++ { - replicas++ + replicas = replicas + 1 err := framework.UpdateDeployment(f.KubeClientSet, f.Namespace, deploymentName, replicas, nil) assert.Nil(ginkgo.GinkgoT(), err) framework.Sleep() @@ -127,7 +119,7 @@ var _ = framework.DescribeAnnotation("affinitymode", func() { framework.Sleep() // validate, there is no backend to serve the request - request.WithCookies(cookies).Expect().Status(http.StatusServiceUnavailable) + response = request.WithCookies(cookies).Expect().Status(http.StatusServiceUnavailable) // create brand new backends replicas = 2 diff --git a/test/e2e/annotations/alias.go b/test/e2e/annotations/alias.go index ca4fe9c31..6b11a9c07 100644 --- a/test/e2e/annotations/alias.go +++ b/test/e2e/annotations/alias.go @@ -21,13 +21,11 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) -const fooHost = "foo" - var _ = framework.DescribeAnnotation("server-alias", func() { f := framework.NewDefaultFramework("alias") @@ -36,7 +34,7 @@ var _ = framework.DescribeAnnotation("server-alias", func() { }) ginkgo.It("should return status code 200 for host 'foo' and 404 for 'bar'", func() { - host := fooHost + host := "foo" ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil) f.EnsureIngress(ing) @@ -62,7 +60,7 @@ var _ = framework.DescribeAnnotation("server-alias", func() { }) ginkgo.It("should return status code 200 for host 'foo' and 'bar'", func() { - host := fooHost + host := "foo" annotations := map[string]string{ "nginx.ingress.kubernetes.io/server-alias": "bar", } @@ -75,7 +73,7 @@ var _ = framework.DescribeAnnotation("server-alias", func() { return strings.Contains(server, fmt.Sprintf("server_name %v", host)) }) - hosts := []string{fooHost, "bar"} + hosts := []string{"foo", "bar"} for _, host := range hosts { f.HTTPTestClient(). GET("/"). @@ -87,7 +85,7 @@ var _ = framework.DescribeAnnotation("server-alias", func() { }) ginkgo.It("should return status code 200 for hosts defined in two ingresses, different path with one alias", func() { - host := fooHost + host := "foo" ing := framework.NewSingleIngress("app-a", "/app-a", host, f.Namespace, framework.EchoService, 80, nil) f.EnsureIngress(ing) @@ -103,7 +101,7 @@ var _ = framework.DescribeAnnotation("server-alias", func() { return strings.Contains(server, fmt.Sprintf("server_name %v bar", host)) }) - hosts := []string{fooHost, "bar"} + hosts := []string{"foo", "bar"} for _, host := range hosts { f.HTTPTestClient(). GET("/app-a"). diff --git a/test/e2e/annotations/approot.go b/test/e2e/annotations/approot.go index 485b4d277..a73c60253 100644 --- a/test/e2e/annotations/approot.go +++ b/test/e2e/annotations/approot.go @@ -20,7 +20,7 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/annotations/auth.go b/test/e2e/annotations/auth.go index ddda1dce5..781a32b1b 100644 --- a/test/e2e/annotations/auth.go +++ b/test/e2e/annotations/auth.go @@ -21,36 +21,30 @@ import ( "fmt" "net/http" "net/url" + "os/exec" "regexp" "strings" + "time" - "golang.org/x/crypto/bcrypt" - - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/test/e2e/framework" ) -const ( - differentHost = "different" - authHost = "auth" - authURL = "http://foo.bar.baz:5000/path" -) - var _ = framework.DescribeAnnotation("auth-*", func() { - f := framework.NewDefaultFramework("auth", framework.WithHTTPBunEnabled()) + f := framework.NewDefaultFramework("auth") ginkgo.BeforeEach(func() { f.NewEchoDeployment() }) ginkgo.It("should return status code 200 when no authentication is configured", func() { - host := authHost + host := "auth" ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil) f.EnsureIngress(ing) @@ -69,7 +63,7 @@ var _ = framework.DescribeAnnotation("auth-*", func() { }) ginkgo.It("should return status code 503 when authentication is configured with an invalid secret", func() { - host := authHost + host := "auth" annotations := map[string]string{ "nginx.ingress.kubernetes.io/auth-type": "basic", "nginx.ingress.kubernetes.io/auth-secret": "something", @@ -93,9 +87,9 @@ var _ = framework.DescribeAnnotation("auth-*", func() { }) ginkgo.It("should return status code 401 when authentication is configured but Authorization header is not configured", func() { - host := authHost + host := "auth" - s := f.EnsureSecret(buildSecret(fooHost, "bar", "test", f.Namespace)) + s := f.EnsureSecret(buildSecret("foo", "bar", "test", f.Namespace)) annotations := map[string]string{ "nginx.ingress.kubernetes.io/auth-type": "basic", @@ -120,9 +114,9 @@ var _ = framework.DescribeAnnotation("auth-*", func() { }) ginkgo.It("should return status code 401 when authentication is configured and Authorization header is sent with invalid credentials", func() { - host := authHost + host := "auth" - s := f.EnsureSecret(buildSecret(fooHost, "bar", "test", f.Namespace)) + s := f.EnsureSecret(buildSecret("foo", "bar", "test", f.Namespace)) annotations := map[string]string{ "nginx.ingress.kubernetes.io/auth-type": "basic", @@ -147,38 +141,10 @@ var _ = framework.DescribeAnnotation("auth-*", func() { Body().Contains("401 Authorization Required") }) - ginkgo.It("should return status code 401 and cors headers when authentication and cors is configured but Authorization header is not configured", func() { - host := authHost - - s := f.EnsureSecret(buildSecret(fooHost, "bar", "test", f.Namespace)) - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/auth-type": "basic", - "nginx.ingress.kubernetes.io/auth-secret": s.Name, - "nginx.ingress.kubernetes.io/auth-realm": "test auth", - "nginx.ingress.kubernetes.io/enable-cors": "true", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "server_name auth") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusUnauthorized). - Header("Access-Control-Allow-Origin").Equal("*") - }) - ginkgo.It("should return status code 200 when authentication is configured and Authorization header is sent", func() { - host := authHost + host := "auth" - s := f.EnsureSecret(buildSecret(fooHost, "bar", "test", f.Namespace)) + s := f.EnsureSecret(buildSecret("foo", "bar", "test", f.Namespace)) annotations := map[string]string{ "nginx.ingress.kubernetes.io/auth-type": "basic", @@ -197,15 +163,15 @@ var _ = framework.DescribeAnnotation("auth-*", func() { f.HTTPTestClient(). GET("/"). WithHeader("Host", host). - WithBasicAuth(fooHost, "bar"). + WithBasicAuth("foo", "bar"). Expect(). Status(http.StatusOK) }) ginkgo.It("should return status code 200 when authentication is configured with a map and Authorization header is sent", func() { - host := authHost + host := "auth" - s := f.EnsureSecret(buildMapSecret(fooHost, "bar", "test", f.Namespace)) + s := f.EnsureSecret(buildMapSecret("foo", "bar", "test", f.Namespace)) annotations := map[string]string{ "nginx.ingress.kubernetes.io/auth-type": "basic", @@ -225,13 +191,13 @@ var _ = framework.DescribeAnnotation("auth-*", func() { f.HTTPTestClient(). GET("/"). WithHeader("Host", host). - WithBasicAuth(fooHost, "bar"). + WithBasicAuth("foo", "bar"). Expect(). Status(http.StatusOK) }) ginkgo.It("should return status code 401 when authentication is configured with invalid content and Authorization header is sent", func() { - host := authHost + host := "auth" s := f.EnsureSecret( &corev1.Secret{ @@ -264,21 +230,19 @@ var _ = framework.DescribeAnnotation("auth-*", func() { f.HTTPTestClient(). GET("/"). WithHeader("Host", host). - WithBasicAuth(fooHost, "bar"). + WithBasicAuth("foo", "bar"). Expect(). Status(http.StatusUnauthorized) }) ginkgo.It(`should set snippet "proxy_set_header My-Custom-Header 42;" when external auth is configured`, func() { - host := authHost + host := "auth" annotations := map[string]string{ "nginx.ingress.kubernetes.io/auth-url": "http://foo.bar/basic-auth/user/password", "nginx.ingress.kubernetes.io/auth-snippet": ` proxy_set_header My-Custom-Header 42;`, } - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -290,9 +254,7 @@ var _ = framework.DescribeAnnotation("auth-*", func() { }) ginkgo.It(`should not set snippet "proxy_set_header My-Custom-Header 42;" when external auth is not configured`, func() { - host := authHost - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() + host := "auth" annotations := map[string]string{ "nginx.ingress.kubernetes.io/auth-snippet": ` @@ -309,7 +271,7 @@ var _ = framework.DescribeAnnotation("auth-*", func() { }) ginkgo.It(`should set "proxy_set_header 'My-Custom-Header' '42';" when auth-headers are set`, func() { - host := authHost + host := "auth" annotations := map[string]string{ "nginx.ingress.kubernetes.io/auth-url": "http://foo.bar/basic-auth/user/password", @@ -330,11 +292,11 @@ var _ = framework.DescribeAnnotation("auth-*", func() { }) ginkgo.It(`should set cache_key when external auth cache is configured`, func() { - host := authHost + host := "auth" annotations := map[string]string{ "nginx.ingress.kubernetes.io/auth-url": "http://foo.bar/basic-auth/user/password", - "nginx.ingress.kubernetes.io/auth-cache-key": fooHost, + "nginx.ingress.kubernetes.io/auth-cache-key": "foo", "nginx.ingress.kubernetes.io/auth-cache-duration": "200 202 401 30m", } @@ -347,15 +309,13 @@ var _ = framework.DescribeAnnotation("auth-*", func() { func(server string) bool { return cacheRegex.MatchString(server) && strings.Contains(server, `proxy_cache_valid 200 202 401 30m;`) + }) }) - ginkgo.Context("cookie set by external authentication server", func() { + ginkgo.It("retains cookie set by external authentication server", func() { host := "auth-check-cookies" - var annotations map[string]string - var ing1, ing2 *networking.Ingress - cfg := `# events { worker_connections 1024; @@ -382,91 +342,63 @@ http { location / { return 200; } - - location /error { - return 503; - } } } ` - ginkgo.BeforeEach(func() { - f.NGINXWithConfigDeployment("http-cookie-with-error", cfg) - e, err := f.KubeClientSet.CoreV1().Endpoints(f.Namespace).Get(context.TODO(), "http-cookie-with-error", metav1.GetOptions{}) + f.NGINXWithConfigDeployment(framework.HTTPBinService, cfg) + + e, err := f.KubeClientSet.CoreV1().Endpoints(f.Namespace).Get(context.TODO(), framework.HTTPBinService, metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err) + + assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets), 1, "expected at least one endpoint") + assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets[0].Addresses), 1, "expected at least one address ready in the endpoint") + + httpbinIP := e.Subsets[0].Addresses[0].IP + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/cookies/set/alma/armud", httpbinIP), + "nginx.ingress.kubernetes.io/auth-signin": "http://$host/auth/start", + } + + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, func(server string) bool { + return strings.Contains(server, "server_name auth") + }) + + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", host). + WithQuery("a", "b"). + WithQuery("c", "d"). + Expect(). + Status(http.StatusOK). + Header("Set-Cookie").Contains("alma=armud") + }) + + ginkgo.Context("when external authentication is configured", func() { + host := "auth" + var annotations map[string]string + var ing *networking.Ingress + + ginkgo.BeforeEach(func() { + f.NewHttpbinDeployment() + + err := framework.WaitForEndpoints(f.KubeClientSet, framework.DefaultTimeout, framework.HTTPBinService, f.Namespace, 1) + assert.Nil(ginkgo.GinkgoT(), err) + + e, err := f.KubeClientSet.CoreV1().Endpoints(f.Namespace).Get(context.TODO(), framework.HTTPBinService, metav1.GetOptions{}) assert.Nil(ginkgo.GinkgoT(), err) assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets), 1, "expected at least one endpoint") assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets[0].Addresses), 1, "expected at least one address ready in the endpoint") - nginxIP := e.Subsets[0].Addresses[0].IP + httpbinIP := e.Subsets[0].Addresses[0].IP annotations = map[string]string{ - "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/cookies/set/alma/armud", nginxIP), - "nginx.ingress.kubernetes.io/auth-signin": "http://$host/auth/start", - } - - ing1 = framework.NewSingleIngress(host, "/", host, f.Namespace, "http-cookie-with-error", 80, annotations) - f.EnsureIngress(ing1) - - ing2 = framework.NewSingleIngress(host+"-error", "/error", host, f.Namespace, "http-cookie-with-error", 80, annotations) - f.EnsureIngress(ing2) - - f.WaitForNginxServer(host, func(server string) bool { - //nolint:goconst //server_name is a constant - return strings.Contains(server, "server_name "+host) - }) - }) - - ginkgo.It("user retains cookie by default", func() { - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithQuery("a", "b"). - WithQuery("c", "d"). - Expect(). - Status(http.StatusOK). - Header("Set-Cookie").Contains("alma=armud") - }) - - ginkgo.It("user does not retain cookie if upstream returns error status code", func() { - f.HTTPTestClient(). - GET("/error"). - WithHeader("Host", host). - WithQuery("a", "b"). - WithQuery("c", "d"). - Expect(). - Status(http.StatusServiceUnavailable). - Header("Set-Cookie").Contains("") - }) - - ginkgo.It("user with annotated ingress retains cookie if upstream returns error status code", func() { - annotations["nginx.ingress.kubernetes.io/auth-always-set-cookie"] = enableAnnotation - f.UpdateIngress(ing1) - f.UpdateIngress(ing2) - - f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "server_name "+host) - }) - - f.HTTPTestClient(). - GET("/error"). - WithHeader("Host", host). - WithQuery("a", "b"). - WithQuery("c", "d"). - Expect(). - Status(http.StatusServiceUnavailable). - Header("Set-Cookie").Contains("alma=armud") - }) - }) - - ginkgo.Context("when external authentication is configured", func() { - host := authHost - var annotations map[string]string - var ing *networking.Ingress - - ginkgo.BeforeEach(func() { - annotations = map[string]string{ - "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", f.HTTPBunIP), + "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", httpbinIP), "nginx.ingress.kubernetes.io/auth-signin": "http://$host/auth/start", } @@ -504,7 +436,7 @@ http { annotations["nginx.ingress.kubernetes.io/auth-realm"] = "test auth" f.UpdateIngress(ing) - anotherHost := differentHost + anotherHost := "different" anotherAnnotations := map[string]string{} anotherIng := framework.NewSingleIngress(anotherHost, "/", anotherHost, f.Namespace, framework.EchoService, 80, anotherAnnotations) @@ -521,171 +453,28 @@ http { Expect(). Status(http.StatusOK) }) - - ginkgo.It("should overwrite Foo header with auth response", func() { - var ( - rewriteHeader = "Foo" - rewriteVal = "bar" - ) - annotations["nginx.ingress.kubernetes.io/auth-response-headers"] = rewriteHeader - f.UpdateIngress(ing) - - f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("proxy_set_header '%s' $authHeader0;", rewriteHeader)) - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader(rewriteHeader, rewriteVal). - WithBasicAuth("user", "password"). - Expect(). - Status(http.StatusOK). - Body(). - NotContainsFold(fmt.Sprintf("%s=%s", rewriteHeader, rewriteVal)) - }) - - ginkgo.It(`should not create additional upstream block when auth-keepalive is not set`, func() { - f.UpdateNginxConfigMapData("use-http2", "false") - defer func() { - f.UpdateNginxConfigMapData("use-http2", "true") - }() - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - - annotations["nginx.ingress.kubernetes.io/auth-url"] = authURL - f.UpdateIngress(ing) - - f.WaitForNginxServer("", - func(server string) bool { - return strings.Contains(server, authURL) && - !strings.Contains(server, `upstream auth-external-auth`) - }) - }) - - ginkgo.It(`should not create additional upstream block when host part of auth-url contains a variable`, func() { - f.UpdateNginxConfigMapData("use-http2", "false") - defer func() { - f.UpdateNginxConfigMapData("use-http2", "true") - }() - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - - annotations["nginx.ingress.kubernetes.io/auth-url"] = "http://$host/path" - annotations["nginx.ingress.kubernetes.io/auth-keepalive"] = "123" - f.UpdateIngress(ing) - - f.WaitForNginxServer("", - func(server string) bool { - return strings.Contains(server, "http://$host/path") && - !strings.Contains(server, `upstream auth-external-auth`) && - !strings.Contains(server, `keepalive 123;`) - }) - }) - - ginkgo.It(`should not create additional upstream block when auth-keepalive is negative`, func() { - f.UpdateNginxConfigMapData("use-http2", "false") - defer func() { - f.UpdateNginxConfigMapData("use-http2", "true") - }() - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - - annotations["nginx.ingress.kubernetes.io/auth-url"] = authURL - annotations["nginx.ingress.kubernetes.io/auth-keepalive"] = "-1" - f.UpdateIngress(ing) - - f.WaitForNginxServer("", - func(server string) bool { - return strings.Contains(server, authURL) && - !strings.Contains(server, `upstream auth-external-auth`) - }) - }) - - ginkgo.It(`should not create additional upstream block when auth-keepalive is set with HTTP/2`, func() { - annotations["nginx.ingress.kubernetes.io/auth-url"] = authURL - annotations["nginx.ingress.kubernetes.io/auth-keepalive"] = "123" - annotations["nginx.ingress.kubernetes.io/auth-keepalive-requests"] = "456" - annotations["nginx.ingress.kubernetes.io/auth-keepalive-timeout"] = "789" - f.UpdateIngress(ing) - - f.WaitForNginxServer("", - func(server string) bool { - return strings.Contains(server, authURL) && - !strings.Contains(server, `upstream auth-external-auth`) - }) - }) - - ginkgo.It(`should create additional upstream block when auth-keepalive is set with HTTP/1.x`, func() { - f.UpdateNginxConfigMapData("use-http2", "false") - defer func() { - f.UpdateNginxConfigMapData("use-http2", "true") - }() - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - - annotations["nginx.ingress.kubernetes.io/auth-keepalive"] = "123" - annotations["nginx.ingress.kubernetes.io/auth-keepalive-requests"] = "456" - annotations["nginx.ingress.kubernetes.io/auth-keepalive-timeout"] = "789" - f.UpdateIngress(ing) - - f.WaitForNginxServer("", - func(server string) bool { - return strings.Contains(server, `upstream auth-external-auth`) && - strings.Contains(server, `keepalive 123;`) && - strings.Contains(server, `keepalive_requests 456;`) && - strings.Contains(server, `keepalive_timeout 789s;`) - }) - }) - - ginkgo.It(`should disable set_all_vars when auth-keepalive-share-vars is not set`, func() { - f.UpdateNginxConfigMapData("use-http2", "false") - defer func() { - f.UpdateNginxConfigMapData("use-http2", "true") - }() - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - - annotations["nginx.ingress.kubernetes.io/auth-keepalive"] = "10" - f.UpdateIngress(ing) - - f.WaitForNginxServer("", - func(server string) bool { - return strings.Contains(server, `upstream auth-external-auth`) && - strings.Contains(server, `keepalive 10;`) && - strings.Contains(server, `set $auth_keepalive_share_vars false;`) - }) - }) - - ginkgo.It(`should enable set_all_vars when auth-keepalive-share-vars is true`, func() { - f.UpdateNginxConfigMapData("use-http2", "false") - defer func() { - f.UpdateNginxConfigMapData("use-http2", "true") - }() - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - - annotations["nginx.ingress.kubernetes.io/auth-keepalive"] = "10" - annotations["nginx.ingress.kubernetes.io/auth-keepalive-share-vars"] = enableAnnotation - f.UpdateIngress(ing) - - f.WaitForNginxServer("", - func(server string) bool { - return strings.Contains(server, `upstream auth-external-auth`) && - strings.Contains(server, `keepalive 10;`) && - strings.Contains(server, `set $auth_keepalive_share_vars true;`) - }) - }) }) ginkgo.Context("when external authentication is configured with a custom redirect param", func() { - host := authHost + host := "auth" var annotations map[string]string var ing *networking.Ingress ginkgo.BeforeEach(func() { + f.NewHttpbinDeployment() + + var httpbinIP string + + err := framework.WaitForEndpoints(f.KubeClientSet, framework.DefaultTimeout, framework.HTTPBinService, f.Namespace, 1) + assert.Nil(ginkgo.GinkgoT(), err) + + e, err := f.KubeClientSet.CoreV1().Endpoints(f.Namespace).Get(context.TODO(), framework.HTTPBinService, metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err) + + httpbinIP = e.Subsets[0].Addresses[0].IP + annotations = map[string]string{ - "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", f.HTTPBunIP), + "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", httpbinIP), "nginx.ingress.kubernetes.io/auth-signin": "http://$host/auth/start", "nginx.ingress.kubernetes.io/auth-signin-redirect-param": "orig", } @@ -724,7 +513,7 @@ http { annotations["nginx.ingress.kubernetes.io/auth-realm"] = "test auth" f.UpdateIngress(ing) - anotherHost := differentHost + anotherHost := "different" anotherAnnotations := map[string]string{} anotherIng := framework.NewSingleIngress(anotherHost, "/", anotherHost, f.Namespace, framework.EchoService, 80, anotherAnnotations) @@ -744,15 +533,30 @@ http { }) ginkgo.Context("when external authentication with caching is configured", func() { - thisHost := authHost - thatHost := differentHost + thisHost := "auth" + thatHost := "different" fooPath := "/foo" barPath := "/bar" ginkgo.BeforeEach(func() { + f.NewHttpbinDeployment() + + err := framework.WaitForEndpoints(f.KubeClientSet, framework.DefaultTimeout, framework.HTTPBinService, f.Namespace, 1) + assert.Nil(ginkgo.GinkgoT(), err) + + framework.Sleep(1 * time.Second) + + e, err := f.KubeClientSet.CoreV1().Endpoints(f.Namespace).Get(context.TODO(), framework.HTTPBinService, metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err) + + assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets), 1, "expected at least one endpoint") + assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets[0].Addresses), 1, "expected at least one address ready in the endpoint") + + httpbinIP := e.Subsets[0].Addresses[0].IP + annotations := map[string]string{ - "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", f.HTTPBunIP), + "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", httpbinIP), "nginx.ingress.kubernetes.io/auth-signin": "http://$host/auth/start", "nginx.ingress.kubernetes.io/auth-cache-key": "fixed", "nginx.ingress.kubernetes.io/auth-cache-duration": "200 201 401 30m", @@ -785,7 +589,7 @@ http { Expect(). Status(http.StatusOK) - err := f.DeleteDeployment(framework.HTTPBunService) + err := f.DeleteDeployment(framework.HTTPBinService) assert.Nil(ginkgo.GinkgoT(), err) framework.Sleep() @@ -805,7 +609,7 @@ http { Expect(). Status(http.StatusOK) - err := f.DeleteDeployment(framework.HTTPBunService) + err := f.DeleteDeployment(framework.HTTPBinService) assert.Nil(ginkgo.GinkgoT(), err) framework.Sleep() @@ -834,7 +638,7 @@ http { Expect(). Status(http.StatusOK) - err := f.DeleteDeployment(framework.HTTPBunService) + err := f.DeleteDeployment(framework.HTTPBinService) assert.Nil(ginkgo.GinkgoT(), err) framework.Sleep() @@ -865,39 +669,6 @@ http { Header("Location").Equal(fmt.Sprintf("http://%s/auth/start?rd=http://%s%s", thisHost, thisHost, url.QueryEscape("/?a=b&c=d"))) }) }) - - ginkgo.Context("with invalid auth-url should deny whole location", func() { - host := authHost - var annotations map[string]string - var ing *networking.Ingress - - ginkgo.BeforeEach(func() { - annotations = map[string]string{ - "nginx.ingress.kubernetes.io/auth-url": "https://invalid..auth.url", - } - - ing = framework.NewSingleIngress(host, "/denied-auth", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "server_name auth") - }) - }) - - ginkgo.It("should return 503 (location was denied)", func() { - f.HTTPTestClient(). - GET("/denied-auth"). - WithHeader("Host", host). - Expect(). - Status(http.StatusServiceUnavailable) - }) - - ginkgo.It("should add error to the config", func() { - f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "could not parse auth-url annotation: invalid url host") - }) - }) - }) }) // TODO: test Digest Auth @@ -907,7 +678,7 @@ http { // Auth error func buildSecret(username, password, name, namespace string) *corev1.Secret { - out, err := bcrypt.GenerateFromPassword([]byte(password), 14) + out, err := exec.Command("openssl", "passwd", "-crypt", password).CombinedOutput() encpass := fmt.Sprintf("%v:%s\n", username, out) assert.Nil(ginkgo.GinkgoT(), err) @@ -925,7 +696,7 @@ func buildSecret(username, password, name, namespace string) *corev1.Secret { } func buildMapSecret(username, password, name, namespace string) *corev1.Secret { - out, err := bcrypt.GenerateFromPassword([]byte(password), 14) + out, err := exec.Command("openssl", "passwd", "-crypt", password).CombinedOutput() assert.Nil(ginkgo.GinkgoT(), err) return &corev1.Secret{ @@ -935,7 +706,7 @@ func buildMapSecret(username, password, name, namespace string) *corev1.Secret { DeletionGracePeriodSeconds: framework.NewInt64(1), }, Data: map[string][]byte{ - username: out, + username: []byte(out), }, Type: corev1.SecretTypeOpaque, } diff --git a/test/e2e/annotations/authtls.go b/test/e2e/annotations/authtls.go index 3315065f1..86c5d9ec5 100644 --- a/test/e2e/annotations/authtls.go +++ b/test/e2e/annotations/authtls.go @@ -17,26 +17,25 @@ limitations under the License. package annotations import ( + "crypto/tls" "fmt" "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" ) -const authTLSFooHost = "authtls.foo.com" - var _ = framework.DescribeAnnotation("auth-tls-*", func() { f := framework.NewDefaultFramework("authtls") ginkgo.BeforeEach(func() { - f.NewEchoDeployment(framework.WithDeploymentReplicas(2)) + f.NewEchoDeploymentWithReplicas(2) }) - ginkgo.It("should set sslClientCertificate, sslVerifyClient and sslVerifyDepth with auth-tls-secret", func() { - host := authTLSFooHost + ginkgo.It("should set valid auth-tls-secret", func() { + host := "authtls.foo.com" nameSpace := f.Namespace clientConfig, err := framework.CreateIngressMASecret( @@ -46,28 +45,16 @@ var _ = framework.DescribeAnnotation("auth-tls-*", func() { nameSpace) assert.Nil(ginkgo.GinkgoT(), err) - annotations := map[string]string{} - - ing := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, nameSpace, framework.EchoService, 80, annotations)) - - f.WaitForNginxServer(host, - func(server string) bool { - return !strings.Contains(server, "ssl_client_certificate") && - !strings.Contains(server, "ssl_verify_client") && - !strings.Contains(server, "ssl_verify_depth") - }) - - annotations = map[string]string{ + annotations := map[string]string{ "nginx.ingress.kubernetes.io/auth-tls-secret": nameSpace + "/" + host, } - ing.SetAnnotations(annotations) - f.UpdateIngress(ing) + f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, nameSpace, framework.EchoService, 80, annotations)) assertSslClientCertificateConfig(f, host, "on", "1") // Send Request without Client Certs - f.HTTPTestClient(). + f.HTTPTestClientWithTLSConfig(&tls.Config{ServerName: host, InsecureSkipVerify: true}). GET("/"). WithURL(f.GetURL(framework.HTTPS)). WithHeader("Host", host). @@ -84,7 +71,7 @@ var _ = framework.DescribeAnnotation("auth-tls-*", func() { }) ginkgo.It("should set valid auth-tls-secret, sslVerify to off, and sslVerifyDepth to 2", func() { - host := authTLSFooHost + host := "authtls.foo.com" nameSpace := f.Namespace _, err := framework.CreateIngressMASecret( @@ -113,8 +100,8 @@ var _ = framework.DescribeAnnotation("auth-tls-*", func() { Status(http.StatusOK) }) - ginkgo.It("should 302 redirect to error page instead of 400 when auth-tls-error-page is set", func() { - host := authTLSFooHost + ginkgo.It("should set valid auth-tls-secret, pass certificate to upstream, and error page", func() { + host := "authtls.foo.com" nameSpace := f.Namespace errorPath := "/error" @@ -127,8 +114,9 @@ var _ = framework.DescribeAnnotation("auth-tls-*", func() { assert.Nil(ginkgo.GinkgoT(), err) annotations := map[string]string{ - "nginx.ingress.kubernetes.io/auth-tls-secret": nameSpace + "/" + host, - "nginx.ingress.kubernetes.io/auth-tls-error-page": f.GetURL(framework.HTTP) + errorPath, + "nginx.ingress.kubernetes.io/auth-tls-secret": nameSpace + "/" + host, + "nginx.ingress.kubernetes.io/auth-tls-error-page": f.GetURL(framework.HTTP) + errorPath, + "nginx.ingress.kubernetes.io/auth-tls-pass-certificate-to-upstream": "true", } f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, nameSpace, framework.EchoService, 80, annotations)) @@ -136,10 +124,12 @@ var _ = framework.DescribeAnnotation("auth-tls-*", func() { assertSslClientCertificateConfig(f, host, "on", "1") sslErrorPage := fmt.Sprintf("error_page 495 496 = %s;", f.GetURL(framework.HTTP)+errorPath) + sslUpstreamClientCert := "proxy_set_header ssl-client-cert $ssl_client_escaped_cert;" f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, sslErrorPage) + return strings.Contains(server, sslErrorPage) && + strings.Contains(server, sslUpstreamClientCert) }) // Send Request without Client Certs @@ -160,53 +150,8 @@ var _ = framework.DescribeAnnotation("auth-tls-*", func() { Status(http.StatusOK) }) - ginkgo.It("should pass URL-encoded certificate to upstream", func() { - host := authTLSFooHost - nameSpace := f.Namespace - - clientConfig, err := framework.CreateIngressMASecret( - f.KubeClientSet, - host, - host, - nameSpace) - assert.Nil(ginkgo.GinkgoT(), err) - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/auth-tls-secret": nameSpace + "/" + host, - "nginx.ingress.kubernetes.io/auth-tls-pass-certificate-to-upstream": "true", - } - - f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, nameSpace, framework.EchoService, 80, annotations)) - - assertSslClientCertificateConfig(f, host, "on", "1") - - sslUpstreamClientCert := "proxy_set_header ssl-client-cert $ssl_client_escaped_cert;" - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, sslUpstreamClientCert) - }) - - // Send Request without Client Certs - f.HTTPTestClient(). - GET("/"). - WithURL(f.GetURL(framework.HTTPS)). - WithHeader("Host", host). - Expect(). - Status(http.StatusBadRequest) - - // Send Request Passing the Client Certs - f.HTTPTestClientWithTLSConfig(clientConfig). - GET("/"). - WithURL(f.GetURL(framework.HTTPS)). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK). - Body().Contains("ssl-client-cert=-----BEGIN%20CERTIFICATE-----%0A") - }) - ginkgo.It("should validate auth-tls-verify-client", func() { - host := authTLSFooHost + host := "authtls.foo.com" nameSpace := f.Namespace clientConfig, err := framework.CreateIngressMASecret( @@ -262,141 +207,11 @@ var _ = framework.DescribeAnnotation("auth-tls-*", func() { WithHeader("Host", host). Expect(). Status(http.StatusOK) - }) - ginkgo.It("should return 403 using auth-tls-match-cn with no matching CN from client", func() { - host := authTLSFooHost - nameSpace := f.Namespace - - clientConfig, err := framework.CreateIngressMASecret( - f.KubeClientSet, - host, - host, - nameSpace) - assert.Nil(ginkgo.GinkgoT(), err) - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/auth-tls-secret": nameSpace + "/" + host, - "nginx.ingress.kubernetes.io/auth-tls-verify-client": "on", - "nginx.ingress.kubernetes.io/auth-tls-match-cn": "CN=notgonnamatch", - } - - f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, nameSpace, framework.EchoService, 80, annotations)) - - assertSslClientCertificateConfig(f, host, "on", "1") - - f.HTTPTestClientWithTLSConfig(clientConfig). - GET("/"). - WithURL(f.GetURL(framework.HTTPS)). - WithHeader("Host", host). - Expect(). - Status(http.StatusForbidden) - }) - - ginkgo.It("should return 200 using auth-tls-match-cn with matching CN from client", func() { - host := authTLSFooHost - nameSpace := f.Namespace - - clientConfig, err := framework.CreateIngressMASecret( - f.KubeClientSet, - host, - host, - nameSpace) - assert.Nil(ginkgo.GinkgoT(), err) - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/auth-tls-secret": nameSpace + "/" + host, - "nginx.ingress.kubernetes.io/auth-tls-verify-client": "on", - "nginx.ingress.kubernetes.io/auth-tls-match-cn": "CN=authtls", - } - - f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, nameSpace, framework.EchoService, 80, annotations)) - - assertSslClientCertificateConfig(f, host, "on", "1") - - f.HTTPTestClientWithTLSConfig(clientConfig). - GET("/"). - WithURL(f.GetURL(framework.HTTPS)). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - }) - - ginkgo.It("should reload the nginx config when auth-tls-match-cn is updated", func() { - host := authTLSFooHost - nameSpace := f.Namespace - - clientConfig, err := framework.CreateIngressMASecret( - f.KubeClientSet, - host, - host, - nameSpace) - assert.Nil(ginkgo.GinkgoT(), err) - - // First add an annotation that forbids our connection - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/auth-tls-secret": nameSpace + "/" + host, - "nginx.ingress.kubernetes.io/auth-tls-verify-client": "on", - "nginx.ingress.kubernetes.io/auth-tls-match-cn": "CN=notvalid", - } - - ingress := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, nameSpace, framework.EchoService, 80, annotations)) - - assertSslClientCertificateConfig(f, host, "on", "1") - - f.HTTPTestClientWithTLSConfig(clientConfig). - GET("/"). - WithURL(f.GetURL(framework.HTTPS)). - WithHeader("Host", host). - Expect(). - Status(http.StatusForbidden) - - // Update the annotation to something that allows the connection - ingress.Annotations["nginx.ingress.kubernetes.io/auth-tls-match-cn"] = "CN=authtls" - f.UpdateIngress(ingress) - - assertSslClientCertificateConfig(f, host, "on", "1") - - f.HTTPTestClientWithTLSConfig(clientConfig). - GET("/"). - WithURL(f.GetURL(framework.HTTPS)). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - }) - - ginkgo.It("should return 200 using auth-tls-match-cn where atleast one of the regex options matches CN from client", func() { - host := authTLSFooHost - nameSpace := f.Namespace - - clientConfig, err := framework.CreateIngressMASecret( - f.KubeClientSet, - host, - host, - nameSpace) - assert.Nil(ginkgo.GinkgoT(), err) - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/auth-tls-secret": nameSpace + "/" + host, - "nginx.ingress.kubernetes.io/auth-tls-verify-client": "on", - "nginx.ingress.kubernetes.io/auth-tls-match-cn": "CN=(itwillmatch|withthenextoption|authtls)", - } - - f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, nameSpace, framework.EchoService, 80, annotations)) - - assertSslClientCertificateConfig(f, host, "on", "1") - - f.HTTPTestClientWithTLSConfig(clientConfig). - GET("/"). - WithURL(f.GetURL(framework.HTTPS)). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) }) }) -//nolint:unparam // Ignore the invariant param: host -func assertSslClientCertificateConfig(f *framework.Framework, host, verifyClient, verifyDepth string) { +func assertSslClientCertificateConfig(f *framework.Framework, host string, verifyClient string, verifyDepth string) { sslClientCertDirective := fmt.Sprintf("ssl_client_certificate /etc/ingress-controller/ssl/%s-%s.pem;", f.Namespace, host) sslVerify := fmt.Sprintf("ssl_verify_client %s;", verifyClient) sslVerifyDepth := fmt.Sprintf("ssl_verify_depth %s;", verifyDepth) diff --git a/test/e2e/annotations/backendprotocol.go b/test/e2e/annotations/backendprotocol.go index 0a181d3c7..db7e50908 100644 --- a/test/e2e/annotations/backendprotocol.go +++ b/test/e2e/annotations/backendprotocol.go @@ -19,13 +19,11 @@ package annotations import ( "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) -const backendProtocolHost = "backendprotocol.foo.com" - var _ = framework.DescribeAnnotation("backend-protocol", func() { f := framework.NewDefaultFramework("backendprotocol") @@ -34,7 +32,7 @@ var _ = framework.DescribeAnnotation("backend-protocol", func() { }) ginkgo.It("should set backend protocol to https:// and use proxy_pass", func() { - host := backendProtocolHost + host := "backendprotocol.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS", } @@ -48,38 +46,8 @@ var _ = framework.DescribeAnnotation("backend-protocol", func() { }) }) - ginkgo.It("should set backend protocol to https:// and use proxy_pass with lowercase annotation", func() { - host := backendProtocolHost - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/backend-protocol": "https", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "proxy_pass https://upstream_balancer;") - }) - }) - - ginkgo.It("should set backend protocol to $scheme:// and use proxy_pass", func() { - host := backendProtocolHost - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/backend-protocol": "AUTO_HTTP", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "proxy_pass $scheme://upstream_balancer;") - }) - }) - ginkgo.It("should set backend protocol to grpc:// and use grpc_pass", func() { - host := backendProtocolHost + host := "backendprotocol.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/backend-protocol": "GRPC", } @@ -94,7 +62,7 @@ var _ = framework.DescribeAnnotation("backend-protocol", func() { }) ginkgo.It("should set backend protocol to grpcs:// and use grpc_pass", func() { - host := backendProtocolHost + host := "backendprotocol.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/backend-protocol": "GRPCS", } @@ -109,7 +77,7 @@ var _ = framework.DescribeAnnotation("backend-protocol", func() { }) ginkgo.It("should set backend protocol to '' and use fastcgi_pass", func() { - host := backendProtocolHost + host := "backendprotocol.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/backend-protocol": "FCGI", } @@ -122,4 +90,19 @@ var _ = framework.DescribeAnnotation("backend-protocol", func() { return strings.Contains(server, "fastcgi_pass upstream_balancer;") }) }) + + ginkgo.It("should set backend protocol to '' and use ajp_pass", func() { + host := "backendprotocol.foo.com" + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/backend-protocol": "AJP", + } + + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "ajp_pass upstream_balancer;") + }) + }) }) diff --git a/test/e2e/annotations/canary.go b/test/e2e/annotations/canary.go index ea733dbf4..c66e9b981 100644 --- a/test/e2e/annotations/canary.go +++ b/test/e2e/annotations/canary.go @@ -19,41 +19,38 @@ package annotations import ( "fmt" "net/http" - "reflect" - "regexp" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/test/e2e/framework" ) const ( - canaryService = "httpbun-canary" + canaryService = "echo-canary" ) var _ = framework.DescribeAnnotation("canary-*", func() { - f := framework.NewDefaultFramework("canary", framework.WithHTTPBunEnabled()) + f := framework.NewDefaultFramework("canary") ginkgo.BeforeEach(func() { + // Deployment for main backend + f.NewEchoDeployment() + // Deployment for canary backend - f.NewHttpbunDeployment(framework.WithDeploymentName(canaryService)) + f.NewEchoDeploymentWithNameAndReplicas(canaryService, 1) }) ginkgo.Context("when canary is created", func() { ginkgo.It("should response with a 200 status from the mainline upstream when requests are made to the mainline ingress", func() { - host := fooHost + host := "foo" annotations := map[string]string{} - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, + framework.EchoService, 80, annotations) + f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { @@ -67,27 +64,20 @@ var _ = framework.DescribeAnnotation("canary-*", func() { canaryIngName := fmt.Sprintf("%v-canary", host) - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, + f.Namespace, canaryService, 80, canaryAnnotations) + f.EnsureIngress(canaryIng) f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). Expect(). Status(http.StatusOK). - Body(). - Contains(framework.HTTPBunService). - NotContains(canaryService) + Body().Contains(framework.EchoService).NotContains(canaryService) }) ginkgo.It("should return 404 status for requests to the canary if no matching ingress is found", func() { - host := fooHost + host := "foo" canaryAnnotations := map[string]string{ "nginx.ingress.kubernetes.io/canary": "true", @@ -96,17 +86,13 @@ var _ = framework.DescribeAnnotation("canary-*", func() { canaryIngName := fmt.Sprintf("%v-canary", host) - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, + f.Namespace, canaryService, 80, canaryAnnotations) + + f.EnsureIngress(canaryIng) f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "always"). Expect(). @@ -118,10 +104,10 @@ var _ = framework.DescribeAnnotation("canary-*", func() { TODO: This test needs improvements made to the e2e framework so that deployment updates work in order to successfully run It("should return the correct status codes when endpoints are unavailable", func() { - host := fooHost + host := "foo" annotations := map[string]string{} - ing := framework.NewSingleIngress(host, "/info", host, f.Namespace, framework.HTTPBunService, 80, annotations) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) f.WaitForNginxServer(host, @@ -136,7 +122,7 @@ var _ = framework.DescribeAnnotation("canary-*", func() { canaryIngName := fmt.Sprintf("%v-canary", host) - canaryIng := framework.NewSingleIngress(canaryIngName, "/info", host, f.Namespace, canaryService, + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, f.Namespace, canaryService, 80, canaryAnnotations) f.EnsureIngress(canaryIng) @@ -144,7 +130,7 @@ var _ = framework.DescribeAnnotation("canary-*", func() { ginkgo.By("returning a 503 status when the mainline deployment has 0 replicas and a request is sent to the canary") - f.NewEchoDeployment(framework.WithDeploymentReplicas(0)) + f.NewEchoDeploymentWithReplicas(0) resp, _, errs := gorequest.New(). Get(f.GetURL(framework.HTTP)). @@ -157,8 +143,8 @@ var _ = framework.DescribeAnnotation("canary-*", func() { ginkgo.By("returning a 200 status when the canary deployment has 0 replicas and a request is sent to the mainline ingress") - f.NewEchoDeployment() - f.NewDeployment(canaryService, "registry.k8s.io/e2e-test-images/echoserver:2.3", 8080, 0) + f.NewEchoDeploymentWithReplicas(1) + f.NewDeployment(canaryService, "gcr.io/kubernetes-e2e-test-images/echoserver:2.2", 8080, 0) resp, _, errs = gorequest.New(). Get(f.GetURL(framework.HTTP)). @@ -172,17 +158,12 @@ var _ = framework.DescribeAnnotation("canary-*", func() { */ ginkgo.It("should route requests to the correct upstream if mainline ingress is created before the canary ingress", func() { - host := fooHost + host := "foo" annotations := map[string]string{} - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, + framework.EchoService, 80, annotations) + f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { @@ -196,41 +177,33 @@ var _ = framework.DescribeAnnotation("canary-*", func() { canaryIngName := fmt.Sprintf("%v-canary", host) - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, + f.Namespace, canaryService, 80, canaryAnnotations) + f.EnsureIngress(canaryIng) ginkgo.By("routing requests destined for the mainline ingress to the maineline upstream") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "never"). Expect(). Status(http.StatusOK). - Body(). - Contains(framework.HTTPBunService). - NotContains(canaryService) + Body().Contains(framework.EchoService).NotContains(canaryService) ginkgo.By("routing requests destined for the canary ingress to the canary upstream") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "always"). Expect(). Status(http.StatusOK). - Body(). - Contains(canaryService) + Body().Contains(canaryService) }) ginkgo.It("should route requests to the correct upstream if mainline ingress is created after the canary ingress", func() { - host := fooHost + host := "foo" canaryAnnotations := map[string]string{ "nginx.ingress.kubernetes.io/canary": "true", @@ -239,25 +212,15 @@ var _ = framework.DescribeAnnotation("canary-*", func() { canaryIngName := fmt.Sprintf("%v-canary", host) - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, + f.Namespace, canaryService, 80, canaryAnnotations) + f.EnsureIngress(canaryIng) annotations := map[string]string{} - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, + framework.EchoService, 80, annotations) + f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { @@ -266,38 +229,30 @@ var _ = framework.DescribeAnnotation("canary-*", func() { ginkgo.By("routing requests destined for the mainline ingress to the mainelin upstream") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "never"). Expect(). Status(http.StatusOK). - Body(). - Contains(framework.HTTPBunService). - NotContains(canaryService) + Body().Contains(framework.EchoService).NotContains(canaryService) ginkgo.By("routing requests destined for the canary ingress to the canary upstream") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "always"). Expect(). Status(http.StatusOK). - Body(). - Contains(canaryService) + Body().Contains(canaryService) }) ginkgo.It("should route requests to the correct upstream if the mainline ingress is modified", func() { - host := fooHost + host := "foo" annotations := map[string]string{} - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, + framework.EchoService, 80, annotations) + f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { @@ -311,27 +266,18 @@ var _ = framework.DescribeAnnotation("canary-*", func() { canaryIngName := fmt.Sprintf("%v-canary", host) - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, + f.Namespace, canaryService, 80, canaryAnnotations) + f.EnsureIngress(canaryIng) modAnnotations := map[string]string{ - fooHost: "bar", + "foo": "bar", } - f.UpdateIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - modAnnotations)) + modIng := framework.NewSingleIngress(host, "/", host, f.Namespace, + framework.EchoService, 80, modAnnotations) + + f.UpdateIngress(modIng) f.WaitForNginxServer(host, func(server string) bool { @@ -340,38 +286,30 @@ var _ = framework.DescribeAnnotation("canary-*", func() { ginkgo.By("routing requests destined fro the mainline ingress to the mainline upstream") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "never"). Expect(). Status(http.StatusOK). - Body(). - Contains(framework.HTTPBunService). - NotContains(canaryService) + Body().Contains(framework.EchoService).NotContains(canaryService) ginkgo.By("routing requests destined for the canary ingress to the canary upstream") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "always"). Expect(). Status(http.StatusOK). - Body(). - Contains(canaryService) + Body().Contains(canaryService) }) ginkgo.It("should route requests to the correct upstream if the canary ingress is modified", func() { - host := fooHost + host := "foo" annotations := map[string]string{} - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, + framework.EchoService, 80, annotations) + f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { @@ -385,74 +323,47 @@ var _ = framework.DescribeAnnotation("canary-*", func() { canaryIngName := fmt.Sprintf("%v-canary", host) - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, + f.Namespace, canaryService, 80, canaryAnnotations) + f.EnsureIngress(canaryIng) - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "server_name foo") - }) - - newAnnotations := map[string]string{ - "nginx.ingress.kubernetes.io/canary": "true", - "nginx.ingress.kubernetes.io/canary-by-header": "CanaryByHeader2", - } - - f.UpdateIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - newAnnotations)) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "server_name foo") + err := framework.UpdateIngress(f.KubeClientSet, f.Namespace, canaryIngName, + func(ingress *networking.Ingress) error { + ingress.ObjectMeta.Annotations = map[string]string{ + "nginx.ingress.kubernetes.io/canary": "true", + "nginx.ingress.kubernetes.io/canary-by-header": "CanaryByHeader2", + } + return nil }) + assert.Nil(ginkgo.GinkgoT(), err) ginkgo.By("routing requests destined for the mainline ingress to the mainline upstream") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader2", "never"). Expect(). Status(http.StatusOK). - Body(). - Contains(framework.HTTPBunService). - NotContains(canaryService) + Body().Contains(framework.EchoService).NotContains(canaryService) ginkgo.By("routing requests destined for the canary ingress to the canary upstream") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader2", "always"). Expect(). Status(http.StatusOK). - Body(). - Contains(canaryService) + Body().Contains(canaryService) }) }) ginkgo.Context("when canaried by header with no value", func() { ginkgo.It("should route requests to the correct upstream", func() { - host := fooHost + host := "foo" - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - nil)) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, + framework.EchoService, 80, nil) + f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { @@ -466,62 +377,48 @@ var _ = framework.DescribeAnnotation("canary-*", func() { canaryIngName := fmt.Sprintf("%v-canary", host) - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, + f.Namespace, canaryService, 80, canaryAnnotations) + f.EnsureIngress(canaryIng) ginkgo.By("routing requests to the canary upstream when header is set to 'always'") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "always"). Expect(). Status(http.StatusOK). - Body(). - Contains(canaryService) + Body().Contains(canaryService) ginkgo.By("routing requests to the mainline upstream when header is set to 'never'") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "never"). Expect(). Status(http.StatusOK). Body(). - Contains(framework.HTTPBunService). - NotContains(canaryService) + Contains(framework.EchoService).NotContains(canaryService) ginkgo.By("routing requests to the mainline upstream when header is set to anything else") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "badheadervalue"). Expect(). Status(http.StatusOK). - Body(). - Contains(framework.HTTPBunService). - NotContains(canaryService) + Body().Contains(framework.EchoService).NotContains(canaryService) }) }) ginkgo.Context("when canaried by header with value", func() { ginkgo.It("should route requests to the correct upstream", func() { - host := fooHost + host := "foo" annotations := map[string]string{} - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) + ing := framework.NewSingleIngress(host, "/", host, + f.Namespace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { @@ -536,73 +433,55 @@ var _ = framework.DescribeAnnotation("canary-*", func() { canaryIngName := fmt.Sprintf("%v-canary", host) - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, + f.Namespace, canaryService, 80, canaryAnnotations) + f.EnsureIngress(canaryIng) ginkgo.By("routing requests to the canary upstream when header is set to 'DoCanary'") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "DoCanary"). Expect(). Status(http.StatusOK). - Body(). - Contains(canaryService) + Body().Contains(canaryService) ginkgo.By("routing requests to the mainline upstream when header is set to 'always'") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "always"). Expect(). Status(http.StatusOK). - Body(). - Contains(framework.HTTPBunService). - NotContains(canaryService) + Body().Contains(framework.EchoService).NotContains(canaryService) ginkgo.By("routing requests to the mainline upstream when header is set to 'never'") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "never"). Expect(). Status(http.StatusOK). - Body(). - Contains(framework.HTTPBunService). - NotContains(canaryService) + Body().Contains(framework.EchoService).NotContains(canaryService) ginkgo.By("routing requests to the mainline upstream when header is set to anything else") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "otherheadervalue"). Expect(). Status(http.StatusOK). - Body(). - Contains(framework.HTTPBunService). - NotContains(canaryService) + Body().Contains(framework.EchoService).NotContains(canaryService) }) }) ginkgo.Context("when canaried by header with value and pattern", func() { ginkgo.It("should route requests to the correct upstream", func() { - host := fooHost + host := "foo" annotations := map[string]string{} - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { @@ -617,18 +496,13 @@ var _ = framework.DescribeAnnotation("canary-*", func() { canaryIngName := fmt.Sprintf("%v-canary", host) - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, f.Namespace, canaryService, + 80, canaryAnnotations) + f.EnsureIngress(canaryIng) ginkgo.By("routing requests to the canary upstream when header pattern is matched") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "DoCanary"). Expect(). @@ -637,25 +511,19 @@ var _ = framework.DescribeAnnotation("canary-*", func() { ginkgo.By("routing requests to the mainline upstream when header failed to match header value") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "Docanary"). Expect(). Status(http.StatusOK). - Body().Contains(framework.HTTPBunService).NotContains(canaryService) + Body().Contains(framework.EchoService).NotContains(canaryService) }) ginkgo.It("should route requests to the correct upstream", func() { - host := fooHost + host := "foo" annotations := map[string]string{} - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { @@ -671,36 +539,25 @@ var _ = framework.DescribeAnnotation("canary-*", func() { canaryIngName := fmt.Sprintf("%v-canary", host) - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, f.Namespace, canaryService, + 80, canaryAnnotations) + f.EnsureIngress(canaryIng) ginkgo.By("routing requests to the mainline upstream when header is set to 'DoCananry' and header-value is 'DoCanary'") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "DoCananry"). Expect(). Status(http.StatusOK). - Body().Contains(framework.HTTPBunService).NotContains(canaryService) + Body().Contains(framework.EchoService).NotContains(canaryService) }) ginkgo.It("should routes to mainline upstream when the given Regex causes error", func() { - host := fooHost + host := "foo" annotations := map[string]string{} - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { @@ -716,40 +573,30 @@ var _ = framework.DescribeAnnotation("canary-*", func() { canaryIngName := fmt.Sprintf("%v-canary", host) - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, f.Namespace, canaryService, + 80, canaryAnnotations) + f.EnsureIngress(canaryIng) ginkgo.By("routing requests to the mainline upstream when the given Regex causes error") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "DoCanary"). WithCookie("CanaryByCookie", "always"). Expect(). Status(http.StatusOK). - Body().Contains(framework.HTTPBunService).NotContains(canaryService) + Body().Contains(framework.EchoService).NotContains(canaryService) }) }) ginkgo.Context("when canaried by header with value and cookie", func() { ginkgo.It("should route requests to the correct upstream", func() { - host := fooHost + host := "foo" annotations := map[string]string{} - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) + ing := framework.NewSingleIngress(host, "/", host, + f.Namespace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { @@ -765,18 +612,13 @@ var _ = framework.DescribeAnnotation("canary-*", func() { canaryIngName := fmt.Sprintf("%v-canary", host) - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, + f.Namespace, canaryService, 80, canaryAnnotations) + f.EnsureIngress(canaryIng) ginkgo.By("routing requests to the canary upstream when header value does not match and cookie is set to 'always'") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). WithHeader("CanaryByHeader", "otherheadervalue"). WithCookie("CanaryByCookie", "always"). @@ -787,18 +629,13 @@ var _ = framework.DescribeAnnotation("canary-*", func() { }) ginkgo.Context("when canaried by cookie", func() { - ginkgo.It("respects always and never values", func() { - host := fooHost + ginkgo.It("should route requests to the correct upstream", func() { + host := "foo" annotations := map[string]string{} - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) + ing := framework.NewSingleIngress(host, "/", host, + f.Namespace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { @@ -812,65 +649,48 @@ var _ = framework.DescribeAnnotation("canary-*", func() { canaryIngName := fmt.Sprintf("%v-canary", host) - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, + f.Namespace, canaryService, 80, canaryAnnotations) + f.EnsureIngress(canaryIng) ginkgo.By("routing requests to the canary upstream when cookie is set to 'always'") - for i := 0; i < 50; i++ { - f.HTTPTestClient(). - GET("/info"). - WithHeader("Host", host). - WithCookie("Canary-By-Cookie", "always"). - Expect(). - Status(http.StatusOK). - Body().Contains(canaryService) - } + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", host). + WithCookie("Canary-By-Cookie", "always"). + Expect(). + Status(http.StatusOK). + Body().Contains(canaryService) ginkgo.By("routing requests to the mainline upstream when cookie is set to 'never'") - for i := 0; i < 50; i++ { - f.HTTPTestClient(). - GET("/info"). - WithHeader("Host", host). - WithCookie("Canary-By-Cookie", "never"). - Expect(). - Status(http.StatusOK). - Body().Contains(framework.HTTPBunService).NotContains(canaryService) - } + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", host). + WithCookie("Canary-By-Cookie", "never"). + Expect(). + Status(http.StatusOK). + Body().Contains(framework.EchoService).NotContains(canaryService) ginkgo.By("routing requests to the mainline upstream when cookie is set to anything else") - for i := 0; i < 50; i++ { - // This test relies on canary cookie not parsing into the valid - // affinity data and canary weight not being specified at all. - f.HTTPTestClient(). - GET("/info"). - WithHeader("Host", host). - WithCookie("Canary-By-Cookie", "badcookievalue"). - Expect(). - Status(http.StatusOK). - Body().Contains(framework.HTTPBunService).NotContains(canaryService) - } + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", host). + WithCookie("Canary-By-Cookie", "badcookievalue"). + Expect(). + Status(http.StatusOK). + Body().Contains(framework.EchoService).NotContains(canaryService) }) }) + // TODO: add testing for canary-weight 0 < weight < 100 ginkgo.Context("when canaried by weight", func() { - ginkgo.It("should route requests only to mainline if canary weight is 0", func() { - host := fooHost + ginkgo.It("should route requests to the correct upstream", func() { + host := "foo" annotations := map[string]string{} - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) + ing := framework.NewSingleIngress(host, "/", host, + f.Namespace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { @@ -883,215 +703,62 @@ var _ = framework.DescribeAnnotation("canary-*", func() { "nginx.ingress.kubernetes.io/canary-weight": "0", } - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "server_name foo") - }) + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, + f.Namespace, canaryService, 80, canaryAnnotations) + f.EnsureIngress(canaryIng) + ginkgo.By("returning requests from the mainline only when weight is equal to 0") f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). Expect(). Status(http.StatusOK). Body(). - Contains(framework.HTTPBunService). + Contains(framework.EchoService). NotContains(canaryService) - }) - ginkgo.It("should route requests only to canary if canary weight is 100", func() { - host := fooHost - annotations := map[string]string{} + ginkgo.By("returning requests from the canary only when weight is equal to 100") - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "server_name foo") + err := framework.UpdateIngress(f.KubeClientSet, f.Namespace, canaryIngName, + func(ingress *networking.Ingress) error { + ingress.ObjectMeta.Annotations = map[string]string{ + "nginx.ingress.kubernetes.io/canary": "true", + "nginx.ingress.kubernetes.io/canary-weight": "100", + } + return nil }) - - canaryIngName := fmt.Sprintf("%v-canary", host) - canaryAnnotations := map[string]string{ - "nginx.ingress.kubernetes.io/canary": "true", - "nginx.ingress.kubernetes.io/canary-weight": "100", - } - - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) + assert.Nil(ginkgo.GinkgoT(), err) f.HTTPTestClient(). - GET("/info"). + GET("/"). WithHeader("Host", host). Expect(). Status(http.StatusOK). Body(). Contains(canaryService) }) - - ginkgo.It("should route requests only to canary if canary weight is equal to canary weight total", func() { - host := fooHost - annotations := map[string]string{} - - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "server_name foo") - }) - - canaryIngName := fmt.Sprintf("%v-canary", host) - canaryAnnotations := map[string]string{ - "nginx.ingress.kubernetes.io/canary": "true", - "nginx.ingress.kubernetes.io/canary-weight": "1000", - "nginx.ingress.kubernetes.io/canary-weight-total": "1000", - } - - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) - - f.HTTPTestClient(). - GET("/info"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK). - Body(). - Contains(canaryService) - }) - - ginkgo.It("should route requests split between mainline and canary if canary weight is 50", func() { - host := fooHost - annotations := map[string]string{} - - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "server_name foo") - }) - - canaryIngName := fmt.Sprintf("%v-canary", host) - canaryAnnotations := map[string]string{ - "nginx.ingress.kubernetes.io/canary": "true", - "nginx.ingress.kubernetes.io/canary-weight": "50", - } - - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) - - TestMainlineCanaryDistribution(f, host) - }) - - ginkgo.It("should route requests split between mainline and canary if canary weight is 100 and weight total is 200", func() { - host := fooHost - annotations := map[string]string{} - - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "server_name foo") - }) - - canaryIngName := fmt.Sprintf("%v-canary", host) - canaryAnnotations := map[string]string{ - "nginx.ingress.kubernetes.io/canary": "true", - "nginx.ingress.kubernetes.io/canary-weight": "100", - "nginx.ingress.kubernetes.io/canary-weight-total": "200", - } - - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) - - TestMainlineCanaryDistribution(f, host) - }) }) ginkgo.Context("Single canary Ingress", func() { ginkgo.It("should not use canary as a catch-all server", func() { - host := fooHost + host := "foo" canaryIngName := fmt.Sprintf("%v-canary", host) annotations := map[string]string{ "nginx.ingress.kubernetes.io/canary": "true", "nginx.ingress.kubernetes.io/canary-by-header": "CanaryByHeader", } - f.EnsureIngress(framework.NewSingleCatchAllIngress( - canaryIngName, - f.Namespace, - canaryService, - 80, - annotations)) + ing := framework.NewSingleCatchAllIngress(canaryIngName, + f.Namespace, canaryService, 80, annotations) + f.EnsureIngress(ing) - f.EnsureIngress(framework.NewSingleCatchAllIngress( - host, - f.Namespace, - framework.HTTPBunService, - 80, - nil)) + ing = framework.NewSingleCatchAllIngress(host, f.Namespace, + framework.EchoService, 80, nil) + f.EnsureIngress(ing) f.WaitForNginxServer("_", func(server string) bool { - upstreamName := fmt.Sprintf(`set $proxy_upstream_name "%s-%s-%s";`, f.Namespace, framework.HTTPBunService, "80") + upstreamName := fmt.Sprintf(`set $proxy_upstream_name "%s-%s-%s";`, f.Namespace, framework.EchoService, "80") canaryUpstreamName := fmt.Sprintf(`set $proxy_upstream_name "%s-%s-%s";`, f.Namespace, canaryService, "80") return strings.Contains(server, fmt.Sprintf(`set $ingress_name "%v";`, host)) && @@ -1102,31 +769,21 @@ var _ = framework.DescribeAnnotation("canary-*", func() { }) ginkgo.It("should not use canary with domain as a server", func() { - host := fooHost + host := "foo" canaryIngName := fmt.Sprintf("%v-canary", host) annotations := map[string]string{ "nginx.ingress.kubernetes.io/canary": "true", "nginx.ingress.kubernetes.io/canary-by-header": "CanaryByHeader", } - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - annotations)) + ing := framework.NewSingleIngress(canaryIngName, "/", host, + f.Namespace, canaryService, 80, annotations) + f.EnsureIngress(ing) otherHost := "bar" - f.EnsureIngress(framework.NewSingleIngress( - otherHost, - "/info", - otherHost, - f.Namespace, - framework.HTTPBunService, - 80, - nil)) + ing = framework.NewSingleIngress(otherHost, "/", otherHost, + f.Namespace, framework.EchoService, 80, nil) + f.EnsureIngress(ing) f.WaitForNginxConfiguration(func(cfg string) bool { return strings.Contains(cfg, "server_name "+otherHost) && @@ -1136,7 +793,7 @@ var _ = framework.DescribeAnnotation("canary-*", func() { }) ginkgo.It("does not crash when canary ingress has multiple paths to the same non-matching backend", func() { - host := fooHost + host := "foo" canaryIngName := fmt.Sprintf("%v-canary", host) annotations := map[string]string{ "nginx.ingress.kubernetes.io/canary": "true", @@ -1144,274 +801,17 @@ var _ = framework.DescribeAnnotation("canary-*", func() { } paths := []string{"/foo", "/bar"} - f.EnsureIngress(framework.NewSingleIngressWithMultiplePaths( - canaryIngName, - paths, - host, - f.Namespace, - "httpy-svc-canary", - 80, - annotations)) + ing := framework.NewSingleIngressWithMultiplePaths(canaryIngName, paths, host, + f.Namespace, "httpy-svc-canary", 80, annotations) + f.EnsureIngress(ing) - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, f.Namespace, - framework.HTTPBunService, - 80, - nil)) + ing = framework.NewSingleIngress(host, "/", host, f.Namespace, + framework.EchoService, 80, nil) + f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "server_name foo") }) }) - - ginkgo.Context("canary affinity behavior", func() { - host := fooHost - affinityCookieName := "aff" - canaryIngName := fmt.Sprintf("%v-canary", host) - - ginkgo.It("always routes traffic to canary if first request was affinitized to canary (default behavior)", func() { - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/affinity": "cookie", - "nginx.ingress.kubernetes.io/session-cookie-name": affinityCookieName, - } - - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "server_name foo") - }) - - // Canary weight is 1% to ensure affinity cookie does its job. - // affinity-canary-behavior annotation is not explicitly configured. - canaryAnnotations := map[string]string{ - "nginx.ingress.kubernetes.io/canary": "true", - "nginx.ingress.kubernetes.io/canary-by-header": "ForceCanary", - "nginx.ingress.kubernetes.io/canary-by-header-value": "yes", - "nginx.ingress.kubernetes.io/canary-weight": "1", - } - - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) - - // This request will produce affinity cookie coming from the canary - // backend. - forcedRequestToCanary := f.HTTPTestClient(). - GET("/info"). - WithHeader("Host", host). - WithHeader("ForceCanary", "yes"). - Expect(). - Status(http.StatusOK) - - // Make sure we got response from canary. - forcedRequestToCanary. - Body().Contains(canaryService) - - affinityCookie := forcedRequestToCanary. - Cookie(affinityCookieName) - - // As long as affinity cookie is present, all requests will be - // routed to a specific backend. - for i := 0; i < 50; i++ { - f.HTTPTestClient(). - GET("/info"). - WithHeader("Host", host). - WithCookie(affinityCookieName, affinityCookie.Raw().Value). - Expect(). - Status(http.StatusOK). - Body().Contains(canaryService) - } - }) - - ginkgo.It("always routes traffic to canary if first request was affinitized to canary (explicit sticky behavior)", func() { - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/affinity": "cookie", - "nginx.ingress.kubernetes.io/session-cookie-name": affinityCookieName, - } - - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "server_name foo") - }) - - // Canary weight is 1% to ensure affinity cookie does its job. - // Explicitly set affinity-canary-behavior annotation to "sticky". - canaryAnnotations := map[string]string{ - "nginx.ingress.kubernetes.io/canary": "true", - "nginx.ingress.kubernetes.io/canary-by-header": "ForceCanary", - "nginx.ingress.kubernetes.io/canary-by-header-value": "yes", - "nginx.ingress.kubernetes.io/canary-weight": "1", - "nginx.ingress.kubernetes.io/affinity-canary-behavior": "sticky", - } - - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) - - // This request will produce affinity cookie coming from the canary - // backend. - forcedRequestToCanary := f.HTTPTestClient(). - GET("/info"). - WithHeader("Host", host). - WithHeader("ForceCanary", "yes"). - Expect(). - Status(http.StatusOK) - - // Make sure we got response from canary. - forcedRequestToCanary. - Body().Contains(canaryService) - - affinityCookie := forcedRequestToCanary. - Cookie(affinityCookieName) - - // As long as affinity cookie is present, all requests will be - // routed to a specific backend. - for i := 0; i < 50; i++ { - f.HTTPTestClient(). - GET("/info"). - WithHeader("Host", host). - WithCookie(affinityCookieName, affinityCookie.Raw().Value). - Expect(). - Status(http.StatusOK). - Body().Contains(canaryService) - } - }) - - ginkgo.It("routes traffic to either mainline or canary backend (legacy behavior)", func() { - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/affinity": "cookie", - "nginx.ingress.kubernetes.io/session-cookie-name": affinityCookieName, - } - - f.EnsureIngress(framework.NewSingleIngress( - host, - "/info", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "server_name foo") - }) - - // Canary weight is 50% to ensure requests are going there. - // Explicitly set affinity-canary-behavior annotation to "legacy". - canaryAnnotations := map[string]string{ - "nginx.ingress.kubernetes.io/canary": "true", - "nginx.ingress.kubernetes.io/canary-by-header": "ForceCanary", - "nginx.ingress.kubernetes.io/canary-by-header-value": "yes", - "nginx.ingress.kubernetes.io/canary-weight": "50", - "nginx.ingress.kubernetes.io/affinity-canary-behavior": "legacy", - } - - f.EnsureIngress(framework.NewSingleIngress( - canaryIngName, - "/info", - host, - f.Namespace, - canaryService, - 80, - canaryAnnotations)) - - // This request will produce affinity cookie coming from the canary - // backend. - forcedRequestToCanary := f.HTTPTestClient(). - GET("/info"). - WithHeader("Host", host). - WithHeader("ForceCanary", "yes"). - Expect(). - Status(http.StatusOK) - - // Make sure we got response from canary. - forcedRequestToCanary. - Body().Contains(canaryService) - - // Legacy behavior results in affinity cookie not being set in - // response. - for _, c := range forcedRequestToCanary.Cookies().Iter() { - if c.String().Raw() == affinityCookieName { - ginkgo.GinkgoT().Error("Affinity cookie is present in response, but was not expected.") - } - } - - TestMainlineCanaryDistribution(f, host) - }) - }) }) - -// This method assumes canary weight being configured at 50%. -func TestMainlineCanaryDistribution(f *framework.Framework, host string) { - re := regexp.MustCompile(fmt.Sprintf(`%s.*`, framework.HTTPBunService)) - replicaRequestCount := map[string]int{} - - // The implementation of choice by weight doesn't guarantee exact - // number of requests, so verify if mainline and canary have at - // least some requests - requestsToGet := 200 - requestsNumberToTest := (40 * requestsToGet) / 100 - - for i := 0; i < requestsToGet; i++ { - body := f.HTTPTestClient(). - GET("/info"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK).Body().Raw() - - replica := re.FindString(body) - assert.NotEmpty(ginkgo.GinkgoT(), replica) - - if _, ok := replicaRequestCount[replica]; !ok { - replicaRequestCount[replica] = 1 - } else { - replicaRequestCount[replica]++ - } - } - - keys := reflect.ValueOf(replicaRequestCount).MapKeys() - - assert.Equal(ginkgo.GinkgoT(), 2, len(keys)) - - assert.GreaterOrEqual( - ginkgo.GinkgoT(), - replicaRequestCount[keys[0].String()], - requestsNumberToTest, - ) - assert.GreaterOrEqual( - ginkgo.GinkgoT(), - replicaRequestCount[keys[1].String()], - requestsNumberToTest, - ) -} diff --git a/test/e2e/annotations/clientbodybuffersize.go b/test/e2e/annotations/clientbodybuffersize.go index 15a3530f3..4091bdad1 100644 --- a/test/e2e/annotations/clientbodybuffersize.go +++ b/test/e2e/annotations/clientbodybuffersize.go @@ -18,15 +18,13 @@ package annotations import ( "fmt" - "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" + "k8s.io/ingress-nginx/test/e2e/framework" ) -const clientBodyBufferSizeHost = "client-body-buffer-size.com" - var _ = framework.DescribeAnnotation("client-body-buffer-size", func() { f := framework.NewDefaultFramework("clientbodybuffersize") @@ -35,7 +33,7 @@ var _ = framework.DescribeAnnotation("client-body-buffer-size", func() { }) ginkgo.It("should set client_body_buffer_size to 1000", func() { - host := clientBodyBufferSizeHost + host := "client-body-buffer-size.com" clientBodyBufferSize := "1000" annotations := make(map[string]string) @@ -48,16 +46,10 @@ var _ = framework.DescribeAnnotation("client-body-buffer-size", func() { func(server string) bool { return strings.Contains(server, fmt.Sprintf("client_body_buffer_size %s;", clientBodyBufferSize)) }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) }) ginkgo.It("should set client_body_buffer_size to 1K", func() { - host := clientBodyBufferSizeHost + host := "client-body-buffer-size.com" clientBodyBufferSize := "1K" annotations := make(map[string]string) @@ -70,16 +62,10 @@ var _ = framework.DescribeAnnotation("client-body-buffer-size", func() { func(server string) bool { return strings.Contains(server, fmt.Sprintf("client_body_buffer_size %s;", clientBodyBufferSize)) }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) }) ginkgo.It("should set client_body_buffer_size to 1k", func() { - host := clientBodyBufferSizeHost + host := "client-body-buffer-size.com" clientBodyBufferSize := "1k" annotations := make(map[string]string) @@ -92,16 +78,10 @@ var _ = framework.DescribeAnnotation("client-body-buffer-size", func() { func(server string) bool { return strings.Contains(server, fmt.Sprintf("client_body_buffer_size %s;", clientBodyBufferSize)) }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) }) ginkgo.It("should set client_body_buffer_size to 1m", func() { - host := clientBodyBufferSizeHost + host := "client-body-buffer-size.com" clientBodyBufferSize := "1m" annotations := make(map[string]string) @@ -114,16 +94,10 @@ var _ = framework.DescribeAnnotation("client-body-buffer-size", func() { func(server string) bool { return strings.Contains(server, fmt.Sprintf("client_body_buffer_size %s;", clientBodyBufferSize)) }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) }) ginkgo.It("should set client_body_buffer_size to 1M", func() { - host := clientBodyBufferSizeHost + host := "client-body-buffer-size.com" clientBodyBufferSize := "1M" annotations := make(map[string]string) @@ -136,16 +110,10 @@ var _ = framework.DescribeAnnotation("client-body-buffer-size", func() { func(server string) bool { return strings.Contains(server, fmt.Sprintf("client_body_buffer_size %s;", clientBodyBufferSize)) }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) }) ginkgo.It("should not set client_body_buffer_size to invalid 1b", func() { - host := clientBodyBufferSizeHost + host := "client-body-buffer-size.com" clientBodyBufferSize := "1b" annotations := make(map[string]string) @@ -158,11 +126,5 @@ var _ = framework.DescribeAnnotation("client-body-buffer-size", func() { func(server string) bool { return !strings.Contains(server, fmt.Sprintf("client_body_buffer_size %s;", clientBodyBufferSize)) }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) }) }) diff --git a/test/e2e/annotations/connection.go b/test/e2e/annotations/connection.go index 428d85876..a3d13702d 100644 --- a/test/e2e/annotations/connection.go +++ b/test/e2e/annotations/connection.go @@ -17,10 +17,11 @@ limitations under the License. package annotations import ( + "fmt" "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -51,6 +52,6 @@ var _ = framework.DescribeAnnotation("connection-proxy-header", func() { WithHeader("Host", host). Expect(). Status(http.StatusOK). - Body().Contains("connection=keep-alive") + Body().Contains(fmt.Sprintf("connection=keep-alive")) }) }) diff --git a/test/e2e/annotations/cors.go b/test/e2e/annotations/cors.go index 58f4445f7..e4cdd32b4 100644 --- a/test/e2e/annotations/cors.go +++ b/test/e2e/annotations/cors.go @@ -20,25 +20,20 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) -const ( - originHost = "http://origin.com:8080" - corsHost = "cors.foo.com" -) - var _ = framework.DescribeAnnotation("cors-*", func() { f := framework.NewDefaultFramework("cors") ginkgo.BeforeEach(func() { - f.NewEchoDeployment(framework.WithDeploymentReplicas(2)) + f.NewEchoDeploymentWithReplicas(2) }) ginkgo.It("should enable cors", func() { - host := corsHost + host := "cors.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/enable-cors": "true", } @@ -49,12 +44,10 @@ var _ = framework.DescribeAnnotation("cors-*", func() { f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "more_set_headers 'Access-Control-Allow-Methods: GET, PUT, POST, DELETE, PATCH, OPTIONS';") && - strings.Contains(server, "more_set_headers 'Access-Control-Allow-Origin: $http_origin';") && - strings.Contains(server, "more_set_headers 'Access-Control-Allow-Headers: DNT,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization';") && + strings.Contains(server, "more_set_headers 'Access-Control-Allow-Origin: *';") && + strings.Contains(server, "more_set_headers 'Access-Control-Allow-Headers: DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization';") && strings.Contains(server, "more_set_headers 'Access-Control-Max-Age: 1728000';") && - strings.Contains(server, "more_set_headers 'Access-Control-Allow-Credentials: true';") && - strings.Contains(server, "set $http_origin *;") && - strings.Contains(server, "$cors 'true';") + strings.Contains(server, "more_set_headers 'Access-Control-Allow-Credentials: true';") }) f.HTTPTestClient(). @@ -65,7 +58,7 @@ var _ = framework.DescribeAnnotation("cors-*", func() { }) ginkgo.It("should set cors methods to only allow POST, GET", func() { - host := corsHost + host := "cors.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/enable-cors": "true", "nginx.ingress.kubernetes.io/cors-allow-methods": "POST, GET", @@ -81,7 +74,7 @@ var _ = framework.DescribeAnnotation("cors-*", func() { }) ginkgo.It("should set cors max-age", func() { - host := corsHost + host := "cors.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/enable-cors": "true", "nginx.ingress.kubernetes.io/cors-max-age": "200", @@ -97,7 +90,7 @@ var _ = framework.DescribeAnnotation("cors-*", func() { }) ginkgo.It("should disable cors allow credentials", func() { - host := corsHost + host := "cors.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/enable-cors": "true", "nginx.ingress.kubernetes.io/cors-allow-credentials": "false", @@ -113,8 +106,7 @@ var _ = framework.DescribeAnnotation("cors-*", func() { }) ginkgo.It("should allow origin for cors", func() { - host := corsHost - origin := "https://origin.cors.com:8080" + host := "cors.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/enable-cors": "true", "nginx.ingress.kubernetes.io/cors-allow-origin": "https://origin.cors.com:8080", @@ -123,24 +115,14 @@ var _ = framework.DescribeAnnotation("cors-*", func() { ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Headers().ContainsKey("Access-Control-Allow-Origin") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{origin}) + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "more_set_headers 'Access-Control-Allow-Origin: https://origin.cors.com:8080';") + }) }) ginkgo.It("should allow headers for cors", func() { - host := corsHost + host := "cors.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/enable-cors": "true", "nginx.ingress.kubernetes.io/cors-allow-headers": "DNT, User-Agent", @@ -156,7 +138,7 @@ var _ = framework.DescribeAnnotation("cors-*", func() { }) ginkgo.It("should expose headers for cors", func() { - host := corsHost + host := "cors.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/enable-cors": "true", "nginx.ingress.kubernetes.io/cors-expose-headers": "X-CustomResponseHeader, X-CustomSecondHeader", @@ -170,532 +152,4 @@ var _ = framework.DescribeAnnotation("cors-*", func() { return strings.Contains(server, "more_set_headers 'Access-Control-Expose-Headers: X-CustomResponseHeader, X-CustomSecondHeader';") }) }) - - ginkgo.It("should allow - single origin for multiple cors values", func() { - host := corsHost - origin := "https://origin.cors.com:8080" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - "nginx.ingress.kubernetes.io/cors-allow-origin": "https://origin.cors.com:8080, https://origin2.cors.com", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Headers().ContainsKey("Access-Control-Allow-Origin") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{origin}) - }) - - ginkgo.It("should not allow - single origin for multiple cors values", func() { - host := corsHost - origin := "http://no.origin.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - "nginx.ingress.kubernetes.io/cors-allow-origin": "http://origin2.cors.com, https://origin.com", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - // the client should still receive a response but browsers should block the request - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Headers().NotContainsKey("Access-Control-Allow-Origin") - }) - - ginkgo.It("should allow correct origins - single origin for multiple cors values", func() { - host := corsHost - badOrigin := "origin.cors.com:8080" - origin1 := "https://origin2.cors.com" - origin2 := "https://origin.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - "nginx.ingress.kubernetes.io/cors-allow-origin": "origin.cors.com:8080, https://origin2.cors.com, https://origin.com", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", badOrigin). - Expect(). - Headers().NotContainsKey("Access-Control-Allow-Origin") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin1). - Expect(). - Headers().ContainsKey("Access-Control-Allow-Origin") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin1). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{origin1}) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin2). - Expect(). - Headers().ContainsKey("Access-Control-Allow-Origin") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin2). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{origin2}) - }) - - ginkgo.It("should not break functionality", func() { - host := corsHost - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - "nginx.ingress.kubernetes.io/cors-allow-origin": "*", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Headers().ContainsKey("Access-Control-Allow-Origin") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{"*"}) - }) - - ginkgo.It("should not break functionality - without `*`", func() { - host := corsHost - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Headers().ContainsKey("Access-Control-Allow-Origin") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{"*"}) - }) - - ginkgo.It("should not break functionality with extra domain", func() { - host := corsHost - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - "nginx.ingress.kubernetes.io/cors-allow-origin": "*, foo.bar.com", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Headers().ContainsKey("Access-Control-Allow-Origin") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{"*"}) - }) - - ginkgo.It("should not match", func() { - host := corsHost - origin := "https://fooxbar.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - "nginx.ingress.kubernetes.io/cors-allow-origin": "https://foo.bar.com", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - // the client should still receive a response but browsers should block the request - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Headers().NotContainsKey("Access-Control-Allow-Origin") - }) - - ginkgo.It("should allow - single origin with required port", func() { - host := corsHost - origin := originHost - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - "nginx.ingress.kubernetes.io/cors-allow-origin": "http://origin.cors.com:8080, http://origin.com:8080", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - // the client should still receive a response but browsers should block the request - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Headers().ContainsKey("Access-Control-Allow-Origin") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{origin}) - }) - - ginkgo.It("should not allow - single origin with port and origin without port", func() { - host := corsHost - origin := originHost - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - "nginx.ingress.kubernetes.io/cors-allow-origin": "https://origin2.cors.com, http://origin.com", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Headers().NotContainsKey("Access-Control-Allow-Origin") - }) - - ginkgo.It("should not allow - single origin without port and origin with required port", func() { - host := corsHost - origin := "http://origin.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - "nginx.ingress.kubernetes.io/cors-allow-origin": "http://origin.cors.com:8080, http://origin.com:8080", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - // the client should still receive a response but browsers should block the request - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Headers().NotContainsKey("Access-Control-Allow-Origin") - }) - - ginkgo.It("should allow - matching origin with wildcard origin (2 subdomains)", func() { - host := corsHost - origin := "http://foo.origin.cors.com" - origin2 := "http://bar-foo.origin.cors.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - "nginx.ingress.kubernetes.io/cors-allow-origin": "http://*.origin.cors.com, http://*.origin.com:8080", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Headers().ContainsKey("Access-Control-Allow-Origin") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{origin}) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin2). - Expect(). - Headers().ContainsKey("Access-Control-Allow-Origin") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin2). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{origin2}) - }) - - ginkgo.It("should not allow - unmatching origin with wildcard origin (2 subdomains)", func() { - host := corsHost - origin := "http://bar.foo.origin.cors.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - "nginx.ingress.kubernetes.io/cors-allow-origin": "http://*.origin.cors.com, http://*.origin.com:8080", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - // the client should still receive a response but browsers should block the request - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Headers().NotContainsKey("Access-Control-Allow-Origin") - }) - - ginkgo.It("should allow - matching origin+port with wildcard origin", func() { - host := corsHost - origin := "http://abc.origin.com:8080" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - "nginx.ingress.kubernetes.io/cors-allow-origin": "http://origin.cors.com:8080, http://*.origin.com:8080", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Headers().ContainsKey("Access-Control-Allow-Origin") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{origin}) - }) - - ginkgo.It("should not allow - portless origin with wildcard origin", func() { - host := corsHost - origin := "http://abc.origin.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - "nginx.ingress.kubernetes.io/cors-allow-origin": "http://origin.cors.com:8080, http://*.origin.com:8080", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - // the client should still receive a response but browsers should block the request - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Headers().NotContainsKey("Access-Control-Allow-Origin") - }) - - ginkgo.It("should allow correct origins - missing subdomain + origin with wildcard origin and correct origin", func() { - host := corsHost - badOrigin := originHost - origin := "http://bar.origin.com:8080" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - "nginx.ingress.kubernetes.io/cors-allow-origin": "http://origin.cors.com:8080, http://*.origin.com:8080", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - // the client should still receive a response but browsers should block the request - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", badOrigin). - Expect(). - Headers().NotContainsKey("Access-Control-Allow-Origin") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Headers().ContainsKey("Access-Control-Allow-Origin") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{origin}) - }) - - ginkgo.It("should allow - missing origins (should allow all origins)", func() { - host := corsHost - origin := "http://origin.com" - origin2 := "http://book.origin.com" - origin3 := "test.origin.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - "nginx.ingress.kubernetes.io/cors-allow-origin": " ", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - // the client should still receive a response but browsers should block the request - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Headers().ContainsKey("Access-Control-Allow-Origin") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{"*"}) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin2). - Expect(). - Headers().ContainsKey("Access-Control-Allow-Origin") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin2). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{"*"}) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin3). - Expect(). - Headers().ContainsKey("Access-Control-Allow-Origin") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin3). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{"*"}) - }) - - ginkgo.It("should allow correct origin but not others - cors allow origin annotations contain trailing comma", func() { - host := corsHost - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - "nginx.ingress.kubernetes.io/cors-allow-origin": "https://origin-123.cors.com:8080, ,https://origin-321.cors.com:8080,", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - origin1 := "https://origin-123.cors.com:8080" - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin1). - Expect(). - Headers().ContainsKey("Access-Control-Allow-Origin") - - origin2 := "https://origin-321.cors.com:8080" - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin2). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{origin2}) - - origin3 := "https://unknown.cors.com:8080" - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin3). - Expect(). - Headers(). - NotContainsKey("Access-Control-Allow-Origin") - }) - - ginkgo.It("should allow - origins with non-http[s] protocols", func() { - host := corsHost - origin := "test://localhost" - origin2 := "tauri://localhost:3000" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-cors": "true", - "nginx.ingress.kubernetes.io/cors-allow-origin": "test://localhost, tauri://localhost:3000", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{"test://localhost"}) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("Origin", origin2). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Access-Control-Allow-Origin", []string{"tauri://localhost:3000"}) - }) }) diff --git a/test/e2e/annotations/customheaders.go b/test/e2e/annotations/customheaders.go deleted file mode 100644 index 274ce8278..000000000 --- a/test/e2e/annotations/customheaders.go +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package annotations - -import ( - "fmt" - "net/http" - "strings" - - "github.com/onsi/ginkgo/v2" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -const ( - customHeaderHost = "custom-headers" -) - -var _ = framework.DescribeAnnotation("custom-headers-*", func() { - f := framework.NewDefaultFramework("custom-headers") - - ginkgo.BeforeEach(func() { - f.NewEchoDeployment() - }) - - ginkgo.It("should return status code 200 when no custom-headers is configured", func() { - ing := framework.NewSingleIngress(customHeaderHost, "/", customHeaderHost, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ing) - - f.WaitForNginxServer(customHeaderHost, - func(server string) bool { - return strings.Contains(server, "server_name custom-headers") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", customHeaderHost). - Expect(). - Status(http.StatusOK). - Body().Contains(fmt.Sprintf("host=%v", customHeaderHost)) - }) - - ginkgo.It("should return status code 503 when custom-headers is configured with an invalid secret", func() { - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/custom-headers": f.Namespace + "/custom-headers", - } - - ing := framework.NewSingleIngress(customHeaderHost, "/", customHeaderHost, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(customHeaderHost, - func(server string) bool { - return strings.Contains(server, "server_name custom-headers") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", customHeaderHost). - Expect(). - Status(http.StatusServiceUnavailable). - Body().Contains("503 Service Temporarily Unavailable") - }) - - ginkgo.It(`should set "more_set_headers 'My-Custom-Header' '42';" when custom-headers are set`, func() { - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/custom-headers": f.Namespace + "/custom-headers", - } - - f.CreateConfigMap("custom-headers", map[string]string{ - "My-Custom-Header": "42", - "My-Custom-Header-Dollar": "$remote_addr", - }) - f.UpdateNginxConfigMapData("global-allowed-response-headers", "My-Custom-Header,My-Custom-Header-Dollar") - - ing := framework.NewSingleIngress(customHeaderHost, "/", customHeaderHost, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(customHeaderHost, - func(server string) bool { - return strings.Contains(server, `more_set_headers "My-Custom-Header: 42";`) - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", customHeaderHost). - Expect(). - Status(http.StatusOK). - Header("My-Custom-Header").Contains("42") - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", customHeaderHost). - Expect(). - Status(http.StatusOK). - Header("My-Custom-Header-Dollar").Contains("$remote_addr") - }) -}) diff --git a/test/e2e/annotations/customhttperrors.go b/test/e2e/annotations/customhttperrors.go index 37a3e9695..7369ebe6e 100644 --- a/test/e2e/annotations/customhttperrors.go +++ b/test/e2e/annotations/customhttperrors.go @@ -20,14 +20,14 @@ import ( "fmt" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/test/e2e/framework" ) -func errorBlockName(upstreamName, errorCode string) string { +func errorBlockName(upstreamName string, errorCode string) string { return fmt.Sprintf("@custom_%s_%s", upstreamName, errorCode) } @@ -101,7 +101,7 @@ var _ = framework.DescribeAnnotation("custom-http-errors", func() { ginkgo.By("using the custom default-backend from annotation for upstream") customDefaultBackend := "from-annotation" - f.NewEchoDeployment(framework.WithDeploymentName(customDefaultBackend)) + f.NewEchoDeploymentWithNameAndReplicas(customDefaultBackend, 1) err = framework.UpdateIngress(f.KubeClientSet, f.Namespace, host, func(ingress *networking.Ingress) error { ingress.ObjectMeta.Annotations["nginx.ingress.kubernetes.io/default-backend"] = customDefaultBackend @@ -116,7 +116,7 @@ var _ = framework.DescribeAnnotation("custom-http-errors", func() { } return false }) - assert.Contains(ginkgo.GinkgoT(), serverConfig, errorBlockName(fmt.Sprintf("custom-default-backend-%s-%s", f.Namespace, customDefaultBackend), "503")) - assert.Contains(ginkgo.GinkgoT(), serverConfig, fmt.Sprintf("error_page %s = %s", "503", errorBlockName(fmt.Sprintf("custom-default-backend-%s-%s", f.Namespace, customDefaultBackend), "503"))) + assert.Contains(ginkgo.GinkgoT(), serverConfig, errorBlockName(fmt.Sprintf("custom-default-backend-%s", customDefaultBackend), "503")) + assert.Contains(ginkgo.GinkgoT(), serverConfig, fmt.Sprintf("error_page %s = %s", "503", errorBlockName(fmt.Sprintf("custom-default-backend-%s", customDefaultBackend), "503"))) }) }) diff --git a/test/e2e/annotations/default_backend.go b/test/e2e/annotations/default_backend.go index 72ca303b5..15a3505af 100644 --- a/test/e2e/annotations/default_backend.go +++ b/test/e2e/annotations/default_backend.go @@ -21,7 +21,7 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -48,18 +48,18 @@ var _ = framework.DescribeAnnotation("default-backend", func() { return strings.Contains(server, fmt.Sprintf("server_name %v", host)) }) - requestID := "something-unique" + requestId := "something-unique" f.HTTPTestClient(). GET("/alma/armud"). WithHeader("Host", host). - WithHeader("x-request-id", requestID). + WithHeader("x-request-id", requestId). Expect(). Status(http.StatusOK). Body().Contains("x-code=503"). Contains(fmt.Sprintf("x-ingress-name=%s", host)). Contains("x-service-name=invalid"). - Contains(fmt.Sprintf("x-request-id=%s", requestID)) + Contains(fmt.Sprintf("x-request-id=%s", requestId)) }) }) }) diff --git a/test/e2e/annotations/disableaccesslog.go b/test/e2e/annotations/disableaccesslog.go index 4d152b45d..dab54c561 100644 --- a/test/e2e/annotations/disableaccesslog.go +++ b/test/e2e/annotations/disableaccesslog.go @@ -17,10 +17,9 @@ limitations under the License. package annotations import ( - "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -42,12 +41,6 @@ var _ = framework.DescribeAnnotation("disable-access-log disable-http-access-log f.WaitForNginxConfiguration(func(ngx string) bool { return strings.Contains(ngx, `access_log off;`) }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) }) ginkgo.It("disable-http-access-log set access_log off", func() { @@ -60,29 +53,5 @@ var _ = framework.DescribeAnnotation("disable-access-log disable-http-access-log f.WaitForNginxConfiguration(func(ngx string) bool { return strings.Contains(ngx, `access_log off;`) }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - }) - - ginkgo.It("disable-stream-access-log set access_log off", func() { - host := "disablehttpaccesslog.foo.com" - - f.UpdateNginxConfigMapData("disable-stream-access-log", "true") - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ing) - - f.WaitForNginxConfiguration(func(ngx string) bool { - return strings.Contains(ngx, `access_log off;`) - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) }) }) diff --git a/test/e2e/annotations/disableproxyintercepterrors.go b/test/e2e/annotations/disableproxyintercepterrors.go deleted file mode 100644 index 17efa4588..000000000 --- a/test/e2e/annotations/disableproxyintercepterrors.go +++ /dev/null @@ -1,100 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package annotations - -import ( - "fmt" - "net/http" - "strings" - - networking "k8s.io/api/networking/v1" - - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.DescribeAnnotation("disable-proxy-intercept-errors", func() { - f := framework.NewDefaultFramework("disable-proxy-intercept-errors") - - ginkgo.BeforeEach(func() { - f.NewHttpbunDeployment() - f.NewEchoDeployment() - }) - - ginkgo.It("configures Nginx correctly", func() { - host := "pie.foo.com" - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/custom-http-errors": "404", - "nginx.ingress.kubernetes.io/disable-proxy-intercept-errors": "true", - "nginx.ingress.kubernetes.io/default-backend": framework.EchoService, - } - - ingHTTPBunService := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.HTTPBunService, 80, annotations) - f.EnsureIngress(ingHTTPBunService) - - var serverConfig string - f.WaitForNginxServer(host, func(sc string) bool { - serverConfig = sc - return strings.Contains(serverConfig, fmt.Sprintf("server_name %s", host)) - }) - - ginkgo.By("turning off proxy_intercept_errors directive") - assert.NotContains(ginkgo.GinkgoT(), serverConfig, "proxy_intercept_errors on;") - - // the plan for client side testing - // create ingress where we disable intercept for code 404 - that error should get to the client - // the same ingress should intercept any other error (>300 and not 404) where we will get intercepted error - ginkgo.By("client test to check response - with intercept disabled") - requestID := "proxy_intercept_errors" - - f.HTTPTestClient(). - GET("/status/404"). - WithHeader("Host", host). - WithHeader("x-request-id", requestID). - Expect(). - Status(http.StatusNotFound). - Body().Empty() - - ginkgo.By("client test to check response - with intercept enabled") - err := framework.UpdateIngress(f.KubeClientSet, f.Namespace, host, func(ingress *networking.Ingress) error { - ingress.ObjectMeta.Annotations["nginx.ingress.kubernetes.io/disable-proxy-intercept-errors"] = "false" - return nil - }) - assert.Nil(ginkgo.GinkgoT(), err) - - f.WaitForNginxServer(host, func(sc string) bool { - if serverConfig != sc { - serverConfig = sc - return true - } - return false - }) - - f.HTTPTestClient(). - GET("/status/404"). - WithHeader("Host", host). - WithHeader("x-request-id", requestID). - Expect(). - Status(http.StatusOK). - Body().Contains("x-code=404"). - Contains(fmt.Sprintf("x-ingress-name=%s", host)). - Contains(fmt.Sprintf("x-service-name=%s", framework.HTTPBunService)). - Contains(fmt.Sprintf("x-request-id=%s", requestID)) - }) -}) diff --git a/test/e2e/annotations/fastcgi.go b/test/e2e/annotations/fastcgi.go index bcf1c3487..f1f6d3ac1 100644 --- a/test/e2e/annotations/fastcgi.go +++ b/test/e2e/annotations/fastcgi.go @@ -20,7 +20,8 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" + "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -75,14 +76,16 @@ var _ = framework.DescribeAnnotation("backend-protocol - FastCGI", func() { Namespace: f.Namespace, }, Data: map[string]string{ - "SCRIPT_FILENAME": "$fastcgi_script_name", + "SCRIPT_FILENAME": "/home/www/scripts/php$fastcgi_script_name", "REDIRECT_STATUS": "200", }, } - f.EnsureConfigMap(configuration) + cm, err := f.EnsureConfigMap(configuration) + assert.Nil(ginkgo.GinkgoT(), err, "creating configmap") + assert.NotNil(ginkgo.GinkgoT(), cm, "expected a configmap but none returned") - host := "fastcgi-params-configmap" //#nosec G101 + host := "fastcgi-params-configmap" annotations := map[string]string{ "nginx.ingress.kubernetes.io/backend-protocol": "FCGI", @@ -94,7 +97,7 @@ var _ = framework.DescribeAnnotation("backend-protocol - FastCGI", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "fastcgi_param SCRIPT_FILENAME \"$fastcgi_script_name\";") && + return strings.Contains(server, "fastcgi_param SCRIPT_FILENAME \"/home/www/scripts/php$fastcgi_script_name\";") && strings.Contains(server, "fastcgi_param REDIRECT_STATUS \"200\";") }) }) diff --git a/test/e2e/annotations/forcesslredirect.go b/test/e2e/annotations/forcesslredirect.go index 4f5316971..8fa2e8f60 100644 --- a/test/e2e/annotations/forcesslredirect.go +++ b/test/e2e/annotations/forcesslredirect.go @@ -19,7 +19,7 @@ package annotations import ( "net/http" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/annotations/fromtowwwredirect.go b/test/e2e/annotations/fromtowwwredirect.go index a3fb3b9b5..4fb1763b2 100644 --- a/test/e2e/annotations/fromtowwwredirect.go +++ b/test/e2e/annotations/fromtowwwredirect.go @@ -22,7 +22,7 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" @@ -58,13 +58,10 @@ var _ = framework.DescribeAnnotation("from-to-www-redirect", func() { WithHeader("Host", fmt.Sprintf("%s.%s", "www", host)). Expect(). Status(http.StatusPermanentRedirect). - Header("Location").Equal("http://fromtowwwredirect.bar.com:80/foo") + Header("Location").Equal("http://fromtowwwredirect.bar.com/foo") }) ginkgo.It("should redirect from www HTTPS to HTTPS", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - ginkgo.By("setting up server for redirect from www") fromHost := fmt.Sprintf("%s.nip.io", f.GetNginxIP()) @@ -93,7 +90,7 @@ var _ = framework.DescribeAnnotation("from-to-www-redirect", func() { ginkgo.By("sending request to www should redirect to domain") f.HTTPTestClientWithTLSConfig(&tls.Config{ - InsecureSkipVerify: true, //nolint:gosec // Ignore the gosec error in testing + InsecureSkipVerify: true, ServerName: toHost, }). GET("/"). @@ -101,11 +98,11 @@ var _ = framework.DescribeAnnotation("from-to-www-redirect", func() { WithHeader("Host", toHost). Expect(). Status(http.StatusPermanentRedirect). - Header("Location").Equal(fmt.Sprintf("https://%v:443", fromHost)) + Header("Location").Equal(fmt.Sprintf("https://%v", fromHost)) ginkgo.By("sending request to domain should not redirect to www") f.HTTPTestClientWithTLSConfig(&tls.Config{ - InsecureSkipVerify: true, //nolint:gosec // Ignore the gosec error in testing + InsecureSkipVerify: true, ServerName: fromHost, }). GET("/"). diff --git a/test/e2e/annotations/globalratelimit.go b/test/e2e/annotations/globalratelimit.go new file mode 100644 index 000000000..dd985c68c --- /dev/null +++ b/test/e2e/annotations/globalratelimit.go @@ -0,0 +1,83 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package annotations + +import ( + "fmt" + "net/http" + "strings" + + "github.com/onsi/ginkgo" + "github.com/stretchr/testify/assert" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.DescribeAnnotation("annotation-global-rate-limit", func() { + f := framework.NewDefaultFramework("global-rate-limit") + host := "global-rate-limit-annotation" + + ginkgo.BeforeEach(func() { + f.NewEchoDeployment() + }) + + ginkgo.It("generates correct configuration", func() { + annotations := make(map[string]string) + annotations["nginx.ingress.kubernetes.io/global-rate-limit"] = "5" + annotations["nginx.ingress.kubernetes.io/global-rate-limit-window"] = "2m" + + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) + ing = f.EnsureIngress(ing) + namespace := strings.Replace(string(ing.UID), "-", "", -1) + + serverConfig := "" + f.WaitForNginxServer(host, func(server string) bool { + serverConfig = server + return true + }) + assert.Contains(ginkgo.GinkgoT(), serverConfig, + fmt.Sprintf(`global_throttle = { namespace = "%v", `+ + `limit = 5, window_size = 120, key = { { nil, nil, "remote_addr", nil, }, }, `+ + `ignored_cidrs = { } }`, + namespace)) + + f.HTTPTestClient().GET("/").WithHeader("Host", host).Expect().Status(http.StatusOK) + + ginkgo.By("regenerating the correct configuration after update") + annotations["nginx.ingress.kubernetes.io/global-rate-limit-key"] = "${remote_addr}${http_x_api_client}" + annotations["nginx.ingress.kubernetes.io/global-rate-limit-ignored-cidrs"] = "192.168.1.1, 234.234.234.0/24" + ing.SetAnnotations(annotations) + + f.WaitForReload(func() { + ing = f.UpdateIngress(ing) + }) + + serverConfig = "" + f.WaitForNginxServer(host, func(server string) bool { + serverConfig = server + return true + }) + assert.Contains(ginkgo.GinkgoT(), serverConfig, + fmt.Sprintf(`global_throttle = { namespace = "%v", `+ + `limit = 5, window_size = 120, `+ + `key = { { nil, "remote_addr", nil, nil, }, { nil, "http_x_api_client", nil, nil, }, }, `+ + `ignored_cidrs = { "192.168.1.1", "234.234.234.0/24", } }`, + namespace)) + + f.HTTPTestClient().GET("/").WithHeader("Host", host).Expect().Status(http.StatusOK) + }) +}) diff --git a/test/e2e/annotations/grpc.go b/test/e2e/annotations/grpc.go index 2a9c5a983..003277885 100644 --- a/test/e2e/annotations/grpc.go +++ b/test/e2e/annotations/grpc.go @@ -17,19 +17,17 @@ limitations under the License. package annotations import ( - "context" "crypto/tls" "fmt" "strings" - delaypb "github.com/Anddd7/pb/grpcbin" pb "github.com/moul/pb/grpcbin/go-grpc" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" + "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/credentials" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/metadata" + core "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -37,17 +35,14 @@ import ( "k8s.io/ingress-nginx/test/e2e/framework" ) -const ( - echoHost = "echo" - host = "grpc" -) - var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { - f := framework.NewDefaultFramework("grpc", framework.WithHTTPBunEnabled()) + f := framework.NewDefaultFramework("grpc") ginkgo.It("should use grpc_pass in the configuration file", func() { f.NewGRPCFortuneTellerDeployment() + host := "grpc" + annotations := map[string]string{ "nginx.ingress.kubernetes.io/backend-protocol": "GRPC", } @@ -71,9 +66,9 @@ var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { ginkgo.It("should return OK for service with backend protocol GRPC", func() { f.NewGRPCBinDeployment() - host := echoHost + host := "echo" - svc := &corev1.Service{ + svc := &core.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "grpcbin-test", Namespace: f.Namespace, @@ -106,16 +101,14 @@ var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { return strings.Contains(server, "grpc_pass grpc://upstream_balancer;") }) - //nolint:goconst //string interpolation - conn, err := grpc.NewClient(f.GetNginxIP()+":443", + conn, _ := grpc.Dial(f.GetNginxIP()+":443", grpc.WithTransportCredentials( credentials.NewTLS(&tls.Config{ - ServerName: echoHost, - InsecureSkipVerify: true, //nolint:gosec // Ignore certificate validation in testing + ServerName: "echo", + InsecureSkipVerify: true, }), ), ) - assert.Nil(ginkgo.GinkgoT(), err, "error creating a connection") defer conn.Close() client := pb.NewGRPCBinClient(conn) @@ -126,79 +119,14 @@ var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { metadata := res.GetMetadata() assert.Equal(ginkgo.GinkgoT(), metadata["content-type"].Values[0], "application/grpc") - assert.Equal(ginkgo.GinkgoT(), metadata[":authority"].Values[0], host) - }) - - ginkgo.It("authorization metadata should be overwritten by external auth response headers", func() { - f.NewGRPCBinDeployment() - host := echoHost - - svc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "grpcbin-test", - Namespace: f.Namespace, - }, - Spec: corev1.ServiceSpec{ - ExternalName: fmt.Sprintf("grpcbin.%v.svc.cluster.local", f.Namespace), - Type: corev1.ServiceTypeExternalName, - Ports: []corev1.ServicePort{ - { - Name: host, - Port: 9000, - TargetPort: intstr.FromInt(9000), - Protocol: "TCP", - }, - }, - }, - } - f.EnsureService(svc) - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/response-headers?authorization=foo", f.HTTPBunIP), - "nginx.ingress.kubernetes.io/auth-response-headers": "Authorization", - "nginx.ingress.kubernetes.io/backend-protocol": "GRPC", - } - - ing := framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, "grpcbin-test", 9000, annotations) - - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "grpc_pass grpc://upstream_balancer;") - }) - - conn, err := grpc.NewClient(f.GetNginxIP()+":443", - grpc.WithTransportCredentials( - credentials.NewTLS(&tls.Config{ - ServerName: echoHost, - InsecureSkipVerify: true, //nolint:gosec // Ignore certificate validation in testing - }), - ), - ) - assert.Nil(ginkgo.GinkgoT(), err) - defer conn.Close() - - client := pb.NewGRPCBinClient(conn) - ctx := metadata.AppendToOutgoingContext(context.Background(), - "authorization", "bar") - - res, err := client.HeadersUnary(ctx, &pb.EmptyMessage{}) - assert.Nil(ginkgo.GinkgoT(), err) - - metadata := res.GetMetadata() - assert.Equal(ginkgo.GinkgoT(), fooHost, metadata["authorization"].Values[0]) }) ginkgo.It("should return OK for service with backend protocol GRPCS", func() { f.NewGRPCBinDeployment() - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() + host := "echo" - host := echoHost - - svc := &corev1.Service{ + svc := &core.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "grpcbin-test", Namespace: f.Namespace, @@ -236,15 +164,14 @@ var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { return strings.Contains(server, "grpc_pass grpcs://upstream_balancer;") }) - conn, err := grpc.NewClient(f.GetNginxIP()+":443", + conn, _ := grpc.Dial(f.GetNginxIP()+":443", grpc.WithTransportCredentials( credentials.NewTLS(&tls.Config{ - ServerName: echoHost, - InsecureSkipVerify: true, //nolint:gosec // Ignore the gosec error in testing + ServerName: "echo", + InsecureSkipVerify: true, }), ), ) - assert.Nil(ginkgo.GinkgoT(), err) defer conn.Close() client := pb.NewGRPCBinClient(conn) @@ -256,89 +183,4 @@ var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { metadata := res.GetMetadata() assert.Equal(ginkgo.GinkgoT(), metadata["content-type"].Values[0], "application/grpc") }) - - ginkgo.It("should return OK when request not exceed timeout", func() { - f.NewGRPCBinDelayDeployment() - - proxyTimeout := "10" - ingressName := "grpcbin-delay" - - annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/backend-protocol"] = "GRPC" - annotations["nginx.ingress.kubernetes.io/proxy-connect-timeout"] = proxyTimeout - annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = proxyTimeout - annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = proxyTimeout - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, ingressName, 50051, annotations) - - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, fmt.Sprintf("grpc_connect_timeout %ss;", proxyTimeout)) && - strings.Contains(server, fmt.Sprintf("grpc_send_timeout %ss;", proxyTimeout)) && - strings.Contains(server, fmt.Sprintf("grpc_read_timeout %ss;", proxyTimeout)) - }) - - conn, err := grpc.NewClient( - f.GetNginxIP()+":80", - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithAuthority(host), - ) - assert.Nil(ginkgo.GinkgoT(), err, "error creating a connection") - defer conn.Close() - - client := delaypb.NewGrpcbinServiceClient(conn) - - res, err := client.Unary(context.Background(), &delaypb.UnaryRequest{ - Data: "hello", - }) - assert.Nil(ginkgo.GinkgoT(), err) - - metadata := res.GetResponseAttributes().RequestHeaders - assert.Equal(ginkgo.GinkgoT(), metadata["content-type"], "application/grpc") - assert.Equal(ginkgo.GinkgoT(), metadata[":authority"], host) - }) - - ginkgo.It("should return Error when request exceed timeout", func() { - f.NewGRPCBinDelayDeployment() - - proxyTimeout := "10" - ingressName := "grpcbin-delay" - - annotations := make(map[string]string) - annotations["nginx.ingress.kubernetes.io/backend-protocol"] = "GRPC" - annotations["nginx.ingress.kubernetes.io/proxy-connect-timeout"] = proxyTimeout - annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = proxyTimeout - annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = proxyTimeout - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, ingressName, 50051, annotations) - - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, fmt.Sprintf("grpc_connect_timeout %ss;", proxyTimeout)) && - strings.Contains(server, fmt.Sprintf("grpc_send_timeout %ss;", proxyTimeout)) && - strings.Contains(server, fmt.Sprintf("grpc_read_timeout %ss;", proxyTimeout)) - }) - - conn, err := grpc.NewClient( - f.GetNginxIP()+":80", - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithAuthority(host), - ) - assert.Nil(ginkgo.GinkgoT(), err, "error creating a connection") - defer conn.Close() - - client := delaypb.NewGrpcbinServiceClient(conn) - - _, err = client.Unary(context.Background(), &delaypb.UnaryRequest{ - Data: "hello", - RequestAttributes: &delaypb.RequestAttributes{ - Delay: 15, - }, - }) - assert.Error(ginkgo.GinkgoT(), err) - }) }) diff --git a/test/e2e/annotations/http2pushpreload.go b/test/e2e/annotations/http2pushpreload.go index b15d2df17..400b77587 100644 --- a/test/e2e/annotations/http2pushpreload.go +++ b/test/e2e/annotations/http2pushpreload.go @@ -19,7 +19,7 @@ package annotations import ( "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/annotations/influxdb.go b/test/e2e/annotations/influxdb.go new file mode 100644 index 000000000..09c831f6f --- /dev/null +++ b/test/e2e/annotations/influxdb.go @@ -0,0 +1,186 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package annotations + +import ( + "bytes" + "context" + "fmt" + "net/http" + "os/exec" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + "github.com/onsi/ginkgo" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.DescribeAnnotation("influxdb-*", func() { + f := framework.NewDefaultFramework("influxdb") + + ginkgo.BeforeEach(func() { + f.NewInfluxDBDeployment() + f.NewEchoDeployment() + }) + + ginkgo.Context("when influxdb is enabled", func() { + ginkgo.It("should send the request metric to the influxdb server", func() { + ifs := createInfluxDBService(f) + + // Ingress configured with InfluxDB annotations + host := "influxdb.e2e.local" + createInfluxDBIngress( + f, + host, + framework.EchoService, + 80, + map[string]string{ + "nginx.ingress.kubernetes.io/enable-influxdb": "true", + "nginx.ingress.kubernetes.io/influxdb-host": ifs.Spec.ClusterIP, + "nginx.ingress.kubernetes.io/influxdb-port": "8089", + "nginx.ingress.kubernetes.io/influxdb-measurement": "requests", + "nginx.ingress.kubernetes.io/influxdb-servername": "e2e-nginx-srv", + }, + ) + + // Do a request to the echo server ingress that sends metrics + // to the InfluxDB backend. + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", host). + Expect(). + Status(http.StatusOK) + + framework.Sleep(10 * time.Second) + + var measurements string + var err error + + err = wait.Poll(time.Second, time.Minute, func() (bool, error) { + measurements, err = extractInfluxDBMeasurements(f) + if err != nil { + return false, nil + } + return true, nil + }) + assert.Nil(ginkgo.GinkgoT(), err) + + var results map[string][]map[string]interface{} + _ = jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal([]byte(measurements), &results) + + assert.NotEqual(ginkgo.GinkgoT(), len(measurements), 0) + for _, elem := range results["results"] { + assert.NotEqual(ginkgo.GinkgoT(), len(elem), 0) + } + }) + }) +}) + +func createInfluxDBService(f *framework.Framework) *corev1.Service { + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "influxdb", + Namespace: f.Namespace, + }, + Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{ + { + Name: "udp", + Port: 8089, + TargetPort: intstr.FromInt(8089), + Protocol: "UDP", + }, + }, + Selector: map[string]string{ + "app": "influxdb", + }, + }, + } + + return f.EnsureService(service) +} + +func createInfluxDBIngress(f *framework.Framework, host, service string, port int, annotations map[string]string) { + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, service, port, annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, fmt.Sprintf("server_name %v", host)) + }) +} + +func extractInfluxDBMeasurements(f *framework.Framework) (string, error) { + l, err := f.KubeClientSet.CoreV1().Pods(f.Namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: "app=influxdb", + }) + if err != nil { + return "", err + } + + if len(l.Items) == 0 { + return "", err + } + + cmd := "influx -database 'nginx' -execute 'select * from requests' -format 'json' -pretty" + + var pod *corev1.Pod + for _, p := range l.Items { + pod = &p + break + } + + if pod == nil { + return "", fmt.Errorf("no influxdb pods found") + } + + o, err := execInfluxDBCommand(pod, cmd) + if err != nil { + return "", err + } + + return o, nil +} + +func execInfluxDBCommand(pod *corev1.Pod, command string) (string, error) { + var ( + execOut bytes.Buffer + execErr bytes.Buffer + ) + + cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v exec --namespace %s %s -- %s", framework.KubectlPath, pod.Namespace, pod.Name, command)) + cmd.Stdout = &execOut + cmd.Stderr = &execErr + + err := cmd.Run() + + if execErr.Len() > 0 { + return "", fmt.Errorf("stderr: %v", execErr.String()) + } + + if err != nil { + return "", fmt.Errorf("could not execute '%s %s': %v", cmd.Path, cmd.Args, err) + } + + return execOut.String(), nil +} diff --git a/test/e2e/annotations/ipdenylist.go b/test/e2e/annotations/ipdenylist.go deleted file mode 100644 index 9c1d45cf5..000000000 --- a/test/e2e/annotations/ipdenylist.go +++ /dev/null @@ -1,147 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package annotations - -import ( - "net/http" - "strings" - - "github.com/onsi/ginkgo/v2" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.DescribeAnnotation("denylist-source-range", func() { - f := framework.NewDefaultFramework("ipdenylist") - - ginkgo.BeforeEach(func() { - f.NewEchoDeployment() - }) - - ginkgo.It("only deny explicitly denied IPs, allow all others", func() { - host := "ipdenylist.foo.com" - namespace := f.Namespace - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/denylist-source-range": "18.0.0.0/8, 56.0.0.1", - } - - ing := framework.NewSingleIngress(host, "/", host, namespace, framework.EchoService, 80, annotations) - - // Temporarily trust forwarded headers so we can test IP based access control - f.UpdateNginxConfigMapData("use-forwarded-headers", "true") - defer func() { - // Return to the original value - f.UpdateNginxConfigMapData("use-forwarded-headers", "false") - }() - - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "deny 18.0.0.0/8;") && - strings.Contains(server, "deny 56.0.0.1;") && - !strings.Contains(server, "deny all;") - }) - - ginkgo.By("sending request from an explicitly denied IP range") - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("X-Forwarded-For", "18.0.0.1"). - Expect(). - Status(http.StatusForbidden) - - ginkgo.By("sending request from an explicitly denied IP address") - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("X-Forwarded-For", "56.0.0.1"). - Expect(). - Status(http.StatusForbidden) - - ginkgo.By("sending request from an implicitly allowed IP range") - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("X-Forwarded-For", "56.0.0.2"). - Expect(). - Status(http.StatusOK) - }) - - ginkgo.It("only allow explicitly allowed IPs, deny all others", func() { - host := "ipdenylist.foo.com" - namespace := f.Namespace - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/denylist-source-range": "18.1.0.0/16, 56.0.0.0/8", - "nginx.ingress.kubernetes.io/whitelist-source-range": "18.0.0.0/8, 55.0.0.0/8", - } - - ing := framework.NewSingleIngress(host, "/", host, namespace, framework.EchoService, 80, annotations) - - // Temporarily trust forwarded headers so we can test IP based access control - f.UpdateNginxConfigMapData("use-forwarded-headers", "true") - defer func() { - // Return to the original value - f.UpdateNginxConfigMapData("use-forwarded-headers", "false") - }() - - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "deny 18.1.0.0/16;") && - strings.Contains(server, "deny 56.0.0.0/8;") && - strings.Contains(server, "allow 18.0.0.0/8;") && - strings.Contains(server, "allow 55.0.0.0/8;") && - strings.Contains(server, "deny all;") - }) - - ginkgo.By("sending request from an explicitly denied IP range") - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("X-Forwarded-For", "18.1.0.1"). - Expect(). - Status(http.StatusForbidden) - - ginkgo.By("sending request from an implicitly denied IP") - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("X-Forwarded-For", "10.10.10.10"). - Expect(). - Status(http.StatusForbidden) - - ginkgo.By("sending request from an explicitly allowed IP range") - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("X-Forwarded-For", "18.4.0.1"). - Expect(). - Status(http.StatusOK) - - ginkgo.By("sending request from an explicitly allowed IP range") - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("X-Forwarded-For", "55.55.55.55"). - Expect(). - Status(http.StatusOK) - }) -}) diff --git a/test/e2e/annotations/ipallowlist.go b/test/e2e/annotations/ipwhitelist.go similarity index 79% rename from test/e2e/annotations/ipallowlist.go rename to test/e2e/annotations/ipwhitelist.go index 79c77b4d0..66e4de3f5 100644 --- a/test/e2e/annotations/ipallowlist.go +++ b/test/e2e/annotations/ipwhitelist.go @@ -19,24 +19,23 @@ package annotations import ( "strings" - "github.com/onsi/ginkgo/v2" - + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) -var _ = framework.DescribeAnnotation("allowlist-source-range", func() { - f := framework.NewDefaultFramework("ipallowlist") +var _ = framework.DescribeAnnotation("whitelist-source-range", func() { + f := framework.NewDefaultFramework("ipwhitelist") ginkgo.BeforeEach(func() { f.NewEchoDeployment() }) - ginkgo.It("should set valid ip allowlist range", func() { - host := "ipallowlist.foo.com" + ginkgo.It("should set valid ip whitelist range", func() { + host := "ipwhitelist.foo.com" nameSpace := f.Namespace annotations := map[string]string{ - "nginx.ingress.kubernetes.io/allowlist-source-range": "18.0.0.0/8, 56.0.0.0/8", + "nginx.ingress.kubernetes.io/whitelist-source-range": "18.0.0.0/8, 56.0.0.0/8", } ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) diff --git a/test/e2e/annotations/limitconnections.go b/test/e2e/annotations/limitconnections.go index 7d00b6df0..2274c574c 100644 --- a/test/e2e/annotations/limitconnections.go +++ b/test/e2e/annotations/limitconnections.go @@ -22,7 +22,7 @@ import ( "strings" "sync" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" diff --git a/test/e2e/annotations/limitrate.go b/test/e2e/annotations/limitrate.go index 21e76dc8d..8c0b06ea9 100644 --- a/test/e2e/annotations/limitrate.go +++ b/test/e2e/annotations/limitrate.go @@ -21,7 +21,7 @@ import ( "strconv" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/annotations/log.go b/test/e2e/annotations/log.go index 2279729a2..65dc9af57 100644 --- a/test/e2e/annotations/log.go +++ b/test/e2e/annotations/log.go @@ -19,7 +19,7 @@ package annotations import ( "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/annotations/mirror.go b/test/e2e/annotations/mirror.go index 787cbfa3b..e904cca1c 100644 --- a/test/e2e/annotations/mirror.go +++ b/test/e2e/annotations/mirror.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -60,7 +60,7 @@ var _ = framework.DescribeAnnotation("mirror-*", func() { func(server string) bool { return strings.Contains(server, fmt.Sprintf("mirror /_mirror-%v;", ing.UID)) && strings.Contains(server, "mirror_request_body on;") && - strings.Contains(server, `proxy_pass "https://test.env.com/$request_uri";`) + strings.Contains(server, "proxy_pass https://test.env.com/$request_uri;") }) }) diff --git a/test/e2e/annotations/modsecurity.go b/test/e2e/annotations/modsecurity.go new file mode 100644 index 000000000..1f5a2f607 --- /dev/null +++ b/test/e2e/annotations/modsecurity.go @@ -0,0 +1,219 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package annotations + +import ( + "net/http" + "strings" + + "github.com/onsi/ginkgo" + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.DescribeAnnotation("modsecurity owasp", func() { + f := framework.NewDefaultFramework("modsecuritylocation") + + ginkgo.BeforeEach(func() { + f.NewEchoDeployment() + }) + + ginkgo.It("should enable modsecurity", func() { + host := "modsecurity.foo.com" + nameSpace := f.Namespace + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/enable-modsecurity": "true", + } + + ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "modsecurity on;") && + strings.Contains(server, "modsecurity_rules_file /etc/nginx/modsecurity/modsecurity.conf;") + }) + }) + + ginkgo.It("should enable modsecurity with transaction ID and OWASP rules", func() { + host := "modsecurity.foo.com" + nameSpace := f.Namespace + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/enable-modsecurity": "true", + "nginx.ingress.kubernetes.io/enable-owasp-core-rules": "true", + "nginx.ingress.kubernetes.io/modsecurity-transaction-id": "modsecurity-$request_id", + } + + ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "modsecurity on;") && + strings.Contains(server, "modsecurity_rules_file /etc/nginx/owasp-modsecurity-crs/nginx-modsecurity.conf;") && + strings.Contains(server, "modsecurity_transaction_id \"modsecurity-$request_id\";") + }) + }) + + ginkgo.It("should disable modsecurity", func() { + host := "modsecurity.foo.com" + nameSpace := f.Namespace + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/enable-modsecurity": "false", + } + + ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return !strings.Contains(server, "modsecurity on;") + }) + }) + + ginkgo.It("should enable modsecurity with snippet", func() { + host := "modsecurity.foo.com" + nameSpace := f.Namespace + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/enable-modsecurity": "true", + "nginx.ingress.kubernetes.io/modsecurity-snippet": "SecRuleEngine On", + } + + ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "modsecurity on;") && + strings.Contains(server, "SecRuleEngine On") + }) + }) + + ginkgo.It("should enable modsecurity without using 'modsecurity on;'", func() { + f.SetNginxConfigMapData(map[string]string{ + "enable-modsecurity": "true"}, + ) + + host := "modsecurity.foo.com" + nameSpace := f.Namespace + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/enable-modsecurity": "true", + } + + ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return !strings.Contains(server, "modsecurity on;") && + !strings.Contains(server, "modsecurity_rules_file /etc/nginx/modsecurity/modsecurity.conf;") + }) + }) + + ginkgo.It("should disable modsecurity using 'modsecurity off;'", func() { + f.SetNginxConfigMapData(map[string]string{ + "enable-modsecurity": "true"}, + ) + + host := "modsecurity.foo.com" + nameSpace := f.Namespace + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/enable-modsecurity": "false", + } + + ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "modsecurity off;") + }) + }) + + ginkgo.It("should enable modsecurity with snippet and block requests", func() { + host := "modsecurity.foo.com" + nameSpace := f.Namespace + + snippet := `SecRuleEngine On + SecRequestBodyAccess On + SecAuditEngine RelevantOnly + SecAuditLogParts ABIJDEFHZ + SecAuditLog /dev/stdout + SecAuditLogType Serial + SecRule REQUEST_HEADERS:User-Agent \"block-ua\" \"log,deny,id:107,status:403,msg:\'UA blocked\'\"` + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/enable-modsecurity": "true", + "nginx.ingress.kubernetes.io/modsecurity-snippet": snippet, + } + + ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "modsecurity on;") && + strings.Contains(server, "SecRuleEngine On") + }) + + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", host). + WithHeader("User-Agent", "block-ua"). + Expect(). + Status(http.StatusForbidden) + }) + + ginkgo.It("should enable modsecurity globally and with modsecurity-snippet block requests", func() { + host := "modsecurity.foo.com" + nameSpace := f.Namespace + + snippet := `SecRuleEngine On + SecRequestBodyAccess On + SecAuditEngine RelevantOnly + SecAuditLogParts ABIJDEFHZ + SecAuditLog /dev/stdout + SecAuditLogType Serial + SecRule REQUEST_HEADERS:User-Agent \"block-ua\" \"log,deny,id:107,status:403,msg:\'UA blocked\'\"` + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/modsecurity-snippet": snippet, + } + + ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) + + f.UpdateNginxConfigMapData("enable-modsecurity", "true") + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "SecRuleEngine On") + }) + + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", host). + WithHeader("User-Agent", "block-ua"). + Expect(). + Status(http.StatusForbidden) + }) +}) diff --git a/test/e2e/annotations/modsecurity/modsecurity.go b/test/e2e/annotations/modsecurity/modsecurity.go deleted file mode 100644 index 730fc76e7..000000000 --- a/test/e2e/annotations/modsecurity/modsecurity.go +++ /dev/null @@ -1,393 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package modsecurity - -import ( - "net/http" - "strings" - - "github.com/onsi/ginkgo/v2" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -const ( - modSecurityFooHost = "modsecurity.foo.com" - defaultSnippet = `SecRuleEngine On - SecRequestBodyAccess On - SecAuditEngine RelevantOnly - SecAuditLogParts ABIJDEFHZ - SecAuditLog /dev/stdout - SecAuditLogType Serial - SecRule REQUEST_HEADERS:User-Agent \"block-ua\" \"log,deny,id:107,status:403,msg:\'UA blocked\'\"` -) - -var _ = framework.DescribeAnnotation("modsecurity owasp", func() { - f := framework.NewDefaultFramework("modsecuritylocation") - - ginkgo.BeforeEach(func() { - f.NewEchoDeployment() - }) - - ginkgo.It("should enable modsecurity", func() { - host := modSecurityFooHost - nameSpace := f.Namespace - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-modsecurity": "true", - } - - ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "modsecurity on;") && - strings.Contains(server, "modsecurity_rules_file /etc/nginx/modsecurity/modsecurity.conf;") - }) - }) - - ginkgo.It("should enable modsecurity with transaction ID and OWASP rules", func() { - host := modSecurityFooHost - nameSpace := f.Namespace - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-modsecurity": "true", - "nginx.ingress.kubernetes.io/enable-owasp-core-rules": "true", - "nginx.ingress.kubernetes.io/modsecurity-transaction-id": "modsecurity-$request_id", - } - - ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "modsecurity on;") && - strings.Contains(server, "modsecurity_rules_file /etc/nginx/owasp-modsecurity-crs/nginx-modsecurity.conf;") && - strings.Contains(server, "modsecurity_transaction_id \"modsecurity-$request_id\";") - }) - }) - - ginkgo.It("should disable modsecurity", func() { - host := modSecurityFooHost - nameSpace := f.Namespace - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-modsecurity": "false", - } - - ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return !strings.Contains(server, "modsecurity on;") - }) - }) - - ginkgo.It("should enable modsecurity with snippet", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - - host := modSecurityFooHost - nameSpace := f.Namespace - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-modsecurity": "true", - "nginx.ingress.kubernetes.io/modsecurity-snippet": "SecRuleEngine On", - } - - ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "modsecurity on;") && - strings.Contains(server, "SecRuleEngine On") - }) - }) - - ginkgo.It("should enable modsecurity without using 'modsecurity on;'", func() { - f.SetNginxConfigMapData(map[string]string{ - "enable-modsecurity": "true", - }, - ) - - host := modSecurityFooHost - nameSpace := f.Namespace - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-modsecurity": "true", - } - - ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return !strings.Contains(server, "modsecurity on;") && - !strings.Contains(server, "modsecurity_rules_file /etc/nginx/modsecurity/modsecurity.conf;") - }) - }) - - ginkgo.It("should disable modsecurity using 'modsecurity off;'", func() { - f.SetNginxConfigMapData(map[string]string{ - "enable-modsecurity": "true", - }, - ) - - host := modSecurityFooHost - nameSpace := f.Namespace - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-modsecurity": "false", - } - - ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "modsecurity off;") - }) - }) - - ginkgo.It("should enable modsecurity with snippet and block requests", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - - host := modSecurityFooHost - nameSpace := f.Namespace - - snippet := defaultSnippet - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-modsecurity": "true", - "nginx.ingress.kubernetes.io/modsecurity-snippet": snippet, - } - f.UpdateNginxConfigMapData("annotation-value-word-blocklist", "load_module, lua_package, _by_lua, location, root, {, }") - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "modsecurity on;") && - strings.Contains(server, "SecRuleEngine On") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("User-Agent", "block-ua"). - Expect(). - Status(http.StatusForbidden) - }) - - ginkgo.It("should enable modsecurity globally and with modsecurity-snippet block requests", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - - host := modSecurityFooHost - nameSpace := f.Namespace - - snippet := defaultSnippet - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/modsecurity-snippet": snippet, - } - f.UpdateNginxConfigMapData("annotation-value-word-blocklist", "load_module, lua_package, _by_lua, location, root, {, }") - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.UpdateNginxConfigMapData("enable-modsecurity", "true") - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "SecRuleEngine On") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("User-Agent", "block-ua"). - Expect(). - Status(http.StatusForbidden) - }) - - ginkgo.It("should enable modsecurity when enable-owasp-modsecurity-crs is set to true", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - - f.UpdateNginxConfigMapData("enable-modsecurity", "true") - f.UpdateNginxConfigMapData("enable-owasp-modsecurity-crs", "true") - - host := modSecurityFooHost - nameSpace := f.Namespace - - snippet := defaultSnippet - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/modsecurity-snippet": snippet, - } - f.UpdateNginxConfigMapData("annotation-value-word-blocklist", "load_module, lua_package, _by_lua, location, root, {, }") - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "SecRuleEngine On") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("User-Agent", "block-ua"). - Expect(). - Status(http.StatusForbidden) - }) - - ginkgo.It("should enable modsecurity through the config map", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - host := modSecurityFooHost - nameSpace := f.Namespace - - snippet := `SecRequestBodyAccess On - SecAuditEngine RelevantOnly - SecAuditLogParts ABIJDEFHZ - SecAuditLog /dev/stdout - SecAuditLogType Serial - SecRule REQUEST_HEADERS:User-Agent \"block-ua\" \"log,deny,id:107,status:403,msg:\'UA blocked\'\"` - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/modsecurity-snippet": snippet, - } - f.UpdateNginxConfigMapData("annotation-value-word-blocklist", "load_module, lua_package, _by_lua, location, root, {, }") - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - expectedComment := "SecRuleEngine On" - f.UpdateNginxConfigMapData("enable-modsecurity", "true") - f.UpdateNginxConfigMapData("enable-owasp-modsecurity-crs", "true") - f.UpdateNginxConfigMapData("modsecurity-snippet", expectedComment) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "SecRequestBodyAccess On") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("User-Agent", "block-ua"). - Expect(). - Status(http.StatusForbidden) - }) - - ginkgo.It("should enable modsecurity through the config map but ignore snippet as disabled by admin", func() { - host := modSecurityFooHost - nameSpace := f.Namespace - - f.UpdateNginxConfigMapData("annotations-risk-level", "Critical") // To enable snippet configurations - defer f.UpdateNginxConfigMapData("annotations-risk-level", "High") - - snippet := `SecRequestBodyAccess On - SecAuditEngine RelevantOnly - SecAuditLogParts ABIJDEFHZ - SecAuditLog /dev/stdout - SecAuditLogType Serial - SecRule REQUEST_HEADERS:User-Agent \"block-ua\" \"log,deny,id:107,status:403,msg:\'UA blocked\'\"` - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/modsecurity-snippet": snippet, - } - f.UpdateNginxConfigMapData("annotation-value-word-blocklist", "load_module, lua_package, _by_lua, location, root, {, }") - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - expectedComment := "SecRuleEngine On" - - f.SetNginxConfigMapData(map[string]string{ - "enable-modsecurity": "true", - "enable-owasp-modsecurity-crs": "true", - "allow-snippet-annotations": "false", - "modsecurity-snippet": expectedComment, - }) - - f.WaitForNginxServer(host, - func(server string) bool { - return !strings.Contains(server, "block-ua") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("User-Agent", "block-ua"). - Expect(). - Status(http.StatusOK) - }) - - ginkgo.It("should disable default modsecurity conf setting when modsecurity-snippet is specified", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - - host := modSecurityFooHost - nameSpace := f.Namespace - - snippet := `SecRuleEngine On - SecRequestBodyAccess On - SecAuditEngine RelevantOnly - SecAuditLogParts ABIJDEFHZ - SecAuditLogType Concurrent - SecAuditLog /var/tmp/modsec_audit.log - SecAuditLogStorageDir /var/tmp/ - SecRule REQUEST_HEADERS:User-Agent \"block-ua\" \"log,deny,id:107,status:403,msg:\'UA blocked\'\"` - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/enable-modsecurity": "true", - "nginx.ingress.kubernetes.io/modsecurity-snippet": snippet, - } - f.UpdateNginxConfigMapData("annotation-value-word-blocklist", "load_module, lua_package, _by_lua, location, root, {, }") - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - ing := framework.NewSingleIngress(host, "/", host, nameSpace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return !strings.Contains(server, "modsecurity_rules_file /etc/nginx/modsecurity/modsecurity.conf;") && - strings.Contains(server, "SecAuditLog /var/tmp/modsec_audit.log") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("User-Agent", "block-ua"). - Expect(). - Status(http.StatusForbidden) - }) -}) diff --git a/test/e2e/annotations/preservetrailingslash.go b/test/e2e/annotations/preservetrailingslash.go index 7e27b25fa..f9129c77e 100644 --- a/test/e2e/annotations/preservetrailingslash.go +++ b/test/e2e/annotations/preservetrailingslash.go @@ -19,7 +19,7 @@ package annotations import ( "net/http" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/annotations/proxy.go b/test/e2e/annotations/proxy.go index 8e9866021..1c749b4dd 100644 --- a/test/e2e/annotations/proxy.go +++ b/test/e2e/annotations/proxy.go @@ -20,13 +20,11 @@ import ( "fmt" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) -const proxyRedirectToHost = "goodbye.com" - var _ = framework.DescribeAnnotation("proxy-*", func() { f := framework.NewDefaultFramework("proxy") host := "proxy.foo.com" @@ -40,7 +38,7 @@ var _ = framework.DescribeAnnotation("proxy-*", func() { annotations := make(map[string]string) annotations["nginx.ingress.kubernetes.io/proxy-redirect-from"] = proxyRedirectFrom - annotations["nginx.ingress.kubernetes.io/proxy-redirect-to"] = proxyRedirectToHost + annotations["nginx.ingress.kubernetes.io/proxy-redirect-to"] = "goodbye.com" ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -56,7 +54,7 @@ var _ = framework.DescribeAnnotation("proxy-*", func() { annotations := make(map[string]string) annotations["nginx.ingress.kubernetes.io/proxy-redirect-from"] = proxyRedirectFrom - annotations["nginx.ingress.kubernetes.io/proxy-redirect-to"] = proxyRedirectToHost + annotations["nginx.ingress.kubernetes.io/proxy-redirect-to"] = "goodbye.com" ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -69,7 +67,7 @@ var _ = framework.DescribeAnnotation("proxy-*", func() { ginkgo.It("should set proxy_redirect to hello.com goodbye.com", func() { proxyRedirectFrom := "hello.com" - proxyRedirectTo := proxyRedirectToHost + proxyRedirectTo := "goodbye.com" annotations := make(map[string]string) annotations["nginx.ingress.kubernetes.io/proxy-redirect-from"] = proxyRedirectFrom @@ -160,13 +158,11 @@ var _ = framework.DescribeAnnotation("proxy-*", func() { proxyBuffering := "on" proxyBuffersNumber := "8" proxyBufferSize := "8k" - proxyBusyBuffersSize := "16k" annotations := make(map[string]string) annotations["nginx.ingress.kubernetes.io/proxy-buffering"] = proxyBuffering annotations["nginx.ingress.kubernetes.io/proxy-buffers-number"] = proxyBuffersNumber annotations["nginx.ingress.kubernetes.io/proxy-buffer-size"] = proxyBufferSize - annotations["nginx.ingress.kubernetes.io/proxy-busy-buffers-size"] = proxyBusyBuffersSize ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -176,7 +172,6 @@ var _ = framework.DescribeAnnotation("proxy-*", func() { return strings.Contains(server, fmt.Sprintf("proxy_buffering %s;", proxyBuffering)) && strings.Contains(server, fmt.Sprintf("proxy_buffer_size %s;", proxyBufferSize)) && strings.Contains(server, fmt.Sprintf("proxy_buffers %s %s;", proxyBuffersNumber, proxyBufferSize)) && - strings.Contains(server, fmt.Sprintf("proxy_busy_buffers_size %s;", proxyBusyBuffersSize)) && strings.Contains(server, fmt.Sprintf("proxy_request_buffering %s;", proxyBuffering)) }) }) @@ -249,4 +244,5 @@ var _ = framework.DescribeAnnotation("proxy-*", func() { return strings.Contains(server, fmt.Sprintf("proxy_http_version %s;", proxyHTTPVersion)) }) }) + }) diff --git a/test/e2e/annotations/proxyssl.go b/test/e2e/annotations/proxyssl.go index 898cbed48..0e928664e 100644 --- a/test/e2e/annotations/proxyssl.go +++ b/test/e2e/annotations/proxyssl.go @@ -21,14 +21,12 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" ) -const proxySSLHost = "proxyssl.foo.com" - var _ = framework.DescribeAnnotation("proxy-ssl-*", func() { f := framework.NewDefaultFramework("proxyssl") @@ -37,7 +35,7 @@ var _ = framework.DescribeAnnotation("proxy-ssl-*", func() { }) ginkgo.It("should set valid proxy-ssl-secret", func() { - host := proxySSLHost + host := "proxyssl.foo.com" annotations := make(map[string]string) annotations["nginx.ingress.kubernetes.io/proxy-ssl-secret"] = f.Namespace + "/" + host @@ -47,7 +45,7 @@ var _ = framework.DescribeAnnotation("proxy-ssl-*", func() { ing := framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) - assertProxySSL(f, host, "", "DEFAULT", "TLSv1.2", "off", 1, "") + assertProxySSL(f, host, "", "DEFAULT", "TLSv1 TLSv1.1 TLSv1.2", "off", 1, "") f.HTTPTestClient(). GET("/"). @@ -64,7 +62,7 @@ var _ = framework.DescribeAnnotation("proxy-ssl-*", func() { }) ginkgo.It("should set valid proxy-ssl-secret, proxy-ssl-verify to on, proxy-ssl-verify-depth to 2, and proxy-ssl-server-name to on", func() { - host := proxySSLHost + host := "proxyssl.foo.com" annotations := make(map[string]string) annotations["nginx.ingress.kubernetes.io/proxy-ssl-secret"] = f.Namespace + "/" + host annotations["nginx.ingress.kubernetes.io/proxy-ssl-verify"] = "on" @@ -77,7 +75,7 @@ var _ = framework.DescribeAnnotation("proxy-ssl-*", func() { ing := framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) - assertProxySSL(f, host, "", "DEFAULT", "TLSv1.2", "on", 2, "on") + assertProxySSL(f, host, "", "DEFAULT", "TLSv1 TLSv1.1 TLSv1.2", "on", 2, "on") f.HTTPTestClient(). GET("/"). @@ -92,9 +90,9 @@ var _ = framework.DescribeAnnotation("proxy-ssl-*", func() { Expect(). Status(http.StatusOK) }) - //nolint:dupl // Ignore dupl errors for similar test case + ginkgo.It("should set valid proxy-ssl-secret, proxy-ssl-ciphers to HIGH:!AES", func() { - host := proxySSLHost + host := "proxyssl.foo.com" annotations := make(map[string]string) annotations["nginx.ingress.kubernetes.io/proxy-ssl-secret"] = f.Namespace + "/" + host annotations["nginx.ingress.kubernetes.io/proxy-ssl-ciphers"] = "HIGH:!AES" @@ -105,7 +103,7 @@ var _ = framework.DescribeAnnotation("proxy-ssl-*", func() { ing := framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) - assertProxySSL(f, host, "", "HIGH:!AES", "TLSv1.2", "off", 1, "") + assertProxySSL(f, host, "", "HIGH:!AES", "TLSv1 TLSv1.1 TLSv1.2", "off", 1, "") f.HTTPTestClient(). GET("/"). @@ -120,9 +118,9 @@ var _ = framework.DescribeAnnotation("proxy-ssl-*", func() { Expect(). Status(http.StatusOK) }) - //nolint:dupl // Ignore dupl errors for similar test case + ginkgo.It("should set valid proxy-ssl-secret, proxy-ssl-protocols", func() { - host := proxySSLHost + host := "proxyssl.foo.com" annotations := make(map[string]string) annotations["nginx.ingress.kubernetes.io/proxy-ssl-secret"] = f.Namespace + "/" + host annotations["nginx.ingress.kubernetes.io/proxy-ssl-protocols"] = "TLSv1.2 TLSv1.3" @@ -152,7 +150,7 @@ var _ = framework.DescribeAnnotation("proxy-ssl-*", func() { ginkgo.It("proxy-ssl-location-only flag should change the nginx config server part", func() { host := "proxyssl.com" - f.NewEchoDeployment(framework.WithDeploymentName("echodeployment")) + f.NewEchoDeploymentWithNameAndReplicas("echodeployment", 1) secretName := "secretone" annotations := make(map[string]string) @@ -171,7 +169,7 @@ var _ = framework.DescribeAnnotation("proxy-ssl-*", func() { wlValue := "true" f.UpdateNginxConfigMapData(wlKey, wlValue) - assertProxySSL(f, host, secretName, "DEFAULT", "TLSv1.2", "on", 1, "on") + assertProxySSL(f, host, secretName, "DEFAULT", "TLSv1 TLSv1.1 TLSv1.2", "on", 1, "on") f.WaitForNginxCustomConfiguration("## start server proxyssl.com", "location ", func(server string) bool { return (!strings.Contains(server, "proxy_ssl_trusted_certificate") && @@ -197,6 +195,7 @@ var _ = framework.DescribeAnnotation("proxy-ssl-*", func() { strings.Contains(server, "proxy_ssl_certificate_key")) }) }) + }) func assertProxySSL(f *framework.Framework, host, sslName, ciphers, protocols, verify string, depth int, proxySSLServerName string) { diff --git a/test/e2e/annotations/redirect.go b/test/e2e/annotations/redirect.go index 633e2eb0b..84c219b71 100644 --- a/test/e2e/annotations/redirect.go +++ b/test/e2e/annotations/redirect.go @@ -22,7 +22,7 @@ import ( "strconv" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/annotations/relativeredirects.go b/test/e2e/annotations/relativeredirects.go deleted file mode 100644 index 430b357e4..000000000 --- a/test/e2e/annotations/relativeredirects.go +++ /dev/null @@ -1,107 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package annotations - -import ( - "fmt" - "net/http" - "strings" - - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" - "k8s.io/ingress-nginx/test/e2e/framework" -) - -const ( - relativeRedirectsHostname = "rr.foo.com" - relativeRedirectsRedirectPath = "/something" - relativeRedirectsRelativeRedirectURL = "/new-location" -) - -var _ = framework.DescribeAnnotation("relative-redirects", func() { - f := framework.NewDefaultFramework("relative-redirects") - - ginkgo.BeforeEach(func() { - f.NewHttpbunDeployment() - f.NewEchoDeployment() - }) - - ginkgo.It("configures Nginx correctly", func() { - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/relative-redirects": "true", - } - - ing := framework.NewSingleIngress(relativeRedirectsHostname, "/", relativeRedirectsHostname, f.Namespace, framework.HTTPBunService, 80, annotations) - f.EnsureIngress(ing) - - var serverConfig string - f.WaitForNginxServer(relativeRedirectsHostname, func(srvCfg string) bool { - serverConfig = srvCfg - return strings.Contains(serverConfig, fmt.Sprintf("server_name %s", relativeRedirectsHostname)) - }) - - ginkgo.By("turning off absolute_redirect directive") - assert.Contains(ginkgo.GinkgoT(), serverConfig, "absolute_redirect off;") - }) - - ginkgo.It("should respond with absolute URL in Location", func() { - absoluteRedirectURL := fmt.Sprintf("http://%s%s", relativeRedirectsHostname, relativeRedirectsRelativeRedirectURL) - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/permanent-redirect": relativeRedirectsRelativeRedirectURL, - "nginx.ingress.kubernetes.io/relative-redirects": "false", - } - - ginkgo.By("setup ingress") - ing := framework.NewSingleIngress(relativeRedirectsHostname, relativeRedirectsRedirectPath, relativeRedirectsHostname, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(relativeRedirectsHostname, func(srvCfg string) bool { - return strings.Contains(srvCfg, fmt.Sprintf("server_name %s", relativeRedirectsHostname)) - }) - - ginkgo.By("sending request to redirected URL path") - f.HTTPTestClient(). - GET(relativeRedirectsRedirectPath). - WithHeader("Host", relativeRedirectsHostname). - Expect(). - Status(http.StatusMovedPermanently). - Header("Location").Equal(absoluteRedirectURL) - }) - - ginkgo.It("should respond with relative URL in Location", func() { - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/permanent-redirect": relativeRedirectsRelativeRedirectURL, - "nginx.ingress.kubernetes.io/relative-redirects": "true", - } - - ginkgo.By("setup ingress") - ing := framework.NewSingleIngress(relativeRedirectsHostname, relativeRedirectsRedirectPath, relativeRedirectsHostname, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(relativeRedirectsHostname, func(srvCfg string) bool { - return strings.Contains(srvCfg, fmt.Sprintf("server_name %s", relativeRedirectsHostname)) - }) - - ginkgo.By("sending request to redirected URL path") - f.HTTPTestClient(). - GET(relativeRedirectsRedirectPath). - WithHeader("Host", relativeRedirectsHostname). - Expect(). - Status(http.StatusMovedPermanently). - Header("Location").Equal(relativeRedirectsRelativeRedirectURL) - }) -}) diff --git a/test/e2e/annotations/rewrite.go b/test/e2e/annotations/rewrite.go index 173df29f0..667df920f 100644 --- a/test/e2e/annotations/rewrite.go +++ b/test/e2e/annotations/rewrite.go @@ -21,14 +21,12 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" ) -const rewriteHost = "rewrite.bar.com" - var _ = framework.DescribeAnnotation("rewrite-target use-regex enable-rewrite-log", func() { f := framework.NewDefaultFramework("rewrite") @@ -39,7 +37,7 @@ var _ = framework.DescribeAnnotation("rewrite-target use-regex enable-rewrite-lo ginkgo.It("should write rewrite logs", func() { ginkgo.By("setting enable-rewrite-log annotation") - host := rewriteHost + host := "rewrite.bar.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/rewrite-target": "/", "nginx.ingress.kubernetes.io/enable-rewrite-log": "true", @@ -66,7 +64,7 @@ var _ = framework.DescribeAnnotation("rewrite-target use-regex enable-rewrite-lo }) ginkgo.It("should use correct longest path match", func() { - host := rewriteHost + host := "rewrite.bar.com" ginkgo.By("creating a regular ingress definition") ing := framework.NewSingleIngress("kube-lego", "/.well-known/acme/challenge", host, f.Namespace, framework.EchoService, 80, nil) @@ -111,10 +109,10 @@ var _ = framework.DescribeAnnotation("rewrite-target use-regex enable-rewrite-lo }) ginkgo.It("should use ~* location modifier if regex annotation is present", func() { - host := rewriteHost + host := "rewrite.bar.com" ginkgo.By("creating a regular ingress definition") - ing := framework.NewSingleIngress(fooHost, "/foo", host, f.Namespace, framework.EchoService, 80, nil) + ing := framework.NewSingleIngress("foo", "/foo", host, f.Namespace, framework.EchoService, 80, nil) f.EnsureIngress(ing) f.WaitForNginxServer(host, @@ -158,10 +156,10 @@ var _ = framework.DescribeAnnotation("rewrite-target use-regex enable-rewrite-lo }) ginkgo.It("should fail to use longest match for documented warning", func() { - host := rewriteHost + host := "rewrite.bar.com" ginkgo.By("creating a regular ingress definition") - ing := framework.NewSingleIngress(fooHost, "/foo/bar/bar", host, f.Namespace, framework.EchoService, 80, nil) + ing := framework.NewSingleIngress("foo", "/foo/bar/bar", host, f.Namespace, framework.EchoService, 80, nil) f.EnsureIngress(ing) ginkgo.By(`creating an ingress definition with the use-regex annotation`) @@ -190,7 +188,7 @@ var _ = framework.DescribeAnnotation("rewrite-target use-regex enable-rewrite-lo }) ginkgo.It("should allow for custom rewrite parameters", func() { - host := rewriteHost + host := "rewrite.bar.com" ginkgo.By(`creating an ingress definition with the use-regex annotation`) annotations := map[string]string{ diff --git a/test/e2e/annotations/satisfy.go b/test/e2e/annotations/satisfy.go index 6ba6db33e..dba710b4b 100644 --- a/test/e2e/annotations/satisfy.go +++ b/test/e2e/annotations/satisfy.go @@ -17,21 +17,23 @@ limitations under the License. package annotations import ( + "context" "fmt" "net/http" "net/url" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/test/e2e/framework" ) var _ = framework.DescribeAnnotation("satisfy", func() { - f := framework.NewDefaultFramework("satisfy", framework.WithHTTPBunEnabled()) + f := framework.NewDefaultFramework("satisfy") ginkgo.BeforeEach(func() { f.NewEchoDeployment() @@ -82,6 +84,17 @@ var _ = framework.DescribeAnnotation("satisfy", func() { ginkgo.It("should allow multiple auth with satisfy any", func() { host := "auth" + // setup external auth + f.NewHttpbinDeployment() + + err := framework.WaitForEndpoints(f.KubeClientSet, framework.DefaultTimeout, framework.HTTPBinService, f.Namespace, 1) + assert.Nil(ginkgo.GinkgoT(), err) + + e, err := f.KubeClientSet.CoreV1().Endpoints(f.Namespace).Get(context.TODO(), framework.HTTPBinService, metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err) + + httpbinIP := e.Subsets[0].Addresses[0].IP + // create basic auth secret at ingress s := f.EnsureSecret(buildSecret("uname", "pwd", "basic-secret", f.Namespace)) @@ -92,7 +105,7 @@ var _ = framework.DescribeAnnotation("satisfy", func() { "nginx.ingress.kubernetes.io/auth-realm": "test basic auth", // annotations for external auth - "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", f.HTTPBunIP), + "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", httpbinIP), "nginx.ingress.kubernetes.io/auth-signin": "http://$host/auth/start", // set satisfy any diff --git a/test/e2e/annotations/serversnippet.go b/test/e2e/annotations/serversnippet.go index c94960a3d..8a4f25ea4 100644 --- a/test/e2e/annotations/serversnippet.go +++ b/test/e2e/annotations/serversnippet.go @@ -17,10 +17,9 @@ limitations under the License. package annotations import ( - "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -33,14 +32,11 @@ var _ = framework.DescribeAnnotation("server-snippet", func() { }) ginkgo.It(`add valid directives to server via server snippet`, func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - host := "serversnippet.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/server-snippet": ` - more_set_headers "Foo: Bar"; - more_set_headers "Xpto: Lalala";`, + more_set_headers "Content-Length: $content_length"; + more_set_headers "Content-Type: $content_type";`, } ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) @@ -48,47 +44,8 @@ var _ = framework.DescribeAnnotation("server-snippet", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, `more_set_headers "Foo: Bar`) && - strings.Contains(server, `more_set_headers "Xpto: Lalala";`) + return strings.Contains(server, `more_set_headers "Content-Length: $content_length`) && + strings.Contains(server, `more_set_headers "Content-Type: $content_type";`) }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Foo", []string{"Bar"}). - ValueEqual("Xpto", []string{"Lalala"}) - }) - - ginkgo.It(`drops server snippet if disabled by the administrator`, func() { - f.UpdateNginxConfigMapData("annotations-risk-level", "Critical") // To enable snippet configurations - defer f.UpdateNginxConfigMapData("annotations-risk-level", "High") - - host := "noserversnippet.foo.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/server-snippet": ` - more_set_headers "Foo: Bar"; - more_set_headers "Xpto: Lalala";`, - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return !strings.Contains(server, `more_set_headers "Foo: Bar`) && - !strings.Contains(server, `more_set_headers "Xpto: Lalala";`) - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK).Headers(). - NotContainsKey("Foo"). - NotContainsKey("Xpto") }) }) diff --git a/test/e2e/annotations/serviceupstream.go b/test/e2e/annotations/serviceupstream.go deleted file mode 100644 index 1d80f304a..000000000 --- a/test/e2e/annotations/serviceupstream.go +++ /dev/null @@ -1,125 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package annotations - -import ( - "fmt" - "net/http" - "strings" - - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -const dbgCmd = "/dbg backends all" - -var _ = framework.DescribeAnnotation("service-upstream", func() { - f := framework.NewDefaultFramework("serviceupstream") - host := "serviceupstream" - - ginkgo.BeforeEach(func() { - f.NewEchoDeployment() - }) - - ginkgo.Context("when using the default value (false) and enabling in the annotations", func() { - ginkgo.It("should use the Service Cluster IP and Port ", func() { - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/service-upstream": "true", - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s", host)) - }) - - ginkgo.By("checking if the service is reached") - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - - ginkgo.By("checking if the Service Cluster IP and Port are used") - s := f.GetService(f.Namespace, framework.EchoService) - output, err := f.ExecIngressPod(dbgCmd) - assert.Nil(ginkgo.GinkgoT(), err) - assert.Contains(ginkgo.GinkgoT(), output, fmt.Sprintf(`"address": %q`, s.Spec.ClusterIP)) - }) - }) - - ginkgo.Context("when enabling in the configmap", func() { - ginkgo.It("should use the Service Cluster IP and Port ", func() { - annotations := map[string]string{} - - f.UpdateNginxConfigMapData("service-upstream", "true") - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s", host)) - }) - - ginkgo.By("checking if the service is reached") - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - - ginkgo.By("checking if the Service Cluster IP and Port are used") - s := f.GetService(f.Namespace, framework.EchoService) - output, err := f.ExecIngressPod(dbgCmd) - assert.Nil(ginkgo.GinkgoT(), err) - assert.Contains(ginkgo.GinkgoT(), output, fmt.Sprintf(`"address": %q`, s.Spec.ClusterIP)) - }) - }) - - ginkgo.Context("when enabling in the configmap and disabling in the annotations", func() { - ginkgo.It("should not use the Service Cluster IP and Port", func() { - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/service-upstream": "false", - } - - f.UpdateNginxConfigMapData("service-upstream", "true") - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s", host)) - }) - - ginkgo.By("checking if the service is reached") - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - - ginkgo.By("checking if the Service Cluster IP and Port are not used") - s := f.GetService(f.Namespace, framework.EchoService) - output, err := f.ExecIngressPod(dbgCmd) - assert.Nil(ginkgo.GinkgoT(), err) - assert.NotContains(ginkgo.GinkgoT(), output, fmt.Sprintf(`"address": %q`, s.Spec.ClusterIP)) - }) - }) -}) diff --git a/test/e2e/annotations/snippet.go b/test/e2e/annotations/snippet.go index 9e3160dcc..61c39fa65 100644 --- a/test/e2e/annotations/snippet.go +++ b/test/e2e/annotations/snippet.go @@ -17,86 +17,33 @@ limitations under the License. package annotations import ( - "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) var _ = framework.DescribeAnnotation("configuration-snippet", func() { - f := framework.NewDefaultFramework( - "configurationsnippet", - framework.WithHTTPBunEnabled(), - ) + f := framework.NewDefaultFramework("configurationsnippet") - ginkgo.It("set snippet more_set_headers in all locations", func() { - host := "configurationsnippet.foo.com" - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_headers "Foo1: Bar1";`, - } - - f.EnsureIngress(framework.NewSingleIngress( - host, - "/", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations)) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, `more_set_headers "Foo1: Bar1";`) - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK). - Headers(). - ValueEqual("Foo1", []string{"Bar1"}) + ginkgo.BeforeEach(func() { + f.NewEchoDeployment() }) - ginkgo.It("drops snippet more_set_header in all locations if disabled by admin", func() { - f.UpdateNginxConfigMapData("annotations-risk-level", "Critical") // To enable snippet configurations - defer f.UpdateNginxConfigMapData("annotations-risk-level", "High") - host := "noconfigurationsnippet.foo.com" + ginkgo.It(`set snippet "more_set_headers "Request-Id: $req_id";" in all locations"`, func() { + host := "configurationsnippet.foo.com" annotations := map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_headers "Foo1: Bar1";`, + "nginx.ingress.kubernetes.io/configuration-snippet": ` + more_set_headers "Request-Id: $req_id";`, } - ing := framework.NewSingleIngress( - host, - "/", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations) - - f.UpdateNginxConfigMapData("allow-snippet-annotations", "false") - - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { - return !strings.Contains(server, `more_set_headers "Foo1: Bar1";`) + return strings.Contains(server, `more_set_headers "Request-Id: $req_id";`) }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK). - Headers(). - NotContainsKey("Foo1") }) }) diff --git a/test/e2e/annotations/sslciphers.go b/test/e2e/annotations/sslciphers.go index aece7fc41..0e2753b52 100644 --- a/test/e2e/annotations/sslciphers.go +++ b/test/e2e/annotations/sslciphers.go @@ -17,10 +17,9 @@ limitations under the License. package annotations import ( - "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -47,34 +46,5 @@ var _ = framework.DescribeAnnotation("ssl-ciphers", func() { return strings.Contains(server, "ssl_ciphers ALL:!aNULL:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP;") && strings.Contains(server, "ssl_prefer_server_ciphers off;") }) - f.HTTPTestClient(). - GET("/something"). - WithURL(f.GetURL(framework.HTTPS)). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - }) - - ginkgo.It("should keep ssl ciphers", func() { - host := "ciphers.foo.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/ssl-ciphers": "ALL:!aNULL:!EXPORT56:RC4+RSA@STRENGTH:+HIGH@SECLEVEL=0:+MEDIUM:+LOW:+SSLv2:+EXP", - "nginx.ingress.kubernetes.io/ssl-prefer-server-ciphers": "true", - } - - ing := framework.NewSingleIngress(host, "/something", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "ssl_ciphers ALL:!aNULL:!EXPORT56:RC4+RSA@STRENGTH:+HIGH@SECLEVEL=0:+MEDIUM:+LOW:+SSLv2:+EXP;") && - strings.Contains(server, "ssl_prefer_server_ciphers on;") - }) - f.HTTPTestClient(). - GET("/something"). - WithURL(f.GetURL(framework.HTTPS)). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) }) }) diff --git a/test/e2e/annotations/streamsnippet.go b/test/e2e/annotations/streamsnippet.go deleted file mode 100644 index f91cdc34e..000000000 --- a/test/e2e/annotations/streamsnippet.go +++ /dev/null @@ -1,141 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package annotations - -import ( - "context" - "fmt" - "net/http" - "strings" - - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.DescribeSetting("stream-snippet", func() { - f := framework.NewDefaultFramework("stream-snippet") - - ginkgo.BeforeEach(func() { - f.NewEchoDeployment() - }) - - ginkgo.It("should add value of stream-snippet to nginx config", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - - host := "foo.com" - - snippet := `server {listen 8000; proxy_pass 127.0.0.1:80;}` - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, map[string]string{ - "nginx.ingress.kubernetes.io/stream-snippet": snippet, - }) - f.EnsureIngress(ing) - - svc, err := f.KubeClientSet. - CoreV1(). - Services(f.Namespace). - Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining ingress-nginx service") - assert.NotNil(ginkgo.GinkgoT(), svc, "expected a service but none returned") - - svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{ - Name: framework.EchoService, - Port: 8000, - TargetPort: intstr.FromInt(8000), - }) - - _, err = f.KubeClientSet. - CoreV1(). - Services(f.Namespace). - Update(context.TODO(), svc, metav1.UpdateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating service") - - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - - f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, snippet) - }) - - f.HTTPTestClient(). - GET("/healthz"). - WithURL(fmt.Sprintf("http://%v:8000/healthz", f.GetNginxIP())). - Expect(). - Status(http.StatusOK) - }) - - ginkgo.It("should add stream-snippet and drop annotations per admin config", func() { - host := "cm.foo.com" - hostAnnot := "annot.foo.com" - - cmSnippet := `server {listen 8000; proxy_pass 127.0.0.1:80;}` - annotSnippet := `server {listen 8001; proxy_pass 127.0.0.1:80;}` - - f.SetNginxConfigMapData(map[string]string{ - "allow-snippet-annotations": "false", - "stream-snippet": cmSnippet, - }) - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ing) - - ing1 := framework.NewSingleIngress(hostAnnot, "/", hostAnnot, f.Namespace, framework.EchoService, 80, map[string]string{ - "nginx.ingress.kubernetes.io/stream-snippet": annotSnippet, - }) - f.EnsureIngress(ing1) - - svc, err := f.KubeClientSet. - CoreV1(). - Services(f.Namespace). - Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining ingress-nginx service") - assert.NotNil(ginkgo.GinkgoT(), svc, "expected a service but none returned") - - svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{ - Name: framework.EchoService, - Port: 8000, - TargetPort: intstr.FromInt(8000), - }) - - _, err = f.KubeClientSet. - CoreV1(). - Services(f.Namespace). - Update(context.TODO(), svc, metav1.UpdateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating service") - - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - - f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, cmSnippet) && !strings.Contains(cfg, annotSnippet) - }) - - f.HTTPTestClient(). - GET("/healthz"). - WithURL(fmt.Sprintf("http://%v:8000/healthz", f.GetNginxIP())). - Expect(). - Status(http.StatusOK) - }) -}) diff --git a/test/e2e/annotations/upstreamhashby.go b/test/e2e/annotations/upstreamhashby.go index 1b8106662..9474f2b2b 100644 --- a/test/e2e/annotations/upstreamhashby.go +++ b/test/e2e/annotations/upstreamhashby.go @@ -22,7 +22,7 @@ import ( "regexp" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/util/wait" @@ -39,13 +39,12 @@ func startIngress(f *framework.Framework, annotations map[string]string) map[str return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) }) - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated err := wait.Poll(framework.Poll, framework.DefaultTimeout, func() (bool, error) { + resp := f.HTTPTestClient(). GET("/"). WithHeader("Host", host). Expect().Raw() - defer resp.Body.Close() if resp.StatusCode == http.StatusOK { return true, nil @@ -56,9 +55,7 @@ func startIngress(f *framework.Framework, annotations map[string]string) map[str assert.Nil(ginkgo.GinkgoT(), err) - re, err := regexp.Compile(fmt.Sprintf(`Hostname: %v.*`, framework.EchoService)) - assert.Nil(ginkgo.GinkgoT(), err, "error compiling regex") - + re, _ := regexp.Compile(fmt.Sprintf(`Hostname: %v.*`, framework.EchoService)) podMap := map[string]bool{} for i := 0; i < 100; i++ { @@ -80,7 +77,7 @@ var _ = framework.DescribeAnnotation("upstream-hash-by-*", func() { f := framework.NewDefaultFramework("upstream-hash-by") ginkgo.BeforeEach(func() { - f.NewEchoDeployment(framework.WithDeploymentReplicas(6)) + f.NewEchoDeploymentWithReplicas(6) }) ginkgo.It("should connect to the same pod", func() { diff --git a/test/e2e/annotations/upstreamvhost.go b/test/e2e/annotations/upstreamvhost.go index e091e7c9f..a0e84726e 100644 --- a/test/e2e/annotations/upstreamvhost.go +++ b/test/e2e/annotations/upstreamvhost.go @@ -19,7 +19,7 @@ package annotations import ( "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/annotations/xforwardedprefix.go b/test/e2e/annotations/xforwardedprefix.go index 35f2eb426..6ae0d93c8 100644 --- a/test/e2e/annotations/xforwardedprefix.go +++ b/test/e2e/annotations/xforwardedprefix.go @@ -20,7 +20,7 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/cgroups/cgroups.go b/test/e2e/cgroups/cgroups.go deleted file mode 100644 index eab194324..000000000 --- a/test/e2e/cgroups/cgroups.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cgroups - -import ( - "log" - "os" - "path/filepath" - - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" - - "k8s.io/ingress-nginx/test/e2e/framework" - - "k8s.io/ingress-nginx/pkg/util/runtime" -) - -var _ = framework.IngressNginxDescribeSerial("[CGroups] cgroups", func() { - f := framework.NewDefaultFramework("cgroups") - - ginkgo.BeforeEach(func() { - f.NewEchoDeployment() - f.NewSlowEchoDeployment() - }) - - ginkgo.It("detects cgroups version v1", func() { - cgroupPath := "/testing/sys/fs/cgroup/" - if err := os.MkdirAll(cgroupPath, os.ModePerm); err != nil { - log.Fatal(err) - } - - quotaFile, err := os.Create(filepath.Join(cgroupPath, "cpu.cfs_quota_us")) - if err != nil { - log.Fatal(err) - } - - periodFile, err := os.Create(filepath.Join(cgroupPath, "cpu.cfs_period_us")) - if err != nil { - log.Fatal(err) - } - - _, err = quotaFile.WriteString("4") - if err != nil { - log.Fatal(err) - } - - err = quotaFile.Sync() - if err != nil { - log.Fatal(err) - } - - _, err = periodFile.WriteString("2") - if err != nil { - log.Fatal(err) - } - - err = periodFile.Sync() - if err != nil { - log.Fatal(err) - } - - assert.Equal(ginkgo.GinkgoT(), runtime.GetCgroupVersion(cgroupPath), int64(1)) - assert.Equal(ginkgo.GinkgoT(), runtime.NumCPUWithCustomPath(cgroupPath), 2) - - os.Remove(filepath.Join(cgroupPath, "cpu.cfs_quota_us")) - os.Remove(filepath.Join(cgroupPath, "cpu.cfs_period_us")) - }) - - ginkgo.It("detect cgroups version v2", func() { - cgroupPath := "/testing/sys/fs/cgroup/" - if err := os.MkdirAll(cgroupPath, os.ModePerm); err != nil { - log.Fatal(err) - } - - _, err := os.Create(filepath.Join(cgroupPath, "cgroup.controllers")) - if err != nil { - log.Fatal(err) - } - - file, err := os.Create(filepath.Join(cgroupPath, "cpu.max")) - if err != nil { - log.Fatal(err) - } - - _, err = file.WriteString("4 2") - if err != nil { - log.Fatal(err) - } - - err = file.Sync() - if err != nil { - log.Fatal(err) - } - - assert.Equal(ginkgo.GinkgoT(), runtime.GetCgroupVersion(cgroupPath), int64(2)) - assert.Equal(ginkgo.GinkgoT(), runtime.NumCPUWithCustomPath(cgroupPath), 2) - - os.Remove(filepath.Join(cgroupPath, "cpu.max")) - os.Remove(filepath.Join(cgroupPath, "cgroup.controllers")) - }) -}) diff --git a/test/e2e/dbg/main.go b/test/e2e/dbg/main.go index 6c0a22f69..2e50ef6af 100644 --- a/test/e2e/dbg/main.go +++ b/test/e2e/dbg/main.go @@ -20,7 +20,7 @@ import ( "encoding/json" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" @@ -49,7 +49,7 @@ var _ = framework.IngressNginxDescribe("Debug CLI", func() { assert.Nil(ginkgo.GinkgoT(), err) // Should be 2: the default and the echo deployment - numUpstreams := len(strings.Split(strings.Trim(output, "\n"), "\n")) + numUpstreams := len(strings.Split(strings.Trim(string(output), "\n"), "\n")) assert.Equal(ginkgo.GinkgoT(), numUpstreams, 2) }) @@ -67,7 +67,7 @@ var _ = framework.IngressNginxDescribe("Debug CLI", func() { output, err := f.ExecIngressPod(cmd) assert.Nil(ginkgo.GinkgoT(), err) - backends := strings.Split(output, "\n") + backends := strings.Split(string(output), "\n") assert.Greater(ginkgo.GinkgoT(), len(backends), 0) getCmd := "/dbg backends get " + backends[0] diff --git a/test/e2e/defaultbackend/custom_default_backend.go b/test/e2e/defaultbackend/custom_default_backend.go index 1e30b9269..f7f40ad8d 100644 --- a/test/e2e/defaultbackend/custom_default_backend.go +++ b/test/e2e/defaultbackend/custom_default_backend.go @@ -22,7 +22,7 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/e2e/defaultbackend/default_backend.go b/test/e2e/defaultbackend/default_backend.go index 936878000..853c04ae8 100644 --- a/test/e2e/defaultbackend/default_backend.go +++ b/test/e2e/defaultbackend/default_backend.go @@ -19,9 +19,8 @@ package defaultbackend import ( "net/http" - "k8s.io/ingress-nginx/test/e2e/framework/httpexpect" - - "github.com/onsi/ginkgo/v2" + "github.com/gavv/httpexpect/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" @@ -49,15 +48,15 @@ var _ = framework.IngressNginxDescribe("[Default Backend]", func() { {"basic HTTPS POST request without host to path / should return 404", "", framework.HTTPS, "POST", "/", http.StatusNotFound}, {"basic HTTPS POST request without host to path /demo should return 404", "", framework.HTTPS, "POST", "/demo", http.StatusNotFound}, - {"basic HTTP GET request to host foo.bar.com and path / should return 404", "foo.bar.com", framework.HTTP, "GET", "/", http.StatusNotFound}, - {"basic HTTP GET request to host foo.bar.com and path /demo should return 404", "foo.bar.com", framework.HTTP, "GET", "/demo", http.StatusNotFound}, - {"basic HTTPS GET request to host foo.bar.com and path / should return 404", "foo.bar.com", framework.HTTPS, "GET", "/", http.StatusNotFound}, - {"basic HTTPS GET request to host foo.bar.com and path /demo should return 404", "foo.bar.com", framework.HTTPS, "GET", "/demo", http.StatusNotFound}, + {"basic HTTP GET request to host foo.bar.com and path / should return 404", " foo.bar.com", framework.HTTP, "GET", "/", http.StatusNotFound}, + {"basic HTTP GET request to host foo.bar.com and path /demo should return 404", " foo.bar.com", framework.HTTP, "GET", "/demo", http.StatusNotFound}, + {"basic HTTPS GET request to host foo.bar.com and path / should return 404", " foo.bar.com", framework.HTTPS, "GET", "/", http.StatusNotFound}, + {"basic HTTPS GET request to host foo.bar.com and path /demo should return 404", " foo.bar.com", framework.HTTPS, "GET", "/demo", http.StatusNotFound}, - {"basic HTTP POST request to host foo.bar.com and path / should return 404", "foo.bar.com", framework.HTTP, "POST", "/", http.StatusNotFound}, - {"basic HTTP POST request to host foo.bar.com and path /demo should return 404", "foo.bar.com", framework.HTTP, "POST", "/demo", http.StatusNotFound}, - {"basic HTTPS POST request to host foo.bar.com and path / should return 404", "foo.bar.com", framework.HTTPS, "POST", "/", http.StatusNotFound}, - {"basic HTTPS POST request to host foo.bar.com and path /demo should return 404", "foo.bar.com", framework.HTTPS, "POST", "/demo", http.StatusNotFound}, + {"basic HTTP POST request to host foo.bar.com and path / should return 404", " foo.bar.com", framework.HTTP, "POST", "/", http.StatusNotFound}, + {"basic HTTP POST request to host foo.bar.com and path /demo should return 404", " foo.bar.com", framework.HTTP, "POST", "/demo", http.StatusNotFound}, + {"basic HTTPS POST request to host foo.bar.com and path / should return 404", " foo.bar.com", framework.HTTPS, "POST", "/", http.StatusNotFound}, + {"basic HTTPS POST request to host foo.bar.com and path /demo should return 404", " foo.bar.com", framework.HTTPS, "POST", "/demo", http.StatusNotFound}, } framework.Sleep() @@ -65,13 +64,15 @@ var _ = framework.IngressNginxDescribe("[Default Backend]", func() { for _, test := range testCases { ginkgo.By(test.Name) - var req *httpexpect.HTTPRequest + var req *httpexpect.Request switch test.Scheme { case framework.HTTP: - req = f.HTTPTestClient().DoRequest(test.Method, test.Path).WithURL(f.GetURL(framework.HTTP) + test.Path) + req = f.HTTPTestClient().Request(test.Method, test.Path) + req.WithURL(f.GetURL(framework.HTTP) + test.Path) case framework.HTTPS: - req = f.HTTPTestClient().DoRequest(test.Method, test.Path).WithURL(f.GetURL(framework.HTTPS) + test.Path) + req = f.HTTPTestClient().Request(test.Method, test.Path) + req.WithURL(f.GetURL(framework.HTTPS) + test.Path) default: ginkgo.Fail("Unexpected request scheme") } diff --git a/test/e2e/defaultbackend/ssl.go b/test/e2e/defaultbackend/ssl.go index 9f44e2e56..f453dd2d3 100644 --- a/test/e2e/defaultbackend/ssl.go +++ b/test/e2e/defaultbackend/ssl.go @@ -17,7 +17,7 @@ limitations under the License. package defaultbackend import ( - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" diff --git a/test/e2e/defaultbackend/with_hosts.go b/test/e2e/defaultbackend/with_hosts.go index c9a2c12a7..9a3d87a27 100644 --- a/test/e2e/defaultbackend/with_hosts.go +++ b/test/e2e/defaultbackend/with_hosts.go @@ -20,10 +20,11 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -47,14 +48,9 @@ var _ = framework.IngressNginxDescribe("[Default Backend] change default setting Annotations: annotations, }, Spec: networking.IngressSpec{ - IngressClassName: &f.IngressClass, - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: framework.EchoService, - Port: networking.ServiceBackendPort{ - Number: int32(80), - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: framework.EchoService, + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { diff --git a/test/e2e/disableleaderelection/disable_leader.go b/test/e2e/disableleaderelection/disable_leader.go deleted file mode 100644 index fd7369dfb..000000000 --- a/test/e2e/disableleaderelection/disable_leader.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright 2024 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package disableleaderelection - -import ( - "net/http" - "strings" - - "github.com/onsi/ginkgo/v2" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.IngressNginxDescribe("[Disable Leader] Routing works when leader election was disabled", func() { - f := framework.NewDefaultFramework("disableleaderelection") - - ginkgo.BeforeEach(func() { - f.NewEchoDeployment() - }) - - ginkgo.It("should create multiple ingress routings rules when leader election has disabled", func() { - host1 := "leader.election.disabled.com" - host2 := "leader.election.disabled2.com" - - ing1 := framework.NewSingleIngress(host1, "/foo", host1, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ing1) - - ing2 := framework.NewSingleIngress(host2, "/ping", host2, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ing2) - - f.WaitForNginxServer(host1, - func(server string) bool { - return strings.Contains(server, host1) && - strings.Contains(server, "location /foo") - }) - - f.WaitForNginxServer(host2, - func(server string) bool { - return strings.Contains(server, host2) && - strings.Contains(server, "location /ping") - }) - - f.HTTPTestClient(). - GET("/foo"). - WithHeader("Host", host1). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/bar"). - WithHeader("Host", host1). - Expect(). - Status(http.StatusNotFound) - - f.HTTPTestClient(). - GET("/foo"). - WithHeader("Host", host2). - Expect(). - Status(http.StatusNotFound) - - f.HTTPTestClient(). - GET("/ping"). - WithHeader("Host", host2). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/pong"). - WithHeader("Host", host2). - Expect(). - Status(http.StatusNotFound) - - f.HTTPTestClient(). - GET("/ping"). - WithHeader("Host", host1). - Expect(). - Status(http.StatusNotFound) - }) -}) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 9bf005164..9bb8937ea 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -20,7 +20,8 @@ import ( "os" "testing" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/config" "k8s.io/component-base/logs" // required @@ -31,25 +32,17 @@ import ( // tests to run _ "k8s.io/ingress-nginx/test/e2e/admission" _ "k8s.io/ingress-nginx/test/e2e/annotations" - _ "k8s.io/ingress-nginx/test/e2e/annotations/modsecurity" - _ "k8s.io/ingress-nginx/test/e2e/cgroups" _ "k8s.io/ingress-nginx/test/e2e/dbg" _ "k8s.io/ingress-nginx/test/e2e/defaultbackend" - _ "k8s.io/ingress-nginx/test/e2e/disableleaderelection" - _ "k8s.io/ingress-nginx/test/e2e/endpointslices" _ "k8s.io/ingress-nginx/test/e2e/gracefulshutdown" _ "k8s.io/ingress-nginx/test/e2e/ingress" _ "k8s.io/ingress-nginx/test/e2e/leaks" _ "k8s.io/ingress-nginx/test/e2e/loadbalance" _ "k8s.io/ingress-nginx/test/e2e/lua" - _ "k8s.io/ingress-nginx/test/e2e/metrics" - _ "k8s.io/ingress-nginx/test/e2e/nginx" _ "k8s.io/ingress-nginx/test/e2e/security" _ "k8s.io/ingress-nginx/test/e2e/servicebackend" _ "k8s.io/ingress-nginx/test/e2e/settings" - _ "k8s.io/ingress-nginx/test/e2e/settings/modsecurity" _ "k8s.io/ingress-nginx/test/e2e/settings/ocsp" - _ "k8s.io/ingress-nginx/test/e2e/settings/validations" _ "k8s.io/ingress-nginx/test/e2e/ssl" _ "k8s.io/ingress-nginx/test/e2e/status" _ "k8s.io/ingress-nginx/test/e2e/tcpudp" @@ -66,6 +59,6 @@ func RunE2ETests(t *testing.T) { framework.Logf("Using kubectl path '%s'", framework.KubectlPath) } - framework.Logf("Starting e2e run %q on Ginkgo node %d", framework.RunID, ginkgo.GinkgoParallelProcess()) + framework.Logf("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode) ginkgo.RunSpecs(t, "nginx-ingress-controller e2e suite") } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 2e556cf6d..4a7a48d92 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -26,7 +26,6 @@ func init() { testing.Init() framework.RegisterParseFlags() } - func TestE2E(t *testing.T) { RunE2ETests(t) } diff --git a/test/e2e/endpointslices/longname.go b/test/e2e/endpointslices/longname.go deleted file mode 100644 index b2242daac..000000000 --- a/test/e2e/endpointslices/longname.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package endpointslices - -import ( - "fmt" - "net/http" - "strings" - - "github.com/onsi/ginkgo/v2" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.IngressNginxDescribe("[Endpointslices] long service name", func() { - f := framework.NewDefaultFramework("endpointslices") - host := "longsvcname.foo.com" - name := "long-name-foobar-foobar-foobar-foobar-foobar-bar-foo-bar-foobuz" - - ginkgo.BeforeEach(func() { - f.NewEchoDeployment(framework.WithName(name)) - }) - - ginkgo.It("should return 200 when service name has max allowed number of characters 63", func() { - annotations := make(map[string]string) - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, name, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s", host)) - }) - - ginkgo.By("checking if the service is reached") - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - }) -}) diff --git a/test/e2e/endpointslices/topology.go b/test/e2e/endpointslices/topology.go deleted file mode 100644 index 70f7ff86b..000000000 --- a/test/e2e/endpointslices/topology.go +++ /dev/null @@ -1,94 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package endpointslices - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "strings" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.IngressNginxDescribeSerial("[TopologyHints] topology aware routing", func() { - f := framework.NewDefaultFramework("topology") - host := "topology-svc.foo.com" - - ginkgo.BeforeEach(func() { - f.NewEchoDeployment(framework.WithDeploymentReplicas(2), framework.WithSvcTopologyAnnotations()) - }) - - ginkgo.It("should return 200 when service has topology hints", func() { - annotations := make(map[string]string) - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s", host)) - }) - - ginkgo.By("checking if the service is reached") - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - - slices, err := f.KubeClientSet.DiscoveryV1().EndpointSlices(f.Namespace).List(context.TODO(), metav1.ListOptions{ - LabelSelector: "kubernetes.io/service-name=echo", - Limit: 1, - }) - assert.Nil(ginkgo.GinkgoT(), err) - - // check if we have hints, really depends on k8s endpoint slice controller - gotHints := true - for _, ep := range slices.Items[0].Endpoints { - if ep.Hints == nil || len(ep.Hints.ForZones) == 0 { - gotHints = false - break - } - } - - dbgCmd := "/dbg backends all" - status, err := f.ExecIngressPod(dbgCmd) - assert.Nil(ginkgo.GinkgoT(), err) - var backends []map[string]interface{} - err = json.Unmarshal([]byte(status), &backends) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error unmarshalling backends") - gotBackends := 0 - for _, bck := range backends { - if strings.Contains(bck["name"].(string), "topology") { - gotBackends = len(bck["endpoints"].([]interface{})) - } - } - - if gotHints { - // we have 2 replicas, if there is just one backend it means that we are routing according slices hints to same zone as controller is - assert.Equal(ginkgo.GinkgoT(), 1, gotBackends) - } else { - // two replicas should have two endpoints without topology hints - assert.Equal(ginkgo.GinkgoT(), 2, gotBackends) - } - }) -}) diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go index f213e2e98..8725e9950 100644 --- a/test/e2e/framework/deployment.go +++ b/test/e2e/framework/deployment.go @@ -18,12 +18,9 @@ package framework import ( "context" - "errors" - "fmt" - "os" "time" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -37,201 +34,40 @@ const EchoService = "echo" // SlowEchoService name of the deployment for the echo app const SlowEchoService = "slow-echo" -// HTTPBunService name of the deployment for the httpbun app -const HTTPBunService = "httpbun" +// HTTPBinService name of the deployment for the httpbin app +const HTTPBinService = "httpbin" -// NipService name of external service using nip.io -const NIPService = "external-nip" +// NginxBaseImage use for testing +const NginxBaseImage = "k8s.gcr.io/ingress-nginx/nginx:v20210530-g6aab4c291@sha256:a7356029dd0c26cc3466bf7a27daec0f4df73aa14ca6c8b871a767022a812c0b" -// HTTPBunImage is the default image that is used to deploy HTTPBun with the framework -var HTTPBunImage = os.Getenv("HTTPBUN_IMAGE") - -// EchoImage is the default image to be used by the echo service -const EchoImage = "registry.k8s.io/ingress-nginx/e2e-test-echo:v1.1.1@sha256:a1e0152e2eeab26e3f6fd3986f3d82b17bc7711717cae5392dcd18dd447ba6ef" //#nosec G101 - -// TODO: change all Deployment functions to use these options -// in order to reduce complexity and have a unified API across the -// framework -type deploymentOptions struct { - name string - namespace string - image string - replicas int - svcAnnotations map[string]string +// NewEchoDeployment creates a new single replica deployment of the echoserver image in a particular namespace +func (f *Framework) NewEchoDeployment() { + f.NewEchoDeploymentWithReplicas(1) } -// WithDeploymentNamespace allows configuring the deployment's namespace -func WithDeploymentNamespace(n string) func(*deploymentOptions) { - return func(o *deploymentOptions) { - o.namespace = n - } +// NewEchoDeploymentWithReplicas creates a new deployment of the echoserver image in a particular namespace. Number of +// replicas is configurable +func (f *Framework) NewEchoDeploymentWithReplicas(replicas int) { + f.NewEchoDeploymentWithNameAndReplicas(EchoService, replicas) } -// WithSvcTopologyAnnotations create svc with topology mode sets to auto -func WithSvcTopologyAnnotations() func(*deploymentOptions) { - return func(o *deploymentOptions) { - o.svcAnnotations = map[string]string{ - corev1.AnnotationTopologyMode: "auto", - } - } -} - -// WithDeploymentName allows configuring the deployment's names -func WithDeploymentName(n string) func(*deploymentOptions) { - return func(o *deploymentOptions) { - o.name = n - } -} - -// WithDeploymentReplicas allows configuring the deployment's replicas count -func WithDeploymentReplicas(r int) func(*deploymentOptions) { - return func(o *deploymentOptions) { - o.replicas = r - } -} - -func WithName(n string) func(*deploymentOptions) { - return func(o *deploymentOptions) { - o.name = n - } -} - -// WithImage allows configuring the image for the deployments -func WithImage(i string) func(*deploymentOptions) { - return func(o *deploymentOptions) { - o.image = i - } -} - -// NewEchoDeployment creates a new single replica deployment of the echo server image in a particular namespace -func (f *Framework) NewEchoDeployment(opts ...func(*deploymentOptions)) { - options := &deploymentOptions{ - namespace: f.Namespace, - name: EchoService, - replicas: 1, - image: EchoImage, - } - for _, o := range opts { - o(options) - } - - f.EnsureDeployment(newDeployment( - options.name, - options.namespace, - options.image, - 80, - int32(options.replicas), - nil, nil, nil, +// NewEchoDeploymentWithNameAndReplicas creates a new deployment of the echoserver image in a particular namespace. Number of +// replicas is configurable and +// name is configurable +func (f *Framework) NewEchoDeploymentWithNameAndReplicas(name string, replicas int) { + deployment := newDeployment(name, f.Namespace, "k8s.gcr.io/ingress-nginx/e2e-test-echo@sha256:b13e44f7bb6852b90633957e743e4a2b34f32f1694da556a9131b43950b8b2b1", 80, int32(replicas), + nil, []corev1.VolumeMount{}, []corev1.Volume{}, - true, - )) - - f.EnsureService(&corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: options.name, - Namespace: options.namespace, - Annotations: options.svcAnnotations, - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "http", - Port: 80, - TargetPort: intstr.FromInt(80), - Protocol: corev1.ProtocolTCP, - }, - }, - Selector: map[string]string{ - "app": options.name, - }, - }, - }) - - err := WaitForEndpoints( - f.KubeClientSet, - DefaultTimeout, - options.name, - options.namespace, - options.replicas, ) - assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready") -} -// BuildNipHost used to generate a nip host for DNS resolving -func BuildNIPHost(ip string) string { - return fmt.Sprintf("%s.nip.io", ip) -} + f.EnsureDeployment(deployment) -// GetNipHost used to generate a nip host for external DNS resolving -// for the instance deployed by the framework -func (f *Framework) GetNIPHost() string { - return BuildNIPHost(f.HTTPBunIP) -} - -// BuildNIPExternalNameService used to generate a service pointing to nip.io to -// help resolve to an IP address -func BuildNIPExternalNameService(f *Framework, ip, portName string) *corev1.Service { - return &corev1.Service{ + service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: NIPService, + Name: name, Namespace: f.Namespace, }, - Spec: corev1.ServiceSpec{ - ExternalName: BuildNIPHost(ip), - Type: corev1.ServiceTypeExternalName, - Ports: []corev1.ServicePort{ - { - Name: portName, - Port: 80, - TargetPort: intstr.FromInt(80), - Protocol: "TCP", - }, - }, - }, - } -} - -// NewHttpbunDeployment creates a new single replica deployment of the httpbun -// server image in a particular namespace we return the ip for testing purposes -func (f *Framework) NewHttpbunDeployment(opts ...func(*deploymentOptions)) string { - options := &deploymentOptions{ - namespace: f.Namespace, - name: HTTPBunService, - replicas: 1, - image: HTTPBunImage, - } - for _, o := range opts { - o(options) - } - - // Create the HTTPBun Deployment - f.EnsureDeployment(newDeployment( - options.name, - options.namespace, - options.image, - 80, - int32(options.replicas), - nil, nil, - // Required to get hostname information - []corev1.EnvVar{ - { - Name: "HTTPBUN_INFO_ENABLED", - Value: "1", - }, - }, - []corev1.VolumeMount{}, - []corev1.Volume{}, - true, - )) - - // Create a service pointing to deployment - f.EnsureService(&corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: options.name, - Namespace: options.namespace, - Annotations: options.svcAnnotations, - }, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ { @@ -242,29 +78,15 @@ func (f *Framework) NewHttpbunDeployment(opts ...func(*deploymentOptions)) strin }, }, Selector: map[string]string{ - "app": options.name, + "app": name, }, }, - }) + } - // Wait for deployment to become available - err := WaitForEndpoints( - f.KubeClientSet, - DefaultTimeout, - options.name, - options.namespace, - options.replicas, - ) + f.EnsureService(service) + + err := WaitForEndpoints(f.KubeClientSet, DefaultTimeout, name, f.Namespace, replicas) assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready") - - // Get cluster ip for HTTPBun to be used in tests - e, err := f.KubeClientSet. - CoreV1(). - Endpoints(f.Namespace). - Get(context.TODO(), HTTPBunService, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "failed to get httpbun endpoint") - - return e.Subsets[0].Addresses[0].IP } // NewSlowEchoDeployment creates a new deployment of the slow echo server image in a particular namespace. @@ -305,37 +127,23 @@ http { f.NGINXWithConfigDeployment(SlowEchoService, cfg) } -func (f *Framework) GetNginxBaseImage() string { - nginxBaseImage := os.Getenv("NGINX_BASE_IMAGE") - - if nginxBaseImage == "" { - assert.NotEmpty(ginkgo.GinkgoT(), errors.New("NGINX_BASE_IMAGE not defined"), "NGINX_BASE_IMAGE not defined") - } - - return nginxBaseImage -} - -// NGINXDeployment creates a new simple NGINX Deployment using NGINX base image -// and passing the desired configuration -func (f *Framework) NGINXDeployment(name, cfg string, waitendpoint bool) { +// NGINXWithConfigDeployment creates an NGINX deployment using a configmap containing the nginx.conf configuration +func (f *Framework) NGINXWithConfigDeployment(name string, cfg string) { cfgMap := map[string]string{ "nginx.conf": cfg, } - _, err := f.KubeClientSet. - CoreV1(). - ConfigMaps(f.Namespace). - Create(context.TODO(), &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: f.Namespace, - }, - Data: cfgMap, - }, metav1.CreateOptions{}) + _, err := f.KubeClientSet.CoreV1().ConfigMaps(f.Namespace).Create(context.TODO(), &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: f.Namespace, + }, + Data: cfgMap, + }, metav1.CreateOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "creating configmap") - deployment := newDeployment(name, f.Namespace, f.GetNginxBaseImage(), 80, 1, - nil, nil, nil, + deployment := newDeployment(name, f.Namespace, NginxBaseImage, 80, 1, + nil, []corev1.VolumeMount{ { Name: name, @@ -355,7 +163,7 @@ func (f *Framework) NGINXDeployment(name, cfg string, waitendpoint bool) { }, }, }, - }, true, + }, ) f.EnsureDeployment(deployment) @@ -382,15 +190,8 @@ func (f *Framework) NGINXDeployment(name, cfg string, waitendpoint bool) { f.EnsureService(service) - if waitendpoint { - err = WaitForEndpoints(f.KubeClientSet, DefaultTimeout, name, f.Namespace, 1) - assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready") - } -} - -// NGINXWithConfigDeployment creates an NGINX deployment using a configmap containing the nginx.conf configuration -func (f *Framework) NGINXWithConfigDeployment(name, cfg string) { - f.NGINXDeployment(name, cfg, true) + err = WaitForEndpoints(f.KubeClientSet, DefaultTimeout, name, f.Namespace, 1) + assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready") } // NewGRPCBinDeployment creates a new deployment of the @@ -403,7 +204,7 @@ func (f *Framework) NewGRPCBinDeployment() { PeriodSeconds: 1, SuccessThreshold: 1, TimeoutSeconds: 1, - ProbeHandler: corev1.ProbeHandler{ + Handler: corev1.Handler{ TCPSocket: &corev1.TCPSocketAction{ Port: intstr.FromInt(9000), }, @@ -486,16 +287,15 @@ func (f *Framework) NewGRPCBinDeployment() { assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready") } -func newDeployment(name, namespace, image string, port int32, replicas int32, command []string, args []string, env []corev1.EnvVar, - volumeMounts []corev1.VolumeMount, volumes []corev1.Volume, setProbe bool, -) *appsv1.Deployment { +func newDeployment(name, namespace, image string, port int32, replicas int32, command []string, + volumeMounts []corev1.VolumeMount, volumes []corev1.Volume) *appsv1.Deployment { probe := &corev1.Probe{ InitialDelaySeconds: 2, PeriodSeconds: 1, SuccessThreshold: 1, TimeoutSeconds: 2, FailureThreshold: 6, - ProbeHandler: corev1.ProbeHandler{ + Handler: corev1.Handler{ HTTPGet: &corev1.HTTPGetAction{ Port: intstr.FromString("http"), Path: "/", @@ -534,7 +334,9 @@ func newDeployment(name, namespace, image string, port int32, replicas int32, co ContainerPort: port, }, }, - VolumeMounts: volumeMounts, + ReadinessProbe: probe, + LivenessProbe: probe, + VolumeMounts: volumeMounts, }, }, Volumes: volumes, @@ -543,30 +345,21 @@ func newDeployment(name, namespace, image string, port int32, replicas int32, co }, } - if setProbe { - d.Spec.Template.Spec.Containers[0].ReadinessProbe = probe - d.Spec.Template.Spec.Containers[0].LivenessProbe = probe - } if len(command) > 0 { d.Spec.Template.Spec.Containers[0].Command = command } - if len(args) > 0 { - d.Spec.Template.Spec.Containers[0].Args = args - } - if len(env) > 0 { - d.Spec.Template.Spec.Containers[0].Env = env - } return d } -func (f *Framework) NewDeployment(name, image string, port, replicas int32) { - f.NewDeploymentWithOpts(name, image, port, replicas, nil, nil, nil, nil, nil, true) +// NewHttpbinDeployment creates a new single replica deployment of the httpbin image in a particular namespace. +func (f *Framework) NewHttpbinDeployment() { + f.NewDeployment(HTTPBinService, "k8s.gcr.io/ingress-nginx/e2e-test-httpbin@sha256:c6372ef57a775b95f18e19d4c735a9819f2e7bb4641e5e3f27287d831dfeb7e8", 80, 1) } // NewDeployment creates a new deployment in a particular namespace. -func (f *Framework) NewDeploymentWithOpts(name, image string, port, replicas int32, command, args []string, env []corev1.EnvVar, volumeMounts []corev1.VolumeMount, volumes []corev1.Volume, setProbe bool) { - deployment := newDeployment(name, f.Namespace, image, port, replicas, command, args, env, volumeMounts, volumes, setProbe) +func (f *Framework) NewDeployment(name, image string, port int32, replicas int32) { + deployment := newDeployment(name, f.Namespace, image, port, replicas, nil, nil, nil) f.EnsureDeployment(deployment) @@ -607,7 +400,7 @@ func (f *Framework) DeleteDeployment(name string) error { }) assert.Nil(ginkgo.GinkgoT(), err, "deleting deployment") - return waitForPodsDeleted(f.KubeClientSet, 2*time.Minute, f.Namespace, &metav1.ListOptions{ + return waitForPodsDeleted(f.KubeClientSet, 2*time.Minute, f.Namespace, metav1.ListOptions{ LabelSelector: labelSelectorToString(d.Spec.Selector.MatchLabels), }) } diff --git a/test/e2e/framework/exec.go b/test/e2e/framework/exec.go index 8d528c37a..8c8c7ddb0 100644 --- a/test/e2e/framework/exec.go +++ b/test/e2e/framework/exec.go @@ -21,7 +21,6 @@ import ( "encoding/json" "fmt" "io" - "os" "os/exec" "regexp" "strconv" @@ -66,7 +65,6 @@ func (f *Framework) ExecCommand(pod *corev1.Pod, command string) (string, error) execErr bytes.Buffer ) - //nolint:gosec // Ignore G204 error cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v exec --namespace %s %s --container controller -- %s", KubectlPath, pod.Namespace, pod.Name, command)) cmd.Stdout = &execOut cmd.Stderr = &execErr @@ -74,6 +72,7 @@ func (f *Framework) ExecCommand(pod *corev1.Pod, command string) (string, error) err := cmd.Run() if err != nil { return "", fmt.Errorf("could not execute '%s %s': %v", cmd.Path, cmd.Args, err) + } if execErr.Len() > 0 { @@ -91,7 +90,6 @@ func (f *Framework) NamespaceContent() (string, error) { execErr bytes.Buffer ) - //nolint:gosec // Ignore G204 error cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v get pods,services,endpoints,deployments --namespace %s", KubectlPath, f.Namespace)) cmd.Stdout = &execOut cmd.Stderr = &execErr @@ -99,10 +97,11 @@ func (f *Framework) NamespaceContent() (string, error) { err := cmd.Run() if err != nil { return "", fmt.Errorf("could not execute '%s %s': %v", cmd.Path, cmd.Args, err) + } eout := strings.TrimSpace(execErr.String()) - if eout != "" { + if len(eout) > 0 { return "", fmt.Errorf("stderr: %v", eout) } @@ -110,14 +109,9 @@ func (f *Framework) NamespaceContent() (string, error) { } // newIngressController deploys a new NGINX Ingress controller in a namespace -func (f *Framework) newIngressController(namespace, namespaceOverlay string) error { +func (f *Framework) newIngressController(namespace string, namespaceOverlay string) error { // Creates an nginx deployment - isChroot, ok := os.LookupEnv("IS_CHROOT") - if !ok { - isChroot = "false" - } - - cmd := exec.Command("./wait-for-nginx.sh", namespace, namespaceOverlay, isChroot) + cmd := exec.Command("./wait-for-nginx.sh", namespace, namespaceOverlay) out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("unexpected error waiting for ingress controller deployment: %v.\nLogs:\n%v", err, string(out)) @@ -126,11 +120,12 @@ func (f *Framework) newIngressController(namespace, namespaceOverlay string) err return nil } -var proxyRegexp = regexp.MustCompile(`Starting to serve on .*:(\d+)`) +var ( + proxyRegexp = regexp.MustCompile("Starting to serve on .*:([0-9]+)") +) // KubectlProxy creates a proxy to kubernetes apiserver func (f *Framework) KubectlProxy(port int) (int, *exec.Cmd, error) { - //nolint:gosec // Ignore G204 error cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s proxy --accept-hosts=.* --address=0.0.0.0 --port=%d", KubectlPath, port)) stdout, stderr, err := startCmdAndStreamOutput(cmd) if err != nil { @@ -157,17 +152,6 @@ func (f *Framework) KubectlProxy(port int) (int, *exec.Cmd, error) { return -1, cmd, fmt.Errorf("failed to parse port from proxy stdout: %s", output) } -func (f *Framework) UninstallChart() error { - //nolint:gosec //Ignore G204 error - cmd := exec.Command("helm", "uninstall", "--namespace", f.Namespace, "nginx-ingress") - _, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("unexpected error uninstalling ingress-nginx release: %v", err) - } - - return nil -} - func startCmdAndStreamOutput(cmd *exec.Cmd) (stdout, stderr io.ReadCloser, err error) { stdout, err = cmd.StdoutPipe() if err != nil { diff --git a/test/e2e/framework/fastcgi_helloserver.go b/test/e2e/framework/fastcgi_helloserver.go index c414c4da3..75a8d098a 100644 --- a/test/e2e/framework/fastcgi_helloserver.go +++ b/test/e2e/framework/fastcgi_helloserver.go @@ -14,11 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -//nolint:dupl // Ignore dupl errors for similar test case package framework import ( - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -59,7 +58,7 @@ func (f *Framework) NewNewFastCGIHelloServerDeploymentWithReplicas(replicas int3 Containers: []corev1.Container{ { Name: "fastcgi-helloserver", - Image: "registry.k8s.io/ingress-nginx/fastcgi-helloserver:v1.1.1@sha256:6af4d8c7745c6727aab759db616a58fd68d784d07ce7a32d1ad149c331fd9a6f", + Image: "k8s.gcr.io/ingress-nginx/e2e-test-fastcgi-helloserver@sha256:723b8187e1768d199b93fd939c37c1ce9427dcbca72ec6415f4d890bca637fcc", Env: []corev1.EnvVar{}, Ports: []corev1.ContainerPort{ { @@ -76,7 +75,7 @@ func (f *Framework) NewNewFastCGIHelloServerDeploymentWithReplicas(replicas int3 d := f.EnsureDeployment(deployment) - err := waitForPodsReady(f.KubeClientSet, DefaultTimeout, int(replicas), f.Namespace, &metav1.ListOptions{ + err := waitForPodsReady(f.KubeClientSet, DefaultTimeout, int(replicas), f.Namespace, metav1.ListOptions{ LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(), }) assert.Nil(ginkgo.GinkgoT(), err, "failed to wait for to become ready") diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 204da7df0..8f5fb3884 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -16,29 +16,31 @@ package framework import ( "context" "crypto/tls" - "encoding/json" "fmt" "net" "net/http" "strings" "time" - "k8s.io/ingress-nginx/test/e2e/framework/httpexpect" - - "github.com/onsi/ginkgo/v2" - ginkgotypes "github.com/onsi/ginkgo/v2/types" + "github.com/gavv/httpexpect/v2" + "github.com/onsi/ginkgo" + "github.com/pkg/errors" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" apiextcs "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + restclient "k8s.io/client-go/rest" + "k8s.io/ingress-nginx/internal/k8s" "k8s.io/klog/v2" ) @@ -51,48 +53,36 @@ const ( HTTPS RequestScheme = "https" ) -// KubectlPath defines the full path of the kubectl binary -var KubectlPath = "/usr/local/bin/kubectl" +var ( + // KubectlPath defines the full path of the kubectl binary + KubectlPath = "/usr/local/bin/kubectl" +) // Framework supports common operations used by e2e tests; it will keep a client & a namespace for you. type Framework struct { BaseName string + IsIngressV1Ready bool + IsIngressV1Beta1Ready bool + // A Kubernetes and Service Catalog client KubeClientSet kubernetes.Interface - KubeConfig *rest.Config + KubeConfig *restclient.Config APIExtensionsClientSet apiextcs.Interface - Namespace string - IngressClass string + Namespace string - pod *v1.Pod - // We use httpbun as a service that we route to in our tests through - // the ingress controller. We add it as part of the framework as it - // is used extensively - HTTPBunIP string - HTTPBunEnabled bool -} - -// WithHTTPBunEnabled deploys an instance of HTTPBun for the specific test -func WithHTTPBunEnabled() func(*Framework) { - return func(f *Framework) { - f.HTTPBunEnabled = true - } + pod *corev1.Pod } // NewDefaultFramework makes a new framework and sets up a BeforeEach/AfterEach for // you (you can write additional before/after each functions). -func NewDefaultFramework(baseName string, opts ...func(*Framework)) *Framework { +func NewDefaultFramework(baseName string) *Framework { defer ginkgo.GinkgoRecover() f := &Framework{ BaseName: baseName, } - // set framework options - for _, o := range opts { - o(f) - } ginkgo.BeforeEach(f.BeforeEach) ginkgo.AfterEach(f.AfterEach) @@ -100,26 +90,8 @@ func NewDefaultFramework(baseName string, opts ...func(*Framework)) *Framework { return f } -// NewSimpleFramework makes a new framework that allows the usage of a namespace -// for arbitrary tests. -func NewSimpleFramework(baseName string, opts ...func(*Framework)) *Framework { - defer ginkgo.GinkgoRecover() - - f := &Framework{ - BaseName: baseName, - } - // set framework options - for _, o := range opts { - o(f) - } - - ginkgo.BeforeEach(f.CreateEnvironment) - ginkgo.AfterEach(f.DestroyEnvironment) - - return f -} - -func (f *Framework) CreateEnvironment() { +// BeforeEach gets a client and makes a namespace. +func (f *Framework) BeforeEach() { var err error if f.KubeClientSet == nil { @@ -131,26 +103,12 @@ func (f *Framework) CreateEnvironment() { f.KubeClientSet, err = kubernetes.NewForConfig(f.KubeConfig) assert.Nil(ginkgo.GinkgoT(), err, "creating a kubernetes client") + + _, f.IsIngressV1Beta1Ready, f.IsIngressV1Ready = k8s.NetworkingIngressAvailable(f.KubeClientSet) } f.Namespace, err = CreateKubeNamespace(f.BaseName, f.KubeClientSet) assert.Nil(ginkgo.GinkgoT(), err, "creating namespace") -} - -func (f *Framework) DestroyEnvironment() { - defer ginkgo.GinkgoRecover() - err := DeleteKubeNamespace(f.KubeClientSet, f.Namespace) - assert.Nil(ginkgo.GinkgoT(), err, "deleting namespace %v", f.Namespace) -} - -// BeforeEach gets a client and makes a namespace. -func (f *Framework) BeforeEach() { - var err error - - f.CreateEnvironment() - - f.IngressClass, err = CreateIngressClass(f.Namespace, f.KubeClientSet) - assert.Nil(ginkgo.GinkgoT(), err, "creating IngressClass") err = f.newIngressController(f.Namespace, f.BaseName) assert.Nil(ginkgo.GinkgoT(), err, "deploying the ingress controller") @@ -159,32 +117,23 @@ func (f *Framework) BeforeEach() { assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller pod information") f.WaitForNginxListening(80) - - // If HTTPBun is enabled deploy an instance to the namespace - if f.HTTPBunEnabled { - f.HTTPBunIP = f.NewHttpbunDeployment() - } } // AfterEach deletes the namespace, after reading its events. func (f *Framework) AfterEach() { - defer f.DestroyEnvironment() + defer func(kubeClient kubernetes.Interface, ns string) { + go func() { + defer ginkgo.GinkgoRecover() + err := deleteKubeNamespace(kubeClient, ns) + assert.Nil(ginkgo.GinkgoT(), err, "deleting namespace %v", f.Namespace) + }() + }(f.KubeClientSet, f.Namespace) - defer func(kubeClient kubernetes.Interface, ingressclass string) { - defer ginkgo.GinkgoRecover() - - err := f.UninstallChart() - assert.Nil(ginkgo.GinkgoT(), err, "uninstalling helm chart") - - err = deleteIngressClass(kubeClient, ingressclass) - assert.Nil(ginkgo.GinkgoT(), err, "deleting IngressClass") - }(f.KubeClientSet, f.IngressClass) - - if !ginkgo.CurrentSpecReport().Failed() || ginkgo.CurrentSpecReport().State.Is(ginkgotypes.SpecStateInterrupted) { + if !ginkgo.CurrentGinkgoTestDescription().Failed { return } - cmd := "cat /etc/nginx/nginx.conf" + cmd := fmt.Sprintf("cat /etc/nginx/nginx.conf") o, err := f.ExecCommand(f.pod, cmd) if err != nil { Logf("Unexpected error obtaining nginx.conf file: %v", err) @@ -218,11 +167,6 @@ func IngressNginxDescribe(text string, body func()) bool { return ginkgo.Describe(text, body) } -// IngressNginxDescribeSerial wrapper function for ginkgo describe. Adds namespacing. -func IngressNginxDescribeSerial(text string, body func()) bool { - return ginkgo.Describe(text, ginkgo.Serial, body) -} - // DescribeAnnotation wrapper function for ginkgo describe. Adds namespacing. func DescribeAnnotation(text string, body func()) bool { return ginkgo.Describe("[Annotations] "+text, body) @@ -233,6 +177,11 @@ func DescribeSetting(text string, body func()) bool { return ginkgo.Describe("[Setting] "+text, body) } +// MemoryLeakIt is wrapper function for ginkgo It. Adds "[MemoryLeak]" tag and makes static analysis easier. +func MemoryLeakIt(text string, body interface{}, timeout ...float64) bool { + return ginkgo.It(text+" [MemoryLeak]", body, timeout...) +} + // GetNginxIP returns the number of TCP port where NGINX is running func (f *Framework) GetNginxIP() string { s, err := f.KubeClientSet. @@ -249,13 +198,13 @@ func (f *Framework) GetNginxPodIP() string { } // GetURL returns the URL should be used to make a request to NGINX -func (f *Framework) GetURL(requestScheme RequestScheme) string { +func (f *Framework) GetURL(scheme RequestScheme) string { ip := f.GetNginxIP() - return fmt.Sprintf("%v://%v", requestScheme, ip) + return fmt.Sprintf("%v://%v", scheme, ip) } // GetIngressNGINXPod returns the ingress controller running pod -func (f *Framework) GetIngressNGINXPod() *v1.Pod { +func (f *Framework) GetIngressNGINXPod() *corev1.Pod { return f.pod } @@ -269,7 +218,6 @@ func (f *Framework) updateIngressNGINXPod() error { // WaitForNginxServer waits until the nginx configuration contains a particular server section. // `cfg` passed to matcher is normalized by replacing all tabs and spaces with single space. func (f *Framework) WaitForNginxServer(name string, matcher func(cfg string) bool) { - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated err := wait.Poll(Poll, DefaultTimeout, f.matchNginxConditions(name, matcher)) assert.Nil(ginkgo.GinkgoT(), err, "waiting for nginx server condition/s") Sleep(1 * time.Second) @@ -278,24 +226,13 @@ func (f *Framework) WaitForNginxServer(name string, matcher func(cfg string) boo // WaitForNginxConfiguration waits until the nginx configuration contains a particular configuration // `cfg` passed to matcher is normalized by replacing all tabs and spaces with single space. func (f *Framework) WaitForNginxConfiguration(matcher func(cfg string) bool) { - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated err := wait.Poll(Poll, DefaultTimeout, f.matchNginxConditions("", matcher)) assert.Nil(ginkgo.GinkgoT(), err, "waiting for nginx server condition/s") Sleep(1 * time.Second) } -// WaitForLuaConfiguration waits until the nginx configuration contains a particular configuration -// `cfg` passed to matcher is normalized by replacing all tabs and spaces with single space. -func (f *Framework) WaitForLuaConfiguration(matcher func(jsonCfg map[string]interface{}) bool) { - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated - err := wait.Poll(Poll, DefaultTimeout, f.matchLuaConditions(matcher)) - assert.Nil(ginkgo.GinkgoT(), err, "waiting for nginx lua configuration condition/s") - Sleep(1 * time.Second) -} - // WaitForNginxCustomConfiguration waits until the nginx configuration given part (from, to) contains a particular configuration -func (f *Framework) WaitForNginxCustomConfiguration(from, to string, matcher func(cfg string) bool) { - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated +func (f *Framework) WaitForNginxCustomConfiguration(from string, to string, matcher func(cfg string) bool) { err := wait.Poll(Poll, DefaultTimeout, f.matchNginxCustomConditions(from, to, matcher)) assert.Nil(ginkgo.GinkgoT(), err, "waiting for nginx server condition/s") } @@ -313,7 +250,7 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b return func() (bool, error) { var cmd string if name == "" { - cmd = "cat /etc/nginx/nginx.conf" + cmd = fmt.Sprintf("cat /etc/nginx/nginx.conf") } else { cmd = fmt.Sprintf("cat /etc/nginx/nginx.conf | awk '/## start server %v/,/## end server %v/'", name, name) } @@ -323,7 +260,7 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b return false, nil } - if klog.V(10).Enabled() && o != "" { + if klog.V(10).Enabled() && len(o) > 0 { klog.InfoS("NGINX", "configuration", o) } @@ -336,30 +273,7 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b } } -func (f *Framework) matchLuaConditions(matcher func(jsonCfg map[string]interface{}) bool) wait.ConditionFunc { - return func() (bool, error) { - cmd := "cat /etc/nginx/lua/cfg.json" - - o, err := f.ExecCommand(f.pod, cmd) - if err != nil { - return false, nil - } - - if klog.V(10).Enabled() && o != "" { - klog.InfoS("Lua", "configuration", o) - } - - luaConfig := make(map[string]interface{}) // Use unstructured so we can walk through JSON - if err := json.Unmarshal([]byte(o), &luaConfig); err != nil { - return false, err - } - - // passes the lua interface to the function - return matcher(luaConfig), nil - } -} - -func (f *Framework) matchNginxCustomConditions(from, to string, matcher func(cfg string) bool) wait.ConditionFunc { +func (f *Framework) matchNginxCustomConditions(from string, to string, matcher func(cfg string) bool) wait.ConditionFunc { return func() (bool, error) { cmd := fmt.Sprintf("cat /etc/nginx/nginx.conf| awk '/%v/,/%v/'", from, to) @@ -368,7 +282,7 @@ func (f *Framework) matchNginxCustomConditions(from, to string, matcher func(cfg return false, nil } - if klog.V(10).Enabled() && o != "" { + if klog.V(10).Enabled() && len(o) > 0 { klog.InfoS("NGINX", "configuration", o) } @@ -416,20 +330,6 @@ func (f *Framework) SetNginxConfigMapData(cmData map[string]string) { f.WaitForReload(fn) } -// SetNginxConfigMapData sets ingress-nginx's nginx-ingress-controller configMap data -func (f *Framework) AllowSnippetConfiguration() func() { - f.SetNginxConfigMapData(map[string]string{ - "allow-snippet-annotations": "true", - "annotations-risk-level": "Critical", // To enable snippet configurations - }) - return func() { - f.SetNginxConfigMapData(map[string]string{ - "allow-snippet-annotations": "false", - "annotations-risk-level": "High", - }) - } -} - // CreateConfigMap creates a new configmap in the current namespace func (f *Framework) CreateConfigMap(name string, data map[string]string) { _, err := f.KubeClientSet.CoreV1().ConfigMaps(f.Namespace).Create(context.TODO(), &v1.ConfigMap{ @@ -443,7 +343,7 @@ func (f *Framework) CreateConfigMap(name string, data map[string]string) { } // UpdateNginxConfigMapData updates single field in ingress-nginx's nginx-ingress-controller map data -func (f *Framework) UpdateNginxConfigMapData(key, value string) { +func (f *Framework) UpdateNginxConfigMapData(key string, value string) { config, err := f.getConfigMap("nginx-ingress-controller") assert.Nil(ginkgo.GinkgoT(), err) assert.NotNil(ginkgo.GinkgoT(), config, "expected a configmap but none returned") @@ -462,14 +362,13 @@ func (f *Framework) UpdateNginxConfigMapData(key, value string) { } // WaitForReload calls the passed function and -// asserts it has caused at least 1 reload. +// asser it has caused at least 1 reload. func (f *Framework) WaitForReload(fn func()) { initialReloadCount := getReloadCount(f.pod, f.Namespace, f.KubeClientSet) fn() count := 0 - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated err := wait.Poll(1*time.Second, DefaultTimeout, func() (bool, error) { reloads := getReloadCount(f.pod, f.Namespace, f.KubeClientSet) // most of the cases reload the ingress controller @@ -485,13 +384,13 @@ func (f *Framework) WaitForReload(fn func()) { assert.Nil(ginkgo.GinkgoT(), err, "while waiting for ingress controller reload") } -func getReloadCount(pod *v1.Pod, namespace string, client kubernetes.Interface) int { +func getReloadCount(pod *corev1.Pod, namespace string, client kubernetes.Interface) int { events, err := client.CoreV1().Events(namespace).Search(scheme.Scheme, pod) assert.Nil(ginkgo.GinkgoT(), err, "obtaining NGINX Pod") reloadCount := 0 - for i := range events.Items { - if events.Items[i].Reason == "RELOAD" && events.Items[i].Type == v1.EventTypeNormal { + for _, e := range events.Items { + if e.Reason == "RELOAD" && e.Type == corev1.EventTypeNormal { reloadCount++ } } @@ -506,7 +405,7 @@ func (f *Framework) DeleteNGINXPod(grace int64) { err := f.KubeClientSet.CoreV1().Pods(ns).Delete(context.TODO(), f.pod.GetName(), *metav1.NewDeleteOptions(grace)) assert.Nil(ginkgo.GinkgoT(), err, "deleting ingress nginx pod") - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated + err = wait.Poll(Poll, DefaultTimeout, func() (bool, error) { err := f.updateIngressNGINXPod() if err != nil || f.pod == nil { @@ -517,52 +416,52 @@ func (f *Framework) DeleteNGINXPod(grace int64) { assert.Nil(ginkgo.GinkgoT(), err, "while waiting for ingress nginx pod to come up again") } -// HTTPDumbTestClient returns a new httpexpect client without BaseURL. -func (f *Framework) HTTPDumbTestClient() *httpexpect.HTTPRequest { - return f.newHTTPTestClient(nil, false) -} - -// HTTPTestClient returns a new HTTPRequest client for end-to-end HTTP testing. -func (f *Framework) HTTPTestClient() *httpexpect.HTTPRequest { - return f.newHTTPTestClient(nil, true) +// HTTPTestClient returns a new httpexpect client for end-to-end HTTP testing. +func (f *Framework) HTTPTestClient() *httpexpect.Expect { + return f.newTestClient(nil) } // HTTPTestClientWithTLSConfig returns a new httpexpect client for end-to-end // HTTP testing with a custom TLS configuration. -func (f *Framework) HTTPTestClientWithTLSConfig(config *tls.Config) *httpexpect.HTTPRequest { - return f.newHTTPTestClient(config, true) +func (f *Framework) HTTPTestClientWithTLSConfig(config *tls.Config) *httpexpect.Expect { + return f.newTestClient(config) } -func (f *Framework) newHTTPTestClient(config *tls.Config, setIngressURL bool) *httpexpect.HTTPRequest { +func (f *Framework) newTestClient(config *tls.Config) *httpexpect.Expect { if config == nil { config = &tls.Config{ - InsecureSkipVerify: true, //nolint:gosec // Ignore the gosec error in testing + InsecureSkipVerify: true, } } - var baseURL string - if setIngressURL { - baseURL = f.GetURL(HTTP) - } - return httpexpect.NewRequest(baseURL, &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: config, + return httpexpect.WithConfig(httpexpect.Config{ + BaseURL: f.GetURL(HTTP), + Client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: config, + }, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, }, - CheckRedirect: func(_ *http.Request, _ []*http.Request) error { - return http.ErrUseLastResponse + Reporter: httpexpect.NewAssertReporter( + httpexpect.NewAssertReporter(ginkgo.GinkgoT()), + ), + Printers: []httpexpect.Printer{ + // TODO: enable conditionally? + // httpexpect.NewDebugPrinter(ginkgo.GinkgoT(), false), }, - }, httpexpect.NewAssertReporter()) + }) } // WaitForNginxListening waits until NGINX starts accepting connections on a port func (f *Framework) WaitForNginxListening(port int) { - err := waitForPodsReady(f.KubeClientSet, DefaultTimeout, 1, f.Namespace, &metav1.ListOptions{ + err := waitForPodsReady(f.KubeClientSet, DefaultTimeout, 1, f.Namespace, metav1.ListOptions{ LabelSelector: "app.kubernetes.io/name=ingress-nginx", }) assert.Nil(ginkgo.GinkgoT(), err, "waiting for ingress pods to be ready") podIP := f.GetNginxIP() - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated err = wait.Poll(500*time.Millisecond, DefaultTimeout, func() (bool, error) { hostPort := net.JoinHostPort(podIP, fmt.Sprintf("%v", port)) conn, err := net.Dial("tcp", hostPort) @@ -577,21 +476,8 @@ func (f *Framework) WaitForNginxListening(port int) { assert.Nil(ginkgo.GinkgoT(), err, "waiting for ingress controller pod listening on port 80") } -// WaitForPod waits for a specific Pod to be ready, using a label selector -func (f *Framework) WaitForPod(selector string, timeout time.Duration, shouldFail bool) { - err := waitForPodsReady(f.KubeClientSet, timeout, 1, f.Namespace, &metav1.ListOptions{ - LabelSelector: selector, - }) - - if shouldFail { - assert.NotNil(ginkgo.GinkgoT(), err, "waiting for pods to be ready") - } else { - assert.Nil(ginkgo.GinkgoT(), err, "waiting for pods to be ready") - } -} - // UpdateDeployment runs the given updateFunc on the deployment and waits for it to be updated -func UpdateDeployment(kubeClientSet kubernetes.Interface, namespace, name string, replicas int, updateFunc func(d *appsv1.Deployment) error) error { +func UpdateDeployment(kubeClientSet kubernetes.Interface, namespace string, name string, replicas int, updateFunc func(d *appsv1.Deployment) error) error { deployment, err := kubeClientSet.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { return err @@ -612,7 +498,7 @@ func UpdateDeployment(kubeClientSet kubernetes.Interface, namespace, name string deployment.Spec.Replicas = NewInt32(int32(replicas)) _, err = kubeClientSet.AppsV1().Deployments(namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) if err != nil { - return fmt.Errorf("scaling the number of replicas to %d: %w", replicas, err) + return errors.Wrapf(err, "scaling the number of replicas to %v", replicas) } err = waitForDeploymentRollout(kubeClientSet, deployment) @@ -621,18 +507,17 @@ func UpdateDeployment(kubeClientSet kubernetes.Interface, namespace, name string } } - err = waitForPodsReady(kubeClientSet, DefaultTimeout, replicas, namespace, &metav1.ListOptions{ + err = waitForPodsReady(kubeClientSet, DefaultTimeout, replicas, namespace, metav1.ListOptions{ LabelSelector: fields.SelectorFromSet(fields.Set(deployment.Spec.Template.ObjectMeta.Labels)).String(), }) if err != nil { - return fmt.Errorf("waiting for nginx-ingress-controller replica count to be %d: %w", replicas, err) + return errors.Wrapf(err, "waiting for nginx-ingress-controller replica count to be %v", replicas) } return nil } func waitForDeploymentRollout(kubeClientSet kubernetes.Interface, resource *appsv1.Deployment) error { - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated return wait.Poll(Poll, 5*time.Minute, func() (bool, error) { d, err := kubeClientSet.AppsV1().Deployments(resource.Namespace).Get(context.TODO(), resource.Name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { @@ -656,8 +541,8 @@ func waitForDeploymentRollout(kubeClientSet kubernetes.Interface, resource *apps } // UpdateIngress runs the given updateFunc on the ingress -func UpdateIngress(kubeClientSet kubernetes.Interface, namespace, name string, updateFunc func(d *networking.Ingress) error) error { - ingress, err := kubeClientSet.NetworkingV1().Ingresses(namespace).Get(context.TODO(), name, metav1.GetOptions{}) +func UpdateIngress(kubeClientSet kubernetes.Interface, namespace string, name string, updateFunc func(d *networking.Ingress) error) error { + ingress, err := kubeClientSet.NetworkingV1beta1().Ingresses(namespace).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { return err } @@ -674,7 +559,7 @@ func UpdateIngress(kubeClientSet kubernetes.Interface, namespace, name string, u return err } - _, err = kubeClientSet.NetworkingV1().Ingresses(namespace).Update(context.TODO(), ingress, metav1.UpdateOptions{}) + _, err = kubeClientSet.NetworkingV1beta1().Ingresses(namespace).Update(context.TODO(), ingress, metav1.UpdateOptions{}) if err != nil { return err } @@ -693,17 +578,9 @@ func NewSingleIngress(name, path, host, ns, service string, port int, annotation return newSingleIngressWithRules(name, path, host, ns, service, port, annotations, nil) } -func NewSingleIngressWithIngressClass(name, path, host, ns, service, ingressClass string, port int, annotations map[string]string) *networking.Ingress { - ing := newSingleIngressWithRules(name, path, host, ns, service, port, annotations, nil) - ing.Spec.IngressClassName = &ingressClass - return ing -} - // NewSingleIngressWithMultiplePaths creates a simple ingress rule with multiple paths func NewSingleIngressWithMultiplePaths(name string, paths []string, host, ns, service string, port int, annotations map[string]string) *networking.Ingress { - pathtype := networking.PathTypePrefix spec := networking.IngressSpec{ - IngressClassName: GetIngressClassName(ns), Rules: []networking.IngressRule{ { Host: host, @@ -716,15 +593,10 @@ func NewSingleIngressWithMultiplePaths(name string, paths []string, host, ns, se for _, path := range paths { spec.Rules[0].IngressRuleValue.HTTP.Paths = append(spec.Rules[0].IngressRuleValue.HTTP.Paths, networking.HTTPIngressPath{ - Path: path, - PathType: &pathtype, + Path: path, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: service, - Port: networking.ServiceBackendPort{ - Number: int32(port), - }, - }, + ServiceName: service, + ServicePort: intstr.FromInt(port), }, }) } @@ -733,24 +605,17 @@ func NewSingleIngressWithMultiplePaths(name string, paths []string, host, ns, se } func newSingleIngressWithRules(name, path, host, ns, service string, port int, annotations map[string]string, tlsHosts []string) *networking.Ingress { - pathtype := networking.PathTypePrefix spec := networking.IngressSpec{ - IngressClassName: GetIngressClassName(ns), Rules: []networking.IngressRule{ { IngressRuleValue: networking.IngressRuleValue{ HTTP: &networking.HTTPIngressRuleValue{ Paths: []networking.HTTPIngressPath{ { - Path: path, - PathType: &pathtype, + Path: path, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: service, - Port: networking.ServiceBackendPort{ - Number: int32(port), - }, - }, + ServiceName: service, + ServicePort: intstr.FromInt(port), }, }, }, @@ -779,16 +644,10 @@ func newSingleIngressWithRules(name, path, host, ns, service string, port int, a // NewSingleIngressWithBackendAndRules creates an ingress with both a default backend and a rule func NewSingleIngressWithBackendAndRules(name, path, host, ns, defaultService string, defaultPort int, service string, port int, annotations map[string]string) *networking.Ingress { - pathtype := networking.PathTypePrefix spec := networking.IngressSpec{ - IngressClassName: GetIngressClassName(ns), - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: defaultService, - Port: networking.ServiceBackendPort{ - Number: int32(defaultPort), - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: defaultService, + ServicePort: intstr.FromInt(defaultPort), }, Rules: []networking.IngressRule{ { @@ -797,15 +656,10 @@ func NewSingleIngressWithBackendAndRules(name, path, host, ns, defaultService st HTTP: &networking.HTTPIngressRuleValue{ Paths: []networking.HTTPIngressPath{ { - Path: path, - PathType: &pathtype, + Path: path, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: service, - Port: networking.ServiceBackendPort{ - Number: int32(port), - }, - }, + ServiceName: service, + ServicePort: intstr.FromInt(port), }, }, }, @@ -821,14 +675,9 @@ func NewSingleIngressWithBackendAndRules(name, path, host, ns, defaultService st // NewSingleCatchAllIngress creates a simple ingress with a catch-all backend func NewSingleCatchAllIngress(name, ns, service string, port int, annotations map[string]string) *networking.Ingress { spec := networking.IngressSpec{ - IngressClassName: GetIngressClassName(ns), - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: service, - Port: networking.ServiceBackendPort{ - Number: int32(port), - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: service, + ServicePort: intstr.FromInt(port), }, } return newSingleIngress(name, ns, annotations, spec) @@ -867,7 +716,7 @@ func Sleep(duration ...time.Duration) { time.Sleep(sleepFor) } -func loadConfig() (*rest.Config, error) { +func loadConfig() (*restclient.Config, error) { config, err := rest.InClusterConfig() if err != nil { return nil, err diff --git a/test/e2e/framework/grpc_delay.go b/test/e2e/framework/grpc_delay.go deleted file mode 100644 index 58d10b2e9..000000000 --- a/test/e2e/framework/grpc_delay.go +++ /dev/null @@ -1,109 +0,0 @@ -/* -Copyright 2024 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package framework - -import ( - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/util/intstr" -) - -// NewGRPCBinDelayDeployment creates a new single replica -// deployment of the grpcbin image in a particular namespace -func (f *Framework) NewGRPCBinDelayDeployment() { - f.NewNewGRPCBinDelayDeploymentWithReplicas(1) -} - -// NewNewGRPCBinDelayDeploymentWithReplicas creates a new deployment of the -// grpcbin image in a particular namespace. Number of replicas is configurable -func (f *Framework) NewNewGRPCBinDelayDeploymentWithReplicas(replicas int32) { - name := "grpcbin-delay" - - deployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: f.Namespace, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: NewInt32(replicas), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": name, - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": name, - }, - }, - Spec: corev1.PodSpec{ - TerminationGracePeriodSeconds: NewInt64(0), - Containers: []corev1.Container{ - { - Name: name, - Image: "ghcr.io/anddd7/grpcbin:v1.0.6", - Env: []corev1.EnvVar{}, - Ports: []corev1.ContainerPort{ - { - Name: "grpc", - ContainerPort: 50051, - }, - }, - }, - }, - }, - }, - }, - } - - d := f.EnsureDeployment(deployment) - - err := waitForPodsReady(f.KubeClientSet, DefaultTimeout, int(replicas), f.Namespace, &metav1.ListOptions{ - LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(), - }) - assert.Nil(ginkgo.GinkgoT(), err, "failed to wait for to become ready") - - service := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: f.Namespace, - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "grpc", - Port: 50051, - TargetPort: intstr.FromInt(50051), - Protocol: "TCP", - }, - }, - Selector: map[string]string{ - "app": name, - }, - }, - } - - f.EnsureService(service) - - err = WaitForEndpoints(f.KubeClientSet, DefaultTimeout, name, f.Namespace, int(replicas)) - assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready") -} diff --git a/test/e2e/framework/grpc_fortune_teller.go b/test/e2e/framework/grpc_fortune_teller.go index a7be46327..012121761 100644 --- a/test/e2e/framework/grpc_fortune_teller.go +++ b/test/e2e/framework/grpc_fortune_teller.go @@ -14,11 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -//nolint:dupl // Ignore dupl errors for similar test case package framework import ( - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -76,7 +75,7 @@ func (f *Framework) NewNewGRPCFortuneTellerDeploymentWithReplicas(replicas int32 d := f.EnsureDeployment(deployment) - err := waitForPodsReady(f.KubeClientSet, DefaultTimeout, int(replicas), f.Namespace, &metav1.ListOptions{ + err := waitForPodsReady(f.KubeClientSet, DefaultTimeout, int(replicas), f.Namespace, metav1.ListOptions{ LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(), }) assert.Nil(ginkgo.GinkgoT(), err, "failed to wait for to become ready") diff --git a/test/e2e/framework/healthz.go b/test/e2e/framework/healthz.go index 3ee297b27..bef093355 100644 --- a/test/e2e/framework/healthz.go +++ b/test/e2e/framework/healthz.go @@ -26,7 +26,7 @@ func (f *Framework) VerifyHealthz(ip string, statusCode int) error { url := fmt.Sprintf("http://%v:10254/healthz", ip) client := &http.Client{} - req, err := http.NewRequest(http.MethodGet, url, http.NoBody) + req, err := http.NewRequest("GET", url, nil) if err != nil { return fmt.Errorf("creating GET request for URL %q failed: %v", url, err) } diff --git a/test/e2e/framework/httpexpect/README.md b/test/e2e/framework/httpexpect/README.md deleted file mode 100644 index 73cfd9dbb..000000000 --- a/test/e2e/framework/httpexpect/README.md +++ /dev/null @@ -1 +0,0 @@ -This module is based in the deprecated library `github.com/gavv/httpexpect`, and contains slightly adaptations. diff --git a/test/e2e/framework/httpexpect/array.go b/test/e2e/framework/httpexpect/array.go deleted file mode 100644 index 432982f4a..000000000 --- a/test/e2e/framework/httpexpect/array.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package httpexpect - -// Array provides methods to inspect attached []interface{} object -// (Go representation of JSON array). -type Array struct { - chain chain - value []interface{} -} - -// Iter returns a new slice of Values attached to array elements. -func (a *Array) Iter() []Value { - if a.chain.failed() { - return []Value{} - } - ret := []Value{} - for n := range a.value { - ret = append(ret, Value{a.chain, a.value[n]}) - } - return ret -} diff --git a/test/e2e/framework/httpexpect/chain.go b/test/e2e/framework/httpexpect/chain.go deleted file mode 100644 index 33c4d15b7..000000000 --- a/test/e2e/framework/httpexpect/chain.go +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package httpexpect - -type chain struct { - reporter Reporter - failbit bool -} - -func makeChain(reporter Reporter) chain { - return chain{reporter, false} -} - -func (c *chain) failed() bool { - return c.failbit -} - -func (c *chain) fail(message string, args ...interface{}) { - if c.failbit { - return - } - c.failbit = true - c.reporter.Errorf(message, args...) -} diff --git a/test/e2e/framework/httpexpect/cookie.go b/test/e2e/framework/httpexpect/cookie.go deleted file mode 100644 index 1cd57a6cc..000000000 --- a/test/e2e/framework/httpexpect/cookie.go +++ /dev/null @@ -1,29 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package httpexpect - -import "net/http" - -// Cookie provides methods to inspect attached http.Cookie value. -type Cookie struct { - chain chain - value *http.Cookie -} - -func (c *Cookie) Raw() *http.Cookie { - return c.value -} diff --git a/test/e2e/framework/httpexpect/match.go b/test/e2e/framework/httpexpect/match.go deleted file mode 100644 index aad10b02e..000000000 --- a/test/e2e/framework/httpexpect/match.go +++ /dev/null @@ -1,37 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package httpexpect - -// Match provides methods to inspect attached regexp match results. -type Match struct { - chain chain - submatches []string - names map[string]int -} - -func makeMatch(chain chain, submatches, names []string) *Match { - if submatches == nil { - submatches = []string{} - } - namemap := map[string]int{} - for n, name := range names { - if name != "" { - namemap[name] = n - } - } - return &Match{chain, submatches, namemap} -} diff --git a/test/e2e/framework/httpexpect/object.go b/test/e2e/framework/httpexpect/object.go deleted file mode 100644 index 49dcf5659..000000000 --- a/test/e2e/framework/httpexpect/object.go +++ /dev/null @@ -1,114 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package httpexpect - -import ( - "reflect" - - "github.com/yudai/gojsondiff" - "github.com/yudai/gojsondiff/formatter" -) - -const unavailableMsg = " (unavailable)" - -// Object provides methods to inspect attached map[string]interface{} object -// (Go representation of JSON object). -type Object struct { - chain chain - value map[string]interface{} -} - -func (o *Object) ValueEqual(key string, value interface{}) *Object { - if !o.containsKey(key) { - o.chain.fail("\nexpected object containing key '%s', but got:\n%s", - key, dumpValue(o.value)) - return o - } - expected, ok := canonValue(&o.chain, value) - if !ok { - return o - } - if !reflect.DeepEqual(expected, o.value[key]) { - o.chain.fail("\nexpected value for key '%s' equal to:\n%s\n\nbut got:\n%s\n\ndiff:\n%s", - key, - dumpValue(expected), - dumpValue(o.value[key]), - diffValues(expected, o.value[key])) - } - return o -} - -func (o *Object) ContainsKey(key string) *Object { - if !o.containsKey(key) { - o.chain.fail("\nexpected object containing key '%s', but got:\n%s", - key, - dumpValue(o.value)) - } - return o -} - -func (o *Object) NotContainsKey(key string) *Object { - if o.containsKey(key) { - o.chain.fail("\nexpected object not containing key '%s', but got:\n%s", - key, dumpValue(o.value)) - } - return o -} - -func (o *Object) containsKey(key string) bool { - for k := range o.value { - if k == key { - return true - } - } - return false -} - -func diffValues(expected, actual interface{}) string { - differ := gojsondiff.New() - - var diff gojsondiff.Diff - - switch ve := expected.(type) { - case map[string]interface{}: - if va, ok := actual.(map[string]interface{}); ok { - diff = differ.CompareObjects(ve, va) - } else { - return unavailableMsg - } - case []interface{}: - if va, ok := actual.([]interface{}); ok { - diff = differ.CompareArrays(ve, va) - } else { - return unavailableMsg - } - default: - return unavailableMsg - } - - config := formatter.AsciiFormatterConfig{ - ShowArrayIndex: true, - } - f := formatter.NewAsciiFormatter(expected, config) - - str, err := f.Format(diff) - if err != nil { - return unavailableMsg - } - - return "--- expected\n+++ actual\n" + str -} diff --git a/test/e2e/framework/httpexpect/reporter.go b/test/e2e/framework/httpexpect/reporter.go deleted file mode 100644 index e80234590..000000000 --- a/test/e2e/framework/httpexpect/reporter.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package httpexpect - -import ( - "fmt" - - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" -) - -// Reporter is used to report failures. -// testing.TB, AssertReporter, and RequireReporter implement this interface. -type Reporter interface { - // Errorf reports failure. - // Allowed to return normally or terminate test using t.FailNow(). - Errorf(message string, args ...interface{}) -} - -// AssertReporter implements Reporter interface using `testify/assert' -// package. Failures are non-fatal with this reporter. -type AssertReporter struct { - backend *assert.Assertions -} - -// NewAssertReporter returns a new AssertReporter object. -func NewAssertReporter() *AssertReporter { - return &AssertReporter{assert.New(ginkgo.GinkgoT())} -} - -// Errorf implements Reporter.Errorf. -func (r *AssertReporter) Errorf(message string, args ...interface{}) { - r.backend.Fail(fmt.Sprintf(message, args...)) -} diff --git a/test/e2e/framework/httpexpect/request.go b/test/e2e/framework/httpexpect/request.go deleted file mode 100644 index 4daba136e..000000000 --- a/test/e2e/framework/httpexpect/request.go +++ /dev/null @@ -1,206 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package httpexpect - -import ( - "context" - "fmt" - "io" - "net" - "net/http" - "net/url" - "path" -) - -type HTTPRequest struct { - chain chain - reporter Reporter - baseURL string - client *http.Client - query url.Values - Request *http.Request - HTTPResponse *HTTPResponse -} - -// NewRequest returns an HTTPRequest object. -func NewRequest(baseURL string, client *http.Client, reporter Reporter) *HTTPRequest { - response := NewResponse(reporter) - return &HTTPRequest{ - baseURL: baseURL, - client: client, - reporter: reporter, - chain: makeChain(reporter), - HTTPResponse: response, - } -} - -// GET creates a new HTTP request with GET method. -func (h *HTTPRequest) GET(rpath string) *HTTPRequest { - if h.chain.failed() { - return h - } - return h.DoRequest("GET", rpath) -} - -// DoRequest creates a new HTTP request object. -func (h *HTTPRequest) DoRequest(method, rpath string) *HTTPRequest { - uri, err := url.Parse(h.baseURL) - if err != nil { - h.chain.fail(err.Error()) - } - - var request *http.Request - uri.Path = path.Join(uri.Path, rpath) - if request, err = http.NewRequest(method, uri.String(), http.NoBody); err != nil { - h.chain.fail(err.Error()) - } - - h.Request = request - return h -} - -// ForceResolve forces the test resolver to point to a specific endpoint -func (h *HTTPRequest) ForceResolve(ip string, port uint16) *HTTPRequest { - addr := net.ParseIP(ip) - if addr == nil { - h.chain.fail(fmt.Sprintf("invalid ip address: %s", ip)) - return h - } - dialer := &net.Dialer{ - Timeout: h.client.Timeout, - KeepAlive: h.client.Timeout, - DualStack: true, - } - resolveAddr := fmt.Sprintf("%s:%d", ip, int(port)) - - oldTransport, ok := h.client.Transport.(*http.Transport) - if !ok { - h.chain.fail("invalid old transport address") - return h - } - newTransport := oldTransport.Clone() - newTransport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) { - return dialer.DialContext(ctx, network, resolveAddr) - } - h.client.Transport = newTransport - return h -} - -// Expect executes the request and returns an HTTP response. -func (h *HTTPRequest) Expect() *HTTPResponse { - if h.query != nil { - h.Request.URL.RawQuery = h.query.Encode() - } - - response, err := h.client.Do(h.Request) - if err != nil { - h.chain.fail(err.Error()) - } - defer response.Body.Close() - - h.HTTPResponse.Response = response // set the HTTP response - - var content []byte - if content, err = getContent(response); err != nil { - h.chain.fail(err.Error()) - } - // set content and cookies from HTTPResponse - h.HTTPResponse.content = content - h.HTTPResponse.cookies = h.HTTPResponse.Response.Cookies() - return h.HTTPResponse -} - -// WithURL sets the request URL appending paths when already exist. -func (h *HTTPRequest) WithURL(urlStr string) *HTTPRequest { - if h.chain.failed() { - return h - } - if u, err := url.Parse(urlStr); err != nil { - h.chain.fail(err.Error()) - } else { - u.Path = path.Join(h.Request.URL.Path, u.Path) - h.Request.URL = u - } - return h -} - -// WithHeader adds given header to request. -func (h *HTTPRequest) WithHeader(key, value string) *HTTPRequest { - if h.chain.failed() { - return h - } - switch http.CanonicalHeaderKey(key) { - case "Host": - h.Request.Host = value - default: - h.Request.Header.Add(key, value) - } - return h -} - -// WithCookies adds given cookies to request. -func (h *HTTPRequest) WithCookies(cookies map[string]string) *HTTPRequest { - if h.chain.failed() { - return h - } - for k, v := range cookies { - h.WithCookie(k, v) - } - return h -} - -// WithCookie adds given single cookie to request. -func (h *HTTPRequest) WithCookie(k, v string) *HTTPRequest { - if h.chain.failed() { - return h - } - h.Request.AddCookie(&http.Cookie{Name: k, Value: v}) - return h -} - -// WithBasicAuth sets the request's Authorization header to use HTTP -// Basic Authentication with the provided username and password. -// -// With HTTP Basic Authentication the provided username and password -// are not encrypted. -func (h *HTTPRequest) WithBasicAuth(username, password string) *HTTPRequest { - if h.chain.failed() { - return h - } - h.Request.SetBasicAuth(username, password) - return h -} - -// WithQuery adds query parameter to request URL. -func (h *HTTPRequest) WithQuery(key string, value interface{}) *HTTPRequest { - if h.chain.failed() { - return h - } - if h.query == nil { - h.query = make(url.Values) - } - h.query.Add(key, fmt.Sprint(value)) - return h -} - -// getContent returns the content from the body response. -func getContent(resp *http.Response) ([]byte, error) { - if resp.Body == nil { - return []byte{}, nil - } - return io.ReadAll(resp.Body) -} diff --git a/test/e2e/framework/httpexpect/response.go b/test/e2e/framework/httpexpect/response.go deleted file mode 100644 index e324e94ff..000000000 --- a/test/e2e/framework/httpexpect/response.go +++ /dev/null @@ -1,274 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package httpexpect - -import ( - "encoding/json" - "fmt" - "mime" - "net/http" - "reflect" - "strconv" - "strings" -) - -// StatusRange is enum for response status ranges. -type StatusRange int - -const ( - // Status1xx defines "Informational" status codes. - Status1xx StatusRange = 100 - - // Status2xx defines "Success" status codes. - Status2xx StatusRange = 200 - - // Status3xx defines "Redirection" status codes. - Status3xx StatusRange = 300 - - // Status4xx defines "Client Error" status codes. - Status4xx StatusRange = 400 - - // Status5xx defines "Server Error" status codes. - Status5xx StatusRange = 500 -) - -type HTTPResponse struct { - chain chain - content []byte - cookies []*http.Cookie - Response *http.Response -} - -// NewResponse returns an empty HTTPResponse object. -func NewResponse(reporter Reporter) *HTTPResponse { - return &HTTPResponse{ - chain: makeChain(reporter), - } -} - -// Body returns the body of the response. -func (r *HTTPResponse) Body() *String { - return &String{value: string(r.content)} -} - -// Raw returns the raw http response. -func (r *HTTPResponse) Raw() *http.Response { - return r.Response -} - -// Status compare the actual http response with the expected one raising and error -// if they don't match. -func (r *HTTPResponse) Status(status int) *HTTPResponse { - if r.chain.failed() { - return r - } - r.checkEqual("status", statusCodeText(status), statusCodeText(r.Response.StatusCode)) - return r -} - -// ContentEncoding succeeds if response has exactly given Content-Encoding -func (r *HTTPResponse) ContentEncoding(encoding ...string) *HTTPResponse { - if r.chain.failed() { - return r - } - r.checkEqual("\"Content-Encoding\" header", encoding, r.Response.Header["Content-Encoding"]) - return r -} - -// ContentType succeeds if response contains Content-Type header with given -// media type and charset. -func (r *HTTPResponse) ContentType(mediaType string, charset ...string) *HTTPResponse { - r.checkContentType(mediaType, charset...) - return r -} - -// Cookies returns a new Array object with all cookie names set by this response. -// Returned Array contains a String value for every cookie name. -func (r *HTTPResponse) Cookies() *Array { - if r.chain.failed() { - return &Array{r.chain, nil} - } - names := []interface{}{} - for _, c := range r.cookies { - names = append(names, c.Name) - } - return &Array{r.chain, names} -} - -// Cookie returns a new Cookie object that may be used to inspect given cookie -// set by this response. -func (r *HTTPResponse) Cookie(name string) *Cookie { - if r.chain.failed() { - return &Cookie{r.chain, nil} - } - names := []string{} - for _, c := range r.cookies { - if c.Name == name { - return &Cookie{r.chain, c} - } - names = append(names, c.Name) - } - r.chain.fail("\nexpected response with cookie:\n %q\n\nbut got only cookies:\n%s", name, dumpValue(names)) - return &Cookie{r.chain, nil} -} - -// Headers returns a new Object that may be used to inspect header map. -func (r *HTTPResponse) Headers() *Object { - var value map[string]interface{} - if !r.chain.failed() { - value, _ = canonMap(&r.chain, r.Response.Header) - } - return &Object{r.chain, value} -} - -// Header returns a new String object that may be used to inspect given header. -func (r *HTTPResponse) Header(header string) *String { - return &String{chain: r.chain, value: r.Response.Header.Get(header)} -} - -func canonMap(chain *chain, in interface{}) (map[string]interface{}, bool) { - var out map[string]interface{} - data, ok := canonValue(chain, in) - if ok { - out, ok = data.(map[string]interface{}) - if !ok { - chain.fail("expected map, got %v", out) - } - } - return out, ok -} - -func canonValue(chain *chain, in interface{}) (interface{}, bool) { - b, err := json.Marshal(in) - if err != nil { - chain.fail(err.Error()) - return nil, false - } - - var out interface{} - if err := json.Unmarshal(b, &out); err != nil { - chain.fail(err.Error()) - return nil, false - } - - return out, true -} - -// StatusRange succeeds if response status belongs to given range. -func (r *HTTPResponse) StatusRange(rn StatusRange) *HTTPResponse { - if r.chain.failed() { - return r - } - status := statusCodeText(r.Response.StatusCode) - - actual := statusRangeText(r.Response.StatusCode) - expected := statusRangeText(int(rn)) - - if actual == "" || actual != expected { - if actual == "" { - r.chain.fail("\nexpected status from range:\n %q\n\nbut got:\n %q", - expected, status) - } else { - r.chain.fail("\nexpected status from range:\n %q\n\nbut got:\n %q (%q)", - expected, actual, status) - } - } - return r -} - -func statusCodeText(code int) string { - if s := http.StatusText(code); s != "" { - return strconv.Itoa(code) + " " + s - } - return strconv.Itoa(code) -} - -func statusRangeText(code int) string { - switch { - case code >= 100 && code < 200: - return "1xx Informational" - case code >= 200 && code < 300: - return "2xx Success" - case code >= 300 && code < 400: - return "3xx Redirection" - case code >= 400 && code < 500: - return "4xx Client Error" - case code >= 500 && code < 600: - return "5xx Server Error" - default: - return "" - } -} - -func (r *HTTPResponse) checkContentType(expectedType string, expectedCharset ...string) bool { - if r.chain.failed() { - return false - } - - contentType := r.Response.Header.Get("Content-Type") - - if expectedType == "" && len(expectedCharset) == 0 { - if contentType == "" { - return true - } - } - - mediaType, params, err := mime.ParseMediaType(contentType) - if err != nil { - r.chain.fail("\ngot invalid \"Content-Type\" header %q", contentType) - return false - } - - if mediaType != expectedType { - //nolint:goconst //string interpolation - r.chain.fail("\nexpected \"Content-Type\" header with %q media type,"+ - "\nbut got %q", expectedType, mediaType) - return false - } - - charset := params["charset"] - - if len(expectedCharset) == 0 { - if charset != "" && !strings.EqualFold(charset, "utf-8") { - r.chain.fail("\nexpected \"Content-Type\" header with \"utf-8\" or empty charset,"+ - "\nbut got %q", charset) - return false - } - } else { - if !strings.EqualFold(charset, expectedCharset[0]) { - r.chain.fail("\nexpected \"Content-Type\" header with %q charset,"+ - "\nbut got %q", expectedCharset[0], charset) - return false - } - } - return true -} - -func (r *HTTPResponse) checkEqual(what string, expected, actual interface{}) { - if !reflect.DeepEqual(expected, actual) { - r.chain.fail("\nexpected %s equal to:\n%s\n\nbut got:\n%s", - what, dumpValue(expected), dumpValue(actual)) - } -} - -func dumpValue(value interface{}) string { - b, err := json.MarshalIndent(value, " ", " ") - if err != nil { - return " " + fmt.Sprintf("%#v", value) - } - return " " + string(b) -} diff --git a/test/e2e/framework/httpexpect/string.go b/test/e2e/framework/httpexpect/string.go deleted file mode 100644 index be2ab0f04..000000000 --- a/test/e2e/framework/httpexpect/string.go +++ /dev/null @@ -1,120 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package httpexpect - -import ( - "regexp" - "strings" -) - -// String provides methods to inspect attached string value -// (Go representation of JSON string). -type String struct { - chain chain - value string -} - -// Raw returns underlying value attached to String. -// This is the value originally passed to NewString. -func (s *String) Raw() string { - return s.value -} - -// Empty succeeds if string is empty. -func (s *String) Empty() *String { - return s.Equal("") -} - -// NotEmpty succeeds if string is non-empty. -func (s *String) NotEmpty() *String { - return s.NotEqual("") -} - -// Equal succeeds if string is equal to given Go string. -func (s *String) Equal(value string) *String { - if !(s.value == value) { - s.chain.fail("\nexpected string equal to:\n %q\n\nbut got:\n %q", value, s.value) - } - return s -} - -// NotEqual succeeds if string is not equal to given Go string. -func (s *String) NotEqual(value string) *String { - if !(s.value != value) { - s.chain.fail("\nexpected string not equal to:\n %q", value) - } - return s -} - -// Contains succeeds if string contains given Go string as a substring. -func (s *String) Contains(value string) *String { - if !strings.Contains(s.value, value) { - s.chain.fail( - "\nexpected string containing substring:\n %q\n\nbut got:\n %q", - value, s.value) - } - return s -} - -// NotContains succeeds if string doesn't contain Go string as a substring. -func (s *String) NotContains(value string) *String { - if strings.Contains(s.value, value) { - s.chain.fail("\nexpected string not containing substring:\n %q\n\nbut got:\n %q", value, s.value) - } - return s -} - -// ContainsFold succeeds if string contains given Go string as a substring after -// applying Unicode case-folding (so it's a case-insensitive match). -func (s *String) ContainsFold(value string) *String { - if !strings.Contains(strings.ToLower(s.value), strings.ToLower(value)) { - s.chain.fail("\nexpected string containing substring (case-insensitive):\n %q"+"\n\nbut got:\n %q", value, s.value) - } - return s -} - -// NotContainsFold succeeds if string doesn't contain given Go string as a substring -// after applying Unicode case-folding (so it's a case-insensitive match). -// -// Example: -// -// str := NewString(t, "Hello") -// str.NotContainsFold("BYE") -func (s *String) NotContainsFold(value string) *String { - if strings.Contains(strings.ToLower(s.value), strings.ToLower(value)) { - s.chain.fail("\nexpected string not containing substring (case-insensitive):\n %q"+"\n\nbut got:\n %q", value, s.value) - } - return s -} - -// Match matches the string with given regexp and returns a new Match object -// with found submatches. -func (s *String) Match(re string) *Match { - r, err := regexp.Compile(re) - if err != nil { - s.chain.fail(err.Error()) - return makeMatch(s.chain, nil, nil) - } - - m := r.FindStringSubmatch(s.value) - if m == nil { - s.chain.fail("\nexpected string matching regexp:\n `%s`\n\nbut got:\n %q", re, s.value) - return makeMatch(s.chain, nil, nil) - } - - return makeMatch(s.chain, m, r.SubexpNames()) -} diff --git a/test/e2e/framework/httpexpect/value.go b/test/e2e/framework/httpexpect/value.go deleted file mode 100644 index fd59091b1..000000000 --- a/test/e2e/framework/httpexpect/value.go +++ /dev/null @@ -1,33 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package httpexpect - -// Value provides methods to inspect attached interface{} object -// (Go representation of arbitrary JSON value) and cast it to -// concrete type. -type Value struct { - chain chain - value interface{} -} - -func (v *Value) String() *String { - data, ok := v.value.(string) - if !ok { - v.chain.fail("\nexpected string value, but got:\n%s", dumpValue(v.value)) - } - return &String{v.chain, data} -} diff --git a/test/e2e/framework/influxdb.go b/test/e2e/framework/influxdb.go new file mode 100644 index 000000000..6721913d6 --- /dev/null +++ b/test/e2e/framework/influxdb.go @@ -0,0 +1,143 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "github.com/onsi/ginkgo" + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" +) + +const influxConfig = ` +reporting-disabled = true +bind-address = "0.0.0.0:8088" + +[meta] + dir = "/var/lib/influxdb/meta" + retention-autocreate = true + logging-enabled = true + +[data] + dir = "/var/lib/influxdb/data" + index-version = "inmem" + wal-dir = "/var/lib/influxdb/wal" + wal-fsync-delay = "0s" + query-log-enabled = true + cache-max-memory-size = 1073741824 + cache-snapshot-memory-size = 26214400 + cache-snapshot-write-cold-duration = "10m0s" + compact-full-write-cold-duration = "4h0m0s" + max-series-per-database = 1000000 + max-values-per-tag = 100000 + max-concurrent-compactions = 0 + trace-logging-enabled = false + +[[udp]] + enabled = true + bind-address = ":8089" + database = "nginx" +` + +// NewInfluxDBDeployment creates an InfluxDB server configured to reply +// on 8086/tcp and 8089/udp +func (f *Framework) NewInfluxDBDeployment() { + configuration := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "influxdb-config", + Namespace: f.Namespace, + }, + Data: map[string]string{ + "influxd.conf": influxConfig, + }, + } + + cm, err := f.EnsureConfigMap(configuration) + assert.Nil(ginkgo.GinkgoT(), err, "creating an Influxdb deployment") + assert.NotNil(ginkgo.GinkgoT(), cm, "expected a configmap but none returned") + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "influxdb", + Namespace: f.Namespace, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: NewInt32(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "influxdb", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "influxdb", + }, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: NewInt64(0), + Volumes: []corev1.Volume{ + { + Name: "influxdb-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "influxdb-config", + }, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "influxdb", + Image: "docker.io/influxdb:1.5", + Env: []corev1.EnvVar{}, + Command: []string{"influxd", "-config", "/influxdb-config/influxd.conf"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "influxdb-config", + ReadOnly: true, + MountPath: "/influxdb-config", + }, + }, + Ports: []corev1.ContainerPort{ + { + Name: "http", + ContainerPort: 8086, + }, + { + Name: "udp", + ContainerPort: 8089, + }, + }, + }, + }, + }, + }, + }, + } + + d := f.EnsureDeployment(deployment) + + err = waitForPodsReady(f.KubeClientSet, DefaultTimeout, 1, f.Namespace, metav1.ListOptions{ + LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(), + }) + assert.Nil(ginkgo.GinkgoT(), err, "waiting for influxdb pod to become ready") +} diff --git a/test/e2e/framework/k8s.go b/test/e2e/framework/k8s.go index 7c067421d..3ca4a3c46 100644 --- a/test/e2e/framework/k8s.go +++ b/test/e2e/framework/k8s.go @@ -22,11 +22,13 @@ import ( "strings" "time" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" + api "k8s.io/api/core/v1" core "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + v1 "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1beta1" k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilnet "k8s.io/apimachinery/pkg/util/net" @@ -34,9 +36,9 @@ import ( "k8s.io/client-go/kubernetes" ) -// EnsureSecret creates a Secret object or returns it. -func (f *Framework) EnsureSecret(secret *core.Secret) *core.Secret { - err := createSecretWithRetries(f.KubeClientSet, secret.Namespace, secret) +// EnsureSecret creates a Secret object or returns it if it already exists. +func (f *Framework) EnsureSecret(secret *api.Secret) *api.Secret { + err := createSecretWithRetries(f.KubeClientSet, f.Namespace, secret) assert.Nil(ginkgo.GinkgoT(), err, "creating secret") s, err := f.KubeClientSet.CoreV1().Secrets(secret.Namespace).Get(context.TODO(), secret.Name, metav1.GetOptions{}) @@ -46,35 +48,22 @@ func (f *Framework) EnsureSecret(secret *core.Secret) *core.Secret { return s } -// GetConfigMap gets a ConfigMap object from the given namespace, name and returns it, throws error if it does not exist. -func (f *Framework) GetConfigMap(namespace, name string) *core.ConfigMap { - cm, err := f.KubeClientSet.CoreV1().ConfigMaps(namespace).Get(context.TODO(), name, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "getting configmap") - assert.NotNil(ginkgo.GinkgoT(), cm, "expected a configmap but none returned") - return cm -} - -// EnsureConfigMap creates or updates an existing ConfigMap object or returns it. -func (f *Framework) EnsureConfigMap(configMap *core.ConfigMap) *core.ConfigMap { - cm := configMap.DeepCopy() - // Clean out ResourceVersion field if present - if cm.ObjectMeta.ResourceVersion != "" { - cm.ObjectMeta.ResourceVersion = "" +// EnsureConfigMap creates a ConfigMap object or returns it if it already exists. +func (f *Framework) EnsureConfigMap(configMap *api.ConfigMap) (*api.ConfigMap, error) { + cm, err := f.KubeClientSet.CoreV1().ConfigMaps(f.Namespace).Create(context.TODO(), configMap, metav1.CreateOptions{}) + if err != nil { + if k8sErrors.IsAlreadyExists(err) { + return f.KubeClientSet.CoreV1().ConfigMaps(f.Namespace).Update(context.TODO(), configMap, metav1.UpdateOptions{}) + } + return nil, err } - res, err := f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Create(context.TODO(), cm, metav1.CreateOptions{}) - if k8sErrors.IsAlreadyExists(err) { - res, err = f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Update(context.TODO(), cm, metav1.UpdateOptions{}) - } - assert.Nil(ginkgo.GinkgoT(), err, "updating configmap") - assert.NotNil(ginkgo.GinkgoT(), res, "updating configmap") - - return res + return cm, nil } // GetIngress gets an Ingress object from the given namespace, name and returns it, throws error if it does not exists. -func (f *Framework) GetIngress(namespace, name string) *networking.Ingress { - ing, err := f.KubeClientSet.NetworkingV1().Ingresses(namespace).Get(context.TODO(), name, metav1.GetOptions{}) +func (f *Framework) GetIngress(namespace string, name string) *networking.Ingress { + ing, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(namespace).Get(context.TODO(), name, metav1.GetOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "getting ingress") assert.NotNil(ginkgo.GinkgoT(), ing, "expected an ingress but none returned") return ing @@ -83,13 +72,13 @@ func (f *Framework) GetIngress(namespace, name string) *networking.Ingress { // EnsureIngress creates an Ingress object and returns it, throws error if it already exists. func (f *Framework) EnsureIngress(ingress *networking.Ingress) *networking.Ingress { fn := func() { - err := createIngressWithRetries(f.KubeClientSet, ingress.Namespace, ingress) + err := createIngressWithRetries(f.KubeClientSet, f.Namespace, ingress) assert.Nil(ginkgo.GinkgoT(), err, "creating ingress") } f.WaitForReload(fn) - ing := f.GetIngress(ingress.Namespace, ingress.Name) + ing := f.GetIngress(f.Namespace, ingress.Name) if ing.Annotations == nil { ing.Annotations = make(map[string]string) } @@ -99,10 +88,10 @@ func (f *Framework) EnsureIngress(ingress *networking.Ingress) *networking.Ingre // UpdateIngress updates an Ingress object and returns the updated object. func (f *Framework) UpdateIngress(ingress *networking.Ingress) *networking.Ingress { - err := updateIngressWithRetries(f.KubeClientSet, ingress.Namespace, ingress) + err := updateIngressWithRetries(f.KubeClientSet, f.Namespace, ingress) assert.Nil(ginkgo.GinkgoT(), err, "updating ingress") - ing := f.GetIngress(ingress.Namespace, ingress.Name) + ing := f.GetIngress(f.Namespace, ingress.Name) if ing.Annotations == nil { ing.Annotations = make(map[string]string) } @@ -113,26 +102,21 @@ func (f *Framework) UpdateIngress(ingress *networking.Ingress) *networking.Ingre return ing } -// GetService gets a Service object from the given namespace, name and returns it, throws error if it does not exist. -func (f *Framework) GetService(namespace, name string) *core.Service { - s, err := f.KubeClientSet.CoreV1().Services(namespace).Get(context.TODO(), name, metav1.GetOptions{}) +// EnsureService creates a Service object and returns it, throws error if it already exists. +func (f *Framework) EnsureService(service *core.Service) *core.Service { + err := createServiceWithRetries(f.KubeClientSet, f.Namespace, service) + assert.Nil(ginkgo.GinkgoT(), err, "creating service") + + s, err := f.KubeClientSet.CoreV1().Services(f.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "getting service") assert.NotNil(ginkgo.GinkgoT(), s, "expected a service but none returned") return s } -// EnsureService creates a Service object and returns it, throws error if it already exists. -func (f *Framework) EnsureService(service *core.Service) *core.Service { - err := createServiceWithRetries(f.KubeClientSet, service.Namespace, service) - assert.Nil(ginkgo.GinkgoT(), err, "creating service") - - return f.GetService(service.Namespace, service.Name) -} - // EnsureDeployment creates a Deployment object and returns it, throws error if it already exists. func (f *Framework) EnsureDeployment(deployment *appsv1.Deployment) *appsv1.Deployment { - err := createDeploymentWithRetries(f.KubeClientSet, deployment.Namespace, deployment) + err := createDeploymentWithRetries(f.KubeClientSet, f.Namespace, deployment) assert.Nil(ginkgo.GinkgoT(), err, "creating deployment") d, err := f.KubeClientSet.AppsV1().Deployments(deployment.Namespace).Get(context.TODO(), deployment.Name, metav1.GetOptions{}) @@ -143,21 +127,16 @@ func (f *Framework) EnsureDeployment(deployment *appsv1.Deployment) *appsv1.Depl } // waitForPodsReady waits for a given amount of time until a group of Pods is running in the given namespace. -func waitForPodsReady(kubeClientSet kubernetes.Interface, timeout time.Duration, expectedReplicas int, namespace string, opts *metav1.ListOptions) error { - //nolint:staticcheck // TODO: will replace it since wait.PollImmediate is deprecated +func waitForPodsReady(kubeClientSet kubernetes.Interface, timeout time.Duration, expectedReplicas int, namespace string, opts metav1.ListOptions) error { return wait.PollImmediate(1*time.Second, timeout, func() (bool, error) { - pl, err := kubeClientSet.CoreV1().Pods(namespace).List(context.TODO(), *opts) + pl, err := kubeClientSet.CoreV1().Pods(namespace).List(context.TODO(), opts) if err != nil { return false, nil } r := 0 - for i := range pl.Items { - isRunning, err := podRunningReady(&pl.Items[i]) - if err != nil { - Logf("error checking if pod is running : %v", err) - } - if isRunning { + for _, p := range pl.Items { + if isRunning, _ := podRunningReady(&p); isRunning { r++ } } @@ -171,10 +150,9 @@ func waitForPodsReady(kubeClientSet kubernetes.Interface, timeout time.Duration, } // waitForPodsDeleted waits for a given amount of time until a group of Pods are deleted in the given namespace. -func waitForPodsDeleted(kubeClientSet kubernetes.Interface, timeout time.Duration, namespace string, opts *metav1.ListOptions) error { - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated +func waitForPodsDeleted(kubeClientSet kubernetes.Interface, timeout time.Duration, namespace string, opts metav1.ListOptions) error { return wait.Poll(Poll, timeout, func() (bool, error) { - pl, err := kubeClientSet.CoreV1().Pods(namespace).List(context.TODO(), *opts) + pl, err := kubeClientSet.CoreV1().Pods(namespace).List(context.TODO(), opts) if err != nil { return false, nil } @@ -192,7 +170,7 @@ func WaitForEndpoints(kubeClientSet kubernetes.Interface, timeout time.Duration, if expectedEndpoints == 0 { return nil } - //nolint:staticcheck // TODO: will replace it since wait.PollImmediate is deprecated + return wait.PollImmediate(Poll, timeout, func() (bool, error) { endpoint, err := kubeClientSet.CoreV1().Endpoints(ns).Get(context.TODO(), name, metav1.GetOptions{}) if k8sErrors.IsNotFound(err) { @@ -254,7 +232,6 @@ func isPodReady(p *core.Pod) bool { // getIngressNGINXPod returns the ingress controller running pod func getIngressNGINXPod(ns string, kubeClientSet kubernetes.Interface) (*core.Pod, error) { var pod *core.Pod - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated err := wait.Poll(1*time.Second, DefaultTimeout, func() (bool, error) { l, err := kubeClientSet.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{ LabelSelector: "app.kubernetes.io/name=ingress-nginx", @@ -263,16 +240,15 @@ func getIngressNGINXPod(ns string, kubeClientSet kubernetes.Interface) (*core.Po return false, nil } - for i := range l.Items { - p := &l.Items[i] + for _, p := range l.Items { if strings.HasPrefix(p.GetName(), "nginx-ingress-controller") { - isRunning, err := podRunningReady(p) + isRunning, err := podRunningReady(&p) if err != nil { continue } if isRunning { - pod = p + pod = &p return true, nil } } @@ -281,7 +257,6 @@ func getIngressNGINXPod(ns string, kubeClientSet kubernetes.Interface) (*core.Po return false, nil }) if err != nil { - //nolint:staticcheck // TODO: will replace it since wait.ErrWaitTimeout is deprecated if err == wait.ErrWaitTimeout { return nil, fmt.Errorf("timeout waiting at least one ingress-nginx pod running in namespace %v", ns) } @@ -294,7 +269,7 @@ func getIngressNGINXPod(ns string, kubeClientSet kubernetes.Interface) (*core.Po func createDeploymentWithRetries(c kubernetes.Interface, namespace string, obj *appsv1.Deployment) error { if obj == nil { - return fmt.Errorf("object provided to create is empty") + return fmt.Errorf("Object provided to create is empty") } createFunc := func() (bool, error) { _, err := c.AppsV1().Deployments(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) @@ -307,15 +282,15 @@ func createDeploymentWithRetries(c kubernetes.Interface, namespace string, obj * if isRetryableAPIError(err) { return false, nil } - return false, fmt.Errorf("failed to create object with non-retriable error: %v", err) + return false, fmt.Errorf("Failed to create object with non-retriable error: %v", err) } return retryWithExponentialBackOff(createFunc) } -func createSecretWithRetries(c kubernetes.Interface, namespace string, obj *core.Secret) error { +func createSecretWithRetries(c kubernetes.Interface, namespace string, obj *v1.Secret) error { if obj == nil { - return fmt.Errorf("object provided to create is empty") + return fmt.Errorf("Object provided to create is empty") } createFunc := func() (bool, error) { _, err := c.CoreV1().Secrets(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) @@ -328,14 +303,14 @@ func createSecretWithRetries(c kubernetes.Interface, namespace string, obj *core if isRetryableAPIError(err) { return false, nil } - return false, fmt.Errorf("failed to create object with non-retriable error: %v", err) + return false, fmt.Errorf("Failed to create object with non-retriable error: %v", err) } return retryWithExponentialBackOff(createFunc) } -func createServiceWithRetries(c kubernetes.Interface, namespace string, obj *core.Service) error { +func createServiceWithRetries(c kubernetes.Interface, namespace string, obj *v1.Service) error { if obj == nil { - return fmt.Errorf("object provided to create is empty") + return fmt.Errorf("Object provided to create is empty") } createFunc := func() (bool, error) { _, err := c.CoreV1().Services(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) @@ -348,7 +323,7 @@ func createServiceWithRetries(c kubernetes.Interface, namespace string, obj *cor if isRetryableAPIError(err) { return false, nil } - return false, fmt.Errorf("failed to create object with non-retriable error: %v", err) + return false, fmt.Errorf("Failed to create object with non-retriable error: %v", err) } return retryWithExponentialBackOff(createFunc) @@ -356,10 +331,10 @@ func createServiceWithRetries(c kubernetes.Interface, namespace string, obj *cor func createIngressWithRetries(c kubernetes.Interface, namespace string, obj *networking.Ingress) error { if obj == nil { - return fmt.Errorf("object provided to create is empty") + return fmt.Errorf("Object provided to create is empty") } createFunc := func() (bool, error) { - _, err := c.NetworkingV1().Ingresses(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + _, err := c.NetworkingV1beta1().Ingresses(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) if err == nil { return true, nil } @@ -369,7 +344,7 @@ func createIngressWithRetries(c kubernetes.Interface, namespace string, obj *net if isRetryableAPIError(err) { return false, nil } - return false, fmt.Errorf("failed to create object with non-retriable error: %v", err) + return false, fmt.Errorf("Failed to create object with non-retriable error: %v", err) } return retryWithExponentialBackOff(createFunc) @@ -377,17 +352,17 @@ func createIngressWithRetries(c kubernetes.Interface, namespace string, obj *net func updateIngressWithRetries(c kubernetes.Interface, namespace string, obj *networking.Ingress) error { if obj == nil { - return fmt.Errorf("object provided to create is empty") + return fmt.Errorf("Object provided to create is empty") } updateFunc := func() (bool, error) { - _, err := c.NetworkingV1().Ingresses(namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) + _, err := c.NetworkingV1beta1().Ingresses(namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}) if err == nil { return true, nil } if isRetryableAPIError(err) { return false, nil } - return false, fmt.Errorf("failed to update object with non-retriable error: %v", err) + return false, fmt.Errorf("Failed to update object with non-retriable error: %v", err) } return retryWithExponentialBackOff(updateFunc) diff --git a/test/e2e/framework/logs.go b/test/e2e/framework/logs.go index db3eb4a6c..a4b645695 100644 --- a/test/e2e/framework/logs.go +++ b/test/e2e/framework/logs.go @@ -18,15 +18,12 @@ package framework import ( "context" - "time" "k8s.io/client-go/kubernetes" ) // Logs returns the log entries of a given Pod. func Logs(client kubernetes.Interface, namespace, podName string) (string, error) { - // Logs from jails take a bigger time to get shipped due to the need of tailing them - Sleep(3 * time.Second) logs, err := client.CoreV1().RESTClient().Get(). Resource("pods"). Namespace(namespace). diff --git a/test/e2e/framework/metrics.go b/test/e2e/framework/metrics.go index 774f1bd7e..349eb4dc3 100644 --- a/test/e2e/framework/metrics.go +++ b/test/e2e/framework/metrics.go @@ -29,7 +29,7 @@ func (f *Framework) GetMetric(metricName, ip string) (*dto.MetricFamily, error) url := fmt.Sprintf("http://%v:10254/metrics", ip) client := &http.Client{} - req, err := http.NewRequest(http.MethodGet, url, http.NoBody) + req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, fmt.Errorf("creating GET request for URL %q failed: %v", url, err) } @@ -44,6 +44,7 @@ func (f *Framework) GetMetric(metricName, ip string) (*dto.MetricFamily, error) var parser expfmt.TextParser metrics, err := parser.TextToMetricFamilies(resp.Body) + if err != nil { return nil, fmt.Errorf("reading text format failed: %v", err) } diff --git a/test/e2e/framework/ssl.go b/test/e2e/framework/ssl.go index d6252fe47..1eea2be2b 100644 --- a/test/e2e/framework/ssl.go +++ b/test/e2e/framework/ssl.go @@ -33,7 +33,7 @@ import ( "strings" "time" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -93,8 +93,8 @@ func CreateIngressTLSSecret(client kubernetes.Interface, hosts []string, secretN // CreateIngressMASecret creates or updates a Secret containing a Mutual Auth // certificate-chain for the given Ingress and returns a TLS configuration suitable // for HTTP clients to use against that particular Ingress. -func CreateIngressMASecret(client kubernetes.Interface, host, secretName, namespace string) (*tls.Config, error) { - if host == "" { +func CreateIngressMASecret(client kubernetes.Interface, host string, secretName, namespace string) (*tls.Config, error) { + if len(host) == 0 { return nil, fmt.Errorf("requires a non-empty host") } @@ -138,13 +138,12 @@ func CreateIngressMASecret(client kubernetes.Interface, host, secretName, namesp return &tls.Config{ ServerName: host, Certificates: []tls.Certificate{clientPair}, - InsecureSkipVerify: true, //nolint:gosec // Ignore the gosec error in testing + InsecureSkipVerify: true, }, nil } // WaitForTLS waits until the TLS handshake with a given server completes successfully. func WaitForTLS(url string, tlsConfig *tls.Config) { - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated err := wait.Poll(Poll, DefaultTimeout, matchTLSServerName(url, tlsConfig)) assert.Nil(ginkgo.GinkgoT(), err, "waiting for TLS configuration in URL %s", url) } @@ -161,6 +160,7 @@ func generateRSACert(host string, isCA bool, keyOut, certOut io.Writer) error { serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { return fmt.Errorf("failed to generate serial number: %s", err) } @@ -329,7 +329,7 @@ func tlsConfig(serverName string, pemCA []byte) (*tls.Config, error) { if !rootCAPool.AppendCertsFromPEM(pemCA) { return nil, fmt.Errorf("error creating CA certificate pool (%s)", serverName) } - return &tls.Config{ //nolint:gosec // Ignore the gosec error in testing + return &tls.Config{ ServerName: serverName, RootCAs: rootCAPool, }, nil diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 961357885..0a79c894f 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -18,12 +18,14 @@ package framework import ( "flag" + + "github.com/onsi/ginkgo/config" ) // TestContextType describes the client context to use in communications with the Kubernetes API. type TestContextType struct { KubeHost string - // KubeConfig string + //KubeConfig string KubeContext string } @@ -32,7 +34,10 @@ var TestContext TestContextType // registerCommonFlags registers flags common to all e2e test suites. func registerCommonFlags() { + config.GinkgoConfig.EmitSpecProgress = true + flag.StringVar(&TestContext.KubeHost, "kubernetes-host", "http://127.0.0.1:8080", "The kubernetes host, or apiserver, to connect to") + //flag.StringVar(&TestContext.KubeConfig, "kubernetes-config", os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to config containing embedded authinfo for kubernetes. Default value is from environment variable "+clientcmd.RecommendedConfigPathEnvVar) flag.StringVar(&TestContext.KubeContext, "kubernetes-context", "", "config context to use for kubernetes. If unset, will use value from 'current-context'") } diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index abdb168e1..e094166f1 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -22,10 +22,8 @@ import ( "os" "time" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" - rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" @@ -33,8 +31,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/clientcmd/api" - - "k8s.io/ingress-nginx/internal/k8s" ) const ( @@ -42,31 +38,31 @@ const ( Poll = 2 * time.Second // DefaultTimeout time to wait for operations to complete - DefaultTimeout = 90 * time.Second + DefaultTimeout = 5 * time.Minute ) func nowStamp() string { return time.Now().Format(time.StampMilli) } -func logf(level, format string, args ...interface{}) { +func log(level string, format string, args ...interface{}) { fmt.Fprintf(ginkgo.GinkgoWriter, nowStamp()+": "+level+": "+format+"\n", args...) } // Logf logs to the INFO logs. func Logf(format string, args ...interface{}) { - logf("INFO", format, args...) + log("INFO", format, args...) } // Failf logs to the INFO logs and fails the test. func Failf(format string, args ...interface{}) { msg := fmt.Sprintf(format, args...) - logf("INFO", msg) + log("INFO", msg) ginkgo.Fail(nowStamp()+": "+msg, 1) } // RestclientConfig deserializes the contents of a kubeconfig file into a Config object. -func RestclientConfig(config, newContext string) (*api.Config, error) { +func RestclientConfig(config, context string) (*api.Config, error) { Logf(">>> config: %s\n", config) if config == "" { return nil, fmt.Errorf("config file must be specified to load client config") @@ -75,9 +71,9 @@ func RestclientConfig(config, newContext string) (*api.Config, error) { if err != nil { return nil, fmt.Errorf("error loading config: %v", err.Error()) } - if newContext != "" { - Logf(">>> context: %s\n", newContext) - c.CurrentContext = newContext + if context != "" { + Logf(">>> context: %s\n", context) + c.CurrentContext = context } return c, nil } @@ -85,20 +81,18 @@ func RestclientConfig(config, newContext string) (*api.Config, error) { // RunID unique identifier of the e2e run var RunID = uuid.NewUUID() -func createNamespace(baseName string, labels map[string]string, c kubernetes.Interface) (string, error) { +// CreateKubeNamespace creates a new namespace in the cluster +func CreateKubeNamespace(baseName string, c kubernetes.Interface) (string, error) { ts := time.Now().UnixNano() ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ GenerateName: fmt.Sprintf("e2e-tests-%v-%v-", baseName, ts), - Labels: labels, }, } - // Be robust about making the namespace creation call. var got *corev1.Namespace var err error - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated err = wait.Poll(Poll, DefaultTimeout, func() (bool, error) { got, err = c.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) if err != nil { @@ -113,18 +107,8 @@ func createNamespace(baseName string, labels map[string]string, c kubernetes.Int return got.Name, nil } -// CreateKubeNamespace creates a new namespace in the cluster -func CreateKubeNamespace(baseName string, c kubernetes.Interface) (string, error) { - return createNamespace(baseName, nil, c) -} - -// CreateKubeNamespaceWithLabel creates a new namespace with given labels in the cluster -func CreateKubeNamespaceWithLabel(baseName string, labels map[string]string, c kubernetes.Interface) (string, error) { - return createNamespace(baseName, labels, c) -} - -// DeleteKubeNamespace deletes a namespace and all the objects inside -func DeleteKubeNamespace(c kubernetes.Interface, namespace string) error { +// deleteKubeNamespace deletes a namespace and all the objects inside +func deleteKubeNamespace(c kubernetes.Interface, namespace string) error { grace := int64(0) pb := metav1.DeletePropagationBackground return c.CoreV1().Namespaces().Delete(context.TODO(), namespace, metav1.DeleteOptions{ @@ -133,96 +117,8 @@ func DeleteKubeNamespace(c kubernetes.Interface, namespace string) error { }) } -// CreateIngressClass creates a new IngressClass related to a test/namespace and also -// the required ClusterRole/ClusterRoleBinding -func CreateIngressClass(namespace string, c kubernetes.Interface) (string, error) { - icname := fmt.Sprintf("ic-%s", namespace) - var err error - - ic, err := c.NetworkingV1().IngressClasses(). - Create(context.TODO(), &networkingv1.IngressClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: icname, - }, - Spec: networkingv1.IngressClassSpec{ - Controller: k8s.IngressNGINXController, - }, - }, metav1.CreateOptions{}) - if err != nil { - return "", fmt.Errorf("unexpected error creating IngressClass %s: %v", icname, err) - } - - _, err = c.RbacV1().ClusterRoles().Create(context.TODO(), &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{Name: icname}, - Rules: []rbacv1.PolicyRule{{ - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"ingressclasses"}, - Verbs: []string{"get", "list", "watch"}, - }}, - }, metav1.CreateOptions{}) - if err != nil { - return "", fmt.Errorf("unexpected error creating IngressClass ClusterRole %s: %v", icname, err) - } - - _, err = c.RbacV1().ClusterRoleBindings().Create(context.TODO(), &rbacv1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: icname, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: icname, - }, - Subjects: []rbacv1.Subject{ - { - APIGroup: "", - Kind: "ServiceAccount", - Namespace: namespace, - Name: "nginx-ingress", - }, - }, - }, metav1.CreateOptions{}) - if err != nil { - return "", fmt.Errorf("unexpected error creating IngressClass ClusterRoleBinding %s: %v", icname, err) - } - return ic.Name, nil -} - -// deleteIngressClass deletes an IngressClass and its related ClusterRole* objects -func deleteIngressClass(c kubernetes.Interface, ingressclass string) error { - var err error - grace := int64(0) - pb := metav1.DeletePropagationBackground - deleteOptions := metav1.DeleteOptions{ - GracePeriodSeconds: &grace, - PropagationPolicy: &pb, - } - err = c.NetworkingV1().IngressClasses().Delete(context.TODO(), ingressclass, deleteOptions) - if err != nil { - return fmt.Errorf("unexpected error deleting IngressClass %s: %v", ingressclass, err) - } - - err = c.RbacV1().ClusterRoleBindings().Delete(context.TODO(), ingressclass, deleteOptions) - if err != nil { - return fmt.Errorf("unexpected error deleting IngressClass ClusterRoleBinding %s: %v", ingressclass, err) - } - err = c.RbacV1().ClusterRoles().Delete(context.TODO(), ingressclass, deleteOptions) - if err != nil { - return fmt.Errorf("unexpected error deleting IngressClass ClusterRole %s: %v", ingressclass, err) - } - - return nil -} - -// GetIngressClassName returns the default IngressClassName given a namespace -func GetIngressClassName(namespace string) *string { - icname := fmt.Sprintf("ic-%s", namespace) - return &icname -} - // WaitForKubeNamespaceNotExist waits until a namespaces is not present in the cluster func WaitForKubeNamespaceNotExist(c kubernetes.Interface, namespace string) error { - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated return wait.Poll(Poll, DefaultTimeout, namespaceNotExist(c, namespace)) } @@ -241,7 +137,6 @@ func namespaceNotExist(c kubernetes.Interface, namespace string) wait.ConditionF // WaitForNoPodsInNamespace waits until there are no pods running in a namespace func WaitForNoPodsInNamespace(c kubernetes.Interface, namespace string) error { - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated return wait.Poll(Poll, DefaultTimeout, noPodsInNamespace(c, namespace)) } @@ -268,17 +163,15 @@ func WaitForPodRunningInNamespace(c kubernetes.Interface, pod *corev1.Pod) error if pod.Status.Phase == corev1.PodRunning { return nil } - return waitTimeoutForPodRunningInNamespace(c, pod.Name, pod.Namespace) + return waitTimeoutForPodRunningInNamespace(c, pod.Name, pod.Namespace, DefaultTimeout) } -func waitTimeoutForPodRunningInNamespace(c kubernetes.Interface, podName, namespace string) error { - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated +func waitTimeoutForPodRunningInNamespace(c kubernetes.Interface, podName, namespace string, timeout time.Duration) error { return wait.Poll(Poll, DefaultTimeout, podRunning(c, podName, namespace)) } // WaitForSecretInNamespace waits a default amount of time for the specified secret is present in a particular namespace func WaitForSecretInNamespace(c kubernetes.Interface, namespace, name string) error { - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated return wait.Poll(Poll, DefaultTimeout, secretInNamespace(c, namespace, name)) } @@ -301,7 +194,6 @@ func secretInNamespace(c kubernetes.Interface, namespace, name string) wait.Cond // WaitForFileInFS waits a default amount of time for the specified file is present in the filesystem func WaitForFileInFS(file string) error { - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated return wait.Poll(Poll, DefaultTimeout, fileInFS(file)) } @@ -326,13 +218,12 @@ func fileInFS(file string) wait.ConditionFunc { // WaitForNoIngressInNamespace waits until there is no ingress object in a particular namespace func WaitForNoIngressInNamespace(c kubernetes.Interface, namespace, name string) error { - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated return wait.Poll(Poll, DefaultTimeout, noIngressInNamespace(c, namespace, name)) } func noIngressInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc { return func() (bool, error) { - ing, err := c.NetworkingV1().Ingresses(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + ing, err := c.NetworkingV1beta1().Ingresses(namespace).Get(context.TODO(), name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { return true, nil } @@ -349,13 +240,12 @@ func noIngressInNamespace(c kubernetes.Interface, namespace, name string) wait.C // WaitForIngressInNamespace waits until a particular ingress object exists namespace func WaitForIngressInNamespace(c kubernetes.Interface, namespace, name string) error { - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated return wait.Poll(Poll, DefaultTimeout, ingressInNamespace(c, namespace, name)) } func ingressInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc { return func() (bool, error) { - ing, err := c.NetworkingV1().Ingresses(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + ing, err := c.NetworkingV1beta1().Ingresses(namespace).Get(context.TODO(), name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { return false, nil } diff --git a/test/e2e/gracefulshutdown/grace_period.go b/test/e2e/gracefulshutdown/grace_period.go index 123892f3a..95c3729a1 100644 --- a/test/e2e/gracefulshutdown/grace_period.go +++ b/test/e2e/gracefulshutdown/grace_period.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,6 +33,7 @@ var _ = framework.IngressNginxDescribe("[Shutdown] Grace period shutdown", func( f := framework.NewDefaultFramework("shutdown-grace-period") ginkgo.It("/healthz should return status code 500 during shutdown grace period", func() { + f.NewSlowEchoDeployment() err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { @@ -82,5 +83,6 @@ var _ = framework.IngressNginxDescribe("[Shutdown] Grace period shutdown", func( for _, err := range <-result { assert.Nil(ginkgo.GinkgoT(), err) } + }) }) diff --git a/test/e2e/gracefulshutdown/shutdown.go b/test/e2e/gracefulshutdown/shutdown.go index e9883338f..c3c0d5234 100644 --- a/test/e2e/gracefulshutdown/shutdown.go +++ b/test/e2e/gracefulshutdown/shutdown.go @@ -17,12 +17,15 @@ limitations under the License. package gracefulshutdown import ( + "context" "net/http" "strings" "time" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -37,7 +40,7 @@ var _ = framework.IngressNginxDescribe("[Shutdown] ingress controller", func() { f.NewSlowEchoDeployment() }) - ginkgo.It("should shutdown in less than 60 seconds without pending connections", func() { + ginkgo.It("should shutdown in less than 60 secons without pending connections", func() { f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.Namespace, framework.SlowEchoService, 80, nil)) f.WaitForNginxServer(host, @@ -57,7 +60,7 @@ var _ = framework.IngressNginxDescribe("[Shutdown] ingress controller", func() { assert.LessOrEqual(ginkgo.GinkgoT(), int(time.Since(startTime).Seconds()), 60, "waiting shutdown") }) - /* @rikatz - Removing this tests as they are failing in GH Actions but not locally. + ginkgo.It("should shutdown after waiting 60 seconds for pending connections to be closed", func(done ginkgo.Done) { defer close(done) @@ -146,5 +149,5 @@ var _ = framework.IngressNginxDescribe("[Shutdown] ingress controller", func() { statusCode := <-result assert.Equal(ginkgo.GinkgoT(), http.StatusOK, statusCode, "expecting a valid response from HTTP request") assert.GreaterOrEqual(ginkgo.GinkgoT(), int(time.Since(startTime).Seconds()), 150, "waiting shutdown") - }, 200) */ + }, 200) }) diff --git a/test/e2e/gracefulshutdown/slow_requests.go b/test/e2e/gracefulshutdown/slow_requests.go index 57578923c..c05ff7b87 100644 --- a/test/e2e/gracefulshutdown/slow_requests.go +++ b/test/e2e/gracefulshutdown/slow_requests.go @@ -17,7 +17,10 @@ limitations under the License. package gracefulshutdown import ( - "github.com/onsi/ginkgo/v2" + "net/http" + "strings" + + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -29,7 +32,7 @@ var _ = framework.IngressNginxDescribe("[Shutdown] Graceful shutdown with pendin f.NewSlowEchoDeployment() f.UpdateNginxConfigMapData("worker-shutdown-timeout", "50s") }) - /* @rikatz - This seems to be failing on GH Actions and needs to be re-checked and re-verified + ginkgo.It("should let slow requests finish before shutting down", func() { host := "graceful-shutdown" @@ -54,5 +57,5 @@ var _ = framework.IngressNginxDescribe("[Shutdown] Graceful shutdown with pendin framework.Sleep() f.DeleteNGINXPod(60) <-done - }) */ + }) }) diff --git a/test/e2e/ingress/deep_inspection.go b/test/e2e/ingress/deep_inspection.go deleted file mode 100644 index 8869665fb..000000000 --- a/test/e2e/ingress/deep_inspection.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ingress - -import ( - "net/http" - "strings" - - "github.com/onsi/ginkgo/v2" - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.IngressNginxDescribe("[Ingress] DeepInspection", func() { - f := framework.NewDefaultFramework("deep-inspection") - - ginkgo.BeforeEach(func() { - f.NewEchoDeployment() - }) - - ginkgo.It("should drop whole ingress if one path matches invalid regex", func() { - host := "inspection123.com" - - ingInvalid := framework.NewSingleIngress("invalidregex", "/bla{alias /var/run/secrets/;}location ~* ^/abcd", host, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ingInvalid) - ingValid := framework.NewSingleIngress("valid", "/xpto", host, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ingValid) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, host) && - strings.Contains(server, "location /xpto") && - !strings.Contains(server, "location /bla") - }) - - f.HTTPTestClient(). - GET("/xpto"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/bla"). - WithHeader("Host", host). - Expect(). - Status(http.StatusNotFound) - - f.HTTPTestClient(). - GET("/abcd/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusNotFound) - }) -}) diff --git a/test/e2e/ingress/multiple_rules.go b/test/e2e/ingress/multiple_rules.go index 9247dc1d3..dd5abd7ae 100644 --- a/test/e2e/ingress/multiple_rules.go +++ b/test/e2e/ingress/multiple_rules.go @@ -20,46 +20,39 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" - networking "k8s.io/api/networking/v1" + networkingv1beta1 "k8s.io/api/networking/v1beta1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/test/e2e/framework" ) var _ = framework.IngressNginxDescribe("single ingress - multiple hosts", func() { f := framework.NewDefaultFramework("simh") - pathprefix := networking.PathTypePrefix + ginkgo.BeforeEach(func() { - f.NewEchoDeployment(framework.WithDeploymentName("first-service")) - f.NewEchoDeployment(framework.WithDeploymentName("second-service")) + f.NewEchoDeploymentWithNameAndReplicas("first-service", 1) + f.NewEchoDeploymentWithNameAndReplicas("second-service", 1) }) ginkgo.It("should set the correct $service_name NGINX variable", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - annotations := map[string]string{ "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_input_headers "service-name: $service_name";`, } ing := framework.NewSingleIngress("simh", "/", "first.host", f.Namespace, "first-service", 80, annotations) - ing.Spec.Rules = append(ing.Spec.Rules, networking.IngressRule{ + ing.Spec.Rules = append(ing.Spec.Rules, networkingv1beta1.IngressRule{ Host: "second.host", - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ + IngressRuleValue: networkingv1beta1.IngressRuleValue{ + HTTP: &networkingv1beta1.HTTPIngressRuleValue{ + Paths: []networkingv1beta1.HTTPIngressPath{ { - Path: "/", - PathType: &pathprefix, - Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: "second-service", - Port: networking.ServiceBackendPort{ - Number: int32(80), - }, - }, + Path: "/", + Backend: networkingv1beta1.IngressBackend{ + ServiceName: "second-service", + ServicePort: intstr.FromInt(80), }, }, }, diff --git a/test/e2e/ingress/pathtype_exact.go b/test/e2e/ingress/pathtype_exact.go index 2660e32a4..01e37836a 100644 --- a/test/e2e/ingress/pathtype_exact.go +++ b/test/e2e/ingress/pathtype_exact.go @@ -20,9 +20,9 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" - networking "k8s.io/api/networking/v1" + networkingv1beta1 "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -35,8 +35,9 @@ var _ = framework.IngressNginxDescribe("[Ingress] [PathType] exact", func() { }) ginkgo.It("should choose exact location for /exact", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() + if !f.IsIngressV1Beta1Ready { + ginkgo.Skip("Test requires Kubernetes v1.18 or higher") + } host := "exact.path" @@ -44,7 +45,7 @@ var _ = framework.IngressNginxDescribe("[Ingress] [PathType] exact", func() { "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_input_headers "pathType: exact";`, } - exactPathType := networking.PathTypeExact + var exactPathType = networkingv1beta1.PathTypeExact ing := framework.NewSingleIngress("exact", "/exact", host, f.Namespace, framework.EchoService, 80, annotations) ing.Spec.Rules[0].IngressRuleValue.HTTP.Paths[0].PathType = &exactPathType f.EnsureIngress(ing) diff --git a/test/e2e/ingress/pathtype_mixed.go b/test/e2e/ingress/pathtype_mixed.go index 3212089c9..aac7d9ffa 100644 --- a/test/e2e/ingress/pathtype_mixed.go +++ b/test/e2e/ingress/pathtype_mixed.go @@ -20,10 +20,10 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" - networking "k8s.io/api/networking/v1" + networkingv1beta1 "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -34,23 +34,24 @@ var _ = framework.IngressNginxDescribe("[Ingress] [PathType] mix Exact and Prefi f.NewEchoDeployment() }) - exactPathType := networking.PathTypeExact + var exactPathType = networkingv1beta1.PathTypeExact ginkgo.It("should choose the correct location", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() + if !f.IsIngressV1Beta1Ready { + ginkgo.Skip("Test requires Kubernetes v1.18 or higher") + } host := "mixed.path" annotations := map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_input_headers "pathType: exact";more_set_input_headers "pathheader: /";`, + "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_input_headers "pathType: exact";more_set_input_headers "pathlocation: /";`, } ing := framework.NewSingleIngress("exact-root", "/", host, f.Namespace, framework.EchoService, 80, annotations) ing.Spec.Rules[0].IngressRuleValue.HTTP.Paths[0].PathType = &exactPathType f.EnsureIngress(ing) annotations = map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_input_headers "pathType: prefix";more_set_input_headers "pathheader: /";`, + "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_input_headers "pathType: prefix";more_set_input_headers "pathlocation: /";`, } ing = framework.NewSingleIngress("prefix-root", "/", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -73,7 +74,7 @@ var _ = framework.IngressNginxDescribe("[Ingress] [PathType] mix Exact and Prefi assert.NotContains(ginkgo.GinkgoT(), body, "pathtype=prefix") assert.Contains(ginkgo.GinkgoT(), body, "pathtype=exact") - assert.Contains(ginkgo.GinkgoT(), body, "pathheader=/") + assert.Contains(ginkgo.GinkgoT(), body, "pathlocation=/") ginkgo.By("Checking prefix request to /bar") body = f.HTTPTestClient(). @@ -86,17 +87,17 @@ var _ = framework.IngressNginxDescribe("[Ingress] [PathType] mix Exact and Prefi assert.Contains(ginkgo.GinkgoT(), body, "pathtype=prefix") assert.NotContains(ginkgo.GinkgoT(), body, "pathtype=exact") - assert.Contains(ginkgo.GinkgoT(), body, "pathheader=/") + assert.Contains(ginkgo.GinkgoT(), body, "pathlocation=/") annotations = map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_input_headers "pathType: exact";more_set_input_headers "pathheader: /foo";`, + "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_input_headers "pathType: exact";more_set_input_headers "pathlocation: /foo";`, } ing = framework.NewSingleIngress("exact-foo", "/foo", host, f.Namespace, framework.EchoService, 80, annotations) ing.Spec.Rules[0].IngressRuleValue.HTTP.Paths[0].PathType = &exactPathType f.EnsureIngress(ing) annotations = map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_input_headers "pathType: prefix";more_set_input_headers "pathheader: /foo";`, + "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_input_headers "pathType: prefix";more_set_input_headers "pathlocation: /foo";`, } ing = framework.NewSingleIngress("prefix-foo", "/foo", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -119,7 +120,7 @@ var _ = framework.IngressNginxDescribe("[Ingress] [PathType] mix Exact and Prefi assert.NotContains(ginkgo.GinkgoT(), body, "pathtype=prefix") assert.Contains(ginkgo.GinkgoT(), body, "pathtype=exact") - assert.Contains(ginkgo.GinkgoT(), body, "pathheader=/foo") + assert.Contains(ginkgo.GinkgoT(), body, "pathlocation=/foo") ginkgo.By("Checking prefix request to /foo/bar") body = f.HTTPTestClient(). @@ -131,7 +132,7 @@ var _ = framework.IngressNginxDescribe("[Ingress] [PathType] mix Exact and Prefi Raw() assert.Contains(ginkgo.GinkgoT(), body, "pathtype=prefix") - assert.Contains(ginkgo.GinkgoT(), body, "pathheader=/foo") + assert.Contains(ginkgo.GinkgoT(), body, "pathlocation=/foo") ginkgo.By("Checking prefix request to /foobar") body = f.HTTPTestClient(). @@ -143,6 +144,6 @@ var _ = framework.IngressNginxDescribe("[Ingress] [PathType] mix Exact and Prefi Raw() assert.Contains(ginkgo.GinkgoT(), body, "pathtype=prefix") - assert.Contains(ginkgo.GinkgoT(), body, "pathheader=/") + assert.Contains(ginkgo.GinkgoT(), body, "pathlocation=/") }) }) diff --git a/test/e2e/ingress/pathtype_prefix.go b/test/e2e/ingress/pathtype_prefix.go index ce11ca8bf..c2c7b0f8d 100644 --- a/test/e2e/ingress/pathtype_prefix.go +++ b/test/e2e/ingress/pathtype_prefix.go @@ -20,7 +20,7 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -68,138 +68,4 @@ var _ = framework.IngressNginxDescribe("[Ingress] [PathType] prefix checks", fun Expect(). Status(http.StatusOK) }) - - ginkgo.It("should test prefix path using simple regex pattern for /id/{int}", func() { - host := "echo.com.br" - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/use-regex": `true`, - } - - ing := framework.NewSingleIngress(host, "/id/[0-9]+", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.HTTPTestClient(). - GET("/id/1"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/id/12"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/id/123"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/id/aaa"). - WithHeader("Host", host). - Expect(). - Status(http.StatusNotFound) - - f.HTTPTestClient(). - GET("/id/123a"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - }) - - ginkgo.It("should test prefix path using regex pattern for /id/{int} ignoring non-digits characters at end of string", func() { - host := "echo.regex.br" - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/use-regex": `true`, - } - - ing := framework.NewSingleIngress(host, "/id/[0-9]+$", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.HTTPTestClient(). - GET("/id/1"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/id/aaa"). - WithHeader("Host", host). - Expect(). - Status(http.StatusNotFound) - - f.HTTPTestClient(). - GET("/id/123a"). - WithHeader("Host", host). - Expect(). - Status(http.StatusNotFound) - }) - - ginkgo.It("should test prefix path using fixed path size regex pattern /id/{int}{3}", func() { - host := "echo.regex.size.br" - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/use-regex": `true`, - } - - ing := framework.NewSingleIngress(host, "/id/[0-9]{3}$", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.HTTPTestClient(). - GET("/id/99"). - WithHeader("Host", host). - Expect(). - Status(http.StatusNotFound) - - f.HTTPTestClient(). - GET("/id/123"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/id/9999"). - WithHeader("Host", host). - Expect(). - Status(http.StatusNotFound) - - f.HTTPTestClient(). - GET("/id/123a"). - WithHeader("Host", host). - Expect(). - Status(http.StatusNotFound) - }) - - ginkgo.It("should correctly route multi-segment path patterns", func() { - host := "echo.multi.segment.br" - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/use-regex": `true`, - } - - ing := framework.NewSingleIngress(host, "/id/[0-9]+/post/[a-zA-Z]+$", host, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing) - - f.HTTPTestClient(). - GET("/id/123/post/abc"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/id/123/post/abc123"). - WithHeader("Host", host). - Expect(). - Status(http.StatusNotFound) - - f.HTTPTestClient(). - GET("/id/abc/post/abc"). - WithHeader("Host", host). - Expect(). - Status(http.StatusNotFound) - }) }) diff --git a/test/e2e/ingress/without_host.go b/test/e2e/ingress/without_host.go index 38f89beda..154767fa4 100644 --- a/test/e2e/ingress/without_host.go +++ b/test/e2e/ingress/without_host.go @@ -21,9 +21,10 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" - networking "k8s.io/api/networking/v1" + "github.com/onsi/ginkgo" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -61,14 +62,9 @@ var _ = framework.IngressNginxDescribe("[Ingress] definition without host", func Namespace: f.Namespace, }, Spec: networking.IngressSpec{ - IngressClassName: &f.IngressClass, - DefaultBackend: &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: framework.EchoService, - Port: networking.ServiceBackendPort{ - Number: int32(80), - }, - }, + Backend: &networking.IngressBackend{ + ServiceName: framework.EchoService, + ServicePort: intstr.FromInt(80), }, Rules: []networking.IngressRule{ { @@ -93,13 +89,5 @@ var _ = framework.IngressNginxDescribe("[Ingress] definition without host", func WithHeader("Host", "only-backend"). Expect(). Status(http.StatusOK) - // Following assertion added with respect to issue https://github.com/kubernetes/ingress-nginx/issues/8823 - // This check ensure that ingress having defaultBackend with rules should only be added as default backend - // for the host mentioned in rule. It should not affect the default catch-all server_name _ block. - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", "only-backend-doesnotexist"). - Expect(). - Status(http.StatusNotFound) }) }) diff --git a/test/e2e/kind.yaml b/test/e2e/kind.yaml index 07a56dae8..6fddba4e2 100644 --- a/test/e2e/kind.yaml +++ b/test/e2e/kind.yaml @@ -2,14 +2,8 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - labels: - topology.kubernetes.io/zone: zone-1 - role: worker - labels: - topology.kubernetes.io/zone: zone-1 - role: worker - labels: - topology.kubernetes.io/zone: zone-2 kubeadmConfigPatches: - | kind: ClusterConfiguration @@ -19,3 +13,4 @@ kubeadmConfigPatches: extraArgs: namespace-sync-period: 10s concurrent-deployment-syncs: "30" + deployment-controller-sync-period: 10s diff --git a/test/e2e/leaks/lua_ssl.go b/test/e2e/leaks/lua_ssl.go index 88285ba4b..6eb8bbc63 100644 --- a/test/e2e/leaks/lua_ssl.go +++ b/test/e2e/leaks/lua_ssl.go @@ -24,7 +24,7 @@ import ( "strings" "time" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" pool "gopkg.in/go-playground/pool.v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -39,7 +39,7 @@ var _ = framework.IngressNginxDescribe("[Memory Leak] Dynamic Certificates", fun f.NewEchoDeployment() }) - ginkgo.It("should not leak memory from ingress SSL certificates or configuration updates", func() { + framework.MemoryLeakIt("should not leak memory from ingress SSL certificates or configuration updates", func() { hostCount := 1000 iterations := 10 @@ -87,14 +87,14 @@ func provisionIngress(hostname string, f *framework.Framework) { func checkIngress(hostname string, f *framework.Framework) { resp := f.HTTPTestClientWithTLSConfig(&tls.Config{ ServerName: hostname, - InsecureSkipVerify: true, //nolint:gosec // Ignore the gosec error in testing + InsecureSkipVerify: true, }). GET("/"). WithURL(f.GetURL(framework.HTTPS)). WithHeader("Host", hostname). Expect(). Raw() - defer resp.Body.Close() + assert.Equal(ginkgo.GinkgoT(), resp.StatusCode, http.StatusOK) // check the returned secret is not the fake one @@ -103,7 +103,7 @@ func checkIngress(hostname string, f *framework.Framework) { } func deleteIngress(hostname string, f *framework.Framework) { - err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Delete(context.TODO(), hostname, metav1.DeleteOptions{}) + err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Delete(context.TODO(), hostname, metav1.DeleteOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "unexpected error deleting ingress") } diff --git a/test/e2e/loadbalance/configmap.go b/test/e2e/loadbalance/configmap.go index 737cd06dd..ea1ead4fe 100644 --- a/test/e2e/loadbalance/configmap.go +++ b/test/e2e/loadbalance/configmap.go @@ -19,14 +19,12 @@ package loadbalance import ( "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" ) -const loadBalanceHost = "load-balance.com" - var _ = framework.DescribeSetting("[Load Balancer] load-balance", func() { f := framework.NewDefaultFramework("lb-configmap") @@ -35,7 +33,7 @@ var _ = framework.DescribeSetting("[Load Balancer] load-balance", func() { }) ginkgo.It("should apply the configmap load-balance setting", func() { - host := loadBalanceHost + host := "load-balance.com" f.UpdateNginxConfigMapData("load-balance", "ewma") diff --git a/test/e2e/loadbalance/ewma.go b/test/e2e/loadbalance/ewma.go index f457e6357..52e235569 100644 --- a/test/e2e/loadbalance/ewma.go +++ b/test/e2e/loadbalance/ewma.go @@ -22,7 +22,7 @@ import ( "regexp" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" @@ -32,16 +32,15 @@ var _ = framework.DescribeSetting("[Load Balancer] EWMA", func() { f := framework.NewDefaultFramework("ewma") ginkgo.BeforeEach(func() { - f.NewEchoDeployment(framework.WithDeploymentReplicas(3)) + f.NewEchoDeploymentWithReplicas(3) f.SetNginxConfigMapData(map[string]string{ "worker-processes": "2", - "load-balance": "ewma", - }, + "load-balance": "ewma"}, ) }) ginkgo.It("does not fail requests", func() { - host := loadBalanceHost + host := "load-balance.com" f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil)) f.WaitForNginxServer(host, @@ -53,9 +52,7 @@ var _ = framework.DescribeSetting("[Load Balancer] EWMA", func() { assert.Nil(ginkgo.GinkgoT(), err) assert.Equal(ginkgo.GinkgoT(), algorithm, "ewma") - re, err := regexp.Compile(fmt.Sprintf(`%v.*`, framework.EchoService)) - assert.Nil(ginkgo.GinkgoT(), err, "error compiling regex") - + re, _ := regexp.Compile(fmt.Sprintf(`%v.*`, framework.EchoService)) replicaRequestCount := map[string]int{} for i := 0; i < 30; i++ { diff --git a/test/e2e/loadbalance/round_robin.go b/test/e2e/loadbalance/round_robin.go index 5f6667143..9e37d1596 100644 --- a/test/e2e/loadbalance/round_robin.go +++ b/test/e2e/loadbalance/round_robin.go @@ -22,7 +22,7 @@ import ( "regexp" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" @@ -32,12 +32,12 @@ var _ = framework.DescribeSetting("[Load Balancer] round-robin", func() { f := framework.NewDefaultFramework("round-robin") ginkgo.BeforeEach(func() { - f.NewEchoDeployment(framework.WithDeploymentReplicas(3)) + f.NewEchoDeploymentWithReplicas(3) f.UpdateNginxConfigMapData("worker-processes", "1") }) ginkgo.It("should evenly distribute requests with round-robin (default algorithm)", func() { - host := loadBalanceHost + host := "load-balance.com" f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil)) f.WaitForNginxServer(host, @@ -45,9 +45,7 @@ var _ = framework.DescribeSetting("[Load Balancer] round-robin", func() { return strings.Contains(server, "server_name load-balance.com") }) - re, err := regexp.Compile(fmt.Sprintf(`%v.*`, framework.EchoService)) - assert.Nil(ginkgo.GinkgoT(), err, "error compiling regex") - + re, _ := regexp.Compile(fmt.Sprintf(`%v.*`, framework.EchoService)) replicaRequestCount := map[string]int{} for i := 0; i < 600; i++ { @@ -68,7 +66,7 @@ var _ = framework.DescribeSetting("[Load Balancer] round-robin", func() { } for _, v := range replicaRequestCount { - assert.Equal(ginkgo.GinkgoT(), 200, v) + assert.Equal(ginkgo.GinkgoT(), v, 200) } }) }) diff --git a/test/e2e/lua/dynamic_certificates.go b/test/e2e/lua/dynamic_certificates.go index 8c9df5e71..455bf0fc2 100644 --- a/test/e2e/lua/dynamic_certificates.go +++ b/test/e2e/lua/dynamic_certificates.go @@ -23,12 +23,12 @@ import ( "strings" "time" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" "github.com/prometheus/common/model" "github.com/stretchr/testify/assert" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/test/e2e/framework" @@ -45,7 +45,7 @@ var _ = framework.IngressNginxDescribe("[Lua] dynamic certificates", func() { ginkgo.It("picks up the certificate when we add TLS spec to existing ingress", func() { ensureIngress(f, host, framework.EchoService) - ing, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) + ing, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) assert.Nil(ginkgo.GinkgoT(), err) ing.Spec.TLS = []networking.IngressTLS{ { @@ -59,7 +59,7 @@ var _ = framework.IngressNginxDescribe("[Lua] dynamic certificates", func() { ing.Namespace) assert.Nil(ginkgo.GinkgoT(), err) - _, err = f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{}) + _, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{}) assert.Nil(ginkgo.GinkgoT(), err) time.Sleep(waitForLuaSync) @@ -147,7 +147,7 @@ var _ = framework.IngressNginxDescribe("[Lua] dynamic certificates", func() { }) ginkgo.It("picks up the updated certificate without reloading", func() { - ing, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) + ing, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) assert.Nil(ginkgo.GinkgoT(), err) ensureHTTPSRequest(f, fmt.Sprintf("%s?id=dummy_log_splitter_foo_bar", f.GetURL(framework.HTTPS)), host, host) @@ -183,7 +183,7 @@ var _ = framework.IngressNginxDescribe("[Lua] dynamic certificates", func() { }) ginkgo.It("falls back to using default certificate when secret gets deleted without reloading", func() { - ing, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) + ing, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) assert.Nil(ginkgo.GinkgoT(), err) ensureHTTPSRequest(f, fmt.Sprintf("%s?id=dummy_log_splitter_foo_bar", f.GetURL(framework.HTTPS)), host, host) @@ -217,11 +217,11 @@ var _ = framework.IngressNginxDescribe("[Lua] dynamic certificates", func() { ginkgo.It("picks up a non-certificate only change", func() { newHost := "foo2.com" - ing, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) + ing, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) assert.Nil(ginkgo.GinkgoT(), err) ing.Spec.Rules[0].Host = newHost - _, err = f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{}) + _, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{}) assert.Nil(ginkgo.GinkgoT(), err) time.Sleep(waitForLuaSync) @@ -231,11 +231,11 @@ var _ = framework.IngressNginxDescribe("[Lua] dynamic certificates", func() { }) ginkgo.It("removes HTTPS configuration when we delete TLS spec", func() { - ing, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) + ing, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) assert.Nil(ginkgo.GinkgoT(), err) ing.Spec.TLS = []networking.IngressTLS{} - _, err = f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{}) + _, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{}) assert.Nil(ginkgo.GinkgoT(), err) time.Sleep(waitForLuaSync) @@ -245,6 +245,7 @@ var _ = framework.IngressNginxDescribe("[Lua] dynamic certificates", func() { WithHeader("Host", host). Expect(). Status(http.StatusOK) + }) }) }) @@ -253,6 +254,7 @@ func extractReloadCount(mf *dto.MetricFamily) (float64, error) { vec, err := expfmt.ExtractSamples(&expfmt.DecodeOptions{ Timestamp: model.Now(), }, mf) + if err != nil { return 0, err } diff --git a/test/e2e/lua/dynamic_configuration.go b/test/e2e/lua/dynamic_configuration.go index a5e2196ce..5fcebac7d 100644 --- a/test/e2e/lua/dynamic_configuration.go +++ b/test/e2e/lua/dynamic_configuration.go @@ -24,8 +24,9 @@ import ( "strings" "time" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/test/e2e/framework" @@ -42,13 +43,18 @@ var _ = framework.IngressNginxDescribe("[Lua] dynamic configuration", func() { f := framework.NewDefaultFramework("dynamic-configuration") ginkgo.BeforeEach(func() { - f.NewEchoDeployment() + f.NewEchoDeploymentWithReplicas(1) ensureIngress(f, "foo.com", framework.EchoService) }) ginkgo.It("configures balancer Lua middleware correctly", func() { f.WaitForNginxConfiguration(func(cfg string) bool { - return strings.Contains(cfg, "balancer_by_lua_file /etc/nginx/lua/nginx/ngx_conf_balancer.lua") + return strings.Contains(cfg, "balancer.init_worker()") && strings.Contains(cfg, "balancer.balance()") + }) + + host := "foo.com" + f.WaitForNginxServer(host, func(server string) bool { + return strings.Contains(server, "balancer.rewrite()") && strings.Contains(server, "balancer.log()") }) }) @@ -118,10 +124,7 @@ var _ = framework.IngressNginxDescribe("[Lua] dynamic configuration", func() { ginkgo.It("handles endpoints only changes consistently (down scaling of replicas vs. empty service)", func() { deploymentName := "scalingecho" - f.NewEchoDeployment( - framework.WithDeploymentName(deploymentName), - framework.WithDeploymentReplicas(0), - ) + f.NewEchoDeploymentWithNameAndReplicas(deploymentName, 0) createIngress(f, "scaling.foo.com", deploymentName) resp := f.HTTPTestClient(). @@ -169,11 +172,11 @@ var _ = framework.IngressNginxDescribe("[Lua] dynamic configuration", func() { return true }) - ingress, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Get(context.TODO(), "foo.com", metav1.GetOptions{}) + ingress, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), "foo.com", metav1.GetOptions{}) assert.Nil(ginkgo.GinkgoT(), err) ingress.ObjectMeta.Annotations["nginx.ingress.kubernetes.io/load-balance"] = "round_robin" - _, err = f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Update(context.TODO(), ingress, metav1.UpdateOptions{}) + _, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Update(context.TODO(), ingress, metav1.UpdateOptions{}) assert.Nil(ginkgo.GinkgoT(), err) f.HTTPTestClient(). @@ -193,18 +196,20 @@ var _ = framework.IngressNginxDescribe("[Lua] dynamic configuration", func() { }) }) -func ensureIngress(f *framework.Framework, host, deploymentName string) { - createIngress(f, host, deploymentName) +func ensureIngress(f *framework.Framework, host string, deploymentName string) *networking.Ingress { + ing := createIngress(f, host, deploymentName) f.HTTPTestClient(). GET("/"). WithHeader("Host", host). Expect(). Status(http.StatusOK) + + return ing } -func createIngress(f *framework.Framework, host, deploymentName string) { - f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.Namespace, deploymentName, 80, +func createIngress(f *framework.Framework, host string, deploymentName string) *networking.Ingress { + ing := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.Namespace, deploymentName, 80, map[string]string{ "nginx.ingress.kubernetes.io/load-balance": "ewma", }, @@ -215,19 +220,21 @@ func createIngress(f *framework.Framework, host, deploymentName string) { return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) && strings.Contains(server, "proxy_pass http://upstream_balancer;") }) + + return ing } -func ensureHTTPSRequest(f *framework.Framework, url, host, expectedDNSName string) { +func ensureHTTPSRequest(f *framework.Framework, url string, host string, expectedDNSName string) { resp := f.HTTPTestClientWithTLSConfig(&tls.Config{ ServerName: host, - InsecureSkipVerify: true, //nolint:gosec // Ignore the gosec error in testing + InsecureSkipVerify: true, }). GET("/"). WithURL(url). WithHeader("Host", host). Expect(). Raw() - defer resp.Body.Close() + assert.Equal(ginkgo.GinkgoT(), resp.StatusCode, http.StatusOK) assert.Equal(ginkgo.GinkgoT(), len(resp.TLS.PeerCertificates), 1) assert.Equal(ginkgo.GinkgoT(), resp.TLS.PeerCertificates[0].DNSNames[0], expectedDNSName) diff --git a/test/e2e/metrics/metrics.go b/test/e2e/metrics/metrics.go deleted file mode 100644 index bec09bb37..000000000 --- a/test/e2e/metrics/metrics.go +++ /dev/null @@ -1,141 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package defaultbackend - -import ( - "context" - "fmt" - "net/http" - "strings" - "time" - - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" - appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -const waitForMetrics = 2 * time.Second - -var _ = framework.IngressNginxDescribe("[metrics] exported prometheus metrics", func() { - f := framework.NewDefaultFramework("metrics") - host := "foo.com" - wildcardHost := "wildcard." + host - - ginkgo.BeforeEach(func() { - f.NewEchoDeployment() - f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil)) - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) && - strings.Contains(server, "proxy_pass http://upstream_balancer;") - }) - }) - - ginkgo.It("exclude socket request metrics are absent", func() { - err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { - args := deployment.Spec.Template.Spec.Containers[0].Args - args = append(args, "--exclude-socket-metrics=nginx_ingress_controller_request_size,nginx_ingress_controller_header_duration_seconds") - deployment.Spec.Template.Spec.Containers[0].Args = args - _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - return err - }) - assert.Nil(ginkgo.GinkgoT(), err, "updating deployment") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - time.Sleep(waitForMetrics) - - ip := f.GetNginxPodIP() - mf, err := f.GetMetric("nginx_ingress_controller_request_size", ip) - assert.ErrorContains(ginkgo.GinkgoT(), err, "nginx_ingress_controller_request_size") - assert.Nil(ginkgo.GinkgoT(), mf) - }) - ginkgo.It("exclude socket request metrics are present", func() { - err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { - args := deployment.Spec.Template.Spec.Containers[0].Args - args = append(args, "--exclude-socket-metrics=non_existing_metric_does_not_affect_existing_metrics") - deployment.Spec.Template.Spec.Containers[0].Args = args - _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - return err - }) - assert.Nil(ginkgo.GinkgoT(), err, "updating deployment") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - time.Sleep(waitForMetrics) - - ip := f.GetNginxPodIP() - mf, err := f.GetMetric("nginx_ingress_controller_request_size", ip) - assert.Nil(ginkgo.GinkgoT(), err) - assert.NotNil(ginkgo.GinkgoT(), mf) - }) - ginkgo.It("request metrics per undefined host are present when flag is set", func() { - err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { - args := deployment.Spec.Template.Spec.Containers[0].Args - args = append(args, "--metrics-per-undefined-host=true") - deployment.Spec.Template.Spec.Containers[0].Args = args - _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - return err - }) - assert.Nil(ginkgo.GinkgoT(), err, "updating deployment") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", wildcardHost). - Expect(). - Status(http.StatusNotFound) - time.Sleep(waitForMetrics) - - ip := f.GetNginxPodIP() - reqMetrics, err := f.GetMetric("nginx_ingress_controller_requests", ip) - assert.Nil(ginkgo.GinkgoT(), err) - assert.NotNil(ginkgo.GinkgoT(), reqMetrics.Metric) - assert.Len(ginkgo.GinkgoT(), reqMetrics.Metric, 1) - - containedLabel := false - for _, label := range reqMetrics.Metric[0].Label { - if *label.Name == "host" && *label.Value == wildcardHost { - containedLabel = true - break - } - } - - assert.Truef(ginkgo.GinkgoT(), containedLabel, "expected reqMetrics to contain label with \"name\"=\"host\" \"value\"=%q, but it did not: %s", wildcardHost, reqMetrics.String()) - }) - ginkgo.It("request metrics per undefined host are not present when flag is not set", func() { - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", wildcardHost). - Expect(). - Status(http.StatusNotFound) - time.Sleep(waitForMetrics) - - ip := f.GetNginxPodIP() - reqMetrics, err := f.GetMetric("nginx_ingress_controller_requests", ip) - assert.EqualError(ginkgo.GinkgoT(), err, "there is no metric with name nginx_ingress_controller_requests") - assert.Nil(ginkgo.GinkgoT(), reqMetrics) - }) -}) diff --git a/test/e2e/nginx/nginx.go b/test/e2e/nginx/nginx.go deleted file mode 100644 index 7e572a976..000000000 --- a/test/e2e/nginx/nginx.go +++ /dev/null @@ -1,127 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package nginx - -import ( - "fmt" - "net/http" - "time" - - "github.com/onsi/ginkgo/v2" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var ( - cfgOK = `# - events { - worker_connections 1024; - multi_accept on; - } - - http { - default_type 'text/plain'; - client_max_body_size 0; - - server { - access_log on; - access_log /dev/stdout; - - listen 80; - - location / { - content_by_lua_block { - ngx.print("ok") - } - } - } - } - ` - - cfgAlias = `# - events { - worker_connections 1024; - multi_accept on; - } - - http { - default_type 'text/plain'; - client_max_body_size 0; - - server { - access_log on; - access_log /dev/stdout; - - listen 80; - - location / { - alias /www/html; - } - } - } - ` - - cfgRoot = `# - events { - worker_connections 1024; - multi_accept on; - } - - http { - default_type 'text/plain'; - client_max_body_size 0; - root /srv/www; - server { - access_log on; - access_log /dev/stdout; - - listen 80; - - } - } - ` -) - -var _ = framework.DescribeSetting("nginx-configuration", func() { - f := framework.NewSimpleFramework("nginxconfiguration") - - ginkgo.It("start nginx with default configuration", func() { - f.NGINXWithConfigDeployment("default-nginx", cfgOK) - f.WaitForPod("app=default-nginx", 60*time.Second, false) - framework.Sleep(5 * time.Second) - - f.HTTPDumbTestClient(). - GET("/"). - WithURL(fmt.Sprintf("http://default-nginx.%s", f.Namespace)). - Expect(). - Status(http.StatusOK).Body().Contains("ok") - }) - - ginkgo.It("fails when using alias directive", func() { - f.NGINXDeployment("alias-nginx", cfgAlias, false) - // This should fail with a crashloopback because our NGINX does not have - // alias directive! - f.WaitForPod("app=alias-nginx", 60*time.Second, true) - }) - - ginkgo.It("fails when using root directive", func() { - f.NGINXDeployment("root-nginx", cfgRoot, false) - // This should fail with a crashloopback because our NGINX does not have - // root directive! - f.WaitForPod("app=root-nginx", 60*time.Second, true) - }) -}) diff --git a/test/e2e/run-chart-test.sh b/test/e2e/run-chart-test.sh index 7e3f2fe9b..841c05e7d 100755 --- a/test/e2e/run-chart-test.sh +++ b/test/e2e/run-chart-test.sh @@ -36,8 +36,6 @@ cleanup() { trap cleanup EXIT -export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:-ingress-nginx-dev} - if ! command -v kind --version &> /dev/null; then echo "kind is not installed. Use the package manager or visit the official site https://kind.sigs.k8s.io/" exit 1 @@ -45,24 +43,14 @@ fi DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -# Use 1.0.0-dev to make sure we use the latest configuration in the helm template -export TAG=1.0.0-dev -export ARCH=${ARCH:-amd64} -export REGISTRY=ingress-controller - -BASEDIR=$(dirname "$0") -NGINX_BASE_IMAGE=$(cat $BASEDIR/../../NGINX_BASE) - -echo "Running e2e with nginx base image ${NGINX_BASE_IMAGE}" - -export NGINX_BASE_IMAGE=$NGINX_BASE_IMAGE +export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:-ingress-nginx-dev} export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/kind-config-$KIND_CLUSTER_NAME}" if [ "${SKIP_CLUSTER_CREATION:-false}" = "false" ]; then echo "[dev-env] creating Kubernetes cluster with kind" - export K8S_VERSION=${K8S_VERSION:-v1.32.0@sha256:c48c62eac5da28cdadcf560d1d8616cfa6783b58f0d94cf63ad1bf49600cb027} + export K8S_VERSION=${K8S_VERSION:-v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab} kind create cluster \ --verbosity=${KIND_LOG_LEVEL} \ @@ -73,46 +61,16 @@ if [ "${SKIP_CLUSTER_CREATION:-false}" = "false" ]; then echo "Kubernetes cluster:" kubectl get nodes -o wide - -fi - -if [ "${SKIP_IMAGE_CREATION:-false}" = "false" ]; then - if ! command -v ginkgo &> /dev/null; then - go install github.com/onsi/ginkgo/v2/ginkgo@v2.22.2 - fi - echo "[dev-env] building image" - make -C ${DIR}/../../ clean-image build image -fi - - -KIND_WORKERS=$(kind get nodes --name="${KIND_CLUSTER_NAME}" | awk '{printf (NR>1?",":"") $1}') -echo "[dev-env] copying docker images to cluster..." - -kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes=${KIND_WORKERS} ${REGISTRY}/controller:${TAG} - -if [ "${SKIP_CERT_MANAGER_CREATION:-false}" = "false" ]; then - echo "[dev-env] deploying cert-manager..." - - # Get OS & platform for downloading cmctl. - os="$(uname -o | tr "[:upper:]" "[:lower:]" | sed "s/gnu\///")" - platform="$(uname -m | sed "s/aarch64/arm64/;s/x86_64/amd64/")" - - # Download cmctl. Cannot validate checksum as OS & platform may vary. - curl --fail --location "https://github.com/cert-manager/cmctl/releases/download/v2.1.1/cmctl_${os}_${platform}.tar.gz" | tar --extract --gzip cmctl - - # Install cert-manager. - ./cmctl x install - ./cmctl check api --wait 1m fi echo "[dev-env] running helm chart e2e tests..." -docker run \ - --name ct \ - --volume "${KUBECONFIG}:/root/.kube/config:ro" \ - --volume "${DIR}/../../:/workdir" \ - --network host \ - --workdir /workdir \ - --entrypoint ct \ - --rm \ - registry.k8s.io/ingress-nginx/e2e-test-runner:v20250112-a188f4eb@sha256:043038b1e30e5a0b64f3f919f096c5c9488ac3f617ac094b07fb9db8215f9441 \ - install --charts charts/ingress-nginx +# Uses a custom chart-testing image to avoid timeouts waiting for namespace deletion. +# The changes can be found here: https://github.com/aledbf/chart-testing/commit/41fe0ae0733d0c9a538099fb3cec522e888e3d82 +docker run --rm --interactive --network host \ + --name ct \ + --volume $KUBECONFIG:/root/.kube/config \ + --volume "${DIR}/../../":/workdir \ + --workdir /workdir \ + aledbf/chart-testing:v3.3.1-next ct install \ + --charts charts/ingress-nginx \ + --helm-extra-args "--timeout 60s" diff --git a/test/e2e/run-kind-e2e.sh b/test/e2e/run.sh similarity index 50% rename from test/e2e/run-kind-e2e.sh rename to test/e2e/run.sh index e41d31afd..d3caf2537 100755 --- a/test/e2e/run-kind-e2e.sh +++ b/test/e2e/run.sh @@ -14,6 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +KIND_LOG_LEVEL="1" + +if ! [ -z $DEBUG ]; then + set -x + KIND_LOG_LEVEL="6" +fi + set -o errexit set -o nounset set -o pipefail @@ -24,56 +31,39 @@ cleanup() { fi kind delete cluster \ - --verbosity="${KIND_LOG_LEVEL}" \ - --name "${KIND_CLUSTER_NAME}" + --verbosity=${KIND_LOG_LEVEL} \ + --name ${KIND_CLUSTER_NAME} } -DEBUG=${DEBUG:=false} +trap cleanup EXIT -if [ "${DEBUG}" = "true" ]; then - set -x - KIND_LOG_LEVEL="6" -else - trap cleanup EXIT -fi - -KIND_LOG_LEVEL="1" -IS_CHROOT="${IS_CHROOT:-false}" export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:-ingress-nginx-dev} -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -# Use 1.0.0-dev to make sure we use the latest configuration in the helm template -export TAG=1.0.0-dev -export ARCH=${ARCH:-amd64} -export REGISTRY=ingress-controller -NGINX_BASE_IMAGE=${NGINX_BASE_IMAGE:-$(cat "$DIR"/../../NGINX_BASE)} -export NGINX_BASE_IMAGE=$NGINX_BASE_IMAGE -export DOCKER_CLI_EXPERIMENTAL=enabled -export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/kind-config-$KIND_CLUSTER_NAME}" -SKIP_INGRESS_IMAGE_CREATION="${SKIP_INGRESS_IMAGE_CREATION:-false}" -SKIP_E2E_IMAGE_CREATION="${SKIP_E2E_IMAGE_CREATION:=false}" -SKIP_CLUSTER_CREATION="${SKIP_CLUSTER_CREATION:-false}" if ! command -v kind --version &> /dev/null; then echo "kind is not installed. Use the package manager or visit the official site https://kind.sigs.k8s.io/" exit 1 fi -echo "Running e2e with nginx base image ${NGINX_BASE_IMAGE}" +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -if [ "${SKIP_CLUSTER_CREATION}" = "false" ]; then +# Use 1.0.0-dev to make sure we use the latest configuration in the helm template +export TAG=1.0.0-dev +export ARCH=${ARCH:-amd64} +export REGISTRY=ingress-controller + +export DOCKER_CLI_EXPERIMENTAL=enabled + +export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/kind-config-$KIND_CLUSTER_NAME}" + +if [ "${SKIP_CLUSTER_CREATION:-false}" = "false" ]; then echo "[dev-env] creating Kubernetes cluster with kind" - export K8S_VERSION=${K8S_VERSION:-v1.32.0@sha256:c48c62eac5da28cdadcf560d1d8616cfa6783b58f0d94cf63ad1bf49600cb027} - - # delete the cluster if it exists - if kind get clusters | grep "${KIND_CLUSTER_NAME}"; then - kind delete cluster --name "${KIND_CLUSTER_NAME}" - fi + export K8S_VERSION=${K8S_VERSION:-v1.20.2@sha256:8f7ea6e7642c0da54f04a7ee10431549c0257315b3a634f6ef2fecaaedb19bab} kind create cluster \ - --verbosity="${KIND_LOG_LEVEL}" \ - --name "${KIND_CLUSTER_NAME}" \ - --config "${DIR}"/kind.yaml \ + --verbosity=${KIND_LOG_LEVEL} \ + --name ${KIND_CLUSTER_NAME} \ + --config ${DIR}/kind.yaml \ --retain \ --image "kindest/node:${K8S_VERSION}" @@ -81,27 +71,14 @@ if [ "${SKIP_CLUSTER_CREATION}" = "false" ]; then kubectl get nodes -o wide fi -if [ "${SKIP_INGRESS_IMAGE_CREATION}" = "false" ]; then - echo "[dev-env] building image" - if [ "${IS_CHROOT}" = "true" ]; then - make BASE_IMAGE="${NGINX_BASE_IMAGE}" -C "${DIR}"/../../ clean-image build image-chroot - docker tag ${REGISTRY}/controller-chroot:${TAG} ${REGISTRY}/controller:${TAG} - else - make BASE_IMAGE="${NGINX_BASE_IMAGE}" -C "${DIR}"/../../ clean-image build image - fi - - echo "[dev-env] .. done building controller images" -fi - -if [ "${SKIP_E2E_IMAGE_CREATION}" = "false" ]; then +if [ "${SKIP_IMAGE_CREATION:-false}" = "false" ]; then if ! command -v ginkgo &> /dev/null; then - go install github.com/onsi/ginkgo/v2/ginkgo@v2.22.2 + go get github.com/onsi/ginkgo/ginkgo fi - echo "[dev-env] .. done building controller images" - echo "[dev-env] now building e2e-image.." - make -C "${DIR}"/../e2e-image image - echo "[dev-env] ..done building e2e-image" + echo "[dev-env] building image" + make -C ${DIR}/../../ clean-image build image + make -C ${DIR}/../e2e-image image fi # Preload images used in e2e tests @@ -109,7 +86,8 @@ KIND_WORKERS=$(kind get nodes --name="${KIND_CLUSTER_NAME}" | grep worker | awk echo "[dev-env] copying docker images to cluster..." -kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" nginx-ingress-controller:e2e -kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" "${REGISTRY}"/controller:"${TAG}" +kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes=${KIND_WORKERS} nginx-ingress-controller:e2e +kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes=${KIND_WORKERS} ${REGISTRY}/controller:${TAG} + echo "[dev-env] running e2e tests..." -make -C "${DIR}"/../../ e2e-test +make -C ${DIR}/../../ e2e-test diff --git a/test/e2e/security/request_smuggling.go b/test/e2e/security/request_smuggling.go index 5ede02d4b..94ee53c64 100644 --- a/test/e2e/security/request_smuggling.go +++ b/test/e2e/security/request_smuggling.go @@ -23,7 +23,7 @@ import ( "strings" "time" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" @@ -50,12 +50,9 @@ server { f.UpdateNginxConfigMapData("http-snippet", snippet) - // TODO: currently using a self hosted HTTPBun instance results in a 499, we - // should move away from using httpbun.com once we have the httpbun - // deployment as part of the framework ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, map[string]string{ - "nginx.ingress.kubernetes.io/auth-signin": "https://httpbun.com/bearer/d4bcba7a-0def-4a31-91a7-47e420adf44b", - "nginx.ingress.kubernetes.io/auth-url": "https://httpbun.com/basic-auth/user/passwd", + "nginx.ingress.kubernetes.io/auth-signin": "https://httpbin.org/uuid", + "nginx.ingress.kubernetes.io/auth-url": "https://httpbin.org/basic-auth/user/passwd", }) f.EnsureIngress(ing) @@ -79,9 +76,7 @@ func smugglingRequest(host, addr string, port int) (string, error) { defer conn.Close() - if err := conn.SetDeadline(time.Now().Add(time.Second * 10)); err != nil { - return "", err - } + conn.SetDeadline(time.Now().Add(time.Second * 10)) _, err = fmt.Fprintf(conn, "GET /echo HTTP/1.1\r\nHost: %v\r\nContent-Length: 56\r\n\r\nGET /_hidden/index.html HTTP/1.1\r\nHost: notlocalhost\r\n\r\n", host) if err != nil { @@ -91,7 +86,7 @@ func smugglingRequest(host, addr string, port int) (string, error) { // wait for /_hidden/index.html response framework.Sleep() - buf := make([]byte, 1024) + var buf = make([]byte, 1024) r := bufio.NewReader(conn) _, err = r.Read(buf) if err != nil { diff --git a/test/e2e/servicebackend/service_backend.go b/test/e2e/servicebackend/service_backend.go index 6ab2f8ad1..86f39b842 100644 --- a/test/e2e/servicebackend/service_backend.go +++ b/test/e2e/servicebackend/service_backend.go @@ -20,59 +20,56 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" corev1 "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/test/e2e/framework" ) -var ( - pathtype = networking.PathTypePrefix - _ = framework.IngressNginxDescribe("[Service] backend status code 503", func() { - f := framework.NewDefaultFramework("service-backend") +var _ = framework.IngressNginxDescribe("[Service] backend status code 503", func() { + f := framework.NewDefaultFramework("service-backend") - ginkgo.It("should return 503 when backend service does not exist", func() { - host := "nonexistent.svc.com" + ginkgo.It("should return 503 when backend service does not exist", func() { + host := "nonexistent.svc.com" - bi := buildIngressWithNonexistentService(host, f.Namespace, "/") - f.EnsureIngress(bi) + bi := buildIngressWithNonexistentService(host, f.Namespace, "/") + f.EnsureIngress(bi) - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "proxy_pass http://upstream_balancer;") - }) + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "proxy_pass http://upstream_balancer;") + }) - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusServiceUnavailable) - }) - - ginkgo.It("should return 503 when all backend service endpoints are unavailable", func() { - host := "unavailable.svc.com" - - bi, bs := buildIngressWithUnavailableServiceEndpoints(host, f.Namespace, "/") - - f.EnsureService(bs) - f.EnsureIngress(bi) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "proxy_pass http://upstream_balancer;") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusServiceUnavailable) - }) + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", host). + Expect(). + Status(http.StatusServiceUnavailable) }) -) + + ginkgo.It("should return 503 when all backend service endpoints are unavailable", func() { + host := "unavailable.svc.com" + + bi, bs := buildIngressWithUnavailableServiceEndpoints(host, f.Namespace, "/") + + f.EnsureService(bs) + f.EnsureIngress(bi) + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "proxy_pass http://upstream_balancer;") + }) + + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", host). + Expect(). + Status(http.StatusServiceUnavailable) + }) +}) func buildIngressWithNonexistentService(host, namespace, path string) *networking.Ingress { backendService := "nonexistent-svc" @@ -82,7 +79,6 @@ func buildIngressWithNonexistentService(host, namespace, path string) *networkin Namespace: namespace, }, Spec: networking.IngressSpec{ - IngressClassName: framework.GetIngressClassName(namespace), Rules: []networking.IngressRule{ { Host: host, @@ -90,15 +86,10 @@ func buildIngressWithNonexistentService(host, namespace, path string) *networkin HTTP: &networking.HTTPIngressRuleValue{ Paths: []networking.HTTPIngressPath{ { - Path: path, - PathType: &pathtype, + Path: path, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: backendService, - Port: networking.ServiceBackendPort{ - Number: int32(80), - }, - }, + ServiceName: backendService, + ServicePort: intstr.FromInt(80), }, }, }, @@ -118,7 +109,6 @@ func buildIngressWithUnavailableServiceEndpoints(host, namespace, path string) ( Namespace: namespace, }, Spec: networking.IngressSpec{ - IngressClassName: framework.GetIngressClassName(namespace), Rules: []networking.IngressRule{ { Host: host, @@ -126,15 +116,10 @@ func buildIngressWithUnavailableServiceEndpoints(host, namespace, path string) ( HTTP: &networking.HTTPIngressRuleValue{ Paths: []networking.HTTPIngressPath{ { - Path: path, - PathType: &pathtype, + Path: path, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: backendService, - Port: networking.ServiceBackendPort{ - Number: int32(80), - }, - }, + ServiceName: backendService, + ServicePort: intstr.FromInt(80), }, }, }, @@ -148,15 +133,14 @@ func buildIngressWithUnavailableServiceEndpoints(host, namespace, path string) ( Name: backendService, Namespace: namespace, }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "tcp", - Port: 80, - TargetPort: intstr.FromInt(80), - Protocol: "TCP", - }, + Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{ + { + Name: "tcp", + Port: 80, + TargetPort: intstr.FromInt(80), + Protocol: "TCP", }, + }, Selector: map[string]string{ "app": backendService, }, diff --git a/test/e2e/servicebackend/service_externalname.go b/test/e2e/servicebackend/service_externalname.go index fccd1cd19..bd10519e7 100644 --- a/test/e2e/servicebackend/service_externalname.go +++ b/test/e2e/servicebackend/service_externalname.go @@ -22,29 +22,29 @@ import ( "net/http" "strings" - "k8s.io/ingress-nginx/test/e2e/framework/httpexpect" - - "github.com/onsi/ginkgo/v2" + "github.com/gavv/httpexpect/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" + core "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/ingress-nginx/internal/nginx" "k8s.io/ingress-nginx/test/e2e/framework" ) -const echoHost = "echo" - var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { - f := framework.NewDefaultFramework("type-externalname", framework.WithHTTPBunEnabled()) + f := framework.NewDefaultFramework("type-externalname") ginkgo.It("works with external name set to incomplete fqdn", func() { f.NewEchoDeployment() - host := echoHost - svc := &corev1.Service{ + host := "echo" + + svc := &core.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: framework.NIPService, + Name: framework.HTTPBinService, Namespace: f.Namespace, }, Spec: corev1.ServiceSpec{ @@ -52,15 +52,10 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { Type: corev1.ServiceTypeExternalName, }, } + f.EnsureService(svc) - ing := framework.NewSingleIngress(host, - "/", - host, - f.Namespace, - framework.NIPService, - 80, - nil) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.HTTPBinService, 80, nil) f.EnsureIngress(ing) f.WaitForNginxServer(host, @@ -76,31 +71,25 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { }) ginkgo.It("should return 200 for service type=ExternalName without a port defined", func() { - host := echoHost + host := "echo" - svc := &corev1.Service{ + svc := &core.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: framework.NIPService, + Name: framework.HTTPBinService, Namespace: f.Namespace, }, Spec: corev1.ServiceSpec{ - ExternalName: f.GetNIPHost(), + ExternalName: "httpbin.org", Type: corev1.ServiceTypeExternalName, }, } + f.EnsureService(svc) annotations := map[string]string{ - "nginx.ingress.kubernetes.io/upstream-vhost": f.GetNIPHost(), + "nginx.ingress.kubernetes.io/upstream-vhost": "httpbin.org", } - - ing := framework.NewSingleIngress(host, - "/", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.HTTPBinService, 80, annotations) f.EnsureIngress(ing) f.WaitForNginxServer(host, @@ -116,21 +105,32 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { }) ginkgo.It("should return 200 for service type=ExternalName with a port defined", func() { - host := echoHost + host := "echo" - svc := framework.BuildNIPExternalNameService(f, f.HTTPBunIP, host) + svc := &core.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: framework.HTTPBinService, + Namespace: f.Namespace, + }, + Spec: corev1.ServiceSpec{ + ExternalName: "httpbin.org", + Type: corev1.ServiceTypeExternalName, + Ports: []corev1.ServicePort{ + { + Name: host, + Port: 80, + TargetPort: intstr.FromInt(80), + Protocol: "TCP", + }, + }, + }, + } f.EnsureService(svc) annotations := map[string]string{ - "nginx.ingress.kubernetes.io/upstream-vhost": f.GetNIPHost(), + "nginx.ingress.kubernetes.io/upstream-vhost": "httpbin.org", } - ing := framework.NewSingleIngress(host, - "/", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.HTTPBinService, 80, annotations) f.EnsureIngress(ing) f.WaitForNginxServer(host, @@ -146,11 +146,11 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { }) ginkgo.It("should return status 502 for service type=ExternalName with an invalid host", func() { - host := echoHost + host := "echo" - svc := &corev1.Service{ + svc := &core.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: framework.NIPService, + Name: framework.HTTPBinService, Namespace: f.Namespace, }, Spec: corev1.ServiceSpec{ @@ -158,15 +158,10 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { Type: corev1.ServiceTypeExternalName, }, } + f.EnsureService(svc) - ing := framework.NewSingleIngress(host, - "/", - host, - f.Namespace, - framework.NIPService, - 80, - nil) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.HTTPBinService, 80, nil) f.EnsureIngress(ing) f.WaitForNginxServer(host, @@ -182,32 +177,33 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { }) ginkgo.It("should return 200 for service type=ExternalName using a port name", func() { - host := echoHost + host := "echo" - svc := framework.BuildNIPExternalNameService(f, f.HTTPBunIP, host) - f.EnsureService(svc) - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/upstream-vhost": f.GetNIPHost(), - } - ing := framework.NewSingleIngress(host, - "/", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations) - - namedBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: framework.NIPService, - Port: networking.ServiceBackendPort{ - Name: host, + svc := &core.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: framework.HTTPBinService, + Namespace: f.Namespace, + }, + Spec: corev1.ServiceSpec{ + ExternalName: "httpbin.org", + Type: corev1.ServiceTypeExternalName, + Ports: []corev1.ServicePort{ + { + Name: host, + Port: 80, + TargetPort: intstr.FromInt(80), + Protocol: "TCP", + }, }, }, } + f.EnsureService(svc) - ing.Spec.Rules[0].HTTP.Paths[0].Backend = namedBackend + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/upstream-vhost": "httpbin.org", + } + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.HTTPBinService, 80, annotations) + ing.Spec.Rules[0].HTTP.Paths[0].Backend.ServicePort = intstr.FromString(host) f.EnsureIngress(ing) f.WaitForNginxServer(host, @@ -223,27 +219,22 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { }) ginkgo.It("should return 200 for service type=ExternalName using FQDN with trailing dot", func() { - host := echoHost + host := "echo" - svc := &corev1.Service{ + svc := &core.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: framework.NIPService, + Name: framework.HTTPBinService, Namespace: f.Namespace, }, Spec: corev1.ServiceSpec{ - ExternalName: f.GetNIPHost(), + ExternalName: "httpbin.org.", Type: corev1.ServiceTypeExternalName, }, } + f.EnsureService(svc) - ing := framework.NewSingleIngress(host, - "/", - host, - f.Namespace, - framework.HTTPBunService, - 80, - nil) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.HTTPBinService, 80, nil) f.EnsureIngress(ing) f.WaitForNginxServer(host, @@ -259,32 +250,33 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { }) ginkgo.It("should update the external name after a service update", func() { - host := echoHost + host := "echo" - svc := framework.BuildNIPExternalNameService(f, f.HTTPBunIP, host) - f.EnsureService(svc) - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/upstream-vhost": f.GetNIPHost(), - } - - ing := framework.NewSingleIngress(host, - "/", - host, - f.Namespace, - framework.HTTPBunService, - 80, - annotations) - - namedBackend := networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: framework.NIPService, - Port: networking.ServiceBackendPort{ - Name: host, + svc := &core.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: framework.HTTPBinService, + Namespace: f.Namespace, + }, + Spec: corev1.ServiceSpec{ + ExternalName: "httpbin.org", + Type: corev1.ServiceTypeExternalName, + Ports: []corev1.ServicePort{ + { + Name: host, + Port: 80, + TargetPort: intstr.FromInt(80), + Protocol: "TCP", + }, }, }, } - ing.Spec.Rules[0].HTTP.Paths[0].Backend = namedBackend + f.EnsureService(svc) + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/upstream-vhost": "httpbin.org", + } + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.HTTPBinService, 80, annotations) + ing.Spec.Rules[0].HTTP.Paths[0].Backend.ServicePort = intstr.FromString(host) f.EnsureIngress(ing) f.WaitForNginxServer(host, @@ -302,21 +294,13 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { assert.Contains(ginkgo.GinkgoT(), body, `"X-Forwarded-Host": "echo"`) - svc, err := f.KubeClientSet. - CoreV1(). - Services(f.Namespace). - Get(context.TODO(), framework.NIPService, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining external service") + svc, err := f.KubeClientSet.CoreV1().Services(f.Namespace).Get(context.TODO(), framework.HTTPBinService, metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining httpbin service") - // Deploy a new instance to switch routing to - ip := f.NewHttpbunDeployment(framework.WithDeploymentName("eu-server")) - svc.Spec.ExternalName = framework.BuildNIPHost(ip) + svc.Spec.ExternalName = "eu.httpbin.org" - _, err = f.KubeClientSet. - CoreV1(). - Services(f.Namespace). - Update(context.Background(), svc, metav1.UpdateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating external service") + _, err = f.KubeClientSet.CoreV1().Services(f.Namespace).Update(context.Background(), svc, metav1.UpdateOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating httpbin service") framework.Sleep() @@ -330,68 +314,10 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { assert.Contains(ginkgo.GinkgoT(), body, `"X-Forwarded-Host": "echo"`) - ginkgo.By("checking the service is updated to use new host") - dbgCmd := "/dbg backends all" - output, err := f.ExecIngressPod(dbgCmd) + ginkgo.By("checking the service is updated to use eu.httpbin.org") + curlCmd := fmt.Sprintf("curl --fail --silent http://localhost:%v/configuration/backends", nginx.StatusPort) + output, err := f.ExecIngressPod(curlCmd) assert.Nil(ginkgo.GinkgoT(), err) - assert.Contains( - ginkgo.GinkgoT(), - output, - fmt.Sprintf(`"address": %q`, framework.BuildNIPHost(ip)), - ) - }) - - ginkgo.It("should sync ingress on external name service addition/deletion", func() { - host := echoHost - - // Create the Ingress first - ing := framework.NewSingleIngress(host, - "/", - host, - f.Namespace, - framework.NIPService, - 80, - nil) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "proxy_pass http://upstream_balancer;") - }) - - // Nginx should return 503 without the underlying service being available - f.HTTPTestClient(). - GET("/get"). - WithHeader("Host", host). - Expect(). - Status(http.StatusServiceUnavailable) - - // Now create the service - svc := framework.BuildNIPExternalNameService(f, f.HTTPBunIP, host) - f.EnsureService(svc) - - framework.Sleep() - - // 503 should change to 200 OK - f.HTTPTestClient(). - GET("/get"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - - // And back to 503 after deleting the service - err := f.KubeClientSet. - CoreV1(). - Services(f.Namespace). - Delete(context.TODO(), framework.NIPService, metav1.DeleteOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error deleting external service") - - framework.Sleep() - - f.HTTPTestClient(). - GET("/get"). - WithHeader("Host", host). - Expect(). - Status(http.StatusServiceUnavailable) + assert.Contains(ginkgo.GinkgoT(), output, `{"address":"eu.httpbin.org"`) }) }) diff --git a/test/e2e/servicebackend/service_nil_backend.go b/test/e2e/servicebackend/service_nil_backend.go deleted file mode 100644 index 9b5b4c7e6..000000000 --- a/test/e2e/servicebackend/service_nil_backend.go +++ /dev/null @@ -1,105 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package servicebackend - -import ( - "net/http" - "strings" - - "github.com/onsi/ginkgo/v2" - corev1 "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.IngressNginxDescribe("[Service] Nil Service Backend", func() { - f := framework.NewDefaultFramework("service-nil-backend") - - ginkgo.BeforeEach(func() { - f.NewEchoDeployment() - }) - - ginkgo.It("should return 404 when backend service is nil", func() { - ginkgo.By("setting an ingress with a nil backend") - validHost := "valid.svc.com" - invalidHost := "nilbackend.svc.com" - - ing := framework.NewSingleIngress(validHost, "/", validHost, f.Namespace, - framework.EchoService, 80, nil) - - bi := buildIngressWithNonServiceBackend(invalidHost, f.Namespace, "/") - - f.EnsureIngress(bi) - f.EnsureIngress(ing) - - f.WaitForNginxConfiguration(func(cfg string) bool { - return strings.Contains(cfg, "server_name nilbackend.svc.com") && - strings.Contains(cfg, "server_name valid.svc.com") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", validHost). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", invalidHost). - Expect(). - Status(http.StatusNotFound) - }) -}) - -func buildIngressWithNonServiceBackend(host, namespace, path string) *networking.Ingress { - apiGroup := "otherobj.testingress.com" - obj := corev1.TypedLocalObjectReference{ - Kind: "Anything", - Name: "mytest", - APIGroup: &apiGroup, - } - - return &networking.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: host, - Namespace: namespace, - }, - Spec: networking.IngressSpec{ - IngressClassName: framework.GetIngressClassName(namespace), - Rules: []networking.IngressRule{ - { - Host: host, - IngressRuleValue: networking.IngressRuleValue{ - HTTP: &networking.HTTPIngressRuleValue{ - Paths: []networking.HTTPIngressPath{ - { - Path: path, - PathType: &pathtype, - Backend: networking.IngressBackend{ - Resource: &obj, - }, - }, - }, - }, - }, - }, - }, - }, - } -} diff --git a/test/e2e/settings/access_log.go b/test/e2e/settings/access_log.go index 65b9dfda4..0e4c1d827 100644 --- a/test/e2e/settings/access_log.go +++ b/test/e2e/settings/access_log.go @@ -19,8 +19,7 @@ package settings import ( "strings" - "github.com/onsi/ginkgo/v2" - + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -28,60 +27,60 @@ var _ = framework.DescribeSetting("access-log", func() { f := framework.NewDefaultFramework("access-log") ginkgo.Context("access-log-path", func() { + ginkgo.It("use the default configuration", func() { f.WaitForNginxConfiguration( func(cfg string) bool { - return (strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo") && - strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream")) || - (strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 upstreaminfo") && - strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 log_stream")) + return strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo") && + strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream") }) }) ginkgo.It("use the specified configuration", func() { - f.UpdateNginxConfigMapData("access-log-path", "/tmp/nginx/access.log") + f.UpdateNginxConfigMapData("access-log-path", "/tmp/access.log") f.WaitForNginxConfiguration( func(cfg string) bool { - return strings.Contains(cfg, "access_log /tmp/nginx/access.log upstreaminfo") && - strings.Contains(cfg, "access_log /tmp/nginx/access.log log_stream") + return strings.Contains(cfg, "access_log /tmp/access.log upstreaminfo") && + strings.Contains(cfg, "access_log /tmp/access.log log_stream") }) }) }) ginkgo.Context("http-access-log-path", func() { + ginkgo.It("use the specified configuration", func() { - f.UpdateNginxConfigMapData("http-access-log-path", "/tmp/nginx/http-access.log") + f.UpdateNginxConfigMapData("http-access-log-path", "/tmp/http-access.log") f.WaitForNginxConfiguration( func(cfg string) bool { - return strings.Contains(cfg, "access_log /tmp/nginx/http-access.log upstreaminfo") && - (strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream") || - strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 log_stream")) + return strings.Contains(cfg, "access_log /tmp/http-access.log upstreaminfo") && + strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream") }) }) }) ginkgo.Context("stream-access-log-path", func() { + ginkgo.It("use the specified configuration", func() { - f.UpdateNginxConfigMapData("stream-access-log-path", "/tmp/nginx/stream-access.log") + f.UpdateNginxConfigMapData("stream-access-log-path", "/tmp/stream-access.log") f.WaitForNginxConfiguration( func(cfg string) bool { - return strings.Contains(cfg, "access_log /tmp/nginx/stream-access.log log_stream") && - (strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo") || - strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 upstreaminfo")) + return strings.Contains(cfg, "access_log /tmp/stream-access.log log_stream") && + strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo") }) }) }) ginkgo.Context("http-access-log-path & stream-access-log-path", func() { + ginkgo.It("use the specified configuration", func() { f.SetNginxConfigMapData(map[string]string{ - "http-access-log-path": "/tmp/nginx/http-access.log", - "stream-access-log-path": "/tmp/nginx/stream-access.log", + "http-access-log-path": "/tmp/http-access.log", + "stream-access-log-path": "/tmp/stream-access.log", }) f.WaitForNginxConfiguration( func(cfg string) bool { - return strings.Contains(cfg, "access_log /tmp/nginx/http-access.log upstreaminfo") && - strings.Contains(cfg, "access_log /tmp/nginx/stream-access.log log_stream") + return strings.Contains(cfg, "access_log /tmp/http-access.log upstreaminfo") && + strings.Contains(cfg, "access_log /tmp/stream-access.log log_stream") }) }) }) diff --git a/test/e2e/settings/aio_write.go b/test/e2e/settings/aio_write.go deleted file mode 100644 index bcfe2a4de..000000000 --- a/test/e2e/settings/aio_write.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package settings - -import ( - "strings" - - "github.com/onsi/ginkgo/v2" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.DescribeSetting("aio-write", func() { - f := framework.NewDefaultFramework("aio-write") - - ginkgo.It("should be enabled by default", func() { - f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, "aio_write on") - }) - }) - - ginkgo.It("should be enabled when setting is true", func() { - f.UpdateNginxConfigMapData("enable-aio-write", "true") - - f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, "aio_write on") - }) - }) - - ginkgo.It("should be disabled when setting is false", func() { - f.UpdateNginxConfigMapData("enable-aio-write", "false") - - f.WaitForNginxConfiguration( - func(cfg string) bool { - return !strings.Contains(cfg, "aio_write on") - }) - }) -}) diff --git a/test/e2e/settings/badannotationvalues.go b/test/e2e/settings/badannotationvalues.go deleted file mode 100644 index aa9906909..000000000 --- a/test/e2e/settings/badannotationvalues.go +++ /dev/null @@ -1,170 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package settings - -import ( - "fmt" - "net/http" - "strings" - - "github.com/onsi/ginkgo/v2" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.DescribeAnnotation("Bad annotation values", func() { - f := framework.NewDefaultFramework("bad-annotation") - - ginkgo.BeforeEach(func() { - f.NewEchoDeployment() - }) - - ginkgo.It("[BAD_ANNOTATIONS] should drop an ingress if there is an invalid character in some annotation", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - host := "invalid-value-test" - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": ` - # abc { }`, - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.UpdateNginxConfigMapData("annotation-value-word-blocklist", "something_forbidden,otherthing_forbidden,{") - - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return !strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) - }) - - f.WaitForNginxServer(host, - func(server string) bool { - return !strings.Contains(server, "# abc { }") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusNotFound) - }) - - ginkgo.It("[BAD_ANNOTATIONS] should drop an ingress if there is a forbidden word in some annotation", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - - host := "forbidden-value-test" - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": ` - default_type text/plain; - content_by_lua_block { - ngx.say("Hello World") - }`, - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.UpdateNginxConfigMapData("annotation-value-word-blocklist", "something_forbidden,otherthing_forbidden,content_by_lua_block") - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return !strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) - }) - - f.WaitForNginxServer(host, - func(server string) bool { - return !strings.Contains(server, `ngx.say("Hello World")`) - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusNotFound) - }) - - ginkgo.It("[BAD_ANNOTATIONS] should allow an ingress if there is a default blocklist config in place", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - - hostValid := "custom-allowed-value-test" - annotationsValid := map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": ` - # bla_by_lua`, - } - - ingValid := framework.NewSingleIngress(hostValid, "/", hostValid, f.Namespace, framework.EchoService, 80, annotationsValid) - - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - f.EnsureIngress(ingValid) - - f.WaitForNginxServer(hostValid, - func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", hostValid)) - }) - - f.WaitForNginxServer(hostValid, - func(server string) bool { - return strings.Contains(server, "# bla_by_lua") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", hostValid). - Expect(). - Status(http.StatusOK) - }) - - ginkgo.It("[BAD_ANNOTATIONS] should drop an ingress if there is a custom blocklist config in place and allow others to pass", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - host := "custom-forbidden-value-test" - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": ` - # something_forbidden`, - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - f.UpdateNginxConfigMapData("annotation-value-word-blocklist", "something_forbidden,otherthing_forbidden") - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return !strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) - }) - - f.WaitForNginxServer(host, - func(server string) bool { - return !strings.Contains(server, "# something_forbidden") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusNotFound) - }) -}) diff --git a/test/e2e/settings/brotli.go b/test/e2e/settings/brotli.go deleted file mode 100644 index aacaddec5..000000000 --- a/test/e2e/settings/brotli.go +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package settings - -import ( - "fmt" - "net/http" - "strconv" - "strings" - - "github.com/onsi/ginkgo/v2" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.IngressNginxDescribe("brotli", func() { - f := framework.NewDefaultFramework( - "brotli", - framework.WithHTTPBunEnabled(), - ) - - host := "brotli" - - ginkgo.It("should only compress responses that meet the `brotli-min-length` condition", func() { - brotliMinLength := 24 - contentEncoding := "application/octet-stream" - f.UpdateNginxConfigMapData("enable-brotli", "true") - f.UpdateNginxConfigMapData("brotli-types", contentEncoding) - f.UpdateNginxConfigMapData("brotli-min-length", strconv.Itoa(brotliMinLength)) - - f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.Namespace, framework.HTTPBunService, 80, nil)) - - f.WaitForNginxConfiguration( - func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %v", host)) && - strings.Contains(server, "brotli on") && - strings.Contains(server, fmt.Sprintf("brotli_types %v", contentEncoding)) && - strings.Contains(server, fmt.Sprintf("brotli_min_length %d", brotliMinLength)) - }) - - f.HTTPTestClient(). - GET(fmt.Sprintf("/bytes/%d", brotliMinLength)). - WithHeader("Host", host). - WithHeader("Accept-Encoding", "br"). - Expect(). - Status(http.StatusOK). - ContentType(contentEncoding). - ContentEncoding("br") - - f.HTTPTestClient(). - GET(fmt.Sprintf("/bytes/%d", brotliMinLength-1)). - WithHeader("Host", host). - WithHeader("Accept-Encoding", "br"). - Expect(). - Status(http.StatusOK). - ContentType(contentEncoding). - ContentEncoding() - }) -}) diff --git a/test/e2e/settings/configmap_change.go b/test/e2e/settings/configmap_change.go index 3e37b62cd..084be1ba7 100644 --- a/test/e2e/settings/configmap_change.go +++ b/test/e2e/settings/configmap_change.go @@ -20,7 +20,7 @@ import ( "regexp" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" @@ -73,9 +73,5 @@ var _ = framework.DescribeSetting("Configmap change", func() { return strings.ContainsAny(cfg, "error_log /var/log/nginx/error.log debug;") }) assert.NotEqual(ginkgo.GinkgoT(), checksum, newChecksum) - - logs, err := f.NginxLogs() - assert.Nil(ginkgo.GinkgoT(), err, "obtaining nginx logs") - assert.Contains(ginkgo.GinkgoT(), logs, "Backend successfully reloaded") }) }) diff --git a/test/e2e/settings/custom_header.go b/test/e2e/settings/custom_header.go index 8341f7ef0..f907e74a5 100644 --- a/test/e2e/settings/custom_header.go +++ b/test/e2e/settings/custom_header.go @@ -21,7 +21,7 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" diff --git a/test/e2e/settings/default_ssl_certificate.go b/test/e2e/settings/default_ssl_certificate.go index c48a1e87f..421a1543b 100644 --- a/test/e2e/settings/default_ssl_certificate.go +++ b/test/e2e/settings/default_ssl_certificate.go @@ -22,7 +22,7 @@ import ( "fmt" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -30,17 +30,15 @@ import ( "k8s.io/ingress-nginx/test/e2e/framework" ) -const fooHost = "foo" - var _ = framework.IngressNginxDescribe("[SSL] [Flag] default-ssl-certificate", func() { f := framework.NewDefaultFramework("default-ssl-certificate") var tlsConfig *tls.Config - secretName := "my-custom-cert" //nolint:gosec // Ignore the gosec error in testing + secretName := "my-custom-cert" service := framework.EchoService port := 80 ginkgo.BeforeEach(func() { - f.NewEchoDeployment(framework.WithDeploymentReplicas(1)) + f.NewEchoDeploymentWithReplicas(1) var err error tlsConfig, err = framework.CreateIngressTLSSecret(f.KubeClientSet, @@ -80,7 +78,7 @@ var _ = framework.IngressNginxDescribe("[SSL] [Flag] default-ssl-certificate", f }) ginkgo.It("uses default ssl certificate for host based ingress when configured certificate does not match host", func() { - host := fooHost + host := "foo" ing := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, service, port, nil)) _, err := framework.CreateIngressTLSSecret(f.KubeClientSet, diff --git a/test/e2e/settings/disable_catch_all.go b/test/e2e/settings/disable_catch_all.go index 4e7a16f4d..1a8791d80 100644 --- a/test/e2e/settings/disable_catch_all.go +++ b/test/e2e/settings/disable_catch_all.go @@ -21,11 +21,12 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -34,7 +35,7 @@ var _ = framework.IngressNginxDescribe("[Flag] disable-catch-all", func() { f := framework.NewDefaultFramework("disabled-catch-all") ginkgo.BeforeEach(func() { - f.NewEchoDeployment(framework.WithDeploymentReplicas(1)) + f.NewEchoDeploymentWithReplicas(1) err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { args := deployment.Spec.Template.Spec.Containers[0].Args @@ -48,7 +49,7 @@ var _ = framework.IngressNginxDescribe("[Flag] disable-catch-all", func() { }) ginkgo.It("should ignore catch all Ingress with backend", func() { - host := fooHost + host := "foo" ing := framework.NewSingleCatchAllIngress("catch-all", f.Namespace, framework.EchoService, 80, nil) f.EnsureIngress(ing) @@ -67,7 +68,7 @@ var _ = framework.IngressNginxDescribe("[Flag] disable-catch-all", func() { }) ginkgo.It("should ignore catch all Ingress with backend and rules", func() { - host := fooHost + host := "foo" ing := framework.NewSingleIngressWithBackendAndRules(host, "/", host, f.Namespace, framework.EchoService, 80, framework.EchoService, 80, nil) f.EnsureIngress(ing) @@ -79,7 +80,7 @@ var _ = framework.IngressNginxDescribe("[Flag] disable-catch-all", func() { }) ginkgo.It("should delete Ingress updated to catch-all", func() { - host := fooHost + host := "foo" ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil) f.EnsureIngress(ing) @@ -97,13 +98,9 @@ var _ = framework.IngressNginxDescribe("[Flag] disable-catch-all", func() { err := framework.UpdateIngress(f.KubeClientSet, f.Namespace, host, func(ingress *networking.Ingress) error { ingress.Spec.Rules = nil - ingress.Spec.DefaultBackend = &networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: framework.EchoService, - Port: networking.ServiceBackendPort{ - Number: int32(80), - }, - }, + ingress.Spec.Backend = &networking.IngressBackend{ + ServiceName: framework.EchoService, + ServicePort: intstr.FromInt(80), } return nil }) @@ -121,7 +118,7 @@ var _ = framework.IngressNginxDescribe("[Flag] disable-catch-all", func() { }) ginkgo.It("should allow Ingress with rules", func() { - host := fooHost + host := "foo" ing := framework.NewSingleIngress("not-catch-all", "/", host, f.Namespace, framework.EchoService, 80, nil) f.EnsureIngress(ing) diff --git a/test/e2e/settings/disable_service_external_name.go b/test/e2e/settings/disable_service_external_name.go deleted file mode 100644 index 602828089..000000000 --- a/test/e2e/settings/disable_service_external_name.go +++ /dev/null @@ -1,99 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package settings - -import ( - "context" - "net/http" - "strings" - - "k8s.io/ingress-nginx/test/e2e/framework/httpexpect" - - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.IngressNginxDescribe("[Flag] disable-service-external-name", func() { - f := framework.NewDefaultFramework( - "disabled-service-external-name", - framework.WithHTTPBunEnabled(), - ) - - ginkgo.BeforeEach(func() { - f.NewEchoDeployment(framework.WithDeploymentReplicas(2)) - - err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { - args := deployment.Spec.Template.Spec.Containers[0].Args - args = append(args, "--disable-svc-external-name=true") - deployment.Spec.Template.Spec.Containers[0].Args = args - _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - - return err - }) - assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags") - }) - - ginkgo.It("should ignore services of external-name type", func() { - nonexternalhost := "echo-svc.com" - - externalhost := "echo-external-svc.com" - - f.EnsureService(framework.BuildNIPExternalNameService(f, f.HTTPBunIP, "echo")) - - f.EnsureService(&corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "external", - Namespace: f.Namespace, - }, - Spec: corev1.ServiceSpec{ - ExternalName: f.GetNIPHost(), - Type: corev1.ServiceTypeExternalName, - }, - }) - - ingexternal := framework.NewSingleIngress(externalhost, "/", externalhost, f.Namespace, "external", 80, nil) - f.EnsureIngress(ingexternal) - - ing := framework.NewSingleIngress(nonexternalhost, "/", nonexternalhost, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ing) - - f.WaitForNginxServer(nonexternalhost, func(cfg string) bool { - return strings.Contains(cfg, "server_name echo-svc.com") - }) - - f.WaitForNginxServer(externalhost, func(cfg string) bool { - return strings.Contains(cfg, "server_name echo-external-svc.com") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", nonexternalhost). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", externalhost). - Expect(). - StatusRange(httpexpect.Status5xx) - }) -}) diff --git a/test/e2e/settings/disable_sync_events.go b/test/e2e/settings/disable_sync_events.go deleted file mode 100644 index 0d55c96e4..000000000 --- a/test/e2e/settings/disable_sync_events.go +++ /dev/null @@ -1,109 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package settings - -import ( - "context" - "fmt" - "strings" - - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" - appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.IngressNginxDescribe("[Flag] disable-sync-events", func() { - f := framework.NewDefaultFramework("disable-sync-events") - - ginkgo.It("should create sync events (default)", func() { - host := "sync-events-default" - f.NewEchoDeployment(framework.WithDeploymentReplicas(1)) - - ing := framework.NewSingleIngressWithIngressClass(host, "/", host, f.Namespace, framework.EchoService, f.IngressClass, 80, nil) - ing = f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %v", host)) - }) - - //nolint:goconst //string interpolation - events, err := f.KubeClientSet.CoreV1().Events(ing.Namespace).List(context.TODO(), metav1.ListOptions{FieldSelector: "reason=Sync,involvedObject.name=" + host}) - assert.Nil(ginkgo.GinkgoT(), err, "listing events") - - assert.NotEmpty(ginkgo.GinkgoT(), events.Items, "got events") - }) - - //nolint:dupl // Ignore dupl errors for similar test case - ginkgo.It("should create sync events", func() { - host := "disable-sync-events-false" - f.NewEchoDeployment(framework.WithDeploymentReplicas(1)) - - err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { - args := deployment.Spec.Template.Spec.Containers[0].Args - args = append(args, "--disable-sync-events=false") - deployment.Spec.Template.Spec.Containers[0].Args = args - _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - return err - }) - assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags") - - ing := framework.NewSingleIngressWithIngressClass(host, "/", host, f.Namespace, framework.EchoService, f.IngressClass, 80, nil) - ing = f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %v", host)) - }) - - events, err := f.KubeClientSet.CoreV1().Events(ing.Namespace).List(context.TODO(), metav1.ListOptions{FieldSelector: "reason=Sync,involvedObject.name=" + host}) - assert.Nil(ginkgo.GinkgoT(), err, "listing events") - - assert.NotEmpty(ginkgo.GinkgoT(), events.Items, "got events") - }) - - //nolint:dupl // Ignore dupl errors for similar test case - ginkgo.It("should not create sync events", func() { - host := "disable-sync-events-true" - f.NewEchoDeployment(framework.WithDeploymentReplicas(1)) - - err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { - args := deployment.Spec.Template.Spec.Containers[0].Args - args = append(args, "--disable-sync-events=true") - deployment.Spec.Template.Spec.Containers[0].Args = args - _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - return err - }) - assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags") - - ing := framework.NewSingleIngressWithIngressClass(host, "/", host, f.Namespace, framework.EchoService, f.IngressClass, 80, nil) - ing = f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %v", host)) - }) - - events, err := f.KubeClientSet.CoreV1().Events(ing.Namespace).List(context.TODO(), metav1.ListOptions{FieldSelector: "reason=Sync,involvedObject.name=" + host}) - assert.Nil(ginkgo.GinkgoT(), err, "listing events") - - assert.Empty(ginkgo.GinkgoT(), events.Items, "got events") - }) -}) diff --git a/test/e2e/settings/enable_real_ip.go b/test/e2e/settings/enable_real_ip.go index bf16e1ea0..027f4c1f8 100644 --- a/test/e2e/settings/enable_real_ip.go +++ b/test/e2e/settings/enable_real_ip.go @@ -21,7 +21,7 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" @@ -47,7 +47,6 @@ var _ = framework.DescribeSetting("enable-real-ip", func() { f.WaitForNginxServer(host, func(server string) bool { - //nolint:goconst //already a const return strings.Contains(server, "server_name "+host) && !strings.Contains(server, "proxy_set_header X-Forwarded-Proto $full_x_forwarded_proto;") }) @@ -65,15 +64,15 @@ var _ = framework.DescribeSetting("enable-real-ip", func() { Body(). Raw() - assert.NotContains(ginkgo.GinkgoT(), body, "host=myhost") - assert.NotContains(ginkgo.GinkgoT(), body, "x-forwarded-host=myhost") - assert.NotContains(ginkgo.GinkgoT(), body, "x-forwarded-proto=myproto") - assert.NotContains(ginkgo.GinkgoT(), body, "x-forwarded-port=1234") + assert.NotContains(ginkgo.GinkgoT(), body, fmt.Sprintf("host=myhost")) + assert.NotContains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-host=myhost")) + assert.NotContains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-proto=myproto")) + assert.NotContains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-port=1234")) assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("host=%s", host)) assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-host=%s", host)) - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-proto=http") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-port=80") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-for=1.2.3.4") + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-proto=http")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-port=80")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-for=1.2.3.4")) }) ginkgo.It("should not trust X-Forwarded-For header when setting is false", func() { @@ -102,13 +101,13 @@ var _ = framework.DescribeSetting("enable-real-ip", func() { Raw() assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("host=%s", host)) - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-port=80") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-proto=http") - assert.Contains(ginkgo.GinkgoT(), body, "x-original-forwarded-for=1.2.3.4") - assert.NotContains(ginkgo.GinkgoT(), body, "host=myhost") - assert.NotContains(ginkgo.GinkgoT(), body, "x-forwarded-host=myhost") - assert.NotContains(ginkgo.GinkgoT(), body, "x-forwarded-proto=myproto") - assert.NotContains(ginkgo.GinkgoT(), body, "x-forwarded-port=1234") - assert.NotContains(ginkgo.GinkgoT(), body, "x-forwarded-for=1.2.3.4") + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-port=80")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-proto=http")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-original-forwarded-for=1.2.3.4")) + assert.NotContains(ginkgo.GinkgoT(), body, fmt.Sprintf("host=myhost")) + assert.NotContains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-host=myhost")) + assert.NotContains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-proto=myproto")) + assert.NotContains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-port=1234")) + assert.NotContains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-for=1.2.3.4")) }) }) diff --git a/test/e2e/settings/forwarded_headers.go b/test/e2e/settings/forwarded_headers.go index 44460aca6..b010a1c75 100644 --- a/test/e2e/settings/forwarded_headers.go +++ b/test/e2e/settings/forwarded_headers.go @@ -17,17 +17,16 @@ limitations under the License. package settings import ( + "fmt" "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" ) -const forwardedHeadersHost = "forwarded-headers" - var _ = framework.DescribeSetting("use-forwarded-headers", func() { f := framework.NewDefaultFramework("forwarded-headers") @@ -39,7 +38,7 @@ var _ = framework.DescribeSetting("use-forwarded-headers", func() { }) ginkgo.It("should trust X-Forwarded headers when setting is true", func() { - host := forwardedHeadersHost + host := "forwarded-headers" f.UpdateNginxConfigMapData(setting, "true") @@ -66,12 +65,12 @@ var _ = framework.DescribeSetting("use-forwarded-headers", func() { Body(). Raw() - assert.Contains(ginkgo.GinkgoT(), body, "host=myhost") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-host=myhost") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-proto=myproto") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-scheme=myproto") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-port=1234") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-for=1.2.3.4") + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("host=myhost")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-host=myhost")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-proto=myproto")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-scheme=myproto")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-port=1234")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-for=1.2.3.4")) ginkgo.By("ensuring that first entry in X-Forwarded-Host is used as the best host") body = f.HTTPTestClient(). @@ -86,12 +85,12 @@ var _ = framework.DescribeSetting("use-forwarded-headers", func() { Body(). Raw() - assert.Contains(ginkgo.GinkgoT(), body, "host=myhost.com") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-host=myhost.com") + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("host=myhost.com")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-host=myhost.com")) }) ginkgo.It("should not trust X-Forwarded headers when setting is false", func() { - host := forwardedHeadersHost + host := "forwarded-headers" f.UpdateNginxConfigMapData(setting, "false") @@ -116,16 +115,16 @@ var _ = framework.DescribeSetting("use-forwarded-headers", func() { Body(). Raw() - assert.Contains(ginkgo.GinkgoT(), body, "host=forwarded-headers") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-port=80") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-proto=http") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-scheme=http") - assert.Contains(ginkgo.GinkgoT(), body, "x-original-forwarded-for=1.2.3.4") - assert.NotContains(ginkgo.GinkgoT(), body, "host=myhost") - assert.NotContains(ginkgo.GinkgoT(), body, "x-forwarded-host=myhost") - assert.NotContains(ginkgo.GinkgoT(), body, "x-forwarded-proto=myproto") - assert.NotContains(ginkgo.GinkgoT(), body, "x-forwarded-scheme=myproto") - assert.NotContains(ginkgo.GinkgoT(), body, "x-forwarded-port=1234") - assert.NotContains(ginkgo.GinkgoT(), body, "x-forwarded-for=1.2.3.4") + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("host=forwarded-headers")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-port=80")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-proto=http")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-scheme=http")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-original-forwarded-for=1.2.3.4")) + assert.NotContains(ginkgo.GinkgoT(), body, fmt.Sprintf("host=myhost")) + assert.NotContains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-host=myhost")) + assert.NotContains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-proto=myproto")) + assert.NotContains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-scheme=myproto")) + assert.NotContains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-port=1234")) + assert.NotContains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-for=1.2.3.4")) }) }) diff --git a/test/e2e/settings/geoip2.go b/test/e2e/settings/geoip2.go index 9c6d59dc5..37f99f216 100644 --- a/test/e2e/settings/geoip2.go +++ b/test/e2e/settings/geoip2.go @@ -17,22 +17,15 @@ limitations under the License. package settings import ( - "context" - "fmt" - "net/http" - "path/filepath" "strings" - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" + "net/http" + + "github.com/onsi/ginkgo" - appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/test/e2e/framework" ) -const testdataURL = "https://github.com/maxmind/MaxMind-DB/blob/5a0be1c0320490b8e4379dbd5295a18a648ff156/test-data/GeoLite2-Country-Test.mmdb?raw=true" - var _ = framework.DescribeSetting("Geoip2", func() { f := framework.NewDefaultFramework("geoip2") @@ -42,38 +35,13 @@ var _ = framework.DescribeSetting("Geoip2", func() { f.NewEchoDeployment() }) - ginkgo.It("should include geoip2 line in config when enabled and db file exists", func() { - edition := "GeoLite2-Country" - - err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { - args := deployment.Spec.Template.Spec.Containers[0].Args - args = append(args, "--maxmind-edition-ids="+edition) - deployment.Spec.Template.Spec.Containers[0].Args = args - _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - return err - }) - assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags") - - filename := fmt.Sprintf("/etc/ingress-controller/geoip/%s.mmdb", edition) - exec, err := f.ExecIngressPod(fmt.Sprintf(`sh -c "mkdir -p '%s' && wget -O '%s' '%s' 2>&1"`, filepath.Dir(filename), filename, testdataURL)) - framework.Logf(exec) - assert.Nil(ginkgo.GinkgoT(), err, fmt.Sprintln("error downloading test geoip2 db", filename)) - - f.UpdateNginxConfigMapData("use-geoip2", "true") - f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, fmt.Sprintf("geoip2 %s", filename)) - }) - }) - ginkgo.It("should only allow requests from specific countries", func() { ginkgo.Skip("GeoIP test are temporarily disabled") - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() f.UpdateNginxConfigMapData("use-geoip2", "true") - httpSnippetAllowingOnlyAustralia := `map $geoip2_city_country_code $blocked_country { + httpSnippetAllowingOnlyAustralia := + `map $geoip2_city_country_code $blocked_country { default 1; AU 0; }` @@ -85,7 +53,8 @@ var _ = framework.DescribeSetting("Geoip2", func() { return strings.Contains(cfg, "map $geoip2_city_country_code $blocked_country") }) - configSnippet := `if ($blocked_country) { + configSnippet := + `if ($blocked_country) { return 403; }` @@ -118,52 +87,4 @@ var _ = framework.DescribeSetting("Geoip2", func() { Expect(). Status(http.StatusOK) }) - - ginkgo.It("should up and running nginx controller using autoreload flag", func() { - edition := "GeoLite2-Country" - - err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { - args := deployment.Spec.Template.Spec.Containers[0].Args - args = append(args, "--maxmind-edition-ids="+edition) - deployment.Spec.Template.Spec.Containers[0].Args = args - _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - return err - }) - assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags") - - filename := fmt.Sprintf("/etc/ingress-controller/geoip/%s.mmdb", edition) - exec, err := f.ExecIngressPod(fmt.Sprintf(`sh -c "mkdir -p '%s' && wget -O '%s' '%s' 2>&1"`, filepath.Dir(filename), filename, testdataURL)) - framework.Logf(exec) - assert.Nil(ginkgo.GinkgoT(), err, fmt.Sprintln("error downloading test geoip2 db", filename)) - - f.SetNginxConfigMapData(map[string]string{ - "use-geoip2": "true", - "geoip2-autoreload-in-minutes": "5", - }) - - // Check Configmap Autoreload Patterns - f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, fmt.Sprintf("geoip2 %s", filename)) && - strings.Contains(cfg, "auto_reload 5m;") - }, - ) - - // Check if Nginx could up, running and routing with auto_reload configs - host := "ping.com" - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, host) && - strings.Contains(server, "location /") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK) - }) }) diff --git a/test/e2e/settings/global_access_block.go b/test/e2e/settings/global_access_block.go index 4d8a2b8e3..5268590b9 100644 --- a/test/e2e/settings/global_access_block.go +++ b/test/e2e/settings/global_access_block.go @@ -20,7 +20,7 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/settings/global_external_auth.go b/test/e2e/settings/global_external_auth.go old mode 100644 new mode 100755 index f589a63e9..7960b6ca7 --- a/test/e2e/settings/global_external_auth.go +++ b/test/e2e/settings/global_external_auth.go @@ -17,30 +17,20 @@ limitations under the License. package settings import ( - "context" "fmt" "net/http" "regexp" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" "k8s.io/ingress-nginx/test/e2e/framework" ) -const ( - disable = "false" - noAuthLocationSetting = "no-auth-locations" -) - var _ = framework.DescribeSetting("[Security] global-auth-url", func() { - f := framework.NewDefaultFramework( - "global-external-auth", - framework.WithHTTPBunEnabled(), - ) + f := framework.NewDefaultFramework("global-external-auth") host := "global-external-auth" @@ -51,18 +41,20 @@ var _ = framework.DescribeSetting("[Security] global-auth-url", func() { fooPath := "/foo" barPath := "/bar" - noAuthSetting := noAuthLocationSetting + noAuthSetting := "no-auth-locations" noAuthLocations := barPath enableGlobalExternalAuthAnnotation := "nginx.ingress.kubernetes.io/enable-global-auth" ginkgo.BeforeEach(func() { f.NewEchoDeployment() + f.NewHttpbinDeployment() }) ginkgo.Context("when global external authentication is configured", func() { + ginkgo.BeforeEach(func() { - globalExternalAuthURL := fmt.Sprintf("http://%s.%s.svc.cluster.local:80/status/401", framework.HTTPBunService, f.Namespace) + globalExternalAuthURL := fmt.Sprintf("http://%s.%s.svc.cluster.local:80/status/401", framework.HTTPBinService, f.Namespace) ginkgo.By("Adding an ingress rule for /foo") fooIng := framework.NewSingleIngress("foo-ingress", fooPath, host, f.Namespace, echoServiceName, 80, nil) @@ -89,6 +81,7 @@ var _ = framework.DescribeSetting("[Security] global-auth-url", func() { }) ginkgo.It("should return status code 401 when request any protected service", func() { + ginkgo.By("Sending a request to protected service /foo") f.HTTPTestClient(). GET(fooPath). @@ -105,6 +98,7 @@ var _ = framework.DescribeSetting("[Security] global-auth-url", func() { }) ginkgo.It("should return status code 200 when request whitelisted (via no-auth-locations) service and 401 when request protected service", func() { + ginkgo.By("Adding a no-auth-locations for /bar to configMap") f.UpdateNginxConfigMapData(noAuthSetting, noAuthLocations) f.WaitForNginxServer(host, @@ -128,9 +122,10 @@ var _ = framework.DescribeSetting("[Security] global-auth-url", func() { }) ginkgo.It("should return status code 200 when request whitelisted (via ingress annotation) service and 401 when request protected service", func() { + ginkgo.By("Adding an ingress rule for /bar with annotation enable-global-auth = false") err := framework.UpdateIngress(f.KubeClientSet, f.Namespace, "bar-ingress", func(ingress *networking.Ingress) error { - ingress.ObjectMeta.Annotations[enableGlobalExternalAuthAnnotation] = disable + ingress.ObjectMeta.Annotations[enableGlobalExternalAuthAnnotation] = "false" return nil }) assert.Nil(ginkgo.GinkgoT(), err) @@ -155,12 +150,13 @@ var _ = framework.DescribeSetting("[Security] global-auth-url", func() { Status(http.StatusOK) }) - ginkgo.It("should still return status code 200 after auth backend is deleted using cache", func() { + ginkgo.It("should still return status code 200 after auth backend is deleted using cache ", func() { + globalExternalAuthCacheKeySetting := "global-auth-cache-key" - globalExternalAuthCacheKey := fooHost + globalExternalAuthCacheKey := "foo" globalExternalAuthCacheDurationSetting := "global-auth-cache-duration" globalExternalAuthCacheDuration := "200 201 401 30m" - globalExternalAuthURL := fmt.Sprintf("http://%s.%s.svc.cluster.local:80/status/200", framework.HTTPBunService, f.Namespace) + globalExternalAuthURL := fmt.Sprintf("http://%s.%s.svc.cluster.local:80/status/200", framework.HTTPBinService, f.Namespace) ginkgo.By("Adding a global-auth-cache-key to configMap") f.SetNginxConfigMapData(map[string]string{ @@ -184,7 +180,7 @@ var _ = framework.DescribeSetting("[Security] global-auth-url", func() { Expect(). Status(http.StatusOK) - err := f.DeleteDeployment(framework.HTTPBunService) + err := f.DeleteDeployment(framework.HTTPBinService) assert.Nil(ginkgo.GinkgoT(), err) framework.Sleep() @@ -197,6 +193,7 @@ var _ = framework.DescribeSetting("[Security] global-auth-url", func() { }) ginkgo.It(`should proxy_method method when global-auth-method is configured`, func() { + globalExternalAuthMethodSetting := "global-auth-method" globalExternalAuthMethod := "GET" @@ -209,6 +206,7 @@ var _ = framework.DescribeSetting("[Security] global-auth-url", func() { }) ginkgo.It(`should add custom error page when global-auth-signin url is configured`, func() { + globalExternalAuthSigninSetting := "global-auth-signin" globalExternalAuthSignin := "http://foo.com/global-error-page" @@ -221,6 +219,7 @@ var _ = framework.DescribeSetting("[Security] global-auth-url", func() { }) ginkgo.It(`should add auth headers when global-auth-response-headers is configured`, func() { + globalExternalAuthResponseHeadersSetting := "global-auth-response-headers" globalExternalAuthResponseHeaders := "Foo, Bar" @@ -234,6 +233,7 @@ var _ = framework.DescribeSetting("[Security] global-auth-url", func() { }) ginkgo.It(`should set request-redirect when global-auth-request-redirect is configured`, func() { + globalExternalAuthRequestRedirectSetting := "global-auth-request-redirect" globalExternalAuthRequestRedirect := "Foo-Redirect" @@ -256,102 +256,7 @@ var _ = framework.DescribeSetting("[Security] global-auth-url", func() { return strings.Contains(server, globalExternalAuthSnippet) }) }) + }) - ginkgo.Context("cookie set by external authentication server", func() { - host := "global-external-auth-check-cookies" - var ing1, ing2 *networking.Ingress - - cfg := `# -events { - worker_connections 1024; - multi_accept on; -} - -http { - default_type 'text/plain'; - client_max_body_size 0; - - server { - access_log on; - access_log /dev/stdout; - - listen 80; - - location ~ ^/cookies/set/(?.*)/(?.*) { - content_by_lua_block { - ngx.header['Set-Cookie'] = {ngx.var.key.."="..ngx.var.value} - ngx.say("OK") - } - } - - location / { - return 200; - } - - location /error { - return 503; - } - } -} -` - ginkgo.BeforeEach(func() { - f.NGINXWithConfigDeployment("http-cookie-with-error", cfg) - - e, err := f.KubeClientSet.CoreV1().Endpoints(f.Namespace).Get(context.TODO(), "http-cookie-with-error", metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets), 1, "expected at least one endpoint") - assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets[0].Addresses), 1, "expected at least one address ready in the endpoint") - - nginxIP := e.Subsets[0].Addresses[0].IP - - f.UpdateNginxConfigMapData(globalExternalAuthURLSetting, fmt.Sprintf("http://%s/cookies/set/alma/armud", nginxIP)) - - ing1 = framework.NewSingleIngress(host, "/", host, f.Namespace, "http-cookie-with-error", 80, nil) - f.EnsureIngress(ing1) - - ing2 = framework.NewSingleIngress(host+"-error", "/error", host, f.Namespace, "http-cookie-with-error", 80, nil) - f.EnsureIngress(ing2) - - f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "server_name "+host) - }) - }) - - ginkgo.It("user retains cookie by default", func() { - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithQuery("a", "b"). - WithQuery("c", "d"). - Expect(). - Status(http.StatusOK). - Header("Set-Cookie").Contains("alma=armud") - }) - - ginkgo.It("user does not retain cookie if upstream returns error status code", func() { - f.HTTPTestClient(). - GET("/error"). - WithHeader("Host", host). - WithQuery("a", "b"). - WithQuery("c", "d"). - Expect(). - Status(http.StatusServiceUnavailable). - Header("Set-Cookie").Contains("") - }) - - ginkgo.It("user with global-auth-always-set-cookie key in configmap retains cookie if upstream returns error status code", func() { - f.UpdateNginxConfigMapData("global-auth-always-set-cookie", "true") - - f.HTTPTestClient(). - GET("/error"). - WithHeader("Host", host). - WithQuery("a", "b"). - WithQuery("c", "d"). - Expect(). - Status(http.StatusServiceUnavailable). - Header("Set-Cookie").Contains("alma=armud") - }) - }) }) diff --git a/test/e2e/settings/global_options.go b/test/e2e/settings/global_options.go deleted file mode 100644 index 1f84ef5d7..000000000 --- a/test/e2e/settings/global_options.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package settings - -import ( - "fmt" - "strings" - "syscall" - - "github.com/onsi/ginkgo/v2" - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.IngressNginxDescribe("global-options", func() { - f := framework.NewDefaultFramework("global-options") - - ginkgo.It("should have worker_rlimit_nofile option", func() { - f.WaitForNginxConfiguration(func(server string) bool { - return strings.Contains(server, fmt.Sprintf("worker_rlimit_nofile %d;", rlimitMaxNumFiles()-1024)) - }) - }) - - ginkgo.It("should have worker_rlimit_nofile option and be independent on amount of worker processes", func() { - f.SetNginxConfigMapData(map[string]string{ - "worker-processes": "11", - }) - - f.WaitForNginxConfiguration(func(server string) bool { - return strings.Contains(server, "worker_processes 11;") && - strings.Contains(server, fmt.Sprintf("worker_rlimit_nofile %d;", rlimitMaxNumFiles()-1024)) - }) - }) -}) - -// rlimitMaxNumFiles returns hard limit for RLIMIT_NOFILE -func rlimitMaxNumFiles() int { - var rLimit syscall.Rlimit - err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit) - if err != nil { - return 0 - } - return int(rLimit.Max) -} diff --git a/test/e2e/settings/globalratelimit.go b/test/e2e/settings/globalratelimit.go new file mode 100644 index 000000000..409cd5d9c --- /dev/null +++ b/test/e2e/settings/globalratelimit.go @@ -0,0 +1,96 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package settings + +import ( + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/onsi/ginkgo" + "github.com/stretchr/testify/assert" + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.DescribeSetting("settings-global-rate-limit", func() { + f := framework.NewDefaultFramework("global-rate-limit") + host := "global-rate-limit" + + ginkgo.BeforeEach(func() { + f.NewEchoDeployment() + }) + + ginkgo.It("generates correct NGINX configuration", func() { + annotations := make(map[string]string) + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) + f.EnsureIngress(ing) + + ginkgo.By("generating correct defaults") + + ngxCfg := "" + f.WaitForNginxConfiguration(func(cfg string) bool { + if strings.Contains(cfg, "global_throttle") { + ngxCfg = cfg + return true + } + return false + }) + + assert.Contains(ginkgo.GinkgoT(), ngxCfg, fmt.Sprintf(`global_throttle = { `+ + `memcached = { host = "%v", port = %d, connect_timeout = %d, max_idle_timeout = %d, `+ + `pool_size = %d, }, status_code = %d, }`, + "", 11211, 50, 10000, 50, 429)) + + f.HTTPTestClient().GET("/").WithHeader("Host", host).Expect().Status(http.StatusOK) + + ginkgo.By("applying customizations") + + memcachedHost := "memc.default.svc.cluster.local" + memcachedPort := 11211 + memcachedConnectTimeout := 100 + memcachedMaxIdleTimeout := 5000 + memcachedPoolSize := 100 + statusCode := 503 + + f.SetNginxConfigMapData(map[string]string{ + "global-rate-limit-memcached-host": memcachedHost, + "global-rate-limit-memcached-port": strconv.Itoa(memcachedPort), + "global-rate-limit-memcached-connect-timeout": strconv.Itoa(memcachedConnectTimeout), + "global-rate-limit-memcached-max-idle-timeout": strconv.Itoa(memcachedMaxIdleTimeout), + "global-rate-limit-memcached-pool-size": strconv.Itoa(memcachedPoolSize), + "global-rate-limit-status-code": strconv.Itoa(statusCode), + }) + + ngxCfg = "" + f.WaitForNginxConfiguration(func(cfg string) bool { + if strings.Contains(cfg, "global_throttle") { + ngxCfg = cfg + return true + } + return false + }) + + assert.Contains(ginkgo.GinkgoT(), ngxCfg, fmt.Sprintf(`global_throttle = { `+ + `memcached = { host = "%v", port = %d, connect_timeout = %d, max_idle_timeout = %d, `+ + `pool_size = %d, }, status_code = %d, }`, + memcachedHost, memcachedPort, memcachedConnectTimeout, memcachedMaxIdleTimeout, + memcachedPoolSize, statusCode)) + + f.HTTPTestClient().GET("/").WithHeader("Host", host).Expect().Status(http.StatusOK) + }) +}) diff --git a/test/e2e/settings/grpc.go b/test/e2e/settings/grpc.go deleted file mode 100644 index ae3175034..000000000 --- a/test/e2e/settings/grpc.go +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright 2024 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package settings - -import ( - "context" - "crypto/tls" - "fmt" - "strings" - - pb "github.com/moul/pb/grpcbin/go-grpc" - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -const echoHost = "echo" - -var _ = framework.DescribeSetting("GRPC", func() { - f := framework.NewDefaultFramework("grpc-buffersize", framework.WithHTTPBunEnabled()) - - ginkgo.It("should set the correct GRPC Buffer Size", func() { - f.SetNginxConfigMapData(map[string]string{ - "grpc-buffer-size-kb": "8", - }) - - f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, "grpc_buffer_size 8k") - }) - - f.NewGRPCBinDeployment() - - host := echoHost - - svc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "grpcbin-test", - Namespace: f.Namespace, - }, - Spec: corev1.ServiceSpec{ - ExternalName: fmt.Sprintf("grpcbin.%v.svc.cluster.local", f.Namespace), - Type: corev1.ServiceTypeExternalName, - Ports: []corev1.ServicePort{ - { - Name: host, - Port: 9000, - TargetPort: intstr.FromInt(9000), - Protocol: "TCP", - }, - }, - }, - } - f.EnsureService(svc) - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/backend-protocol": "GRPC", - } - - ing := framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, "grpcbin-test", 9000, annotations) - - f.EnsureIngress(ing) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "grpc_pass grpc://upstream_balancer;") - }) - - conn, err := grpc.NewClient(f.GetNginxIP()+":443", - grpc.WithTransportCredentials( - credentials.NewTLS(&tls.Config{ - ServerName: echoHost, - InsecureSkipVerify: true, //nolint:gosec // Ignore certificate validation in testing - }), - ), - ) - assert.Nil(ginkgo.GinkgoT(), err, "error creating a connection") - defer conn.Close() - - client := pb.NewGRPCBinClient(conn) - ctx := context.Background() - - res, err := client.HeadersUnary(ctx, &pb.EmptyMessage{}) - assert.Nil(ginkgo.GinkgoT(), err) - - metadata := res.GetMetadata() - assert.Equal(ginkgo.GinkgoT(), metadata["content-type"].Values[0], "application/grpc") - assert.Equal(ginkgo.GinkgoT(), metadata[":authority"].Values[0], host) - }) -}) diff --git a/test/e2e/settings/gzip.go b/test/e2e/settings/gzip.go deleted file mode 100644 index c7e580e07..000000000 --- a/test/e2e/settings/gzip.go +++ /dev/null @@ -1,193 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package settings - -import ( - "fmt" - "net/http" - "strings" - - "github.com/onsi/ginkgo/v2" - - "k8s.io/ingress-nginx/internal/ingress/controller/config" - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.DescribeSetting("gzip", func() { - f := framework.NewDefaultFramework("gzip") - - host := "gzip" - - ginkgo.BeforeEach(func() { - f.NewHttpbunDeployment() - f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.Namespace, framework.HTTPBunService, 80, nil)) - }) - - ginkgo.It("should be disabled by default", func() { - f.WaitForNginxConfiguration( - func(cfg string) bool { - return !strings.Contains(cfg, "gzip on;") - }, - ) - - f.HTTPTestClient(). - GET("/xml"). - WithHeader("Host", host). - WithHeader("Accept-Encoding", "gzip"). - Expect(). - Status(http.StatusOK). - ContentEncoding() - }) - - ginkgo.It("should be enabled with default settings", func() { - f.UpdateNginxConfigMapData("use-gzip", "true") - - f.WaitForNginxConfiguration( - func(cfg string) bool { - defaultCfg := config.NewDefault() - return strings.Contains(cfg, "gzip on;") && - strings.Contains(cfg, fmt.Sprintf("gzip_comp_level %d;", defaultCfg.GzipLevel)) && - !strings.Contains(cfg, "gzip_disable") && - strings.Contains(cfg, "gzip_http_version 1.1;") && - strings.Contains(cfg, fmt.Sprintf("gzip_min_length %d;", defaultCfg.GzipMinLength)) && - strings.Contains(cfg, fmt.Sprintf("gzip_types %s;", defaultCfg.GzipTypes)) && - strings.Contains(cfg, "gzip_proxied any;") && - strings.Contains(cfg, "gzip_vary on;") - }, - ) - - f.HTTPTestClient(). - GET("/xml"). - WithHeader("Host", host). - WithHeader("Accept-Encoding", "gzip"). - Expect(). - Status(http.StatusOK). - ContentEncoding("gzip") - }) - - ginkgo.It("should set gzip_comp_level to 4", func() { - f.UpdateNginxConfigMapData("use-gzip", "true") - f.UpdateNginxConfigMapData("gzip-level", "4") - - f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, "gzip on;") && - strings.Contains(cfg, "gzip_comp_level 4;") - }, - ) - - f.HTTPTestClient(). - GET("/xml"). - WithHeader("Host", host). - WithHeader("Accept-Encoding", "gzip"). - Expect(). - Status(http.StatusOK). - ContentEncoding("gzip") - }) - - ginkgo.It("should set gzip_disable to msie6", func() { - f.UpdateNginxConfigMapData("use-gzip", "true") - f.UpdateNginxConfigMapData("gzip-disable", "msie6") - - f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, "gzip on;") && - strings.Contains(cfg, `gzip_disable "msie6";`) - }, - ) - - f.HTTPTestClient(). - GET("/xml"). - WithHeader("Host", host). - WithHeader("Accept-Encoding", "gzip"). - WithHeader("User-Agent", "Mozilla/4.8 [en] (Windows NT 5.1; U)"). - Expect(). - Status(http.StatusOK). - ContentEncoding("gzip") - - f.HTTPTestClient(). - GET("/xml"). - WithHeader("Host", host). - WithHeader("Accept-Encoding", "gzip"). - WithHeader("User-Agent", "Mozilla/45.0 (compatible; MSIE 6.0; Windows NT 5.1)"). - Expect(). - Status(http.StatusOK). - ContentEncoding() - }) - - ginkgo.It("should set gzip_min_length to 100", func() { - f.UpdateNginxConfigMapData("use-gzip", "true") - f.UpdateNginxConfigMapData("gzip-min-length", "100") - f.UpdateNginxConfigMapData("gzip-types", "application/octet-stream") - - f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, "gzip on;") && - strings.Contains(cfg, "gzip_min_length 100;") && - strings.Contains(cfg, "gzip_types application/octet-stream;") - }, - ) - - f.HTTPTestClient(). - GET("/bytes/99"). - WithHeader("Host", host). - WithHeader("Accept-Encoding", "gzip"). - Expect(). - Status(http.StatusOK). - ContentType("application/octet-stream"). - ContentEncoding() - - f.HTTPTestClient(). - GET("/bytes/100"). - WithHeader("Host", host). - WithHeader("Accept-Encoding", "gzip"). - Expect(). - Status(http.StatusOK). - ContentType("application/octet-stream"). - ContentEncoding("gzip") - }) - - ginkgo.It("should set gzip_types to text/html", func() { - f.UpdateNginxConfigMapData("use-gzip", "true") - f.UpdateNginxConfigMapData("gzip-types", "text/html") - - f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, "gzip on;") && - strings.Contains(cfg, "gzip_types text/html;") - }, - ) - - f.HTTPTestClient(). - GET("/xml"). - WithHeader("Host", host). - WithHeader("Accept-Encoding", "gzip"). - Expect(). - Status(http.StatusOK). - ContentType("application/xml"). - ContentEncoding() - - f.HTTPTestClient(). - GET("/html"). - WithHeader("Host", host). - WithHeader("Accept-Encoding", "gzip"). - Expect(). - Status(http.StatusOK). - ContentType("text/html"). - ContentEncoding("gzip") - }) -}) diff --git a/test/e2e/settings/hash-size.go b/test/e2e/settings/hash-size.go index 5aa5f7c95..42c9241bb 100644 --- a/test/e2e/settings/hash-size.go +++ b/test/e2e/settings/hash-size.go @@ -19,7 +19,7 @@ package settings import ( "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -36,6 +36,7 @@ var _ = framework.DescribeSetting("hash size", func() { }) ginkgo.Context("Check server names hash size", func() { + ginkgo.It("should set server_names_hash_bucket_size", func() { f.UpdateNginxConfigMapData("server-name-hash-bucket-size", "512") @@ -51,9 +52,11 @@ var _ = framework.DescribeSetting("hash size", func() { return strings.Contains(server, "server_names_hash_max_size 4096;") }) }) + }) ginkgo.Context("Check proxy header hash size", func() { + ginkgo.It("should set proxy-headers-hash-bucket-size", func() { f.UpdateNginxConfigMapData("proxy-headers-hash-bucket-size", "512") @@ -69,9 +72,11 @@ var _ = framework.DescribeSetting("hash size", func() { return strings.Contains(server, "proxy_headers_hash_max_size 4096;") }) }) + }) ginkgo.Context("Check the variable hash size", func() { + ginkgo.It("should set variables-hash-bucket-size", func() { f.UpdateNginxConfigMapData("variables-hash-bucket-size", "512") @@ -87,9 +92,11 @@ var _ = framework.DescribeSetting("hash size", func() { return strings.Contains(server, "variables_hash_max_size 512;") }) }) + }) ginkgo.Context("Check the map hash size", func() { + ginkgo.It("should set vmap-hash-bucket-size", func() { f.UpdateNginxConfigMapData("map-hash-bucket-size", "512") @@ -97,5 +104,7 @@ var _ = framework.DescribeSetting("hash size", func() { return strings.Contains(server, "map_hash_bucket_size 512;") }) }) + }) + }) diff --git a/test/e2e/settings/ingress_class.go b/test/e2e/settings/ingress_class.go index 80c09f80c..1fa02183f 100644 --- a/test/e2e/settings/ingress_class.go +++ b/test/e2e/settings/ingress_class.go @@ -23,40 +23,61 @@ import ( "strings" "sync" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" - networkingv1 "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" rbacv1 "k8s.io/api/rbac/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/ingress-nginx/internal/ingress/controller/ingressclass" + "k8s.io/ingress-nginx/internal/ingress/annotations/class" + "k8s.io/ingress-nginx/internal/k8s" "k8s.io/ingress-nginx/test/e2e/framework" ) -const barHost = "bar" - var _ = framework.IngressNginxDescribe("[Flag] ingress-class", func() { f := framework.NewDefaultFramework("ingress-class") var doOnce sync.Once - otherIngressClassName := "test-new-ingress-class" - otherController := "k8s.io/other-class" + testIngressClassName := "test-new-ingress-class" ginkgo.BeforeEach(func() { - f.NewEchoDeployment(framework.WithDeploymentReplicas(1)) + f.NewEchoDeploymentWithReplicas(1) doOnce.Do(func() { - _, err := f.KubeClientSet.NetworkingV1().IngressClasses(). - Create(context.TODO(), &networkingv1.IngressClass{ + f.KubeClientSet.RbacV1().ClusterRoles().Create(context.TODO(), &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{Name: "ingress-nginx-class"}, + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"ingressclasses"}, + Verbs: []string{"get", "list", "watch"}, + }}, + }, metav1.CreateOptions{}) + + f.KubeClientSet.RbacV1().ClusterRoleBindings().Create(context.TODO(), &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ingress-nginx-class", + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: "ingress-nginx-class", + }, + }, metav1.CreateOptions{}) + + if !f.IsIngressV1Beta1Ready { + return + } + + _, err := f.KubeClientSet.NetworkingV1beta1().IngressClasses(). + Create(context.TODO(), &networking.IngressClass{ ObjectMeta: metav1.ObjectMeta{ - Name: otherIngressClassName, + Name: testIngressClassName, }, - Spec: networkingv1.IngressClassSpec{ - Controller: otherController, + Spec: networking.IngressClassSpec{ + Controller: k8s.IngressNGINXController, }, }, metav1.CreateOptions{}) @@ -66,23 +87,17 @@ var _ = framework.IngressNginxDescribe("[Flag] ingress-class", func() { }) }) - ginkgo.Context("With default ingress class config", func() { - ginkgo.It("should ignore Ingress with a different class annotation", func() { - invalidHost := fooHost + ginkgo.Context("Without a specific ingress-class", func() { + ginkgo.It("should ignore Ingress with class", func() { + invalidHost := "foo" annotations := map[string]string{ - ingressclass.IngressKey: "testclass", + class.IngressKey: "testclass", } ing := framework.NewSingleIngress(invalidHost, "/", invalidHost, f.Namespace, framework.EchoService, 80, annotations) - // We should drop the ingressClassName here as we just want to rely on the annotation in this test - ing.Spec.IngressClassName = nil f.EnsureIngress(ing) - validHost := barHost - annotationClass := map[string]string{ - ingressclass.IngressKey: ingressclass.DefaultAnnotationValue, - } - ing = framework.NewSingleIngress(validHost, "/", validHost, f.Namespace, framework.EchoService, 80, annotationClass) - ing.Spec.IngressClassName = nil + validHost := "bar" + ing = framework.NewSingleIngress(validHost, "/", validHost, f.Namespace, framework.EchoService, 80, nil) f.EnsureIngress(ing) f.WaitForNginxConfiguration(func(cfg string) bool { @@ -102,564 +117,225 @@ var _ = framework.IngressNginxDescribe("[Flag] ingress-class", func() { Expect(). Status(http.StatusOK) }) + }) + + ginkgo.Context("With a specific ingress-class", func() { + ginkgo.BeforeEach(func() { + err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { + args := []string{} + for _, v := range deployment.Spec.Template.Spec.Containers[0].Args { + if strings.Contains(v, "--ingress-class") { + continue + } + + args = append(args, v) + } + + args = append(args, "--ingress-class=testclass") + deployment.Spec.Template.Spec.Containers[0].Args = args + _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) + + return err + }) + assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags") + }) + + ginkgo.It("should ignore Ingress with no class", func() { + invalidHost := "bar" - ginkgo.It("should ignore Ingress with different controller class", func() { - invalidHost := "foo-1" ing := framework.NewSingleIngress(invalidHost, "/", invalidHost, f.Namespace, framework.EchoService, 80, nil) - ing.Spec.IngressClassName = &otherIngressClassName f.EnsureIngress(ing) - validHost := "bar-1" - ing = framework.NewSingleIngress(validHost, "/", validHost, f.Namespace, framework.EchoService, 80, nil) + validHost := "foo" + annotations := map[string]string{ + class.IngressKey: "testclass", + } + ing = framework.NewSingleIngress(validHost, "/", validHost, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) - f.WaitForNginxConfiguration(func(cfg string) bool { - return !strings.Contains(cfg, "server_name foo-1") && - strings.Contains(cfg, "server_name bar-1") + f.WaitForNginxServer(validHost, func(cfg string) bool { + return strings.Contains(cfg, "server_name foo") }) - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", invalidHost). - Expect(). - Status(http.StatusNotFound) + f.WaitForNginxConfiguration(func(cfg string) bool { + return !strings.Contains(cfg, "server_name bar") + }) f.HTTPTestClient(). GET("/"). WithHeader("Host", validHost). Expect(). Status(http.StatusOK) - }) - - ginkgo.It("should accept both Ingresses with default IngressClassName and IngressClass annotation", func() { - validHostAnnotation := "foo-ok" - annotationClass := map[string]string{ - ingressclass.IngressKey: ingressclass.DefaultAnnotationValue, - } - ing := framework.NewSingleIngress(validHostAnnotation, "/", validHostAnnotation, f.Namespace, framework.EchoService, 80, annotationClass) - // We need to drop the Class here as we just want the annotation - ing.Spec.IngressClassName = nil - f.EnsureIngress(ing) - - validHostClass := "bar-ok" - ing = framework.NewSingleIngress(validHostClass, "/", validHostClass, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ing) - - f.WaitForNginxConfiguration(func(cfg string) bool { - return strings.Contains(cfg, "server_name foo-ok") && - strings.Contains(cfg, "server_name bar-ok") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", validHostAnnotation). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", validHostClass). - Expect(). - Status(http.StatusOK) - }) - - ginkgo.It("should ignore Ingress without IngressClass configuration", func() { - invalidHost := "foo-invalid" - ing := framework.NewSingleIngress(invalidHost, "/", invalidHost, f.Namespace, framework.EchoService, 80, nil) - ing.Spec.IngressClassName = nil - f.EnsureIngress(ing) - - validHostClass := "bar-valid" - ing = framework.NewSingleIngress(validHostClass, "/", validHostClass, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ing) - - f.WaitForNginxConfiguration(func(cfg string) bool { - return !strings.Contains(cfg, "server_name foo-invalid") && - strings.Contains(cfg, "server_name bar-valid") - }) f.HTTPTestClient(). GET("/"). WithHeader("Host", invalidHost). Expect(). Status(http.StatusNotFound) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", validHostClass). - Expect(). - Status(http.StatusOK) }) ginkgo.It("should delete Ingress when class is removed", func() { - hostAnnotation := "foo-annotation" - + host := "foo" annotations := map[string]string{ - ingressclass.IngressKey: ingressclass.DefaultAnnotationValue, + class.IngressKey: "testclass", } - ing := framework.NewSingleIngress(hostAnnotation, "/", hostAnnotation, f.Namespace, framework.EchoService, 80, annotations) - ing.Spec.IngressClassName = nil + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) - hostClass := "foo-class" - ing = framework.NewSingleIngress(hostClass, "/", hostClass, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ing) - - f.WaitForNginxConfiguration(func(cfg string) bool { - return strings.Contains(cfg, "server_name foo-annotation") && - strings.Contains(cfg, "server_name foo-class") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", hostAnnotation). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", hostClass). - Expect(). - Status(http.StatusOK) - - ing, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Get(context.TODO(), hostAnnotation, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - delete(ing.Annotations, ingressclass.IngressKey) - _, err = f.KubeClientSet.NetworkingV1().Ingresses(ing.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - ingWithClass, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Get(context.TODO(), hostClass, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - ingWithClass.Spec.IngressClassName = nil - _, err = f.KubeClientSet.NetworkingV1().Ingresses(ingWithClass.Namespace).Update(context.TODO(), ingWithClass, metav1.UpdateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - framework.Sleep() - - f.WaitForNginxConfiguration(func(cfg string) bool { - return !strings.Contains(cfg, "server_name foo-annotation") && - !strings.Contains(cfg, "server_name foo-class") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", hostAnnotation). - Expect(). - Status(http.StatusNotFound) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", hostClass). - Expect(). - Status(http.StatusNotFound) - }) - - ginkgo.It("should serve Ingress when class is added", func() { - hostNoAnnotation := "foo-no-annotation" - - ing := framework.NewSingleIngress(hostNoAnnotation, "/", hostNoAnnotation, f.Namespace, framework.EchoService, 80, nil) - ing.Spec.IngressClassName = nil - f.EnsureIngress(ing) - - hostNoClass := "foo-no-class" - ing = framework.NewSingleIngress(hostNoClass, "/", hostNoClass, f.Namespace, framework.EchoService, 80, nil) - ing.Spec.IngressClassName = nil - f.EnsureIngress(ing) - - f.WaitForNginxConfiguration(func(cfg string) bool { - return !strings.Contains(cfg, "server_name foo-no-nnotation") && - !strings.Contains(cfg, "server_name foo-no-class") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", hostNoAnnotation). - Expect(). - Status(http.StatusNotFound) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", hostNoClass). - Expect(). - Status(http.StatusNotFound) - - ing, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Get(context.TODO(), hostNoAnnotation, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - annotation := map[string]string{ - ingressclass.IngressKey: ingressclass.DefaultAnnotationValue, - } - ing.Annotations = annotation - _, err = f.KubeClientSet.NetworkingV1().Ingresses(ing.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - ingWithClass, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Get(context.TODO(), hostNoClass, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - ingWithClass.Spec.IngressClassName = framework.GetIngressClassName(f.Namespace) - _, err = f.KubeClientSet.NetworkingV1().Ingresses(ingWithClass.Namespace).Update(context.TODO(), ingWithClass, metav1.UpdateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - framework.Sleep() - - f.WaitForNginxConfiguration(func(cfg string) bool { - return strings.Contains(cfg, "server_name foo-no-annotation") && - strings.Contains(cfg, "server_name foo-no-class") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", hostNoAnnotation). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", hostNoClass). - Expect(). - Status(http.StatusOK) - }) - - ginkgo.It("should serve Ingress when class is updated between annotation and ingressClassName", func() { - hostAnnotation2class := "foo-annotation2class" - annotationClass := map[string]string{ - ingressclass.IngressKey: ingressclass.DefaultAnnotationValue, - } - ing := framework.NewSingleIngress(hostAnnotation2class, "/", hostAnnotation2class, f.Namespace, framework.EchoService, 80, annotationClass) - ing.Spec.IngressClassName = nil - f.EnsureIngress(ing) - - hostClass2Annotation := "foo-class2annotation" - ing = framework.NewSingleIngress(hostClass2Annotation, "/", hostClass2Annotation, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ing) - - f.WaitForNginxConfiguration(func(cfg string) bool { - return strings.Contains(cfg, "server_name foo-annotation2class") && - strings.Contains(cfg, "server_name foo-class2annotation") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", hostAnnotation2class). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", hostClass2Annotation). - Expect(). - Status(http.StatusOK) - - ingAnnotation2Class, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Get(context.TODO(), hostAnnotation2class, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - delete(ingAnnotation2Class.Annotations, ingressclass.IngressKey) - ingAnnotation2Class.Spec.IngressClassName = framework.GetIngressClassName(ingAnnotation2Class.Namespace) - _, err = f.KubeClientSet.NetworkingV1().Ingresses(ingAnnotation2Class.Namespace).Update(context.TODO(), ingAnnotation2Class, metav1.UpdateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - ingClass2Annotation, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Get(context.TODO(), hostClass2Annotation, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - ingClass2Annotation.Spec.IngressClassName = nil - ingClass2Annotation.Annotations = annotationClass - _, err = f.KubeClientSet.NetworkingV1().Ingresses(ingClass2Annotation.Namespace).Update(context.TODO(), ingClass2Annotation, metav1.UpdateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - framework.Sleep() - - f.WaitForNginxConfiguration(func(cfg string) bool { - return strings.Contains(cfg, "server_name foo-annotation2class") && - strings.Contains(cfg, "server_name foo-class2annotation") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", hostAnnotation2class). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", hostClass2Annotation). - Expect(). - Status(http.StatusOK) - }) - }) - - ginkgo.Context("With specific ingress-class flags", func() { - ginkgo.BeforeEach(func() { - err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { - args := []string{} - for _, v := range deployment.Spec.Template.Spec.Containers[0].Args { - if strings.Contains(v, "--ingress-class") && strings.Contains(v, "--controller-class") { - continue - } - - args = append(args, v) - } - - args = append(args, "--ingress-class=testclass") - args = append(args, "--controller-class=k8s.io/other-class") - deployment.Spec.Template.Spec.Containers[0].Args = args - _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - - return err - }) - assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags") - }) - - ginkgo.It("should ignore Ingress with no class and accept the correctly configured Ingresses", func() { - invalidHost := barHost - - ing := framework.NewSingleIngress(invalidHost, "/", invalidHost, f.Namespace, framework.EchoService, 80, nil) - ing.Spec.IngressClassName = nil - f.EnsureIngress(ing) - - validHost := fooHost - annotations := map[string]string{ - ingressclass.IngressKey: "testclass", - } - ing = framework.NewSingleIngress(validHost, "/", validHost, f.Namespace, framework.EchoService, 80, annotations) - // Delete the IngressClass as we want just the annotation here - ing.Spec.IngressClassName = nil - f.EnsureIngress(ing) - - validHostClass := "foobar123" - ing = framework.NewSingleIngress(validHostClass, "/", validHostClass, f.Namespace, framework.EchoService, 80, nil) - ing.Spec.IngressClassName = &otherIngressClassName - f.EnsureIngress(ing) - - f.WaitForNginxConfiguration(func(cfg string) bool { - return !strings.Contains(cfg, "server_name bar") && - strings.Contains(cfg, "server_name foo") && - strings.Contains(cfg, "server_name foobar123") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", validHost). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", validHostClass). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", invalidHost). - Expect(). - Status(http.StatusNotFound) - }) - }) - - ginkgo.Context("With watch-ingress-without-class flag", func() { - ginkgo.BeforeEach(func() { - err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { - args := []string{} - for _, v := range deployment.Spec.Template.Spec.Containers[0].Args { - if strings.Contains(v, "--watch-ingress-without-class") && strings.Contains(v, "--controller-class") { - continue - } - - args = append(args, v) - } - - args = append(args, "--watch-ingress-without-class") - deployment.Spec.Template.Spec.Containers[0].Args = args - _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - - return err - }) - assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags") - }) - - ginkgo.It("should watch Ingress with no class and ignore ingress with a different class", func() { - validHost := barHost - - ing := framework.NewSingleIngress(validHost, "/", validHost, f.Namespace, framework.EchoService, 80, nil) - ing.Spec.IngressClassName = nil - f.EnsureIngress(ing) - - invalidHost := fooHost - annotations := map[string]string{ - ingressclass.IngressKey: "testclass123", - } - ing = framework.NewSingleIngress(invalidHost, "/", invalidHost, f.Namespace, framework.EchoService, 80, annotations) - ing.Spec.IngressClassName = nil - f.EnsureIngress(ing) - - f.WaitForNginxConfiguration(func(cfg string) bool { - return strings.Contains(cfg, "server_name bar") && - !strings.Contains(cfg, "server_name foo") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", validHost). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", invalidHost). - Expect(). - Status(http.StatusNotFound) - }) - }) - - ginkgo.Context("With ingress-class-by-name flag", func() { - ginkgo.BeforeEach(func() { - err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { - args := []string{} - for _, v := range deployment.Spec.Template.Spec.Containers[0].Args { - if strings.Contains(v, "--ingress-class-by-name") && - strings.Contains(v, "--ingress-class=test-new-ingress-class") { - continue - } - - args = append(args, v) - } - args = append(args, "--ingress-class=test-new-ingress-class") - args = append(args, "--ingress-class-by-name") - deployment.Spec.Template.Spec.Containers[0].Args = args - _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - - return err - }) - assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags") - }) - - ginkgo.It("should watch Ingress that uses the class name even if spec is different", func() { - validHostClassName := "validhostclassname" - - ing := framework.NewSingleIngress(validHostClassName, "/", validHostClassName, f.Namespace, framework.EchoService, 80, nil) - ing.Spec.IngressClassName = &otherIngressClassName - f.EnsureIngress(ing) - - validHostClass := "validhostclassspec" - ing = framework.NewSingleIngress(validHostClass, "/", validHostClass, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ing) - - invalidHost := "invalidannotation" - annotations := map[string]string{ - ingressclass.IngressKey: "testclass123", - } - ing = framework.NewSingleIngress(invalidHost, "/", invalidHost, f.Namespace, framework.EchoService, 80, annotations) - ing.Spec.IngressClassName = nil - f.EnsureIngress(ing) - - f.WaitForNginxConfiguration(func(cfg string) bool { - return strings.Contains(cfg, "server_name validhostclassname") && - strings.Contains(cfg, "server_name validhostclassspec") && - !strings.Contains(cfg, "server_name invalidannotation") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", validHostClass). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", validHostClassName). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", invalidHost). - Expect(). - Status(http.StatusNotFound) - }) - }) - - ginkgo.Context("Without IngressClass Cluster scoped Permission", func() { - ginkgo.BeforeEach(func() { - icname := fmt.Sprintf("ic-%s", f.Namespace) - - newRole := &rbacv1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: icname, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: icname, - }, - Subjects: []rbacv1.Subject{ - { - APIGroup: "", - Kind: "ServiceAccount", - Namespace: f.Namespace, - Name: "blablabla", - }, - }, - } - _, err := f.KubeClientSet.RbacV1().ClusterRoleBindings().Update(context.TODO(), newRole, metav1.UpdateOptions{}) - - assert.Nil(ginkgo.GinkgoT(), err, "Updating IngressClass ClusterRoleBinding") - - // Force the correct annotation value just for the re-deployment - err = f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { - args := []string{} - for _, v := range deployment.Spec.Template.Spec.Containers[0].Args { - if strings.Contains(v, "--ingress-class=testclass") { - continue - } - - args = append(args, v) - } - args = append(args, "--ingress-class=testclass") - deployment.Spec.Template.Spec.Containers[0].Args = args - _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - - return err - }) - assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags") - }) - - ginkgo.It("should watch Ingress with correct annotation", func() { - validHost := fooHost - annotations := map[string]string{ - ingressclass.IngressKey: "testclass", - } - ing := framework.NewSingleIngress(validHost, "/", validHost, f.Namespace, framework.EchoService, 80, annotations) - ing.Spec.IngressClassName = nil - f.EnsureIngress(ing) - - f.WaitForNginxConfiguration(func(cfg string) bool { + f.WaitForNginxServer(host, func(cfg string) bool { return strings.Contains(cfg, "server_name foo") }) f.HTTPTestClient(). GET("/"). - WithHeader("Host", validHost). + WithHeader("Host", host). Expect(). Status(http.StatusOK) - }) - ginkgo.It("should ignore Ingress with only IngressClassName", func() { - invalidHost := "noclassforyou" + ing, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err) - ing := framework.NewSingleIngress(invalidHost, "/", invalidHost, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ing) + delete(ing.Annotations, class.IngressKey) + _, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(ing.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{}) + assert.Nil(ginkgo.GinkgoT(), err) + + framework.Sleep() f.WaitForNginxConfiguration(func(cfg string) bool { - return !strings.Contains(cfg, "server_name noclassforyou") + return !strings.Contains(cfg, "server_name foo") }) f.HTTPTestClient(). GET("/"). - WithHeader("Host", invalidHost). + WithHeader("Host", host). Expect(). Status(http.StatusNotFound) }) }) + + ginkgo.It("check scenarios for IngressClass and ingress.class annotation", func() { + if !f.IsIngressV1Beta1Ready { + ginkgo.Skip("Test requires Kubernetes v1.18 or higher") + } + + pod := f.GetIngressNGINXPod() + + crb, err := f.KubeClientSet.RbacV1().ClusterRoleBindings().Get(context.Background(), "ingress-nginx-class", metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "searching cluster role binding") + + // add service of current namespace + crb.Subjects = append(crb.Subjects, rbacv1.Subject{ + APIGroup: "", + Kind: "ServiceAccount", + Name: pod.Spec.ServiceAccountName, + Namespace: f.Namespace, + }) + + _, err = f.KubeClientSet.RbacV1().ClusterRoleBindings().Update(context.Background(), crb, metav1.UpdateOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "searching cluster role binding") + + err = f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { + args := []string{} + for _, v := range deployment.Spec.Template.Spec.Containers[0].Args { + if strings.Contains(v, "--ingress-class") { + continue + } + + args = append(args, v) + } + + args = append(args, fmt.Sprintf("--ingress-class=%v", testIngressClassName)) + deployment.Spec.Template.Spec.Containers[0].Args = args + _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) + return err + }) + assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags") + + host := "ingress.class" + + ginkgo.By("only having IngressClassName") + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil) + ing.Spec.IngressClassName = &testIngressClassName + f.EnsureIngress(ing) + + f.WaitForNginxConfiguration(func(cfg string) bool { + return strings.Contains(cfg, fmt.Sprintf("server_name %v", host)) + }) + + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", host). + Expect(). + Status(http.StatusOK) + + ginkgo.By("only having ingress.class annotation") + ing, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err) + + ing.Annotations = map[string]string{ + class.IngressKey: testIngressClassName, + } + ing.Spec.IngressClassName = nil + + _, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{}) + assert.Nil(ginkgo.GinkgoT(), err) + + f.WaitForNginxConfiguration(func(cfg string) bool { + return strings.Contains(cfg, fmt.Sprintf("server_name %v", host)) + }) + + framework.Sleep() + + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", host). + Expect(). + Status(http.StatusOK) + + ginkgo.By("having an invalid ingress.class annotation and no IngressClassName") + ing, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err) + + ing.Annotations = map[string]string{ + class.IngressKey: "invalid", + } + ing.Spec.IngressClassName = nil + + _, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{}) + assert.Nil(ginkgo.GinkgoT(), err) + + framework.Sleep() + + f.WaitForNginxConfiguration(func(cfg string) bool { + return !strings.Contains(cfg, fmt.Sprintf("server_name %v", host)) + }) + + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", host). + Expect(). + Status(http.StatusNotFound) + + ginkgo.By("not having ingress.class annotation and invalid IngressClassName") + ing, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err) + ing.Annotations = map[string]string{} + invalidClassName := "invalidclass" + ing.Spec.IngressClassName = &invalidClassName + + _, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{}) + assert.Nil(ginkgo.GinkgoT(), err) + + framework.Sleep() + + f.WaitForNginxConfiguration(func(cfg string) bool { + return !strings.Contains(cfg, fmt.Sprintf("server_name %v", host)) + }) + + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", host). + Expect(). + Status(http.StatusNotFound) + }) }) diff --git a/test/e2e/settings/keep-alive.go b/test/e2e/settings/keep-alive.go index 167f5ac18..5a2b5189e 100644 --- a/test/e2e/settings/keep-alive.go +++ b/test/e2e/settings/keep-alive.go @@ -17,11 +17,11 @@ limitations under the License. package settings import ( + "fmt" "regexp" "strings" - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -41,7 +41,7 @@ var _ = framework.DescribeSetting("keep-alive keep-alive-requests", func() { f.UpdateNginxConfigMapData("keep-alive", "140") f.WaitForNginxConfiguration(func(server string) bool { - return strings.Contains(server, `keepalive_timeout 140s;`) + return strings.Contains(server, fmt.Sprintf(`keepalive_timeout 140s;`)) }) }) @@ -49,8 +49,9 @@ var _ = framework.DescribeSetting("keep-alive keep-alive-requests", func() { f.UpdateNginxConfigMapData("keep-alive-requests", "200") f.WaitForNginxConfiguration(func(server string) bool { - return strings.Contains(server, `keepalive_requests 200;`) + return strings.Contains(server, fmt.Sprintf(`keepalive_requests 200;`)) }) + }) }) @@ -59,8 +60,7 @@ var _ = framework.DescribeSetting("keep-alive keep-alive-requests", func() { f.UpdateNginxConfigMapData("upstream-keepalive-connections", "128") f.WaitForNginxConfiguration(func(server string) bool { - match, err := regexp.MatchString(`upstream\supstream_balancer\s\{[\s\S]*keepalive 128;`, server) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error matching the upstream keepalive time") + match, _ := regexp.MatchString(`upstream\supstream_balancer\s\{[\s\S]*keepalive 128;`, server) return match }) }) @@ -69,18 +69,7 @@ var _ = framework.DescribeSetting("keep-alive keep-alive-requests", func() { f.UpdateNginxConfigMapData("upstream-keepalive-timeout", "120") f.WaitForNginxConfiguration(func(server string) bool { - match, err := regexp.MatchString(`upstream\supstream_balancer\s\{[\s\S]*keepalive_timeout\s*120s;`, server) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error matching the upstream keepalive time") - return match - }) - }) - - ginkgo.It("should set keepalive time to upstream server", func() { - f.UpdateNginxConfigMapData("upstream-keepalive-time", "75s") - - f.WaitForNginxConfiguration(func(server string) bool { - match, err := regexp.MatchString(`upstream\supstream_balancer\s\{[\s\S]*keepalive_time\s*75s;`, server) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error matching the upstream keepalive time") + match, _ := regexp.MatchString(`upstream\supstream_balancer\s\{[\s\S]*keepalive_timeout\s*120s;`, server) return match }) }) @@ -89,8 +78,7 @@ var _ = framework.DescribeSetting("keep-alive keep-alive-requests", func() { f.UpdateNginxConfigMapData("upstream-keepalive-requests", "200") f.WaitForNginxConfiguration(func(server string) bool { - match, err := regexp.MatchString(`upstream\supstream_balancer\s\{[\s\S]*keepalive_requests\s*200;`, server) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error matching the upstream keepalive time") + match, _ := regexp.MatchString(`upstream\supstream_balancer\s\{[\s\S]*keepalive_requests\s*200;`, server) return match }) }) diff --git a/test/e2e/settings/limit_rate.go b/test/e2e/settings/limit_rate.go index 9d79dc358..d51408335 100644 --- a/test/e2e/settings/limit_rate.go +++ b/test/e2e/settings/limit_rate.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/settings/listen_nondefault_ports.go b/test/e2e/settings/listen_nondefault_ports.go index 9d3952227..28e0b86e9 100644 --- a/test/e2e/settings/listen_nondefault_ports.go +++ b/test/e2e/settings/listen_nondefault_ports.go @@ -17,20 +17,23 @@ limitations under the License. package settings import ( + "context" "fmt" "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/test/e2e/framework" ) var _ = framework.IngressNginxDescribe("[Flag] custom HTTP and HTTPS ports", func() { + host := "forwarded-headers" - f := framework.NewDefaultFramework("forwarded-port-headers", framework.WithHTTPBunEnabled()) + f := framework.NewDefaultFramework("forwarded-port-headers") ginkgo.BeforeEach(func() { f.NewEchoDeployment() @@ -43,6 +46,7 @@ var _ = framework.IngressNginxDescribe("[Flag] custom HTTP and HTTPS ports", fun ginkgo.Context("with a plain HTTP ingress", func() { ginkgo.It("should set X-Forwarded-Port headers accordingly when listening on a non-default HTTP port", func() { + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil) f.EnsureIngress(ing) @@ -62,7 +66,9 @@ var _ = framework.IngressNginxDescribe("[Flag] custom HTTP and HTTPS ports", fun }) ginkgo.Context("with a TLS enabled ingress", func() { + ginkgo.It("should set X-Forwarded-Port header to 443", func() { + ing := framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, framework.EchoService, 80, nil) f.EnsureIngress(ing) @@ -86,13 +92,27 @@ var _ = framework.IngressNginxDescribe("[Flag] custom HTTP and HTTPS ports", fun Expect(). Status(http.StatusOK). Body(). - Contains("x-forwarded-port=443") + Contains(fmt.Sprintf("x-forwarded-port=443")) }) ginkgo.Context("when external authentication is configured", func() { + ginkgo.It("should set the X-Forwarded-Port header to 443", func() { + f.NewHttpbinDeployment() + + err := framework.WaitForEndpoints(f.KubeClientSet, framework.DefaultTimeout, framework.HTTPBinService, f.Namespace, 1) + assert.Nil(ginkgo.GinkgoT(), err) + + e, err := f.KubeClientSet.CoreV1().Endpoints(f.Namespace).Get(context.TODO(), framework.HTTPBinService, metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err) + + assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets), 1, "expected at least one endpoint") + assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets[0].Addresses), 1, "expected at least one address ready in the endpoint") + + httpbinIP := e.Subsets[0].Addresses[0].IP + annotations := map[string]string{ - "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", f.HTTPBunIP), + "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", httpbinIP), "nginx.ingress.kubernetes.io/auth-signin": "http://$host/auth/start", } @@ -121,7 +141,7 @@ var _ = framework.IngressNginxDescribe("[Flag] custom HTTP and HTTPS ports", fun Expect(). Status(http.StatusOK). Body(). - Contains("x-forwarded-port=443") + Contains(fmt.Sprintf("x-forwarded-port=443")) }) }) }) diff --git a/test/e2e/settings/log-format.go b/test/e2e/settings/log-format.go index 18835cb66..aaa5d0430 100644 --- a/test/e2e/settings/log-format.go +++ b/test/e2e/settings/log-format.go @@ -20,7 +20,7 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -35,11 +35,12 @@ var _ = framework.DescribeSetting("log-format-*", func() { f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil)) }) - ginkgo.Context("Check log-format-escape-json and log-format-escape-none", func() { - ginkgo.It("should not configure log-format escape by default", func() { + ginkgo.Context("Check log-format-escape-json", func() { + + ginkgo.It("should disable the log-format-escape-json by default", func() { f.WaitForNginxConfiguration( func(cfg string) bool { - return !strings.Contains(cfg, "log_format upstreaminfo escape") + return !strings.Contains(cfg, "log_format upstreaminfo escape=json") }) }) @@ -58,25 +59,10 @@ var _ = framework.DescribeSetting("log-format-*", func() { return !strings.Contains(cfg, "log_format upstreaminfo escape=json") }) }) - - ginkgo.It("should enable the log-format-escape-none", func() { - f.UpdateNginxConfigMapData("log-format-escape-none", "true") - f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, "log_format upstreaminfo escape=none") - }) - }) - - ginkgo.It("should disable the log-format-escape-none", func() { - f.UpdateNginxConfigMapData("log-format-escape-none", "false") - f.WaitForNginxConfiguration( - func(cfg string) bool { - return !strings.Contains(cfg, "log_format upstreaminfo escape=none") - }) - }) }) - ginkgo.Context("Check log-format-upstream with log-format-escape-json and log-format-escape-none", func() { + ginkgo.Context("Check log-format-upstream with log-format-escape-json", func() { + ginkgo.It("log-format-escape-json enabled", func() { f.SetNginxConfigMapData(map[string]string{ "log-format-escape-json": "true", @@ -100,7 +86,7 @@ var _ = framework.DescribeSetting("log-format-*", func() { assert.Contains(ginkgo.GinkgoT(), logs, `{"my_header1":"Here is \"header1\" with json escape", "my_header2":""}`) }) - ginkgo.It("log-format default escape", func() { + ginkgo.It("log-format-escape-json disabled", func() { f.SetNginxConfigMapData(map[string]string{ "log-format-escape-json": "false", "log-format-upstream": "\"{\"my_header3\":\"$http_header3\", \"my_header4\":\"$http_header4\"}\"", @@ -108,42 +94,19 @@ var _ = framework.DescribeSetting("log-format-*", func() { f.WaitForNginxConfiguration( func(cfg string) bool { - return !strings.Contains(cfg, "log_format upstreaminfo escape") + return !strings.Contains(cfg, "log_format upstreaminfo escape=json") }) f.HTTPTestClient(). GET("/"). WithHeader("Host", host). - WithHeader("header3", `Here is "header3" with default escape`). + WithHeader("header3", `Here is "header3" with json escape`). Expect(). Status(http.StatusOK) logs, err := f.NginxLogs() assert.Nil(ginkgo.GinkgoT(), err, "obtaining nginx logs") - assert.Contains(ginkgo.GinkgoT(), logs, `{"my_header3":"Here is \x22header3\x22 with default escape", "my_header4":"-"}`) - }) - - ginkgo.It("log-format-escape-none enabled", func() { - f.SetNginxConfigMapData(map[string]string{ - "log-format-escape-none": "true", - "log-format-upstream": "\"{\"my_header5\":\"$http_header5\", \"my_header6\":\"$http_header6\"}\"", - }) - - f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, "log_format upstreaminfo escape=none") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - WithHeader("header5", `Here is "header5" with none escape`). - Expect(). - Status(http.StatusOK) - - logs, err := f.NginxLogs() - assert.Nil(ginkgo.GinkgoT(), err, "obtaining nginx logs") - assert.Contains(ginkgo.GinkgoT(), logs, `{"my_header5":"Here is "header5" with none escape", "my_header6":""}`) + assert.Contains(ginkgo.GinkgoT(), logs, `{"my_header3":"Here is \x22header3\x22 with json escape", "my_header4":"-"}`) }) }) }) diff --git a/test/e2e/settings/lua_shared_dicts.go b/test/e2e/settings/lua_shared_dicts.go index 53f999c94..8c5ea1d33 100644 --- a/test/e2e/settings/lua_shared_dicts.go +++ b/test/e2e/settings/lua_shared_dicts.go @@ -19,7 +19,7 @@ package settings import ( "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/settings/main_snippet.go b/test/e2e/settings/main_snippet.go index ff14bab9d..18027f199 100644 --- a/test/e2e/settings/main_snippet.go +++ b/test/e2e/settings/main_snippet.go @@ -19,7 +19,7 @@ package settings import ( "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/settings/modsecurity/modsecurity_snippet.go b/test/e2e/settings/modsecurity_snippet.go similarity index 95% rename from test/e2e/settings/modsecurity/modsecurity_snippet.go rename to test/e2e/settings/modsecurity_snippet.go index 2dd92ced2..c0b962236 100644 --- a/test/e2e/settings/modsecurity/modsecurity_snippet.go +++ b/test/e2e/settings/modsecurity_snippet.go @@ -14,12 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -package modsecurity +package settings import ( "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/settings/multi_accept.go b/test/e2e/settings/multi_accept.go index 211c168aa..a6b4ffd5b 100644 --- a/test/e2e/settings/multi_accept.go +++ b/test/e2e/settings/multi_accept.go @@ -19,7 +19,7 @@ package settings import ( "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/settings/namespace_selector.go b/test/e2e/settings/namespace_selector.go deleted file mode 100644 index 1da62ee86..000000000 --- a/test/e2e/settings/namespace_selector.go +++ /dev/null @@ -1,114 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package settings - -import ( - "context" - "net/http" - "strings" - - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.IngressNginxDescribeSerial("[Flag] watch namespace selector", func() { - f := framework.NewDefaultFramework("namespace-selector") - notMatchedHost, matchedHost := barHost, fooHost - var notMatchedNs string - var matchedNs string - - // create a test namespace, under which create an ingress and backend deployment - prepareTestIngress := func(host string, labels map[string]string) string { - ns, err := framework.CreateKubeNamespaceWithLabel(f.BaseName, labels, f.KubeClientSet) - assert.Nil(ginkgo.GinkgoT(), err, "creating test namespace") - f.NewEchoDeployment(framework.WithDeploymentNamespace(ns)) - ing := framework.NewSingleIngressWithIngressClass(host, "/", host, ns, framework.EchoService, f.IngressClass, 80, nil) - f.EnsureIngress(ing) - return ns - } - - cleanupNamespace := func(ns string) { - err := framework.DeleteKubeNamespace(f.KubeClientSet, ns) - assert.Nil(ginkgo.GinkgoT(), err, "deleting temporarily created namespace") - } - - ginkgo.BeforeEach(func() { - notMatchedNs = prepareTestIngress(notMatchedHost, nil) // create namespace without label "foo=bar" - matchedNs = prepareTestIngress(matchedHost, map[string]string{fooHost: barHost}) - }) - - ginkgo.AfterEach(func() { - cleanupNamespace(notMatchedNs) - cleanupNamespace(matchedNs) - }) - - ginkgo.Context("With specific watch-namespace-selector flags", func() { - ginkgo.It("should ignore Ingress of namespace without label foo=bar and accept those of namespace with label foo=bar", func() { - f.WaitForNginxConfiguration(func(cfg string) bool { - return !strings.Contains(cfg, "server_name bar") && - strings.Contains(cfg, "server_name foo") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", matchedHost). - Expect(). - Status(http.StatusOK) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", notMatchedHost). - Expect(). - Status(http.StatusNotFound) - - // should accept Ingress when namespace labeled with foo=bar - ns, err := f.KubeClientSet.CoreV1().Namespaces().Get(context.TODO(), notMatchedNs, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - if ns.Labels == nil { - ns.Labels = make(map[string]string) - } - ns.Labels[fooHost] = barHost - - _, err = f.KubeClientSet.CoreV1().Namespaces().Update(context.TODO(), ns, metav1.UpdateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "labeling not matched namespace") - - // update ingress to trigger reconciliation - ing, err := f.KubeClientSet.NetworkingV1().Ingresses(notMatchedNs).Get(context.TODO(), notMatchedHost, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "retrieve test ingress") - if ing.Labels == nil { - ing.Labels = make(map[string]string) - } - ing.Labels[fooHost] = barHost - - _, err = f.KubeClientSet.NetworkingV1().Ingresses(notMatchedNs).Update(context.TODO(), ing, metav1.UpdateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "updating ingress") - - f.WaitForNginxConfiguration(func(cfg string) bool { - return strings.Contains(cfg, "server_name bar") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", notMatchedHost). - Expect(). - Status(http.StatusOK) - }) - }) -}) diff --git a/test/e2e/settings/no_auth_locations.go b/test/e2e/settings/no_auth_locations.go index 103c057d7..dd7112b5a 100644 --- a/test/e2e/settings/no_auth_locations.go +++ b/test/e2e/settings/no_auth_locations.go @@ -19,14 +19,15 @@ package settings import ( "fmt" "net/http" + "os/exec" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" - "golang.org/x/crypto/bcrypt" corev1 "k8s.io/api/core/v1" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -34,7 +35,7 @@ var _ = framework.DescribeSetting("[Security] no-auth-locations", func() { f := framework.NewDefaultFramework("no-auth-locations") setting := "no-auth-locations" - username := fooHost + username := "foo" password := "bar" secretName := "test-secret" host := "no-auth-locations" @@ -95,19 +96,16 @@ var _ = framework.DescribeSetting("[Security] no-auth-locations", func() { }) func buildBasicAuthIngressWithSecondPath(host, namespace, secretName, pathName string) *networking.Ingress { - pathtype := networking.PathTypePrefix return &networking.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: host, Namespace: namespace, - Annotations: map[string]string{ - "nginx.ingress.kubernetes.io/auth-type": "basic", + Annotations: map[string]string{"nginx.ingress.kubernetes.io/auth-type": "basic", "nginx.ingress.kubernetes.io/auth-secret": secretName, "nginx.ingress.kubernetes.io/auth-realm": "test auth", }, }, Spec: networking.IngressSpec{ - IngressClassName: framework.GetIngressClassName(namespace), Rules: []networking.IngressRule{ { Host: host, @@ -115,27 +113,17 @@ func buildBasicAuthIngressWithSecondPath(host, namespace, secretName, pathName s HTTP: &networking.HTTPIngressRuleValue{ Paths: []networking.HTTPIngressPath{ { - Path: "/", - PathType: &pathtype, + Path: "/", Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: framework.EchoService, - Port: networking.ServiceBackendPort{ - Number: int32(80), - }, - }, + ServiceName: framework.EchoService, + ServicePort: intstr.FromInt(80), }, }, { - Path: pathName, - PathType: &pathtype, + Path: pathName, Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: framework.EchoService, - Port: networking.ServiceBackendPort{ - Number: int32(80), - }, - }, + ServiceName: framework.EchoService, + ServicePort: intstr.FromInt(80), }, }, }, @@ -148,7 +136,7 @@ func buildBasicAuthIngressWithSecondPath(host, namespace, secretName, pathName s } func buildSecret(username, password, name, namespace string) *corev1.Secret { - out, err := bcrypt.GenerateFromPassword([]byte(password), 14) + out, err := exec.Command("openssl", "passwd", "-crypt", password).CombinedOutput() assert.Nil(ginkgo.GinkgoT(), err, "creating password") encpass := fmt.Sprintf("%v:%s\n", username, out) diff --git a/test/e2e/settings/no_tls_redirect_locations.go b/test/e2e/settings/no_tls_redirect_locations.go index 18fd09e26..64c9de516 100644 --- a/test/e2e/settings/no_tls_redirect_locations.go +++ b/test/e2e/settings/no_tls_redirect_locations.go @@ -17,9 +17,10 @@ limitations under the License. package settings import ( + "fmt" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -33,7 +34,7 @@ var _ = framework.DescribeSetting("Add no tls redirect locations", func() { f.EnsureIngress(ing) f.WaitForNginxConfiguration(func(server string) bool { - return strings.Contains(server, "set $force_no_ssl_redirect \"false\"") + return !strings.Contains(server, fmt.Sprintf("force_no_ssl_redirect = true,")) }) wlKey := "no-tls-redirect-locations" @@ -42,7 +43,8 @@ var _ = framework.DescribeSetting("Add no tls redirect locations", func() { f.UpdateNginxConfigMapData(wlKey, wlValue) f.WaitForNginxConfiguration(func(server string) bool { - return strings.Contains(server, "set $force_no_ssl_redirect \"true\"") + return strings.Contains(server, fmt.Sprintf("force_no_ssl_redirect = true,")) }) + }) }) diff --git a/test/e2e/settings/ocsp/ocsp.go b/test/e2e/settings/ocsp/ocsp.go index b2d50292a..873d10f2b 100644 --- a/test/e2e/settings/ocsp/ocsp.go +++ b/test/e2e/settings/ocsp/ocsp.go @@ -22,19 +22,18 @@ import ( "crypto/tls" "crypto/x509" "fmt" + "io/ioutil" "net/http" - "os" "os/exec" "strings" "syscall" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "golang.org/x/crypto/ocsp" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/test/e2e/framework" @@ -53,27 +52,23 @@ var _ = framework.DescribeSetting("OCSP", func() { f.UpdateNginxConfigMapData("enable-ocsp", "true") err := prepareCertificates(f.Namespace) - if err != nil { - ginkgo.By(fmt.Sprintf("Prepare Certs error %v", err.Error())) - } - assert.Nil(ginkgo.GinkgoT(), err) ing := framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, framework.EchoService, 80, nil) f.EnsureIngress(ing) - leafCert, err := os.ReadFile("leaf.pem") + leafCert, err := ioutil.ReadFile("leaf.pem") assert.Nil(ginkgo.GinkgoT(), err) - leafKey, err := os.ReadFile("leaf-key.pem") + leafKey, err := ioutil.ReadFile("leaf-key.pem") assert.Nil(ginkgo.GinkgoT(), err) - intermediateCa, err := os.ReadFile("intermediate_ca.pem") + intermediateCa, err := ioutil.ReadFile("intermediate_ca.pem") assert.Nil(ginkgo.GinkgoT(), err) var pemCertBuffer bytes.Buffer pemCertBuffer.Write(leafCert) - pemCertBuffer.WriteString("\n") + pemCertBuffer.Write([]byte("\n")) pemCertBuffer.Write(intermediateCa) f.EnsureSecret(&corev1.Secret{ @@ -87,10 +82,10 @@ var _ = framework.DescribeSetting("OCSP", func() { }, }) - cfsslDB, err := os.ReadFile("empty.db") + cfsslDB, err := ioutil.ReadFile("empty.db") assert.Nil(ginkgo.GinkgoT(), err) - f.EnsureConfigMap(&corev1.ConfigMap{ + cmap, err := f.EnsureConfigMap(&corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "ocspserve", Namespace: f.Namespace, @@ -100,6 +95,8 @@ var _ = framework.DescribeSetting("OCSP", func() { "db-config.json": []byte(`{"driver":"sqlite3","data_source":"/data/empty.db"}`), }, }) + assert.Nil(ginkgo.GinkgoT(), err) + assert.NotNil(ginkgo.GinkgoT(), cmap) d, s := ocspserveDeployment(f.Namespace) f.EnsureDeployment(d) @@ -108,9 +105,8 @@ var _ = framework.DescribeSetting("OCSP", func() { err = framework.WaitForEndpoints(f.KubeClientSet, framework.DefaultTimeout, "ocspserve", f.Namespace, 1) assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready") - f.WaitForLuaConfiguration(func(jsonCfg map[string]interface{}) bool { - val, ok, err := unstructured.NestedBool(jsonCfg, "enable_ocsp") - return err == nil && ok && val + f.WaitForNginxConfiguration(func(cfg string) bool { + return strings.Contains(cfg, "certificate.is_ocsp_stapling_enabled = true") }) f.WaitForNginxServer(host, @@ -118,7 +114,7 @@ var _ = framework.DescribeSetting("OCSP", func() { return strings.Contains(server, fmt.Sprintf(`server_name %v`, host)) }) - tlsConfig := &tls.Config{ServerName: host, InsecureSkipVerify: true} //nolint:gosec // Ignore the gosec error in testing + tlsConfig := &tls.Config{ServerName: host, InsecureSkipVerify: true} f.HTTPTestClientWithTLSConfig(tlsConfig). GET("/"). WithURL(f.GetURL(framework.HTTPS)). @@ -201,8 +197,7 @@ const configTemplate = ` func prepareCertificates(namespace string) error { config := fmt.Sprintf(configTemplate, namespace) - //nolint:gosec // Not change permission to avoid possible issues - err := os.WriteFile("cfssl_config.json", []byte(config), 0o644) + err := ioutil.WriteFile("cfssl_config.json", []byte(config), 0644) if err != nil { return fmt.Errorf("creating cfssl_config.json file: %v", err) } @@ -297,7 +292,7 @@ func ocspserveDeployment(namespace string) (*appsv1.Deployment, *corev1.Service) Containers: []corev1.Container{ { Name: name, - Image: "registry.k8s.io/ingress-nginx/cfssl:v1.1.1@sha256:bcd576c6d0a01d4710969195e804c02da62b71b5c35c6816df9b7584d5445437", + Image: "k8s.gcr.io/ingress-nginx/e2e-test-cfssl@sha256:be2f69024f7b7053f35b86677de16bdaa5d3ff0f81b17581ef0b0c6804188b03", Command: []string{ "/bin/bash", "-c", diff --git a/test/e2e/settings/opentelemetry.go b/test/e2e/settings/opentelemetry.go deleted file mode 100644 index b5fc6ff4e..000000000 --- a/test/e2e/settings/opentelemetry.go +++ /dev/null @@ -1,120 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package settings - -import ( - "strings" - - "github.com/onsi/ginkgo/v2" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -const ( - enableOpentelemetry = "enable-opentelemetry" - opentelemetryTrustIncomingSpan = "opentelemetry-trust-incoming-span" - - opentelemetryOperationName = "opentelemetry-operation-name" - opentelemetryLocationOperationName = "opentelemetry-location-operation-name" - opentelemetryConfig = "opentelemetry-config" - opentelemetryConfigPath = "/etc/ingress-controller/telemetry/opentelemetry.toml" - - enable = "true" -) - -var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() { - f := framework.NewDefaultFramework("enable-opentelemetry") - - ginkgo.BeforeEach(func() { - f.NewEchoDeployment() - }) - - ginkgo.AfterEach(func() { - }) - - ginkgo.It("should not exists opentelemetry directive", func() { - config := map[string]string{} - config[enableOpentelemetry] = disable - f.SetNginxConfigMapData(config) - - f.EnsureIngress(framework.NewSingleIngress(enableOpentelemetry, "/", enableOpentelemetry, f.Namespace, "http-svc", 80, nil)) - - f.WaitForNginxConfiguration( - func(cfg string) bool { - return !strings.Contains(cfg, "opentelemetry on") - }) - }) - - ginkgo.It("should exists opentelemetry directive when is enabled", func() { - config := map[string]string{} - config[enableOpentelemetry] = enable - config[opentelemetryConfig] = opentelemetryConfigPath - f.SetNginxConfigMapData(config) - - f.EnsureIngress(framework.NewSingleIngress(enableOpentelemetry, "/", enableOpentelemetry, f.Namespace, "http-svc", 80, nil)) - - f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, "opentelemetry on") - }) - }) - - ginkgo.It("should include opentelemetry_trust_incoming_spans on directive when enabled", func() { - config := map[string]string{} - config[enableOpentelemetry] = enable - config[opentelemetryConfig] = opentelemetryConfigPath - config[opentelemetryTrustIncomingSpan] = enable - f.SetNginxConfigMapData(config) - - f.EnsureIngress(framework.NewSingleIngress(enableOpentelemetry, "/", enableOpentelemetry, f.Namespace, "http-svc", 80, nil)) - - f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, "opentelemetry_trust_incoming_spans on") - }) - }) - - ginkgo.It("should not exists opentelemetry_operation_name directive when is empty", func() { - config := map[string]string{} - config[enableOpentelemetry] = enable - config[opentelemetryConfig] = opentelemetryConfigPath - config[opentelemetryOperationName] = "" - f.SetNginxConfigMapData(config) - - f.EnsureIngress(framework.NewSingleIngress(enableOpentelemetry, "/", enableOpentelemetry, f.Namespace, "http-svc", 80, nil)) - - f.WaitForNginxConfiguration( - func(cfg string) bool { - return !strings.Contains(cfg, "opentelemetry_operation_name") - }) - }) - - ginkgo.It("should exists opentelemetry_operation_name directive when is configured", func() { - config := map[string]string{} - config[enableOpentelemetry] = enable - config[opentelemetryConfig] = opentelemetryConfigPath - config[opentelemetryOperationName] = "HTTP $request_method $uri" - f.SetNginxConfigMapData(config) - - f.EnsureIngress(framework.NewSingleIngress(enableOpentelemetry, "/", enableOpentelemetry, f.Namespace, "http-svc", 80, nil)) - - f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, `opentelemetry_operation_name "HTTP $request_method $uri"`) - }) - }) -}) diff --git a/test/e2e/settings/opentracing.go b/test/e2e/settings/opentracing.go new file mode 100644 index 000000000..0fe4e62f6 --- /dev/null +++ b/test/e2e/settings/opentracing.go @@ -0,0 +1,236 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package settings + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/onsi/ginkgo" + "github.com/stretchr/testify/assert" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +const ( + enableOpentracing = "enable-opentracing" + + zipkinCollectorHost = "zipkin-collector-host" + + jaegerCollectorHost = "jaeger-collector-host" + jaegerSamplerHost = "jaeger-sampler-host" + jaegerPropagationFormat = "jaeger-propagation-format" + // jaegerEndpoint = "jaeger-endpoint" + + datadogCollectorHost = "datadog-collector-host" + + opentracingOperationName = "opentracing-operation-name" + opentracingLocationOperationName = "opentracing-location-operation-name" +) + +var _ = framework.IngressNginxDescribe("Configure OpenTracing", func() { + f := framework.NewDefaultFramework("enable-opentracing") + + ginkgo.BeforeEach(func() { + f.NewEchoDeployment() + }) + + ginkgo.AfterEach(func() { + }) + + ginkgo.It("should not exists opentracing directive", func() { + config := map[string]string{} + config[enableOpentracing] = "false" + f.SetNginxConfigMapData(config) + + f.EnsureIngress(framework.NewSingleIngress(enableOpentracing, "/", enableOpentracing, f.Namespace, "http-svc", 80, nil)) + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return !strings.Contains(cfg, "opentracing on") + }) + }) + + ginkgo.It("should exists opentracing directive when is enabled", func() { + config := map[string]string{} + config[enableOpentracing] = "true" + config[zipkinCollectorHost] = "127.0.0.1" + f.SetNginxConfigMapData(config) + + f.EnsureIngress(framework.NewSingleIngress(enableOpentracing, "/", enableOpentracing, f.Namespace, "http-svc", 80, nil)) + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, "opentracing on") + }) + }) + + ginkgo.It("should not exists opentracing_operation_name directive when is empty", func() { + config := map[string]string{} + config[enableOpentracing] = "true" + config[zipkinCollectorHost] = "127.0.0.1" + config[opentracingOperationName] = "" + f.SetNginxConfigMapData(config) + + f.EnsureIngress(framework.NewSingleIngress(enableOpentracing, "/", enableOpentracing, f.Namespace, "http-svc", 80, nil)) + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return !strings.Contains(cfg, "opentracing_operation_name") + }) + }) + + ginkgo.It("should exists opentracing_operation_name directive when is configured", func() { + config := map[string]string{} + config[enableOpentracing] = "true" + config[zipkinCollectorHost] = "127.0.0.1" + config[opentracingOperationName] = "HTTP $request_method $uri" + f.SetNginxConfigMapData(config) + + f.EnsureIngress(framework.NewSingleIngress(enableOpentracing, "/", enableOpentracing, f.Namespace, "http-svc", 80, nil)) + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, `opentracing_operation_name "HTTP $request_method $uri"`) + }) + }) + + ginkgo.It("should not exists opentracing_location_operation_name directive when is empty", func() { + config := map[string]string{} + config[enableOpentracing] = "true" + config[zipkinCollectorHost] = "127.0.0.1" + config[opentracingLocationOperationName] = "" + f.SetNginxConfigMapData(config) + + f.EnsureIngress(framework.NewSingleIngress(enableOpentracing, "/", enableOpentracing, f.Namespace, "http-svc", 80, nil)) + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return !strings.Contains(cfg, "opentracing_location_operation_name") + }) + }) + + ginkgo.It("should exists opentracing_location_operation_name directive when is configured", func() { + config := map[string]string{} + config[enableOpentracing] = "true" + config[zipkinCollectorHost] = "127.0.0.1" + config[opentracingLocationOperationName] = "HTTP $request_method $uri" + f.SetNginxConfigMapData(config) + + f.EnsureIngress(framework.NewSingleIngress(enableOpentracing, "/", enableOpentracing, f.Namespace, "http-svc", 80, nil)) + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, "opentracing_location_operation_name \"HTTP $request_method $uri\"") + }) + }) + + ginkgo.It("should enable opentracing using zipkin", func() { + config := map[string]string{} + config[enableOpentracing] = "true" + config[zipkinCollectorHost] = "127.0.0.1" + f.SetNginxConfigMapData(config) + + framework.Sleep(10 * time.Second) + log, err := f.NginxLogs() + assert.Nil(ginkgo.GinkgoT(), err, "obtaining nginx logs") + assert.NotContains(ginkgo.GinkgoT(), log, "Unexpected failure reloading the backend", "reloading nginx after a configmap change") + }) + + ginkgo.It("should enable opentracing using jaeger", func() { + config := map[string]string{} + config[enableOpentracing] = "true" + config[jaegerCollectorHost] = "127.0.0.1" + f.SetNginxConfigMapData(config) + + framework.Sleep(10 * time.Second) + log, err := f.NginxLogs() + assert.Nil(ginkgo.GinkgoT(), err, "obtaining nginx logs") + assert.NotContains(ginkgo.GinkgoT(), log, "Unexpected failure reloading the backend", "reloading nginx after a configmap change") + }) + + ginkgo.It("should enable opentracing using jaeger with sampler host", func() { + config := map[string]string{} + config[enableOpentracing] = "true" + config[jaegerCollectorHost] = "127.0.0.1" + config[jaegerSamplerHost] = "127.0.0.1" + f.SetNginxConfigMapData(config) + + framework.Sleep(10 * time.Second) + log, err := f.NginxLogs() + assert.Nil(ginkgo.GinkgoT(), err, "obtaining nginx logs") + assert.NotContains(ginkgo.GinkgoT(), log, "Unexpected failure reloading the backend", "reloading nginx after a configmap change") + }) + + ginkgo.It("should propagate the w3c header when configured with jaeger", func() { + host := "jaeger-w3c" + config := map[string]string{} + config[enableOpentracing] = "true" + config[jaegerCollectorHost] = "127.0.0.1" + config[jaegerPropagationFormat] = "w3c" + f.SetNginxConfigMapData(config) + + framework.Sleep(10 * time.Second) + log, err := f.NginxLogs() + assert.Nil(ginkgo.GinkgoT(), err, "obtaining nginx logs") + assert.NotContains(ginkgo.GinkgoT(), log, "Unexpected failure reloading the backend", "reloading nginx after a configmap change") + + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + }) + + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", host). + Expect(). + Status(http.StatusOK). + Body(). + Match("traceparent=[0-9a-f]{2}-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}") + }) + + /* + ginkgo.It("should enable opentracing using jaeger with an HTTP endpoint", func() { + config := map[string]string{} + config[enableOpentracing] = "true" + config[jaegerEndpoint] = "http://127.0.0.1/api/traces" + f.SetNginxConfigMapData(config) + + framework.Sleep(10 * time.Second) + log, err := f.NginxLogs() + assert.Nil(ginkgo.GinkgoT(), err, "obtaining nginx logs") + assert.NotContains(ginkgo.GinkgoT(), log, "Unexpected failure reloading the backend", "reloading nginx after a configmap change") + }) + */ + + ginkgo.It("should enable opentracing using datadog", func() { + config := map[string]string{} + config[enableOpentracing] = "true" + config[datadogCollectorHost] = "http://127.0.0.1" + f.SetNginxConfigMapData(config) + + framework.Sleep(10 * time.Second) + log, err := f.NginxLogs() + assert.Nil(ginkgo.GinkgoT(), err, "obtaining nginx logs") + assert.NotContains(ginkgo.GinkgoT(), log, "Unexpected failure reloading the backend", "reloading nginx after a configmap change") + }) +}) diff --git a/test/e2e/settings/plugins.go b/test/e2e/settings/plugins.go new file mode 100644 index 000000000..1af04e9d2 --- /dev/null +++ b/test/e2e/settings/plugins.go @@ -0,0 +1,55 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package settings + +import ( + "fmt" + "net/http" + "strings" + + "github.com/onsi/ginkgo" + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("plugins", func() { + f := framework.NewDefaultFramework("plugins") + + ginkgo.BeforeEach(func() { + f.NewEchoDeployment() + }) + + ginkgo.It("should exist a x-hello-world header", func() { + f.UpdateNginxConfigMapData("plugins", "hello_world, invalid") + + host := "example.com" + f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil)) + + f.WaitForNginxConfiguration( + func(server string) bool { + return strings.Contains(server, fmt.Sprintf("server_name %v", host)) && + strings.Contains(server, `plugins.init({ "hello_world","invalid" })`) + }) + + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", host). + WithHeader("User-Agent", "hello"). + Expect(). + Status(http.StatusOK). + Body().Contains("x-hello-world=1") + }) +}) diff --git a/test/e2e/settings/pod_security_policy.go b/test/e2e/settings/pod_security_policy.go new file mode 100644 index 000000000..75c7698d0 --- /dev/null +++ b/test/e2e/settings/pod_security_policy.go @@ -0,0 +1,127 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package settings + +import ( + "context" + "net/http" + "strings" + + "github.com/onsi/ginkgo" + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + policyv1beta1 "k8s.io/api/policy/v1beta1" + rbacv1 "k8s.io/api/rbac/v1" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +const ( + ingressControllerPSP = "ingress-controller-psp" +) + +var _ = framework.IngressNginxDescribe("[Security] Pod Security Policies", func() { + f := framework.NewDefaultFramework("pod-security-policies") + + ginkgo.It("should be running with a Pod Security Policy", func() { + psp := createPodSecurityPolicy() + _, err := f.KubeClientSet.PolicyV1beta1().PodSecurityPolicies().Create(context.TODO(), psp, metav1.CreateOptions{}) + if !k8sErrors.IsAlreadyExists(err) { + assert.Nil(ginkgo.GinkgoT(), err, "creating Pod Security Policy") + } + + role, err := f.KubeClientSet.RbacV1().Roles(f.Namespace).Get(context.TODO(), "nginx-ingress", metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "getting ingress controller cluster role") + assert.NotNil(ginkgo.GinkgoT(), role) + + role.Rules = append(role.Rules, rbacv1.PolicyRule{ + APIGroups: []string{"policy"}, + Resources: []string{"podsecuritypolicies"}, + ResourceNames: []string{ingressControllerPSP}, + Verbs: []string{"use"}, + }) + + _, err = f.KubeClientSet.RbacV1().Roles(f.Namespace).Update(context.TODO(), role, metav1.UpdateOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller cluster role to use a pod security policy") + + // update the deployment just to trigger a rolling update and the use of the security policy + err = f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { + args := deployment.Spec.Template.Spec.Containers[0].Args + args = append(args, "--v=2") + deployment.Spec.Template.Spec.Containers[0].Args = args + _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) + + return err + }) + assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating ingress controller deployment flags") + + f.WaitForNginxListening(80) + + f.NewEchoDeployment() + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, "server_tokens off") + }) + + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", "foo.bar.com"). + Expect(). + Status(http.StatusNotFound) + }) +}) + +func createPodSecurityPolicy() *policyv1beta1.PodSecurityPolicy { + trueValue := true + return &policyv1beta1.PodSecurityPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: ingressControllerPSP, + }, + Spec: policyv1beta1.PodSecurityPolicySpec{ + AllowPrivilegeEscalation: &trueValue, + RequiredDropCapabilities: []corev1.Capability{"All"}, + RunAsUser: policyv1beta1.RunAsUserStrategyOptions{ + Rule: "RunAsAny", + }, + SELinux: policyv1beta1.SELinuxStrategyOptions{ + Rule: "RunAsAny", + }, + FSGroup: policyv1beta1.FSGroupStrategyOptions{ + Ranges: []policyv1beta1.IDRange{ + { + Min: 1, + Max: 65535, + }, + }, + Rule: "MustRunAs", + }, + SupplementalGroups: policyv1beta1.SupplementalGroupsStrategyOptions{ + Ranges: []policyv1beta1.IDRange{ + { + Min: 1, + Max: 65535, + }, + }, + Rule: "MustRunAs", + }, + }, + } +} diff --git a/test/e2e/settings/pod_security_policy_volumes.go b/test/e2e/settings/pod_security_policy_volumes.go new file mode 100644 index 000000000..da88ed5be --- /dev/null +++ b/test/e2e/settings/pod_security_policy_volumes.go @@ -0,0 +1,113 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package settings + +import ( + "context" + "net/http" + "strings" + + "github.com/onsi/ginkgo" + "github.com/stretchr/testify/assert" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("[Security] Pod Security Policies with volumes", func() { + f := framework.NewDefaultFramework("pod-security-policies-volumes") + + ginkgo.It("should be running with a Pod Security Policy", func() { + psp := createPodSecurityPolicy() + _, err := f.KubeClientSet.PolicyV1beta1().PodSecurityPolicies().Create(context.TODO(), psp, metav1.CreateOptions{}) + if !k8sErrors.IsAlreadyExists(err) { + assert.Nil(ginkgo.GinkgoT(), err, "creating Pod Security Policy") + } + + role, err := f.KubeClientSet.RbacV1().Roles(f.Namespace).Get(context.TODO(), "nginx-ingress", metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "getting ingress controller cluster role") + assert.NotNil(ginkgo.GinkgoT(), role) + + role.Rules = append(role.Rules, rbacv1.PolicyRule{ + APIGroups: []string{"policy"}, + Resources: []string{"podsecuritypolicies"}, + ResourceNames: []string{ingressControllerPSP}, + Verbs: []string{"use"}, + }) + + _, err = f.KubeClientSet.RbacV1().Roles(f.Namespace).Update(context.TODO(), role, metav1.UpdateOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller cluster role to use a pod security policy") + + err = f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { + args := deployment.Spec.Template.Spec.Containers[0].Args + args = append(args, "--v=2") + deployment.Spec.Template.Spec.Containers[0].Args = args + + deployment.Spec.Template.Spec.Volumes = []corev1.Volume{ + { + Name: "ssl", VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "tmp", VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } + + fsGroup := int64(33) + deployment.Spec.Template.Spec.SecurityContext = &corev1.PodSecurityContext{ + FSGroup: &fsGroup, + } + + deployment.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ + { + Name: "ssl", MountPath: "/etc/ingress-controller", + }, + { + Name: "tmp", MountPath: "/tmp", + }, + } + + _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) + + return err + }) + assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment") + + f.WaitForNginxListening(80) + + f.NewEchoDeployment() + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, "server_tokens off") + }) + + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", "foo.bar.com"). + Expect(). + Status(http.StatusNotFound) + }) +}) diff --git a/test/e2e/settings/proxy_connect_timeout.go b/test/e2e/settings/proxy_connect_timeout.go index 185ac1dd1..8d2fd5fda 100644 --- a/test/e2e/settings/proxy_connect_timeout.go +++ b/test/e2e/settings/proxy_connect_timeout.go @@ -14,14 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -//nolint:dupl // Ignore dupl errors for similar test case package settings import ( "fmt" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -65,4 +64,5 @@ var _ = framework.DescribeSetting("proxy-connect-timeout", func() { return !strings.Contains(server, fmt.Sprintf("proxy_connect_timeout %ss;", proxyConnectTimeout)) }) }) + }) diff --git a/test/e2e/settings/proxy_host.go b/test/e2e/settings/proxy_host.go index bb5dc9c01..de72f9d5b 100644 --- a/test/e2e/settings/proxy_host.go +++ b/test/e2e/settings/proxy_host.go @@ -21,7 +21,7 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -34,9 +34,6 @@ var _ = framework.IngressNginxDescribe("Dynamic $proxy_host", func() { }) ginkgo.It("should exist a proxy_host", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - upstreamName := fmt.Sprintf("%v-%v-80", f.Namespace, framework.EchoService) annotations := map[string]string{ "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_headers "Custom-Header: $proxy_host"`, @@ -58,9 +55,6 @@ var _ = framework.IngressNginxDescribe("Dynamic $proxy_host", func() { }) ginkgo.It("should exist a proxy_host using the upstream-vhost annotation value", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - upstreamName := fmt.Sprintf("%v-%v-80", f.Namespace, framework.EchoService) upstreamVHost := "different.host" annotations := map[string]string{ @@ -72,7 +66,7 @@ var _ = framework.IngressNginxDescribe("Dynamic $proxy_host", func() { f.WaitForNginxConfiguration( func(server string) bool { return strings.Contains(server, fmt.Sprintf("server_name %v", test)) && - strings.Contains(server, "set $proxy_host $proxy_upstream_name") + strings.Contains(server, fmt.Sprintf("set $proxy_host $proxy_upstream_name")) }) f.HTTPTestClient(). diff --git a/test/e2e/settings/proxy_next_upstream.go b/test/e2e/settings/proxy_next_upstream.go index b59d5edb7..bc17cd0c7 100644 --- a/test/e2e/settings/proxy_next_upstream.go +++ b/test/e2e/settings/proxy_next_upstream.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/settings/proxy_protocol.go b/test/e2e/settings/proxy_protocol.go index cfce68bf8..3b85cf428 100644 --- a/test/e2e/settings/proxy_protocol.go +++ b/test/e2e/settings/proxy_protocol.go @@ -20,11 +20,11 @@ import ( "context" "crypto/tls" "fmt" - "io" + "io/ioutil" "net" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,10 +33,8 @@ import ( "k8s.io/ingress-nginx/test/e2e/framework" ) -const proxyProtocol = "proxy-protocol" - var _ = framework.DescribeSetting("use-proxy-protocol", func() { - f := framework.NewDefaultFramework(proxyProtocol) + f := framework.NewDefaultFramework("proxy-protocol") setting := "use-proxy-protocol" @@ -44,9 +42,9 @@ var _ = framework.DescribeSetting("use-proxy-protocol", func() { f.NewEchoDeployment() f.UpdateNginxConfigMapData(setting, "false") }) - //nolint:dupl // Ignore dupl errors for similar test case + ginkgo.It("should respect port passed by the PROXY Protocol", func() { - host := proxyProtocol + host := "proxy-protocol" f.UpdateNginxConfigMapData(setting, "true") @@ -65,25 +63,21 @@ var _ = framework.DescribeSetting("use-proxy-protocol", func() { defer conn.Close() header := "PROXY TCP4 192.168.0.1 192.168.0.11 56324 1234\r\n" - _, err = conn.Write([]byte(header)) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error writing header") + conn.Write([]byte(header)) + conn.Write([]byte("GET / HTTP/1.1\r\nHost: proxy-protocol\r\n\r\n")) - _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: proxy-protocol\r\n\r\n")) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error writing request") - - data, err := io.ReadAll(conn) + data, err := ioutil.ReadAll(conn) assert.Nil(ginkgo.GinkgoT(), err, "unexpected error reading connection data") body := string(data) - assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("host=%v", proxyProtocol)) - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-port=1234") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-proto=http") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-for=192.168.0.1") + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("host=%v", "proxy-protocol")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-port=1234")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-proto=http")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-for=192.168.0.1")) }) - //nolint:dupl // Ignore dupl errors for similar test case ginkgo.It("should respect proto passed by the PROXY Protocol server port", func() { - host := proxyProtocol + host := "proxy-protocol" f.UpdateNginxConfigMapData(setting, "true") @@ -102,24 +96,21 @@ var _ = framework.DescribeSetting("use-proxy-protocol", func() { defer conn.Close() header := "PROXY TCP4 192.168.0.1 192.168.0.11 56324 443\r\n" - _, err = conn.Write([]byte(header)) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error writing header") + conn.Write([]byte(header)) + conn.Write([]byte("GET / HTTP/1.1\r\nHost: proxy-protocol\r\n\r\n")) - _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: proxy-protocol\r\n\r\n")) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error writing request") - - data, err := io.ReadAll(conn) + data, err := ioutil.ReadAll(conn) assert.Nil(ginkgo.GinkgoT(), err, "unexpected error reading connection data") body := string(data) - assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("host=%v", proxyProtocol)) - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-port=443") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-proto=https") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-for=192.168.0.1") + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("host=%v", "proxy-protocol")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-port=443")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-proto=https")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-for=192.168.0.1")) }) ginkgo.It("should enable PROXY Protocol for HTTPS", func() { - host := proxyProtocol + host := "proxy-protocol" f.UpdateNginxConfigMapData(setting, "true") @@ -150,15 +141,15 @@ var _ = framework.DescribeSetting("use-proxy-protocol", func() { _, err = tlsConn.Write([]byte("GET / HTTP/1.1\r\nHost: proxy-protocol\r\n\r\n")) assert.Nil(ginkgo.GinkgoT(), err, "writing HTTP request") - data, err := io.ReadAll(tlsConn) + data, err := ioutil.ReadAll(tlsConn) assert.Nil(ginkgo.GinkgoT(), err, "unexpected error reading connection data") body := string(data) - assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("host=%v", proxyProtocol)) - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-port=1234") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-proto=https") - assert.Contains(ginkgo.GinkgoT(), body, "x-scheme=https") - assert.Contains(ginkgo.GinkgoT(), body, "x-forwarded-for=192.168.0.1") + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("host=%v", "proxy-protocol")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-port=1234")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-proto=https")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-scheme=https")) + assert.Contains(ginkgo.GinkgoT(), body, fmt.Sprintf("x-forwarded-for=192.168.0.1")) }) ginkgo.It("should enable PROXY Protocol for TCP", func() { @@ -214,13 +205,10 @@ var _ = framework.DescribeSetting("use-proxy-protocol", func() { defer conn.Close() header := "PROXY TCP4 192.168.0.1 192.168.0.11 56324 8080\r\n" - _, err = conn.Write([]byte(header)) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error writing header") + conn.Write([]byte(header)) + conn.Write([]byte("GET / HTTP/1.1\r\nHost: proxy-protocol\r\n\r\n")) - _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: proxy-protocol\r\n\r\n")) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error writing request") - - _, err = io.ReadAll(conn) + _, err = ioutil.ReadAll(conn) assert.Nil(ginkgo.GinkgoT(), err, "unexpected error reading connection data") logs, err := f.NginxLogs() diff --git a/test/e2e/settings/proxy_read_timeout.go b/test/e2e/settings/proxy_read_timeout.go index 484b44f24..41171e2c6 100644 --- a/test/e2e/settings/proxy_read_timeout.go +++ b/test/e2e/settings/proxy_read_timeout.go @@ -14,14 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -//nolint:dupl // Ignore dupl errors for similar test case package settings import ( "fmt" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -65,4 +64,5 @@ var _ = framework.DescribeSetting("proxy-read-timeout", func() { return !strings.Contains(server, fmt.Sprintf("proxy_read_timeout %ss;", proxyReadtimeout)) }) }) + }) diff --git a/test/e2e/settings/proxy_send_timeout.go b/test/e2e/settings/proxy_send_timeout.go index bdcd46f0f..031f4361f 100644 --- a/test/e2e/settings/proxy_send_timeout.go +++ b/test/e2e/settings/proxy_send_timeout.go @@ -14,14 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -//nolint:dupl // Ignore dupl errors for similar test case package settings import ( "fmt" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -65,4 +64,5 @@ var _ = framework.DescribeSetting("proxy-send-timeout", func() { return !strings.Contains(server, fmt.Sprintf("proxy_send_timeout %ss;", proxySendTimeout)) }) }) + }) diff --git a/test/e2e/settings/reuse-port.go b/test/e2e/settings/reuse-port.go index 29d52c4f1..b8b2c5611 100644 --- a/test/e2e/settings/reuse-port.go +++ b/test/e2e/settings/reuse-port.go @@ -19,7 +19,7 @@ package settings import ( "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/settings/server_snippet.go b/test/e2e/settings/server_snippet.go deleted file mode 100644 index 1e2084bd8..000000000 --- a/test/e2e/settings/server_snippet.go +++ /dev/null @@ -1,164 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package settings - -import ( - "net/http" - "strings" - - "github.com/onsi/ginkgo/v2" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.DescribeSetting("configmap server-snippet", func() { - f := framework.NewDefaultFramework("cm-server-snippet") - - ginkgo.BeforeEach(func() { - f.NewEchoDeployment() - }) - - ginkgo.It("should add value of server-snippet setting to all ingress config", func() { - host := "serverglobalsnippet1.foo.com" - hostAnnots := "serverannotssnippet1.foo.com" - - f.SetNginxConfigMapData(map[string]string{ - "allow-snippet-annotations": "true", - "annotations-risk-level": "Critical", - "server-snippet": ` - more_set_headers "Globalfoo: Foooo";`, - }) - - defer func() { - f.SetNginxConfigMapData(map[string]string{ - "allow-snippet-annotations": "false", - "annotations-risk-level": "High", - }) - }() - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/server-snippet": ` - more_set_headers "Foo: Bar"; - more_set_headers "Xpto: Lalala";`, - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ing) - - ing1 := framework.NewSingleIngress(hostAnnots, "/", hostAnnots, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing1) - - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, `more_set_headers "Globalfoo: Foooo`) && - !strings.Contains(server, `more_set_headers "Foo: Bar";`) && - !strings.Contains(server, `more_set_headers "Xpto: Lalala";`) - }) - - f.WaitForNginxServer(hostAnnots, - func(server string) bool { - return strings.Contains(server, `more_set_headers "Globalfoo: Foooo`) && - strings.Contains(server, `more_set_headers "Foo: Bar";`) && - strings.Contains(server, `more_set_headers "Xpto: Lalala";`) - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Globalfoo", []string{"Foooo"}). - NotContainsKey("Foo"). - NotContainsKey("Xpto") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", hostAnnots). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Foo", []string{"Bar"}). - ValueEqual("Xpto", []string{"Lalala"}). - ValueEqual("Globalfoo", []string{"Foooo"}) - }) - - ginkgo.It("should add global server-snippet and drop annotations per admin config", func() { - host := "serverglobalsnippet2.foo.com" - hostAnnots := "serverannotssnippet2.foo.com" - - f.SetNginxConfigMapData(map[string]string{ - "allow-snippet-annotations": "false", - "annotations-risk-level": "Critical", // To allow Configuration Snippet - "server-snippet": ` - more_set_headers "Globalfoo: Foooo";`, - }) - - defer func() { - f.SetNginxConfigMapData(map[string]string{ - "allow-snippet-annotations": "false", - "annotations-risk-level": "High", - }) - }() - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/server-snippet": ` - more_set_headers "Foo: Bar"; - more_set_headers "Xpto: Lalala";`, - } - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ing) - - ing1 := framework.NewSingleIngress(hostAnnots, "/", hostAnnots, f.Namespace, framework.EchoService, 80, annotations) - f.EnsureIngress(ing1) - - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, `more_set_headers "Globalfoo: Foooo`) && - !strings.Contains(server, `more_set_headers "Foo: Bar";`) && - !strings.Contains(server, `more_set_headers "Xpto: Lalala";`) - }) - - f.WaitForNginxServer(hostAnnots, - func(server string) bool { - return strings.Contains(server, `more_set_headers "Globalfoo: Foooo`) && - !strings.Contains(server, `more_set_headers "Foo: Bar";`) && - !strings.Contains(server, `more_set_headers "Xpto: Lalala";`) - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", host). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Globalfoo", []string{"Foooo"}). - NotContainsKey("Foo"). - NotContainsKey("Xpto") - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", hostAnnots). - Expect(). - Status(http.StatusOK).Headers(). - ValueEqual("Globalfoo", []string{"Foooo"}). - NotContainsKey("Foo"). - NotContainsKey("Xpto") - }) -}) diff --git a/test/e2e/settings/server_tokens.go b/test/e2e/settings/server_tokens.go index 015554f07..43ccc86d3 100644 --- a/test/e2e/settings/server_tokens.go +++ b/test/e2e/settings/server_tokens.go @@ -19,15 +19,15 @@ package settings import ( "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" - networking "k8s.io/api/networking/v1" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/test/e2e/framework" ) var _ = framework.DescribeSetting("server-tokens", func() { - pathtype := networking.PathTypePrefix f := framework.NewDefaultFramework("server-tokens") serverTokens := "server-tokens" @@ -57,7 +57,6 @@ var _ = framework.DescribeSetting("server-tokens", func() { Annotations: map[string]string{}, }, Spec: networking.IngressSpec{ - IngressClassName: &f.IngressClass, Rules: []networking.IngressRule{ { Host: serverTokens, @@ -65,15 +64,10 @@ var _ = framework.DescribeSetting("server-tokens", func() { HTTP: &networking.HTTPIngressRuleValue{ Paths: []networking.HTTPIngressPath{ { - Path: "/", - PathType: &pathtype, + Path: "/", Backend: networking.IngressBackend{ - Service: &networking.IngressServiceBackend{ - Name: framework.EchoService, - Port: networking.ServiceBackendPort{ - Number: int32(80), - }, - }, + ServiceName: framework.EchoService, + ServicePort: intstr.FromInt(80), }, }, }, diff --git a/test/e2e/settings/ssl_ciphers.go b/test/e2e/settings/ssl_ciphers.go index 241dfc92b..9aa4b8e14 100644 --- a/test/e2e/settings/ssl_ciphers.go +++ b/test/e2e/settings/ssl_ciphers.go @@ -20,7 +20,7 @@ import ( "fmt" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "k8s.io/ingress-nginx/test/e2e/framework" ) diff --git a/test/e2e/settings/ssl_passthrough.go b/test/e2e/settings/ssl_passthrough.go deleted file mode 100644 index b10511bde..000000000 --- a/test/e2e/settings/ssl_passthrough.go +++ /dev/null @@ -1,188 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package settings - -import ( - "context" - "crypto/tls" - "net" - "net/http" - "strings" - - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.IngressNginxDescribe("[Flag] enable-ssl-passthrough", func() { - f := framework.NewDefaultFramework("ssl-passthrough", framework.WithHTTPBunEnabled()) - - ginkgo.BeforeEach(func() { - err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { - args := deployment.Spec.Template.Spec.Containers[0].Args - args = append(args, "--enable-ssl-passthrough") - deployment.Spec.Template.Spec.Containers[0].Args = args - _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - return err - }) - assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags") - - f.WaitForNginxServer("_", - func(server string) bool { - return strings.Contains(server, "listen 442") - }) - }) - - ginkgo.Describe("With enable-ssl-passthrough enabled", func() { - ginkgo.It("should enable ssl-passthrough-proxy-port on a different port", func() { - err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { - args := deployment.Spec.Template.Spec.Containers[0].Args - args = append(args, "--ssl-passthrough-proxy-port=1442") - deployment.Spec.Template.Spec.Containers[0].Args = args - _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - return err - }) - assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags") - - f.WaitForNginxServer("_", - func(server string) bool { - return strings.Contains(server, "listen 1442") - }) - - f.HTTPTestClient(). - GET("/"). - WithHeader("Host", "something"). - Expect(). - Status(http.StatusNotFound) - }) - - ginkgo.It("should pass unknown traffic to default backend and handle known traffic", func() { - host := "testpassthrough.com" - echoName := "echopass" - - /* Even with enable-ssl-passthrough enabled, only annotated ingresses may receive the traffic */ - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/ssl-passthrough": "true", - } - - ingressDef := framework.NewSingleIngressWithTLS(host, - "/", - host, - []string{host}, - f.Namespace, - echoName, - 80, - annotations) - tlsConfig, err := framework.CreateIngressTLSSecret(f.KubeClientSet, - ingressDef.Spec.TLS[0].Hosts, - ingressDef.Spec.TLS[0].SecretName, - ingressDef.Namespace) - - volumeMount := []corev1.VolumeMount{ - { - Name: "certs", - ReadOnly: true, - MountPath: "/certs", - }, - } - volume := []corev1.Volume{ - { - Name: "certs", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: ingressDef.Spec.TLS[0].SecretName, - }, - }, - }, - } - envs := []corev1.EnvVar{ - { - Name: "HTTPBUN_SSL_CERT", - Value: "/certs/tls.crt", - }, - { - Name: "HTTPBUN_SSL_KEY", - Value: "/certs/tls.key", - }, - } - - f.NewDeploymentWithOpts("echopass", - framework.HTTPBunImage, - 80, - 1, - nil, - nil, - envs, - volumeMount, - volume, - false) - - f.EnsureIngress(ingressDef) - - assert.Nil(ginkgo.GinkgoT(), err) - framework.WaitForTLS(f.GetURL(framework.HTTPS), tlsConfig) - - f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "listen 442") - }) - - /* This one should not receive traffic as it does not contain passthrough annotation */ - hostBad := "noannotationnopassthrough.com" - ingBad := f.EnsureIngress(framework.NewSingleIngressWithTLS(hostBad, - "/", - hostBad, - []string{hostBad}, - f.Namespace, - echoName, - 80, - nil)) - tlsConfigBad, err := framework.CreateIngressTLSSecret(f.KubeClientSet, - ingBad.Spec.TLS[0].Hosts, - ingBad.Spec.TLS[0].SecretName, - ingBad.Namespace) - assert.Nil(ginkgo.GinkgoT(), err) - framework.WaitForTLS(f.GetURL(framework.HTTPS), tlsConfigBad) - - f.WaitForNginxServer(hostBad, - func(server string) bool { - return strings.Contains(server, "listen 442") - }) - - //nolint:gosec // Ignore the gosec error in testing - f.HTTPTestClientWithTLSConfig(&tls.Config{ServerName: host, InsecureSkipVerify: true}). - GET("/"). - WithURL("https://"+net.JoinHostPort(host, "443")). - ForceResolve(f.GetNginxIP(), 443). - Expect(). - Status(http.StatusOK) - - //nolint:gosec // Ignore the gosec error in testing - f.HTTPTestClientWithTLSConfig(&tls.Config{ServerName: hostBad, InsecureSkipVerify: true}). - GET("/"). - WithURL("https://"+net.JoinHostPort(hostBad, "443")). - ForceResolve(f.GetNginxIP(), 443). - Expect(). - Status(http.StatusNotFound) - }) - }) -}) diff --git a/test/e2e/settings/stream_snippet.go b/test/e2e/settings/stream_snippet.go deleted file mode 100644 index e1552afd3..000000000 --- a/test/e2e/settings/stream_snippet.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package settings - -import ( - "context" - "fmt" - "net/http" - "strings" - - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.DescribeSetting("configmap stream-snippet", func() { - f := framework.NewDefaultFramework("cm-stream-snippet") - - ginkgo.BeforeEach(func() { - f.NewEchoDeployment() - }) - - ginkgo.It("should add value of stream-snippet via config map to nginx config", func() { - host := "foo.com" - snippet := `server {listen 8000; proxy_pass 127.0.0.1:80;}` - - f.SetNginxConfigMapData(map[string]string{ - "stream-snippet": snippet, - }) - - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil) - f.EnsureIngress(ing) - - svc, err := f.KubeClientSet. - CoreV1(). - Services(f.Namespace). - Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining ingress-nginx service") - assert.NotNil(ginkgo.GinkgoT(), svc, "expected a service but none returned") - - svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{ - Name: framework.EchoService, - Port: 8000, - TargetPort: intstr.FromInt(8000), - }) - - _, err = f.KubeClientSet. - CoreV1(). - Services(f.Namespace). - Update(context.TODO(), svc, metav1.UpdateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating service") - - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - - f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, snippet) - }) - - f.HTTPTestClient(). - GET("/healthz"). - WithURL(fmt.Sprintf("http://%v:8000/healthz", f.GetNginxIP())). - Expect(). - Status(http.StatusOK) - }) -}) diff --git a/test/e2e/settings/tls.go b/test/e2e/settings/tls.go index 1ebf358c1..b3026aa8c 100644 --- a/test/e2e/settings/tls.go +++ b/test/e2e/settings/tls.go @@ -22,14 +22,13 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/ingress-nginx/test/e2e/framework" ) -var _ = framework.DescribeSetting("[SSL] TLS protocols, ciphers and headers", func() { +var _ = framework.DescribeSetting("[SSL] TLS protocols, ciphers and headers)", func() { f := framework.NewDefaultFramework("settings-tls") host := "settings-tls" @@ -85,10 +84,34 @@ var _ = framework.DescribeSetting("[SSL] TLS protocols, ciphers and headers", fu assert.Equal(ginkgo.GinkgoT(), int(resp.TLS.Version), tls.VersionTLS12) assert.Equal(ginkgo.GinkgoT(), resp.TLS.CipherSuite, tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) }) + ginkgo.It("enforcing TLS v1.0", func() { + f.SetNginxConfigMapData(map[string]string{ + sslCiphers: testCiphers, + sslProtocols: "TLSv1", + }) + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, "ssl_protocols TLSv1;") + }) + + resp := f.HTTPTestClientWithTLSConfig(tlsConfig). + GET("/"). + WithURL(f.GetURL(framework.HTTPS)). + WithHeader("Host", host). + Expect(). + Status(http.StatusOK). + Raw() + + assert.Equal(ginkgo.GinkgoT(), int(resp.TLS.Version), tls.VersionTLS10) + assert.Equal(ginkgo.GinkgoT(), resp.TLS.CipherSuite, tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA) + }) }) ginkgo.Context("should configure HSTS policy header", func() { - var tlsConfig *tls.Config + var ( + tlsConfig *tls.Config + ) const ( hstsMaxAge = "hsts-max-age" @@ -110,9 +133,8 @@ var _ = framework.DescribeSetting("[SSL] TLS protocols, ciphers and headers", fu ginkgo.It("setting max-age parameter", func() { f.UpdateNginxConfigMapData(hstsMaxAge, "86400") - f.WaitForLuaConfiguration(func(jsonCfg map[string]interface{}) bool { - val, ok, err := unstructured.NestedString(jsonCfg, "hsts_max_age") - return err == nil && ok && val == "86400" + f.WaitForNginxConfiguration(func(server string) bool { + return strings.Contains(server, fmt.Sprintf(`hsts_max_age = 86400,`)) }) f.HTTPTestClientWithTLSConfig(tlsConfig). @@ -130,9 +152,8 @@ var _ = framework.DescribeSetting("[SSL] TLS protocols, ciphers and headers", fu hstsIncludeSubdomains: "false", }) - f.WaitForLuaConfiguration(func(jsonCfg map[string]interface{}) bool { - val, ok, err := unstructured.NestedBool(jsonCfg, "hsts_include_subdomains") - return err == nil && ok && !val + f.WaitForNginxConfiguration(func(server string) bool { + return strings.Contains(server, fmt.Sprintf(`hsts_include_subdomains = false,`)) }) f.HTTPTestClientWithTLSConfig(tlsConfig). @@ -151,9 +172,8 @@ var _ = framework.DescribeSetting("[SSL] TLS protocols, ciphers and headers", fu hstsIncludeSubdomains: "false", }) - f.WaitForLuaConfiguration(func(jsonCfg map[string]interface{}) bool { - val, ok, err := unstructured.NestedBool(jsonCfg, "hsts_preload") - return err == nil && ok && val + f.WaitForNginxConfiguration(func(server string) bool { + return strings.Contains(server, fmt.Sprintf(`hsts_preload = true,`)) }) f.HTTPTestClientWithTLSConfig(tlsConfig). @@ -172,18 +192,17 @@ var _ = framework.DescribeSetting("[SSL] TLS protocols, ciphers and headers", fu hstsIncludeSubdomains: "false", }) - expectResponse := f.HTTPTestClientWithTLSConfig(tlsConfig). - GET("/"). - WithURL(f.GetURL(framework.HTTPS)). - WithHeader("Host", host). - WithQuery("hsts", "true"). - Expect() - - expectResponse.Header("Strict-Transport-Security").Equal("max-age=86400; preload") - header := expectResponse.Raw().Header - got := header["Strict-Transport-Security"] - assert.Equal(ginkgo.GinkgoT(), 1, len(got)) + // we can not use gorequest here because it flattens the duplicate headers + // and specifically in case of Strict-Transport-Security it ignore extra headers + // intead of concatenating, rightfully. And I don't know of any API it provides for getting raw headers. + curlCmd := fmt.Sprintf("curl -I -k --fail --silent --resolve settings-tls:443:127.0.0.1 https://settings-tls%v", "?hsts=true") + output, err := f.ExecIngressPod(curlCmd) + assert.Nil(ginkgo.GinkgoT(), err) + assert.Contains(ginkgo.GinkgoT(), output, "strict-transport-security: max-age=86400; preload") + // this is what the upstream sets + assert.NotContains(ginkgo.GinkgoT(), output, "strict-transport-security: max-age=3600; preload") }) + }) ginkgo.Context("ports or X-Forwarded-Host check during HTTP tp HTTPS redirection", func() { diff --git a/test/e2e/settings/validations/validations.go b/test/e2e/settings/validations/validations.go deleted file mode 100644 index 881de39b4..000000000 --- a/test/e2e/settings/validations/validations.go +++ /dev/null @@ -1,101 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package annotations - -import ( - "context" - - "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "k8s.io/ingress-nginx/test/e2e/framework" -) - -var _ = framework.IngressNginxDescribeSerial("annotation validations", func() { - f := framework.NewDefaultFramework("validations") - //nolint:dupl // Ignore dupl errors for similar test case - ginkgo.It("should allow ingress based on their risk on webhooks", func() { - f.SetNginxConfigMapData(map[string]string{ - "allow-snippet-annotations": "true", - }) - defer func() { - f.SetNginxConfigMapData(map[string]string{ - "allow-snippet-annotations": "false", - }) - }() - - host := "annotation-validations" - - // Low and Medium Risk annotations should be allowed, the rest should be denied - f.UpdateNginxConfigMapData("annotations-risk-level", "Medium") - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/default-backend": "bla", // low risk - "nginx.ingress.kubernetes.io/denylist-source-range": "1.1.1.1/32", // medium risk - } - - ginkgo.By("allow ingress with low/medium risk annotations") - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - _, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Create(context.TODO(), ing, metav1.CreateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "creating ingress with allowed annotations should not trigger an error") - - ginkgo.By("block ingress with risky annotations") - annotations["nginx.ingress.kubernetes.io/modsecurity-transaction-id"] = "bla123" // High should be blocked - annotations["nginx.ingress.kubernetes.io/modsecurity-snippet"] = "some random stuff;" // High should be blocked - ing = framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - _, err = f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{}) - assert.NotNil(ginkgo.GinkgoT(), err, "creating ingress with risky annotations should trigger an error") - }) - //nolint:dupl // Ignore dupl errors for similar test case - ginkgo.It("should allow ingress based on their risk on webhooks", func() { - f.SetNginxConfigMapData(map[string]string{ - "allow-snippet-annotations": "true", - }) - defer func() { - f.SetNginxConfigMapData(map[string]string{ - "allow-snippet-annotations": "false", - }) - }() - host := "annotation-validations" - - // Low and Medium Risk annotations should be allowed, the rest should be denied - f.UpdateNginxConfigMapData("annotations-risk-level", "Medium") - // Sleep a while just to guarantee that the configmap is applied - framework.Sleep() - - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/default-backend": "bla", // low risk - "nginx.ingress.kubernetes.io/denylist-source-range": "1.1.1.1/32", // medium risk - } - - ginkgo.By("allow ingress with low/medium risk annotations") - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - _, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Create(context.TODO(), ing, metav1.CreateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "creating ingress with allowed annotations should not trigger an error") - - ginkgo.By("block ingress with risky annotations") - annotations["nginx.ingress.kubernetes.io/modsecurity-transaction-id"] = "bla123" // High should be blocked - annotations["nginx.ingress.kubernetes.io/modsecurity-snippet"] = "some random stuff;" // High should be blocked - ing = framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) - _, err = f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{}) - assert.NotNil(ginkgo.GinkgoT(), err, "creating ingress with risky annotations should trigger an error") - }) -}) diff --git a/test/e2e/ssl/http_redirect.go b/test/e2e/ssl/http_redirect.go index 982dd4f6c..5968397cb 100644 --- a/test/e2e/ssl/http_redirect.go +++ b/test/e2e/ssl/http_redirect.go @@ -20,7 +20,7 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" "k8s.io/ingress-nginx/test/e2e/framework" diff --git a/test/e2e/ssl/secret_update.go b/test/e2e/ssl/secret_update.go index 8e81f09f9..05c73b9b8 100644 --- a/test/e2e/ssl/secret_update.go +++ b/test/e2e/ssl/secret_update.go @@ -23,7 +23,7 @@ import ( "net/http" "strings" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -73,8 +73,7 @@ var _ = framework.IngressNginxDescribe("[SSL] secret update", func() { dummySecret.Data["some-key"] = []byte("some value") - _, err = f.KubeClientSet.CoreV1().Secrets(f.Namespace).Update(context.TODO(), dummySecret, metav1.UpdateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "updating secret") + f.KubeClientSet.CoreV1().Secrets(f.Namespace).Update(context.TODO(), dummySecret, metav1.UpdateOptions{}) assert.NotContains(ginkgo.GinkgoT(), log, fmt.Sprintf("starting syncing of secret %v/dummy", f.Namespace)) assert.NotContains(ginkgo.GinkgoT(), log, fmt.Sprintf("error obtaining PEM from secret %v/dummy", f.Namespace)) @@ -98,7 +97,7 @@ var _ = framework.IngressNginxDescribe("[SSL] secret update", func() { return strings.Contains(server, "server_name invalid-ssl") && strings.Contains(server, "listen 443") }) - //nolint:gosec // Ignore certificate validation in testing + resp := f.HTTPTestClientWithTLSConfig(&tls.Config{ServerName: host, InsecureSkipVerify: true}). GET("/"). WithURL(f.GetURL(framework.HTTPS)). diff --git a/test/e2e/status/update.go b/test/e2e/status/update.go index c3c48f8d2..af0c6b9a4 100644 --- a/test/e2e/status/update.go +++ b/test/e2e/status/update.go @@ -24,11 +24,11 @@ import ( "strings" "time" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/networking/v1" + apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -69,9 +69,9 @@ var _ = framework.IngressNginxDescribe("[Status] status update", func() { }) assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating ingress controller deployment flags") - f.NewEchoDeployment() + f.NewEchoDeploymentWithReplicas(1) - f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil)) + ing := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil)) f.WaitForNginxConfiguration( func(cfg string) bool { @@ -84,18 +84,18 @@ var _ = framework.IngressNginxDescribe("[Status] status update", func() { err = cmd.Process.Kill() assert.Nil(ginkgo.GinkgoT(), err, "unexpected error terminating kubectl proxy") - ing, err := f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) + ing, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "unexpected error getting %s/%v Ingress", f.Namespace, host) - ing.Status.LoadBalancer.Ingress = []v1.IngressLoadBalancerIngress{} - _, err = f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).UpdateStatus(context.TODO(), ing, metav1.UpdateOptions{}) + ing.Status.LoadBalancer.Ingress = []apiv1.LoadBalancerIngress{} + _, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).UpdateStatus(context.TODO(), ing, metav1.UpdateOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "unexpected error cleaning Ingress status") framework.Sleep(10 * time.Second) - err = f.KubeClientSet.CoordinationV1(). - Leases(f.Namespace). - Delete(context.TODO(), "ingress-controller-leader", metav1.DeleteOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error deleting leader election lease") + err = f.KubeClientSet.CoreV1(). + ConfigMaps(f.Namespace). + Delete(context.TODO(), "ingress-controller-leader-nginx", metav1.DeleteOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "unexpected error deleting leader election configmap") _, cmd, err = f.KubectlProxy(port) assert.Nil(ginkgo.GinkgoT(), err, "unexpected error starting kubectl proxy") @@ -108,9 +108,8 @@ var _ = framework.IngressNginxDescribe("[Status] status update", func() { } }() - //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated err = wait.Poll(5*time.Second, 4*time.Minute, func() (done bool, err error) { - ing, err = f.KubeClientSet.NetworkingV1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) + ing, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{}) if err != nil { return false, nil } @@ -122,9 +121,9 @@ var _ = framework.IngressNginxDescribe("[Status] status update", func() { return true, nil }) assert.Nil(ginkgo.GinkgoT(), err, "unexpected error waiting for ingress status") - assert.Equal(ginkgo.GinkgoT(), ing.Status.LoadBalancer.Ingress, []v1.IngressLoadBalancerIngress{ + assert.Equal(ginkgo.GinkgoT(), ing.Status.LoadBalancer.Ingress, ([]apiv1.LoadBalancerIngress{ {IP: "1.1.0.0"}, - }) + })) }) }) @@ -135,8 +134,7 @@ func getHostIP() net.IP { } defer conn.Close() - localAddr, ok := conn.LocalAddr().(*net.UDPAddr) - assert.True(ginkgo.GinkgoT(), ok, "unexpected type: %T", conn.LocalAddr()) + localAddr := conn.LocalAddr().(*net.UDPAddr) return localAddr.IP } diff --git a/test/e2e/tcpudp/tcp.go b/test/e2e/tcpudp/tcp.go index 856184d18..9b1885510 100644 --- a/test/e2e/tcpudp/tcp.go +++ b/test/e2e/tcpudp/tcp.go @@ -21,11 +21,10 @@ import ( "fmt" "net" "net/http" - "regexp" "strings" "time" - "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -37,39 +36,58 @@ import ( var _ = framework.IngressNginxDescribe("[TCP] tcp-services", func() { f := framework.NewDefaultFramework("tcp") - var ip string - - ginkgo.BeforeEach(func() { - ip = f.GetNginxIP() - }) ginkgo.It("should expose a TCP service", func() { - f.NewEchoDeployment() + f.NewEchoDeploymentWithReplicas(1) - cm := f.GetConfigMap(f.Namespace, "tcp-services") - cm.Data = map[string]string{ - "8080": fmt.Sprintf("%v/%v:80", f.Namespace, framework.EchoService), + config, err := f.KubeClientSet. + CoreV1(). + ConfigMaps(f.Namespace). + Get(context.TODO(), "tcp-services", metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining tcp-services configmap") + assert.NotNil(ginkgo.GinkgoT(), config, "expected a configmap but none returned") + + if config.Data == nil { + config.Data = map[string]string{} } - f.EnsureConfigMap(cm) - svc := f.GetService(f.Namespace, "nginx-ingress-controller") + config.Data["8080"] = fmt.Sprintf("%v/%v:80", f.Namespace, framework.EchoService) + + _, err = f.KubeClientSet. + CoreV1(). + ConfigMaps(f.Namespace). + Update(context.TODO(), config, metav1.UpdateOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating configmap") + + svc, err := f.KubeClientSet. + CoreV1(). + Services(f.Namespace). + Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining ingress-nginx service") + assert.NotNil(ginkgo.GinkgoT(), svc, "expected a service but none returned") + svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{ Name: framework.EchoService, Port: 8080, TargetPort: intstr.FromInt(8080), }) - _, err := f.KubeClientSet. + _, err = f.KubeClientSet. CoreV1(). Services(f.Namespace). Update(context.TODO(), svc, metav1.UpdateOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating service") + // wait for update and nginx reload and new endpoint is available + framework.Sleep() + f.WaitForNginxConfiguration( func(cfg string) bool { return strings.Contains(cfg, fmt.Sprintf(`ngx.var.proxy_upstream_name="tcp-%v-%v-80"`, f.Namespace, framework.EchoService)) }) + ip := f.GetNginxIP() + f.HTTPTestClient(). GET("/"). WithURL(fmt.Sprintf("http://%v:8080", ip)). @@ -104,25 +122,44 @@ var _ = framework.IngressNginxDescribe("[TCP] tcp-services", func() { } f.EnsureService(externalService) - // Expose the `external name` port on the `ingress-nginx-controller` service - svc := f.GetService(f.Namespace, "nginx-ingress-controller") + // Expose the `external name` port on the `ingress-nginx` service + svc, err := f.KubeClientSet. + CoreV1(). + Services(f.Namespace). + Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining ingress-nginx service") + assert.NotNil(ginkgo.GinkgoT(), svc, "expected a service but none returned") + svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{ Name: "dns-svc", Port: 5353, TargetPort: intstr.FromInt(5353), }) - _, err := f.KubeClientSet. + _, err = f.KubeClientSet. CoreV1(). Services(f.Namespace). Update(context.TODO(), svc, metav1.UpdateOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating service") // Update the TCP configmap to link port 5353 to the DNS external name service - config := f.GetConfigMap(f.Namespace, "tcp-services") - config.Data = map[string]string{ - "5353": fmt.Sprintf("%v/dns-external-name-svc:5353", f.Namespace), + config, err := f.KubeClientSet. + CoreV1(). + ConfigMaps(f.Namespace). + Get(context.TODO(), "tcp-services", metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining tcp-services configmap") + assert.NotNil(ginkgo.GinkgoT(), config, "expected a configmap but none returned") + + if config.Data == nil { + config.Data = map[string]string{} } - f.EnsureConfigMap(config) + + config.Data["5353"] = fmt.Sprintf("%v/dns-external-name-svc:5353", f.Namespace) + + _, err = f.KubeClientSet. + CoreV1(). + ConfigMaps(f.Namespace). + Update(context.TODO(), config, metav1.UpdateOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating configmap") // Validate that the generated nginx config contains the expected `proxy_upstream_name` value f.WaitForNginxConfiguration( @@ -131,9 +168,10 @@ var _ = framework.IngressNginxDescribe("[TCP] tcp-services", func() { }) // Execute the test. Use the `external name` service to resolve a domain name. + ip := f.GetNginxIP() resolver := net.Resolver{ PreferGo: true, - Dial: func(ctx context.Context, _, _ string) (net.Conn, error) { + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { d := net.Dialer{} return d.DialContext(ctx, "tcp", fmt.Sprintf("%v:5353", ip)) }, @@ -148,72 +186,21 @@ var _ = framework.IngressNginxDescribe("[TCP] tcp-services", func() { } var ips []string - var errRetry error + var retryErr error err = wait.ExponentialBackoff(retry, func() (bool, error) { - ips, errRetry = resolver.LookupHost(context.Background(), "google-public-dns-b.google.com") - if errRetry == nil { + ips, retryErr = resolver.LookupHost(context.Background(), "google-public-dns-b.google.com") + if retryErr == nil { return true, nil } return false, nil }) - //nolint:staticcheck // TODO: will replace it since wait.ErrWaitTimeout is deprecated + if err == wait.ErrWaitTimeout { - err = errRetry + err = retryErr } assert.Nil(ginkgo.GinkgoT(), err, "unexpected error from DNS resolver") assert.Contains(ginkgo.GinkgoT(), ips, "8.8.4.4") }) - - ginkgo.It("should reload after an update in the configuration", func() { - ginkgo.By("setting up a first deployment") - f.NewEchoDeployment(framework.WithDeploymentName("first-service")) - - cm := f.GetConfigMap(f.Namespace, "tcp-services") - cm.Data = map[string]string{ - "8080": fmt.Sprintf("%v/first-service:80", f.Namespace), - } - f.EnsureConfigMap(cm) - - checksumRegex := regexp.MustCompile(`Configuration checksum:\s+(\d+)`) - checksum := "" - - f.WaitForNginxConfiguration( - func(cfg string) bool { - // before returning, extract the current checksum - match := checksumRegex.FindStringSubmatch(cfg) - if len(match) > 0 { - checksum = match[1] - } - - return strings.Contains(cfg, fmt.Sprintf(`ngx.var.proxy_upstream_name="tcp-%v-first-service-80"`, - f.Namespace)) - }) - assert.NotEmpty(ginkgo.GinkgoT(), checksum) - - ginkgo.By("updating the tcp service to a second deployment") - f.NewEchoDeployment(framework.WithDeploymentName("second-service")) - - cm = f.GetConfigMap(f.Namespace, "tcp-services") - cm.Data["8080"] = fmt.Sprintf("%v/second-service:80", f.Namespace) - f.EnsureConfigMap(cm) - - newChecksum := "" - f.WaitForNginxConfiguration( - func(cfg string) bool { - match := checksumRegex.FindStringSubmatch(cfg) - if len(match) > 0 { - newChecksum = match[1] - } - - return strings.Contains(cfg, fmt.Sprintf(`ngx.var.proxy_upstream_name="tcp-%v-second-service-80"`, - f.Namespace)) - }) - assert.NotEqual(ginkgo.GinkgoT(), checksum, newChecksum) - - logs, err := f.NginxLogs() - assert.Nil(ginkgo.GinkgoT(), err, "obtaining nginx logs") - assert.Contains(ginkgo.GinkgoT(), logs, "Backend successfully reloaded") - }) }) diff --git a/test/e2e/wait-for-nginx.sh b/test/e2e/wait-for-nginx.sh index 73023aba1..1eb8b32cc 100755 --- a/test/e2e/wait-for-nginx.sh +++ b/test/e2e/wait-for-nginx.sh @@ -23,7 +23,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export NAMESPACE=$1 export NAMESPACE_OVERLAY=$2 -export IS_CHROOT=$3 echo "deploying NGINX Ingress controller in namespace $NAMESPACE" @@ -60,10 +59,8 @@ fullnameOverride: nginx-ingress controller: image: repository: ingress-controller/controller - chroot: ${IS_CHROOT} tag: 1.0.0-dev digest: - digestChroot: scope: enabled: true config: @@ -76,10 +73,6 @@ controller: periodSeconds: 1 service: type: NodePort - electionID: ingress-controller-leader - ingressClassResource: - # We will create and remove each IC/ClusterRole/ClusterRoleBinding per test so there's no conflict - enabled: false extraArgs: tcp-services-configmap: $NAMESPACE/tcp-services # e2e tests do not require information about ingress status @@ -87,8 +80,6 @@ controller: terminationGracePeriodSeconds: 1 admissionWebhooks: enabled: false - metrics: - enabled: true # ulimit -c unlimited # mkdir -p /tmp/coredump @@ -103,8 +94,6 @@ controller: hostPath: path: /tmp/coredump -${OTEL_MODULE} - rbac: create: true scope: true diff --git a/test/k6/README.md b/test/k6/README.md deleted file mode 100644 index 81169d3f6..000000000 --- a/test/k6/README.md +++ /dev/null @@ -1,83 +0,0 @@ -# Performance testing ingress-nginx-controller in GithubAction-CI -This README will evolve as the development of testing occurs. - -## INFORMATION -### 1. No CPU/Memory for stress -- Github-Actions job runner is a 2core 7Gig VM so that limits what/how we test https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners - - image - - - Need to eventually get our own beefy runner, with enough cpu/memory to handle stress level load - -### 2. Scale is work-in-progress -- We are grateful to have got a free account on K6.io, as part of their OSS Program. But it is limited to 600 tests per year. - -### 3. No Testplans -- Testplan discussion and coding is needed for more practical real-world testing reports - -## DESCRIPTION - -### What -- An issue was created for performance tests, for the ingress-nginx-controller builds, https://github.com/kubernetes/ingress-nginx/issues/8033 . - -### How -- A step by step guide to using https://k6.io with GithubActions is here https://k6.io/blog/load-testing-using-github-actions/ - - - The link above contains sample code - image - - - - Copy sample test code from website and edit to taste - image - - - The CI launches a ubuntu environment and uses `make dev-env` to create a kind cluster. The popular https://httpbin.org api docker image is used to create a workload - - image - - - We don't want the test to block CI so this syntax from Github-Actions creates a button to run the test - image - - - The button looks like this (the `Run Workflow` dropdown at bottom right of screenshot) - image - - image - - -### fqdn -- Obtained a freenom domain `ingress-nginx-controller.ga` - - - The test uses a fqdn `test.ingress-nginx-controller.ga` - - - The K6 api has configuration options for dns resolution of (above mentioned fqdn) to localhost/loopback/127.0.0.1 (`make dev-env` cluster) - image - - - Will need to discuss and decide on fqdn, as it relates to tls secret - -### tls -- Procured a letsencrypt wildcard certificate for `*.ingress-nginx-controller.ga` - - - base64 encoded hash of the cert + key is stored in the `Github Project Settings Secrets` as a variable - - - The `GithubActions secrets` variables are decoded in the CI to create the TLS secret - - image - - -### Visualization -- Plan is to run tests locally on a kind cluster, in the CI pipeline, but push results to K6-cloud - - - Pushing and visualization on K6 cloud is as simple as executing `k6 run -o cloud test.js` - - - Currently there is a personal account in trial period (50 tests or 1 year limit) bing used - - - Pushing test-results from K6 tests on laptop, to K6-cloud personal trial account on K6-Cloud, to see what the graphs look like - - image - - image - - - The cli result looks like this - image - -- Before merging the PR, the testing is being done on personal Github project with exact same code as this PR here https://github.com/longwuyuan/k6-loadtest-example/runs/6545706269?check_suite_focus=true - diff --git a/test/k6/loadtest.js b/test/k6/loadtest.js deleted file mode 100644 index 51801765e..000000000 --- a/test/k6/loadtest.js +++ /dev/null @@ -1,49 +0,0 @@ -// This is a loadtest under development -// Test here is spec'd to have 100virtual-users -// Other specs currently similar to smoketest -// But loadtest needs testplan that likely uses auth & data-transfer - -import http from 'k6/http'; -import { sleep } from 'k6'; - -export const options = { - hosts: { - 'test.ingress-nginx-controller.ga:80': '127.0.0.1:80', - 'test.ingress-nginx-controller.ga:443': '127.0.0.1:443', - }, - duration: '1m', - vus: 100, - thresholds: { - http_req_failed: ['rate<0.01'], // http errors should be less than 1% - http_req_duration: ['p(95)<500'], // 95 percent of response times must be below 500ms - http_req_duration: ['p(99)<1500'], // 99 percent of response times must be below 1500ms - }, -}; - -export default function () { - const params = { - headers: {'host': 'test.ingress-nginx-controller.ga'}, - }; - const req1 = { - method: 'GET', - url: 'http://test.ingress-nginx-controller.ga/ip', - }; - const req2 = { - method: 'GET', - url: 'http://test.ingress-nginx-controller.ga/image/svg', - }; - const req3 = { - params: { - headers: { - 'Content-Type': 'application/x-www-form-urlencoded' - }, - }, - method: 'POST', - url: 'https://test.ingress-nginx-controller.ga/post', - body: { - hello: 'world!', - }, - }; - const res = http.batch([req1, req2, req3], params); - sleep(1); -} diff --git a/test/k6/smoketest.js b/test/k6/smoketest.js deleted file mode 100644 index 12691b63a..000000000 --- a/test/k6/smoketest.js +++ /dev/null @@ -1,64 +0,0 @@ -// smoketest.js edited after copy/pasting from https://k6.io docs -// Using this like loadtest because of limited cpu/memory/other - -import http from 'k6/http'; -import { sleep } from 'k6'; - -export const options = { - // testbed created with "make dev-env" requires this name resolution - // this does not set the host header - hosts: { - 'test.ingress-nginx-controller.ga:80': '127.0.0.1:80', - 'test.ingress-nginx-controller.ga:443': '127.0.0.1:443', - }, - // below 3 lines documented at https://k6.io - duration: '1m', - vus: 50, - thresholds: { - http_req_failed: ['rate<0.01'], // http errors should be less than 1% - http_req_duration: ['p(95)<500'], // 95 percent of response times must be below 500ms - http_req_duration: ['p(99)<1500'], // 99 percent of response times must be below 1500ms - }, -}; - -export default function () { - // docs of k6 say this is how to add host header - // needed as ingress is created with this host value - const params = { - headers: {'host': 'test.ingress-nginx-controller.ga'}, - }; - // httpbun.com documents these requests - const req1 = { - method: 'GET', - url: 'http://test.ingress-nginx-controller.ga/ip', - }; - const req2 = { - method: 'GET', - url: 'http://test.ingress-nginx-controller.ga/image/svg', - }; - const req3 = { - params: { - headers: { - 'Content-Type': 'application/json' - }, - }, - method: 'POST', - url: 'https://test.ingress-nginx-controller.ga/post', - body: { - 'key1': 'Hello World!', - }, - }; - const req4 = { - method: 'GET', - url: 'https://test.ingress-nginx-controller.ga/basic-auth/admin/admin', - params: { - headers: { - 'accept': 'application/jsom', - } - } - } - for(let i=0; i<20; i++){ - const res = http.batch([req0, req1, req2, req3, req4], params); - sleep(1); - } -} diff --git a/test/manifests/configuration-a.json b/test/manifests/configuration-a.json index f9599d77c..ba513c616 100644 --- a/test/manifests/configuration-a.json +++ b/test/manifests/configuration-a.json @@ -302,12 +302,7 @@ "validationDepth": 0 }, "use-port-in-redirects": false, - "configuration-snippet": "", - "customHeaders": { - "headers": { - "Server": "HAL9000" - } - } + "configuration-snippet": "" }] }, { "hostname": "dev.mycompany.com", diff --git a/test/manifests/configuration-b.json b/test/manifests/configuration-b.json index d2e71bb29..9e40785b4 100644 --- a/test/manifests/configuration-b.json +++ b/test/manifests/configuration-b.json @@ -302,12 +302,7 @@ "validationDepth": 0 }, "use-port-in-redirects": false, - "configuration-snippet": "", - "customHeaders": { - "headers": { - "Server": "HAL9000" - } - } + "configuration-snippet": "" }] }, { "hostname": "dev.mycompany.com",