Chart: Tighten securityContexts and Pod Security Policies. (#10491)

* Values: Fix docs of `controller.podSecurityContext` & `controller.sysctls`.

* Values: Add missing `controller.containerSecurityContext`.

Already in use, but has never been added to values.

* Values: Fix docs of `defaultBackend.podSecurityContext` & `defaultBackend.containerSecurityContext`.

* Helpers: Rename `controller.containerSecurityContext` to `ingress-nginx.controller.containerSecurityContext`.

Due to alignment with other templates.

* Helpers: Improve `extraModules`.

- Make `command` a multiline list.
- Fix `toYaml` usage.
- Remove `toYaml` where not necessary.

* Helpers: Move `ingress-nginx.defaultBackend.fullname`.

* Helpers: Add `ingress-nginx.defaultBackend.containerSecurityContext`.

Extracts the default backend `securityContext` into a template, as for the controller.

* Controller: Fix indentation of `controller.podSecurityContext` & `controller.sysctls`.

* Controller: Improve `controller.extraModules` & `controller.opentelemetry`.

- Add `controller.extraModules.distroless` & `controller.extraModules.resources`.
- Add `controller.opentelemetry.name` & `controller.opentelemetry.distroless`.
- Align `extraModules` inclusion for `controller.extraModules` & `controller.opentelemetry`.
- Remove redundant whitespaces.

* Controller/PSP: Align indentation.

* Controller/PSP: Remove quotes.

* Controller/PSP: Improve comments.

* Controller/PSP: Reorder fields.

See https://v1-24.docs.kubernetes.io/docs/concepts/security/pod-security-policy.

* Admission Webhooks: Fix indentation of `controller.admissionWebhooks.patch.securityContext`.

* Admission Webhooks/PSP: Align indentation.

* Admission Webhooks/PSP: Reorder fields.

* Admission Webhooks/PSP: Align condition.

* Admission Webhooks/ClusterRole: Align PSP rule.

* Default Backend/PSP: Align indentation.

* Default Backend/PSP: Reorder fields.

See https://v1-24.docs.kubernetes.io/docs/concepts/security/pod-security-policy.

* Values: Tighten `controller.image`.

Due to recent changes, the controller image can be run without privilege escalation:

- https://github.com/kubernetes/ingress-nginx/issues/8499
- https://github.com/kubernetes/ingress-nginx/pull/7449

* Values: Tighten `controller.extraModules.containerSecurityContext`.

* Values: Tighten `controller.opentelemetry.containerSecurityContext`.

* Values: Tighten `controller.admissionWebhooks.*.securityContext`.

Moves the pod `securityContext` to the containers to not interfere with injected containers.

* Values: Tighten `defaultBackend.image`.

charts/ingress-nginx/templates/controller-deployment.yaml

@@ -63,16 +63,16 @@ spec:
     {{- end }}
     {{- if or .Values.controller.podSecurityContext .Values.controller.sysctls }}
       securityContext:
-    {{- end }}
-    {{- if .Values.controller.podSecurityContext }}
+      {{- if .Values.controller.podSecurityContext }}
         {{- toYaml .Values.controller.podSecurityContext | nindent 8 }}
-    {{- end }}
-    {{- if .Values.controller.sysctls }}
+      {{- end }}
+      {{- if .Values.controller.sysctls }}
         sysctls:
-    {{- range $sysctl, $value := .Values.controller.sysctls }}
-        - name: {{ $sysctl | quote }}
-          value: {{ $value | quote }}
-    {{- end }}
+        {{- range $sysctl, $value := .Values.controller.sysctls }}
+          - name: {{ $sysctl | quote }}
+            value: {{ $value | quote }}
+        {{- end }}
+      {{- end }}
     {{- end }}
     {{- if .Values.controller.shareProcessNamespace }}
       shareProcessNamespace: {{ .Values.controller.shareProcessNamespace }}
@@ -86,9 +86,8 @@ spec:
         {{- if .Values.controller.lifecycle }}
           lifecycle: {{ toYaml .Values.controller.lifecycle | nindent 12 }}
         {{- end }}
-          args:
-            {{- include "ingress-nginx.params" . | nindent 12 }}
-          securityContext: {{ include "controller.containerSecurityContext" . | nindent 12 }}
+          args: {{ include "ingress-nginx.params" . | nindent 12 }}
+          securityContext: {{ include "ingress-nginx.controller.containerSecurityContext" . | nindent 12 }}
           env:
             - name: POD_NAME
               valueFrom:
@@ -153,11 +152,11 @@ spec:
           volumeMounts:
           {{- if (or .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }}
             - name: modules
-            {{ if .Values.controller.image.chroot }}
+            {{- if .Values.controller.image.chroot }}
               mountPath: /chroot/modules_mount
-            {{ else }}
+            {{- else }}
               mountPath: /modules_mount
-            {{ end }}
+            {{- end }}
           {{- end }}
           {{- if .Values.controller.customTemplate.configMapName }}
             - mountPath: /etc/nginx/template
@@ -177,24 +176,25 @@ spec:
           resources: {{ toYaml .Values.controller.resources | nindent 12 }}
         {{- end }}
       {{- if .Values.controller.extraContainers }}
-        {{ toYaml .Values.controller.extraContainers | nindent 8 }}
+        {{- toYaml .Values.controller.extraContainers | nindent 8 }}
       {{- end }}
     {{- if (or .Values.controller.extraInitContainers .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }}
       initContainers:
       {{- if .Values.controller.extraInitContainers }}
-        {{ toYaml .Values.controller.extraInitContainers | nindent 8 }}
+        {{- toYaml .Values.controller.extraInitContainers | nindent 8 }}
       {{- end }}
       {{- if .Values.controller.extraModules }}
         {{- range .Values.controller.extraModules }}
           {{- $containerSecurityContext := .containerSecurityContext | default $.Values.controller.containerSecurityContext }}
-          {{- include "extraModules" (dict "name" .name "image" .image "containerSecurityContext" $containerSecurityContext) | nindent 8 }}
+          {{- include "extraModules" (dict "name" .name "image" .image "distroless" .distroless "containerSecurityContext" $containerSecurityContext "resources" .resources) | nindent 8 }}
+        {{- end }}
+      {{- end }}
+      {{- if .Values.controller.opentelemetry.enabled }}
+        {{- with .Values.controller.opentelemetry }}
+          {{- $containerSecurityContext := .containerSecurityContext | default $.Values.controller.containerSecurityContext }}
+          {{- include "extraModules" (dict "name" .name "image" .image "distroless" .distroless "containerSecurityContext" $containerSecurityContext "resources" .resources) | nindent 8 }}
         {{- end }}
       {{- end }}
-      {{- if .Values.controller.opentelemetry.enabled}}
-          {{- $otelContainerSecurityContext := $.Values.controller.opentelemetry.containerSecurityContext | default $.Values.controller.containerSecurityContext }}
-          {{ $otelResources := $.Values.controller.opentelemetry.resources | default dict }}
-          {{- include "extraModules" (dict "name" "opentelemetry" "image" .Values.controller.opentelemetry.image "containerSecurityContext" $otelContainerSecurityContext "distroless" true "resources" $otelResources) | nindent 8}}
-      {{- end}}
     {{- end }}
     {{- if .Values.controller.hostNetwork }}
       hostNetwork: {{ .Values.controller.hostNetwork }}