Jail/chroot nginx process inside controller container (#8337)

* Initial work on chrooting nginx process * More improvements in chroot * Fix charts and some file locations * Fix symlink on non chrooted container * fix psp test * Add e2e tests to chroot image * Fix logger * Add internal logger in controller * Fix overlay for chrooted tests * Fix tests * fix boilerplates * Fix unittest to point to the right pid * Fix PR review
2022-04-09 01:48:04 -03:00 · 2022-04-09 01:48:04 -03:00 · 3def835a6a
commit 3def835a6a
parent 83ce21b4dd
41 changed files with 456 additions and 49 deletions
--- a/test/data/cleanConf.expected.conf
+++ b/test/data/cleanConf.expected.conf
@ -1,7 +1,7 @@
 # Configuration checksum: 

 # setup custom paths that do not require root access
-pid /tmp/nginx.pid;
+pid /tmp/nginx/nginx.pid;

 daemon off;

--- a/test/data/cleanConf.src.conf
+++ b/test/data/cleanConf.src.conf
@ -1,7 +1,7 @@
 # Configuration checksum: 

 # setup custom paths that do not require root access
-pid /tmp/nginx.pid;
+pid /tmp/nginx/nginx.pid;



--- a/test/e2e-image/namespace-overlays/admission/values.yaml
+++ b/test/e2e-image/namespace-overlays/admission/values.yaml
@ -3,6 +3,7 @@ fullnameOverride: nginx-ingress
 controller:
  image:
    repository: ingress-controller/controller
+    chroot: true
    tag: 1.0.0-dev
    digest:
  containerPort:
--- a/test/e2e-image/namespace-overlays/custom-health-check-path/values.yaml
+++ b/test/e2e-image/namespace-overlays/custom-health-check-path/values.yaml
@ -3,6 +3,7 @@ fullnameOverride: nginx-ingress
 controller:
  image:
    repository: ingress-controller/controller
+    chroot: true
    tag: 1.0.0-dev
    digest:
  extraArgs:
--- a/test/e2e-image/namespace-overlays/forwarded-port-headers/values.yaml
+++ b/test/e2e-image/namespace-overlays/forwarded-port-headers/values.yaml
@ -3,6 +3,7 @@ fullnameOverride: nginx-ingress
 controller:
  image:
    repository: ingress-controller/controller
+    chroot: true
    tag: 1.0.0-dev
    digest:
  containerPort:
--- a/test/e2e-image/namespace-overlays/namespace-selector/values.yaml
+++ b/test/e2e-image/namespace-overlays/namespace-selector/values.yaml
@ -3,6 +3,7 @@ fullnameOverride: nginx-ingress
 controller:
  image:
    repository: ingress-controller/controller
+    chroot: true
    tag: 1.0.0-dev
    digest:
  containerPort:
--- a/test/e2e/framework/logs.go
+++ b/test/e2e/framework/logs.go
@ -18,12 +18,15 @@ package framework

 import (
 	"context"
+	"time"

 	"k8s.io/client-go/kubernetes"
 )

 // Logs returns the log entries of a given Pod.
 func Logs(client kubernetes.Interface, namespace, podName string) (string, error) {
+	// Logs from jails take a bigger time to get shipped due to the need of tailing them
+	Sleep(3 * time.Second)
 	logs, err := client.CoreV1().RESTClient().Get().
 		Resource("pods").
 		Namespace(namespace).
--- a/test/e2e/run.sh
+++ b/test/e2e/run.sh
@ -58,7 +58,7 @@ export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/kind-config-$KIND_CLUSTER_NAME}"
 if [ "${SKIP_CLUSTER_CREATION:-false}" = "false" ]; then
  echo "[dev-env] creating Kubernetes cluster with kind"

-  export K8S_VERSION=${K8S_VERSION:-v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6}
+  export K8S_VERSION=${K8S_VERSION:-v1.21.10@sha256:84709f09756ba4f863769bdcabe5edafc2ada72d3c8c44d6515fc581b66b029c}

  kind create cluster \
    --verbosity=${KIND_LOG_LEVEL} \
@ -77,7 +77,7 @@ if [ "${SKIP_IMAGE_CREATION:-false}" = "false" ]; then
  fi

  echo "[dev-env] building image"
-  make -C ${DIR}/../../ clean-image build image
+  make -C ${DIR}/../../ clean-image build image image-chroot
  make -C ${DIR}/../e2e-image image
 fi

@ -87,6 +87,11 @@ KIND_WORKERS=$(kind get nodes --name="${KIND_CLUSTER_NAME}" | grep worker | awk
 echo "[dev-env] copying docker images to cluster..."

 kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes=${KIND_WORKERS} nginx-ingress-controller:e2e
+
+if [ "${IS_CHROOT:-false}" = "true" ]; then
+   docker tag ${REGISTRY}/controller-chroot:${TAG} ${REGISTRY}/controller:${TAG}
+fi
+
 kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes=${KIND_WORKERS} ${REGISTRY}/controller:${TAG}

 echo "[dev-env] running e2e tests..."
--- a/test/e2e/settings/access_log.go
+++ b/test/e2e/settings/access_log.go
@ -31,17 +31,19 @@ var _ = framework.DescribeSetting("access-log", func() {
 		ginkgo.It("use the default configuration", func() {
 			f.WaitForNginxConfiguration(
 				func(cfg string) bool {
-					return strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo") &&
-						strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream")
+					return (strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo") &&
+						strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream")) ||
+						(strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 upstreaminfo") &&
+							strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 log_stream"))
 				})
 		})

 		ginkgo.It("use the specified configuration", func() {
-			f.UpdateNginxConfigMapData("access-log-path", "/tmp/access.log")
+			f.UpdateNginxConfigMapData("access-log-path", "/tmp/nginx/access.log")
 			f.WaitForNginxConfiguration(
 				func(cfg string) bool {
-					return strings.Contains(cfg, "access_log /tmp/access.log upstreaminfo") &&
-						strings.Contains(cfg, "access_log /tmp/access.log log_stream")
+					return strings.Contains(cfg, "access_log /tmp/nginx/access.log upstreaminfo") &&
+						strings.Contains(cfg, "access_log /tmp/nginx/access.log log_stream")
 				})
 		})
 	})
@ -49,11 +51,12 @@ var _ = framework.DescribeSetting("access-log", func() {
 	ginkgo.Context("http-access-log-path", func() {

 		ginkgo.It("use the specified configuration", func() {
-			f.UpdateNginxConfigMapData("http-access-log-path", "/tmp/http-access.log")
+			f.UpdateNginxConfigMapData("http-access-log-path", "/tmp/nginx/http-access.log")
 			f.WaitForNginxConfiguration(
 				func(cfg string) bool {
-					return strings.Contains(cfg, "access_log /tmp/http-access.log upstreaminfo") &&
-						strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream")
+					return strings.Contains(cfg, "access_log /tmp/nginx/http-access.log upstreaminfo") &&
+						(strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream") ||
+							strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 log_stream"))
 				})
 		})
 	})
@ -61,11 +64,12 @@ var _ = framework.DescribeSetting("access-log", func() {
 	ginkgo.Context("stream-access-log-path", func() {

 		ginkgo.It("use the specified configuration", func() {
-			f.UpdateNginxConfigMapData("stream-access-log-path", "/tmp/stream-access.log")
+			f.UpdateNginxConfigMapData("stream-access-log-path", "/tmp/nginx/stream-access.log")
 			f.WaitForNginxConfiguration(
 				func(cfg string) bool {
-					return strings.Contains(cfg, "access_log /tmp/stream-access.log log_stream") &&
-						strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo")
+					return strings.Contains(cfg, "access_log /tmp/nginx/stream-access.log log_stream") &&
+						(strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo") ||
+							strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 upstreaminfo"))
 				})
 		})
 	})
@ -74,13 +78,13 @@ var _ = framework.DescribeSetting("access-log", func() {

 		ginkgo.It("use the specified configuration", func() {
 			f.SetNginxConfigMapData(map[string]string{
-				"http-access-log-path":   "/tmp/http-access.log",
-				"stream-access-log-path": "/tmp/stream-access.log",
+				"http-access-log-path":   "/tmp/nginx/http-access.log",
+				"stream-access-log-path": "/tmp/nginx/stream-access.log",
 			})
 			f.WaitForNginxConfiguration(
 				func(cfg string) bool {
-					return strings.Contains(cfg, "access_log /tmp/http-access.log upstreaminfo") &&
-						strings.Contains(cfg, "access_log /tmp/stream-access.log log_stream")
+					return strings.Contains(cfg, "access_log /tmp/nginx/http-access.log upstreaminfo") &&
+						strings.Contains(cfg, "access_log /tmp/nginx/stream-access.log log_stream")
 				})
 		})
 	})
--- a/test/e2e/settings/pod_security_policy_volumes.go
+++ b/test/e2e/settings/pod_security_policy_volumes.go
@ -82,10 +82,10 @@ var _ = framework.IngressNginxDescribe("[Security] Pod Security Policies with vo

 			deployment.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
 				{
-					Name: "ssl", MountPath: "/etc/ingress-controller",
+					Name: "ssl", MountPath: "/etc/my-amazing-ssl",
 				},
 				{
-					Name: "tmp", MountPath: "/tmp",
+					Name: "tmp", MountPath: "/my-other-tmp",
 				},
 			}

--- a/test/e2e/wait-for-nginx.sh
+++ b/test/e2e/wait-for-nginx.sh
@ -59,6 +59,7 @@ fullnameOverride: nginx-ingress
 controller:
  image:
    repository: ingress-controller/controller
+    chroot: true
    tag: 1.0.0-dev
    digest:
  scope: