Jail/chroot nginx process inside controller container (#8337)

* Initial work on chrooting nginx process

* More improvements in chroot

* Fix charts and some file locations

* Fix symlink on non chrooted container

* fix psp test

* Add e2e tests to chroot image

* Fix logger

* Add internal logger in controller

* Fix overlay for chrooted tests

* Fix tests

* fix boilerplates

* Fix unittest to point to the right pid

* Fix PR review

rootfs/Dockerfile

@@ -54,6 +54,7 @@ RUN bash -xeu -c ' \
     /etc/ingress-controller/auth \
     /var/log \
     /var/log/nginx \
+    /tmp/nginx \
   ); \
   for dir in "${writeDirs[@]}"; do \
     mkdir -p ${dir}; \
@@ -67,7 +68,8 @@ RUN apk add --no-cache libcap \
   && setcap -v cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \
   && setcap    cap_net_bind_service=+ep /usr/bin/dumb-init \
   && setcap -v cap_net_bind_service=+ep /usr/bin/dumb-init \
-  && apk del libcap
+  && apk del libcap \
+  && ln -sf /usr/local/nginx/sbin/nginx /usr/bin/nginx
 
 USER www-data
 

rootfs/Dockerfile.chroot

@@ -0,0 +1,112 @@
+# Copyright 2022 The Kubernetes Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+ARG BASE_IMAGE
+
+FROM ${BASE_IMAGE} as chroot
+
+# This intermediary image will be used only to copy all the required files to the chroot
+# TODO: Simplify in a future to a single Dockerfile
+COPY chroot.sh /chroot.sh
+RUN apk update \
+  && apk upgrade \
+  && /chroot.sh
+
+FROM alpine:3.14.2
+
+ARG TARGETARCH
+ARG VERSION
+ARG COMMIT_SHA
+ARG BUILD_ID=UNSET
+
+LABEL org.opencontainers.image.title="NGINX Ingress Controller for Kubernetes"
+LABEL org.opencontainers.image.documentation="https://kubernetes.github.io/ingress-nginx/"
+LABEL org.opencontainers.image.source="https://github.com/kubernetes/ingress-nginx"
+LABEL org.opencontainers.image.vendor="The Kubernetes Authors"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
+LABEL org.opencontainers.image.version="${VERSION}"
+LABEL org.opencontainers.image.revision="${COMMIT_SHA}"
+
+LABEL build_id="${BUILD_ID}"
+
+# This will be injected in the chroot. Don't change :)
+ENV LUA_PATH="/usr/local/share/luajit-2.1.0-beta3/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/lib/lua/?.lua;;"
+ENV LUA_CPATH="/usr/local/lib/lua/?/?.so;/usr/local/lib/lua/?.so;;"
+ENV PATH=$PATH:/usr/local/luajit/bin:/usr/local/nginx/sbin:/usr/local/nginx/bin
+
+RUN apk update \
+  && apk upgrade \
+  && apk add -U --no-cache \
+     bash \
+     curl \
+     openssl \
+     ca-certificates \
+     dumb-init \
+     tzdata \
+     diffutils \
+     util-linux \
+  && ln -s /usr/local/nginx/sbin/nginx /sbin/nginx \
+  && adduser -S -D -H -u 101 -h /usr/local/nginx \
+    -s /sbin/nologin -G www-data -g www-data www-data 
+
+COPY --from=chroot /chroot /chroot
+
+COPY --chown=www-data:www-data etc /chroot/etc
+
+COPY --chown=www-data:www-data bin/${TARGETARCH}/dbg /
+COPY --chown=www-data:www-data bin/${TARGETARCH}/nginx-ingress-controller /
+COPY --chown=www-data:www-data bin/${TARGETARCH}/wait-shutdown /
+COPY --chown=www-data:www-data nginx-chroot-wrapper.sh /usr/bin/nginx
+
+WORKDIR /chroot/etc/nginx
+
+# Fix permission during the build to avoid issues at runtime
+# with volumes (custom templates)
+RUN bash -xeu -c ' \
+  writeDirs=( \
+    /var/log \
+  ); \
+  for dir in "${writeDirs[@]}"; do \
+    mkdir -p ${dir}; \
+    chown -R www-data.www-data ${dir}; \
+  done'
+
+RUN apk add --no-cache libcap \
+  && setcap    cap_sys_chroot,cap_net_bind_service=+ep /nginx-ingress-controller \
+  && setcap -v cap_sys_chroot,cap_net_bind_service=+ep /nginx-ingress-controller \
+  && setcap    cap_sys_chroot,cap_net_bind_service=+ep /usr/bin/unshare \
+  && setcap -v cap_sys_chroot,cap_net_bind_service=+ep /usr/bin/unshare \
+  && setcap    cap_net_bind_service=+ep /chroot/usr/local/nginx/sbin/nginx \
+  && setcap -v cap_net_bind_service=+ep /chroot/usr/local/nginx/sbin/nginx \
+  && setcap    cap_sys_chroot,cap_net_bind_service=+ep /usr/bin/dumb-init \
+  && setcap -v cap_sys_chroot,cap_net_bind_service=+ep /usr/bin/dumb-init \
+  && apk del libcap
+
+RUN  ln -sf /chroot/etc/nginx /etc/nginx \
+  && ln -sf /chroot/tmp/nginx /tmp/nginx \
+  && ln -sf /chroot/etc/ingress-controller /etc/ingress-controller \
+  && ln -sf /chroot/var/log/nginx /var/log/nginx \
+  && touch /chroot/var/log/nginx/access.log \
+  && chown www-data:www-data /chroot/var/log/nginx/access.log \
+  && echo "" > /chroot/etc/resolv.conf \
+  && chown -R www-data.www-data /chroot/var/log/nginx /chroot/etc/resolv.conf
+
+USER www-data
+
+EXPOSE 80 443
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+CMD ["/nginx-ingress-controller"]
+ 

rootfs/chroot.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# Copyright 2022 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+writeDirs=( \
+  /chroot/etc/nginx \
+  /chroot/usr/local/ \
+  /chroot/etc/ingress-controller \
+  /chroot/etc/ingress-controller/ssl \
+  /chroot/etc/ingress-controller/auth \
+  /chroot/opt/modsecurity/var/log \
+  /chroot/opt/modsecurity/var/upload \
+  /chroot/opt/modsecurity/var/audit \
+  /chroot/var/log/audit \
+  /chroot/var/lib/nginx \
+  /chroot/var/log/nginx \
+  /chroot/var/lib/nginx/body \
+  /chroot/var/lib/nginx/fastcgi \
+  /chroot/var/lib/nginx/proxy \
+  /chroot/var/lib/nginx/scgi \
+  /chroot/var/lib/nginx/uwsgi \
+  /chroot/tmp/nginx
+);
+
+for dir in "${writeDirs[@]}"; do
+  mkdir -p ${dir};
+  chown -R www-data.www-data ${dir};
+done
+
+mkdir -p  /chroot/lib /chroot/proc /chroot/usr /chroot/bin /chroot/dev /chroot/run 
+cp /etc/passwd /etc/group /chroot/etc/
+cp -a /usr/* /chroot/usr/
+mv /var/log/nginx /chroot/var/log/
+cp -a /etc/nginx/* /chroot/etc/nginx/
+cp /lib/ld-musl-* /lib/libcrypto* /lib/libssl* /lib/libz* /chroot/lib/
+mknod -m 0666 /chroot/dev/null c 1 3
+mknod -m 0666 /chroot/dev/random c 1 8
+mknod -m 0666 /chroot/dev/urandom c 1 9
+mknod -m 0666 /chroot/dev/full c 1 7
+mknod -m 0666 /chroot/dev/ptmx c 5 2
+mknod -m 0666 /chroot/dev/zero c 1 5
+mknod -m 0666 /chroot/dev/tty c 5 0

rootfs/etc/nginx/lua/monitor.lua

@@ -26,7 +26,7 @@ local _M = {}
 
 local function send(payload)
   local s = assert(socket())
-  assert(s:connect("unix:/tmp/prometheus-nginx.socket"))
+  assert(s:connect("unix:/tmp/nginx/prometheus-nginx.socket"))
   assert(s:send(payload))
   assert(s:close())
 end

rootfs/etc/nginx/lua/test/monitor_test.lua

@@ -148,7 +148,7 @@ describe("Monitor", function()
         },
       })
 
-      assert.stub(tcp_mock.connect).was_called_with(tcp_mock, "unix:/tmp/prometheus-nginx.socket")
+      assert.stub(tcp_mock.connect).was_called_with(tcp_mock, "unix:/tmp/nginx/prometheus-nginx.socket")
       assert.stub(tcp_mock.send).was_called_with(tcp_mock, expected_payload)
       assert.stub(tcp_mock.close).was_called_with(tcp_mock)
     end)

rootfs/etc/nginx/nginx.conf

@@ -1,6 +1,8 @@
 # A very simple nginx configuration file that forces nginx to start.
-pid /tmp/nginx.pid;
+pid /tmp/nginx/nginx.pid;
+
+error_log stderr;
 
 events {}
 http {}
-daemon off;
+daemon off;

rootfs/etc/nginx/template/nginx.tmpl

@@ -285,10 +285,10 @@ http {
     keepalive_timeout  {{ $cfg.KeepAlive }}s;
     keepalive_requests {{ $cfg.KeepAliveRequests }};
 
-    client_body_temp_path           /tmp/client-body;
-    fastcgi_temp_path               /tmp/fastcgi-temp;
-    proxy_temp_path                 /tmp/proxy-temp;
-    ajp_temp_path                   /tmp/ajp-temp;
+    client_body_temp_path           /tmp/nginx/client-body;
+    fastcgi_temp_path               /tmp/nginx/fastcgi-temp;
+    proxy_temp_path                 /tmp/nginx/proxy-temp;
+    ajp_temp_path                   /tmp/nginx/ajp-temp;
 
     client_header_buffer_size       {{ $cfg.ClientHeaderBufferSize }};
     client_header_timeout           {{ $cfg.ClientHeaderTimeout }}s;
@@ -536,7 +536,7 @@ http {
     {{ end }}
 
     # Cache for internal auth checks
-    proxy_cache_path /tmp/nginx-cache-auth levels=1:2 keys_zone=auth_cache:10m max_size=128m inactive=30m use_temp_path=off;
+    proxy_cache_path /tmp/nginx/nginx-cache-auth levels=1:2 keys_zone=auth_cache:10m max_size=128m inactive=30m use_temp_path=off;
 
     # Global filters
     {{ range $ip := $cfg.BlockCIDRs }}deny {{ trimSpace $ip }};
@@ -773,8 +773,8 @@ stream {
     access_log {{ or $cfg.StreamAccessLogPath $cfg.AccessLogPath }} log_stream {{ $cfg.AccessLogParams }};
     {{ end }}
 
+   
     error_log  {{ $cfg.ErrorLogPath }} {{ $cfg.ErrorLogLevel }};
-
     {{ if $cfg.EnableRealIp }}
     {{ range $trusted_ip := $cfg.ProxyRealIPCIDR }}
     set_real_ip_from    {{ $trusted_ip }};

rootfs/nginx-chroot-wrapper.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Copyright 2022 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cat /etc/resolv.conf > /chroot/etc/resolv.conf
+unshare  -S 101 -R /chroot nginx "$@"