Jail/chroot nginx process inside controller container (#8337)

* Initial work on chrooting nginx process

* More improvements in chroot

* Fix charts and some file locations

* Fix symlink on non chrooted container

* fix psp test

* Add e2e tests to chroot image

* Fix logger

* Add internal logger in controller

* Fix overlay for chrooted tests

* Fix tests

* fix boilerplates

* Fix unittest to point to the right pid

* Fix PR review

rootfs/etc/nginx/lua/monitor.lua

@@ -26,7 +26,7 @@ local _M = {}
 
 local function send(payload)
   local s = assert(socket())
-  assert(s:connect("unix:/tmp/prometheus-nginx.socket"))
+  assert(s:connect("unix:/tmp/nginx/prometheus-nginx.socket"))
   assert(s:send(payload))
   assert(s:close())
 end

rootfs/etc/nginx/lua/test/monitor_test.lua

@@ -148,7 +148,7 @@ describe("Monitor", function()
         },
       })
 
-      assert.stub(tcp_mock.connect).was_called_with(tcp_mock, "unix:/tmp/prometheus-nginx.socket")
+      assert.stub(tcp_mock.connect).was_called_with(tcp_mock, "unix:/tmp/nginx/prometheus-nginx.socket")
       assert.stub(tcp_mock.send).was_called_with(tcp_mock, expected_payload)
       assert.stub(tcp_mock.close).was_called_with(tcp_mock)
     end)

rootfs/etc/nginx/nginx.conf

@@ -1,6 +1,8 @@
 # A very simple nginx configuration file that forces nginx to start.
-pid /tmp/nginx.pid;
+pid /tmp/nginx/nginx.pid;
+
+error_log stderr;
 
 events {}
 http {}
-daemon off;
+daemon off;

rootfs/etc/nginx/template/nginx.tmpl

@@ -285,10 +285,10 @@ http {
     keepalive_timeout  {{ $cfg.KeepAlive }}s;
     keepalive_requests {{ $cfg.KeepAliveRequests }};
 
-    client_body_temp_path           /tmp/client-body;
-    fastcgi_temp_path               /tmp/fastcgi-temp;
-    proxy_temp_path                 /tmp/proxy-temp;
-    ajp_temp_path                   /tmp/ajp-temp;
+    client_body_temp_path           /tmp/nginx/client-body;
+    fastcgi_temp_path               /tmp/nginx/fastcgi-temp;
+    proxy_temp_path                 /tmp/nginx/proxy-temp;
+    ajp_temp_path                   /tmp/nginx/ajp-temp;
 
     client_header_buffer_size       {{ $cfg.ClientHeaderBufferSize }};
     client_header_timeout           {{ $cfg.ClientHeaderTimeout }}s;
@@ -536,7 +536,7 @@ http {
     {{ end }}
 
     # Cache for internal auth checks
-    proxy_cache_path /tmp/nginx-cache-auth levels=1:2 keys_zone=auth_cache:10m max_size=128m inactive=30m use_temp_path=off;
+    proxy_cache_path /tmp/nginx/nginx-cache-auth levels=1:2 keys_zone=auth_cache:10m max_size=128m inactive=30m use_temp_path=off;
 
     # Global filters
     {{ range $ip := $cfg.BlockCIDRs }}deny {{ trimSpace $ip }};
@@ -773,8 +773,8 @@ stream {
     access_log {{ or $cfg.StreamAccessLogPath $cfg.AccessLogPath }} log_stream {{ $cfg.AccessLogParams }};
     {{ end }}
 
+   
     error_log  {{ $cfg.ErrorLogPath }} {{ $cfg.ErrorLogLevel }};
-
     {{ if $cfg.EnableRealIp }}
     {{ range $trusted_ip := $cfg.ProxyRealIPCIDR }}
     set_real_ip_from    {{ $trusted_ip }};