Jail/chroot nginx process inside controller container (#8337)

* Initial work on chrooting nginx process

* More improvements in chroot

* Fix charts and some file locations

* Fix symlink on non chrooted container

* fix psp test

* Add e2e tests to chroot image

* Fix logger

* Add internal logger in controller

* Fix overlay for chrooted tests

* Fix tests

* fix boilerplates

* Fix unittest to point to the right pid

* Fix PR review

internal/ingress/controller/checker_test.go

@@ -76,7 +76,7 @@ func TestNginxCheck(t *testing.T) {
 			})
 
 			// create pid file
-			os.MkdirAll("/tmp", file.ReadWriteByUser)
+			os.MkdirAll("/tmp/nginx", file.ReadWriteByUser)
 			pidFile, err := os.Create(nginx.PID)
 			if err != nil {
 				t.Fatalf("unexpected error: %v", err)

internal/ingress/controller/config/config.go

@@ -958,12 +958,11 @@ type TemplateConfig struct {
 	EnableMetrics            bool
 	MaxmindEditionFiles      *[]string
 	MonitorMaxBatchSize      int
-
-	PID            string
-	StatusPath     string
-	StatusPort     int
-	StreamPort     int
-	StreamSnippets []string
+	PID                      string
+	StatusPath               string
+	StatusPort               int
+	StreamPort               int
+	StreamSnippets           []string
 }
 
 // ListenPorts describe the ports required to run the

internal/ingress/controller/controller.go

@@ -120,6 +120,9 @@ type Configuration struct {
 
 	PostShutdownGracePeriod int
 	ShutdownGracePeriod     int
+
+	InternalLoggerAddress string
+	IsChroot              bool
 }
 
 // GetPublishService returns the Service used to set the load-balancer status of Ingresses.

internal/ingress/controller/nginx.go

@@ -575,6 +575,15 @@ func (n NGINXController) generateTemplate(cfg ngx_config.Configuration, ingressC
 
 	cfg.DefaultSSLCertificate = n.getDefaultSSLCertificate()
 
+	if n.cfg.IsChroot {
+		if cfg.AccessLogPath == "/var/log/nginx/access.log" {
+			cfg.AccessLogPath = fmt.Sprintf("syslog:server=%s", n.cfg.InternalLoggerAddress)
+		}
+		if cfg.ErrorLogPath == "/var/log/nginx/error.log" {
+			cfg.ErrorLogPath = fmt.Sprintf("syslog:server=%s", n.cfg.InternalLoggerAddress)
+		}
+	}
+
 	tc := ngx_config.TemplateConfig{
 		ProxySetHeaders:          setHeaders,
 		AddHeaders:               addHeaders,
@@ -614,7 +623,8 @@ func (n NGINXController) testTemplate(cfg []byte) error {
 	if len(cfg) == 0 {
 		return fmt.Errorf("invalid NGINX configuration (empty)")
 	}
-	tmpfile, err := os.CreateTemp("", tempNginxPattern)
+	tmpDir := os.TempDir() + "/nginx"
+	tmpfile, err := os.CreateTemp(tmpDir, tempNginxPattern)
 	if err != nil {
 		return err
 	}

internal/ingress/controller/process/nginx.go

@@ -20,7 +20,7 @@ import (
 	"os/exec"
 	"syscall"
 
-	"k8s.io/klog/v2"
+	klog "k8s.io/klog/v2"
 )
 
 // IsRespawnIfRequired checks if error type is exec.ExitError or not

internal/ingress/controller/util.go

@@ -29,7 +29,7 @@ import (
 	networking "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/ingress-nginx/internal/ingress"
-	"k8s.io/klog/v2"
+	klog "k8s.io/klog/v2"
 )
 
 // newUpstream creates an upstream without servers.
@@ -98,7 +98,7 @@ func rlimitMaxNumFiles() int {
 }
 
 const (
-	defBinary = "/usr/local/nginx/sbin/nginx"
+	defBinary = "/usr/bin/nginx"
 	cfgPath   = "/etc/nginx/nginx.conf"
 )
 

internal/ingress/metric/collectors/socket.go

@@ -111,7 +111,7 @@ var defObjectives = map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}
 // NewSocketCollector creates a new SocketCollector instance using
 // the ingress watch namespace and class used by the controller
 func NewSocketCollector(pod, namespace, class string, metricsPerHost bool, buckets HistogramBuckets) (*SocketCollector, error) {
-	socket := "/tmp/prometheus-nginx.socket"
+	socket := "/tmp/nginx/prometheus-nginx.socket"
 	// unix sockets must be unlink()ed before being used
 	_ = syscall.Unlink(socket)
 

internal/nginx/main.go

@@ -28,7 +28,7 @@ import (
 	"time"
 
 	ps "github.com/mitchellh/go-ps"
-	"k8s.io/klog/v2"
+	klog "k8s.io/klog/v2"
 )
 
 // TODO: Check https://github.com/kubernetes/kubernetes/blob/master/pkg/master/ports/ports.go for ports already being used
@@ -40,7 +40,7 @@ var ProfilerPort = 10245
 var TemplatePath = "/etc/nginx/template/nginx.tmpl"
 
 // PID defines the location of the pid file used by NGINX
-var PID = "/tmp/nginx.pid"
+var PID = "/tmp/nginx/nginx.pid"
 
 // StatusPort port used by NGINX for the status server
 var StatusPort = 10246