Allow configuring nginx worker reload behaviour, to prevent multiple concurrent worker reloads which can lead to high resource usage and OOMKill (#10884)

* feat: allow configuring nginx worker reload behaviour, to prevent multiple concurrent worker reloads

Signed-off-by: Rafael da Fonseca <rafael.fonseca@wildlifestudios.com>

* appease linter, remove unnecessary log line

Signed-off-by: Rafael da Fonseca <rafael.fonseca@wildlifestudios.com>

* Flip to using a positive behaviour flag instead of negative

Signed-off-by: Rafael da Fonseca <rafael.fonseca@wildlifestudios.com>

* Update helm-docs

Signed-off-by: Rafael da Fonseca <rafael.fonseca@wildlifestudios.com>

* Avoid calling GetBackendConfiguration() twice, use clearer name for helm chart option

Signed-off-by: Rafael da Fonseca <rafael.fonseca@wildlifestudios.com>

* Fix helm-docs ordering

Signed-off-by: Rafael da Fonseca <rafael.fonseca@wildlifestudios.com>

---------

Signed-off-by: Rafael da Fonseca <rafael.fonseca@wildlifestudios.com>
This commit is contained in:
Rafael da Fonseca 2024-05-14 22:45:25 +01:00 committed by GitHub
parent 689b993a3c
commit 4e11074323
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 69 additions and 3 deletions

View file

@ -477,6 +477,13 @@ type Configuration struct {
// http://nginx.org/en/docs/ngx_core_module.html#worker_processes
WorkerProcesses string `json:"worker-processes,omitempty"`
// Defines whether multiple concurrent reloads of worker processes should occur.
// Set this to false to prevent more than n x 2 workers to exist at any time, to avoid potential OOM situations and high CPU load
// With this setting on false, configuration changes in the queue will be re-queued with an exponential backoff, until the number of worker process is the expected value.
// By default new worker processes are spawned every time there's a change that cannot be applied dynamically with no upper limit to the number of running workers
// http://nginx.org/en/docs/ngx_core_module.html#worker_processes
WorkerSerialReloads bool `json:"enable-serial-reloads,omitempty"`
// Defines a timeout for a graceful shutdown of worker processes
// http://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout
WorkerShutdownTimeout string `json:"worker-shutdown-timeout,omitempty"`
@ -851,6 +858,7 @@ func NewDefault() Configuration {
UseGeoIP2: false,
GeoIP2AutoReloadMinutes: 0,
WorkerProcesses: strconv.Itoa(runtime.NumCPU()),
WorkerSerialReloads: false,
WorkerShutdownTimeout: "240s",
VariablesHashBucketSize: 256,
VariablesHashMaxSize: 2048,