DevFW-CICD/ingress-nginx-helm
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor health checks and wait until NGINX process ends

Browse source
This commit is contained in:
Manuel Alejandro de Brito Fontes 2019-09-01 14:21:24 -04:00
parent c85450c1e7
commit c2935ca35c
No known key found for this signature in database
GPG key ID: 786136016A8BA02A
15 changed files with 126 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

56
internal/ingress/controller/checker.go
View file

@ -25,7 +25,6 @@ import (
"github.com/ncabatoff/process-exporter/proc"
"github.com/pkg/errors"
"k8s.io/klog"
"k8s.io/ingress-nginx/internal/nginx"
)
@ -37,41 +36,48 @@ func (n NGINXController) Name() string {
// Check returns if the nginx healthz endpoint is returning ok (status code 200)
func (n *NGINXController) Check(_ *http.Request) error {
statusCode, _, err := nginx.NewGetStatusRequest(nginx.HealthPath)
if err != nil {
klog.Errorf("healthcheck error: %v", err)
return err
}
if statusCode != 200 {
klog.Errorf("healthcheck error: %v", statusCode)
return fmt.Errorf("ingress controller is not healthy")
}
statusCode, _, err = nginx.NewGetStatusRequest("/is-dynamic-lb-initialized")
if err != nil {
klog.Errorf("healthcheck error: %v", err)
return err
}
if statusCode != 200 {
klog.Errorf("healthcheck error: %v", statusCode)
return fmt.Errorf("dynamic load balancer not started")
if n.isShuttingDown {
return fmt.Errorf("the ingress controller is shutting down")
}
// check the nginx master process is running
fs, err := proc.NewFS("/proc", false)
if err != nil {
return errors.Wrap(err, "unexpected error reading /proc directory")
return errors.Wrap(err, "reading /proc directory")
}
f, err := ioutil.ReadFile(nginx.PID)
if err != nil {
return errors.Wrapf(err, "unexpected error reading %v", nginx.PID)
return errors.Wrapf(err, "reading %v", nginx.PID)
}
pid, err := strconv.Atoi(strings.TrimRight(string(f), "\r\n"))
if err != nil {
return errors.Wrapf(err, "unexpected error reading the nginx PID from %v", nginx.PID)
return errors.Wrapf(err, "reading NGINX PID from file %v", nginx.PID)
}
_, err = fs.NewProc(pid)
return err
if err != nil {
return errors.Wrapf(err, "checking for NGINX process with PID %v", pid)
}
statusCode, _, err := nginx.NewGetStatusRequest(nginx.HealthPath)
if err != nil {
return errors.Wrapf(err, "checking if NGINX is running")
}
if statusCode != 200 {
return fmt.Errorf("ingress controller is not healthy (%v)", statusCode)
}
statusCode, _, err = nginx.NewGetStatusRequest("/is-dynamic-lb-initialized")
if err != nil {
return errors.Wrapf(err, "checking if the dynamic load balancer started")
}
if statusCode != 200 {
return fmt.Errorf("dynamic load balancer not started")
}
return nil
}

5
internal/ingress/controller/checker_test.go
View file

@ -35,12 +35,11 @@ import (
func TestNginxCheck(t *testing.T) {
mux := http.NewServeMux()
listener, err := net.Listen("unix", nginx.StatusSocket)
listener, err := net.Listen("tcp", fmt.Sprintf(":%v", nginx.StatusPort))
if err != nil {
t.Fatalf("crating unix listener: %s", err)
t.Fatalf("crating tcp listener: %s", err)
}
defer listener.Close()
defer os.Remove(nginx.StatusSocket)
server := &httptest.Server{
Listener: listener,

2
internal/ingress/controller/config/config.go
View file

@ -796,8 +796,8 @@ type TemplateConfig struct {
EnableMetrics bool
PID string
StatusSocket string
StatusPath string
StatusPort int
StreamSocket string
}

3
internal/ingress/controller/controller.go
View file

@ -37,6 +37,7 @@ import (
"k8s.io/ingress-nginx/internal/ingress/annotations/proxy"
ngx_config "k8s.io/ingress-nginx/internal/ingress/controller/config"
"k8s.io/ingress-nginx/internal/k8s"
"k8s.io/ingress-nginx/internal/nginx"
"k8s.io/klog"
)
@ -268,6 +269,8 @@ func (n *NGINXController) getStreamServices(configmapName string, proto apiv1.Pr
n.cfg.ListenPorts.SSLProxy,
n.cfg.ListenPorts.Health,
n.cfg.ListenPorts.Default,
10255, // profiling port
nginx.StatusPort,
}
reserverdPorts := sets.NewInt(rp...)
// svcRef format: <(str)namespace>/<(str)service>:<(intstr)port>[:<("PROXY")decode>:<("PROXY")encode>]

2
internal/ingress/controller/nginx.go
View file

@ -605,8 +605,8 @@ func (n NGINXController) generateTemplate(cfg ngx_config.Configuration, ingressC
HealthzURI: nginx.HealthPath,
PID: nginx.PID,
StatusSocket: nginx.StatusSocket,
StatusPath: nginx.StatusPath,
StatusPort: nginx.StatusPort,
StreamSocket: nginx.StreamSocket,
}

11
internal/ingress/controller/nginx_test.go
View file

@ -17,6 +17,7 @@ limitations under the License.
package controller
import (
"fmt"
"io"
"io/ioutil"
"net"
@ -148,16 +149,15 @@ func TestIsDynamicConfigurationEnough(t *testing.T) {
}
func TestConfigureDynamically(t *testing.T) {
listener, err := net.Listen("unix", nginx.StatusSocket)
listener, err := net.Listen("tcp", fmt.Sprintf(":%v", nginx.StatusPort))
if err != nil {
t.Errorf("crating unix listener: %s", err)
t.Fatalf("crating unix listener: %s", err)
}
defer listener.Close()
defer os.Remove(nginx.StatusSocket)
streamListener, err := net.Listen("unix", nginx.StreamSocket)
if err != nil {
t.Errorf("crating unix listener: %s", err)
t.Fatalf("crating unix listener: %s", err)
}
defer streamListener.Close()
defer os.Remove(nginx.StreamSocket)
@ -319,12 +319,11 @@ func TestConfigureDynamically(t *testing.T) {
}
func TestConfigureCertificates(t *testing.T) {
listener, err := net.Listen("unix", nginx.StatusSocket)
listener, err := net.Listen("tcp", fmt.Sprintf(":%v", nginx.StatusPort))
if err != nil {
t.Fatalf("crating unix listener: %s", err)
}
defer listener.Close()
defer os.Remove(nginx.StatusSocket)
streamListener, err := net.Listen("unix", nginx.StreamSocket)
if err != nil {

4
internal/ingress/metric/collectors/nginx_status_test.go
View file

@ -21,7 +21,6 @@ import (
"net"
"net/http"
"net/http/httptest"
"os"
"testing"
"time"
@ -97,7 +96,7 @@ func TestStatusCollector(t *testing.T) {
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
listener, err := net.Listen("unix", nginx.StatusSocket)
listener, err := net.Listen("tcp", fmt.Sprintf(":%v", nginx.StatusPort))
if err != nil {
t.Fatalf("crating unix listener: %s", err)
}
@ -145,7 +144,6 @@ func TestStatusCollector(t *testing.T) {
cm.Stop()
listener.Close()
os.Remove(nginx.StatusSocket)
})
}
}

34
internal/nginx/main.go
View file

@ -28,7 +28,6 @@ import (
"time"
ps "github.com/mitchellh/go-ps"
"github.com/tv42/httpunix"
"k8s.io/klog"
)
@ -38,8 +37,8 @@ var TemplatePath = "/etc/nginx/template/nginx.tmpl"
// PID defines the location of the pid file used by NGINX
var PID = "/tmp/nginx.pid"
// StatusSocket defines the location of the unix socket used by NGINX for the status server
var StatusSocket = "/tmp/nginx-status-server.sock"
// StatusPort port used by NGINX for the status server
var StatusPort = 10256
// HealthPath defines the path used to define the health check location in NGINX
var HealthPath = "/healthz"
@ -56,17 +55,12 @@ var StreamSocket = "/tmp/ingress-stream.sock"
var statusLocation = "nginx-status"
var httpClient *http.Client
func init() {
httpClient = buildUnixSocketClient(HealthCheckTimeout)
}
// NewGetStatusRequest creates a new GET request to the internal NGINX status server
func NewGetStatusRequest(path string) (int, []byte, error) {
url := fmt.Sprintf("%v://%v%v", httpunix.Scheme, statusLocation, path)
url := fmt.Sprintf("http://127.0.0.1:%v%v", StatusPort, path)
res, err := httpClient.Get(url)
client := http.Client{}
res, err := client.Get(url)
if err != nil {
return 0, nil, err
}
@ -82,14 +76,15 @@ func NewGetStatusRequest(path string) (int, []byte, error) {
// NewPostStatusRequest creates a new POST request to the internal NGINX status server
func NewPostStatusRequest(path, contentType string, data interface{}) (int, []byte, error) {
url := fmt.Sprintf("%v://%v%v", httpunix.Scheme, statusLocation, path)
url := fmt.Sprintf("http://127.0.0.1:%v%v", StatusPort, path)
buf, err := json.Marshal(data)
if err != nil {
return 0, nil, err
}
res, err := httpClient.Post(url, contentType, bytes.NewReader(buf))
client := http.Client{}
res, err := client.Post(url, contentType, bytes.NewReader(buf))
if err != nil {
return 0, nil, err
}
@ -142,19 +137,6 @@ func readFileToString(path string) (string, error) {
return string(contents), nil
}
func buildUnixSocketClient(timeout time.Duration) *http.Client {
u := &httpunix.Transport{
DialTimeout: 1 * time.Second,
RequestTimeout: timeout,
ResponseHeaderTimeout: timeout,
}
u.RegisterLocation(statusLocation, StatusSocket)
return &http.Client{
Transport: u,
}
}
// Version return details about NGINX
func Version() string {
flag := "-v"