Improve event handling using a workqueue

2016-03-22 15:01:04 -03:00 · 2016-03-22 15:01:04 -03:00 · 13c21386e2
commit 13c21386e2
parent f5892e06fe
18 changed files with 1384 additions and 206 deletions
--- a/controllers/nginx-third-party/README.md
+++ b/controllers/nginx-third-party/README.md
@ -6,7 +6,7 @@ This is a nginx Ingress controller that uses [ConfigMap](https://github.com/kube
 ## What it provides?

 - Ingress controller
- nginx 1.9.x with [lua-nginx-module](https://github.com/openresty/lua-nginx-module)
+- nginx 1.9.x with
 - SSL support
 - custom ssl_dhparam (optional). Just mount a secret with a file named `dhparam.pem`.
 - support for TCP services (flag `--tcp-services-configmap`)
@ -17,11 +17,43 @@ This is a nginx Ingress controller that uses [ConfigMap](https://github.com/kube
 ## Requirements
 - default backend [404-server](https://github.com/kubernetes/contrib/tree/master/404-server) (or a custom compatible image)

-## SSL
+## TLS

+You can secure an Ingress by specifying a secret that contains a TLS private key and certificate. Currently the Ingress only supports a single TLS port, 443, and assumes TLS termination. This controller supports SNI. The TLS secret must contain keys named tls.crt and tls.key that contain the certificate and private key to use for TLS, eg:
+
+```
+apiVersion: v1
+data:
+  tls.crt: base64 encoded cert
+  tls.key: base64 encoded key
+kind: Secret
+metadata:
+  name: testsecret
+  namespace: default
+type: Opaque
+```
+
+Referencing this secret in an Ingress will tell the Ingress controller to secure the channel from the client to the loadbalancer using TLS:
+
+```
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: no-rules-map
+spec:
+  tls:
+    secretName: testsecret
+  backend:
+    serviceName: s1
+    servicePort: 80
+```
 Please follow [test.sh](https://github.com/bprashanth/Ingress/blob/master/examples/sni/nginx/test.sh) as a guide on how to generate secrets containing SSL certificates. The name of the secret can be different than the name of the certificate.

-Currently Ingress does not support HTTPS. To bypass this the controller will check if there's a certificate for the the host in `Spec.Rules.Host` checking for a certificate in each of the mounted secrets. If exists it will create a nginx server listening in the port 443.
+
+## Optimizing TLS Time To First Byte (TTTFB)
+
+NGINX provides the configuration option (`ssl_buffer_size)[http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_buffer_size] to allow the optimization of the TLS record size. This improves the (Time To First Byte)[https://www.igvita.com/2013/12/16/optimizing-nginx-tls-time-to-first-byte/] (TTTFB). The default value in the Ingress controller it is `4k` (nginx default is `16k`);
+

 ## Examples:

@ -149,8 +181,9 @@ To configure which services and ports will be exposed
 kubectl create -f examples/tcp-configmap-example.yaml
 ```

-The file `examples/tcp-configmap-example.yaml` uses a ConfigMap where the key is the external port to use and the value is <namespace/service name>:<service port>.
-(Is possible to use a number or the name of the port)
+The file `examples/tcp-configmap-example.yaml` uses a ConfigMap where the key is the external port to use and the value is 
+`<namespace/service name>:<service port>`
+It is possible to use a number or the name of the port.


 ```
@ -307,6 +340,93 @@ The route `/error` expects two arguments: code and format

 Using a volume pointing to `/var/www/html` directory is possible to use a custom error

+
+## Debug
+
+Using the flag `--v=XX` it is possible to increase the level of logging.
+In particular:
+- `--v=2` shows details using `diff` about the changes in the configuration in nginx
+
+```
+I0316 12:24:37.581267       1 utils.go:148] NGINX configuration diff a//etc/nginx/nginx.conf b//etc/nginx/nginx.conf
+I0316 12:24:37.581356       1 utils.go:149] --- /tmp/922554809  2016-03-16 12:24:37.000000000 +0000
+++ /tmp/079811012  2016-03-16 12:24:37.000000000 +0000
+@@ -235,7 +235,6 @@
+
+     upstream default-echoheadersx {
+         least_conn;
+-        server 10.2.112.124:5000;
+         server 10.2.208.50:5000;
+
+     }
+I0316 12:24:37.610073       1 command.go:69] change in configuration detected. Reloading...
+```
+
+- `--v=3` shows details about the service, Ingress rule, endpoint changes and it dumps the nginx configuration in JSON format
+- `--v=5` configures NGINX in [debug mode](http://nginx.org/en/docs/debugging_log.html)
+
+## Custom NGINX configuration
+
+Using a ConfigMap it is possible to customize the defaults in nginx.
+The next command shows the defaults:
+```
+$ ./nginx-third-party-lb --dump-nginx—configuration
+Example of ConfigMap to customize NGINX configuration:
+data:
+  body-size: 1m
+  error-log-level: info
+  gzip-types: application/atom+xml application/javascript application/json application/rss+xml
+    application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json
+    application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon
+    text/css text/plain text/x-component
+  hts-include-subdomains: "true"
+  hts-max-age: "15724800"
+  keep-alive: "75"
+  max-worker-connections: "16384"
+  proxy-connect-timeout: "30"
+  proxy-read-timeout: "30"
+  proxy-real-ip-cidr: 0.0.0.0/0
+  proxy-send-timeout: "30"
+  server-name-hash-bucket-size: "64"
+  server-name-hash-max-size: "512"
+  ssl-buffer-size: 4k
+  ssl-ciphers: ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
+  ssl-protocols: TLSv1 TLSv1.1 TLSv1.2
+  ssl-session-cache: "true"
+  ssl-session-cache-size: 10m
+  ssl-session-tickets: "true"
+  ssl-session-timeout: 10m
+  use-gzip: "true"
+  use-hts: "true"
+  worker-processes: "8"
+metadata:
+  name: custom-name
+  namespace: a-valid-namespace
+```
+
+For instance, if we want to change the timeouts we need to create a ConfigMap:
+```
+$ cat nginx-load-balancer-conf.yaml
+apiVersion: v1
+data:
+  proxy-connect-timeout: "10"
+  proxy-read-timeout: "120"
+  proxy-send-imeout: "120"
+kind: ConfigMap
+metadata:
+  name: nginx-load-balancer-conf
+
+```
+
+```
+$ kubectl create -f nginx-load-balancer-conf.yaml
+```
+
+Please check the example `rc-custom-configuration.yaml`
+
+If the Configmap it is updated, NGINX will be reloaded with the new configuration
+
+
 ## Troubleshooting

 Problems encountered during [1.2.0-alpha7 deployment](https://github.com/kubernetes/kubernetes/blob/master/docs/getting-started-guides/docker.md):
--- a/controllers/nginx-third-party/controller.go
+++ b/controllers/nginx-third-party/controller.go
@ -18,6 +18,7 @@ package main

 import (
 	"fmt"
+	"reflect"
 	"sort"
 	"strconv"
 	"strings"
@ -33,7 +34,6 @@ import (
 	"k8s.io/kubernetes/pkg/controller/framework"
 	"k8s.io/kubernetes/pkg/runtime"
 	"k8s.io/kubernetes/pkg/util/intstr"
-	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/watch"

 	"k8s.io/contrib/ingress/controllers/nginx-third-party/nginx"
@ -43,6 +43,10 @@ const (
 	defUpstreamName = "upstream-default-backend"
 )

+var (
+	keyFunc = framework.DeletionHandlingMetaNamespaceKeyFunc
+)
+
 // loadBalancerController watches the kubernetes api and adds/removes services
 // from the loadbalancer
 type loadBalancerController struct {
@ -59,6 +63,8 @@ type loadBalancerController struct {
 	nxgConfigMap   string
 	tcpConfigMap   string

+	syncQueue *taskQueue
+
 	// stopLock is used to enforce only a single call to Stop is active.
 	// Needed because we allow stopping through an http endpoint and
 	// allowing concurrent stoppers leads to stack traces.
@ -80,19 +86,35 @@ func newLoadBalancerController(kubeClient *client.Client, resyncPeriod time.Dura
 		defaultSvc:   defaultSvc,
 	}

+	lbc.syncQueue = NewTaskQueue(lbc.sync)
+
+	eventHandler := framework.ResourceEventHandlerFuncs{
+		AddFunc: func(obj interface{}) {
+			lbc.syncQueue.enqueue(obj)
+		},
+		DeleteFunc: func(obj interface{}) {
+			lbc.syncQueue.enqueue(obj)
+		},
+		UpdateFunc: func(old, cur interface{}) {
+			if !reflect.DeepEqual(old, cur) {
+				lbc.syncQueue.enqueue(cur)
+			}
+		},
+	}
+
 	lbc.ingLister.Store, lbc.ingController = framework.NewInformer(
 		&cache.ListWatch{
 			ListFunc:  ingressListFunc(lbc.client, namespace),
 			WatchFunc: ingressWatchFunc(lbc.client, namespace),
 		},
-		&extensions.Ingress{}, resyncPeriod, framework.ResourceEventHandlerFuncs{})
+		&extensions.Ingress{}, resyncPeriod, eventHandler)

 	lbc.endpLister.Store, lbc.endpController = framework.NewInformer(
 		&cache.ListWatch{
 			ListFunc:  endpointsListFunc(lbc.client, namespace),
 			WatchFunc: endpointsWatchFunc(lbc.client, namespace),
 		},
-		&api.Endpoints{}, resyncPeriod, framework.ResourceEventHandlerFuncs{})
+		&api.Endpoints{}, resyncPeriod, eventHandler)

 	lbc.svcLister.Store, lbc.svcController = framework.NewInformer(
 		&cache.ListWatch{
@ -140,6 +162,10 @@ func endpointsWatchFunc(c *client.Client, ns string) func(options api.ListOption
 	}
 }

+func (lbc *loadBalancerController) controllersInSync() bool {
+	return lbc.ingController.HasSynced() && lbc.svcController.HasSynced() && lbc.endpController.HasSynced()
+}
+
 func (lbc *loadBalancerController) getConfigMap(ns, name string) (*api.ConfigMap, error) {
 	return lbc.client.ConfigMaps(ns).Get(name)
 }
@ -148,7 +174,12 @@ func (lbc *loadBalancerController) getTCPConfigMap(ns, name string) (*api.Config
 	return lbc.client.ConfigMaps(ns).Get(name)
 }

-func (lbc *loadBalancerController) sync() {
+func (lbc *loadBalancerController) sync(key string) {
+	if !lbc.controllersInSync() {
+		lbc.syncQueue.requeue(key, fmt.Errorf("deferring sync till endpoints controller has synced"))
+		return
+	}
+
 	ings := lbc.ingLister.Store.List()
 	upstreams, servers := lbc.getUpstreamServers(ings)

@ -160,11 +191,7 @@ func (lbc *loadBalancerController) sync() {
 		cfg = &api.ConfigMap{}
 	}

-	ngxConfig, err := lbc.nginx.ReadConfig(cfg)
-	if err != nil {
-		glog.Warningf("%v", err)
-	}
-
+	ngxConfig := lbc.nginx.ReadConfig(cfg)
 	tcpServices := lbc.getTCPServices()
 	lbc.nginx.CheckAndReload(ngxConfig, nginx.IngressConfig{
 		Upstreams:    upstreams,
@ -239,6 +266,13 @@ func (lbc *loadBalancerController) getTCPServices() []*nginx.Location {
 			}
 		}

+		// tcp upstreams cannot contain empty upstreams and there is no
+		// default backend equivalent for TCP
+		if len(endps) == 0 {
+			glog.Warningf("service %v/%v does no have any active endpoints", svcNs, svcName)
+			continue
+		}
+
 		tcpSvcs = append(tcpSvcs, &nginx.Location{
 			Path: k,
 			Upstream: nginx.Upstream{
@ -441,14 +475,13 @@ func (lbc *loadBalancerController) getPemsFromIngress(data []interface{}) map[st
 				continue
 			}

-			cn, err := lbc.nginx.CheckSSLCertificate(secretName)
+			pemFileName := lbc.nginx.AddOrUpdateCertAndKey(secretName, string(cert), string(key))
+			cn, err := lbc.nginx.CheckSSLCertificate(pemFileName)
 			if err != nil {
 				glog.Warningf("No valid SSL certificate found in secret %v", secretName)
 				continue
 			}

-			pemFileName := lbc.nginx.AddOrUpdateCertAndKey(secretName, string(cert), string(key))
-
 			for _, host := range tls.Hosts {
 				if isHostValid(host, cn) {
 					pems[host] = pemFileName
@ -513,6 +546,7 @@ func (lbc *loadBalancerController) Stop() {
 		close(lbc.stopCh)
 		glog.Infof("shutting down controller queues")
 		lbc.shutdown = true
+		lbc.syncQueue.shutdown()
 	}
 }

@ -525,8 +559,7 @@ func (lbc *loadBalancerController) Run() {
 	go lbc.endpController.Run(lbc.stopCh)
 	go lbc.svcController.Run(lbc.stopCh)

-	// periodic check for changes in configuration
-	go wait.Until(lbc.sync, 5*time.Second, wait.NeverStop)
+	go lbc.syncQueue.run(time.Second, lbc.stopCh)

 	<-lbc.stopCh
 	glog.Infof("shutting down NGINX loadbalancer controller")
--- a/controllers/nginx-third-party/examples/certs.sh
+++ b/controllers/nginx-third-party/examples/certs.sh
@ -1,71 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2015 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# This test is for dev purposes.
-
-set -e
-
-SECRET_NAME=${SECRET_NAME:-ssl-secret}
-# Name of the app in the .yaml
-APP=${APP:-nginxsni}
-# SNI hostnames
-HOSTS=${HOSTS:-foo.bar.com}
-# Should the test build and push the container via make push?
-PUSH=${PUSH:-false}
-
-# makeCerts makes certificates applying the given hostnames as CNAMEs
-# $1 Name of the app that will use this secret, applied as a app= label
-# $2... hostnames as described below
-# Eg: makeCerts nginxsni nginx1 nginx2 nginx3
-# Will generate nginx{1,2,3}.crt,.key,.json file in cwd. It's upto the caller
-# to execute kubectl -f on the json file. The secret will have a label of
-# app=nginxsni, so you can delete it via the cleanup function.
-function makeCerts {
-    local label=$1
-    shift
-    for h in ${@}; do
-        if [ ! -f $h.json ] || [ ! -f $h.crt ] || [ ! -f $h.key ]; then
-            printf "\nCreating new secrets for $h, will take ~30s\n\n"
-            local cert=$h.crt key=$h.key host=$h secret=$h.json cname=$h
-            if [ $h == "wildcard" ]; then
-                cname=*.$h.com
-            fi
-
-            # Generate crt and key
-            openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
-                    -keyout "${key}" -out "${cert}" -subj "/CN=${cname}/O=${cname}"
-        fi
-
-        cat <<EOF > secret-$SECRET_NAME-$h.json
-{
-    "kind": "Secret",
-    "apiVersion": "v1",
-    "metadata": {
-        "name": "$SECRET_NAME"
-    },
-    "data": {
-        "$h.crt": "$(cat ./$h.crt | base64)",
-        "$h.key": "$(cat ./$h.key | base64)"
-    }
-}
-
-EOF
-
-    done
-}
-
-makeCerts ${APP} ${HOSTS[*]}
--- a/controllers/nginx-third-party/examples/rc-custom-configuration.yaml
+++ b/controllers/nginx-third-party/examples/rc-custom-configuration.yaml
@ -0,0 +1,54 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: nginx-ingress-3rdpartycfg
+  labels:
+    k8s-app: nginx-ingress-lb
+spec:
+  replicas: 1
+  selector:
+    k8s-app: nginx-ingress-lb
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-ingress-lb
+        name: nginx-ingress-lb
+    spec:
+      containers:
+      - image: gcr.io/google_containers/nginx-third-party:0.4
+        name: nginx-ingress-lb
+        imagePullPolicy: Always
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 10249
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        # use downward API
+        env:
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+        ports:
+        - containerPort: 80
+          hostPort: 80
+        - containerPort: 443
+          hostPort: 4444
+        # we expose 8080 to access nginx stats in url /nginx-status
+        # this is optional
+        - containerPort: 8080
+          hostPort: 8081
+        args:
+        - /nginx-third-party-lb
+        - --default-backend-service=default/default-http-backend
+        - --nginx-configmap=default/nginx-load-balancer-conf
--- a/controllers/nginx-third-party/lua/error_page.lua
+++ b/controllers/nginx-third-party/lua/error_page.lua
@ -1,14 +1,23 @@
 http = require "resty.http"
-def_backend = "http://upstream-default-backend"
+def_backend = "upstream-default-backend"
+
+local concat = table.concat
+local upstream = require "ngx.upstream"
+local get_servers = upstream.get_servers
+local get_upstreams = upstream.get_upstreams
+local random = math.random
+local us = get_upstreams()

 function openURL(status)
    local httpc = http.new()

-    local res, err = httpc:request_uri(def_backend, {
+    local random_backend = get_destination()
+    local res, err = httpc:request_uri(random_backend, {
        path = "/",
        method = "GET",
        headers = {
-          ["Content-Type"] = ngx.var.httpAccept or "html",
+            ["X-Code"] = status or "404",
+            ["X-Format"] = ngx.var.httpAccept or "html",
        }
    })

@ -17,10 +26,42 @@ function openURL(status)
        ngx.exit(500)
    end

-    ngx.status = tonumber(status)
    if ngx.var.http_cookie then
        ngx.header["Cookie"] = ngx.var.http_cookie
    end
-    
+
+    ngx.status = tonumber(status)
    ngx.say(res.body)
 end
+
+function get_destination()
+    for _, u in ipairs(us) do
+        if u == def_backend then
+            local srvs, err = get_servers(u)
+            local us_table = {}
+            if not srvs then
+                return "http://127.0.0.1:8181"
+            else
+                for _, srv in ipairs(srvs) do
+                    us_table[srv["name"]] = srv["weight"]
+                end
+            end
+            local destination = random_weight(us_table)
+            return "http://"..destination
+        end
+    end
+end
+
+function random_weight(tbl)
+    local total = 0
+    for k, v in pairs(tbl) do
+        total = total + v
+    end
+    local offset = random(0, total - 1)
+    for k1, v1 in pairs(tbl) do
+        if offset < v1 then
+            return k1
+        end
+        offset = offset - v1
+    end
+end
--- a/controllers/nginx-third-party/nginx.tmpl
+++ b/controllers/nginx-third-party/nginx.tmpl
@ -7,8 +7,12 @@ pid /run/nginx.pid;

 worker_rlimit_nofile 131072;

+pcre_jit on;
+
 events {
-    worker_connections {{ $cfg.maxWorkerConnections }};
+    multi_accept        on;
+    worker_connections  {{ $cfg.maxWorkerConnections }};
+    use                 epoll; 
 }

 http {
@ -20,9 +24,14 @@ http {
        require("error_page")
    }

-    sendfile on;
-    tcp_nopush on;
-    tcp_nodelay on;
+    sendfile            on;
+    aio                 threads;
+    tcp_nopush          on;
+    tcp_nodelay         on;
+    
+    log_subrequest      on;
+
+    reset_timedout_connection on;

    keepalive_timeout {{ $cfg.keepAlive }}s;

@ -45,8 +54,8 @@ http {
    client_max_body_size "{{ $cfg.bodySize }}";

    {{ if $cfg.useProxyProtocol }}
-    set_real_ip_from {{ $cfg.proxyRealIpCidr }};
-    real_ip_header proxy_protocol;
+    set_real_ip_from    {{ $cfg.proxyRealIpCidr }};
+    real_ip_header      proxy_protocol;
    {{ end }}

    log_format upstreaminfo '{{ if $cfg.useProxyProtocol }}$proxy_protocol_addr{{ else }}$remote_addr{{ end }} - '
@ -120,42 +129,19 @@ http {

    # Custom error pages
    proxy_intercept_errors on;
-    error_page 403 @custom_403;
-    error_page 404 @custom_404;
-    error_page 405 @custom_405;
-    error_page 408 @custom_408;
-    error_page 413 @custom_413;
-    error_page 501 @custom_501;
-    error_page 502 @custom_502;
-    error_page 503 @custom_503;
-    error_page 504 @custom_504;

-    # Reverse Proxy configuration
-    # pass original Host header
-    proxy_set_header Host                   $host;
-    # Pass Real IP
-    proxy_set_header X-Real-IP              $remote_addr;
-
-    proxy_set_header X-Forwarded-For        $proxy_add_x_forwarded_for;
-    proxy_set_header X-Forwarded-Host       $host;
-    proxy_set_header X-Forwarded-Server     $host;
-    proxy_set_header X-Forwarded-Port       $http_x_forwarded_port;
-    proxy_set_header X-Forwarded-Proto      $scheme;
-
-    proxy_connect_timeout                   {{ $cfg.proxyConnectTimeout }}s;
-    proxy_send_timeout                      {{ $cfg.proxySendTimeout }}s;
-    proxy_read_timeout                      {{ $cfg.proxyReadTimeout }}s;
-
-    proxy_buffering                         off;
-    
-    proxy_http_version                      1.1;
-
-    # Allow websocket connections
-    proxy_set_header            Upgrade           $http_upgrade;
-    proxy_set_header            Connection        $connection_upgrade;
+    error_page 403 = @custom_403;
+    error_page 404 = @custom_404;
+    error_page 405 = @custom_405;
+    error_page 408 = @custom_408;
+    error_page 413 = @custom_413;
+    error_page 501 = @custom_501;
+    error_page 502 = @custom_502;
+    error_page 503 = @custom_503;
+    error_page 504 = @custom_504;

    # In case of errors try the next upstream server before returning an error
-    proxy_next_upstream                     error timeout http_502 http_503 http_504;
+    proxy_next_upstream                     error timeout invalid_header http_502 http_503 http_504;

    server {
        listen 80 default_server{{ if $cfg.useProxyProtocol }} proxy_protocol{{ end }};
@ -183,6 +169,7 @@ http {
        ssl_certificate_key {{ $server.SSLCertificateKey }};{{ end }}

        server_name {{ $server.Name }};
+
        {{ if $server.SSL }}
        if ($scheme = http) {
            return 301 https://$host$request_uri;
@ -190,11 +177,32 @@ http {
        {{ end }}
        {{ range $location := $server.Locations }}
        location {{ $location.Path }} {
-            proxy_set_header Host $host;
+            proxy_set_header Host                   $host;
+
+            # Pass Real IP
+            proxy_set_header X-Real-IP              $remote_addr;
+
+            # Allow websocket connections
+            proxy_set_header                        Upgrade           $http_upgrade;
+            proxy_set_header                        Connection        $connection_upgrade;
+
+            proxy_set_header X-Forwarded-For        $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Host       $host;
+            proxy_set_header X-Forwarded-Server     $host;
+            proxy_set_header X-Forwarded-Proto      $scheme;
+
+            proxy_connect_timeout                   {{ $cfg.proxyConnectTimeout }}s;
+            proxy_send_timeout                      {{ $cfg.proxySendTimeout }}s;
+            proxy_read_timeout                      {{ $cfg.proxyReadTimeout }}s;
+
+            proxy_redirect                          off;
+            proxy_buffering                         off;
+
+            proxy_http_version                      1.1;
+
            proxy_pass http://{{ $location.Upstream.Name }};
        }
        {{ end }}
-
        {{ template "CUSTOM_ERRORS" $cfg }}
    }
    {{ end }}
@ -217,6 +225,7 @@ http {
        location /nginx-status {
            #vhost_traffic_status_display;
            #vhost_traffic_status_display_format html;
+            access_log off;
            stub_status on;
        }

@ -248,8 +257,8 @@ stream {

    server {
        listen {{ $tcpServer.Path }};
-        proxy_connect_timeout  {{ $cfg.proxyConnectTimeout }}s;
-        proxy_timeout          {{ $cfg.proxyReadTimeout }}s;
+        proxy_connect_timeout  {{ $cfg.proxyConnectTimeout }};
+        proxy_timeout          {{ $cfg.proxyReadTimeout }};
        proxy_pass             tcp-{{ $tcpServer.Upstream.Name }};
    }
 {{ end }}
@ -258,48 +267,56 @@ stream {
 {{/* definition of templates to avoid repetitions */}}
 {{ define "CUSTOM_ERRORS" }}
        location @custom_403 {
+            internal;
            content_by_lua_block {
                openURL(403)
            }
        }

        location @custom_404 {
+            internal;
            content_by_lua_block {
                openURL(404)
            }
        }

        location @custom_405 {
+            internal;
            content_by_lua_block {
                openURL(405)
            }
        }

        location @custom_408 {
+            internal;
            content_by_lua_block {
                openURL(408)
            }
        }

        location @custom_413 {
+            internal;
            content_by_lua_block {
                openURL(413)
            }
        }

        location @custom_502 {
+            internal;
            content_by_lua_block {
                openURL(502)
            }
        }

        location @custom_503 {
+            internal;
            content_by_lua_block {
                openURL(503)
            }
        }

        location @custom_504 {
+            internal;
            content_by_lua_block {
                openURL(504)
            }
--- a/controllers/nginx-third-party/nginx/command.go
+++ b/controllers/nginx-third-party/nginx/command.go
@ -54,7 +54,9 @@ func (ngx *Manager) Start() {
 // shut down, stop accepting new connections and continue to service current requests
 // until all such requests are serviced. After that, the old worker processes exit.
 // http://nginx.org/en/docs/beginners_guide.html#control
-func (ngx *Manager) CheckAndReload(cfg *nginxConfiguration, ingressCfg IngressConfig) {
+func (ngx *Manager) CheckAndReload(cfg nginxConfiguration, ingressCfg IngressConfig) {
+	ngx.reloadRateLimiter.Accept()
+
 	ngx.reloadLock.Lock()
 	defer ngx.reloadLock.Unlock()

--- a/controllers/nginx-third-party/nginx/main.go
+++ b/controllers/nginx-third-party/nginx/main.go
@ -32,6 +32,7 @@ import (

 	"k8s.io/kubernetes/pkg/api"
 	client "k8s.io/kubernetes/pkg/client/unversioned"
+	"k8s.io/kubernetes/pkg/util"
 )

 const (
@ -42,7 +43,7 @@ const (
 	// http://nginx.org/en/docs/ngx_core_module.html#error_log
 	// Configures logging level [debug | info | notice | warn | error | crit | alert | emerg]
 	// Log levels above are listed in the order of increasing severity
-	errorLevel = "info"
+	errorLevel = "notice"

 	// HTTP Strict Transport Security (often abbreviated as HSTS) is a security feature (HTTP header)
 	// that tell browsers that it should only be communicated with using HTTPS, instead of using HTTP.
@ -84,135 +85,137 @@ const (
 type nginxConfiguration struct {
 	// http://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size
 	// Sets the maximum allowed size of the client request body
-	BodySize string `json:"bodySize,omitempty" structs:"bodySize,omitempty"`
+	BodySize string `structs:"body-size,omitempty"`

 	// http://nginx.org/en/docs/ngx_core_module.html#error_log
 	// Configures logging level [debug | info | notice | warn | error | crit | alert | emerg]
 	// Log levels above are listed in the order of increasing severity
-	ErrorLogLevel string `json:"errorLogLevel,omitempty" structs:"errorLogLevel,omitempty"`
+	ErrorLogLevel string `structs:"error-log-level,omitempty"`

 	// Enables or disables the header HTS in servers running SSL
-	UseHTS bool `json:"useHTS,omitempty" structs:"useHTS,omitempty"`
+	UseHTS bool `structs:"use-hts,omitempty"`

 	// Enables or disables the use of HTS in all the subdomains of the servername
-	HTSIncludeSubdomains bool `json:"htsIncludeSubdomains,omitempty" structs:"htsIncludeSubdomains,omitempty"`
+	HTSIncludeSubdomains bool `structs:"hts-include-subdomains,omitempty"`

 	// HTTP Strict Transport Security (often abbreviated as HSTS) is a security feature (HTTP header)
 	// that tell browsers that it should only be communicated with using HTTPS, instead of using HTTP.
 	// https://developer.mozilla.org/en-US/docs/Web/Security/HTTP_strict_transport_security
 	// max-age is the time, in seconds, that the browser should remember that this site is only to be
 	// accessed using HTTPS.
-	HTSMaxAge string `json:"htsMaxAge,omitempty" structs:"htsMaxAge,omitempty"`
+	HTSMaxAge string `structs:"hts-max-age,omitempty"`

 	// Time during which a keep-alive client connection will stay open on the server side.
 	// The zero value disables keep-alive client connections
 	// http://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_timeout
-	KeepAlive int `json:"keepAlive,omitempty" structs:"keepAlive,omitempty"`
+	KeepAlive int `structs:"keep-alive,omitempty"`

 	// Maximum number of simultaneous connections that can be opened by each worker process
 	// http://nginx.org/en/docs/ngx_core_module.html#worker_connections
-	MaxWorkerConnections int `json:"maxWorkerConnections,omitempty" structs:"maxWorkerConnections,omitempty"`
+	MaxWorkerConnections int `structs:"max-worker-connections,omitempty"`

 	// Defines a timeout for establishing a connection with a proxied server.
 	// It should be noted that this timeout cannot usually exceed 75 seconds.
 	// http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout
-	ProxyConnectTimeout int `json:"proxyConnectTimeout,omitempty" structs:"proxyConnectTimeout,omitempty"`
+	ProxyConnectTimeout int `structs:"proxy-connect-timeout,omitempty"`

 	// If UseProxyProtocol is enabled ProxyRealIPCIDR defines the default the IP/network address
 	// of your external load balancer
-	ProxyRealIPCIDR string `json:"proxyRealIPCIDR,omitempty" structs:"proxyRealIPCIDR,omitempty"`
+	ProxyRealIPCIDR string `structs:"proxy-real-ip-cidr,omitempty"`

 	// Timeout in seconds for reading a response from the proxied server. The timeout is set only between
 	// two successive read operations, not for the transmission of the whole response
 	// http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout
-	ProxyReadTimeout int `json:"proxyReadTimeout,omitempty" structs:"proxyReadTimeout,omitempty"`
+	ProxyReadTimeout int `structs:"proxy-read-timeout,omitempty"`

 	// Timeout in seconds for transmitting a request to the proxied server. The timeout is set only between
 	// two successive write operations, not for the transmission of the whole request.
 	// http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_send_timeout
-	ProxySendTimeout int `json:"proxySendTimeout,omitempty" structs:"proxySendTimeout,omitempty"`
+	ProxySendTimeout int `structs:"proxy-send-timeout,omitempty"`

 	// Configures name servers used to resolve names of upstream servers into addresses
 	// http://nginx.org/en/docs/http/ngx_http_core_module.html#resolver
-	Resolver string `json:"resolver,omitempty" structs:"resolver,omitempty"`
+	Resolver string `structs:"resolver,omitempty"`

 	// Maximum size of the server names hash tables used in server names, map directive’s values,
 	// MIME types, names of request header strings, etcd.
 	// http://nginx.org/en/docs/hash.html
 	// http://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_max_size
-	ServerNameHashMaxSize int `json:"serverNameHashMaxSize,omitempty" structs:"serverNameHashMaxSize,omitempty"`
+	ServerNameHashMaxSize int `structs:"server-name-hash-max-size,omitempty"`

 	// Size of the bucker for the server names hash tables
 	// http://nginx.org/en/docs/hash.html
 	// http://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_bucket_size
-	ServerNameHashBucketSize int `json:"serverNameHashBucketSize,omitempty" structs:"serverNameHashBucketSize,omitempty"`
+	ServerNameHashBucketSize int `structs:"server-name-hash-bucket-size,omitempty"`

 	// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_buffer_size
 	// Sets the size of the buffer used for sending data.
 	// 4k helps NGINX to improve TLS Time To First Byte (TTTFB)
 	// https://www.igvita.com/2013/12/16/optimizing-nginx-tls-time-to-first-byte/
-	SSLBufferSize string `json:"sslBufferSize,omitempty" structs:"sslBufferSize,omitempty"`
+	SSLBufferSize string `structs:"ssl-buffer-size,omitempty"`

 	// Enabled ciphers list to enabled. The ciphers are specified in the format understood by
 	// the OpenSSL library
 	// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_ciphers
-	SSLCiphers string `json:"sslCiphers,omitempty" structs:"sslCiphers,omitempty"`
+	SSLCiphers string `structs:"ssl-ciphers,omitempty"`

 	// Base64 string that contains Diffie-Hellman key to help with "Perfect Forward Secrecy"
 	// https://www.openssl.org/docs/manmaster/apps/dhparam.html
 	// https://wiki.mozilla.org/Security/Server_Side_TLS#DHE_handshake_and_dhparam
 	// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_dhparam
-	SSLDHParam string `json:"sslDHParam,omitempty" structs:"sslDHParam,omitempty"`
+	SSLDHParam string `structs:"ssl-dh-param,omitempty"`

 	// SSL enabled protocols to use
 	// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_protocols
-	SSLProtocols string `json:"sslProtocols,omitempty" structs:"sslProtocols,omitempty"`
+	SSLProtocols string `structs:"ssl-protocols,omitempty"`

 	// Enables or disables the use of shared SSL cache among worker processes.
 	// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_cache
-	SSLSessionCache bool `json:"sslSessionCache,omitempty" structs:"sslSessionCache,omitempty"`
+	SSLSessionCache bool `structs:"ssl-session-cache,omitempty"`

 	// Size of the SSL shared cache between all worker processes.
 	// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_cache
-	SSLSessionCacheSize string `json:"sslSessionCacheSize,omitempty" structs:"sslSessionCacheSize,omitempty"`
+	SSLSessionCacheSize string `structs:"ssl-session-cache-size,omitempty"`

 	// Enables or disables session resumption through TLS session tickets.
 	// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_tickets
-	SSLSessionTickets bool `json:"sslSessionTickets,omitempty" structs:"sslSessionTickets,omitempty"`
+	SSLSessionTickets bool `structs:"ssl-session-tickets,omitempty"`

 	// Time during which a client may reuse the session parameters stored in a cache.
 	// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_timeout
-	SSLSessionTimeout string `json:"sslSessionTimeout,omitempty" structs:"sslSessionTimeout,omitempty"`
+	SSLSessionTimeout string `structs:"ssl-session-timeout,omitempty"`

 	// Enables or disables the use of the PROXY protocol to receive client connection
 	// (real IP address) information passed through proxy servers and load balancers
 	// such as HAproxy and Amazon Elastic Load Balancer (ELB).
 	// https://www.nginx.com/resources/admin-guide/proxy-protocol/
-	UseProxyProtocol bool `json:"useProxyProtocol,omitempty" structs:"useProxyProtocol,omitempty"`
+	UseProxyProtocol bool `structs:"use-proxy-protocol,omitempty"`

 	// Enables or disables the use of the nginx module that compresses responses using the "gzip" method
 	// http://nginx.org/en/docs/http/ngx_http_gzip_module.html
-	UseGzip bool `json:"useGzip,omitempty" structs:"useGzip,omitempty"`
+	UseGzip bool `structs:"use-gzip,omitempty"`

 	// MIME types in addition to "text/html" to compress. The special value “*” matches any MIME type.
 	// Responses with the “text/html” type are always compressed if UseGzip is enabled
-	GzipTypes string `json:"gzipTypes,omitempty" structs:"gzipTypes,omitempty"`
+	GzipTypes string `structs:"gzip-types,omitempty"`

 	// Defines the number of worker processes. By default auto means number of available CPU cores
 	// http://nginx.org/en/docs/ngx_core_module.html#worker_processes
-	WorkerProcesses string `json:"workerProcesses,omitempty" structs:"workerProcesses,omitempty"`
+	WorkerProcesses string `structs:"worker-processes,omitempty"`
 }

 // Manager ...
 type Manager struct {
 	ConfigFile string

-	defCfg *nginxConfiguration
+	defCfg nginxConfiguration

 	defResolver string

 	sslDHParam string

+	reloadRateLimiter util.RateLimiter
+
 	// template loaded ready to be used to generate the nginx configuration file
 	template *template.Template

@ -221,7 +224,7 @@ type Manager struct {

 // defaultConfiguration returns the default configuration contained
 // in the file default-conf.json
-func newDefaultNginxCfg() *nginxConfiguration {
+func newDefaultNginxCfg() nginxConfiguration {
 	cfg := nginxConfiguration{
 		BodySize:                 bodySize,
 		ErrorLogLevel:            errorLevel,
@ -231,10 +234,10 @@ func newDefaultNginxCfg() *nginxConfiguration {
 		GzipTypes:                gzipTypes,
 		KeepAlive:                75,
 		MaxWorkerConnections:     16384,
-		ProxyConnectTimeout:      30,
+		ProxyConnectTimeout:      5,
 		ProxyRealIPCIDR:          defIPCIDR,
-		ProxyReadTimeout:         30,
-		ProxySendTimeout:         30,
+		ProxyReadTimeout:         60,
+		ProxySendTimeout:         60,
 		ServerNameHashMaxSize:    512,
 		ServerNameHashBucketSize: 64,
 		SSLBufferSize:            sslBufferSize,
@ -253,16 +256,17 @@ func newDefaultNginxCfg() *nginxConfiguration {
 		cfg.ErrorLogLevel = "debug"
 	}

-	return &cfg
+	return cfg
 }

 // NewManager ...
 func NewManager(kubeClient *client.Client) *Manager {
 	ngx := &Manager{
-		ConfigFile:  "/etc/nginx/nginx.conf",
-		defCfg:      newDefaultNginxCfg(),
-		defResolver: strings.Join(getDNSServers(), " "),
-		reloadLock:  &sync.Mutex{},
+		ConfigFile:        "/etc/nginx/nginx.conf",
+		defCfg:            newDefaultNginxCfg(),
+		defResolver:       strings.Join(getDNSServers(), " "),
+		reloadLock:        &sync.Mutex{},
+		reloadRateLimiter: util.NewTokenBucketRateLimiter(0.1, 1),
 	}

 	ngx.createCertsDir(sslDirectory)
--- a/controllers/nginx-third-party/nginx/ssl.go
+++ b/controllers/nginx-third-party/nginx/ssl.go
@ -47,8 +47,7 @@ func (nginx *Manager) AddOrUpdateCertAndKey(name string, cert string, key string
 // CheckSSLCertificate checks if the certificate and key file are valid
 // returning the result of the validation and the list of hostnames
 // contained in the common name/s
-func (nginx *Manager) CheckSSLCertificate(secretName string) ([]string, error) {
-	pemFileName := sslDirectory + "/" + secretName + ".pem"
+func (nginx *Manager) CheckSSLCertificate(pemFileName string) ([]string, error) {
 	pemCerts, err := ioutil.ReadFile(pemFileName)
 	if err != nil {
 		return []string{}, err
--- a/controllers/nginx-third-party/nginx/template.go
+++ b/controllers/nginx-third-party/nginx/template.go
@ -20,29 +20,34 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"regexp"
 	"text/template"

 	"github.com/fatih/structs"
 	"github.com/golang/glog"
 )

-var funcMap = template.FuncMap{
-	"empty": func(input interface{}) bool {
-		check, ok := input.(string)
-		if ok {
-			return len(check) == 0
-		}
+var (
+	camelRegexp = regexp.MustCompile("[0-9A-Za-z]+")

-		return true
-	},
-}
+	funcMap = template.FuncMap{
+		"empty": func(input interface{}) bool {
+			check, ok := input.(string)
+			if ok {
+				return len(check) == 0
+			}
+
+			return true
+		},
+	}
+)

 func (ngx *Manager) loadTemplate() {
 	tmpl, _ := template.New("nginx.tmpl").Funcs(funcMap).ParseFiles("./nginx.tmpl")
 	ngx.template = tmpl
 }

-func (ngx *Manager) writeCfg(cfg *nginxConfiguration, ingressCfg IngressConfig) (bool, error) {
+func (ngx *Manager) writeCfg(cfg nginxConfiguration, ingressCfg IngressConfig) (bool, error) {
 	fromMap := structs.Map(cfg)
 	toMap := structs.Map(ngx.defCfg)
 	curNginxCfg := merge(toMap, fromMap)
@ -53,7 +58,7 @@ func (ngx *Manager) writeCfg(cfg *nginxConfiguration, ingressCfg IngressConfig)
 	conf["tcpUpstreams"] = ingressCfg.TCPUpstreams
 	conf["defResolver"] = ngx.defResolver
 	conf["sslDHParam"] = ngx.sslDHParam
-	conf["cfg"] = curNginxCfg
+	conf["cfg"] = fixKeyNames(curNginxCfg)

 	buffer := new(bytes.Buffer)
 	err := ngx.template.Execute(buffer, conf)
@ -77,3 +82,23 @@ func (ngx *Manager) writeCfg(cfg *nginxConfiguration, ingressCfg IngressConfig)

 	return changed, nil
 }
+
+func fixKeyNames(data map[string]interface{}) map[string]interface{} {
+	fixed := make(map[string]interface{})
+	for k, v := range data {
+		fixed[toCamelCase(k)] = v
+	}
+
+	return fixed
+}
+
+func toCamelCase(src string) string {
+	byteSrc := []byte(src)
+	chunks := camelRegexp.FindAll(byteSrc, -1)
+	for idx, val := range chunks {
+		if idx > 0 {
+			chunks[idx] = bytes.Title(val)
+		}
+	}
+	return string(bytes.Join(chunks, nil))
+}
--- a/controllers/nginx-third-party/nginx/utils.go
+++ b/controllers/nginx-third-party/nginx/utils.go
@ -18,7 +18,6 @@ package nginx

 import (
 	"bytes"
-	"encoding/json"
 	"io/ioutil"
 	"os"
 	"os/exec"
@ -27,7 +26,7 @@ import (

 	"github.com/golang/glog"

-	"github.com/imdario/mergo"
+	"github.com/mitchellh/mapstructure"
 	"k8s.io/kubernetes/pkg/api"
 )

@ -61,22 +60,25 @@ func getDNSServers() []string {
 }

 // ReadConfig obtains the configuration defined by the user merged with the defaults.
-func (ngx *Manager) ReadConfig(config *api.ConfigMap) (*nginxConfiguration, error) {
+func (ngx *Manager) ReadConfig(config *api.ConfigMap) nginxConfiguration {
 	if len(config.Data) == 0 {
-		return newDefaultNginxCfg(), nil
+		return newDefaultNginxCfg()
 	}

 	cfg := newDefaultNginxCfg()

-	data, err := json.Marshal(config.Data)
+	decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
+		TagName:          "structs",
+		Result:           &cfg,
+		WeaklyTypedInput: true,
+	})
+
+	err = decoder.Decode(config.Data)
 	if err != nil {
-		err = mergo.Merge(cfg, data)
-		if err != nil {
-			return cfg, nil
-		}
+		glog.Infof("%v", err)
 	}

-	return cfg, nil
+	return cfg
 }

 func (ngx *Manager) needsReload(data *bytes.Buffer) (bool, error) {
--- a/controllers/nginx-third-party/utils.go
+++ b/controllers/nginx-third-party/utils.go
@ -20,13 +20,14 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"time"
+
+	"github.com/golang/glog"

 	"k8s.io/kubernetes/pkg/client/cache"
 	"k8s.io/kubernetes/pkg/client/unversioned"
-)
-
-var (
-	errMissingPodInfo = fmt.Errorf("Unable to get POD information")
+	"k8s.io/kubernetes/pkg/util/wait"
+	"k8s.io/kubernetes/pkg/util/workqueue"
 )

 // StoreToIngressLister makes a Store that lists Ingress.
@ -34,6 +35,66 @@ type StoreToIngressLister struct {
 	cache.Store
 }

+// taskQueue manages a work queue through an independent worker that
+// invokes the given sync function for every work item inserted.
+type taskQueue struct {
+	// queue is the work queue the worker polls
+	queue *workqueue.Type
+	// sync is called for each item in the queue
+	sync func(string)
+	// workerDone is closed when the worker exits
+	workerDone chan struct{}
+}
+
+func (t *taskQueue) run(period time.Duration, stopCh <-chan struct{}) {
+	wait.Until(t.worker, period, stopCh)
+}
+
+// enqueue enqueues ns/name of the given api object in the task queue.
+func (t *taskQueue) enqueue(obj interface{}) {
+	key, err := keyFunc(obj)
+	if err != nil {
+		glog.Infof("could not get key for object %+v: %v", obj, err)
+		return
+	}
+	t.queue.Add(key)
+}
+
+func (t *taskQueue) requeue(key string, err error) {
+	glog.V(3).Infof("requeuing %v, err %v", key, err)
+	t.queue.Add(key)
+}
+
+// worker processes work in the queue through sync.
+func (t *taskQueue) worker() {
+	for {
+		key, quit := t.queue.Get()
+		if quit {
+			close(t.workerDone)
+			return
+		}
+		glog.V(3).Infof("syncing %v", key)
+		t.sync(key.(string))
+		t.queue.Done(key)
+	}
+}
+
+// shutdown shuts down the work queue and waits for the worker to ACK
+func (t *taskQueue) shutdown() {
+	t.queue.ShutDown()
+	<-t.workerDone
+}
+
+// NewTaskQueue creates a new task queue with the given sync function.
+// The sync function is called for every element inserted into the queue.
+func NewTaskQueue(syncFn func(string)) *taskQueue {
+	return &taskQueue{
+		queue:      workqueue.New(),
+		sync:       syncFn,
+		workerDone: make(chan struct{}),
+	}
+}
+
 // getLBDetails returns runtime information about the pod (name, IP) and replication
 // controller or daemonset (namespace and name).
 // This is required to watch for changes in annotations or configuration (ConfigMap)
@ -44,7 +105,7 @@ func getLBDetails(kubeClient *unversioned.Client) (*lbInfo, error) {

 	pod, _ := kubeClient.Pods(podNs).Get(podName)
 	if pod == nil {
-		return nil, errMissingPodInfo
+		return nil, fmt.Errorf("Unable to get POD information")
 	}

 	return &lbInfo{
@ -56,12 +117,12 @@ func getLBDetails(kubeClient *unversioned.Client) (*lbInfo, error) {

 func isValidService(kubeClient *unversioned.Client, name string) error {
 	if name == "" {
-		return fmt.Errorf("Empty string is not a valid service name")
+		return fmt.Errorf("empty string is not a valid service name")
 	}

 	parts := strings.Split(name, "/")
 	if len(parts) != 2 {
-		return fmt.Errorf("Invalid name format (namespace/name) in service '%v'", name)
+		return fmt.Errorf("invalid name format (namespace/name) in service '%v'", name)
 	}

 	_, err := kubeClient.Services(parts[0]).Get(parts[1])