Add e2e test for OCSP and new configmap setting

test/e2e-image/.gitignore

@@ -2,3 +2,6 @@ e2e.test
 ginkgo
 kubectl
 charts/*
+*.json
+*.db
+*.go

test/e2e-image/Dockerfile

@@ -17,12 +17,6 @@ RUN curl -sSL https://raw.githubusercontent.com/helm/helm/master/scripts/get-hel
 COPY --from=BASE /go/bin/ginkgo /usr/local/bin/
 COPY --from=BASE /usr/local/bin/kubectl /usr/local/bin/
 
-COPY e2e.sh             /e2e.sh
-COPY namespace-overlays /namespace-overlays
-
-COPY wait-for-nginx.sh  /
-COPY e2e.test           /
-
-COPY charts /charts
+COPY . /
 
 CMD [ "/e2e.sh" ]

test/e2e-image/Makefile

@@ -24,6 +24,9 @@ endif
 	cp $(DIR)/../e2e/wait-for-nginx.sh .
 	cp -R $(DIR)/../../charts .
 
+	# TODO: avoid manual copy
+	cp -R $(DIR)/../../test/e2e/settings/ocsp/* .
+
 	docker buildx build \
 		--load \
 		--progress plain \

test/e2e/e2e.go

@@ -40,6 +40,7 @@ import (
 	_ "k8s.io/ingress-nginx/test/e2e/security"
 	_ "k8s.io/ingress-nginx/test/e2e/servicebackend"
 	_ "k8s.io/ingress-nginx/test/e2e/settings"
+	_ "k8s.io/ingress-nginx/test/e2e/settings/ocsp"
 	_ "k8s.io/ingress-nginx/test/e2e/ssl"
 	_ "k8s.io/ingress-nginx/test/e2e/status"
 	_ "k8s.io/ingress-nginx/test/e2e/tcpudp"

test/e2e/run.sh

@@ -104,6 +104,7 @@ docker tag ${REGISTRY}/nginx-ingress-controller-${ARCH}:${TAG} ${REGISTRY}/nginx
 # Preload images used in e2e tests
 docker pull openresty/openresty:1.15.8.2-alpine
 docker pull moul/grpcbin
+docker pull cfssl/cfssl:1.3.2
 
 echo "[dev-env] copying docker images to cluster..."
 export EXIT_CODE=-1
@@ -115,6 +116,7 @@ kind load docker-image --name="${KIND_CLUSTER_NAME}" openresty/openresty:1.15.8.
 kind load docker-image --name="${KIND_CLUSTER_NAME}" ${REGISTRY}/httpbin:${TAG}
 kind load docker-image --name="${KIND_CLUSTER_NAME}" ${REGISTRY}/echo:${TAG}
 kind load docker-image --name="${KIND_CLUSTER_NAME}" moul/grpcbin
+kind load docker-image --name="${KIND_CLUSTER_NAME}" cfssl/cfssl:1.3.2
 " | parallel --joblog /tmp/log {} || EXIT_CODE=$?
 if [ ${EXIT_CODE} -eq 0 ] || [ ${EXIT_CODE} -eq -1 ];
 then

test/e2e/settings/ocsp/ca_csr.json

@@ -0,0 +1,14 @@
+{
+	"CN": "Root CA",
+	"key": {
+		"algo": "rsa",
+		"size": 2048
+	},
+	"names": [{
+		"C": "US",
+		"L": "SF",
+		"O": "kubernetes",
+		"OU": "ingress-nginx",
+		"ST": "CA"
+	}]
+}

test/e2e/settings/ocsp/db-config.json

@@ -0,0 +1 @@
+{"driver":"sqlite3","data_source":"empty.db"}

test/e2e/settings/ocsp/intermediate_ca_csr.json

@@ -0,0 +1,14 @@
+{
+	"CN": "Intermediate CA",
+	"key": {
+		"algo": "rsa",
+		"size": 2048
+	},
+	"names": [{
+		"C": "US",
+		"L": "SF",
+		"O": "kubernetes",
+		"OU": "ingress-nginx",
+		"ST": "CA"
+	}]
+}

test/e2e/settings/ocsp/leaf_csr.json

@@ -0,0 +1,18 @@
+{
+	"CN": "example.com",
+	"hosts": [
+		"example.com",
+		"www.example.com"
+	],
+	"key": {
+		"algo": "rsa",
+		"size": 2048
+	},
+	"names": [{
+		"C": "US",
+		"L": "SF",
+		"O": "kubernetes",
+		"OU": "ingress-nginx",
+		"ST": "CA"
+	}]
+}

test/e2e/settings/ocsp/ocsp.go

@@ -0,0 +1,423 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ocsp
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/crypto/ocsp"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+
+	"k8s.io/ingress-nginx/test/e2e/framework"
+)
+
+var _ = framework.DescribeSetting("OCSP", func() {
+	f := framework.NewDefaultFramework("ocsp")
+
+	ginkgo.BeforeEach(func() {
+		f.NewEchoDeployment()
+	})
+
+	ginkgo.It("should enable OCSP and contain stapling information in the connection", func() {
+		host := "www.example.com"
+
+		f.UpdateNginxConfigMapData("enable-ocsp", "true")
+
+		err := prepareCertificates(f.Namespace)
+		assert.Nil(ginkgo.GinkgoT(), err)
+
+		ing := framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, framework.EchoService, 80, nil)
+		f.EnsureIngress(ing)
+
+		leafCert, err := ioutil.ReadFile("leaf.pem")
+		assert.Nil(ginkgo.GinkgoT(), err)
+
+		leafKey, err := ioutil.ReadFile("leaf-key.pem")
+		assert.Nil(ginkgo.GinkgoT(), err)
+
+		intermediateCa, err := ioutil.ReadFile("intermediate_ca.pem")
+		assert.Nil(ginkgo.GinkgoT(), err)
+
+		var pemCertBuffer bytes.Buffer
+		pemCertBuffer.Write(leafCert)
+		pemCertBuffer.Write([]byte("\n"))
+		pemCertBuffer.Write(intermediateCa)
+
+		f.EnsureSecret(&corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      ing.Spec.TLS[0].SecretName,
+				Namespace: f.Namespace,
+			},
+			Data: map[string][]byte{
+				corev1.TLSCertKey:       pemCertBuffer.Bytes(),
+				corev1.TLSPrivateKeyKey: leafKey,
+			},
+		})
+
+		cfsslDB, err := ioutil.ReadFile("empty.db")
+		assert.Nil(ginkgo.GinkgoT(), err)
+
+		cmap, err := f.EnsureConfigMap(&corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "ocspserve",
+				Namespace: f.Namespace,
+			},
+			BinaryData: map[string][]byte{
+				"empty.db":       cfsslDB,
+				"db-config.json": []byte(`{"driver":"sqlite3","data_source":"/data/empty.db"}`),
+			},
+		})
+		assert.Nil(ginkgo.GinkgoT(), err)
+		assert.NotNil(ginkgo.GinkgoT(), cmap)
+
+		d, s := ocspserveDeployment(f.Namespace)
+		f.EnsureDeployment(d)
+		f.EnsureService(s)
+
+		err = framework.WaitForEndpoints(f.KubeClientSet, framework.DefaultTimeout, "ocspserve", f.Namespace, 1)
+		assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready")
+
+		f.WaitForNginxConfiguration(func(cfg string) bool {
+			return strings.Contains(cfg, "certificate.is_ocsp_stapling_enabled = true")
+		})
+
+		f.WaitForNginxServer(host,
+			func(server string) bool {
+				return strings.Contains(server, fmt.Sprintf(`server_name %v`, host))
+			})
+
+		tlsConfig := &tls.Config{ServerName: host, InsecureSkipVerify: true}
+		resp := f.HTTPTestClientWithTLSConfig(tlsConfig).
+			GET("/").
+			WithURL(f.GetURL(framework.HTTPS)).
+			WithHeader("Host", host).
+			Expect().
+			Status(http.StatusOK).
+			Raw()
+
+		// TODO: avoid second request
+		resp = f.HTTPTestClientWithTLSConfig(tlsConfig).
+			GET("/").
+			WithURL(f.GetURL(framework.HTTPS)).
+			WithHeader("Host", host).
+			Expect().
+			Status(http.StatusOK).
+			Raw()
+
+		state := resp.TLS
+		assert.NotNil(ginkgo.GinkgoT(), state.OCSPResponse, "unexpected connection without OCSP response")
+
+		var issuerCertificate *x509.Certificate
+		var leafAuthorityKeyID string
+		for index, certificate := range state.PeerCertificates {
+			if index == 0 {
+				leafAuthorityKeyID = string(certificate.AuthorityKeyId)
+				continue
+			}
+
+			if leafAuthorityKeyID == string(certificate.SubjectKeyId) {
+				issuerCertificate = certificate
+			}
+		}
+
+		response, err := ocsp.ParseResponse(state.OCSPResponse, issuerCertificate)
+		assert.Nil(ginkgo.GinkgoT(), err)
+		assert.Equal(ginkgo.GinkgoT(), ocsp.Good, response.Status)
+	})
+})
+
+const configTemplate = `
+{
+	"signing": {
+		"default": {
+			"ocsp_url": "http://ocspserve.%v.svc.cluster.local",
+			"expiry": "219000h",
+			"usages": [
+				"signing",
+				"key encipherment",
+				"client auth"
+			]
+		},
+		"profiles": {
+			"ocsp": {
+				"usages": ["digital signature", "ocsp signing"],
+				"expiry": "8760h"
+			},
+			"intermediate": {
+				"usages": ["cert sign", "crl sign"],
+				"expiry": "219000h",
+				"ca_constraint": {
+					"is_ca": true
+				}
+			},
+			"server": {
+				"usages": ["signing", "key encipherment", "server auth"],
+				"expiry": "8760h"
+			},
+			"client": {
+				"usages": ["signing", "key encipherment", "client auth"],
+				"expiry": "8760h"
+			}
+		}
+	}
+}
+`
+
+func prepareCertificates(namespace string) error {
+	config := fmt.Sprintf(configTemplate, namespace)
+	err := ioutil.WriteFile("cfssl_config.json", []byte(config), 0644)
+	if err != nil {
+		return fmt.Errorf("creating cfssl_config.json file: %v", err)
+	}
+
+	cfssl := "cfssl"
+	cfssljson := "cfssljson"
+	if !commandExists(cfssl) {
+		ginkgo.By("downloading cfssl...")
+		cfssl, err = downloadBinary(cfssl)
+		if err != nil {
+			return fmt.Errorf("downloading cfssl: %v", err)
+		}
+	}
+
+	if !commandExists(cfssljson) {
+		ginkgo.By("downloading cfssljson...")
+		cfssljson, err = downloadBinary(cfssljson)
+		if err != nil {
+			return fmt.Errorf("downloading cfssljson: %v", err)
+		}
+	}
+
+	commands := []string{
+		fmt.Sprintf("%v gencert -initca ca_csr.json | %v -bare ca", cfssl, cfssljson),
+		fmt.Sprintf("%v gencert -ca ca.pem -ca-key ca-key.pem -config=cfssl_config.json -profile=intermediate intermediate_ca_csr.json | %v -bare intermediate_ca", cfssl, cfssljson),
+		fmt.Sprintf("%v gencert -ca intermediate_ca.pem -ca-key intermediate_ca-key.pem -config=cfssl_config.json -profile=ocsp ocsp_csr.json | %v -bare ocsp", cfssl, cfssljson),
+	}
+
+	for _, command := range commands {
+		ginkgo.By(fmt.Sprintf("running %v", command))
+		out, err := exec.Command("bash", "-c", command).Output()
+		if err != nil {
+			framework.Logf("Command error: %v\n%v\n%v", command, err, out)
+			return err
+		}
+	}
+
+	ctx, canc := context.WithCancel(context.Background())
+	defer canc()
+
+	command := fmt.Sprintf("%v serve -db-config=db-config.json -ca-key=intermediate_ca-key.pem -ca=intermediate_ca.pem -config=cfssl_config.json -responder=ocsp.pem -responder-key=ocsp-key.pem", cfssl)
+	ginkgo.By(fmt.Sprintf("running %v", command))
+	serve := exec.CommandContext(ctx, "bash", "-c", command)
+	if err := serve.Start(); err != nil {
+		framework.Logf("Command start error: %v\n%v", command, err)
+		return err
+	}
+
+	time.Sleep(1 * time.Second)
+
+	command = fmt.Sprintf("%v gencert -remote=localhost -profile=server leaf_csr.json | %v -bare leaf", cfssl, cfssljson)
+	ginkgo.By(fmt.Sprintf("running %v", command))
+	out, err := exec.Command("bash", "-c", command).Output()
+	if err != nil {
+		framework.Logf("Command error: %v\n%v\n%v", command, err, out)
+		return err
+	}
+
+	err = serve.Process.Signal(syscall.SIGTERM)
+	if err != nil {
+		framework.Logf("Command error: %v", err)
+		return err
+	}
+
+	command = fmt.Sprintf("%v ocsprefresh -ca intermediate_ca.pem -responder=ocsp.pem -responder-key=ocsp-key.pem -db-config=db-config.json", cfssl)
+	ginkgo.By(fmt.Sprintf("running %v", command))
+	out, err = exec.Command("bash", "-c", command).Output()
+	if err != nil {
+		framework.Logf("Command error: %v\n%v\n%v", command, err, out)
+		return err
+	}
+
+	/*
+		Example:
+		cfssl ocspserve -port=8080 -db-config=db-config.json
+		openssl ocsp -issuer intermediate_ca.pem -no_nonce -cert leaf.pem -CAfile ca.pem -text -url http://localhost:8080
+	*/
+
+	return nil
+}
+
+func commandExists(name string) bool {
+	_, err := exec.Command(name, "version").Output()
+	if err != nil {
+		return false
+	}
+
+	return true
+}
+
+func getBinary(filepath string, url string) error {
+	out, err := os.Create(filepath)
+	if err != nil {
+		return err
+	}
+
+	defer out.Close()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("unexpected error downloading CFSSL")
+	}
+
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func downloadBinary(name string) (string, error) {
+	f, err := ioutil.TempFile("", "fw")
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	arch := runtime.GOARCH
+	goos := runtime.GOOS
+	cfsslURL := "https://github.com/cloudflare/cfssl/releases/download/v1.4.1/" + name + "_1.4.1_" + goos + "_" + arch
+
+	err = getBinary(f.Name(), cfsslURL)
+	if err != nil {
+		return "", err
+	}
+
+	err = os.Chmod(f.Name(), 0755)
+	if err != nil {
+		return "", err
+	}
+
+	return f.Name(), nil
+}
+
+func ocspserveDeployment(namespace string) (*appsv1.Deployment, *corev1.Service) {
+	name := "ocspserve"
+	return &appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      name,
+				Namespace: namespace,
+			},
+			Spec: appsv1.DeploymentSpec{
+				Replicas: framework.NewInt32(1),
+				Selector: &metav1.LabelSelector{
+					MatchLabels: map[string]string{
+						"app": name,
+					},
+				},
+				Template: corev1.PodTemplateSpec{
+					ObjectMeta: metav1.ObjectMeta{
+						Labels: map[string]string{
+							"app": name,
+						},
+					},
+					Spec: corev1.PodSpec{
+						TerminationGracePeriodSeconds: framework.NewInt64(0),
+						Containers: []corev1.Container{
+							{
+								Name:  name,
+								Image: "cfssl/cfssl:1.3.2",
+								Command: []string{
+									"/bin/bash",
+									"-c",
+									"cfssl ocspserve -port=80 -address=0.0.0.0 -db-config=/data/db-config.json -loglevel=0",
+								},
+								Ports: []corev1.ContainerPort{
+									{
+										Name:          "http",
+										ContainerPort: 80,
+									},
+								},
+								VolumeMounts: []corev1.VolumeMount{
+									{
+										Name:      name,
+										MountPath: "/data",
+										ReadOnly:  true,
+									},
+								},
+							},
+						},
+						Volumes: []corev1.Volume{
+							{
+								Name: name,
+								VolumeSource: corev1.VolumeSource{
+									ConfigMap: &corev1.ConfigMapVolumeSource{
+										LocalObjectReference: corev1.LocalObjectReference{
+											Name: name,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		&corev1.Service{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      name,
+				Namespace: namespace,
+			},
+			Spec: corev1.ServiceSpec{
+				Ports: []corev1.ServicePort{
+					{
+						Name:       "http",
+						Port:       80,
+						TargetPort: intstr.FromInt(80),
+						Protocol:   corev1.ProtocolTCP,
+					},
+				},
+				Selector: map[string]string{
+					"app": name,
+				},
+			},
+		}
+}

test/e2e/settings/ocsp/ocsp_csr.json

@@ -0,0 +1,14 @@
+{
+	"CN": "OCSP Responder",
+	"key": {
+		"algo": "rsa",
+		"size": 2048
+	},
+	"names": [{
+		"C": "US",
+		"L": "SF",
+		"O": "kubernetes",
+		"OU": "ingress-nginx",
+		"ST": "CA"
+	}]
+}