Rearrange deployment files into kustomizations

test/e2e-image/.gitignore

@@ -1,3 +1,5 @@
 e2e.test
 ginkgo
 kubectl
+/cloud-generic/
+/cluster-wide/

test/e2e-image/Dockerfile

@@ -15,7 +15,10 @@ RUN curl -Lo /usr/local/bin/kubectl \
 COPY --from=BASE /go/bin/ginkgo /usr/local/bin/
 
 COPY e2e.sh             /e2e.sh
-COPY manifests          /manifests
+COPY cloud-generic      /cloud-generic
+COPY cluster-wide       /cluster-wide
+COPY overlay            /overlay
+RUN sed -E -i 's|^- .*deploy/cloud-generic$|- ../cloud-generic|' /overlay/kustomization.yaml
 COPY wait-for-nginx.sh  /
 COPY e2e.test           /
 

test/e2e-image/Makefile

@@ -9,6 +9,8 @@ container:
 
 	cp ../e2e/e2e.test .
 	cp ../e2e/wait-for-nginx.sh .
+	cp -r ../../deploy/cloud-generic .
+	cp -r ../../deploy/cluster-wide .
 
 	docker build -t $(IMAGE) .
 

test/e2e-image/e2e.sh

@@ -27,8 +27,6 @@ if [ ! -f ${HOME}/.kube/config ]; then
     kubectl config use-context default
 fi
 
-kubectl apply -f manifests/rbac.yaml
-
 ginkgo_args=(
     "-randomizeSuites"
     "-randomizeAllSpecs"

test/e2e-image/manifests/mandatory.yaml

@@ -1,205 +0,0 @@
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: nginx-configuration
-  labels:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/part-of: ingress-nginx
-data:
-  worker-processes: "1"
-
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: tcp-services
-  labels:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/part-of: ingress-nginx
-
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: udp-services
-  labels:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/part-of: ingress-nginx
-
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: nginx-ingress-serviceaccount
-  labels:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/part-of: ingress-nginx
-
----
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: Role
-metadata:
-  name: nginx-ingress-role
-  labels:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/part-of: ingress-nginx
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - configmaps
-      - pods
-      - secrets
-      - namespaces
-    verbs:
-      - get
-  - apiGroups:
-      - ""
-    resources:
-      - configmaps
-    resourceNames:
-      # Defaults to "<election-id>-<ingress-class>"
-      # Here: "<ingress-controller-leader>-<nginx>"
-      # This has to be adapted if you change either parameter
-      # when launching the nginx-ingress-controller.
-      - "ingress-controller-leader-nginx"
-    verbs:
-      - get
-      - update
-  - apiGroups:
-      - ""
-    resources:
-      - configmaps
-    verbs:
-      - create
-  - apiGroups:
-      - ""
-    resources:
-      - endpoints
-    verbs:
-      - get
-
----
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: RoleBinding
-metadata:
-  name: nginx-ingress-role-${NAMESPACE}
-  labels:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/part-of: ingress-nginx
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: nginx-ingress-role
-subjects:
-  - kind: ServiceAccount
-    name: nginx-ingress-serviceaccount
-    namespace: ${NAMESPACE}
-
----
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRoleBinding
-metadata:
-  name: nginx-ingress-clusterrole-${NAMESPACE}
-  labels:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/part-of: ingress-nginx
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: nginx-ingress-clusterrole
-subjects:
-  - kind: ServiceAccount
-    name: nginx-ingress-serviceaccount
-    namespace: ${NAMESPACE}
-
----
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: nginx-ingress-controller
-  labels:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/part-of: ingress-nginx
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: ingress-nginx
-      app.kubernetes.io/part-of: ingress-nginx
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: ingress-nginx
-        app.kubernetes.io/part-of: ingress-nginx
-      annotations:
-        prometheus.io/port: "10254"
-        prometheus.io/scrape: "true"
-    spec:
-      terminationGracePeriodSeconds: 0
-      serviceAccountName: nginx-ingress-serviceaccount
-      initContainers:
-        - name: enable-coredump
-          image: busybox
-          command:
-            - /bin/sh
-            - -c
-            - |
-              ulimit -c unlimited
-              echo "/tmp/core.%e.%p" > /proc/sys/kernel/core_pattern
-              sysctl -w fs.suid_dumpable=2
-          securityContext:
-            privileged: true
-      containers:
-        - name: nginx-ingress-controller
-          image: ingress-controller/nginx-ingress-controller:dev
-          args:
-            - /nginx-ingress-controller
-            - --configmap=$(POD_NAMESPACE)/nginx-configuration
-            - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
-            - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
-            - --publish-service=$(POD_NAMESPACE)/ingress-nginx
-            - --annotations-prefix=nginx.ingress.kubernetes.io
-            - --watch-namespace=${NAMESPACE}
-          securityContext:
-            allowPrivilegeEscalation: true
-            capabilities:
-              drop:
-                - ALL
-              add:
-                - NET_BIND_SERVICE
-            # www-data -> 33
-            runAsUser: 33
-          env:
-            - name: POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-          ports:
-            - name: http
-              containerPort: 80
-            - name: https
-              containerPort: 443
-          livenessProbe:
-            failureThreshold: 3
-            httpGet:
-              path: /healthz
-              port: 10254
-              scheme: HTTP
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            successThreshold: 1
-            timeoutSeconds: 1
-          readinessProbe:
-            failureThreshold: 3
-            httpGet:
-              path: /healthz
-              port: 10254
-              scheme: HTTP
-            periodSeconds: 10
-            successThreshold: 1
-            timeoutSeconds: 1

test/e2e-image/manifests/rbac.yaml

@@ -1,54 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRole
-metadata:
-  name: nginx-ingress-clusterrole
-  labels:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/part-of: ingress-nginx
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - configmaps
-      - endpoints
-      - nodes
-      - pods
-      - secrets
-    verbs:
-      - list
-      - watch
-  - apiGroups:
-      - ""
-    resources:
-      - nodes
-    verbs:
-      - get
-  - apiGroups:
-      - ""
-    resources:
-      - services
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups:
-      - "extensions"
-    resources:
-      - ingresses
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups:
-      - ""
-    resources:
-      - events
-    verbs:
-      - create
-      - patch
-  - apiGroups:
-      - "extensions"
-    resources:
-      - ingresses/status
-    verbs:
-      - update

test/e2e-image/overlay/deployment-e2e.yaml

@@ -0,0 +1,26 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-ingress-controller
+spec:
+  template:
+    spec:
+      terminationGracePeriodSeconds: 0
+      initContainers:
+        - name: enable-coredump
+          image: busybox
+          command:
+            - /bin/sh
+            - -c
+            - |
+              ulimit -c unlimited
+              echo "/tmp/core.%e.%p" > /proc/sys/kernel/core_pattern
+              sysctl -w fs.suid_dumpable=2
+          securityContext:
+            privileged: true
+      containers:
+        - name: nginx-ingress-controller
+          livenessProbe:
+            timeoutSeconds: 1
+          readinessProbe:
+            timeoutSeconds: 1

test/e2e-image/overlay/deployment-extension-group-patch.yaml

@@ -0,0 +1,3 @@
+- op: replace
+  path: /apiVersion
+  value: extensions/v1beta1

test/e2e-image/overlay/deployment-namespace-patch.yaml

@@ -0,0 +1,3 @@
+- op: add
+  path: /spec/template/spec/containers/0/args/-1
+  value: "--watch-namespace=$(POD_NAMESPACE)"

test/e2e-image/overlay/kustomization.yaml

@@ -0,0 +1,34 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+- ../../../deploy/cloud-generic
+configMapGenerator:
+- name: nginx-configuration
+  behavior: merge
+  literals:
+  - worker-processes=1
+patchesStrategicMerge:
+- deployment-e2e.yaml
+- service-protocol-tcp.yaml
+patchesJson6902:
+- path: deployment-namespace-patch.yaml
+  target:
+    group: apps
+    kind: Deployment
+    name: nginx-ingress-controller
+    version: v1
+- path: service-cluster-patch.yaml
+  target:
+    kind: Service
+    name: ingress-nginx
+    version: v1
+- path: deployment-extension-group-patch.yaml
+  target:
+    group: apps
+    kind: Deployment
+    name: nginx-ingress-controller
+    version: v1
+images:
+- name: quay.io/kubernetes-ingress-controller/nginx-ingress-controller
+  newName: ingress-controller/nginx-ingress-controller
+  newTag: dev

test/e2e-image/overlay/service-cluster-patch.yaml

@@ -0,0 +1,4 @@
+- op: remove
+  path: /spec/externalTrafficPolicy
+- op: remove
+  path: /spec/type

test/e2e-image/overlay/service-protocol-tcp.yaml

@@ -12,6 +12,3 @@ spec:
       port: 443
       targetPort: 443
       protocol: TCP
-  selector:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/part-of: ingress-nginx

test/e2e/defaultbackend/custom_default_backend.go

@@ -17,7 +17,6 @@ limitations under the License.
 package defaultbackend
 
 import (
-	"fmt"
 	"net/http"
 	"strings"
 
@@ -40,7 +39,7 @@ var _ = framework.IngressNginxDescribe("Custom Default Backend", func() {
 		framework.UpdateDeployment(f.KubeClientSet, f.Namespace, "nginx-ingress-controller", 1,
 			func(deployment *appsv1beta1.Deployment) error {
 				args := deployment.Spec.Template.Spec.Containers[0].Args
-				args = append(args, fmt.Sprintf("--default-backend-service=%s/%s", f.Namespace, "http-svc"))
+				args = append(args, "--default-backend-service=$(POD_NAMESPACE)/http-svc")
 				deployment.Spec.Template.Spec.Containers[0].Args = args
 				_, err := f.KubeClientSet.AppsV1beta1().Deployments(f.Namespace).Update(deployment)
 

test/e2e/settings/pod_security_policy.go

@@ -17,6 +17,7 @@ limitations under the License.
 package settings
 
 import (
+	"fmt"
 	"net/http"
 	"strings"
 
@@ -48,7 +49,7 @@ var _ = framework.IngressNginxDescribe("[Serial] Pod Security Policies", func()
 			Expect(err).NotTo(HaveOccurred(), "creating Pod Security Policy")
 		}
 
-		role, err := f.KubeClientSet.RbacV1().ClusterRoles().Get("nginx-ingress-clusterrole", metav1.GetOptions{})
+		role, err := f.KubeClientSet.RbacV1().ClusterRoles().Get(fmt.Sprintf("nginx-ingress-clusterrole-%v", f.Namespace), metav1.GetOptions{})
 		Expect(err).NotTo(HaveOccurred(), "getting ingress controller cluster role")
 		Expect(role).NotTo(BeNil())
 
@@ -78,7 +79,7 @@ var _ = framework.IngressNginxDescribe("[Serial] Pod Security Policies", func()
 	})
 
 	AfterEach(func() {
-		role, err := f.KubeClientSet.RbacV1().ClusterRoles().Get("nginx-ingress-clusterrole", metav1.GetOptions{})
+		role, err := f.KubeClientSet.RbacV1().ClusterRoles().Get(fmt.Sprintf("nginx-ingress-clusterrole-%v", f.Namespace), metav1.GetOptions{})
 		Expect(err).NotTo(HaveOccurred(), "getting ingress controller cluster role")
 		Expect(role).NotTo(BeNil())
 

test/e2e/wait-for-nginx.sh

@@ -35,10 +35,32 @@ function on_exit {
 }
 trap on_exit EXIT
 
-kubectl apply --namespace=$NAMESPACE -f $DIR/manifests/service.yaml
+CLUSTER_WIDE="$DIR/cluster-wide-$NAMESPACE"
 
-sed "s@\${NAMESPACE}@${NAMESPACE}@" $DIR/manifests/mandatory.yaml | kubectl apply --namespace=$NAMESPACE -f -
-cat $DIR/manifests/service.yaml | kubectl apply --namespace=$NAMESPACE -f -
+mkdir "$CLUSTER_WIDE"
+
+cat << EOF > "$CLUSTER_WIDE/kustomization.yaml"
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+- ../cluster-wide
+nameSuffix: "-$NAMESPACE"
+EOF
+
+OVERLAY="$DIR/overlay-$NAMESPACE"
+
+mkdir "$OVERLAY"
+
+cat << EOF > "$OVERLAY/kustomization.yaml"
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: $NAMESPACE
+bases:
+- ../overlay
+- ../cluster-wide-$NAMESPACE
+EOF
+
+kubectl apply --kustomize "$OVERLAY"
 
 # wait for the deployment and fail if there is an error before starting the execution of any test
 kubectl rollout status \