ci(plugins): add kubernetes test with deployments

.github/workflows/plugins.yml

@@ -32,7 +32,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        provider: [docker_classic, docker_swarm]
+        provider: [docker_classic, docker_swarm, kubernetes]
     steps:
       - name: Set up Go 1.18
         uses: actions/setup-go@v2

plugins/traefik/e2e/docker-kubernetes.yml

@@ -0,0 +1,25 @@
+version: '3'
+services:
+  server:
+    image: "rancher/k3s:v1.23.12-k3s1"
+    command: server --no-deploy traefik
+    tmpfs:
+      - /run
+      - /var/run
+    ulimits:
+      nproc: 65535
+      nofile:
+        soft: 65535
+        hard: 65535
+    privileged: true
+    restart: always
+    environment:
+      - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml
+      - K3S_KUBECONFIG_MODE=666
+    volumes:
+      # This is just so that we get the kubeconfig file out
+      - .:/output
+      - '../../..:/plugins-local/src/github.com/acouvreur/sablier'
+    ports:
+      - 6443:6443  # Kubernetes API Server
+      - 8080:80  # Ingress controller port 80

plugins/traefik/e2e/docker_classic.sh

@@ -1,12 +1,10 @@
 #!/bin/bash
 
-TRAEFIK_VERSION=2.9.4
 DOCKER_COMPOSE_FILE=docker-compose.yml
 DOCKER_COMPOSE_PROJECT_NAME=docker_classic_e2e
 
 errors=0
 
-echo "Using Traefik version ${TRAEFIK_VERSION}"
 echo "Using Docker version:"
 docker version
 
@@ -24,7 +22,10 @@ run_docker_classic_test() {
   prepare_docker_classic
   sleep 2
   go clean -testcache
-  go test -count=1 -tags e2e -timeout 30s -run ^${1}$ github.com/acouvreur/sablier/e2e || (docker compose -f ${DOCKER_COMPOSE_FILE} -p ${DOCKER_COMPOSE_PROJECT_NAME}"" logs sablier traefik && errors=1)
+  if ! go test -count=1 -tags e2e -timeout 30s -run ^${1}$ github.com/acouvreur/sablier/e2e; then
+    errors=1
+    docker compose -f ${DOCKER_COMPOSE_FILE} -p ${DOCKER_COMPOSE_PROJECT_NAME} logs sablier traefik
+  fi
   destroy_docker_classic
 }
 

plugins/traefik/e2e/docker_swarm.sh

@@ -1,12 +1,10 @@
 #!/bin/bash
 
-TRAEFIK_VERSION=2.9.4
 DOCKER_STACK_FILE=docker-stack.yml
 DOCKER_STACK_NAME=DOCKER_SWARM_E2E
 
 errors=0
 
-echo "Using Traefik version ${TRAEFIK_VERSION}"
 echo "Using Docker version:"
 docker version
 
@@ -34,8 +32,11 @@ run_docker_swarm_test() {
   prepare_docker_stack
   sleep 10
   go clean -testcache
-  go test -count=1 -tags e2e -timeout 30s -run ^${1}$ github.com/acouvreur/sablier/e2e || (docker service logs ${DOCKER_STACK_NAME}_sablier && docker service logs ${DOCKER_STACK_NAME}_traefik && errors=1)
-  destroy_docker_stack
+  if ! go test -count=1 -tags e2e -timeout 30s -run ^${1}$ github.com/acouvreur/sablier/e2e; then
+    errors=1
+    docker service logs ${DOCKER_STACK_NAME}_sablier
+    docker service logs ${DOCKER_STACK_NAME}_traefik
+  fi
 }
 
 trap destroy_docker_swarm EXIT

plugins/traefik/e2e/kubernetes.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+export DOCKER_COMPOSE_FILE=docker-kubernetes.yml
+export DOCKER_COMPOSE_PROJECT_NAME=kubernetes_e2e
+
+errors=0
+
+export KUBECONFIG=./kubeconfig.yaml
+
+echo "Using Docker version:"
+docker version
+
+prepare_kubernetes() {
+  docker compose -f $DOCKER_COMPOSE_FILE -p $DOCKER_COMPOSE_PROJECT_NAME up -d
+  until kubectl get nodes | grep " Ready "; do sleep 1; done
+  echo "Loading ghcr.io/acouvreur/sablier:local into k3s..."
+  docker save ghcr.io/acouvreur/sablier:local | docker exec -i ${DOCKER_COMPOSE_PROJECT_NAME}-server-1 ctr images import -
+  echo "Loading succeeded."
+}
+
+destroy_kubernetes() {
+  docker compose -f $DOCKER_COMPOSE_FILE -p $DOCKER_COMPOSE_PROJECT_NAME down --volumes
+}
+
+prepare_traefik_and_sablier() {
+  helm repo add traefik https://helm.traefik.io/traefik
+  helm repo update
+  helm install traefik traefik/traefik -f values.yaml --namespace kube-system
+  kubectl apply -f ./manifests/sablier.yml
+}
+
+prepare_deployment() {
+  kubectl apply -f ./manifests/deployment.yml
+}
+
+destroy_deployment() {
+  kubectl delete -f ./manifests/deployment.yml
+}
+
+prepare_stateful_set() {
+  kubectl apply -f ./manifests/statefulset.yml
+}
+
+destroy_stateful_set() {
+  kubectl delete -f ./manifests/statefulset.yml
+}
+
+run_kubernetes_deployment_test() {
+  echo "Running Kubernetes Test: $1"
+  prepare_deployment
+  sleep 10
+  go clean -testcache
+  if ! go test -count=1 -tags e2e -timeout 30s -run ^${1}$ github.com/acouvreur/sablier/e2e; then
+    errors=1
+    kubectl -n kube-system logs deployments/sablier-deployment
+    kubectl -n kube-system logs deployments/traefik
+  fi
+
+  destroy_deployment
+}
+
+trap destroy_kubernetes EXIT
+
+prepare_kubernetes
+prepare_traefik_and_sablier
+run_kubernetes_deployment_test Test_Dynamic
+run_kubernetes_deployment_test Test_Blocking
+run_kubernetes_deployment_test Test_Multiple
+run_kubernetes_deployment_test Test_Healthy
+
+exit $errors

plugins/traefik/e2e/manifests/deployment.yml

@@ -0,0 +1,217 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: whoami-deployment
+  labels:
+    app: whoami
+spec:
+  replicas: 0
+  selector:
+    matchLabels:
+      app: whoami
+  template:
+    metadata:
+      labels:
+        app: whoami
+    spec:
+      containers:
+      - name: whoami
+        image: containous/whoami:v1.5.0
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: whoami-service
+spec:
+  ports:
+  - name: http
+    targetPort: 80
+    port: 80
+  selector:
+    app: whoami
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: Middleware
+metadata:
+  name: dynamic
+  namespace: default
+spec:
+  plugin:
+    sablier:
+      name: deployment_default_whoami-deployment_1
+      serviceUrl: 'http://sablier:10000'
+      timeout: 1m
+      displayname: 'Dynamic Whoami'
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: Middleware
+metadata:
+  name: blocking
+  namespace: default
+spec:
+  plugin:
+    sablier:
+      name: deployment_default_whoami-deployment_1
+      serviceUrl: 'http://sablier:10000'
+      timeout: 1m
+      waitui: false
+
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: Middleware
+metadata:
+  name: multiple
+  namespace: default
+spec:
+  plugin:
+    sablier:
+      names: deployment_default_whoami-deployment_1,deployment_default_nginx-deployment_1
+      serviceUrl: 'http://sablier:10000'
+      timeout: 1m
+      displayname: 'Multiple Whoami'
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: whoami-dynamic-ingress
+  annotations:
+    kubernetes.io/ingress.class: traefik
+    traefik.ingress.kubernetes.io/router.middlewares: default-dynamic@kubernetescrd
+spec:
+  rules:
+  - host: localhost
+    http:
+      paths:
+      - path: /dynamic/whoami
+        pathType: Prefix
+        backend:
+          service:
+            name: whoami-service
+            port:
+              number: 80
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: whoami-blocking-ingress
+  annotations:
+    kubernetes.io/ingress.class: traefik
+    traefik.ingress.kubernetes.io/router.middlewares: default-blocking@kubernetescrd
+spec:
+  rules:
+  - host: localhost
+    http:
+      paths:
+      - path: /blocking/whoami
+        pathType: Prefix
+        backend:
+          service:
+            name: whoami-service
+            port:
+              number: 80
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: whoami-multiple-ingress
+  annotations:
+    kubernetes.io/ingress.class: traefik
+    traefik.ingress.kubernetes.io/router.middlewares: default-multiple@kubernetescrd
+spec:
+  rules:
+  - host: localhost
+    http:
+      paths:
+      - path: /multiple/whoami
+        pathType: Prefix
+        backend:
+          service:
+            name: whoami-service
+            port:
+              number: 80
+---
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+  labels:
+    app: nginx
+spec:
+  replicas: 0
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.23.1
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-service
+spec:
+  ports:
+  - name: http
+    targetPort: 80
+    port: 80
+  selector:
+    app: nginx
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: Middleware
+metadata:
+  name: healthy
+  namespace: default
+spec:
+  plugin:
+    sablier:
+      name: deployment_default_nginx-deployment_1
+      serviceUrl: 'http://sablier:10000'
+      timeout: 1m
+      displayname: 'Healthy Nginx'
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: nginx-multiple-ingress
+  annotations:
+    kubernetes.io/ingress.class: traefik
+    traefik.ingress.kubernetes.io/router.middlewares: default-multiple@kubernetescrd
+spec:
+  rules:
+  - host: localhost
+    http:
+      paths:
+      - path: /multiple/nginx
+        pathType: Prefix
+        backend:
+          service:
+            name: nginx-service
+            port:
+              number: 80
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: nginx-healthy-ingress
+  annotations:
+    kubernetes.io/ingress.class: traefik
+    traefik.ingress.kubernetes.io/router.middlewares: default-healthy@kubernetescrd
+spec:
+  rules:
+  - host: localhost
+    http:
+      paths:
+      - path: /healthy/nginx
+        pathType: Prefix
+        backend:
+          service:
+            name: nginx-service
+            port:
+              number: 80

plugins/traefik/e2e/manifests/sablier.yml

@@ -0,0 +1,78 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: sablier-deployment
+  namespace: kube-system
+  labels:
+    app: sablier
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: sablier
+  template:
+    metadata:
+      labels:
+        app: sablier
+    spec:
+      serviceAccountName: sablier
+      serviceAccount: sablier
+      containers:
+      - name: sablier
+        image: ghcr.io/acouvreur/sablier:local
+        args: ["start", "--provider.name=kubernetes"]
+        ports:
+        - containerPort: 10000
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: sablier
+  namespace: kube-system
+spec:
+  selector:
+    app: sablier
+  ports:
+    - protocol: TCP
+      port: 10000
+      targetPort: 10000
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: sablier
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: sablier
+  namespace: kube-system
+rules:
+  - apiGroups:
+      - apps
+      - ""
+    resources:
+      - deployments
+      - deployments/scale
+      - statefulsets
+      - statefulsets/scale
+      - endpoints
+    verbs:
+      - patch
+      - get
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: sablier
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: sablier
+subjects:
+  - kind: ServiceAccount
+    name: sablier
+    namespace: kube-system

plugins/traefik/e2e/values.yaml

@@ -0,0 +1,25 @@
+# traefik helm values
+image:
+  tag: "2.9.1"
+
+additionalArguments:
+  - "--experimental.localPlugins.sablier.moduleName=github.com/acouvreur/sablier"
+
+providers:
+  kubernetesIngress:
+    allowEmptyServices: true
+  kubernetesCRD:
+    allowEmptyServices: true
+
+additionalVolumeMounts:
+  - name: local-sablier-plugin
+    mountPath: /plugins-local/src/github.com/acouvreur/sablier
+
+deployment:
+  additionalVolumes:
+    - name: local-sablier-plugin
+      hostPath:
+        # directory location on host
+        path: /plugins-local/src/github.com/acouvreur/sablier
+        # this field is optional
+        type: Directory