From ff5b510dd1594b1062cf90deb8cbbfdb915d0311 Mon Sep 17 00:00:00 2001 From: Alexis Couvreur Date: Fri, 7 Oct 2022 19:38:30 +0000 Subject: [PATCH] ci(plugins): add kubernetes test with deployments --- .github/workflows/plugins.yml | 2 +- plugins/traefik/e2e/docker-kubernetes.yml | 25 +++ plugins/traefik/e2e/docker_classic.sh | 7 +- plugins/traefik/e2e/docker_swarm.sh | 9 +- plugins/traefik/e2e/kubernetes.sh | 71 ++++++ plugins/traefik/e2e/manifests/deployment.yml | 217 +++++++++++++++++++ plugins/traefik/e2e/manifests/sablier.yml | 78 +++++++ plugins/traefik/e2e/values.yaml | 25 +++ 8 files changed, 426 insertions(+), 8 deletions(-) create mode 100644 plugins/traefik/e2e/docker-kubernetes.yml create mode 100644 plugins/traefik/e2e/kubernetes.sh create mode 100644 plugins/traefik/e2e/manifests/deployment.yml create mode 100644 plugins/traefik/e2e/manifests/sablier.yml create mode 100644 plugins/traefik/e2e/values.yaml diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 107aed1..5e0b991 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -32,7 +32,7 @@ jobs: strategy: fail-fast: false matrix: - provider: [docker_classic, docker_swarm] + provider: [docker_classic, docker_swarm, kubernetes] steps: - name: Set up Go 1.18 uses: actions/setup-go@v2 diff --git a/plugins/traefik/e2e/docker-kubernetes.yml b/plugins/traefik/e2e/docker-kubernetes.yml new file mode 100644 index 0000000..fb68599 --- /dev/null +++ b/plugins/traefik/e2e/docker-kubernetes.yml @@ -0,0 +1,25 @@ +version: '3' +services: + server: + image: "rancher/k3s:v1.23.12-k3s1" + command: server --no-deploy traefik + tmpfs: + - /run + - /var/run + ulimits: + nproc: 65535 + nofile: + soft: 65535 + hard: 65535 + privileged: true + restart: always + environment: + - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml + - K3S_KUBECONFIG_MODE=666 + volumes: + # This is just so that we get the kubeconfig file out + - .:/output + - '../../..:/plugins-local/src/github.com/acouvreur/sablier' + ports: + - 6443:6443 # Kubernetes API Server + - 8080:80 # Ingress controller port 80 diff --git a/plugins/traefik/e2e/docker_classic.sh b/plugins/traefik/e2e/docker_classic.sh index fe3d97d..49b2327 100644 --- a/plugins/traefik/e2e/docker_classic.sh +++ b/plugins/traefik/e2e/docker_classic.sh @@ -1,12 +1,10 @@ #!/bin/bash -TRAEFIK_VERSION=2.9.4 DOCKER_COMPOSE_FILE=docker-compose.yml DOCKER_COMPOSE_PROJECT_NAME=docker_classic_e2e errors=0 -echo "Using Traefik version ${TRAEFIK_VERSION}" echo "Using Docker version:" docker version @@ -24,7 +22,10 @@ run_docker_classic_test() { prepare_docker_classic sleep 2 go clean -testcache - go test -count=1 -tags e2e -timeout 30s -run ^${1}$ github.com/acouvreur/sablier/e2e || (docker compose -f ${DOCKER_COMPOSE_FILE} -p ${DOCKER_COMPOSE_PROJECT_NAME}"" logs sablier traefik && errors=1) + if ! go test -count=1 -tags e2e -timeout 30s -run ^${1}$ github.com/acouvreur/sablier/e2e; then + errors=1 + docker compose -f ${DOCKER_COMPOSE_FILE} -p ${DOCKER_COMPOSE_PROJECT_NAME} logs sablier traefik + fi destroy_docker_classic } diff --git a/plugins/traefik/e2e/docker_swarm.sh b/plugins/traefik/e2e/docker_swarm.sh index 41cc573..d01aea0 100644 --- a/plugins/traefik/e2e/docker_swarm.sh +++ b/plugins/traefik/e2e/docker_swarm.sh @@ -1,12 +1,10 @@ #!/bin/bash -TRAEFIK_VERSION=2.9.4 DOCKER_STACK_FILE=docker-stack.yml DOCKER_STACK_NAME=DOCKER_SWARM_E2E errors=0 -echo "Using Traefik version ${TRAEFIK_VERSION}" echo "Using Docker version:" docker version @@ -34,8 +32,11 @@ run_docker_swarm_test() { prepare_docker_stack sleep 10 go clean -testcache - go test -count=1 -tags e2e -timeout 30s -run ^${1}$ github.com/acouvreur/sablier/e2e || (docker service logs ${DOCKER_STACK_NAME}_sablier && docker service logs ${DOCKER_STACK_NAME}_traefik && errors=1) - destroy_docker_stack + if ! go test -count=1 -tags e2e -timeout 30s -run ^${1}$ github.com/acouvreur/sablier/e2e; then + errors=1 + docker service logs ${DOCKER_STACK_NAME}_sablier + docker service logs ${DOCKER_STACK_NAME}_traefik + fi } trap destroy_docker_swarm EXIT diff --git a/plugins/traefik/e2e/kubernetes.sh b/plugins/traefik/e2e/kubernetes.sh new file mode 100644 index 0000000..e6f00ef --- /dev/null +++ b/plugins/traefik/e2e/kubernetes.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +export DOCKER_COMPOSE_FILE=docker-kubernetes.yml +export DOCKER_COMPOSE_PROJECT_NAME=kubernetes_e2e + +errors=0 + +export KUBECONFIG=./kubeconfig.yaml + +echo "Using Docker version:" +docker version + +prepare_kubernetes() { + docker compose -f $DOCKER_COMPOSE_FILE -p $DOCKER_COMPOSE_PROJECT_NAME up -d + until kubectl get nodes | grep " Ready "; do sleep 1; done + echo "Loading ghcr.io/acouvreur/sablier:local into k3s..." + docker save ghcr.io/acouvreur/sablier:local | docker exec -i ${DOCKER_COMPOSE_PROJECT_NAME}-server-1 ctr images import - + echo "Loading succeeded." +} + +destroy_kubernetes() { + docker compose -f $DOCKER_COMPOSE_FILE -p $DOCKER_COMPOSE_PROJECT_NAME down --volumes +} + +prepare_traefik_and_sablier() { + helm repo add traefik https://helm.traefik.io/traefik + helm repo update + helm install traefik traefik/traefik -f values.yaml --namespace kube-system + kubectl apply -f ./manifests/sablier.yml +} + +prepare_deployment() { + kubectl apply -f ./manifests/deployment.yml +} + +destroy_deployment() { + kubectl delete -f ./manifests/deployment.yml +} + +prepare_stateful_set() { + kubectl apply -f ./manifests/statefulset.yml +} + +destroy_stateful_set() { + kubectl delete -f ./manifests/statefulset.yml +} + +run_kubernetes_deployment_test() { + echo "Running Kubernetes Test: $1" + prepare_deployment + sleep 10 + go clean -testcache + if ! go test -count=1 -tags e2e -timeout 30s -run ^${1}$ github.com/acouvreur/sablier/e2e; then + errors=1 + kubectl -n kube-system logs deployments/sablier-deployment + kubectl -n kube-system logs deployments/traefik + fi + + destroy_deployment +} + +trap destroy_kubernetes EXIT + +prepare_kubernetes +prepare_traefik_and_sablier +run_kubernetes_deployment_test Test_Dynamic +run_kubernetes_deployment_test Test_Blocking +run_kubernetes_deployment_test Test_Multiple +run_kubernetes_deployment_test Test_Healthy + +exit $errors \ No newline at end of file diff --git a/plugins/traefik/e2e/manifests/deployment.yml b/plugins/traefik/e2e/manifests/deployment.yml new file mode 100644 index 0000000..c782d9a --- /dev/null +++ b/plugins/traefik/e2e/manifests/deployment.yml @@ -0,0 +1,217 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: whoami-deployment + labels: + app: whoami +spec: + replicas: 0 + selector: + matchLabels: + app: whoami + template: + metadata: + labels: + app: whoami + spec: + containers: + - name: whoami + image: containous/whoami:v1.5.0 +--- +apiVersion: v1 +kind: Service +metadata: + name: whoami-service +spec: + ports: + - name: http + targetPort: 80 + port: 80 + selector: + app: whoami +--- +apiVersion: traefik.containo.us/v1alpha1 +kind: Middleware +metadata: + name: dynamic + namespace: default +spec: + plugin: + sablier: + name: deployment_default_whoami-deployment_1 + serviceUrl: 'http://sablier:10000' + timeout: 1m + displayname: 'Dynamic Whoami' +--- +apiVersion: traefik.containo.us/v1alpha1 +kind: Middleware +metadata: + name: blocking + namespace: default +spec: + plugin: + sablier: + name: deployment_default_whoami-deployment_1 + serviceUrl: 'http://sablier:10000' + timeout: 1m + waitui: false + +--- +apiVersion: traefik.containo.us/v1alpha1 +kind: Middleware +metadata: + name: multiple + namespace: default +spec: + plugin: + sablier: + names: deployment_default_whoami-deployment_1,deployment_default_nginx-deployment_1 + serviceUrl: 'http://sablier:10000' + timeout: 1m + displayname: 'Multiple Whoami' +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: whoami-dynamic-ingress + annotations: + kubernetes.io/ingress.class: traefik + traefik.ingress.kubernetes.io/router.middlewares: default-dynamic@kubernetescrd +spec: + rules: + - host: localhost + http: + paths: + - path: /dynamic/whoami + pathType: Prefix + backend: + service: + name: whoami-service + port: + number: 80 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: whoami-blocking-ingress + annotations: + kubernetes.io/ingress.class: traefik + traefik.ingress.kubernetes.io/router.middlewares: default-blocking@kubernetescrd +spec: + rules: + - host: localhost + http: + paths: + - path: /blocking/whoami + pathType: Prefix + backend: + service: + name: whoami-service + port: + number: 80 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: whoami-multiple-ingress + annotations: + kubernetes.io/ingress.class: traefik + traefik.ingress.kubernetes.io/router.middlewares: default-multiple@kubernetescrd +spec: + rules: + - host: localhost + http: + paths: + - path: /multiple/whoami + pathType: Prefix + backend: + service: + name: whoami-service + port: + number: 80 +--- +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + app: nginx +spec: + replicas: 0 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.23.1 +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-service +spec: + ports: + - name: http + targetPort: 80 + port: 80 + selector: + app: nginx +--- +apiVersion: traefik.containo.us/v1alpha1 +kind: Middleware +metadata: + name: healthy + namespace: default +spec: + plugin: + sablier: + name: deployment_default_nginx-deployment_1 + serviceUrl: 'http://sablier:10000' + timeout: 1m + displayname: 'Healthy Nginx' +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: nginx-multiple-ingress + annotations: + kubernetes.io/ingress.class: traefik + traefik.ingress.kubernetes.io/router.middlewares: default-multiple@kubernetescrd +spec: + rules: + - host: localhost + http: + paths: + - path: /multiple/nginx + pathType: Prefix + backend: + service: + name: nginx-service + port: + number: 80 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: nginx-healthy-ingress + annotations: + kubernetes.io/ingress.class: traefik + traefik.ingress.kubernetes.io/router.middlewares: default-healthy@kubernetescrd +spec: + rules: + - host: localhost + http: + paths: + - path: /healthy/nginx + pathType: Prefix + backend: + service: + name: nginx-service + port: + number: 80 \ No newline at end of file diff --git a/plugins/traefik/e2e/manifests/sablier.yml b/plugins/traefik/e2e/manifests/sablier.yml new file mode 100644 index 0000000..fea733f --- /dev/null +++ b/plugins/traefik/e2e/manifests/sablier.yml @@ -0,0 +1,78 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sablier-deployment + namespace: kube-system + labels: + app: sablier +spec: + replicas: 1 + selector: + matchLabels: + app: sablier + template: + metadata: + labels: + app: sablier + spec: + serviceAccountName: sablier + serviceAccount: sablier + containers: + - name: sablier + image: ghcr.io/acouvreur/sablier:local + args: ["start", "--provider.name=kubernetes"] + ports: + - containerPort: 10000 +--- +apiVersion: v1 +kind: Service +metadata: + name: sablier + namespace: kube-system +spec: + selector: + app: sablier + ports: + - protocol: TCP + port: 10000 + targetPort: 10000 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sablier + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: sablier + namespace: kube-system +rules: + - apiGroups: + - apps + - "" + resources: + - deployments + - deployments/scale + - statefulsets + - statefulsets/scale + - endpoints + verbs: + - patch + - get + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: sablier + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: sablier +subjects: + - kind: ServiceAccount + name: sablier + namespace: kube-system \ No newline at end of file diff --git a/plugins/traefik/e2e/values.yaml b/plugins/traefik/e2e/values.yaml new file mode 100644 index 0000000..3a63aaa --- /dev/null +++ b/plugins/traefik/e2e/values.yaml @@ -0,0 +1,25 @@ +# traefik helm values +image: + tag: "2.9.1" + +additionalArguments: + - "--experimental.localPlugins.sablier.moduleName=github.com/acouvreur/sablier" + +providers: + kubernetesIngress: + allowEmptyServices: true + kubernetesCRD: + allowEmptyServices: true + +additionalVolumeMounts: + - name: local-sablier-plugin + mountPath: /plugins-local/src/github.com/acouvreur/sablier + +deployment: + additionalVolumes: + - name: local-sablier-plugin + hostPath: + # directory location on host + path: /plugins-local/src/github.com/acouvreur/sablier + # this field is optional + type: Directory \ No newline at end of file