E2E Playwright Testing (#466)

* Add e2e testing for frontend

* Hopefully working CI/CD for playwright

* Fix run name

* Trying to fix the CI/CD stuff

* Try this again, although Vite apparently has playwright?

* Fix vitetest

* Add registration tests

* Safer kill of testing dependencies

* These might not last.

* feat: Add iPhone and Android device testing

* fix: Minor fixes, set registration to "fixme" as it fails frequently for some reason.

* fix: Make sure the OS dependencies get installed

* fix: For now remove mobile, they seem to be very hit or miss.

* Use sharding based testing

* Fix some minor mess ups

* Forgot PNPM for the merge

.github/workflows/e2e-partial.yaml

@@ -0,0 +1,94 @@
+name: E2E (Playwright)
+
+on:
+  workflow_call:
+
+
+jobs:
+  playwright-tests:
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        shardIndex: [1,2,3,4]
+        shardTotal: [4]
+    name: E2E Playwright Testing ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install Task
+        uses: arduino/setup-task@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.21"
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+
+      - uses: pnpm/action-setup@v3.0.0
+        with:
+          version: 9.12.2
+
+      - name: Install dependencies
+        run: pnpm install
+        working-directory: frontend
+
+      - name: Install Go Dependencies
+        run: go mod download
+        working-directory: backend
+
+      - name: Run E2E Tests
+        run: task test:e2e -- --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+
+      - uses: actions/upload-artifact@v4
+        name: Upload partial Playwright report
+        if: ${{ !cancelled() }}
+        with:
+          name: blob-report-${{ matrix.shardIndex }}
+          path: frontend/blob-report/
+          retention-days: 2
+
+  merge-reports:
+    # Merge reports after playwright-tests, even if some shards have failed
+    if: ${{ !cancelled() }}
+    needs: [playwright-tests]
+    name: Merge Playwright Reports
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+      - uses: pnpm/action-setup@v3.0.0
+        with:
+          version: 9.12.2
+
+      - name: Install dependencies
+        run: pnpm install
+        working-directory: frontend
+
+      - name: Download blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: frontend/all-blob-reports
+          pattern: blob-report-*
+          merge-multiple: true
+
+      - name: Merge into HTML Report
+        run: pnpm exec playwright merge-reports --reporter html,github ./all-blob-reports
+        working-directory: frontend
+
+      - name: Upload HTML report
+        uses: actions/upload-artifact@v4
+        with:
+          name: html-report--attempt-${{ github.run_attempt }}
+          path: frontend/playwright-report
+          retention-days: 30

.github/workflows/partial-frontend.yaml

@@ -1,4 +1,4 @@
-name: Frontend / E2E
+name: Frontend
 
 on:
   workflow_call:
@@ -114,7 +114,7 @@ jobs:
 
       - uses: actions/setup-node@v4
         with:
-          node-version: 18
+          node-version: lts/*
 
       - uses: pnpm/action-setup@v3.0.0
         with:

.github/workflows/pull-requests.yaml

@@ -17,5 +17,9 @@ jobs:
     uses: ./.github/workflows/partial-backend.yaml
 
   frontend-tests:
-    name: "Frontend and End-to-End Tests"
-    uses: ./.github/workflows/partial-frontend.yaml
+    name: "Frontend Tests"
+    uses: ./.github/workflows/partial-frontend.yaml
+
+  e2e-tests:
+    name: "End-to-End Playwright Tests"
+    uses: ./.github/workflows/e2e-partial.yaml

.gitignore

@@ -61,3 +61,9 @@ backend/api
 
 docs/.vitepress/cache/
 /.data/
+
+# Playwright
+frontend/test-results/
+frontend/playwright-report/
+frontend/blob-report/
+frontend/playwright/.cache/

Taskfile.yml

@@ -81,6 +81,15 @@ tasks:
       - go run ./app/api/ {{ .CLI_ARGS }}
     silent: false
 
+  go:ci:
+    env:
+      HBOX_DEMO: true
+    desc: Runs all go test and lint related tasks
+    dir: backend
+    cmds:
+      - go run ./app/api/ {{ .CLI_ARGS }} &
+    silent: true
+
   go:test:
     desc: Runs all go tests using gotestsum - supports passing gotestsum args
     dir: backend
@@ -162,6 +171,13 @@ tasks:
     cmds:
       - pnpm dev
 
+  ui:ci:
+    desc: Run frontend build in CI mode
+    dir: frontend
+    cmds:
+      - pnpm dev &
+    silent: true
+
   ui:fix:
     desc: Runs prettier and eslint on the frontend
     dir: frontend
@@ -200,6 +216,17 @@ tasks:
       - cd frontend && pnpm run test:ci
     silent: true
 
+  test:e2e:
+    desc: Runs end-to-end test on a live server
+    dir: frontend
+    cmds:
+      - task: go:ci
+      - task: ui:ci
+      - pnpm exec playwright install-deps
+      - pnpm exec playwright install
+      - sleep 30
+      - TEST_SHUTDOWN_API_SERVER=true pnpm exec playwright test -c ./test/playwright.config.ts {{ .CLI_ARGS }}
+
   pr:
     desc: Runs all tasks required for a PR
     cmds:

frontend/package.json

@@ -18,6 +18,7 @@
     "@iconify-json/mdi": "^1.2.3",
     "@intlify/unplugin-vue-i18n": "^4.0.0",
     "@nuxtjs/eslint-config-typescript": "^12.1.0",
+    "@playwright/test": "^1.49.1",
     "@types/markdown-it": "^13.0.9",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^6.21.0",

frontend/pnpm-lock.yaml

@@ -129,6 +129,9 @@ importers:
       '@nuxtjs/eslint-config-typescript':
         specifier: ^12.1.0
         version: 12.1.0(eslint@8.57.1)(typescript@5.6.2)
+      '@playwright/test':
+        specifier: ^1.49.1
+        version: 1.49.1
       '@types/markdown-it':
         specifier: ^13.0.9
         version: 13.0.9
@@ -1586,6 +1589,11 @@ packages:
     resolution: {integrity: sha512-cq8o4cWH0ibXh9VGi5P20Tu9XF/0fFXl9EUinr9QfTM7a7p0oTA4iJRCQWppXR1Pg8dSM0UCItCkPwsk9qWWYA==}
     engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0}
 
+  '@playwright/test@1.49.1':
+    resolution: {integrity: sha512-Ky+BVzPz8pL6PQxHqNRW1k3mIyv933LML7HktS8uik0bUXNCdPhoS/kLihiO1tMf/egaJb4IutXd7UywvXEW+g==}
+    engines: {node: '>=18'}
+    hasBin: true
+
   '@polka/url@1.0.0-next.28':
     resolution: {integrity: sha512-8LduaNlMZGwdZ6qWrKlfa+2M4gahzFkprZiAt2TF8uS0qQgBizKXpXURqvTJ4WtmupWxaLqjRb2UCTe72mu+Aw==}
 
@@ -3364,6 +3372,11 @@ packages:
   fs.realpath@1.0.0:
     resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==}
 
+  fsevents@2.3.2:
+    resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==}
+    engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
+    os: [darwin]
+
   fsevents@2.3.3:
     resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==}
     engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
@@ -4556,6 +4569,16 @@ packages:
   pkg-types@2.1.0:
     resolution: {integrity: sha512-wmJwA+8ihJixSoHKxZJRBQG1oY8Yr9pGLzRmSsNms0iNWyHHAlZCa7mmKiFR10YPZuz/2k169JiS/inOjBCZ2A==}
 
+  playwright-core@1.49.1:
+    resolution: {integrity: sha512-BzmpVcs4kE2CH15rWfzpjzVGhWERJfmnXmniSyKeRZUs9Ws65m+RGIi7mjJK/euCegfn3i7jvqWeWyHe9y3Vgg==}
+    engines: {node: '>=18'}
+    hasBin: true
+
+  playwright@1.49.1:
+    resolution: {integrity: sha512-VYL8zLoNTBxVOrJBbDuRgDWa3i+mfQgDTrL8Ah9QXZ7ax4Dsj0MSq5bYgytRnDVVe+njoKnfsYkH3HzqVj5UZA==}
+    engines: {node: '>=18'}
+    hasBin: true
+
   pluralize@8.0.0:
     resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==}
     engines: {node: '>=4'}
@@ -7745,6 +7768,10 @@ snapshots:
 
   '@pkgr/core@0.1.1': {}
 
+  '@playwright/test@1.49.1':
+    dependencies:
+      playwright: 1.49.1
+
   '@polka/url@1.0.0-next.28': {}
 
   '@redocly/ajv@8.11.2':
@@ -9885,6 +9912,9 @@ snapshots:
 
   fs.realpath@1.0.0: {}
 
+  fsevents@2.3.2:
+    optional: true
+
   fsevents@2.3.3:
     optional: true
 
@@ -11270,6 +11300,14 @@ snapshots:
       exsolve: 1.0.1
       pathe: 2.0.3
 
+  playwright-core@1.49.1: {}
+
+  playwright@1.49.1:
+    dependencies:
+      playwright-core: 1.49.1
+    optionalDependencies:
+      fsevents: 2.3.2
+
   pluralize@8.0.0: {}
 
   portfinder@1.0.33:

frontend/test/e2e/login.browser.spec.ts

@@ -0,0 +1,57 @@
+import { test, expect } from "@playwright/test";
+
+test("valid login", async ({ page }) => {
+  await page.goto("/home");
+  await expect(page).toHaveURL("/");
+  await page.fill("input[type='text']", "demo@example.com");
+  await page.fill("input[placeholder='Password']", "demo");
+  await page.click("button[type='submit']");
+  await expect(page).toHaveURL("/home");
+});
+
+test("invalid login", async ({ page }) => {
+  await page.goto("/home");
+  await expect(page).toHaveURL("/");
+  await page.fill("input[type='text']", "dummy@example.com");
+  await page.fill("input[placeholder='Password']", "dummy");
+  await page.click("button[type='submit']");
+  await expect(page.locator("div[class*='top-2']")).toHaveText("Invalid email or password");
+  await expect(page).toHaveURL("/");
+});
+
+test("registration", async ({ page }) => {
+  test.slow();
+  test.fixme();
+  // Register a new user
+  await page.goto("/home");
+  await expect(page).toHaveURL("/");
+  await page.click("button[class$='btn-wide']");
+  await page.fill(
+    "html > body > div:nth-of-type(1) > div > div:nth-of-type(2) > div:nth-of-type(2) > div > div > form > div > div > div:nth-of-type(1) > input",
+    "test@example.com"
+  );
+  await page.fill(
+    "html > body > div:nth-of-type(1) > div > div:nth-of-type(2) > div:nth-of-type(2) > div > div > form > div > div > div:nth-of-type(2) > input",
+    "Test User"
+  );
+  await page.fill("input[placeholder='Password']", "ThisIsAStrongDemoPass");
+  await page.click("button[class$='mt-2']");
+  await expect(page).toHaveURL("/");
+
+  // Try to register the same user again (it should fail)
+  await page.goto("/home");
+  await expect(page).toHaveURL("/");
+  await page.click("button[class$='btn-wide']");
+  await page.fill(
+    "html > body > div:nth-of-type(1) > div > div:nth-of-type(2) > div:nth-of-type(2) > div > div > form > div > div > div:nth-of-type(1) > input",
+    "test@example.com"
+  );
+  await page.fill(
+    "html > body > div:nth-of-type(1) > div > div:nth-of-type(2) > div:nth-of-type(2) > div > div > form > div > div > div:nth-of-type(2) > input",
+    "Test User"
+  );
+  await page.fill("input[placeholder='Password']", "ThisIsAStrongDemoPass");
+  await page.click("button[class$='mt-2']");
+  await expect(page).toHaveURL("/");
+  await expect(page.locator("div[class*='top-2']")).toHaveText("Problem registering user");
+});

frontend/test/playwright.config.ts

@@ -0,0 +1,29 @@
+import { defineConfig, devices } from "@playwright/test";
+
+export default defineConfig({
+  testDir: "./e2e",
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 1,
+  reporter: process.env.CI ? "blob" : "html",
+  use: {
+    baseURL: process.env.E2E_BASE_URL || "http://localhost:3000",
+    trace: "on-all-retries",
+    video: "retry-with-video",
+  },
+  projects: [
+    {
+      name: "chromium",
+      use: { ...devices["Desktop Chrome"] },
+    },
+    {
+      name: "firefox",
+      use: { ...devices["Desktop Firefox"] },
+    },
+    {
+      name: "webkit",
+      use: { ...devices["Desktop Safari"] },
+    },
+  ],
+  globalTeardown: require.resolve("./playwright.teardown"),
+});

frontend/test/playwright.teardown.ts

@@ -0,0 +1,16 @@
+import { exec } from "child_process";
+
+function globalTeardown() {
+  if (process.env.TEST_SHUTDOWN_API_SERVER) {
+    const pc = exec("pkill -SIGTERM api"); // Kill background API process
+    const fr = exec("pkill -SIGTERM task"); // Kill background Frontend process
+    pc.stdout?.on("data", (data: void) => {
+      console.log(`stdout: ${data}`);
+    });
+    fr.stdout?.on("data", (data: void) => {
+      console.log(`stdout: ${data}`);
+    });
+  }
+}
+
+export default globalTeardown;

frontend/test/vitest.config.ts

@@ -5,6 +5,7 @@ export default defineConfig({
   // @ts-ignore
   test: {
     globalSetup: "./test/setup.ts",
+    include: ["**/*.test.ts"],
   },
   resolve: {
     alias: {