test(ui): added minimal automated KopiaUI test (#2434)

* test(ui): added minimal automated KopiaUI test

```
$ make kopia-ui-test
```

This currently executes super minimal E2E test on pre-built KopiaUI
using Playwright https://playwright.dev

* better os/arch detection

* remove unwanted log

* fixed executable path on linux

* fix for linux, misc

Makefile

@@ -118,6 +118,15 @@ website:
 kopia-ui: $(kopia_ui_embedded_exe)
 	$(MAKE) -C app build-electron
 
+MAYBE_XVFB=
+ifeq ($(GOOS),linux)
+# on Linux
+MAYBE_XVFB=xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" --
+endif
+
+kopia-ui-test:
+	$(MAYBE_XVFB) $(MAKE) -C app e2e-test
+
 # use this to test htmlui changes in full build of KopiaUI, this is rarely needed
 # except when testing htmlui specific features that only light up when running under Electron.
 #
@@ -188,6 +197,7 @@ ci-build:
 	$(MAKE) kopia
 ifeq ($(GOARCH),amd64)
 	$(retry) $(MAKE) kopia-ui
+	$(retry) $(MAKE) kopia-ui-test
 endif
 ifeq ($(GOOS)/$(GOARCH),linux/amd64)
 	$(MAKE) generate-change-log

app/Makefile

@@ -68,6 +68,9 @@ dev: node_modules/.up-to-date
 run:
 	$(npm) $(npm_flags) run start-electron-prebuilt
 
+e2e-test:
+	$(npm) $(npm_flags) run e2e
+
 build-electron: ../dist/kopia-ui/.up-to-date
 
 # rebuild packages if HTML, embedded EXE or build config changed.

app/package-lock.json

@@ -18,12 +18,15 @@
         "uuid": "^8.3.2"
       },
       "devDependencies": {
+        "@playwright/test": "^1.26.0",
         "asar": "^3.2.0",
         "concurrently": "^7.3.0",
         "dotenv": "^16.0.2",
         "electron": "^19.0.8",
         "electron-builder": "^23.3.3",
-        "electron-notarize": "^1.2.1"
+        "electron-notarize": "^1.2.1",
+        "playwright": "^1.26.0",
+        "playwright-core": "^1.26.0"
       }
     },
     "node_modules/@develar/schema-utils": {
@@ -192,6 +195,22 @@
         "node": ">= 10.0.0"
       }
     },
+    "node_modules/@playwright/test": {
+      "version": "1.26.0",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.26.0.tgz",
+      "integrity": "sha512-D24pu1k/gQw3Lhbpc38G5bXlBjGDrH5A52MsrH12wz6ohGDeQ+aZg/JFSEsT/B3G8zlJe/EU4EkJK74hpqsjEg==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*",
+        "playwright-core": "1.26.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=14"
+      }
+    },
     "node_modules/@sindresorhus/is": {
       "version": "0.14.0",
       "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz",
@@ -2785,6 +2804,34 @@
         "node": ">=4"
       }
     },
+    "node_modules/playwright": {
+      "version": "1.26.0",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.26.0.tgz",
+      "integrity": "sha512-XxTVlvFEYHdatxUkh1KiPq9BclNtFKMi3BgQnl/aactmhN4G9AkZUXwt0ck6NDAOrDFlfibhbM7A1kZwQJKSBw==",
+      "dev": true,
+      "hasInstallScript": true,
+      "dependencies": {
+        "playwright-core": "1.26.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=14"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.26.0",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.26.0.tgz",
+      "integrity": "sha512-p8huU8eU4gD3VkJd3DA1nA7R3XA6rFvFL+1RYS96cSljCF2yJE9CWEHTPF4LqX8KN9MoWCrAfVKP5381X3CZqg==",
+      "dev": true,
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=14"
+      }
+    },
     "node_modules/plist": {
       "version": "3.0.6",
       "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.6.tgz",
@@ -3796,6 +3843,16 @@
         }
       }
     },
+    "@playwright/test": {
+      "version": "1.26.0",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.26.0.tgz",
+      "integrity": "sha512-D24pu1k/gQw3Lhbpc38G5bXlBjGDrH5A52MsrH12wz6ohGDeQ+aZg/JFSEsT/B3G8zlJe/EU4EkJK74hpqsjEg==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*",
+        "playwright-core": "1.26.0"
+      }
+    },
     "@sindresorhus/is": {
       "version": "0.14.0",
       "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz",
@@ -5834,6 +5891,21 @@
       "dev": true,
       "optional": true
     },
+    "playwright": {
+      "version": "1.26.0",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.26.0.tgz",
+      "integrity": "sha512-XxTVlvFEYHdatxUkh1KiPq9BclNtFKMi3BgQnl/aactmhN4G9AkZUXwt0ck6NDAOrDFlfibhbM7A1kZwQJKSBw==",
+      "dev": true,
+      "requires": {
+        "playwright-core": "1.26.0"
+      }
+    },
+    "playwright-core": {
+      "version": "1.26.0",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.26.0.tgz",
+      "integrity": "sha512-p8huU8eU4gD3VkJd3DA1nA7R3XA6rFvFL+1RYS96cSljCF2yJE9CWEHTPF4LqX8KN9MoWCrAfVKP5381X3CZqg==",
+      "dev": true
+    },
     "plist": {
       "version": "3.0.6",
       "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.6.tgz",

app/package.json

@@ -110,12 +110,15 @@
     "afterSign": "notarize.js"
   },
   "devDependencies": {
+    "@playwright/test": "^1.26.0",
     "asar": "^3.2.0",
     "concurrently": "^7.3.0",
     "dotenv": "^16.0.2",
     "electron": "^19.0.8",
     "electron-builder": "^23.3.3",
-    "electron-notarize": "^1.2.1"
+    "electron-notarize": "^1.2.1",
+    "playwright": "^1.26.0",
+    "playwright-core": "^1.26.0"
   },
   "homepage": "./",
   "description": "Fast and secure open source backup.",
@@ -124,7 +127,7 @@
   "scripts": {
     "start": "react-scripts start",
     "build-html": "react-scripts build",
-    "test": "react-scripts test",
+    "e2e": "playwright test",
     "eject": "react-scripts eject",
     "start-electron": "electron .",
     "build-electron": "electron-builder",

app/public/electron.js

@@ -273,6 +273,10 @@ function isOutsideOfApplicationsFolderOnMac() {
 }
 
 function maybeMoveToApplicationsFolder() {
+  if (process.env["KOPIA_UI_TESTING"]) {
+    return;
+  }
+
   dialog.showMessageBox({
     buttons: ["Yes", "No"],
     message: "For best experience, Kopia needs to be installed in Applications folder.\n\nDo you want to move it now?"
@@ -337,6 +341,14 @@ app.on('ready', () => {
 
   tray.setToolTip('Kopia');
 
+  // hooks exposed to tests
+  if (process.env["KOPIA_UI_TESTING"]) {
+    app.testHooks = {
+      tray: tray,
+      showRepoWindow: showRepoWindow,
+    }
+  }
+
   safeTrayHandler("click", () => tray.popUpContextMenu());
   safeTrayHandler("right-click", () => tray.popUpContextMenu());
   safeTrayHandler("double-click", () => showAllRepoWindows());

app/public/preload.js

@@ -1,7 +1,5 @@
 const { contextBridge, shell, ipcRenderer } = require("electron");
 
-console.log('preloading...', contextBridge, shell);
-
 contextBridge.exposeInMainWorld("kopiaUI", {
     "selectDirectory": function (onSelected) {
         ipcRenderer.invoke('select-dir').then(v => {

app/tests/main.spec.js

@@ -0,0 +1,99 @@
+import { test, expect } from '@playwright/test'
+import { _electron as electron } from 'playwright'
+
+import path from 'path';
+
+let electronApp
+
+function getKopiaUIUnpackedDir() {
+  switch (process.platform + "/" + process.arch) {
+    case "darwin/x64":
+      return path.resolve("../dist/kopia-ui/mac");
+    case "darwin/arm64":
+      return path.resolve("../dist/kopia-ui/mac-arm64");
+    case "linux/x64":
+      return path.resolve("../dist/kopia-ui/linux-unpacked");
+    case "linux/arm64":
+      return path.resolve("../dist/kopia-ui/linux-arm64-unpacked");
+    case "win32/x64":
+      return path.resolve("../dist/kopia-ui/win-unpacked");
+    default:
+      return null;
+  }
+}
+ 
+function getMainPath(unpackedDir) {
+  switch (process.platform) {
+    case "darwin":
+      return path.join(unpackedDir, "KopiaUI.app", "Contents", "Resources", "app.asar", "public", "electron.js");
+    default:
+      return path.join(unpackedDir, "resources", "app.asar", "public", "electron.js");
+  }
+}
+
+function getExecutablePath(unpackedDir) {
+  switch (process.platform) {
+    case "win32":
+      return path.join(unpackedDir, "KopiaUI.exe");
+    case "darwin":
+      return path.join(unpackedDir, "KopiaUI.app", "Contents", "MacOS", "KopiaUI");
+    default:
+      return path.join(unpackedDir, "kopia-ui");
+  }
+}
+
+test.beforeAll(async () => {
+  const unpackedDir = getKopiaUIUnpackedDir();
+  expect(unpackedDir).not.toBeNull();
+
+  const mainPath = getMainPath(unpackedDir);
+  const executablePath = getExecutablePath(unpackedDir);
+
+  console.log('main path', mainPath);
+  console.log('executable path', executablePath);
+
+  process.env.CI = 'e2e'
+  process.env.KOPIA_UI_TESTING = '1'
+  electronApp = await electron.launch({
+    args: [mainPath],
+    executablePath: executablePath,
+  })
+  electronApp.on('window', async (page) => {
+    const filename = page.url()?.split('/').pop()
+    console.log(`Window opened: ${filename}`)
+
+    // capture errors
+    page.on('pageerror', (error) => {
+      console.error(error)
+    })
+    // capture console messages
+    page.on('console', (msg) => {
+      console.log(msg.text())
+    })
+  })
+})
+ 
+test.afterAll(async () => {
+  await electronApp.close()
+})
+ 
+test('opens repository window', async () => {
+  await electronApp.evaluate(async ({app}) => {
+    app.testHooks.showRepoWindow('repository');
+  });
+
+  const page = await electronApp.firstWindow();
+
+  expect(page).toBeTruthy();
+  expect(await page.title()).toMatch(/KopiaUI v\d+/);
+
+  // TODO - we can exercise some UI scenario using 'page'
+
+  await electronApp.evaluate(async ({app}) => {
+    return app.testHooks.tray.popUpContextMenu();
+  })
+
+  await electronApp.evaluate(async ({app}) => {
+    return app.testHooks.tray.closeContextMenu();
+  })
+});