[INS-2022] Initial exploration multiple test projects (#5244)

* Initial exploration multiple test projects

* Replace duplicate scenario with one to be filled in

* Simplify number of playwright project configs

* Fix testMatch setting on playwright config

* Edit github actions to distinguish between CI and Release runs

.github/workflows/test.yml

@@ -53,8 +53,15 @@ jobs:
           NODE_OPTIONS: '--max_old_space_size=6144'
         run: npm run app-build
 
-      - name: Run app smoke tests
-        run: npm run test:smoke:build
+      - name: Run Smoke tests (CI)
+        # Partial Smoke test run, for regular CI triggers
+        if: "!startsWith(github.head_ref, 'release/')"
+        run: npm run test:build --prefix packages/insomnia-smoke-test -- --project=Smoke
+
+      - name: Run Smoke tests (Release)
+        # Full Smoke test run, for Release PRs
+        if: "startsWith(github.head_ref, 'release/')"
+        run: npm run test:build --prefix packages/insomnia-smoke-test
 
       - name: Set Inso CLI variables
         id: inso-variables

package.json

@@ -29,9 +29,9 @@
     "app-package": "npm run build --prefix packages/insomnia && npm run package --prefix packages/insomnia",
     "app-bump-version": "npm run bump-version --prefix packages/insomnia",
     "preinstall": "node ./scripts/check-version.js",
-    "test:smoke:dev": "npm run test:dev --prefix packages/insomnia-smoke-test",
-    "test:smoke:build": "npm run test:build --prefix packages/insomnia-smoke-test",
-    "test:smoke:package": "npm run test:package --prefix packages/insomnia-smoke-test",
+    "test:smoke:dev": "npm run test:dev --prefix packages/insomnia-smoke-test -- --project=Smoke",
+    "test:smoke:build": "npm run test:build --prefix packages/insomnia-smoke-test -- --project=Smoke",
+    "test:smoke:package": "npm run test:package --prefix packages/insomnia-smoke-test -- --project=Smoke",
     "test:smoke:cli": "npm run test:cli --prefix packages/insomnia-smoke-test",
     "hard-reset": "npm run clean && npm run bootstrap && npm run app-start",
     "type-check": "lerna run type-check",

packages/insomnia-smoke-test/playwright.config.ts

@@ -1,6 +1,20 @@
 /* eslint-disable filenames/match-exported */
 import { PlaywrightTestConfig } from '@playwright/test';
 const config: PlaywrightTestConfig = {
+  projects: [
+    {
+      // Run all tests
+      name: 'Default',
+      testMatch: /.*.test.ts/,
+      retries: 0,
+    },
+    {
+      // High-confidence smoke/sanity checks
+      name: 'Smoke',
+      testMatch: /smoke\/.*.test.ts/,
+      retries: 0,
+    },
+  ],
   webServer: {
     command: 'npm run serve',
     url: 'http://127.0.0.1:4010',

packages/insomnia-smoke-test/tests/prerelease/basic-interactions.test.ts

@@ -0,0 +1,9 @@
+import { expect } from '@playwright/test';
+
+import { test } from '../../playwright/test';
+
+test('Basic interactions', async ({ }) => {
+  // TODO(filfreire): add a few scenarios ported from the release checklist
+
+  await expect(true).toBeTruthy();
+});

packages/insomnia-smoke-test/tests/smoke/app.test.ts

@@ -1,7 +1,7 @@
 import { expect } from '@playwright/test';
 
-import { loadFixture } from '../playwright/paths';
-import { test } from '../playwright/test';
+import { loadFixture } from '../../playwright/paths';
+import { test } from '../../playwright/test';
 
 test('can send requests', async ({ app, page }) => {
   test.slow(process.platform === 'darwin' || process.platform === 'win32', 'Slow app start on these platforms');

packages/insomnia-smoke-test/tests/smoke/git-sync.test.ts

@@ -1,4 +1,4 @@
-import { test } from '../playwright/test';
+import { test } from '../../playwright/test';
 
 test('Sign in with GitHub', async ({ app, page }) => {
   await page.locator('text=Setup Git Sync').click();

packages/insomnia-smoke-test/tests/smoke/graphql.test.ts

@@ -1,7 +1,7 @@
 import { expect } from '@playwright/test';
 
-import { loadFixture } from '../playwright/paths';
-import { test } from '../playwright/test';
+import { loadFixture } from '../../playwright/paths';
+import { test } from '../../playwright/test';
 
 test('can render schema and send GraphQL requests', async ({ app, page }) => {
   test.slow(process.platform === 'darwin' || process.platform === 'win32', 'Slow app start on these platforms');

packages/insomnia-smoke-test/tests/smoke/grpc.test.ts

@@ -1,7 +1,7 @@
 import { expect } from '@playwright/test';
 
-import { loadFixture } from '../playwright/paths';
-import { test } from '../playwright/test';
+import { loadFixture } from '../../playwright/paths';
+import { test } from '../../playwright/test';
 
 test('can send gRPC requests', async ({ app, page }) => {
   test.slow(process.platform === 'darwin' || process.platform === 'win32', 'Slow app start on these platforms');

packages/insomnia-smoke-test/tests/smoke/oauth-gitlab.test.ts

@@ -1,4 +1,4 @@
-import { test } from '../playwright/test';
+import { test } from '../../playwright/test';
 
 test('Sign in with Gitlab', async ({ app, page }) => {
   await page.locator('text=Setup Git Sync').click();

packages/insomnia-smoke-test/tests/smoke/oauth.test.ts

@@ -1,7 +1,7 @@
 import { expect } from '@playwright/test';
 
-import { loadFixture } from '../playwright/paths';
-import { test } from '../playwright/test';
+import { loadFixture } from '../../playwright/paths';
+import { test } from '../../playwright/test';
 
 test('can make oauth2 requests', async ({ app, page }) => {
   if (process.platform === 'darwin') {

packages/insomnia-smoke-test/tests/smoke/openapi.test.ts

@@ -1,6 +1,6 @@
 import { expect } from '@playwright/test';
 
-import { test } from '../playwright/test';
+import { test } from '../../playwright/test';
 
 test('can render Spectral OpenAPI lint errors', async ({ page }) => {
   await page.click('text=Design');

packages/insomnia-smoke-test/tests/smoke/websocket.test.ts

@@ -1,7 +1,7 @@
 import { expect } from '@playwright/test';
 
-import { loadFixture } from '../playwright/paths';
-import { test } from '../playwright/test';
+import { loadFixture } from '../../playwright/paths';
+import { test } from '../../playwright/test';
 
 test('can make websocket connection', async ({ app, page }) => {
   test.slow(process.platform === 'darwin' || process.platform === 'win32', 'Slow app start on these platforms');