feat: after-response cli (#7931)

* wip * add after response example and wiring * add tests and tap output * check error case * add reporters * test reporters * only list tests with results * add failing test * rename config function * add test title
2026-04-23 15:49:42 -04:00 · 2024-09-12 11:30:24 +02:00
parent 7021a0b037
commit deccf97204
10 changed files with 215 additions and 36 deletions
--- a/packages/insomnia-inso/src/cli.test.ts
+++ b/packages/insomnia-inso/src/cli.test.ts
@@ -39,6 +39,8 @@ const shouldReturnSuccessCode = [
  '$PWD/packages/insomnia-inso/bin/inso run collection -w packages/insomnia-smoke-test/fixtures/simple.yaml -e env_2eecf85b7f wrk_0702a5',
  // with regex filter
  '$PWD/packages/insomnia-inso/bin/inso run collection -w packages/insomnia-smoke-test/fixtures/simple.yaml -e env_2eecf85b7f --requestNamePattern "example http" wrk_0702a5',
+  // after-response script and test
+  '$PWD/packages/insomnia-inso/bin/inso run collection -w packages/insomnia-inso/src/examples/after-response.yml wrk_616795 --verbose',
 ];

 const shouldReturnErrorCode = [
@@ -46,6 +48,8 @@ const shouldReturnErrorCode = [
  '$PWD/packages/insomnia-inso/bin/inso run test -w packages/insomnia-inso/src/db/fixtures/git-repo -e env_env_ca046a uts_7f0f85',
  '$PWD/packages/insomnia-inso/bin/inso lint spec -w packages/insomnia-inso/src/db/fixtures/git-repo-malformed-spec spc_46c5a4',
  '$PWD/packages/insomnia-inso/bin/inso lint spec packages/insomnia-inso/src/db/fixtures/insomnia-v4/malformed.yaml',
+  // after-response script and test
+  '$PWD/packages/insomnia-inso/bin/inso run collection -w packages/insomnia-inso/src/examples/after-response-failed-test.yml wrk_616795 --verbose',
 ];

 describe('inso dev bundle', () => {
--- a/packages/insomnia-inso/src/cli.ts
+++ b/packages/insomnia-inso/src/cli.ts
@@ -7,6 +7,7 @@ import consola, { BasicReporter, FancyReporter, LogLevel, logType } from 'consol
 import { cosmiconfig } from 'cosmiconfig';
 import fs from 'fs';
 import { getSendRequestCallbackMemDb } from 'insomnia/src/common/send-request';
+import type { RequestTestResult } from 'insomnia-sdk';
 import { generate, runTestsCli } from 'insomnia-testing';
 import { parseArgsStringToArgv } from 'string-argv';

@@ -28,11 +29,10 @@ export interface GlobalOptions {
  workingDir: string;
 };

-export type TestReporter = 'dot' | 'list' | 'spec' | 'min' | 'progress';
-export const reporterTypes: TestReporter[] = ['dot', 'list', 'min', 'progress', 'spec'];
-export const reporterTypesSet = new Set(reporterTypes);
+export type TestReporter = 'dot' | 'list' | 'spec' | 'min' | 'progress' | 'tap';
+export const reporterTypes: TestReporter[] = ['dot', 'list', 'min', 'progress', 'spec', 'tap'];

-export const loadCosmiConfig = async (configFile?: string, workingDir?: string) => {
+export const tryToReadInsoConfigFile = async (configFile?: string, workingDir?: string) => {
  try {
    const explorer = await cosmiconfig('inso');
    // set or detect .insorc in workingDir or cwd https://github.com/cosmiconfig/cosmiconfig?tab=readme-ov-file#explorersearch
@@ -164,7 +164,40 @@ const resolveSpecInDatabase = async (identifier: string, options: GlobalOptions)
 };

 const localAppDir = getAppDataDir(getDefaultProductName());
+const logTestResult = (reporter: TestReporter, testResults?: RequestTestResult[]) => {
+  if (!testResults || testResults.length === 0) {
+    return '';
+  }
+  const fallbackReporter = testResults.map(r => `${r.status === 'passed' ? '✅' : '❌'} ${r.testCase}`).join('\n');

+  const reporterMap = {
+    dot: testResults.map(r => r.status === 'passed' ? '.' : 'F').join(''),
+    list: fallbackReporter,
+    min: ' ',
+    progress: `[${testResults.map(r => r.status === 'passed' ? '-' : 'x').join('')}]`,
+    spec: fallbackReporter,
+    tap: convertToTAP(testResults),
+  };
+
+  return `${reporterMap[reporter] || fallbackReporter}
+
+Total tests: ${testResults.length}
+Passed: ${testResults.filter(r => r.status === 'passed').length}
+Failed: ${testResults.filter(r => r.status === 'failed').length}`;
+};
+function convertToTAP(testCases: RequestTestResult[]): string {
+  let tapOutput = 'TAP version 13\n';
+  const totalTests = testCases.length;
+  // Add the number of test cases
+  tapOutput += `1..${totalTests}\n`;
+  // Iterate through each test case and format it in TAP
+  testCases.forEach((test, index) => {
+    const testNumber = index + 1;
+    const testStatus = test.status === 'passed' ? 'ok' : 'not ok';
+    tapOutput += `${testStatus} ${testNumber} - ${test.testCase}\n`;
+  });
+  return tapOutput;
+}
 export const go = (args?: string[]) => {

  const program = new commander.Command();
@@ -213,22 +246,18 @@ export const go = (args?: string[]) => {
    .description('Run Insomnia unit test suites, identifier can be a test suite id or a API Spec id')
    .option('-e, --env <identifier>', 'environment to use', '')
    .option('-t, --testNamePattern <regex>', 'run tests that match the regex', '')
-    .option(
-      '-r, --reporter <reporter>',
-      `reporter to use, options are [${reporterTypes.join(', ')}] (default: ${defaultReporter})`, defaultReporter
-    )
+    .option('-r, --reporter <reporter>', `reporter to use, options are [${reporterTypes.join(', ')}] (default: ${defaultReporter})`, defaultReporter)
    .option('-b, --bail', 'abort ("bail") after first test failure', false)
    .option('--keepFile', 'do not delete the generated test file', false)
    .option('--disableCertValidation', 'disable certificate validation for requests with SSL', false)
    .action(async (identifier, cmd: { env: string; testNamePattern: string; reporter: TestReporter; bail: true; keepFile: true; disableCertValidation: true }) => {
      const globals: GlobalOptions = program.optsWithGlobals();
      const commandOptions = { ...globals, ...cmd };
-      const __configFile = await loadCosmiConfig(commandOptions.config, commandOptions.workingDir);
+      const __configFile = await tryToReadInsoConfigFile(commandOptions.config, commandOptions.workingDir);

      const options = {
        ...__configFile?.options || {},
        ...commandOptions,
-        ...(__configFile ? { __configFile } : {}),
      };
      logger.level = options.verbose ? LogLevel.Verbose : LogLevel.Info;
      options.ci && logger.setReporters([new BasicReporter()]);
@@ -241,7 +270,7 @@ export const go = (args?: string[]) => {
      } else {
        pathToSearch = path.resolve(options.workingDir || process.cwd(), options.exportFile || '');
      }
-      if (options.reporter && !reporterTypesSet.has(options.reporter)) {
+      if (options.reporter && !reporterTypes.find(r => r === options.reporter)) {
        logger.fatal(`Reporter "${options.reporter}" not unrecognized. Options are [${reporterTypes.join(', ')}].`);
        return process.exit(1);
      }
@@ -300,19 +329,19 @@ export const go = (args?: string[]) => {
    .description('Run Insomnia request collection, identifier can be a workspace id')
    .option('-t, --requestNamePattern <regex>', 'run requests that match the regex', '')
    .option('-e, --env <identifier>', 'environment to use', '')
+    .option('-r, --reporter <reporter>', `reporter to use, options are [${reporterTypes.join(', ')}] (default: ${defaultReporter})`, defaultReporter)
    .option('-b, --bail', 'abort ("bail") after first non-200 response', false)
    .option('--disableCertValidation', 'disable certificate validation for requests with SSL', false)
    .action(async (identifier, cmd: { env: string; disableCertValidation: true; requestNamePattern: string; bail: boolean }) => {
      const globals: { config: string; workingDir: string; exportFile: string; ci: boolean; printOptions: boolean; verbose: boolean } = program.optsWithGlobals();

      const commandOptions = { ...globals, ...cmd };
-      const __configFile = await loadCosmiConfig(commandOptions.config, commandOptions.workingDir);
+      const __configFile = await tryToReadInsoConfigFile(commandOptions.config, commandOptions.workingDir);

      const options = {
        reporter: defaultReporter,
        ...__configFile?.options || {},
        ...commandOptions,
-        ...(__configFile ? { __configFile } : {}),
      };
      logger.level = options.verbose ? LogLevel.Verbose : LogLevel.Info;
      options.ci && logger.setReporters([new BasicReporter()]);
@@ -325,10 +354,6 @@ export const go = (args?: string[]) => {
      } else {
        pathToSearch = path.resolve(options.workingDir || process.cwd(), options.exportFile || '');
      }
-      if (options.reporter && !reporterTypesSet.has(options.reporter)) {
-        logger.fatal(`Reporter "${options.reporter}" not unrecognized. Options are [${reporterTypes.join(', ')}].`);
-        return process.exit(1);
-      }

      const db = await loadDb({
        pathToSearch,
@@ -384,6 +409,16 @@ export const go = (args?: string[]) => {
          const timelineString = await readFile(res.timelinePath, 'utf8');
          const timeline = timelineString.split('\n').filter(e => e?.trim()).map(e => JSON.parse(e).value).join(' ');
          logger.trace(timeline);
+          if (res.testResults?.length) {
+            console.log(`
+Test results:`);
+            console.log(logTestResult(options.reporter, res.testResults));
+            const hasFailedTests = res.testResults.some(t => t.status === 'failed');
+            if (hasFailedTests) {
+              success = false;
+            }
+          }
+
          if (res.status !== 200) {
            success = false;
            logger.error(`Request failed with status ${res.status}`);
@@ -403,11 +438,10 @@ export const go = (args?: string[]) => {
    .action(async identifier => {
      const globals: GlobalOptions = program.optsWithGlobals();
      const commandOptions = globals;
-      const __configFile = await loadCosmiConfig(commandOptions.config, commandOptions.workingDir);
+      const __configFile = await tryToReadInsoConfigFile(commandOptions.config, commandOptions.workingDir);
      const options = {
        ...__configFile?.options || {},
        ...commandOptions,
-        ...(__configFile ? { __configFile } : {}),
      };
      logger.level = options.verbose ? LogLevel.Verbose : LogLevel.Info;
      options.ci && logger.setReporters([new BasicReporter()]);
@@ -460,11 +494,10 @@ export const go = (args?: string[]) => {
    .action(async (identifier, cmd: { output: string; skipAnnotations: boolean }) => {
      const globals: GlobalOptions = program.optsWithGlobals();
      const commandOptions = { ...globals, ...cmd };
-      const __configFile = await loadCosmiConfig(commandOptions.config, commandOptions.workingDir);
+      const __configFile = await tryToReadInsoConfigFile(commandOptions.config, commandOptions.workingDir);
      const options = {
        ...__configFile?.options || {},
        ...commandOptions,
-        ...(__configFile ? { __configFile } : {}),
      };
      options.printOptions && logger.log('Loaded options', options, '\n');
      let specContent = '';
@@ -499,12 +532,11 @@ export const go = (args?: string[]) => {
    .action(async (scriptName: string, cmd) => {
      const commandOptions = { ...program.optsWithGlobals(), ...cmd };
      // TODO: getAbsolutePath to working directory and use it to check from config file
-      const __configFile = await loadCosmiConfig(commandOptions.config, commandOptions.workingDir);
+      const __configFile = await tryToReadInsoConfigFile(commandOptions.config, commandOptions.workingDir);

      const options = {
        ...__configFile?.options || {},
        ...commandOptions,
-        ...(__configFile ? { __configFile } : {}),
      };
      logger.level = options.verbose ? LogLevel.Verbose : LogLevel.Info;
      options.ci && logger.setReporters([new BasicReporter()]);
--- a/packages/insomnia-inso/src/examples/after-response-failed-test.yml
+++ b/packages/insomnia-inso/src/examples/after-response-failed-test.yml
@@ -0,0 +1,68 @@
+_type: export
+__export_format: 4
+__export_date: 2024-09-09T13:31:03.526Z
+__export_source: insomnia.desktop.app:v10.0.0-beta.2
+resources:
+  - _id: req_65f6ea71a1634669b5ddb74528145de7
+    parentId: wrk_6167950ae1354e41988471465940c1b5
+    modified: 1725888648103
+    created: 1725888487017
+    url: localhost:4010/echo
+    name: New Request
+    description: ""
+    method: GET
+    body: {}
+    parameters: []
+    headers:
+      - name: User-Agent
+        value: insomnia/10.0.0-beta.2
+    authentication: {}
+    metaSortKey: -1725888487017
+    isPrivate: false
+    pathParameters: []
+    afterResponseScript: |+
+      insomnia.test('Check if status is 200', () => {
+          insomnia.expect(insomnia.response.code).to.eql(200);
+          insomnia.expect(JSON.parse(insomnia.response.body).headers.host)
+      			.to.eql('localhost:4010');
+      });
+      insomnia.test('Check if true still true', () => {
+          insomnia.expect(true).to.eql(true);
+      });
+      insomnia.test('Check if true is falsy', () => {
+          insomnia.expect(true).to.eql(!!0);
+      });
+
+    settingStoreCookies: true
+    settingSendCookies: true
+    settingDisableRenderRequestBody: false
+    settingEncodeUrl: true
+    settingRebuildPath: true
+    settingFollowRedirects: global
+    _type: request
+  - _id: wrk_6167950ae1354e41988471465940c1b5
+    parentId: null
+    modified: 1725888480476
+    created: 1725888480476
+    name: after response test
+    description: ""
+    scope: collection
+    _type: workspace
+  - _id: env_af8d61e3630269b8a124b18968608f85797d57fc
+    parentId: wrk_6167950ae1354e41988471465940c1b5
+    modified: 1725888649920
+    created: 1725888480477
+    name: Base Environment
+    data: {}
+    dataPropertyOrder: {}
+    color: null
+    isPrivate: false
+    metaSortKey: 1725888480477
+    _type: environment
+  - _id: jar_af8d61e3630269b8a124b18968608f85797d57fc
+    parentId: wrk_6167950ae1354e41988471465940c1b5
+    modified: 1725888649919
+    created: 1725888480477
+    name: Default Jar
+    cookies: []
+    _type: cookie_jar
--- a/packages/insomnia-inso/src/examples/after-response.yml
+++ b/packages/insomnia-inso/src/examples/after-response.yml
@@ -0,0 +1,62 @@
+_type: export
+__export_format: 4
+__export_date: 2024-09-09T13:31:03.526Z
+__export_source: insomnia.desktop.app:v10.0.0-beta.2
+resources:
+  - _id: req_65f6ea71a1634669b5ddb74528145de7
+    parentId: wrk_6167950ae1354e41988471465940c1b5
+    modified: 1725888648103
+    created: 1725888487017
+    url: localhost:4010/echo
+    name: New Request
+    description: ""
+    method: GET
+    body: {}
+    parameters: []
+    headers:
+      - name: User-Agent
+        value: insomnia/10.0.0-beta.2
+    authentication: {}
+    metaSortKey: -1725888487017
+    isPrivate: false
+    pathParameters: []
+    afterResponseScript: |+
+      insomnia.test('Check if status is 200', () => {
+          insomnia.expect(insomnia.response.code).to.eql(200);
+          insomnia.expect(JSON.parse(insomnia.response.body).headers.host)
+      			.to.eql('localhost:4010');
+      });
+
+    settingStoreCookies: true
+    settingSendCookies: true
+    settingDisableRenderRequestBody: false
+    settingEncodeUrl: true
+    settingRebuildPath: true
+    settingFollowRedirects: global
+    _type: request
+  - _id: wrk_6167950ae1354e41988471465940c1b5
+    parentId: null
+    modified: 1725888480476
+    created: 1725888480476
+    name: after response test
+    description: ""
+    scope: collection
+    _type: workspace
+  - _id: env_af8d61e3630269b8a124b18968608f85797d57fc
+    parentId: wrk_6167950ae1354e41988471465940c1b5
+    modified: 1725888649920
+    created: 1725888480477
+    name: Base Environment
+    data: {}
+    dataPropertyOrder: {}
+    color: null
+    isPrivate: false
+    metaSortKey: 1725888480477
+    _type: environment
+  - _id: jar_af8d61e3630269b8a124b18968608f85797d57fc
+    parentId: wrk_6167950ae1354e41988471465940c1b5
+    modified: 1725888649919
+    created: 1725888480477
+    name: Default Jar
+    cookies: []
+    _type: cookie_jar
--- a/packages/insomnia-inso/src/get-options.test.ts
+++ b/packages/insomnia-inso/src/get-options.test.ts
@@ -1,15 +1,15 @@
 import path from 'path';
 import { describe, expect, it, vi } from 'vitest';

-import { loadCosmiConfig } from './cli';
+import { tryToReadInsoConfigFile } from './cli';

 vi.unmock('cosmiconfig');

 const fixturesDir = path.join('src', 'fixtures');

-describe('loadCosmiConfig()', () => {
+describe('tryToReadInsoConfigFile()', () => {
  it('should load .insorc-test.yaml config file in fixtures dir', async () => {
-    const result = await loadCosmiConfig(path.join(fixturesDir, '.insorc-test.yaml'));
+    const result = await tryToReadInsoConfigFile(path.join(fixturesDir, '.insorc-test.yaml'));
    expect(result).toEqual({
      options: {
      },
@@ -25,19 +25,19 @@ describe('loadCosmiConfig()', () => {
  it('should return empty object and report error if specified config file not found', async () => {
    const consoleLogSpy = vi.spyOn(console, 'log').mockImplementation(() => { });
    const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => { });
-    const result = await loadCosmiConfig('not-found.yaml');
+    const result = await tryToReadInsoConfigFile('not-found.yaml');
    expect(result).toEqual({});
    expect(consoleLogSpy).toHaveBeenCalledWith('Could not find config file at not-found.yaml.');
    expect(consoleErrorSpy).toHaveBeenCalled();
  });

  it('should return empty object if config file is blank', async () => {
-    const result = await loadCosmiConfig(path.join(fixturesDir, '.insorc-blank.yaml'));
+    const result = await tryToReadInsoConfigFile(path.join(fixturesDir, '.insorc-blank.yaml'));
    expect(result).toEqual({});
  });

  it('should return blank properties and ignore extra items if settings and scripts not found in file', async () => {
-    const result = await loadCosmiConfig(path.join(fixturesDir, '.insorc-missing-properties.yaml'));
+    const result = await tryToReadInsoConfigFile(path.join(fixturesDir, '.insorc-missing-properties.yaml'));
    expect(result).toEqual({
      options: {},
      scripts: {},
--- a/packages/insomnia-sdk/src/objects/response.ts
+++ b/packages/insomnia-sdk/src/objects/response.ts
@@ -1,6 +1,7 @@
 import Ajv from 'ajv';
 import deepEqual from 'deep-equal';
 import { RESPONSE_CODE_REASONS } from 'insomnia/src/common/constants';
+import { readCurlResponse } from 'insomnia/src/models/response';
 import type { sendCurlAndWriteTimelineError, sendCurlAndWriteTimelineResponse } from 'insomnia/src/network/network';

 import { Cookie, type CookieOptions } from './cookies';
@@ -317,8 +318,8 @@ export async function readBodyFromPath(response: sendCurlAndWriteTimelineRespons
    } else if (!response.bodyPath) {
        return '';
    }
-
-    const readResponseResult = await window.bridge.readCurlResponse({
+    const nodejsReadCurlResponse = process.type === 'renderer' ? window.bridge.readCurlResponse : readCurlResponse;
+    const readResponseResult = await nodejsReadCurlResponse({
        bodyPath: response.bodyPath,
        bodyCompression: response.bodyCompression,
    });
--- a/packages/insomnia/src/common/send-request.ts
+++ b/packages/insomnia/src/common/send-request.ts
@@ -12,6 +12,7 @@ import {
  getOrInheritHeaders,
  responseTransform,
  sendCurlAndWriteTimeline,
+  tryToExecuteAfterResponseScript,
  tryToExecutePreRequestScript,
  tryToInterpolateRequest,
 } from '../network/network';
@@ -115,13 +116,25 @@ export async function getSendRequestCallbackMemDb(environmentId: string, memDB:
      requestData.responseId
    );
    const res = await responseTransform(response, environmentId, renderedRequest, renderedResult.context);
-
+    const postMutatedContext = await tryToExecuteAfterResponseScript({
+      ...requestData,
+      ...mutatedContext,
+      response,
+    });
+    // TODO: figure out how to handle this error
+    if ('error' in postMutatedContext) {
+      console.error('[network] An error occurred while running after-response script for request named:', renderedRequest.name);
+      throw {
+        error: postMutatedContext.error,
+        response: await responseTransform(response, requestData.activeEnvironmentId, renderedRequest, renderedResult.context),
+      };
+    }
    const { statusCode: status, statusMessage, headers: headerArray, elapsedTime: responseTime } = res;

    const headers = headerArray?.reduce((acc, { name, value }) => ({ ...acc, [name.toLowerCase() || '']: value || '' }), []);
    const bodyBuffer = await getBodyBuffer(res) as Buffer;
    const data = bodyBuffer ? bodyBuffer.toString('utf8') : undefined;

-    return { status, statusMessage, data, headers, responseTime, timelinePath: requestData.timelinePath };
+    return { status, statusMessage, data, headers, responseTime, timelinePath: requestData.timelinePath, testResults: postMutatedContext.requestTestResults };
  };
 }
--- a/packages/insomnia/src/network/network.ts
+++ b/packages/insomnia/src/network/network.ts
@@ -346,8 +346,6 @@ export const tryToExecuteScript = async (context: RequestAndContextAndOptionalRe
    if ('error' in originalOutput) {
      return { error: `Script executor returns error: ${originalOutput.error}` };
    }
-    console.log('[network] script execution succeeded', originalOutput);
-
    const output = originalOutput as RequestContext;

    const envPropertyOrder = orderedJSON.parse(
--- a/packages/insomnia/src/plugins/index.ts
+++ b/packages/insomnia/src/plugins/index.ts
@@ -119,7 +119,7 @@ async function _traversePluginPath(
      continue;
    }
    const folders = (await fs.promises.readdir(p)).filter(f => f.startsWith('insomnia-plugin-'));
-    console.log('[plugin] Loading', folders.map(f => f.replace('insomnia-plugin-', '')).join(', '));
+    folders.length && console.log('[plugin] Loading', folders.map(f => f.replace('insomnia-plugin-', '')).join(', '));
    for (const filename of fs.readdirSync(p)) {
      try {
        const modulePath = path.join(p, filename);
--- a/packages/insomnia/src/scriptExecutor.ts
+++ b/packages/insomnia/src/scriptExecutor.ts
@@ -81,6 +81,7 @@ export const runScript = async (
    clientCertificates: updatedCertificates,
    cookieJar: updatedCookieJar,
    globals: mutatedContextObject.globals,
+    requestTestResults: mutatedContextObject.requestTestResults,
  };
 };