tor-browser

devtools-node-test-runner.js (9622B)

---

/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at <http://mozilla.org/MPL/2.0/>. */

/* global __dirname, process */

"use strict";

/**
* This is a test runner dedicated to run DevTools node tests continuous integration
* platforms. It will parse the logs to output errors compliant with treeherder tooling.
*
* See taskcluster/kinds/source-test/node.yml for the definition of the task running those
* tests on try.
*/

const { execFileSync } = require("child_process");
const { writeFileSync } = require("fs");
const { chdir } = require("process");
const path = require("path");
const os = require("os");

const REPOSITORY_ROOT = __dirname.replace("devtools/client/bin", "");

// All Windows platforms report "win32", even for 64bit editions.
const isWin = os.platform() === "win32";

// On Windows, the ".cmd" suffix is mandatory to invoke yarn ; or executables in
// general.
const YARN_PROCESS = isWin ? "yarn.cmd" : "yarn";

// Supported node test suites for DevTools
const TEST_TYPES = {
 JEST: "jest",
 TYPESCRIPT: "typescript",
};

const SUITES = {
 aboutdebugging: {
   path: "../aboutdebugging/test/node",
   type: TEST_TYPES.JEST,
 },
 accessibility: {
   path: "../accessibility/test/node",
   type: TEST_TYPES.JEST,
 },
 application: {
   path: "../application/test/node",
   type: TEST_TYPES.JEST,
 },
 compatibility: {
   path: "../inspector/compatibility/test/node",
   type: TEST_TYPES.JEST,
 },
 debugger: {
   path: "../debugger",
   type: TEST_TYPES.JEST,
 },
 framework: {
   path: "../framework/test/node",
   type: TEST_TYPES.JEST,
 },
 netmonitor: {
   path: "../netmonitor/test/node",
   type: TEST_TYPES.JEST,
 },
 performance: {
   path: "../performance-new",
   type: TEST_TYPES.TYPESCRIPT,
 },
 shared_components: {
   path: "../shared/components/test/node",
   type: TEST_TYPES.JEST,
 },
 webconsole: {
   path: "../webconsole/test/node",
   type: TEST_TYPES.JEST,
   dependencies: ["../debugger"],
 },
};

function execOut(...args) {
 let out;
 let err;
 try {
   out = execFileSync(...args);
 } catch (e) {
   out = e.stdout;
   err = e.stderr;
 }
 return { out: out.toString(), err: err && err.toString() };
}

function getErrors(suite, out, err, testPath) {
 switch (SUITES[suite].type) {
   case TEST_TYPES.JEST:
     return getJestErrors(out, err);
   case TEST_TYPES.TYPESCRIPT:
     return getTypescriptErrors(out, err, testPath);
   default:
     throw new Error("Unsupported suite type: " + SUITES[suite].type);
 }
}

const JEST_ERROR_SUMMARY_REGEX = /\s●\s/;

function getJestErrors(out) {
 // The string out has extra content before the JSON object starts.
 const jestJsonOut = out.substring(out.indexOf("{"), out.lastIndexOf("}") + 1);
 const results = JSON.parse(jestJsonOut);

 /**
  * We don't have individual information, but a multiple line string in testResult.message,
  * which looks like
  *
  *  ● Simple function
  *
  *        expect(received).toEqual(expected) // deep equality
  *
  *        Expected: false
  *        Received: true
  *
  *          391 |       url: "test.js",
  *          392 |     });
  *        > 393 |     expect(true).toEqual(false);
  *              |                  ^
  *          394 |     expect(actual.code).toMatchSnapshot();
  *          395 |
  *          396 |     const smc = await new SourceMapConsumer(actual.map.toJSON());
  *
  *          at Object.<anonymous> (src/workers/pretty-print/tests/prettyFast.spec.js:393:18)
  *          at asyncGeneratorStep (src/workers/pretty-print/tests/prettyFast.spec.js:7:103)
  *          at _next (src/workers/pretty-print/tests/prettyFast.spec.js:9:194)
  *          at src/workers/pretty-print/tests/prettyFast.spec.js:9:364
  *          at Object.<anonymous> (src/workers/pretty-print/tests/prettyFast.spec.js:9:97)
  *
  */

 const errors = [];
 for (const testResult of results.testResults) {
   if (testResult.status != "failed") {
     continue;
   }
   let currentError;
   let errorLine;

   const lines = testResult.message.split("\n");
   lines.forEach((line, i) => {
     if (line.match(JEST_ERROR_SUMMARY_REGEX) || i == lines.length - 1) {
       // This is the name of the test, if we were gathering information from a previous
       // error, we add it to the errors
       if (currentError) {
         errors.push({
           // The file should be relative from the repository
           file: testResult.name.replace(REPOSITORY_ROOT, ""),
           line: errorLine,
           // we don't have information for the column
           column: 0,
           message: currentError.trim(),
         });
       }

       // Handle the new error
       currentError = line;
     } else {
       // We put any line that is not a test name in the error message as it may be
       // valuable for the user.
       currentError += "\n" + line;

       // The actual line of the error is marked with " > XXX |"
       const res = line.match(/> (?<line>\d+) \|/);
       if (res) {
         errorLine = parseInt(res.groups.line, 10);
       }
     }
   });
 }

 return errors;
}

function getTypescriptErrors(out, err, testPath) {
 console.log(out);
 // Typescript error lines look like:
 //   popup/panel.jsm.js(103,7): error TS2531: Object is possibly 'null'.
 // Which means:
 //   {file_path}({line},{col}): error TS{error_code}: {message}
 const tsErrorRegex =
   /(?<file>(\w|\/|\.)+)\((?<line>\d+),(?<column>\d+)\): (?<message>error TS\d+\:.*)/;
 const errors = [];
 for (const line of out.split("\n")) {
   const res = line.match(tsErrorRegex);
   if (!res) {
     continue;
   }
   // TypeScript gives us the path from the directory the command is executed in, so we
   // need to prepend the directory path.
   const fileAbsPath = testPath + res.groups.file;
   errors.push({
     // The file should be relative from the repository.
     file: fileAbsPath.replace(REPOSITORY_ROOT, ""),
     line: parseInt(res.groups.line, 10),
     column: parseInt(res.groups.column, 10),
     message: res.groups.message.trim(),
   });
 }
 return errors;
}

function runTests() {
 console.log("[devtools-node-test] Extract suite argument");
 const suiteArg = process.argv.find(arg => arg.includes("suite="));
 const suite = suiteArg.split("=")[1];
 if (suite !== "all" && !SUITES[suite]) {
   throw new Error("Invalid suite argument to devtools-node-test: " + suite);
 }

 console.log("[devtools-node-test] Check `yarn` is available");
 try {
   // This will throw if yarn is unavailable
   execFileSync(YARN_PROCESS, ["--version"]);
 } catch (e) {
   console.log(
     "[devtools-node-test] ERROR: `yarn` is not installed. " +
       "See https://yarnpkg.com/docs/install/ "
   );
   return false;
 }

 const artifactArg = process.argv.find(arg => arg.includes("artifact="));
 const artifactFilePath = artifactArg && artifactArg.split("=")[1];
 const artifactErrors = {};

 const failedSuites = [];
 const suites = suite == "all" ? SUITES : { [suite]: SUITES[suite] };
 for (const [suiteName, suiteData] of Object.entries(suites)) {
   console.log("[devtools-node-test] Running suite: " + suiteName);

   if (suiteData.dependencies) {
     console.log("[devtools-node-test] Running `yarn` for dependencies");
     for (const dep of suiteData.dependencies) {
       const depPath = path.join(__dirname, dep);
       chdir(depPath);

       console.log("[devtools-node-test] Run `yarn` in " + depPath);
       execOut(YARN_PROCESS);
     }
   }

   const testPath = path.join(__dirname, suiteData.path);
   chdir(testPath);

   console.log("[devtools-node-test] Run `yarn` in test folder");
   execOut(YARN_PROCESS);

   console.log(`TEST START | ${suiteData.type} | ${suiteName}`);

   console.log("[devtools-node-test] Run `yarn test` in test folder");
   const { out, err } = execOut(YARN_PROCESS, ["test-ci"]);

   if (err) {
     console.log("[devtools-node-test] Error log");
     console.log(err);
   }

   console.log("[devtools-node-test] Parse errors from the test logs");
   const errors = getErrors(suiteName, out, err, testPath) || [];
   if (errors.length) {
     failedSuites.push(suiteName);
   }
   for (const error of errors) {
     if (!artifactErrors[error.file]) {
       artifactErrors[error.file] = [];
     }
     artifactErrors[error.file].push({
       path: error.file,
       line: error.line,
       column: error.column,
       level: "error",
       message: error.message,
       analyzer: suiteName,
     });

     console.log(
       `TEST-UNEXPECTED-FAIL | ${suiteData.type} | ${suiteName} | ${error.file}:${error.line}: ${error.message}`
     );
   }
 }

 if (artifactFilePath) {
   console.log(`[devtools-node-test] Writing artifact to ${artifactFilePath}`);
   writeFileSync(artifactFilePath, JSON.stringify(artifactErrors, null, 2));
 }

 const success = failedSuites.length === 0;
 if (success) {
   console.log(
     `[devtools-node-test] Test suites [${Object.keys(suites).join(
       ", "
     )}] succeeded`
   );
 } else {
   console.log(
     `[devtools-node-test] Test suites [${failedSuites.join(", ")}] failed`
   );
   console.log(
     "TEST-UNEXPECTED-FAIL | mach devtools-node-test failed. Documentation " +
       "at https://firefox-source-docs.mozilla.org/devtools/tests/node-tests.html"
   );
 }
 return success;
}

process.exitCode = runTests() ? 0 : 1;