Improve output for QUnit tests in CI:

* Uses fewer lines of output
* Always outputs error at the end even if timeout
* Support for QUNIT_FAIL_FAST ENV var to quit at first failure
This commit is contained in:
Robin Ward 2020-08-05 13:43:14 -04:00
parent 18e3339580
commit 0c8365a1a2

View File

@ -91,8 +91,8 @@ async function runAllTests() {
Runtime.consoleAPICalled(response => {
const message = response["args"][0].value;
// If it's a simple test result, write without newline
if (message === "." || message === "F") {
// Not finished yet, don't add a newline
if (message && message.startsWith && message.startsWith("↪")) {
process.stdout.write(message);
} else if (
message &&
@ -110,7 +110,13 @@ async function runAllTests() {
Page.loadEventFired(async () => {
await Runtime.evaluate({
expression: `(${qunit_script})()`
expression:
`const QUNIT_FAIL_FAST = ` +
(process.env.QUNIT_FAIL_FAST === "true").toString() +
";"
});
await Runtime.evaluate({
expression: `(${qunit_script})();`
});
if (args[0].indexOf("report_requests=1") > -1) {
@ -161,43 +167,47 @@ runAllTests().catch(e => {
// The following functions are converted to strings
// And then sent to chrome to be evalaluated
function logQUnit() {
var moduleErrors = [];
var testErrors = [];
var assertionErrors = [];
let testErrors = [];
let assertionErrors = [];
console.log("\nRunning: " + JSON.stringify(QUnit.urlParams) + "\n");
QUnit.config.testTimeout = 10000;
QUnit.moduleDone(function(context) {
if (context.failed) {
var msg = "Module Failed: " + context.name + "\n" + testErrors.join("\n");
moduleErrors.push(msg);
testErrors = [];
}
});
let durations = {};
let inTest = false;
QUnit.testStart(function(context) {
console.log("\n" + context.module + "::" + context.name);
console.log("↪ " + context.module + "::" + context.name);
inTest = true;
});
QUnit.testDone(function(context) {
durations[context.module + "::" + context.name] = context.runtime;
if (context.failed) {
var msg = " Test Failed: " + context.name + assertionErrors.join(" ");
/* QUNIT_RESULT */
const msg =
" Test Failed: " +
context.name +
assertionErrors.join(" ") +
"\n" +
context.source;
testErrors.push(msg);
assertionErrors = [];
console.log("F");
QUnit.config.queue.length = 0;
// Pass QUNIT_FAIL_FAST on the command line to quit after the first failure
if (QUNIT_FAIL_FAST) {
QUnit.config.queue.length = 0;
}
if (inTest) {
console.log(" [✘]");
}
} else {
console.log(".");
if (inTest) {
console.log(" [✔]");
}
}
inTest = false;
});
QUnit.log(function(context) {
@ -221,10 +231,13 @@ function logQUnit() {
QUnit.done(function(context) {
console.log("\n");
if (moduleErrors.length > 0) {
for (var idx = 0; idx < moduleErrors.length; idx++) {
console.error(moduleErrors[idx] + "\n");
}
if (testErrors.length) {
console.log("Test Errors");
console.log("----------------------------------------------");
testErrors.forEach(e => {
console.error(e);
});
console.log("\n");
}
console.log("Slowest tests");
@ -248,7 +261,11 @@ function logQUnit() {
if (context.failed) {
console.log("\nUse this filter to run in the same order:");
console.log("QUNIT_SEED=" + QUnit.config.seed + " rake qunit:test\n");
console.log(
"QUNIT_FAIL_FAST=true QUNIT_SEED=" +
QUnit.config.seed +
" rake qunit:test\n"
);
console.log("If you have a web environment running, you can visit:");
console.log(
"http://localhost:3000/qunit?hidepassed&seed=" +