/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
const {interfaces: Ci, utils: Cu} = Components;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://gre/modules/Preferences.jsm");
Cu.import("chrome://marionette/content/assert.js");
Cu.import("chrome://marionette/content/capture.js");
const {InvalidArgumentError} =
Cu.import("chrome://marionette/content/error.js", {});
this.EXPORTED_SYMBOLS = ["reftest"];
const XUL_NS = "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul";
const PREF_E10S = "browser.tabs.remote.autostart";
const logger = Log.repository.getLogger("Marionette");
const SCREENSHOT_MODE = {
unexpected: 0,
fail: 1,
always: 2,
};
const STATUS = {
PASS: "PASS",
FAIL: "FAIL",
ERROR: "ERROR",
TIMEOUT: "TIMEOUT",
};
/**
* Implements an fast runner for web-platform-tests format reftests
* c.f. http://web-platform-tests.org/writing-tests/reftests.html.
*
* @namespace
*/
this.reftest = {};
/**
* @memberof reftest
* @class Runner
*/
reftest.Runner = class {
constructor(driver) {
this.driver = driver;
this.canvasCache = new Map([[null, []]]);
this.windowUtils = null;
this.lastURL = null;
this.remote = Preferences.get(PREF_E10S);
}
/**
* Setup the required environment for running reftests.
*
* This will open a non-browser window in which the tests will
* be loaded, and set up various caches for the reftest run.
*
* @param {Object.<Number>} urlCount
* Object holding a map of URL: number of times the URL
* will be opened during the reftest run, where that's
* greater than 1.
* @param {string} screenshotMode
* String enum representing when screenshots should be taken
*/
async setup(urlCount, screenshotMode) {
this.parentWindow = assert.window(this.driver.getCurrentWindow());
this.screenshotMode = SCREENSHOT_MODE[screenshotMode] ||
SCREENSHOT_MODE.unexpected;
this.urlCount = Object.keys(urlCount || {})
.reduce((map, key) => map.set(key, urlCount[key]), new Map());
await this.ensureWindow();
}
async ensureWindow() {
if (this.reftestWin && !this.reftestWin.closed) {
return this.reftestWin;
}
let reftestWin = await this.openWindow();
let found = this.driver.findWindow([reftestWin], () => true);
await this.driver.setWindowHandle(found, true);
this.windowUtils = reftestWin.QueryInterface(Ci.nsIInterfaceRequestor)
.getInterface(Ci.nsIDOMWindowUtils);
this.reftestWin = reftestWin;
return reftestWin;
}
async openWindow() {
let reftestWin;
await new Promise(resolve => {
reftestWin = this.parentWindow.openDialog(
"chrome://marionette/content/reftest.xul",
"reftest",
"chrome,dialog,height=600,width=600,all",
resolve);
});
let browser = reftestWin.document.createElementNS(XUL_NS, "xul:browser");
browser.permanentKey = {};
browser.setAttribute("id", "browser");
browser.setAttribute("anonid", "initialBrowser");
browser.setAttribute("type", "content");
browser.setAttribute("primary", "true");
if (this.remote) {
browser.setAttribute("remote", "true");
browser.setAttribute("remoteType", "web");
}
// Make sure the browser element is exactly 600x600, no matter
// what size our window is
const windowStyle = `padding: 0px; margin: 0px; border:none;
min-width: 600px; min-height: 600px; max-width: 600px; max-height: 600px`;
browser.setAttribute("style", windowStyle);
let doc = reftestWin.document.documentElement;
while (doc.firstChild) {
doc.firstChild.remove();
}
doc.appendChild(browser);
reftestWin.gBrowser = browser;
return reftestWin;
}
abort() {
if (this.reftestWin) {
this.driver.close();
}
this.reftestWin = null;
}
/**
* Run a specific reftest.
*
* The assumed semantics are those of web-platform-tests where
* references form a tree and each test must meet all the conditions
* to reach one leaf node of the tree in order for the overall test
* to pass.
*
* @param {string} testUrl
* URL of the test itself.
* @param {Array.<Array>} references
* Array representing a tree of references to try.
*
* Each item in the array represents a single reference node and
* has the form <code>[referenceUrl, references, relation]</code>,
* where <var>referenceUrl</var> is a string to the URL, relation
* is either <code>==</code> or <code>!=</code> depending on the
* type of reftest, and references is another array containing
* items of the same form, representing further comparisons treated
* as AND with the current item. Sibling entries are treated as OR.
*
* For example with testUrl of T:
*
* <pre><code>
* references = [[A, [[B, [], ==]], ==]]
* Must have T == A AND A == B to pass
*
* references = [[A, [], ==], [B, [], !=]
* Must have T == A OR T != B
*
* references = [[A, [[B, [], ==], [C, [], ==]], ==], [D, [], ]]
* Must have (T == A AND A == B) OR (T == A AND A == C) OR (T == D)
* </code></pre>
*
* @param {string} expected
* Expected test outcome (e.g. <tt>PASS</tt>, <tt>FAIL</tt>).
* @param {number} timeout
* Test timeout in milliseconds.
*
* @return {Object}
* Result object with fields status, message and extra.
*/
async run(testUrl, references, expected, timeout) {
let timeoutHandle;
let timeoutPromise = new Promise(resolve => {
timeoutHandle = this.parentWindow.setTimeout(() => {
resolve({status: STATUS.TIMEOUT, message: null, extra: {}});
}, timeout);
});
let testRunner = (async () => {
let result;
try {
result = await this.runTest(testUrl, references, expected, timeout);
} catch (e) {
result = {status: STATUS.ERROR, message: e.stack, extra: {}};
}
return result;
})();
let result = await Promise.race([testRunner, timeoutPromise]);
this.parentWindow.clearTimeout(timeoutHandle);
if (result.status === STATUS.TIMEOUT) {
this.abort();
}
return result;
}
async runTest(testUrl, references, expected, timeout) {
let win = await this.ensureWindow();
function toBase64(screenshot) {
let dataURL = screenshot.canvas.toDataURL();
return dataURL.split(",")[1];
}
win.innerWidth = 600;
win.innerHeight = 600;
let message = "";
let screenshotData = [];
let stack = [];
for (let i = references.length - 1; i >= 0; i--) {
let item = references[i];
stack.push([testUrl, item[0], item[1], item[2]]);
}
let status = STATUS.FAIL;
while (stack.length) {
let [lhsUrl, rhsUrl, references, relation] = stack.pop();
message += `Testing ${lhsUrl} ${relation} ${rhsUrl}\n`;
let comparison = await this.compareUrls(
win, lhsUrl, rhsUrl, relation, timeout);
function recordScreenshot() {
let encodedLHS = toBase64(comparison.lhs);
let encodedRHS = toBase64(comparison.rhs);
screenshotData.push([{url: lhsUrl, screenshot: encodedLHS},
relation,
{url: rhsUrl, screenshot: encodedRHS}]);
}
if (this.screenshotMode === SCREENSHOT_MODE.always) {
recordScreenshot();
}
if (comparison.passed) {
if (references.length) {
for (let i = references.length - 1; i >= 0; i--) {
let item = references[i];
stack.push([testUrl, item[0], item[1], item[2]]);
}
} else {
// Reached a leaf node so all of one reference chain passed
status = STATUS.PASS;
if (this.screenshotMode <= SCREENSHOT_MODE.fail &&
expected != status) {
recordScreenshot();
}
break;
}
} else if (!stack.length) {
// If we don't have any alternatives to try then this will be
// the last iteration, so save the failing screenshots if required.
let isFail = this.screenshotMode === SCREENSHOT_MODE.fail;
let isUnexpected = this.screenshotMode === SCREENSHOT_MODE.unexpected;
if (isFail || (isUnexpected && expected != status)) {
recordScreenshot();
}
}
// Return any reusable canvases to the pool
let canvasPool = this.canvasCache.get(null);
[comparison.lhs, comparison.rhs].map(screenshot => {
if (screenshot.reuseCanvas) {
canvasPool.push(screenshot.canvas);
}
});
logger.debug(`Canvas pool is of length ${canvasPool.length}`);
}
let result = {status, message, extra: {}};
if (screenshotData.length) {
// For now the tbpl formatter only accepts one screenshot, so just
// return the last one we took.
let lastScreenshot = screenshotData[screenshotData.length - 1];
// eslint-disable-next-line camelcase
result.extra.reftest_screenshots = lastScreenshot;
}
return result;
}
async compareUrls(win, lhsUrl, rhsUrl, relation, timeout) {
logger.info(`Testing ${lhsUrl} ${relation} ${rhsUrl}`);
// Take the reference screenshot first so that if we pause
// we see the test rendering
let rhs = await this.screenshot(win, rhsUrl, timeout);
let lhs = await this.screenshot(win, lhsUrl, timeout);
let maxDifferences = {};
let differences = this.windowUtils.compareCanvases(
lhs.canvas, rhs.canvas, maxDifferences);
let passed;
switch (relation) {
case "==":
passed = differences === 0;
if (!passed) {
logger.info(`Found ${differences} pixels different, ` +
`maximum difference per channel ${maxDifferences.value}`);
}
break;
case "!=":
passed = differences !== 0;
break;
default:
throw new InvalidArgumentError("Reftest operator should be '==' or '!='");
}
return {lhs, rhs, passed};
}
async screenshot(win, url, timeout) {
let canvas = null;
let remainingCount = this.urlCount.get(url) || 1;
let cache = remainingCount > 1;
logger.debug(`screenshot ${url} remainingCount: ` +
`${remainingCount} cache: ${cache}`);
let reuseCanvas = false;
if (this.canvasCache.has(url)) {
logger.debug(`screenshot ${url} taken from cache`);
canvas = this.canvasCache.get(url);
if (!cache) {
this.canvasCache.delete(url);
}
} else {
let canvases = this.canvasCache.get(null);
if (canvases.length) {
canvas = canvases.pop();
} else {
canvas = null;
}
reuseCanvas = !cache;
let ctxInterface = win.CanvasRenderingContext2D;
let flags = ctxInterface.DRAWWINDOW_DRAW_CARET |
ctxInterface.DRAWWINDOW_USE_WIDGET_LAYERS |
ctxInterface.DRAWWINDOW_DRAW_VIEW;
logger.debug(`Starting load of ${url}`);
let navigateOpts = {
commandId: this.driver.listener.activeMessageId,
pageTimeout: timeout,
};
if (this.lastURL === url) {
logger.debug(`Refreshing page`);
await this.driver.listener.refresh(navigateOpts);
} else {
navigateOpts.url = url;
navigateOpts.loadEventExpected = false;
await this.driver.listener.get(navigateOpts);
this.lastURL = url;
}
this.driver.curBrowser.contentBrowser.focus();
await this.driver.listener.reftestWait(url, this.remote);
canvas = capture.canvas(
win,
0, // left
0, // top
win.innerWidth,
win.innerHeight,
{canvas, flags});
}
if (cache) {
this.canvasCache.set(url, canvas);
}
this.urlCount.set(url, remainingCount - 1);
return {canvas, reuseCanvas};
}
};