Comparison of JavaScript browser automation and test specification libraries

As part of my trial for my current role at Automattic, I was tasked with implementing some e2e acceptance tests using my choice of library/framework/language.

I very much recommend writing automated acceptance tests in the same language as your app, even though I have described some benefits of using a different language, and since WordPress is moving towards JavaScript from PHP, JavaScript seems the most suitable language for Automattic.

Since I had never written acceptance tests in JavaScript (only ruby, Java and C#), I spent some time researching the browser automation/test specification options. I found the choice isn’t as straightforward as other languages were the official Selenium WebDriver bindings are almost ubiquitous (although I still prefer watir-webdriver in ruby).

Interestingly, as at September 2015, I found there’s at lest four browser automation libraries (one official) that allow you to use the Selenium/WebDriver API asynchronously (WebDriverJs, wd, and Nightwatch.js), and at least two more that allow you to use the Selenium/WebDriver API synchronously (webdriver-sync and webdriver-http-sync) – I personally like the synchronous approach (it avoids all the callbacks and/or promises), but I’m possibly biased because my prior experience in ruby/C#/Java has all been with synchronous approaches to browser automation scripts.

Of the non-WebDriver browser automation frameworks: zombie.js and casper.js allow headless only testing; zombie uses its own headless browser whereas Casper supports PhantomJS (WebKit) and Slimer (Gecko). DalekJS automates real and headless browsers but is in a developer preview and warns against production release. The Intern library also automates real and headless browsers in an asynchronous way, but doesn’t look as appealing as other options.

I also found three JavaScript test specification frameworks. Mocha is by far the most popular and uses a “describe/it” specification style, followed by Cucumber.js which uses the “Given/When/Then” specification style, and Testium which uses “describe/it” but tightly couples this to webdriver-http-sync browser automation (which is also used by Groupon).

After a fairly comprehensive shortlist/comparison I selected WebDriverJs (official) and Mocha as the tools of choice for Automattic for various reasons including ease of use with CI and longevity. I’ll probably go into this further in a separate post.

I have included short examples of each library below.

JavaScript Browser Automation Libraries – As at September 2015

Library Browsers Async /
Runtime Node.js Releases Node.js Sep 2015 Installs
WebDriverJS (Official Selenium) Real + PhantomJS Async Node.js 31 537,161
Headless browser testing
Own headless Async IO.js 123 23,754
Casper.js (Beta) PhantomJS & SlimerJS Async Node.js 2 67,815
DalekJS (Developer Preview 0.03) Real + PhantomJS Async Node.js 9 1,010
webdriver-sync Real + PhantomJS Sync Node.js 83 1,104
webdriver-http-sync Real + PhantomJS Sync Node.js 12 444
Real + PhantomJS Async Node.js 81 117,083 Real + PhantomJS Async Node.js 16 22,990
An implementation of WebDriver API
Real + PhantomJS Async Node.js 95 121,756
The Intern Real + PhantomJS Sync Node.js 25 9,729

JavaScript Test Specification Libraries

Library Runtime Specification Style Node.js Releases Node.js Monthly Installs
Cucumber.js Node.js Given/When/Then 44 41,650
Mocha.js Node.js Describe/It 98 1,865,460
Testium – requires with webdriver-http-sync Node.js Describe/It 34 1758

WebDriverJS Example

var webdriver = require('selenium-webdriver');
var driver = new webdriver.Builder().
driver.wait(function() {
 return driver.getTitle().then(function(title) {
   return title === 'webdriver - Google Search';
}, 1000);

Zombie.js Example

const Browser = require('zombie');
// We're going to make requests to 
// Which will be routed to our test server localhost:3000 
Browser.localhost('', 3000);
describe('User visits signup page', function() {
  const browser = new Browser();
  before(function() {
    return browser.visit('/signup');
  describe('submits form', function() {
    before(function() {
        .fill('email',    'zombie@underworld.dead')
        .fill('password', 'eat-the-living');
      return browser.pressButton('Sign Me Up!');
    it('should be successful', function() {
    it('should see welcome page', function() {
      browser.assert.text('title', 'Welcome To Brains Depot');


// googletesting.js
casper.test.begin('Google search retrieves 10 or more results', 5, function suite(test) {
    casper.start("", function() {
        test.assertTitle("Google", "google homepage title is the one expected");
        test.assertExists('form[action="/search"]', "main form is found");
        this.fill('form[action="/search"]', {
            q: "casperjs"
        }, true);
    casper.then(function() {
        test.assertTitle("casperjs - Recherche Google", "google title is ok");
        test.assertUrlMatch(/q=casperjs/, "search term has been submitted");
        test.assertEval(function() {
            return __utils__.findAll("h3.r").length >= 10;
        }, "google search for \"casperjs\" retrieves 10 or more results");
    }); {


module.exports = {
'Page title is correct': function (test) {
    .assert.title().is('Google', 'It has title')


title = driver.getTitle();
link  = driver.findElement('i am a link'));;
assert(driver.getCurrentUrl().indexOf('foo title 2') > -1);
title.should.equal('foo title');


var WebDriver = require('webdriver-http-sync');
var desiredCapabilities = {browserName: 'firefox'};
var driver = new WebDriver('', desiredCapabilities);


    .should.become('WD Tests')
  .elementById('i am a link')
  .fin(function() { return browser.quit(); })


var webdriverio = require('../index');
var options = {
    desiredCapabilities: {
        browserName: 'chrome'
    .getTitle(function(err, title) {
        console.log('Title was: ' + title);


module.exports = {
  'Demo test Google' : function (client) {
      .waitForElementVisible('body', 1000)
      .setValue('input[type=text]', 'rembrandt van rijn')
      .waitForElementVisible('button[name=btnG]', 1000)
      .assert.containsText('ol#rso li:first-child',
        'Rembrandt - Wikipedia')

The Intern

define(function (require) {
  var registerSuite = require('intern!object');
  var assert = require('intern/chai!assert');
    name: 'index',
    'greeting form': function () {
      return this.remote
        .findByCssSelector('#loginForm input[type=submit]')
          .then(function (text) {
            assert.strictEqual(text, 'Hello, Elaine!',
              'Greeting should be displayed when the form is submitted');

Testium Example

injectBrowser = require 'testium/mocha'
assert = require 'assertive' # or whatever assert library you prefer 
describe 'browse', ->
  before injectBrowser()
  before ->
    @browser.navigateTo '/my-account'
    @browser.assert.httpStatus 200
  it 'is serving up gzipped content', ->
    assert.equal 'gzip', @browser.getHeader('Content-Encoding')

Mocha Example

var assert = require("assert")
describe('Array', function(){
  describe('#indexOf()', function(){
    it('should return -1 when the value is not present', function(){
      assert.equal(-1, [1,2,3].indexOf(5));
      assert.equal(-1, [1,2,3].indexOf(0));


Cucumber Example

Feature: Example feature
  As a user of cucumber.js
  I want to have documentation on cucumber
  So that I can concentrate on building awesome applications

  Scenario: Reading documentation
    Given I am on the Cucumber.js GitHub repository
    When I go to the README file
    Then I should see "Usage" as the page title
this.Given(/^I am on the Cucumber.js GitHub repository$/, function () {
  // Notice how `callback` is omitted from the parameters 
  return this.visit('');
  // A promise, returned by zombie.js's `visit` method is returned to Cucumber. 

Author: Alister Scott

Alister is an Excellence Wrangler for Automattic.