Book Cover

Chapter 11

Essential Knowledge: Testing

Note: The examples below are abridged; the book contains more details.

  1. White-Box Testing
  2. Gray-Box Testing
  3. Black-Box Testing
  4. Mocking
  5. EasyMock
  6. Test Hooks - Part I
  7. Test Hooks - Part II
  8. Dealing with Randomness - Part I
  9. Dealing with Randomness - Part II
  10. Test Case Anatomy - Part I
  11. Test Case Anatomy - Part II
  12. Data-Driven Testing

White-Box Testing

Here’s an example in Python for a unit test that verifies the behavior of putting a value in a key-value store in a white-box fashion:

import unittest

class KeyValueStore(object):
  ''' A key-value store example '''
  def __init__(self):
    self._store = {}

  def put(self, key, value):
    ''' Puts a value in the store '''
    self._store[key] = value

  def get(self, key):
    ''' Gets a value from the store by its key '''
    return self._store.get(key)


class TestKeyValueStore(unittest.TestCase):
  ''' A unit test for KeyValueStore '''
  def test_put(self):
    ''' Tests put '''
    store = KeyValueStore()
    store.put('white-box', 'testing')
    self.assertEqual(store._store.get('white-box'), 'testing')


if __name__ == '__main__':
  unittest.main()

Gray-Box Testing

Here’s an example of testing the put method of our key-value store in a scenario that requires some knowledge about how the store is implemented to verify the post-conditions of the method:

import unittest

class KeyValueStore(object):
  ''' A key-value store example '''
  def __init__(self):
    self._store = {}

  def put(self, key, value):
    ''' Puts a value in the store '''
    self._store[key] = value

  def get(self, key):
    ''' Gets a value from the store by its key '''
    return self._store.get(key)


class TestKeyValueStore(unittest.TestCase):
  ''' A unit test for KeyValueStore '''
  def test_put_key_already_exists(self):
    ''' Tests put when key already exists '''
    store = KeyValueStore()
    store.put('same key twice', 'first time')
    store.put('same key twice', 'second time')
    # This a black-box assertion
    self.assertEqual(store.get('same key twice'), 'second time')
    self.assertEqual(len(store._store), 1)  # a white-box one


if __name__ == '__main__':
  unittest.main()

Black-Box Testing

import unittest

class KeyValueStore(object):
  ''' A key-value store example '''
  def __init__(self):
    self._store = {}

  def put(self, key, value):
    ''' Puts a value in the store '''
    self._store[key] = value

  def get(self, key):
    ''' Gets a value from the store by its key '''
    return self._store.get(key)


class TestKeyValueStore(unittest.TestCase):
  ''' A unit test for KeyValueStore '''
  def test_put(self):
    ''' Tests put '''
    store = KeyValueStore()
    store.put('black-box', 'testing')
    self.assertEqual(store.get('black-box'), 'testing')


if __name__ == '__main__':
  unittest.main()

Mocking

Here’s an example of mocking in Java:

import org.testng.annotations.Test;
import org.testng.Assert;
import org.testng.collections.Lists;

import java.util.ArrayList;
import java.util.List;

interface DataLayer {
  void insertReminder(final Reminder reminder);

  Reminder getReminder(final String reminderID);

  void updateReminder(
    final String reminderID, final Reminder reminder);

  void deleteReminder(final String reminderID);
}

class Reminder {
  // Reminder fields go here
}

class Orchestrator {
  private final DataLayer dataLayer;

  public Orchestrator(final DataLayer dataLayer) {
    this.dataLayer = dataLayer;
  }

  public void addReminder(final Reminder reminder) {
    // Validation and pre-processing code goes here
    dataLayer.insertReminder(reminder);
    // ...
  }
}

public class OrchestratorTest {
  @Test
  public void testAddReminder() throws Exception {
    final Reminder reminderToAdd = new Reminder();
    final List<String> methodsCalled = new ArrayList<>();
    final Orchestrator orchestrator =
      new Orchestrator(new DataLayer() {
      @Override
      public void insertReminder(final Reminder reminder) {
        methodsCalled.add("dataLayer.insertReminder");
        Assert.assertSame(
          reminder,
          reminderToAdd,
          "reminder mismatch in dataLayer.insertReminder");
      }

      @Override
      public Reminder getReminder(String reminderID) {
        Assert.fail("dataLayer.getReminder is not expected");
        return null;
      }

      @Override
      public void updateReminder(
        String reminderID, Reminder reminder) {
        Assert.fail("dataLayer.updateReminder is not expected");
      }

      @Override
      public void deleteReminder(String reminderID) {
        Assert.fail("dataLayer.deleteReminder is not expected");
      }
    });

    orchestrator.addReminder(reminderToAdd);

    Assert.assertEquals(
      methodsCalled,
      Lists.newArrayList("dataLayer.insertReminder"),
      "expected methods not called");
  }
}

EasyMock

Here’s an example of mocking in Java using EasyMock:

import org.easymock.EasyMock;
import org.testng.annotations.Test;

interface DataLayer {
  void insertReminder(final Reminder reminder);

  Reminder getReminder(final String reminderID);

  void updateReminder(
    final String reminderID, final Reminder reminder);

  void deleteReminder(final String reminderID);
}

class Reminder {
  // Reminder fields go here
}

class Orchestrator {
  private final DataLayer dataLayer;

  public Orchestrator(final DataLayer dataLayer) {
    this.dataLayer = dataLayer;
  }

  public void addReminder(final Reminder reminder) {
    // Validation and pre-processing code goes here
    dataLayer.insertReminder(reminder);
    // ...
  }
}

public class OrchestratorTest {
  @Test
  public void testAddReminder() throws Exception {
    final Reminder reminderToAdd = new Reminder();
    final DataLayer dataLayerMock =
      EasyMock.createStrictMock(DataLayer.class);
    final Orchestrator orchestrator =
      new Orchestrator(dataLayerMock);

    dataLayerMock.insertReminder(reminderToAdd); // expect
    EasyMock.replay(dataLayerMock); // switch to replay state

    orchestrator.addReminder(reminderToAdd);
    // Assert the above mock conditions
    EasyMock.verify(dataLayerMock);
  }
}

Test Hooks - Part I

For example, if we want to record the timestamp at which a reminder is inserted in the database, we need to control what value is sent to the database at the time of insertion. The below example shows an incomplete way of verifying the behavior:

import org.easymock.EasyMock;
import org.testng.annotations.Test;


interface DataLayer {
  void insertReminder(
    final Reminder reminder, long creationTimestampInMillis);
}

class Reminder {
  // Reminder fields go here
}

class Orchestrator {
  private final DataLayer dataLayer;

  public Orchestrator(final DataLayer dataLayer) {
    this.dataLayer = dataLayer;
  }

  public void addReminder(final Reminder reminder) {
    // Validation and pre-processing code goes here
    dataLayer.insertReminder(
      reminder,
      System.currentTimeMillis());
    // ...
  }
}

public class OrchestratorTest {
  @Test
  public void testAddReminder() throws Exception {
    final Reminder reminderToAdd = new Reminder();
    final DataLayer dataLayerMock =
      EasyMock.createStrictMock(DataLayer.class);
    final Orchestrator orchestrator =
      new Orchestrator(dataLayerMock);

    dataLayerMock.insertReminder(
      EasyMock.same(reminderToAdd), EasyMock.anyLong());
    EasyMock.replay(dataLayerMock);

    orchestrator.addReminder(reminderToAdd);
    EasyMock.verify(dataLayerMock);
  }
}

Test Hooks - Part II

To properly test the use of the timestamp, we follow the dependency inversion principle again here and a test hook to set an instance of the Clock abstract class we use for generating the creation timestamp:

import com.google.common.annotations.VisibleForTesting;
import org.easymock.EasyMock;
import org.testng.annotations.Test;

import java.time.Clock;
import java.time.Instant;
import java.time.ZoneId;

interface DataLayer {
  void insertReminder(
    final Reminder reminder,
    final long creationTimestampInMillis);
}

class Reminder {
  // Reminder fields go here
}

class Orchestrator {
  private final DataLayer dataLayer;
  private Clock clock = Clock.systemUTC(); // for testability

  public Orchestrator(final DataLayer dataLayer) {
    this.dataLayer = dataLayer;
  }

  @VisibleForTesting
  void setClock(final Clock clock) {
    this.clock = clock;
  }

  public void addReminder(final Reminder reminder) {
    // Validation and pre-processing code goes here
    dataLayer.insertReminder(reminder, this.clock.millis());
    // ...
  }
}

public class OrchestratorTest {
  @Test
  public void testAddReminder() throws Exception {
    final Reminder reminderToAdd = new Reminder();
    final DataLayer dataLayerMock =
      EasyMock.createStrictMock(DataLayer.class);
    final Orchestrator orchestrator =
      new Orchestrator(dataLayerMock);
    final long creationTimestampInMillis =
      System.currentTimeMillis();

    orchestrator.setClock(
      Clock.fixed(Instant.now(),
      ZoneId.systemDefault()));

    dataLayerMock.insertReminder(
      reminderToAdd,
      creationTimestampInMillis);
    EasyMock.replay(dataLayerMock);

    orchestrator.addReminder(reminderToAdd);
    EasyMock.verify(dataLayerMock);
  }
}

Dealing with Randomness - Part I

Here’s an example of a Java unit test that verifies the behavior of a coin-flipping Bernoulli trial:

import org.testng.annotations.Test;

import java.util.Random;

import static org.testng.Assert.*;

class CoinFlipper {
  public enum Face {HEAD, TAIL}
  
  private final Random random = new Random();
  
  public Face flip() {
    return random.nextBoolean() ? Face.HEAD : Face.TAIL;
  }
}

public class CoinFlipperTest {
  @Test
  public void testFlip() {
    final CoinFlipper flipper = new CoinFlipper();
    final int trialsCount = 100;
    int headsCount = 0;
    
    for (int i = 0; i < trialsCount; i++) {
      if (flipper.flip() == CoinFlipper.Face.HEAD) {
        ++headsCount;
      }
    }
    
    assertTrue(Math.abs(headsCount - trialsCount / 2) <= 5);
  }
}

Dealing with Randomness - Part II

we can set the seed of the random number generator so that each time the test case runs it’s predictable instead of pseudo-random; here’s an example:

import com.google.common.annotations.VisibleForTesting;
import org.testng.annotations.Test;

import java.util.Random;

import static org.testng.Assert.*;

class CoinFlipper {
  public enum Face {HEAD, TAIL}
  
  private Random random = new Random();
  
  public Face flip() {
    return random.nextBoolean() ? Face.HEAD : Face.TAIL;
  }
  
  @VisibleForTesting
  void setRandom(final Random random) {
    this.random = random;
  }
}

public class CoinFlipperTest {
  @Test
  public void testFlip() {
    final CoinFlipper flipper = new CoinFlipper();
    final Random random = new Random(42);
    
    flipper.setRandom(random);
    assertEquals(flipper.flip(), CoinFlipper.Face.HEAD);
    assertEquals(flipper.flip(), CoinFlipper.Face.TAIL);
    assertEquals(flipper.flip(), CoinFlipper.Face.HEAD);
  }
}

Test Case Anatomy - Part I

In our key-value store test example, we can see the anatomy of the unit test as the following:

import unittest

class KeyValueStore(object):
  ''' A key-value store example '''
  def __init__(self):
    self._store = {}

  def put(self, key, value):
    ''' Puts a value in the store '''
    self._store[key] = value

  def get(self, key):
    ''' Gets a value from the store by its key '''
    return self._store.get(key)


class TestKeyValueStore(unittest.TestCase):
  ''' A unit test for KeyValueStore '''
  def test_put_key_already_exists(self):
    ''' Tests put when key already exists '''

    # Set up
    store = KeyValueStore()
    store.put('same key twice', 'first time')

    # Execute
    store.put('same key twice', 'second time')
    
    # Verify
    self.assertEqual(store.get('same key twice'), 'second time')
    self.assertEqual(len(store._store), 1)

    # No shared state between tests => no cleanup required


if __name__ == '__main__':
  unittest.main()

Test Case Anatomy - Part II

Here’s an example that makes use of the setUp method to create a new instance of the key-value store in a clean state for each test case:

import unittest

class KeyValueStore(object):
  ''' A key-value store example '''
  def __init__(self):
    self._store = {}

  def put(self, key, value):
    ''' Puts a value in the store '''
    self._store[key] = value

  def get(self, key):
    ''' Gets a value from the store by its key '''
    return self._store.get(key)


class TestKeyValueStore(unittest.TestCase):
  ''' A unit test for KeyValueStore '''
  def setUp(self):
    ''' Runs before each test method '''
    self.store = KeyValueStore()

  def test_init(self):
    ''' Tests __init__ '''
    self.assertEqual(self.store.get('key'), None)
    self.assertEqual(len(self.store._store), 0)

  def test_put_with_value(self):
    ''' Tests put '''
    self.store.put('key', 'value')
    self.assertEqual(self.store.get('key'), 'value')
    self.assertEqual(len(self.store._store), 1)

  def test_put_with_none_value(self):
    ''' Tests put with none value '''
    self.store.put('key', None)
    self.assertEqual(self.store.get('key'), None)
    self.assertEqual(len(self.store._store), 1)


if __name__ == '__main__':
  unittest.main()

Data-Driven Testing

Here’s an example of table-driven testing in Java using TestNG:

import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;

import static org.testng.Assert.*;

class Adder {
  public int add(int x, int y) {
    return x + y;
  }
}

public class AdderTest {
  @DataProvider(name = "testAdd")
  private static Object[][] provideTestData() {
    return new Integer[][]{
      new Integer[]{0, 0, 0},
      new Integer[]{2, 3, 5},
      new Integer[]{-3, 5, 2},
      new Integer[]{-5, -3, -8},
      new Integer[]{-5, 3, -2},
      new Integer[]{0, Integer.MAX_VALUE, Integer.MAX_VALUE},
      new Integer[]{0, Integer.MIN_VALUE, Integer.MIN_VALUE},
      // ...
    };
  }
  
  @Test(dataProvider = "testAdd")
  public void testAdd(int x, int y, int expected) {
    final int z = new Adder().add(x, y);
    assertEquals(z, expected, "failed for " + x + " and " + y);
  }
}