support running integration and performance tests

* Infrastructure for running performance and integration tests
* Basic tests of both types for the MongoDB backend
* Ensure timestamps include tzinfo

Uses "make" to simplify the execution of the various commands.  We could probably integrate with manage.py or setup.py or something more sophisticated in the future if necessary.  For now I wanted to KISS.

.travis.yml

@@ -2,9 +2,7 @@ language: python
 python:
   - "2.7"
 install:
-  - "pip install -r test-requirements.txt --use-mirrors"
-  - "python setup.py install"
+  - make test.setup install
 script:
-  - DJANGO_SETTINGS_MODULE=eventtracking.django.tests.settings nosetests --cover-erase --with-coverage --cover-min-percentage=95
-  - pep8
-  - pylint --reports=y eventtracking
+  - make ci
+services: mongodb

Makefile

+export DJANGO_SETTINGS_MODULE=eventtracking.django.tests.settings
+
+.PHONY: test.unit style lint
+
+ci: test.unit test.integration style lint
+
+test.setup:
+	pip install -r test-requirements.txt -q
+
+test: test.unit test.integration test.performance
+
+test.unit: test.setup
+	nosetests --cover-erase --with-coverage -A 'not integration and not performance' --cover-min-percentage=95
+
+test.integration: test.setup
+	nosetests --verbose --nocapture -a 'integration'
+
+test.performance: test.setup
+	nosetests --verbose --nocapture -a 'performance'
+
+style:
+	pep8
+
+lint:
+	pylint --reports=y eventtracking
+
+install:
+	python setup.py install

eventtracking/backends/tests/__init__.py

-"""Tests for backends"""
+"""
+Helper classes for backend tests
+"""
+from __future__ import absolute_import
+
+from unittest import TestCase
+from contextlib import contextmanager
+import time
+import os
+import random
+import string  # pylint: disable=deprecated-module
+
+
+class InMemoryBackend(object):
+    """A backend that simply stores all events in memory"""
+
+    def __init__(self):
+        super(InMemoryBackend, self).__init__()
+        self.events = []
+
+    def send(self, event):
+        """Store the event in a list"""
+        self.events.append(event)
+
+
+class IntegrationTestCase(TestCase):
+    """
+    Tests the integration between a backend and any external systems
+    it makes use of.
+    """
+
+    # This is equivalent to decorating all subclasses with attr('integration')
+    # which allows us to selectively run integration tests.
+    integration = 1
+
+
+class PerformanceTestCase(TestCase):
+    """
+    Reads parameters from the following environment variables:
+
+    * EVENT_TRACKING_PERF_EVENTS - Number of events to send to the backend
+    * EVENT_TRACKING_PERF_PAYLOAD_SIZE - Approximate size (in bytes) of a
+      payload field to include in every event.
+    * EVENT_TRACKING_PERF_THRESHOLD_SECONDS - Fail the test if it takes
+      longer than this number of seconds to save all of the events.
+    """
+
+    # This is equivalent to decorating all subclasses with attr('performance')
+    performance = 1
+
+    def __init__(self, *args, **kwargs):
+        super(PerformanceTestCase, self).__init__(*args, **kwargs)
+        self.num_events = int(os.getenv('EVENT_TRACKING_PERF_EVENTS', 20000))
+        self.payload_size = int(os.getenv('EVENT_TRACKING_PERF_PAYLOAD_SIZE', 600))
+        self.random_payload = ''.join(random.choice(string.ascii_letters) for _ in range(self.payload_size))
+        self.threshold = float(os.getenv('EVENT_TRACKING_PERF_THRESHOLD_SECONDS', 1))
+
+    @contextmanager
+    def assert_execution_time_less_than_threshold(self):
+        """
+        Times the execution of the block within the context and raises an
+        `AssertionError` if it is longer than `self.threshold`
+        """
+        start_time = time.time()
+        yield
+        elapsed_time = time.time() - start_time
+
+        print ''
+        print 'Elapsed Time: {0} seconds'.format(elapsed_time)
+        print 'Threshold: {0} seconds'.format(self.threshold)
+        print 'Number of Events: {0}'.format(self.num_events)
+        print 'Payload Size: {0} bytes'.format(self.payload_size)
+        print 'Events per second: {0}'.format(self.num_events / elapsed_time)
+
+        if self.threshold >= 0:
+            self.assertLessEqual(elapsed_time, self.threshold)

eventtracking/backends/tests/test_mongodb_integration.py

+"""
+Runs invasive tests to ensure that the tracking system can communicate with
+an actual MongoDB instance.
+"""
+
+from __future__ import absolute_import
+
+from uuid import uuid4
+from datetime import datetime
+
+from pytz import UTC
+
+from eventtracking.backends.tests import IntegrationTestCase
+from eventtracking.backends.tests import InMemoryBackend
+from eventtracking.backends.mongodb import MongoBackend
+from eventtracking.track import Tracker
+
+
+class TestMongoIntegration(IntegrationTestCase):
+    """
+    Makes use of a real MongoDB instance to ensure the backend is wired up
+    properly to the external system.  These tests require a mongodb instance
+    to be running on localhost listening on the default port.
+    """
+
+    def setUp(self):
+        self.database_name = 'test_eventtracking_' + str(uuid4())
+        self.mongo_backend = MongoBackend(database=self.database_name)
+        self.memory_backend = InMemoryBackend()
+        self.tracker = Tracker({
+            'mongo': self.mongo_backend,
+            'mem': self.memory_backend
+        })
+
+    def tearDown(self):
+        self.mongo_backend.connection.drop_database(self.database_name)
+
+    def test_sequential_events(self):
+        now = datetime.now(UTC)
+        for i in range(10):
+            self.tracker.event('org.test.user.login', {
+                'username': 'tester',
+                'user_id': 10,
+                'email': 'tester@eventtracking.org',
+                'sequence': i,
+                'current_time': now
+            })
+
+        # Ensure MongoDB has finished writing out the events before we
+        # run our query.
+        self.mongo_backend.connection.fsync()
+
+        mem_events = {}
+        for event in self.memory_backend.events:
+            mem_events[event['data']['sequence']] = event
+        self.assertEquals(len(mem_events), 10)
+
+        cursor = self.mongo_backend.collection.find()
+        self.assertEquals(cursor.count(), 10)
+
+        for event in cursor:
+            mem_event = mem_events[event['data']['sequence']]
+            mem_event['_id'] = event['_id']
+            if self.are_results_equal(mem_event, event):
+                del mem_events[event['data']['sequence']]
+        self.assertEquals(len(mem_events), 0)
+
+    def are_results_equal(self, left, right):
+        """
+        Ensure two events are equivalent.
+
+        We use a bit of special logic here when comparing timestamps since
+        MongoDB apparently only stores millisecond precision timestamps.
+        Thus, when comparing we ignore the datetime.millisecond property.
+        """
+        for event in [left, right]:
+            self.remove_microseconds_from_timestamps(event)
+
+        return left == right
+
+    def remove_microseconds_from_timestamps(self, event):
+        """Truncate the microseconds from the event timestamp"""
+        event['timestamp'] = event['timestamp'].replace(microsecond=0)
+        event['data']['current_time'] = event['data']['current_time'].replace(microsecond=0)

eventtracking/backends/tests/test_mongodb_performance.py

+"""
+Runs performance tests to determine if a significant performance regression has
+been introduced to the MongoBackend.
+"""
+
+from __future__ import absolute_import
+
+from uuid import uuid4
+
+from eventtracking.backends.tests import PerformanceTestCase
+from eventtracking.backends.mongodb import MongoBackend
+from eventtracking.track import Tracker
+
+
+class TestBackendPerformance(PerformanceTestCase):
+    """
+    Makes use of real backend systems to see how long it takes for the backend
+    to commit events to stable storage.
+    """
+
+    def setUp(self):
+        self.database_name = 'perf_test_eventtracking_' + str(uuid4())
+        self.mongo_backend = MongoBackend(database=self.database_name)
+        self.tracker = Tracker({
+            'mongo': self.mongo_backend
+        })
+
+    def tearDown(self):
+        self.mongo_backend.connection.drop_database(self.database_name)
+
+    def test_sequential_events(self):
+        with self.assert_execution_time_less_than_threshold():
+            for i in range(self.num_events):
+                self.tracker.event('perf.event', {
+                    'sequence': i,
+                    'payload': self.random_payload
+                })

eventtracking/tests/test_track.py

@@ -10,6 +10,7 @@ from unittest import TestCase
 from mock import MagicMock
 from mock import patch
 from mock import sentinel
+from pytz import UTC
 
 from eventtracking import track
 
@@ -22,11 +23,11 @@ class TestTrack(TestCase):  # pylint: disable=missing-docstring
         self.tracker = None
         self.configure_mock_backends(1)
 
-        self._expected_timestamp = datetime.utcnow()
+        self._expected_timestamp = datetime.now(UTC)
         self._datetime_patcher = patch('eventtracking.track.datetime')
         self.addCleanup(self._datetime_patcher.stop)
         mock_datetime = self._datetime_patcher.start()
-        mock_datetime.utcnow.return_value = self._expected_timestamp  # pylint: disable=maybe-no-member
+        mock_datetime.now.return_value = self._expected_timestamp  # pylint: disable=maybe-no-member
 
     def configure_mock_backends(self, number_of_mocks):
         """Ensure the tracking module has the requisite number of mock backends"""

eventtracking/track.py

@@ -19,6 +19,7 @@ from __future__ import absolute_import
 from datetime import datetime
 import logging
 
+from pytz import UTC
 
 LOG = logging.getLogger(__name__)
 
@@ -47,7 +48,7 @@ class Tracker(object):
         """
         full_event = {
             'event_type': event_type,
-            'timestamp': datetime.utcnow(),
+            'timestamp': datetime.now(UTC),
             'data': data or {}
         }
 

requirements.txt

 Django >= 1.4, < 1.5
 pymongo==2.4.1
+pytz