Merge pull request #3 from mulby/gabe/backends

port mongodb backend from the legacy app

.coveragerc

 [run]
 branch = True
-omit = *mock*
+include = eventtracking*

eventtracking/backends/__init__.py

+"""
+Event tracking backend module.
+
+Contains the base class for event trackers, and implementation of some
+backends.
+
+"""
+
+from __future__ import absolute_import
+
+import abc
+
+
+# pylint: disable=unused-argument
+class BaseBackend(object):
+    """
+    Abstract Base Class for event tracking backends.
+
+    """
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self, **kwargs):
+        pass
+
+    @abc.abstractmethod
+    def send(self, event):  # pragma: no cover
+        """Send event to tracker."""
+        pass

eventtracking/backends/mongodb.py

+"""MongoDB event tracker backend."""
+
+from __future__ import absolute_import
+
+import logging
+
+import pymongo
+from pymongo import MongoClient
+from pymongo.errors import PyMongoError
+
+from eventtracking.backends import BaseBackend
+
+
+log = logging.getLogger(__name__)
+
+
+class MongoBackend(BaseBackend):
+    """Class for a MongoDB event tracker Backend"""
+
+    def __init__(self, **kwargs):
+        """
+        Connect to a MongoDB.
+
+        :Parameters:
+
+          - `host`: hostname
+          - `port`: port
+          - `user`: collection username
+          - `password`: collection user password
+          - `database`: name of the database
+          - `collection`: name of the collection
+          - `extra`: parameters to pymongo.MongoClient not listed above
+
+        """
+
+        super(MongoBackend, self).__init__(**kwargs)
+
+        # Extract connection parameters from kwargs
+
+        host = kwargs.get('host', 'localhost')
+        port = kwargs.get('port', 27017)
+
+        user = kwargs.get('user', '')
+        password = kwargs.get('password', '')
+
+        db_name = kwargs.get('database', 'eventtracking')
+        collection_name = kwargs.get('collection', 'events')
+
+        # Other mongo connection arguments
+        extra = kwargs.get('extra', {})
+
+        # By default disable write acknowledgments, reducing the time
+        # blocking during an insert
+        extra['w'] = extra.get('w', 0)
+
+        # Make timezone aware by default
+        extra['tz_aware'] = extra.get('tz_aware', True)
+
+        # Connect to database and get collection
+
+        self.connection = MongoClient(
+            host=host,
+            port=port,
+            **extra
+        )
+
+        self.collection = self.connection[db_name][collection_name]
+
+        if user or password:
+            self.collection.database.authenticate(user, password)
+
+        self._create_indexes()
+
+    def _create_indexes(self):
+        """Ensures the proper fields are indexed"""
+        # WARNING: The collection will be locked during the index
+        # creation. If the collection has a large number of
+        # documents in it, the operation can take a long time.
+
+        # TODO: The creation of indexes can be moved to a Django
+        # management command or equivalent. There is also an option to
+        # run the indexing on the background, without locking.
+        self.collection.ensure_index([('time', pymongo.DESCENDING)])
+        self.collection.ensure_index('event_type')
+
+    def send(self, event):
+        try:
+            self.collection.insert(event, manipulate=False)
+        except PyMongoError:
+            msg = 'Error inserting to MongoDB event tracker backend'
+            log.exception(msg)

eventtracking/backends/tests/test_mongodb.py

+from __future__ import absolute_import
+
+from unittest import TestCase
+from mock import patch
+from mock import sentinel
+
+from pymongo.errors import PyMongoError
+
+from eventtracking.backends.mongodb import MongoBackend
+
+
+class TestMongoBackend(TestCase):  # pylint: disable=missing-docstring
+
+    def setUp(self):
+        self.mongo_patcher = patch('eventtracking.backends.mongodb.MongoClient')
+        self.addCleanup(self.mongo_patcher.stop)
+        self.mongo_patcher.start()
+
+        self.backend = MongoBackend()
+
+    def test_mongo_backend(self):
+        events = [{'test': 1}, {'test': 2}]
+
+        self.backend.send(events[0])
+        self.backend.send(events[1])
+
+        # Check if we inserted events into the database
+
+        calls = self.backend.collection.insert.mock_calls
+
+        self.assertEqual(len(calls), 2)
+
+        # Unpack the arguments and check if the events were used
+        # as the first argument to collection.insert
+
+        def first_argument(call):
+            _, args, _ = call
+            return args[0]
+
+        self.assertEqual(events[0], first_argument(calls[0]))
+        self.assertEqual(events[1], first_argument(calls[1]))
+
+    def test_authentication_settings(self):
+        backend = MongoBackend(user=sentinel.user, password=sentinel.password)
+        backend.collection.database.authenticate.assert_called_once_with(sentinel.user, sentinel.password)
+
+    def test_mongo_insertion_error(self):
+        self.backend.collection.insert.side_effect = PyMongoError
+
+        self.backend.send({'test': 1})
+        # Ensure this error is caught

eventtracking/tests/test_track.py

 """
 Test the event tracking module
 """
+from __future__ import absolute_import
 
 from datetime import datetime
 from unittest import TestCase
@@ -10,14 +11,18 @@ from mock import patch
 from mock import sentinel
 
 from eventtracking import track
+from eventtracking.backends import BaseBackend
 
 
 class TestTrack(TestCase):  # pylint: disable=missing-docstring
 
     def setUp(self):
-        self._mock_backend = MagicMock()
-        self.addCleanup(track.BACKENDS.remove, self._mock_backend)
-        track.BACKENDS.append(self._mock_backend)
+        # Ensure all backends are removed after executing
+        self.addCleanup(track.configure, {})
+
+        self._mock_backends = []
+        self._mock_backend = None
+        self.configure_mock_backends(1)
 
         self._expected_timestamp = datetime.utcnow()
         self._datetime_patcher = patch('eventtracking.track.datetime')
@@ -25,17 +30,39 @@ class TestTrack(TestCase):  # pylint: disable=missing-docstring
         mock_datetime = self._datetime_patcher.start()
         mock_datetime.utcnow.return_value = self._expected_timestamp  # pylint: disable=maybe-no-member
 
+    def configure_mock_backends(self, number_of_mocks):
+        """Ensure the tracking module has the requisite number of mock backends"""
+        config = {}
+        for i in range(number_of_mocks):
+            name = 'mock{0}'.format(i)
+            config[name] = {
+                'ENGINE': 'eventtracking.tests.test_track.TrivialFakeBackend'
+            }
+
+        track.configure(config)
+
+        self._mock_backends = []
+        for i in range(number_of_mocks):
+            backend = self.get_mock_backend(i)
+            backend.send = MagicMock()
+            self._mock_backends.append(backend)
+        self._mock_backend = self._mock_backends[0]
+
+    def get_mock_backend(self, index):
+        """Get the mock backend created by `configure_mock_backends`"""
+        return track.BACKENDS['mock{0}'.format(index)]
+
     def test_event_simple_event_without_data(self):
         track.event(sentinel.event_type)
 
-        self.__assert_backend_called_with(sentinel.event_type)
+        self.assert_backend_called_with(sentinel.event_type)
 
-    def __assert_backend_called_with(self, event_type, data=None, backend=None):
+    def assert_backend_called_with(self, event_type, data=None, backend=None):
         """Ensures the backend is called exactly once with the expected data."""
         if not backend:
             backend = self._mock_backend
 
-        backend.event.assert_called_once_with(
+        backend.send.assert_called_once_with(
             {
                 'event_type': event_type,
                 'timestamp': self._expected_timestamp,
@@ -51,7 +78,7 @@ class TestTrack(TestCase):  # pylint: disable=missing-docstring
             }
         )
 
-        self.__assert_backend_called_with(
+        self.assert_backend_called_with(
             sentinel.event_type,
             {
                 sentinel.key: sentinel.value
@@ -59,26 +86,149 @@ class TestTrack(TestCase):  # pylint: disable=missing-docstring
         )
 
     def test_multiple_backends(self):
-        another_backend = MagicMock()
-        track.BACKENDS.append(another_backend)
-        try:
-            track.event(sentinel.event_type)
+        self.configure_mock_backends(2)
+        track.event(sentinel.event_type)
 
-            self.__assert_backend_called_with(sentinel.event_type)
-            self.__assert_backend_called_with(
-                sentinel.event_type, backend=another_backend)
-        finally:
-            track.BACKENDS.remove(another_backend)
+        for backend in self._mock_backends:
+            self.assert_backend_called_with(
+                sentinel.event_type, backend=backend)
 
     def test_single_backend_failure(self):
-        self._mock_backend.event.side_effect = Exception
+        self.configure_mock_backends(2)
+        self.get_mock_backend(0).send.side_effect = Exception
 
-        another_backend = MagicMock()
-        track.BACKENDS.append(another_backend)
+        track.event(sentinel.event_type)
+
+        self.assert_backend_called_with(
+            sentinel.event_type, backend=self.get_mock_backend(1))
+
+    def test_configure(self):
+        track.configure({
+            "fake": {
+                'ENGINE': 'eventtracking.tests.test_track.TrivialFakeBackend'
+            }
+        })
+        fake_backend = track.BACKENDS['fake']
+        self.assertTrue(isinstance(fake_backend, TrivialFakeBackend))
+
+    def test_ignore_no_engine(self):
+        track.configure({
+            "no_engine": {
+                'OPTIONS': {}
+            }
+        })
+        self.assertEquals(len(track.BACKENDS), 0)
+
+    def test_configure_empty_engine(self):
+        try:
+            track.configure({
+                "empty_engine": {
+                    'ENGINE': ''
+                }
+            })
+            self.fail('Expected exception to be thrown when attempting to add a backend with an empty engine')
+        except ValueError:
+            pass
+
+    def test_configure_invalid_package(self):
         try:
-            track.event(sentinel.event_type)
+            track.configure({
+                "invalid_package": {
+                    'ENGINE': 'foo.BarBackend'
+                }
+            })
+            self.fail('Expected exception to be thrown when attempting to add a backend from a non-existent package')
+        except ValueError:
+            pass
+
+    def test_configure_no_package_invalid_class(self):
+        try:
+            track.configure({
+                "no_package_invalid_class": {
+                    'ENGINE': 'BarBackend'
+                }
+            })
+            self.fail('Expected exception to be thrown when attempting to add a non-existent backend class')
+        except ValueError:
+            pass
+
+    def test_configure_invalid_class(self):
+        try:
+            track.configure({
+                "invalid_class": {
+                    'ENGINE': 'eventtracking.tests.test_track.BarBackend'
+                }
+            })
+            self.fail('Expected exception to be thrown when attempting to add a non-existent backend class')
+        except ValueError:
+            pass
+
+    def test_configure_engine_not_a_backend(self):
+        try:
+            track.configure({
+                "not_a_backend": {
+                    'ENGINE': 'eventtracking.tests.test_track.NotABackend'
+                }
+            })
+            self.fail(
+                'Expected exception to be thrown when attempting to add a backend class that'
+                ' does not subclass BaseBackend'
+            )
+        except ValueError:
+            pass
+
+    def test_configure_engine_with_options(self):
+        track.configure({
+            'with_options': {
+                'ENGINE': 'eventtracking.tests.test_track.FakeBackendWithOptions',
+                'OPTIONS': {
+                    'option': sentinel.option_value
+                }
+            }
+        })
+        self.assertEquals(track.BACKENDS['with_options'].option, sentinel.option_value)
+
+    def test_configure_engine_missing_options(self):
+        track.configure({
+            'without_options': {
+                'ENGINE': 'eventtracking.tests.test_track.FakeBackendWithOptions'
+            }
+        })
+        self.assertEquals(track.BACKENDS['without_options'].option, None)
+
+    def test_configure_engine_with_extra_options(self):
+        track.configure({
+            'extra_options': {
+                'ENGINE': 'eventtracking.tests.test_track.FakeBackendWithOptions',
+                'OPTIONS': {
+                    'option': sentinel.option_value,
+                    'extra_option': sentinel.extra_option_value
+                }
+            }
+        })
+        self.assertEquals(track.BACKENDS['extra_options'].option, sentinel.option_value)
+
+
+class TrivialFakeBackend(BaseBackend):
+    """A trivial fake backend without any options"""
+
+    def send(self, event):
+        pass
+
+
+class NotABackend(object):
+    """
+    A class that is not a backend
+    """
+    pass
+
+
+class FakeBackendWithOptions(BaseBackend):
+    """A trivial fake backend with options"""
+
+    def __init__(self, **kwargs):
+        super(FakeBackendWithOptions, self).__init__()
+        self.option = kwargs.get('option', None)
 
-            self.__assert_backend_called_with(
-                sentinel.event_type, backend=another_backend)
-        finally:
-            track.BACKENDS.remove(another_backend)
+    def send(self, event):
+        pass

eventtracking/track.py

@@ -3,30 +3,99 @@ Track application events.  Supports persisting events to multiple backends.
 
 Best Practices:
 * It is recommended that event types are namespaced using dot notation to
-  avoid naming collisions, similar to DNS names.  For example:
-  org.edx.video.stop, edu.mit.audio.stop
+    avoid naming collisions, similar to DNS names.  For example:
+    org.edx.video.stop, edu.mit.audio.stop
 * Avoid using event type names that may cause collisions.  The burden is
-  on the analyst to decide whether your event is equivalent to another
-  and should be grouped accordingly etc.
+    on the analyst to decide whether your event is equivalent to another
+    and should be grouped accordingly etc.
 * Do not emit events that you don't own.  This could negatively impact
-  the analysis of the event stream.  If you suspect your event is
-  equivalent to another, say so in your documenation, and the analyst
-  can decide whether or not to group them.
+    the analysis of the event stream.  If you suspect your event is
+    equivalent to another, say so in your documenation, and the analyst
+    can decide whether or not to group them.
 """
+from __future__ import absolute_import
+
 from datetime import datetime
+from importlib import import_module
+import inspect
 import logging
 
+from eventtracking.backends import BaseBackend
+
+
+__all__ = ['configure', 'event']
+
 
 LOG = logging.getLogger(__name__)
-BACKENDS = []
+BACKENDS = {}
+
+
+def configure(config):
+    """
+    Configure event tracking.  `config` is expected to be a dictionary of backend engines.
+
+    Example::
+
+        config = {
+            'default': {
+                'ENGINE': 'some.arbitrary.Backend',
+                'OPTIONS': {
+                    'endpoint': 'http://something/event'
+                }
+            },
+            'anoter_engine': {
+                'ENGINE': 'some.arbitrary.OtherBackend',
+                'OPTIONS': {
+                    'user': 'foo'
+                }
+            },
+        }
+    """
+    BACKENDS.clear()
+
+    for name, values in config.iteritems():
+        # Ignore empty values to turn-off default tracker backends
+        if values and 'ENGINE' in values:
+            engine = values['ENGINE']
+            options = values.get('OPTIONS', {})
+            BACKENDS[name] = _instantiate_backend_from_name(engine, options)
+
+
+def _instantiate_backend_from_name(name, options):
+    """
+    Instantiate an event tracker backend from the full module path to
+    the backend class. Useful when setting backends from configuration
+    files.
+
+    """
+    # Parse backend name
+
+    parts = name.split('.')
+    module_name = '.'.join(parts[:-1])
+    class_name = parts[-1]
+
+    # Get and verify the backend class
+
+    try:
+        module = import_module(module_name)
+        cls = getattr(module, class_name)
+        if not inspect.isclass(cls) or not issubclass(cls, BaseBackend):
+            raise TypeError
+    except (ValueError, AttributeError, TypeError, ImportError):
+        raise ValueError('Cannot find event track backend %s' % name)
+
+    backend = cls(**options)
+
+    return backend
 
 
 def event(event_type, data=None):
     """
     Emit an event annotated with the UTC time when this function was called.
 
-    :event_type: A unique identification string for an event that has already been registered.
-    :data: A dictionary mapping field names to the value to include in the event.  Note that all values provided must be serializable.
+    `event_type` is a unique identification string for an event that has already been registered.
+    `data` is a dictionary mapping field names to the value to include in the event.
+        Note that all values provided must be serializable.
 
     """
     full_event = {
@@ -35,10 +104,10 @@ def event(event_type, data=None):
         'data': data or {}
     }
 
-    for backend in BACKENDS:
+    for name, backend in BACKENDS.iteritems():
         try:
-            backend.event(full_event)
-        except Exception:  # pylint: disable=W0703
+            backend.send(full_event)
+        except Exception:  # pylint: disable=broad-except
             LOG.exception(
-                'Unable to commit event to backend: {0}'.format(backend)
+                'Unable to send event to backend: {0}'.format(name)
             )

requirements.txt

 # 3rd-party needs
 Django >= 1.4, < 1.5
+pymongo==2.4.1
 
 # For Tests
 mock