Integrate encryption in to event exports

Encrypt all output from the task.

Change-Id: Ie394d489784f95c1f792cf4e52b5a83ba073f9b9

edx/analytics/tasks/encrypt.py

@@ -29,6 +29,7 @@ def make_encrypted_file(output_file, key_file_targets, recipients=None):
     with make_temp_directory(prefix="encrypt") as temp_dir:
         # Use temp directory to hold gpg keys.
         gpg = gnupg.GPG(gnupghome=temp_dir)
+        gpg.encoding = 'utf-8'
         _import_key_files(gpg, key_file_targets)
 
         # Create a temp file to contain the unencrypted output, in the same temp directory.
@@ -77,61 +78,3 @@ def _copy_file_to_open_file(filepath, output_file):
                 output_file.write(transfer_buffer)
             else:
                 break
-
-
-class FakeEventExportWithEncryptionTask(MultiOutputMapReduceJobTask):
-    """Example class to demonstrate use of encryption of files for export from multi-output."""
-    source = luigi.Parameter()
-    config = luigi.Parameter()
-    # TODO: these parameters could be moved into the config file.
-    gpg_key_dir = luigi.Parameter()
-    gpg_master_key = luigi.Parameter(default=None)
-
-    def init_reducer(self):
-        self._get_organization_info()
-
-    def requires(self):
-        return {
-            'source': ExternalURL(self.source),
-            'config': ExternalURL(self.config),
-        }
-
-    def requires_local(self):
-        return self.requires()['config']
-
-    def requires_hadoop(self):
-        return self.requires()['source']
-
-    def mapper(self, line):
-        org_id = "edx"
-        server_id = "prod-edxapp-011"
-        yield (org_id, server_id), line
-
-    def extra_modules(self):
-        return [gnupg, yaml]
-
-    def output_path_for_key(self, key):
-        org_id, server_id = key
-        return url_path_join(self.output_root, org_id, server_id, 'tracking.log.gpg')
-
-    def multi_output_reducer(self, key, values, output_file):
-        org_id, _server_id = key
-        recipients = self._get_recipients(org_id)
-        key_file_targets = [get_target_from_url(url_path_join(self.gpg_key_dir, recipient)) for recipient in recipients]
-        with make_encrypted_file(output_file, key_file_targets) as encrypted_output_file:
-            for value in values:
-                encrypted_output_file.write(value)
-                encrypted_output_file.write('\n')
-
-    def _get_organization_info(self):
-        """Get the organization configuration from the configuration yaml file."""
-        with self.input()['config'].open() as config_input:
-            config_data = yaml.load(config_input)
-        self.organizations = config_data['organizations']  # pylint: disable=attribute-defined-outside-init
-
-    def _get_recipients(self, org_id):
-        """Get the correct recipients for the specified organization."""
-        recipients = [self.organizations[org_id]['recipient']]
-        if self.gpg_master_key is not None:
-            recipients.append(self.gpg_master_key)
-        return recipients

edx/analytics/tasks/event_exports.py

@@ -3,13 +3,14 @@
 import logging
 import os
 
+import gnupg
 import luigi
-import luigi.configuration
 import yaml
 
+from edx.analytics.tasks.encrypt import make_encrypted_file
 from edx.analytics.tasks.mapreduce import MultiOutputMapReduceJobTask
 from edx.analytics.tasks.pathutil import EventLogSelectionTask
-from edx.analytics.tasks.url import url_path_join, ExternalURL
+from edx.analytics.tasks.url import url_path_join, ExternalURL, get_target_from_url
 from edx.analytics.tasks.util import eventlog
 
 
@@ -45,6 +46,13 @@ class EventExportTask(MultiOutputMapReduceJobTask):
     interval = luigi.DateIntervalParameter()
     pattern = luigi.Parameter(default=None)
 
+    gpg_key_dir = luigi.Parameter(
+        default_from_config={'section': 'event-export', 'name': 'gpg_key_dir'}
+    )
+    gpg_master_key = luigi.Parameter(
+        default_from_config={'section': 'event-export', 'name': 'gpg_master_key'}
+    )
+
     def requires(self):
         tasks = []
         for env in self.environment:
@@ -61,16 +69,25 @@ class EventExportTask(MultiOutputMapReduceJobTask):
     def requires_local(self):
         return ExternalURL(url=self.config)
 
-    def init_mapper(self):
+    def extra_modules(self):
+        return [gnupg, yaml]
+
+    def init_local(self):
         with self.input_local().open() as config_input:
             config_data = yaml.load(config_input)
             self.organizations = config_data['organizations']
 
-        self.org_id_whitelist = set(self.organizations.keys())
-        for _org_id, org_config in self.organizations.iteritems():
+        # Map org_ids to recipient names, taking in to account org_id aliases. For example, if an org_id Foo is also
+        # known as FooX then two entries will appear in this dictionary ('Foo', 'recipient@foo.org') and
+        # ('FooX', 'recipient@foo.org'). Note that both aliases map to the same recipient.
+        self.recipient_for_org_id = {}
+        for org_id, org_config in self.organizations.iteritems():
+            recipient = org_config['recipient']
+            self.recipient_for_org_id[org_id] = recipient
             for alias in org_config.get('other_names', []):
-                self.org_id_whitelist.add(alias)
+                self.recipient_for_org_id[alias] = recipient
 
+        self.org_id_whitelist = self.recipient_for_org_id.keys()
         log.debug('Using org_id whitelist ["%s"]', '", "'.join(self.org_id_whitelist))
 
         self.server_name_whitelist = set()
@@ -101,18 +118,23 @@ class EventExportTask(MultiOutputMapReduceJobTask):
         if date_string < self.lower_bound_date_string or date_string >= self.upper_bound_date_string:
             return
 
-        server_id = self.get_server_id()
-
         org_id = self.get_org_id(event)
         if org_id not in self.org_id_whitelist:
-            log.debug('Unrecognized organization: server_id=%s org_id=%s', server_id or '', org_id or '')
+            log.debug('Unrecognized organization: org_id=%s', org_id or '')
             return
 
+        server_id = self.get_server_id()
         if server_id not in self.server_name_whitelist:
-            log.debug('Unrecognized server: server_id=%s org_id=%s', server_id or '', org_id or '')
+            log.debug('Unrecognized server: server_id=%s', server_id or '')
             return
 
-        yield (date_string, org_id, server_id), line
+        key = (date_string, org_id, server_id)
+        # Enforce a standard encoding for the parts of the key. Without this a part of the key might appear differently
+        # in the key string when it is coerced to a string by luigi. For example, if the same org_id appears in two
+        # different records, one as a str() type and the other a unicode() then without this change they would appear as
+        # u'FooX' and 'FooX' in the final key string. Although python doesn't care about this difference, hadoop does,
+        # and will bucket the values separately. Which is not what we want.
+        yield tuple([value.encode('utf8') for value in key]), line.strip()
 
     def get_server_id(self):
         """
@@ -190,13 +212,24 @@ class EventExportTask(MultiOutputMapReduceJobTask):
             self.output_root,
             org_id,
             server_id,
-            '{date}_{org}.log'.format(
+            '{date}_{org}.log.gpg'.format(
                 date=date,
                 org=org_id,
             )
         )
 
-    def multi_output_reducer(self, _key, values, output_file):
+    def multi_output_reducer(self, key, values, output_file):
+        _date_string, org_id, _server_id = key
+        recipients = self._get_recipients(org_id)
+        key_file_targets = [get_target_from_url(url_path_join(self.gpg_key_dir, recipient)) for recipient in recipients]
+        with make_encrypted_file(output_file, key_file_targets) as encrypted_output_file:
             for value in values:
-            output_file.write(value.strip())
-            output_file.write('\n')
+                encrypted_output_file.write(value.strip())
+                encrypted_output_file.write('\n')
+
+    def _get_recipients(self, org_id):
+        """Get the correct recipients for the specified organization."""
+        recipients = [self.recipient_for_org_id[org_id]]
+        if self.gpg_master_key is not None:
+            recipients.append(self.gpg_master_key)
+        return recipients

edx/analytics/tasks/tests/acceptance/__init__.py

 import boto
+import gnupg
 import json
 import logging
 import os
+import shutil
 import subprocess
 import sys
+import tempfile
 if sys.version_info[:2] <= (2, 6):
     import unittest2 as unittest
 else:
@@ -38,3 +41,26 @@ class AcceptanceTestCase(unittest.TestCase):
         """Execute a subprocess and log the command before running it."""
         log.info('Running subprocess {0}'.format(command))
         subprocess.check_call(command)
+
+    def decrypt_file(self, encrypted_filename, decrypted_filename, key_filename='insecure_secret.key'):
+        """
+        Decrypts an encrypted file.
+
+        Arguments:
+            encrypted_filename (str): The full path to the PGP encrypted file.
+            decrypted_filename (str): The full path of the the file to write the decrypted data to.
+            key_filename (str): The name of the key file to use to decrypt the data. It should correspond to one of the
+                keys found in the gpg-keys directory.
+
+        """
+        gpg_home_dir = tempfile.mkdtemp()
+        try:
+            gpg = gnupg.GPG(gnupghome=gpg_home_dir)
+            gpg.encoding = 'utf-8'
+            with open(os.path.join('gpg-keys', key_filename), 'r') as key_file:
+                gpg.import_keys(key_file.read())
+
+            with open(encrypted_filename, 'r') as encrypted_file:
+                gpg.decrypt_file(encrypted_file, output=decrypted_filename)
+        finally:
+            shutil.rmtree(gpg_home_dir)

edx/analytics/tasks/tests/acceptance/test_database_export.py

@@ -14,6 +14,7 @@ import textwrap
 import shutil
 import subprocess
 
+import gnupg
 import oursql
 
 from edx.analytics.tasks.url import get_target_from_url
@@ -287,22 +288,13 @@ class ExportAcceptanceTest(AcceptanceTestCase):
         os.makedirs(gpg_dir)
         os.chmod(gpg_dir, 0700)
 
-        import_key_command = [
-            'gpg',
-            '--homedir', gpg_dir,
-            '--armor',
-            '--import', 'gpg-keys/insecure_secret.key'
-        ]
-        self.call_subprocess(import_key_command)
+        gpg = gnupg.GPG(gnupghome=gpg_dir)
+        with open(os.path.join('gpg-keys', 'insecure_secret.key'), 'r') as key_file:
+            gpg.import_keys(key_file.read())
 
         exported_file_path = os.path.join(validation_dir, self.exported_filename)
-        decrypt_file_command = [
-            'gpg',
-            '--homedir', gpg_dir,
-            '--output', exported_file_path,
-            '--decrypt', os.path.join(validation_dir, export_id, self.exported_filename + '.gpg'),
-        ]
-        self.call_subprocess(decrypt_file_command)
+        with open(os.path.join(validation_dir, export_id, self.exported_filename + '.gpg'), 'r') as encrypted_file:
+            gpg.decrypt_file(encrypted_file, output=exported_file_path)
 
         sorted_filename = exported_file_path + '.sorted'
         self.call_subprocess(['sort', '-o', sorted_filename, exported_file_path])

edx/analytics/tasks/tests/acceptance/test_event_export.py

@@ -8,6 +8,7 @@ import logging
 import tempfile
 import textwrap
 import time
+import shutil
 
 from luigi.s3 import S3Client, S3Target
 
@@ -57,7 +58,10 @@ class EventExportAcceptanceTest(AcceptanceTestCase):
 
         self.test_src = url_path_join(self.test_root, 'src')
         self.test_out = url_path_join(self.test_root, 'out')
-        self.test_config = url_path_join(self.test_root, 'config', 'default.yaml')
+        self.test_config_root = url_path_join(self.test_root, 'config')
+
+        self.test_config = url_path_join(self.test_config_root, 'default.yaml')
+        self.test_gpg_key_dir = url_path_join(self.test_config_root, 'gpg-keys')
 
         self.input_paths = {
             'prod': url_path_join(self.test_src, self.PROD_SERVER_NAME, 'tracking.log-20140515.gz'),
@@ -66,6 +70,7 @@ class EventExportAcceptanceTest(AcceptanceTestCase):
 
         self.upload_data()
         self.write_config()
+        self.upload_public_keys()
 
     def upload_data(self):
         src = os.path.join(self.data_dir, 'input', self.INPUT_FILE)
@@ -74,8 +79,7 @@ class EventExportAcceptanceTest(AcceptanceTestCase):
             gzip_file = gzip.open(temp_file.name, 'wb')
             try:
                 with open(src, 'r') as input_file:
-                    for line in input_file:
-                        gzip_file.write(line)
+                    shutil.copyfileobj(input_file, gzip_file)
             finally:
                 gzip_file.close()
 
@@ -100,9 +104,9 @@ class EventExportAcceptanceTest(AcceptanceTestCase):
                           - {server_2}
                     organizations:
                       edX:
-                        recipient: automation@example.com
+                        recipient: daemon@edx.org
                       AcceptanceX:
-                        recipient: automation@example.com
+                        recipient: daemon+2@edx.org
                     """
                     .format(
                         server_1=self.PROD_SERVER_NAME,
@@ -111,6 +115,15 @@ class EventExportAcceptanceTest(AcceptanceTestCase):
                 )
             )
 
+    def upload_public_keys(self):
+        gpg_key_dir = os.path.join('gpg-keys')
+        for key_filename in os.listdir(gpg_key_dir):
+            full_local_path = os.path.join(gpg_key_dir, key_filename)
+            remote_url = url_path_join(self.test_gpg_key_dir, key_filename)
+
+            if not key_filename.endswith('.key'):
+                self.s3_client.put(full_local_path, remote_url)
+
     def test_event_log_exports_using_manifest(self):
         with tempfile.NamedTemporaryFile() as temp_config_file:
             temp_config_file.write(
@@ -152,6 +165,8 @@ class EventExportAcceptanceTest(AcceptanceTestCase):
                 '--environment', 'prod',
                 '--environment', 'edge',
                 '--interval', '2014-05',
+                '--gpg-key-dir', self.test_gpg_key_dir,
+                '--gpg-master-key', 'daemon+master@edx.org',
                 '--n-reduce-tasks', str(self.NUM_REDUCERS),
             ]
         )
@@ -159,35 +174,46 @@ class EventExportAcceptanceTest(AcceptanceTestCase):
         self.call_subprocess(command)
 
     def validate_output(self):
-        # TODO: a lot of duplication here
-        comparisons = [
-            ('2014-05-15_edX.log', url_path_join(self.test_out, 'edX', self.PROD_SERVER_NAME, '2014-05-15_edX.log')),
-            ('2014-05-16_edX.log', url_path_join(self.test_out, 'edX', self.PROD_SERVER_NAME, '2014-05-16_edX.log')),
-            ('2014-05-15_edX.log', url_path_join(self.test_out, 'edX', self.EDGE_SERVER_NAME, '2014-05-15_edX.log')),
-            ('2014-05-16_edX.log', url_path_join(self.test_out, 'edX', self.EDGE_SERVER_NAME, '2014-05-16_edX.log')),
-            ('2014-05-15_AcceptanceX.log', url_path_join(self.test_out, 'AcceptanceX', self.EDGE_SERVER_NAME, '2014-05-15_AcceptanceX.log')),
-            ('2014-05-15_AcceptanceX.log', url_path_join(self.test_out, 'AcceptanceX', self.PROD_SERVER_NAME, '2014-05-15_AcceptanceX.log')),
-        ]
+        for server_id in [self.PROD_SERVER_NAME, self.EDGE_SERVER_NAME]:
+            for use_master_key in [False, True]:
+                self.validate_output_file('2014-05-15', 'edX', server_id, use_master_key)
+                self.validate_output_file('2014-05-16', 'edX', server_id, use_master_key)
+                self.validate_output_file('2014-05-15', 'AcceptanceX', server_id, use_master_key)
+
+    def validate_output_file(self, day, org_id, server_id, use_master_key=False):
+        if use_master_key:
+            key_filename = 'insecure_master_secret.key'
+        else:
+            if org_id == 'edX':
+                key_filename = 'insecure_secret.key'
+            else:
+                key_filename = 'insecure_secret_2.key'
+
+        self.temporary_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.temporary_dir)
 
-        for local_file_name, remote_url in comparisons:
-            with open(os.path.join(self.data_dir, 'output', local_file_name), 'r') as local_file:
-                remote_target = S3Target(remote_url)
+        self.downloaded_outputs = os.path.join(self.temporary_dir, 'output')
+        os.makedirs(self.downloaded_outputs)
+
+        local_file_name = '{day}_{org_id}.log'.format(day=day, org_id=org_id)
+        remote_url = url_path_join(self.test_out, org_id, server_id, local_file_name + '.gpg')
 
         # Files won't appear in S3 instantaneously, wait for the files to appear.
         # TODO: exponential backoff
-                found = False
         for _i in range(30):
-                    if remote_target.exists():
-                        found = True
+            key = self.s3_client.get_key(remote_url)
+            if key is not None:
                 break
             else:
                 time.sleep(2)
 
-                if not found:
+        if key is None:
             self.fail('Unable to find expected output file {0}'.format(remote_url))
 
-                with remote_target.open('r') as remote_file:
-                    local_contents = local_file.read()
-                    remote_contents = remote_file.read()
+        downloaded_output_path = os.path.join(self.downloaded_outputs, remote_url.split('/')[-1])
+        key.get_contents_to_filename(downloaded_output_path)
+
+        decrypted_file_name = downloaded_output_path[:-len('.gpg')]
+        self.decrypt_file(downloaded_output_path, decrypted_file_name, key_filename)
 
-                    self.assertEquals(local_contents, remote_contents)
+        self.call_subprocess(['diff', decrypted_file_name, os.path.join(self.data_dir, 'output', local_file_name)])

edx/analytics/tasks/tests/test_event_exports.py

@@ -7,7 +7,7 @@ from textwrap import dedent
 from cStringIO import StringIO
 
 from luigi.date_interval import Year
-from mock import MagicMock, patch
+from mock import MagicMock, patch, call
 import yaml
 
 from edx.analytics.tasks.event_exports import EventExportTask
@@ -37,10 +37,10 @@ class EventExportTestCase(unittest.TestCase):
         },
         'organizations': {
             'FooX': {
-                'recipient': 'automation@example.com'
+                'recipient': 'automation@foox.com'
             },
             'BarX': {
-                'recipient': 'automation@example.com',
+                'recipient': 'automation@barx.com',
                 'other_names': [
                     'BazX',
                     'bar'
@@ -58,16 +58,18 @@ class EventExportTestCase(unittest.TestCase):
             source='test://input/',
             environment=['edge', 'prod'],
             interval=Year.parse('2014'),
+            gpg_key_dir='test://config/gpg-keys/',
+            gpg_master_key='skeleton.key@example.com'
         )
 
         self.task.input_local = MagicMock(return_value=FakeTarget(self.CONFIGURATION))
 
     def test_org_whitelist_capture(self):
-        self.task.init_mapper()
+        self.task.init_local()
         self.assertItemsEqual(self.task.org_id_whitelist, ['FooX', 'BarX', 'BazX', 'bar'])
 
     def test_server_whitelist_capture(self):
-        self.task.init_mapper()
+        self.task.init_local()
         self.assertItemsEqual(self.task.server_name_whitelist, [self.SERVER_NAME_1, self.SERVER_NAME_2])
 
     def test_mapper(self):
@@ -111,7 +113,7 @@ class EventExportTestCase(unittest.TestCase):
 
         input_events = expected_output + excluded_events
 
-        self.task.init_mapper()
+        self.task.init_local()
 
         results = []
         for key, event_string in input_events:
@@ -207,17 +209,18 @@ class EventExportTestCase(unittest.TestCase):
 
     def test_output_path_for_key(self):
         path = self.task.output_path_for_key((datetime.date(2015, 1, 1), 'OrgX', 'prod-app-001'))
-        self.assertEquals('test://output/OrgX/prod-app-001/2015-01-01_OrgX.log', path)
+        self.assertEquals('test://output/OrgX/prod-app-001/2015-01-01_OrgX.log.gpg', path)
 
     def test_output_path_for_key_casing(self):
         path = self.task.output_path_for_key((datetime.date(2015, 1, 1), 'orgX', 'prod-app-001'))
-        self.assertEquals('test://output/orgX/prod-app-001/2015-01-01_orgX.log', path)
+        self.assertEquals('test://output/orgX/prod-app-001/2015-01-01_orgX.log.gpg', path)
 
-    def test_multi_output_reducer(self):
-        output = StringIO()
-        self.task.multi_output_reducer(None, ['a\t', 'b', 'c'], output)
-        output.seek(0)
-        self.assertEquals('a\nb\nc\n', output.read())
+    @patch('edx.analytics.tasks.event_exports.make_encrypted_file')
+    def test_multi_output_reducer(self, mock_make_encrypted_file):
+        self.task.multi_output_reducer((None, 'FooX', None), ['a\t', 'b', 'c'], None)
+
+        mock_encrypted_file = mock_make_encrypted_file.return_value.__enter__.return_value
+        self.assertEquals(mock_encrypted_file.write.mock_calls, [call(s) for s in ['a', '\n', 'b', '\n', 'c', '\n']])
 
     def test_local_requirements(self):
         self.assertEquals(self.task.requires_local().url, 'test://config/default.yaml')
@@ -239,7 +242,7 @@ class EventExportTestCase(unittest.TestCase):
             # Some coverage missing here, but it's probably good enough for now
 
     def test_unrecognized_environment(self):
-        self.task.init_mapper()
+        self.task.init_local()
 
         for server in ['prod-app-001', 'prod-app-002']:
             expected_output = [((self.EXAMPLE_DATE, 'FooX', server), self.EXAMPLE_EVENT)]
@@ -248,11 +251,11 @@ class EventExportTestCase(unittest.TestCase):
         self.assertItemsEqual(self.run_mapper_for_server_file('foobar', self.EXAMPLE_EVENT), [])
 
     def test_odd_file_paths(self):
-        self.task.init_mapper()
+        self.task.init_local()
 
         for path in ['something.gz', 'test://input/something.gz']:
             self.assertItemsEqual(self.run_mapper_for_file_path(path, self.EXAMPLE_EVENT), [])
 
     def test_missing_environment_variable(self):
-        self.task.init_mapper()
+        self.task.init_local()
         self.assertItemsEqual([output for output in self.task.mapper(self.EXAMPLE_EVENT) if output is not None], [])

gpg-keys/daemon+2@edx.org

+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.11 (GNU/Linux)
+
+mQENBFOEpb0BCADBVcxd+O3QreY0J00OIoy+s79QMzJUxhLr8y6HX5dpy/W2rBx6
+OkurkJm3ljGCYsV7U02Ri1Gn467DSw0YRuj4EUKsPL4BMB5n9wV6NbK185e5ar8M
+STr8gOIbpmCL8X21wGvWqWqp1mXVeWw62dnWwWrZHe9vGm6eemzimAHy6WWmpKeW
+riS/yUjOeUAIJLT7ecDZx/RkMAD3HF+b7vMyh2EK15QlgKWmHqNNZjgMKTYGHp4h
+OKvt1PSPegPzVrpPpIpDL1zqwyAiJwIE2T+dfjrJq+lcKzprBuQVO1pY5luu/nr+
+UgY+OkhtwkhnDXmk+VzQI5yj+Uwd2549XT1BABEBAAG0V1NlY29uZCBBY2NlcHRh
+bmNlIFRlc3RlciAoSW5zZWN1cmUga2V5IHVzZWQgZm9yIGFjY2VwdGFuY2UgdGVz
+dGluZy4pIDxkYWVtb24rMkBlZHgub3JnPokBOAQTAQIAIgUCU4SlvQIbAwYLCQgH
+AwIGFQgCCQoLBBYCAwECHgECF4AACgkQSf/Gdh00iKyoYAf/Wg3ahOjE3eIY6b12
+f0SWUhIBXoGzwwlrI3E2nnBmC7c428p40I4LcByogPD6YGGu15DSZGzm4ERyGvp2
+PP+hwXXI6kiHpWQ1jsShqvcY5kG0IULn62puFIrZ49NxlfiEAGcehIQ7ksHLD+7P
+/RMjy/Ho97Hk3g5f0hU2XMoZknkg3VyR7lFi6r64MhFKweH+m+/MGmwngXYnKU0q
++gTaha7zHGeAvX+29uMbd2bzD5q3H43EcOf+5vU/TGLfGwOeZc5n4fCQWidDDAIt
+grbrcjsVPK/PiR6eXGnki2+f0d1Pja9v232hiKw5Ram6nLC/9qttEITUTWRJS0M8
+jOn+rLkBDQRThKW9AQgAs0yQqVQwE/qbzMN4hHwGB3AfpeuErAtMcRVEQcJscPUB
+58ALjN+M7LxMos27AeuFaM9isg2JDcPG9AAc/s8i+2BVjSC/OOSEcoACAECOPN10
+gn/xM9l3Ap9rwcaQGT4WocYIvosw0F6MDdpASh887qqvh5lu4QUmb/0R5Ou+ak9H
+Y3/ILiWqznloMkVhwMRUsink0p+OIsOdc8UfnIZnJ7VPwip/HF3LgBDAMZsFvP2R
+FwVlQNoZJkP1y9toOq6AbbEf0ed9LhevRoR1Tmx5m0BtY7oUEveFpBqmBSqrUZ0c
+t8SEX57QwAtsHy4oI4c3R8tFGvmalOovjAX7bKGwCwARAQABiQEfBBgBAgAJBQJT
+hKW9AhsMAAoJEEn/xnYdNIisEOIIAIJ+2d5s+5tnIGdhEEVu0cNKaBVfR6RmbjQV
+Gv074uUz9tTu71DZrTYkoa1awILLi/vSps5FzywKcflEd4jRLvLxtMAv0e9wFq8L
+/fqqBrKDzN9RuuGAaW74zUiJMleDfvHeZsshV1uYdE2+WvVB8qL2oIGg7eATuDG9
+Y2RLcYPb/iqoB4kzJA674Sup+2QoAB9n7dS6lUDmLX7KtJn8mURuWSCy3yHUGy9m
+Y+6C6oDw5R6uJYCTR/EeH1dFqr3uDRKj+7NLKu5GuVH0uSMr4vnQ+wjPqTlYPCoA
+hfJLJWU0JQNj0bofD9Cd1+I0iIUbToCbZYhwXs8LV17T5/rufhg=
+=q/aw
+-----END PGP PUBLIC KEY BLOCK-----

gpg-keys/daemon+master@edx.org

+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.11 (GNU/Linux)
+
+mQENBFOEpJEBCAC00MeA2FMUnKwtYUPp1pRCW0k2Fs8zJGySZ+7E2sUPDTM/fb79
+84tJ2pJvuaAm7KXJPDjZyMYPCe2VEDF+QDBx+xJMUfJ32wsBfIZFUaLe6VNj6usE
+vAHb37qr13yU29Rtge1vvqh4HAqoTuSkfSdGjHzSjy/aKkteLHvvXCoKf3/W4YNr
+Iiij55oJ48v/d8G2bPdE3i7hhHr/CsIatMDItxAm8BV2bs9Pc2kIzn80CcN4OEsN
+au/hy5HOAKiuMIWtbXiYVPF2D9f9OSWhwry6/pW6JVwCU03KqDFp6zRL2ql+tSat
+f2X8lE2GUJriyYaEwAQmUc7pOVaONslh6j7xABEBAAG0PkF1dG9tYXRlZCBNYXN0
+ZXIgKEluc2VjdXJlIE1hc3RlciBLZXkpIDxkYWVtb24rbWFzdGVyQGVkeC5vcmc+
+iQE4BBMBAgAiBQJThKSRAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRBb
+xJowfVpQ0q7+CACa/EKS3eOO80Jy6joV2SFebu3nN7Owkbz9kONgAdbNYGOPgOh6
+eNi7pDjRN8JWKkmeNDLFIdehw1I90qLla0HZ8xMfRW73ZA/C/Q9O3/C/Hivz2A9H
+p9Jott3TUqpdF7TuZaqBNtljyCCLA8pnbjpkO1GluaNP/KcR/8un58/gnubcQ8ya
+YLYKl50+TqF64ovzJdUHIWN2HOhz6andrI4VE+//tRX09uhpbGHs9aQjmveLltUy
+o043SBp2kIWGppeHS4SvUr/qRKEWHXW+P61envZdKXfn3MTEIeozK0PeSRDaYDEL
+A8FYOSS7cbycuHiopqNy8LDQIMBwnHDrpQE0uQENBFOEpJEBCACkrs1aZKmmijrr
++GzxrnLfQtgFVHilNyFYY99bEdyaMPRK4MiaHQlhHm0eKy/bTg3+3WhTAvlW6F52
+hPqQL1QDdidgUwhTMQ2bGvkdzYw0rGiPCvV2Jy3fd807+QvJJ0iWvYzrn0BQ9q4c
+o2Mff3+hdQMmQUawshqHGcbRgHLIsWU8YPLj+J7IVgRHJyCHoQ9ARzJpqsheejPo
+7+5d4YZCJho2T/cgx/9Ft/wWeLgD7VqFg6RydoXLj5jNYJSB00TxSQHcxHZSBdpd
+lbOkQopJJXQEQ6CCN3eNGKXqJjhBMnr8QBXn5ahL67FF1CGCiwSJQRE3vaTKK5wF
+6nB77s8tABEBAAGJAR8EGAECAAkFAlOEpJECGwwACgkQW8SaMH1aUNK81AgAqwip
+OPRoUu5TjI88Pvfi8AMy1NQg74Nc+2yWaJ+yr58Hrk20eaFqhDyCxN6UMcKi3Dab
+60tsYzXaw/wqEclGAXlf95HCXYId0I+qvkvsAHoTJiURxWhP0vyRVYmy5VvNeCWq
+sO0hdMxkQUhWKF9IA3YzB1od+2ibYOlh18BBdS42irSapF0Y3MR/spVI7cOyf02C
+pa6oXrJqMcUqGfQPfRMKMIaFNPJLF3sRVIbWduzrmUNWDkNizM+Jp/ZdhtaIpXv0
+sF2Y8yc77HqMwh8no8h2AvoWOFhWvo+22gPimupN0LVpvZKxf7hxycB4DVosK3GT
+7BUXDA0tieXsYjoZcg==
+=wr9g
+-----END PGP PUBLIC KEY BLOCK-----

gpg-keys/insecure_master_secret.key

+-----BEGIN PGP PRIVATE KEY BLOCK-----
+Version: GnuPG v1.4.11 (GNU/Linux)
+
+lQOYBFOEpJEBCAC00MeA2FMUnKwtYUPp1pRCW0k2Fs8zJGySZ+7E2sUPDTM/fb79
+84tJ2pJvuaAm7KXJPDjZyMYPCe2VEDF+QDBx+xJMUfJ32wsBfIZFUaLe6VNj6usE
+vAHb37qr13yU29Rtge1vvqh4HAqoTuSkfSdGjHzSjy/aKkteLHvvXCoKf3/W4YNr
+Iiij55oJ48v/d8G2bPdE3i7hhHr/CsIatMDItxAm8BV2bs9Pc2kIzn80CcN4OEsN
+au/hy5HOAKiuMIWtbXiYVPF2D9f9OSWhwry6/pW6JVwCU03KqDFp6zRL2ql+tSat
+f2X8lE2GUJriyYaEwAQmUc7pOVaONslh6j7xABEBAAEAB/wL8mV+z7bsPDkrcGtG
+e906OyzYVuPO1b/EtqWcj8szEOS2HmQBryygrxaQ15DLhu73Sl7ZPj5JMmyRC6EG
+3ffgIr4qmRmDriSJLxtHYOWkLoIfg8nj0LDwwFS6/kG8K4AwJ28IPJkwrRhOgZPy
+2FVsKw548mlaKVmRzy+geOmgMtWjBRLYaIu7tMHyAlGRf7ngRX2cSSZ9tm1/ddX4
+SDH79u3m46uVcOuev0rFfXAE7VHhcMxu7PKLoWiNyUKNfNa3BF3zvrcqcQPYgNyp
+XamY6KgCWpaPxoxeQzFnAABc6mfIl5YGAYpFZ+ATCh51QCPwpOvh29l+lfyDMNai
+jJXHBADQIV20fMDevgmIMR7nKSA5Pt3hypioB92KXGC89acTRfCP4g0IVv7J2c3r
+LJYJjta1f2ZvO/M+LVN4kQnaSYRwKNsZItZHNQ2YvLEsiOus1Scpkz4UCEJXEOUV
+SsI84ksAEAWOPpdCd/YEZLxalGsn4bsOLEAT8KVmUoUubq62SwQA3mcg/LX+54bf
+Lx17DFwpojlNry73eMmnsjOJuZUOfV4lTt2AUn5iXu2OnvNpr7WTSzcA5/G+jST3
+tVVVJFRU9+rEEzUDrkZRo2aF9ATZIivsWxhftDlWh6FhnvFyNuwQFqqzV72SfKKN
+0mc9oWimzN6Anm7T4TIr/9mh1JvXCjMD/2+3Ql6btZOWipu/eCSD5mY0JKyeNE2l
+xcYtJdkmDOp7Ygz0Z236qL0dGAKD7qifZBX5ZR6gxu9SAMv7Hu1j5KAOugAs2N9z
+2oiZcCbRgB8bOnQ4r7mOTp94JGB37z53dKdJm0XMwnw4euDqbt624hDBLNX8P2o3
+7Te5Qp0LUN0uQyu0PkF1dG9tYXRlZCBNYXN0ZXIgKEluc2VjdXJlIE1hc3RlciBL
+ZXkpIDxkYWVtb24rbWFzdGVyQGVkeC5vcmc+iQE4BBMBAgAiBQJThKSRAhsDBgsJ
+CAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRBbxJowfVpQ0q7+CACa/EKS3eOO80Jy
+6joV2SFebu3nN7Owkbz9kONgAdbNYGOPgOh6eNi7pDjRN8JWKkmeNDLFIdehw1I9
+0qLla0HZ8xMfRW73ZA/C/Q9O3/C/Hivz2A9Hp9Jott3TUqpdF7TuZaqBNtljyCCL
+A8pnbjpkO1GluaNP/KcR/8un58/gnubcQ8yaYLYKl50+TqF64ovzJdUHIWN2HOhz
+6andrI4VE+//tRX09uhpbGHs9aQjmveLltUyo043SBp2kIWGppeHS4SvUr/qRKEW
+HXW+P61envZdKXfn3MTEIeozK0PeSRDaYDELA8FYOSS7cbycuHiopqNy8LDQIMBw
+nHDrpQE0nQOYBFOEpJEBCACkrs1aZKmmijrr+GzxrnLfQtgFVHilNyFYY99bEdya
+MPRK4MiaHQlhHm0eKy/bTg3+3WhTAvlW6F52hPqQL1QDdidgUwhTMQ2bGvkdzYw0
+rGiPCvV2Jy3fd807+QvJJ0iWvYzrn0BQ9q4co2Mff3+hdQMmQUawshqHGcbRgHLI
+sWU8YPLj+J7IVgRHJyCHoQ9ARzJpqsheejPo7+5d4YZCJho2T/cgx/9Ft/wWeLgD
+7VqFg6RydoXLj5jNYJSB00TxSQHcxHZSBdpdlbOkQopJJXQEQ6CCN3eNGKXqJjhB
+Mnr8QBXn5ahL67FF1CGCiwSJQRE3vaTKK5wF6nB77s8tABEBAAEAB/kBl64lMGGC
+moYY15FoIeV6+ri/jnJPOLICGo8joI/XTt9h5PwTn3HChmqMNuMy1fWlMAts+BOk
+r0EQmNcac1a25NrbH9puAYSt6gwcKWtwa/Vj4rl+b8EODujFanJeetqFGKam9aaH
+0ebboInaGV/I1iqWYV23YjWG9m0ZhAo4HhkRqhMjCldDK0cGnxDMkEV3CHVOTdyA
+WInvh0j+C9SL7d3M3cQLUytkEHLttZAPfNWzaIt+3UHzmA1clDbt6telqpwYIk6r
+WspAnCDAPqX4Uw2IitB3+ZCYNkudnVu3SIDHayeqX9ow+DuYiavbb2QKTugRH4ZU
+e30uzq6zDp+JBADNCMftNBqoew+ASecNKflyu5rS+e4xh1Isy6w6mfx09eHhpyJI
+cXz3CH6/4DvXfe5XKnVBj1Xl6/vtEk9/kgAX0EPF0bFAMju+zcshSl98S3ossrsY
+wBNfmQJDHrdWKQIlOvEDm76542u6TntCH8JxkJI1fuJfvGQ6MNEXS70aGQQAzZ5I
+NmqzUxqqT+vX/bPeVZqeTL/ATaSLyhdrR8cysD8Si0YjhNb+zSs2NWHEMMl/SloM
+Tes0fgB/MEXKfTPHo0d+UAqCRlFKA5FA4frMcq9lQGm7BN1K8fjG+rbXrJ2apfhF
+pVJOFl6iRgezMlqA4kQkJ1lJXRXuS8x3+D7oqDUD/iFa2hHSFuoMr693dFmnbqL2
+xC1Oytr7DR+NSVVjie1BKubGMNNwfQu0k8YilE/tUnuj9wZRt/hejZZS7ZPWEbo9
+Au8FsHyfkQWTSvA0eUaxcyI8xbFPQ8KAG1Bq2GVqC/yiZvU9Lrco1KVKw31Wiuwj
+MZ6H+RDpOG1Oxt15m2AlO8qJAR8EGAECAAkFAlOEpJECGwwACgkQW8SaMH1aUNK8
+1AgAqwipOPRoUu5TjI88Pvfi8AMy1NQg74Nc+2yWaJ+yr58Hrk20eaFqhDyCxN6U
+McKi3Dab60tsYzXaw/wqEclGAXlf95HCXYId0I+qvkvsAHoTJiURxWhP0vyRVYmy
+5VvNeCWqsO0hdMxkQUhWKF9IA3YzB1od+2ibYOlh18BBdS42irSapF0Y3MR/spVI
+7cOyf02Cpa6oXrJqMcUqGfQPfRMKMIaFNPJLF3sRVIbWduzrmUNWDkNizM+Jp/Zd
+htaIpXv0sF2Y8yc77HqMwh8no8h2AvoWOFhWvo+22gPimupN0LVpvZKxf7hxycB4
+DVosK3GT7BUXDA0tieXsYjoZcg==
+=vBsX
+-----END PGP PRIVATE KEY BLOCK-----

gpg-keys/insecure_secret_2.key

+-----BEGIN PGP PRIVATE KEY BLOCK-----
+Version: GnuPG v1.4.11 (GNU/Linux)
+
+lQOYBFOEpb0BCADBVcxd+O3QreY0J00OIoy+s79QMzJUxhLr8y6HX5dpy/W2rBx6
+OkurkJm3ljGCYsV7U02Ri1Gn467DSw0YRuj4EUKsPL4BMB5n9wV6NbK185e5ar8M
+STr8gOIbpmCL8X21wGvWqWqp1mXVeWw62dnWwWrZHe9vGm6eemzimAHy6WWmpKeW
+riS/yUjOeUAIJLT7ecDZx/RkMAD3HF+b7vMyh2EK15QlgKWmHqNNZjgMKTYGHp4h
+OKvt1PSPegPzVrpPpIpDL1zqwyAiJwIE2T+dfjrJq+lcKzprBuQVO1pY5luu/nr+
+UgY+OkhtwkhnDXmk+VzQI5yj+Uwd2549XT1BABEBAAEAB/oCDL2oHhGwvYnsHKHk
+V/kZCSvVQa2zbg7Y9zuTji+7R0EkKNVRIG7D82UF0sRUnGUj8RaonTWw6344rizb
+MzPuHrS6bGxxSR18FFziVlETknEV3gFc3nvkcx4H6q7UXQDsXTgSJGVam/N5chpJ
+J3J3+etUQPfUZuSBrqJFPBUCinKO2tLCDndLL1wi7S/IJklFeza1//YZvwMdx7rU
+hoTB5KX5ZmBnHI9snJMtZvJ8i2EW8u6km0DjTjUnpZZZcojRVwhG3UQeQtSUxWRv
+2wh9gsbh3Cnz9rqLTDZiK2gmIlx23n2mHMQR+ELO23RDFQHwE/BEpYf0usJ9/jr5
+0YtJBADYlgyRUkxLog3XjOHmDk27sHS1ZDamDokzNXMPYMsDj2Hn4t6Pq/1nvS+e
+WkEdCLaZvQ/TajN+aT13IAVOImyrABemGcs+Eo7PklzYB2c8UIb/XTpTWHGt3i5u
+1tzrgP+JJRZdliYS9JnRizIDLSDOrc25zlEQiAUlV5JuHyevRQQA5ISSVa6XwYrL
+Gu/s739fM1h5F4nzw3tAbgpMrE6J8wwP2JAMfo2q1dNRtulln1h7hrvg05rAGTt7
+3z8py/iAlxA8KUf0jYyCmoSfp/biz3VFr7yfVzYCriEbYn86o5NehuzJSFpTTjQy
+ciy00v+c1U3MROcAMf4Xr17L/kzPB80EAKR6ZYl8+4g0OPZuxKISIH/8RaRx5aq6
+12rTERWHOQm0Kns/u8fZBKhwdjOAGEfi/KGlaISND9aSHc5QDrm7luK8y1vKVI3R
+LYPjh9BpB6TCVe547h4lCy8ps2MOFiqdpAnQv5dpEA/rbFZgPCF+PtA/fyw/DUMj
+h95nB5bTgFqzNhC0V1NlY29uZCBBY2NlcHRhbmNlIFRlc3RlciAoSW5zZWN1cmUg
+a2V5IHVzZWQgZm9yIGFjY2VwdGFuY2UgdGVzdGluZy4pIDxkYWVtb24rMkBlZHgu
+b3JnPokBOAQTAQIAIgUCU4SlvQIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AA
+CgkQSf/Gdh00iKyoYAf/Wg3ahOjE3eIY6b12f0SWUhIBXoGzwwlrI3E2nnBmC7c4
+28p40I4LcByogPD6YGGu15DSZGzm4ERyGvp2PP+hwXXI6kiHpWQ1jsShqvcY5kG0
+IULn62puFIrZ49NxlfiEAGcehIQ7ksHLD+7P/RMjy/Ho97Hk3g5f0hU2XMoZknkg
+3VyR7lFi6r64MhFKweH+m+/MGmwngXYnKU0q+gTaha7zHGeAvX+29uMbd2bzD5q3
+H43EcOf+5vU/TGLfGwOeZc5n4fCQWidDDAItgrbrcjsVPK/PiR6eXGnki2+f0d1P
+ja9v232hiKw5Ram6nLC/9qttEITUTWRJS0M8jOn+rJ0DmARThKW9AQgAs0yQqVQw
+E/qbzMN4hHwGB3AfpeuErAtMcRVEQcJscPUB58ALjN+M7LxMos27AeuFaM9isg2J
+DcPG9AAc/s8i+2BVjSC/OOSEcoACAECOPN10gn/xM9l3Ap9rwcaQGT4WocYIvosw
+0F6MDdpASh887qqvh5lu4QUmb/0R5Ou+ak9HY3/ILiWqznloMkVhwMRUsink0p+O
+IsOdc8UfnIZnJ7VPwip/HF3LgBDAMZsFvP2RFwVlQNoZJkP1y9toOq6AbbEf0ed9
+LhevRoR1Tmx5m0BtY7oUEveFpBqmBSqrUZ0ct8SEX57QwAtsHy4oI4c3R8tFGvma
+lOovjAX7bKGwCwARAQABAAf8CmlIPyTd09MqB9ZT4bGH8hefJtXJsunHCwv2ljDx
+dg+sCCZ9JTV36+k1NADpOn/QE9ly/C7QBgYXv+RfyqYBfIJIBXVg1S9jx3hKZXRa
+LguX79n5Cg83G2ZhbfOXO3gA/WzP5iyT7Y2H6WgCcqtksOElZyTQLO3NIRSbXOUZ
+ffe6onqbaOT3VHAWwVhESCIo+ctbzNgHSelf/Gcbh8W30Pk6hpDK/XBLYED5MGmM
+ufUuhIkL/4YGNEI+x2bJNgf4e8GsSMTMNzZTbZSnv4+BVo5+WE17Wp1ne/TTAfYF
+JyXnBZVKLK4L6jeVWK15svKg9YXkNrzQ2Qa553Y7+PKSGQQAzttdhRBGSWnl/0ND
+/R33Amhrw2Lj8ENT/Uzc9j9lx6+WYVpRaaPKQYpsHwONaVM5l9lCqI7sJxmJiPjx
+3I0ZUP6mYbK0V+4gMPYU54MKkD2g9OOWsfCslqE4rYrtz5CM7E63E3453MsdnIr0
+6QRqHKcIg+5NcJ13v1PAB7a0a3MEAN3lL+RdhPP7o3Do4d48LxeysloQqCH/RyKj
+nwJA9TGJEfKw8z+iZlhgpdgl5bW9fZN6q5f11o2yJrIAlcB/hgPCcpKikxvUzRps
+R8ZJohhRFpSqzLpJQkhbSi0H12fvizdoU0A+2kRtyBFJBs00A0/URE1+RTMo4uL4
+ufchFjMJA/9lNSEGuiK+4s2EbhdwqeABXcD/9wsSdOH/o8MtexLHOri7XoAnVKYn
+3pBBClLtixW2874MCPwDh95uvDIcj1zBOl8W4liArJeidHhp/xWP8wmu4h1mkK58
+5qZrEWL+6MXgUMOIFBVPApkpKJ/FPP+FGN5uuDVMM0ZEKj/sYCUvIUCLiQEfBBgB
+AgAJBQJThKW9AhsMAAoJEEn/xnYdNIisEOIIAIJ+2d5s+5tnIGdhEEVu0cNKaBVf
+R6RmbjQVGv074uUz9tTu71DZrTYkoa1awILLi/vSps5FzywKcflEd4jRLvLxtMAv
+0e9wFq8L/fqqBrKDzN9RuuGAaW74zUiJMleDfvHeZsshV1uYdE2+WvVB8qL2oIGg
+7eATuDG9Y2RLcYPb/iqoB4kzJA674Sup+2QoAB9n7dS6lUDmLX7KtJn8mURuWSCy
+3yHUGy9mY+6C6oDw5R6uJYCTR/EeH1dFqr3uDRKj+7NLKu5GuVH0uSMr4vnQ+wjP
+qTlYPCoAhfJLJWU0JQNj0bofD9Cd1+I0iIUbToCbZYhwXs8LV17T5/rufhg=
+=tWj+
+-----END PGP PRIVATE KEY BLOCK-----

setup.cfg

@@ -31,7 +31,6 @@ edx.analytics.tasks =
     export-student-module = edx.analytics.tasks.database_exports:StudentModulePerCourseAfterImportWorkflow
     last-country = edx.analytics.tasks.user_location:LastCountryForEachUser
     export-events = edx.analytics.tasks.event_exports:EventExportTask
-    encrypt-exports = edx.analytics.tasks.encrypt:FakeEventExportWithEncryptionTask
 
 mapreduce.engine =
     hadoop = edx.analytics.tasks.mapreduce:MapReduceJobRunner