"""End to end test of obfuscation."""

import datetime
import json
import logging
import os
import shutil
import tarfile
import tempfile

from edx.analytics.tasks.tests.acceptance import AcceptanceTestCase, as_list_param, when_geolocation_data_available
from edx.analytics.tasks.tests.acceptance.services import fs, shell
from edx.analytics.tasks.util.file_util import copy_file_to_file
from edx.analytics.tasks.util.opaque_key_util import get_filename_safe_course_id
from edx.analytics.tasks.util.url import url_path_join

log = logging.getLogger(__name__)


class ObfuscationAcceptanceTest(AcceptanceTestCase):
    """Validate data flow for obfuscation process."""

    COURSE_ID = 'course-v1:edX+DemoX+Demo_Course'
    INPUT_FILE = 'obfuscation_tracking.log'
    EXPORT_DATE = '2015-11-13'
    INTERVAL = '2015-10-01-2015-12-31'
    FORMAT_VERSION = 'rdx-test'
    PIPELINE_VERSION = '9c3b6bc9cf1c1b26e39b6eba2ca0d1e76829862d'

    def setUp(self):
        super(ObfuscationAcceptanceTest, self).setUp()
        self.temporary_dir = tempfile.mkdtemp()
        self.addCleanup(shutil.rmtree, self.temporary_dir)
        self.dump_root = url_path_join(self.test_src, 'course_exports', 'raw')
        self.filename_safe_course_id = get_filename_safe_course_id(self.COURSE_ID)
        self.test_gpg_key_dir = url_path_join(self.test_root, 'gpg-keys')

    def setup_state_files(self):
        """Upload input fixture data files, needed to mimic the output produced by course-exporter which is not a part of this test."""

        state_files_dir = os.path.join(self.data_dir, 'input', 'obfuscation', 'state')
        for filename in os.listdir(state_files_dir):
            local_filepath = os.path.join(state_files_dir, filename)
            dst_url = url_path_join(self.dump_root, self.filename_safe_course_id, 'state', self.EXPORT_DATE, filename)
            self.upload_file(local_filepath, dst_url)

    def upload_gpg_keys(self):
        """Uploads test gpg keys, needed for encryption."""
        gpg_key_dir = os.path.join('gpg-keys')
        for key_filename in os.listdir(gpg_key_dir):
            local_filepath = os.path.join(gpg_key_dir, key_filename)
            destination_url = url_path_join(self.test_gpg_key_dir, key_filename)

            if not key_filename.endswith('.key'):
                self.upload_file(local_filepath, destination_url)

    def upload_profile_data(self):
        obfuscation_dir = os.path.join(self.data_dir, 'input', 'obfuscation')
        for filename in ['auth_user', 'auth_userprofile']:
            local_filepath = os.path.join(obfuscation_dir, filename)
            dst_url = url_path_join(self.test_root, 'warehouse', filename)
            self.upload_file(local_filepath, dst_url)

    def run_event_export_task(self):
        self.upload_tracking_log(self.INPUT_FILE, datetime.date(2015, 10, 15))
        self.task.launch([
            'EventExportByCourseTask',
            '--output-root', self.dump_root,
            '--interval', self.INTERVAL,
            '--n-reduce-tasks', str(self.NUM_REDUCERS)
        ])

    @when_geolocation_data_available
    def test_obfuscation(self):
        """Test obfuscation workflow."""
        self.run_event_export_task()
        self.setup_state_files()
        self.upload_profile_data()
        self.upload_gpg_keys()
        self.run_obfuscation_task()
        self.run_obfuscated_package_task()
        self.maxDiff = None
        self.validate_obfuscation()

    def run_obfuscation_task(self):
        """Run ObfuscatedCourseTask."""
        self.task.launch([
            'ObfuscatedCourseTask',
            '--course', self.filename_safe_course_id,
            '--dump-root', self.dump_root,
            '--obfuscated-output-root', url_path_join(self.test_root, 'obfuscated-output'),
            '--format-version', self.FORMAT_VERSION,
            '--pipeline-version', self.PIPELINE_VERSION,
            '--auth-user-path', url_path_join(self.test_root, 'warehouse', 'auth_user'),
            '--auth-userprofile-path', url_path_join(self.test_root, 'warehouse', 'auth_userprofile')
        ])

    def run_obfuscated_package_task(self):
        """Run ObfuscatedPackageTask."""
        self.task.launch([
            'ObfuscatedPackageTask',
            '--course', self.filename_safe_course_id,
            '--obfuscated-output-root', url_path_join(self.test_root, 'obfuscated-output'),
            '--gpg-key-dir', self.test_gpg_key_dir,
            '--gpg-master-key', 'daemon+master@edx.org',
            '--output-root', self.test_out,
            '--recipient', as_list_param('daemon@edx.org'),
            '--format-version', self.FORMAT_VERSION
        ])

    def validate_obfuscation(self):
        """Validates obfuscation workflow."""
        output_target = self.get_targets_from_remote_path(self.test_out, '*.tar.gz.gpg')[0]
        output_filename = os.path.basename(output_target.path)
        temp_output_filepath = os.path.join(self.temporary_dir, output_filename)

        with output_target.open('r') as input_file:
            with open(temp_output_filepath, 'w') as output_file:
                copy_file_to_file(input_file, output_file)

        decrypted_filepath = temp_output_filepath[:-len('.gpg')]
        fs.decrypt_file(temp_output_filepath, decrypted_filepath, 'insecure_secret.key')

        with tarfile.open(decrypted_filepath, 'r:gz') as tfile:
            tfile.extractall(self.temporary_dir)

        # Validate package metadata info.
        metadata_filepath = os.path.join(self.temporary_dir, 'metadata_file.json')
        with open(metadata_filepath) as metadata_file:
            metadata_info = json.load(metadata_file)
        self.assertItemsEqual(metadata_info['format_version'], self.FORMAT_VERSION)
        self.assertItemsEqual(metadata_info['pipeline_version'], self.PIPELINE_VERSION)

        self.validate_data_obfuscation()
        self.validate_events_obfuscation()

    def validate_data_obfuscation(self):
        """Validates data obfuscation."""
        data_dir = os.path.join(self.temporary_dir, 'state', self.EXPORT_DATE)
        for data_filename in os.listdir(data_dir):
            data_filepath = os.path.join(data_dir, data_filename)
            expected_output_filepath = os.path.join(self.data_dir, 'output', 'obfuscation', 'state', data_filename)

            if data_filename.endswith('mongo'):
                with open(data_filepath) as mongo_output_file:
                    output_json = [json.loads(line) for line in mongo_output_file]
                with open(expected_output_filepath) as expected_output_file:
                    expected_output_json = [json.loads(line) for line in expected_output_file]
                self.assertItemsEqual(output_json, expected_output_json)
            elif data_filename.endswith('.json'):
                with open(data_filepath) as actual_output_file:
                    output_json = json.load(actual_output_file)
                with open(expected_output_filepath) as expected_output_file:
                    expected_output_json = json.load(expected_output_file)
                self.assertDictEqual(output_json, expected_output_json)
            elif data_filename.endswith('.tar.gz'):
                pass
            else:
                shell.run(['diff', data_filepath, expected_output_filepath])

    def validate_events_obfuscation(self):
        """Validates events obfuscation."""
        events_dir = os.path.join(self.temporary_dir, 'events')
        for events_filename in os.listdir(events_dir):
            events_filepath = os.path.join(events_dir, events_filename)
            decompressed_filepath = events_filepath[:-len('.gz')]
            fs.decompress_file(events_filepath, decompressed_filepath)

            expected_events_filepath = os.path.join(self.data_dir, 'output', 'obfuscation', 'events', events_filename[:-len('.gz')])

            with open(decompressed_filepath) as events_output_file:
                events_output = [json.loads(line) for line in events_output_file]
            with open(expected_events_filepath) as expected_events_file:
                expected_events = [json.loads(line) for line in expected_events_file]
            self.assertItemsEqual(events_output, expected_events)
