Merge pull request #22 from open-craft/data-export

Data Export block prototype

problem_builder/__init__.py

@@ -2,6 +2,7 @@ from .mentoring import MentoringBlock
 from .answer import AnswerBlock, AnswerRecapBlock
 from .choice import ChoiceBlock
 from .dashboard import DashboardBlock
+from .data_export import DataExportBlock
 from .mcq import MCQBlock, RatingBlock
 from .mrq import MRQBlock
 from .message import MentoringMessageBlock

problem_builder/data_export.py

+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2014-2015 Harvard, edX & OpenCraft
+#
+# This software's license gives you freedom; you can copy, convey,
+# propagate, redistribute and/or modify this program under the terms of
+# the GNU Affero General Public License (AGPL) as published by the Free
+# Software Foundation (FSF), either version 3 of the License, or (at your
+# option) any later version of the AGPL published by the FSF.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program in a file in the toplevel directory called
+# "AGPLv3".  If not, see <http://www.gnu.org/licenses/>.
+#
+"""
+Data Export: An XBlock for instructors to export student answers from a course.
+
+All processing is done offline.
+"""
+import json
+from xblock.core import XBlock
+from xblock.fields import Scope, String, Dict
+from xblock.fragment import Fragment
+from xblockutils.resources import ResourceLoader
+
+loader = ResourceLoader(__name__)
+
+
+@XBlock.wants('user')
+class DataExportBlock(XBlock):
+    """
+    DataExportBlock: An XBlock for instructors to export student answers from a course.
+
+    All processing is done offline.
+    """
+    active_export_task_id = String(
+        # The UUID of the celery AsyncResult for the most recent export,
+        # IF we are sill waiting for it to finish
+        default="",
+        scope=Scope.user_state,
+    )
+    last_export_result = Dict(
+        # The info dict returned by the most recent successful export.
+        # If the export failed, it will have an "error" key set.
+        default=None,
+        scope=Scope.user_state,
+    )
+    has_author_view = True
+
+    @property
+    def display_name_with_default(self):
+        return "Data Export"
+
+    def author_view(self, context=None):
+        """ Studio View """
+        # Warn the user that this block will only work from the LMS. (Since the CMS uses
+        # different celery queues; our task listener is waiting for tasks on the LMS queue)
+        return Fragment(u'<p>Data Export Block</p><p>This block only works from the LMS.</p>')
+
+    def studio_view(self, context=None):
+        """ 'Edit' form view in Studio """
+        return Fragment(u'<p>This block has no configuration options.</p>')
+
+    def check_pending_export(self):
+        """
+        If we're waiting for an export, see if it has finished, and if so, get the result.
+        """
+        from .tasks import export_data as export_data_task  # Import here since this is edX LMS specific
+        if self.active_export_task_id:
+            async_result = export_data_task.AsyncResult(self.active_export_task_id)
+            if async_result.ready():
+                self._save_result(async_result)
+
+    def _save_result(self, task_result):
+        """ Given an AsyncResult or EagerResult, save it. """
+        self.active_export_task_id = ''
+        if task_result.successful():
+            if isinstance(task_result.result, dict) and not task_result.result.get('error'):
+                self.last_export_result = task_result.result
+            else:
+                self.last_export_result = {'error': u'Unexpected result: {}'.format(repr(task_result.result))}
+        else:
+            self.last_export_result = {'error': unicode(task_result.result)}
+
+    def student_view(self, context=None):
+        """ Normal View """
+        if not self.user_is_staff():
+            return Fragment(u'<p>This interface can only be used by course staff.</p>')
+        html = loader.render_template('templates/html/data_export.html')
+        fragment = Fragment(html)
+        fragment.add_css_url(self.runtime.local_resource_url(self, 'public/css/data_export.css'))
+        fragment.add_javascript_url(self.runtime.local_resource_url(self, 'public/js/data_export.js'))
+        fragment.initialize_js('DataExportBlock')
+        return fragment
+
+    @property
+    def download_url_for_last_report(self):
+        """ Get the URL for the last report, if any """
+        # Unfortunately this is a bit inefficient due to the ReportStore API
+        if not self.last_export_result or self.last_export_result['error'] is not None:
+            return None
+        from instructor_task.models import ReportStore
+        report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD')
+        course_key = getattr(self.scope_ids.usage_id, 'course_key', None)
+        return dict(report_store.links_for(course_key)).get(self.last_export_result['report_filename'])
+
+    def _get_status(self):
+        self.check_pending_export()
+        return {
+            'export_pending': bool(self.active_export_task_id),
+            'last_export_result': self.last_export_result,
+            'download_url': self.download_url_for_last_report,
+        }
+
+    @XBlock.json_handler
+    def get_status(self, request, suffix=''):
+        return self._get_status()
+
+    @XBlock.json_handler
+    def delete_export(self, request, suffix=''):
+        self._delete_export()
+        return self._get_status()
+
+    def _delete_export(self):
+        self.last_export_result = None
+        self.active_export_task_id = ''
+
+    @XBlock.json_handler
+    def start_export(self, request, suffix=''):
+        """ Start a new asynchronous export """
+        if not self.user_is_staff():
+            return {'error': 'permission denied'}
+        from .tasks import export_data as export_data_task  # Import here since this is edX LMS specific
+        self._delete_export()
+        async_result = export_data_task.delay(unicode(self.scope_ids.usage_id), self.get_user_id())
+        if async_result.ready():
+            # In development mode, the task may have executed synchronously.
+            # Store the result now, because we won't be able to retrieve it later :-/
+            if async_result.successful():
+                # Make sure the result can be represented as JSON, since the non-eager celery
+                # requires that
+                json.dumps(async_result.result)
+            self._save_result(async_result)
+        else:
+            # The task is running asynchronously. Store the result ID so we can query its progress:
+            self.active_export_task_id = async_result.id
+        return self._get_status()
+        return {'result': 'started'}
+
+    @XBlock.json_handler
+    def cancel_export(self, request, suffix=''):
+        from .tasks import export_data as export_data_task  # Import here since this is edX LMS specific
+        if self.active_export_task_id:
+            async_result = export_data_task.AsyncResult(self.active_export_task_id)
+            async_result.revoke()
+            self._delete_export()
+
+    def _get_user_attr(self, attr):
+        """Get an attribute of the current user."""
+        user_service = self.runtime.service(self, 'user')
+        if user_service:
+            # May be None when creating bok choy test fixtures
+            return user_service.get_current_user().opt_attrs.get(attr)
+        return None
+
+    def user_is_staff(self):
+        """Return a Boolean value indicating whether the current user is a member of staff."""
+        return self._get_user_attr('edx-platform.user_is_staff')
+
+    def get_user_id(self):
+        """Get the edx-platform user_id of the current user."""
+        return self._get_user_attr('edx-platform.user_id')

problem_builder/public/css/data_export.css

+.data-export-actions button {
+    display: none;
+}
+.data-export-status {
+    height: 8em;
+}

problem_builder/public/js/data_export.js

+function DataExportBlock(runtime, element) {
+    'use strict';
+    var $startButton = $('.data-export-start', element);
+    var $cancelButton = $('.data-export-cancel', element);
+    var $downloadButton = $('.data-export-download', element);
+    var $deleteButton = $('.data-export-delete', element);
+    var status;
+    function getStatus() {
+        $.ajax({
+            type: 'POST',
+            url: runtime.handlerUrl(element, 'get_status'),
+            data: '{}',
+            success: updateStatus,
+            dataType: 'json',
+        });
+    }
+    function updateStatus(newStatus) {
+        var statusChanged = newStatus !== status;
+        status = newStatus;
+        if (status.export_pending) {
+            // Keep polling for status updates when an export is running.
+            setTimeout(getStatus, 1000);
+        }
+        if (statusChanged) updateView();
+    }
+    function showSpinner() {
+        $startButton.prop('disabled', true);
+        $cancelButton.prop('disabled', true);
+        $downloadButton.prop('disabled', true);
+        $deleteButton.prop('disabled', true);
+        $('.data-export-status', element).empty().append(
+            $('<i>').addClass('icon fa fa-spinner fa-spin')
+        );
+    }
+    function updateView() {
+        var $statusArea = $('.data-export-status', element), startTime;
+        $statusArea.empty();
+        $startButton.toggle(!status.export_pending).prop('disabled', false);
+        $cancelButton.toggle(status.export_pending).prop('disabled', false);
+        $downloadButton.toggle(Boolean(status.download_url)).prop('disabled', false);
+        $deleteButton.toggle(Boolean(status.last_export_result)).prop('disabled', false);
+        if (status.last_export_result) {
+            if (status.last_export_result.error) {
+                $statusArea.append($('<p>').text(
+                    'Data export failed. Reason: ' + status.last_export_result.error
+                ));
+            } else {
+                startTime = new Date(status.last_export_result.start_timestamp * 1000);
+                $statusArea.append($('<p>').text(
+                    'A report is available for download.'
+                ));
+                $statusArea.append($('<p>').text(
+                    'It was created at ' + startTime.toString() +
+                    ' and took ' + status.last_export_result.generation_time_s.toFixed(1) +
+                    ' seconds to finish.'
+                ));
+            }
+        } else {
+            if (status.export_pending) {
+                $statusArea.append($('<p>').text(
+                    'The report is currently being generated…'
+                ));
+            } else {
+                $statusArea.append($('<p>').text(
+                    'No report data available.'
+                ));
+            }
+        }
+    }
+    function addHandler($button, handlerName) {
+        $button.on('click', function() {
+            $.ajax({
+                type: 'POST',
+                url: runtime.handlerUrl(element, handlerName),
+                data: '{}',
+                success: updateStatus,
+                dataType: 'json',
+            });
+            showSpinner();
+        });
+    }
+    addHandler($startButton, 'start_export');
+    addHandler($cancelButton, 'cancel_export');
+    addHandler($deleteButton, 'delete_export');
+    $downloadButton.on('click', function() {
+        window.location.href = status.download_url;
+    });
+    showSpinner();
+    getStatus();
+}

problem_builder/tasks.py

+"""
+Celery task for CSV student answer export.
+"""
+import time
+
+from celery.task import task
+from celery.utils.log import get_task_logger
+from instructor_task.models import ReportStore
+from opaque_keys.edx.keys import UsageKey
+from xmodule.modulestore.django import modulestore
+
+from .mcq import MCQBlock, RatingBlock
+from .sub_api import sub_api
+
+logger = get_task_logger(__name__)
+
+
+@task()
+def export_data(source_block_id_str, user_id):
+    """
+    Exports student answers to all MCQ questions to a CSV file.
+    """
+    start_timestamp = time.time()
+
+    logger.debug("Beginning data export")
+
+    block_key = UsageKey.from_string(source_block_id_str)
+    src_block = modulestore().get_item(block_key)
+    course_key = src_block.scope_ids.usage_id.course_key.replace(branch=None, version_guid=None)
+    course_key_str = unicode(course_key)
+
+    # Get the root block:
+    root = src_block
+    while root.parent:
+        root = root.get_parent()
+
+    # Build an ordered list of blocks to include in the export - each block is a column in the CSV file
+    blocks_to_include = []
+
+    def scan_for_blocks(block):
+        """ Recursively scan the course tree for blocks of interest """
+        if isinstance(block, (MCQBlock, RatingBlock)):
+            blocks_to_include.append(block)
+        elif block.has_children:
+            for child_id in block.children:
+                scan_for_blocks(block.runtime.get_block(child_id))
+
+    scan_for_blocks(root)
+
+    # Define the header rows of our CSV:
+    rows = []
+    rows.append(["Student"] + [block.display_name_with_default for block in blocks_to_include])
+    rows.append([""] + [block.scope_ids.block_type for block in blocks_to_include])
+    rows.append([""] + [block.scope_ids.usage_id for block in blocks_to_include])
+
+    # Load the actual student submissions for each block in blocks_to_include.
+    # Note this requires one giant query per block (all student submissions for each block, one block at a time)
+    student_submissions = {}  # Key is student ID, value is a list with same length as blocks_to_include
+    for idx, block in enumerate(blocks_to_include, start=1):  # start=1 since first column is stuent ID
+        # Get all of the most recent student submissions for this block:
+        block_id = unicode(block.scope_ids.usage_id.replace(branch=None, version_guid=None))
+        block_type = block.scope_ids.block_type
+        for submission in sub_api.get_all_submissions(course_key_str, block_id, block_type):
+            student_id = submission['student_id']
+            if student_id not in student_submissions:
+                student_submissions[student_id] = [student_id] + [""] * len(blocks_to_include)
+            student_submissions[student_id][idx] = submission['answer']
+
+    # Now change from a dict to an array ordered by student ID as we generate the remaining rows:
+    for student_id in sorted(student_submissions.iterkeys()):
+        rows.append(student_submissions[student_id])
+        del student_submissions[student_id]
+
+    # Generate the CSV:
+    filename = u"pb-data-export-{}.csv".format(time.strftime("%Y-%m-%d-%H%M%S", time.gmtime(start_timestamp)))
+    report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD')
+    report_store.store_rows(course_key, filename, rows)
+
+    generation_time_s = time.time() - start_timestamp
+    logger.debug("Done data export - took {} seconds".format(generation_time_s))
+
+    return {
+        "error": None,
+        "report_filename": filename,
+        "start_timestamp": start_timestamp,
+        "generation_time_s": generation_time_s,
+    }

problem_builder/templates/html/data_export.html

+<h3>Data Export</h3>
+
+<p>You can export all student answers to multiple-choice questions to a CSV file here.</p>
+
+<div class="data-export-status"></div>
+
+<div class="data-export-actions">
+  <button class="data-export-download">Download result</button>
+  <button class="data-export-start">Start a new export</button>
+  <button class="data-export-cancel">Cancel current export</button>
+  <button class="data-export-delete">Delete result</button>
+</div>

problem_builder/tests/integration/test_data_export.py

+# -*- coding: utf-8 -*-
+
+import time
+import pdb
+import sys
+
+from mock import patch, Mock
+from xblockutils.base_test import SeleniumXBlockTest
+
+from problem_builder.data_export import DataExportBlock
+
+
+class MockTasksModule(object):
+    """Mock for the tasks module, which can only be meaningfully import in the LMS."""
+
+    def __init__(self, successful=True):
+        self.export_data = Mock()
+        async_result = self.export_data.async_result
+        async_result.ready.side_effect = [False, False, True, True]
+        async_result.id = "export_task_id"
+        async_result.successful.return_value = successful
+        if successful:
+            async_result.result = dict(
+                error=None,
+                report_filename='/file/report.csv',
+                start_timestamp=time.time(),
+                generation_time_s=23.4,
+            )
+        else:
+            async_result.result = 'error'
+        self.export_data.AsyncResult.return_value = async_result
+        self.export_data.delay.return_value = async_result
+
+
+class MockInstructorTaskModelsModule(object):
+
+    def __init__(self):
+        self.ReportStore = Mock()
+        self.ReportStore.from_config.return_value.links_for.return_value = [
+            ('/file/report.csv', '/url/report.csv')
+        ]
+
+
+class DataExportTest(SeleniumXBlockTest):
+
+    def setUp(self):
+        super(DataExportTest, self).setUp()
+        self.set_scenario_xml("""
+        <vertical_demo>
+          <pb-data-export url_name="data_export"/>
+        </vertical_demo>
+        """)
+
+    def test_students_dont_see_interface(self):
+        data_export = self.go_to_view()
+        self.assertIn('This interface can only be used by course staff.', data_export.text)
+
+    @patch.dict('sys.modules', {
+        'problem_builder.tasks': MockTasksModule(successful=True),
+        'instructor_task': True,
+        'instructor_task.models': MockInstructorTaskModelsModule(),
+    })
+    @patch.object(DataExportBlock, 'user_is_staff', Mock(return_value=True))
+    def test_data_export(self):
+        data_export = self.go_to_view()
+        start_button = data_export.find_element_by_class_name('data-export-start')
+        cancel_button = data_export.find_element_by_class_name('data-export-cancel')
+        download_button = data_export.find_element_by_class_name('data-export-download')
+        delete_button = data_export.find_element_by_class_name('data-export-delete')
+        status_area = data_export.find_element_by_class_name('data-export-status')
+
+        start_button.click()
+
+        self.wait_until_hidden(start_button)
+        self.wait_until_visible(cancel_button)
+        self.wait_until_hidden(download_button)
+        self.wait_until_hidden(delete_button)
+        self.assertIn('The report is currently being generated', status_area.text)
+
+        self.wait_until_visible(start_button)
+        self.wait_until_hidden(cancel_button)
+        self.wait_until_visible(download_button)
+        self.wait_until_visible(delete_button)
+        self.assertIn('A report is available for download.', status_area.text)

setup.py

@@ -54,23 +54,7 @@ BLOCKS = [
     'pb-choice = problem_builder:ChoiceBlock',
 
     'pb-dashboard = problem_builder:DashboardBlock',
-    # Deprecated. You can temporarily uncomment and run 'python setup.py develop' if you have these blocks
-    # installed from testing mentoring v2 and need to get past an error message.
-    #'mentoring = problem_builder:MentoringBlock',  # Deprecated alias for problem-builder
-    #'answer = problem_builder:AnswerBlock',
-    #'mentoring-answer = problem_builder:AnswerBlock',
-    #'answer-recap = problem_builder:AnswerRecapBlock',
-    #'mentoring-answer-recap = problem_builder:AnswerRecapBlock',
-    #'mcq = problem_builder:MCQBlock',
-    #'mentoring-mcq = problem_builder:MCQBlock',
-    #'rating = problem_builder:RatingBlock',
-    #'mentoring-rating = problem_builder:RatingBlock',
-    #'mrq = problem_builder:MRQBlock',
-    #'mentoring-mrq = problem_builder:MRQBlock',
-    #'tip = problem_builder:TipBlock',
-    #'mentoring-tip = problem_builder:TipBlock',
-    #'choice = problem_builder:ChoiceBlock',
-    #'mentoring-choice = problem_builder:ChoiceBlock',
+    'pb-data-export = problem_builder:DataExportBlock',
 ]
 
 setup(