Fix+test data export, fix multiple <mentoring> blocks on one page

mentoring/components/answer.py

@@ -73,7 +73,7 @@ class AnswerBlock(XBlock, StepMixin):
         default=""
     )
     weight = Float(
-        help="Defines the maximum total grade of the light child block.",
+        help="Defines the maximum total grade of the answer block.",
         default=1,
         scope=Scope.settings,
         enforce_type=True

mentoring/dataexport.py

@@ -23,10 +23,10 @@
 
 # Imports ###########################################################
 
+import json
 import logging
 import unicodecsv
 
-from itertools import groupby
 from StringIO import StringIO
 from webob import Response
 from xblock.core import XBlock
@@ -34,10 +34,12 @@ from xblock.fields import String, Scope
 from xblock.fragment import Fragment
 from xblockutils.resources import ResourceLoader
 
+from .components import AnswerBlock
 
 # Globals ###########################################################
 
 log = logging.getLogger(__name__)
+loader = ResourceLoader(__name__)
 
 # Utils ###########################################################
 
@@ -46,10 +48,12 @@ def list2csv(row):
     """
     Convert a list to a CSV string (single row)
     """
-    with StringIO() as f:
+    f = StringIO()
     writer = unicodecsv.writer(f, encoding='utf-8')
     writer.writerow(row)
-        return f.getvalue()
+    result = f.getvalue()
+    f.close()
+    return result
 
 
 # Classes ###########################################################
@@ -63,20 +67,16 @@ class MentoringDataExportBlock(XBlock):
                           scope=Scope.settings)
 
     def student_view(self, context):
-        html = ResourceLoader(__name__).render_template('templates/html/dataexport.html', {
-            'self': self,
+        """
+        Main view of the data export block
+        """
+        # Count how many 'Answer' blocks are in this course:
+        num_answer_blocks = sum(1 for i in self._get_answer_blocks())
+        html = loader.render_template('templates/html/dataexport.html', {
+            'download_url': self.runtime.handler_url(self, 'download_csv'),
+            'num_answer_blocks': num_answer_blocks,
         })
-
-        fragment = Fragment(html)
-        fragment.add_javascript_url(self.runtime.local_resource_url(self, 'public/js/dataexport.js'))
-        fragment.add_css_url(self.runtime.local_resource_url(self, 'public/css/dataexport.css'))
-
-        fragment.initialize_js('MentoringDataExportBlock')
-
-        return fragment
-
-    def studio_view(self, context):
-        return Fragment(u'Studio view body')
+        return Fragment(html)
 
     @XBlock.handler
     def download_csv(self, request, suffix=''):
@@ -86,31 +86,86 @@ class MentoringDataExportBlock(XBlock):
         return response
 
     def get_csv(self):
-        # course_id = self.xmodule_runtime.course_id
+        """
+        Download all student answers as a CSV.
 
-        # TODO: Fix this method - not working yet with rewrite away from LightChildren.
-        raise NotImplementedError
-        answers = []  # Answer.objects.filter(course_id=course_id).order_by('student_id', 'name')
-        answers_names = answers.values_list('name', flat=True).distinct().order_by('name')
+        Columns are: student_id, [name of each answer block is a separate column]
+        """
+        answers_names = []  # List of the '.name' of each answer property
+        student_answers = {}  # Dict of student ID: {answer_name: answer, ...}
+        for answer_block in self._get_answer_blocks():
+            answers_names.append(answer_block.name)
+            student_data = self._get_students_data(answer_block)  # Tuples of (student ID, student input)
+            for student_id, student_answer in student_data:
+                if student_id not in student_answers:
+                    student_answers[student_id] = {}
+                student_answers[student_id][answer_block.name] = student_answer
+
+        # Sort things:
+        answers_names.sort()
+        student_answers_sorted = list(student_answers.iteritems())
+        student_answers_sorted.sort(key=lambda entry: entry[0])  # Sort by student ID
 
         # Header line
         yield list2csv([u'student_id'] + list(answers_names))
 
         if answers_names:
-            for _, student_answers in groupby(answers, lambda x: x.student_id):
-                row = []
-                next_answer_idx = 0
-                for answer in student_answers:
-                    if not row:
-                        row = [answer.student_id]
-
-                    while answer.name != answers_names[next_answer_idx]:
-                        # Still add answer row to CSV when they don't exist in DB
-                        row.append('')
-                        next_answer_idx += 1
-
-                    row.append(answer.student_input)
-                    next_answer_idx += 1
-
-                if row:
+            for student_id, answers in student_answers_sorted:
+                row = [student_id]
+                for name in answers_names:
+                    row.append(answers.get(name, u""))
                 yield list2csv(row)
+
+    def _get_students_data(self, answer_block):
+        """
+        Efficiently query for the answers entered by ALL students.
+        (Note: The XBlock API only allows querying for the current
+        student, so we have to use other APIs)
+
+        Yields tuples of (student_id, student_answer)
+        """
+        usage_id = answer_block.scope_ids.usage_id
+        # Check if we're in edX:
+        try:
+            from courseware.models import StudentModule
+            usage_id = usage_id.for_branch(None).version_agnostic()
+            entries = StudentModule.objects.filter(module_state_key=unicode(usage_id)).values('student_id', 'state')
+            for entry in entries:
+                state = json.loads(entry['state'])
+                if 'student_input_raw' in state:
+                    yield (entry['student_id'], state['student_input_raw'])
+        except ImportError:
+            pass
+        # Check if we're in the XBlock SDK:
+        try:
+            from workbench.models import XBlockState
+            rows = XBlockState.objects.filter(scope="usage", scope_id=usage_id).exclude(user_id=None)
+            for entry in rows.values('user_id', 'state'):
+                state = json.loads(entry['state'])
+                if 'student_input_raw' in state:
+                    yield (entry['user_id'], state['student_input_raw'])
+        except ImportError:
+            pass
+        # Something else - return only the data
+        # for the current user.
+        yield (answer_block.scope_ids.user_id, answer_block.student_input_raw)
+
+    def _get_answer_blocks(self):
+        """
+        Generator.
+        Searches the tree of XBlocks that includes this data export block
+        (i.e. search the current course)
+        and returns all the AnswerBlock blocks that we can see.
+        """
+        root_block = self
+        while root_block.parent:
+            root_block = root_block.get_parent()
+
+        block_ids_left = set([root_block.scope_ids.usage_id])
+
+        while block_ids_left:
+            block = self.runtime.get_block(block_ids_left.pop())
+            if isinstance(block, AnswerBlock):
+                yield block
+            elif block.has_children:
+                block_ids_left |= set(block.children)

mentoring/public/css/dataexport.css

-.mentoring-dataexport {
-    margin: 10px;
-}

mentoring/public/js/dataexport.js

-function MentoringDataExportBlock(runtime, element) {
-    var downloadUrl = runtime.handlerUrl(element, 'download_csv');
-
-    $('button.download', element).click(function(ev) {
-        ev.preventDefault();
-        window.location = downloadUrl;
-    });
-}

mentoring/public/js/mentoring_assessment_view.js

@@ -14,7 +14,7 @@ function MentoringAssessmentView(runtime, element, mentoring) {
         checkmark.removeClass('checkmark-incorrect icon-exclamation fa-exclamation');
 
         // Clear all selections
-        $('input[type=radio], input[type=checkbox]').prop('checked', false);
+        $('input[type=radio], input[type=checkbox]', element).prop('checked', false);
 
         // hide all children
         mentoring.hideAllChildren();
@@ -79,13 +79,18 @@ function MentoringAssessmentView(runtime, element, mentoring) {
         reviewDOM.bind('click', renderGrade);
         tryAgainDOM.bind('click', tryAgain);
 
-        active_child = mentoring.step-1;
+        active_child = mentoring.step;
 
         var options = {
             onChange: onChange
         };
         mentoring.initChildren(options);
+        if (isDone()) {
+            renderGrade();
+        } else {
+            active_child = active_child - 1;
             displayNextChild();
+        }
 
         mentoring.renderDependency();
     }

mentoring/templates/html/dataexport.html

 <div class="mentoring-dataexport">
   <h3>Answers data dump</h3>
-  <button class="download">Download CSV</button>
+  <p><a style="font-weight: bold;" href="{{ download_url }}">Download CSV Export</a> for the {{ num_answer_blocks }} answer{{ num_answer_blocks|pluralize }} in this course.</p>
 </div>

mentoring/templates/html/mentoring.html

@@ -6,7 +6,7 @@
 
   {% if title or header %}
   <div class="title">
-    {% if title %} <h2 class="main">{{ title }}</h2> {% endif %}
+    {% if title %} <h2>{{ title }}</h2> {% endif %}
     {% if header %} {{ header|safe }} {% endif %}
   </div>
   {% endif %}

mentoring/templates/xml/mentoring_data_export.xml

+<vertical_demo>
+    <html>
+        <p>Below are two mentoring blocks and a grade export block that
+        should be able to export their data as CSV.</p>
+    </html>
+    <mentoring display_name="Mentoring Block 1" mode="standard">
+        <title>Mentoring Block 1</title>
+        <html>
+            <p>Please answer the question below.</p>
+        </html>
+
+        <answer name="goal">
+            <question>What is your goal?</question>
+        </answer>
+    </mentoring>
+    <mentoring display_name="Mentoring Block 1" mode="assessment">
+        <title>Mentoring Block 2 (Assessment)</title>
+        <html>
+            <p>Please answer the question below.</p>
+        </html>
+
+        <answer name="inspired">
+            <question>Who has inspired you the most?</question>
+        </answer>
+        <answer name="meaning">
+            <question>What is the meaning of life?</question>
+        </answer>
+    </mentoring>
+
+    <mentoring-dataexport display_name="Data Export Test" />
+</vertical_demo>

mentoring/tests/integration/test_data_export.py

+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2014-2015 Harvard, edX & OpenCraft
+#
+# This software's license gives you freedom; you can copy, convey,
+# propagate, redistribute and/or modify this program under the terms of
+# the GNU Affero General Public License (AGPL) as published by the Free
+# Software Foundation (FSF), either version 3 of the License, or (at your
+# option) any later version of the AGPL published by the FSF.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program in a file in the toplevel directory called
+# "AGPLv3".  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# Imports ###########################################################
+
+import ddt
+import urllib2
+from .base_test import MentoringBaseTest
+
+
+# Classes ###########################################################
+
+
+@ddt.ddt
+class AnswerBlockTest(MentoringBaseTest):
+    """
+    Test mentoring's data export tool.
+    """
+    default_css_selector = 'body'
+
+    def go_to_page_as_student(self, page_name, student_id):
+        """
+        Navigate to the page `page_name`, as listed on the workbench home
+        but override the student_id used
+        """
+        self.browser.get(self.live_server_url)
+        href = self.browser.find_element_by_link_text(page_name).get_attribute("href")
+        href += "?student={}".format(student_id)
+        self.browser.get(href)
+        block = self.browser.find_element_by_css_selector(self._default_css_selector)
+        return block
+
+    def click_submit_button(self, page, mentoring_block_index):
+        """
+        Click on one of the submit buttons on the page
+        """
+        mentoring_div = page.find_elements_by_css_selector('div.mentoring')[mentoring_block_index]
+        submit = mentoring_div.find_element_by_css_selector('.submit .input-main')
+        self.assertTrue(submit.is_enabled())
+        submit.click()
+        self.wait_until_disabled(submit)
+
+    @ddt.data(
+        # student submissions, expected CSV text
+        (
+            [
+                ("student10", "Essay1", "Essay2", "Essay3"),
+                ("student20", "I didn't answer the last two questions", None, None),
+            ],
+            (
+                u"student_id,goal,inspired,meaning\r\n"
+                "student10,Essay1,Essay2,Essay3\r\n"
+                "student20,I didn't answer the last two questions,,\r\n"
+            )
+        ),
+    )
+    @ddt.unpack
+    def test_data_export_edit(self, submissions, expected_csv):
+        """
+        Have students submit answers, then run an export and validate the output
+        """
+        for student_id, answer1, answer2, answer3 in submissions:
+            page = self.go_to_page_as_student('Data Export', student_id)
+
+            answer1_field = page.find_element_by_css_selector('div[data-name=goal] textarea')
+            self.assertEqual(answer1_field.text, '')
+            answer1_field.send_keys(answer1)
+            self.click_submit_button(page, 0)
+
+            if answer2:
+                answer2_field = page.find_element_by_css_selector('div[data-name=inspired] textarea')
+                self.assertEqual(answer2_field.text, '')
+                answer2_field.send_keys(answer2)
+                self.click_submit_button(page, 1)
+                mentoring_div = page.find_elements_by_css_selector('div.mentoring')[1]
+                next = mentoring_div.find_element_by_css_selector('.submit .input-next')
+                next.click()
+                self.wait_until_disabled(next)
+
+            if answer3:
+                answer3_field = page.find_element_by_css_selector('div[data-name=meaning] textarea')
+                self.assertEqual(answer3_field.text, '')
+                answer3_field.send_keys(answer3)
+                self.click_submit_button(page, 1)
+
+        export_div = page.find_element_by_css_selector('.mentoring-dataexport')
+        self.assertIn("for the 3 answers in this course", export_div.text)
+
+        # Now "click" on the export link:
+        download_url = self.browser.find_element_by_link_text('Download CSV Export').get_attribute("href")
+        response = urllib2.urlopen(download_url)
+        headers = response.info()
+        self.assertTrue(headers['Content-Type'].startswith('text/csv'))
+        csv_data = response.read()
+        self.assertEqual(csv_data, expected_csv)

mentoring/tests/integration/xml/data_export.xml

+<vertical_demo>
+    <html>
+        <p>Below are two mentoring blocks and a grade export block that
+        should be able to export their data as CSV.</p>
+    </html>
+    <mentoring display_name="Mentoring Block 1" mode="standard">
+        <title>Mentoring Block 1</title>
+        <html>
+            <p>Please answer the question below.</p>
+        </html>
+
+        <answer name="goal">
+            <question>What is your goal?</question>
+        </answer>
+    </mentoring>
+    <mentoring display_name="Mentoring Block 1" mode="assessment">
+        <title>Mentoring Block 2 (Assessment)</title>
+        <html>
+            <p>Please answer the question below.</p>
+        </html>
+
+        <answer name="inspired">
+            <question>Who has inspired you the most?</question>
+        </answer>
+        <answer name="meaning">
+            <question>What is the meaning of life?</question>
+        </answer>
+    </mentoring>
+
+    <mentoring-dataexport display_name="Data Export Test" />
+</vertical_demo>