Restore Django models - preserve Answer data

README.md

@@ -483,11 +483,29 @@ Install to the workbench's virtualenv by running the following command form the
 pip install -r requirements.txt
 ```
 
+In the main XBlock repository, create the following configuration file
+in `workbench/settings_mentoring.py` in the XBlock repository:
+
+```python
+from settings import *
+
+INSTALLED_APPS += ('mentoring',)
+DATABASES['default']['NAME'] = 'workbench.sqlite'
+```
+
+Because this XBlock uses a Django model, you need to sync the database
+before starting the workbench. Run this from the XBlock repository
+root:
+
+```bash
+$ ./manage.py syncdb --settings=workbench.settings_mentoring
+```
+
 Running the workbench
 ---------------------
 
 ```bash
-$ ./manage.py runserver 8000
+$ ./manage.py runserver 8000 --settings=workbench.settings_mentoring
 ```
 
 Access it at [http://localhost:8000/](http://localhost:8000).

mentoring/components/answer.py

@@ -24,9 +24,12 @@
 # Imports ###########################################################
 
 import logging
+from lazy import lazy
+
+from mentoring.models import Answer
 
 from xblock.core import XBlock
-from xblock.fields import Scope, Boolean, Dict, Float, Integer, String
+from xblock.fields import Scope, Boolean, Float, Integer, String
 from xblock.fragment import Fragment
 from xblockutils.resources import ResourceLoader
 from .step import StepMixin
@@ -78,15 +81,6 @@ class AnswerBlock(XBlock, StepMixin):
         scope=Scope.settings,
         enforce_type=True
     )
-    # This is the internal value of student_input. Don't access directly - use student_input instead.
-    student_input_raw = String(
-        scope=Scope.user_state,
-        default=""
-    )
-    # Shared student input - share answers among all Answer blocks in the course with the same name
-    student_input_shared = Dict(
-        scope=Scope.preferences,
-    )
 
     @classmethod
     def parse_xml(cls, node, runtime, keys, id_generator):
@@ -108,19 +102,29 @@ class AnswerBlock(XBlock, StepMixin):
         """ Get a course ID if available """
         return getattr(self.runtime, 'course_id', 'all')
 
-    @property
+    def _get_student_id(self):
+        """ Get student anonymous ID or normal ID """
+        try:
+            return self.runtime.anonymous_student_id
+        except AttributeError:
+            return self.scope_ids.user_id
+
+    @lazy
     def student_input(self):
         """
         The student input value, or a default which may come from another block.
+        Read from a Django model, since XBlock API doesn't yet support course-scoped
+        fields or generating instructor reports across many student responses.
         """
-        course_id = self._get_course_id()
-        if self.name and self.name in self.student_input_shared.get(course_id, {}):
-            self.student_input_raw = self.student_input_shared[course_id][self.name]
-        student_input = self.student_input_raw
+        # Only attempt to locate a model object for this block when the answer has a name
+        if not self.name:
+            return ''
+
+        student_input = self.get_model_object().student_input
 
         # Default value can be set from another answer's current value
         if not student_input and self.default_from:
-            student_input = self.runtime.get_block(self.default_from).student_input
+            student_input = self.get_model_object(name=self.default_from).student_input
 
         return student_input
 
@@ -150,11 +154,7 @@ class AnswerBlock(XBlock, StepMixin):
 
     def submit(self, submission):
         if not self.read_only:
-            self.student_input_raw = submission[0]['value'].strip()
-            course_id = self._get_course_id()
-            if not self.student_input_shared.get(course_id):
-                self.student_input_shared[course_id] = {}
-            self.student_input_shared[course_id][self.name] = self.student_input_raw
+            self.student_input = submission[0]['value'].strip()
             self.save()
             log.info(u'Answer submitted for`{}`: "{}"'.format(self.name, self.student_input))
         return {
@@ -175,3 +175,41 @@ class AnswerBlock(XBlock, StepMixin):
     @property
     def completed(self):
         return self.status == 'correct'
+
+    def save(self):
+        """
+        Replicate data changes on the related Django model used for sharing of data accross XBlocks
+        """
+        super(AnswerBlock, self).save()
+
+        student_id = self._get_student_id()
+        if not student_id:
+            return  # save() gets called from the workbench homepage sometimes when there is no student ID
+
+        # Only attempt to locate a model object for this block when the answer has a name
+        if self.name:
+            answer_data = self.get_model_object()
+            if answer_data.student_input != self.student_input and not self.read_only:
+                answer_data.student_input = self.student_input
+                answer_data.save()
+
+    def get_model_object(self, name=None):
+        """
+        Fetches the Answer model object for the answer named `name`
+        """
+        # By default, get the model object for the current answer's name
+        if not name:
+            name = self.name
+        # Consistency check - we should have a name by now
+        if not name:
+            raise ValueError('AnswerBlock.name field need to be set to a non-null/empty value')
+
+        student_id = self._get_student_id()
+        course_id = self._get_course_id()
+
+        answer_data, _ = Answer.objects.get_or_create(
+            student_id=student_id,
+            course_id=course_id,
+            name=name,
+        )
+        return answer_data

mentoring/dataexport.py

@@ -23,10 +23,10 @@
 
 # Imports ###########################################################
 
-import json
 import logging
 import unicodecsv
 
+from itertools import groupby
 from StringIO import StringIO
 from webob import Response
 from xblock.core import XBlock
@@ -34,7 +34,7 @@ from xblock.fields import String, Scope
 from xblock.fragment import Fragment
 from xblockutils.resources import ResourceLoader
 
-from .components import AnswerBlock
+from .components.answer import AnswerBlock, Answer
 
 # Globals ###########################################################
 
@@ -86,69 +86,31 @@ class MentoringDataExportBlock(XBlock):
         return response
 
     def get_csv(self):
-        """
-        Download all student answers as a CSV.
-
-        Columns are: student_id, [name of each answer block is a separate column]
-        """
-        answers_names = []  # List of the '.name' of each answer property
-        student_answers = {}  # Dict of student ID: {answer_name: answer, ...}
-        for answer_block in self._get_answer_blocks():
-            answers_names.append(answer_block.name)
-            student_data = self._get_students_data(answer_block)  # Tuples of (student ID, student input)
-            for student_id, student_answer in student_data:
-                if student_id not in student_answers:
-                    student_answers[student_id] = {}
-                student_answers[student_id][answer_block.name] = student_answer
-
-        # Sort things:
-        answers_names.sort()
-        student_answers_sorted = list(student_answers.iteritems())
-        student_answers_sorted.sort(key=lambda entry: entry[0])  # Sort by student ID
+        course_id = getattr(self.runtime, "course_id", "all")
+        answers = Answer.objects.filter(course_id=course_id).order_by('student_id', 'name')
+        answers_names = answers.values_list('name', flat=True).distinct().order_by('name')
 
         # Header line
         yield list2csv([u'student_id'] + list(answers_names))
 
         if answers_names:
-            for student_id, answers in student_answers_sorted:
-                row = [student_id]
-                for name in answers_names:
-                    row.append(answers.get(name, u""))
-                yield list2csv(row)
-
-    def _get_students_data(self, answer_block):
-        """
-        Efficiently query for the answers entered by ALL students.
-        (Note: The XBlock API only allows querying for the current
-        student, so we have to use other APIs)
-
-        Yields tuples of (student_id, student_answer)
-        """
-        usage_id = answer_block.scope_ids.usage_id
-        # Check if we're in edX:
-        try:
-            from courseware.models import StudentModule
-            usage_id = usage_id.for_branch(None).version_agnostic()
-            entries = StudentModule.objects.filter(module_state_key=unicode(usage_id)).values('student_id', 'state')
-            for entry in entries:
-                state = json.loads(entry['state'])
-                if 'student_input_raw' in state:
-                    yield (entry['student_id'], state['student_input_raw'])
-        except ImportError:
-            pass
-        # Check if we're in the XBlock SDK:
-        try:
-            from workbench.models import XBlockState
-            rows = XBlockState.objects.filter(scope="usage", scope_id=usage_id).exclude(user_id=None)
-            for entry in rows.values('user_id', 'state'):
-                state = json.loads(entry['state'])
-                if 'student_input_raw' in state:
-                    yield (entry['user_id'], state['student_input_raw'])
-        except ImportError:
-            pass
-        # Something else - return only the data
-        # for the current user.
-        yield (answer_block.scope_ids.user_id, answer_block.student_input_raw)
+            for _, student_answers in groupby(answers, lambda x: x.student_id):
+                row = []
+                next_answer_idx = 0
+                for answer in student_answers:
+                    if not row:
+                        row = [answer.student_id]
+
+                    while answer.name != answers_names[next_answer_idx]:
+                        # Still add answer row to CSV when they don't exist in DB
+                        row.append('')
+                        next_answer_idx += 1
+
+                    row.append(answer.student_input)
+                    next_answer_idx += 1
+
+                if row:
+                    yield list2csv(row)
 
     def _get_answer_blocks(self):
         """

mentoring/migrations/0001_initial.py

+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'Answer'
+        db.create_table('mentoring_answer', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('name', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)),
+            ('student_id', self.gf('django.db.models.fields.CharField')(max_length=20, db_index=True)),
+            ('student_input', self.gf('django.db.models.fields.TextField')(default='', blank=True)),
+            ('created_on', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+            ('modified_on', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)),
+        ))
+        db.send_create_signal('mentoring', ['Answer'])
+
+        # Adding unique constraint on 'Answer', fields ['student_id', 'name']
+        db.create_unique('mentoring_answer', ['student_id', 'name'])
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'Answer', fields ['student_id', 'name']
+        db.delete_unique('mentoring_answer', ['student_id', 'name'])
+
+        # Deleting model 'Answer'
+        db.delete_table('mentoring_answer')
+
+    models = {
+        'mentoring.answer': {
+            'Meta': {'unique_together': "(('student_id', 'name'),)", 'object_name': 'Answer'},
+            'created_on': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified_on': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'student_id': ('django.db.models.fields.CharField', [], {'max_length': '20', 'db_index': 'True'}),
+            'student_input': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['mentoring']

mentoring/migrations/0002_auto__add_field_answer_course_id__chg_field_answer_student_id.py

+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'Answer.course_id'
+        db.add_column('mentoring_answer', 'course_id',
+                      self.gf('django.db.models.fields.CharField')(default='default', max_length=50, db_index=True),
+                      keep_default=False)
+
+        # Changing field 'Answer.student_id'
+        db.alter_column('mentoring_answer', 'student_id', self.gf('django.db.models.fields.CharField')(max_length=32))
+
+    def backwards(self, orm):
+        # Deleting field 'Answer.course_id'
+        db.delete_column('mentoring_answer', 'course_id')
+
+        # Changing field 'Answer.student_id'
+        db.alter_column('mentoring_answer', 'student_id', self.gf('django.db.models.fields.CharField')(max_length=20))
+
+    models = {
+        'mentoring.answer': {
+            'Meta': {'unique_together': "(('student_id', 'name'),)", 'object_name': 'Answer'},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'created_on': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified_on': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'student_id': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
+            'student_input': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['mentoring']

mentoring/migrations/0003_auto__del_unique_answer_student_id_name__add_unique_answer_course_id_s.py

+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Removing unique constraint on 'Answer', fields ['student_id', 'name']
+        db.delete_unique('mentoring_answer', ['student_id', 'name'])
+
+        # Adding unique constraint on 'Answer', fields ['course_id', 'student_id', 'name']
+        db.create_unique('mentoring_answer', ['course_id', 'student_id', 'name'])
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'Answer', fields ['course_id', 'student_id', 'name']
+        db.delete_unique('mentoring_answer', ['course_id', 'student_id', 'name'])
+
+        # Adding unique constraint on 'Answer', fields ['student_id', 'name']
+        db.create_unique('mentoring_answer', ['student_id', 'name'])
+
+    models = {
+        'mentoring.answer': {
+            'Meta': {'unique_together': "(('student_id', 'course_id', 'name'),)", 'object_name': 'Answer'},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'created_on': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified_on': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'student_id': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
+            'student_input': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['mentoring']

mentoring/migrations/0004_auto__add_lightchild__add_unique_lightchild_student_id_course_id_name.py

+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'LightChild'
+        db.create_table('mentoring_lightchild', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('name', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)),
+            ('student_id', self.gf('django.db.models.fields.CharField')(max_length=32, db_index=True)),
+            ('course_id', self.gf('django.db.models.fields.CharField')(max_length=50, db_index=True)),
+            ('student_data', self.gf('django.db.models.fields.TextField')(default='', blank=True)),
+            ('created_on', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+            ('modified_on', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)),
+        ))
+        db.send_create_signal('mentoring', ['LightChild'])
+
+        # Adding unique constraint on 'LightChild', fields ['student_id', 'course_id', 'name']
+        db.create_unique('mentoring_lightchild', ['student_id', 'course_id', 'name'])
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'LightChild', fields ['student_id', 'course_id', 'name']
+        db.delete_unique('mentoring_lightchild', ['student_id', 'course_id', 'name'])
+
+        # Deleting model 'LightChild'
+        db.delete_table('mentoring_lightchild')
+
+    models = {
+        'mentoring.answer': {
+            'Meta': {'unique_together': "(('student_id', 'course_id', 'name'),)", 'object_name': 'Answer'},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'created_on': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified_on': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'student_id': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
+            'student_input': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'})
+        },
+        'mentoring.lightchild': {
+            'Meta': {'unique_together': "(('student_id', 'course_id', 'name'),)", 'object_name': 'LightChild'},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'created_on': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'student_data': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified_on': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'student_id': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'})
+        }
+    }
+
+    complete_apps = ['mentoring']

mentoring/migrations/0005_auto__chg_field_lightchild_name.py

+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+
+        # Changing field 'LightChild.name'
+        db.alter_column('mentoring_lightchild', 'name', self.gf('django.db.models.fields.CharField')(max_length=100))
+
+    def backwards(self, orm):
+
+        # Changing field 'LightChild.name'
+        db.alter_column('mentoring_lightchild', 'name', self.gf('django.db.models.fields.CharField')(max_length=50))
+
+    models = {
+        'mentoring.answer': {
+            'Meta': {'unique_together': "(('student_id', 'course_id', 'name'),)", 'object_name': 'Answer'},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'created_on': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified_on': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'student_id': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'}),
+            'student_input': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'})
+        },
+        'mentoring.lightchild': {
+            'Meta': {'unique_together': "(('student_id', 'course_id', 'name'),)", 'object_name': 'LightChild'},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'created_on': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'modified_on': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'db_index': 'True'}),
+            'student_data': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
+            'student_id': ('django.db.models.fields.CharField', [], {'max_length': '32', 'db_index': 'True'})
+        }
+    }
+
+    complete_apps = ['mentoring']

mentoring/models.py

+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2014 Harvard
+#
+# Authors:
+#          Xavier Antoviaque <xavier@antoviaque.org>
+#
+# This software's license gives you freedom; you can copy, convey,
+# propagate, redistribute and/or modify this program under the terms of
+# the GNU Affero General Public License (AGPL) as published by the Free
+# Software Foundation (FSF), either version 3 of the License, or (at your
+# option) any later version of the AGPL published by the FSF.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program in a file in the toplevel directory called
+# "AGPLv3".  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# Imports ###########################################################
+
+from django.db import models
+
+
+# Classes ###########################################################
+
+class Answer(models.Model):
+    """
+    Django model used to store AnswerBlock data that need to be shared
+    and queried accross XBlock instances (workaround).
+    """
+
+    class Meta:
+        app_label = 'mentoring'
+        unique_together = (('student_id', 'course_id', 'name'),)
+
+    name = models.CharField(max_length=50, db_index=True)
+    student_id = models.CharField(max_length=32, db_index=True)
+    course_id = models.CharField(max_length=50, db_index=True)
+    student_input = models.TextField(blank=True, default='')
+    created_on = models.DateTimeField('created on', auto_now_add=True)
+    modified_on = models.DateTimeField('modified on', auto_now=True)
+
+    def save(self, *args, **kwargs):
+        # Force validation of max_length
+        self.full_clean()
+        super(Answer, self).save(*args, **kwargs)
+
+
+class LightChild(models.Model):
+    """
+    DEPRECATED.
+    Django model previously used to store LightChild student data.
+
+    This is not used at all by any of the mentoring blocks but will
+    be kept here for the purpose of migrating data for other
+    LightChildren that are converted to XBlocks and need to migrate
+    data from Django to native XBlock fields.
+    """
+
+    class Meta:
+        app_label = 'mentoring'
+        managed = False  # Don't create this table. This class is only to migrate data from an existing table.
+        unique_together = (('student_id', 'course_id', 'name'),)
+
+    name = models.CharField(max_length=100, db_index=True)
+    student_id = models.CharField(max_length=32, db_index=True)
+    course_id = models.CharField(max_length=50, db_index=True)
+    student_data = models.TextField(blank=True, default='')
+    created_on = models.DateTimeField('created on', auto_now_add=True)
+    modified_on = models.DateTimeField('modified on', auto_now=True)

run_tests.py

@@ -22,6 +22,9 @@ if __name__ == "__main__":
     # Configure a range of ports in case the default port of 8081 is in use
     os.environ.setdefault("DJANGO_LIVE_TEST_SERVER_ADDRESS", "localhost:8081-8099")
 
+    from django.conf import settings
+    settings.INSTALLED_APPS += ("mentoring", )
+
     from django.core.management import execute_from_command_line
     args = sys.argv[1:]
     paths = [arg for arg in args if arg[0] != '-']