resolved merges

.gitignore

@@ -25,3 +25,6 @@ Gemfile.lock
 .env/
 lms/static/sass/*.css
 cms/static/sass/*.css
+lms/lib/comment_client/python
+nosetests.xml
+cover_html/

cms/static/sass/_base.scss

@@ -14,6 +14,10 @@ $yellow: #fff8af;
 $cream: #F6EFD4;
 $border-color: #ddd;
 
+// edX colors
+$blue: rgb(29,157,217);
+$pink: rgb(182,37,104);
+
 @mixin hide-text {
   background-color: transparent;
   border:           0;

common/djangoapps/student/management/commands/sync_user_info.py

+##
+## One-off script to sync all user information to the discussion service (later info will be synced automatically)
+
+
+from django.core.management.base import BaseCommand
+from django.contrib.auth.models import User
+import comment_client as cc
+
+
+class Command(BaseCommand):
+    help = \
+'''
+Sync all user ids, usernames, and emails to the discussion
+service'''
+
+    def handle(self, *args, **options):
+        for user in User.objects.all().iterator():
+            cc_user = cc.User.from_django_user(user)
+            cc_user.save()

common/djangoapps/student/models.py

@@ -45,6 +45,15 @@ from django.db.models.signals import post_delete, post_save
 from django.dispatch import receiver
 from django_countries import CountryField
 
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+
+from functools import partial
+
+import comment_client as cc
+
+import logging
+
 from xmodule.modulestore.django import modulestore
 
 #from cache_toolbox import cache_model, cache_relation
@@ -254,8 +263,18 @@ def add_user_to_default_group(user, group):
     utg.users.add(User.objects.get(username=user))
     utg.save()
 
+# @receiver(post_save, sender=User)
+def update_user_information(sender, instance, created, **kwargs):
+    try:
+        cc_user = cc.User.from_django_user(instance)
+        cc_user.save()
+    except Exception as e:
+        log = logging.getLogger("mitx.discussion")
+        log.error(unicode(e))
+        log.error("update user info to discussion failed for user with id: " + str(instance.id))
+
 ########################## REPLICATION SIGNALS #################################
-@receiver(post_save, sender=User)
+# @receiver(post_save, sender=User)
 def replicate_user_save(sender, **kwargs):
     user_obj = kwargs['instance']
     if not should_replicate(user_obj):
@@ -263,7 +282,7 @@ def replicate_user_save(sender, **kwargs):
     for course_db_name in db_names_to_replicate_to(user_obj.id):
         replicate_user(user_obj, course_db_name)
 
-@receiver(post_save, sender=CourseEnrollment)
+# @receiver(post_save, sender=CourseEnrollment)
 def replicate_enrollment_save(sender, **kwargs):
     """This is called when a Student enrolls in a course. It has to do the 
     following:
@@ -289,12 +308,12 @@ def replicate_enrollment_save(sender, **kwargs):
     user_profile = UserProfile.objects.get(user_id=enrollment_obj.user_id)
     replicate_model(UserProfile.save, user_profile, enrollment_obj.user_id)
  
-@receiver(post_delete, sender=CourseEnrollment)
+# @receiver(post_delete, sender=CourseEnrollment)
 def replicate_enrollment_delete(sender, **kwargs):
     enrollment_obj = kwargs['instance']
     return replicate_model(CourseEnrollment.delete, enrollment_obj, enrollment_obj.user_id)
  
-@receiver(post_save, sender=UserProfile)
+# @receiver(post_save, sender=UserProfile)
 def replicate_userprofile_save(sender, **kwargs):
     """We just updated the UserProfile (say an update to the name), so push that
     change to all Course DBs that we're enrolled in."""
@@ -404,4 +423,3 @@ def should_replicate(instance):
                   .format(instance))
         return False
     return True
-

common/djangoapps/student/tests.py

@@ -8,6 +8,7 @@ import logging
 from datetime import datetime
 
 from django.test import TestCase
+from nose.plugins.skip import SkipTest
 
 from .models import User, UserProfile, CourseEnrollment, replicate_user, USER_FIELDS_TO_COPY
 
@@ -22,6 +23,7 @@ class ReplicationTest(TestCase):
 
     def test_user_replication(self):
         """Test basic user replication."""
+        raise SkipTest()
         portal_user = User.objects.create_user('rusty', 'rusty@edx.org', 'fakepass')
         portal_user.first_name='Rusty'
         portal_user.last_name='Skids'
@@ -80,6 +82,7 @@ class ReplicationTest(TestCase):
     def test_enrollment_for_existing_user_info(self):
         """Test the effect of Enrolling in a class if you've already got user
         data to be copied over."""
+        raise SkipTest()
         # Create our User
         portal_user = User.objects.create_user('jack', 'jack@edx.org', 'fakepass')
         portal_user.first_name = "Jack"
@@ -143,6 +146,8 @@ class ReplicationTest(TestCase):
 
     def test_enrollment_for_user_info_after_enrollment(self):
         """Test the effect of modifying User data after you've enrolled."""
+        raise SkipTest()
+
         # Create our User
         portal_user = User.objects.create_user('patty', 'patty@edx.org', 'fakepass')
         portal_user.first_name = "Patty"

common/djangoapps/student/views.py

@@ -140,7 +140,20 @@ def dashboard(request):
     if not user.is_active:
         message = render_to_string('registration/activate_account_notice.html', {'email': user.email})
 
-    context = {'courses': courses, 'message': message}
+
+    # Global staff can see what courses errored on their dashboard
+    staff_access = False
+    errored_courses = {}
+    if has_access(user, 'global', 'staff'):
+        # Show any courses that errored on load
+        staff_access = True
+        errored_courses = modulestore().get_errored_courses()
+
+    context = {'courses': courses,
+               'message': message,
+               'staff_access': staff_access,
+               'errored_courses': errored_courses,}
+
     return render_to_response('dashboard.html', context)
 
 

common/djangoapps/track/views.py

 import json
 import logging
 import os
+import pytz
 import datetime
 import dateutil.parser
 
@@ -84,15 +85,33 @@ def server_track(request, event_type, event, page=None):
         "time": datetime.datetime.utcnow().isoformat(),
         }
 
-    if event_type=="/event_logs" and request.user.is_staff:	# don't log
+    if event_type.startswith("/event_logs") and request.user.is_staff:	# don't log
         return
     log_event(event)
 
 @login_required
 @ensure_csrf_cookie
-def view_tracking_log(request):
+def view_tracking_log(request,args=''):
     if not request.user.is_staff:
         return redirect('/')
-    record_instances = TrackingLog.objects.all().order_by('-time')[0:100]
+    nlen = 100
+    username = ''
+    if args:
+        for arg in args.split('/'):
+            if arg.isdigit():
+                nlen = int(arg)
+            if arg.startswith('username='):
+                username = arg[9:]
+            
+    record_instances = TrackingLog.objects.all().order_by('-time')
+    if username:
+        record_instances = record_instances.filter(username=username)
+    record_instances = record_instances[0:nlen]
+    
+    # fix dtstamp
+    fmt = '%a %d-%b-%y %H:%M:%S'  # "%Y-%m-%d %H:%M:%S %Z%z"
+    for rinst in record_instances:
+        rinst.dtstr = rinst.time.replace(tzinfo=pytz.utc).astimezone(pytz.timezone('US/Eastern')).strftime(fmt)
+
     return render_to_response('tracking_log.html',{'records':record_instances})
 

common/djangoapps/util/cache.py

@@ -9,6 +9,7 @@ from functools import wraps
 
 from django.core import cache
 
+
 # If we can't find a 'general' CACHE defined in settings.py, we simply fall back
 # to returning the default cache. This will happen with dev machines.
 try:
@@ -41,7 +42,10 @@ def cache_if_anonymous(view_func):
     def _decorated(request, *args, **kwargs):
         if not request.user.is_authenticated():
             #Use the cache
-            cache_key = "cache_if_anonymous." + request.path
+            # same view accessed through different domain names may
+            # return different things, so include the domain name in the key.
+            domain = str(request.META.get('HTTP_HOST')) + '.'
+            cache_key = domain + "cache_if_anonymous." + request.path
             response = cache.get(cache_key)
             if not response:
                 response = view_func(request, *args, **kwargs)

common/lib/capa/capa/inputtypes.py

@@ -29,6 +29,7 @@ Each input type takes the xml tree as 'element', the previous answer as 'value',
 import logging
 import re
 import shlex  # for splitting quoted strings
+import json
 
 from lxml import etree
 import xml.sax.saxutils as saxutils
@@ -149,6 +150,7 @@ def optioninput(element, value, status, render_template, msg=''):
              'state': status,
              'msg': msg,
              'options': osetdict,
+             'inline': element.get('inline',''),
              }
 
     html = render_template("optioninput.html", context)
@@ -205,7 +207,7 @@ def extract_choices(element):
             raise Exception("[courseware.capa.inputtypes.extract_choices] \
                              Expected a <choice> tag; got %s instead"
                              % choice.tag)
-        choice_text = ''.join([x.text for x in choice])
+        choice_text = ''.join([etree.tostring(x) for x in choice])
 
         choices.append((choice.get("name"), choice_text))
 
@@ -293,7 +295,9 @@ def textline(element, value, status, render_template, msg=""):
     hidden = element.get('hidden', '')	 # if specified, then textline is hidden and id is stored in div of name given by hidden
     escapedict = {'"': '&quot;'}
     value = saxutils.escape(value, escapedict)	 # otherwise, answers with quotes in them crashes the system!
-    context = {'id': eid, 'value': value, 'state': status, 'count': count, 'size': size, 'msg': msg, 'hidden': hidden}
+    context = {'id': eid, 'value': value, 'state': status, 'count': count, 'size': size, 'msg': msg, 'hidden': hidden,
+               'inline': element.get('inline',''),
+               }
     html = render_template("textinput.html", context)
     try:
         xhtml = etree.XML(html)
@@ -336,6 +340,11 @@ def filesubmission(element, value, status, render_template, msg=''):
     Upload a single file (e.g. for programming assignments)
     '''
     eid = element.get('id')
+    escapedict = {'"': '&quot;'}
+    allowed_files  = json.dumps(element.get('allowed_files', '').split())
+    allowed_files  = saxutils.escape(allowed_files, escapedict)
+    required_files = json.dumps(element.get('required_files', '').split())
+    required_files = saxutils.escape(required_files, escapedict)
 
     # Check if problem has been queued
     queue_len = 0
@@ -345,7 +354,8 @@ def filesubmission(element, value, status, render_template, msg=''):
         msg = 'Submitted to grader. (Queue length: %s)' % queue_len
 
     context = { 'id': eid, 'state': status, 'msg': msg, 'value': value,
-                'queue_len': queue_len
+            'queue_len': queue_len, 'allowed_files': allowed_files,
+            'required_files': required_files
               }
     html = render_template("filesubmission.html", context)
     return etree.XML(html)

common/lib/capa/capa/responsetypes.py

@@ -557,7 +557,7 @@ class ChoiceResponse(LoncapaResponse):
             return CorrectMap(self.answer_id, 'incorrect')
 
     def get_answers(self):
-        return {self.answer_id: self.correct_choices}
+        return {self.answer_id: list(self.correct_choices)}
 
 #-----------------------------------------------------------------------------
 

common/lib/capa/capa/templates/filesubmission.html

 <section id="filesubmission_${id}" class="filesubmission">
-    <input type="file" name="input_${id}" id="input_${id}" value="${value}" multiple="multiple" /><br />
+    <input type="file" name="input_${id}" id="input_${id}" value="${value}" multiple="multiple" data-required_files="${required_files}" data-allowed_files="${allowed_files}"/><br />
     % if state == 'unsubmitted':
     <span class="unanswered" style="display:inline-block;" id="status_${id}"></span>
     % elif state == 'correct':

common/lib/capa/capa/templates/textinput.html

-<section id="textinput_${id}" class="textinput">
+<% doinline = "inline" if inline else "" %>
+
+<section id="textinput_${id}" class="textinput ${doinline}" >
     % if state == 'unsubmitted':
-    <div class="unanswered" id="status_${id}">
+    <div class="unanswered ${doinline}" id="status_${id}">
     % elif state == 'correct':
-    <div class="correct" id="status_${id}">
+    <div class="correct ${doinline}" id="status_${id}">
     % elif state == 'incorrect':
-    <div class="incorrect" id="status_${id}">
+    <div class="incorrect ${doinline}" id="status_${id}">
     % elif state == 'incomplete':
-    <div class="incorrect" id="status_${id}">
+    <div class="incorrect ${doinline}" id="status_${id}">
     % endif
     % if hidden:
       <div style="display:none;" name="${hidden}" inputid="input_${id}" />

common/lib/xmodule/setup.py

@@ -34,6 +34,7 @@ setup(
             "video = xmodule.video_module:VideoDescriptor",
             "videodev = xmodule.backcompat_module:TranslateCustomTagDescriptor",
             "videosequence = xmodule.seq_module:SequenceDescriptor",
+            "discussion = xmodule.discussion_module:DiscussionDescriptor",
         ]
     }
 )

common/lib/xmodule/xmodule/backcompat_module.py

@@ -68,7 +68,7 @@ class SemanticSectionDescriptor(XModuleDescriptor):
         the child element
         """
         xml_object = etree.fromstring(xml_data)
-        system.error_tracker("WARNING: the <{0}> tag is deprecated.  Please do not use in new content."
+        system.error_tracker("WARNING: the <{0}> tag is deprecated.  Please do not use in new content."
                              .format(xml_object.tag))
 
         if len(xml_object) == 1:

common/lib/xmodule/xmodule/capa_module.py

@@ -390,9 +390,19 @@ class CapaModule(XModule):
             raise NotFoundError('Answer is not available')
         else:
             answers = self.lcp.get_question_answers()
+
 	    # answers (eg <solution>) may have embedded images
-	    answers = dict( (k,self.system.replace_urls(answers[k], self.metadata['data_dir'])) for k in answers )
-            return {'answers': answers}
+        #   but be careful, some problems are using non-string answer dicts
+        new_answers = dict()
+        for answer_id in answers:
+            try:
+                new_answer = {answer_id: self.system.replace_urls(answers[answer_id], self.metadata['data_dir'])}
+            except TypeError:
+                log.debug('Unable to perform URL substitution on answers[%s]: %s' % (answer_id, answers[answer_id]))
+                new_answer = {answer_id: answers[answer_id]}
+            new_answers.update(new_answer)
+
+        return {'answers': new_answers}
 
     # Figure out if we should move these to capa_problem?
     def get_problem(self, get):

common/lib/xmodule/xmodule/course_module.py

@@ -18,7 +18,7 @@ class CourseDescriptor(SequenceDescriptor):
     class Textbook:
         def __init__(self, title, book_url):
             self.title = title
-            self.book_url = book_url 
+            self.book_url = book_url
             self.table_of_contents = self._get_toc_from_s3()
 
         @classmethod
@@ -30,11 +30,11 @@ class CourseDescriptor(SequenceDescriptor):
             return self.table_of_contents
 
         def _get_toc_from_s3(self):
-            '''
+            """
             Accesses the textbook's table of contents (default name "toc.xml") at the URL self.book_url
 
             Returns XML tree representation of the table of contents
-            '''
+            """
             toc_url = self.book_url + 'toc.xml'
 
             # Get the table of contents from S3
@@ -60,6 +60,8 @@ class CourseDescriptor(SequenceDescriptor):
     def __init__(self, system, definition=None, **kwargs):
         super(CourseDescriptor, self).__init__(system, definition, **kwargs)
         self.textbooks = self.definition['data']['textbooks']
+        
+        self.wiki_slug = self.definition['data']['wiki_slug'] or self.location.course
 
         msg = None
         if self.start is None:
@@ -72,14 +74,41 @@ class CourseDescriptor(SequenceDescriptor):
         self.enrollment_start = self._try_parse_time("enrollment_start")
         self.enrollment_end = self._try_parse_time("enrollment_end")
 
+        # NOTE: relies on the modulestore to call set_grading_policy() right after
+        # init.  (Modulestore is in charge of figuring out where to load the policy from)
+
+
+    def set_grading_policy(self, policy_str):
+        """Parse the policy specified in policy_str, and save it"""
+        try:
+            self._grading_policy = load_grading_policy(policy_str)
+        except:
+            self.system.error_tracker("Failed to load grading policy")
+            # Setting this to an empty dictionary will lead to errors when
+            # grading needs to happen, but should allow course staff to see
+            # the error log.
+            self._grading_policy = {}
+
+
     @classmethod
     def definition_from_xml(cls, xml_object, system):
         textbooks = []
         for textbook in xml_object.findall("textbook"):
             textbooks.append(cls.Textbook.from_xml_object(textbook))
             xml_object.remove(textbook)
+        
+        #Load the wiki tag if it exists
+        wiki_slug = None
+        wiki_tag = xml_object.find("wiki")
+        if wiki_tag is not None:
+            wiki_slug = wiki_tag.attrib.get("slug", default=None)
+            xml_object.remove(wiki_tag)
+        
         definition =  super(CourseDescriptor, cls).definition_from_xml(xml_object, system)
+        
         definition.setdefault('data', {})['textbooks'] = textbooks
+        definition['data']['wiki_slug'] = wiki_slug
+        
         return definition
 
     def has_started(self):
@@ -87,25 +116,11 @@ class CourseDescriptor(SequenceDescriptor):
 
     @property
     def grader(self):
-        return self.__grading_policy['GRADER']
+        return self._grading_policy['GRADER']
 
     @property
     def grade_cutoffs(self):
-        return self.__grading_policy['GRADE_CUTOFFS']
-
-    @lazyproperty
-    def __grading_policy(self):
-        policy_string = ""
-
-        try:
-            with self.system.resources_fs.open("grading_policy.json") as grading_policy_file:
-                policy_string = grading_policy_file.read()
-        except (IOError, ResourceNotFoundError):
-            log.warning("Unable to load course settings file from grading_policy.json in course " + self.id)
-
-        grading_policy = load_grading_policy(policy_string)
-
-        return grading_policy
+        return self._grading_policy['GRADE_CUTOFFS']
 
     @lazyproperty
     def grading_context(self):
@@ -195,6 +210,19 @@ class CourseDescriptor(SequenceDescriptor):
     def start_date_text(self):
         return time.strftime("%b %d, %Y", self.start)
 
+    # An extra property is used rather than the wiki_slug/number because
+    # there are courses that change the number for different runs. This allows
+    # courses to share the same css_class across runs even if they have
+    # different numbers.
+    # 
+    # TODO get rid of this as soon as possible or potentially build in a robust
+    # way to add in course-specific styling. There needs to be a discussion
+    # about the right way to do this, but arjun will address this ASAP. Also
+    # note that the courseware template needs to change when this is removed.
+    @property
+    def css_class(self):
+        return self.metadata.get('css_class', '')
+
     @property
     def title(self):
         return self.display_name
@@ -204,10 +232,6 @@ class CourseDescriptor(SequenceDescriptor):
         return self.location.course
 
     @property
-    def wiki_slug(self):
-        return self.location.course
-
-    @property
     def org(self):
         return self.location.org
 

common/lib/xmodule/xmodule/css/capa/display.scss

@@ -27,6 +27,10 @@ section.problem {
     }
   }
 
+  .inline {
+      display: inline;
+  }
+
   div {
     p {
       &.answer {

common/lib/xmodule/xmodule/css/sequence/display.scss

@@ -2,18 +2,54 @@ nav.sequence-nav {
   // TODO (cpennington): This doesn't work anymore. XModules aren't able to
   // import from external sources.
   @extend .topbar;
-  border-bottom: 1px solid $border-color;
-  @include border-top-right-radius(4px);
-  margin: 0 0 lh() (-(lh()));
+  margin: -4px 0 30px;
   position: relative;
+  border-bottom: none;
+
+  .left-shadow {
+    position: absolute;
+    top: 0;
+    left: 0;
+    z-index: 9999;
+    width: 20px;
+    height: 46px;
+    @include linear-gradient(left, rgba(0, 0, 0, .2), rgba(0, 0, 0, 0));
+    background-color: transparent;
+    pointer-events: none;
+  }
+
+  .right-shadow {
+    position: absolute;
+    top: 0;
+    right: 0;
+    z-index: 9999;
+    width: 20px;
+    height: 46px;
+    @include linear-gradient(right, rgba(0, 0, 0, .2), rgba(0, 0, 0, 0));
+    background-color: transparent;
+    pointer-events: none;
+  }
+
+  .sequence-list-wrapper {
+    position: relative;
+    z-index: 9999;
+    border: 1px solid #ccc;
+    height: 44px;
+    margin: 0 30px;
+    @include linear-gradient(top, #ddd, #eee);
+    overflow: hidden;
+    @include box-shadow(0 1px 3px rgba(0, 0, 0, .1) inset);
+  }
 
   ol {
+    position: absolute;
+    top: 0;
+    left: 0;
     @include box-sizing(border-box);
     display: table;
     height: 100%;
     margin: 0;
-    padding-left: 3px;
-    padding-right: flex-grid(1, 9);
+    padding: 0 10px;
     width: 100%;
 
     a {
@@ -25,42 +61,44 @@ nav.sequence-nav {
       min-width: 20px;
 
       a {
-        background-position: center;
+        width: 34px;
+        height: 34px;
+        margin: 4px auto;
+        background-position: center 10px;
         background-repeat: no-repeat;
         border: 1px solid transparent;
-        border-bottom: none;
-        @include border-radius(3px 3px 0 0);
+        @include border-radius(35px);
         cursor: pointer;
         display: block;
-        height: 10px;
-        padding: 15px 0 14px;
+        padding: 0;
         position: relative;
         @include transition();
-        width: 100%;
 
         &:hover {
+          background-color: #fff;
           background-repeat: no-repeat;
-          background-position: center;
-          background-color: #F3F3F3;
-        }
-
-        &.visited {
-          background-color: #F3F3F3;
-
-          &:hover {
-            background-position: center center;
-          }
+          background-position: center 10px;
         }
 
         &.active {
-          border-color: $border-color;
-          @include box-shadow(0 2px 0 #fff);
           background-color: #fff;
           z-index: 9;
 
+          // &:after {
+          //   content: '▲';
+          //   position: absolute;
+          //   top: 28px;
+          //   left: 50%;
+          //   z-index: 9999;
+          //   margin-left: -5px;
+          //   font-size: 12px;
+          //   color: #aaa;
+          // }
+
           &:hover {
-            background-position: center;
             background-color: #fff;
+            background-repeat: no-repeat;
+            background-position: center 10px;
           }
         }
 
@@ -171,20 +209,24 @@ nav.sequence-nav {
   }
 
   ul {
+    position: absolute;
+    top: 0;
     list-style: none;
     height: 100%;
-    position: absolute;
     right: 0;
     top: 0;
-    width: flex-grid(1, 9);
-    border: 1px solid $border-color;
-    border-bottom: 0;
-    @include border-radius(3px 3px 0 0);
+    width: 100%;
+    margin: 0;
+    border: none;
 
     li {
-      float: left;
+      position: absolute;
       margin-bottom: 0;
-      width: 50%;
+      height: 44px;
+      width: 70px;
+      border: 1px solid #ccc;
+      @include linear-gradient(top, #eee, #ddd);
+      @include box-shadow(0 1px 0 rgba(255, 255, 255, .7) inset);
 
       &.prev, &.next {
 
@@ -192,14 +234,13 @@ nav.sequence-nav {
           background-position: center;
           background-repeat: no-repeat;
           display: block;
-          height: 10px;
-          padding: 15px 0 14px;
+          height: 34px;
+          width: 40px;
           text-indent: -9999px;
           @include transition(all, .2s, $ease-in-out-quad);
 
           &:hover {
             opacity: .5;
-            background-color: #f4f4f4;
           }
 
           &.disabled {
@@ -210,15 +251,23 @@ nav.sequence-nav {
       }
 
       &.prev {
+        left: -10px;
+        border-radius: 35px 0 0 35px;
+
         a {
           background-image: url('../images/sequence-nav/previous-icon.png');
+          background-position: center 15px;
         }
       }
 
       &.next {
+        right: -10px;
+        border-radius: 0 35px 35px 0;
+
         a {
-          border-left: 1px solid lighten($border-color, 10%);
+          margin-left: 30px;
           background-image: url('../images/sequence-nav/next-icon.png');
+          background-position: center 15px;
         }
       }
     }

common/lib/xmodule/xmodule/css/video/display.scss

 div.video {
   @include clearfix();
   background: #f3f3f3;
-  border-bottom: 1px solid #e1e1e1;
-  border-top: 1px solid #e1e1e1;
   display: block;
-  margin: 0 0 0 (-(lh()));
-  padding: 6px lh();
+  margin: 0 -12px;
+  padding: 12px;
+  border-radius: 5px;
 
   article.video-wrapper {
     float: left;
@@ -401,6 +400,7 @@ div.video {
     overflow: auto;
     width: flex-grid(3, 9);
     margin: 0;
+    font-size: 14px;
 
     li {
       border: 0;

common/lib/xmodule/xmodule/discussion_module.py

+from lxml import etree
+
+from xmodule.x_module import XModule
+from xmodule.raw_module import RawDescriptor
+
+import json
+
+class DiscussionModule(XModule):
+    def get_html(self):
+        context = {
+            'discussion_id': self.discussion_id,
+        }
+        return self.system.render_template('discussion/_discussion_module.html', context)
+
+    def __init__(self, system, location, definition, descriptor, instance_state=None, shared_state=None, **kwargs):
+        XModule.__init__(self, system, location, definition, descriptor, instance_state, shared_state, **kwargs)
+
+        if isinstance(instance_state, str):
+            instance_state = json.loads(instance_state)
+        xml_data = etree.fromstring(definition['data'])
+        self.discussion_id = xml_data.attrib['id']
+        self.title = xml_data.attrib['for']
+        self.discussion_category = xml_data.attrib['discussion_category']
+
+class DiscussionDescriptor(RawDescriptor):
+    module_class = DiscussionModule

common/lib/xmodule/xmodule/error_module.py

@@ -15,18 +15,39 @@ from xmodule.errortracker import exc_info_to_str
 
 log = logging.getLogger(__name__)
 
+# NOTE: This is not the most beautiful design in the world, but there's no good
+# way to tell if the module is being used in a staff context or not.  Errors that get discovered
+# at course load time are turned into ErrorDescriptor objects, and automatically hidden from students.
+# Unfortunately, we can also have errors when loading modules mid-request, and then we need to decide
+# what to show, and the logic for that belongs in the LMS (e.g. in get_module), so the error handler
+# decides whether to create a staff or not-staff module.
+
 class ErrorModule(XModule):
     def get_html(self):
-        '''Show an error.
+        '''Show an error to staff.
         TODO (vshnayder): proper style, divs, etc.
         '''
         # staff get to see all the details
         return self.system.render_template('module-error.html', {
+            'staff_access' : True,
             'data' : self.definition['data']['contents'],
             'error' : self.definition['data']['error_msg'],
             })
 
 
+class NonStaffErrorModule(XModule):
+    def get_html(self):
+        '''Show an error to a student.
+        TODO (vshnayder): proper style, divs, etc.
+        '''
+        # staff get to see all the details
+        return self.system.render_template('module-error.html', {
+            'staff_access' : False,
+            'data' : "",
+            'error' : "",
+            })
+
+
 class ErrorDescriptor(EditingDescriptor):
     """
     Module that provides a raw editing view of broken xml.
@@ -99,3 +120,9 @@ class ErrorDescriptor(EditingDescriptor):
             err_node = etree.SubElement(root, 'error_msg')
             err_node.text = self.definition['data']['error_msg']
             return etree.tostring(root)
+
+class NonStaffErrorDescriptor(ErrorDescriptor):
+    """
+    Module that provides non-staff error messages.
+    """
+    module_class = NonStaffErrorModule

common/lib/xmodule/xmodule/graders.py

 import abc
 import json
 import logging
+import sys
 
 from collections import namedtuple
 
@@ -14,11 +15,11 @@ def load_grading_policy(course_policy_string):
     """
     This loads a grading policy from a string (usually read from a file),
     which can be a JSON object or an empty string.
-    
+
     The JSON object can have the keys GRADER and GRADE_CUTOFFS. If either is
     missing, it reverts to the default.
     """
-    
+
     default_policy_string = """
     {
         "GRADER" : [
@@ -56,7 +57,7 @@ def load_grading_policy(course_policy_string):
         }
     }
     """
-    
+
     # Load the global settings as a dictionary
     grading_policy = json.loads(default_policy_string)
 
@@ -64,15 +65,15 @@ def load_grading_policy(course_policy_string):
     course_policy = {}
     if course_policy_string:
         course_policy = json.loads(course_policy_string)
-    
+
     # Override any global settings with the course settings
     grading_policy.update(course_policy)
 
     # Here is where we should parse any configurations, so that we can fail early
     grading_policy['GRADER'] = grader_from_conf(grading_policy['GRADER'])
-    
+
     return grading_policy
-    
+
 
 def aggregate_scores(scores, section_name="summary"):
     """
@@ -130,9 +131,11 @@ def grader_from_conf(conf):
                 raise ValueError("Configuration has no appropriate grader class.")
 
         except (TypeError, ValueError) as error:
-            errorString = "Unable to parse grader configuration:\n    " + str(subgraderconf) + "\n    Error was:\n    " + str(error)
-            log.critical(errorString)
-            raise ValueError(errorString)
+            # Add info and re-raise
+            msg = ("Unable to parse grader configuration:\n    " +
+                   str(subgraderconf) +
+                   "\n    Error was:\n    " + str(error))
+            raise ValueError(msg), None, sys.exc_info()[2]
 
     return WeightedSubsectionsGrader(subgraders)
 

common/lib/xmodule/xmodule/html_module.py

@@ -86,7 +86,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
             # online and has imported all current (fall 2012) courses from xml
             if not system.resources_fs.exists(filepath):
                 candidates = cls.backcompat_paths(filepath)
-                log.debug("candidates = {0}".format(candidates))
+                #log.debug("candidates = {0}".format(candidates))
                 for candidate in candidates:
                     if system.resources_fs.exists(candidate):
                         filepath = candidate

common/lib/xmodule/xmodule/js/src/capa/display.coffee

@@ -160,24 +160,42 @@ class @Problem
     max_filesize = 4*1000*1000 # 4 MB
     file_too_large = false
     file_not_selected = false
+    required_files_not_submitted = false
+    unallowed_file_submitted = false
+
+    errors = []
 
     @inputs.each (index, element) ->
       if element.type is 'file'
+        required_files = $(element).data("required_files")
+        allowed_files  = $(element).data("allowed_files")
         for file in element.files
+          if allowed_files.length != 0 and file.name not in allowed_files
+              unallowed_file_submitted = true
+              errors.push "You submitted #{file.name}; only #{allowed_files} are allowed."
+          if file.name in required_files
+              required_files.splice(required_files.indexOf(file.name), 1)
           if file.size > max_filesize
             file_too_large = true
-            alert 'Submission aborted! Your file "' + file.name '" is too large (max size: ' + max_filesize/(1000*1000) + ' MB)'
+            errors.push 'Your file "' + file.name '" is too large (max size: ' + max_filesize/(1000*1000) + ' MB)'
           fd.append(element.id, file)
         if element.files.length == 0 
           file_not_selected = true
           fd.append(element.id, '') # In case we want to allow submissions with no file
+        if required_files.length != 0
+          required_files_not_submitted = true
+          errors.push "You did not submit the required files: #{required_files}."
       else
         fd.append(element.id, element.value)
+
     
     if file_not_selected
-      alert 'Submission aborted! You did not select any files to submit'
+      errors.push 'You did not select any files to submit'
+
+    if errors.length > 0
+        alert errors.join("\n")
 
-    abort_submission = file_too_large or file_not_selected
+    abort_submission = file_too_large or file_not_selected or unallowed_file_submitted or required_files_not_submitted
 
     settings = 
       type: "POST"

common/lib/xmodule/xmodule/js/src/sequence/display.coffee

@@ -3,6 +3,7 @@ class @Sequence
     @el = $(element).find('.sequence')
     @contents = @$('.seq_contents')
     @id = @el.data('id')
+    @modx_url = @el.data('course_modx_root')
     @initProgress()
     @bind()
     @render parseInt(@el.data('position'))
@@ -76,13 +77,14 @@ class @Sequence
     if @position != new_position
       if @position != undefined
         @mark_visited @position
-        $.postWithPrefix "/modx/#{@id}/goto_position", position: new_position
+        modx_full_url = @modx_url + '/' + @id + '/goto_position'
+        $.postWithPrefix modx_full_url, position: new_position
 
       @mark_active new_position
       @$('#seq_content').html @contents.eq(new_position - 1).text()
       XModule.loadModules('display', @$('#seq_content'))
 
-      MathJax.Hub.Queue(["Typeset", MathJax.Hub])
+      MathJax.Hub.Queue(["Typeset", MathJax.Hub, "seq_content"]) # NOTE: Actually redundant. Some other MathJax call also being performed
       @position = new_position
       @toggleArrows()
       @hookUpProgressEvent()
@@ -91,7 +93,7 @@ class @Sequence
     event.preventDefault()
     new_position = $(event.target).data('element')
     Logger.log "seq_goto", old: @position, new: new_position, id: @id
-
+    
     # On Sequence chage, destroy any existing polling thread 
     #   for queued submissions, see ../capa/display.coffee
     if window.queuePollerID

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -4,16 +4,17 @@ import os
 import re
 
 from collections import defaultdict
+from cStringIO import StringIO
 from fs.osfs import OSFS
 from importlib import import_module
 from lxml import etree
 from lxml.html import HtmlComment
 from path import path
+
 from xmodule.errortracker import ErrorLog, make_error_tracker
-from xmodule.x_module import XModuleDescriptor, XMLParsingSystem
 from xmodule.course_module import CourseDescriptor
 from xmodule.mako_module import MakoDescriptorSystem
-from cStringIO import StringIO
+from xmodule.x_module import XModuleDescriptor, XMLParsingSystem
 
 from . import ModuleStoreBase, Location
 from .exceptions import ItemNotFoundError
@@ -134,12 +135,12 @@ class XMLModuleStore(ModuleStoreBase):
         self.data_dir = path(data_dir)
         self.modules = defaultdict(dict)  # course_id -> dict(location -> XModuleDescriptor)
         self.courses = {}  # course_dir -> XModuleDescriptor for the course
+        self.errored_courses = {}  # course_dir -> errorlog, for dirs that failed to load
 
         if default_class is None:
             self.default_class = None
         else:
             module_path, _, class_name = default_class.rpartition('.')
-            #log.debug('module_path = %s' % module_path)
             class_ = getattr(import_module(module_path), class_name)
             self.default_class = class_
 
@@ -162,18 +163,27 @@ class XMLModuleStore(ModuleStoreBase):
         '''
         Load a course, keeping track of errors as we go along.
         '''
+        # Special-case code here, since we don't have a location for the
+        # course before it loads.
+        # So, make a tracker to track load-time errors, then put in the right
+        # place after the course loads and we have its location
+        errorlog = make_error_tracker()
+        course_descriptor = None
         try:
-            # Special-case code here, since we don't have a location for the
-            # course before it loads.
-            # So, make a tracker to track load-time errors, then put in the right
-            # place after the course loads and we have its location
-            errorlog = make_error_tracker()
             course_descriptor = self.load_course(course_dir, errorlog.tracker)
+        except Exception as e:
+            msg = "Failed to load course '{0}': {1}".format(course_dir, str(e))
+            log.exception(msg)
+            errorlog.tracker(msg)
+
+        if course_descriptor is not None:
             self.courses[course_dir] = course_descriptor
             self._location_errors[course_descriptor.location] = errorlog
-        except:
-            msg = "Failed to load course '%s'" % course_dir
-            log.exception(msg)
+        else:
+            # Didn't load course.  Instead, save the errors elsewhere.
+            self.errored_courses[course_dir] = errorlog
+
+
 
     def __unicode__(self):
         '''
@@ -202,6 +212,28 @@ class XMLModuleStore(ModuleStoreBase):
         return {}
 
 
+    def read_grading_policy(self, paths, tracker):
+        """Load a grading policy from the specified paths, in order, if it exists."""
+        # Default to a blank policy
+        policy_str = ""
+
+        for policy_path in paths:
+            if not os.path.exists(policy_path):
+                continue
+            log.debug("Loading grading policy from {0}".format(policy_path))
+            try:
+                with open(policy_path) as grading_policy_file:
+                    policy_str = grading_policy_file.read()
+                    # if we successfully read the file, stop looking at backups
+                    break
+            except (IOError):
+                msg = "Unable to load course settings file from '{0}'".format(policy_path)
+                tracker(msg)
+                log.warning(msg)
+
+        return policy_str
+
+
     def load_course(self, course_dir, tracker):
         """
         Load a course into this module store
@@ -242,9 +274,16 @@ class XMLModuleStore(ModuleStoreBase):
                 course = course_dir
 
             url_name = course_data.get('url_name', course_data.get('slug'))
+            policy_dir = None
             if url_name:
-                policy_path = self.data_dir / course_dir / 'policies' / '{0}.json'.format(url_name)
+                policy_dir = self.data_dir / course_dir / 'policies' / url_name
+                policy_path = policy_dir / 'policy.json'
                 policy = self.load_policy(policy_path, tracker)
+
+                # VS[compat]: remove once courses use the policy dirs.
+                if policy == {}:
+                    old_policy_path = self.data_dir / course_dir / 'policies' / '{0}.json'.format(url_name)
+                    policy = self.load_policy(old_policy_path, tracker)
             else:
                 policy = {}
                 # VS[compat] : 'name' is deprecated, but support it for now...
@@ -268,6 +307,15 @@ class XMLModuleStore(ModuleStoreBase):
             # after we have the course descriptor.
             XModuleDescriptor.compute_inherited_metadata(course_descriptor)
 
+            # Try to load grading policy
+            paths = [self.data_dir / course_dir / 'grading_policy.json']
+            if policy_dir:
+                paths = [policy_dir / 'grading_policy.json'] + paths
+
+            policy_str = self.read_grading_policy(paths, tracker)
+            course_descriptor.set_grading_policy(policy_str)
+
+
             log.debug('========> Done with course import from {0}'.format(course_dir))
             return course_descriptor
 
@@ -318,6 +366,12 @@ class XMLModuleStore(ModuleStoreBase):
         """
         return self.courses.values()
 
+    def get_errored_courses(self):
+        """
+        Return a dictionary of course_dir -> [(msg, exception_str)], for each
+        course_dir where course loading failed.
+        """
+        return dict( (k, self.errored_courses[k].errors) for k in self.errored_courses)
 
     def create_item(self, location):
         raise NotImplementedError("XMLModuleStores are read-only")

common/lib/xmodule/xmodule/tests/test_import.py

@@ -233,6 +233,9 @@ class ImportTestCase(unittest.TestCase):
         self.assertEqual(toy_ch.display_name, "Overview")
         self.assertEqual(two_toys_ch.display_name, "Two Toy Overview")
 
+        # Also check that the grading policy loaded
+        self.assertEqual(two_toys.grade_cutoffs['C'], 0.5999)
+
 
     def test_definition_loading(self):
         """When two courses share the same org and course name and

common/lib/xmodule/xmodule/xml_module.py

@@ -38,7 +38,7 @@ def get_metadata_from_xml(xml_object, remove=True):
     if meta is None:
         return ''
     dmdata = meta.text
-    log.debug('meta for %s loaded: %s' % (xml_object,dmdata))
+    #log.debug('meta for %s loaded: %s' % (xml_object,dmdata))
     if remove:
         xml_object.remove(meta)
     return dmdata

common/test/data/test_start_date/README.md

+Simple course.  If start dates are on, non-staff users should see Overview, but not Ch 2.

common/test/data/test_start_date/course.xml

+roots/2012_Fall.xml
\ No newline at end of file

common/test/data/test_start_date/course/2012_Fall.xml

+<course>
+  <chapter url_name="Overview">
+    <videosequence url_name="Toy_Videos">
+      <html url_name="toylab"/>
+      <video url_name="Video_Resources" youtube="1.0:1bK-WdDi6Qw"/>
+    </videosequence>
+    <video url_name="Welcome" youtube="1.0:p2Q6BrNhdh8"/>
+  </chapter>
+  <chapter url_name="Ch2">
+     <html url_name="test_html">
+         <h2>Welcome</h2>
+     </html>
+  </chapter>
+
+</course>

common/test/data/test_start_date/html/toylab.html

+<b>Lab 2A: Superposition Experiment</b>
+
+<p>Isn't the toy course great?</p>

common/test/data/test_start_date/html/toylab.xml

+<html filename="toylab.html"/>
\ No newline at end of file

common/test/data/test_start_date/policies/2012_Fall.json

+{
+    "course/2012_Fall": {
+        "graceperiod": "2 days 5 hours 59 minutes 59 seconds",
+        "start": "2011-07-17T12:00",
+        "display_name": "Toy Course"
+    },
+    "chapter/Overview": {
+        "display_name": "Overview"
+    },
+    "chapter/Ch2": {
+       "display_name": "Chapter 2",
+       "start": "2015-07-17T12:00"
+    },
+    "videosequence/Toy_Videos": {
+        "display_name": "Toy Videos",
+        "format": "Lecture Sequence"
+    },
+    "html/toylab": {
+        "display_name": "Toy lab"
+    },
+    "video/Video_Resources": {
+        "display_name": "Video Resources"
+    },
+    "video/Welcome": {
+        "display_name": "Welcome"
+    }
+}

common/test/data/test_start_date/roots/2012_Fall.xml

+<course org="edX" course="test_start_date" url_name="2012_Fall"/>
\ No newline at end of file

common/test/data/toy/html/toylab.xml

+<html filename="toylab.html"/>
\ No newline at end of file

common/test/data/two_toys/policies/TT_2012_Fall/grading_policy.json

+{
+  "GRADER" : [
+    {
+      "type" : "Homework",
+      "min_count" : 12,
+      "drop_count" : 2,
+      "short_label" : "HW",
+      "weight" : 0.15
+    },
+    {
+      "type" : "Lab",
+      "min_count" : 12,
+      "drop_count" : 2,
+      "category" : "Labs",
+      "weight" : 0.15
+    },
+    {
+      "type" : "Midterm",
+      "name" : "Midterm Exam",
+      "short_label" : "Midterm",
+      "weight" : 0.3
+    },
+    {
+      "type" : "Final",
+      "name" : "Final Exam",
+      "short_label" : "Final",
+      "weight" : 0.4
+    }
+  ],
+  "GRADE_CUTOFFS" : {
+    "A" : 0.87,
+    "B" : 0.7,
+    "C" : 0.5999
+  }
+}

doc/discussion.md

+# Running the discussion service
+
+## Instruction for Mac
+
+## Installing Mongodb
+
+If you haven't done so already:
+
+    brew install mongodb
+
+Make sure that you have mongodb running. You can simply open a new terminal tab and type:
+ 
+    mongod
+
+## Installing elasticsearch
+
+    brew install elasticsearch
+
+For debugging, it's often more convenient to have elasticsearch running in a terminal tab instead of in background. To do so, simply open a new terminal tab and then type:
+
+    elasticsearch -f
+
+## Setting up the discussion service
+
+First, make sure that you have access to the [github repository](https://github.com/rll/cs_comments_service). If this were not the case, send an email to dementrock@gmail.com.
+
+First go into the mitx_all directory. Then type
+
+    git clone git@github.com:rll/cs_comments_service.git
+    cd cs_comments_service/
+
+If you see a prompt asking "Do you wish to trust this .rvmrc file?", type "y"
+
+Now if you see this error "Gemset 'cs_comments_service' does not exist," run the following command to create the gemset and then use the rvm environment manually:
+
+    rvm gemset create 'cs_comments_service'
+    rvm use 1.9.3@cs_comments_service
+
+Now use the following command to install required packages:
+
+    bundle install
+
+The following command creates database indexes:
+
+    bundle exec rake db:init
+
+Now use the following command to generate seeds (basically some random comments in Latin):
+
+    bundle exec rake db:seed
+
+It's done! Launch the app now:
+
+    ruby app.rb
+
+## Running the delayed job worker
+
+In the discussion service, notifications are handled asynchronously using a third party gem called delayed_job. If you want to test this functionality, run the following command in a separate tab:
+
+    bundle exec rake jobs:work
+
+## Initialize roles and permissions
+
+To fully test the discussion forum, you might want to act as a moderator or an administrator. Currently, moderators can manage everything in the forum, and administrator can manage everything plus assigning and revoking moderator status of other users.
+
+First make sure that the database is up-to-date:
+
+    rake django-admin[syncdb]
+    rake django-admin[migrate]
+
+For convenience, add the following environment variables to the terminal (assuming that you're using configuration set lms.envs.dev):
+
+    export DJANGO_SETTINGS_MODULE=lms.envs.dev
+    export PYTHONPATH=.
+
+Now initialzie roles and permissions:
+
+    django-admin.py seed_permissions_roles
+
+To assign yourself as a moderator, use the following command (assuming your username is "test", and the course id is "MITx/6.002x/2012_Fall"):
+
+    django-admin.py assign_role test Moderator "MITx/6.002x/2012_Fall"
+
+To assign yourself as an administrator, use the following command
+
+    django-admin.py assign_role test Administrator "MITx/6.002x/2012_Fall"
+
+## Some other useful commands
+
+### generate seeds for a specific forum
+The seed generating command above assumes that you have the following discussion tags somewhere in the course data:
+
+    <discussion for="Welcome Video" id="video_1" discussion_category="Video"/>
+    <discussion for="Lab 0: Using the Tools" id="lab_1" discussion_category="Lab"/>
+    <discussion for="Lab Circuit Sandbox" id="lab_2" discussion_category="Lab"/>
+
+For example, you can insert them into overview section as following:
+
+    <chapter name="Overview">
+      <section format="Video" name="Welcome">
+        <vertical>
+          <video youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8"/>
+          <discussion for="Welcome Video" id="video_1" discussion_category="Video"/>
+        </vertical>
+      </section>
+      <section format="Lecture Sequence" name="System Usage Sequence">
+        <%include file="sections/introseq.xml"/>
+      </section>
+      <section format="Lab" name="Lab0: Using the tools">
+        <vertical>
+          <html> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab </html>
+          <problem name="Lab 0: Using the Tools" filename="Lab0" rerandomize="false"/>
+          <discussion for="Lab 0: Using the Tools" id="lab_1" discussion_category="Lab"/>
+        </vertical>
+      </section>
+      <section format="Lab" name="Circuit Sandbox">
+        <vertical>
+          <problem name="Circuit Sandbox" filename="Lab_sandbox" rerandomize="false"/>
+          <discussion for="Lab Circuit Sandbox" id="lab_2" discussion_category="Lab"/>
+        </vertical>
+      </section>
+    </chapter>
+
+Currently, only the attribute "id" is actually used, which identifies discussion forum. In the code for the data generator, the corresponding lines are:
+
+    generate_comments_for("video_1")
+    generate_comments_for("lab_1")
+    generate_comments_for("lab_2")
+
+We also have a command for generating comments within a forum with the specified id:
+
+    bundle exec rake db:generate_comments[type_the_discussion_id_here]
+
+For instance, if you want to generate comments for a new discussion tab named "lab_3", then use the following command
+
+    bundle exec rake db:generate_comments[lab_3]
+
+### Running tests for the service
+
+    bundle exec rspec
+
+Warning: the development and test environments share the same elasticsearch index. After running tests, search may not work in the development environment. You simply need to reindex:
+
+    bundle exec rake db:reindex_search
+
+### debugging the service
+
+You can use the following command to launch a console within the service environment:
+
+    bundle exec rake console
+
+### show user roles and permissions
+
+Use the following command to see the roles and permissions of a user in a given course (assuming, again, that the username is "test"):
+
+    django-admin.py show_permissions moderator
+
+You need to make sure that the environment variables are exported. Otherwise you would need to do
+
+    django-admin.py show_permissions moderator --settings=lms.envs.dev --pythonpath=.

install.txt

@@ -74,5 +74,4 @@ There is also a script "create-dev-env.sh" that automates these steps.
         $ django-admin.py syncdb --settings=envs.dev --pythonpath=.
         $ django-admin.py migrate --settings=envs.dev --pythonpath=.
         $ django-admin.py runserver --settings=envs.dev --pythonpath=.   
- 
 

lms/djangoapps/acceptance_testing/features/static/terrain.py

@@ -9,4 +9,4 @@ def setup_browser():
 
 @after.all
 def teardown_browser(total):
-  world.browser.quit()
+  world.browser.quit()
\ No newline at end of file

lms/djangoapps/course_wiki/course_nav.py

@@ -5,6 +5,7 @@ from django.http import Http404
 from django.shortcuts import redirect
 
 from wiki.models import reverse as wiki_reverse
+from courseware.access import has_access
 from courseware.courses import get_course_with_access
 
 
@@ -135,7 +136,9 @@ def context_processor(request):
         
         try:
             course = get_course_with_access(request.user, course_id, 'load')
-            return {'course' : course}
+            staff_access = has_access(request.user, course, 'staff')
+            return {'course' : course,
+                    'staff_access': staff_access}
         except Http404:
             # We couldn't access the course for whatever reason. It is too late to change
             # the URL here, so we just leave the course context. The middleware shouldn't

lms/djangoapps/course_wiki/views.py

@@ -80,8 +80,8 @@ def course_wiki_redirect(request, course_id):
         urlpath = URLPath.create_article(
             root,
             course_slug,
-            title=course.title,
-            content="This is the wiki for " + course.title + ".",
+            title=course_slug,
+            content="This is the wiki for **{0}**'s _{1}_.".format(course.org, course.title),
             user_message="Course page automatically created.",
             user=None,
             ip_address=None,
@@ -114,7 +114,7 @@ def get_or_create_root():
     "===",
     "Visit a course wiki to add an article."))
     
-    root = URLPath.create_root(title="edX Wiki",
+    root = URLPath.create_root(title="Wiki",
                         content=starting_content)
     article = root.article
     article.group = None

lms/djangoapps/courseware/access.py

@@ -13,7 +13,6 @@ from xmodule.modulestore import Location
 from xmodule.timeparse import parse_time
 from xmodule.x_module import XModule, XModuleDescriptor
 
-
 DEBUG_ACCESS = False
 
 log = logging.getLogger(__name__)
@@ -63,6 +62,9 @@ def has_access(user, obj, action):
     if isinstance(obj, Location):
         return _has_access_location(user, obj, action)
 
+    if isinstance(obj, basestring):
+        return _has_access_string(user, obj, action)
+
     # Passing an unknown object here is a coding error, so rather than
     # returning a default, complain.
     raise TypeError("Unknown object type in has_access(): '{0}'"
@@ -238,6 +240,30 @@ def _has_access_location(user, location, action):
     return _dispatch(checkers, action, user, location)
 
 
+def _has_access_string(user, perm, action):
+    """
+    Check if user has certain special access, specified as string.  Valid strings:
+
+    'global'
+
+    Valid actions:
+
+    'staff' -- global staff access.
+    """
+
+    def check_staff():
+        if perm != 'global':
+            debug("Deny: invalid permission '%s'", perm)
+            return False
+        return _has_global_staff_access(user)
+
+    checkers = {
+        'staff': check_staff
+        }
+
+    return _dispatch(checkers, action, user, perm)
+
+
 #####  Internal helper methods below
 
 def _dispatch(table, action, user, obj):
@@ -266,6 +292,15 @@ def _course_staff_group_name(location):
     """
     return 'staff_%s' % Location(location).course
 
+def _has_global_staff_access(user):
+    if user.is_staff:
+        debug("Allow: user.is_staff")
+        return True
+    else:
+        debug("Deny: not user.is_staff")
+        return False
+
+
 def _has_staff_access_to_location(user, location):
     '''
     Returns True if the given user has staff access to a location.  For now this

lms/djangoapps/courseware/courses.py

@@ -30,7 +30,6 @@ def get_course_by_id(course_id):
         raise Http404("Course not found.")
 
 
-
 def get_course_with_access(user, course_id, action):
     """
     Given a course_id, look up the corresponding course descriptor,
@@ -142,6 +141,35 @@ def get_course_info_section(course, section_key):
 
     raise KeyError("Invalid about key " + str(section_key))
 
+# TODO: Fix this such that these are pulled in as extra course-specific tabs.
+#       arjun will address this by the end of October if no one does so prior to
+#       then. 
+def get_course_syllabus_section(course, section_key):
+    """
+    This returns the snippet of html to be rendered on the syllabus page,
+    given the key for the section.
+
+    Valid keys:
+    - syllabus
+    - guest_syllabus
+    """
+
+    # Many of these are stored as html files instead of some semantic
+    # markup. This can change without effecting this interface when we find a
+    # good format for defining so many snippets of text/html.
+
+    if section_key in ['syllabus', 'guest_syllabus']:
+        try:
+            with course.system.resources_fs.open(path("syllabus") / section_key + ".html") as htmlFile:
+                return replace_urls(htmlFile.read().decode('utf-8'),
+                                    course.metadata['data_dir'])
+        except ResourceNotFoundError:
+            log.exception("Missing syllabus section {key} in course {url}".format(
+                key=section_key, url=course.location.url()))
+            return "! Syllabus missing !"
+
+    raise KeyError("Invalid about key " + str(section_key))
+
 
 def get_courses_by_university(user, domain=None):
     '''

lms/djangoapps/courseware/grades.py

@@ -29,6 +29,8 @@ def grade(student, request, course, student_module_cache=None):
     output from the course grader, augmented with the final letter
     grade. The keys in the output are:
 
+    course: a CourseDescriptor
+
     - grade : A final letter grade.
     - percent : The final percent for the class (rounded up).
     - section_breakdown : A breakdown of each section that makes
@@ -42,7 +44,7 @@ def grade(student, request, course, student_module_cache=None):
     grading_context = course.grading_context
 
     if student_module_cache == None:
-        student_module_cache = StudentModuleCache(student, grading_context['all_descriptors'])
+        student_module_cache = StudentModuleCache(course.id, student, grading_context['all_descriptors'])
 
     totaled_scores = {}
     # This next complicated loop is just to collect the totaled_scores, which is
@@ -56,7 +58,8 @@ def grade(student, request, course, student_module_cache=None):
             should_grade_section = False
             # If we haven't seen a single problem in the section, we don't have to grade it at all! We can assume 0%
             for moduledescriptor in section['xmoduledescriptors']:
-                if student_module_cache.lookup(moduledescriptor.category, moduledescriptor.location.url() ):
+                if student_module_cache.lookup(
+                        course.id, moduledescriptor.category, moduledescriptor.location.url()):
                     should_grade_section = True
                     break
 
@@ -64,10 +67,9 @@ def grade(student, request, course, student_module_cache=None):
                 scores = []
                 # TODO: We need the request to pass into here. If we could forgo that, our arguments
                 # would be simpler
-                course_id = CourseDescriptor.location_to_id(course.location)
                 section_module = get_module(student, request,
                                             section_descriptor.location, student_module_cache,
-                                            course_id)
+                                            course.id)
                 if section_module is None:
                     # student doesn't have access to this module, or something else
                     # went wrong.
@@ -76,7 +78,7 @@ def grade(student, request, course, student_module_cache=None):
                 # TODO: We may be able to speed this up by only getting a list of children IDs from section_module
                 # Then, we may not need to instatiate any problems if they are already in the database
                 for module in yield_module_descendents(section_module):
-                    (correct, total) = get_score(student, module, student_module_cache)
+                    (correct, total) = get_score(course.id, student, module, student_module_cache)
                     if correct is None and total is None:
                         continue
 
@@ -161,7 +163,9 @@ def progress_summary(student, course, grader, student_module_cache):
             graded = s.metadata.get('graded', False)
             scores = []
             for module in yield_module_descendents(s):
-                (correct, total) = get_score(student, module, student_module_cache)
+                # course is a module, not a descriptor...
+                course_id = course.descriptor.id
+                (correct, total) = get_score(course_id, student, module, student_module_cache)
                 if correct is None and total is None:
                     continue
 
@@ -190,7 +194,7 @@ def progress_summary(student, course, grader, student_module_cache):
     return chapters
 
 
-def get_score(user, problem, student_module_cache):
+def get_score(course_id, user, problem, student_module_cache):
     """
     Return the score for a user on a problem, as a tuple (correct, total).
 
@@ -205,10 +209,11 @@ def get_score(user, problem, student_module_cache):
     correct = 0.0
 
     # If the ID is not in the cache, add the item
-    instance_module = get_instance_module(user, problem, student_module_cache)
+    instance_module = get_instance_module(course_id, user, problem, student_module_cache)
     # instance_module = student_module_cache.lookup(problem.category, problem.id)
     # if instance_module is None:
     #     instance_module = StudentModule(module_type=problem.category,
+    #                                     course_id=????,
     #                                     module_state_key=problem.id,
     #                                     student=user,
     #                                     state=None,

lms/djangoapps/courseware/management/commands/check_course.py

@@ -84,6 +84,7 @@ class Command(BaseCommand):
         # TODO (cpennington): Get coursename in a legitimate way
         course_location = 'i4x://edx/6002xs12/course/6.002_Spring_2012'
         student_module_cache = StudentModuleCache.cache_for_descriptor_descendents(
+        course_id,
             sample_user, modulestore().get_item(course_location))
         course = get_module(sample_user, None, course_location, student_module_cache)
 

lms/djangoapps/courseware/migrations/0003_done_grade_cache.py

@@ -9,6 +9,8 @@ class Migration(SchemaMigration):
 
     def forwards(self, orm):
 
+        # NOTE (vshnayder): This constraint has the wrong field order, so it doesn't actually
+        # do anything in sqlite.  Migration 0004 actually removes this index for sqlite.
         # Removing unique constraint on 'StudentModule', fields ['module_id', 'module_type', 'student']
         db.delete_unique('courseware_studentmodule', ['module_id', 'module_type', 'student_id'])
 

lms/djangoapps/courseware/migrations/0004_add_field_studentmodule_course_id.py

+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'StudentModule.course_id'
+        db.add_column('courseware_studentmodule', 'course_id',
+                      self.gf('django.db.models.fields.CharField')(default="", max_length=255, db_index=True),
+                      keep_default=False)
+
+        # Removing unique constraint on 'StudentModule', fields ['module_id', 'student']
+        db.delete_unique('courseware_studentmodule', ['module_id', 'student_id'])
+
+        # NOTE: manually remove this constaint (from 0001)--0003 tries, but fails for sqlite.
+        # Removing unique constraint on 'StudentModule', fields ['module_id', 'module_type', 'student']
+        if db.backend_name == "sqlite3":
+            db.delete_unique('courseware_studentmodule', ['student_id', 'module_id', 'module_type'])
+
+        # Adding unique constraint on 'StudentModule', fields ['course_id', 'module_state_key', 'student']
+        db.create_unique('courseware_studentmodule', ['student_id', 'module_id', 'course_id'])
+
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'StudentModule', fields ['studnet_id', 'module_state_key', 'course_id']
+        db.delete_unique('courseware_studentmodule', ['student_id', 'module_id', 'course_id'])
+
+        # Deleting field 'StudentModule.course_id'
+        db.delete_column('courseware_studentmodule', 'course_id')
+
+        # Adding unique constraint on 'StudentModule', fields ['module_id', 'student']
+        db.create_unique('courseware_studentmodule', ['module_id', 'student_id'])
+
+        # Adding unique constraint on 'StudentModule', fields ['module_id', 'module_type', 'student']
+        db.create_unique('courseware_studentmodule', ['student_id', 'module_id', 'module_type'])
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'about': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'avatar_type': ('django.db.models.fields.CharField', [], {'default': "'n'", 'max_length': '1'}),
+            'bronze': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'consecutive_days_visit_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'country': ('django_countries.fields.CountryField', [], {'max_length': '2', 'blank': 'True'}),
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'date_of_birth': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'display_tag_filter_strategy': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'email_isvalid': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'email_key': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}),
+            'email_tag_filter_strategy': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'gold': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'gravatar': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'ignored_tags': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'interesting_tags': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'last_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'location': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'new_response_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'questions_per_page': ('django.db.models.fields.SmallIntegerField', [], {'default': '10'}),
+            'real_name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'reputation': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}),
+            'seen_response_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'show_country': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'silver': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'status': ('django.db.models.fields.CharField', [], {'default': "'w'", 'max_length': '2'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}),
+            'website': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'courseware.studentmodule': {
+            'Meta': {'unique_together': "(('course_id', 'student', 'module_state_key'),)", 'object_name': 'StudentModule'},
+            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'done': ('django.db.models.fields.CharField', [], {'default': "'na'", 'max_length': '8', 'db_index': 'True'}),
+            'grade': ('django.db.models.fields.FloatField', [], {'db_index': 'True', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'max_grade': ('django.db.models.fields.FloatField', [], {'null': 'True', 'blank': 'True'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'db_index': 'True', 'blank': 'True'}),
+            'module_state_key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_column': "'module_id'", 'db_index': 'True'}),
+            'module_type': ('django.db.models.fields.CharField', [], {'default': "'problem'", 'max_length': '32', 'db_index': 'True'}),
+            'state': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'student': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
+        }
+    }
+
+    complete_apps = ['courseware']

lms/djangoapps/courseware/models.py

@@ -22,6 +22,9 @@ from django.contrib.auth.models import User
 
 
 class StudentModule(models.Model):
+    """
+    Keeps student state for a particular module in a particular course.
+    """
     # For a homework problem, contains a JSON
     # object consisting of state
     MODULE_TYPES = (('problem', 'problem'),
@@ -37,9 +40,10 @@ class StudentModule(models.Model):
     # Filename for homeworks, etc.
     module_state_key = models.CharField(max_length=255, db_index=True, db_column='module_id')
     student = models.ForeignKey(User, db_index=True)
+    course_id = models.CharField(max_length=255, db_index=True)
 
     class Meta:
-        unique_together = (('student', 'module_state_key'),)
+        unique_together = (('student', 'module_state_key', 'course_id'),)
 
     ## Internal state of the object
     state = models.TextField(null=True, blank=True)
@@ -57,7 +61,8 @@ class StudentModule(models.Model):
     modified = models.DateTimeField(auto_now=True, db_index=True)
 
     def __unicode__(self):
-        return '/'.join([self.module_type, self.student.username, self.module_state_key, str(self.state)[:20]])
+        return '/'.join([self.course_id, self.module_type,
+                         self.student.username, self.module_state_key, str(self.state)[:20]])
 
 
 # TODO (cpennington): Remove these once the LMS switches to using XModuleDescriptors
@@ -67,20 +72,20 @@ class StudentModuleCache(object):
     """
     A cache of StudentModules for a specific student
     """
-    def __init__(self, user, descriptors, select_for_update=False):
+    def __init__(self, course_id, user, descriptors, select_for_update=False):
         '''
         Find any StudentModule objects that are needed by any descriptor
         in descriptors. Avoids making multiple queries to the database.
         Note: Only modules that have store_state = True or have shared
         state will have a StudentModule.
-        
+
         Arguments
         user: The user for which to fetch maching StudentModules
         descriptors: An array of XModuleDescriptors.
         select_for_update: Flag indicating whether the rows should be locked until end of transaction
         '''
         if user.is_authenticated():
-            module_ids = self._get_module_state_keys(descriptors) 
+            module_ids = self._get_module_state_keys(descriptors)
 
             # This works around a limitation in sqlite3 on the number of parameters
             # that can be put into a single query
@@ -89,78 +94,86 @@ class StudentModuleCache(object):
             for id_chunk in [module_ids[i:i + chunk_size] for i in xrange(0, len(module_ids), chunk_size)]:
                 if select_for_update:
                     self.cache.extend(StudentModule.objects.select_for_update().filter(
+                        course_id=course_id,
                         student=user,
                         module_state_key__in=id_chunk)
                     )
                 else:
                     self.cache.extend(StudentModule.objects.filter(
+                        course_id=course_id,
                         student=user,
                         module_state_key__in=id_chunk)
                     )
 
         else:
             self.cache = []
-    
-    
+
+
     @classmethod
-    def cache_for_descriptor_descendents(cls, user, descriptor, depth=None, descriptor_filter=lambda descriptor: True, select_for_update=False):
+    def cache_for_descriptor_descendents(cls, course_id, user, descriptor, depth=None,
+                                         descriptor_filter=lambda descriptor: True,
+                                         select_for_update=False):
         """
+        course_id: the course in the context of which we want StudentModules.
+        user: the django user for whom to load modules.
         descriptor: An XModuleDescriptor
         depth is the number of levels of descendent modules to load StudentModules for, in addition to
             the supplied descriptor. If depth is None, load all descendent StudentModules
-        descriptor_filter is a function that accepts a descriptor and return wether the StudentModule 
+        descriptor_filter is a function that accepts a descriptor and return wether the StudentModule
             should be cached
         select_for_update: Flag indicating whether the rows should be locked until end of transaction
         """
-        
+
         def get_child_descriptors(descriptor, depth, descriptor_filter):
             if descriptor_filter(descriptor):
                 descriptors = [descriptor]
             else:
                 descriptors = []
-            
+
             if depth is None or depth > 0:
                 new_depth = depth - 1 if depth is not None else depth
 
                 for child in descriptor.get_children():
                     descriptors.extend(get_child_descriptors(child, new_depth, descriptor_filter))
-                
+
             return descriptors
-        
-        
+
+
         descriptors = get_child_descriptors(descriptor, depth, descriptor_filter)
-        
-        return StudentModuleCache(user, descriptors, select_for_update)
-    
+
+        return StudentModuleCache(course_id, user, descriptors, select_for_update)
+
     def _get_module_state_keys(self, descriptors):
         '''
         Get a list of the state_keys needed for StudentModules
         required for this module descriptor
-        
-        descriptor_filter is a function that accepts a descriptor and return wether the StudentModule 
+
+        descriptor_filter is a function that accepts a descriptor and return wether the StudentModule
             should be cached
         '''
         keys = []
         for descriptor in descriptors:
             if descriptor.stores_state:
                 keys.append(descriptor.location.url())
-            
+
             shared_state_key = getattr(descriptor, 'shared_state_key', None)
             if shared_state_key is not None:
                 keys.append(shared_state_key)
 
         return keys
 
-    def lookup(self, module_type, module_state_key):
+    def lookup(self, course_id, module_type, module_state_key):
         '''
-        Look for a student module with the given type and id in the cache.
+        Look for a student module with the given course_id, type, and id in the cache.
 
         cache -- list of student modules
 
         returns first found object, or None
         '''
         for o in self.cache:
-            if o.module_type == module_type and o.module_state_key == module_state_key:
+            if (o.course_id == course_id and
+                o.module_type == module_type and
+                o.module_state_key == module_state_key):
                 return o
         return None
 

lms/djangoapps/courseware/views.py

@@ -3,6 +3,10 @@ import logging
 import urllib
 import itertools
 
+from functools import partial
+
+from functools import partial
+
 from django.conf import settings
 from django.core.context_processors import csrf
 from django.core.urlresolvers import reverse
@@ -21,6 +25,11 @@ from courseware.courses import (get_course_with_access, get_courses_by_universit
 from models import StudentModuleCache
 from module_render import toc_for_course, get_module, get_section
 from student.models import UserProfile
+
+from multicourse import multicourse_settings
+
+from django_comment_client.utils import get_discussion_title
+
 from student.models import UserTestGroup, CourseEnrollment
 from util.cache import cache, cache_if_anonymous
 from xmodule.course_module import CourseDescriptor
@@ -29,6 +38,11 @@ from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, NoPathToItem
 from xmodule.modulestore.search import path_to_location
 
+import comment_client
+
+
+
+
 log = logging.getLogger("mitx.courseware")
 
 template_imports = {'urllib': urllib}
@@ -135,8 +149,7 @@ def index(request, course_id, chapter=None, section=None,
             section_descriptor = get_section(course, chapter, section)
             if section_descriptor is not None:
                 student_module_cache = StudentModuleCache.cache_for_descriptor_descendents(
-                                                          request.user,
-                                                          section_descriptor)
+                    course_id, request.user, section_descriptor)
                 module = get_module(request.user, request,
                                     section_descriptor.location,
                                     student_module_cache, course_id)
@@ -219,6 +232,19 @@ def course_info(request, course_id):
     return render_to_response('courseware/info.html', {'course': course,
                                             'staff_access': staff_access,})
 
+# TODO arjun: remove when custom tabs in place, see courseware/syllabus.py
+@ensure_csrf_cookie
+def syllabus(request, course_id):
+    """
+    Display the course's syllabus.html, or 404 if there is no such course.
+
+    Assumes the course_id is in a valid format.
+    """
+    course = get_course_with_access(request.user, course_id, 'load')
+    staff_access = has_access(request.user, course, 'staff')
+
+    return render_to_response('courseware/syllabus.html', {'course': course,
+                                            'staff_access': staff_access,})
 
 def registered_for_course(course, user):
     '''Return CourseEnrollment if user is registered for course, else False'''
@@ -256,6 +282,26 @@ def university_profile(request, org_id):
 
     return render_to_response(template_file, context)
 
+def render_notifications(request, course, notifications):
+    context = {
+        'notifications': notifications,
+        'get_discussion_title': partial(get_discussion_title, request=request, course=course),
+        'course': course,
+    }
+    return render_to_string('notifications.html', context)
+
+@login_required
+def news(request, course_id):
+    course = get_course_with_access(request.user, course_id, 'load')
+
+    notifications = comment_client.get_notifications(request.user.id)
+
+    context = {
+        'course': course,
+        'content': render_notifications(request, course, notifications),
+    }
+
+    return render_to_response('news.html', context)
 
 @login_required
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
@@ -276,7 +322,8 @@ def progress(request, course_id, student_id=None):
             raise Http404
         student = User.objects.get(id=int(student_id))
 
-    student_module_cache = StudentModuleCache.cache_for_descriptor_descendents(request.user, course)
+    student_module_cache = StudentModuleCache.cache_for_descriptor_descendents(
+        course_id, request.user, course)
     course_module = get_module(request.user, request, course.location,
                                student_module_cache, course_id)
 
@@ -346,4 +393,3 @@ def instructor_dashboard(request, course_id):
     context = {'course': course,
                'staff_access': True,}
     return render_to_response('courseware/instructor_dashboard.html', context)
-

lms/djangoapps/django_comment_client/__init__.py

+# call some function from permissions so that the post_save hook is imported
+from permissions import assign_default_role

lms/djangoapps/django_comment_client/base/urls.py

+from django.conf.urls.defaults import url, patterns
+import django_comment_client.base.views
+
+urlpatterns = patterns('django_comment_client.base.views',
+
+    url(r'upload$', 'upload', name='upload'),
+    url(r'users/(?P<user_id>\w+)/update_moderator_status$', 'update_moderator_status', name='update_moderator_status'),
+    url(r'threads/tags/autocomplete$', 'tags_autocomplete', name='tags_autocomplete'),
+    url(r'threads/(?P<thread_id>[\w\-]+)/update$', 'update_thread', name='update_thread'),
+    url(r'threads/(?P<thread_id>[\w\-]+)/reply$', 'create_comment', name='create_comment'),
+    url(r'threads/(?P<thread_id>[\w\-]+)/delete', 'delete_thread', name='delete_thread'),
+    url(r'threads/(?P<thread_id>[\w\-]+)/upvote$', 'vote_for_thread', {'value': 'up'}, name='upvote_thread'),
+    url(r'threads/(?P<thread_id>[\w\-]+)/downvote$', 'vote_for_thread', {'value': 'down'}, name='downvote_thread'),
+    url(r'threads/(?P<thread_id>[\w\-]+)/unvote$', 'undo_vote_for_thread', name='undo_vote_for_thread'),
+    url(r'threads/(?P<thread_id>[\w\-]+)/follow$', 'follow_thread', name='follow_thread'),
+    url(r'threads/(?P<thread_id>[\w\-]+)/unfollow$', 'unfollow_thread', name='unfollow_thread'),
+    url(r'threads/(?P<thread_id>[\w\-]+)/close$', 'openclose_thread', name='openclose_thread'),
+
+    url(r'comments/(?P<comment_id>[\w\-]+)/update$', 'update_comment', name='update_comment'),
+    url(r'comments/(?P<comment_id>[\w\-]+)/endorse$', 'endorse_comment', name='endorse_comment'),
+    url(r'comments/(?P<comment_id>[\w\-]+)/reply$', 'create_sub_comment', name='create_sub_comment'),
+    url(r'comments/(?P<comment_id>[\w\-]+)/delete$', 'delete_comment', name='delete_comment'),
+    url(r'comments/(?P<comment_id>[\w\-]+)/upvote$', 'vote_for_comment', {'value': 'up'}, name='upvote_comment'),
+    url(r'comments/(?P<comment_id>[\w\-]+)/downvote$', 'vote_for_comment', {'value': 'down'}, name='downvote_comment'),
+    url(r'comments/(?P<comment_id>[\w\-]+)/unvote$', 'undo_vote_for_comment', name='undo_vote_for_comment'),
+    
+    url(r'(?P<commentable_id>[\w\-]+)/threads/create$', 'create_thread', name='create_thread'),
+    # TODO should we search within the board?
+    url(r'(?P<commentable_id>[\w\-]+)/threads/search_similar$', 'search_similar_threads', name='search_similar_threads'),
+    url(r'(?P<commentable_id>[\w\-]+)/follow$', 'follow_commentable', name='follow_commentable'),
+    url(r'(?P<commentable_id>[\w\-]+)/unfollow$', 'unfollow_commentable', name='unfollow_commentable'),
+)

lms/djangoapps/django_comment_client/forum/urls.py

+from django.conf.urls.defaults import url, patterns
+import django_comment_client.forum.views
+
+urlpatterns = patterns('django_comment_client.forum.views',
+    url(r'users/(?P<user_id>\w+)$', 'user_profile', name='user_profile'),
+    url(r'(?P<discussion_id>\w+)/threads/(?P<thread_id>\w+)$', 'single_thread', name='single_thread'),
+    url(r'(?P<discussion_id>\w+)/inline$', 'inline_discussion', name='inline_discussion'),
+    url(r'', 'forum_form_discussion', name='forum_form_discussion'),
+)

lms/djangoapps/django_comment_client/forum/views.py

+from django.contrib.auth.decorators import login_required
+from django.views.decorators.http import require_POST
+from django.http import HttpResponse
+from django.utils import simplejson
+from django.core.context_processors import csrf
+from django.core.urlresolvers import reverse
+from django.contrib.auth.models import User
+
+from mitxmako.shortcuts import render_to_response, render_to_string
+from courseware.courses import get_course_with_access
+from courseware.access import has_access
+
+from urllib import urlencode
+from django_comment_client.permissions import check_permissions_by_view
+from django_comment_client.utils import merge_dict, extract, strip_none
+
+import json
+import dateutil
+import django_comment_client.utils as utils
+import comment_client as cc
+
+
+THREADS_PER_PAGE = 5
+PAGES_NEARBY_DELTA = 2
+
+
+def _general_discussion_id(course_id):
+    return course_id.replace('/', '_').replace('.', '_')
+
+def _should_perform_search(request):
+    return bool(request.GET.get('text', False) or \
+            request.GET.get('tags', False))
+        
+
+def render_accordion(request, course, discussion_id):
+
+    discussion_info = utils.get_categorized_discussion_info(request, course)
+
+    context = {
+        'course': course,
+        'discussion_info': discussion_info,
+        'active': discussion_id,
+        'csrf': csrf(request)['csrf_token'],
+    }
+
+    return render_to_string('discussion/_accordion.html', context)
+
+def render_discussion(request, course_id, threads, *args, **kwargs):
+
+    discussion_id = kwargs.get('discussion_id')
+    user_id = kwargs.get('user_id')
+    discussion_type = kwargs.get('discussion_type', 'inline')
+    query_params = kwargs.get('query_params', {})
+
+    template = {
+        'inline': 'discussion/_inline.html',
+        'forum': 'discussion/_forum.html',
+        'user': 'discussion/_user_active_threads.html',
+    }[discussion_type]
+
+    base_url = {
+        'inline': (lambda: reverse('django_comment_client.forum.views.inline_discussion', args=[course_id, discussion_id])), 
+        'forum': (lambda: reverse('django_comment_client.forum.views.forum_form_discussion', args=[course_id])),
+        'user': (lambda: reverse('django_comment_client.forum.views.user_profile', args=[course_id, user_id])),
+    }[discussion_type]()
+
+    annotated_content_infos = map(lambda x: utils.get_annotated_content_infos(course_id, x, request.user), threads)
+    annotated_content_info = reduce(merge_dict, annotated_content_infos, {})
+
+    context = {
+        'threads': threads,
+        'discussion_id': discussion_id,
+        'user_id': user_id,
+        'user_info': json.dumps(cc.User.from_django_user(request.user).to_dict()),
+        'course_id': course_id,
+        'request': request,
+        'performed_search': _should_perform_search(request),
+        'pages_nearby_delta': PAGES_NEARBY_DELTA,
+        'discussion_type': discussion_type,
+        'base_url': base_url,
+        'query_params': strip_none(extract(query_params, ['page', 'sort_key', 'sort_order', 'tags', 'text'])),
+        'annotated_content_info': json.dumps(annotated_content_info),
+    }
+    context = dict(context.items() + query_params.items())
+    return render_to_string(template, context)
+
+def render_inline_discussion(*args, **kwargs):
+    return render_discussion(discussion_type='inline', *args, **kwargs)
+
+def render_forum_discussion(*args, **kwargs):
+    return render_discussion(discussion_type='forum', *args, **kwargs)
+
+def render_user_discussion(*args, **kwargs):
+    return render_discussion(discussion_type='user', *args, **kwargs)
+
+def get_threads(request, course_id, discussion_id=None):
+
+    default_query_params = {
+        'page': 1,
+        'per_page': THREADS_PER_PAGE,
+        'sort_key': 'activity',
+        'sort_order': 'desc',
+        'text': '',
+        'tags': '',
+        'commentable_id': discussion_id,
+        'course_id': course_id,
+    }
+
+    query_params = merge_dict(default_query_params,
+                              strip_none(extract(request.GET, ['page', 'sort_key', 'sort_order', 'text', 'tags'])))
+
+    threads, page, num_pages = cc.Thread.search(query_params)
+
+    query_params['page'] = page
+    query_params['num_pages'] = num_pages
+
+    return threads, query_params
+
+# discussion per page is fixed for now
+def inline_discussion(request, course_id, discussion_id):
+    threads, query_params = get_threads(request, course_id, discussion_id)
+    html = render_inline_discussion(request, course_id, threads, discussion_id=discussion_id,  \
+                                                                 query_params=query_params)
+    return utils.HtmlResponse(html)
+
+def render_search_bar(request, course_id, discussion_id=None, text=''):
+    if not discussion_id:
+        return ''
+    context = {
+        'discussion_id': discussion_id,
+        'text': text,
+        'course_id': course_id,
+    }
+    return render_to_string('discussion/_search_bar.html', context)
+
+def forum_form_discussion(request, course_id):
+    course = get_course_with_access(request.user, course_id, 'load')
+    threads, query_params = get_threads(request, course_id)
+    content = render_forum_discussion(request, course_id, threads, discussion_id=_general_discussion_id(course_id), query_params=query_params)
+
+    recent_active_threads = cc.search_recent_active_threads(
+        course_id,
+        recursive=False,
+        query_params={'follower_id': request.user.id},
+    )
+
+    trending_tags = cc.search_trending_tags(
+        course_id,
+    )
+
+    if request.is_ajax():
+        return utils.HtmlResponse(content)
+    else:
+        context = {
+            'csrf': csrf(request)['csrf_token'],
+            'course': course,
+            'content': content,
+            'recent_active_threads': recent_active_threads,
+            'trending_tags': trending_tags,
+            'staff_access' : has_access(request.user, course, 'staff'),
+        }
+        # print "start rendering.."
+        return render_to_response('discussion/index.html', context)
+
+def render_single_thread(request, discussion_id, course_id, thread_id):
+    
+    thread = cc.Thread.find(thread_id).retrieve(recursive=True)
+
+    annotated_content_info = utils.get_annotated_content_infos(course_id, thread=thread.to_dict(), user=request.user)
+
+    context = {
+        'discussion_id': discussion_id,
+        'thread': thread,
+        'user_info': json.dumps(cc.User.from_django_user(request.user).to_dict()),
+        'annotated_content_info': json.dumps(annotated_content_info),
+        'course_id': course_id,
+        'request': request,
+    }
+    return render_to_string('discussion/_single_thread.html', context)
+
+def single_thread(request, course_id, discussion_id, thread_id):
+
+    if request.is_ajax():
+        
+        thread = cc.Thread.find(thread_id).retrieve(recursive=True)
+        annotated_content_info = utils.get_annotated_content_infos(course_id, thread, request.user)
+        context = {'thread': thread.to_dict(), 'course_id': course_id}
+        html = render_to_string('discussion/_ajax_single_thread.html', context)
+
+        return utils.JsonResponse({
+            'html': html,
+            'annotated_content_info': annotated_content_info,
+        })
+
+    else:
+        course = get_course_with_access(request.user, course_id, 'load')
+
+        context = {
+            'discussion_id': discussion_id,
+            'csrf': csrf(request)['csrf_token'],
+            'init': '',
+            'content': render_single_thread(request, discussion_id, course_id, thread_id),
+            'accordion': render_accordion(request, course, discussion_id),
+            'course': course,
+            'course_id': course.id,
+        }
+
+        return render_to_response('discussion/index.html', context)
+
+def user_profile(request, course_id, user_id):
+
+    course = get_course_with_access(request.user, course_id, 'load')
+    profiled_user = cc.User(id=user_id, course_id=course_id)
+
+    query_params = {
+        'page': request.GET.get('page', 1),
+        'per_page': THREADS_PER_PAGE, # more than threads_per_page to show more activities
+    }
+
+    threads, page, num_pages = profiled_user.active_threads(query_params)
+
+    query_params['page'] = page
+    query_params['num_pages'] = num_pages
+
+    content = render_user_discussion(request, course_id, threads, user_id=user_id, query_params=query_params)
+
+    if request.is_ajax():
+        return utils.HtmlResponse(content)
+    else:
+        context = {
+            'course': course, 
+            'user': request.user,
+            'django_user': User.objects.get(id=user_id),
+            'profiled_user': profiled_user.to_dict(),
+            'content': content,
+        }
+
+        return render_to_response('discussion/user_profile.html', context)

lms/djangoapps/django_comment_client/helpers.py

+from django.core.urlresolvers import reverse
+from mitxmako.shortcuts import render_to_string
+from utils import *
+from mustache_helpers import mustache_helpers
+from functools import partial
+
+import pystache_custom as pystache
+import urllib
+
+def pluralize(singular_term, count):
+    if int(count) >= 2:
+        return singular_term + 's'
+    return singular_term
+
+def show_if(text, condition):
+    if condition:
+        return text
+    else:
+        return ''
+
+def render_content(content, additional_context={}):
+    content_info = {
+        'displayed_title': content.get('highlighted_title') or content.get('title', ''),
+        'displayed_body': content.get('highlighted_body') or content.get('body', ''),
+        'raw_tags': ','.join(content.get('tags', [])),
+    }
+    context = {
+        'content': merge_dict(content, content_info),
+        content['type']: True,
+    }
+    context = merge_dict(context, additional_context)
+    partial_mustache_helpers = {k: partial(v, content) for k, v in mustache_helpers.items()}
+    context = merge_dict(context, partial_mustache_helpers)
+    return render_mustache('discussion/_content.mustache', context)

lms/djangoapps/django_comment_client/management/commands/assign_role.py

+from django.core.management.base import BaseCommand, CommandError
+from django_comment_client.models import Permission, Role
+from django.contrib.auth.models import User
+
+
+class Command(BaseCommand):
+    args = 'user role course_id'
+    help = 'Assign a role to a user'
+
+    def handle(self, *args, **options):
+        role = Role.objects.get(name=args[1], course_id=args[2])
+
+        if '@' in args[0]:
+            user = User.objects.get(email=args[0])
+        else:
+            user = User.objects.get(username=args[0])
+
+        user.roles.add(role)
\ No newline at end of file

lms/djangoapps/django_comment_client/management/commands/create_roles_for_existing.py

+"""
+This must be run only after seed_permissions_roles.py!
+
+Creates default roles for all users currently in the database. Just runs through
+Enrollments.
+"""
+from django.core.management.base import BaseCommand, CommandError
+
+from student.models import CourseEnrollment
+from django_comment_client.permissions import assign_default_role
+
+
+class Command(BaseCommand):
+    args = 'course_id'
+    help = 'Seed default permisssions and roles'
+
+    def handle(self, *args, **options):
+        if len(args) != 0:
+            raise CommandError("This Command takes no arguments")
+
+        print "Updated roles for ",
+        for i, enrollment in enumerate(CourseEnrollment.objects.all(), start=1):
+            assign_default_role(None, enrollment)
+            if i % 1000 == 0:
+                print "{0}...".format(i),
+        print
\ No newline at end of file

lms/djangoapps/django_comment_client/management/commands/seed_permissions_roles.py

+from django.core.management.base import BaseCommand, CommandError
+from django_comment_client.models import Permission, Role
+
+
+class Command(BaseCommand):
+    args = 'course_id'
+    help = 'Seed default permisssions and roles'
+
+    def handle(self, *args, **options):
+        if len(args) != 1:
+            raise CommandError("The number of arguments does not match. ")
+        course_id = args[0]
+        administrator_role = Role.objects.get_or_create(name="Administrator", course_id=course_id)[0]
+        moderator_role = Role.objects.get_or_create(name="Moderator", course_id=course_id)[0]
+        student_role = Role.objects.get_or_create(name="Student", course_id=course_id)[0]
+
+        for per in ["vote", "update_thread", "follow_thread", "unfollow_thread",
+                       "update_comment", "create_sub_comment", "unvote" , "create_thread",
+                       "follow_commentable", "unfollow_commentable", "create_comment", ]:
+            student_role.add_permission(per)
+
+        for per in ["edit_content", "delete_thread", "openclose_thread",
+                        "endorse_comment", "delete_comment"]:
+            moderator_role.add_permission(per)
+
+        for per in ["manage_moderator"]:
+            administrator_role.add_permission(per)
+
+        moderator_role.inherit_permissions(student_role)
+
+        administrator_role.inherit_permissions(moderator_role)

lms/djangoapps/django_comment_client/management/commands/show_permissions.py

+from django.core.management.base import BaseCommand, CommandError
+from django_comment_client.models import Permission, Role
+from django.contrib.auth.models import User
+
+
+class Command(BaseCommand):
+    args = 'user'
+    help = "Show a user's roles and permissions"
+
+    def handle(self, *args, **options):
+        print args
+        if len(args) != 1:
+            raise CommandError("The number of arguments does not match. ")
+        try:
+            if '@' in args[0]:
+                user = User.objects.get(email=args[0])
+            else:
+                user = User.objects.get(username=args[0])
+        except User.DoesNotExist:
+            print "User %s does not exist. " % args[0]
+            print "Available users: "
+            print User.objects.all()
+            return
+
+        roles = user.roles.all()
+        print "%s has %d roles:" % (user, len(roles))
+        for role in roles:
+            print "\t%s" % role
+
+        for role in roles:
+            print "%s has permissions: " % role
+            print role.permissions.all()

lms/djangoapps/django_comment_client/middleware.py

+from comment_client import CommentClientError
+from django_comment_client.utils import JsonError
+import json
+
+class AjaxExceptionMiddleware(object): 
+    def process_exception(self, request, exception):
+        if isinstance(exception, CommentClientError) and request.is_ajax():
+            return JsonError(json.loads(exception.message))
+        return None

lms/djangoapps/django_comment_client/migrations/0001_initial.py

+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'Role'
+        db.create_table('django_comment_client_role', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('name', self.gf('django.db.models.fields.CharField')(max_length=30)),
+            ('course_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=255, blank=True)),
+        ))
+        db.send_create_signal('django_comment_client', ['Role'])
+
+        # Adding M2M table for field users on 'Role'
+        db.create_table('django_comment_client_role_users', (
+            ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
+            ('role', models.ForeignKey(orm['django_comment_client.role'], null=False)),
+            ('user', models.ForeignKey(orm['auth.user'], null=False))
+        ))
+        db.create_unique('django_comment_client_role_users', ['role_id', 'user_id'])
+
+        # Adding model 'Permission'
+        db.create_table('django_comment_client_permission', (
+            ('name', self.gf('django.db.models.fields.CharField')(max_length=30, primary_key=True)),
+        ))
+        db.send_create_signal('django_comment_client', ['Permission'])
+
+        # Adding M2M table for field roles on 'Permission'
+        db.create_table('django_comment_client_permission_roles', (
+            ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
+            ('permission', models.ForeignKey(orm['django_comment_client.permission'], null=False)),
+            ('role', models.ForeignKey(orm['django_comment_client.role'], null=False))
+        ))
+        db.create_unique('django_comment_client_permission_roles', ['permission_id', 'role_id'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'Role'
+        db.delete_table('django_comment_client_role')
+
+        # Removing M2M table for field users on 'Role'
+        db.delete_table('django_comment_client_role_users')
+
+        # Deleting model 'Permission'
+        db.delete_table('django_comment_client_permission')
+
+        # Removing M2M table for field roles on 'Permission'
+        db.delete_table('django_comment_client_permission_roles')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'about': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'avatar_type': ('django.db.models.fields.CharField', [], {'default': "'n'", 'max_length': '1'}),
+            'bronze': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'consecutive_days_visit_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'country': ('django_countries.fields.CountryField', [], {'max_length': '2', 'blank': 'True'}),
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'date_of_birth': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'display_tag_filter_strategy': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'email_isvalid': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'email_key': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}),
+            'email_tag_filter_strategy': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'gold': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'gravatar': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'ignored_tags': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'interesting_tags': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'last_seen': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'location': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'new_response_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'questions_per_page': ('django.db.models.fields.SmallIntegerField', [], {'default': '10'}),
+            'real_name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'reputation': ('django.db.models.fields.PositiveIntegerField', [], {'default': '1'}),
+            'seen_response_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'show_country': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'silver': ('django.db.models.fields.SmallIntegerField', [], {'default': '0'}),
+            'status': ('django.db.models.fields.CharField', [], {'default': "'w'", 'max_length': '2'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}),
+            'website': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'django_comment_client.permission': {
+            'Meta': {'object_name': 'Permission'},
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'primary_key': 'True'}),
+            'roles': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'permissions'", 'symmetrical': 'False', 'to': "orm['django_comment_client.Role']"})
+        },
+        'django_comment_client.role': {
+            'Meta': {'object_name': 'Role'},
+            'course_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '30'}),
+            'users': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'roles'", 'symmetrical': 'False', 'to': "orm['auth.User']"})
+        }
+    }
+
+    complete_apps = ['django_comment_client']
\ No newline at end of file

lms/djangoapps/django_comment_client/models.py

+from django.db import models
+from django.contrib.auth.models import User
+import logging
+
+
+class Role(models.Model):
+    name = models.CharField(max_length=30, null=False, blank=False)
+    users = models.ManyToManyField(User, related_name="roles")
+    course_id = models.CharField(max_length=255, blank=True, db_index=True)
+
+    def __unicode__(self):
+        return self.name + " for " + (self.course_id if self.course_id else "all courses")
+
+    def inherit_permissions(self, role): # TODO the name of this method is a little bit confusing,
+                                         # since it's one-off and doesn't handle inheritance later
+        if role.course_id and role.course_id != self.course_id:
+            logging.warning("%s cannot inheret permissions from %s due to course_id inconsistency" % 
+                            (self, role))
+        for per in role.permissions.all():
+            self.add_permission(per)
+
+    def add_permission(self, permission):
+        self.permissions.add(Permission.objects.get_or_create(name=permission)[0])
+
+    def has_permission(self, permission):
+        return self.permissions.filter(name=permission).exists()
+
+
+class Permission(models.Model):
+    name = models.CharField(max_length=30, null=False, blank=False, primary_key=True)
+    roles = models.ManyToManyField(Role, related_name="permissions")
+
+    def __unicode__(self):
+        return self.name

lms/djangoapps/django_comment_client/mustache_helpers.py

+import django.core.urlresolvers as urlresolvers
+import urllib
+import sys
+import inspect
+
+def pluralize(content, text):
+    num, word = text.split(' ')
+    if int(num or '0') >= 2:
+        return num + ' ' + word + 's'
+    else:
+        return num + ' ' + word
+
+def url_for_user(content, user_id):
+    return urlresolvers.reverse('django_comment_client.forum.views.user_profile', args=[content['course_id'], user_id])
+
+def url_for_tags(content, tags): # assume that attribute 'tags' is in the format u'a, b, c'
+    return urlresolvers.reverse('django_comment_client.forum.views.forum_form_discussion', args=[content['course_id']]) + '?' + urllib.urlencode({'tags': tags})
+
+def close_thread_text(content):
+    if content.get('closed'):
+        return 'Re-open thread'
+    else:
+        return 'Close thread'
+
+current_module = sys.modules[__name__]
+all_functions = inspect.getmembers(current_module, inspect.isfunction)
+
+mustache_helpers = {k: v for k, v in all_functions if not k.startswith('_')}

lms/djangoapps/django_comment_client/permissions.py

+from .models import Role, Permission
+from django.contrib.auth.models import User
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+from student.models import CourseEnrollment
+
+import logging
+from util.cache import cache
+
+
+@receiver(post_save, sender=CourseEnrollment)
+def assign_default_role(sender, instance, **kwargs):
+    if instance.user.is_staff:
+        role = Role.objects.get_or_create(course_id=instance.course_id, name="Moderator")[0]
+    else:
+        role = Role.objects.get_or_create(course_id=instance.course_id, name="Student")[0]
+
+    logging.info("assign_default_role: adding %s as %s" % (instance.user, role))
+    instance.user.roles.add(role)
+
+def cached_has_permission(user, permission, course_id=None):
+    """
+    Call has_permission if it's not cached. A change in a user's role or
+    a role's permissions will only become effective after CACHE_LIFESPAN seconds. 
+    """
+    CACHE_LIFESPAN = 60
+    key = "permission_%d_%s_%s" % (user.id, str(course_id), permission)
+    val = cache.get(key, None)
+    if val not in [True, False]:
+        val = has_permission(user, permission, course_id=course_id)
+        cache.set(key, val, CACHE_LIFESPAN)
+    return val
+
+def has_permission(user, permission, course_id=None):
+    for role in user.roles.filter(course_id=course_id):
+        if role.has_permission(permission):
+            return True
+    return False
+
+
+CONDITIONS = ['is_open', 'is_author']
+def check_condition(user, condition, course_id, data):
+    def check_open(user, condition, course_id, data):
+        try:
+            return data and not data['content']['closed']
+        except KeyError:
+            return False
+
+    def check_author(user, condition, course_id, data):
+        try:
+            return data and data['content']['user_id'] == str(user.id)
+        except KeyError:
+            return False
+
+    handlers = {
+        'is_open'      : check_open,
+        'is_author'    : check_author,
+    }
+
+    return handlers[condition](user, condition, course_id, data)
+
+
+def check_conditions_permissions(user, permissions, course_id, **kwargs):
+    """
+    Accepts a list of permissions and proceed if any of the permission is valid.
+    Note that ["can_view", "can_edit"] will proceed if the user has either
+    "can_view" or "can_edit" permission. To use AND operator in between, wrap them in 
+    a list. 
+    """
+
+    def test(user, per, operator="or"):
+        if isinstance(per, basestring):
+            if per in CONDITIONS:
+                return check_condition(user, per, course_id, kwargs)
+            return cached_has_permission(user, per, course_id=course_id)
+        elif isinstance(per, list) and operator in ["and", "or"]:
+            results = [test(user, x, operator="and") for x in per]
+            if operator == "or":
+                return True in results
+            elif operator == "and":
+                return not False in results
+
+    return test(user, permissions, operator="or")
+
+
+VIEW_PERMISSIONS = {
+    'update_thread'     :       ['edit_content', ['update_thread', 'is_open', 'is_author']],
+    'create_comment'    :       [["create_comment", "is_open"]],
+    'delete_thread'     :       ['delete_thread'],
+    'update_comment'    :       ['edit_content', ['update_comment', 'is_open', 'is_author']],
+    'endorse_comment'   :       ['endorse_comment'],
+    'openclose_thread'  :       ['openclose_thread'],
+    'create_sub_comment':       [['create_sub_comment', 'is_open']],
+    'delete_comment'    :       ['delete_comment'],
+    'vote_for_comment'  :       [['vote', 'is_open']],
+    'undo_vote_for_comment':    [['unvote', 'is_open']],
+    'vote_for_thread'   :       [['vote', 'is_open']],
+    'undo_vote_for_thread':     [['unvote', 'is_open']],
+    'follow_thread'     :       ['follow_thread'],
+    'follow_commentable':       ['follow_commentable'], 
+    'follow_user'       :       ['follow_user'],
+    'unfollow_thread'   :       ['unfollow_thread'],
+    'unfollow_commentable':     ['unfollow_commentable'],
+    'unfollow_user'     :       ['unfollow_user'],
+    'create_thread'     :       ['create_thread'],
+    'update_moderator_status' : ['manage_moderator'],
+}
+
+
+def check_permissions_by_view(user, course_id, content, name):
+    try:
+        p = VIEW_PERMISSIONS[name]
+    except KeyError:
+        logging.warning("Permission for view named %s does not exist in permissions.py" % name)
+    return check_conditions_permissions(user, p, course_id, content=content)

lms/djangoapps/django_comment_client/tests.py

+from django.contrib.auth.models import User
+from django.test import TestCase
+from student.models import CourseEnrollment, \
+                           replicate_enrollment_save, \
+                           replicate_enrollment_delete, \
+                           update_user_information, \
+                           replicate_user_save
+                           
+from django.db.models.signals import m2m_changed, pre_delete, pre_save, post_delete, post_save
+from django.dispatch.dispatcher import _make_id
+import string
+import random
+from .permissions import has_permission
+from .models import Role, Permission
+
+class PermissionsTestCase(TestCase):
+    def random_str(self, length=15, chars=string.ascii_uppercase + string.digits):
+        return ''.join(random.choice(chars) for x in range(length))
+
+    def setUp(self):
+        self.course_id = "MITx/6.002x/2012_Fall"
+
+        self.moderator_role = Role.objects.get_or_create(name="Moderator", course_id=self.course_id)[0]
+        self.student_role = Role.objects.get_or_create(name="Student", course_id=self.course_id)[0]
+
+        self.student = User.objects.create(username=self.random_str(),
+                            password="123456", email="john@yahoo.com")
+        self.moderator = User.objects.create(username=self.random_str(),
+                            password="123456", email="staff@edx.org")
+        self.moderator.is_staff = True
+        self.moderator.save()
+        self.student_enrollment = CourseEnrollment.objects.create(user=self.student, course_id=self.course_id)
+        self.moderator_enrollment = CourseEnrollment.objects.create(user=self.moderator, course_id=self.course_id)
+
+    def tearDown(self):
+        self.student_enrollment.delete()
+        self.moderator_enrollment.delete()
+
+# Do we need to have this? We shouldn't be deleting students, ever
+#        self.student.delete()
+#        self.moderator.delete()
+
+    def testDefaultRoles(self):
+        self.assertTrue(self.student_role in self.student.roles.all())
+        self.assertTrue(self.moderator_role in self.moderator.roles.all())
+
+    def testPermission(self):
+        name = self.random_str()
+        self.moderator_role.add_permission(name)
+        self.assertTrue(has_permission(self.moderator, name, self.course_id))
+
+        self.student_role.add_permission(name)
+        self.assertTrue(has_permission(self.student, name, self.course_id))

lms/djangoapps/django_comment_client/urls.py

+from django.conf.urls.defaults import url, patterns, include
+
+urlpatterns = patterns('',
+    url(r'forum/', include('django_comment_client.forum.urls')),
+    url(r'', include('django_comment_client.base.urls')),
+)

lms/djangoapps/django_comment_client/utils.py

+from importlib import import_module
+from courseware.models import StudentModuleCache
+from courseware.module_render import get_module
+from xmodule.modulestore import Location
+from xmodule.modulestore.django import modulestore
+from django.http import HttpResponse
+from django.utils import simplejson
+from django.db import connection
+from django.conf import settings
+from django_comment_client.permissions import check_permissions_by_view
+from mitxmako import middleware
+
+import logging
+import operator
+import itertools
+import pystache_custom as pystache
+
+
+_FULLMODULES = None
+_DISCUSSIONINFO = None
+
+def extract(dic, keys):
+    return {k: dic.get(k) for k in keys}
+
+def strip_none(dic):
+    return dict([(k, v) for k, v in dic.iteritems() if v is not None])
+
+def strip_blank(dic):
+    def _is_blank(v):
+        return isinstance(v, str) and len(v.strip()) == 0
+    return dict([(k, v) for k, v in dic.iteritems() if not _is_blank(v)])
+
+def merge_dict(dic1, dic2):
+    return dict(dic1.items() + dic2.items())
+
+def get_full_modules():
+    global _FULLMODULES
+    if not _FULLMODULES:
+        class_path = settings.MODULESTORE['default']['ENGINE']
+        module_path, _, class_name = class_path.rpartition('.')
+        class_ = getattr(import_module(module_path), class_name)
+        modulestore = class_(**dict(settings.MODULESTORE['default']['OPTIONS'].items() + [('eager', True)]))
+        _FULLMODULES = modulestore.modules
+    return _FULLMODULES
+
+def get_categorized_discussion_info(request, course):
+    """
+        return a dict of the form {category: modules}
+    """
+    global _DISCUSSIONINFO
+    if not _DISCUSSIONINFO:
+        initialize_discussion_info(request, course)
+    return _DISCUSSIONINFO['categorized']
+
+def get_discussion_title(request, course, discussion_id):
+    global _DISCUSSIONINFO
+    if not _DISCUSSIONINFO:
+        initialize_discussion_info(request, course)
+    title = _DISCUSSIONINFO['by_id'].get(discussion_id, {}).get('title', '(no title)')
+    return title
+
+def initialize_discussion_info(request, course):
+
+    global _DISCUSSIONINFO
+    if _DISCUSSIONINFO:
+        return
+
+    course_id = course.id
+    _, course_name, _ = course_id.split('/')
+    user = request.user
+    url_course_id = course_id.replace('/', '_').replace('.', '_')
+
+    _is_course_discussion = lambda x: x[0].dict()['category'] == 'discussion' \
+                         and x[0].dict()['course'] == course_name
+    
+    _get_module_descriptor = operator.itemgetter(1)
+
+    def _get_module(module_descriptor):
+        print module_descriptor
+        module = get_module(user, request, module_descriptor.location, student_module_cache)
+        return module
+
+    def _extract_info(module):
+        return {
+            'title': module.title,
+            'discussion_id': module.discussion_id,
+            'category': module.discussion_category,
+        }
+
+    def _pack_with_id(info):
+        return (info['discussion_id'], info)
+
+    discussion_module_descriptors = map(_get_module_descriptor,
+                                        filter(_is_course_discussion,
+                                               get_full_modules().items()))
+
+    student_module_cache = StudentModuleCache.cache_for_descriptor_descendents(user, course)
+
+    discussion_info = map(_extract_info, map(_get_module, discussion_module_descriptors))
+
+    _DISCUSSIONINFO = {}
+
+    _DISCUSSIONINFO['by_id'] = dict(map(_pack_with_id, discussion_info))
+
+    _DISCUSSIONINFO['categorized'] = dict((category, list(l)) \
+                for category, l in itertools.groupby(discussion_info, operator.itemgetter('category')))
+
+    _DISCUSSIONINFO['categorized']['General'] = [{
+        'title': 'General discussion',
+        'discussion_id': url_course_id,
+        'category': 'General',
+    }]
+
+class JsonResponse(HttpResponse):
+    def __init__(self, data=None):
+        content = simplejson.dumps(data)
+        super(JsonResponse, self).__init__(content,
+                                           mimetype='application/json; charset=utf8')
+
+class JsonError(HttpResponse):
+    def __init__(self, error_messages=[]):
+        if isinstance(error_messages, str):
+            error_messages = [error_messages]
+        content = simplejson.dumps({'errors': error_messages},
+                                   indent=2,
+                                   ensure_ascii=False)
+        super(JsonError, self).__init__(content,
+                                        mimetype='application/json; charset=utf8', status=400)
+
+class HtmlResponse(HttpResponse):
+    def __init__(self, html=''):
+        super(HtmlResponse, self).__init__(html, content_type='text/plain')
+
+class ViewNameMiddleware(object):  
+    def process_view(self, request, view_func, view_args, view_kwargs):  
+        request.view_name = view_func.__name__
+
+class QueryCountDebugMiddleware(object):
+    """
+    This middleware will log the number of queries run
+    and the total time taken for each request (with a
+    status code of 200). It does not currently support
+    multi-db setups.
+    """
+    def process_response(self, request, response):
+        if response.status_code == 200:
+            total_time = 0
+
+            for query in connection.queries:
+                query_time = query.get('time')
+                if query_time is None:
+                    # django-debug-toolbar monkeypatches the connection
+                    # cursor wrapper and adds extra information in each
+                    # item in connection.queries. The query time is stored
+                    # under the key "duration" rather than "time" and is
+                    # in milliseconds, not seconds.
+                    query_time = query.get('duration', 0) / 1000
+                total_time += float(query_time)
+
+            logging.info('%s queries run, total %s seconds' % (len(connection.queries), total_time))
+        return response
+
+def get_annotated_content_info(course_id, content, user):
+    return {
+        'editable': check_permissions_by_view(user, course_id, content, "update_thread" if content['type'] == 'thread' else "update_comment"),
+        'can_reply': check_permissions_by_view(user, course_id, content, "create_comment" if content['type'] == 'thread' else "create_sub_comment"),
+        'can_endorse': check_permissions_by_view(user, course_id, content, "endorse_comment") if content['type'] == 'comment' else False,
+        'can_delete': check_permissions_by_view(user, course_id, content, "delete_thread" if content['type'] == 'thread' else "delete_comment"),
+        'can_openclose': check_permissions_by_view(user, course_id, content, "openclose_thread") if content['type'] == 'thread' else False,
+        'can_vote': check_permissions_by_view(user, course_id, content, "vote_for_thread" if content['type'] == 'thread' else "vote_for_comment"),
+    }
+
+def get_annotated_content_infos(course_id, thread, user):
+    infos = {}
+    def _annotate(content):
+        infos[str(content['id'])] = get_annotated_content_info(course_id, content, user)
+        for child in content.get('children', []):
+            _annotate(child)
+    _annotate(thread)
+    return infos
+
+def render_mustache(template_name, dictionary, *args, **kwargs):
+    template = middleware.lookup['main'].get_template(template_name).source
+    return pystache.render(template, dictionary)

utility-scripts/create_groups.py → lms/djangoapps/lms_migration/management/commands/create_groups.py

@@ -6,41 +6,53 @@
 
 import os, sys, string, re
 
-sys.path.append(os.path.abspath('.'))
-os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev'
-
-try:
-    from lms.envs.dev import *
-except Exception as err:
-    print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory."
-    sys.exit(-1)
-
+from django.core.management.base import BaseCommand
 from django.conf import settings
 from django.contrib.auth.models import User, Group
 from path import path
 from lxml import etree
 
-data_dir = settings.DATA_DIR
-print "data_dir = %s" % data_dir
+def create_groups():
+    '''
+    Create staff and instructor groups for all classes in the data_dir
+    '''
+
+    data_dir = settings.DATA_DIR
+    print "data_dir = %s" % data_dir
+
+    for course_dir in os.listdir(data_dir):
 
-for course_dir in os.listdir(data_dir):
-    # print course_dir
-    if not os.path.isdir(path(data_dir) / course_dir):
-        continue
+        if course_dir.startswith('.'):
+            continue
+        if not os.path.isdir(path(data_dir) / course_dir):
+            continue
+        
+        cxfn = path(data_dir) / course_dir / 'course.xml'
+        try:
+            coursexml = etree.parse(cxfn)
+        except Exception as err:
+            print "Oops, cannot read %s, skipping" % cxfn
+            continue
+        cxmlroot = coursexml.getroot()
+        course = cxmlroot.get('course')		# TODO (vshnayder!!): read metadata from policy file(s) instead of from course.xml
+        if course is None:
+            print "oops, can't get course id for %s" % course_dir
+            continue
+        print "course=%s for course_dir=%s" % (course,course_dir)
     
-    cxfn = path(data_dir) / course_dir / 'course.xml'
-    coursexml = etree.parse(cxfn)
-    cxmlroot = coursexml.getroot()
-    course = cxmlroot.get('course')
-    if course is None:
-        print "oops, can't get course id for %s" % course_dir
-        continue
-    print "course=%s for course_dir=%s" % (course,course_dir)
-
-    gname = 'staff_%s' % course
+        create_group('staff_%s' % course)		# staff group
+        create_group('instructor_%s' % course)		# instructor group (can manage staff group list)
+
+def create_group(gname):
     if Group.objects.filter(name=gname):
-        print "group exists for %s" % gname
-        continue
+        print "    group exists for %s" % gname
+        return
     g = Group(name=gname)
     g.save()
-    print "created group %s" % gname
+    print "    created group %s" % gname
+
+class Command(BaseCommand):
+    help = "Create groups associated with all courses in data_dir."
+
+    def handle(self, *args, **options):
+        create_groups()

utility-scripts/create_user.py → lms/djangoapps/lms_migration/management/commands/create_user.py

@@ -8,21 +8,13 @@ import os, sys, string, re
 import datetime
 from getpass import getpass
 import json
+from random import choice
 import readline
 
-sys.path.append(os.path.abspath('.'))
-os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev'
-
-try:
-    from lms.envs.dev import *
-except Exception as err:
-    print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory."
-    sys.exit(-1)
-
+from django.core.management.base import BaseCommand
 from student.models import UserProfile, Registration
 from external_auth.models import ExternalAuthMap
 from django.contrib.auth.models import User, Group
-from random import choice
 
 class MyCompleter(object):  # Custom completer
 
@@ -47,103 +39,108 @@ def GenPasswd(length=8, chars=string.letters + string.digits):
     return ''.join([choice(chars) for i in range(length)])
 
 #-----------------------------------------------------------------------------
-# main
+# main command
 
-while True:
-    uname = raw_input('username: ')
-    if User.objects.filter(username=uname):
-        print "username %s already taken" % uname
-    else:
-        break
+class Command(BaseCommand):
+    help = "Create user, interactively; can add ExternalAuthMap for MIT user if email@MIT.EDU resolves properly."
 
-make_eamap = False
-if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y':
-    email = '%s@MIT.EDU' % uname
-    if not email.endswith('@MIT.EDU'):
-        print "Failed - email must be @MIT.EDU"
-        sys.exit(-1)
-    mit_domain = 'ssl:MIT'
-    if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain):
-        print "Failed - email %s already exists as external_id" % email
-        sys.exit(-1)
-    make_eamap = True
-    password = GenPasswd(12)
-    
-    # get name from kerberos
-    kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip()
-    name = raw_input('Full name: [%s] ' % kname).strip()
-    if name=='':
-        name = kname
-    print "name = %s" % name
-else:
-    while True:
-        password = getpass()
-        password2 = getpass()
-        if password == password2:
-            break
-        print "Oops, passwords do not match, please retry"
+    def handle(self, *args, **options):
 
-    while True:
-        email = raw_input('email: ')
-        if User.objects.filter(email=email):
-            print "email %s already taken" % email
+        while True:
+            uname = raw_input('username: ')
+            if User.objects.filter(username=uname):
+                print "username %s already taken" % uname
+            else:
+                break
+        
+        make_eamap = False
+        if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y':
+            email = '%s@MIT.EDU' % uname
+            if not email.endswith('@MIT.EDU'):
+                print "Failed - email must be @MIT.EDU"
+                sys.exit(-1)
+            mit_domain = 'ssl:MIT'
+            if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain):
+                print "Failed - email %s already exists as external_id" % email
+                sys.exit(-1)
+            make_eamap = True
+            password = GenPasswd(12)
+            
+            # get name from kerberos
+            kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip()
+            name = raw_input('Full name: [%s] ' % kname).strip()
+            if name=='':
+                name = kname
+            print "name = %s" % name
         else:
-            break
+            while True:
+                password = getpass()
+                password2 = getpass()
+                if password == password2:
+                    break
+                print "Oops, passwords do not match, please retry"
         
-    name = raw_input('Full name: ')
-
+            while True:
+                email = raw_input('email: ')
+                if User.objects.filter(email=email):
+                    print "email %s already taken" % email
+                else:
+                    break
+                
+            name = raw_input('Full name: ')
         
-user = User(username=uname, email=email, is_active=True)
-user.set_password(password)
-try:
-    user.save()
-except IntegrityError:
-    print "Oops, failed to create user %s, IntegrityError" % user
-    raise
-    
-r = Registration()
-r.register(user)
-
-up = UserProfile(user=user)
-up.name = name
-up.save()
-
-if make_eamap:
-    credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email)
-    eamap = ExternalAuthMap(external_id = email,
-                            external_email = email,
-                            external_domain = mit_domain,
-                            external_name = name,
-                            internal_password = password,
-                            external_credentials = json.dumps(credentials),
-        )
-    eamap.user = user
-    eamap.dtsignup = datetime.datetime.now()
-    eamap.save()
-
-print "User %s created successfully!" % user
-
-if not raw_input('Add user %s to any groups? [n] ' % user).lower()=='y':
-    sys.exit(0)
-
-print "Here are the groups available:"
-
-groups = [str(g.name) for g in Group.objects.all()]
-print groups
-
-completer = MyCompleter(groups)
-readline.set_completer(completer.complete)
-readline.parse_and_bind('tab: complete')
-
-while True:
-    gname = raw_input("Add group (tab to autocomplete, empty line to end): ")
-    if not gname:
-        break
-    if not gname in groups:
-        print "Unknown group %s" % gname
-        continue
-    g = Group.objects.get(name=gname)
-    user.groups.add(g)
-    print "Added %s to group %s" % (user,g)
-
-print "Done!"
+                
+        user = User(username=uname, email=email, is_active=True)
+        user.set_password(password)
+        try:
+            user.save()
+        except IntegrityError:
+            print "Oops, failed to create user %s, IntegrityError" % user
+            raise
+            
+        r = Registration()
+        r.register(user)
+        
+        up = UserProfile(user=user)
+        up.name = name
+        up.save()
+        
+        if make_eamap:
+            credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email)
+            eamap = ExternalAuthMap(external_id = email,
+                                    external_email = email,
+                                    external_domain = mit_domain,
+                                    external_name = name,
+                                    internal_password = password,
+                                    external_credentials = json.dumps(credentials),
+                )
+            eamap.user = user
+            eamap.dtsignup = datetime.datetime.now()
+            eamap.save()
+        
+        print "User %s created successfully!" % user
+        
+        if not raw_input('Add user %s to any groups? [n] ' % user).lower()=='y':
+            sys.exit(0)
+        
+        print "Here are the groups available:"
+        
+        groups = [str(g.name) for g in Group.objects.all()]
+        print groups
+        
+        completer = MyCompleter(groups)
+        readline.set_completer(completer.complete)
+        readline.parse_and_bind('tab: complete')
+        
+        while True:
+            gname = raw_input("Add group (tab to autocomplete, empty line to end): ")
+            if not gname:
+                break
+            if not gname in groups:
+                print "Unknown group %s" % gname
+                continue
+            g = Group.objects.get(name=gname)
+            user.groups.add(g)
+            print "Added %s to group %s" % (user,g)
+        
+        print "Done!"

lms/djangoapps/lms_migration/migrate.py

@@ -2,13 +2,21 @@
 # migration tools for content team to go from stable-edx4edx to LMS+CMS
 #
 
+import json
 import logging
+import os
 from pprint import pprint
 import xmodule.modulestore.django as xmodule_django
 from xmodule.modulestore.django import modulestore
 
 from django.http import HttpResponse
 from django.conf import settings
+import track.views
+
+try:
+    from django.views.decorators.csrf import csrf_exempt
+except ImportError:
+    from django.contrib.csrf.middleware import csrf_exempt
 
 log = logging.getLogger("mitx.lms_migrate")
 LOCAL_DEBUG = True
@@ -18,6 +26,15 @@ def escape(s):
     """escape HTML special characters in string"""
     return str(s).replace('<','&lt;').replace('>','&gt;')
 
+def getip(request):
+    '''
+    Extract IP address of requester from header, even if behind proxy
+    '''
+    ip = request.META.get('HTTP_X_REAL_IP','')	# nginx reverse proxy
+    if not ip:
+        ip = request.META.get('REMOTE_ADDR','None')
+    return ip
+
 def manage_modulestores(request,reload_dir=None):
     '''
     Manage the static in-memory modulestores.
@@ -32,9 +49,7 @@ def manage_modulestores(request,reload_dir=None):
     #----------------------------------------
     # check on IP address of requester
 
-    ip = request.META.get('HTTP_X_REAL_IP','')	# nginx reverse proxy
-    if not ip:
-        ip = request.META.get('REMOTE_ADDR','None')
+    ip = getip(request)
 
     if LOCAL_DEBUG:
         html += '<h3>IP address: %s ' % ip
@@ -48,7 +63,7 @@ def manage_modulestores(request,reload_dir=None):
             html += 'Permission denied'
             html += "</body></html>"
             log.debug('request denied, ALLOWED_IPS=%s' % ALLOWED_IPS)
-            return HttpResponse(html)
+            return HttpResponse(html, status=403)
 
     #----------------------------------------
     # reload course if specified
@@ -108,3 +123,66 @@ def manage_modulestores(request,reload_dir=None):
 
     html += "</body></html>"
     return HttpResponse(html)
+
+@csrf_exempt
+def gitreload(request, reload_dir=None):
+    '''
+    This can be used as a github WebHook Service Hook, for reloading of the content repo used by the LMS.
+
+    If reload_dir is not None, then instruct the xml loader to reload that course directory.
+    '''
+    html = "<html><body>"
+    ip = getip(request)
+
+    html += '<h3>IP address: %s ' % ip
+    html += '<h3>User: %s ' % request.user
+
+    ALLOWED_IPS = []	# allow none by default
+    if hasattr(settings,'ALLOWED_GITRELOAD_IPS'):	# allow override in settings
+        ALLOWED_IPS = ALLOWED_GITRELOAD_IPS
+
+    if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
+        if request.user and request.user.is_staff:
+            log.debug('request allowed because user=%s is staff' % request.user)
+        else:
+            html += 'Permission denied'
+            html += "</body></html>"
+            log.debug('request denied from %s, ALLOWED_IPS=%s' % (ip,ALLOWED_IPS))
+            return HttpResponse(html)    
+        
+    #----------------------------------------
+    # see if request is from github (POST with JSON)
+
+    if reload_dir is None and 'payload' in request.POST:
+        payload = request.POST['payload']
+        log.debug("payload=%s" % payload)
+        gitargs = json.loads(payload)
+        log.debug("gitargs=%s" % gitargs)
+        reload_dir = gitargs['repository']['name']
+        log.debug("github reload_dir=%s" % reload_dir)
+        gdir = settings.DATA_DIR / reload_dir 
+        if not os.path.exists(gdir):
+            log.debug("====> ERROR in gitreload - no such directory %s" % reload_dir)
+            return HttpResponse('Error')
+        cmd = "cd %s; git reset --hard HEAD; git clean -f -d; git pull origin; chmod g+w course.xml" % gdir
+        log.debug(os.popen(cmd).read())
+        if hasattr(settings,'GITRELOAD_HOOK'):	# hit this hook after reload, if set
+            gh = settings.GITRELOAD_HOOK
+            if gh:
+                ghurl = '%s/%s' % (gh,reload_dir)
+                r = requests.get(ghurl)
+                log.debug("GITRELOAD_HOOK to %s: %s" % (ghurl, r.text))
+        
+    #----------------------------------------
+    # reload course if specified
+
+    if reload_dir is not None:
+        def_ms = modulestore()
+        if reload_dir not in def_ms.courses:
+            html += "<h2><font color='red'>Error: '%s' is not a valid course directory</font></h2>" % reload_dir
+        else:
+            html += "<h2><font color='blue'>Reloaded course directory '%s'</font></h2>" % reload_dir
+            def_ms.try_load_course(reload_dir)
+            track.views.server_track(request, 'reloaded %s' % reload_dir, {}, page='migrate')
+
+    return HttpResponse(html)    

lms/djangoapps/simplewiki/views.py

@@ -39,7 +39,7 @@ def update_template_dictionary(dictionary, request=None, course=None, article=No
     if course:
         dictionary['course'] = course
         if 'namespace' not in dictionary:
-            dictionary['namespace'] = course.wiki_namespace
+            dictionary['namespace'] = "edX"
     else:
         dictionary['course'] = None
 
@@ -99,7 +99,7 @@ def root_redirect(request, course_id=None):
     course = get_opt_course_with_access(request.user, course_id, 'load')
 
     #TODO: Add a default namespace to settings.
-    namespace = course.wiki_namespace if course else "edX"
+    namespace = "edX"
 
     try:
         root = Article.get_root(namespace)
@@ -479,7 +479,7 @@ def not_found(request, article_path, course):
     """Generate a NOT FOUND message for some URL"""
     d = {'wiki_err_notfound': True,
          'article_path': article_path,
-         'namespace': course.wiki_namespace}
+         'namespace': "edX"}
     update_template_dictionary(d, request, course)
     return render_to_response('simplewiki/simplewiki_error.html', d)
 

lms/envs/aws.py

@@ -19,6 +19,11 @@ EMAIL_BACKEND = 'django_ses.SESBackend'
 SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
 DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage'
 
+# Disable askbot, enable Berkeley forums
+MITX_FEATURES['ENABLE_DISCUSSION'] = False
+MITX_FEATURES['ENABLE_DISCUSSION_SERVICE'] = True
+
+
 ########################### NON-SECURE ENV CONFIG ##############################
 # Things like server locations, ports, etc.
 with open(ENV_ROOT / "env.json") as env_file:
@@ -42,6 +47,7 @@ LOGGING = get_logger_config(LOG_DIR,
                             syslog_addr=(ENV_TOKENS['SYSLOG_SERVER'], 514),
                             debug=False)
 
+COURSE_LISTINGS = ENV_TOKENS['COURSE_LISTINGS']
 
 ############################## SECURE AUTH ITEMS ###############################
 # Secret things: passwords, access keys, etc.
@@ -60,3 +66,5 @@ XQUEUE_INTERFACE = AUTH_TOKENS['XQUEUE_INTERFACE']
 if 'COURSE_ID' in ENV_TOKENS:
     ASKBOT_URL = "courses/{0}/discussions/".format(ENV_TOKENS['COURSE_ID'])
 
+COMMENTS_SERVICE_URL = ENV_TOKENS["COMMENTS_SERVICE_URL"]
+

lms/envs/common.py

@@ -30,10 +30,11 @@ import djcelery
 from path import path
 
 from .askbotsettings import * # this is where LIVESETTINGS_OPTIONS comes from
+from .discussionsettings import *
 
 ################################### FEATURES ###################################
 COURSEWARE_ENABLED = True
-ASKBOT_ENABLED = True
+ASKBOT_ENABLED = False
 GENERATE_RANDOM_USER_CREDENTIALS = False
 PERFSTATS = False
 
@@ -54,8 +55,13 @@ MITX_FEATURES = {
     # course_ids (see dev_int.py for an example)
     'SUBDOMAIN_COURSE_LISTINGS' : False,
 
+    # TODO: This will be removed once course-specific tabs are in place. see
+    # courseware/courses.py
+    'ENABLE_SYLLABUS' : True, 
+
     'ENABLE_TEXTBOOK' : True,
-    'ENABLE_DISCUSSION' : True,
+    'ENABLE_DISCUSSION' : False,
+    'ENABLE_DISCUSSION_SERVICE': True,
 
     'ENABLE_SQL_TRACKING_LOGS': False,
     'ENABLE_LMS_MIGRATION': False,
@@ -260,6 +266,14 @@ USE_L10N = True
 # Messages
 MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage'
 
+#################################### GITHUB #######################################
+# gitreload is used in LMS-workflow to pull content from github
+# gitreload requests are only allowed from these IP addresses, which are
+# the advertised public IPs of the github WebHook servers.
+# These are listed, eg at https://github.com/MITx/mitx/admin/hooks
+
+ALLOWED_GITRELOAD_IPS = ['207.97.227.253', '50.57.128.197', '108.171.174.178']
+
 #################################### AWS #######################################
 # S3BotoStorage insists on a timeout for uploaded assets. We should make it
 # permanent instead, but rather than trying to figure out exactly where that
@@ -304,6 +318,9 @@ SIMPLE_WIKI_REQUIRE_LOGIN_VIEW = False
 ################################# WIKI ###################################
 WIKI_ACCOUNT_HANDLING = False
 WIKI_EDITOR = 'course_wiki.editors.CodeMirror'
+WIKI_SHOW_MAX_CHILDREN = 0 # We don't use the little menu that shows children of an article in the breadcrumb
+WIKI_ANONYMOUS = False # Don't allow anonymous access until the styling is figured out
+WIKI_CAN_CHANGE_PERMISSIONS = lambda article, user: user.has_perm('wiki.assign')
 
 ################################# Jasmine ###################################
 JASMINE_TEST_DIRECTORY = PROJECT_ROOT + '/static/coffee'
@@ -329,6 +346,7 @@ TEMPLATE_LOADERS = (
 )
 
 MIDDLEWARE_CLASSES = (
+    'django_comment_client.middleware.AjaxExceptionMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
@@ -351,6 +369,9 @@ MIDDLEWARE_CLASSES = (
     'askbot.middleware.spaceless.SpacelessMiddleware',
     # 'askbot.middleware.pagesize.QuestionsPageSizeMiddleware',
     # 'debug_toolbar.middleware.DebugToolbarMiddleware',
+
+    'django_comment_client.utils.ViewNameMiddleware',
+    'django_comment_client.utils.QueryCountDebugMiddleware',
 )
 
 ############################### Pipeline #######################################
@@ -565,13 +586,16 @@ INSTALLED_APPS = (
     'course_wiki', # Our customizations
     'mptt',
     'sekizai',
-    'wiki.plugins.attachments',
+    #'wiki.plugins.attachments',
     'wiki.plugins.notifications',
     'course_wiki.plugins.markdownedx',
 
     # For testing
     'django_jasmine',
 
+    # Discussion
+    'django_comment_client',
+
     # For Askbot
     'django.contrib.sitemaps',
     'django.contrib.admin',

lms/envs/dev.py

@@ -73,6 +73,8 @@ MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True
 MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = False   # require that user be in the staff_* group to be able to enroll
 MITX_FEATURES['USE_XQA_SERVER'] = 'http://xqa:server@content-qa.mitx.mit.edu/xqa'
 
+INSTALLED_APPS += ('lms_migration',)
+
 LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1']
 
 ################################ OpenID Auth #################################

lms/envs/dev_ike.py

@@ -17,14 +17,19 @@ MITX_FEATURES['ENABLE_TEXTBOOK'] = False
 MITX_FEATURES['ENABLE_DISCUSSION'] = False
 MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True	  # require that user be in the staff_* group to be able to enroll
 
+MITX_FEATURES['DISABLE_START_DATES'] = True
+# MITX_FEATURES['USE_DJANGO_PIPELINE']=False      # don't recompile scss
+
 myhost = socket.gethostname()
 if ('edxvm' in myhost) or ('ocw' in myhost):
     MITX_FEATURES['DISABLE_LOGIN_BUTTON'] = True	# auto-login with MIT certificate
     MITX_FEATURES['USE_XQA_SERVER'] = 'https://qisx.mit.edu/xqa'	# needs to be ssl or browser blocks it
+    MITX_FEATURES['USE_DJANGO_PIPELINE']=False      # don't recompile scss
 
 if ('domU' in myhost):
     EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
     MITX_FEATURES['REROUTE_ACTIVATION_EMAIL'] = 'ichuang@mitx.mit.edu'	# nonempty string = address for all activation emails
+    MITX_FEATURES['USE_DJANGO_PIPELINE']=False      # don't recompile scss
 
 SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https')	# django 1.4 for nginx ssl proxy
 
@@ -33,4 +38,9 @@ SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https')	# django 1.4 fo
 
 INSTALLED_APPS = tuple([ app for app in INSTALLED_APPS if not app.startswith('debug_toolbar') ])
 MIDDLEWARE_CLASSES = tuple([ mcl for mcl in MIDDLEWARE_CLASSES if not mcl.startswith('debug_toolbar') ])
-TEMPLATE_LOADERS = tuple([ app for app in TEMPLATE_LOADERS if not app.startswith('askbot') ])
+#TEMPLATE_LOADERS = tuple([ app for app in TEMPLATE_LOADERS if not app.startswith('askbot') ])
+#TEMPLATE_LOADERS = tuple([ app for app in TEMPLATE_LOADERS if not app.startswith('mitxmako') ])
+TEMPLATE_LOADERS = (
+    'django.template.loaders.filesystem.Loader',
+    'django.template.loaders.app_directories.Loader',
+    )

lms/envs/discussionsettings.py

+DISCUSSION_ALLOWED_UPLOAD_FILE_TYPES = ('.jpg', '.jpeg', '.gif', '.bmp', '.png', '.tiff')

lms/lib/comment_client/__init__.py

+from comment_client import *
+from utils import CommentClientError, CommentClientUnknownError

lms/lib/comment_client/comment.py

+from utils import *
+
+import models
+import settings
+
+class Comment(models.Model):
+
+    accessible_fields = [
+        'id', 'body', 'anonymous', 'course_id',
+        'endorsed', 'parent_id', 'thread_id',
+        'username', 'votes', 'user_id', 'closed',
+        'created_at', 'updated_at', 'depth',
+        'at_position_list', 'type',
+    ]
+
+    updatable_fields = [
+        'body', 'anonymous', 'course_id', 'closed',
+        'user_id', 'endorsed',
+    ]
+
+    initializable_fields = updatable_fields
+
+    base_url = "{prefix}/comments".format(prefix=settings.PREFIX)
+    type = 'comment'
+
+    @classmethod
+    def url_for_comments(cls, params={}):
+        if params.get('thread_id'):
+            return _url_for_thread_comments(params['thread_id'])
+        else:
+            return _url_for_comment(params['parent_id'])
+
+    @classmethod
+    def url(cls, action, params={}):
+        if action in ['post']:
+            return cls.url_for_comments(params)
+        else:
+            return super(Comment, cls).url(action, params)
+
+def _url_for_thread_comments(thread_id):
+    return "{prefix}/threads/{thread_id}/comments".format(prefix=settings.PREFIX, thread_id=thread_id)
+
+def _url_for_comment(comment_id):
+    return "{prefix}/comments/{comment_id}".format(prefix=settings.PREFIX, comment_id=comment_id)

lms/lib/comment_client/comment_client.py

+from comment import Comment
+from thread import Thread
+from user import User
+from commentable import Commentable
+
+from utils import *
+
+import settings
+
+def search_similar_threads(course_id, recursive=False, query_params={}, *args, **kwargs):
+    default_params = {'course_id': course_id, 'recursive': recursive}
+    attributes = dict(default_params.items() + query_params.items())
+    return perform_request('get', _url_for_search_similar_threads(), attributes, *args, **kwargs)
+
+def search_recent_active_threads(course_id, recursive=False, query_params={}, *args, **kwargs):
+    default_params = {'course_id': course_id, 'recursive': recursive}
+    attributes = dict(default_params.items() + query_params.items())
+    return perform_request('get', _url_for_search_recent_active_threads(), attributes, *args, **kwargs)
+
+def search_trending_tags(course_id, query_params={}, *args, **kwargs):
+    default_params = {'course_id': course_id}
+    attributes = dict(default_params.items() + query_params.items())
+    return perform_request('get', _url_for_search_trending_tags(), attributes, *args, **kwargs)
+
+def tags_autocomplete(value, *args, **kwargs):
+    return perform_request('get', _url_for_threads_tags_autocomplete(), {'value': value}, *args, **kwargs)
+
+def _url_for_search_similar_threads():
+    return "{prefix}/search/threads/more_like_this".format(prefix=settings.PREFIX)
+
+def _url_for_search_recent_active_threads():
+    return "{prefix}/search/threads/recent_active".format(prefix=settings.PREFIX)
+
+def _url_for_search_trending_tags():
+    return "{prefix}/search/tags/trending".format(prefix=settings.PREFIX)
+
+def _url_for_threads_tags_autocomplete():
+    return "{prefix}/threads/tags/autocomplete".format(prefix=settings.PREFIX)

lms/lib/comment_client/commentable.py

+from utils import *
+
+import models
+import settings
+
+class Commentable(models.Model):
+
+    base_url = "{prefix}/commentables".format(prefix=settings.PREFIX)
+    type = 'commentable'

lms/lib/comment_client/legacy.py

+def delete_threads(commentable_id, *args, **kwargs):
+    return _perform_request('delete', _url_for_commentable_threads(commentable_id), *args, **kwargs)
+
+def get_threads(commentable_id, recursive=False, query_params={}, *args, **kwargs):
+    default_params = {'page': 1, 'per_page': 20, 'recursive': recursive}
+    attributes = dict(default_params.items() + query_params.items())
+    response = _perform_request('get', _url_for_threads(commentable_id), \
+            attributes, *args, **kwargs)
+    return response.get('collection', []), response.get('page', 1), response.get('num_pages', 1)
+
+def search_threads(course_id, recursive=False, query_params={}, *args, **kwargs):
+    default_params = {'page': 1, 'per_page': 20, 'course_id': course_id, 'recursive': recursive}
+    attributes = dict(default_params.items() + query_params.items())
+    response = _perform_request('get', _url_for_search_threads(), \
+            attributes, *args, **kwargs)
+    return response.get('collection', []), response.get('page', 1), response.get('num_pages', 1)
+
+def search_similar_threads(course_id, recursive=False, query_params={}, *args, **kwargs):
+    default_params = {'course_id': course_id, 'recursive': recursive}
+    attributes = dict(default_params.items() + query_params.items())
+    return _perform_request('get', _url_for_search_similar_threads(), attributes, *args, **kwargs)
+
+def search_recent_active_threads(course_id, recursive=False, query_params={}, *args, **kwargs):
+    default_params = {'course_id': course_id, 'recursive': recursive}
+    attributes = dict(default_params.items() + query_params.items())
+    return _perform_request('get', _url_for_search_recent_active_threads(), attributes, *args, **kwargs)
+
+def search_trending_tags(course_id, query_params={}, *args, **kwargs):
+    default_params = {'course_id': course_id}
+    attributes = dict(default_params.items() + query_params.items())
+    return _perform_request('get', _url_for_search_trending_tags(), attributes, *args, **kwargs)
+
+def create_user(attributes, *args, **kwargs):
+    return _perform_request('post', _url_for_users(), attributes, *args, **kwargs)
+
+def update_user(user_id, attributes, *args, **kwargs):
+    return _perform_request('put', _url_for_user(user_id), attributes, *args, **kwargs)
+
+def get_threads_tags(*args, **kwargs):
+    return _perform_request('get', _url_for_threads_tags(), {}, *args, **kwargs)
+
+def tags_autocomplete(value, *args, **kwargs):
+    return _perform_request('get', _url_for_threads_tags_autocomplete(), {'value': value}, *args, **kwargs)
+
+def create_thread(commentable_id, attributes, *args, **kwargs):
+    return _perform_request('post', _url_for_threads(commentable_id), attributes, *args, **kwargs)
+    
+def get_thread(thread_id, recursive=False, *args, **kwargs):
+    return _perform_request('get', _url_for_thread(thread_id), {'recursive': recursive}, *args, **kwargs)
+
+def update_thread(thread_id, attributes, *args, **kwargs):
+    return _perform_request('put', _url_for_thread(thread_id), attributes, *args, **kwargs)
+
+def create_comment(thread_id, attributes, *args, **kwargs):
+    return _perform_request('post', _url_for_thread_comments(thread_id), attributes, *args, **kwargs)
+
+def delete_thread(thread_id, *args, **kwargs):
+    return _perform_request('delete', _url_for_thread(thread_id), *args, **kwargs)
+
+def get_comment(comment_id, recursive=False, *args, **kwargs):
+    return _perform_request('get', _url_for_comment(comment_id), {'recursive': recursive}, *args, **kwargs)
+
+def update_comment(comment_id, attributes, *args, **kwargs):
+    return _perform_request('put', _url_for_comment(comment_id), attributes, *args, **kwargs)
+
+def create_sub_comment(comment_id, attributes, *args, **kwargs):
+    return _perform_request('post', _url_for_comment(comment_id), attributes, *args, **kwargs)
+
+def delete_comment(comment_id, *args, **kwargs):
+    return _perform_request('delete', _url_for_comment(comment_id), *args, **kwargs)
+
+def vote_for_comment(comment_id, user_id, value, *args, **kwargs):
+    return _perform_request('put', _url_for_vote_comment(comment_id), {'user_id': user_id, 'value': value}, *args, **kwargs)
+
+def undo_vote_for_comment(comment_id, user_id, *args, **kwargs):
+    return _perform_request('delete', _url_for_vote_comment(comment_id), {'user_id': user_id}, *args, **kwargs)
+
+def vote_for_thread(thread_id, user_id, value, *args, **kwargs):
+    return _perform_request('put', _url_for_vote_thread(thread_id), {'user_id': user_id, 'value': value}, *args, **kwargs)
+
+def undo_vote_for_thread(thread_id, user_id, *args, **kwargs):
+    return _perform_request('delete', _url_for_vote_thread(thread_id), {'user_id': user_id}, *args, **kwargs)
+
+def get_notifications(user_id, *args, **kwargs):
+    return _perform_request('get', _url_for_notifications(user_id), *args, **kwargs)
+
+def get_user_info(user_id, complete=True, *args, **kwargs):
+    return _perform_request('get', _url_for_user(user_id), {'complete': complete}, *args, **kwargs)
+
+def subscribe(user_id, subscription_detail, *args, **kwargs):
+    return _perform_request('post', _url_for_subscription(user_id), subscription_detail, *args, **kwargs)
+
+def subscribe_user(user_id, followed_user_id, *args, **kwargs):
+    return subscribe(user_id, {'source_type': 'user', 'source_id': followed_user_id})
+
+follow = subscribe_user
+
+def subscribe_thread(user_id, thread_id, *args, **kwargs):
+    return subscribe(user_id, {'source_type': 'thread', 'source_id': thread_id})
+
+def subscribe_commentable(user_id, commentable_id, *args, **kwargs):
+    return subscribe(user_id, {'source_type': 'other', 'source_id': commentable_id})
+
+def unsubscribe(user_id, subscription_detail, *args, **kwargs):
+    return _perform_request('delete', _url_for_subscription(user_id), subscription_detail, *args, **kwargs)
+
+def unsubscribe_user(user_id, followed_user_id, *args, **kwargs):
+    return unsubscribe(user_id, {'source_type': 'user', 'source_id': followed_user_id})
+
+unfollow = unsubscribe_user
+
+def unsubscribe_thread(user_id, thread_id, *args, **kwargs):
+    return unsubscribe(user_id, {'source_type': 'thread', 'source_id': thread_id})
+
+def unsubscribe_commentable(user_id, commentable_id, *args, **kwargs):
+    return unsubscribe(user_id, {'source_type': 'other', 'source_id': commentable_id})
+
+def _perform_request(method, url, data_or_params=None, *args, **kwargs):
+    if method in ['post', 'put', 'patch']:
+        response = requests.request(method, url, data=data_or_params)
+    else:
+        response = requests.request(method, url, params=data_or_params)
+    if 200 < response.status_code < 500:
+        raise CommentClientError(response.text)
+    elif response.status_code == 500:
+        raise CommentClientUnknownError(response.text)
+    else:
+        if kwargs.get("raw", False):
+            return response.text
+        else:
+            return json.loads(response.text)
+
+def _url_for_threads(commentable_id):
+    return "{prefix}/{commentable_id}/threads".format(prefix=PREFIX, commentable_id=commentable_id)
+
+def _url_for_thread(thread_id):
+    return "{prefix}/threads/{thread_id}".format(prefix=PREFIX, thread_id=thread_id)
+
+def _url_for_thread_comments(thread_id):
+    return "{prefix}/threads/{thread_id}/comments".format(prefix=PREFIX, thread_id=thread_id)
+
+def _url_for_comment(comment_id):
+    return "{prefix}/comments/{comment_id}".format(prefix=PREFIX, comment_id=comment_id)
+
+def _url_for_vote_comment(comment_id):
+    return "{prefix}/comments/{comment_id}/votes".format(prefix=PREFIX, comment_id=comment_id)
+
+def _url_for_vote_thread(thread_id):
+    return "{prefix}/threads/{thread_id}/votes".format(prefix=PREFIX, thread_id=thread_id)
+
+def _url_for_notifications(user_id):
+    return "{prefix}/users/{user_id}/notifications".format(prefix=PREFIX, user_id=user_id)
+
+def _url_for_subscription(user_id):
+    return "{prefix}/users/{user_id}/subscriptions".format(prefix=PREFIX, user_id=user_id)
+
+def _url_for_user(user_id):
+    return "{prefix}/users/{user_id}".format(prefix=PREFIX, user_id=user_id)
+
+def _url_for_search_threads():
+    return "{prefix}/search/threads".format(prefix=PREFIX)
+
+def _url_for_search_similar_threads():
+    return "{prefix}/search/threads/more_like_this".format(prefix=PREFIX)
+
+def _url_for_search_recent_active_threads():
+    return "{prefix}/search/threads/recent_active".format(prefix=PREFIX)
+
+def _url_for_search_trending_tags():
+    return "{prefix}/search/tags/trending".format(prefix=PREFIX)
+
+def _url_for_threads_tags():
+    return "{prefix}/threads/tags".format(prefix=PREFIX)
+
+def _url_for_threads_tags_autocomplete():
+    return "{prefix}/threads/tags/autocomplete".format(prefix=PREFIX)
+
+def _url_for_users():
+    return "{prefix}/users".format(prefix=PREFIX)
+

lms/lib/comment_client/models.py

+from utils import *
+
+class Model(object):
+
+    accessible_fields = ['id']
+    updatable_fields = ['id']
+    initializable_fields = ['id']
+    base_url = None
+    default_retrieve_params = {}
+
+    DEFAULT_ACTIONS_WITH_ID = ['get', 'put', 'delete']
+    DEFAULT_ACTIONS_WITHOUT_ID = ['get_all', 'post']
+    DEFAULT_ACTIONS = DEFAULT_ACTIONS_WITH_ID + DEFAULT_ACTIONS_WITHOUT_ID
+
+    def __init__(self, *args, **kwargs):
+        self.attributes = extract(kwargs, self.accessible_fields)
+        self.retrieved = False
+
+    def __getattr__(self, name):
+        if name == 'id':
+            return self.attributes.get('id', None)
+        try:
+            return self.attributes[name]
+        except KeyError:
+            if self.retrieved or self.id == None:
+                raise AttributeError("Field {0} does not exist".format(name))
+            self.retrieve()
+            return self.__getattr__(name)
+    
+    def __setattr__(self, name, value):
+        if name == 'attributes' or name not in self.accessible_fields:
+            super(Model, self).__setattr__(name, value)
+        else:
+            self.attributes[name] = value
+
+    def __getitem__(self, key):
+        if key not in self.accessible_fields:
+            raise KeyError("Field {0} does not exist".format(key))
+        return self.attributes.get(key)
+
+    def __setitem__(self, key, value):
+        if key not in self.accessible_fields:
+            raise KeyError("Field {0} does not exist".format(key))
+        self.attributes.__setitem__(key, value)
+
+    def get(self, *args, **kwargs):
+        return self.attributes.get(*args, **kwargs)
+
+    def to_dict(self):
+        self.retrieve()
+        return self.attributes
+
+    def retrieve(self, *args, **kwargs):
+        if not self.retrieved:
+            self._retrieve(*args, **kwargs)
+            self.retrieved = True
+        return self
+
+    def _retrieve(self, *args, **kwargs):
+        url = self.url(action='get', params=self.attributes)
+        response = perform_request('get', url, self.default_retrieve_params)
+        self.update_attributes(**response)
+
+    @classmethod
+    def find(cls, id):
+        return cls(id=id)
+
+    def update_attributes(self, *args, **kwargs):
+        for k, v in kwargs.items():
+            if k in self.accessible_fields:
+                self.__setattr__(k, v)
+            else:
+                raise AttributeError("Field {0} does not exist".format(k))
+
+    def updatable_attributes(self):
+        return extract(self.attributes, self.updatable_fields)
+
+    def initializable_attributes(self):
+        return extract(self.attributes, self.initializable_fields)
+        
+    @classmethod
+    def before_save(cls, instance):
+        pass
+
+    @classmethod
+    def after_save(cls, instance):
+        pass
+
+    def save(self):
+        self.__class__.before_save(self)
+        if self.id: # if we have id already, treat this as an update
+            url = self.url(action='put', params=self.attributes)
+            response = perform_request('put', url, self.updatable_attributes())
+        else: # otherwise, treat this as an insert
+            url = self.url(action='post', params=self.attributes)
+            response = perform_request('post', url, self.initializable_attributes())
+        self.retrieved = True
+        self.update_attributes(**response)
+        self.__class__.after_save(self)
+
+    def delete(self):
+        url = self.url(action='delete', params=self.attributes)
+        response = perform_request('delete', url)
+        self.retrieved = True
+        self.update_attributes(**response)
+
+    @classmethod
+    def url_with_id(cls, params={}):
+        return cls.base_url + '/' + str(params['id'])
+
+    @classmethod
+    def url_without_id(cls, params={}):
+        return cls.base_url
+
+    @classmethod
+    def url(cls, action, params={}):
+        if cls.base_url is None:
+            raise CommentClientError("Must provide base_url when using default url function")
+        if action not in cls.DEFAULT_ACTIONS:
+            raise ValueError("Invalid action {0}. The supported action must be in {1}".format(action, str(cls.DEFAULT_ACTIONS)))
+        elif action in cls.DEFAULT_ACTIONS_WITH_ID:
+            try:
+                return cls.url_with_id(params)
+            except KeyError:
+                raise CommentClientError("Cannot perform action {0} without id".format(action))
+        else: # action must be in DEFAULT_ACTIONS_WITHOUT_ID now
+            return cls.url_without_id()

lms/lib/comment_client/requirements.txt

+requests

lms/lib/comment_client/settings.py

+from django.conf import settings
+
+if hasattr(settings, "COMMENTS_SERVICE_URL"):
+    SERVICE_HOST = settings.COMMENTS_SERVICE_URL
+else:
+    SERVICE_HOST = 'http://localhost:4567'
+
+PREFIX = SERVICE_HOST + '/api/v1'
+
+API_KEY = "PUT_YOUR_API_KEY_HERE"

lms/lib/comment_client/thread.py

+from utils import *
+
+import models
+import settings
+
+class Thread(models.Model):
+
+    accessible_fields = [
+        'id', 'title', 'body', 'anonymous',
+        'course_id', 'closed', 'tags', 'votes',
+        'commentable_id', 'username', 'user_id',
+        'created_at', 'updated_at', 'comments_count',
+        'at_position_list', 'children', 'type',
+    ]
+
+    updatable_fields = [
+        'title', 'body', 'anonymous', 'course_id', 
+        'closed', 'tags', 'user_id', 'commentable_id',
+    ]
+
+    initializable_fields = updatable_fields
+
+    base_url = "{prefix}/threads".format(prefix=settings.PREFIX)
+    default_retrieve_params = {'recursive': False}
+    type = 'thread'
+
+    @classmethod
+    def search(cls, query_params, *args, **kwargs):
+        default_params = {'page': 1,
+                          'per_page': 20,
+                          'course_id': query_params['course_id'],
+                          'recursive': False}
+        params = merge_dict(default_params, strip_blank(strip_none(query_params)))
+        if query_params.get('text') or query_params.get('tags'):
+            url = cls.url(action='search')
+        else:
+            url = cls.url(action='get_all', params=extract(params, 'commentable_id'))
+            if params.get('commentable_id'):
+                del params['commentable_id']
+        response = perform_request('get', url, params, *args, **kwargs)
+        return response.get('collection', []), response.get('page', 1), response.get('num_pages', 1)
+        
+    @classmethod
+    def url_for_threads(cls, params={}):
+        if params.get('commentable_id'):
+            return "{prefix}/{commentable_id}/threads".format(prefix=settings.PREFIX, commentable_id=params['commentable_id'])
+        else:
+            return "{prefix}/threads".format(prefix=settings.PREFIX)
+
+    @classmethod
+    def url_for_search_threads(cls, params={}):
+        return "{prefix}/search/threads".format(prefix=settings.PREFIX)
+
+    @classmethod
+    def url(cls, action, params={}):
+        if action in ['get_all', 'post']:
+            return cls.url_for_threads(params)
+        elif action == 'search':
+            return cls.url_for_search_threads(params)
+        else:
+            return super(Thread, cls).url(action, params)
+
+    def _retrieve(self, *args, **kwargs):
+        url = self.url(action='get', params=self.attributes)
+        response = perform_request('get', url, {'recursive': kwargs.get('recursive')})
+        self.update_attributes(**response)

lms/lib/comment_client/user.py

+from utils import *
+
+import models
+import settings
+
+class User(models.Model):
+
+    accessible_fields = ['username', 'email', 'follower_ids', 'upvoted_ids', 'downvoted_ids',
+                         'id', 'external_id', 'subscribed_user_ids', 'children', 'course_id',
+                         'subscribed_thread_ids', 'subscribed_commentable_ids',
+                         'threads_count', 'comments_count',
+                        ]
+
+    updatable_fields = ['username', 'external_id', 'email']
+    initializable_fields = updatable_fields
+
+    base_url = "{prefix}/users".format(prefix=settings.PREFIX)
+    default_retrieve_params = {'complete': True}
+    type = 'user'
+
+    @classmethod
+    def from_django_user(cls, user):
+        return cls(id=str(user.id),
+                   external_id=str(user.id),
+                   username=user.username,
+                   email=user.email)
+
+    def follow(self, source):
+        params = {'source_type': source.type, 'source_id': source.id}
+        response = perform_request('post', _url_for_subscription(self.id), params)
+
+    def unfollow(self, source):
+        params = {'source_type': source.type, 'source_id': source.id}
+        response = perform_request('delete', _url_for_subscription(self.id), params)
+
+    def vote(self, voteable, value):
+        if voteable.type == 'thread':
+            url = _url_for_vote_thread(voteable.id)
+        elif voteable.type == 'comment':
+            url = _url_for_vote_comment(voteable.id)
+        else:
+            raise CommentClientError("Can only vote / unvote for threads or comments")
+        params = {'user_id': self.id, 'value': value}
+        request = perform_request('put', url, params)
+        voteable.update_attributes(request)
+
+    def unvote(self, voteable):
+        if voteable.type == 'thread':
+            url = _url_for_vote_thread(voteable.id)
+        elif voteable.type == 'comment':
+            url = _url_for_vote_comment(voteable.id)
+        else:
+            raise CommentClientError("Can only vote / unvote for threads or comments")
+        params = {'user_id': self.id}
+        request = perform_request('delete', url, params)
+        voteable.update_attributes(request)
+
+    def active_threads(self, query_params={}):
+        if not self.course_id:
+            raise CommentClientError("Must provide course_id when retrieving active threads for the user")
+        url = _url_for_user_active_threads(self.id)
+        params = {'course_id': self.course_id}
+        params = merge_dict(params, query_params)
+        response = perform_request('get', url, params)
+        return response.get('collection', []), response.get('page', 1), response.get('num_pages', 1)
+
+    def _retrieve(self, *args, **kwargs):
+        url = self.url(action='get', params=self.attributes)
+        retrieve_params = self.default_retrieve_params
+        if self.attributes.get('course_id'):
+            retrieve_params['course_id'] = self.course_id
+        response = perform_request('get', url, retrieve_params)
+        self.update_attributes(**response)
+
+def _url_for_vote_comment(comment_id):
+    return "{prefix}/comments/{comment_id}/votes".format(prefix=settings.PREFIX, comment_id=comment_id)
+
+def _url_for_vote_thread(thread_id):
+    return "{prefix}/threads/{thread_id}/votes".format(prefix=settings.PREFIX, thread_id=thread_id)
+
+def _url_for_subscription(user_id):
+    return "{prefix}/users/{user_id}/subscriptions".format(prefix=settings.PREFIX, user_id=user_id)
+
+def _url_for_user_active_threads(user_id):
+    return "{prefix}/users/{user_id}/active_threads".format(prefix=settings.PREFIX, user_id=user_id)

lms/lib/comment_client/utils.py

+import requests
+import json
+import settings
+
+def strip_none(dic):
+    return dict([(k, v) for k, v in dic.iteritems() if v is not None])
+
+def strip_blank(dic):
+    def _is_blank(v):
+        return isinstance(v, str) and len(v.strip()) == 0
+    return dict([(k, v) for k, v in dic.iteritems() if not _is_blank(v)])
+
+def extract(dic, keys):
+    if isinstance(keys, str):
+        return strip_none({keys: dic.get(keys)})
+    else:
+        return strip_none({k: dic.get(k) for k in keys})
+
+def merge_dict(dic1, dic2):
+    return dict(dic1.items() + dic2.items())
+    
+def perform_request(method, url, data_or_params=None, *args, **kwargs):
+    data_or_params['api_key'] = settings.API_KEY
+    if method in ['post', 'put', 'patch']:
+        response = requests.request(method, url, data=data_or_params)
+    else:
+        response = requests.request(method, url, params=data_or_params)
+    if 200 < response.status_code < 500:
+        raise CommentClientError(response.text)
+    elif response.status_code == 500:
+        raise CommentClientUnknownError(response.text)
+    else:
+        if kwargs.get("raw", False):
+            return response.text
+        else:
+            return json.loads(response.text)
+
+class CommentClientError(Exception):
+    def __init__(self, msg):
+        self.message = msg
+
+    def __str__(self):
+        return repr(self.message)
+
+class CommentClientUnknownError(CommentClientError):
+    pass

lms/lib/dogfood/views.py

@@ -24,7 +24,6 @@ from mitxmako.shortcuts import render_to_response, render_to_string
 import track.views
 from lxml import etree
 
-
 from courseware.module_render import make_track_function, ModuleSystem, get_module
 from courseware.models import StudentModule
 from multicourse import multicourse_settings