Merge branch 'master' into drupal-new

cms/djangoapps/contentstore/tests/test_contentstore.py

@@ -11,8 +11,10 @@ import json
 from fs.osfs import OSFS
 import copy
 from json import loads
+import traceback
 
 from django.contrib.auth.models import User
+from django.dispatch import Signal
 from contentstore.utils import get_modulestore
 
 from .utils import ModuleStoreTestCase, parse_json
@@ -215,13 +217,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         module_store = modulestore('direct')
         found = False
 
-        item = None
         items = module_store.get_items(['i4x', 'edX', 'full', 'poll_question', None, None])
         found = len(items) > 0
 
         self.assertTrue(found)
         # check that there's actually content in the 'question' field
-        self.assertGreater(len(items[0].question),0)
+        self.assertGreater(len(items[0].question), 0)
 
     def test_xlint_fails(self):
         err_cnt = perform_xlint('common/test/data', ['full'])
@@ -234,7 +235,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
 
         sequential = module_store.get_item(Location(['i4x', 'edX', 'full', 'sequential', 'Administrivia_and_Circuit_Elements', None]))
 
-        chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter','Week_1', None]))
+        chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter', 'Week_1', None]))
 
         # make sure the parent no longer points to the child object which was deleted
         self.assertTrue(sequential.location.url() in chapter.children)
@@ -252,7 +253,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
 
         self.assertFalse(found)
 
-        chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter','Week_1', None]))
+        chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter', 'Week_1', None]))
 
         # make sure the parent no longer points to the child object which was deleted
         self.assertFalse(sequential.location.url() in chapter.children)
@@ -275,7 +276,6 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         import_from_xml(modulestore(), 'common/test/data/', ['full'])
 
         module_store = modulestore('direct')
-        content_store = contentstore()
 
         source_location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012')
         course = module_store.get_item(source_location)
@@ -347,17 +347,44 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
 
     def test_export_course(self):
         module_store = modulestore('direct')
+        draft_store = modulestore('draft')
         content_store = contentstore()
 
         import_from_xml(module_store, 'common/test/data/', ['full'])
         location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012')
 
+        # get a vertical (and components in it) to put into 'draft'
+        vertical = module_store.get_item(Location(['i4x', 'edX', 'full',
+                                         'vertical', 'vertical_66', None]), depth=1)
+
+        draft_store.clone_item(vertical.location, vertical.location)
+
+        for child in vertical.get_children():
+            draft_store.clone_item(child.location, child.location)           
+
         root_dir = path(mkdtemp_clean())
 
+        # now create a private vertical
+        private_vertical = draft_store.clone_item(vertical.location,
+                                                  Location(['i4x', 'edX', 'full', 'vertical', 'a_private_vertical', None]))
+
+        # add private to list of children
+        sequential = module_store.get_item(Location(['i4x', 'edX', 'full',
+                                           'sequential', 'Administrivia_and_Circuit_Elements', None]))
+        private_location_no_draft = private_vertical.location._replace(revision=None)
+        module_store.update_children(sequential.location, sequential.children +
+                                     [private_location_no_draft.url()])
+
+        # read back the sequential, to make sure we have a pointer to 
+        sequential = module_store.get_item(Location(['i4x', 'edX', 'full',
+                                                     'sequential', 'Administrivia_and_Circuit_Elements', None]))
+
+        self.assertIn(private_location_no_draft.url(), sequential.children)
+
         print 'Exporting to tempdir = {0}'.format(root_dir)
 
         # export out to a tempdir
-        export_to_xml(module_store, content_store, location, root_dir, 'test_export')
+        export_to_xml(module_store, content_store, location, root_dir, 'test_export', draft_modulestore=draft_store)
 
         # check for static tabs
         self.verify_content_existence(module_store, root_dir, location, 'tabs', 'static_tab', '.html')
@@ -391,20 +418,36 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         delete_course(module_store, content_store, location)
 
         # reimport
-        import_from_xml(module_store, root_dir, ['test_export'])
+        import_from_xml(module_store, root_dir, ['test_export'], draft_store=draft_store)
 
         items = module_store.get_items(Location(['i4x', 'edX', 'full', 'vertical', None]))
         self.assertGreater(len(items), 0)
         for descriptor in items:
+            # don't try to look at private verticals. Right now we're running
+            # the service in non-draft aware
+            if getattr(descriptor, 'is_draft', False):
                 print "Checking {0}....".format(descriptor.location.url())
                 resp = self.client.get(reverse('edit_unit', kwargs={'location': descriptor.location.url()}))
                 self.assertEqual(resp.status_code, 200)
 
+        # verify that we have the content in the draft store as well
+        vertical = draft_store.get_item(Location(['i4x', 'edX', 'full',
+                                                  'vertical', 'vertical_66', None]), depth=1)
+
+        self.assertTrue(getattr(vertical, 'is_draft', False))
+        for child in vertical.get_children():
+            self.assertTrue(getattr(child, 'is_draft', False))
+
+        # verify that we have the private vertical
+        test_private_vertical = draft_store.get_item(Location(['i4x', 'edX', 'full',
+                                                               'vertical', 'vertical_66', None]))
+
+        self.assertTrue(getattr(test_private_vertical, 'is_draft', False))
+
         shutil.rmtree(root_dir)
 
     def test_course_handouts_rewrites(self):
         module_store = modulestore('direct')
-        content_store = contentstore()
 
         # import a test course
         import_from_xml(module_store, 'common/test/data/', ['full'])
@@ -440,8 +483,8 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
                                   'Administrivia_and_Circuit_Elements', None]) in course.system.module_data)
 
         # make sure we don't have a specific vertical which should be at depth=3
-        self.assertFalse(Location(['i4x', 'edX', 'full', 'vertical', 'vertical_58',
-            None]) in course.system.module_data)
+        self.assertFalse(Location(['i4x', 'edX', 'full', 'vertical', 'vertical_58', None])
+                         in course.system.module_data)
 
     def test_export_course_with_unknown_metadata(self):
         module_store = modulestore('direct')
@@ -468,10 +511,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
             export_to_xml(module_store, content_store, location, root_dir, 'test_export')
             exported = True
         except Exception:
+            print 'Exception thrown: {0}'.format(traceback.format_exc())
             pass
 
         self.assertTrue(exported)
 
+
 class ContentStoreTest(ModuleStoreTestCase):
     """
     Tests for the CMS ContentStore application.
@@ -748,6 +793,45 @@ class ContentStoreTest(ModuleStoreTestCase):
         # make sure we found the item (e.g. it didn't error while loading)
         self.assertTrue(did_load_item)
 
+    def test_forum_id_generation(self):
+        import_from_xml(modulestore(), 'common/test/data/', ['full'])
+        module_store = modulestore('direct')
+        new_component_location = Location('i4x', 'edX', 'full', 'discussion', 'new_component')
+        source_template_location = Location('i4x', 'edx', 'templates', 'discussion', 'Discussion_Tag')
+
+        # crate a new module and add it as a child to a vertical
+        module_store.clone_item(source_template_location, new_component_location)
+
+        new_discussion_item = module_store.get_item(new_component_location)
+
+        self.assertNotEquals(new_discussion_item.discussion_id, '$$GUID$$')
+
+    def test_update_modulestore_signal_did_fire(self):
+
+        import_from_xml(modulestore(), 'common/test/data/', ['full'])
+        module_store = modulestore('direct')
+
+        try:
+            module_store.modulestore_update_signal = Signal(providing_args=['modulestore', 'course_id', 'location'])
+
+            self.got_signal = False
+
+            def _signal_hander(modulestore=None, course_id=None, location=None, **kwargs):
+                self.got_signal = True
+
+            module_store.modulestore_update_signal.connect(_signal_hander)
+
+            new_component_location = Location('i4x', 'edX', 'full', 'html', 'new_component')
+            source_template_location = Location('i4x', 'edx', 'templates', 'html', 'Blank_HTML_Page')
+
+            # crate a new module
+            module_store.clone_item(source_template_location, new_component_location)
+
+        finally:
+            module_store.modulestore_update_signal = None
+
+        self.assertTrue(self.got_signal)
+
     def test_metadata_inheritance(self):
         import_from_xml(modulestore(), 'common/test/data/', ['full'])
 

cms/djangoapps/contentstore/utils.py

@@ -11,6 +11,7 @@ DIRECT_ONLY_CATEGORIES = ['course', 'chapter', 'sequential', 'about', 'static_ta
 #In order to instantiate an open ended tab automatically, need to have this data
 OPEN_ENDED_PANEL = {"name" : "Open Ended Panel", "type" : "open_ended"}
 
+
 def get_modulestore(location):
     """
     Returns the correct modulestore to use for modifying the specified location

cms/djangoapps/contentstore/views.py

@@ -1586,7 +1586,8 @@ def import_course(request, org, course, name):
                 shutil.move(r / fname, course_dir)
 
         module_store, course_items = import_from_xml(modulestore('direct'), settings.GITHUB_REPO_ROOT,
-            [course_subdir], load_error_modules=False, static_content_store=contentstore(), target_location_namespace=Location(location))
+            [course_subdir], load_error_modules=False, static_content_store=contentstore(), 
+            target_location_namespace=Location(location), draft_store=modulestore())
 
         # we can blow this away when we're done importing.
         shutil.rmtree(course_dir)
@@ -1620,8 +1621,8 @@ def generate_export_course(request, org, course, name):
 
     logging.debug('root = {0}'.format(root_dir))
 
-    export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name)
-    # filename = root_dir / name + '.tar.gz'
+    export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name, modulestore())
+    #filename = root_dir / name + '.tar.gz'
 
     logging.debug('tar file being generated at {0}'.format(export_file.name))
     tf = tarfile.open(name=export_file.name, mode='w:gz')

cms/one_time_startup.py

 from dogapi import dog_http_api, dog_stats_api
 from django.conf import settings
 from xmodule.modulestore.django import modulestore
+from django.dispatch import Signal
 from request_cache.middleware import RequestCache
 
-from django.core.cache import get_cache, InvalidCacheBackendError
+from django.core.cache import get_cache
 
 cache = get_cache('mongo_metadata_inheritance')
 for store_name in settings.MODULESTORE:
@@ -11,6 +12,8 @@ for store_name in settings.MODULESTORE:
     store.metadata_inheritance_cache_subsystem = cache
     store.request_cache = RequestCache.get_request_cache()
 
+    modulestore_update_signal = Signal(providing_args=['modulestore', 'course_id', 'location'])
+    store.modulestore_update_signal = modulestore_update_signal
 if hasattr(settings, 'DATADOG_API'):
     dog_http_api.api_key = settings.DATADOG_API
     dog_stats_api.start(api_key=settings.DATADOG_API, statsd=True)

cms/static/js/base.js

@@ -225,7 +225,6 @@ function toggleSections(e) {
 function editSectionPublishDate(e) {
     e.preventDefault();
     $modal = $('.edit-subsection-publish-settings').show();
-    $modal = $('.edit-subsection-publish-settings').show();
     $modal.attr('data-id', $(this).attr('data-id'));
     $modal.find('.start-date').val($(this).attr('data-date'));
     $modal.find('.start-time').val($(this).attr('data-time'));

cms/static/sass/elements/_controls.scss

@@ -97,7 +97,7 @@
   color: $blue;
 
   &:hover, &:active {
-    background: $blue-l3;
+    background: $blue-l4;
     color: $blue-s2;
   }
 

cms/static/sass/elements/_forms.scss

@@ -8,11 +8,11 @@ input[type="password"],
 textarea.text {
   padding: 6px 8px 8px;
   @include box-sizing(border-box);
-  border: 1px solid $mediumGrey;
+  border: 1px solid $gray-l2;
   border-radius: 2px;
-  @include linear-gradient($lightGrey, tint($lightGrey, 90%));
-  background-color: $lightGrey;
-  @include box-shadow(0 1px 2px rgba(0, 0, 0, .1) inset);
+  @include linear-gradient($gray-l5, $white);
+  background-color: $gray-l5;
+  @include box-shadow(inset 0 1px 2px $shadow-l1);
   font-family: 'Open Sans', sans-serif;
   font-size: 11px;
   color: $baseFontColor;
@@ -21,7 +21,7 @@ textarea.text {
   &::-webkit-input-placeholder,
   &:-moz-placeholder,
   &:-ms-input-placeholder {
-      color: #979faf;
+      color: $gray-l2;
   }
 
   &:focus {
@@ -30,7 +30,72 @@ textarea.text {
   }
 }
 
-// forms - specific
+// ====================
+
+// forms - fields - not editable
+.field.is-not-editable {
+
+  & label.is-focused {
+   color: $gray-d2;
+  }
+
+  label, input, textarea {
+    pointer-events: none;
+  }
+}
+
+// ====================
+
+// field with error
+.field.error {
+
+  input, textarea {
+    border-color: $red;
+  }
+}
+
+// ====================
+
+// forms - additional UI
+form {
+
+  .note {
+    @include box-sizing(border-box);
+
+    .title {
+
+    }
+
+    .copy {
+
+    }
+
+    // note with actions
+    &.has-actions {
+      @include clearfix();
+
+      .title {
+
+      }
+
+      .copy {
+
+      }
+
+      .list-actions {
+
+      }
+    }
+  }
+
+  .note-promotion {
+
+  }
+}
+
+// ====================
+
+// forms - grandfathered
 input.search {
   padding: 6px 15px 8px 30px;
   @include box-sizing(border-box);

cms/static/sass/views/_account.scss

@@ -4,7 +4,7 @@
 body.signup, body.signin {
 
   .wrapper-content {
-    margin: 0;
+    margin: ($baseline*1.5) 0 0 0;
     padding: 0 $baseline;
     position: relative;
     width: 100%;

cms/static/sass/views/_settings.scss

@@ -147,7 +147,7 @@ body.course.settings {
         }
 
         label {
-          @include font-size(14);
+          @extend .t-copy-sub1;
           @include transition(color, 0.15s, ease-in-out);
           margin: 0 0 ($baseline/4) 0;
           font-weight: 400;
@@ -238,33 +238,36 @@ body.course.settings {
       }
     }
 
-    // not editable fields
-    .field.is-not-editable {
-     
-      & label.is-focused {
-       color: $gray-d2;
-      }
-    }
-
-    // field with error
-    .field.error {
-
-      input, textarea {
-        border-color: $red;
-      }
-    }
-     
     // specific fields - basic
     &.basic {
 
       .list-input {
         @include clearfix();
+        padding: 0 ($baseline/2);
 
         .field {
           margin-bottom: 0;
         }
       }
 
+      // course details that should appear more like content than elements to change
+      .field.is-not-editable {
+
+        label {
+
+        }
+
+        input, textarea {
+          @extend .t-copy-lead1;
+          @include box-shadow(none);
+          border: none;
+          background: none;
+          padding: 0;
+          margin: 0;
+          font-weight: 600;
+        }
+      }
+
       #field-course-organization {
         float: left;
         width: flex-grid(2, 9);
@@ -281,6 +284,58 @@ body.course.settings {
         float: left;
         width: flex-grid(5, 9);
       }
+
+      // course link note
+      .note-promotion-courseURL {
+        @include box-shadow(0 2px 1px $shadow-l1);
+        @include border-radius(($baseline/5));
+        margin-top: ($baseline*1.5);
+        border: 1px solid $gray-l2;
+        padding: ($baseline/2) 0 0 0;
+
+        .title {
+          @extend .t-copy-sub1;
+          margin: 0 0 ($baseline/10) 0;
+          padding: 0 ($baseline/2);
+
+          .tip {
+            display: inline;
+            margin-left: ($baseline/4);
+          }
+        }
+
+        .copy {
+          padding: 0 ($baseline/2) ($baseline/2) ($baseline/2);
+
+          .link-courseURL {
+            @extend .t-copy-lead1;
+
+            &:hover {
+
+            }
+          }
+        }
+
+        .list-actions {
+          @include box-shadow(inset 0 1px 1px $shadow-l1);
+          border-top: 1px solid $gray-l2;
+          padding:  ($baseline/2);
+          background: $gray-l5;
+
+          .action-primary {
+            @include blue-button();
+            @include font-size(13);
+            font-weight: 600;
+
+            .icon {
+              @extend .t-icon;
+              @include font-size(16);
+              display: inline-block;
+              vertical-align: middle;
+            }
+          }
+        }
+      }
     }
 
     // specific fields - schedule

cms/templates/settings.html

@@ -83,7 +83,19 @@ from contentstore import utils
               <input title="This field is disabled: this information cannot be changed." type="text" class="long" id="course-name" value="[Course Name]" readonly />
             </li>
           </ol>
-          <span class="tip tip-stacked">These are used in <a rel="external" href="${utils.get_lms_link_for_about_page(course_location)}" />your course URL</a>, and cannot be changed</span>
+
+          <div class="note note-promotion note-promotion-courseURL has-actions">
+            <h3 class="title">Course Summary Page <span class="tip">(for student enrollment and access)</span></h3>
+            <div class="copy">
+              <p><a class="link-courseURL" rel="external" href="${utils.get_lms_link_for_about_page(course_location)}" />${utils.get_lms_link_for_about_page(course_location)}</a></p>
+            </div>
+
+            <ul class="list-actions">
+              <li class="action-item">
+                <a title="Send a note to students via email" href="mailto:john.doe@gmail.com?Subject=Enroll%20in%20COURSENAME&body=Hi,%20COURSENAME,%20provided%20by%20edX,%20is%20almost%20ready%20to%20begin.%20Please%20enroll%20for%20this%20course%20at%20${utils.get_lms_link_for_about_page(course_location)}." class="action action-primary"><i class="ss-icon icon ss-symbolicons-standard icon icon-inline icon-announcement">&#x2709;</i> Send an invitation to your students</a>
+              </li>
+            </ul>
+          </div>
         </section>
 
         <hr class="divide" />
@@ -167,7 +179,7 @@ from contentstore import utils
             <li class="field text" id="field-course-overview">
               <label for="course-overview">Course Overview</label>
               <textarea class="tinymce text-editor" id="course-overview"></textarea>
-              <span class="tip tip-stacked">Introductions, prerequisites, FAQs that are used on <a href="${utils.get_lms_link_for_about_page(course_location)}">your course summary page</a></span>
+              <span class="tip tip-stacked">Introductions, prerequisites, FAQs that are used on <a class="link-courseURL" rel="external" href="${utils.get_lms_link_for_about_page(course_location)}">your course summary page</a></span>
             </li>
 
             <li class="field video" id="field-course-introduction-video">
@@ -175,7 +187,6 @@ from contentstore import utils
               <div class="input input-existing">
                 <div class="current current-course-introduction-video">
                   <iframe width="618" height="350" src="" frameborder="0" allowfullscreen></iframe>
-
                 </div>
                 <div class="actions">
                   <a href="#" class="remove-item remove-course-introduction-video remove-video-data"><span class="delete-icon"></span> Delete Current Video</a>

cms/templates/widgets/metadata-only-edit.html

+<%include file="metadata-edit.html" />

cms/templates/widgets/source-edit.html

@@ -12,7 +12,7 @@
 
     <form id="hls-form" enctype="multipart/form-data">
       <section class="source-edit">
-          <textarea name="" data-metadata-name="source_code" class="source-edit-box hls-data" rows="8" cols="40">${metadata['source_code']|h}</textarea>
+          <textarea name="" data-metadata-name="source_code" class="source-edit-box hls-data" rows="8" cols="40">${editable_metadata_fields['source_code']|h}</textarea>
       </section>
       <div class="submit">
         <button type="reset" class="hls-compile">Save &amp; Compile to edX XML</button>

common/lib/xmodule/xmodule/capa_module.py

@@ -99,6 +99,7 @@ class CapaFields(object):
     seed = StringyInteger(help="Random seed for this student", scope=Scope.user_state)
     weight = StringyFloat(help="How much to weight this problem by", scope=Scope.settings)
     markdown = String(help="Markdown source of this module", scope=Scope.settings)
+    source_code = String(help="Source code for LaTeX and Word problems. This feature is not well-supported.", scope=Scope.settings)
 
 
 class CapaModule(CapaFields, XModule):

common/lib/xmodule/xmodule/discussion_module.py

@@ -3,6 +3,7 @@ from pkg_resources import resource_string, resource_listdir
 
 from xmodule.x_module import XModule
 from xmodule.raw_module import RawDescriptor
+from xmodule.editing_module import MetadataOnlyEditingDescriptor
 from xblock.core import String, Scope
 
 
@@ -28,7 +29,7 @@ class DiscussionModule(DiscussionFields, XModule):
         return self.system.render_template('discussion/_discussion_module.html', context)
 
 
-class DiscussionDescriptor(DiscussionFields, RawDescriptor):
+class DiscussionDescriptor(DiscussionFields, MetadataOnlyEditingDescriptor, RawDescriptor):
     module_class = DiscussionModule
     template_dir_name = "discussion"
 

common/lib/xmodule/xmodule/editing_module.py

@@ -41,6 +41,18 @@ class XMLEditingDescriptor(EditingDescriptor):
     js_module_name = "XMLEditingDescriptor"
 
 
+class MetadataOnlyEditingDescriptor(EditingDescriptor):
+    """
+    Module which only provides an editing interface for the metadata, it does
+    not expose a UI for editing the module data
+    """
+
+    js = {'coffee': [resource_string(__name__, 'js/src/raw/edit/metadata-only.coffee')]}
+    js_module_name = "MetadataOnlyEditingDescriptor"
+
+    mako_template = "widgets/metadata-only-edit.html"
+
+
 class JSONEditingDescriptor(EditingDescriptor):
     """
     Module that provides a raw editing view of its data as XML. It does not perform

common/lib/xmodule/xmodule/html_module.py

@@ -118,8 +118,8 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor):
                 with system.resources_fs.open(filepath) as file:
                     html = file.read().decode('utf-8')
                     # Log a warning if we can't parse the file, but don't error
-                    if not check_html(html):
-                        msg = "Couldn't parse html in {0}.".format(filepath)
+                    if not check_html(html) and len(html) > 0:
+                        msg = "Couldn't parse html in {0}, content = {1}".format(filepath, html)
                         log.warning(msg)
                         system.error_tracker("Warning: " + msg)
 
@@ -156,7 +156,8 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor):
 
         resource_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True)
         with resource_fs.open(filepath, 'w') as file:
-            file.write(self.data.encode('utf-8'))
+            html_data = self.data.encode('utf-8')
+            file.write(html_data)
 
         # write out the relative name
         relname = path(pathname).basename()

common/lib/xmodule/xmodule/js/src/raw/edit/metadata-only.coffee

+class @MetadataOnlyEditingDescriptor extends XModule.Descriptor
+  constructor: (@element) ->
+
+  save: ->
+    data: null

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -252,7 +252,6 @@ class Location(_LocationBase):
     def __repr__(self):
         return "Location%s" % repr(tuple(self))
 
-
     @property
     def course_id(self):
         """Return the ID of the Course that this item belongs to by looking
@@ -414,7 +413,6 @@ class ModuleStore(object):
         return courses
 
 
-
 class ModuleStoreBase(ModuleStore):
     '''
     Implement interface functionality that can be shared.
@@ -425,6 +423,7 @@ class ModuleStoreBase(ModuleStore):
         '''
         self._location_errors = {}    # location -> ErrorLog
         self.metadata_inheritance_cache = None
+        self.modulestore_update_signal = None  # can be set by runtime to route notifications of datastore changes
 
     def _get_errorlog(self, location):
         """

common/lib/xmodule/xmodule/modulestore/draft.py

@@ -3,7 +3,6 @@ from datetime import datetime
 from . import ModuleStoreBase, Location, namedtuple_to_son
 from .exceptions import ItemNotFoundError
 from .inheritance import own_metadata
-import logging
 
 DRAFT = 'draft'
 
@@ -107,7 +106,7 @@ class DraftModuleStore(ModuleStoreBase):
         """
         return wrap_draft(super(DraftModuleStore, self).clone_item(source, as_draft(location)))
 
-    def update_item(self, location, data):
+    def update_item(self, location, data, allow_not_found=False):
         """
         Set the data in the item specified by the location to
         data
@@ -116,9 +115,13 @@ class DraftModuleStore(ModuleStoreBase):
         data: A nested dictionary of problem data
         """
         draft_loc = as_draft(location)
+        try:
             draft_item = self.get_item(location)
             if not getattr(draft_item, 'is_draft', False):
                 self.clone_item(location, draft_loc)
+        except ItemNotFoundError, e:
+            if not allow_not_found:
+                raise e
 
         return super(DraftModuleStore, self).update_item(draft_loc, data)
 
@@ -164,7 +167,6 @@ class DraftModuleStore(ModuleStoreBase):
         """
         return super(DraftModuleStore, self).delete_item(as_draft(location))
 
-
     def get_parent_locations(self, location, course_id):
         '''Find all locations that are the parents of this location.  Needed
         for path_to_location().
@@ -178,6 +180,7 @@ class DraftModuleStore(ModuleStoreBase):
         Save a current draft to the underlying modulestore
         """
         draft = self.get_item(location)
+
         draft.cms.published_date = datetime.utcnow()
         draft.cms.published_by = published_by_id
         super(DraftModuleStore, self).update_item(location, draft._model_data._kvs._data)

common/lib/xmodule/xmodule/modulestore/mongo.py

@@ -9,6 +9,7 @@ from itertools import repeat
 from path import path
 from datetime import datetime
 from operator import attrgetter
+from uuid import uuid4
 
 from importlib import import_module
 from xmodule.errortracker import null_error_tracker, exc_info_to_str
@@ -30,6 +31,10 @@ log = logging.getLogger(__name__)
 # there is only one revision for each item. Once we start versioning inside the CMS,
 # that assumption will have to change
 
+def get_course_id_no_run(location):
+    '''
+    '''
+    return "/".join([location.org, location.course])
 
 class MongoKeyValueStore(KeyValueStore):
     """
@@ -541,8 +546,15 @@ class MongoModuleStore(ModuleStoreBase):
         Clone a new item that is a copy of the item at the location `source`
         and writes it to `location`
         """
+        item = None
         try:
             source_item = self.collection.find_one(location_to_query(source))
+
+            # allow for some programmatically generated substitutions in metadata, e.g. Discussion_id's should be auto-generated
+            for key in source_item['metadata'].keys():
+                if source_item['metadata'][key] == '$$GUID$$':
+                    source_item['metadata'][key] = uuid4().hex
+
             source_item['_id'] = Location(location).dict()
             self.collection.insert(
                 source_item,
@@ -566,12 +578,19 @@ class MongoModuleStore(ModuleStoreBase):
                 course.tabs = existing_tabs
                 self.update_metadata(course.location, course._model_data._kvs._metadata)
 
-            return item
         except pymongo.errors.DuplicateKeyError:
             raise DuplicateItemError(location)
 
         # recompute (and update) the metadata inheritance tree which is cached
         self.refresh_cached_metadata_inheritance_tree(Location(location))
+        self.fire_updated_modulestore_signal(get_course_id_no_run(Location(location)), Location(location))
+
+        return item
+
+    def fire_updated_modulestore_signal(self, course_id, location):
+        if self.modulestore_update_signal is not None:
+            self.modulestore_update_signal.send(self, modulestore=self, course_id=course_id,
+                                                location=location)
 
     def get_course_for_item(self, location, depth=0):
         '''
@@ -643,6 +662,8 @@ class MongoModuleStore(ModuleStoreBase):
         self._update_single_item(location, {'definition.children': children})
         # recompute (and update) the metadata inheritance tree which is cached
         self.refresh_cached_metadata_inheritance_tree(Location(location))
+        # fire signal that we've written to DB
+        self.fire_updated_modulestore_signal(get_course_id_no_run(Location(location)), Location(location))
 
     def update_metadata(self, location, metadata):
         """
@@ -669,6 +690,7 @@ class MongoModuleStore(ModuleStoreBase):
         self._update_single_item(location, {'metadata': metadata})
         # recompute (and update) the metadata inheritance tree which is cached
         self.refresh_cached_metadata_inheritance_tree(loc)
+        self.fire_updated_modulestore_signal(get_course_id_no_run(Location(location)), Location(location))
 
     def delete_item(self, location):
         """
@@ -692,6 +714,7 @@ class MongoModuleStore(ModuleStoreBase):
             safe=self.collection.safe)
         # recompute (and update) the metadata inheritance tree which is cached
         self.refresh_cached_metadata_inheritance_tree(Location(location))
+        self.fire_updated_modulestore_signal(get_course_id_no_run(Location(location)), Location(location))
 
     def get_parent_locations(self, location, course_id):
         '''Find all locations that are the parents of this location in this

common/lib/xmodule/xmodule/modulestore/xml_exporter.py

 import logging
 from xmodule.modulestore import Location
-from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.inheritance import own_metadata
 from fs.osfs import OSFS
 from json import dumps
 
 
-def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir):
+def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir, draft_modulestore=None):
 
     course = modulestore.get_item(course_location)
 
@@ -40,6 +39,24 @@ def export_to_xml(modulestore, contentstore, course_location, root_dir, course_d
         policy = {'course/' + course.location.name: own_metadata(course)}
         course_policy.write(dumps(policy))
 
+    # export draft content
+    # NOTE: this code assumes that verticals are the top most draftable container
+    # should we change the application, then this assumption will no longer
+    # be valid
+    if draft_modulestore is not None:
+        draft_verticals = draft_modulestore.get_items([None, course_location.org, course_location.course,
+                                                       'vertical', None, 'draft'])
+        if len(draft_verticals) > 0:
+            draft_course_dir = export_fs.makeopendir('drafts')
+            for draft_vertical in draft_verticals:
+                parent_locs = draft_modulestore.get_parent_locations(draft_vertical.location, course.location.course_id)
+                logging.debug('parent_locs = {0}'.format(parent_locs))
+                draft_vertical.xml_attributes['parent_sequential_url'] = Location(parent_locs[0]).url()
+                sequential = modulestore.get_item(Location(parent_locs[0]))
+                index = sequential.children.index(draft_vertical.location.url())
+                draft_vertical.xml_attributes['index_in_children_list'] = str(index)
+                draft_vertical.export_to_xml(draft_course_dir)
+
 
 def export_extra_content(export_fs, modulestore, course_location, category_type, dirname, file_suffix=''):
     query_loc = Location('i4x', course_location.org, course_location.course, category_type, None)

common/lib/xmodule/xmodule/templates/discussion/default.yaml

@@ -2,8 +2,8 @@
 metadata:
     display_name: Discussion Tag
     for: Topic-Level Student-Visible Label 
-    id: 6002x_group_discussion_by_this 
+    id: $$GUID$$
     discussion_category: Week 1
 data: |
-  <discussion for="Topic-Level Student-Visible Label" id="6002x_group_discussion_by_this" discussion_category="Week 1" />
+  <discussion />
 children: []

common/lib/xmodule/xmodule/templates/html/latex_html.yaml

 ---
 metadata:
     display_name: E-text Written in LaTeX
-    source_processor_url: https://qisx.mit.edu:5443/latex2edx
     source_code: |
         \subsection{Example of E-text in LaTeX}
 

common/lib/xmodule/xmodule/templates/problem/latex_problem.yaml

 ---
 metadata:
     display_name: Problem Written in LaTeX
-    source_processor_url: https://studio-input-filter.mitx.mit.edu/latex2edx
     source_code: |
         % Nearly any kind of edX problem can be authored using Latex as
         % the source language.  Write latex as usual, including equations.  The

common/lib/xmodule/xmodule/templates/problem/problem_with_hint.yaml

 ---
 metadata:
     display_name: Problem with Adaptive Hint
-    source_processor_url: https://qisx.mit.edu:5443/latex2edx
     source_code: |
         \subsection{Problem With Adaptive Hint}
 

common/lib/xmodule/xmodule/x_module.py

@@ -340,7 +340,9 @@ class XModuleDescriptor(XModuleFields, HTMLSnippet, ResourceTemplates, XBlock):
 
     # cdodge: this is a list of metadata names which are 'system' metadata
     # and should not be edited by an end-user
-    system_metadata_fields = ['data_dir', 'published_date', 'published_by', 'is_draft', 'xml_attributes']
+
+    system_metadata_fields = ['data_dir', 'published_date', 'published_by', 'is_draft', 
+        'discussion_id', 'xml_attributes']
 
     # A list of descriptor attributes that must be equal for the descriptors to
     # be equal

common/lib/xmodule/xmodule/xml_module.py

@@ -110,8 +110,7 @@ class XmlDescriptor(XModuleDescriptor):
         'name', 'slug')
 
     metadata_to_strip = ('data_dir',
-            # cdodge: @TODO: We need to figure out a way to export out 'tabs' and 'grading_policy' which is on the course
-            'tabs', 'grading_policy', 'is_draft', 'published_by', 'published_date',
+            'tabs', 'grading_policy', 'published_by', 'published_date',
             'discussion_blackouts', 'testcenter_info',
            # VS[compat] -- remove the below attrs once everything is in the CMS
            'course', 'org', 'url_name', 'filename',
@@ -135,7 +134,7 @@ class XmlDescriptor(XModuleDescriptor):
         'graded': bool_map,
         'hide_progress_tab': bool_map,
         'allow_anonymous': bool_map,
-        'allow_anonymous_to_peers': bool_map
+        'allow_anonymous_to_peers': bool_map,
     }
 
 

lms/djangoapps/courseware/tests/test_login.py

 from django.test import TestCase
 from django.test.client import Client
 from django.core.urlresolvers import reverse
-from django.contrib.auth.models import User
-from student.models import Registration, UserProfile
+from factories import UserFactory, RegistrationFactory, UserProfileFactory
 import json
 
+
 class LoginTest(TestCase):
     '''
     Test student.views.login_user() view
     '''
 
     def setUp(self):
-
         # Create one user and save it to the database
-        self.user = User.objects.create_user('test', 'test@edx.org', 'test_password')
-        self.user.is_active = True
+        self.user = UserFactory.build(username='test', email='test@edx.org')
+        self.user.set_password('test_password')
         self.user.save()
 
         # Create a registration for the user
-        Registration().register(self.user)
+        registration = RegistrationFactory(user=self.user)
+        registration.register(self.user)
+        registration.activate()
 
         # Create a profile for the user
-        UserProfile(user=self.user).save()
+        UserProfileFactory(user=self.user)
 
         # Create the test client
         self.client = Client()
@@ -42,7 +43,6 @@ class LoginTest(TestCase):
         response = self._login_response(unicode_email, 'test_password')
         self._assert_response(response, success=True)
 
-
     def test_login_fail_no_user_exists(self):
         response = self._login_response('not_a_user@edx.org', 'test_password')
         self._assert_response(response, success=False,
@@ -54,7 +54,6 @@ class LoginTest(TestCase):
                               value='Email or password is incorrect')
 
     def test_login_not_activated(self):
-
         # De-activate the user
         self.user.is_active = False
         self.user.save()
@@ -64,7 +63,6 @@ class LoginTest(TestCase):
         self._assert_response(response, success=False,
                               value="This account has not been activated")
 
-
     def test_login_unicode_email(self):
         unicode_email = u'test@edx.org' + unichr(40960)
         response = self._login_response(unicode_email, 'test_password')

lms/djangoapps/django_comment_client/management/commands/seed_permissions_roles.py

 from django.core.management.base import BaseCommand, CommandError
-from django_comment_client.models import Permission, Role
+from django_comment_client.models import Role
 
 
 class Command(BaseCommand):
@@ -12,6 +12,7 @@ class Command(BaseCommand):
         if len(args) > 1:
             raise CommandError("Too many arguments")
         course_id = args[0]
+
         administrator_role = Role.objects.get_or_create(name="Administrator", course_id=course_id)[0]
         moderator_role = Role.objects.get_or_create(name="Moderator", course_id=course_id)[0]
         community_ta_role = Role.objects.get_or_create(name="Community TA", course_id=course_id)[0]

lms/djangoapps/django_comment_client/tests/mock_cs_server/mock_cs_server.py

+from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
+import json
+from logging import getLogger
+logger = getLogger(__name__)
+
+
+class MockCommentServiceRequestHandler(BaseHTTPRequestHandler):
+    '''
+    A handler for Comment Service POST requests.
+    '''
+    protocol = "HTTP/1.0"
+
+    def do_POST(self):
+        '''
+        Handle a POST request from the client
+        Used by the APIs for comment threads, commentables, comments,
+        subscriptions, commentables, users
+        '''
+        # Retrieve the POST data into a dict.
+        # It should have been sent in json format
+        length = int(self.headers.getheader('content-length'))
+        data_string = self.rfile.read(length)
+        post_dict = json.loads(data_string)
+
+        # Log the request
+        logger.debug("Comment Service received POST request %s to path %s" %
+                    (json.dumps(post_dict), self.path))
+
+        # Every good post has at least an API key
+        if 'api_key' in post_dict:
+            response = self.server._response_str
+            # Log the response
+            logger.debug("Comment Service: sending response %s" % json.dumps(response))
+
+            # Send a response back to the client
+            self.send_response(200)
+            self.send_header('Content-type', 'application/json')
+            self.end_headers()
+            self.wfile.write(response)
+
+        else:
+            # Respond with failure
+            self.send_response(500, 'Bad Request: does not contain API key')
+            self.send_header('Content-type', 'text/plain')
+            self.end_headers()
+            return False
+
+
+class MockCommentServiceServer(HTTPServer):
+    '''
+    A mock Comment Service server that responds
+    to POST requests to localhost.
+    '''
+    def __init__(self, port_num,
+                 response={'username': 'new', 'external_id': 1}):
+        '''
+        Initialize the mock Comment Service server instance.
+        *port_num* is the localhost port to listen to
+        *response* is a dictionary that will be JSON-serialized
+            and sent in response to comment service requests.
+        '''
+        self._response_str = json.dumps(response)
+
+        handler = MockCommentServiceRequestHandler
+        address = ('', port_num)
+        HTTPServer.__init__(self, address, handler)
+
+    def shutdown(self):
+        '''
+        Stop the server and free up the port
+        '''
+        # First call superclass shutdown()
+        HTTPServer.shutdown(self)
+
+        # We also need to manually close the socket
+        self.socket.close()

lms/djangoapps/django_comment_client/tests/mock_cs_server/test_mock_cs_server.py

+import unittest
+import threading
+import json
+import urllib2
+from mock_cs_server import MockCommentServiceServer
+from nose.plugins.skip import SkipTest
+
+
+class MockCommentServiceServerTest(unittest.TestCase):
+    '''
+    A mock version of the Comment Service server that listens on a local
+    port and responds with pre-defined grade messages.
+    '''
+
+    def setUp(self):
+        # This is a test of the test setup,
+        # so it does not need to run as part of the unit test suite
+        # You can re-enable it by commenting out the line below
+        raise SkipTest
+
+        # Create the server
+        server_port = 4567
+        self.server_url = 'http://127.0.0.1:%d' % server_port
+
+        # Start up the server and tell it that by default it should
+        # return this as its json response
+        self.expected_response = {'username': 'user100', 'external_id': '4'}
+        self.server = MockCommentServiceServer(port_num=server_port,
+                                               response=self.expected_response)
+
+        # Start the server in a separate daemon thread
+        server_thread = threading.Thread(target=self.server.serve_forever)
+        server_thread.daemon = True
+        server_thread.start()
+
+    def tearDown(self):
+        # Stop the server, freeing up the port
+        self.server.shutdown()
+
+    def test_new_user_request(self):
+        """
+        Test the mock comment service using an example
+        of how you would create a new user
+        """
+        # Send a request
+        values = {'username': u'user100', 'api_key': 'TEST_API_KEY',
+                  'external_id': '4', 'email': u'user100@edx.org'}
+        data = json.dumps(values)
+        headers = {'Content-Type': 'application/json', 'Content-Length': len(data)}
+        req = urllib2.Request(self.server_url + '/api/v1/users/4', data, headers)
+
+        # Send the request to the mock cs server
+        response = urllib2.urlopen(req)
+
+        # Receive the reply from the mock cs server
+        response_dict = json.loads(response.read())
+
+        # You should have received the response specified in the setup above
+        self.assertEqual(response_dict, self.expected_response)

lms/djangoapps/django_comment_client/utils.py

@@ -146,28 +146,16 @@ def sort_map_entries(category_map):
 
 
 def initialize_discussion_info(course):
-
     global _DISCUSSIONINFO
 
-    # only cache in-memory discussion information for 10 minutes
-    # this is because we need a short-term hack fix for
-    # mongo-backed courseware whereby new discussion modules can be added
-    # without LMS service restart
-
-    if _DISCUSSIONINFO[course.id]:
-        timestamp = _DISCUSSIONINFO[course.id].get('timestamp', datetime.now())
-        age = datetime.now() - timestamp
-        # expire every 5 minutes
-        if age.seconds < 300:
-            return
-
     course_id = course.id
 
     discussion_id_map = {}
     unexpanded_category_map = defaultdict(list)
 
     # get all discussion models within this course_id
-    all_modules = modulestore().get_items(['i4x', course.location.org, course.location.course, 'discussion', None], course_id=course_id)
+    all_modules = modulestore().get_items(['i4x', course.location.org, course.location.course,
+        'discussion', None], course_id=course_id)
 
     for module in all_modules:
         skip_module = False