Merge pull request #1840 from MITx/fix/cdodge/export-draft-modules

Fix/cdodge/export draft modules

cms/djangoapps/contentstore/tests/test_contentstore.py

@@ -11,6 +11,7 @@ import json
 from fs.osfs import OSFS
 import copy
 from json import loads
+import traceback
 
 from django.contrib.auth.models import User
 from contentstore.utils import get_modulestore
@@ -215,13 +216,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         module_store = modulestore('direct')
         found = False
 
-        item = None
         items = module_store.get_items(['i4x', 'edX', 'full', 'poll_question', None, None])
         found = len(items) > 0
 
         self.assertTrue(found)
         # check that there's actually content in the 'question' field
-        self.assertGreater(len(items[0].question),0)
+        self.assertGreater(len(items[0].question), 0)
 
     def test_xlint_fails(self):
         err_cnt = perform_xlint('common/test/data', ['full'])
@@ -234,7 +234,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
 
         sequential = module_store.get_item(Location(['i4x', 'edX', 'full', 'sequential', 'Administrivia_and_Circuit_Elements', None]))
 
-        chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter','Week_1', None]))
+        chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter', 'Week_1', None]))
 
         # make sure the parent no longer points to the child object which was deleted
         self.assertTrue(sequential.location.url() in chapter.children)
@@ -252,7 +252,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
 
         self.assertFalse(found)
 
-        chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter','Week_1', None]))
+        chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter', 'Week_1', None]))
 
         # make sure the parent no longer points to the child object which was deleted
         self.assertFalse(sequential.location.url() in chapter.children)
@@ -275,7 +275,6 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         import_from_xml(modulestore(), 'common/test/data/', ['full'])
 
         module_store = modulestore('direct')
-        content_store = contentstore()
 
         source_location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012')
         course = module_store.get_item(source_location)
@@ -347,17 +346,44 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
 
     def test_export_course(self):
         module_store = modulestore('direct')
+        draft_store = modulestore('draft')
         content_store = contentstore()
 
         import_from_xml(module_store, 'common/test/data/', ['full'])
         location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012')
 
+        # get a vertical (and components in it) to put into 'draft'
+        vertical = module_store.get_item(Location(['i4x', 'edX', 'full',
+                                         'vertical', 'vertical_66', None]), depth=1)
+
+        draft_store.clone_item(vertical.location, vertical.location)
+
+        for child in vertical.get_children():
+            draft_store.clone_item(child.location, child.location)           
+
         root_dir = path(mkdtemp_clean())
 
+        # now create a private vertical
+        private_vertical = draft_store.clone_item(vertical.location,
+                                                  Location(['i4x', 'edX', 'full', 'vertical', 'a_private_vertical', None]))
+
+        # add private to list of children
+        sequential = module_store.get_item(Location(['i4x', 'edX', 'full',
+                                           'sequential', 'Administrivia_and_Circuit_Elements', None]))
+        private_location_no_draft = private_vertical.location._replace(revision=None)
+        module_store.update_children(sequential.location, sequential.children +
+                                     [private_location_no_draft.url()])
+
+        # read back the sequential, to make sure we have a pointer to 
+        sequential = module_store.get_item(Location(['i4x', 'edX', 'full',
+                                                     'sequential', 'Administrivia_and_Circuit_Elements', None]))
+
+        self.assertIn(private_location_no_draft.url(), sequential.children)
+
         print 'Exporting to tempdir = {0}'.format(root_dir)
 
         # export out to a tempdir
-        export_to_xml(module_store, content_store, location, root_dir, 'test_export')
+        export_to_xml(module_store, content_store, location, root_dir, 'test_export', draft_modulestore=draft_store)
 
         # check for static tabs
         self.verify_content_existence(module_store, root_dir, location, 'tabs', 'static_tab', '.html')
@@ -391,20 +417,36 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         delete_course(module_store, content_store, location)
 
         # reimport
-        import_from_xml(module_store, root_dir, ['test_export'])
+        import_from_xml(module_store, root_dir, ['test_export'], draft_store=draft_store)
 
         items = module_store.get_items(Location(['i4x', 'edX', 'full', 'vertical', None]))
         self.assertGreater(len(items), 0)
         for descriptor in items:
+            # don't try to look at private verticals. Right now we're running
+            # the service in non-draft aware
+            if getattr(descriptor, 'is_draft', False):
                 print "Checking {0}....".format(descriptor.location.url())
                 resp = self.client.get(reverse('edit_unit', kwargs={'location': descriptor.location.url()}))
                 self.assertEqual(resp.status_code, 200)
 
+        # verify that we have the content in the draft store as well
+        vertical = draft_store.get_item(Location(['i4x', 'edX', 'full',
+                                                  'vertical', 'vertical_66', None]), depth=1)
+
+        self.assertTrue(getattr(vertical, 'is_draft', False))
+        for child in vertical.get_children():
+            self.assertTrue(getattr(child, 'is_draft', False))
+
+        # verify that we have the private vertical
+        test_private_vertical = draft_store.get_item(Location(['i4x', 'edX', 'full',
+                                                               'vertical', 'vertical_66', None]))
+
+        self.assertTrue(getattr(test_private_vertical, 'is_draft', False))
+
         shutil.rmtree(root_dir)
 
     def test_course_handouts_rewrites(self):
         module_store = modulestore('direct')
-        content_store = contentstore()
 
         # import a test course
         import_from_xml(module_store, 'common/test/data/', ['full'])
@@ -440,8 +482,8 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
                                   'Administrivia_and_Circuit_Elements', None]) in course.system.module_data)
 
         # make sure we don't have a specific vertical which should be at depth=3
-        self.assertFalse(Location(['i4x', 'edX', 'full', 'vertical', 'vertical_58',
-            None]) in course.system.module_data)
+        self.assertFalse(Location(['i4x', 'edX', 'full', 'vertical', 'vertical_58', None])
+                         in course.system.module_data)
 
     def test_export_course_with_unknown_metadata(self):
         module_store = modulestore('direct')
@@ -468,10 +510,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
             export_to_xml(module_store, content_store, location, root_dir, 'test_export')
             exported = True
         except Exception:
+            print 'Exception thrown: {0}'.format(traceback.format_exc())
             pass
 
         self.assertTrue(exported)
 
+
 class ContentStoreTest(ModuleStoreTestCase):
     """
     Tests for the CMS ContentStore application.

cms/djangoapps/contentstore/views.py

@@ -1586,7 +1586,8 @@ def import_course(request, org, course, name):
                 shutil.move(r / fname, course_dir)
 
         module_store, course_items = import_from_xml(modulestore('direct'), settings.GITHUB_REPO_ROOT,
-            [course_subdir], load_error_modules=False, static_content_store=contentstore(), target_location_namespace=Location(location))
+            [course_subdir], load_error_modules=False, static_content_store=contentstore(), 
+            target_location_namespace=Location(location), draft_store=modulestore())
 
         # we can blow this away when we're done importing.
         shutil.rmtree(course_dir)
@@ -1620,8 +1621,8 @@ def generate_export_course(request, org, course, name):
 
     logging.debug('root = {0}'.format(root_dir))
 
-    export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name)
-    # filename = root_dir / name + '.tar.gz'
+    export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name, modulestore())
+    #filename = root_dir / name + '.tar.gz'
 
     logging.debug('tar file being generated at {0}'.format(export_file.name))
     tf = tarfile.open(name=export_file.name, mode='w:gz')

common/lib/xmodule/xmodule/html_module.py

@@ -118,8 +118,8 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor):
                 with system.resources_fs.open(filepath) as file:
                     html = file.read().decode('utf-8')
                     # Log a warning if we can't parse the file, but don't error
-                    if not check_html(html):
-                        msg = "Couldn't parse html in {0}.".format(filepath)
+                    if not check_html(html) and len(html) > 0:
+                        msg = "Couldn't parse html in {0}, content = {1}".format(filepath, html)
                         log.warning(msg)
                         system.error_tracker("Warning: " + msg)
 
@@ -156,7 +156,8 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor):
 
         resource_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True)
         with resource_fs.open(filepath, 'w') as file:
-            file.write(self.data.encode('utf-8'))
+            html_data = self.data.encode('utf-8')
+            file.write(html_data)
 
         # write out the relative name
         relname = path(pathname).basename()

common/lib/xmodule/xmodule/modulestore/draft.py

@@ -3,7 +3,6 @@ from datetime import datetime
 from . import ModuleStoreBase, Location, namedtuple_to_son
 from .exceptions import ItemNotFoundError
 from .inheritance import own_metadata
-import logging
 
 DRAFT = 'draft'
 
@@ -107,7 +106,7 @@ class DraftModuleStore(ModuleStoreBase):
         """
         return wrap_draft(super(DraftModuleStore, self).clone_item(source, as_draft(location)))
 
-    def update_item(self, location, data):
+    def update_item(self, location, data, allow_not_found=False):
         """
         Set the data in the item specified by the location to
         data
@@ -116,9 +115,13 @@ class DraftModuleStore(ModuleStoreBase):
         data: A nested dictionary of problem data
         """
         draft_loc = as_draft(location)
+        try:
             draft_item = self.get_item(location)
             if not getattr(draft_item, 'is_draft', False):
                 self.clone_item(location, draft_loc)
+        except ItemNotFoundError, e:
+            if not allow_not_found:
+                raise e
 
         return super(DraftModuleStore, self).update_item(draft_loc, data)
 
@@ -164,7 +167,6 @@ class DraftModuleStore(ModuleStoreBase):
         """
         return super(DraftModuleStore, self).delete_item(as_draft(location))
 
-
     def get_parent_locations(self, location, course_id):
         '''Find all locations that are the parents of this location.  Needed
         for path_to_location().
@@ -178,6 +180,7 @@ class DraftModuleStore(ModuleStoreBase):
         Save a current draft to the underlying modulestore
         """
         draft = self.get_item(location)
+
         draft.cms.published_date = datetime.utcnow()
         draft.cms.published_by = published_by_id
         super(DraftModuleStore, self).update_item(location, draft._model_data._kvs._data)

common/lib/xmodule/xmodule/modulestore/xml_exporter.py

 import logging
 from xmodule.modulestore import Location
-from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.inheritance import own_metadata
 from fs.osfs import OSFS
 from json import dumps
 
 
-def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir):
+def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir, draft_modulestore=None):
 
     course = modulestore.get_item(course_location)
 
@@ -40,6 +39,24 @@ def export_to_xml(modulestore, contentstore, course_location, root_dir, course_d
         policy = {'course/' + course.location.name: own_metadata(course)}
         course_policy.write(dumps(policy))
 
+    # export draft content
+    # NOTE: this code assumes that verticals are the top most draftable container
+    # should we change the application, then this assumption will no longer
+    # be valid
+    if draft_modulestore is not None:
+        draft_verticals = draft_modulestore.get_items([None, course_location.org, course_location.course,
+                                                       'vertical', None, 'draft'])
+        if len(draft_verticals) > 0:
+            draft_course_dir = export_fs.makeopendir('drafts')
+            for draft_vertical in draft_verticals:
+                parent_locs = draft_modulestore.get_parent_locations(draft_vertical.location, course.location.course_id)
+                logging.debug('parent_locs = {0}'.format(parent_locs))
+                draft_vertical.xml_attributes['parent_sequential_url'] = Location(parent_locs[0]).url()
+                sequential = modulestore.get_item(Location(parent_locs[0]))
+                index = sequential.children.index(draft_vertical.location.url())
+                draft_vertical.xml_attributes['index_in_children_list'] = str(index)
+                draft_vertical.export_to_xml(draft_course_dir)
+
 
 def export_extra_content(export_fs, modulestore, course_location, category_type, dirname, file_suffix=''):
     query_loc = Location('i4x', course_location.org, course_location.course, category_type, None)

common/lib/xmodule/xmodule/xml_module.py

@@ -110,8 +110,7 @@ class XmlDescriptor(XModuleDescriptor):
         'name', 'slug')
 
     metadata_to_strip = ('data_dir',
-            # cdodge: @TODO: We need to figure out a way to export out 'tabs' and 'grading_policy' which is on the course
-            'tabs', 'grading_policy', 'is_draft', 'published_by', 'published_date',
+            'tabs', 'grading_policy', 'published_by', 'published_date',
             'discussion_blackouts', 'testcenter_info',
            # VS[compat] -- remove the below attrs once everything is in the CMS
            'course', 'org', 'url_name', 'filename',
@@ -135,7 +134,7 @@ class XmlDescriptor(XModuleDescriptor):
         'graded': bool_map,
         'hide_progress_tab': bool_map,
         'allow_anonymous': bool_map,
-        'allow_anonymous_to_peers': bool_map
+        'allow_anonymous_to_peers': bool_map,
     }