Merge pull request #1211 from MITx/feature/cdodge/export

Feature/cdodge/export

cms/djangoapps/contentstore/management/commands/export.py

+###
+### Script for exporting courseware from Mongo to a tar.gz file
+###
+import os
+
+from django.core.management.base import BaseCommand, CommandError
+from xmodule.modulestore.xml_exporter import export_to_xml
+from xmodule.modulestore.django import modulestore
+from xmodule.contentstore.django import contentstore
+from xmodule.modulestore import Location
+from xmodule.course_module import CourseDescriptor
+
+
+unnamed_modules = 0
+
+
+class Command(BaseCommand):
+    help = \
+'''Import the specified data directory into the default ModuleStore'''
+
+    def handle(self, *args, **options):
+        if len(args) != 2:
+            raise CommandError("import requires two arguments: <course location> <output path>")
+
+        course_id = args[0]
+        output_path = args[1]
+
+        print "Exporting course id = {0} to {1}".format(course_id, output_path)
+
+        location = CourseDescriptor.id_to_location(course_id)
+
+        root_dir = os.path.dirname(output_path)
+        course_dir = os.path.splitext(os.path.basename(output_path))[0]
+
+        export_to_xml(modulestore('direct'), contentstore(), location, root_dir, course_dir)

cms/djangoapps/contentstore/tests/tests.py

 import json
+import shutil
 from django.test import TestCase
 from django.test.client import Client
 from override_settings import override_settings
 from django.conf import settings
 from django.core.urlresolvers import reverse
 from path import path
+from tempfile import mkdtemp
 
 from student.models import Registration
 from django.contrib.auth.models import User
@@ -18,6 +20,7 @@ from xmodule.modulestore.store_utilities import delete_course
 from xmodule.modulestore.django import modulestore
 from xmodule.contentstore.django import contentstore
 from xmodule.course_module import CourseDescriptor
+from xmodule.modulestore.xml_exporter import export_to_xml
 
 def parse_json(response):
     """Parse response, which is assumed to be json"""
@@ -385,4 +388,35 @@ class ContentStoreTest(TestCase):
         items = ms.get_items(Location(['i4x','edX', 'full', 'vertical', None]))
         self.assertEqual(len(items), 0)
 
+    def test_export_course(self):
+        ms = modulestore('direct')
+        cs = contentstore() 
+
+        import_from_xml(ms, 'common/test/data/', ['full'])
+        location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012')
+
+        root_dir = path(mkdtemp())
+
+        print 'Exporting to tempdir = {0}'.format(root_dir)
+
+        # export out to a tempdir
+        export_to_xml(ms, cs, location, root_dir, 'test_export')
+
+        # remove old course
+        delete_course(ms, cs, location)
+
+        # reimport
+        import_from_xml(ms, root_dir, ['test_export'])
+
+        items = ms.get_items(Location(['i4x','edX', 'full', 'vertical', None]))
+        self.assertGreater(len(items), 0)
+        for descriptor in items:
+            print "Checking {0}....".format(descriptor.location.url())
+            resp = self.client.get(reverse('edit_unit', kwargs={'location': descriptor.location.url()}))
+            self.assertEqual(resp.status_code, 200)
+
+        shutil.rmtree(root_dir)        
+
+
+
 

cms/djangoapps/contentstore/views.py

@@ -10,6 +10,10 @@ from datetime import datetime
 from collections import defaultdict
 from uuid import uuid4
 from path import path
+from xmodule.modulestore.xml_exporter import export_to_xml
+from tempfile import mkdtemp
+from django.core.servers.basehttp import FileWrapper
+from django.core.files.temp import NamedTemporaryFile
 
 # to install PIL on MacOSX: 'easy_install http://dist.repoze.org/PIL-1.1.6.tar.gz'
 from PIL import Image
@@ -1302,3 +1306,55 @@ def import_course(request, org, course, name):
                         course_module.location.course,
                         course_module.location.name])
         })
+
+@ensure_csrf_cookie
+@login_required
+def generate_export_course(request, org, course, name):
+    location = ['i4x', org, course, 'course', name]
+    course_module = modulestore().get_item(location)
+    # check that logged in user has permissions to this item
+    if not has_access(request.user, location):
+        raise PermissionDenied()
+
+    loc = Location(location)
+    export_file = NamedTemporaryFile(prefix=name+'.', suffix=".tar.gz")
+
+    root_dir = path(mkdtemp())
+
+    # export out to a tempdir
+    
+    logging.debug('root = {0}'.format(root_dir))
+
+    export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name)
+    #filename = root_dir / name + '.tar.gz'
+
+    logging.debug('tar file being generated at {0}'.format(export_file.name))
+    tf = tarfile.open(name=export_file.name, mode='w:gz')
+    tf.add(root_dir/name, arcname=name)
+    tf.close()
+
+    # remove temp dir
+    shutil.rmtree(root_dir/name) 
+
+    wrapper = FileWrapper(export_file)
+    response = HttpResponse(wrapper, content_type='application/x-tgz')
+    response['Content-Disposition'] = 'attachment; filename=%s' % os.path.basename(export_file.name)
+    response['Content-Length'] = os.path.getsize(export_file.name)
+    return response
+
+
+@ensure_csrf_cookie
+@login_required
+def export_course(request, org, course, name):
+
+    location = ['i4x', org, course, 'course', name]
+    course_module = modulestore().get_item(location)
+    # check that logged in user has permissions to this item
+    if not has_access(request.user, location):
+        raise PermissionDenied()
+
+    return render_to_response('export.html', {
+        'context_course': course_module,
+        'active_tab': 'export',
+        'successful_import_redirect_url' : ''
+    })

cms/static/sass/_export.scss

+.export {
+  .export-overview {
+    @extend .window;
+    @include clearfix;
+    padding: 30px 40px;
+  }
+
+  .description {
+    float: left;
+    width: 62%;
+    margin-right: 3%;
+    font-size: 14px;
+
+    h2 {
+      font-weight: 700;
+      font-size: 19px;
+      margin-bottom: 20px;
+    }
+
+    strong {
+      font-weight: 700;
+    }
+
+    p + p {
+      margin-top: 20px;
+    }
+
+    ul {
+      margin: 20px 0;
+      list-style: disc inside;
+
+      li {
+        margin: 0 0 5px 0;
+      }
+    }
+  }
+
+  .export-form-wrapper {
+  
+    .export-form {
+      float: left;
+      width: 35%;
+      padding: 25px 30px 35px;
+      @include box-sizing(border-box);
+      border: 1px solid $mediumGrey;
+      border-radius: 3px;
+      background: $lightGrey;
+      text-align: center;
+
+      h2 {
+        margin-bottom: 30px;
+        font-size: 26px;
+        font-weight: 300;
+      }
+
+      .error-block {
+        display: none;
+        margin-bottom: 15px;
+        font-size: 13px;
+      }
+
+      .error-block {
+        color: $error-red;
+      }
+
+      .button-export {
+        @include green-button;
+        padding: 10px 50px 11px;
+        font-size: 17px;
+      }
+
+      .message-status {
+        margin-top: 10px;
+        font-size: 12px;
+      }
+
+      .progress-bar {
+        display: none;
+        width: 350px;
+        height: 30px;
+        margin: 30px auto 10px;
+        border: 1px solid $blue;
+
+        &.loaded {
+          border-color: #66b93d;
+
+          .progress-fill {
+            background: #66b93d;
+          }
+        }
+      }
+
+      .progress-fill {
+        width: 0%;
+        height: 30px;
+        background: $blue;
+        color: #fff;
+        line-height: 48px;
+      }
+    }
+
+    // downloading state
+    &.is-downloading {
+
+      .progress-bar {
+        display: block;
+      }
+
+      .button-export {
+        padding: 10px 50px 11px;
+        font-size: 17px;
+
+        &.disabled {
+
+          pointer-events: none;
+          cursor: default;
+        }
+      }
+    }
+  }
+
+
+}
\ No newline at end of file

cms/static/sass/base-style.scss

@@ -18,6 +18,7 @@
 @import "static-pages";
 @import "users";
 @import "import";
+@import "export";
 @import "settings";
 @import "course-info";
 @import "landing";

cms/templates/export.html

+<%inherit file="base.html" />
+<%namespace name='static' file='static_content.html'/>
+
+<%! from django.core.urlresolvers import reverse %>
+<%block name="title">Export</%block>
+<%block name="bodyclass">export</%block>
+
+<%block name="content">
+<div class="main-wrapper">
+  <div class="inner-wrapper">
+    <article class="export-overview">
+      <div class="description">
+        <h2>About Exporting Courses</h2>
+        <p>When exporting your course, you will receive a .tar.gz formatted file that contains the following course data:</p>
+
+        <ul>
+          <li>Course Structure (Sections and sub-section ordering)</li>
+          <li>Individual Units</li>
+          <li>Individual Problems</li>
+          <li>Static Pages</li>
+          <li>Course Assets</li>
+        </ul>
+
+        <p>Your course export <strong>will not include</strong>: student data, forum/discussion data, course settings, certificates, grading information, or user data.</p>
+      </div>
+
+      <!-- default state -->
+      <div class="export-form-wrapper">
+        <form action="${reverse('generate_export_course', kwargs=dict(org=context_course.location.org, course=context_course.location.course, name=context_course.location.name))}" method="post" enctype="multipart/form-data" class="export-form">
+          <h2>Export Course:</h2>
+          
+          <p class="error-block"></p>
+          
+          <a href="${reverse('generate_export_course', kwargs=dict(org=context_course.location.org, course=context_course.location.course, name=context_course.location.name))}" class="button-export">Download Files</a>
+        </form>
+      </div>
+
+      <!-- download state: after user clicks download buttons -->
+      <%doc>
+      <div class="export-form-wrapper is-downloading">
+        <form action="${reverse('export_course', kwargs=dict(org=context_course.location.org, course=context_course.location.course, name=context_course.location.name))}" method="post" enctype="multipart/form-data" class="export-form">
+          <h2>Export Course:</h2>
+          
+          <p class="error-block"></p>
+          
+          <a href="#" class="button-export disabled">Files Downloading</a>
+          <p class="message-status">Download not start? <a href="#" class="text-export">Try again</a></p>          
+        </form>
+      </div>
+      </%doc>
+    </article>
+  </div>
+</div>
+</%block>
+
+<%block name="jsextra">
+<script>
+(function() {
+    
+var bar = $('.progress-bar');
+var fill = $('.progress-fill');
+var percent = $('.percent');
+var status = $('#status');
+var submitBtn = $('.submit-button');
+   
+$('form').ajaxForm({
+    beforeSend: function() {
+        status.empty();
+        var percentVal = '0%';
+        bar.show();
+        fill.width(percentVal);
+        percent.html(percentVal);
+        submitBtn.hide();
+    },
+    uploadProgress: function(event, position, total, percentComplete) {
+        var percentVal = percentComplete + '%';
+        fill.width(percentVal);
+        percent.html(percentVal);
+    },
+    complete: function(xhr) {
+      if (xhr.status == 200) {
+        alert('Your import was successful.');
+        window.location = '${successful_import_redirect_url}';
+      }
+      else
+        alert('Your import has failed.\n\n' + xhr.responseText);
+        submitBtn.show();
+        bar.hide();
+    }
+  }); 
+})();       
+</script>
+</%block>
\ No newline at end of file

cms/templates/widgets/header.html

@@ -33,6 +33,7 @@
       <li><a href="${reverse('asset_index', kwargs=dict(org=ctx_loc.org, course=ctx_loc.course, name=ctx_loc.name))}" id='assets-tab'>Assets</a></li>
       <li><a href="${reverse('course_settings', kwargs=dict(org=ctx_loc.org, course=ctx_loc.course, name=ctx_loc.name))}" id='settings-tab'>Settings</a></li>
       <li><a href="${reverse('import_course', kwargs=dict(org=ctx_loc.org, course=ctx_loc.course, name=ctx_loc.name))}" id='import-tab'>Import</a></li>
+      <li><a href="${reverse('export_course', kwargs=dict(org=ctx_loc.org, course=ctx_loc.course, name=ctx_loc.name))}" id='export-tab'>Export</a></li>
     </ul>
     % endif
   </nav>

cms/urls.py

@@ -23,6 +23,11 @@ urlpatterns = ('',
     url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/import/(?P<name>[^/]+)$',
         'contentstore.views.import_course', name='import_course'),
 
+    url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/export/(?P<name>[^/]+)$',
+        'contentstore.views.export_course', name='export_course'),
+    url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/generate_export/(?P<name>[^/]+)$',
+        'contentstore.views.generate_export_course', name='generate_export_course'),
+
     url(r'^preview/modx/(?P<preview_id>[^/]*)/(?P<location>.*?)/(?P<dispatch>[^/]*)$',
         'contentstore.views.preview_dispatch', name='preview_dispatch'),
     url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/course/(?P<coursename>[^/]+)/upload_asset$',

common/lib/xmodule/xmodule/contentstore/content.py

@@ -12,13 +12,16 @@ from .django import contentstore
 from PIL import Image
 
 class StaticContent(object):
-    def __init__(self, loc, name, content_type, data, last_modified_at=None, thumbnail_location=None):
+    def __init__(self, loc, name, content_type, data, last_modified_at=None, thumbnail_location=None, import_path=None):
         self.location = loc
         self.name = name #a display string which can be edited, and thus not part of the location which needs to be fixed
         self.content_type = content_type
         self.data = data
         self.last_modified_at = last_modified_at
         self.thumbnail_location = Location(thumbnail_location)
+        # optional information about where this file was imported from. This is needed to support import/export 
+        # cycles
+        self.import_path = import_path
 
     @property
     def is_thumbnail(self):

common/lib/xmodule/xmodule/contentstore/mongo.py

@@ -11,6 +11,8 @@ import logging
 
 from .content import StaticContent, ContentStore
 from xmodule.exceptions import NotFoundError
+from fs.osfs import OSFS
+import os
 
 
 class MongoContentStore(ContentStore):
@@ -32,7 +34,7 @@ class MongoContentStore(ContentStore):
         self.delete(id)
 
         with self.fs.new_file(_id = id, filename=content.get_url_path(), content_type=content.content_type, 
-            displayname=content.name, thumbnail_location=content.thumbnail_location) as fp:
+            displayname=content.name, thumbnail_location=content.thumbnail_location, import_path=content.import_path) as fp:
 
             fp.write(content.data)
         
@@ -47,10 +49,32 @@ class MongoContentStore(ContentStore):
         try:
             with self.fs.get(id) as fp:
                 return StaticContent(location, fp.displayname, fp.content_type, fp.read(), 
-                    fp.uploadDate, thumbnail_location = fp.thumbnail_location if hasattr(fp, 'thumbnail_location') else None)
+                    fp.uploadDate, thumbnail_location = fp.thumbnail_location if hasattr(fp, 'thumbnail_location') else None,
+                    import_path = fp.import_path if hasattr(fp, 'import_path') else None)
         except NoFile:
             raise NotFoundError()
 
+    def export(self, location, output_directory):
+        content = self.find(location)
+
+        if content.import_path is not None:
+            output_directory = output_directory + '/' + os.path.dirname(content.import_path)
+
+        if not os.path.exists(output_directory):
+            os.makedirs(output_directory)
+
+        disk_fs = OSFS(output_directory)
+
+        with disk_fs.open(content.name, 'wb') as asset_file:
+            asset_file.write(content.data)
+
+    def export_all_for_course(self, course_location, output_directory):
+        assets = self.get_all_content_for_course(course_location)
+
+        for asset in assets:
+            asset_location = Location(asset['_id'])
+            self.export(asset_location, output_directory)
+
     def get_all_content_thumbnails_for_course(self, location):
         return self._get_all_content_for_course(location, get_thumbnails = True)
 

common/lib/xmodule/xmodule/html_module.py

@@ -160,7 +160,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
         filepath = u'{category}/{pathname}.html'.format(category=self.category,
                                                     pathname=pathname)
 
-        resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
+        resource_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True)
         with resource_fs.open(filepath, 'w') as file:
             file.write(self.definition['data'])
 

common/lib/xmodule/xmodule/modulestore/xml_exporter.py

+import logging
+from xmodule.modulestore import Location
+from xmodule.modulestore.django import modulestore
+from fs.osfs import OSFS
+
+def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir):
+
+  course = modulestore.get_item(course_location)
+
+  fs = OSFS(root_dir)
+  export_fs = fs.makeopendir(course_dir)
+
+  xml = course.export_to_xml(export_fs)
+  with export_fs.open('course.xml', 'w') as course_xml:
+      course_xml.write(xml)
+
+  # export the static assets
+  contentstore.export_all_for_course(course_location, root_dir + '/' + course_dir + '/static/')
+
+  
\ No newline at end of file

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -32,7 +32,7 @@ def import_static_content(modules, course_loc, course_data_path, static_content_
                 with open(content_path, 'rb') as f:
                     data = f.read()
 
-                content = StaticContent(content_loc, filename, mime_type, data)
+                content = StaticContent(content_loc, filename, mime_type, data, import_path = fullname_with_subpath)
 
                 # first let's save a thumbnail so we can get back a thumbnail location
                 thumbnail_content = static_content_store.generate_thumbnail(content)
@@ -66,7 +66,7 @@ def verify_content_links(module, base_dir, static_content_store, link, remap_dic
                 with open(static_pathname, 'rb') as f:
                     data = f.read()
 
-                content = StaticContent(content_loc, filename, mime_type, data) 
+                content = StaticContent(content_loc, filename, mime_type, data, import_path = path) 
 
                 # first let's save a thumbnail so we can get back a thumbnail location
                 thumbnail_content = static_content_store.generate_thumbnail(content)

common/lib/xmodule/xmodule/xml_module.py

@@ -98,20 +98,30 @@ class XmlDescriptor(XModuleDescriptor):
 
     metadata_to_strip = ('data_dir', 
             # cdodge: @TODO: We need to figure out a way to export out 'tabs' and 'grading_policy' which is on the course
-            'tabs', 'grading_policy',
+            'tabs', 'grading_policy', 'is_draft', 'published_by', 'published_date', 
+            'discussion_blackouts',
            # VS[compat] -- remove the below attrs once everything is in the CMS
            'course', 'org', 'url_name', 'filename')
 
+    metadata_to_export_to_policy = ('discussion_topics')
+
     # A dictionary mapping xml attribute names AttrMaps that describe how
     # to import and export them
     # Allow json to specify either the string "true", or the bool True.  The string is preferred.
     to_bool = lambda val: val == 'true' or val == True
     from_bool = lambda val: str(val).lower()
     bool_map = AttrMap(to_bool, from_bool)
+
+    to_int = lambda val: int(val)
+    from_int = lambda val: str(val) 
+    int_map = AttrMap(to_int, from_int)
     xml_attribute_map = {
         # type conversion: want True/False in python, "true"/"false" in xml
         'graded': bool_map,
         'hide_progress_tab': bool_map,
+        'allow_anonymous': bool_map,
+        'allow_anonymous_to_peers': bool_map,
+        'weight':int_map
     }
 
 
@@ -359,8 +369,9 @@ class XmlDescriptor(XModuleDescriptor):
         # Add the non-inherited metadata
         for attr in sorted(self.own_metadata):
             # don't want e.g. data_dir
-            if attr not in self.metadata_to_strip:
+            if attr not in self.metadata_to_strip and attr not in self.metadata_to_export_to_policy:
                 val = val_for_xml(attr)
+                #logging.debug('location.category = {0}, attr = {1}'.format(self.location.category, attr))
                 xml_object.set(attr, val)
 
         if self.export_to_file():

lms/djangoapps/courseware/courses.py

@@ -148,7 +148,7 @@ def get_course_about_section(course, section_key):
             request = get_request_for_thread()
 
             loc = course.location._replace(category='about', name=section_key)
-            course_module = get_module(request.user, request, loc, None, course.id, not_found_ok = True, wrap_xmodule_display = False)
+            course_module = get_module(request.user, request, loc, None, course.id, not_found_ok = True, wrap_xmodule_display = True)
 
             html = ''
 
@@ -186,7 +186,7 @@ def get_course_info_section(request, cache, course, section_key):
 
 
     loc = Location(course.location.tag, course.location.org, course.location.course, 'course_info', section_key)
-    course_module = get_module(request.user, request, loc, cache, course.id, wrap_xmodule_display = False)
+    course_module = get_module(request.user, request, loc, cache, course.id, wrap_xmodule_display = True)
     html = ''
 
     if course_module is not None:

rakefile

@@ -445,6 +445,18 @@ namespace :cms do
   end
 end
 
+namespace :cms do
+  desc "Export course data to a tar.gz file"
+  task :export do
+    if ENV['COURSE_ID'] and ENV['OUTPUT_PATH']
+      sh(django_admin(:cms, :dev, :export, ENV['COURSE_ID'], ENV['OUTPUT_PATH']))
+    else
+      raise "Please specify a COURSE_ID and OUTPUT_PATH.\n" +
+        "Example: \`rake cms:export COURSE_ID=MITx/12345/name OUTPUT_PATH=foo.tar.gz\`"
+    end
+  end
+end
+
 desc "Build a properties file used to trigger autodeploy builds"
 task :autodeploy_properties do
     File.open("autodeploy.properties", "w") do |file|