Merge pull request #877 from MITx/feature/cdodge/cas-tarball-import

allow importing of .tar.gz packages of courseware

cms/djangoapps/auth/authz.py

@@ -46,7 +46,7 @@ def create_all_course_groups(creator, location):
 
 def create_new_course_group(creator, location, role):
     groupname = get_course_groupname_for_role(location, role)
-    (group, created) =Group.get_or_create(name=groupname)
+    (group, created) =Group.objects.get_or_create(name=groupname)
     if created:
         group.save()
 

cms/djangoapps/contentstore/views.py

@@ -8,6 +8,8 @@ import os
 import StringIO
 import sys
 import time
+import tarfile
+import shutil
 from collections import defaultdict
 from uuid import uuid4
 
@@ -44,10 +46,11 @@ from xmodule.contentstore.content import StaticContent
 from cache_toolbox.core import set_cached_content, get_cached_content, del_cached_content
 from auth.authz import is_user_in_course_group_role, get_users_in_course_group_by_role
 from auth.authz import get_user_by_email, add_user_to_course_group, remove_user_from_course_group
-from auth.authz import INSTRUCTOR_ROLE_NAME, STAFF_ROLE_NAME
+from auth.authz import INSTRUCTOR_ROLE_NAME, STAFF_ROLE_NAME, create_all_course_groups
 from .utils import get_course_location_for_item, get_lms_link_for_item, compute_unit_state, get_date_display
 
 from xmodule.templates import all_templates
+from xmodule.modulestore.xml_importer import import_from_xml
 
 log = logging.getLogger(__name__)
 
@@ -809,3 +812,46 @@ def asset_index(request, org, course, name):
 # points to the temporary edge page
 def edge(request):
     return render_to_response('university_profiles/edge.html', {})
+
+def import_course(request):
+    if request.method != 'POST':
+        # (cdodge) @todo: Is there a way to do a - say - 'raise Http400'?
+        return HttpResponseBadRequest()
+
+    filename = request.FILES['file'].name
+
+    if not filename.endswith('.tar.gz'):
+        return HttpResponse(json.dumps({'ErrMsg': 'We only support uploading a .tar.gz file.'}))
+
+    temp_filepath = settings.GITHUB_REPO_ROOT + '/' + filename
+
+    logging.debug('importing course to {0}'.format(temp_filepath))
+
+    # stream out the uploaded files in chunks to disk
+    temp_file = open(temp_filepath, 'wb+')
+    for chunk in request.FILES['file'].chunks():
+        temp_file.write(chunk)
+    temp_file.close()
+
+    tf = tarfile.open(temp_filepath)
+    tf.extractall(settings.GITHUB_REPO_ROOT + '/')
+
+    os.remove(temp_filepath)    # remove the .tar.gz file
+
+    # @todo: don't assume the top-level directory that was unziped was the same name (but without .tar.gz)
+
+    course_dir = filename.replace('.tar.gz','')
+
+    module_store, course_items = import_from_xml(modulestore('direct'), settings.GITHUB_REPO_ROOT, 
+        [course_dir], load_error_modules=False,static_content_store=contentstore())
+
+    # remove content directory - we *shouldn't* need this any longer :-)
+    shutil.rmtree(temp_filepath.replace('.tar.gz', ''))
+
+    logging.debug('new course at {0}'.format(course_items[0].location))
+
+    create_all_course_groups(request.user, course_items[0].location)
+
+
+
+    return HttpResponse(json.dumps({'Status' : 'OK'}))

cms/static/sass/base-style.scss

@@ -17,7 +17,6 @@
 @import "static-pages";
 @import "users";
 @import "course-info";
-@import "edge";
 @import "landing";
 @import "graphics";
 @import "modal";

cms/templates/index.html

@@ -27,4 +27,7 @@
 	</div>
 
 </section>
+
+<%include file="widgets/import-course.html"/>
+
 </%block>

cms/templates/widgets/import-course.html

+<%! from django.core.urlresolvers import reverse %>
+
+<section>
+  <div class="course-upload">
+    You can import an existing .tar.gz file of your course
+    <form action="${reverse('import_course')}" method="post" enctype="multipart/form-data">
+      <input type="file" name="file">
+      <input type="submit" value="Upload File">
+    </form>
+    <div class="progress" style="position:relative; width:400px; border: 1px solid #ddd; padding: 1px; border-radius: 3px;">
+      <div class="bar" style="background-color: #B4F5B4; width:0%; height:20px; border-radius: 3px;"></div>
+      <div class="percent">0%</div>
+    </div>
+
+    <div id="status"></div>
+  </div>
+
+</section>
+
+<script src="http://malsup.github.com/jquery.form.js"></script>
+<script>
+(function() {
+    
+var bar = $('.bar');
+var percent = $('.percent');
+var status = $('#status');
+   
+$('form').ajaxForm({
+    beforeSend: function() {
+        status.empty();
+        var percentVal = '0%';
+        bar.width(percentVal)
+        percent.html(percentVal);
+    },
+    uploadProgress: function(event, position, total, percentComplete) {
+        var percentVal = percentComplete + '%';
+        bar.width(percentVal)
+        percent.html(percentVal);
+    },
+    complete: function(xhr) {
+      status.html(xhr.responseText);
+    }
+  }); 
+})();       
+</script>
\ No newline at end of file

cms/urls.py

@@ -46,6 +46,8 @@ urlpatterns = ('',
     url(r'^edge$', 'contentstore.views.edge', name='edge'),
 
     url(r'^heartbeat$', include('heartbeat.urls')),
+
+    url(r'import_course$', 'contentstore.views.import_course', name='import_course'),
 )
 
 # User creation and updating views

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -40,9 +40,6 @@ def import_static_content(modules, data_dir, static_content_store):
                 content_loc = StaticContent.compute_location(course_loc.org, course_loc.course, fullname_with_subpath)
                 mime_type = mimetypes.guess_type(filename)[0]
 
-                print 'importing static asset {0} of mime-type {1} from path {2}'.format(content_loc, 
-                    mime_type, content_path)
-
                 f = open(content_path, 'rb')
                 data = f.read()
                 f.close()
@@ -87,6 +84,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
     # NOTE: the XmlModuleStore does not implement get_items() which would be a preferable means
     # to enumerate the entire collection of course modules. It will be left as a TBD to implement that
     # method on XmlModuleStore.
+    course_items = []
     for course_id in module_store.modules.keys():
         remap_dict = {}
         if static_content_store is not None:
@@ -97,6 +95,7 @@ def import_from_xml(store, data_dir, course_dirs=None,
             if module.category == 'course':
                 # HACK: for now we don't support progress tabs. There's a special metadata configuration setting for this.
                 module.metadata['hide_progress_tab'] = True
+                course_items.append(module)
 
             if 'data' in module.definition:
                 module_data = module.definition['data']
@@ -124,4 +123,4 @@ def import_from_xml(store, data_dir, course_dirs=None,
             store.update_metadata(module.location, dict(module.own_metadata))
             
 
-    return module_store
+    return module_store, course_items