wip: static asset import

cms/djangoapps/contentstore/management/commands/import.py

@@ -5,6 +5,7 @@
 from django.core.management.base import BaseCommand, CommandError
 from xmodule.modulestore.xml_importer import import_from_xml
 from xmodule.modulestore.django import modulestore
+from xmodule.contentstore.django import contentstore
 
 
 unnamed_modules = 0
@@ -26,4 +27,4 @@ class Command(BaseCommand):
         print "Importing.  Data_dir={data}, course_dirs={courses}".format(
             data=data_dir,
             courses=course_dirs)
-        import_from_xml(modulestore('direct'), data_dir, course_dirs, load_error_modules=False)
+        import_from_xml(modulestore('direct'), data_dir, course_dirs, load_error_modules=False,static_content_store=contentstore())

common/lib/xmodule/xmodule/contentstore/content.py

@@ -5,7 +5,11 @@ XASSET_THUMBNAIL_TAIL_NAME = '.thumbnail.jpg'
 
 import os
 import logging
+import StringIO
+
 from xmodule.modulestore import Location
+from .django import contentstore
+from PIL import Image
 
 class StaticContent(object):
     def __init__(self, loc, name, content_type, data, last_modified_at=None):
@@ -24,6 +28,10 @@ class StaticContent(object):
 
     @staticmethod
     def compute_location(org, course, name, revision=None):
+        # replace some illegal characters
+        # for example, when importing courseware static assets, typically the source repository has subdirectories. 
+        # right now the content store does not support a hierarchy structure, so collapse those subpaths
+        name = name.replace('/', '_')
         return Location([XASSET_LOCATION_TAG, org, course, 'asset', name, revision])
 
     def get_id(self):
@@ -66,3 +74,43 @@ class ContentStore(object):
 
     def get_all_content_for_course(self, location):
         raise NotImplementedError
+
+    def generate_thumbnail(self, content):
+        thumbnail_content = None
+        # if we're uploading an image, then let's generate a thumbnail so that we can
+        # serve it up when needed without having to rescale on the fly
+        if content.content_type is not None and content.content_type.split('/')[0] == 'image':
+            try:
+                # use PIL to do the thumbnail generation (http://www.pythonware.com/products/pil/)
+                # My understanding is that PIL will maintain aspect ratios while restricting
+                # the max-height/width to be whatever you pass in as 'size'
+                # @todo: move the thumbnail size to a configuration setting?!?
+                im = Image.open(StringIO.StringIO(content.data))
+
+                # I've seen some exceptions from the PIL library when trying to save palletted 
+                # PNG files to JPEG. Per the google-universe, they suggest converting to RGB first.
+                im = im.convert('RGB')
+                size = 128, 128
+                im.thumbnail(size, Image.ANTIALIAS)
+                thumbnail_file = StringIO.StringIO()
+                im.save(thumbnail_file, 'JPEG')
+                thumbnail_file.seek(0)
+            
+                # use a naming convention to associate originals with the thumbnail
+                thumbnail_name = content.generate_thumbnail_name()
+
+                # then just store this thumbnail as any other piece of content
+                thumbnail_file_location = StaticContent.compute_location(content.location.org, content.location.course, 
+                                                                                  thumbnail_name)
+                thumbnail_content = StaticContent(thumbnail_file_location, thumbnail_name, 
+                                                  'image/jpeg', thumbnail_file)
+
+                contentstore().save(thumbnail_content)
+            except:
+                raise
+
+        return thumbnail_content
+
+
+
+

common/lib/xmodule/xmodule/modulestore/xml_importer.py

 import logging
+import os
+import mimetypes
 
 from .xml import XMLModuleStore
 from .exceptions import DuplicateItemError
+from xmodule.modulestore import Location
+from xmodule.contentstore.content import StaticContent
 
 log = logging.getLogger(__name__)
 
 
 def import_from_xml(store, data_dir, course_dirs=None, 
                     default_class='xmodule.raw_module.RawDescriptor',
-                    load_error_modules=True):
+                    load_error_modules=True, static_content_store=None):
     """
     Import the specified xml data_dir into the "store" modulestore,
     using org and course as the location org and course.
@@ -23,9 +27,16 @@ def import_from_xml(store, data_dir, course_dirs=None,
         course_dirs=course_dirs,
         load_error_modules=load_error_modules,
     )
+
     for course_id in module_store.modules.keys():
+        course_data_dir = None
+        course_loc = None
+
         for module in module_store.modules[course_id].itervalues():
 
+            if module.category == 'course':
+                course_loc = module.location
+
             if 'data' in module.definition:
                 store.update_item(module.location, module.definition['data'])
             if 'children' in module.definition:
@@ -33,5 +44,39 @@ def import_from_xml(store, data_dir, course_dirs=None,
             # NOTE: It's important to use own_metadata here to avoid writing
             # inherited metadata everywhere.
             store.update_metadata(module.location, dict(module.own_metadata))
+            course_data_dir = module.metadata['data_dir']
+
+        if static_content_store is not None:
+            ''' 
+            now import all static assets
+            '''
+            static_dir = '{0}/{1}/static/'.format(data_dir, course_data_dir)
+
+            for dirname, dirnames, filenames in os.walk(static_dir):
+                for filename in filenames:
+
+                    try:
+                        content_path = os.path.join(dirname, filename)
+                        fullname_with_subpath = content_path.replace(static_dir, '')  # strip away leading path from the name
+                        content_loc = StaticContent.compute_location(course_loc.org, course_loc.course, fullname_with_subpath)
+                        mime_type = mimetypes.guess_type(filename)[0]
+
+                        print 'importing static asset {0} of mime-type {1} from path {2}'.format(content_loc, 
+                            mime_type, content_path)
+
+                        f = open(content_path, 'rb')
+                        data = f.read()
+                        f.close()
+
+                        content = StaticContent(content_loc, filename, mime_type, data)
+
+                        static_content_store.save(content)
+
+                        # this will be a NOP if content is not an image
+                        thumbnail_content = static_content_store.generate_thumbnail(content)
+
+                    except:
+                        raise
+
 
     return module_store