Merge pull request #652 from edx/feature/ichuang/import-with-no-static

Enable mongodb to be used as back-end for git-based authoring workflow

cms/djangoapps/contentstore/management/commands/import.py

@@ -2,7 +2,7 @@
 Script for importing courseware from XML format
 """
 
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError, make_option
 from xmodule.modulestore.xml_importer import import_from_xml
 from xmodule.modulestore.django import modulestore
 from xmodule.contentstore.django import contentstore
@@ -14,18 +14,26 @@ class Command(BaseCommand):
     """
     help = 'Import the specified data directory into the default ModuleStore'
 
+    option_list = BaseCommand.option_list + (
+        make_option('--nostatic',
+                    action='store_true',
+                    help='Skip import of static content'),
+    )
+
     def handle(self, *args, **options):
         "Execute the command"
         if len(args) == 0:
-            raise CommandError("import requires at least one argument: <data directory> [<course dir>...]")
+            raise CommandError("import requires at least one argument: <data directory> [--nostatic] [<course dir>...]")
 
         data_dir = args[0]
+        do_import_static = not (options.get('nostatic', False))
         if len(args) > 1:
             course_dirs = args[1:]
         else:
             course_dirs = None
         print("Importing.  Data_dir={data}, course_dirs={courses}".format(
             data=data_dir,
-            courses=course_dirs))
+            courses=course_dirs,
+            dis=do_import_static))
         import_from_xml(modulestore('direct'), data_dir, course_dirs, load_error_modules=False,
-                        static_content_store=contentstore(), verbose=True)
+                        static_content_store=contentstore(), verbose=True, do_import_static=do_import_static)

cms/djangoapps/contentstore/tests/test_contentstore.py

@@ -476,7 +476,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase):
         content_store = contentstore()
 
         module_store = modulestore('direct')
-        import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store)
+        import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store, verbose=True)
 
         course_location = CourseDescriptor.id_to_location('edX/toy/2012_Fall')
         course = module_store.get_item(course_location)

cms/djangoapps/contentstore/tests/test_import_nostatic.py

+#pylint: disable=E1101
+'''
+Tests for importing with no static
+'''
+
+from django.test.client import Client
+from django.test.utils import override_settings
+from django.conf import settings
+from path import path
+import copy
+
+from django.contrib.auth.models import User
+
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+
+from xmodule.modulestore import Location
+from xmodule.modulestore.django import modulestore
+from xmodule.contentstore.django import contentstore
+from xmodule.modulestore.xml_importer import import_from_xml
+from xmodule.contentstore.content import StaticContent
+
+from xmodule.course_module import CourseDescriptor
+
+from xmodule.exceptions import NotFoundError
+from uuid import uuid4
+
+
+TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
+TEST_DATA_CONTENTSTORE['OPTIONS']['db'] = 'test_xcontent_%s' % uuid4().hex
+
+
+@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
+class ContentStoreImportNoStaticTest(ModuleStoreTestCase):
+    """
+    Tests that rely on the toy and test_import_course courses.
+    NOTE: refactor using CourseFactory so they do not.
+    """
+    def setUp(self):
+
+        settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
+        settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data')
+        uname = 'testuser'
+        email = 'test+courses@edx.org'
+        password = 'foo'
+
+        # Create the use so we can log them in.
+        self.user = User.objects.create_user(uname, email, password)
+
+        # Note that we do not actually need to do anything
+        # for registration if we directly mark them active.
+        self.user.is_active = True
+        # Staff has access to view all courses
+        self.user.is_staff = True
+
+        # Save the data that we've just changed to the db.
+        self.user.save()
+
+        self.client = Client()
+        self.client.login(username=uname, password=password)
+
+    def load_test_import_course(self):
+        '''
+        Load the standard course used to test imports (for do_import_static=False behavior).
+        '''
+        content_store = contentstore()
+        module_store = modulestore('direct')
+        import_from_xml(module_store, 'common/test/data/', ['test_import_course'], static_content_store=content_store, do_import_static=False, verbose=True)
+        course_location = CourseDescriptor.id_to_location('edX/test_import_course/2012_Fall')
+        course = module_store.get_item(course_location)
+        self.assertIsNotNone(course)
+
+        return module_store, content_store, course, course_location
+
+    def test_static_import(self):
+        '''
+        Stuff in static_import should always be imported into contentstore
+        '''
+        _, content_store, course, course_location = self.load_test_import_course()
+
+        # make sure we have ONE asset in our contentstore ("should_be_imported.html")
+        all_assets = content_store.get_all_content_for_course(course_location)
+        print "len(all_assets)=%d" % len(all_assets)
+        self.assertEqual(len(all_assets), 1)
+
+        content = None
+        try:
+            location = StaticContent.get_location_from_path('/c4x/edX/test_import_course/asset/should_be_imported.html')
+            content = content_store.find(location)
+        except NotFoundError:
+            pass
+
+        self.assertIsNotNone(content)
+
+        # make sure course.lms.static_asset_path is correct
+        print "static_asset_path = {0}".format(course.lms.static_asset_path)
+        self.assertEqual(course.lms.static_asset_path, 'test_import_course')
+
+    def test_asset_import_nostatic(self):
+        '''
+        This test validates that an image asset is NOT imported when do_import_static=False
+        '''
+        content_store = contentstore()
+
+        module_store = modulestore('direct')
+        import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store, do_import_static=False, verbose=True)
+
+        course_location = CourseDescriptor.id_to_location('edX/toy/2012_Fall')
+        module_store.get_item(course_location)
+
+        # make sure we have NO assets in our contentstore
+        all_assets = content_store.get_all_content_for_course(course_location)
+        print "len(all_assets)=%d" % len(all_assets)
+        self.assertEqual(len(all_assets), 0)
+
+    def test_no_static_link_rewrites_on_import(self):
+        module_store = modulestore('direct')
+        import_from_xml(module_store, 'common/test/data/', ['toy'], do_import_static=False, verbose=True)
+
+        handouts = module_store.get_item(Location(['i4x', 'edX', 'toy', 'course_info', 'handouts', None]))
+        self.assertIn('/static/', handouts.data)
+
+        handouts = module_store.get_item(Location(['i4x', 'edX', 'toy', 'html', 'toyhtml', None]))
+        self.assertIn('/static/', handouts.data)

common/djangoapps/static_replace/__init__.py

@@ -90,7 +90,7 @@ def replace_course_urls(text, course_id):
     return re.sub(_url_replace_regex('/course/'), replace_course_url, text)
 
 
-def replace_static_urls(text, data_directory, course_id=None):
+def replace_static_urls(text, data_directory, course_id=None, static_asset_path=''):
     """
     Replace /static/$stuff urls either with their correct url as generated by collectstatic,
     (/static/$md5_hashed_stuff) or by the course-specific content static url
@@ -100,6 +100,7 @@ def replace_static_urls(text, data_directory, course_id=None):
     text: The source text to do the substitution in
     data_directory: The directory in which course data is stored
     course_id: The course identifier used to distinguish static content for this course in studio
+    static_asset_path: Path for static assets, which overrides data_directory and course_namespace, if nonempty
     """
 
     def replace_static_url(match):
@@ -116,7 +117,7 @@ def replace_static_urls(text, data_directory, course_id=None):
         if settings.DEBUG and finders.find(rest, True):
             return original
         # if we're running with a MongoBacked store course_namespace is not None, then use studio style urls
-        elif course_id and modulestore().get_modulestore_type(course_id) != XML_MODULESTORE_TYPE:
+        elif (not static_asset_path) and course_id and modulestore().get_modulestore_type(course_id) != XML_MODULESTORE_TYPE:
             # first look in the static file pipeline and see if we are trying to reference
             # a piece of static content which is in the mitx repo (e.g. JS associated with an xmodule)
 
@@ -135,7 +136,7 @@ def replace_static_urls(text, data_directory, course_id=None):
                 url = StaticContent.convert_legacy_static_url_with_course_id(rest, course_id)
         # Otherwise, look the file up in staticfiles_storage, and append the data directory if needed
         else:
-            course_path = "/".join((data_directory, rest))
+            course_path = "/".join((static_asset_path or data_directory, rest))
 
             try:
                 if staticfiles_storage.exists(rest):
@@ -152,7 +153,7 @@ def replace_static_urls(text, data_directory, course_id=None):
 
 
     return re.sub(
-        _url_replace_regex('/static/(?!{data_dir})'.format(data_dir=data_directory)),
+        _url_replace_regex('/static/(?!{data_dir})'.format(data_dir=static_asset_path or data_directory)),
         replace_static_url,
         text
     )

common/djangoapps/xmodule_modifiers.py

@@ -76,7 +76,7 @@ def replace_course_urls(get_html, course_id):
     return _get_html
 
 
-def replace_static_urls(get_html, data_dir, course_id=None):
+def replace_static_urls(get_html, data_dir, course_id=None, static_asset_path=''):
     """
     Updates the supplied module with a new get_html function that wraps
     the old get_html function and substitutes urls of the form /static/...
@@ -85,7 +85,7 @@ def replace_static_urls(get_html, data_dir, course_id=None):
 
     @wraps(get_html)
     def _get_html():
-        return static_replace.replace_static_urls(get_html(), data_dir, course_id)
+        return static_replace.replace_static_urls(get_html(), data_dir, course_id, static_asset_path=static_asset_path)
     return _get_html
 
 

common/lib/xmodule/xmodule/modulestore/inheritance.py

@@ -9,7 +9,8 @@ INHERITABLE_METADATA = (
     # intended to be set per-course, but can be overridden in for specific
     # elements.  Can be a float.
     'days_early_for_beta',
-    'giturl'  # for git edit link
+    'giturl',  # for git edit link
+    'static_asset_path',       # for static assets placed outside xcontent contentstore
 )
 
 

common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py

@@ -10,7 +10,9 @@ from xblock.runtime import KeyValueStore, InvalidScopeError
 from xmodule.tests import DATA_DIR
 from xmodule.modulestore import Location
 from xmodule.modulestore.mongo import MongoModuleStore, MongoKeyValueStore
-from xmodule.modulestore.xml_importer import import_from_xml
+from xmodule.modulestore.draft import DraftModuleStore
+from xmodule.modulestore.xml_importer import import_from_xml, perform_xlint
+from xmodule.contentstore.mongo import MongoContentStore
 
 from xmodule.modulestore.tests.test_modulestore import check_path_to_location
 
@@ -35,7 +37,7 @@ class TestMongoModuleStore(object):
         # is ok only as long as none of the tests modify the db.
         # If (when!) that changes, need to either reload the db, or load
         # once and copy over to a tmp db for each test.
-        cls.store = cls.initdb()
+        cls.store, cls.content_store, cls.draft_store = cls.initdb()
 
     @classmethod
     def teardownClass(cls):
@@ -46,10 +48,28 @@ class TestMongoModuleStore(object):
     def initdb():
         # connect to the db
         store = MongoModuleStore(HOST, DB, COLLECTION, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS)
+        # since MongoModuleStore and MongoContentStore are basically assumed to be together, create this class
+        # as well
+        content_store = MongoContentStore(HOST, DB)
+        #
+        # Also test draft store imports
+        #
+        draft_store = DraftModuleStore(HOST, DB, COLLECTION, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS)
         # Explicitly list the courses to load (don't want the big one)
-        courses = ['toy', 'simple']
-        import_from_xml(store, DATA_DIR, courses)
-        return store
+        courses = ['toy', 'simple', 'simple_with_draft']
+        import_from_xml(store, DATA_DIR, courses, draft_store=draft_store, static_content_store=content_store)
+
+        # also test a course with no importing of static content
+        import_from_xml(
+            store,
+            DATA_DIR,
+            ['test_import_course'],
+            static_content_store=content_store,
+            do_import_static=False,
+            verbose=True
+        )
+
+        return store, content_store, draft_store
 
     @staticmethod
     def destroy_db(connection):
@@ -77,10 +97,12 @@ class TestMongoModuleStore(object):
     def test_get_courses(self):
         '''Make sure the course objects loaded properly'''
         courses = self.store.get_courses()
-        assert_equals(len(courses), 2)
+        assert_equals(len(courses), 4)
         courses.sort(key=lambda c: c.id)
         assert_equals(courses[0].id, 'edX/simple/2012_Fall')
-        assert_equals(courses[1].id, 'edX/toy/2012_Fall')
+        assert_equals(courses[1].id, 'edX/simple_with_draft/2012_Fall')
+        assert_equals(courses[2].id, 'edX/test_import_course/2012_Fall')
+        assert_equals(courses[3].id, 'edX/toy/2012_Fall')
 
     def test_loads(self):
         assert_not_equals(
@@ -112,6 +134,13 @@ class TestMongoModuleStore(object):
         '''Make sure that path_to_location works'''
         check_path_to_location(self.store)
 
+    def test_xlinter(self):
+        '''
+        Run through the xlinter, we know the 'toy' course has violations, but the
+        number will continue to grow over time, so just check > 0
+        '''
+        assert_not_equals(perform_xlint(DATA_DIR, ['toy']), 0)
+
     def test_get_courses_has_no_templates(self):
         courses = self.store.get_courses()
         for course in courses:
@@ -129,7 +158,7 @@ class TestMongoModuleStore(object):
 
             Assumes the information is desired for courses[1] ('toy' course).
             """
-            return courses[1].tabs[index]['name']
+            return courses[2].tabs[index]['name']
 
         # There was a bug where model.save was not getting called after the static tab name
         # was set set for tabs that have a URL slug. 'Syllabus' and 'Resources' fall into that

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -51,7 +51,10 @@ def import_static_content(modules, course_loc, course_data_path, static_content_
                     content.thumbnail_location = thumbnail_location
 
                 #then commit the content
-                static_content_store.save(content)
+                try:
+                    static_content_store.save(content)
+                except Exception as err:
+                    log.exception('Error importing {0}, error={1}'.format(fullname_with_subpath, err))
 
                 #store the remapping information which will be needed to subsitute in the module data
                 remap_dict[fullname_with_subpath] = content_loc.name
@@ -64,7 +67,8 @@ def import_static_content(modules, course_loc, course_data_path, static_content_
 def import_from_xml(store, data_dir, course_dirs=None,
                     default_class='xmodule.raw_module.RawDescriptor',
                     load_error_modules=True, static_content_store=None, target_location_namespace=None,
-                    verbose=False, draft_store=None):
+                    verbose=False, draft_store=None,
+                    do_import_static=True):
     """
     Import the specified xml data_dir into the "store" modulestore,
     using org and course as the location org and course.
@@ -77,6 +81,10 @@ def import_from_xml(store, data_dir, course_dirs=None,
     expects a 'url_name' as an identifier to where things are on disk e.g. ../policies/<url_name>/policy.json as well as metadata keys in
     the policy.json. so we need to keep the original url_name during import
 
+    do_import_static: if False, then static files are not imported into the static content store.  This can be employed for courses which
+                      have substantial unchanging static content, which is to inefficient to import every time the course is loaded.
+                      Static content for some courses may also be served directly by nginx, instead of going through django.
+
     """
 
     xml_module_store = XMLModuleStore(
@@ -116,8 +124,17 @@ def import_from_xml(store, data_dir, course_dirs=None,
                     course_data_path = path(data_dir) / module.data_dir
                     course_location = module.location
 
+                    log.debug('======> IMPORTING course to location {0}'.format(course_location))
+
                     module = remap_namespace(module, target_location_namespace)
 
+                    if not do_import_static:
+                        module.lms.static_asset_path = module.data_dir			# for old-style xblock where this was actually linked to kvs
+                        module._model_data['static_asset_path'] = module.data_dir
+                        log.debug('course static_asset_path={0}'.format(module.lms.static_asset_path))
+
+                    log.debug('course data_dir={0}'.format(module.data_dir))
+
                     # cdodge: more hacks (what else). Seems like we have a problem when importing a course (like 6.002) which
                     # does not have any tabs defined in the policy file. The import goes fine and then displays fine in LMS,
                     # but if someone tries to add a new tab in the CMS, then the LMS barfs because it expects that -
@@ -129,18 +146,35 @@ def import_from_xml(store, data_dir, course_dirs=None,
                                        {"type": "wiki", "name": "Wiki"}]  # note, add 'progress' when we can support it on Edge
 
                     import_module(module, store, course_data_path, static_content_store, course_location,
-                                  target_location_namespace or course_location)
+                                  target_location_namespace or course_location, do_import_static=do_import_static)
 
                     course_items.append(module)
 
             # then import all the static content
-            if static_content_store is not None:
+            if static_content_store is not None and do_import_static:
                 _namespace_rename = target_location_namespace if target_location_namespace is not None else course_location
 
                 # first pass to find everything in /static/
                 import_static_content(xml_module_store.modules[course_id], course_location, course_data_path, static_content_store,
                                       _namespace_rename, subpath='static', verbose=verbose)
 
+            elif verbose and not do_import_static:
+                log.debug('Skipping import of static content, since do_import_static={0}'.format(do_import_static))
+
+            # no matter what do_import_static is, import "static_import" directory
+
+            # This is needed because the "about" pages (eg "overview") are loaded via load_extra_content, and
+            # do not inherit the lms metadata from the course module, and thus do not get "static_content_store"
+            # properly defined.   Static content referenced in those extra pages thus need to come through the
+            # c4x:// contentstore, unfortunately.  Tell users to copy that content into the "static_import" subdir.
+
+            simport = 'static_import'
+            if os.path.exists(course_data_path / simport):
+                _namespace_rename = target_location_namespace if target_location_namespace is not None else course_location
+
+                import_static_content(xml_module_store.modules[course_id], course_location, course_data_path, static_content_store,
+                                      _namespace_rename, subpath=simport, verbose=verbose)
+
             # finally loop through all the modules
             for module in xml_module_store.modules[course_id].itervalues():
                 if module.category == 'course':
@@ -156,7 +190,8 @@ def import_from_xml(store, data_dir, course_dirs=None,
                     log.debug('importing module location {0}'.format(module.location))
 
                 import_module(module, store, course_data_path, static_content_store, course_location,
-                              target_location_namespace if target_location_namespace else course_location)
+                              target_location_namespace if target_location_namespace else course_location,
+                              do_import_static=do_import_static)
 
             # now import any 'draft' items
             if draft_store is not None:
@@ -176,7 +211,8 @@ def import_from_xml(store, data_dir, course_dirs=None,
 
 
 def import_module(module, store, course_data_path, static_content_store,
-                  source_course_location, dest_course_location, allow_not_found=False):
+                  source_course_location, dest_course_location, allow_not_found=False,
+                  do_import_static=True):
 
     logging.debug('processing import of module {0}...'.format(module.location.url()))
 
@@ -196,7 +232,7 @@ def import_module(module, store, course_data_path, static_content_store,
     else:
         module_data = content
 
-    if isinstance(module_data, basestring):
+    if isinstance(module_data, basestring) and do_import_static:
         # we want to convert all 'non-portable' links in the module_data (if it is a string) to
         # portable strings (e.g. /static/)
         module_data = rewrite_nonportable_content_links(

common/test/data/simple_with_draft/README.md

+This is a simple, but non-trivial, course using multiple module types and some nested structure.
+ 

common/test/data/simple_with_draft/course.xml

+<course name="A Simple Course" org="edX" course="simple_with_draft" graceperiod="1 day 5 hours 59 minutes 59 seconds" slug="2012_Fall">
+  <chapter name="Overview">
+    <video name="Welcome" youtube_id_0_75="izygArpw-Qo" youtube_id_1_0="p2Q6BrNhdh8" youtube_id_1_25="1EeWXzPdhSA" youtube_id_1_5="rABDYkeK0x8"/>
+    <videosequence format="Lecture Sequence" name="A simple sequence">
+      <html name="toylab" filename="toylab"/>
+      <video name="S0V1: Video Resources" youtube_id_0_75="EuzkdzfR0i8" youtube_id_1_0="1bK-WdDi6Qw" youtube_id_1_25="0v1VzoDVUTM" youtube_id_1_5="Bxk_-ZJb240"/>
+    </videosequence>
+    <section name="Lecture 2">
+    <sequential>
+        <video youtube_id_1_0="TBvX7HzxexQ"/>
+        <problem name="L1 Problem 1" points="1" type="lecture" showanswer="attempted" filename="L1_Problem_1" rerandomize="never"/>
+      </sequential>
+    </section>
+  </chapter>
+  <chapter name="Chapter 2" url_name='chapter_2'>
+    <section name="Problem Set 1">
+      <sequential>
+        <problem type="lecture" showanswer="attempted" rerandomize="true" display_name="A simple coding problem" name="Simple coding problem" filename="ps01-simple" url_name="ps01-simple"/>
+      </sequential>
+    </section>
+    <video name="Lost Video" youtube_id_1_0="TBvX7HzxexQ"/>
+    <sequential format="Lecture Sequence" url_name='test_sequence'>
+        <vertical url_name='test_vertical'>
+            <html url_name='test_html'>
+                Foobar
+            </html>
+        </vertical>
+    </sequential>
+  </chapter>
+
+</course>

common/test/data/simple_with_draft/drafts/vertical/test_vertical.xml

+<vertical url_name='test_vertical' parent_sequential_url='i4x://edX/simple_with_draft/sequential/test_sequence' index_in_children_list="0">
+    <html url_name='test_html'>
+        Foobar - edit in draft
+    </html>
+</vertical>
\ No newline at end of file

common/test/data/simple_with_draft/html/toylab.html

+<b>Lab 2A: Superposition Experiment</b>
+
+<p>Isn't the toy course great?</p>

common/test/data/simple_with_draft/problem/L1_Problem_1.xml

+<?xml version="1.0"?>
+<problem>
+  <p>
+    <h1>Finger Exercise 1</h1>
+  </p>
+  <p>
+Here are two definitions: </p>
+  <ol class="enumerate">
+    <li>
+      <p>
+Declarative knowledge refers to statements of fact. </p>
+    </li>
+    <li>
+      <p>
+Imperative knowledge refers to 'how to' methods. </p>
+    </li>
+  </ol>
+  <p>
+Which of the following choices is correct? </p>
+  <ol class="enumerate">
+    <li>
+      <p>
+Statement 1 is true, Statement 2 is false </p>
+    </li>
+    <li>
+      <p>
+Statement 1 is false, Statement 2 is true </p>
+    </li>
+    <li>
+      <p>
+Statement 1 and Statement 2 are both false </p>
+    </li>
+    <li>
+      <p>
+Statement 1 and Statement 2 are both true </p>
+    </li>
+  </ol>
+  <p>
+    <symbolicresponse answer="4">
+      <textline size="90" math="1"/>
+    </symbolicresponse>
+  </p>
+</problem>

common/test/data/simple_with_draft/problem/ps01-simple.xml

+<problem><style media="all" type="text/css"/>
+<text><h2>Paying Off Credit Card Debt</h2>
+<p> Each month, a credit
+      card statement will come with the option for you to pay a
+      minimum amount of your charge, usually 2% of the balance due.
+      However, the credit card company earns money by charging
+      interest on the balance that you don't pay. So even if you
+      pay credit card payments on time, interest is still accruing
+      on the outstanding balance.</p>
+<p >Say you've made a
+      $5,000 purchase on a credit card with 18% annual interest
+      rate and 2% minimum monthly payment rate. After a year, how
+      much is the remaining balance? Use the following
+      equations.</p>
+<blockquote>
+<p><strong>Minimum monthly payment</strong>
+= (Minimum monthly payment rate) x (Balance)<br/>
+        (Minimum monthly payment gets split into interest paid and
+        principal paid)<br/>
+<strong>Interest Paid</strong> = (Annual interest rate) / (12
+        months) x (Balance)<br/>
+<strong>Principal paid</strong> = (Minimum monthly payment) -
+        (Interest paid)<br/>
+<strong>Remaining balance</strong> = Balance - (Principal
+        paid)</p>
+</blockquote>
+<p >For month 1, compute the minimum monthly payment by taking 2% of the balance.</p>
+<blockquote xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:v="urn:schemas-microsoft-com:vml" xmlns:x="urn:schemas-microsoft-com:office:excel">
+<p><strong>Minimum monthly payment</strong>
+= .02 x $5000 = $100</p>
+<p>We can't simply deduct this from the balance because
+        there is compounding interest. Of this $100 monthly
+        payment, compute how much will go to paying off interest
+        and how much will go to paying off the principal. Remember
+        that it's the annual interest rate that is given, so we
+        need to divide it by 12 to get the monthly interest
+        rate.</p>
+<p><strong>Interest paid</strong> = .18/12 x $5000 =
+        $75<br/>
+<strong>Principal paid</strong> = $100 - $75 = $25</p>
+<p>The remaining balance at the end of the first month will
+        be the principal paid this month subtracted from the
+        balance at the start of the month.</p>
+<p><strong>Remaining balance</strong> = $5000 - $25 =
+        $4975</p>
+</blockquote>
+<p xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:v="urn:schemas-microsoft-com:vml" xmlns:x="urn:schemas-microsoft-com:office:excel">For month 2, we
+      repeat the same steps.</p>
+<blockquote xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:v="urn:schemas-microsoft-com:vml" xmlns:x="urn:schemas-microsoft-com:office:excel">
+<p><strong>Minimum monthly payment</strong>
+= .02 x $4975 = $99.50<br/>
+<strong>Interest Paid</strong> = .18/12 x $4975 =
+        $74.63<br/>
+<strong>Principal Paid</strong> = $99.50 - $74.63 =
+        $24.87<br/>
+<strong>Remaining Balance</strong> = $4975 - $24.87 =
+        $4950.13</p>
+</blockquote>
+<p xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:v="urn:schemas-microsoft-com:vml" xmlns:x="urn:schemas-microsoft-com:office:excel">After 12 months, the
+      total amount paid is $1167.55, leaving an outstanding balance
+      of $4708.10. Pretty depressing!</p>
+</text></problem>

common/test/data/test_import_course/about/end_date.html

+TBD

common/test/data/test_import_course/chapter/vertical_container.xml

+<sequential>
+    <sequential filename='vertical_sequential' slug='vertical_sequential' />
+</sequential>
\ No newline at end of file

common/test/data/test_import_course/course.xml

+<course org="edX" course="test_import_course" url_name="2012_Fall"/>
\ No newline at end of file

common/test/data/test_import_course/course/2012_Fall.xml

+<course>
+    <textbook title="Textbook" book_url="https://s3.amazonaws.com/edx-textbooks/guttag_computation_v3/"/>
+  <chapter url_name="Overview">
+    <videosequence url_name="Toy_Videos">
+      <html url_name="secret:toylab"/>
+      <html url_name="toyjumpto"/>
+      <html url_name="toyhtml"/>
+      <html url_name="nonportable"/>
+      <html url_name="nonportable_link"/>
+      <video url_name="Video_Resources" youtube_id_1_0="1bK-WdDi6Qw" display_name="Video Resources"/>
+    </videosequence>
+    <video url_name="Welcome" youtube_id_1_0="p2Q6BrNhdh8" display_name="Welcome"/>
+    <video url_name="video_123456789012" youtube_id_1_0="p2Q6BrNhdh8" display_name='Test Video'/>
+    <video url_name="video_4f66f493ac8f" youtube_id_1_0="p2Q6BrNhdh8"/>
+  </chapter>
+  <chapter url_name="secret:magic"/>
+  <chapter url_name="poll_test"/>
+  <chapter url_name="vertical_container"/>
+  <chapter url_name="handout_container"/>
+</course>

common/test/data/test_import_course/info/handouts.html

+<a href='/static/handouts/sample_handout.txt'>Sample</a>
\ No newline at end of file

common/test/data/test_import_course/policies/2012_Fall.json

+{
+    "course/2012_Fall": {
+        "graceperiod": "2 days 5 hours 59 minutes 59 seconds",
+        "start": "2015-07-17T12:00",
+        "display_name": "Toy Course",
+        "graded": "true",
+	"tabs": [
+          {"type": "courseware"},
+          {"type": "course_info", "name": "Course Info"},
+          {"type": "static_tab", "url_slug": "syllabus", "name": "Syllabus"},
+          {"type": "static_tab", "url_slug": "resources", "name": "Resources"},
+          {"type": "discussion", "name": "Discussion"},
+          {"type": "wiki", "name": "Wiki"},
+          {"type": "progress", "name": "Progress"}
+        ]
+    },
+    "chapter/Overview": {
+        "display_name": "Overview"
+    },
+    "videosequence/Toy_Videos": {
+        "display_name": "Toy Videos",
+        "format": "Lecture Sequence"
+    },
+    "html/secret:toylab": {
+        "display_name": "Toy lab"
+    },
+    "video/Video_Resources": {
+        "display_name": "Video Resources"
+    },
+    "video/Welcome": {
+        "display_name": "Welcome"
+    }
+}

common/test/data/test_import_course/sequential/vertical_sequential.xml

+<sequential>
+    <vertical filename="vertical_test" slug="vertical_test" />
+    <html slug="unicode">…</html>
+</sequential>
\ No newline at end of file

common/test/data/test_import_course/static_import/should_be_imported.html

+<p>this file should be in the contentstore</p>

common/test/data/test_import_course/vertical/vertical_test.xml

+<sequential>
+        <video display_name="default" youtube_id_0_75="JMD_ifUUfsU" youtube_id_1_0="OEoXaMPEzfM" youtube_id_1_25="AKqURZnYqpk" youtube_id_1_5="DYpADpL7jAY" name="sample_video"/>
+        <video url_name="separate_file_video"/>
+        <poll_question name="T1_changemind_poll_foo_2" display_name="Change your answer" reset="false">
+        <p>Have you changed your mind?</p>
+        <answer id="yes">Yes</answer>
+        <answer id="no">No</answer>
+    </poll_question>
+</sequential>

common/test/data/test_import_course/video/separate_file_video.xml

+<video display_name="default" youtube_id_0_75="JMD_ifUUfsU" youtube_id_1_0="OEoXaMPEzfM" youtube_id_1_25="AKqURZnYqpk" youtube_id_1_5="DYpADpL7jAY" name="sample_video"/>

docs/internal/testing.md

@@ -136,6 +136,25 @@ To run a single nose test:
 
     nosetests common/lib/xmodule/xmodule/tests/test_stringify.py:test_stringify
 
+To run a single test and get stdout, with proper env config:
+
+    python manage.py cms --settings test test contentstore.tests.test_import_nostatic -s
+
+To run a single test and get stdout and get coverage:
+
+    python -m coverage run --rcfile=./common/lib/xmodule/.coveragerc which ./manage.py cms --settings test test --traceback --logging-clear-handlers --liveserver=localhost:8000-9000 contentstore.tests.test_import_nostatic -s # cms example 
+    python -m coverage run --rcfile=./lms/.coveragerc which ./manage.py lms --settings test test --traceback --logging-clear-handlers --liveserver=localhost:8000-9000  courseware.tests.test_module_render -s # lms example
+
+generate coverage report:
+
+    coverage report --rcfile=./common/lib/xmodule/.coveragerc 
+
+or to get html report:
+
+    coverage html --rcfile=./common/lib/xmodule/.coveragerc
+    
+then browse reports/common/lib/xmodule/cover/index.html    
+
 
 Very handy: if you uncomment the `pdb=1` line in `setup.cfg`, it will drop you into pdb on error.  This lets you go up and down the stack and see what the values of the variables are.  Check out [the pdb documentation](http://docs.python.org/library/pdb.html)
 

lms/djangoapps/courseware/courses.py

@@ -81,8 +81,8 @@ def get_opt_course_with_access(user, course_id, action):
 def course_image_url(course):
     """Try to look up the image url for the course.  If it's not found,
     log an error and return the dead link"""
-    if modulestore().get_modulestore_type(course.location.course_id) == XML_MODULESTORE_TYPE:
-        return '/static/' + course.data_dir + "/images/course_image.jpg"
+    if course.lms.static_asset_path or modulestore().get_modulestore_type(course.location.course_id) == XML_MODULESTORE_TYPE:
+        return '/static/' + (course.lms.static_asset_path or getattr(course, 'data_dir', '')) + "/images/course_image.jpg"
     else:
         loc = course.location._replace(tag='c4x', category='asset', name=course.course_image)
         _path = StaticContent.get_url_path_from_location(loc)
@@ -156,7 +156,8 @@ def get_course_about_section(course, section_key):
                 model_data_cache,
                 course.id,
                 not_found_ok=True,
-                wrap_xmodule_display=False
+                wrap_xmodule_display=False,
+                static_asset_path=course.lms.static_asset_path
             )
 
             html = ''
@@ -204,7 +205,8 @@ def get_course_info_section(request, course, section_key):
         loc,
         model_data_cache,
         course.id,
-        wrap_xmodule_display=False
+        wrap_xmodule_display=False,
+        static_asset_path=course.lms.static_asset_path
     )
 
     html = ''
@@ -242,7 +244,8 @@ def get_course_syllabus_section(course, section_key):
                 return replace_static_urls(
                     htmlFile.read().decode('utf-8'),
                     getattr(course, 'data_dir', None),
-                    course_id=course.location.course_id
+                    course_id=course.location.course_id,
+                    static_asset_path=course.lms.static_asset_path,
                 )
         except ResourceNotFoundError:
             log.exception("Missing syllabus section {key} in course {url}".format(

lms/djangoapps/courseware/module_render.py

@@ -124,7 +124,8 @@ def toc_for_course(user, request, course, active_chapter, active_section, model_
 
 def get_module(user, request, location, model_data_cache, course_id,
                position=None, not_found_ok=False, wrap_xmodule_display=True,
-               grade_bucket_type=None, depth=0):
+               grade_bucket_type=None, depth=0,
+               static_asset_path=''):
     """
     Get an instance of the xmodule class identified by location,
     setting the state based on an existing StudentModule, or creating one if none
@@ -141,6 +142,10 @@ def get_module(user, request, location, model_data_cache, course_id,
                                 position within module
       - depth                 : number of levels of descendents to cache when loading this module.
                                 None means cache all descendents
+      - static_asset_path     : static asset path to use (overrides descriptor's value); needed
+                                by get_course_info_section, because info section modules
+                                do not have a course as the parent module, and thus do not
+                                inherit this lms key value.
 
     Returns: xmodule instance, or None if the user does not have access to the
     module.  If there's an error, will try to return an instance of ErrorModule
@@ -152,7 +157,8 @@ def get_module(user, request, location, model_data_cache, course_id,
         return get_module_for_descriptor(user, request, descriptor, model_data_cache, course_id,
                                          position=position,
                                          wrap_xmodule_display=wrap_xmodule_display,
-                                         grade_bucket_type=grade_bucket_type)
+                                         grade_bucket_type=grade_bucket_type,
+                                         static_asset_path=static_asset_path)
     except ItemNotFoundError:
         if not not_found_ok:
             log.exception("Error in get_module")
@@ -179,7 +185,8 @@ def get_xqueue_callback_url_prefix(request):
 
 
 def get_module_for_descriptor(user, request, descriptor, model_data_cache, course_id,
-                              position=None, wrap_xmodule_display=True, grade_bucket_type=None):
+                              position=None, wrap_xmodule_display=True, grade_bucket_type=None,
+                              static_asset_path=''):
     """
     Implements get_module, extracting out the request-specific functionality.
 
@@ -194,12 +201,14 @@ def get_module_for_descriptor(user, request, descriptor, model_data_cache, cours
 
     return get_module_for_descriptor_internal(user, descriptor, model_data_cache, course_id,
                                               track_function, xqueue_callback_url_prefix,
-                                              position, wrap_xmodule_display, grade_bucket_type)
+                                              position, wrap_xmodule_display, grade_bucket_type,
+                                              static_asset_path)
 
 
 def get_module_for_descriptor_internal(user, descriptor, model_data_cache, course_id,
                                        track_function, xqueue_callback_url_prefix,
-                                       position=None, wrap_xmodule_display=True, grade_bucket_type=None):
+                                       position=None, wrap_xmodule_display=True, grade_bucket_type=None,
+                                       static_asset_path=''):
     """
     Actually implement get_module, without requiring a request.
 
@@ -282,7 +291,8 @@ def get_module_for_descriptor_internal(user, descriptor, model_data_cache, cours
         # inner_get_module, not the parent's callback.  Add it as an argument....
         return get_module_for_descriptor_internal(user, descriptor, model_data_cache, course_id,
                                                   track_function, make_xqueue_callback,
-                                                  position, wrap_xmodule_display, grade_bucket_type)
+                                                  position, wrap_xmodule_display, grade_bucket_type,
+                                                  static_asset_path)
 
     def xblock_model_data(descriptor):
         return DbModel(
@@ -349,6 +359,7 @@ def get_module_for_descriptor_internal(user, descriptor, model_data_cache, cours
             static_replace.replace_static_urls,
             data_directory=getattr(descriptor, 'data_dir', None),
             course_id=course_id,
+            static_asset_path=static_asset_path or descriptor.lms.static_asset_path,
         ),
         replace_course_urls=partial(
             static_replace.replace_course_urls,
@@ -407,7 +418,8 @@ def get_module_for_descriptor_internal(user, descriptor, model_data_cache, cours
     module.get_html = replace_static_urls(
         _get_html,
         getattr(descriptor, 'data_dir', None),
-        course_id=course_id
+        course_id=course_id,
+        static_asset_path=static_asset_path or descriptor.lms.static_asset_path
     )
 
     # Allow URLs of the form '/course/' refer to the root of multicourse directory

lms/djangoapps/courseware/tabs.py

@@ -380,7 +380,8 @@ def get_static_tab_contents(request, course, tab):
     loc = Location(course.location.tag, course.location.org, course.location.course, 'static_tab', tab['url_slug'])
     model_data_cache = ModelDataCache.cache_for_descriptor_descendents(course.id,
         request.user, modulestore().get_instance(course.id, loc), depth=0)
-    tab_module = get_module(request.user, request, loc, model_data_cache, course.id)
+    tab_module = get_module(request.user, request, loc, model_data_cache, course.id,
+                            static_asset_path=course.lms.static_asset_path)
 
     logging.debug('course_module = {0}'.format(tab_module))
 

lms/djangoapps/courseware/tests/test_module_render.py

@@ -19,7 +19,7 @@ from courseware.tests.tests import LoginEnrollmentTestCase, TEST_DATA_MONGO_MODU
 from courseware.model_data import ModelDataCache
 from modulestore_config import TEST_DATA_XML_MODULESTORE
 
-from courseware.courses import get_course_with_access
+from courseware.courses import get_course_with_access, course_image_url, get_course_info_section
 
 from .factories import UserFactory
 
@@ -83,7 +83,7 @@ class ModuleRenderTestCase(LoginEnrollmentTestCase):
 
         # See if the url got rewritten to the target link
         # note if the URL mapping changes then this assertion will break
-        self.assertIn('/courses/'+self.course_id+'/jump_to_id/vertical_test', html)
+        self.assertIn('/courses/' + self.course_id + '/jump_to_id/vertical_test', html)
 
     def test_modx_dispatch(self):
         self.assertRaises(Http404, render.modx_dispatch, 'dummy', 'dummy',
@@ -355,6 +355,38 @@ class TestHtmlModifiers(ModuleStoreTestCase):
             result_fragment.content
         )
 
+    def test_static_asset_path_use(self):
+        '''
+        when a course is loaded with do_import_static=False (see xml_importer.py), then
+        static_asset_path is set as an lms kv in course.  That should make static paths
+        not be mangled (ie not changed to c4x://).
+        '''
+        module = render.get_module(
+            self.user,
+            self.request,
+            self.location,
+            self.model_data_cache,
+            self.course.id,
+            static_asset_path="toy_course_dir",
+        )
+        result_fragment = module.runtime.render(module, None, 'student_view')
+        self.assertIn('href="/static/toy_course_dir', result_fragment.content)
+
+    def test_course_image(self):
+        url = course_image_url(self.course)
+        self.assertTrue(url.startswith('/c4x/'))
+
+        self.course.lms.static_asset_path = "toy_course_dir"
+        url = course_image_url(self.course)
+        self.assertTrue(url.startswith('/static/toy_course_dir/'))
+        self.course.lms.static_asset_path = ""
+
+    def test_get_course_info_section(self):
+        self.course.lms.static_asset_path = "toy_course_dir"
+        get_course_info_section(self.request, self.course, "handouts")
+        # NOTE: check handouts output...right now test course seems to have no such content
+        # at least this makes sure get_course_info_section returns without exception
+
     def test_course_link_rewrite(self):
         module = render.get_module(
             self.user,

lms/xmodule_namespace.py

@@ -56,3 +56,4 @@ class LmsNamespace(Namespace):
         default=None,
         scope=Scope.settings
     )
+    static_asset_path = String(help="Path to use for static assets - overrides Studio c4x://", scope=Scope.settings, default='')