Merge pull request #1397 from MITx/feature/cale/no-course-collectstatic

Free courses from the confines of collectstatic

cms/.coveragerc

 # .coveragerc for cms
 [run]
 data_file = reports/cms/.coverage
-source = cms
+source = cms,common/djangoapps
 omit = cms/envs/*, cms/manage.py
 
 [report]

cms/djangoapps/contentstore/module_info_model.py

 import logging
-from static_replace import replace_urls
+from static_replace import replace_static_urls
 from xmodule.modulestore.exceptions import ItemNotFoundError
 from xmodule.modulestore import Location
 from xmodule.modulestore.django import modulestore
@@ -18,7 +18,17 @@ def get_module_info(store, location, parent_location = None, rewrite_static_link
 
   data = module.definition['data']
   if rewrite_static_links:
-    data = replace_urls(module.definition['data'], course_namespace = Location([module.location.tag, module.location.org, module.location.course, None, None]))
+    data = replace_static_urls(
+        module.definition['data'],
+        None,
+        course_namespace=Location([
+            module.location.tag,
+            module.location.org,
+            module.location.course,
+            None,
+            None
+        ])
+    )
 
   return {
         'id': module.location.url(),
@@ -47,7 +57,7 @@ def set_module_info(store, location, post_data):
   if post_data.get('data') is not None:
       data = post_data['data']
       store.update_item(location, data)
-     
+
   # cdodge: note calling request.POST.get('children') will return None if children is an empty array
   # so it lead to a bug whereby the last component to be deleted in the UI was not actually
   # deleting the children object from the children collection

cms/envs/common.py

@@ -285,4 +285,5 @@ INSTALLED_APPS = (
     # For asset pipelining
     'pipeline',
     'staticfiles',
+    'static_replace',
 )

common/djangoapps/static_replace.py

-import logging
-import re
-
-from staticfiles.storage import staticfiles_storage
-from staticfiles import finders
-from django.conf import settings
-
-from xmodule.modulestore.django import modulestore
-from xmodule.modulestore.xml import XMLModuleStore
-from xmodule.contentstore.content import StaticContent
-
-log = logging.getLogger(__name__)
-
-def try_staticfiles_lookup(path):
-    """
-    Try to lookup a path in staticfiles_storage.  If it fails, return
-    a dead link instead of raising an exception.
-    """
-    try:
-        url = staticfiles_storage.url(path)
-    except Exception as err:
-        log.warning("staticfiles_storage couldn't find path {0}: {1}".format(
-            path, str(err)))
-        # Just return the original path; don't kill everything.
-        url = path
-    return url
-
-
-def replace(static_url, prefix=None, course_namespace=None):
-    if prefix is None:
-        prefix = ''
-    else:
-        prefix = prefix + '/'
-
-    quote = static_url.group('quote')
-
-    servable = (
-        # If in debug mode, we'll serve up anything that the finders can find
-        (settings.DEBUG and finders.find(static_url.group('rest'), True)) or
-        # Otherwise, we'll only serve up stuff that the storages can find
-        staticfiles_storage.exists(static_url.group('rest'))
-    )
-
-    if servable:
-        return static_url.group(0)
-    else:
-        # don't error if file can't be found
-        # cdodge: to support the change over to Mongo backed content stores, lets
-        # use the utility functions in StaticContent.py
-        if static_url.group('prefix') == '/static/' and not isinstance(modulestore(), XMLModuleStore):
-            if course_namespace is None:
-                raise BaseException('You must pass in course_namespace when remapping static content urls with MongoDB stores')
-            url = StaticContent.convert_legacy_static_url(static_url.group('rest'), course_namespace)
-        else:
-            url = try_staticfiles_lookup(prefix + static_url.group('rest'))
-
-        new_link = "".join([quote, url, quote])
-        return new_link
-
-
-
-def replace_urls(text, staticfiles_prefix=None, replace_prefix='/static/', course_namespace=None):
-
-    def replace_url(static_url):
-        return replace(static_url, staticfiles_prefix, course_namespace = course_namespace)
-
-    return re.sub(r"""
-        (?x)                 # flags=re.VERBOSE
-        (?P<quote>\\?['"])   # the opening quotes
-        (?P<prefix>{prefix}) # the prefix
-        (?P<rest>.*?)        # everything else in the url
-        (?P=quote)           # the first matching closing quote
-        """.format(prefix=replace_prefix), replace_url, text)

common/djangoapps/static_replace/__init__.py

+import logging
+import re
+
+from staticfiles.storage import staticfiles_storage
+from staticfiles import finders
+from django.conf import settings
+
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.xml import XMLModuleStore
+from xmodule.contentstore.content import StaticContent
+
+log = logging.getLogger(__name__)
+
+
+def _url_replace_regex(prefix):
+    return r"""
+        (?x)                 # flags=re.VERBOSE
+        (?P<quote>\\?['"])   # the opening quotes
+        (?P<prefix>{prefix}) # theeprefix
+        (?P<rest>.*?)        # everything else in the url
+        (?P=quote)           # the first matching closing quote
+        """.format(prefix=prefix)
+
+def try_staticfiles_lookup(path):
+    """
+    Try to lookup a path in staticfiles_storage.  If it fails, return
+    a dead link instead of raising an exception.
+    """
+    try:
+        url = staticfiles_storage.url(path)
+    except Exception as err:
+        log.warning("staticfiles_storage couldn't find path {0}: {1}".format(
+            path, str(err)))
+        # Just return the original path; don't kill everything.
+        url = path
+    return url
+
+
+def replace_course_urls(text, course_id):
+    """
+    Replace /course/$stuff urls with /courses/$course_id/$stuff urls
+
+    text: The text to replace
+    course_module: A CourseDescriptor
+
+    returns: text with the links replaced
+    """
+
+
+    def replace_course_url(match):
+        quote = match.group('quote')
+        rest = match.group('rest')
+        return "".join([quote, '/courses/' + course_id + '/', rest, quote])
+
+    return re.sub(_url_replace_regex('/course/'), replace_course_url, text)
+
+
+def replace_static_urls(text, data_directory, course_namespace=None):
+    """
+    Replace /static/$stuff urls either with their correct url as generated by collectstatic,
+    (/static/$md5_hashed_stuff) or by the course-specific content static url
+    /static/$course_data_dir/$stuff, or, if course_namespace is not None, by the
+    correct url in the contentstore (c4x://)
+
+    text: The source text to do the substitution in
+    data_directory: The directory in which course data is stored
+    course_namespace: The course identifier used to distinguish static content for this course in studio
+    """
+
+    def replace_static_url(match):
+        original = match.group(0)
+        prefix = match.group('prefix')
+        quote = match.group('quote')
+        rest = match.group('rest')
+
+        # course_namespace is not None, then use studio style urls
+        if course_namespace is not None and not isinstance(modulestore(), XMLModuleStore):
+            url = StaticContent.convert_legacy_static_url(rest, course_namespace)
+        # If we're in debug mode, and the file as requested exists, then don't change the links
+        elif (settings.DEBUG and finders.find(rest, True)):
+            return original
+        # Otherwise, look the file up in staticfiles_storage without the data directory
+        else:
+            try:
+                url = staticfiles_storage.url(rest)
+            # And if that fails, assume that it's course content, and add manually data directory
+            except Exception as err:
+                log.warning("staticfiles_storage couldn't find path {0}: {1}".format(
+                    rest, str(err)))
+                url = "".join([prefix, data_directory, '/', rest])
+
+        return "".join([quote, url, quote])
+
+    return re.sub(
+        _url_replace_regex('/static/(?!{data_dir})'.format(data_dir=data_directory)),
+        replace_static_url,
+        text
+    )

common/djangoapps/static_replace/management/commands/clear_collectstatic_cache.py

+###
+### Script for importing courseware from XML format
+###
+
+from django.core.management.base import NoArgsCommand
+from django.core.cache import get_cache
+
+
+class Command(NoArgsCommand):
+    help = \
+'''Import the specified data directory into the default ModuleStore'''
+
+    def handle_noargs(self, **options):
+        staticfiles_cache = get_cache('staticfiles')
+        staticfiles_cache.clear()

common/djangoapps/static_replace/test/test_static_replace.py

+from nose.tools import assert_equals
+from static_replace import replace_static_urls, replace_course_urls
+from mock import patch, Mock
+from xmodule.modulestore import Location
+from xmodule.modulestore.mongo import MongoModuleStore
+from xmodule.modulestore.xml import XMLModuleStore
+
+DATA_DIRECTORY = 'data_dir'
+COURSE_ID = 'org/course/run'
+NAMESPACE = Location('org', 'course', 'run', None, None)
+STATIC_SOURCE = '"/static/file.png"'
+
+
+def test_multi_replace():
+    course_source = '"/course/file.png"'
+
+    assert_equals(
+        replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY),
+        replace_static_urls(replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY), DATA_DIRECTORY)
+    )
+    assert_equals(
+        replace_course_urls(course_source, COURSE_ID),
+        replace_course_urls(replace_course_urls(course_source, COURSE_ID), COURSE_ID)
+    )
+
+
+@patch('static_replace.finders')
+@patch('static_replace.settings')
+def test_debug_no_modify(mock_settings, mock_finders):
+    mock_settings.DEBUG = True
+    mock_finders.find.return_value = True
+
+    assert_equals(STATIC_SOURCE, replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY))
+
+    mock_finders.find.assert_called_once_with('file.png', True)
+
+
+@patch('static_replace.StaticContent')
+@patch('static_replace.modulestore')
+def test_mongo_filestore(mock_modulestore, mock_static_content):
+
+    mock_modulestore.return_value = Mock(MongoModuleStore)
+    mock_static_content.convert_legacy_static_url.return_value = "c4x://mock_url"
+
+    # No namespace => no change to path
+    assert_equals('"/static/data_dir/file.png"', replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY))
+
+    # Namespace => content url
+    assert_equals(
+        '"' + mock_static_content.convert_legacy_static_url.return_value + '"',
+        replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY, NAMESPACE)
+    )
+
+    mock_static_content.convert_legacy_static_url.assert_called_once_with('file.png', NAMESPACE)
+
+@patch('static_replace.settings')
+@patch('static_replace.modulestore')
+@patch('static_replace.staticfiles_storage')
+def test_data_dir_fallback(mock_storage, mock_modulestore, mock_settings):
+    mock_modulestore.return_value = Mock(XMLModuleStore)
+    mock_settings.DEBUG = False
+    mock_storage.url.side_effect = Exception
+
+    assert_equals('"/static/data_dir/file.png"', replace_static_urls(STATIC_SOURCE, DATA_DIRECTORY))

common/djangoapps/xmodule_modifiers.py

@@ -2,10 +2,10 @@ import re
 import json
 import logging
 import time
+import static_replace
 
 from django.conf import settings
 from functools import wraps
-from static_replace import replace_urls
 from mitxmako.shortcuts import render_to_string
 from xmodule.seq_module import SequenceModule
 from xmodule.vertical_module import VerticalModule
@@ -49,10 +49,10 @@ def replace_course_urls(get_html, course_id):
     """
     @wraps(get_html)
     def _get_html():
-        return replace_urls(get_html(), staticfiles_prefix='/courses/'+course_id, replace_prefix='/course/')
+        return static_replace.replace_course_urls(get_html(), course_id)
     return _get_html
 
-def replace_static_urls(get_html, prefix, course_namespace=None):
+def replace_static_urls(get_html, data_dir, course_namespace=None):
     """
     Updates the supplied module with a new get_html function that wraps
     the old get_html function and substitutes urls of the form /static/...
@@ -61,7 +61,7 @@ def replace_static_urls(get_html, prefix, course_namespace=None):
 
     @wraps(get_html)
     def _get_html():
-        return replace_urls(get_html(), staticfiles_prefix=prefix, course_namespace = course_namespace)
+        return static_replace.replace_static_urls(get_html(), data_dir, course_namespace)
     return _get_html
 
 

common/lib/xmodule/xmodule/capa_module.py

@@ -369,7 +369,7 @@ class CapaModule(XModule):
                 id=self.location.html_id(), ajax_url=self.system.ajax_url) + html + "</div>"
 
         # now do the substitutions which are filesystem based, e.g. '/static/' prefixes
-        return self.system.replace_urls(html, self.metadata['data_dir'], course_namespace=self.location)
+        return self.system.replace_urls(html)
 
     def handle_ajax(self, dispatch, get):
         '''
@@ -490,7 +490,7 @@ class CapaModule(XModule):
         new_answers = dict()
         for answer_id in answers:
             try:
-                new_answer = {answer_id: self.system.replace_urls(answers[answer_id], self.metadata['data_dir'], course_namespace=self.location)}
+                new_answer = {answer_id: self.system.replace_urls(answers[answer_id])}
             except TypeError:
                 log.debug('Unable to perform URL substitution on answers[%s]: %s' % (answer_id, answers[answer_id]))
                 new_answer = {answer_id: answers[answer_id]}

common/lib/xmodule/xmodule/video_module.py

@@ -6,7 +6,7 @@ from pkg_resources import resource_string, resource_listdir
 
 from xmodule.x_module import XModule
 from xmodule.raw_module import RawDescriptor
-from xmodule.modulestore.mongo import MongoModuleStore
+from xmodule.modulestore.xml import XMLModuleStore
 from xmodule.modulestore.django import modulestore
 from xmodule.contentstore.content import StaticContent
 
@@ -121,12 +121,12 @@ class VideoModule(XModule):
         return self.youtube
 
     def get_html(self):
-        if isinstance(modulestore(), MongoModuleStore) :
-            caption_asset_path = StaticContent.get_base_url_path_for_course_assets(self.location) + '/subs_'
-        else:
+        if isinstance(modulestore(), XMLModuleStore) :
             # VS[compat]
             # cdodge: filesystem static content support.
             caption_asset_path = "/static/{0}/subs/".format(self.metadata['data_dir'])
+        else:
+            caption_asset_path = StaticContent.get_base_url_path_for_course_assets(self.location) + '/subs_'
 
         return self.system.render_template('video.html', {
             'streams': self.video_list(),

lms/.coveragerc

 # .coveragerc for lms
 [run]
 data_file = reports/lms/.coverage
-source = lms
+source = lms,common/djangoapps
 omit = lms/envs/*
 
 [report]

lms/djangoapps/courseware/courses.py

@@ -19,7 +19,7 @@ from xmodule.contentstore.content import StaticContent
 from xmodule.modulestore.xml import XMLModuleStore
 from xmodule.modulestore.exceptions import ItemNotFoundError
 from xmodule.x_module import XModule
-from static_replace import replace_urls, try_staticfiles_lookup
+from static_replace import replace_static_urls
 from courseware.access import has_access
 import branding
 from courseware.models import StudentModuleCache
@@ -83,13 +83,12 @@ def get_opt_course_with_access(user, course_id, action):
         return None
     return get_course_with_access(user, course_id, action)
 
-    
+
 def course_image_url(course):
     """Try to look up the image url for the course.  If it's not found,
     log an error and return the dead link"""
     if isinstance(modulestore(), XMLModuleStore):
-        path = course.metadata['data_dir'] + "/images/course_image.jpg"
-        return try_staticfiles_lookup(path)
+        return '/static/' + course.metadata['data_dir'] + "/images/course_image.jpg"
     else:
         loc = course.location._replace(tag='c4x', category='asset', name='images_course_image.jpg')
         path = StaticContent.get_url_path_from_location(loc)
@@ -224,8 +223,11 @@ def get_course_syllabus_section(course, section_key):
             dirs = [path("syllabus") / course.url_name, path("syllabus")]
             filepath = find_file(fs, dirs, section_key + ".html")
             with fs.open(filepath) as htmlFile:
-                return replace_urls(htmlFile.read().decode('utf-8'),
-                                    course.metadata['data_dir'], course_namespace=course.location)
+                return replace_static_urls(
+                    htmlFile.read().decode('utf-8'),
+                    course.metadata['data_dir'],
+                    course_namespace=course.location
+                )
         except ResourceNotFoundError:
             log.exception("Missing syllabus section {key} in course {url}".format(
                 key=section_key, url=course.location.url()))

lms/djangoapps/courseware/module_render.py

@@ -2,6 +2,9 @@ import json
 import logging
 import pyparsing
 import sys
+import static_replace
+
+from functools import partial
 
 from django.conf import settings
 from django.contrib.auth.models import User
@@ -18,7 +21,6 @@ from courseware.access import has_access
 from mitxmako.shortcuts import render_to_string
 from models import StudentModule, StudentModuleCache
 from psychometrics.psychoanalyze import make_psychometrics_data_update_handler
-from static_replace import replace_urls
 from student.models import unique_id_for_user
 from xmodule.errortracker import exc_info_to_str
 from xmodule.exceptions import NotFoundError
@@ -244,7 +246,11 @@ def _get_module(user, request, descriptor, student_module_cache, course_id,
                           # TODO (cpennington): This should be removed when all html from
                           # a module is coming through get_html and is therefore covered
                           # by the replace_static_urls code below
-                          replace_urls=replace_urls,
+                          replace_urls=partial(
+                              static_replace.replace_static_urls,
+                              data_directory=descriptor.metadata.get('data_dir', ''),
+                              course_namespace=descriptor.location._replace(category=None, name=None),
+                          ),
                           node_path=settings.NODE_PATH,
                           anonymous_student_id=unique_id_for_user(user),
                           course_id=course_id,
@@ -280,7 +286,7 @@ def _get_module(user, request, descriptor, student_module_cache, course_id,
 
     module.get_html = replace_static_urls(
         _get_html,
-        module.metadata['data_dir'] if 'data_dir' in module.metadata else '', 
+        module.metadata.get('data_dir', ''),
         course_namespace = module.location._replace(category=None, name=None))
 
     # Allow URLs of the form '/course/' refer to the root of multicourse directory

lms/djangoapps/courseware/tabs.py

@@ -19,12 +19,10 @@ from django.core.urlresolvers import reverse
 from fs.errors import ResourceNotFoundError
 
 from courseware.access import has_access
-from static_replace import replace_urls
 
 from lxml.html import rewrite_links
 from module_render import get_module
 from courseware.access import has_access
-from static_replace import replace_urls
 from xmodule.modulestore import Location
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.xml import XMLModuleStore
@@ -322,4 +320,4 @@ def get_static_tab_contents(request, cache, course, tab):
     if tab_module is not None:
         html = tab_module.get_html()
 
-    return html
\ No newline at end of file
+    return html

lms/envs/common.py

@@ -266,24 +266,6 @@ STATICFILES_DIRS = [
     COMMON_ROOT / "static",
     PROJECT_ROOT / "static",
 ]
-if os.path.isdir(DATA_DIR):
-    # Add the full course repo if there is no static directory
-    STATICFILES_DIRS += [
-        # TODO (cpennington): When courses are stored in a database, this
-        # should no longer be added to STATICFILES
-        (course_dir, DATA_DIR / course_dir)
-        for course_dir in os.listdir(DATA_DIR)
-        if (os.path.isdir(DATA_DIR / course_dir) and
-            not os.path.isdir(DATA_DIR / course_dir / 'static'))
-    ]
-    # Otherwise, add only the static directory from the course dir
-    STATICFILES_DIRS += [
-        # TODO (cpennington): When courses are stored in a database, this
-        # should no longer be added to STATICFILES
-        (course_dir, DATA_DIR / course_dir / 'static')
-        for course_dir in os.listdir(DATA_DIR)
-        if (os.path.isdir(DATA_DIR / course_dir / 'static'))
-    ]
 
 # Locale/Internationalization
 TIME_ZONE = 'America/New_York' # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
@@ -566,6 +548,7 @@ INSTALLED_APPS = (
     # For asset pipelining
     'pipeline',
     'staticfiles',
+    'static_replace',
 
     # Our courseware
     'circuit',

lms/envs/dev.py

@@ -106,6 +106,27 @@ VIRTUAL_UNIVERSITIES = []
 
 COMMENTS_SERVICE_KEY = "PUT_YOUR_API_KEY_HERE"
 
+############################## Course static files ##########################
+if os.path.isdir(DATA_DIR):
+    # Add the full course repo if there is no static directory
+    STATICFILES_DIRS += [
+        # TODO (cpennington): When courses are stored in a database, this
+        # should no longer be added to STATICFILES
+        (course_dir, DATA_DIR / course_dir)
+        for course_dir in os.listdir(DATA_DIR)
+        if (os.path.isdir(DATA_DIR / course_dir) and
+            not os.path.isdir(DATA_DIR / course_dir / 'static'))
+    ]
+    # Otherwise, add only the static directory from the course dir
+    STATICFILES_DIRS += [
+        # TODO (cpennington): When courses are stored in a database, this
+        # should no longer be added to STATICFILES
+        (course_dir, DATA_DIR / course_dir / 'static')
+        for course_dir in os.listdir(DATA_DIR)
+        if (os.path.isdir(DATA_DIR / course_dir / 'static'))
+    ]
+
+
 ################################# mitx revision string  #####################
 
 MITX_VERSION_STRING = os.popen('cd %s; git describe' % REPO_ROOT).read().strip()