Rejigger how /course and /static urls are replaced, to make the logic slightly more comprehensible

cms/djangoapps/contentstore/module_info_model.py

 import logging
-from static_replace import replace_urls
+from static_replace import replace_static_urls
 from xmodule.modulestore.exceptions import ItemNotFoundError
 from xmodule.modulestore import Location
 from xmodule.modulestore.django import modulestore
@@ -18,7 +18,17 @@ def get_module_info(store, location, parent_location = None, rewrite_static_link
 
   data = module.definition['data']
   if rewrite_static_links:
-    data = replace_urls(module.definition['data'], course_namespace = Location([module.location.tag, module.location.org, module.location.course, None, None]))
+    data = replace_static_urls(
+        module.definition['data'],
+        None,
+        course_namespace=Location([
+            module.location.tag,
+            module.location.org,
+            module.location.course,
+            None,
+            None
+        ])
+    )
 
   return {
         'id': module.location.url(),
@@ -47,7 +57,7 @@ def set_module_info(store, location, post_data):
   if post_data.get('data') is not None:
       data = post_data['data']
       store.update_item(location, data)
-     
+
   # cdodge: note calling request.POST.get('children') will return None if children is an empty array
   # so it lead to a bug whereby the last component to be deleted in the UI was not actually
   # deleting the children object from the children collection

cms/djangoapps/contentstore/views.py

@@ -31,7 +31,7 @@ from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationErr
 from xmodule.x_module import ModuleSystem
 from xmodule.error_module import ErrorDescriptor
 from xmodule.errortracker import exc_info_to_str
-from static_replace import replace_urls
+from static_replace import replace_static_urls
 from external_auth.views import ssl_login_shortcut
 
 from mitxmako.shortcuts import render_to_response, render_to_string
@@ -473,7 +473,7 @@ def preview_module_system(request, preview_id, descriptor):
         get_module=partial(get_preview_module, request, preview_id),
         render_template=render_from_lms,
         debug=True,
-        replace_urls=replace_urls,
+        replace_urls=partial(replace_static_urls, data_directory=None, course_namespace=descriptor.location),
         user=request.user,
     )
 
@@ -915,7 +915,7 @@ def reorder_static_tabs(request):
     # get list of existing static tabs in course
     # make sure they are the same lengths (i.e. the number of passed in tabs equals the number
     # that we know about) otherwise we can drop some!
-        
+
     existing_static_tabs = [t for t in course.tabs if t['type'] == 'static_tab']
     if len(existing_static_tabs) != len(tabs):
         return HttpResponseBadRequest()
@@ -934,15 +934,15 @@ def reorder_static_tabs(request):
     static_tab_idx = 0
     for tab in course.tabs:
         if tab['type'] == 'static_tab':
-            reordered_tabs.append({'type': 'static_tab', 
-                'name' : tab_items[static_tab_idx].metadata.get('display_name'), 
+            reordered_tabs.append({'type': 'static_tab',
+                'name' : tab_items[static_tab_idx].metadata.get('display_name'),
                 'url_slug' : tab_items[static_tab_idx].location.name})
             static_tab_idx += 1
         else:
             reordered_tabs.append(tab)
 
 
-    # OK, re-assemble the static tabs in the new order        
+    # OK, re-assemble the static tabs in the new order
     course.tabs = reordered_tabs
     modulestore('direct').update_metadata(course.location, course.metadata)
     return HttpResponse()

common/djangoapps/static_replace.py

@@ -11,6 +11,16 @@ from xmodule.contentstore.content import StaticContent
 
 log = logging.getLogger(__name__)
 
+
+def _url_replace_regex(prefix):
+    return r"""
+        (?x)                 # flags=re.VERBOSE
+        (?P<quote>\\?['"])   # the opening quotes
+        (?P<prefix>{prefix}) # theeprefix
+        (?P<rest>.*?)        # everything else in the url
+        (?P=quote)           # the first matching closing quote
+        """.format(prefix=prefix)
+
 def try_staticfiles_lookup(path):
     """
     Try to lookup a path in staticfiles_storage.  If it fails, return
@@ -26,48 +36,67 @@ def try_staticfiles_lookup(path):
     return url
 
 
-def replace(static_url, prefix=None, course_namespace=None):
-    if prefix is None:
-        prefix = ''
-    else:
-        prefix = prefix + '/'
+def replace_course_urls(text, course_id):
+    """
+    Replace /course/$stuff urls with /courses/$course_id/$stuff urls
 
-    quote = static_url.group('quote')
+    text: The text to replace
+    course_module: A CourseDescriptor
 
-    servable = (
-        # If in debug mode, we'll serve up anything that the finders can find
-        (settings.DEBUG and finders.find(static_url.group('rest'), True)) or
-        # Otherwise, we'll only serve up stuff that the storages can find
-        staticfiles_storage.exists(static_url.group('rest'))
-    )
+    returns: text with the links replaced
+    """
 
-    if servable:
-        return static_url.group(0)
-    else:
-        # don't error if file can't be found
-        # cdodge: to support the change over to Mongo backed content stores, lets
-        # use the utility functions in StaticContent.py
-        if static_url.group('prefix') == '/static/' and not isinstance(modulestore(), XMLModuleStore):
-            if course_namespace is None:
-                raise BaseException('You must pass in course_namespace when remapping static content urls with MongoDB stores')
-            url = StaticContent.convert_legacy_static_url(static_url.group('rest'), course_namespace)
-        else:
-            url = try_staticfiles_lookup(prefix + static_url.group('rest'))
 
-        new_link = "".join([quote, url, quote])
-        return new_link
+    def replace_course_url(match):
+        log.warning("Course match: %s", match.groupdict())
+        quote = match.group('quote')
+        rest = match.group('rest')
+        return "".join([quote, '/courses/' + course_id + '/', rest, quote])
 
+    return re.sub(_url_replace_regex('/courses/'), replace_course_url, text)
 
 
-def replace_urls(text, staticfiles_prefix=None, replace_prefix='/static/', course_namespace=None):
+def replace_static_urls(text, data_directory, course_namespace=None):
+    """
+    Replace /static/$stuff urls either with their correct url as generated by collectstatic,
+    (/static/$md5_hashed_stuff) or by the course-specific content static url
+    /static/$course_data_dir/$stuff, or, if course_namespace is not None, by the
+    correct url in the contentstore (c4x://)
 
-    def replace_url(static_url):
-        return replace(static_url, staticfiles_prefix, course_namespace = course_namespace)
+    text: The source text to do the substitution in
+    data_directory: The directory in which course data is stored
+    course_namespace: The course identifier used to distinguish static content for this course in studio
+    """
 
-    return re.sub(r"""
-        (?x)                 # flags=re.VERBOSE
-        (?P<quote>\\?['"])   # the opening quotes
-        (?P<prefix>{prefix}) # the prefix
-        (?P<rest>.*?)        # everything else in the url
-        (?P=quote)           # the first matching closing quote
-        """.format(prefix=replace_prefix), replace_url, text)
+    def replace_static_url(match):
+        log.warning(match.groupdict())
+        quote = match.group('quote')
+        rest = match.group('rest')
+
+        # course_namespace is not None, then use studio style urls
+        if course_namespace is not None and not isinstance(modulestore(), XMLModuleStore):
+            url = StaticContent.convert_legacy_static_url(rest, course_namespace)
+            log.warning("From modulestore: %s", url)
+        # If we're in debug mode, and the file as requested exists, then don't change the links
+        elif (settings.DEBUG and finders.find(rest, True)):
+            url = match.group('prefix') + rest
+            log.warning("From finder: %s", url)
+        # Otherwise, look the file up in staticfiles_storage
+        else:
+            try:
+                url = staticfiles_storage.url(data_directory + '/' + rest)
+                log.warning("From staticfiles_storage: %s", url)
+            # And if that fails, return the path unmodified
+            except Exception as err:
+                log.warning("staticfiles_storage couldn't find path {0}: {1}".format(
+                    path, str(err)))
+                url = path
+                log.warning("Fallback: %s", url)
+        log.warning("".join([quote, url, quote]))
+        return "".join([quote, url, quote])
+
+    return re.sub(
+        _url_replace_regex('/static/(?!{data_dir}'.format(data_dir=data_directory)),
+        replace_static_url,
+        text
+    )

common/djangoapps/xmodule_modifiers.py

@@ -2,10 +2,10 @@ import re
 import json
 import logging
 import time
+import static_replace
 
 from django.conf import settings
 from functools import wraps
-from static_replace import replace_urls
 from mitxmako.shortcuts import render_to_string
 from xmodule.seq_module import SequenceModule
 from xmodule.vertical_module import VerticalModule
@@ -49,10 +49,10 @@ def replace_course_urls(get_html, course_id):
     """
     @wraps(get_html)
     def _get_html():
-        return replace_urls(get_html(), staticfiles_prefix='/courses/'+course_id, replace_prefix='/course/')
+        return static_replace.replace_course_urls(get_html(), course_id)
     return _get_html
 
-def replace_static_urls(get_html, prefix, course_namespace=None):
+def replace_static_urls(get_html, data_dir, course_namespace=None):
     """
     Updates the supplied module with a new get_html function that wraps
     the old get_html function and substitutes urls of the form /static/...
@@ -61,7 +61,7 @@ def replace_static_urls(get_html, prefix, course_namespace=None):
 
     @wraps(get_html)
     def _get_html():
-        return replace_urls(get_html(), staticfiles_prefix=prefix, course_namespace = course_namespace)
+        return static_replace.replace_static_urls(get_html(), data_dir, course_namespace)
     return _get_html
 
 

common/lib/xmodule/xmodule/video_module.py

@@ -6,7 +6,7 @@ from pkg_resources import resource_string, resource_listdir
 
 from xmodule.x_module import XModule
 from xmodule.raw_module import RawDescriptor
-from xmodule.modulestore.mongo import MongoModuleStore
+from xmodule.modulestore.xml import XMLModuleStore
 from xmodule.modulestore.django import modulestore
 from xmodule.contentstore.content import StaticContent
 
@@ -121,12 +121,12 @@ class VideoModule(XModule):
         return self.youtube
 
     def get_html(self):
-        if isinstance(modulestore(), MongoModuleStore) :
-            caption_asset_path = StaticContent.get_base_url_path_for_course_assets(self.location) + '/subs_'
-        else:
+        if isinstance(modulestore(), XMLModuleStore) :
             # VS[compat]
             # cdodge: filesystem static content support.
             caption_asset_path = "/static/{0}/subs/".format(self.metadata['data_dir'])
+        else:
+            caption_asset_path = StaticContent.get_base_url_path_for_course_assets(self.location) + '/subs_'
 
         return self.system.render_template('video.html', {
             'streams': self.video_list(),

lms/djangoapps/courseware/courses.py

@@ -19,7 +19,7 @@ from xmodule.contentstore.content import StaticContent
 from xmodule.modulestore.xml import XMLModuleStore
 from xmodule.modulestore.exceptions import ItemNotFoundError
 from xmodule.x_module import XModule
-from static_replace import replace_urls, try_staticfiles_lookup
+from static_replace import replace_static_urls
 from courseware.access import has_access
 import branding
 from courseware.models import StudentModuleCache
@@ -223,8 +223,11 @@ def get_course_syllabus_section(course, section_key):
             dirs = [path("syllabus") / course.url_name, path("syllabus")]
             filepath = find_file(fs, dirs, section_key + ".html")
             with fs.open(filepath) as htmlFile:
-                return replace_urls(htmlFile.read().decode('utf-8'),
-                                    course.metadata['data_dir'], course_namespace=course.location)
+                return replace_static_urls(
+                    htmlFile.read().decode('utf-8'),
+                    course.metadata['data_dir'],
+                    course_namespace=course.location
+                )
         except ResourceNotFoundError:
             log.exception("Missing syllabus section {key} in course {url}".format(
                 key=section_key, url=course.location.url()))

lms/djangoapps/courseware/module_render.py

@@ -2,6 +2,7 @@ import json
 import logging
 import pyparsing
 import sys
+import static_replace
 
 from functools import partial
 
@@ -20,7 +21,6 @@ from courseware.access import has_access
 from mitxmako.shortcuts import render_to_string
 from models import StudentModule, StudentModuleCache
 from psychometrics.psychoanalyze import make_psychometrics_data_update_handler
-from static_replace import replace_urls
 from student.models import unique_id_for_user
 from xmodule.errortracker import exc_info_to_str
 from xmodule.exceptions import NotFoundError
@@ -247,8 +247,8 @@ def _get_module(user, request, descriptor, student_module_cache, course_id,
                           # a module is coming through get_html and is therefore covered
                           # by the replace_static_urls code below
                           replace_urls=partial(
-                              replace_urls,
-                              staticfiles_prefix='/static/' + descriptor.metadata.get('data_dir', ''),
+                              static_replace.replace_static_urls,
+                              data_directory=descriptor.metadata.get('data_dir', ''),
                               course_namespace=descriptor.location._replace(category=None, name=None),
                           ),
                           node_path=settings.NODE_PATH,

lms/djangoapps/courseware/tabs.py

@@ -24,7 +24,6 @@ from static_replace import replace_urls
 from lxml.html import rewrite_links
 from module_render import get_module
 from courseware.access import has_access
-from static_replace import replace_urls
 from xmodule.modulestore import Location
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.xml import XMLModuleStore
@@ -322,4 +321,4 @@ def get_static_tab_contents(request, cache, course, tab):
     if tab_module is not None:
         html = tab_module.get_html()
 
-    return html
\ No newline at end of file
+    return html