Support asset library references in CapaModule. Also some code…

Support asset library references in CapaModule. Also some code refactoring/cleanup. Preparing for review and submit

cms/djangoapps/contentstore/views.py

@@ -29,11 +29,11 @@ from itertools import groupby
 from operator import attrgetter
 
 from xmodule.contentstore.django import contentstore
-from xmodule.contentstore import StaticContent
+from xmodule.contentstore.content import StaticContent
 
 #from django.core.cache import cache
 
-from cache_toolbox.core import set_cached_content, get_cached_content
+from cache_toolbox.core import set_cached_content, get_cached_content, del_cached_content
 
 log = logging.getLogger(__name__)
 
@@ -450,10 +450,15 @@ def upload_asset(request, org, course, coursename):
     content = StaticContent(file_location, name, mime_type, filedata)
 
     # first commit to the DB
-    contentstore().update(content)
-
-    # then update the cache so we're not serving up stale content
-    set_cached_content(content)
+    contentstore().save(content)
+
+    # then remove the cache so we're not serving up stale content
+    # NOTE: we're not re-populating the cache here as the DB owns the last-modified timestamp
+    # which is used when serving up static content. This integrity is needed for
+    # browser-side caching support. We *could* re-fetch the saved content so that we have the
+    # timestamp populated, but we might as well wait for the first real request to come in
+    # to re-populate the cache.
+    del_cached_content(file_location)
 
     return HttpResponse('Upload completed')
 

cms/templates/widgets/upload_assets.html

@@ -6,7 +6,7 @@
       <input type="submit" value="Upload File">
     </form>
   </div>
-  <div class="asset-upload-progress" style="position:relative; width:400px; border: 1px solid #ddd; padding: 1px; border-radius: 3px;">
+  <div class="progress" style="position:relative; width:400px; border: 1px solid #ddd; padding: 1px; border-radius: 3px;">
     <div class="bar" style="background-color: #B4F5B4; width:0%; height:20px; border-radius: 3px;"></div>
     <div class="percent">0%</div>
   </div>

common/djangoapps/cache_toolbox/core.py

@@ -10,7 +10,6 @@ Core methods
 
 from django.core.cache import cache
 from django.db import DEFAULT_DB_ALIAS
-from xmodule.contentstore import StaticContent
 
 from . import app_settings
 
@@ -118,9 +117,5 @@ def set_cached_content(content):
 def get_cached_content(filename):
     return cache.get(content_key(filename))
 
-
-#def set_cached_content(filename, content_type, data):
-#    cache.set(content_key(filename), (filename, content_type, data))
-
-#def get_cached_content(filename):
-#    return cache.get(content_key(filename))
+def del_cached_content(filename):
+    cache.delete(content_key(filename))

common/djangoapps/contentserver/middleware.py

@@ -4,18 +4,15 @@ import time
 from django.http import HttpResponse, Http404, HttpResponseNotModified
 
 from xmodule.contentstore.django import contentstore
-from xmodule.contentstore import StaticContent
+from xmodule.contentstore.content import StaticContent, XASSET_LOCATION_TAG
 from cache_toolbox.core import get_cached_content, set_cached_content
 from xmodule.exceptions import NotFoundError
 
 
 class StaticContentServer(object):
-    def __init__(self):
-        self.match_tag = StaticContent.get_location_tag()
-
     def process_request(self, request):
         # look to see if the request is prefixed with 'c4x' tag
-        if request.path.startswith('/' + self.match_tag):
+        if request.path.startswith('/' + XASSET_LOCATION_TAG):
 
             # first look in our cache so we don't have to round-trip to the DB
             content = get_cached_content(request.path)

common/lib/xmodule/xmodule/capa_module.py

@@ -10,6 +10,7 @@ import sys
 
 from datetime import timedelta
 from lxml import etree
+from lxml.html import rewrite_links
 from pkg_resources import resource_string
 
 from capa.capa_problem import LoncapaProblem
@@ -332,6 +333,15 @@ class CapaModule(XModule):
             html = '<div id="problem_{id}" class="problem" data-url="{ajax_url}">'.format(
                 id=self.location.html_id(), ajax_url=self.system.ajax_url) + html + "</div>"
 
+        # cdodge: OK, we have to do two rounds of url reference subsitutions
+        # one which uses the 'asset library' that is served by the contentstore and the
+        # more global /static/ filesystem based static content.
+        # NOTE: rewrite_content_links is defined in XModule
+        # This is a bit unfortunate and I'm sure we'll try to considate this into
+        # a one step process.
+        html = rewrite_links(html, self.rewrite_content_links)
+
+        # now do the substitutions which are filesystem based, e.g. '/static/' prefixes
         return self.system.replace_urls(html, self.metadata['data_dir'])
 
     def handle_ajax(self, dispatch, get):

common/lib/xmodule/xmodule/contentstore/__init__.py

-class StaticContent(object):
-    def __init__(self, filename, name, content_type, data, last_modified_at=None):
-        self.filename = filename
-        self.name = name
-        self.content_type = content_type
-        self.data = data
-        self.last_modified_at = last_modified_at
-
-    @staticmethod
-    def get_location_tag():
-        return 'c4x'
-
-    @staticmethod
-    def compute_location_filename(org, course, name):
-        return '/{0}/{1}/{2}/asset/{3}'.format(StaticContent.get_location_tag(), org, course, name)
-

common/lib/xmodule/xmodule/contentstore/content.py

+XASSET_LOCATION_TAG = 'c4x'
+XASSET_SRCREF_PREFIX = 'xasset:'
+
+class StaticContent(object):
+    def __init__(self, filename, name, content_type, data, last_modified_at=None):
+        self.filename = filename
+        self.name = name
+        self.content_type = content_type
+        self.data = data
+        self.last_modified_at = last_modified_at
+
+    @staticmethod
+    def compute_location_filename(org, course, name):
+        return '/{0}/{1}/{2}/asset/{3}'.format(XASSET_LOCATION_TAG, org, course, name)
+
+'''
+Abstraction for all ContentStore providers (e.g. MongoDB)
+'''
+class ContentStore(object):
+    def save(self, content):
+        raise NotImplementedError
+
+    def find(self, filename):
+        raise NotImplementedError
+
+            

common/lib/xmodule/xmodule/contentstore/mongo.py

@@ -5,20 +5,21 @@ from gridfs.errors import NoFile
 import sys
 import logging
 
-from . import StaticContent
+from .content import StaticContent, ContentStore
 from xmodule.exceptions import NotFoundError
 
 
-class MongoContentStore(object):
+class MongoContentStore(ContentStore):
     def __init__(self, host, db, port=27017):
         logging.debug( 'Using MongoDB for static content serving at host={0} db={1}'.format(host,db))
         _db = Connection(host=host, port=port)[db]
         self.fs = gridfs.GridFS(_db)
 
-    def update(self, content):
+    def save(self, content):
         with self.fs.new_file(filename=content.filename, content_type=content.content_type, displayname=content.name) as fp:
             fp.write(content.data)
             return content
+        
     
     def find(self, filename):
         try:

common/lib/xmodule/xmodule/html_module.py

@@ -13,6 +13,9 @@ from .xml_module import XmlDescriptor, name_to_pathname
 from .editing_module import EditingDescriptor
 from .stringify import stringify_children
 from .html_checker import check_html
+from xmodule.modulestore import Location
+
+from xmodule.contentstore.content import XASSET_SRCREF_PREFIX, StaticContent
 
 log = logging.getLogger("mitx.courseware")
 
@@ -20,8 +23,7 @@ log = logging.getLogger("mitx.courseware")
 class HtmlModule(XModule):
     def get_html(self):
         # cdodge: perform link substitutions for any references to course static content (e.g. images)
-        return rewrite_links(self.html, self.rewrite_content_links, self)
-        #return self.html
+        return rewrite_links(self.html, self.rewrite_content_links)
 
     def __init__(self, system, location, definition, descriptor,
                  instance_state=None, shared_state=None, **kwargs):
@@ -29,10 +31,6 @@ class HtmlModule(XModule):
                          instance_state, shared_state, **kwargs)
         self.html = self.definition['data']
 
-    def rewrite_content_links(link, self):
-        if link.startswith('xasset:'):
-            logging.debug('found link: {0}'.format(link))
-        return link
 
 
 class HtmlDescriptor(XmlDescriptor, EditingDescriptor):

common/lib/xmodule/xmodule/x_module.py

@@ -12,6 +12,8 @@ from pkg_resources import resource_listdir, resource_string, resource_isdir
 from xmodule.modulestore import Location
 from xmodule.timeparse import parse_time
 
+from xmodule.contentstore.content import StaticContent, XASSET_SRCREF_PREFIX
+
 log = logging.getLogger('mitx.' + __name__)
 
 
@@ -317,6 +319,20 @@ class XModule(HTMLSnippet):
             get is a dictionary-like object '''
         return ""
 
+    # cdodge: added to support dynamic substitutions of 
+    # links for courseware assets (e.g. images). <link> is passed through from lxml.html parser
+    def rewrite_content_links(self, link):
+        # see if we start with our format, e.g. 'xasset:<filename>'
+        if link.startswith(XASSET_SRCREF_PREFIX):
+            # yes, then parse out the name
+            name = link[len(XASSET_SRCREF_PREFIX):]
+            loc = Location(self.location)
+            # resolve the reference to our internal 'filepath' which
+            link = StaticContent.compute_location_filename(loc.org, loc.course, name)
+
+        return link
+
+
 
 def policy_key(location):
     """