Commit 2497f0a0 by Christina Roberts Committed by Toby Lawrence

WIP: xblock pipeline work (#10176)

[PERF-303] Integer XBlocks/XModules into the static asset pipeline.

This PR, based on hackathon work from Christina/Andy, implements a way to discover all installed XBlocks and XModules and to enumerate their public assets, then pulling them in during the collectstatic phase and hashing them.  In turn, the methods for generating URLs to resources will then returned the hashed name for assets, allowing them to be served from nginx/CDNs, and cached heavily.
parent a77e6ea2
...@@ -148,6 +148,7 @@ ...@@ -148,6 +148,7 @@
"JSON", "JSON",
// edX globals // edX globals
"edx" "edx",
"XBlock"
] ]
} }
...@@ -9,8 +9,9 @@ from django.http import Http404, HttpResponseBadRequest ...@@ -9,8 +9,9 @@ from django.http import Http404, HttpResponseBadRequest
from django.contrib.auth.decorators import login_required from django.contrib.auth.decorators import login_required
from edxmako.shortcuts import render_to_string from edxmako.shortcuts import render_to_string
from openedx.core.lib.xblock_utils import replace_static_urls, wrap_xblock, wrap_fragment, wrap_xblock_aside,\ from openedx.core.lib.xblock_utils import (
request_token replace_static_urls, wrap_xblock, wrap_fragment, wrap_xblock_aside, request_token, xblock_local_resource_url,
)
from xmodule.x_module import PREVIEW_VIEWS, STUDENT_VIEW, AUTHOR_VIEW from xmodule.x_module import PREVIEW_VIEWS, STUDENT_VIEW, AUTHOR_VIEW
from xmodule.contentstore.django import contentstore from xmodule.contentstore.django import contentstore
from xmodule.error_module import ErrorDescriptor from xmodule.error_module import ErrorDescriptor
...@@ -31,7 +32,6 @@ from xblock_django.user_service import DjangoXBlockUserService ...@@ -31,7 +32,6 @@ from xblock_django.user_service import DjangoXBlockUserService
from lms.djangoapps.lms_xblock.field_data import LmsFieldData from lms.djangoapps.lms_xblock.field_data import LmsFieldData
from cms.lib.xblock.field_data import CmsFieldData from cms.lib.xblock.field_data import CmsFieldData
from cms.lib.xblock.runtime import local_resource_url
from util.sandboxing import can_execute_unsafe_code, get_python_lib_zip from util.sandboxing import can_execute_unsafe_code, get_python_lib_zip
...@@ -115,7 +115,7 @@ class PreviewModuleSystem(ModuleSystem): # pylint: disable=abstract-method ...@@ -115,7 +115,7 @@ class PreviewModuleSystem(ModuleSystem): # pylint: disable=abstract-method
}) + '?' + query }) + '?' + query
def local_resource_url(self, block, uri): def local_resource_url(self, block, uri):
return local_resource_url(block, uri) return xblock_local_resource_url(block, uri)
def applicable_aside_types(self, block): def applicable_aside_types(self, block):
""" """
......
...@@ -524,6 +524,7 @@ STATICFILES_FINDERS = [ ...@@ -524,6 +524,7 @@ STATICFILES_FINDERS = [
'openedx.core.djangoapps.theming.finders.ComprehensiveThemeFinder', 'openedx.core.djangoapps.theming.finders.ComprehensiveThemeFinder',
'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.FileSystemFinder',
'django.contrib.staticfiles.finders.AppDirectoriesFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder',
'openedx.core.lib.xblock_pipeline.finder.XBlockPipelineFinder',
'pipeline.finders.PipelineFinder', 'pipeline.finders.PipelineFinder',
] ]
......
...@@ -32,6 +32,9 @@ DEBUG = True ...@@ -32,6 +32,9 @@ DEBUG = True
# Set REQUIRE_DEBUG to false so that it behaves like production # Set REQUIRE_DEBUG to false so that it behaves like production
REQUIRE_DEBUG = False REQUIRE_DEBUG = False
# Fetch static files out of the pipeline's static root
STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'
# Serve static files at /static directly from the staticfiles directory under test root. # Serve static files at /static directly from the staticfiles directory under test root.
# Note: optimized files for testing are generated with settings from test_static_optimized # Note: optimized files for testing are generated with settings from test_static_optimized
STATIC_URL = "/static/" STATIC_URL = "/static/"
......
...@@ -31,6 +31,7 @@ STATICFILES_STORAGE = 'openedx.core.lib.django_require.staticstorage.OptimizedCa ...@@ -31,6 +31,7 @@ STATICFILES_STORAGE = 'openedx.core.lib.django_require.staticstorage.OptimizedCa
STATICFILES_FINDERS = [ STATICFILES_FINDERS = [
'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.FileSystemFinder',
'django.contrib.staticfiles.finders.AppDirectoriesFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder',
'openedx.core.lib.xblock_pipeline.finder.XBlockPipelineFinder',
] ]
# Redirect to the test_root folder within the repo # Redirect to the test_root folder within the repo
......
...@@ -23,13 +23,3 @@ def handler_url(block, handler_name, suffix='', query='', thirdparty=False): ...@@ -23,13 +23,3 @@ def handler_url(block, handler_name, suffix='', query='', thirdparty=False):
url += '?' + query url += '?' + query
return url return url
def local_resource_url(block, uri):
"""
local_resource_url for Studio
"""
return reverse('xblock_resource_url', kwargs={
'block_type': block.scope_ids.block_type,
'uri': uri,
})
...@@ -13,6 +13,7 @@ from monkey_patch import ( ...@@ -13,6 +13,7 @@ from monkey_patch import (
third_party_auth, third_party_auth,
django_db_models_options django_db_models_options
) )
from openedx.core.lib.xblock_utils import xblock_local_resource_url
import xmodule.x_module import xmodule.x_module
import cms.lib.xblock.runtime import cms.lib.xblock.runtime
...@@ -46,7 +47,7 @@ def run(): ...@@ -46,7 +47,7 @@ def run():
# TODO: Remove this code when Runtimes are no longer created by modulestores # TODO: Remove this code when Runtimes are no longer created by modulestores
# https://openedx.atlassian.net/wiki/display/PLAT/Convert+from+Storage-centric+runtimes+to+Application-centric+runtimes # https://openedx.atlassian.net/wiki/display/PLAT/Convert+from+Storage-centric+runtimes+to+Application-centric+runtimes
xmodule.x_module.descriptor_global_handler_url = cms.lib.xblock.runtime.handler_url xmodule.x_module.descriptor_global_handler_url = cms.lib.xblock.runtime.handler_url
xmodule.x_module.descriptor_global_local_resource_url = cms.lib.xblock.runtime.local_resource_url xmodule.x_module.descriptor_global_local_resource_url = xblock_local_resource_url
def add_mimetypes(): def add_mimetypes():
......
...@@ -162,3 +162,4 @@ class AnnotatableModule(AnnotatableFields, XModule): ...@@ -162,3 +162,4 @@ class AnnotatableModule(AnnotatableFields, XModule):
class AnnotatableDescriptor(AnnotatableFields, RawDescriptor): class AnnotatableDescriptor(AnnotatableFields, RawDescriptor):
module_class = AnnotatableModule module_class = AnnotatableModule
mako_template = "widgets/raw-edit.html" mako_template = "widgets/raw-edit.html"
resources_dir = None
/* JavaScript for editing operations that can be done on the split test author view. */ /* JavaScript for editing operations that can be done on the split test author view. */
window.SplitTestAuthorView = function (runtime, element) { window.SplitTestAuthorView = function (runtime, element) {
"use strict";
var $element = $(element); var $element = $(element);
var splitTestLocator = $element.closest('.studio-xblock-wrapper').data('locator'); var splitTestLocator = $element.closest('.studio-xblock-wrapper').data('locator');
......
/* Creates a new selector for managing toggling which child to show. */
/** window.ABTestSelector = function (runtime, elem) {
* Creates a new selector for managing toggling which child to show "use strict";
* @constructor var _this = {};
*/
function ABTestSelector(runtime, elem) {
var _this = this;
_this.elem = $(elem); _this.elem = $(elem);
_this.children = _this.elem.find('.split-test-child'); _this.children = _this.elem.find('.split-test-child');
_this.content_container = _this.elem.find('.split-test-child-container'); _this.content_container = _this.elem.find('.split-test-child-container');
...@@ -23,17 +19,13 @@ function ABTestSelector(runtime, elem) { ...@@ -23,17 +19,13 @@ function ABTestSelector(runtime, elem) {
}); });
} }
select = _this.elem.find('.split-test-select'); var select = _this.elem.find('.split-test-select');
cur_group_id = select.val(); var cur_group_id = select.val();
select_child(cur_group_id); select_child(cur_group_id);
// bind the change event to the dropdown // bind the change event to the dropdown
select.change(function() { select.change(function() {
group_id = $(this).val() var group_id = $(this).val();
select_child(group_id); select_child(group_id);
}); });
};
}
/* Javascript for the Split Test XBlock. */ /* Javascript for the Split Test XBlock. */
function SplitTestStudentView(runtime, element) { window.SplitTestStudentView = function (runtime, element) {
"use strict";
$.post(runtime.handlerUrl(element, 'log_child_render')); $.post(runtime.handlerUrl(element, 'log_child_render'));
return {}; return {};
} };
/* JavaScript for Vertical Student View. */ /* JavaScript for Vertical Student View. */
window.VerticalStudentView = function (runtime, element) { window.VerticalStudentView = function (runtime, element) {
"use strict";
'use strict';
RequireJS.require(['js/bookmarks/views/bookmark_button'], function (BookmarkButton) { RequireJS.require(['js/bookmarks/views/bookmark_button'], function (BookmarkButton) {
var $element = $(element); var $element = $(element);
var $bookmarkButtonElement = $element.find('.bookmark-button'); var $bookmarkButtonElement = $element.find('.bookmark-button');
......
...@@ -60,6 +60,8 @@ def process_includes(fn): ...@@ -60,6 +60,8 @@ def process_includes(fn):
class SemanticSectionDescriptor(XModuleDescriptor): class SemanticSectionDescriptor(XModuleDescriptor):
resources_dir = None
@classmethod @classmethod
@process_includes @process_includes
def from_xml(cls, xml_data, system, id_generator): def from_xml(cls, xml_data, system, id_generator):
...@@ -82,6 +84,8 @@ class SemanticSectionDescriptor(XModuleDescriptor): ...@@ -82,6 +84,8 @@ class SemanticSectionDescriptor(XModuleDescriptor):
class TranslateCustomTagDescriptor(XModuleDescriptor): class TranslateCustomTagDescriptor(XModuleDescriptor):
resources_dir = None
@classmethod @classmethod
def from_xml(cls, xml_data, system, id_generator): def from_xml(cls, xml_data, system, id_generator):
""" """
......
...@@ -131,6 +131,7 @@ class CapaDescriptor(CapaFields, RawDescriptor): ...@@ -131,6 +131,7 @@ class CapaDescriptor(CapaFields, RawDescriptor):
INDEX_CONTENT_TYPE = 'CAPA' INDEX_CONTENT_TYPE = 'CAPA'
module_class = CapaModule module_class = CapaModule
resources_dir = None
has_score = True has_score = True
show_in_read_only_mode = True show_in_read_only_mode = True
......
...@@ -184,6 +184,8 @@ class ConditionalDescriptor(ConditionalFields, SequenceDescriptor): ...@@ -184,6 +184,8 @@ class ConditionalDescriptor(ConditionalFields, SequenceDescriptor):
module_class = ConditionalModule module_class = ConditionalModule
resources_dir = None
filename_extension = "xml" filename_extension = "xml"
has_score = False has_score = False
......
...@@ -789,6 +789,8 @@ class CourseDescriptor(CourseFields, SequenceDescriptor, LicenseMixin): ...@@ -789,6 +789,8 @@ class CourseDescriptor(CourseFields, SequenceDescriptor, LicenseMixin):
""" """
module_class = CourseModule module_class = CourseModule
resources_dir = None
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
""" """
Expects the same arguments as XModuleDescriptor.__init__ Expects the same arguments as XModuleDescriptor.__init__
......
...@@ -108,8 +108,9 @@ class DiscussionModule(DiscussionFields, XModule): ...@@ -108,8 +108,9 @@ class DiscussionModule(DiscussionFields, XModule):
class DiscussionDescriptor(DiscussionFields, MetadataOnlyEditingDescriptor, RawDescriptor): class DiscussionDescriptor(DiscussionFields, MetadataOnlyEditingDescriptor, RawDescriptor):
module_class = DiscussionModule module_class = DiscussionModule
resources_dir = None
# The discussion XML format uses `id` and `for` attributes, # The discussion XML format uses `id` and `for` attributes,
# but these would overload other module attributes, so we prefix them # but these would overload other module attributes, so we prefix them
# for actual use in the code # for actual use in the code
......
...@@ -20,6 +20,8 @@ class EditingDescriptor(EditingFields, MakoModuleDescriptor): ...@@ -20,6 +20,8 @@ class EditingDescriptor(EditingFields, MakoModuleDescriptor):
This class is intended to be used as a mixin. This class is intended to be used as a mixin.
""" """
resources_dir = None
mako_template = "widgets/raw-edit.html" mako_template = "widgets/raw-edit.html"
@property @property
......
...@@ -75,6 +75,7 @@ class ErrorDescriptor(ErrorFields, XModuleDescriptor): ...@@ -75,6 +75,7 @@ class ErrorDescriptor(ErrorFields, XModuleDescriptor):
Module that provides a raw editing view of broken xml. Module that provides a raw editing view of broken xml.
""" """
module_class = ErrorModule module_class = ErrorModule
resources_dir = None
def get_html(self): def get_html(self):
return u'' return u''
......
...@@ -15,3 +15,4 @@ class HiddenModule(XModule): ...@@ -15,3 +15,4 @@ class HiddenModule(XModule):
class HiddenDescriptor(RawDescriptor): class HiddenDescriptor(RawDescriptor):
module_class = HiddenModule module_class = HiddenModule
resources_dir = None
...@@ -118,6 +118,7 @@ class HtmlDescriptor(HtmlBlock, XmlDescriptor, EditingDescriptor): # pylint: di ...@@ -118,6 +118,7 @@ class HtmlDescriptor(HtmlBlock, XmlDescriptor, EditingDescriptor): # pylint: di
""" """
mako_template = "widgets/html-edit.html" mako_template = "widgets/html-edit.html"
module_class = HtmlModule module_class = HtmlModule
resources_dir = None
filename_extension = "xml" filename_extension = "xml"
template_dir_name = "html" template_dir_name = "html"
show_in_read_only_mode = True show_in_read_only_mode = True
......
...@@ -151,6 +151,7 @@ class ImageAnnotationModule(AnnotatableFields, XModule): ...@@ -151,6 +151,7 @@ class ImageAnnotationModule(AnnotatableFields, XModule):
class ImageAnnotationDescriptor(AnnotatableFields, RawDescriptor): class ImageAnnotationDescriptor(AnnotatableFields, RawDescriptor):
''' Image annotation descriptor ''' ''' Image annotation descriptor '''
module_class = ImageAnnotationModule module_class = ImageAnnotationModule
resources_dir = None
mako_template = "widgets/raw-edit.html" mako_template = "widgets/raw-edit.html"
@property @property
......
../public/
\ No newline at end of file
../../../assets/library_content/public/js/library_content_edit.js
\ No newline at end of file
../../../assets/split_test/public/js/split_test_author_view.js
\ No newline at end of file
../../../assets/split_test/public/js/split_test_staff.js
\ No newline at end of file
../../../assets/split_test/public/js/split_test_student.js
\ No newline at end of file
../../../assets/vertical/public/js/vertical_student_view.js
\ No newline at end of file
...@@ -367,6 +367,9 @@ class LibraryContentDescriptor(LibraryContentFields, MakoModuleDescriptor, XmlDe ...@@ -367,6 +367,9 @@ class LibraryContentDescriptor(LibraryContentFields, MakoModuleDescriptor, XmlDe
""" """
Descriptor class for LibraryContentModule XBlock. Descriptor class for LibraryContentModule XBlock.
""" """
resources_dir = 'assets/library_content'
module_class = LibraryContentModule module_class = LibraryContentModule
mako_template = 'widgets/metadata-edit.html' mako_template = 'widgets/metadata-edit.html'
js = {'coffee': [resource_string(__name__, 'js/src/vertical/edit.coffee')]} js = {'coffee': [resource_string(__name__, 'js/src/vertical/edit.coffee')]}
......
...@@ -22,6 +22,8 @@ class LibraryRoot(XBlock): ...@@ -22,6 +22,8 @@ class LibraryRoot(XBlock):
the library are its children. It contains metadata such as the library's the library are its children. It contains metadata such as the library's
display_name. display_name.
""" """
resources_dir = None
display_name = String( display_name = String(
help=_("Enter the name of the library as it should appear in Studio."), help=_("Enter the name of the library as it should appear in Studio."),
default="Library", default="Library",
......
...@@ -899,6 +899,7 @@ class LTIDescriptor(LTIFields, MetadataOnlyEditingDescriptor, EmptyDataRawDescri ...@@ -899,6 +899,7 @@ class LTIDescriptor(LTIFields, MetadataOnlyEditingDescriptor, EmptyDataRawDescri
Descriptor for LTI Xmodule. Descriptor for LTI Xmodule.
""" """
module_class = LTIModule module_class = LTIModule
resources_dir = None
grade_handler = module_attr('grade_handler') grade_handler = module_attr('grade_handler')
preview_handler = module_attr('preview_handler') preview_handler = module_attr('preview_handler')
lti_2_0_result_rest_handler = module_attr('lti_2_0_result_rest_handler') lti_2_0_result_rest_handler = module_attr('lti_2_0_result_rest_handler')
......
...@@ -59,5 +59,7 @@ class MakoModuleDescriptor(MakoTemplateBlockBase, XModuleDescriptor): # pylint: ...@@ -59,5 +59,7 @@ class MakoModuleDescriptor(MakoTemplateBlockBase, XModuleDescriptor): # pylint:
""" """
Mixin to use for XModule descriptors. Mixin to use for XModule descriptors.
""" """
resources_dir = None
def get_html(self): def get_html(self):
return self.studio_view(None).content return self.studio_view(None).content
...@@ -145,6 +145,7 @@ class PollDescriptor(PollFields, MakoModuleDescriptor, XmlDescriptor): ...@@ -145,6 +145,7 @@ class PollDescriptor(PollFields, MakoModuleDescriptor, XmlDescriptor):
_child_tag_name = 'answer' _child_tag_name = 'answer'
module_class = PollModule module_class = PollModule
resources_dir = None
@classmethod @classmethod
def definition_from_xml(cls, xml_object, system): def definition_from_xml(cls, xml_object, system):
......
...@@ -98,6 +98,7 @@ class RandomizeModule(RandomizeFields, XModule): ...@@ -98,6 +98,7 @@ class RandomizeModule(RandomizeFields, XModule):
class RandomizeDescriptor(RandomizeFields, SequenceDescriptor): class RandomizeDescriptor(RandomizeFields, SequenceDescriptor):
# the editing interface can be the same as for sequences -- just a container # the editing interface can be the same as for sequences -- just a container
module_class = RandomizeModule module_class = RandomizeModule
resources_dir = None
filename_extension = "xml" filename_extension = "xml"
......
...@@ -13,6 +13,8 @@ class RawDescriptor(XmlDescriptor, XMLEditingDescriptor): ...@@ -13,6 +13,8 @@ class RawDescriptor(XmlDescriptor, XMLEditingDescriptor):
Module that provides a raw editing view of its data and children. It Module that provides a raw editing view of its data and children. It
requires that the definition xml is valid. requires that the definition xml is valid.
""" """
resources_dir = None
data = String(help="XML data for the module", default="", scope=Scope.content) data = String(help="XML data for the module", default="", scope=Scope.content)
@classmethod @classmethod
...@@ -42,6 +44,8 @@ class EmptyDataRawDescriptor(XmlDescriptor, XMLEditingDescriptor): ...@@ -42,6 +44,8 @@ class EmptyDataRawDescriptor(XmlDescriptor, XMLEditingDescriptor):
Version of RawDescriptor for modules which may have no XML data, Version of RawDescriptor for modules which may have no XML data,
but use XMLEditingDescriptor for import/export handling. but use XMLEditingDescriptor for import/export handling.
""" """
resources_dir = None
data = String(default='', scope=Scope.content) data = String(default='', scope=Scope.content)
@classmethod @classmethod
......
...@@ -402,6 +402,7 @@ class SequenceDescriptor(SequenceFields, ProctoringFields, MakoModuleDescriptor, ...@@ -402,6 +402,7 @@ class SequenceDescriptor(SequenceFields, ProctoringFields, MakoModuleDescriptor,
""" """
mako_template = 'widgets/sequence-edit.html' mako_template = 'widgets/sequence-edit.html'
module_class = SequenceModule module_class = SequenceModule
resources_dir = None
show_in_read_only_mode = True show_in_read_only_mode = True
......
...@@ -376,6 +376,8 @@ class SplitTestDescriptor(SplitTestFields, SequenceDescriptor, StudioEditableDes ...@@ -376,6 +376,8 @@ class SplitTestDescriptor(SplitTestFields, SequenceDescriptor, StudioEditableDes
# the editing interface can be the same as for sequences -- just a container # the editing interface can be the same as for sequences -- just a container
module_class = SplitTestModule module_class = SplitTestModule
resources_dir = 'assets/split_test'
filename_extension = "xml" filename_extension = "xml"
mako_template = "widgets/metadata-only-edit.html" mako_template = "widgets/metadata-only-edit.html"
......
...@@ -37,6 +37,7 @@ class CustomTagModule(XModule): ...@@ -37,6 +37,7 @@ class CustomTagModule(XModule):
class CustomTagDescriptor(RawDescriptor): class CustomTagDescriptor(RawDescriptor):
""" Descriptor for custom tags. Loads the template when created.""" """ Descriptor for custom tags. Loads the template when created."""
module_class = CustomTagModule module_class = CustomTagModule
resources_dir = None
template_dir_name = 'customtag' template_dir_name = 'customtag'
def render_template(self, system, xml_data): def render_template(self, system, xml_data):
......
...@@ -147,6 +147,7 @@ class TextAnnotationModule(AnnotatableFields, XModule): ...@@ -147,6 +147,7 @@ class TextAnnotationModule(AnnotatableFields, XModule):
class TextAnnotationDescriptor(AnnotatableFields, RawDescriptor): class TextAnnotationDescriptor(AnnotatableFields, RawDescriptor):
''' Text Annotation Descriptor ''' ''' Text Annotation Descriptor '''
module_class = TextAnnotationModule module_class = TextAnnotationModule
resources_dir = None
mako_template = "widgets/raw-edit.html" mako_template = "widgets/raw-edit.html"
@property @property
......
...@@ -24,6 +24,9 @@ class VerticalBlock(SequenceFields, XModuleFields, StudioEditableBlock, XmlParse ...@@ -24,6 +24,9 @@ class VerticalBlock(SequenceFields, XModuleFields, StudioEditableBlock, XmlParse
""" """
Layout XBlock for rendering subblocks vertically. Layout XBlock for rendering subblocks vertically.
""" """
resources_dir = 'assets/vertical'
mako_template = 'widgets/sequence-edit.html' mako_template = 'widgets/sequence-edit.html'
js_module_name = "VerticalBlock" js_module_name = "VerticalBlock"
......
...@@ -154,6 +154,7 @@ class VideoAnnotationModule(AnnotatableFields, XModule): ...@@ -154,6 +154,7 @@ class VideoAnnotationModule(AnnotatableFields, XModule):
class VideoAnnotationDescriptor(AnnotatableFields, RawDescriptor): class VideoAnnotationDescriptor(AnnotatableFields, RawDescriptor):
''' Video annotation descriptor ''' ''' Video annotation descriptor '''
module_class = VideoAnnotationModule module_class = VideoAnnotationModule
resources_dir = None
mako_template = "widgets/raw-edit.html" mako_template = "widgets/raw-edit.html"
@property @property
......
...@@ -248,4 +248,5 @@ class WordCloudModule(WordCloudFields, XModule): ...@@ -248,4 +248,5 @@ class WordCloudModule(WordCloudFields, XModule):
class WordCloudDescriptor(WordCloudFields, MetadataOnlyEditingDescriptor, EmptyDataRawDescriptor): class WordCloudDescriptor(WordCloudFields, MetadataOnlyEditingDescriptor, EmptyDataRawDescriptor):
"""Descriptor for WordCloud Xmodule.""" """Descriptor for WordCloud Xmodule."""
module_class = WordCloudModule module_class = WordCloudModule
resources_dir = None
template_dir_name = 'word_cloud' template_dir_name = 'word_cloud'
...@@ -503,6 +503,8 @@ class XmlDescriptor(XmlParserMixin, XModuleDescriptor): # pylint: disable=abstr ...@@ -503,6 +503,8 @@ class XmlDescriptor(XmlParserMixin, XModuleDescriptor): # pylint: disable=abstr
""" """
Mixin class for standardized parsing of XModule xml. Mixin class for standardized parsing of XModule xml.
""" """
resources_dir = None
@classmethod @classmethod
def from_xml(cls, xml_data, system, id_generator): def from_xml(cls, xml_data, system, id_generator):
""" """
......
...@@ -3,12 +3,13 @@ Module implementing `xblock.runtime.Runtime` functionality for the LMS ...@@ -3,12 +3,13 @@ Module implementing `xblock.runtime.Runtime` functionality for the LMS
""" """
import re import re
from django.core.urlresolvers import reverse
from django.conf import settings from django.conf import settings
from django.core.urlresolvers import reverse
from badges.service import BadgingService from badges.service import BadgingService
from badges.utils import badges_enabled from badges.utils import badges_enabled
from openedx.core.djangoapps.user_api.course_tag import api as user_course_tag_api from openedx.core.djangoapps.user_api.course_tag import api as user_course_tag_api
from openedx.core.lib.xblock_utils import xblock_local_resource_url
from request_cache.middleware import RequestCache from request_cache.middleware import RequestCache
import xblock.reference.plugins import xblock.reference.plugins
from xmodule.library_tools import LibraryToolsService from xmodule.library_tools import LibraryToolsService
...@@ -126,11 +127,7 @@ def local_resource_url(block, uri): ...@@ -126,11 +127,7 @@ def local_resource_url(block, uri):
""" """
local_resource_url for Studio local_resource_url for Studio
""" """
path = reverse('xblock_resource_url', kwargs={ return xblock_local_resource_url(block, uri)
'block_type': block.scope_ids.block_type,
'uri': uri,
})
return '//{}{}'.format(settings.SITE_NAME, path)
class LmsPartitionService(PartitionService): class LmsPartitionService(PartitionService):
......
...@@ -1185,6 +1185,7 @@ STATICFILES_FINDERS = [ ...@@ -1185,6 +1185,7 @@ STATICFILES_FINDERS = [
'openedx.core.djangoapps.theming.finders.ComprehensiveThemeFinder', 'openedx.core.djangoapps.theming.finders.ComprehensiveThemeFinder',
'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.FileSystemFinder',
'django.contrib.staticfiles.finders.AppDirectoriesFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder',
'openedx.core.lib.xblock_pipeline.finder.XBlockPipelineFinder',
'pipeline.finders.PipelineFinder', 'pipeline.finders.PipelineFinder',
] ]
......
...@@ -32,6 +32,9 @@ DEBUG = True ...@@ -32,6 +32,9 @@ DEBUG = True
# Set REQUIRE_DEBUG to false so that it behaves like production # Set REQUIRE_DEBUG to false so that it behaves like production
REQUIRE_DEBUG = False REQUIRE_DEBUG = False
# Fetch static files out of the pipeline's static root
STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'
# Serve static files at /static directly from the staticfiles directory under test root. # Serve static files at /static directly from the staticfiles directory under test root.
# Note: optimized files for testing are generated with settings from test_static_optimized # Note: optimized files for testing are generated with settings from test_static_optimized
STATIC_URL = "/static/" STATIC_URL = "/static/"
......
...@@ -44,6 +44,7 @@ STATICFILES_STORAGE = 'openedx.core.lib.django_require.staticstorage.OptimizedCa ...@@ -44,6 +44,7 @@ STATICFILES_STORAGE = 'openedx.core.lib.django_require.staticstorage.OptimizedCa
STATICFILES_FINDERS = [ STATICFILES_FINDERS = [
'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.FileSystemFinder',
'django.contrib.staticfiles.finders.AppDirectoriesFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder',
'openedx.core.lib.xblock_pipeline.finder.XBlockPipelineFinder',
] ]
# Redirect to the test_root folder within the repo # Redirect to the test_root folder within the repo
......
...@@ -21,14 +21,10 @@ def xblock_resource(request, block_type, uri): # pylint: disable=unused-argumen ...@@ -21,14 +21,10 @@ def xblock_resource(request, block_type, uri): # pylint: disable=unused-argumen
Return a package resource for the specified XBlock. Return a package resource for the specified XBlock.
""" """
try: try:
# Figure out what the XBlock class is from the block type, and
# then open whatever resource has been requested.
xblock_class = XBlock.load_class(block_type, select=settings.XBLOCK_SELECT_FUNCTION) xblock_class = XBlock.load_class(block_type, select=settings.XBLOCK_SELECT_FUNCTION)
# Note: in debug mode, return any file rather than going through the XBlock which content = xblock_class.open_local_resource(uri)
# will only return public files. This allows unbundled files to be served up
# during development.
if settings.DEBUG:
content = pkg_resources.resource_stream(xblock_class.__module__, uri)
else:
content = xblock_class.open_local_resource(uri)
except IOError: except IOError:
log.info('Failed to load xblock resource', exc_info=True) log.info('Failed to load xblock resource', exc_info=True)
raise Http404 raise Http404
......
...@@ -4,9 +4,10 @@ ...@@ -4,9 +4,10 @@
from pipeline.storage import PipelineCachedStorage from pipeline.storage import PipelineCachedStorage
from require.storage import OptimizedFilesMixin from require.storage import OptimizedFilesMixin
from django_pipeline_forgiving.storages import PipelineForgivingStorage
class OptimizedCachedRequireJsStorage(OptimizedFilesMixin, PipelineCachedStorage): class OptimizedCachedRequireJsStorage(OptimizedFilesMixin, PipelineForgivingStorage):
""" """
Custom storage backend that is used by Django-require. Custom storage backend that is used by Django-require.
""" """
......
"""
Django pipeline finder for handling static assets required by XBlocks.
"""
from datetime import datetime
import os
from pkg_resources import resource_exists, resource_listdir, resource_isdir, resource_filename
from xblock.core import XBlock
from django.contrib.staticfiles import utils
from django.contrib.staticfiles.finders import BaseFinder
from django.contrib.staticfiles.storage import FileSystemStorage
from django.core.files.storage import Storage
class XBlockPackageStorage(Storage):
"""
Storage implementation for accessing XBlock package resources.
"""
RESOURCE_PREFIX = 'xblock/resources/'
def __init__(self, module, base_dir, *args, **kwargs):
"""
Returns a static file storage if available in the given app.
"""
super(XBlockPackageStorage, self).__init__(*args, **kwargs)
self.module = module
self.base_dir = base_dir
# Register a prefix that collectstatic will add to each path
self.prefix = os.path.join(self.RESOURCE_PREFIX, module)
def path(self, name):
"""
Returns a file system filename for the specified file name.
"""
return resource_filename(self.module, os.path.join(self.base_dir, name))
def exists(self, path):
"""
Returns True if the specified path exists.
"""
if self.base_dir is None:
return False
return resource_exists(self.module, os.path.join(self.base_dir, path))
def listdir(self, path):
"""
Lists the directories beneath the specified path.
"""
directories = []
files = []
for item in resource_listdir(self.module, os.path.join(self.base_dir, path)):
__, file_extension = os.path.splitext(item)
if file_extension not in [".py", ".pyc", ".scss"]:
if resource_isdir(self.module, os.path.join(self.base_dir, path, item)):
directories.append(item)
else:
files.append(item)
return directories, files
def open(self, name, mode='rb'):
"""
Retrieves the specified file from storage.
"""
path = self.path(name)
return FileSystemStorage(path).open(path, mode)
def size(self, name):
"""
Returns the size of the package resource.
"""
return os.path.getsize(self.path(name))
def accessed_time(self, name):
"""
Returns a URL to the package resource.
"""
return datetime.fromtimestamp(os.path.getatime(self.path(name)))
def created_time(self, name):
"""
Returns the created time of the package resource.
"""
return datetime.fromtimestamp(os.path.getctime(self.path(name)))
def modified_time(self, name):
"""
Returns the modified time of the resource.
"""
return datetime.fromtimestamp(os.path.getmtime(self.path(name)))
def url(self, name):
"""
Note: package resources do not support URLs
"""
raise NotImplementedError("Package resources do not support URLs")
def delete(self, name):
"""
Note: deleting files from a package is not supported.
"""
raise NotImplementedError("Deleting files from a package is not supported")
class XBlockPipelineFinder(BaseFinder):
"""
A static files finder that gets static assets from xblocks.
"""
def __init__(self, *args, **kwargs):
super(XBlockPipelineFinder, self).__init__(*args, **kwargs)
xblock_classes = set()
for __, xblock_class in XBlock.load_classes():
xblock_classes.add(xblock_class)
self.package_storages = [
XBlockPackageStorage(xblock_class.__module__, xblock_class.get_resources_dir())
for xblock_class in xblock_classes
]
def list(self, ignore_patterns):
"""
List all static files in all xblock packages.
"""
for storage in self.package_storages:
if storage.exists(''): # check if storage location exists
for path in utils.get_files(storage, ignore_patterns):
yield path, storage
def find(self, path, all=False): # pylint: disable=redefined-builtin
"""
Looks for files in the xblock package directories.
"""
matches = []
for storage in self.package_storages:
if storage.exists(path):
match = storage.path(path)
if not all:
return match
matches.append(match)
return matches
...@@ -13,6 +13,8 @@ from lxml import html, etree ...@@ -13,6 +13,8 @@ from lxml import html, etree
from contracts import contract from contracts import contract
from django.conf import settings from django.conf import settings
from django.contrib.staticfiles.storage import staticfiles_storage
from django.core.urlresolvers import reverse
from django.utils.timezone import UTC from django.utils.timezone import UTC
from django.utils.html import escape from django.utils.html import escape
from django.contrib.auth.models import User from django.contrib.auth.models import User
...@@ -445,3 +447,23 @@ def get_course_update_items(course_updates, provided_index=0): ...@@ -445,3 +447,23 @@ def get_course_update_items(course_updates, provided_index=0):
return payload return payload
return course_update_items return course_update_items
def xblock_local_resource_url(block, uri):
"""
Returns the URL for an XBlock's local resource.
Note: when running with the full Django pipeline, the file will be accessed
as a static asset which will use a CDN in production.
"""
xblock_class = getattr(block.__class__, 'unmixed_class', block.__class__)
if settings.PIPELINE_ENABLED or not settings.REQUIRE_DEBUG:
return staticfiles_storage.url('xblock/resources/{package_name}/{path}'.format(
package_name=xblock_class.__module__,
path=uri
))
else:
return reverse('xblock_resource_url', kwargs={
'block_type': block.scope_ids.block_type,
'uri': uri,
})
""" """
Django storage backends for Open edX. Django storage backends for Open edX.
""" """
from django_pipeline_forgiving.storages import PipelineForgivingStorage
from django.contrib.staticfiles.storage import StaticFilesStorage, CachedFilesMixin from django.contrib.staticfiles.storage import StaticFilesStorage, CachedFilesMixin
from pipeline.storage import PipelineMixin, NonPackagingMixin from pipeline.storage import PipelineMixin, NonPackagingMixin
from require.storage import OptimizedFilesMixin from require.storage import OptimizedFilesMixin
...@@ -8,6 +9,7 @@ from openedx.core.djangoapps.theming.storage import ComprehensiveThemingAwareMix ...@@ -8,6 +9,7 @@ from openedx.core.djangoapps.theming.storage import ComprehensiveThemingAwareMix
class ProductionStorage( class ProductionStorage(
PipelineForgivingStorage,
ComprehensiveThemingAwareMixin, ComprehensiveThemingAwareMixin,
OptimizedFilesMixin, OptimizedFilesMixin,
PipelineMixin, PipelineMixin,
......
...@@ -24,6 +24,7 @@ django-model-utils==2.3.1 ...@@ -24,6 +24,7 @@ django-model-utils==2.3.1
django-mptt==0.7.4 django-mptt==0.7.4
django-oauth-plus==2.2.8 django-oauth-plus==2.2.8
django-oauth-toolkit==0.10.0 django-oauth-toolkit==0.10.0
django-pipeline-forgiving==1.0.0
django-sekizai==0.8.2 django-sekizai==0.8.2
django-ses==0.7.0 django-ses==0.7.0
django-simple-history==1.6.3 django-simple-history==1.6.3
......
...@@ -70,7 +70,7 @@ git+https://github.com/edx/lettuce.git@0.2.20.002#egg=lettuce==0.2.20.002 ...@@ -70,7 +70,7 @@ git+https://github.com/edx/lettuce.git@0.2.20.002#egg=lettuce==0.2.20.002
git+https://github.com/edx/pa11ycrawler.git@0.0.1#egg=pa11ycrawler git+https://github.com/edx/pa11ycrawler.git@0.0.1#egg=pa11ycrawler
# Our libraries: # Our libraries:
git+https://github.com/edx/XBlock.git@xblock-0.4.8#egg=XBlock==0.4.8 git+https://github.com/edx/XBlock.git@xblock-0.4.10#egg=XBlock==0.4.10
-e git+https://github.com/edx/codejail.git@6b17c33a89bef0ac510926b1d7fea2748b73aadd#egg=codejail -e git+https://github.com/edx/codejail.git@6b17c33a89bef0ac510926b1d7fea2748b73aadd#egg=codejail
-e git+https://github.com/edx/event-tracking.git@0.2.1#egg=event-tracking==0.2.1 -e git+https://github.com/edx/event-tracking.git@0.2.1#egg=event-tracking==0.2.1
-e git+https://github.com/edx/django-splash.git@v0.2#egg=django-splash==0.2 -e git+https://github.com/edx/django-splash.git@v0.2#egg=django-splash==0.2
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment