Merge pull request #4889 from edx/danielli/pyfs

Adds pyfilesystem service from XBlocks to edx-platform

cms/envs/dev.py

@@ -27,6 +27,12 @@ update_module_store_settings(
     }
 )
 
+DJFS = {
+    'type': 'osfs',
+    'directory_root': 'cms/static/djpyfs',
+    'url_root': '/static/djpyfs'
+}
+
 # cdodge: This is the specifier for the MongoDB (using GridFS) backed static content store
 # This is for static content for courseware, not system static content (e.g. javascript, css, edX branding, etc)
 CONTENTSTORE = {
@@ -129,7 +135,7 @@ CELERY_ALWAYS_EAGER = True
 
 ################################ DEBUG TOOLBAR #################################
 INSTALLED_APPS += ('debug_toolbar', 'debug_toolbar_mongo')
-MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',)
+MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware', 'djpyfs')
 INTERNAL_IPS = ('127.0.0.1',)
 
 DEBUG_TOOLBAR_PANELS = (

common/lib/xmodule/xmodule/modulestore/django.py

@@ -19,6 +19,7 @@ import threading
 from xmodule.util.django import get_current_request_hostname
 import xmodule.modulestore  # pylint: disable=unused-import
 from xmodule.contentstore.django import contentstore
+import xblock.reference.plugins
 
 # We may not always have the request_cache module available
 try:
@@ -41,7 +42,7 @@ def load_function(path):
     return getattr(import_module(module_path), name)
 
 
-def create_modulestore_instance(engine, content_store, doc_store_config, options, i18n_service=None):
+def create_modulestore_instance(engine, content_store, doc_store_config, options, i18n_service=None, fs_service=None):
     """
     This will return a new instance of a modulestore given an engine and options
     """
@@ -73,6 +74,7 @@ def create_modulestore_instance(engine, content_store, doc_store_config, options
         xblock_select=getattr(settings, 'XBLOCK_SELECT_FUNCTION', None),
         doc_store_config=doc_store_config,
         i18n_service=i18n_service or ModuleI18nService(),
+        fs_service=fs_service or xblock.reference.plugins.FSService(),
         branch_setting_func=_get_modulestore_branch_setting,
         create_modulestore_instance=create_modulestore_instance,
         **_options

common/lib/xmodule/xmodule/modulestore/mixed.py

@@ -91,7 +91,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
     """
     ModuleStore knows how to route requests to the right persistence ms
     """
-    def __init__(self, contentstore, mappings, stores, i18n_service=None, create_modulestore_instance=None, **kwargs):
+    def __init__(self, contentstore, mappings, stores, i18n_service=None, fs_service=None, create_modulestore_instance=None, **kwargs):
         """
         Initialize a MixedModuleStore. Here we look into our passed in kwargs which should be a
         collection of other modulestore configuration information
@@ -130,6 +130,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
                 store_settings.get('DOC_STORE_CONFIG', {}),
                 store_settings.get('OPTIONS', {}),
                 i18n_service=i18n_service,
+                fs_service=fs_service,
             )
             # replace all named pointers to the store into actual pointers
             for course_key, store_name in self.mappings.iteritems():

common/lib/xmodule/xmodule/modulestore/mongo/base.py

@@ -366,6 +366,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
                  default_class=None,
                  error_tracker=null_error_tracker,
                  i18n_service=None,
+                 fs_service=None,
                  **kwargs):
         """
         :param doc_store_config: must have a host, db, and collection entries. Other common entries: port, tz_aware.
@@ -409,6 +410,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
         self.error_tracker = error_tracker
         self.render_template = render_template
         self.i18n_service = i18n_service
+        self.fs_service = fs_service
 
         # performance optimization to prevent updating the meta-data inheritance tree during
         # bulk write operations
@@ -696,6 +698,9 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
         if self.i18n_service:
             services["i18n"] = self.i18n_service
 
+        if self.fs_service:
+            services["fs"] = self.fs_service
+
         system = CachingDescriptorSystem(
             modulestore=self,
             course_key=course_key,
@@ -989,6 +994,9 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
             if self.i18n_service:
                 services["i18n"] = self.i18n_service
 
+            if self.fs_service:
+                services["fs"] = self.fs_service
+
             runtime = CachingDescriptorSystem(
                 modulestore=self,
                 module_data={},

common/lib/xmodule/xmodule/modulestore/split_mongo/split.py

@@ -120,8 +120,8 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
     def __init__(self, contentstore, doc_store_config, fs_root, render_template,
                  default_class=None,
                  error_tracker=null_error_tracker,
-                 i18n_service=None, services=None,
-                 **kwargs):
+                 i18n_service=None, fs_service=None,
+                 services=None, **kwargs):
         """
         :param doc_store_config: must have a host, db, and collection entries. Other common entries: port, tz_aware.
         """
@@ -148,6 +148,9 @@ class SplitMongoModuleStore(ModuleStoreWriteBase):
         if i18n_service is not None:
             self.services["i18n"] = i18n_service
 
+        if fs_service is not None:
+            self.services["fs"] = fs_service
+
     def close_connections(self):
         """
         Closes any open connections to the underlying databases

common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py

@@ -1297,7 +1297,7 @@ def load_function(path):
 
 
 # pylint: disable=unused-argument
-def create_modulestore_instance(engine, contentstore, doc_store_config, options, i18n_service=None):
+def create_modulestore_instance(engine, contentstore, doc_store_config, options, i18n_service=None, fs_service=None):
     """
     This will return a new instance of a modulestore given an engine and options
     """

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -370,7 +370,7 @@ class XMLModuleStore(ModuleStoreReadBase):
     """
     def __init__(
         self, data_dir, default_class=None, course_dirs=None, course_ids=None,
-        load_error_modules=True, i18n_service=None, **kwargs
+        load_error_modules=True, i18n_service=None, pyfs_service=None, **kwargs
     ):
         """
         Initialize an XMLModuleStore from data_dir
@@ -409,6 +409,7 @@ class XMLModuleStore(ModuleStoreReadBase):
         self.field_data = inheriting_field_data(kvs=DictKeyValueStore())
 
         self.i18n_service = i18n_service
+        self.pyfs_service = pyfs_service
 
         # If we are specifically asked for missing courses, that should
         # be an error.  If we are asked for "all" courses, find the ones
@@ -554,6 +555,9 @@ class XMLModuleStore(ModuleStoreReadBase):
             if self.i18n_service:
                 services['i18n'] = self.i18n_service
 
+            if self.pyfs_service:
+                services['fs'] = self.pyfs_service
+
             system = ImportSystem(
                 xmlstore=self,
                 course_id=course_id,

lms/djangoapps/courseware/module_render.py

@@ -3,6 +3,7 @@ import logging
 import mimetypes
 
 import static_replace
+import xblock.reference.plugins
 
 from functools import partial
 from requests.auth import HTTPBasicAuth
@@ -535,6 +536,7 @@ def get_module_system_for_user(user, field_data_cache,
         get_real_user=user_by_anonymous_id,
         services={
             'i18n': ModuleI18nService(),
+            'fs': xblock.reference.plugins.FSService(),
         },
         get_user_role=lambda: get_user_role(user, course_id),
         descriptor_runtime=descriptor.runtime,

lms/envs/dev.py

@@ -50,6 +50,12 @@ LOGGING = get_logger_config(ENV_ROOT / "log",
                             dev_env=True,
                             debug=True)
 
+DJFS = {
+    'type': 'osfs',
+    'directory_root': 'lms/static/djpyfs',
+    'url_root': '/static/djpyfs'
+}
+
 # If there is a database called 'read_replica', you can use the use_read_replica_if_available
 # function in util/query.py, which is useful for very large database reads
 DATABASES = {
@@ -216,8 +222,11 @@ CELERY_ALWAYS_EAGER = True
 ################################ DEBUG TOOLBAR ################################
 
 INSTALLED_APPS += ('debug_toolbar',)
-MIDDLEWARE_CLASSES += ('django_comment_client.utils.QueryCountDebugMiddleware',
-                       'debug_toolbar.middleware.DebugToolbarMiddleware',)
+MIDDLEWARE_CLASSES += (
+    'django_comment_client.utils.QueryCountDebugMiddleware',
+    'debug_toolbar.middleware.DebugToolbarMiddleware',
+    'djpyfs',
+)
 INTERNAL_IPS = ('127.0.0.1',)
 
 DEBUG_TOOLBAR_PANELS = (

lms/lib/xblock/runtime.py

@@ -3,6 +3,7 @@ Module implementing `xblock.runtime.Runtime` functionality for the LMS
 """
 
 import re
+import xblock.reference.plugins
 
 from django.core.urlresolvers import reverse
 from django.conf import settings
@@ -193,4 +194,5 @@ class LmsModuleSystem(LmsHandlerUrls, ModuleSystem):  # pylint: disable=abstract
             course_id=kwargs.get('course_id', None),
             track_function=kwargs.get('track_function', None),
         )
+        services['fs'] = xblock.reference.plugins.FSService()
         super(LmsModuleSystem, self).__init__(**kwargs)

requirements/edx/base.txt

@@ -138,3 +138,5 @@ git+https://github.com/mitocw/django-cas.git
 
 # edX packages
 edx-submissions==0.0.6
+
+-e git+https://github.com/pmitros/django-pyfs.git@514607d78535fd80bfd23184cd292ee5799b500d#egg=djpyfs

requirements/edx/edx-private.txt

@@ -15,4 +15,3 @@
 # It is an R&D prototype, intended for roll-out one location in one course. 
 # It should not be used without learning sciences support in the current state. 
 -e git+https://github.com/pmitros/RecommenderXBlock.git@fae9e5bc8a8297cb15001f0d674430e3d22ffa35#egg=recommender-xblock
-