Merge pull request #640 from openfun/edx/multiple-fileupload-backends

Allow file upload backends other than S3. All tests are passing - merging.

openassessment/fileupload/api.py

@@ -4,180 +4,17 @@ associated with submissions. This can be used to upload new files and provide
 URLs to the new location.
 
 """
-import boto
-import logging
-from django.conf import settings
-
-logger = logging.getLogger("openassessment.fileupload.api")
-
-
-class FileUploadError(Exception):
-    """An error related to uploading files
-
-    This is the generic error raised when a file could not be uploaded.
-
-    """
-    pass
-
-
-class FileUploadInternalError(FileUploadError):
-    """An error internal to the File Upload API.
-
-    This is an error raised when file upload failed due to internal problems in
-    the File Upload API, beyond the intervention of the requester.
-
-    """
-    pass
-
-
-class FileUploadRequestError(FileUploadError):
-    """This error is raised when the request has invalid parameters for upload.
-
-    This error will be raised if the file being uploaded is somehow invalid,
-    based on type restrictions, size restrictions, upload limits, etc.
-
-    """
-    pass
-
-
-# The setting used to find the name of the AWS Bucket used for uploading
-# content.
-BUCKET_SETTING = "FILE_UPLOAD_STORAGE_BUCKET_NAME"
-
-# The setting used to prefix uploaded files using this service.
-FILE_STORAGE_SETTING = "FILE_UPLOAD_STORAGE_PREFIX"
-
-# The default file storage prefix.
-FILE_STORAGE = "submissions_attachments"
 
+from . import backends
 
 def get_upload_url(key, content_type):
-    """Request a one-time upload URL to upload files.
-
-    Requests a URL for a one-time file upload.
-
-    Args:
-        key (str): A unique identifier used to construct the upload location and
-            later, can be used to retrieve the same information. This service
-            must be able to identify data for both upload and download using
-            this key.
-        content_type (str): The content type for the file.
-
-    Returns:
-        A URL (str) to use for a one-time upload.
-
-    Raises:
-        FileUploadInternalError: Raised when an internal error occurs while
-            retrieving a one-time URL.
-        FileUploadRequestError: Raised when the request failed due to
-            request restrictions
-
     """
-    bucket_name, key_name = _retrieve_parameters(key)
-    try:
-        conn = _connect_to_s3()
-        upload_url = conn.generate_url(
-            3600,
-            'PUT',
-            bucket_name,
-            key_name,
-            headers={'Content-Length': '5242880', 'Content-Type': content_type}
-        )
-        return upload_url
-    except Exception as ex:
-        logger.exception(
-            u"An internal exception occurred while generating an upload URL."
-        )
-        raise FileUploadInternalError(ex)
-
-
-def get_download_url(key):
-    """Requests a URL to download the related file from.
-
-    Requests a URL for the given student_item.
-
-    Args:
-        key (str): A unique identifier used to identify the data requested for
-            download. This service must be able to identify data for both
-            upload and download using this key.
-
-    Returns:
-        A URL (str) to use for downloading related files. If no file is found,
-        returns an empty string.
-
+    Returns a url (absolute or relative, depending on the endpoint) which can be used to upload a file to.
     """
-    bucket_name, key_name = _retrieve_parameters(key)
-    try:
-        conn = _connect_to_s3()
-        bucket = conn.get_bucket(bucket_name)
-        s3_key = bucket.get_key(key_name)
-        return s3_key.generate_url(expires_in=1000) if s3_key else ""
-    except Exception as ex:
-        logger.exception(
-            u"An internal exception occurred while generating a download URL."
-        )
-        raise FileUploadInternalError(ex)
-
-
-def _connect_to_s3():
-    """Connect to s3
-
-    Creates a connection to s3 for file URLs.
-
-    """
-    # Try to get the AWS credentials from settings if they are available
-    # If not, these will default to `None`, and boto will try to use
-    # environment vars or configuration files instead.
-    aws_access_key_id = getattr(settings, 'AWS_ACCESS_KEY_ID', None)
-    aws_secret_access_key = getattr(settings, 'AWS_SECRET_ACCESS_KEY', None)
-
-    return boto.connect_s3(
-        aws_access_key_id=aws_access_key_id,
-        aws_secret_access_key=aws_secret_access_key
-    )
-
-
-def _retrieve_parameters(key):
-    """
-    Simple utility function to validate settings and arguments before compiling
-    bucket names and key names.
-
-    Args:
-        key (str): Custom key passed in with the request.
-
-    Returns:
-        A tuple of the bucket name and the complete key.
-
-    Raises:
-        FileUploadRequestError
-        FileUploadInternalError
+    return backends.get_backend().get_upload_url(key, content_type)
 
+def get_download_url(key):
     """
-    if not key:
-        raise FileUploadRequestError("Key required for URL request")
-
-    bucket_name = getattr(settings, BUCKET_SETTING, None)
-    if not bucket_name:
-        raise FileUploadInternalError("No bucket name configured for FileUpload Service.")
-    return bucket_name, _get_key_name(key)
-
-
-def _get_key_name(key):
-    """Construct a key name with the given string and configured prefix.
-
-    Constructs a unique key with the specified path and the service-specific
-    configured prefix.
-
-    Args:
-        key (str): Key to identify data for both upload and download.
-
-    Returns:
-        A key name (str) to use constructing URLs.
+    Returns the url at which the file that corresponds to the key can be downloaded.
     """
-    # The specified file prefix for the storage must be publicly viewable
-    # or all uploaded images will not be seen.
-    prefix = getattr(settings, FILE_STORAGE_SETTING, FILE_STORAGE)
-    return u"{prefix}/{key}".format(
-        prefix=prefix,
-        key=key
-    )
+    return backends.get_backend().get_download_url(key)
\ No newline at end of file

openassessment/fileupload/backends/__init__.py

+from . import s3
+from . import filesystem
+
+from django.conf import settings
+
+
+def get_backend():
+    # Use S3 backend by default (current behaviour)
+    backend_setting = getattr(settings, "ORA2_FILEUPLOAD_BACKEND", "s3")
+    if backend_setting == "s3":
+        return s3.Backend()
+    elif backend_setting == "filesystem":
+        return filesystem.Backend()
+    else:
+        raise ValueError("Invalid ORA2_FILEUPLOAD_BACKEND setting value: %s" % backend_setting)

openassessment/fileupload/backends/base.py

+import abc
+
+from django.conf import settings
+
+from ..exceptions import FileUploadInternalError
+from ..exceptions import FileUploadRequestError
+
+
+class Settings(object):
+    """Store settings related to file upload
+
+    The following settings are used:
+
+        FILE_UPLOAD_STORAGE_BUCKET_NAME (str, required): name of the bucket
+        (AWS or local file directory) to which content will be uploaded.
+
+        FILE_STORAGE_STORAGE_PREFIX (str, defaults to
+        DEFAULT_FILE_UPLOAD_STORAGE_PREFIX): this will be used to prefix all
+        stored file names. The specified file prefix for the storage must be
+        publicly viewable or all uploaded files will not be seen.
+    """
+    DEFAULT_FILE_UPLOAD_STORAGE_PREFIX = "submissions_attachments"
+
+    @classmethod
+    def get_bucket_name(cls):
+        bucket_name = getattr(settings, "FILE_UPLOAD_STORAGE_BUCKET_NAME", None)
+        if not bucket_name:
+            raise FileUploadInternalError("No bucket name configured for FileUpload Service.")
+        return bucket_name
+
+    @classmethod
+    def get_prefix(cls):
+        """Return the prefix for stored files.
+
+        Defaults to the DEFAULT_FILE_UPLOAD_STORAGE_PREFIX class attribute.
+        """
+        return getattr(settings, "FILE_UPLOAD_STORAGE_PREFIX", cls.DEFAULT_FILE_UPLOAD_STORAGE_PREFIX)
+
+
+class BaseBackend(object):
+
+    __metaclass__ = abc.ABCMeta
+
+    # Time (in seconds) before an upload url expires
+    UPLOAD_URL_TIMEOUT = 3600
+
+    # Time (in seconds) before a download url expires
+    DOWNLOAD_URL_TIMEOUT = 1000
+
+    @abc.abstractmethod
+    def get_upload_url(self, key, content_type):
+        """Request a one-time upload URL to upload files.
+
+        Requests a URL for a one-time file upload.
+
+        Args:
+            key (str): A unique identifier used to construct the upload location and
+                later, can be used to retrieve the same information. This service
+                must be able to identify data for both upload and download using
+                this key.
+            content_type (str): The content type for the file.
+
+        Returns:
+            A URL (str) to use for a one-time upload.
+
+        Raises:
+            FileUploadInternalError: Raised when an internal error occurs while
+                retrieving a one-time URL.
+            FileUploadRequestError: Raised when the request failed due to
+                request restrictions
+
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def get_download_url(self, key):
+        """Requests a URL to download the related file from.
+
+        Requests a URL for the given student_item.
+
+        Args:
+            key (str): A unique identifier used to identify the data requested for
+                download. This service must be able to identify data for both
+                upload and download using this key.
+
+        Returns:
+            A URL (str) to use for downloading related files. If no file is found,
+            returns an empty string.
+
+        """
+        raise NotImplementedError
+
+    def _retrieve_parameters(self, key):
+        """
+        Simple utility function to validate settings and arguments before compiling
+        bucket names and key names.
+
+        Args:
+            key (str): Custom key passed in with the request.
+
+        Returns:
+            A tuple of the bucket name and the complete key.
+
+        Raises:
+            FileUploadRequestError
+            FileUploadInternalError
+
+        """
+        if not key:
+            raise FileUploadRequestError("Key required for URL request")
+        return Settings.get_bucket_name(), self._get_key_name(key)
+
+
+    def _get_key_name(self, key):
+        """Construct a key name with the given string and configured prefix.
+
+        Constructs a unique key with the specified path and the service-specific
+        configured prefix.
+
+        Args:
+            key (str): Key to identify data for both upload and download.
+
+        Returns:
+            A key name (str) to use constructing URLs.
+        """
+        return "{prefix}/{key}".format(
+            prefix=Settings.get_prefix(),
+            key=key
+        )
+

openassessment/fileupload/backends/filesystem.py

+
+from .base import BaseBackend
+from .. import exceptions
+
+from django.conf import settings
+import django.core.cache
+from django.core.urlresolvers import reverse
+
+
+class Backend(BaseBackend):
+    """
+    Upload openassessment student files (images) to a local filesystem. Note
+    that in order to use this file storage backend, you need to include the
+    urls from openassessment.fileupload in your urls.py file:
+
+    E.g:
+        url(r'^openassessment/storage', include(openassessment.fileupload.urls)),
+
+    The ORA2_FILEUPLOAD_CACHE_NAME setting will also have to be defined for the
+    name of the django.core.cache instance which will maintain the list of
+    active storage URLs.
+
+    E.g:
+
+        ORA2_FILEUPLOAD_CACHE_NAME = "ora2-storage"
+        CACHES = {
+            ...
+            'ora2-storage': {
+                'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+                ...
+            },
+            ...
+        }
+    """
+
+    def get_upload_url(self, key, content_type):
+        make_upload_url_available(self._get_key_name(key), self.UPLOAD_URL_TIMEOUT)
+        return self._get_url(key)
+
+    def get_download_url(self, key):
+        make_download_url_available(self._get_key_name(key), self.DOWNLOAD_URL_TIMEOUT)
+        return self._get_url(key)
+
+    def _get_url(self, key):
+        key_name = self._get_key_name(key)
+        url = reverse("openassessment-filesystem-storage", kwargs={'key': key_name})
+        return url
+
+
+def get_cache():
+    """
+    Returns a django.core.cache instance in charge of maintaining the
+    authorized upload and download URL.
+
+    Raises:
+        FileUploadInternalError if the cache name setting is not defined.
+        InvalidCacheBackendError if the corresponding cache backend has not
+        been configured.
+    """
+    cache_name = getattr(settings, "ORA2_FILEUPLOAD_CACHE_NAME", None)
+    if cache_name is None:
+        raise exceptions.FileUploadInternalError("Undefined cache backend for file upload")
+    return django.core.cache.get_cache(cache_name)
+
+
+def make_upload_url_available(url_key_name, timeout):
+    """
+    Authorize an upload URL.
+
+    Arguments:
+        url_key_name (str): key that uniquely identifies the upload url
+        timeout (int): time in seconds before the url expires
+    """
+    return get_cache().set(
+        get_upload_cache_key(url_key_name),
+        1, timeout
+    )
+
+def make_download_url_available(url_key_name, timeout):
+    """
+    Authorize a download URL.
+
+    Arguments:
+        url_key_name (str): key that uniquely identifies the url
+        timeout (int): time in seconds before the url expires
+    """
+    return get_cache().set(
+        get_download_cache_key(url_key_name),
+        1, timeout
+    )
+
+def is_upload_url_available(url_key_name):
+    """
+    Return True if the corresponding upload URL is available.
+    """
+    return get_cache().get(get_upload_cache_key(url_key_name)) is not None
+
+def is_download_url_available(url_key_name):
+    """
+    Return True if the corresponding download URL is available.
+    """
+    return get_cache().get(get_download_cache_key(url_key_name)) is not None
+
+def get_upload_cache_key(url_key_name):
+    return "upload/" + url_key_name
+
+def get_download_cache_key(url_key_name):
+    return "download/" + url_key_name

openassessment/fileupload/backends/s3.py

+import boto
+import logging
+from django.conf import settings
+
+logger = logging.getLogger("openassessment.fileupload.api")
+
+from .base import BaseBackend
+from ..exceptions import FileUploadInternalError
+
+
+class Backend(BaseBackend):
+
+    def get_upload_url(self, key, content_type):
+        bucket_name, key_name = self._retrieve_parameters(key)
+        try:
+            conn = _connect_to_s3()
+            upload_url = conn.generate_url(
+                expires_in=self.UPLOAD_URL_TIMEOUT,
+                method='PUT',
+                bucket=bucket_name,
+                key=key_name,
+                headers={'Content-Length': '5242880', 'Content-Type': content_type}
+            )
+            return upload_url
+        except Exception as ex:
+            logger.exception(
+                u"An internal exception occurred while generating an upload URL."
+            )
+            raise FileUploadInternalError(ex)
+
+
+    def get_download_url(self, key):
+        bucket_name, key_name = self._retrieve_parameters(key)
+        try:
+            conn = _connect_to_s3()
+            bucket = conn.get_bucket(bucket_name)
+            s3_key = bucket.get_key(key_name)
+            return s3_key.generate_url(expires_in=self.DOWNLOAD_URL_TIMEOUT) if s3_key else ""
+        except Exception as ex:
+            logger.exception(
+                u"An internal exception occurred while generating a download URL."
+            )
+            raise FileUploadInternalError(ex)
+
+
+def _connect_to_s3():
+    """Connect to s3
+
+    Creates a connection to s3 for file URLs.
+
+    """
+    # Try to get the AWS credentials from settings if they are available
+    # If not, these will default to `None`, and boto will try to use
+    # environment vars or configuration files instead.
+    aws_access_key_id = getattr(settings, 'AWS_ACCESS_KEY_ID', None)
+    aws_secret_access_key = getattr(settings, 'AWS_SECRET_ACCESS_KEY', None)
+
+    return boto.connect_s3(
+        aws_access_key_id=aws_access_key_id,
+        aws_secret_access_key=aws_secret_access_key
+    )
+
+

openassessment/fileupload/exceptions.py

+class FileUploadError(Exception):
+    """An error related to uploading files
+
+    This is the generic error raised when a file could not be uploaded.
+
+    """
+    pass
+
+
+class FileUploadInternalError(FileUploadError):
+    """An error internal to the File Upload API.
+
+    This is an error raised when file upload failed due to internal problems in
+    the File Upload API, beyond the intervention of the requester.
+
+    """
+    pass
+
+
+class FileUploadRequestError(FileUploadError):
+    """This error is raised when the request has invalid parameters for upload.
+
+    This error will be raised if the file being uploaded is somehow invalid,
+    based on type restrictions, size restrictions, upload limits, etc.
+
+    """
+    pass

openassessment/fileupload/tests/test_api.py

+# -*- coding: utf-8 -*-
+
 import boto
 from boto.s3.key import Key
 import ddt
+
+import json
+import os
+import shutil
+import tempfile
+
+from django.conf import settings
 from django.test import TestCase
 from django.test.utils import override_settings
+from django.core.urlresolvers import reverse
+
 from moto import mock_s3
 from mock import patch
 from nose.tools import raises
+
 from openassessment.fileupload import api
+from openassessment.fileupload import exceptions
+from openassessment.fileupload import views_filesystem as views
+from openassessment.fileupload.backends.base import Settings as FileUploadSettings
+from openassessment.fileupload.backends.filesystem import get_cache as get_filesystem_cache
 
 @ddt.ddt
 class TestFileUploadService(TestCase):
@@ -38,11 +54,11 @@ class TestFileUploadService(TestCase):
         downloadUrl = api.get_download_url("foo")
         self.assertIn("https://mybucket.s3.amazonaws.com/submissions_attachments/foo", downloadUrl)
 
-    @raises(api.FileUploadInternalError)
+    @raises(exceptions.FileUploadInternalError)
     def test_get_upload_url_no_bucket(self):
         api.get_upload_url("foo", "bar")
 
-    @raises(api.FileUploadRequestError)
+    @raises(exceptions.FileUploadRequestError)
     def test_get_upload_url_no_key(self):
         api.get_upload_url("", "bar")
 
@@ -53,7 +69,7 @@ class TestFileUploadService(TestCase):
         FILE_UPLOAD_STORAGE_BUCKET_NAME="mybucket"
     )
     @patch.object(boto, 'connect_s3')
-    @raises(api.FileUploadInternalError)
+    @raises(exceptions.FileUploadInternalError)
     def test_get_upload_url_error(self, mock_s3):
         mock_s3.side_effect = Exception("Oh noes")
         api.get_upload_url("foo", "bar")
@@ -65,8 +81,199 @@ class TestFileUploadService(TestCase):
         FILE_UPLOAD_STORAGE_BUCKET_NAME="mybucket"
     )
     @patch.object(boto, 'connect_s3')
-    @raises(api.FileUploadInternalError, mock_s3)
+    @raises(exceptions.FileUploadInternalError, mock_s3)
     def test_get_download_url_error(self, mock_s3):
         mock_s3.side_effect = Exception("Oh noes")
         api.get_download_url("foo")
 
+
+@override_settings(
+    ORA2_FILEUPLOAD_BACKEND="filesystem",
+    ORA2_FILEUPLOAD_ROOT='/tmp',
+    ORA2_FILEUPLOAD_CACHE_NAME='default',
+    FILE_UPLOAD_STORAGE_BUCKET_NAME="testbucket",
+)
+class TestFileUploadServiceWithFilesystemBackend(TestCase):
+    """
+    Test open assessment file upload to local file storage.
+    """
+
+    def setUp(self):
+        self.backend = api.backends.get_backend()
+
+        self.content = tempfile.TemporaryFile()
+        self.content.write("foobar content")
+        self.content.seek(0)
+
+        self.key = None
+        self.key_name = None
+        self.set_key("myfile.jpg")
+        self.content_type = "image/jpeg"
+
+        get_filesystem_cache().clear()
+        self.delete_data(self.key_name)
+
+
+    def tearDown(self):
+        self.delete_data(self.key_name)
+
+    def set_key(self, key):
+        self.key = key
+        self.key_name = os.path.join(FileUploadSettings.get_prefix(), self.key)
+
+    def delete_data(self, key_name):
+        try:
+            path = views.get_data_path(key_name)
+            if os.path.exists(path):
+                shutil.rmtree(path)
+        except exceptions.FileUploadInternalError:
+            pass
+
+    def test_get_backend(self):
+        self.assertTrue(isinstance(self.backend, api.backends.filesystem.Backend))
+
+    def test_get_file_path(self):
+        path1 = views.get_file_path("mykey1")
+        path2 = views.get_file_path("mykey2")
+        self.assertEqual(
+            os.path.join(
+                settings.ORA2_FILEUPLOAD_ROOT,
+                settings.FILE_UPLOAD_STORAGE_BUCKET_NAME,
+                "mykey1"
+            ),
+            os.path.dirname(path1)
+        )
+        self.assertNotEqual(path1, path2)
+
+    def test_hack_get_file_path(self):
+        expected_path = os.path.join(
+            settings.ORA2_FILEUPLOAD_ROOT,
+            settings.FILE_UPLOAD_STORAGE_BUCKET_NAME,
+            "key",
+            "content"
+        )
+        self.assertEqual(
+            expected_path,
+            os.path.abspath(views.get_file_path("../key"))
+        )
+        self.assertEqual(
+            expected_path,
+            os.path.abspath(views.get_file_path("../key/"))
+        )
+        self.assertEqual(
+            expected_path,
+            os.path.abspath(views.get_file_path(" ../key/ "))
+        )
+
+    def test_safe_save(self):
+        self.assertRaises(
+            exceptions.FileUploadRequestError,
+            views.safe_save,
+            "/tmp/nonauthorisedbucket/file.txt",
+            "content"
+        )
+
+    def test_delete_file_data_on_metadata_saving_error(self):
+        key = "key"
+        file_path = views.get_file_path(key)
+        non_existing_path = "/non/existing/path"
+
+        with patch('openassessment.fileupload.views_filesystem.get_metadata_path') as mock_get_metadata_path:
+            mock_get_metadata_path.return_value = non_existing_path
+            self.assertRaises(
+                exceptions.FileUploadRequestError,
+                views.save_to_file,
+                "key", "content", "metadata"
+            )
+
+        self.assertFalse(os.path.exists(file_path))
+        self.assertFalse(os.path.exists(non_existing_path))
+
+    @override_settings(ORA2_FILEUPLOAD_ROOT='')
+    def test_undefined_file_upload_root(self):
+        self.assertRaises(exceptions.FileUploadInternalError, views.get_file_path, self.key)
+
+    @override_settings(ORA2_FILEUPLOAD_ROOT='/tmp/nonexistingdirectory')
+    def test_file_upload_root_does_not_exist(self):
+        if os.path.exists(settings.ORA2_FILEUPLOAD_ROOT):
+            shutil.rmtree(settings.ORA2_FILEUPLOAD_ROOT)
+        self.assertRaises(exceptions.FileUploadInternalError, views.save_to_file, self.key, "content")
+
+    def test_post_is_405(self):
+        upload_url = self.backend.get_upload_url(self.key, "bar")
+        response = self.client.post(upload_url, data={"attachment": self.content})
+        self.assertEqual(405, response.status_code)
+
+    def test_metadata(self):
+        self.content_type = "image/bmp"
+        upload_url = self.backend.get_upload_url(self.key, self.content_type)
+
+        self.client.put(upload_url, data=self.content.read(), content_type=self.content_type)
+        metadata_path = views.get_metadata_path(self.key_name)
+        metadata = json.load(open(metadata_path))
+
+        self.assertIsNotNone(metadata_path)
+        self.assertTrue(os.path.exists(metadata_path), "No metadata found at %s" % metadata_path)
+        self.assertIn("Content-Type", metadata)
+        self.assertIn("Date", metadata)
+        self.assertIn("Content-MD5", metadata)
+        self.assertIn("Content-Length", metadata)
+
+    def test_upload_download(self):
+        upload_url = self.backend.get_upload_url(self.key, self.content_type)
+        download_url = self.backend.get_download_url(self.key)
+        file_path = views.get_file_path(self.key_name)
+
+        upload_response = self.client.put(upload_url, data=self.content.read(), content_type=self.content_type)
+        download_response = self.client.get(download_url)
+        self.content.seek(0)
+
+        self.assertIn("/" + self.key, upload_url)
+        self.assertEqual(200, upload_response.status_code)
+        self.assertEqual("", upload_response.content)
+        self.assertEqual(200, download_response.status_code)
+        self.assertEqual(
+            "attachment; filename=" + self.key,
+            download_response.get('Content-Disposition')
+        )
+        self.assertEqual(self.content_type, download_response.get('Content-Type'))
+        self.assertIn("foobar content", download_response.content)
+        self.assertTrue(os.path.exists(file_path), "File %s does not exist" % file_path)
+        with open(file_path) as f:
+            self.assertEqual(self.content.read(), f.read())
+
+    def test_download_content_with_no_content_type(self):
+        views.save_to_file(self.key_name, "uploaded content", metadata=None)
+        download_url = self.backend.get_download_url(self.key)
+
+        download_response = self.client.get(download_url)
+        self.assertEqual(200, download_response.status_code)
+        self.assertEqual('application/octet-stream', download_response["Content-Type"])
+
+    def test_upload_with_unauthorized_key(self):
+        upload_url = reverse("openassessment-filesystem-storage", kwargs={'key': self.key_name})
+
+        cache_before_request = get_filesystem_cache().get(self.key_name)
+        upload_response = self.client.put(upload_url, data=self.content.read(), content_type=self.content_type)
+        cache_after_request = get_filesystem_cache().get(self.key_name)
+        self.assertIsNone(cache_before_request)
+        self.assertEqual(404, upload_response.status_code)
+        self.assertIsNone(cache_after_request)
+
+    def test_download_url_with_unauthorized_key(self):
+        download_url = reverse("openassessment-filesystem-storage", kwargs={'key': self.key_name})
+        views.save_to_file(self.key_name, "uploaded content")
+        download_response = self.client.get(download_url)
+
+        self.assertEqual(404, download_response.status_code)
+
+    def test_upload_download_with_accented_key(self):
+        self.set_key("noël.jpg")
+        upload_url = self.backend.get_upload_url(self.key, self.content_type)
+        download_url = self.backend.get_download_url(self.key)
+
+        upload_response = self.client.put(upload_url, data=self.content.read(), content_type=self.content_type)
+        download_response = self.client.get(download_url)
+
+        self.assertEqual(200, upload_response.status_code)
+        self.assertEqual(200, download_response.status_code)

openassessment/fileupload/urls.py

+from django.conf.urls import patterns, url
+
+
+urlpatterns = patterns('openassessment.fileupload.views_filesystem',
+    url(r'^(?P<key>.+)/$', 'filesystem_storage', name='openassessment-filesystem-storage'),
+)

openassessment/fileupload/views_filesystem.py

+import hashlib
+import json
+import os
+
+from django.conf import settings
+from django.shortcuts import HttpResponse, Http404
+from django.utils import timezone
+from django.views.decorators.http import require_http_methods
+
+from . import exceptions
+from .backends.filesystem import is_upload_url_available, is_download_url_available
+from .backends.base import Settings
+
+
+@require_http_methods(["PUT", "GET"])
+def filesystem_storage(request, key):
+    """
+    Uploading and download files to the local filesystem backend.
+    """
+    if isinstance(key, unicode):
+        key = key.encode("utf-8")
+    if request.method == "PUT":
+        if not is_upload_url_available(key):
+            raise Http404()
+        content, metadata = get_content_metadata(request)
+        save_to_file(key, content, metadata)
+        return HttpResponse()
+    elif request.method == "GET":
+        if not is_download_url_available(key):
+            raise Http404()
+        return download_file(key)
+
+
+def download_file(key):
+    """Returns an HttpResponse to download the corresponding file"""
+
+    file_path = get_file_path(key)
+    metadata_path = get_metadata_path(key)
+    if not os.path.exists(file_path):
+        raise Http404()
+    with open(metadata_path) as f:
+        metadata = json.load(f)
+        content_type = metadata.get("Content-Type", 'application/octet-stream')
+    with open(file_path, 'r') as f:
+        response = HttpResponse(f.read(), content_type=content_type)
+        file_name = os.path.basename(os.path.dirname(file_path))
+        response['Content-Disposition'] = 'attachment; filename=' + file_name
+    return response
+
+
+def get_content_metadata(request):
+    """
+    Read the content and metadata associated to an HttpRequest.
+
+    Returns:
+        request body (str)
+        request metadata (dict)
+    """
+
+    metadata = {
+        "Content-Type": request.META["CONTENT_TYPE"],
+        "Date": str(timezone.now()),
+        "Content-MD5": hashlib.md5(request.body).hexdigest(),
+        "Content-Length": request.META["CONTENT_LENGTH"],
+    }
+    return request.body, metadata
+
+
+def save_to_file(key, content, metadata=None):
+    """
+    Save the content and metadata to a local file determined by the given key.
+
+    Arguments:
+        key (str): unique file identifier
+        content (str): uploaded file content
+        metadata (dict): json-dumpable data
+    """
+    file_path = get_file_path(key)
+    metadata_path = get_metadata_path(key)
+    if metadata is None:
+        metadata = {}
+
+    safe_save(file_path, content)
+    try:
+        safe_save(metadata_path, json.dumps(metadata))
+    except:
+        safe_remove(file_path)
+        safe_remove(metadata_path)
+        raise
+
+def safe_save(path, content):
+    """
+    Save content to path. Creates the appropriate directories, if required.
+
+    Raises:
+        FileUploadInternalError if the root directory does not exist or if we
+        try to save in an unauthorized directory.
+    """
+    dir_path = os.path.abspath(os.path.dirname(path))
+    if not dir_path.startswith(get_bucket_path()):
+        raise exceptions.FileUploadRequestError("Uploaded file name not allowed: '%s'" % path)
+    root_directory = get_root_directory_path()
+    if not os.path.exists(root_directory):
+        raise exceptions.FileUploadInternalError("File upload root directory does not exist: %s" % root_directory)
+    if not os.path.exists(dir_path):
+        os.makedirs(dir_path)
+    with open(path, 'w') as f:
+        f.write(content)
+
+def safe_remove(path):
+    """Remove a file if it exists.
+
+    Note that an exception will be raised if the file is not writable.
+    """
+    if os.path.exists(path):
+        os.remove(path)
+
+
+def get_file_path(key):
+    """
+    Returns the path to the content file.
+    """
+    return os.path.join(get_data_path(key), "content")
+
+
+def get_metadata_path(key):
+    """
+    Returns the path to the metadata file.
+    """
+    return os.path.join(get_data_path(key), "metadata.json")
+
+
+def get_data_path(key):
+    """
+    Returns the path to the directory which will store the content and metadata
+    files.
+    """
+    subdirectory = key.replace("..", "").strip("/ ")
+    return os.path.join(get_bucket_path(), subdirectory)
+
+
+def get_bucket_path():
+    """
+    Returns the path to the bucket directory.
+    """
+    dir_path = os.path.join(
+        get_root_directory_path(),
+        Settings.get_bucket_name(),
+    )
+    return os.path.abspath(dir_path)
+
+
+def get_root_directory_path():
+    """
+    Returns the path to the root directory in which bucket directories are stored.
+
+    Raises:
+        FileUploadInternalError if the root directory setting does not exist.
+    """
+    root_dir = getattr(settings, "ORA2_FILEUPLOAD_ROOT", None)
+    if not root_dir:
+        raise exceptions.FileUploadInternalError("Undefined file upload root directory setting")
+    return root_dir

openassessment/xblock/staff_info_mixin.py

@@ -24,6 +24,7 @@ from openassessment.assessment.api import self as self_api
 from openassessment.assessment.api import ai as ai_api
 from openassessment.fileupload import api as file_api
 from openassessment.workflow import api as workflow_api
+from openassessment.fileupload import exceptions as file_exceptions
 
 
 logger = logging.getLogger(__name__)
@@ -256,7 +257,7 @@ class StaffInfoMixin(object):
 
                 try:
                     submission['image_url'] = file_api.get_download_url(file_key)
-                except file_api.FileUploadError:
+                except file_exceptions.FileUploadError:
                     # Log the error, but do not prevent the rest of the student info
                     # from being displayed.
                     msg = (

openassessment/xblock/submission_mixin.py

@@ -5,7 +5,7 @@ from xblock.core import XBlock
 
 from submissions import api
 from openassessment.fileupload import api as file_upload_api
-from openassessment.fileupload.api import FileUploadError
+from openassessment.fileupload.exceptions import FileUploadError
 from openassessment.workflow import api as workflow_api
 from openassessment.workflow.errors import AssessmentWorkflowError
 from .resolve_dates import DISTANT_FUTURE

openassessment/xblock/test/test_leaderboard.py

@@ -228,6 +228,11 @@ class TestLeaderboardRender(XBlockHandlerTransactionTestCase):
                 is a dictionary of with keys 'content' (the submission text)
                 and 'score' (the integer number of points earned)
         """
+        # We temporarily set the maxDiff attribute to infinite so that we can
+        # have a full diff
+        maxDiff = self.maxDiff
+        self.maxDiff = None
+
         self._assert_path_and_context(
             xblock,
             'openassessmentblock/leaderboard/oa_leaderboard_show.html',
@@ -238,6 +243,8 @@ class TestLeaderboardRender(XBlockHandlerTransactionTestCase):
             workflow_status='done'
         )
 
+        self.maxDiff = maxDiff
+
     def _assert_path_and_context(self, xblock, expected_path, expected_context, workflow_status=None):
         """
         Render the leaderboard and verify:

openassessment/xblock/test/test_staff_info.py

@@ -12,7 +12,7 @@ from openassessment.assessment.api import self as self_api
 from openassessment.assessment.api import ai as ai_api
 from openassessment.workflow import api as workflow_api
 from openassessment.assessment.errors.ai import AIError, AIGradingInternalError
-from openassessment.fileupload.api import FileUploadInternalError
+from openassessment.fileupload.exceptions import FileUploadInternalError
 from submissions import api as sub_api
 
 from openassessment.xblock.data_conversion import prepare_submission_for_serialization

urls.py

@@ -4,6 +4,7 @@ from django.views.i18n import javascript_catalog
 from django.contrib import admin
 
 import openassessment.assessment.urls
+import openassessment.fileupload.urls
 import workbench.urls
 
 admin.autodiscover()
@@ -26,6 +27,9 @@ urlpatterns = patterns(
 
     # JavaScript i18n
     (r'^jsi18n/$', 'django.views.i18n.javascript_catalog', JS_INFO_DICT),
+
+    # File upload to local filesystem
+    url(r'^openassessment/storage', include(openassessment.fileupload.urls)),
 )
 
 # We need to do explicit setup of the Django debug toolbar because autodiscovery