Merge pull request #930 from ObjectifLibre/master

Add new backend for swift

openassessment/fileupload/backends/__init__.py

 from . import s3
 from . import filesystem
+from . import swift
 
 from django.conf import settings
 
@@ -11,5 +12,7 @@ def get_backend():
         return s3.Backend()
     elif backend_setting == "filesystem":
         return filesystem.Backend()
+    elif backend_setting == "swift":
+        return swift.Backend()
     else:
         raise ValueError("Invalid ORA2_FILEUPLOAD_BACKEND setting value: %s" % backend_setting)

openassessment/fileupload/backends/swift.py

+'''
+Add in /edx/app/edxapp/edx-platform/lms/envs/aws.py:
+ORA2_SWIFT_URL = AUTH_TOKENS["ORA2_SWIFT_URL"]
+ORA2_SWIFT_KEY = AUTH_TOKENS["ORA2_SWIFT_KEY"]
+
+Add in /edx/app/edxapp/lms.auth.json
+"ORA2_SWIFT_URL": "https://EXAMPLE",
+"ORA2_SWIFT_KEY": "EXAMPLE",
+
+ORA2_SWIFT_KEY should correspond to Meta Temp-Url-Key configure in swift. Run
+'swift stat -v' to get it.
+'''
+
+import logging
+from django.conf import settings
+import swiftclient
+import urlparse
+import requests
+
+
+logger = logging.getLogger("openassessment.fileupload.api")
+
+from .base import BaseBackend
+from ..exceptions import FileUploadInternalError
+
+
+class Backend(BaseBackend):
+    """
+    Upload openassessment student files to swift
+    """
+
+    def get_upload_url(self, key, content_type):
+        bucket_name, key_name = self._retrieve_parameters(key)
+        key, url = get_settings()
+        try:
+            temp_url = swiftclient.utils.generate_temp_url(
+                path='%s/%s/%s' % (url.path, bucket_name, key_name),
+                key=key,
+                method='PUT',
+                seconds=self.UPLOAD_URL_TIMEOUT)
+            return '%s://%s%s' % (url.scheme, url.netloc, temp_url)
+        except Exception as ex:
+            logger.exception(
+                u"An internal exception occurred while generating an upload URL."
+            )
+            raise FileUploadInternalError(ex)
+
+    def get_download_url(self, key):
+        bucket_name, key_name = self._retrieve_parameters(key)
+        key, url = get_settings()
+        try:
+            temp_url = swiftclient.utils.generate_temp_url(
+                path='%s/%s/%s' % (url.path, bucket_name, key_name),
+                key=key,
+                method='GET',
+                seconds=self.DOWNLOAD_URL_TIMEOUT)
+            download_url = '%s://%s%s' % (url.scheme, url.netloc, temp_url)
+            response = requests.get(download_url)
+            return download_url if response.status_code == 200 else ""
+        except Exception as ex:
+            logger.exception(
+                u"An internal exception occurred while generating a download URL."
+            )
+            raise FileUploadInternalError(ex)
+
+
+def get_settings():
+    """
+    Returns the swift key and a parsed url.
+    Both are generated from django settings.
+    """
+
+    url = getattr(settings, 'ORA2_SWIFT_URL', None)
+    key = getattr(settings, 'ORA2_SWIFT_KEY', None)
+    url = urlparse.urlparse(url)
+    return key, url

openassessment/fileupload/tests/test_api.py

@@ -5,9 +5,11 @@ from boto.s3.key import Key
 import ddt
 
 import json
+from mock import patch, Mock
 import os
 import shutil
 import tempfile
+from urlparse import urlparse
 
 from django.conf import settings
 from django.test import TestCase
@@ -24,6 +26,7 @@ from openassessment.fileupload import views_filesystem as views
 from openassessment.fileupload.backends.base import Settings as FileUploadSettings
 from openassessment.fileupload.backends.filesystem import get_cache as get_filesystem_cache
 
+
 @ddt.ddt
 class TestFileUploadService(TestCase):
 
@@ -113,7 +116,6 @@ class TestFileUploadServiceWithFilesystemBackend(TestCase):
         get_filesystem_cache().clear()
         self.delete_data(self.key_name)
 
-
     def tearDown(self):
         self.delete_data(self.key_name)
 
@@ -277,3 +279,62 @@ class TestFileUploadServiceWithFilesystemBackend(TestCase):
 
         self.assertEqual(200, upload_response.status_code)
         self.assertEqual(200, download_response.status_code)
+
+
+@override_settings(
+    ORA2_FILEUPLOAD_BACKEND='swift',
+    ORA2_SWIFT_URL='http://www.example.com:12345',
+    ORA2_SWIFT_KEY='bar',
+    FILE_UPLOAD_STORAGE_BUCKET_NAME='bucket_name'
+)
+class TestSwiftBackend(TestCase):
+    """
+    Test open assessment file upload to swift object storage.
+    """
+    def setUp(self):
+        super(TestSwiftBackend, self).setUp()
+        self.backend = api.backends.get_backend()
+
+    def _verify_url(self, url):
+        result = urlparse(url)
+        self.assertEqual(result.scheme, u'http')
+        self.assertEqual(result.netloc, u'www.example.com:12345')
+        self.assertEqual(result.path, u'/bucket_name/submissions_attachments/foo')
+        self.assertIn(result.params, 'temp_url_sig=')
+        self.assertIn(result.params, 'temp_url_expires=')
+
+    def test_get_backend(self):
+        """
+        Verify that there are no errors setting up swift as a backend.
+        """
+        self.assertTrue(isinstance(self.backend, api.backends.swift.Backend))
+
+    def test_get_upload_url(self):
+        """
+        Verify the upload URL.
+        """
+        url = self.backend.get_upload_url('foo', '_text')
+        self._verify_url(url)
+
+    @patch('openassessment.fileupload.backends.swift.requests.get')
+    def test_get_download_url_success(self, requests_get_mock):
+        """
+        Verify the download URL when the object already exists in storage.
+        """
+        fake_resp = Mock()
+        fake_resp.status_code = 200  # always return a 200 status code
+        requests_get_mock.return_value = fake_resp
+        url = self.backend.get_download_url('foo')
+        self._verify_url(url)
+
+    @patch('openassessment.fileupload.backends.swift.requests.get')
+    def test_get_download_url_no_object(self, requests_get_mock):
+        """
+        Verify the download URL is empty when the object
+        cannot be found in storage.
+        """
+        fake_resp = Mock()
+        fake_resp.status_code = 404  # always return a 404 status code
+        requests_get_mock.return_value = fake_resp
+        url = self.backend.get_download_url('foo')
+        self.assertEqual(url, '')

requirements/base.txt

@@ -10,6 +10,7 @@ git+https://github.com/edx/edx-submissions.git@1.1.5#egg=edx-submissions==1.1.5
 
 # Third Party Requirements
 boto>=2.32.1,<3.0.0
+python-swiftclient==3.1.0
 celery==3.1.18
 defusedxml==0.4.1
 django<1.9a0 # Resolves known bug on gemnasium. See TNL-6266