Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
E
edx-ora2
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
edx
edx-ora2
Commits
dd59d943
Commit
dd59d943
authored
Sep 12, 2016
by
Jacques Roussel
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Add new backend for swift + tests + rebase
parent
7a50f4f4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
142 additions
and
1 deletions
+142
-1
openassessment/fileupload/backends/__init__.py
+3
-0
openassessment/fileupload/backends/swift.py
+76
-0
openassessment/fileupload/tests/test_api.py
+62
-1
requirements/base.txt
+1
-0
No files found.
openassessment/fileupload/backends/__init__.py
View file @
dd59d943
from
.
import
s3
from
.
import
s3
from
.
import
filesystem
from
.
import
filesystem
from
.
import
swift
from
django.conf
import
settings
from
django.conf
import
settings
...
@@ -11,5 +12,7 @@ def get_backend():
...
@@ -11,5 +12,7 @@ def get_backend():
return
s3
.
Backend
()
return
s3
.
Backend
()
elif
backend_setting
==
"filesystem"
:
elif
backend_setting
==
"filesystem"
:
return
filesystem
.
Backend
()
return
filesystem
.
Backend
()
elif
backend_setting
==
"swift"
:
return
swift
.
Backend
()
else
:
else
:
raise
ValueError
(
"Invalid ORA2_FILEUPLOAD_BACKEND setting value:
%
s"
%
backend_setting
)
raise
ValueError
(
"Invalid ORA2_FILEUPLOAD_BACKEND setting value:
%
s"
%
backend_setting
)
openassessment/fileupload/backends/swift.py
0 → 100644
View file @
dd59d943
'''
Add in /edx/app/edxapp/edx-platform/lms/envs/aws.py:
ORA2_SWIFT_URL = AUTH_TOKENS["ORA2_SWIFT_URL"]
ORA2_SWIFT_KEY = AUTH_TOKENS["ORA2_SWIFT_KEY"]
Add in /edx/app/edxapp/lms.auth.json
"ORA2_SWIFT_URL": "https://EXAMPLE",
"ORA2_SWIFT_KEY": "EXAMPLE",
ORA2_SWIFT_KEY should correspond to Meta Temp-Url-Key configure in swift. Run
'swift stat -v' to get it.
'''
import
logging
from
django.conf
import
settings
import
swiftclient
import
urlparse
import
requests
logger
=
logging
.
getLogger
(
"openassessment.fileupload.api"
)
from
.base
import
BaseBackend
from
..exceptions
import
FileUploadInternalError
class
Backend
(
BaseBackend
):
"""
Upload openassessment student files to swift
"""
def
get_upload_url
(
self
,
key
,
content_type
):
bucket_name
,
key_name
=
self
.
_retrieve_parameters
(
key
)
key
,
url
=
get_settings
()
try
:
temp_url
=
swiftclient
.
utils
.
generate_temp_url
(
path
=
'
%
s/
%
s/
%
s'
%
(
url
.
path
,
bucket_name
,
key_name
),
key
=
key
,
method
=
'PUT'
,
seconds
=
self
.
UPLOAD_URL_TIMEOUT
)
return
'
%
s://
%
s
%
s'
%
(
url
.
scheme
,
url
.
netloc
,
temp_url
)
except
Exception
as
ex
:
logger
.
exception
(
u"An internal exception occurred while generating an upload URL."
)
raise
FileUploadInternalError
(
ex
)
def
get_download_url
(
self
,
key
):
bucket_name
,
key_name
=
self
.
_retrieve_parameters
(
key
)
key
,
url
=
get_settings
()
try
:
temp_url
=
swiftclient
.
utils
.
generate_temp_url
(
path
=
'
%
s/
%
s/
%
s'
%
(
url
.
path
,
bucket_name
,
key_name
),
key
=
key
,
method
=
'GET'
,
seconds
=
self
.
DOWNLOAD_URL_TIMEOUT
)
download_url
=
'
%
s://
%
s
%
s'
%
(
url
.
scheme
,
url
.
netloc
,
temp_url
)
response
=
requests
.
get
(
download_url
)
return
download_url
if
response
.
status_code
==
200
else
""
except
Exception
as
ex
:
logger
.
exception
(
u"An internal exception occurred while generating a download URL."
)
raise
FileUploadInternalError
(
ex
)
def
get_settings
():
"""
Returns the swift key and a parsed url.
Both are generated from django settings.
"""
url
=
getattr
(
settings
,
'ORA2_SWIFT_URL'
,
None
)
key
=
getattr
(
settings
,
'ORA2_SWIFT_KEY'
,
None
)
url
=
urlparse
.
urlparse
(
url
)
return
key
,
url
openassessment/fileupload/tests/test_api.py
View file @
dd59d943
...
@@ -5,9 +5,11 @@ from boto.s3.key import Key
...
@@ -5,9 +5,11 @@ from boto.s3.key import Key
import
ddt
import
ddt
import
json
import
json
from
mock
import
patch
,
Mock
import
os
import
os
import
shutil
import
shutil
import
tempfile
import
tempfile
from
urlparse
import
urlparse
from
django.conf
import
settings
from
django.conf
import
settings
from
django.test
import
TestCase
from
django.test
import
TestCase
...
@@ -24,6 +26,7 @@ from openassessment.fileupload import views_filesystem as views
...
@@ -24,6 +26,7 @@ from openassessment.fileupload import views_filesystem as views
from
openassessment.fileupload.backends.base
import
Settings
as
FileUploadSettings
from
openassessment.fileupload.backends.base
import
Settings
as
FileUploadSettings
from
openassessment.fileupload.backends.filesystem
import
get_cache
as
get_filesystem_cache
from
openassessment.fileupload.backends.filesystem
import
get_cache
as
get_filesystem_cache
@ddt.ddt
@ddt.ddt
class
TestFileUploadService
(
TestCase
):
class
TestFileUploadService
(
TestCase
):
...
@@ -113,7 +116,6 @@ class TestFileUploadServiceWithFilesystemBackend(TestCase):
...
@@ -113,7 +116,6 @@ class TestFileUploadServiceWithFilesystemBackend(TestCase):
get_filesystem_cache
()
.
clear
()
get_filesystem_cache
()
.
clear
()
self
.
delete_data
(
self
.
key_name
)
self
.
delete_data
(
self
.
key_name
)
def
tearDown
(
self
):
def
tearDown
(
self
):
self
.
delete_data
(
self
.
key_name
)
self
.
delete_data
(
self
.
key_name
)
...
@@ -277,3 +279,62 @@ class TestFileUploadServiceWithFilesystemBackend(TestCase):
...
@@ -277,3 +279,62 @@ class TestFileUploadServiceWithFilesystemBackend(TestCase):
self
.
assertEqual
(
200
,
upload_response
.
status_code
)
self
.
assertEqual
(
200
,
upload_response
.
status_code
)
self
.
assertEqual
(
200
,
download_response
.
status_code
)
self
.
assertEqual
(
200
,
download_response
.
status_code
)
@override_settings
(
ORA2_FILEUPLOAD_BACKEND
=
'swift'
,
ORA2_SWIFT_URL
=
'http://www.example.com:12345'
,
ORA2_SWIFT_KEY
=
'bar'
,
FILE_UPLOAD_STORAGE_BUCKET_NAME
=
'bucket_name'
)
class
TestSwiftBackend
(
TestCase
):
"""
Test open assessment file upload to swift object storage.
"""
def
setUp
(
self
):
super
(
TestSwiftBackend
,
self
)
.
setUp
()
self
.
backend
=
api
.
backends
.
get_backend
()
def
_verify_url
(
self
,
url
):
result
=
urlparse
(
url
)
self
.
assertEqual
(
result
.
scheme
,
u'http'
)
self
.
assertEqual
(
result
.
netloc
,
u'www.example.com:12345'
)
self
.
assertEqual
(
result
.
path
,
u'/bucket_name/submissions_attachments/foo'
)
self
.
assertIn
(
result
.
params
,
'temp_url_sig='
)
self
.
assertIn
(
result
.
params
,
'temp_url_expires='
)
def
test_get_backend
(
self
):
"""
Verify that there are no errors setting up swift as a backend.
"""
self
.
assertTrue
(
isinstance
(
self
.
backend
,
api
.
backends
.
swift
.
Backend
))
def
test_get_upload_url
(
self
):
"""
Verify the upload URL.
"""
url
=
self
.
backend
.
get_upload_url
(
'foo'
,
'_text'
)
self
.
_verify_url
(
url
)
@patch
(
'openassessment.fileupload.backends.swift.requests.get'
)
def
test_get_download_url_success
(
self
,
requests_get_mock
):
"""
Verify the download URL when the object already exists in storage.
"""
fake_resp
=
Mock
()
fake_resp
.
status_code
=
200
# always return a 200 status code
requests_get_mock
.
return_value
=
fake_resp
url
=
self
.
backend
.
get_download_url
(
'foo'
)
self
.
_verify_url
(
url
)
@patch
(
'openassessment.fileupload.backends.swift.requests.get'
)
def
test_get_download_url_no_object
(
self
,
requests_get_mock
):
"""
Verify the download URL is empty when the object
cannot be found in storage.
"""
fake_resp
=
Mock
()
fake_resp
.
status_code
=
404
# always return a 404 status code
requests_get_mock
.
return_value
=
fake_resp
url
=
self
.
backend
.
get_download_url
(
'foo'
)
self
.
assertEqual
(
url
,
''
)
requirements/base.txt
View file @
dd59d943
...
@@ -10,6 +10,7 @@ git+https://github.com/edx/edx-submissions.git@1.1.5#egg=edx-submissions==1.1.5
...
@@ -10,6 +10,7 @@ git+https://github.com/edx/edx-submissions.git@1.1.5#egg=edx-submissions==1.1.5
# Third Party Requirements
# Third Party Requirements
boto>=2.32.1,<3.0.0
boto>=2.32.1,<3.0.0
python-swiftclient==3.1.0
celery==3.1.18
celery==3.1.18
defusedxml==0.4.1
defusedxml==0.4.1
django<1.9a0 # Resolves known bug on gemnasium. See TNL-6266
django<1.9a0 # Resolves known bug on gemnasium. See TNL-6266
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment