Skip to content

E
edx-platform
Commit cd84a9e7 by Renzo Lucioni
Options

Merge pull request #11714 from edx/renzo/sysadmin-xml 

Whittling away at XML modulestore code
parents 95007a83 32a29f4f
Showing with 3 additions and 377 deletions
lms/djangoapps/dashboard/sysadmin.py
...@@ -59,10 +59,6 @@ class SysadminDashboardView(TemplateView): ...@@ -59,10 +59,6 @@ class SysadminDashboardView(TemplateView):
""" """
self.def_ms = modulestore() self.def_ms = modulestore()
self.is_using_mongo = True
if self.def_ms.get_modulestore_type(None) == 'xml':
self.is_using_mongo = False
self.msg = u'' self.msg = u''
self.datatable = [] self.datatable = []
super(SysadminDashboardView, self).__init__(**kwargs) super(SysadminDashboardView, self).__init__(**kwargs)
...@@ -374,11 +370,8 @@ class Courses(SysadminDashboardView): ...@@ -374,11 +370,8 @@ class Courses(SysadminDashboardView):
return _("The git repo location should end with '.git', " return _("The git repo location should end with '.git', "
"and be a valid url") "and be a valid url")
if self.is_using_mongo:
return self.import_mongo_course(gitloc, branch) return self.import_mongo_course(gitloc, branch)
return self.import_xml_course(gitloc, branch)
def import_mongo_course(self, gitloc, branch): def import_mongo_course(self, gitloc, branch):
""" """
Imports course using management command and captures logging output Imports course using management command and captures logging output
...@@ -429,80 +422,6 @@ class Courses(SysadminDashboardView): ...@@ -429,80 +422,6 @@ class Courses(SysadminDashboardView):
msg += u"<pre>{0}</pre>".format(escape(ret)) msg += u"<pre>{0}</pre>".format(escape(ret))
return msg return msg
def import_xml_course(self, gitloc, branch):
"""Imports a git course into the XMLModuleStore"""
msg = u''
if not getattr(settings, 'GIT_IMPORT_WITH_XMLMODULESTORE', False):
# Translators: "GIT_IMPORT_WITH_XMLMODULESTORE" is a variable name.
# "XMLModuleStore" and "MongoDB" are database systems. You should not
# translate these names.
return _('Refusing to import. GIT_IMPORT_WITH_XMLMODULESTORE is '
'not turned on, and it is generally not safe to import '
'into an XMLModuleStore with multithreaded. We '
'recommend you enable the MongoDB based module store '
'instead, unless this is a development environment.')
cdir = (gitloc.rsplit('/', 1)[1])[:-4]
gdir = settings.DATA_DIR / cdir
if os.path.exists(gdir):
msg += _("The course {0} already exists in the data directory! "
"(reloading anyway)").format(cdir)
cmd = ['git', 'pull', ]
cwd = gdir
else:
cmd = ['git', 'clone', gitloc, ]
cwd = settings.DATA_DIR
cwd = os.path.abspath(cwd)
try:
cmd_output = escape(
subprocess.check_output(cmd, stderr=subprocess.STDOUT, cwd=cwd)
)
except subprocess.CalledProcessError as ex:
log.exception('Git pull or clone output was: %r', ex.output)
# Translators: unable to download the course content from
# the source git repository. Clone occurs if this is brand
# new, and pull is when it is being updated from the
# source.
return _('Unable to clone or pull repository. Please check '
'your url. Output was: {0!r}').format(ex.output)
msg += u'<pre>{0}</pre>'.format(cmd_output)
if not os.path.exists(gdir):
msg += _('Failed to clone repository to {directory_name}').format(directory_name=gdir)
return msg
# Change branch if specified
if branch:
try:
git_import.switch_branch(branch, gdir)
except GitImportError as ex:
return str(ex)
# Translators: This is a git repository branch, which is a
# specific version of a courses content
msg += u'<p>{0}</p>'.format(
_('Successfully switched to branch: '
'{branch_name}').format(branch_name=branch))
self.def_ms.try_load_course(os.path.abspath(gdir))
errlog = self.def_ms.errored_courses.get(cdir, '')
if errlog:
msg += u'<hr width="50%"><pre>{0}</pre>'.format(escape(errlog))
else:
course = self.def_ms.courses[os.path.abspath(gdir)]
msg += _('Loaded course {course_name}<br/>Errors:').format(
course_name="{} {}".format(cdir, course.display_name)
)
errors = self.def_ms.get_course_errors(course.id)
if not errors:
msg += u'None'
else:
msg += u'<ul>'
for (summary, err) in errors:
msg += u'<li><pre>{0}: {1}</pre></li>'.format(escape(summary),
escape(err))
msg += u'</ul>'
return msg
def make_datatable(self): def make_datatable(self):
"""Creates course information datatable""" """Creates course information datatable"""
......
lms/djangoapps/dashboard/tests/test_sysadmin.py
...@@ -20,8 +20,6 @@ from django.utils.translation import ugettext as _ ...@@ -20,8 +20,6 @@ from django.utils.translation import ugettext as _
import mongoengine import mongoengine
from opaque_keys.edx.locations import SlashSeparatedCourseKey from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.modulestore.tests.django_utils import TEST_DATA_XML_MODULESTORE
from dashboard.models import CourseImportLog from dashboard.models import CourseImportLog
from dashboard.sysadmin import Users from dashboard.sysadmin import Users
from dashboard.git_import import GitImportError from dashboard.git_import import GitImportError
...@@ -30,7 +28,7 @@ from external_auth.models import ExternalAuthMap ...@@ -30,7 +28,7 @@ from external_auth.models import ExternalAuthMap
from student.roles import CourseStaffRole, GlobalStaff from student.roles import CourseStaffRole, GlobalStaff
from student.tests.factories import UserFactory from student.tests.factories import UserFactory
from xmodule.modulestore.django import modulestore from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST
...@@ -46,7 +44,7 @@ FEATURES_WITH_SSL_AUTH = settings.FEATURES.copy() ...@@ -46,7 +44,7 @@ FEATURES_WITH_SSL_AUTH = settings.FEATURES.copy()
FEATURES_WITH_SSL_AUTH['AUTH_USE_CERTIFICATES'] = True FEATURES_WITH_SSL_AUTH['AUTH_USE_CERTIFICATES'] = True
class SysadminBaseTestCase(ModuleStoreTestCase): class SysadminBaseTestCase(SharedModuleStoreTestCase):
""" """
Base class with common methods used in XML and Mongo tests Base class with common methods used in XML and Mongo tests
""" """
...@@ -57,7 +55,7 @@ class SysadminBaseTestCase(ModuleStoreTestCase): ...@@ -57,7 +55,7 @@ class SysadminBaseTestCase(ModuleStoreTestCase):
def setUp(self): def setUp(self):
"""Setup test case by adding primary user.""" """Setup test case by adding primary user."""
super(SysadminBaseTestCase, self).setUp(create_user=False) super(SysadminBaseTestCase, self).setUp()
self.user = UserFactory.create(username='test_user', self.user = UserFactory.create(username='test_user',
email='test_user+sysadmin@edx.org', email='test_user+sysadmin@edx.org',
password='foo') password='foo')
...@@ -116,297 +114,6 @@ class SysadminBaseTestCase(ModuleStoreTestCase): ...@@ -116,297 +114,6 @@ class SysadminBaseTestCase(ModuleStoreTestCase):
@attr('shard_1') @attr('shard_1')
@unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'),
"ENABLE_SYSADMIN_DASHBOARD not set")
@override_settings(GIT_IMPORT_WITH_XMLMODULESTORE=True)
class TestSysadmin(SysadminBaseTestCase):
"""
Test sysadmin dashboard features using XMLModuleStore
"""
MODULESTORE = TEST_DATA_XML_MODULESTORE
def test_staff_access(self):
"""Test access controls."""
test_views = ['sysadmin', 'sysadmin_courses', 'sysadmin_staffing', ]
for view in test_views:
response = self.client.get(reverse(view))
self.assertEqual(response.status_code, 302)
self.user.is_staff = False
self.user.save()
logged_in = self.client.login(username=self.user.username,
password='foo')
self.assertTrue(logged_in)
for view in test_views:
response = self.client.get(reverse(view))
self.assertEqual(response.status_code, 404)
response = self.client.get(reverse('gitlogs'))
self.assertEqual(response.status_code, 404)
self.user.is_staff = True
self.user.save()
self.client.logout()
self.client.login(username=self.user.username, password='foo')
for view in test_views:
response = self.client.get(reverse(view))
self.assertTrue(response.status_code, 200)
response = self.client.get(reverse('gitlogs'))
self.assertTrue(response.status_code, 200)
def test_user_mod(self):
"""Create and delete a user"""
self._setstaff_login()
self.client.login(username=self.user.username, password='foo')
# Create user tests
# No uname
response = self.client.post(reverse('sysadmin'),
{'action': 'create_user',
'student_fullname': 'blah',
'student_password': 'foozor', })
self.assertIn('Must provide username', response.content.decode('utf-8'))
# no full name
response = self.client.post(reverse('sysadmin'),
{'action': 'create_user',
'student_uname': 'test_cuser+sysadmin@edx.org',
'student_password': 'foozor', })
self.assertIn('Must provide full name', response.content.decode('utf-8'))
# Test create valid user
self.client.post(reverse('sysadmin'),
{'action': 'create_user',
'student_uname': 'test_cuser+sysadmin@edx.org',
'student_fullname': 'test cuser',
'student_password': 'foozor', })
self.assertIsNotNone(
User.objects.get(username='test_cuser+sysadmin@edx.org',
email='test_cuser+sysadmin@edx.org'))
# login as new user to confirm
self.assertTrue(self.client.login(
username='test_cuser+sysadmin@edx.org', password='foozor'))
self.client.logout()
self.client.login(username=self.user.username, password='foo')
# Delete user tests
# Try no username
response = self.client.post(reverse('sysadmin'),
{'action': 'del_user', })
self.assertIn('Must provide username', response.content.decode('utf-8'))
# Try bad usernames
response = self.client.post(reverse('sysadmin'),
{'action': 'del_user',
'student_uname': 'flabbergast@example.com',
'student_fullname': 'enigma jones', })
self.assertIn('Cannot find user with email address', response.content.decode('utf-8'))
response = self.client.post(reverse('sysadmin'),
{'action': 'del_user',
'student_uname': 'flabbergast',
'student_fullname': 'enigma jones', })
self.assertIn('Cannot find user with username', response.content.decode('utf-8'))
self.client.post(reverse('sysadmin'),
{'action': 'del_user',
'student_uname': 'test_cuser+sysadmin@edx.org',
'student_fullname': 'test cuser', })
self.assertEqual(0, len(User.objects.filter(
username='test_cuser+sysadmin@edx.org',
email='test_cuser+sysadmin@edx.org')))
self.assertEqual(1, len(User.objects.all()))
def test_user_csv(self):
"""Download and validate user CSV"""
num_test_users = 100
self._setstaff_login()
# Stuff full of users to test streaming
for user_num in xrange(num_test_users):
Users().create_user('testingman_with_long_name{}'.format(user_num),
'test test')
response = self.client.post(reverse('sysadmin'),
{'action': 'download_users', })
self.assertIn('attachment', response['Content-Disposition'])
self.assertEqual('text/csv', response['Content-Type'])
self.assertIn('test_user', response.content)
self.assertTrue(num_test_users + 2, len(response.content.splitlines()))
# Clean up
User.objects.filter(
username__startswith='testingman_with_long_name').delete()
@override_settings(FEATURES=FEATURES_WITH_SSL_AUTH)
def test_authmap_repair(self):
"""Run authmap check and repair"""
self._setstaff_login()
Users().create_user('test0', 'test test')
# Will raise exception, so no assert needed
eamap = ExternalAuthMap.objects.get(external_name='test test')
mitu = User.objects.get(username='test0')
self.assertTrue(check_password(eamap.internal_password, mitu.password))
mitu.set_password('not autogenerated')
mitu.save()
self.assertFalse(check_password(eamap.internal_password, mitu.password))
# Create really non user AuthMap
ExternalAuthMap(external_id='ll',
external_domain='ll',
external_credentials='{}',
external_email='a@b.c',
external_name='c',
internal_password='').save()
response = self.client.post(reverse('sysadmin'),
{'action': 'repair_eamap', })
self.assertIn('{0} test0'.format('Failed in authenticating'),
response.content)
self.assertIn('fixed password', response.content.decode('utf-8'))
self.assertTrue(self.client.login(username='test0',
password=eamap.internal_password))
# Check for all OK
self._setstaff_login()
response = self.client.post(reverse('sysadmin'),
{'action': 'repair_eamap', })
self.assertIn('All ok!', response.content.decode('utf-8'))
def test_xml_course_add_delete(self):
"""add and delete course from xml module store"""
self._setstaff_login()
# Try bad git repo
response = self.client.post(reverse('sysadmin_courses'), {
'repo_location': 'github.com/mitocw/edx4edx_lite',
'action': 'add_course', })
self.assertIn(_("The git repo location should end with '.git', "
"and be a valid url"), response.content.decode('utf-8'))
response = self.client.post(reverse('sysadmin_courses'), {
'repo_location': 'http://example.com/not_real.git',
'action': 'add_course', })
self.assertIn('Unable to clone or pull repository',
response.content.decode('utf-8'))
# Create git loaded course
response = self._add_edx4edx()
def_ms = modulestore()
self.assertEqual('xml', def_ms.get_modulestore_type(None))
course = def_ms.courses.get('{0}/edx4edx_lite'.format(
os.path.abspath(settings.DATA_DIR)), None)
self.assertIsNotNone(course)
# Delete a course
self._rm_edx4edx()
course = def_ms.courses.get('{0}/edx4edx_lite'.format(
os.path.abspath(settings.DATA_DIR)), None)
self.assertIsNone(course)
# Load a bad git branch
response = self._add_edx4edx('asdfasdfasdf')
self.assertIn(GitImportError.REMOTE_BRANCH_MISSING,
response.content.decode('utf-8'))
# Load a course from a git branch
self._add_edx4edx(self.TEST_BRANCH)
course = def_ms.courses.get('{0}/edx4edx_lite'.format(
os.path.abspath(settings.DATA_DIR)), None)
self.assertIsNotNone(course)
self.assertEqual(self.TEST_BRANCH_COURSE, course.id)
self._rm_edx4edx()
# Try and delete a non-existent course
response = self.client.post(reverse('sysadmin_courses'),
{'course_id': 'foobar/foo/blah',
'action': 'del_course', })
self.assertIn('Error - cannot get course with ID',
response.content.decode('utf-8'))
@override_settings(GIT_IMPORT_WITH_XMLMODULESTORE=False)
def test_xml_safety_flag(self):
"""Make sure the settings flag to disable xml imports is working"""
self._setstaff_login()
response = self._add_edx4edx()
self.assertIn('GIT_IMPORT_WITH_XMLMODULESTORE', response.content)
def_ms = modulestore()
course = def_ms.courses.get('{0}/edx4edx_lite'.format(
os.path.abspath(settings.DATA_DIR)), None)
self.assertIsNone(course)
def test_git_pull(self):
"""Make sure we can pull"""
self._setstaff_login()
response = self._add_edx4edx()
response = self._add_edx4edx()
self.assertIn(_("The course {0} already exists in the data directory! "
"(reloading anyway)").format('edx4edx_lite'),
response.content.decode('utf-8'))
self._rm_edx4edx()
def test_staff_csv(self):
"""Download and validate staff CSV"""
self._setstaff_login()
self._add_edx4edx()
def_ms = modulestore()
course = def_ms.get_course(SlashSeparatedCourseKey('MITx', 'edx4edx', 'edx4edx'))
CourseStaffRole(course.id).add_users(self.user)
response = self.client.post(reverse('sysadmin_staffing'),
{'action': 'get_staff_csv', })
self.assertIn('attachment', response['Content-Disposition'])
self.assertEqual('text/csv', response['Content-Type'])
columns = ['course_id', 'role', 'username',
'email', 'full_name', ]
self.assertIn(','.join('"' + c + '"' for c in columns),
response.content)
self._rm_edx4edx()
def test_enrollment_page(self):
"""
Adds a course and makes sure that it shows up on the staffing and
enrollment page
"""
self._setstaff_login()
self._add_edx4edx()
response = self.client.get(reverse('sysadmin_staffing'))
self.assertIn('edx4edx', response.content)
self._rm_edx4edx()
@attr('shard_1')
@override_settings(MONGODB_LOG=TEST_MONGODB_LOG) @override_settings(MONGODB_LOG=TEST_MONGODB_LOG)
@unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), @unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'),
"ENABLE_SYSADMIN_DASHBOARD not set") "ENABLE_SYSADMIN_DASHBOARD not set")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment