Skip to content

E
edx-platform
Unverified Commit 319aa877 by Troy Sankey Committed by GitHub
Options

Merge pull request #16487 from edx/pwnage101/courseware_mgmt_cleanup 

courseware management commands cleanup for django 1.11
parents f1f5a7dd 150096e3
Showing with 124 additions and 429 deletions
cms/djangoapps/contentstore/management/commands/import.py
"""
Script for importing courseware from XML format
"""
from optparse import make_option
from django.core.management.base import BaseCommand
from django_comment_common.utils import are_permissions_roles_seeded, seed_permissions_roles
from xmodule.contentstore.django import contentstore
from xmodule.modulestore import ModuleStoreEnum
......@@ -16,25 +13,24 @@ class Command(BaseCommand):
"""
Import the specified data directory into the default ModuleStore
"""
help = 'Import the specified data directory into the default ModuleStore'
help = 'Import the specified data directory into the default ModuleStore.'
option_list = BaseCommand.option_list + (
make_option('--nostatic',
action='store_true',
help='Skip import of static content'),
)
def add_arguments(self, parser):
parser.add_argument('data_directory')
parser.add_argument('course_dirs',
nargs='*',
metavar='course_dir')
parser.add_argument('--nostatic',
action='store_true',
help='skip import of static content')
def handle(self, *args, **options):
"Execute the command"
if len(args) == 0:
raise CommandError("import requires at least one argument: <data directory> [--nostatic] [<course dir>...]")
data_dir = args[0]
do_import_static = not options.get('nostatic', False)
if len(args) > 1:
source_dirs = args[1:]
else:
data_dir = options['data_directory']
do_import_static = not options['nostatic']
source_dirs = options['course_dirs']
if len(source_dirs) == 0:
source_dirs = None
self.stdout.write("Importing. Data_dir={data}, source_dirs={courses}\n".format(
data=data_dir,
courses=source_dirs,
......
lms/djangoapps/courseware/management/commands/clean_xml.py
from __future__ import print_function
import os
import sys
import traceback
import lxml.etree
from django.core.management.base import BaseCommand
from fs.osfs import OSFS
from path import Path as path
from xmodule.modulestore.xml import XMLModuleStore
def traverse_tree(course):
"""Load every descriptor in course. Return bool success value."""
"""
Load every descriptor in course. Return bool success value.
"""
queue = [course]
while len(queue) > 0:
node = queue.pop()
......@@ -21,13 +25,13 @@ def traverse_tree(course):
def export(course, export_dir):
"""Export the specified course to course_dir. Creates dir if it doesn't exist.
Overwrites files, does not clean out dir beforehand.
"""
Export the specified course to course_dir. Creates dir if it doesn't
exist. Overwrites files, does not clean out dir beforehand.
"""
fs = OSFS(export_dir, create=True)
if not fs.isdirempty('.'):
print ('WARNING: Directory {dir} not-empty.'
' May clobber/confuse things'.format(dir=export_dir))
print('WARNING: Directory {dir} not-empty. May clobber/confuse things'.format(dir=export_dir))
try:
course.runtime.export_fs = fs
......@@ -38,7 +42,7 @@ def export(course, export_dir):
return True
except:
print 'Export failed!'
print('Export failed!')
traceback.print_exc()
return False
......@@ -47,7 +51,7 @@ def export(course, export_dir):
def import_with_checks(course_dir):
all_ok = True
print "Attempting to load '{0}'".format(course_dir)
print('Attempting to load "{}"'.format(course_dir))
course_dir = path(course_dir)
data_dir = course_dir.dirname()
......@@ -69,88 +73,85 @@ def import_with_checks(course_dir):
n = len(courses)
if n != 1:
print 'ERROR: Expect exactly 1 course. Loaded {n}: {lst}'.format(
n=n, lst=courses)
print('ERROR: Expect exactly 1 course. Loaded {n}: {lst}'.format(n=n, lst=courses))
return (False, None)
course = courses[0]
errors = modulestore.get_course_errors(course.id)
if len(errors) != 0:
all_ok = False
print '\n'
print "=" * 40
print 'ERRORs during import:'
print '\n'.join(map(str_of_err, errors))
print "=" * 40
print '\n'
print(
'\n' +
'========================================' +
'ERRORs during import:' +
'\n'.join(map(str_of_err, errors)) +
'========================================' +
'\n'
)
# print course
validators = (
traverse_tree,
)
print "=" * 40
print "Running validators..."
print('========================================')
print('Running validators...')
for validate in validators:
print 'Running {0}'.format(validate.__name__)
print('Running {}'.format(validate.__name__))
all_ok = validate(course) and all_ok
if all_ok:
print 'Course passes all checks!'
print('Course passes all checks!')
else:
print "Course fails some checks. See above for errors."
print('Course fails some checks. See above for errors.')
return all_ok, course
def check_roundtrip(course_dir):
"""Check that import->export leaves the course the same"""
"""
Check that import->export leaves the course the same
"""
print "====== Roundtrip import ======="
print('====== Roundtrip import =======')
(ok, course) = import_with_checks(course_dir)
if not ok:
raise Exception("Roundtrip import failed!")
raise Exception('Roundtrip import failed!')
print "====== Roundtrip export ======="
export_dir = course_dir + ".rt"
print('====== Roundtrip export =======')
export_dir = course_dir + '.rt'
export(course, export_dir)
# dircmp doesn't do recursive diffs.
# diff = dircmp(course_dir, export_dir, ignore=[], hide=[])
print "======== Roundtrip diff: ========="
print('======== Roundtrip diff: =========')
sys.stdout.flush() # needed to make diff appear in the right place
os.system("diff -r {0} {1}".format(course_dir, export_dir))
print "======== ideally there is no diff above this ======="
def clean_xml(course_dir, export_dir, force):
(ok, course) = import_with_checks(course_dir)
if ok or force:
if not ok:
print "WARNING: Exporting despite errors"
export(course, export_dir)
check_roundtrip(export_dir)
else:
print "Did NOT export"
os.system('diff -r {} {}'.format(course_dir, export_dir))
print('======== ideally there is no diff above this =======')
class Command(BaseCommand):
help = """Imports specified course.xml, validate it, then exports in
a canonical format.
Usage: clean_xml PATH-TO-COURSE-DIR PATH-TO-OUTPUT-DIR [force]
help = 'Imports specified course, validates it, then exports it in a canonical format.'
If 'force' is specified as the last argument, exports even if there
were import errors.
"""
def add_arguments(self, parser):
parser.add_argument('course_dir',
help='path to the input course directory')
parser.add_argument('output_dir',
help='path to the output course directory')
parser.add_argument('--force',
action='store_true',
help='export course even if there were import errors')
def handle(self, *args, **options):
n = len(args)
if n < 2 or n > 3:
print Command.help
return
force = False
if n == 3 and args[2] == 'force':
force = True
clean_xml(args[0], args[1], force)
course_dir = options['course_dir']
output_dir = options['output_dir']
force = options['force']
(ok, course) = import_with_checks(course_dir)
if ok or force:
if not ok:
print('WARNING: Exporting despite errors')
export(course, output_dir)
check_roundtrip(output_dir)
else:
print('Did NOT export')
lms/djangoapps/courseware/management/commands/dump_course_ids.py
# pylint: disable=missing-docstring
"""
Dump the course_ids available to the lms.
Output is UTF-8 encoded by default.
"""
from __future__ import unicode_literals
from optparse import make_option
from textwrap import dedent
from django.core.management.base import BaseCommand
from six import text_type
from django.core.management.base import BaseCommand
from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
class Command(BaseCommand):
"""
Simple command to dump the course_ids available to the lms.
Output is UTF-8 encoded by default.
"""
help = dedent(__doc__).strip()
option_list = BaseCommand.option_list + (
make_option('--modulestore',
action='store',
default='default',
help='Name of the modulestore to use'),
)
def add_arguments(self, parser):
parser.add_argument('--modulestore',
default='default',
help='name of the modulestore to use')
def handle(self, *args, **options):
output = u'\n'.join(unicode(course_overview.id) for course_overview in CourseOverview.get_all_courses()) + '\n'
output = '\n'.join(text_type(course_overview.id) for course_overview in CourseOverview.get_all_courses()) + '\n'
return output
lms/djangoapps/courseware/management/commands/dump_course_structure.py
"""
A Django command that dumps the structure of a course as a JSON object.
Dump the structure of a course as a JSON object.
The resulting JSON object has one entry for each module in the course:
......@@ -24,7 +24,6 @@ from django.core.management.base import BaseCommand, CommandError
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
from xblock.fields import Scope
from xblock_discussion import DiscussionXBlock
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.inheritance import compute_inherited_metadata, own_metadata
......@@ -34,30 +33,22 @@ INHERITED_FILTER_LIST = ['children', 'xml_attributes']
class Command(BaseCommand):
"""
Write out to stdout a structural and metadata information for a
course as a JSON object
"""
args = "<course_id>"
help = dedent(__doc__).strip()
option_list = BaseCommand.option_list + (
make_option('--modulestore',
action='store',
default='default',
help='Name of the modulestore'),
make_option('--inherited',
action='store_true',
default=False,
help='Whether to include inherited metadata'),
make_option('--inherited_defaults',
action='store_true',
default=False,
help='Whether to include default values of inherited metadata'),
)
def add_arguments(self, parser):
parser.add_argument('course_id',
help='specifies the course to dump')
parser.add_argument('--modulestore',
default='default',
help='name of the modulestore')
parser.add_argument('--inherited',
action='store_true',
help='include inherited metadata'),
parser.add_argument('--inherited_defaults',
action='store_true',
help='include default values of inherited metadata'),
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("course_id not specified")
# Get the modulestore
......@@ -66,7 +57,7 @@ class Command(BaseCommand):
# Get the course data
try:
course_key = CourseKey.from_string(args[0])
course_key = CourseKey.from_string(options['course_id'])
except InvalidKeyError:
raise CommandError("Invalid course_id")
......
lms/djangoapps/courseware/management/commands/export_course.py deleted 100644 → 0
"""
A Django command that exports a course to a tar.gz file.
If <filename> is '-', it pipes the file to stdout
NOTE: This used to be used by Analytics research exports to provide
researchers with course content. It is now DEPRECATED, and
functionality has moved to export_olx.py in
cms/djangoapps/contentstore/management/commands.
Note: when removing this file, also remove references to it
from test_dump_course.
"""
import os
import re
import shutil
import tarfile
from tempfile import mkdtemp, mktemp
from textwrap import dedent
from django.core.management.base import BaseCommand, CommandError
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
from path import Path as path
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.xml_exporter import export_course_to_xml
class Command(BaseCommand):
"""
Export a course to XML. The output is compressed as a tar.gz file
"""
args = "<course_id> <output_filename>"
help = dedent(__doc__).strip()
def handle(self, *args, **options):
course_key, filename, pipe_results = self._parse_arguments(args)
export_course_to_tarfile(course_key, filename)
results = self._get_results(filename) if pipe_results else None
self.stdout.write(results, ending="")
def _parse_arguments(self, args):
"""Parse command line arguments"""
try:
course_key = CourseKey.from_string(args[0])
filename = args[1]
except InvalidKeyError:
raise CommandError("Unparsable course_id")
except IndexError:
raise CommandError("Insufficient arguments")
# If filename is '-' save to a temp file
pipe_results = False
if filename == '-':
filename = mktemp()
pipe_results = True
return course_key, filename, pipe_results
def _get_results(self, filename):
"""Load results from file"""
with open(filename) as f:
results = f.read()
os.remove(filename)
return results
def export_course_to_tarfile(course_key, filename):
"""Exports a course into a tar.gz file"""
tmp_dir = mkdtemp()
try:
course_dir = export_course_to_directory(course_key, tmp_dir)
compress_directory(course_dir, filename)
finally:
shutil.rmtree(tmp_dir, ignore_errors=True)
def export_course_to_directory(course_key, root_dir):
"""Export course into a directory"""
store = modulestore()
course = store.get_course(course_key)
if course is None:
raise CommandError("Invalid course_id")
# The safest characters are A-Z, a-z, 0-9, <underscore>, <period> and <hyphen>.
# We represent the first four with \w.
# TODO: Once we support courses with unicode characters, we will need to revisit this.
replacement_char = u'-'
course_dir = replacement_char.join([course.id.org, course.id.course, course.id.run])
course_dir = re.sub(r'[^\w\.\-]', replacement_char, course_dir)
export_course_to_xml(store, None, course.id, root_dir, course_dir)
export_dir = path(root_dir) / course_dir
return export_dir
def compress_directory(directory, filename):
"""Compress a directory into a tar.gz file"""
mode = 'w:gz'
name = path(directory).name
with tarfile.open(filename, mode) as tar_file:
tar_file.add(directory, arcname=name)
lms/djangoapps/courseware/management/commands/regrade_partial.py deleted 100644 → 0
'''
This is a one-off command aimed at fixing a temporary problem encountered where partial credit was awarded for
code problems, but the resulting score (or grade) was mistakenly set to zero because of a bug in
CorrectMap.get_npoints().
'''
import json
import logging
from optparse import make_option
from django.core.management.base import BaseCommand
from capa.correctmap import CorrectMap
from courseware.models import StudentModule
LOG = logging.getLogger(__name__)
class Command(BaseCommand):
'''
The fix here is to recalculate the score/grade based on the partial credit.
To narrow down the set of problems that might need fixing, the StudentModule
objects to be checked is filtered down to those:
created < '2013-03-08 15:45:00' (the problem must have been answered before the fix was installed,
on Prod and Edge)
modified > '2013-03-07 20:18:00' (the problem must have been visited after the bug was introduced)
state like '%"npoints": 0.%' (the problem must have some form of partial credit).
'''
num_visited = 0
num_changed = 0
option_list = BaseCommand.option_list + (
make_option('--save',
action='store_true',
dest='save_changes',
default=False,
help='Persist the changes that were encountered. If not set, no changes are saved.'), )
def fix_studentmodules(self, save_changes):
'''Identify the list of StudentModule objects that might need fixing, and then fix each one'''
modules = StudentModule.objects.filter(modified__gt='2013-03-07 20:18:00',
created__lt='2013-03-08 15:45:00',
state__contains='"npoints": 0.')
for module in modules:
self.fix_studentmodule_grade(module, save_changes)
def fix_studentmodule_grade(self, module, save_changes):
''' Fix the grade assigned to a StudentModule'''
module_state = module.state
if module_state is None:
# not likely, since we filter on it. But in general...
LOG.info(
u"No state found for %s module %s for student %s in course %s",
module.module_type,
module.module_state_key,
module.student.username,
module.course_id,
)
return
state_dict = json.loads(module_state)
self.num_visited += 1
# LoncapaProblem.get_score() checks student_answers -- if there are none, we will return a grade of 0
# Check that this is the case, but do so sooner, before we do any of the other grading work.
student_answers = state_dict['student_answers']
if (not student_answers) or len(student_answers) == 0:
# we should not have a grade here:
if module.grade != 0:
log_msg = (
u"No answer found but grade %(grade)s exists for %(type)s module %(id)s for student %(student)s " +
u"in course %(course_id)s"
)
LOG.error(log_msg, {
"grade": module.grade,
"type": module.module_type,
"id": module.module_state_key,
"student": module.student.username,
"course_id": module.course_id,
})
else:
log_msg = (
u"No answer and no grade found for %(type)s module %(id)s for student %(student)s " +
u"in course %(course_id)s"
)
LOG.debug(log_msg, {
"grade": module.grade,
"type": module.module_type,
"id": module.module_state_key,
"student": module.student.username,
"course_id": module.course_id,
})
return
# load into a CorrectMap, as done in LoncapaProblem.__init__():
correct_map = CorrectMap()
if 'correct_map' in state_dict:
correct_map.set_dict(state_dict['correct_map'])
# calculate score the way LoncapaProblem.get_score() works, by deferring to
# CorrectMap's get_npoints implementation.
correct = 0
for key in correct_map:
correct += correct_map.get_npoints(key)
if module.grade == correct:
# nothing to change
log_msg = u"Grade matches for %(type)s module %(id)s for student %(student)s in course %(course_id)s"
LOG.debug(log_msg, {
"type": module.module_type,
"id": module.module_state_key,
"student": module.student.username,
"course_id": module.course_id,
})
elif save_changes:
# make the change
log_msg = (
u"Grade changing from %(grade)s to %(correct)s for %(type)s module " +
u"%(id)s for student %(student)s in course %(course_id)s"
)
LOG.debug(log_msg, {
"grade": module.grade,
"correct": correct,
"type": module.module_type,
"id": module.module_state_key,
"student": module.student.username,
"course_id": module.course_id,
})
module.grade = correct
module.save()
self.num_changed += 1
else:
# don't make the change, but log that the change would be made
log_msg = (
u"Grade would change from %(grade)s to %(correct)s for %(type)s module %(id)s for student " +
u"%(student)s in course %(course_id)s"
)
LOG.debug(log_msg, {
"grade": module.grade,
"correct": correct,
"type": module.module_type,
"id": module.module_state_key,
"student": module.student.username,
"course_id": module.course_id,
})
self.num_changed += 1
def handle(self, **options):
'''Handle management command request'''
save_changes = options['save_changes']
LOG.info("Starting run: save_changes = %s", save_changes)
self.fix_studentmodules(save_changes)
LOG.info("Finished run: updating %s of %s modules", self.num_changed, self.num_visited)
lms/djangoapps/courseware/management/commands/tests/test_dump_course.py
# coding=utf-8
"""Tests for Django management commands"""
"""
Tests for Django management commands
"""
import json
import shutil
import tarfile
from StringIO import StringIO
from tempfile import mkdtemp
from nose.plugins.attrib import attr
from six import text_type
import factory
from django.conf import settings
from django.core.management import call_command
from nose.plugins.attrib import attr
from path import Path as path
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import (
......@@ -49,7 +48,9 @@ class CommandsTestBase(SharedModuleStoreTestCase):
@classmethod
def load_courses(cls):
"""Load test courses and return list of ids"""
"""
Load test courses and return list of ids
"""
store = modulestore()
unique_org = factory.Sequence(lambda n: 'edX.%d' % n)
......@@ -76,7 +77,9 @@ class CommandsTestBase(SharedModuleStoreTestCase):
return [course.id for course in store.get_courses()]
def call_command(self, name, *args, **kwargs):
"""Call management command and return output"""
"""
Call management command and return output
"""
out = StringIO() # To Capture the output of the command
call_command(name, *args, stdout=out, **kwargs)
out.seek(0)
......@@ -85,12 +88,12 @@ class CommandsTestBase(SharedModuleStoreTestCase):
def test_dump_course_ids(self):
output = self.call_command('dump_course_ids')
dumped_courses = output.decode('utf-8').strip().split('\n')
course_ids = {unicode(course_id) for course_id in self.loaded_courses}
course_ids = {text_type(course_id) for course_id in self.loaded_courses}
dumped_ids = set(dumped_courses)
self.assertEqual(course_ids, dumped_ids)
def test_correct_course_structure_metadata(self):
course_id = unicode(self.test_course_key)
course_id = text_type(self.test_course_key)
args = [course_id]
kwargs = {'modulestore': 'default'}
......@@ -103,7 +106,7 @@ class CommandsTestBase(SharedModuleStoreTestCase):
self.assertGreater(len(dump.values()), 0)
def test_dump_course_structure(self):
args = [unicode(self.test_course_key)]
args = [text_type(self.test_course_key)]
kwargs = {'modulestore': 'default'}
output = self.call_command('dump_course_structure', *args, **kwargs)
......@@ -119,7 +122,7 @@ class CommandsTestBase(SharedModuleStoreTestCase):
# Check a few elements in the course dump
test_course_key = self.test_course_key
parent_id = unicode(test_course_key.make_usage_key('chapter', 'Overview'))
parent_id = text_type(test_course_key.make_usage_key('chapter', 'Overview'))
self.assertEqual(dump[parent_id]['category'], 'chapter')
self.assertEqual(len(dump[parent_id]['children']), 3)
......@@ -127,7 +130,7 @@ class CommandsTestBase(SharedModuleStoreTestCase):
self.assertEqual(dump[child_id]['category'], 'videosequence')
self.assertEqual(len(dump[child_id]['children']), 2)
video_id = unicode(test_course_key.make_usage_key('video', 'Welcome'))
video_id = text_type(test_course_key.make_usage_key('video', 'Welcome'))
self.assertEqual(dump[video_id]['category'], 'video')
self.assertItemsEqual(
dump[video_id]['metadata'].keys(),
......@@ -140,7 +143,7 @@ class CommandsTestBase(SharedModuleStoreTestCase):
self.assertEqual(len(dump), 17)
def test_dump_inherited_course_structure(self):
args = [unicode(self.test_course_key)]
args = [text_type(self.test_course_key)]
kwargs = {'modulestore': 'default', 'inherited': True}
output = self.call_command('dump_course_structure', *args, **kwargs)
dump = json.loads(output)
......@@ -155,7 +158,7 @@ class CommandsTestBase(SharedModuleStoreTestCase):
self.assertNotIn('due', element['inherited_metadata'])
def test_dump_inherited_course_structure_with_defaults(self):
args = [unicode(self.test_course_key)]
args = [text_type(self.test_course_key)]
kwargs = {'modulestore': 'default', 'inherited': True, 'inherited_defaults': True}
output = self.call_command('dump_course_structure', *args, **kwargs)
dump = json.loads(output)
......@@ -170,37 +173,18 @@ class CommandsTestBase(SharedModuleStoreTestCase):
self.assertIsNone(element['inherited_metadata']['due'])
def test_export_discussion_ids(self):
output = self.call_command('dump_course_structure', unicode(self.course.id))
output = self.call_command('dump_course_structure', text_type(self.course.id))
dump = json.loads(output)
dumped_id = dump[unicode(self.discussion.location)]['metadata']['discussion_id']
dumped_id = dump[text_type(self.discussion.location)]['metadata']['discussion_id']
self.assertEqual(dumped_id, self.discussion.discussion_id)
def test_export_discussion_id_custom_id(self):
output = self.call_command('dump_course_structure', unicode(self.test_course_key))
output = self.call_command('dump_course_structure', text_type(self.test_course_key))
dump = json.loads(output)
discussion_key = unicode(self.test_course_key.make_usage_key('discussion', 'custom_id'))
dumped_id = dump[unicode(discussion_key)]['metadata']['discussion_id']
discussion_key = text_type(self.test_course_key.make_usage_key('discussion', 'custom_id'))
dumped_id = dump[text_type(discussion_key)]['metadata']['discussion_id']
self.assertEqual(dumped_id, "custom")
def test_export_course(self):
tmp_dir = path(mkdtemp())
self.addCleanup(shutil.rmtree, tmp_dir)
filename = tmp_dir / 'test.tar.gz'
self.run_export_course(filename)
with tarfile.open(filename) as tar_file:
self.check_export_file(tar_file)
def test_export_course_stdout(self):
output = self.run_export_course('-')
with tarfile.open(fileobj=StringIO(output)) as tar_file:
self.check_export_file(tar_file)
def run_export_course(self, filename): # pylint: disable=missing-docstring
args = [unicode(self.test_course_key), filename]
kwargs = {'modulestore': 'default'}
return self.call_command('export_course', *args, **kwargs)
def check_export_file(self, tar_file): # pylint: disable=missing-docstring
names = tar_file.getnames()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment