Skip to content

E
edx-platform
Commit bdb078d0 by Nimisha Asthagiri Committed by GitHub
Options

Merge pull request #16075 from edx/naa/grades-remove-old-commands 

Grades: remove unneeded management commands
parents 00dadc3a b2c6a534
Showing with 2 additions and 633 deletions
lms/djangoapps/grades/management/commands/get_grades.py deleted 100644 → 0
"""
Management command to generate a list of grades for
all students that are enrolled in a course.
"""
import csv
import datetime
import os
from optparse import make_option
from django.contrib.auth.models import User
from django.core.handlers.base import BaseHandler
from django.core.management.base import BaseCommand, CommandError
from django.test.client import RequestFactory
from opaque_keys.edx.keys import CourseKey
from lms.djangoapps.certificates.models import GeneratedCertificate
from lms.djangoapps.courseware import courses
from lms.djangoapps.grades.new.course_grade_factory import CourseGradeFactory
class RequestMock(RequestFactory):
"""
Class to create a mock request.
"""
def request(self, **request):
"Construct a generic request object."
request = RequestFactory.request(self, **request)
handler = BaseHandler()
handler.load_middleware()
for middleware_method in handler._request_middleware: # pylint: disable=protected-access
if middleware_method(request):
raise Exception("Couldn't create request mock object - "
"request middleware returned a response")
return request
class Command(BaseCommand):
"""
Management command for get_grades
"""
help = """
Generate a list of grades for all students
that are enrolled in a course.
CSV will include the following:
- username
- email
- grade in the certificate table if it exists
- computed grade
- grade breakdown
Outputs grades to a csv file.
Example:
sudo -u www-data SERVICE_VARIANT=lms /opt/edx/bin/django-admin.py get_grades \
-c MITx/Chi6.00intro/A_Taste_of_Python_Programming -o /tmp/20130813-6.00x.csv \
--settings=lms.envs.aws --pythonpath=/opt/wwc/edx-platform
"""
option_list = BaseCommand.option_list + (
make_option('-c', '--course',
metavar='COURSE_ID',
dest='course',
default=False,
help='Course ID for grade distribution'),
make_option('-o', '--output',
metavar='FILE',
dest='output',
default=False,
help='Filename for grade output'))
def handle(self, *args, **options):
if os.path.exists(options['output']):
raise CommandError("File {0} already exists".format(
options['output']))
status_interval = 100
# parse out the course into a coursekey
if options['course']:
course_key = CourseKey.from_string(options['course'])
print "Fetching enrolled students for {0}".format(course_key)
enrolled_students = User.objects.filter(
courseenrollment__course_id=course_key
)
factory = RequestMock()
request = factory.get('/')
total = enrolled_students.count()
print "Total enrolled: {0}".format(total)
course = courses.get_course_by_id(course_key)
total = enrolled_students.count()
start = datetime.datetime.now()
rows = []
header = None
print "Fetching certificate data"
cert_grades = {
cert.user.username: cert.grade
for cert in list(
GeneratedCertificate.objects.filter( # pylint: disable=no-member
course_id=course_key
).prefetch_related('user')
)
}
print "Grading students"
for count, student in enumerate(enrolled_students):
count += 1
if count % status_interval == 0:
# Print a status update with an approximation of
# how much time is left based on how long the last
# interval took
diff = datetime.datetime.now() - start
timeleft = diff * (total - count) / status_interval
hours, remainder = divmod(timeleft.seconds, 3600)
minutes, __ = divmod(remainder, 60)
print "{0}/{1} completed ~{2:02}:{3:02}m remaining".format(
count, total, hours, minutes)
start = datetime.datetime.now()
request.user = student
grade = CourseGradeFactory().create(student, course)
if not header:
header = [section['label'] for section in grade.summary[u'section_breakdown']]
rows.append(["email", "username", "certificate-grade", "grade"] + header)
percents = {section['label']: section['percent'] for section in grade.summary[u'section_breakdown']}
row_percents = [percents[label] for label in header]
if student.username in cert_grades:
rows.append(
[student.email, student.username, cert_grades[student.username], grade.percent] + row_percents,
)
else:
rows.append([student.email, student.username, "N/A", grade.percent] + row_percents)
with open(options['output'], 'wb') as f:
writer = csv.writer(f)
writer.writerows(rows)
lms/djangoapps/grades/management/commands/reset_grades.py deleted 100644 → 0
"""
Reset persistent grades for learners.
"""
import logging
from datetime import datetime
from textwrap import dedent
from django.core.management.base import BaseCommand, CommandError
from django.db.models import Count
from pytz import utc
from lms.djangoapps.grades.models import PersistentCourseGrade, PersistentSubsectionGrade
from openedx.core.lib.command_utils import get_mutually_exclusive_required_option, parse_course_keys
log = logging.getLogger(__name__)
DATE_FORMAT = "%Y-%m-%d %H:%M"
class Command(BaseCommand):
"""
Reset persistent grades for learners.
"""
help = dedent(__doc__).strip()
def add_arguments(self, parser):
"""
Add arguments to the command parser.
"""
parser.add_argument(
'--dry_run',
action='store_true',
default=False,
dest='dry_run',
help="Output what we're going to do, but don't actually do it. To actually delete, use --delete instead."
)
parser.add_argument(
'--delete',
action='store_true',
default=False,
dest='delete',
help="Actually perform the deletions of the course. For a Dry Run, use --dry_run instead."
)
parser.add_argument(
'--courses',
dest='courses',
nargs='+',
help='Reset persistent grades for the list of courses provided.',
)
parser.add_argument(
'--all_courses',
action='store_true',
dest='all_courses',
default=False,
help='Reset persistent grades for all courses.',
)
parser.add_argument(
'--modified_start',
dest='modified_start',
help='Starting range for modified date (inclusive): e.g. "2016-08-23 16:43"; expected in UTC.',
)
parser.add_argument(
'--modified_end',
dest='modified_end',
help='Ending range for modified date (inclusive): e.g. "2016-12-23 16:43"; expected in UTC.',
)
parser.add_argument(
'--db_table',
dest='db_table',
help='Specify "subsection" to reset subsection grades or "course" to reset course grades. If absent, both '
'are reset.',
)
def handle(self, *args, **options):
course_keys = None
modified_start = None
modified_end = None
run_mode = get_mutually_exclusive_required_option(options, 'delete', 'dry_run')
courses_mode = get_mutually_exclusive_required_option(options, 'courses', 'all_courses')
db_table = options.get('db_table')
if db_table not in {'subsection', 'course', None}:
raise CommandError('Invalid value for db_table. Valid options are "subsection" or "course" only.')
if options.get('modified_start'):
modified_start = utc.localize(datetime.strptime(options['modified_start'], DATE_FORMAT))
if options.get('modified_end'):
if not modified_start:
raise CommandError('Optional value for modified_end provided without a value for modified_start.')
modified_end = utc.localize(datetime.strptime(options['modified_end'], DATE_FORMAT))
if courses_mode == 'courses':
course_keys = parse_course_keys(options['courses'])
log.info("reset_grade: Started in %s mode!", run_mode)
operation = self._query_grades if run_mode == 'dry_run' else self._delete_grades
if db_table == 'subsection' or db_table is None:
operation(PersistentSubsectionGrade, course_keys, modified_start, modified_end)
if db_table == 'course' or db_table is None:
operation(PersistentCourseGrade, course_keys, modified_start, modified_end)
log.info("reset_grade: Finished in %s mode!", run_mode)
def _delete_grades(self, grade_model_class, *args, **kwargs):
"""
Deletes the requested grades in the given model, filtered by the provided args and kwargs.
"""
grades_query_set = grade_model_class.query_grades(*args, **kwargs)
num_rows_to_delete = grades_query_set.count()
log.info("reset_grade: Deleting %s: %d row(s).", grade_model_class.__name__, num_rows_to_delete)
grade_model_class.delete_grades(*args, **kwargs)
log.info("reset_grade: Deleted %s: %d row(s).", grade_model_class.__name__, num_rows_to_delete)
def _query_grades(self, grade_model_class, *args, **kwargs):
"""
Queries the requested grades in the given model, filtered by the provided args and kwargs.
"""
total_for_all_courses = 0
grades_query_set = grade_model_class.query_grades(*args, **kwargs)
grades_stats = grades_query_set.values('course_id').order_by().annotate(total=Count('course_id'))
for stat in grades_stats:
total_for_all_courses += stat['total']
log.info(
"reset_grade: Would delete %s for COURSE %s: %d row(s).",
grade_model_class.__name__,
stat['course_id'],
stat['total'],
)
log.info(
"reset_grade: Would delete %s in TOTAL: %d row(s).",
grade_model_class.__name__,
total_for_all_courses,
)
lms/djangoapps/grades/management/commands/tests/test_reset_grades.py deleted 100644 → 0
"""
Tests for reset_grades management command.
"""
from datetime import datetime, timedelta
import ddt
from django.core.management.base import CommandError
from django.test import TestCase
from freezegun import freeze_time
from mock import MagicMock, patch
from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
from lms.djangoapps.grades.management.commands import reset_grades
from lms.djangoapps.grades.models import PersistentCourseGrade, PersistentSubsectionGrade
@ddt.ddt
class TestResetGrades(TestCase):
"""
Tests generate course blocks management command.
"""
num_users = 3
num_courses = 5
num_subsections = 7
def setUp(self):
super(TestResetGrades, self).setUp()
self.command = reset_grades.Command()
self.user_ids = [user_id for user_id in range(self.num_users)]
self.course_keys = []
for course_index in range(self.num_courses):
self.course_keys.append(
CourseLocator(
org='some_org',
course='some_course',
run=unicode(course_index),
)
)
self.subsection_keys_by_course = {}
for course_key in self.course_keys:
subsection_keys_in_course = []
for subsection_index in range(self.num_subsections):
subsection_keys_in_course.append(
BlockUsageLocator(
course_key=course_key,
block_type='sequential',
block_id=unicode(subsection_index),
)
)
self.subsection_keys_by_course[course_key] = subsection_keys_in_course
def _update_or_create_grades(self, courses_keys=None):
"""
Creates grades for all courses and subsections.
"""
if courses_keys is None:
courses_keys = self.course_keys
course_grade_params = {
"course_version": "JoeMcEwing",
"course_edited_timestamp": datetime(
year=2016,
month=8,
day=1,
hour=18,
minute=53,
second=24,
microsecond=354741,
),
"percent_grade": 77.7,
"letter_grade": "Great job",
"passed": True,
}
subsection_grade_params = {
"course_version": "deadbeef",
"subtree_edited_timestamp": "2016-08-01 18:53:24.354741",
"earned_all": 6.0,
"possible_all": 12.0,
"earned_graded": 6.0,
"possible_graded": 8.0,
"visible_blocks": MagicMock(),
"first_attempted": datetime.now(),
}
for course_key in courses_keys:
for user_id in self.user_ids:
course_grade_params['user_id'] = user_id
course_grade_params['course_id'] = course_key
PersistentCourseGrade.update_or_create(**course_grade_params)
for subsection_key in self.subsection_keys_by_course[course_key]:
subsection_grade_params['user_id'] = user_id
subsection_grade_params['usage_key'] = subsection_key
PersistentSubsectionGrade.update_or_create_grade(**subsection_grade_params)
def _assert_grades_exist_for_courses(self, course_keys, db_table=None):
"""
Assert grades for given courses exist.
"""
for course_key in course_keys:
if db_table == "course" or db_table is None:
self.assertIsNotNone(PersistentCourseGrade.read(self.user_ids[0], course_key))
if db_table == "subsection" or db_table is None:
for subsection_key in self.subsection_keys_by_course[course_key]:
self.assertIsNotNone(PersistentSubsectionGrade.read_grade(self.user_ids[0], subsection_key))
def _assert_grades_absent_for_courses(self, course_keys, db_table=None):
"""
Assert grades for given courses do not exist.
"""
for course_key in course_keys:
if db_table == "course" or db_table is None:
with self.assertRaises(PersistentCourseGrade.DoesNotExist):
PersistentCourseGrade.read(self.user_ids[0], course_key)
if db_table == "subsection" or db_table is None:
for subsection_key in self.subsection_keys_by_course[course_key]:
with self.assertRaises(PersistentSubsectionGrade.DoesNotExist):
PersistentSubsectionGrade.read_grade(self.user_ids[0], subsection_key)
def _assert_stat_logged(self, mock_log, num_rows, grade_model_class, message_substring, log_offset):
self.assertIn('reset_grade: ' + message_substring, mock_log.info.call_args_list[log_offset][0][0])
self.assertEqual(grade_model_class.__name__, mock_log.info.call_args_list[log_offset][0][1])
self.assertEqual(num_rows, mock_log.info.call_args_list[log_offset][0][2])
def _assert_course_delete_stat_logged(self, mock_log, num_rows):
self._assert_stat_logged(mock_log, num_rows, PersistentCourseGrade, 'Deleted', log_offset=4)
def _assert_subsection_delete_stat_logged(self, mock_log, num_rows):
self._assert_stat_logged(mock_log, num_rows, PersistentSubsectionGrade, 'Deleted', log_offset=2)
def _assert_course_query_stat_logged(self, mock_log, num_rows, num_courses=None):
if num_courses is None:
num_courses = self.num_courses
log_offset = num_courses + 1 + num_courses + 1
self._assert_stat_logged(mock_log, num_rows, PersistentCourseGrade, 'Would delete', log_offset)
def _assert_subsection_query_stat_logged(self, mock_log, num_rows, num_courses=None):
if num_courses is None:
num_courses = self.num_courses
log_offset = num_courses + 1
self._assert_stat_logged(mock_log, num_rows, PersistentSubsectionGrade, 'Would delete', log_offset)
def _date_from_now(self, days=None):
return datetime.now() + timedelta(days=days)
def _date_str_from_now(self, days=None):
future_date = self._date_from_now(days=days)
return future_date.strftime(reset_grades.DATE_FORMAT)
@patch('lms.djangoapps.grades.management.commands.reset_grades.log')
def test_reset_all_courses(self, mock_log):
self._update_or_create_grades()
self._assert_grades_exist_for_courses(self.course_keys)
with self.assertNumQueries(7):
self.command.handle(delete=True, all_courses=True)
self._assert_grades_absent_for_courses(self.course_keys)
self._assert_subsection_delete_stat_logged(
mock_log,
num_rows=self.num_users * self.num_courses * self.num_subsections,
)
self._assert_course_delete_stat_logged(
mock_log,
num_rows=self.num_users * self.num_courses,
)
@patch('lms.djangoapps.grades.management.commands.reset_grades.log')
@ddt.data(1, 2, 3)
def test_reset_some_courses(self, num_courses_to_reset, mock_log):
self._update_or_create_grades()
self._assert_grades_exist_for_courses(self.course_keys)
with self.assertNumQueries(6):
self.command.handle(
delete=True,
courses=[unicode(course_key) for course_key in self.course_keys[:num_courses_to_reset]]
)
self._assert_grades_absent_for_courses(self.course_keys[:num_courses_to_reset])
self._assert_grades_exist_for_courses(self.course_keys[num_courses_to_reset:])
self._assert_subsection_delete_stat_logged(
mock_log,
num_rows=self.num_users * num_courses_to_reset * self.num_subsections,
)
self._assert_course_delete_stat_logged(
mock_log,
num_rows=self.num_users * num_courses_to_reset,
)
def test_reset_by_modified_start_date(self):
self._update_or_create_grades()
self._assert_grades_exist_for_courses(self.course_keys)
num_courses_with_updated_grades = 2
with freeze_time(self._date_from_now(days=4)):
self._update_or_create_grades(self.course_keys[:num_courses_with_updated_grades])
with self.assertNumQueries(6):
self.command.handle(delete=True, modified_start=self._date_str_from_now(days=2), all_courses=True)
self._assert_grades_absent_for_courses(self.course_keys[:num_courses_with_updated_grades])
self._assert_grades_exist_for_courses(self.course_keys[num_courses_with_updated_grades:])
def test_reset_by_modified_start_end_date(self):
self._update_or_create_grades()
self._assert_grades_exist_for_courses(self.course_keys)
with freeze_time(self._date_from_now(days=3)):
self._update_or_create_grades(self.course_keys[:2])
with freeze_time(self._date_from_now(days=5)):
self._update_or_create_grades(self.course_keys[2:4])
with self.assertNumQueries(6):
self.command.handle(
delete=True,
modified_start=self._date_str_from_now(days=2),
modified_end=self._date_str_from_now(days=4),
all_courses=True,
)
# Only grades for courses modified within the 2->4 days
# should be deleted.
self._assert_grades_absent_for_courses(self.course_keys[:2])
self._assert_grades_exist_for_courses(self.course_keys[2:])
@ddt.data('subsection', 'course')
def test_specify_db_table(self, db_table):
self._update_or_create_grades()
self._assert_grades_exist_for_courses(self.course_keys)
self.command.handle(delete=True, all_courses=True, db_table=db_table)
self._assert_grades_absent_for_courses(self.course_keys, db_table=db_table)
if db_table == "subsection":
self._assert_grades_exist_for_courses(self.course_keys, db_table='course')
else:
self._assert_grades_exist_for_courses(self.course_keys, db_table='subsection')
@patch('lms.djangoapps.grades.management.commands.reset_grades.log')
def test_dry_run_all_courses(self, mock_log):
self._update_or_create_grades()
self._assert_grades_exist_for_courses(self.course_keys)
with self.assertNumQueries(2):
self.command.handle(dry_run=True, all_courses=True)
self._assert_grades_exist_for_courses(self.course_keys)
self._assert_subsection_query_stat_logged(
mock_log,
num_rows=self.num_users * self.num_courses * self.num_subsections,
)
self._assert_course_query_stat_logged(
mock_log,
num_rows=self.num_users * self.num_courses,
)
@patch('lms.djangoapps.grades.management.commands.reset_grades.log')
@ddt.data(1, 2, 3)
def test_dry_run_some_courses(self, num_courses_to_query, mock_log):
self._update_or_create_grades()
self._assert_grades_exist_for_courses(self.course_keys)
with self.assertNumQueries(2):
self.command.handle(
dry_run=True,
courses=[unicode(course_key) for course_key in self.course_keys[:num_courses_to_query]]
)
self._assert_grades_exist_for_courses(self.course_keys)
self._assert_subsection_query_stat_logged(
mock_log,
num_rows=self.num_users * num_courses_to_query * self.num_subsections,
num_courses=num_courses_to_query,
)
self._assert_course_query_stat_logged(
mock_log,
num_rows=self.num_users * num_courses_to_query,
num_courses=num_courses_to_query,
)
@patch('lms.djangoapps.grades.management.commands.reset_grades.log')
def test_reset_no_existing_grades(self, mock_log):
self._assert_grades_absent_for_courses(self.course_keys)
with self.assertNumQueries(4):
self.command.handle(delete=True, all_courses=True)
self._assert_grades_absent_for_courses(self.course_keys)
self._assert_subsection_delete_stat_logged(mock_log, num_rows=0)
self._assert_course_delete_stat_logged(mock_log, num_rows=0)
def test_invalid_key(self):
with self.assertRaisesRegexp(CommandError, 'Invalid key specified.*invalid/key'):
self.command.handle(dry_run=True, courses=['invalid/key'])
def test_invalid_db_table(self):
with self.assertRaisesMessage(
CommandError,
'Invalid value for db_table. Valid options are "subsection" or "course" only.'
):
self.command.handle(delete=True, all_courses=True, db_table="not course or subsection")
def test_no_run_mode(self):
with self.assertRaisesMessage(CommandError, 'Must specify exactly one of --delete, --dry_run'):
self.command.handle(all_courses=True)
def test_both_run_modes(self):
with self.assertRaisesMessage(CommandError, 'Must specify exactly one of --delete, --dry_run'):
self.command.handle(all_courses=True, dry_run=True, delete=True)
def test_no_course_mode(self):
with self.assertRaisesMessage(CommandError, 'Must specify exactly one of --courses, --all_courses'):
self.command.handle(dry_run=True)
def test_both_course_modes(self):
with self.assertRaisesMessage(CommandError, 'Must specify exactly one of --courses, --all_courses'):
self.command.handle(dry_run=True, all_courses=True, courses=['some/course/key'])
lms/djangoapps/grades/models.py
......@@ -39,38 +39,6 @@ BLOCK_RECORD_LIST_VERSION = 1
BlockRecord = namedtuple('BlockRecord', ['locator', 'weight', 'raw_possible', 'graded'])
class DeleteGradesMixin(object):
"""
A Mixin class that provides functionality to delete grades.
"""
@classmethod
def query_grades(cls, course_ids=None, modified_start=None, modified_end=None):
"""
Queries all the grades in the table, filtered by the provided arguments.
"""
kwargs = {}
if course_ids:
kwargs['course_id__in'] = [course_id for course_id in course_ids]
if modified_start:
if modified_end:
kwargs['modified__range'] = (modified_start, modified_end)
else:
kwargs['modified__gt'] = modified_start
return cls.objects.filter(**kwargs)
@classmethod
def delete_grades(cls, *args, **kwargs):
"""
Deletes all the grades in the table, filtered by the provided arguments.
"""
query = cls.query_grades(*args, **kwargs)
query.delete()
class BlockRecordList(tuple):
"""
An immutable ordered list of BlockRecord objects.
......@@ -285,7 +253,7 @@ class VisibleBlocks(models.Model):
return u"visible_blocks_cache.{}".format(course_key)
class PersistentSubsectionGrade(DeleteGradesMixin, TimeStampedModel):
class PersistentSubsectionGrade(TimeStampedModel):
"""
A django model tracking persistent grades at the subsection level.
"""
......@@ -546,7 +514,7 @@ class PersistentSubsectionGrade(DeleteGradesMixin, TimeStampedModel):
)
class PersistentCourseGrade(DeleteGradesMixin, TimeStampedModel):
class PersistentCourseGrade(TimeStampedModel):
"""
A django model tracking persistent course grades.
"""
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment