Commit e6240c56 by Nimisha Asthagiri

Update generate_course_blocks management command.

parent 8a784eef
""" """
Command to load course blocks. Command to load course blocks.
""" """
from collections import defaultdict
import logging import logging
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand, CommandError
...@@ -45,11 +46,32 @@ class Command(BaseCommand): ...@@ -45,11 +46,32 @@ class Command(BaseCommand):
action='store_true', action='store_true',
default=False, default=False,
) )
parser.add_argument(
'--verbose',
help='Enable verbose logging.',
action='store_true',
default=False,
)
parser.add_argument(
'--start',
help='Starting index of course.',
default=0,
type=int,
)
parser.add_argument(
'--end',
help='Ending index of course.',
default=0,
type=int,
)
def handle(self, *args, **options): def handle(self, *args, **options):
if options.get('all'): if options.get('all'):
course_keys = [course.id for course in modulestore().get_course_summaries()] course_keys = [course.id for course in modulestore().get_course_summaries()]
if options.get('start'):
end = options.get('end') or len(course_keys)
course_keys = course_keys[options['start']:end]
else: else:
if len(args) < 1: if len(args) < 1:
raise CommandError('At least one course or --all must be specified.') raise CommandError('At least one course or --all must be specified.')
...@@ -59,8 +81,13 @@ class Command(BaseCommand): ...@@ -59,8 +81,13 @@ class Command(BaseCommand):
raise CommandError('Invalid key specified.') raise CommandError('Invalid key specified.')
log.info('Generating course blocks for %d courses.', len(course_keys)) log.info('Generating course blocks for %d courses.', len(course_keys))
log.debug('Generating course blocks for the following courses: %s', course_keys)
if options.get('verbose'):
log.setLevel(logging.DEBUG)
else:
log.setLevel(logging.CRITICAL)
dag_info = _DAGInfo()
for course_key in course_keys: for course_key in course_keys:
try: try:
if options.get('force'): if options.get('force'):
...@@ -68,7 +95,7 @@ class Command(BaseCommand): ...@@ -68,7 +95,7 @@ class Command(BaseCommand):
else: else:
block_structure = get_course_in_cache(course_key) block_structure = get_course_in_cache(course_key)
if options.get('dags'): if options.get('dags'):
self._find_and_log_dags(block_structure, course_key) self._find_and_log_dags(block_structure, course_key, dag_info)
except Exception as ex: # pylint: disable=broad-except except Exception as ex: # pylint: disable=broad-except
log.exception( log.exception(
'An error occurred while generating course blocks for %s: %s', 'An error occurred while generating course blocks for %s: %s',
...@@ -78,20 +105,83 @@ class Command(BaseCommand): ...@@ -78,20 +105,83 @@ class Command(BaseCommand):
log.info('Finished generating course blocks.') log.info('Finished generating course blocks.')
def _find_and_log_dags(self, block_structure, course_key): if options.get('dags'):
log.critical('DAG data: %s', unicode(dag_info))
def _find_and_log_dags(self, block_structure, course_key, dag_info):
""" """
Finds all DAGs within the given block structure. Finds all DAGs within the given block structure.
Arguments: Arguments:
BlockStructureBlockData - The block structure in which to find DAGs. BlockStructureBlockData - The block structure in which to find DAGs.
""" """
log.info('DAG check starting for course %s.', unicode(course_key))
for block_key in block_structure.get_block_keys(): for block_key in block_structure.get_block_keys():
parents = block_structure.get_parents(block_key) parents = block_structure.get_parents(block_key)
if len(parents) > 1: if len(parents) > 1:
dag_info.on_dag_found(course_key, block_key)
log.warning( log.warning(
'DAG alert - %s has multiple parents: %s.', 'DAG alert - %s has multiple parents: %s.',
unicode(block_key), unicode(block_key),
[unicode(parent) for parent in parents], [unicode(parent) for parent in parents],
) )
log.info('DAG check complete for course %s.', unicode(course_key))
class PrettyDefaultDict(defaultdict):
"""
Wraps defaultdict to provide a better string representation.
"""
__repr__ = dict.__repr__
class _DAGBlockTypeInfo(object):
"""
Class for aggregated DAG data for a specific block type.
"""
def __init__(self):
self.num_of_dag_blocks = 0
def __repr__(self):
return repr(vars(self))
class _DAGCourseInfo(object):
"""
Class for aggregated DAG data for a specific course run.
"""
def __init__(self):
self.num_of_dag_blocks = 0
self.dag_data_by_block_type = PrettyDefaultDict(_DAGBlockTypeInfo)
def __repr__(self):
return repr(vars(self))
def on_dag_found(self, block_key):
"""
Updates DAG collected data for the given block.
"""
self.num_of_dag_blocks += 1
self.dag_data_by_block_type[block_key.category].num_of_dag_blocks += 1
class _DAGInfo(object):
"""
Class for aggregated DAG data.
"""
def __init__(self):
self.total_num_of_dag_blocks = 0
self.total_num_of_dag_courses = 0
self.dag_data_by_course = PrettyDefaultDict(_DAGCourseInfo)
self.dag_data_by_block_type = PrettyDefaultDict(_DAGBlockTypeInfo)
def __repr__(self):
return repr(vars(self))
def on_dag_found(self, course_key, block_key):
"""
Updates DAG collected data for the given block.
"""
self.total_num_of_dag_blocks += 1
if course_key not in self.dag_data_by_course:
self.total_num_of_dag_courses += 1
self.dag_data_by_course[unicode(course_key)].on_dag_found(block_key)
self.dag_data_by_block_type[block_key.category].num_of_dag_blocks += 1
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment