Commit 86683400 by Zia Fazal

Merge pull request #652 from edx-solutions/ziafazal/YONK-280

ziafazal/YONK-280: add course aggregate meta date model
parents 7e98f454 6d778118
......@@ -11,7 +11,7 @@ from django.conf import settings
import ddt
import copy
from openedx.core.djangoapps.content.course_structures.tests import SignalDisconnectTestMixin
from openedx.core.djangoapps.util.testing import SignalDisconnectTestMixin
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.django import modulestore
......
......@@ -21,7 +21,7 @@ from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from mock import Mock
from opaque_keys.edx.locator import CourseKey, LibraryLocator
from openedx.core.djangoapps.content.course_structures.tests import SignalDisconnectTestMixin
from openedx.core.djangoapps.util.testing import SignalDisconnectTestMixin
class LibraryTestCase(ModuleStoreTestCase):
......
......@@ -14,6 +14,7 @@ from xmodule.modulestore.tests.factories import ItemFactory
from xmodule.validation import StudioValidation, StudioValidationMessage
from xmodule.modulestore.django import modulestore
from xmodule.modulestore import ModuleStoreEnum
from openedx.core.djangoapps.util.testing import SignalDisconnectTestMixin
GROUP_CONFIGURATION_JSON = {
u'name': u'Test name',
......@@ -204,7 +205,8 @@ class GroupConfigurationsBaseTestCase(object):
# pylint: disable=no-member
class GroupConfigurationsListHandlerTestCase(CourseTestCase, GroupConfigurationsBaseTestCase, HelperMethods):
class GroupConfigurationsListHandlerTestCase(SignalDisconnectTestMixin, CourseTestCase,
GroupConfigurationsBaseTestCase, HelperMethods):
"""
Test cases for group_configurations_list_handler.
"""
......
......@@ -769,6 +769,7 @@ INSTALLED_APPS = (
'openedx.core.djangoapps.content.course_overviews',
'openedx.core.djangoapps.content.course_structures',
'openedx.core.djangoapps.content.course_metadata',
# Credit courses
'openedx.core.djangoapps.credit',
......
......@@ -25,12 +25,10 @@ from capa.tests.response_xml_factory import StringResponseXMLFactory
from courseware import module_render
from courseware.tests.factories import StudentModuleFactory
from courseware.model_data import FieldDataCache
from courseware.models import StudentModule
from django_comment_common.models import Role, FORUM_ROLE_MODERATOR
from gradebook.models import StudentGradebook
from instructor.access import allow_access
from student.tests.factories import UserFactory, CourseEnrollmentFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from student.models import CourseEnrollment
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, mixed_store_config
......@@ -40,6 +38,7 @@ from api_manager.courseware_access import get_course_key
from .content import TEST_COURSE_OVERVIEW_CONTENT, TEST_COURSE_UPDATES_CONTENT, TEST_COURSE_UPDATES_CONTENT_LEGACY
from .content import TEST_STATIC_TAB1_CONTENT, TEST_STATIC_TAB2_CONTENT
MODULESTORE_CONFIG = mixed_store_config(settings.COMMON_TEST_DATA_ROOT, {}, include_xml=False)
TEST_API_KEY = str(uuid.uuid4())
USER_COUNT = 6
......
......@@ -37,10 +37,15 @@ from student.models import CourseEnrollment, CourseEnrollmentAllowed
from student.roles import CourseRole, CourseAccessRole, CourseInstructorRole, CourseStaffRole, CourseObserverRole, CourseAssistantRole, UserBasedRole, get_aggregate_exclusion_user_ids
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.search import path_to_location
from openedx.core.djangoapps.content.course_metadata.models import CourseAggregatedMetaData
from api_manager.courseware_access import get_course, get_course_child, get_course_leaf_nodes, get_course_key, \
from api_manager.courseware_access import get_course, get_course_child, get_course_key, \
course_exists, get_modulestore, get_course_descriptor
from api_manager.models import CourseGroupRelationship, CourseContentGroupRelationship, GroupProfile
from api_manager.models import (
CourseGroupRelationship,
CourseContentGroupRelationship,
GroupProfile,
)
from progress.models import CourseModuleCompletion
from api_manager.permissions import SecureAPIView, SecureListAPIView
from api_manager.users.serializers import UserSerializer, UserCountByCitySerializer
......@@ -1736,7 +1741,8 @@ class CoursesMetricsCompletionsLeadersList(SecureAPIView):
if not course_exists(request, request.user, course_id):
return Response({}, status=status.HTTP_404_NOT_FOUND)
course_key = get_course_key(course_id)
total_possible_completions = float(len(get_course_leaf_nodes(course_key)))
course_metadata = CourseAggregatedMetaData.get_from_id(course_key)
total_possible_completions = float(course_metadata.total_assessments)
exclude_users = get_aggregate_exclusion_user_ids(course_key)
orgs_filter = self.request.QUERY_PARAMS.get('organizations', None)
if orgs_filter:
......@@ -1758,7 +1764,7 @@ class CoursesMetricsCompletionsLeadersList(SecureAPIView):
if orgs_filter:
total_users_qs = total_users_qs.filter(organizations__in=orgs_filter)
total_users = total_users_qs.count()
if total_users and total_actual_completions:
if total_users and total_actual_completions and total_possible_completions:
course_avg = total_actual_completions / float(total_users)
course_avg = min(100 * (course_avg / total_possible_completions), 100) # avg in percentage
data['course_avg'] = course_avg
......
""" Centralized access to LMS courseware app """
from django.contrib.auth.models import AnonymousUser
from django.utils import timezone
from django.conf import settings
from courseware import courses, module_render
from courseware.model_data import FieldDataCache
......@@ -64,22 +62,6 @@ def get_course_total_score(course_summary):
return score
def get_course_leaf_nodes(course_key):
"""
Get count of the leaf nodes with ability to exclude some categories
"""
nodes = []
detached_categories = getattr(settings, 'PROGRESS_DETACHED_CATEGORIES', [])
store = get_modulestore()
verticals = store.get_items(course_key, qualifiers={'category': 'vertical'})
orphans = store.get_orphans(course_key)
for vertical in verticals:
if hasattr(vertical, 'children') and vertical.location not in orphans:
nodes.extend([unit for unit in vertical.children
if getattr(unit, 'category') not in detached_categories])
return nodes
def get_course_key(course_id, slashseparated=False):
try:
course_key = CourseKey.from_string(course_id)
......
......@@ -3,8 +3,6 @@
""" Database ORM models managed by this Django app """
from django.contrib.auth.models import Group, User
from django.db import models
from django.db.models import Q
from django.conf import settings
from model_utils.models import TimeStampedModel
from .utils import is_int
......
"""
Signal handlers supporting various gradebook use cases
Signal handlers supporting various course metadata use cases
"""
from django.dispatch import receiver
from util.signals import course_deleted
from .models import CourseGroupRelationship, CourseContentGroupRelationship
......
......@@ -1961,6 +1961,7 @@ INSTALLED_APPS = (
'openedx.core.djangoapps.content.course_overviews',
'openedx.core.djangoapps.content.course_structures',
'openedx.core.djangoapps.content.course_metadata',
'course_structure_api',
# Mailchimp Syncing
......
......@@ -2,3 +2,4 @@
Setup the signals on startup.
"""
import openedx.core.djangoapps.content.course_structures.signals
import openedx.core.djangoapps.content.course_metadata.signals
# -*- coding: utf-8 -*-
from south.utils import datetime_utils as datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'CourseAggregatedMetaData'
db.create_table('course_metadata_courseaggregatedmetadata', (
('created', self.gf('model_utils.fields.AutoCreatedField')(default=datetime.datetime.now)),
('modified', self.gf('model_utils.fields.AutoLastModifiedField')(default=datetime.datetime.now)),
('id', self.gf('xmodule_django.models.CourseKeyField')(max_length=255, primary_key=True, db_index=True)),
('total_modules', self.gf('django.db.models.fields.IntegerField')(default=0)),
('total_assessments', self.gf('django.db.models.fields.IntegerField')(default=0)),
))
db.send_create_signal('course_metadata', ['CourseAggregatedMetaData'])
def backwards(self, orm):
# Deleting model 'CourseAggregatedMetaData'
db.delete_table('course_metadata_courseaggregatedmetadata')
models = {
'course_metadata.courseaggregatedmetadata': {
'Meta': {'object_name': 'CourseAggregatedMetaData'},
'created': ('model_utils.fields.AutoCreatedField', [], {'default': 'datetime.datetime.now'}),
'id': ('xmodule_django.models.CourseKeyField', [], {'max_length': '255', 'primary_key': 'True', 'db_index': 'True'}),
'modified': ('model_utils.fields.AutoLastModifiedField', [], {'default': 'datetime.datetime.now'}),
'total_assessments': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'total_modules': ('django.db.models.fields.IntegerField', [], {'default': '0'})
}
}
complete_apps = ['course_metadata']
\ No newline at end of file
"""
Models for course_metadata app
"""
from django.db import models
from model_utils.models import TimeStampedModel
from xmodule_django.models import CourseKeyField
from openedx.core.djangoapps.content.course_metadata.utils import get_course_leaf_nodes
class CourseAggregatedMetaData(TimeStampedModel):
"""
Model for storing and caching aggregated metadata about a course.
This model contains aggregated metadata about a course such as
total modules, total assessments.
"""
id = CourseKeyField(db_index=True, primary_key=True, max_length=255) # pylint: disable=invalid-name
total_modules = models.IntegerField(default=0)
total_assessments = models.IntegerField(default=0)
@staticmethod
def get_from_id(course_id):
"""
Load a CourseAggregatedMetaData object for a given course ID.
First, we try to load the CourseAggregatedMetaData from the database. If it
doesn't exist, we create CourseAggregatedMetaData in the database for
future use.
Arguments:
course_id (CourseKey): the ID of the course aggregated data to be loaded
Returns:
CourseAggregatedMetaData: aggregated data of the requested course
"""
try:
course_metadata = CourseAggregatedMetaData.objects.get(id=course_id)
except CourseAggregatedMetaData.DoesNotExist:
course_metadata = CourseAggregatedMetaData(id=course_id)
course_metadata.total_assessments = len(get_course_leaf_nodes(course_id))
course_metadata.save()
return course_metadata
"""
This module has definition of receivers for signals
"""
from django.dispatch.dispatcher import receiver
from xmodule.modulestore.django import SignalHandler
from openedx.core.djangoapps.content.course_metadata.tasks import update_course_aggregate_metadata
@receiver(SignalHandler.course_published)
def listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable=unused-argument
"""
Receives signal and kicks off celery task to update course aggregate metadata
"""
# Note: The countdown=0 kwarg is set to to ensure the method below does not attempt to access the course
# before the signal emitter has finished all operations. This is also necessary to ensure all tests pass.
update_course_aggregate_metadata.apply_async([unicode(course_key)], countdown=0)
"""
This module has implementation of celery tasks to compute course aggregate metadata
"""
import logging
from celery.task import task
from opaque_keys.edx.keys import CourseKey
from openedx.core.djangoapps.content.course_metadata.models import CourseAggregatedMetaData
from openedx.core.djangoapps.content.course_metadata.utils import get_course_leaf_nodes
log = logging.getLogger('edx.celery.task')
@task(name=u'lms.djangoapps.api_manager.tasks.update_course_metadata')
def update_course_aggregate_metadata(course_key): # pylint: disable=invalid-name
"""
Regenerates and updates the course aggregate metadata (in the database) for the specified course.
"""
if not isinstance(course_key, basestring):
raise ValueError('course_key must be a string. {} is not acceptable.'.format(type(course_key)))
course_key = CourseKey.from_string(course_key)
try:
course_leaf_nodes = get_course_leaf_nodes(course_key)
except Exception as ex:
log.exception('An error occurred while retrieving course assessments: %s', ex.message)
raise
course_metadata, __ = CourseAggregatedMetaData.objects.get_or_create(id=course_key)
course_metadata.total_assessments = len(course_leaf_nodes)
course_metadata.save()
"""
This module has tests for utils.py
"""
# pylint: disable=no-member
import ddt
from django.test.utils import override_settings
from student.tests.factories import UserFactory
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from openedx.core.djangoapps.content.course_metadata.utils import get_course_leaf_nodes
@ddt.ddt
class UtilsTests(ModuleStoreTestCase):
""" Test suite to test operation in utils"""
def setUp(self):
super(UtilsTests, self).setUp()
self.course = CourseFactory.create()
self.test_data = '<html>Test data</html>'
self.chapter = ItemFactory.create(
category="chapter",
parent_location=self.course.location,
data=self.test_data,
display_name="Overview",
)
self.sub_section = ItemFactory.create(
parent_location=self.chapter.location,
category="sequential",
display_name=u"test subsection",
)
self.sub_section2 = ItemFactory.create(
parent_location=self.chapter.location,
category="sequential",
display_name=u"test subsection 2",
)
self.vertical = ItemFactory.create(
parent_location=self.sub_section.location,
category="vertical",
metadata={'graded': True, 'format': 'Homework'},
display_name=u"test vertical",
)
self.vertical2 = ItemFactory.create(
parent_location=self.sub_section2.location,
category="vertical",
metadata={'graded': True, 'format': 'FinalExam'},
display_name=u"test vertical 2",
)
self.content_child1 = ItemFactory.create(
category="html",
parent_location=self.vertical.location,
data=self.test_data,
display_name="Html component"
)
self.content_child2 = ItemFactory.create(
category="video",
parent_location=self.vertical.location,
data=self.test_data,
display_name="Video component"
)
self.content_child3 = ItemFactory.create(
category="group-project",
parent_location=self.vertical.location,
data=self.test_data,
display_name="group project component"
)
self.content_child4 = ItemFactory.create(
category="html",
parent_location=self.vertical2.location,
data=self.test_data,
display_name="Html component 2"
)
self.user = UserFactory()
@override_settings(PROGRESS_DETACHED_CATEGORIES=[])
@ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
def test_get_course_leaf_nodes(self, module_store_type):
"""
Tests get_course_leaf_nodes works as expected
"""
with modulestore().default_store(module_store_type):
nodes = get_course_leaf_nodes(self.course.id)
self.assertEqual(len(nodes), 4)
@override_settings(PROGRESS_DETACHED_CATEGORIES=["group-project"])
@ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
def test_get_course_leaf_nodes_with_detached_categories(self, module_store_type):
"""
Tests get_course_leaf_nodes with detached categories
"""
with modulestore().default_store(module_store_type):
nodes = get_course_leaf_nodes(self.course.id)
# group-project project node should not be counted
self.assertEqual(len(nodes), 3)
@override_settings(PROGRESS_DETACHED_CATEGORIES=[])
def test_get_course_leaf_nodes_with_orphan_nodes(self):
"""
Tests get_course_leaf_nodes if some nodes are orphan
"""
with modulestore().default_store(ModuleStoreEnum.Type.mongo):
with modulestore().branch_setting(ModuleStoreEnum.Branch.draft_preferred):
# delete sub_section2 to make vertical2 orphan
store = modulestore()
store.delete_item(self.sub_section2.location, self.user.id)
nodes = get_course_leaf_nodes(self.course.id)
self.assertEqual(len(nodes), 3)
"""
Tests for course_metadata app
"""
from mock_django import mock_signal_receiver
from xmodule.modulestore.django import SignalHandler
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from openedx.core.djangoapps.content.course_metadata.signals import listen_for_course_publish
from openedx.core.djangoapps.content.course_metadata.models import CourseAggregatedMetaData
class CoursesMetaDataTests(ModuleStoreTestCase):
""" Test suite for Course Meta Data """
def setUp(self):
super(CoursesMetaDataTests, self).setUp()
self.course = CourseFactory.create()
self.test_data = '<html>Test data</html>'
self.chapter = ItemFactory.create(
category="chapter",
parent_location=self.course.location,
data=self.test_data,
display_name="Overview",
)
self.sub_section = ItemFactory.create(
parent_location=self.chapter.location,
category="sequential",
display_name=u"test subsection",
)
self.unit = ItemFactory.create(
parent_location=self.sub_section.location,
category="vertical",
metadata={'graded': True, 'format': 'Homework'},
display_name=u"test unit",
)
self.content_child1 = ItemFactory.create(
category="html",
parent_location=self.unit.location,
data=self.test_data,
display_name="Html component"
)
def test_course_aggregate_metadata_update_on_course_published(self):
"""
Test course aggregate metadata update receiver is called on course_published signal
and CourseAggregatedMetaData is updated
"""
with mock_signal_receiver(SignalHandler.course_published, wraps=listen_for_course_publish) as receiver:
self.assertEqual(receiver.call_count, 0)
# adding new video unit to course should fire the signal
ItemFactory.create(
category="video",
parent_location=self.unit.location,
data=self.test_data,
display_name="Video to test aggregates"
)
self.assertEqual(receiver.call_count, 1)
total_assessments = CourseAggregatedMetaData.objects.get(id=self.course.id).total_assessments
self.assertEqual(total_assessments, 2)
def test_get_course_aggregate_metadata_by_course_key(self):
"""
Test course aggregate metadata should compute and return metadata
when called by get_from_id
"""
course_metadata = CourseAggregatedMetaData.get_from_id(self.course.id)
self.assertEqual(course_metadata.total_assessments, 1)
"""
Utility methods for course metadata app
"""
from django.conf import settings
from xmodule.modulestore.django import modulestore
def get_course_leaf_nodes(course_key):
"""
Get count of the leaf nodes with ability to exclude some categories
"""
nodes = []
detached_categories = getattr(settings, 'PROGRESS_DETACHED_CATEGORIES', [])
store = modulestore()
verticals = store.get_items(course_key, qualifiers={'category': 'vertical'})
orphans = store.get_orphans(course_key)
for vertical in verticals:
if hasattr(vertical, 'children') and vertical.location not in orphans:
nodes.extend([unit for unit in vertical.children
if getattr(unit, 'category') not in detached_categories])
return nodes
import json
from xmodule.modulestore.django import SignalHandler
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from openedx.core.djangoapps.content.course_structures.models import CourseStructure
from openedx.core.djangoapps.content.course_structures.signals import listen_for_course_publish
from openedx.core.djangoapps.content.course_structures.tasks import _generate_course_structure, update_course_structure
class SignalDisconnectTestMixin(object):
"""
Mixin for tests to disable calls to signals.listen_for_course_publish when the course_published signal is fired.
"""
def setUp(self):
super(SignalDisconnectTestMixin, self).setUp()
SignalHandler.course_published.disconnect(listen_for_course_publish)
class CourseStructureTaskTests(ModuleStoreTestCase):
def setUp(self, **kwargs):
super(CourseStructureTaskTests, self).setUp()
......
......@@ -3,14 +3,20 @@
from datetime import datetime
from pytz import UTC
from openedx.core.djangoapps.course_groups.models import CourseUserGroupPartitionGroup
from openedx.core.djangoapps.course_groups.tests.helpers import CohortFactory
from openedx.core.djangoapps.user_api.tests.factories import UserCourseTagFactory
from xmodule.modulestore.django import SignalHandler
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.partitions.partitions import UserPartition, Group
from student.tests.factories import CourseEnrollmentFactory, UserFactory
from openedx.core.djangoapps.course_groups.models import CourseUserGroupPartitionGroup
from openedx.core.djangoapps.course_groups.tests.helpers import CohortFactory
from openedx.core.djangoapps.content.course_structures.signals import listen_for_course_publish
from openedx.core.djangoapps.content.course_metadata.signals import (
listen_for_course_publish as course_publish_listener
)
from openedx.core.djangoapps.user_api.tests.factories import UserCourseTagFactory
class ContentGroupTestCase(ModuleStoreTestCase):
"""
......@@ -207,3 +213,14 @@ class TestConditionalContent(ModuleStoreTestCase):
display_name='Group B problem container',
location=vertical_b_url
)
class SignalDisconnectTestMixin(object):
"""
Mixin for tests to disable calls to signals.listen_for_course_publish when the course_published signal is fired.
"""
def setUp(self):
super(SignalDisconnectTestMixin, self).setUp()
SignalHandler.course_published.disconnect(listen_for_course_publish)
SignalHandler.course_published.disconnect(course_publish_listener)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment