Merge pull request #194 from edx/remove-demographics-switch

Removed Demographics Switch

Makefile

@@ -64,7 +64,6 @@ validate: validate_python validate_js
 
 demo:
 	cd analytics_dashboard && ./manage.py switch show_engagement_forum_activity on --create
-	cd analytics_dashboard && ./manage.py switch show_navbar_demographics on --create
 	cd analytics_dashboard && ./manage.py switch display_verified_enrollment on --create
 
 compile_translations:

README.md

@@ -53,7 +53,6 @@ The following switches are available:
 | Switch                            | Purpose                                                  |
 |-----------------------------------|----------------------------------------------------------|
 | show_engagement_forum_activity    | Show the forum activity on the course engagement page    |
-| show_navbar_demographics          | Show the enrollment demographics navbar item             |
 
 
 Authentication & Authorization

acceptance_tests/__init__.py

@@ -39,4 +39,3 @@ DOC_BASE_URL = os.environ.get('DOC_BASE_URL', 'http://edx-insights.readthedocs.o
 
 ENABLE_ENROLLMENT_MODES = str2bool(os.environ.get('ENABLE_ENROLLMENT_MODES', False))
 ENABLE_FORUM_POSTS = str2bool(os.environ.get('ENABLE_FORUM_POSTS', False))
-ENABLE_DEMOGRAPHICS_TESTS = str2bool(os.environ.get('ENABLE_DEMOGRAPHICS_TESTS', False))

acceptance_tests/mixins.py

@@ -20,7 +20,7 @@ class AnalyticsApiClientMixin(object):
 
         api_url = API_SERVER_URL
         auth_token = API_AUTH_TOKEN
-        self.api_client = Client(api_url, auth_token=auth_token, timeout=5)
+        self.api_client = Client(api_url, auth_token=auth_token, timeout=10)
 
 
 class AssertMixin(object):
@@ -167,6 +167,13 @@ class AnalyticsDashboardWebAppTestMixin(PrimaryNavMixin, ContextSensitiveHelpMix
         PrimaryNavMixin.test_page(self)
         ContextSensitiveHelpMixin.test_page(self)
 
+    def date_strip_leading_zeroes(self, s):
+        """
+        Remove the leading 0 on formatted date strings.
+        :param s: Date formatted as string
+        """
+        return s.replace(' 0', ' ')
+
 
 class CoursePageTestsMixin(AnalyticsApiClientMixin, FooterMixin, AnalyticsDashboardWebAppTestMixin):
     """ Mixin for common course page assertions and tests. """
@@ -320,6 +327,3 @@ class CourseDemographicsPageTestsMixin(CoursePageTestsMixin):
         last_updated = datetime.datetime.strptime(current_data['created'], self.api_datetime_format)
         return 'Demographic student data was last updated %(update_date)s at %(update_time)s UTC.' % \
                self.format_last_updated_date_and_time(last_updated)
-
-    def _get_total_demographics(self):
-        return float(sum([datum['count'] for datum in self.demographic_data]))

acceptance_tests/test_course_engagement.py

@@ -90,8 +90,10 @@ class CourseEngagementTests(CoursePageTestsMixin, WebAppTest):
         for i, row in enumerate(rows):
             columns = row.find_elements_by_css_selector('td')
             weekly_activity = trend_activity[i]
-            expected_date = self.format_time_as_dashboard((
-                datetime.datetime.strptime(weekly_activity['interval_end'], date_time_format)) - datetime.timedelta(days=1)).replace(' 0', ' ')
+            expected_date = self.format_time_as_dashboard(
+                (datetime.datetime.strptime(weekly_activity['interval_end'], date_time_format)) - datetime.timedelta(
+                    days=1))
+            expected_date = self.date_strip_leading_zeroes(expected_date)
             expected = [expected_date,
                         self.format_number(weekly_activity[at.ANY]),
                         self.format_number(weekly_activity[at.PLAYED_VIDEO]),

acceptance_tests/test_course_enrollment.py

@@ -93,8 +93,9 @@ class CourseEnrollmentActivityTests(CoursePageTestsMixin, WebAppTest):
         for i, row in enumerate(rows):
             columns = row.find_elements_by_css_selector('td')
             enrollment = enrollment_data[i]
-            expected_date = datetime.datetime.strptime(enrollment['date'], self.api_date_format).strftime(
-                "%B %d, %Y").replace(' 0', ' ')
+            expected_date = datetime.datetime.strptime(enrollment['date'], self.api_date_format).strftime("%B %d, %Y")
+            expected_date = self.date_strip_leading_zeroes(expected_date)
+
             expected = [expected_date, self.format_number(enrollment['count'])]
             actual = [columns[0].text, columns[1].text]
             self.assertListEqual(actual, expected)

acceptance_tests/test_course_enrollment_demographics.py

 import datetime
-from unittest import skipUnless
+
 from bok_choy.web_app_test import WebAppTest
 
 import analyticsclient.constants.education_level as EDUCATION_LEVEL
 from analyticsclient.constants import demographic
-from acceptance_tests import ENABLE_DEMOGRAPHICS_TESTS
+import analyticsclient.constants.gender as GENDER
 from acceptance_tests.mixins import CourseDemographicsPageTestsMixin
-
 from acceptance_tests.pages import CourseEnrollmentDemographicsAgePage, CourseEnrollmentDemographicsEducationPage, \
     CourseEnrollmentDemographicsGenderPage
 
@@ -14,7 +13,6 @@ from acceptance_tests.pages import CourseEnrollmentDemographicsAgePage, CourseEn
 _multiprocess_can_split_ = True
 
 
-@skipUnless(ENABLE_DEMOGRAPHICS_TESTS, 'Demographics tests are not enabled.')
 class CourseEnrollmentDemographicsAgeTests(CourseDemographicsPageTestsMixin, WebAppTest):
     help_path = 'enrollment/Demographics_Age.html'
 
@@ -25,22 +23,27 @@ class CourseEnrollmentDemographicsAgeTests(CourseDemographicsPageTestsMixin, Web
         super(CourseEnrollmentDemographicsAgeTests, self).setUp()
         self.page = CourseEnrollmentDemographicsAgePage(self.browser)
         self.course = self.api_client.courses(self.page.course_id)
+
         self.demographic_data = sorted(self.course.enrollment(self.demographic_type),
                                        key=lambda item: item['count'], reverse=True)
 
+        # Remove items with no birth year
+        self.demographic_data_without_none = [datum for datum in self.demographic_data if datum['birth_year']]
+
     def test_page(self):
         super(CourseEnrollmentDemographicsAgeTests, self).test_page()
         self._test_metrics()
 
     def _calculate_median_age(self, current_year):
-        total_enrollment = sum([datum['count'] for datum in self.demographic_data])
+        demographic_data = self.demographic_data_without_none
+
+        total_enrollment = sum([datum['count'] for datum in demographic_data])
         half_enrollments = total_enrollment * 0.5
         count_enrollments = 0
 
-        sorted_by_year = sorted(self.course.enrollment(self.demographic_type),
-                                key=lambda item: item['birth_year'], reverse=False)
+        data = sorted(demographic_data, key=lambda item: item['birth_year'], reverse=False)
 
-        for index, datum in enumerate(sorted_by_year):
+        for index, datum in enumerate(data):
             age = current_year - datum['birth_year']
             count_enrollments += datum['count']
 
@@ -48,7 +51,7 @@ class CourseEnrollmentDemographicsAgeTests(CourseDemographicsPageTestsMixin, Web
                 return age
             elif count_enrollments == half_enrollments:
                 if total_enrollment % 2 == 0:
-                    next_age = current_year - sorted_by_year[index + 1]['birth_year']
+                    next_age = current_year - data[index + 1]['birth_year']
                     return (next_age + age) * 0.5
                 else:
                     return age
@@ -60,7 +63,7 @@ class CourseEnrollmentDemographicsAgeTests(CourseDemographicsPageTestsMixin, Web
         Returns the number of enrollments between min_age (exclusive) and
         max_age (inclusive).
         """
-        filtered_ages = self.demographic_data
+        filtered_ages = self.demographic_data_without_none
 
         if min_age:
             filtered_ages = ([datum for datum in filtered_ages
@@ -73,7 +76,7 @@ class CourseEnrollmentDemographicsAgeTests(CourseDemographicsPageTestsMixin, Web
 
     def _test_metrics(self):
         current_year = datetime.date.today().year
-        total = float(sum([datum['count'] for datum in self.demographic_data]))
+        total = float(sum([datum['count'] for datum in self.demographic_data_without_none]))
         age_metrics = [
             {
                 'stat_type': 'median_age',
@@ -107,7 +110,6 @@ class CourseEnrollmentDemographicsAgeTests(CourseDemographicsPageTestsMixin, Web
         self.assertIn('text-right', column[2].get_attribute('class'))
 
 
-@skipUnless(ENABLE_DEMOGRAPHICS_TESTS, 'Demographics tests are not enabled.')
 class CourseEnrollmentDemographicsGenderTests(CourseDemographicsPageTestsMixin, WebAppTest):
     help_path = 'enrollment/Demographics_Gender.html'
 
@@ -126,19 +128,26 @@ class CourseEnrollmentDemographicsGenderTests(CourseDemographicsPageTestsMixin,
                                        reverse=True)
 
     def _test_table_row(self, datum, column, sum_count):
-        expected_date = datetime.datetime.strptime(datum['date'], self.api_date_format).strftime(
-            "%B %d, %Y").replace(' 0', ' ')
-        gender_total = sum([value for key, value in datum.iteritems() if
-                            value and key in ['female', 'male', 'other', 'unknown']])
-        expected = [unicode(expected_date), unicode(gender_total), unicode(datum.get('female', 0)),
-                    unicode(datum.get('male', 0)), unicode(datum.get('other', 0)), unicode(datum.get('u', 0))]
-        actual = [column[0].text, column[1].text, column[2].text, column[3].text, column[4].text, column[5].text]
+        genders = [GENDER.FEMALE, GENDER.MALE, GENDER.OTHER, GENDER.UNKNOWN]
+        expected_date = datetime.datetime.strptime(datum['date'], self.api_date_format).strftime("%B %d, %Y")
+        expected_date = self.date_strip_leading_zeroes(expected_date)
+        gender_total = sum([value for key, value in datum.iteritems() if value and key in genders])
+
+        expected = [unicode(expected_date), unicode(gender_total)]
+
+        for gender in genders:
+            expected.append(unicode(datum.get(gender, 0)))
+
+        actual = []
+        for i in range(6):
+            actual.append(column[i].text)
+
         self.assertListEqual(actual, expected)
+
         for i in range(1, 6):
             self.assertIn('text-right', column[i].get_attribute('class'))
 
 
-@skipUnless(ENABLE_DEMOGRAPHICS_TESTS, 'Demographics tests are not enabled.')
 class CourseEnrollmentDemographicsEducationTests(CourseDemographicsPageTestsMixin, WebAppTest):
 
     EDUCATION_NAMES = {
@@ -171,7 +180,9 @@ class CourseEnrollmentDemographicsEducationTests(CourseDemographicsPageTestsMixi
         self._test_metrics()
 
     def _test_metrics(self):
-        total = self._get_total_demographics()
+        # The total should not include users who did not provide an education level
+        total = sum([datum['count'] for datum in self.demographic_data if datum['education_level']])
+
         education_groups = [
             {
                 'levels': ['primary', 'junior_secondary', 'secondary'],

analytics_dashboard/courses/presenters.py

@@ -186,7 +186,7 @@ class CourseEngagementPresenter(BasePresenter):
 
 
 class BaseCourseEnrollmentPresenter(BasePresenter):
-    def _calculate_total_enrollment(self, data):
+    def _sum_counts(self, data):
         return sum([datum['count'] for datum in data])
 
     def _calculate_percent(self, count, total):
@@ -301,7 +301,7 @@ class CourseEnrollmentPresenter(BaseCourseEnrollmentPresenter):
             api_response = self._translate_country_names(api_response)
 
             # get the sum as a float so we can divide by it to get a percent
-            total_enrollment = self._calculate_total_enrollment(api_response)
+            total_enrollment = self._sum_counts(api_response)
 
             # formatting this data for easy access in the table UI
             data = [{'countryCode': datum['country']['alpha3'],
@@ -468,11 +468,11 @@ class CourseEnrollmentDemographicsPresenter(BaseCourseEnrollmentPresenter):
         if max_age:
             filtered_ages = ([datum for datum in filtered_ages
                               if datum['birth_year'] and (current_year - datum['birth_year']) <= max_age])
-        return self._calculate_total_enrollment(filtered_ages)
+        return self._sum_counts(filtered_ages)
 
     def _calculate_median_age(self, api_response):
         current_year = datetime.date.today().year
-        total_enrollment = self._calculate_total_enrollment(api_response)
+        total_enrollment = self._sum_counts(api_response)
         half_enrollments = total_enrollment * 0.5
         count_enrollments = 0
         for index, datum in enumerate(api_response):
@@ -524,7 +524,7 @@ class CourseEnrollmentDemographicsPresenter(BaseCourseEnrollmentPresenter):
     def _build_binned_ages(self, api_response):
         current_year = datetime.date.today().year
         known_ages = [i for i in api_response if i['birth_year']]
-        enrollment_total = self._calculate_total_enrollment(api_response)
+        enrollment_total = self._sum_counts(api_response)
 
         binned_ages = [{'age': current_year - int(datum['birth_year']),
                         'count': datum['count'],
@@ -570,19 +570,19 @@ class CourseEnrollmentDemographicsPresenter(BaseCourseEnrollmentPresenter):
 
     def _calculate_known_total_enrollment(self, api_response, enrollment_key):
         known = [i for i in api_response if i[enrollment_key]]
-        return self._calculate_total_enrollment(known)
+        return self._sum_counts(known)
 
     def _calculate_known_total_percent(self, api_response, enrollment_key):
         known_count = self._calculate_known_total_enrollment(api_response, enrollment_key)
-        total_count = self._calculate_total_enrollment(api_response)
+        total_count = self._sum_counts(api_response)
         return self._calculate_percent(known_count, total_count)
 
     def _calculate_education_percent(self, api_response, levels):
         """ Aggregates levels of education and returns the percent of the total. """
         filtered_levels = ([education for education in api_response
                             if education['education_level'] in levels])
-        subset_enrollment = self._calculate_total_enrollment(filtered_levels)
-        return self._calculate_percent(subset_enrollment, self._calculate_total_enrollment(api_response))
+        subset_enrollment = self._sum_counts(filtered_levels)
+        return self._calculate_percent(subset_enrollment, self._sum_counts(api_response))
 
     def _build_education_summary(self, api_response):
         known_education = [i for i in api_response if i['education_level']]
@@ -600,7 +600,7 @@ class CourseEnrollmentDemographicsPresenter(BaseCourseEnrollmentPresenter):
 
     def _build_education_levels(self, api_response):
         known_education = [i for i in api_response if i['education_level']]
-        known_enrollment_total = self._calculate_total_enrollment(known_education)
+        known_enrollment_total = self._sum_counts(known_education)
         levels = [{'educationLevel': EDUCATION_NAMES[datum['education_level']],
                    'count': datum['count'],
                    'percent': self._calculate_percent(datum['count'], known_enrollment_total),

analytics_dashboard/courses/tests/test_views/__init__.py

@@ -148,13 +148,6 @@ class CourseEnrollmentViewTestMixin(CourseViewTestMixin):
     active_secondary_nav_label = None
     api_method = 'analyticsclient.course.Course.enrollment'
 
-    def setUp(self):
-        super(CourseEnrollmentViewTestMixin, self).setUp()
-
-        switch, _created = Switch.objects.get_or_create(name='show_navbar_demographics')
-        switch.active = True
-        switch.save()
-
     def assertPrimaryNav(self, nav, course_id):
         expected = {
             'icon': 'fa-child',

analytics_dashboard/courses/views.py

@@ -326,8 +326,7 @@ class EnrollmentTemplateView(CourseTemplateView):
     """
     secondary_nav_items = [
         {'name': 'activity', 'label': _('Activity'), 'view': 'courses:enrollment_activity'},
-        {'name': 'demographics', 'label': _('Demographics'), 'view': 'courses:enrollment_demographics_age',
-         'switch': 'show_navbar_demographics'},
+        {'name': 'demographics', 'label': _('Demographics'), 'view': 'courses:enrollment_demographics_age'},
         {'name': 'geography', 'label': _('Geography'), 'view': 'courses:enrollment_geography'},
     ]
     active_primary_nav_item = 'enrollment'