Add --all option to social stats export.

lms/djangoapps/django_comment_client/management/commands/export_discussion_participation.py

@@ -5,6 +5,8 @@ from datetime import datetime
 from optparse import make_option
 
 from django.core.management.base import BaseCommand, CommandError
+import os
+from path import path
 
 from opaque_keys import InvalidKeyError
 from opaque_keys.edx.keys import CourseKey
@@ -15,6 +17,7 @@ from student.models import CourseEnrollment
 
 from lms.lib.comment_client.user import User
 import django_comment_client.utils as utils
+from xmodule.modulestore.django import modulestore
 
 
 class DiscussionExportFields(object):
@@ -39,15 +42,18 @@ class Command(BaseCommand):
 
     Usage:
         ./manage.py lms export_discussion_participation course_key [dest_file] [OPTIONS]
+        ./manage.py lms export_discussion_participation [dest_directory] --all [OPTIONS]
 
         * course_key - target course key (e.g. edX/DemoX/T1)
         * dest_file - location of destination file (created if missing, overwritten if exists)
+        * dest_directory - location to store all dumped files to. Will dump into the current directory otherwise.
 
     OPTIONS:
 
         * thread-type - one of {discussion, question}. Filters discussion participation stats by discussion thread type.
         * end-date - date time in iso8601 format (YYYY-MM-DD hh:mm:ss). Filters discussion participation stats
           by creation date: no threads/comments/replies created *after* this date is included in calculation
+        * all - Dump all social stats at once into a particular directory.
 
     Examples:
 
@@ -65,6 +71,7 @@ class Command(BaseCommand):
     """
     THREAD_TYPE_PARAMETER = 'thread_type'
     END_DATE_PARAMETER = 'end_date'
+    ALL_PARAMETER = 'all'
 
     args = "<course_id> <output_file_location>"
 
@@ -86,6 +93,12 @@ class Command(BaseCommand):
             default=None,
             help='Include threads, comments and replies created before the supplied date (iso8601 format)'
         ),
+        make_option(
+            '--all',
+            action='store_true',
+            dest=ALL_PARAMETER,
+            default=False,
+        )
     )
 
     def _get_filter_string_representation(self, options):
@@ -103,8 +116,34 @@ class Command(BaseCommand):
             "social_stats_{course}_{date:%Y_%m_%d_%H_%M_%S}.csv".format(course=course_key, date=datetime.utcnow())
         )
 
-    def handle(self, *args, **options):
-        """ Executes command """
+    @staticmethod
+    def get_all_courses():
+        """
+        Gets all courses. Made into a separate function because patch isn't cooperating.
+        """
+        return modulestore().get_courses()
+
+    def dump_all(self, *args, **options):
+        if len(args) > 1:
+            raise CommandError("May not specify course and destination root directory with the --all option.")
+        args = list(args)
+        try:
+            dir_name = path(args.pop())
+        except IndexError:
+            dir_name = path('social_stats')
+
+        if not os.path.exists(dir_name):
+            os.makedirs(dir_name)
+
+        for course in self.get_all_courses():
+            raw_course_key = unicode(course.location.course_key)
+            args = [
+                raw_course_key,
+                dir_name / self.get_default_file_location(raw_course_key)
+            ]
+            self.dump_one(*args, **options)
+
+    def dump_one(self, *args, **options):
         if not args:
             raise CommandError("Course id not specified")
         if len(args) > 2:
@@ -139,8 +178,14 @@ class Command(BaseCommand):
         with open(output_file_location, 'wb') as output_stream:
             Exporter(output_stream).export(data)
 
-        self.stdout.write("Success!\n")
+    def handle(self, *args, **options):
+        """ Executes command """
+        if options.get(self.ALL_PARAMETER, False):
+            self.dump_all(*args, **options)
+        else:
+            self.dump_one(*args, **options)
 
+        self.stdout.write("Success!\n")
 
 class Extractor(object):
     """ Extracts discussion participation data from db and cs_comments_service """

lms/djangoapps/django_comment_client/tests/management/test_export_discussion_participation.py

@@ -42,38 +42,62 @@ class CommandTest(TestCase):
         self.command.stdout = mock.Mock()
         self.command.stderr = mock.Mock()
 
-    def set_up_default_mocks(self, patched_get_courses):
+    def set_up_default_mocks(self, patched_get_course):
         """ Sets up default mocks passed via class decorator """
-        patched_get_courses.return_value = CourseLocator("edX", "demoX", "now")
+        patched_get_course.return_value = CourseLocator("edX", "demoX", "now")
 
     # pylint:disable=unused-argument
-    def test_handle_given_no_arguments_raises_command_error(self, patched_get_courses):
+    def test_handle_given_no_arguments_raises_command_error(self, patched_get_course):
         """ Tests that raises error if invoked with no arguments """
         with self.assertRaises(CommandError):
             self.command.handle()
 
     # pylint:disable=unused-argument
-    def test_handle_given_more_than_two_args_raises_command_error(self, patched_get_courses):
+    def test_handle_given_more_than_two_args_raises_command_error(self, patched_get_course):
         """ Tests that raises error if invoked with too many arguments """
         with self.assertRaises(CommandError):
             self.command.handle(1, 2, 3)
 
-    def test_handle_given_invalid_course_key_raises_invalid_key_error(self, patched_get_courses):
+    def test_handle_given_invalid_course_key_raises_invalid_key_error(self, patched_get_course):
         """ Tests that invalid key errors are propagated """
-        patched_get_courses.return_value = None
+        patched_get_course.return_value = None
         with self.assertRaises(InvalidKeyError):
             self.command.handle("I'm invalid key")
 
-    def test_handle_given_missing_course_raises_command_error(self, patched_get_courses):
+    def test_handle_given_missing_course_raises_command_error(self, patched_get_course):
         """ Tests that raises command error if missing course key was provided """
-        patched_get_courses.return_value = None
+        patched_get_course.return_value = None
         with self.assertRaises(CommandError):
             self.command.handle("edX/demoX/now")
 
+    # pylint: disable=unused-argument
+    def test_all_option(self, patched_get_course):
+        """ Tests that the 'all' option does run the dump command for all courses """
+        self.command.dump_one = mock.Mock()
+        self.command.get_all_courses = mock.Mock()
+        course_list = [mock.Mock() for __ in range(0, 3)]
+        locator_list = [
+            CourseLocator(org="edX", course="demoX", run="now"),
+            CourseLocator(org="Sandbox", course="Sandbox", run="Sandbox"),
+            CourseLocator(org="Test", course="Testy", run="Testify"),
+        ]
+        for index, course in enumerate(course_list):
+            course.location.course_key = locator_list[index]
+        self.command.get_all_courses.return_value = course_list
+        self.command.handle("test_dir", all=True, dummy='test')
+        calls = self.command.dump_one.call_args_list
+        self.assertEqual(len(calls), 3)
+        self.assertEqual(calls[0][0][0], 'course-v1:edX+demoX+now')
+        self.assertEqual(calls[1][0][0], 'course-v1:Sandbox+Sandbox+Sandbox')
+        self.assertEqual(calls[2][0][0], 'course-v1:Test+Testy+Testify')
+        self.assertIn('test_dir/social_stats_course-v1edXdemoXnow', calls[0][0][1])
+        self.assertIn('test_dir/social_stats_course-v1SandboxSandboxSandbox', calls[1][0][1])
+        self.assertIn('test_dir/social_stats_course-v1TestTestyTestify', calls[2][0][1])
+
     @ddt.data("edX/demoX/now", "otherX/CourseX/later")
-    def test_handle_writes_to_correct_location_when_output_file_not_specified(self, course_key, patched_get_courses):
+    def test_handle_writes_to_correct_location_when_output_file_not_specified(self, course_key, patched_get_course):
         """ Tests that when no explicit filename is given data is exported to default location """
-        self.set_up_default_mocks(patched_get_courses)
+        self.set_up_default_mocks(patched_get_course)
         expected_filename = utils.format_filename(
             "social_stats_{course}_{date:%Y_%m_%d_%H_%M_%S}.csv".format(course=course_key, date=datetime.utcnow())
         )
@@ -85,9 +109,9 @@ class CommandTest(TestCase):
             patched_open.assert_called_with(expected_filename, 'wb')
 
     @ddt.data("test.csv", "other_file.csv")
-    def test_handle_writes_to_correct_location_when_output_file_is_specified(self, location, patched_get_courses):
+    def test_handle_writes_to_correct_location_when_output_file_is_specified(self, location, patched_get_course):
         """ Tests that when explicit filename is given data is exported to chosen location """
-        self.set_up_default_mocks(patched_get_courses)
+        self.set_up_default_mocks(patched_get_course)
         patched_open = mock.mock_open()
         with mock.patch("{}.open".format(_target_module), patched_open, create=True), \
                 mock.patch(_target_module + ".Extractor.extract") as patched_extractor:
@@ -95,9 +119,9 @@ class CommandTest(TestCase):
             self.command.handle("irrelevant/course/key", location)
             patched_open.assert_called_with(location, 'wb')
 
-    def test_handle_creates_correct_exporter(self, patched_get_courses):
+    def test_handle_creates_correct_exporter(self, patched_get_course):
         """ Tests that creates correct exporter """
-        self.set_up_default_mocks(patched_get_courses)
+        self.set_up_default_mocks(patched_get_course)
         patched_open = mock.mock_open()
         with mock.patch("{}.open".format(_target_module), patched_open, create=True), \
                 mock.patch(_target_module + ".Extractor.extract") as patched_extractor, \
@@ -112,9 +136,9 @@ class CommandTest(TestCase):
         {"1": {"num_threads": 12}},
         {"1": {"num_threads": 14, "num_comments": 7}}
     )
-    def test_handle_exports_correct_data(self, extracted, patched_get_courses):
+    def test_handle_exports_correct_data(self, extracted, patched_get_course):
         """ Tests that invokes export with correct data """
-        self.set_up_default_mocks(patched_get_courses)
+        self.set_up_default_mocks(patched_get_course)
         patched_open = mock.mock_open()
         with mock.patch("{}.open".format(_target_module), patched_open, create=True), \
                 mock.patch(_target_module + ".Extractor.extract") as patched_extractor, \
@@ -126,10 +150,10 @@ class CommandTest(TestCase):
     @ddt.unpack
     @ddt.data(*_std_parameters_list)
     def test_handle_passes_correct_parameters_to_extractor(
-        self, course_key, end_date, thread_type, patched_get_courses
+        self, course_key, end_date, thread_type, patched_get_course
     ):
         """ Tests that when no explicit filename is given data is exported to default location """
-        self.set_up_default_mocks(patched_get_courses)
+        self.set_up_default_mocks(patched_get_course)
         patched_open = mock.mock_open()
         with mock.patch("{}.open".format(_target_module), patched_open, create=True), \
                 mock.patch(_target_module + ".Extractor.extract") as patched_extractor: