Merge pull request #15211 from edx/adam/add-precedes-to-coursegraph-2

add precedes relationship to coursegraph (EDUCATOR-484)

openedx/core/djangoapps/coursegraph/management/commands/dump_to_neo4j.py

@@ -33,16 +33,25 @@ class ModuleStoreSerializer(object):
     one graph per course.
     """
 
-    def __init__(self, courses=None, skip=None):
+    def __init__(self, course_keys):
+        self.course_keys = course_keys
+
+    @classmethod
+    def create(cls, courses=None, skip=None):
         """
         Sets the object's course_keys attribute from the `courses` parameter.
         If that parameter isn't furnished, loads all course_keys from the
         modulestore.
+
         Filters out course_keys in the `skip` parameter, if provided.
-        Args:
+
+        Arguments:
             courses: A list of string serializations of course keys.
                 For example, ["course-v1:org+course+run"].
             skip: Also a list of string serializations of course keys.
+
+        Returns:
+            a ModulestoreSerializer instance
         """
         if courses:
             course_keys = [CourseKey.from_string(course.strip()) for course in courses]
@@ -51,14 +60,14 @@ class ModuleStoreSerializer(object):
                 course.id for course in modulestore().get_course_summaries()
             ]
         if skip is not None:
-            skip_keys = [CourseKey.from_string(course.strip()) for course in skip]
+            skip_keys = set([CourseKey.from_string(course.strip()) for course in skip])
             course_keys = [course_key for course_key in course_keys if course_key not in skip_keys]
-        self.course_keys = course_keys
+        return cls(course_keys)
 
     @staticmethod
     def serialize_item(item):
         """
-        Args:
+        Arguments:
             item: an XBlock
 
         Returns:
@@ -100,7 +109,7 @@ class ModuleStoreSerializer(object):
     def serialize_course(self, course_id):
         """
         Serializes a course into py2neo Nodes and Relationships
-        Args:
+        Arguments:
             course_id: CourseKey of the course we want to serialize
 
         Returns:
@@ -125,6 +134,7 @@ class ModuleStoreSerializer(object):
         # create relationships
         relationships = []
         for item in items:
+            previous_child_node = None
             for index, child_loc in enumerate(item.get_children()):
                 parent_node = location_to_node.get(item.location)
                 child_node = location_to_node.get(child_loc.location)
@@ -133,13 +143,20 @@ class ModuleStoreSerializer(object):
                     relationship = Relationship(parent_node, "PARENT_OF", child_node)
                     relationships.append(relationship)
 
+                    if previous_child_node:
+                        ordering_relationship = Relationship(
+                            previous_child_node, "PRECEDES", child_node
+                        )
+                        relationships.append(ordering_relationship)
+                    previous_child_node = child_node
+
         nodes = location_to_node.values()
         return nodes, relationships
 
     @staticmethod
     def coerce_types(value):
         """
-        Args:
+        Arguments:
             value: the value of an xblock's field
 
         Returns: either the value, a text version of the value, or, if the
@@ -159,7 +176,7 @@ class ModuleStoreSerializer(object):
     @staticmethod
     def add_to_transaction(neo4j_entities, transaction):
         """
-        Args:
+        Arguments:
             neo4j_entities: a list of Nodes or Relationships
             transaction: a neo4j transaction
         """
@@ -170,7 +187,7 @@ class ModuleStoreSerializer(object):
     def get_command_last_run(course_key, graph):
         """
         This information is stored on the course node of a course in neo4j
-        Args:
+        Arguments:
             course_key: a CourseKey
             graph: a py2neo Graph
 
@@ -195,7 +212,7 @@ class ModuleStoreSerializer(object):
         """
         We use the CourseStructure table to get when this course was last
         published.
-        Args:
+        Arguments:
             course_key: a CourseKey
 
         Returns: The datetime the course was last published at, converted into
@@ -214,7 +231,7 @@ class ModuleStoreSerializer(object):
         """
         Only dump the course if it's been changed since the last time it's been
         dumped.
-        Args:
+        Arguments:
             course_key: a CourseKey object.
             graph: a py2neo Graph object.
 
@@ -240,39 +257,15 @@ class ModuleStoreSerializer(object):
         # before the course's last published event
         return last_this_command_was_run < course_last_published_date
 
-    def dump_courses_to_neo4j(self, graph, override_cache=False):
+    def dump_course_to_neo4j(self, course_key, graph):
         """
-        Method that iterates through a list of courses in a modulestore,
-        serializes them, then writes them to neo4j
-        Args:
-            graph: py2neo graph object
-            override_cache: serialize the courses even if they'be been recently
-                serialized
+        Serializes a course and writes it to neo4j.
 
-        Returns: two lists--one of the courses that were successfully written
-            to neo4j and one of courses that were not.
+        Arguments:
+            course_key: course key for the course to be exported
+            graph: py2neo graph object
         """
 
-        total_number_of_courses = len(self.course_keys)
-
-        successful_courses = []
-        unsuccessful_courses = []
-
-        for index, course_key in enumerate(self.course_keys):
-            # first, clear the request cache to prevent memory leaks
-            RequestCache.clear_request_cache()
-
-            log.info(
-                "Now exporting %s to neo4j: course %d of %d total courses",
-                course_key,
-                index + 1,
-                total_number_of_courses,
-            )
-
-            if not (override_cache or self.should_dump_course(course_key, graph)):
-                log.info("skipping dumping %s, since it hasn't changed", course_key)
-                continue
-
         nodes, relationships = self.serialize_course(course_key)
         log.info(
             "%d nodes and %d relationships in %s",
@@ -302,12 +295,45 @@ class ModuleStoreSerializer(object):
                 course_string
             )
             transaction.rollback()
-                unsuccessful_courses.append(course_string)
+
+    def dump_courses_to_neo4j(self, graph, override_cache=False):
+        """
+        Method that iterates through a list of courses in a modulestore,
+        serializes them, then submits tasks to write them to neo4j.
+        Arguments:
+            graph: py2neo graph object
+            override_cache: serialize the courses even if they'be been recently
+                serialized
+
+        Returns: two lists--one of the courses that were successfully written
+            to neo4j and one of courses that were not.
+        """
+
+        total_number_of_courses = len(self.course_keys)
+
+        submitted_courses = []
+        skipped_courses = []
+
+        for index, course_key in enumerate(self.course_keys):
+            # first, clear the request cache to prevent memory leaks
+            RequestCache.clear_request_cache()
+
+            log.info(
+                "Now exporting %s to neo4j: course %d of %d total courses",
+                course_key,
+                index + 1,
+                total_number_of_courses,
+            )
+
+            if not (override_cache or self.should_dump_course(course_key, graph)):
+                log.info("skipping dumping %s, since it hasn't changed", course_key)
+                skipped_courses.append(unicode(course_key))
 
             else:
-                successful_courses.append(course_string)
+                self.dump_course_to_neo4j(course_key, graph)
+                submitted_courses.append(unicode(course_key))
 
-        return successful_courses, unsuccessful_courses
+        return submitted_courses, skipped_courses
 
 
 class Command(BaseCommand):
@@ -374,28 +400,24 @@ class Command(BaseCommand):
             secure=secure,
         )
 
-        mss = ModuleStoreSerializer(options['courses'], options['skip'])
+        mss = ModuleStoreSerializer.create(options['courses'], options['skip'])
 
-        successful_courses, unsuccessful_courses = mss.dump_courses_to_neo4j(
+        submitted_courses, skipped_courses = mss.dump_courses_to_neo4j(
             graph, override_cache=options['override']
         )
 
-        if not successful_courses and not unsuccessful_courses:
+        log.info(
+            "%d courses submitted for export to neo4j. %d courses skipped.",
+            len(submitted_courses),
+            len(skipped_courses),
+        )
+
+        if not submitted_courses:
             print("No courses exported to neo4j at all!")
             return
 
-        if successful_courses:
-            print(
-                "These courses exported to neo4j successfully:\n\t" +
-                "\n\t".join(successful_courses)
-            )
-        else:
-            print("No courses exported to neo4j successfully.")
-
-        if unsuccessful_courses:
+        if submitted_courses:
             print(
-                "These courses did not export to neo4j successfully:\n\t" +
-                "\n\t".join(unsuccessful_courses)
+                "These courses were submitted for export to neo4j successfully:\n\t" +
+                "\n\t".join(submitted_courses)
             )
-        else:
-            print("All courses exported to neo4j successfully.")