Merge pull request #7354 from edx/ned/clean-test-turds

Clean up temp file management, mostly in tests

.gitignore

@@ -59,8 +59,6 @@ jscover.log.*
 .tddium*
 common/test/data/test_unicode/static/
 django-pyfs
-test_root/uploads/*.txt
-test_root/uploads/badges/*.png
 
 ### Installation artifacts
 *.egg-info

common/lib/xmodule/xmodule/modulestore/tests/django_utils.py

@@ -4,7 +4,6 @@ Modulestore configuration for test cases.
 """
 import datetime
 import pytz
-from tempfile import mkdtemp
 from uuid import uuid4
 
 from mock import patch
@@ -16,6 +15,8 @@ from django.test.utils import override_settings
 from request_cache.middleware import RequestCache
 
 from courseware.field_overrides import OverrideFieldData  # pylint: disable=import-error
+from openedx.core.lib.tempdir import mkdtemp_clean
+
 from xmodule.contentstore.django import _CONTENTSTORE
 from xmodule.modulestore import ModuleStoreEnum
 from xmodule.modulestore.django import modulestore, clear_existing_modulestores
@@ -184,13 +185,13 @@ TEST_DATA_MIXED_GRADED_MODULESTORE = mixed_store_config(
 # All store requests now go through mixed
 # Use this modulestore if you specifically want to test mongo and not a mocked modulestore.
 # This modulestore definition below will not load any xml courses.
-TEST_DATA_MONGO_MODULESTORE = mixed_store_config(mkdtemp(), {}, include_xml=False)
+TEST_DATA_MONGO_MODULESTORE = mixed_store_config(mkdtemp_clean(), {}, include_xml=False)
 
 # All store requests now go through mixed
 # Use this modulestore if you specifically want to test split-mongo and not a mocked modulestore.
 # This modulestore definition below will not load any xml courses.
 TEST_DATA_SPLIT_MODULESTORE = mixed_store_config(
-    mkdtemp(),
+    mkdtemp_clean(),
     {},
     include_xml=False,
     store_order=[StoreConstructors.split, StoreConstructors.draft]
@@ -235,7 +236,7 @@ class ModuleStoreTestCase(TestCase):
           your `setUp()` method.
     """
 
-    MODULESTORE = mixed_store_config(mkdtemp(), {}, include_xml=False)
+    MODULESTORE = mixed_store_config(mkdtemp_clean(), {}, include_xml=False)
 
     def setUp(self, **kwargs):
         """

common/lib/xmodule/xmodule/modulestore/tests/test_modulestore_settings.py

@@ -3,7 +3,8 @@ Tests for testing the modulestore settings migration code.
 """
 import copy
 import ddt
-from tempfile import mkdtemp
+
+from openedx.core.lib.tempdir import mkdtemp_clean
 
 from unittest import TestCase
 from xmodule.modulestore.modulestore_settings import (
@@ -37,7 +38,7 @@ class ModuleStoreSettingsMigration(TestCase):
                 "collection": "modulestore",
                 "db": "edxapp",
                 "default_class": "xmodule.hidden_module.HiddenDescriptor",
-                "fs_root": mkdtemp(),
+                "fs_root": mkdtemp_clean(),
                 "host": "localhost",
                 "password": "password",
                 "port": 27017,

common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py

@@ -571,12 +571,10 @@ class TestMongoModuleStore(TestMongoModuleStoreBase):
         self.content_store.find(location)
 
         root_dir = path(mkdtemp())
-        try:
-            export_course_to_xml(self.draft_store, self.content_store, course_key, root_dir, 'test_export')
-            assert_true(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
-            assert_true(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
-        finally:
-            shutil.rmtree(root_dir)
+        self.addCleanup(shutil.rmtree, root_dir)
+        export_course_to_xml(self.draft_store, self.content_store, course_key, root_dir, 'test_export')
+        self.assertTrue(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
+        self.assertTrue(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
 
     @patch('xmodule.tabs.CourseTab.from_json', side_effect=mock_tab_from_json)
     def test_export_course_image_nondefault(self, _from_json):
@@ -588,12 +586,10 @@ class TestMongoModuleStore(TestMongoModuleStoreBase):
         assert_true(course.course_image, 'just_a_test.jpg')
 
         root_dir = path(mkdtemp())
-        try:
-            export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
-            assert_true(path(root_dir / 'test_export/static/just_a_test.jpg').isfile())
-            assert_false(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
-        finally:
-            shutil.rmtree(root_dir)
+        self.addCleanup(shutil.rmtree, root_dir)
+        export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
+        self.assertTrue(path(root_dir / 'test_export/static/just_a_test.jpg').isfile())
+        self.assertFalse(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
 
     def test_course_without_image(self):
         """
@@ -602,12 +598,10 @@ class TestMongoModuleStore(TestMongoModuleStoreBase):
         """
         course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'simple_with_draft', '2012_Fall'))
         root_dir = path(mkdtemp())
-        try:
-            export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
-            assert_false(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
-            assert_false(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
-        finally:
-            shutil.rmtree(root_dir)
+        self.addCleanup(shutil.rmtree, root_dir)
+        export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
+        self.assertFalse(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
+        self.assertFalse(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
 
     def _create_test_tree(self, name, user_id=None):
         """
@@ -728,15 +722,13 @@ class TestMongoModuleStore(TestMongoModuleStoreBase):
         self.assertEqual(unicode(component.link_to_location), unicode(problem_location))
 
         root_dir = path(mkdtemp())
+        self.addCleanup(shutil.rmtree, root_dir)
 
         # export_course_to_xml should work.
-        try:
-            export_course_to_xml(
-                self.draft_store, self.content_store, interface_location.course_key,
-                root_dir, 'test_export'
-            )
-        finally:
-            shutil.rmtree(root_dir)
+        export_course_to_xml(
+            self.draft_store, self.content_store, interface_location.course_key,
+            root_dir, 'test_export'
+        )
 
     def test_draft_modulestore_create_child_with_position(self):
         """

lms/djangoapps/courseware/management/commands/tests/test_dump_course.py

@@ -175,14 +175,12 @@ class CommandsTestBase(ModuleStoreTestCase):
 
     def test_export_course(self):
         tmp_dir = path(mkdtemp())
+        self.addCleanup(shutil.rmtree, tmp_dir)
         filename = tmp_dir / 'test.tar.gz'
-        try:
-            self.run_export_course(filename)
-            with tarfile.open(filename) as tar_file:
-                self.check_export_file(tar_file)
 
-        finally:
-            shutil.rmtree(tmp_dir)
+        self.run_export_course(filename)
+        with tarfile.open(filename) as tar_file:
+            self.check_export_file(tar_file)
 
     def test_export_course_stdout(self):
         output = self.run_export_course('-')

lms/djangoapps/instructor_task/tests/test_tasks.py

@@ -40,10 +40,10 @@ class TestTaskFailure(Exception):
 class TestInstructorTasks(InstructorTaskModuleTestCase):
 
     def setUp(self):
-        super(InstructorTaskModuleTestCase, self).setUp()
+        super(TestInstructorTasks, self).setUp()
         self.initialize_course()
         self.instructor = self.create_instructor('instructor')
-        self.location = InstructorTaskModuleTestCase.problem_location(PROBLEM_URL_NAME)
+        self.location = self.problem_location(PROBLEM_URL_NAME)
 
     def _create_input_entry(self, student_ident=None, use_problem_url=True, course_id=None):
         """Creates a InstructorTask entry for testing."""

lms/envs/test.py

@@ -21,10 +21,11 @@ sessions. Assumes structure:
 from .common import *
 import os
 from path import path
-from tempfile import mkdtemp
 from uuid import uuid4
 from warnings import filterwarnings, simplefilter
 
+from openedx.core.lib.tempdir import mkdtemp_clean
+
 # Silence noisy logs to make troubleshooting easier when tests fail.
 import logging
 LOG_OVERRIDES = [
@@ -151,7 +152,7 @@ update_module_store_settings(
         'fs_root': TEST_ROOT / "data",
     },
     xml_store_options={
-        'data_dir': mkdtemp(dir=TEST_ROOT),  # never inadvertently load all the XML courses
+        'data_dir': mkdtemp_clean(dir=TEST_ROOT),  # never inadvertently load all the XML courses
     },
     doc_store_settings={
         'host': MONGO_HOST,

openedx/core/djangoapps/user_api/management/tests/test_email_opt_in_list.py

@@ -331,11 +331,7 @@ class EmailOptInListTest(ModuleStoreTestCase):
         # Create a temporary directory for the output
         # Delete it when we're finished
         temp_dir_path = tempfile.mkdtemp()
-
-        def _cleanup():  # pylint: disable=missing-docstring
-            shutil.rmtree(temp_dir_path)
-
-        self.addCleanup(_cleanup)
+        self.addCleanup(shutil.rmtree, temp_dir_path)
 
         # Sanitize the arguments
         if other_names is None:

test_root/uploads/.gitignore

 *.csv
 *.jpg
 *.png
+*.txt