Use separate git repos for separate TestCases

lms/djangoapps/dashboard/git_import.py

@@ -13,7 +13,7 @@ from django.conf import settings
 from django.core import management
 from django.core.management.base import CommandError
 from django.utils import timezone
-from django.utils.translation import ugettext as _
+from django.utils.translation import ugettext_lazy as _
 import mongoengine
 
 from dashboard.models import CourseImportLog
@@ -23,33 +23,91 @@ from opaque_keys.edx.locations import SlashSeparatedCourseKey
 
 log = logging.getLogger(__name__)
 
-GIT_REPO_DIR = getattr(settings, 'GIT_REPO_DIR', '/edx/var/app/edxapp/course_repos')
-GIT_IMPORT_STATIC = getattr(settings, 'GIT_IMPORT_STATIC', True)
+DEFAULT_GIT_REPO_DIR = '/edx/var/app/edxapp/course_repos'
 
 
 class GitImportError(Exception):
     """
     Exception class for handling the typical errors in a git import.
     """
+    MESSAGE = None
 
-    NO_DIR = _("Path {0} doesn't exist, please create it, "
-               "or configure a different path with "
-               "GIT_REPO_DIR").format(GIT_REPO_DIR)
-    URL_BAD = _('Non usable git url provided. Expecting something like:'
-                ' git@github.com:mitocw/edx4edx_lite.git')
-    BAD_REPO = _('Unable to get git log')
-    CANNOT_PULL = _('git clone or pull failed!')
-    XML_IMPORT_FAILED = _('Unable to run import command.')
-    UNSUPPORTED_STORE = _('The underlying module store does not support import.')
+    def __init__(self, message=None):
+        if message is None:
+            message = self.message
+        super(GitImportError, self).__init__(message)
+
+
+class GitImportErrorNoDir(GitImportError):
+    """
+    GitImportError when no directory exists at the specified path.
+    """
+    def __init__(self, repo_dir):
+        super(GitImportErrorNoDir, self).__init__(
+            _(
+                "Path {0} doesn't exist, please create it, "
+                "or configure a different path with "
+                "GIT_REPO_DIR"
+            ).format(repo_dir)
+        )
+
+
+class GitImportErrorUrlBad(GitImportError):
+    """
+    GitImportError when the git url provided wasn't usable.
+    """
+    MESSAGE = _(
+        'Non usable git url provided. Expecting something like:'
+        ' git@github.com:mitocw/edx4edx_lite.git'
+    )
+
+
+class GitImportErrorBadRepo(GitImportError):
+    """
+    GitImportError when the cloned repository was malformed.
+    """
+    MESSAGE = _('Unable to get git log')
+
+
+class GitImportErrorCannotPull(GitImportError):
+    """
+    GitImportError when the clone of the repository failed.
+    """
+    MESSAGE = _('git clone or pull failed!')
+
+
+class GitImportErrorXmlImportFailed(GitImportError):
+    """
+    GitImportError when the course import command failed.
+    """
+    MESSAGE = _('Unable to run import command.')
+
+
+class GitImportErrorUnsupportedStore(GitImportError):
+    """
+    GitImportError when the modulestore doesn't support imports.
+    """
+    MESSAGE = _('The underlying module store does not support import.')
+
+
+class GitImportErrorRemoteBranchMissing(GitImportError):
+    """
+    GitImportError when the remote branch doesn't exist.
+    """
     # Translators: This is an error message when they ask for a
     # particular version of a git repository and that version isn't
     # available from the remote source they specified
-    REMOTE_BRANCH_MISSING = _('The specified remote branch is not available.')
+    MESSAGE = _('The specified remote branch is not available.')
+
+
+class GitImportErrorCannotBranch(GitImportError):
+    """
+    GitImportError when the local branch doesn't exist.
+    """
     # Translators: Error message shown when they have asked for a git
     # repository branch, a specific version within a repository, that
     # doesn't exist, or there is a problem changing to it.
-    CANNOT_BRANCH = _('Unable to switch to specified branch. Please check '
-                      'your branch name.')
+    MESSAGE = _('Unable to switch to specified branch. Please check your branch name.')
 
 
 def cmd_log(cmd, cwd):
@@ -78,7 +136,7 @@ def switch_branch(branch, rdir):
         cmd_log(['git', 'fetch', ], rdir)
     except subprocess.CalledProcessError as ex:
         log.exception('Unable to fetch remote: %r', ex.output)
-        raise GitImportError(GitImportError.CANNOT_BRANCH)
+        raise GitImportErrorCannotBranch()
 
     # Check if the branch is available from the remote.
     cmd = ['git', 'ls-remote', 'origin', '-h', 'refs/heads/{0}'.format(branch), ]
@@ -86,16 +144,16 @@ def switch_branch(branch, rdir):
         output = cmd_log(cmd, rdir)
     except subprocess.CalledProcessError as ex:
         log.exception('Getting a list of remote branches failed: %r', ex.output)
-        raise GitImportError(GitImportError.CANNOT_BRANCH)
+        raise GitImportErrorCannotBranch()
     if branch not in output:
-        raise GitImportError(GitImportError.REMOTE_BRANCH_MISSING)
+        raise GitImportErrorRemoteBranchMissing()
     # Check it the remote branch has already been made locally
     cmd = ['git', 'branch', '-a', ]
     try:
         output = cmd_log(cmd, rdir)
     except subprocess.CalledProcessError as ex:
         log.exception('Getting a list of local branches failed: %r', ex.output)
-        raise GitImportError(GitImportError.CANNOT_BRANCH)
+        raise GitImportErrorCannotBranch()
     branches = []
     for line in output.split('\n'):
         branches.append(line.replace('*', '').strip())
@@ -108,14 +166,14 @@ def switch_branch(branch, rdir):
             cmd_log(cmd, rdir)
         except subprocess.CalledProcessError as ex:
             log.exception('Unable to checkout remote branch: %r', ex.output)
-            raise GitImportError(GitImportError.CANNOT_BRANCH)
+            raise GitImportErrorCannotBranch()
     # Go ahead and reset hard to the newest version of the branch now that we know
     # it is local.
     try:
         cmd_log(['git', 'reset', '--hard', 'origin/{0}'.format(branch), ], rdir)
     except subprocess.CalledProcessError as ex:
         log.exception('Unable to reset to branch: %r', ex.output)
-        raise GitImportError(GitImportError.CANNOT_BRANCH)
+        raise GitImportErrorCannotBranch()
 
 
 def add_repo(repo, rdir_in, branch=None):
@@ -126,6 +184,9 @@ def add_repo(repo, rdir_in, branch=None):
     """
     # pylint: disable=too-many-statements
 
+    git_repo_dir = getattr(settings, 'GIT_REPO_DIR', DEFAULT_GIT_REPO_DIR)
+    git_import_static = getattr(settings, 'GIT_IMPORT_STATIC', True)
+
     # Set defaults even if it isn't defined in settings
     mongo_db = {
         'host': 'localhost',
@@ -141,12 +202,12 @@ def add_repo(repo, rdir_in, branch=None):
             mongo_db[config_item] = settings.MONGODB_LOG.get(
                 config_item, mongo_db[config_item])
 
-    if not os.path.isdir(GIT_REPO_DIR):
-        raise GitImportError(GitImportError.NO_DIR)
+    if not os.path.isdir(git_repo_dir):
+        raise GitImportErrorNoDir(git_repo_dir)
     # pull from git
     if not (repo.endswith('.git') or
             repo.startswith(('http:', 'https:', 'git:', 'file:'))):
-        raise GitImportError(GitImportError.URL_BAD)
+        raise GitImportErrorUrlBad()
 
     if rdir_in:
         rdir = os.path.basename(rdir_in)
@@ -154,7 +215,7 @@ def add_repo(repo, rdir_in, branch=None):
         rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0]
     log.debug('rdir = %s', rdir)
 
-    rdirp = '{0}/{1}'.format(GIT_REPO_DIR, rdir)
+    rdirp = '{0}/{1}'.format(git_repo_dir, rdir)
     if os.path.exists(rdirp):
         log.info('directory already exists, doing a git pull instead '
                  'of git clone')
@@ -162,14 +223,14 @@ def add_repo(repo, rdir_in, branch=None):
         cwd = rdirp
     else:
         cmd = ['git', 'clone', repo, ]
-        cwd = GIT_REPO_DIR
+        cwd = git_repo_dir
 
     cwd = os.path.abspath(cwd)
     try:
         ret_git = cmd_log(cmd, cwd=cwd)
     except subprocess.CalledProcessError as ex:
         log.exception('Error running git pull: %r', ex.output)
-        raise GitImportError(GitImportError.CANNOT_PULL)
+        raise GitImportErrorCannotPull()
 
     if branch:
         switch_branch(branch, rdirp)
@@ -180,7 +241,7 @@ def add_repo(repo, rdir_in, branch=None):
         commit_id = cmd_log(cmd, cwd=rdirp)
     except subprocess.CalledProcessError as ex:
         log.exception('Unable to get git log: %r', ex.output)
-        raise GitImportError(GitImportError.BAD_REPO)
+        raise GitImportErrorBadRepo()
 
     ret_git += '\nCommit ID: {0}'.format(commit_id)
 
@@ -192,7 +253,7 @@ def add_repo(repo, rdir_in, branch=None):
         # I can't discover a way to excercise this, but git is complex
         # so still logging and raising here in case.
         log.exception('Unable to determine branch: %r', ex.output)
-        raise GitImportError(GitImportError.BAD_REPO)
+        raise GitImportErrorBadRepo()
 
     ret_git += '{0}Branch: {1}'.format('   \n', branch)
 
@@ -212,12 +273,12 @@ def add_repo(repo, rdir_in, branch=None):
         loggers.append(logger)
 
     try:
-        management.call_command('import', GIT_REPO_DIR, rdir,
-                                nostatic=not GIT_IMPORT_STATIC)
+        management.call_command('import', git_repo_dir, rdir,
+                                nostatic=not git_import_static)
     except CommandError:
-        raise GitImportError(GitImportError.XML_IMPORT_FAILED)
+        raise GitImportErrorXmlImportFailed()
     except NotImplementedError:
-        raise GitImportError(GitImportError.UNSUPPORTED_STORE)
+        raise GitImportErrorUnsupportedStore()
 
     ret_import = output.getvalue()
 
@@ -238,7 +299,7 @@ def add_repo(repo, rdir_in, branch=None):
             course_key = CourseKey.from_string(course_id)
         except InvalidKeyError:
             course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id)
-        cdir = '{0}/{1}'.format(GIT_REPO_DIR, course_key.course)
+        cdir = '{0}/{1}'.format(git_repo_dir, course_key.course)
         log.debug('Studio course dir = %s', cdir)
 
         if os.path.exists(cdir) and not os.path.islink(cdir):

lms/djangoapps/dashboard/management/commands/tests/test_git_add_course.py

@@ -7,7 +7,7 @@ import shutil
 import StringIO
 import subprocess
 import unittest
-
+from uuid import uuid4
 from nose.plugins.attrib import attr
 
 from django.conf import settings
@@ -17,7 +17,14 @@ from django.test.utils import override_settings
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
 
 import dashboard.git_import as git_import
-from dashboard.git_import import GitImportError
+from dashboard.git_import import (
+    GitImportError,
+    GitImportErrorNoDir,
+    GitImportErrorUrlBad,
+    GitImportErrorCannotPull,
+    GitImportErrorBadRepo,
+    GitImportErrorRemoteBranchMissing,
+)
 from xmodule.modulestore import ModuleStoreEnum
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
@@ -37,7 +44,10 @@ FEATURES_WITH_SSL_AUTH['AUTH_USE_CERTIFICATES'] = True
 
 
 @attr('shard_3')
-@override_settings(MONGODB_LOG=TEST_MONGODB_LOG)
+@override_settings(
+    MONGODB_LOG=TEST_MONGODB_LOG,
+    GIT_REPO_DIR=settings.TEST_ROOT / "course_repos_{}".format(uuid4().hex)
+)
 @unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'),
                      "ENABLE_SYSADMIN_DASHBOARD not set")
 class TestGitAddCourse(SharedModuleStoreTestCase):
@@ -49,10 +59,13 @@ class TestGitAddCourse(SharedModuleStoreTestCase):
     TEST_COURSE = 'MITx/edx4edx/edx4edx'
     TEST_BRANCH = 'testing_do_not_delete'
     TEST_BRANCH_COURSE = SlashSeparatedCourseKey('MITx', 'edx4edx_branch', 'edx4edx')
-    GIT_REPO_DIR = settings.GIT_REPO_DIR
 
     ENABLED_CACHES = ['default', 'mongo_metadata_inheritance', 'loc_cache']
 
+    def setUp(self):
+        super(TestGitAddCourse, self).setUp()
+        self.git_repo_dir = settings.GIT_REPO_DIR
+
     def assertCommandFailureRegexp(self, regex, *args):
         """
         Convenience function for testing command failures
@@ -72,40 +85,40 @@ class TestGitAddCourse(SharedModuleStoreTestCase):
             'blah', 'blah', 'blah', 'blah')
         # Not a valid path.
         self.assertCommandFailureRegexp(
-            'Path {0} doesn\'t exist, please create it,'.format(self.GIT_REPO_DIR),
+            'Path {0} doesn\'t exist, please create it,'.format(self.git_repo_dir),
             'blah')
         # Test successful import from command
-        if not os.path.isdir(self.GIT_REPO_DIR):
-            os.mkdir(self.GIT_REPO_DIR)
-        self.addCleanup(shutil.rmtree, self.GIT_REPO_DIR)
+        if not os.path.isdir(self.git_repo_dir):
+            os.mkdir(self.git_repo_dir)
+        self.addCleanup(shutil.rmtree, self.git_repo_dir)
 
         # Make a course dir that will be replaced with a symlink
         # while we are at it.
-        if not os.path.isdir(self.GIT_REPO_DIR / 'edx4edx'):
-            os.mkdir(self.GIT_REPO_DIR / 'edx4edx')
+        if not os.path.isdir(self.git_repo_dir / 'edx4edx'):
+            os.mkdir(self.git_repo_dir / 'edx4edx')
 
         call_command('git_add_course', self.TEST_REPO,
-                     directory_path=self.GIT_REPO_DIR / 'edx4edx_lite')
+                     directory_path=self.git_repo_dir / 'edx4edx_lite')
 
         # Test with all three args (branch)
         call_command('git_add_course', self.TEST_REPO,
-                     directory_path=self.GIT_REPO_DIR / 'edx4edx_lite',
+                     directory_path=self.git_repo_dir / 'edx4edx_lite',
                      repository_branch=self.TEST_BRANCH)
 
     def test_add_repo(self):
         """
         Various exit path tests for test_add_repo
         """
-        with self.assertRaisesRegexp(GitImportError, GitImportError.NO_DIR):
+        with self.assertRaises(GitImportErrorNoDir):
             git_import.add_repo(self.TEST_REPO, None, None)
 
-        os.mkdir(self.GIT_REPO_DIR)
-        self.addCleanup(shutil.rmtree, self.GIT_REPO_DIR)
+        os.mkdir(self.git_repo_dir)
+        self.addCleanup(shutil.rmtree, self.git_repo_dir)
 
-        with self.assertRaisesRegexp(GitImportError, GitImportError.URL_BAD):
+        with self.assertRaises(GitImportErrorUrlBad):
             git_import.add_repo('foo', None, None)
 
-        with self.assertRaisesRegexp(GitImportError, GitImportError.CANNOT_PULL):
+        with self.assertRaises(GitImportErrorCannotPull):
             git_import.add_repo('file:///foobar.git', None, None)
 
         # Test git repo that exists, but is "broken"
@@ -115,14 +128,14 @@ class TestGitAddCourse(SharedModuleStoreTestCase):
         subprocess.check_output(['git', '--bare', 'init', ], stderr=subprocess.STDOUT,
                                 cwd=bare_repo)
 
-        with self.assertRaisesRegexp(GitImportError, GitImportError.BAD_REPO):
+        with self.assertRaises(GitImportErrorBadRepo):
             git_import.add_repo('file://{0}'.format(bare_repo), None, None)
 
     def test_detached_repo(self):
         """
         Test repo that is in detached head state.
         """
-        repo_dir = self.GIT_REPO_DIR
+        repo_dir = self.git_repo_dir
         # Test successful import from command
         try:
             os.mkdir(repo_dir)
@@ -133,21 +146,21 @@ class TestGitAddCourse(SharedModuleStoreTestCase):
         subprocess.check_output(['git', 'checkout', 'HEAD~2', ],
                                 stderr=subprocess.STDOUT,
                                 cwd=repo_dir / 'edx4edx_lite')
-        with self.assertRaisesRegexp(GitImportError, GitImportError.CANNOT_PULL):
+        with self.assertRaises(GitImportErrorCannotPull):
             git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite', None)
 
     def test_branching(self):
         """
         Exercise branching code of import
         """
-        repo_dir = self.GIT_REPO_DIR
+        repo_dir = self.git_repo_dir
         # Test successful import from command
         if not os.path.isdir(repo_dir):
             os.mkdir(repo_dir)
         self.addCleanup(shutil.rmtree, repo_dir)
 
         # Checkout non existent branch
-        with self.assertRaisesRegexp(GitImportError, GitImportError.REMOTE_BRANCH_MISSING):
+        with self.assertRaises(GitImportErrorRemoteBranchMissing):
             git_import.add_repo(self.TEST_REPO, repo_dir / 'edx4edx_lite', 'asdfasdfasdf')
 
         # Checkout new branch
@@ -185,13 +198,13 @@ class TestGitAddCourse(SharedModuleStoreTestCase):
                                 cwd=bare_repo)
 
         # Build repo dir
-        repo_dir = self.GIT_REPO_DIR
+        repo_dir = self.git_repo_dir
         if not os.path.isdir(repo_dir):
             os.mkdir(repo_dir)
         self.addCleanup(shutil.rmtree, repo_dir)
 
         rdir = '{0}/bare'.format(repo_dir)
-        with self.assertRaisesRegexp(GitImportError, GitImportError.BAD_REPO):
+        with self.assertRaises(GitImportErrorBadRepo):
             git_import.add_repo('file://{0}'.format(bare_repo), None, None)
 
         # Get logger for checking strings in logs

lms/djangoapps/dashboard/sysadmin.py

@@ -346,7 +346,8 @@ class Courses(SysadminDashboardView):
 
         # Try the data dir, then try to find it in the git import dir
         if not gdir.exists():
-            gdir = path(git_import.GIT_REPO_DIR) / cdir
+            git_repo_dir = getattr(settings, 'GIT_REPO_DIR', git_import.DEFAULT_GIT_REPO_DIR)
+            gdir = path(git_repo_dir / cdir)
             if not gdir.exists():
                 return info
 

lms/djangoapps/dashboard/tests/test_sysadmin.py

@@ -6,6 +6,7 @@ import os
 import re
 import shutil
 import unittest
+from uuid import uuid4
 from util.date_utils import get_time_display, DEFAULT_DATE_TIME_FORMAT
 from nose.plugins.attrib import attr
 
@@ -18,7 +19,7 @@ import mongoengine
 from opaque_keys.edx.locations import SlashSeparatedCourseKey
 
 from dashboard.models import CourseImportLog
-from dashboard.git_import import GitImportError
+from dashboard.git_import import GitImportErrorNoDir
 from datetime import datetime
 from student.roles import CourseStaffRole, GlobalStaff
 from student.tests.factories import UserFactory
@@ -109,7 +110,10 @@ class SysadminBaseTestCase(SharedModuleStoreTestCase):
 
 
 @attr('shard_1')
-@override_settings(MONGODB_LOG=TEST_MONGODB_LOG)
+@override_settings(
+    MONGODB_LOG=TEST_MONGODB_LOG,
+    GIT_REPO_DIR=settings.TEST_ROOT / "course_repos_{}".format(uuid4().hex)
+)
 @unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'),
                      "ENABLE_SYSADMIN_DASHBOARD not set")
 class TestSysAdminMongoCourseImport(SysadminBaseTestCase):
@@ -149,7 +153,7 @@ class TestSysAdminMongoCourseImport(SysadminBaseTestCase):
 
         # Create git loaded course
         response = self._add_edx4edx()
-        self.assertIn(GitImportError.NO_DIR,
+        self.assertIn(GitImportErrorNoDir(settings.GIT_REPO_DIR).message,
                       response.content.decode('UTF-8'))
 
     def test_mongo_course_add_delete(self):