Merge pull request #16345 from edx/jmbowman/PLAT-1780

PLAT-1780 Remove dependency on django-extensions

cms/envs/common.py

@@ -942,7 +942,7 @@ INSTALLED_APPS = [
 
     # Maintenance tools
     'maintenance',
-    'django_extensions',
+    'openedx.core.djangoapps.util.apps.UtilConfig',
 
     # Tracking
     'track',

lms/djangoapps/certificates/migrations/0001_initial.py

@@ -3,9 +3,8 @@ from __future__ import unicode_literals
 
 from django.db import migrations, models
 import certificates.models
+import jsonfield.fields
 import model_utils.fields
-import django_extensions.db.fields
-import django_extensions.db.fields.json
 import django.db.models.deletion
 import django.utils.timezone
 from badges.models import validate_badge_image
@@ -26,7 +25,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('course_id', CourseKeyField(default=None, max_length=255, blank=True)),
                 ('mode', models.CharField(max_length=100)),
-                ('data', django_extensions.db.fields.json.JSONField()),
+                ('data', jsonfield.fields.JSONField()),
                 ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
             ],
         ),
@@ -116,7 +115,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('course_id', CourseKeyField(default=None, max_length=255, blank=True)),
                 ('whitelist', models.BooleanField(default=0)),
-                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
+                ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)),
                 ('notes', models.TextField(default=None, null=True)),
                 ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
             ],

lms/djangoapps/certificates/models.py

@@ -58,8 +58,8 @@ from django.db import models, transaction
 from django.db.models import Count
 from django.dispatch import receiver
 from django.utils.translation import ugettext_lazy as _
-from django_extensions.db.fields import CreationDateTimeField
 from model_utils import Choices
+from model_utils.fields import AutoCreatedField
 from model_utils.models import TimeStampedModel
 
 from badges.events.course_complete import course_badge_check
@@ -135,7 +135,7 @@ class CertificateWhitelist(models.Model):
     user = models.ForeignKey(User)
     course_id = CourseKeyField(max_length=255, blank=True, default=None)
     whitelist = models.BooleanField(default=0)
-    created = CreationDateTimeField(_('created'))
+    created = AutoCreatedField(_('created'))
     notes = models.TextField(default=None, null=True)
 
     @classmethod

lms/djangoapps/experiments/migrations/0001_initial.py

@@ -3,7 +3,8 @@ from __future__ import unicode_literals
 
 from django.db import migrations, models
 from django.conf import settings
-import django_extensions.db.fields
+import django.utils.timezone
+import model_utils.fields
 
 
 class Migration(migrations.Migration):
@@ -17,8 +18,8 @@ class Migration(migrations.Migration):
             name='ExperimentData',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
-                ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
+                ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)),
+                ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)),
                 ('experiment_id', models.PositiveSmallIntegerField(verbose_name=b'Experiment ID', db_index=True)),
                 ('key', models.CharField(max_length=255)),
                 ('value', models.TextField()),

lms/djangoapps/experiments/migrations/0002_auto_20170627_1402.py

@@ -2,7 +2,8 @@
 from __future__ import unicode_literals
 
 from django.db import migrations, models
-import django_extensions.db.fields
+import django.utils.timezone
+import model_utils.fields
 
 
 class Migration(migrations.Migration):
@@ -16,8 +17,8 @@ class Migration(migrations.Migration):
             name='ExperimentKeyValue',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
-                ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
+                ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)),
+                ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)),
                 ('experiment_id', models.PositiveSmallIntegerField(verbose_name=b'Experiment ID', db_index=True)),
                 ('key', models.CharField(max_length=255)),
                 ('value', models.TextField()),

lms/djangoapps/experiments/models.py

 from django.conf import settings
 from django.db import models
-from django_extensions.db.models import TimeStampedModel
+from model_utils.models import TimeStampedModel
 
 
 class ExperimentData(TimeStampedModel):

lms/envs/common.py

@@ -2093,7 +2093,7 @@ INSTALLED_APPS = [
     # For testing
     'django.contrib.admin',  # only used in DEBUG mode
     'debug',
-    'django_extensions',
+    'openedx.core.djangoapps.util.apps.UtilConfig',
 
     # Discussion forums
     'django_comment_client',

openedx/core/djangoapps/api_admin/migrations/0001_initial.py

@@ -4,7 +4,8 @@ from __future__ import unicode_literals
 from django.db import migrations, models
 import django.db.models.deletion
 from django.conf import settings
-import django_extensions.db.fields
+import django.utils.timezone
+import model_utils.fields
 
 
 class Migration(migrations.Migration):
@@ -18,8 +19,8 @@ class Migration(migrations.Migration):
             name='ApiAccessRequest',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
-                ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
+                ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)),
+                ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)),
                 ('status', models.CharField(default=b'pending', help_text='Status of this API access request', max_length=255, db_index=True, choices=[(b'pending', 'Pending'), (b'denied', 'Denied'), (b'approved', 'Approved')])),
                 ('website', models.URLField(help_text='The URL of the website associated with this API user.')),
                 ('reason', models.TextField(help_text='The reason this user wants to access the API.')),
@@ -35,8 +36,8 @@ class Migration(migrations.Migration):
             name='HistoricalApiAccessRequest',
             fields=[
                 ('id', models.IntegerField(verbose_name='ID', db_index=True, auto_created=True, blank=True)),
-                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
-                ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
+                ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)),
+                ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)),
                 ('status', models.CharField(default=b'pending', help_text='Status of this API access request', max_length=255, db_index=True, choices=[(b'pending', 'Pending'), (b'denied', 'Denied'), (b'approved', 'Approved')])),
                 ('website', models.URLField(help_text='The URL of the website associated with this API user.')),
                 ('reason', models.TextField(help_text='The reason this user wants to access the API.')),

openedx/core/djangoapps/api_admin/models.py

@@ -13,9 +13,9 @@ from django.db import models
 from django.db.models.signals import post_save, pre_save
 from django.dispatch import receiver
 from django.utils.translation import ugettext as _
-from django_extensions.db.models import TimeStampedModel
 
 from edxmako.shortcuts import render_to_string
+from model_utils.models import TimeStampedModel
 from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers
 
 log = logging.getLogger(__name__)
@@ -47,6 +47,10 @@ class ApiAccessRequest(TimeStampedModel):
     site = models.ForeignKey(Site)
     contacted = models.BooleanField(default=False)
 
+    class Meta:
+        get_latest_by = 'modified'
+        ordering = ('-modified', '-created',)
+
     @classmethod
     def has_api_access(cls, user):
         """Returns whether or not this user has been granted API access.

openedx/core/djangoapps/schedules/migrations/0001_initial.py

@@ -2,7 +2,8 @@
 from __future__ import unicode_literals
 
 from django.db import migrations, models
-import django_extensions.db.fields
+import django.utils.timezone
+import model_utils.fields
 
 
 class Migration(migrations.Migration):
@@ -16,8 +17,8 @@ class Migration(migrations.Migration):
             name='Schedule',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
-                ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
+                ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)),
+                ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)),
                 ('active', models.BooleanField(default=True, help_text='Indicates if this schedule is actively used')),
                 ('start', models.DateTimeField(help_text='Date this schedule went into effect')),
                 ('upgrade_deadline', models.DateTimeField(help_text='Deadline by which the learner must upgrade to a verified seat', null=True, blank=True)),

openedx/core/djangoapps/schedules/models.py

 from django.db import models
 from django.utils.translation import ugettext_lazy as _
-from django_extensions.db.models import TimeStampedModel
 from django.contrib.sites.models import Site
+from model_utils.models import TimeStampedModel
 
 from config_models.models import ConfigurationModel
 

openedx/core/djangoapps/site_configuration/migrations/0001_initial.py

@@ -2,8 +2,9 @@
 from __future__ import unicode_literals
 
 from django.db import migrations, models
+import django.utils.timezone
 import jsonfield.fields
-import django_extensions.db.fields
+import model_utils.fields
 
 
 class Migration(migrations.Migration):
@@ -25,8 +26,8 @@ class Migration(migrations.Migration):
             name='SiteConfigurationHistory',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
-                ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
+                ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)),
+                ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)),
                 ('values', jsonfield.fields.JSONField(blank=True)),
                 ('site', models.ForeignKey(related_name='configuration_histories', to='sites.Site')),
             ],

openedx/core/djangoapps/site_configuration/models.py

@@ -8,8 +8,8 @@ from django.contrib.sites.models import Site
 from django.db import models
 from django.db.models.signals import post_save
 from django.dispatch import receiver
-from django_extensions.db.models import TimeStampedModel
 from jsonfield.fields import JSONField
+from model_utils.models import TimeStampedModel
 
 logger = getLogger(__name__)  # pylint: disable=invalid-name
 
@@ -130,6 +130,10 @@ class SiteConfigurationHistory(TimeStampedModel):
         load_kwargs={'object_pairs_hook': collections.OrderedDict}
     )
 
+    class Meta:
+        get_latest_by = 'modified'
+        ordering = ('-modified', '-created',)
+
     def __unicode__(self):
         return u"<SiteConfigurationHistory: {site}, Last Modified: {modified} >".format(
             modified=self.modified,

openedx/core/djangoapps/util/apps.py

+"""
+Configuration for the openedx.core.djangoapps.util Django application
+"""
+
+from django.apps import AppConfig
+
+
+class UtilConfig(AppConfig):
+    """
+    Let Django know that this is an app with management commands.
+    """
+    label = 'open_edx_util'
+    name = 'openedx.core.djangoapps.util'
+    verbose_name = 'Open edX Utilities'

openedx/core/djangoapps/util/management/commands/print_setting.py

+# -*- coding: utf-8 -*-
+"""
+print_setting
+=============
+
+Django command to output a single Django setting.
+Useful when paver or a shell script needs such a value.
+
+This handles the one specific use case of the "print_settings" command from
+django-extensions that we were actually using.
+"""
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+
+
+class Command(BaseCommand):
+    """print_setting command"""
+
+    help = "Print the value of a single Django setting."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            'setting',
+            help='Specifies the setting to be printed.'
+        )
+
+    def handle(self, *args, **options):
+        setting = options.get('setting')
+
+        if not hasattr(settings, setting):
+            raise CommandError('%s not found in settings.' % setting)
+
+        print(getattr(settings, setting))

openedx/core/djangoapps/util/management/commands/reset_db.py

+# -*- coding: utf-8 -*-
+"""
+reset_db
+========
+
+Django command to drop and recreate a database.
+Useful when running tests against a database which may previously have
+had different migrations applied to it.
+
+This handles the one specific use case of the "reset_db" command from
+django-extensions that we were actually using.
+
+originally from http://www.djangosnippets.org/snippets/828/ by dnordberg
+"""
+import logging
+
+import django
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from six.moves import configparser, input
+
+
+class Command(BaseCommand):
+    help = "Resets the database for this project."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '-R', '--router', action='store', dest='router', default='default',
+            help='Use this router-database other than defined in settings.py')
+
+    def handle(self, *args, **options):
+        """
+        Resets the database for this project.
+
+        Note: Transaction wrappers are in reverse as a work around for
+        autocommit, anybody know how to do this the right way?
+        """
+        router = options.get('router')
+        dbinfo = settings.DATABASES.get(router)
+        if dbinfo is None:
+            raise CommandError("Unknown database router %s" % router)
+
+        engine = dbinfo.get('ENGINE').split('.')[-1]
+
+        user = password = database_name = database_host = database_port = ''
+        if engine == 'mysql':
+            (user, password, database_name, database_host, database_port) = parse_mysql_cnf(dbinfo)
+
+        user = dbinfo.get('USER') or user
+        password = dbinfo.get('PASSWORD') or password
+        owner = user
+
+        database_name = dbinfo.get('NAME') or database_name
+        if database_name == '':
+            raise CommandError("You need to specify DATABASE_NAME in your Django settings file.")
+
+        database_host = dbinfo.get('HOST') or database_host
+        database_port = dbinfo.get('PORT') or database_port
+
+        verbosity = int(options.get('verbosity', 1))
+
+        if engine in ('sqlite3', 'spatialite'):
+            import os
+            try:
+                logging.info("Unlinking %s database" % engine)
+                os.unlink(database_name)
+            except OSError:
+                pass
+
+        elif engine in ('mysql',):
+            import MySQLdb as Database
+            kwargs = {
+                'user': user,
+                'passwd': password,
+            }
+            if database_host.startswith('/'):
+                kwargs['unix_socket'] = database_host
+            else:
+                kwargs['host'] = database_host
+
+            if database_port:
+                kwargs['port'] = int(database_port)
+
+            connection = Database.connect(**kwargs)
+            drop_query = 'DROP DATABASE IF EXISTS `%s`' % database_name
+            utf8_support = 'CHARACTER SET utf8'
+            create_query = 'CREATE DATABASE `%s` %s' % (database_name, utf8_support)
+            logging.info('Executing... "' + drop_query + '"')
+            connection.query(drop_query)
+            logging.info('Executing... "' + create_query + '"')
+            connection.query(create_query)
+
+        elif engine in ('postgresql', 'postgresql_psycopg2', 'postgis'):
+            if engine == 'postgresql' and django.VERSION < (1, 9):
+                import psycopg as Database  # NOQA
+            elif engine in ('postgresql', 'postgresql_psycopg2', 'postgis'):
+                import psycopg2 as Database  # NOQA
+
+            conn_params = {'database': 'template1'}
+            if user:
+                conn_params['user'] = user
+            if password:
+                conn_params['password'] = password
+            if database_host:
+                conn_params['host'] = database_host
+            if database_port:
+                conn_params['port'] = database_port
+
+            connection = Database.connect(**conn_params)
+            connection.set_isolation_level(0)  # autocommit false
+            cursor = connection.cursor()
+
+            drop_query = "DROP DATABASE \"%s\";" % database_name
+            logging.info('Executing... "' + drop_query + '"')
+            try:
+                cursor.execute(drop_query)
+            except Database.ProgrammingError as e:
+                logging.exception("Error: %s" % str(e))
+
+            create_query = "CREATE DATABASE \"%s\"" % database_name
+            if owner:
+                create_query += " WITH OWNER = \"%s\" " % owner
+            create_query += " ENCODING = 'UTF8'"
+
+            if engine == 'postgis' and django.VERSION < (1, 9):
+                # For PostGIS 1.5, fetch template name if it exists
+                from django.contrib.gis.db.backends.postgis.base import DatabaseWrapper
+                postgis_template = DatabaseWrapper(dbinfo).template_postgis
+                if postgis_template is not None:
+                    create_query += ' TEMPLATE = %s' % postgis_template
+
+            if settings.DEFAULT_TABLESPACE:
+                create_query += ' TABLESPACE = %s;' % settings.DEFAULT_TABLESPACE
+            else:
+                create_query += ';'
+
+            logging.info('Executing... "' + create_query + '"')
+            cursor.execute(create_query)
+
+        else:
+            raise CommandError("Unknown database engine %s" % engine)
+
+        if verbosity >= 2:
+            print("Reset successful.")
+
+
+def parse_mysql_cnf(dbinfo):
+    """
+    Attempt to parse mysql database config file for connection settings.
+    Ideally we would hook into django's code to do this, but read_default_file is handled by the mysql C libs
+    so we have to emulate the behaviour
+
+    Settings that are missing will return ''
+    returns (user, password, database_name, database_host, database_port)
+    """
+    read_default_file = dbinfo.get('OPTIONS', {}).get('read_default_file')
+    if read_default_file:
+        config = configparser.RawConfigParser({
+            'user': '',
+            'password': '',
+            'database': '',
+            'host': '',
+            'port': '',
+            'socket': '',
+        })
+        import os
+        config.read(os.path.expanduser(read_default_file))
+        try:
+            user = config.get('client', 'user')
+            password = config.get('client', 'password')
+            database_name = config.get('client', 'database')
+            database_host = config.get('client', 'host')
+            database_port = config.get('client', 'port')
+            socket = config.get('client', 'socket')
+
+            if database_host == 'localhost' and socket:
+                # mysql actually uses a socket if host is localhost
+                database_host = socket
+
+            return user, password, database_name, database_host, database_port
+
+        except configparser.NoSectionError:
+            pass
+
+    return '', '', '', '', ''

openedx/core/djangoapps/util/tests/test_print_setting.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import pytest
+from django.core.management import call_command, CommandError
+
+
+def test_without_args(capsys):
+    with pytest.raises(CommandError, message='Error: too few arguments'):
+        call_command('print_setting')
+
+
+def test_with_setting_args(capsys):
+    call_command('print_setting', 'DEBUG')
+
+    out, err = capsys.readouterr()
+    assert 'False' in out
+    assert 'INSTALLED_APPS' not in out

pavelib/paver_tests/test_servers.py

@@ -42,9 +42,9 @@ EXPECTED_INDEX_COURSE_COMMAND = (
     u"python manage.py {system} --settings={settings} reindex_course --setup"
 )
 EXPECTED_PRINT_SETTINGS_COMMAND = [
-    u"python manage.py lms --settings={settings} print_settings STATIC_ROOT --format=value 2>/dev/null",
-    u"python manage.py cms --settings={settings} print_settings STATIC_ROOT --format=value 2>/dev/null",
-    u"python manage.py lms --settings={settings} print_settings WEBPACK_CONFIG_PATH --format=value 2>/dev/null"
+    u"python manage.py lms --settings={settings} print_setting STATIC_ROOT 2>/dev/null",
+    u"python manage.py cms --settings={settings} print_setting STATIC_ROOT 2>/dev/null",
+    u"python manage.py lms --settings={settings} print_setting WEBPACK_CONFIG_PATH 2>/dev/null"
 ]
 EXPECTED_WEBPACK_COMMAND = (
     u"NODE_ENV={node_env} STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} "

pavelib/utils/envs.py

@@ -239,7 +239,7 @@ class Env(object):
             django_cmd(
                 system,
                 settings,
-                "print_settings {django_setting} --format=value 2>/dev/null".format(
+                "print_setting {django_setting} 2>/dev/null".format(
                     django_setting=django_setting
                 )
             ),

requirements/edx/base.txt

@@ -19,7 +19,6 @@ django-babel-underscore==0.5.2
 markey==0.8  # From django-babel-underscore
 django-config-models==0.1.8
 django-countries==4.6.1
-django-extensions==1.5.9
 django-filter==1.0.4
 django-ipware==1.1.0
 django-model-utils==3.0.0

scripts/reset-test-db.sh

@@ -46,12 +46,12 @@ for db in "${database_order[@]}"; do
 
     # Clear out the test database
     #
-    # We are using the django-extensions's reset_db command which uses "DROP DATABASE" and
+    # We are using the reset_db command which uses "DROP DATABASE" and
     # "CREATE DATABASE" in case the tests are being run in an environment (e.g. devstack
     # or a jenkins worker environment) that already ran tests on another commit that had
     # different migrations that created, dropped, or altered tables.
     echo "Issuing a reset_db command to the $db bok_choy MySQL database."
-    ./manage.py lms --settings $SETTINGS reset_db --traceback --noinput --router $db
+    ./manage.py lms --settings $SETTINGS reset_db --traceback --router $db
 
     # If there are cached database schemas/data, load them
     if [[ ! -f $DB_CACHE_DIR/bok_choy_schema_$db.sql || ! -f $DB_CACHE_DIR/bok_choy_data_$db.json || ! -f $DB_CACHE_DIR/bok_choy_migrations_data_$db.sql ]]; then