Merge pull request #12726 from edx/christina/deserialize_config_models

ConfigurationModel deserializer and management command

common/djangoapps/config_models/management/commands/populate_model.py

+"""
+Populates a ConfigurationModel by deserializing JSON data contained in a file.
+"""
+import os
+from optparse import make_option
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext_lazy as _
+
+from config_models.utils import deserialize_json
+
+
+class Command(BaseCommand):
+    """
+    This command will deserialize the JSON data in the supplied file to populate
+    a ConfigurationModel. Note that this will add new entries to the model, but it
+    will not delete any entries (ConfigurationModel entries are read-only).
+    """
+    help = """
+    Populates a ConfigurationModel by deserializing the supplied JSON.
+
+    JSON should be in a file, with the following format:
+
+    { "model": "config_models.ExampleConfigurationModel",
+      "data":
+        [
+          { "enabled": True,
+            "color": "black"
+            ...
+          },
+          { "enabled": False,
+            "color": "yellow"
+            ...
+          },
+          ...
+        ]
+    }
+
+    A username corresponding to an existing user must be specified to indicate who
+    is executing the command.
+
+        $ ... populate_model -f path/to/file.json -u username
+    """
+
+    option_list = BaseCommand.option_list + (
+        make_option('-f', '--file',
+                    metavar='JSON_FILE',
+                    dest='file',
+                    default=False,
+                    help='JSON file to import ConfigurationModel data'),
+        make_option('-u', '--username',
+                    metavar='USERNAME',
+                    dest='username',
+                    default=False,
+                    help='username to specify who is executing the command'),
+    )
+
+    def handle(self, *args, **options):
+        if 'file' not in options or not options['file']:
+            raise CommandError(_("A file containing JSON must be specified."))
+
+        if 'username' not in options or not options['username']:
+            raise CommandError(_("A valid username must be specified."))
+
+        json_file = options['file']
+        if not os.path.exists(json_file):
+            raise CommandError(_("File {0} does not exist").format(json_file))
+
+        self.stdout.write(_("Importing JSON data from file {0}").format(json_file))
+        with open(json_file) as data:
+            created_entries = deserialize_json(data, options['username'])
+            self.stdout.write(_("Import complete, {0} new entries created").format(created_entries))

common/djangoapps/config_models/models.py

@@ -6,6 +6,9 @@ from django.contrib.auth.models import User
 from django.core.cache import caches, InvalidCacheBackendError
 from django.utils.translation import ugettext_lazy as _
 
+from rest_framework.utils import model_meta
+
+
 try:
     cache = caches['configuration']  # pylint: disable=invalid-name
 except InvalidCacheBackendError:
@@ -176,3 +179,58 @@ class ConfigurationModel(models.Model):
         values = list(cls.objects.values_list(*key_fields, flat=flat).order_by().distinct())
         cache.set(cache_key, values, cls.cache_timeout)
         return values
+
+    def fields_equal(self, instance, fields_to_ignore=("id", "change_date", "changed_by")):
+        """
+        Compares this instance's fields to the supplied instance to test for equality.
+        This will ignore any fields in `fields_to_ignore`.
+
+        Note that this method ignores many-to-many fields.
+
+        Args:
+            instance: the model instance to compare
+            fields_to_ignore: List of fields that should not be compared for equality. By default
+            includes `id`, `change_date`, and `changed_by`.
+
+        Returns: True if the checked fields are all equivalent, else False
+        """
+        for field in self._meta.get_fields():
+            if not field.many_to_many and field.name not in fields_to_ignore:
+                if getattr(instance, field.name) != getattr(self, field.name):
+                    return False
+
+        return True
+
+    @classmethod
+    def equal_to_current(cls, json, fields_to_ignore=("id", "change_date", "changed_by")):
+        """
+        Compares for equality this instance to a model instance constructed from the supplied JSON.
+        This will ignore any fields in `fields_to_ignore`.
+
+        Note that this method cannot handle fields with many-to-many associations, as those can only
+        be set on a saved model instance (and saving the model instance will create a new entry).
+        All many-to-many field entries will be removed before the equality comparison is done.
+
+        Args:
+            json: json representing an entry to compare
+            fields_to_ignore: List of fields that should not be compared for equality. By default
+            includes `id`, `change_date`, and `changed_by`.
+
+        Returns: True if the checked fields are all equivalent, else False
+        """
+
+        # Remove many-to-many relationships from json.
+        # They require an instance to be already saved.
+        info = model_meta.get_field_info(cls)
+        for field_name, relation_info in info.relations.items():
+            if relation_info.to_many and (field_name in json):
+                json.pop(field_name)
+
+        new_instance = cls(**json)
+        key_field_args = tuple(getattr(new_instance, key) for key in cls.KEY_FIELDS)
+        current = cls.current(*key_field_args)
+        # If current.id is None, no entry actually existed and the "current" method created it.
+        if current.id is not None:
+            return current.fields_equal(new_instance, fields_to_ignore)
+
+        return False

common/djangoapps/config_models/tests/data/data.json

+{
+  "model": "config_models.ExampleDeserializeConfig",
+  "data": [
+    {
+      "name": "betty",
+      "enabled": true,
+      "int_field": 5
+    },
+    {
+      "name": "fred",
+      "enabled": false
+    }
+  ]
+}

common/djangoapps/config_models/tests/test_model_deserialization.py

+"""
+Tests of the populate_model management command and its helper utils.deserialize_json method.
+"""
+
+import textwrap
+import os.path
+
+from django.utils import timezone
+from django.utils.six import BytesIO
+
+from django.contrib.auth.models import User
+from django.core.management.base import CommandError
+from django.db import models
+
+from config_models.management.commands import populate_model
+from config_models.models import ConfigurationModel
+from config_models.utils import deserialize_json
+from openedx.core.djangolib.testing.utils import CacheIsolationTestCase
+
+
+class ExampleDeserializeConfig(ConfigurationModel):
+    """
+    Test model for testing deserialization of ``ConfigurationModels`` with keyed configuration.
+    """
+    KEY_FIELDS = ('name',)
+
+    name = models.TextField()
+    int_field = models.IntegerField(default=10)
+
+    def __unicode__(self):
+        return "ExampleDeserializeConfig(enabled={}, name={}, int_field={})".format(
+            self.enabled, self.name, self.int_field
+        )
+
+
+class DeserializeJSONTests(CacheIsolationTestCase):
+    """
+    Tests of deserializing the JSON representation of ConfigurationModels.
+    """
+    def setUp(self):
+        super(DeserializeJSONTests, self).setUp()
+        self.test_username = 'test_worker'
+        User.objects.create_user(username=self.test_username)
+        self.fixture_path = os.path.join(os.path.dirname(__file__), 'data', 'data.json')
+
+    def test_deserialize_models(self):
+        """
+        Tests the "happy path", where 2 instances of the test model should be created.
+        A valid username is supplied for the operation.
+        """
+        start_date = timezone.now()
+        with open(self.fixture_path) as data:
+            entries_created = deserialize_json(data, self.test_username)
+            self.assertEquals(2, entries_created)
+
+        self.assertEquals(2, ExampleDeserializeConfig.objects.count())
+
+        betty = ExampleDeserializeConfig.current('betty')
+        self.assertTrue(betty.enabled)
+        self.assertEquals(5, betty.int_field)
+        self.assertGreater(betty.change_date, start_date)
+        self.assertEquals(self.test_username, betty.changed_by.username)
+
+        fred = ExampleDeserializeConfig.current('fred')
+        self.assertFalse(fred.enabled)
+        self.assertEquals(10, fred.int_field)
+        self.assertGreater(fred.change_date, start_date)
+        self.assertEquals(self.test_username, fred.changed_by.username)
+
+    def test_existing_entries_not_removed(self):
+        """
+        Any existing configuration model entries are retained
+        (though they may be come history)-- deserialize_json is purely additive.
+        """
+        ExampleDeserializeConfig(name="fred", enabled=True).save()
+        ExampleDeserializeConfig(name="barney", int_field=200).save()
+
+        with open(self.fixture_path) as data:
+            entries_created = deserialize_json(data, self.test_username)
+            self.assertEquals(2, entries_created)
+
+        self.assertEquals(4, ExampleDeserializeConfig.objects.count())
+        self.assertEquals(3, len(ExampleDeserializeConfig.objects.current_set()))
+
+        self.assertEquals(5, ExampleDeserializeConfig.current('betty').int_field)
+        self.assertEquals(200, ExampleDeserializeConfig.current('barney').int_field)
+
+        # The JSON file changes "enabled" to False for Fred.
+        fred = ExampleDeserializeConfig.current('fred')
+        self.assertFalse(fred.enabled)
+
+    def test_duplicate_entries_not_made(self):
+        """
+        If there is no change in an entry (besides changed_by and change_date),
+        a new entry is not made.
+        """
+        with open(self.fixture_path) as data:
+            entries_created = deserialize_json(data, self.test_username)
+            self.assertEquals(2, entries_created)
+
+        with open(self.fixture_path) as data:
+            entries_created = deserialize_json(data, self.test_username)
+            self.assertEquals(0, entries_created)
+
+        # Importing twice will still only result in 2 records (second import a no-op).
+        self.assertEquals(2, ExampleDeserializeConfig.objects.count())
+
+        # Change Betty.
+        betty = ExampleDeserializeConfig.current('betty')
+        betty.int_field = -8
+        betty.save()
+
+        self.assertEquals(3, ExampleDeserializeConfig.objects.count())
+        self.assertEquals(-8, ExampleDeserializeConfig.current('betty').int_field)
+
+        # Now importing will add a new entry for Betty.
+        with open(self.fixture_path) as data:
+            entries_created = deserialize_json(data, self.test_username)
+            self.assertEquals(1, entries_created)
+
+        self.assertEquals(4, ExampleDeserializeConfig.objects.count())
+        self.assertEquals(5, ExampleDeserializeConfig.current('betty').int_field)
+
+    def test_bad_username(self):
+        """
+        Tests the error handling when the specified user does not exist.
+        """
+        test_json = textwrap.dedent("""
+            {
+                "model": "config_models.ExampleDeserializeConfig",
+                "data": [{"name": "dino"}]
+            }
+            """)
+        with self.assertRaisesRegexp(Exception, "User matching query does not exist"):
+            deserialize_json(BytesIO(test_json), "unknown_username")
+
+    def test_invalid_json(self):
+        """
+        Tests the error handling when there is invalid JSON.
+        """
+        test_json = textwrap.dedent("""
+            {
+                "model": "config_models.ExampleDeserializeConfig",
+                "data": [{"name": "dino"
+            """)
+        with self.assertRaisesRegexp(Exception, "JSON parse error"):
+            deserialize_json(BytesIO(test_json), self.test_username)
+
+    def test_invalid_model(self):
+        """
+        Tests the error handling when the configuration model specified does not exist.
+        """
+        test_json = textwrap.dedent("""
+            {
+                "model": "xxx.yyy",
+                "data":[{"name": "dino"}]
+            }
+            """)
+        with self.assertRaisesRegexp(Exception, "No installed app"):
+            deserialize_json(BytesIO(test_json), self.test_username)
+
+
+class PopulateModelTestCase(CacheIsolationTestCase):
+    """
+    Tests of populate model management command.
+    """
+    def setUp(self):
+        super(PopulateModelTestCase, self).setUp()
+        self.file_path = os.path.join(os.path.dirname(__file__), 'data', 'data.json')
+        self.test_username = 'test_management_worker'
+        User.objects.create_user(username=self.test_username)
+
+    def test_run_command(self):
+        """
+        Tests the "happy path", where 2 instances of the test model should be created.
+        A valid username is supplied for the operation.
+        """
+        _run_command(file=self.file_path, username=self.test_username)
+        self.assertEquals(2, ExampleDeserializeConfig.objects.count())
+
+        betty = ExampleDeserializeConfig.current('betty')
+        self.assertEquals(self.test_username, betty.changed_by.username)
+
+        fred = ExampleDeserializeConfig.current('fred')
+        self.assertEquals(self.test_username, fred.changed_by.username)
+
+    def test_no_user_specified(self):
+        """
+        Tests that a username must be specified.
+        """
+        with self.assertRaisesRegexp(CommandError, "A valid username must be specified"):
+            _run_command(file=self.file_path)
+
+    def test_bad_user_specified(self):
+        """
+        Tests that a username must be specified.
+        """
+        with self.assertRaisesRegexp(Exception, "User matching query does not exist"):
+            _run_command(file=self.file_path, username="does_not_exist")
+
+    def test_no_file_specified(self):
+        """
+        Tests the error handling when no JSON file is supplied.
+        """
+        with self.assertRaisesRegexp(CommandError, "A file containing JSON must be specified"):
+            _run_command(username=self.test_username)
+
+    def test_bad_file_specified(self):
+        """
+        Tests the error handling when the path to the JSON file is incorrect.
+        """
+        with self.assertRaisesRegexp(CommandError, "File does/not/exist.json does not exist"):
+            _run_command(file="does/not/exist.json", username=self.test_username)
+
+
+def _run_command(*args, **kwargs):
+    """Run the management command to deserializer JSON ConfigurationModel data. """
+    command = populate_model.Command()
+    return command.handle(*args, **kwargs)

common/djangoapps/config_models/tests.py → common/djangoapps/config_models/tests/tests.py

@@ -25,6 +25,24 @@ class ExampleConfig(ConfigurationModel):
     string_field = models.TextField()
     int_field = models.IntegerField(default=10)
 
+    def __unicode__(self):
+        return "ExampleConfig(enabled={}, string_field={}, int_field={})".format(
+            self.enabled, self.string_field, self.int_field
+        )
+
+
+class ManyToManyExampleConfig(ConfigurationModel):
+    """
+    Test model configuration with a many-to-many field.
+    """
+    cache_timeout = 300
+
+    string_field = models.TextField()
+    many_user_field = models.ManyToManyField(User, related_name='topic_many_user_field')
+
+    def __unicode__(self):
+        return "ManyToManyExampleConfig(enabled={}, string_field={})".format(self.enabled, self.string_field)
+
 
 @patch('config_models.models.cache')
 class ConfigurationModelTests(TestCase):
@@ -40,7 +58,7 @@ class ConfigurationModelTests(TestCase):
         ExampleConfig(changed_by=self.user).save()
         mock_cache.delete.assert_called_with(ExampleConfig.cache_key_name())
 
-    def test_cache_key_name(self, _mock_cache):
+    def test_cache_key_name(self, __):
         self.assertEquals(ExampleConfig.cache_key_name(), 'configuration/ExampleConfig/current')
 
     def test_no_config_empty_cache(self, mock_cache):
@@ -103,6 +121,64 @@ class ConfigurationModelTests(TestCase):
 
         self.assertEquals(2, ExampleConfig.objects.all().count())
 
+    def test_equality(self, mock_cache):
+        mock_cache.get.return_value = None
+
+        config = ExampleConfig(changed_by=self.user, string_field='first')
+        config.save()
+
+        self.assertTrue(ExampleConfig.equal_to_current({"string_field": "first"}))
+        self.assertTrue(ExampleConfig.equal_to_current({"string_field": "first", "enabled": False}))
+        self.assertTrue(ExampleConfig.equal_to_current({"string_field": "first", "int_field": 10}))
+
+        self.assertFalse(ExampleConfig.equal_to_current({"string_field": "first", "enabled": True}))
+        self.assertFalse(ExampleConfig.equal_to_current({"string_field": "first", "int_field": 20}))
+        self.assertFalse(ExampleConfig.equal_to_current({"string_field": "second"}))
+
+        self.assertFalse(ExampleConfig.equal_to_current({}))
+
+    def test_equality_custom_fields_to_ignore(self, mock_cache):
+        mock_cache.get.return_value = None
+
+        config = ExampleConfig(changed_by=self.user, string_field='first')
+        config.save()
+
+        # id, change_date, and changed_by will all be different for a newly created entry
+        self.assertTrue(ExampleConfig.equal_to_current({"string_field": "first"}))
+        self.assertFalse(
+            ExampleConfig.equal_to_current({"string_field": "first"}, fields_to_ignore=("change_date", "changed_by"))
+        )
+        self.assertFalse(
+            ExampleConfig.equal_to_current({"string_field": "first"}, fields_to_ignore=("id", "changed_by"))
+        )
+        self.assertFalse(
+            ExampleConfig.equal_to_current({"string_field": "first"}, fields_to_ignore=("change_date", "id"))
+        )
+
+        # Test the ability to ignore a different field ("int_field").
+        self.assertFalse(ExampleConfig.equal_to_current({"string_field": "first", "int_field": 20}))
+        self.assertTrue(
+            ExampleConfig.equal_to_current(
+                {"string_field": "first", "int_field": 20},
+                fields_to_ignore=("id", "change_date", "changed_by", "int_field")
+            )
+        )
+
+    def test_equality_ignores_many_to_many(self, mock_cache):
+        mock_cache.get.return_value = None
+        config = ManyToManyExampleConfig(changed_by=self.user, string_field='first')
+        config.save()
+
+        second_user = User(username="second_user")
+        second_user.save()
+        config.many_user_field.add(second_user)  # pylint: disable=no-member
+        config.save()
+
+        # The many-to-many field is ignored in comparison.
+        self.assertTrue(
+            ManyToManyExampleConfig.equal_to_current({"string_field": "first", "many_user_field": "removed"})
+        )
+
 
 class ExampleKeyedConfig(ConfigurationModel):
     """
@@ -120,6 +196,11 @@ class ExampleKeyedConfig(ConfigurationModel):
     string_field = models.TextField()
     int_field = models.IntegerField(default=10)
 
+    def __unicode__(self):
+        return "ExampleKeyedConfig(enabled={}, left={}, right={}, string_field={}, int_field={})".format(
+            self.enabled, self.left, self.right, self.string_field, self.int_field
+        )
+
 
 @ddt.ddt
 @patch('config_models.models.cache')
@@ -294,6 +375,45 @@ class KeyedConfigurationModelTests(TestCase):
         mock_cache.get.return_value = fake_result
         self.assertEquals(ExampleKeyedConfig.key_values(), fake_result)
 
+    def test_equality(self, mock_cache):
+        mock_cache.get.return_value = None
+
+        config1 = ExampleKeyedConfig(left='left_a', right='right_a', int_field=1, changed_by=self.user)
+        config1.save()
+
+        config2 = ExampleKeyedConfig(left='left_b', right='right_b', int_field=2, changed_by=self.user, enabled=True)
+        config2.save()
+
+        config3 = ExampleKeyedConfig(left='left_c', changed_by=self.user)
+        config3.save()
+
+        self.assertTrue(
+            ExampleKeyedConfig.equal_to_current({"left": "left_a", "right": "right_a", "int_field": 1})
+        )
+        self.assertTrue(
+            ExampleKeyedConfig.equal_to_current({"left": "left_b", "right": "right_b", "int_field": 2, "enabled": True})
+        )
+        self.assertTrue(
+            ExampleKeyedConfig.equal_to_current({"left": "left_c"})
+        )
+
+        self.assertFalse(
+            ExampleKeyedConfig.equal_to_current(
+                {"left": "left_a", "right": "right_a", "int_field": 1, "string_field": "foo"}
+            )
+        )
+        self.assertFalse(
+            ExampleKeyedConfig.equal_to_current({"left": "left_a", "int_field": 1})
+        )
+        self.assertFalse(
+            ExampleKeyedConfig.equal_to_current({"left": "left_b", "right": "right_b", "int_field": 2})
+        )
+        self.assertFalse(
+            ExampleKeyedConfig.equal_to_current({"left": "left_c", "int_field": 11})
+        )
+
+        self.assertFalse(ExampleKeyedConfig.equal_to_current({}))
+
 
 @ddt.ddt
 class ConfigurationModelAPITests(TestCase):

common/djangoapps/config_models/utils.py

+"""
+Utilities for working with ConfigurationModels.
+"""
+from django.apps import apps
+from rest_framework.parsers import JSONParser
+from rest_framework.serializers import ModelSerializer
+from django.contrib.auth.models import User
+
+
+def get_serializer_class(configuration_model):
+    """ Returns a ConfigurationModel serializer class for the supplied configuration_model. """
+    class AutoConfigModelSerializer(ModelSerializer):
+        """Serializer class for configuration models."""
+
+        class Meta(object):
+            """Meta information for AutoConfigModelSerializer."""
+            model = configuration_model
+
+        def create(self, validated_data):
+            if "changed_by_username" in self.context:
+                validated_data['changed_by'] = User.objects.get(username=self.context["changed_by_username"])
+            return super(AutoConfigModelSerializer, self).create(validated_data)
+
+    return AutoConfigModelSerializer
+
+
+def deserialize_json(stream, username):
+    """
+    Given a stream containing JSON, deserializers the JSON into ConfigurationModel instances.
+
+    The stream is expected to be in the following format:
+        { "model": "config_models.ExampleConfigurationModel",
+          "data":
+            [
+              { "enabled": True,
+                "color": "black"
+                ...
+              },
+              { "enabled": False,
+                "color": "yellow"
+                ...
+              },
+              ...
+            ]
+        }
+
+    If the provided stream does not contain valid JSON for the ConfigurationModel specified,
+    an Exception will be raised.
+
+    Arguments:
+            stream: The stream of JSON, as described above.
+            username: The username of the user making the change. This must match an existing user.
+
+    Returns: the number of created entries
+    """
+    parsed_json = JSONParser().parse(stream)
+    serializer_class = get_serializer_class(apps.get_model(parsed_json["model"]))
+    list_serializer = serializer_class(data=parsed_json["data"], context={"changed_by_username": username}, many=True)
+    if list_serializer.is_valid():
+        model_class = serializer_class.Meta.model
+        for data in reversed(list_serializer.validated_data):
+            if model_class.equal_to_current(data):
+                list_serializer.validated_data.remove(data)
+
+        entries_created = len(list_serializer.validated_data)
+        list_serializer.save()
+        return entries_created
+    else:
+        raise Exception(list_serializer.error_messages)

common/djangoapps/config_models/views.py

@@ -4,9 +4,10 @@ API view to allow manipulation of configuration models.
 from rest_framework.generics import CreateAPIView, RetrieveAPIView
 from rest_framework.permissions import DjangoModelPermissions
 from rest_framework.authentication import SessionAuthentication
-from rest_framework.serializers import ModelSerializer
 from django.db import transaction
 
+from config_models.utils import get_serializer_class
+
 
 class ReadableOnlyByAuthors(DjangoModelPermissions):
     """Only allow access by users with `add` permissions on the model."""
@@ -58,13 +59,7 @@ class ConfigurationModelCurrentAPIView(AtomicMixin, CreateAPIView, RetrieveAPIVi
 
     def get_serializer_class(self):
         if self.serializer_class is None:
-            class AutoConfigModelSerializer(ModelSerializer):
-                """Serializer class for configuration models."""
-                class Meta(object):
-                    """Meta information for AutoConfigModelSerializer."""
-                    model = self.model
-
-            self.serializer_class = AutoConfigModelSerializer
+            self.serializer_class = get_serializer_class(self.model)
 
         return self.serializer_class